def demodata_infr2(defaultdb='PZ_MTEST'):
import ibeis
from graphid.core.annot_inference import AnnotInference
defaultdb = 'PZ_MTEST'
ibs = ibeis.opendb(defaultdb=defaultdb)
annots = ibs.annots()
names = list(annots.group_items(annots.nids).values())[0:20]
def dummy_phi(c, n):
x = np.arange(n)
phi = c * x / (c * x + 1)
phi = phi / phi.sum()
phi = np.diff(phi)
return phi
phis = {c: dummy_phi(c, 30) for c in range(1, 4)}
aids = list(ub.flatten(names))
infr = AnnotInference(ibs, aids, autoinit=True)
infr.init_termination_criteria(phis)
infr.init_refresh_criteria()
# Partially review
n1, n2, n3, n4 = names[0:4]
for name in names[4:]:
for a, b in ub.iter_window(name.aids, 2):
infr.add_feedback((a, b), POSTV)
for name1, name2 in it.combinations(names[4:], 2):
infr.add_feedback((name1.aids[0], name2.aids[0]), NEGTV)
return infr
def _warp_imgaug(self, augmenter, input_dims, inplace=False):
"""
Warps by applying an augmenter from the imgaug library
Args:
augmenter (imgaug.augmenters.Augmenter):
input_dims (Tuple): h/w of the input image
inplace (bool, default=False): if True, modifies data inplace
Example:
>>> # xdoctest: +REQUIRES(module:imgaug)
>>> from kwimage.structs.polygon import * # NOQA
>>> import imgaug
>>> input_dims = np.array((10, 10))
>>> self = Polygon.random(10, n_holes=1, rng=0).scale(input_dims)
>>> augmenter = imgaug.augmenters.Fliplr(p=1)
>>> new = self._warp_imgaug(augmenter, input_dims)
>>> assert np.allclose(self.data['exterior'].data[:, 1], new.data['exterior'].data[:, 1])
>>> assert np.allclose(input_dims[0] - self.data['exterior'].data[:, 0], new.data['exterior'].data[:, 0])
>>> # xdoc: +REQUIRES(--show)
>>> import kwplot
>>> kwplot.autompl()
>>> kwplot.figure(fnum=1, doclf=True)
>>> from matplotlib import pyplot as pl
>>> ax = plt.gca()
>>> ax.set_xlim(0, 10)
>>> ax.set_ylim(0, 10)
>>> self.draw(color='red', alpha=.4)
>>> new.draw(color='blue', alpha=.4)
"""
import kwimage
new = self if inplace else self.__class__(self.data.copy())
# current version of imgaug doesnt fully support polygons
# coerce to and from points instead
dtype = self.data['exterior'].data.dtype
parts = [self.data['exterior']] + self.data.get('interiors', [])
parts = [p.data for p in parts]
cs = [0] + np.cumsum(np.array(list(map(len, parts)))).tolist()
flat_kps = np.concatenate(parts, axis=0)
flat_coords = kwimage.Coords(flat_kps)
flat_coords = flat_coords._warp_imgaug(augmenter, input_dims, inplace=True)
flat_parts = flat_coords.data
new_parts = []
for a, b in ub.iter_window(cs, 2):
new_part = np.array(flat_parts[a:b], dtype=dtype)
new_parts.append(new_part)
new_exterior = kwimage.Coords(new_parts[0])
new_interiors = [kwimage.Coords(p) for p in new_parts[1:]]
new.data['exterior'] = new_exterior
new.data['interiors'] = new_interiors
return new
def paths_to_otree(paths):
tree = nx.OrderedDiGraph()
for path in sorted(paths):
parts = tuple(path.split(sep))
node_path = []
for i in range(1, len(parts) + 1):
node = parts[0:i]
tree.add_node(node)
tree.nodes[node]['label'] = node[-1]
node_path.append(node)
for u, v in ub.iter_window(node_path, 2):
tree.add_edge(u, v)
return tree
def demodata_tarjan_bridge():
"""
Example:
>>> from graphid import util
>>> G = demodata_tarjan_bridge()
>>> # xdoc: +REQUIRES(--show)
>>> util.show_nx(G)
>>> util.show_if_requested()
"""
# define 2-connected compoments and bridges
cc2 = [(1, 2, 4, 3, 1, 4), (5, 6, 7, 5), (8, 9, 10, 8),
(17, 18, 16, 15, 17), (11, 12, 14, 13, 11, 14)]
bridges = [(4, 8), (3, 5), (3, 17)]
G = nx.Graph(ub.flatten(ub.iter_window(path, 2) for path in cc2 + bridges))
return G
def test_convT_rf():
"""
CommandLine:
xdoctest -m ~/code/netharn/tests/test_receptive_feild.py test_convT_rf
"""
# Test that we always invert whatever weird crazy thing we do
import netharn as nh
rng = np.random.RandomState(3668028386)
ntrials = 100
M = 9
for _ in ub.ProgIter(range(ntrials), desc='testing rand convT instances'):
depth = rng.randint(1, M)
params = []
for i in range(depth):
k = rng.randint(0, 1 + M // 2) * 2 + 1
s = rng.randint(1, 1 + M)
d = rng.randint(1, 1 + M)
p = rng.randint(0, 1 + M)
params.append((i, (k, s, d)))
# Construct a series of forward convolutions and the tranpose
# convolutions that should "invert" them. Assert that the strides and
# crop of the RF are the same on every layer. Furthremote the RF size
# should strictly increase.
layers = ub.odict()
for i, (k, s, d) in params:
key = 'c{}'.format(i)
conv = nn.Conv2d(1,
1,
kernel_size=k,
stride=s,
padding=p,
dilation=d)
layers[key] = conv
for i, (k, s, d) in reversed(params):
key = 'c{}T'.format(i)
convT = nn.ConvTranspose2d(1,
1,
kernel_size=k,
stride=s,
padding=p,
dilation=d)
layers[key] = convT
module = nn.Sequential(layers)
field = nh.ReceptiveFieldFor(module)()
input_rf = nh.ReceptiveFieldFor.input()
symmetric = [('input', input_rf)] + list(field.hidden.items())
for a, b, in ub.iter_window(symmetric, 2):
k1, v1 = a
k2, v2 = b
assert np.all(v1['shape'] <= v2['shape']), 'v1={} v2={}'.format(
v1, v2)
for a, b in zip(symmetric, symmetric[::-1]):
k1, v1 = a
k2, v2 = b
assert np.all(v1['stride'] == v2['stride']), 'v1={} v2={}'.format(
v1, v2)
assert np.all(v1['crop'] == v2['crop']), 'v1={} v2={}'.format(
v1, v2)
def _fix_keys(model_state_dict):
"""
Hack around DataParallel wrapper. If there is nothing in common between
the two models check to see if prepending 'module.' to other keys fixes
it.
"""
other_keys = set(model_state_dict)
self_keys = set(self_state)
if 0:
# Automatic way to reduce nodes in the trees?
# If node b always follows node a, can we contract it?
nodes1 = [n for p in other_keys for n in p.split('.')]
nodes2 = [n for p in self_keys for n in p.split('.')]
tups1 = list(tup for key in other_keys
for tup in ub.iter_window(key.split('.'), 2))
tups2 = list(tup for key in self_keys
for tup in ub.iter_window(key.split('.'), 2))
x = ub.ddict(list)
for a, b in tups1:
x[a].append(b)
for a, b in tups2:
x[a].append(b)
nodehist = ub.dict_hist(nodes1 + nodes2)
for k, v in x.items():
print('----')
print(k)
print(nodehist[k])
follow_hist = ub.dict_hist(v)
print(follow_hist)
total = sum(follow_hist.values())
if ub.allsame(follow_hist.values()) and total == nodehist[k]:
print('CONTRACT')
# pair_freq = ub.dict_hist(ub.flatten([tups1, tups2]))
# print(forest_str(paths_to_otree(other_keys, '.')))
# common_keys = other_keys.intersection(self_keys)
# if not common_keys:
if not other_keys.issubset(self_keys):
if association == 'strict':
pass
elif association == 'module-hack':
# If there are no common keys try a hack
prefix = 'module.'
def smap(f, ss):
return set(map(f, ss))
def fix1(k):
return prefix + k
def fix2(k):
if k.startswith(prefix):
return k[len(prefix):]
if smap(fix1, other_keys).intersection(self_keys):
model_state_dict = ub.map_keys(fix1, model_state_dict)
elif smap(fix2, other_keys).intersection(self_keys):
model_state_dict = ub.map_keys(fix2, model_state_dict)
elif association == 'prefix-hack':
import functools
def add_prefix(k, prefix):
return prefix + k
def remove_prefix(k, prefix):
if k.startswith(prefix):
return k[len(prefix):]
# set1 = other_keys
# target_set2 = self_keys
found = _best_prefix_transform(other_keys, self_keys)
if found is not None:
for action, prefix in found['transform']:
if action == 'add':
func = functools.partial(add_prefix, prefix=prefix)
elif action == 'remove':
func = functools.partial(remove_prefix,
prefix=prefix)
else:
raise AssertionError
model_state_dict = ub.map_keys(func, model_state_dict)
elif association in {'embedding', 'isomorphism'}:
if verbose > 1:
print('Using subpath {} association, may take some time'.
format(association))
# I believe this is the correct way to solve the problem
paths1 = sorted(other_keys)
paths2 = sorted(self_state)
if 1:
# hack to filter to reduce tree size in embedding problem
def shrink_paths(paths):
new_paths = []
for p in paths:
p = p.replace('.0', ':0')
p = p.replace('.1', ':1')
p = p.replace('.2', ':2')
p = p.replace('.3', ':3')
p = p.replace('.4', ':4')
p = p.replace('.5', ':5')
p = p.replace('.6', ':6')
p = p.replace('.7', ':7')
p = p.replace('.8', ':8')
p = p.replace('.9', ':9')
p = p.replace('.weight', ':weight')
p = p.replace('.bias', ':bias')
p = p.replace('.num_batches_tracked',
':num_batches_tracked')
p = p.replace('.running_mean', ':running_mean')
p = p.replace('.running_var', ':running_var')
# p = p.replace('.conv1', ':conv1')
# p = p.replace('.conv2', ':conv2')
# p = p.replace('.conv3', ':conv3')
# p = p.replace('.bn1', ':bn1')
# p = p.replace('.bn2', ':bn2')
# p = p.replace('.bn3', ':bn3')
new_paths.append(p)
return new_paths
# Reducing the depth saves a lot of time
paths1_ = shrink_paths(paths1)
paths2_ = shrink_paths(paths2)
subpaths1, subpaths2 = maximum_common_ordered_subpaths(
paths1_, paths2_, sep='.', mode=association)
subpaths1 = [p.replace(':', '.') for p in subpaths1]
subpaths2 = [p.replace(':', '.') for p in subpaths2]
mapping = ub.dzip(subpaths1, subpaths2)
if verbose > 1:
other_unmapped = sorted(other_keys - set(mapping.keys()))
self_unmapped = sorted(self_keys - set(mapping.values()))
print('-- embed association (other -> self) --')
print('mapping = {}'.format(ub.repr2(mapping, nl=1)))
print('self_unmapped = {}'.format(
ub.repr2(self_unmapped, nl=1)))
print('other_unmapped = {}'.format(
ub.repr2(other_unmapped, nl=1)))
print('len(mapping) = {}'.format(
ub.repr2(len(mapping), nl=1)))
print('len(self_unmapped) = {}'.format(
ub.repr2(len(self_unmapped), nl=1)))
print('len(other_unmapped) = {}'.format(
ub.repr2(len(other_unmapped), nl=1)))
print('-- end embed association --')
# HACK: something might be wrong, there was an instance with
# HRNet_w32 where multiple keys mapped to the same key
# bad keys were incre_modules.3.0.conv1.weight and conv1.weight
#
# This will not error, but may produce bad output
try:
model_state_dict = ub.map_keys(lambda k: mapping.get(k, k),
model_state_dict)
except Exception as ex:
HACK = 1
if HACK:
new_state_dict_ = {}
for k, v in model_state_dict.items():
new_state_dict_[mapping.get(k, k)] = v
model_state_dict = new_state_dict_
warnings.warn('ex = {!r}'.format(ex))
else:
raise
else:
raise KeyError(association)
return model_state_dict
def demodata_bridge():
# define 2-connected compoments and bridges
cc2 = [(1, 2, 4, 3, 1, 4), (8, 9, 10, 8), (11, 12, 13, 11)]
bridges = [(4, 8), (3, 5), (20, 21), (22, 23, 24)]
G = nx.Graph(ub.flatten(ub.iter_window(path, 2) for path in cc2 + bridges))
return G
def _bench_put():
B, C, H, W = (32, 100, 32, 32)
idxs = [2, 3, 5, 7, 11, 13, 17, 21]
B, C, H, W = (32, 100, 64, 64)
idxs = [2, 3, 5, 7, 11, 13, 17, 21]
# B, C, H, W = (1, 7, 3, 3)
# idxs = [2, 3, 5]
dim = 1
class_energy = torch.rand(B, C, H, W)
class_energy = class_energy.to(nh.XPU.cast('auto').main_device)
ti = CudaTimerit(1000, bestof=100, verbose=1)
outputs = ub.odict()
for timer in ti.reset('put multi-index (fancy_index)'):
class_logprobs = torch.zeros_like(class_energy)
with timer:
fancy_prefix = [slice(None)] * dim
fancy_index = tuple(fancy_prefix + [idxs])
index = torch.LongTensor(idxs).to(class_energy.device)
selected = torch.index_select(class_energy, dim=dim, index=index)
class_logprobs[fancy_index] = selected
outputs[ti.label] = class_logprobs.clone()
assert not torch.all(class_logprobs[fancy_index] == 0)
for timer in ti.reset('put multi-index (loop, select)'):
class_logprobs = torch.zeros_like(class_energy)
with timer:
for idx in idxs:
class_logprobs.select(dim,
idx)[:] = class_energy.select(dim, idx)
outputs[ti.label] = class_logprobs.clone()
assert torch.all(
class_logprobs.select(dim, idx) == class_energy.select(dim, idx))
for timer in ti.reset('index-copy multi-index'):
class_logprobs = torch.zeros_like(class_energy)
with timer:
index = torch.LongTensor(idxs).to(class_energy.device)
selected = torch.index_select(class_energy, dim=dim, index=index)
class_logprobs.index_copy_(dim, index, selected)
outputs[ti.label] = class_logprobs.clone()
for k1, k2 in ub.iter_window(outputs, 2):
if torch.all(outputs[k1] == outputs[k2]):
print('MATCH: k1={} matches k2={}'.format(k1, k2))
else:
print('DIFF: k1={} DIFFERS k2={}'.format(k1, k2))
print((torch.abs(outputs[k1] - outputs[k2])).sum())
# for timer in ti.reset('put multi-index 1 (fancy_index)'):
# class_logprobs = torch.zeros_like(class_energy)
# with timer:
# fancy_prefix = [slice(None)] * dim
# fancy_index = tuple(fancy_prefix + [idxs])
# class_logprobs[fancy_index] = class_logprobs[fancy_index] + 1
# outputs[ti.label] = class_logprobs.clone()
# assert not torch.all(class_logprobs[fancy_index] == 0)
# for timer in ti.reset('put multi-index 1 (loop, select)'):
# class_logprobs = torch.zeros_like(class_energy)
# with timer:
# for idx in idxs:
# class_logprobs.select(dim, idx)[:] = class_logprobs.select(dim, idx) + 1
# assert torch.all(class_logprobs.select(dim, idx) == 1)
# outputs[ti.label] = class_logprobs.clone()
for timer in ti.reset('index-copy multi-index (just-copy)'):
class_logprobs = torch.zeros_like(class_energy)
index = torch.LongTensor(idxs).to(class_energy.device)
selected = torch.index_select(class_energy, dim=dim, index=index)
with timer:
class_logprobs.index_copy_(dim, index, selected)
outputs[ti.label] = class_logprobs.clone()
for timer in ti.reset('put multi-index (fancy_index) (just-copy)'):
class_logprobs = torch.zeros_like(class_energy)
fancy_prefix = [slice(None)] * dim
fancy_index = tuple(fancy_prefix + [idxs])
index = torch.LongTensor(idxs).to(class_energy.device)
selected = torch.index_select(class_energy, dim=dim, index=index)
with timer:
class_logprobs[fancy_index] = selected
outputs[ti.label] = class_logprobs.clone()
assert not torch.all(class_logprobs[fancy_index] == 0)
def _bench_catgraph_sink_log_softmax():
from ovharn import category_tree
sink_nodes = category_tree.sink_nodes
def sink_log_softmax_method1(self, class_energy, dim):
leaf_idxs = sorted(self.node_to_idx[node]
for node in sink_nodes(self.graph))
class_logprobs = torch.empty_like(class_energy)
fancy_prefix = [slice(None)] * dim
fancy_index = tuple(fancy_prefix + [leaf_idxs])
class_logprobs[fancy_index] = F.log_softmax(class_energy[fancy_index],
dim=dim)
@ub.memoize
def populate1(node):
""" dynamic program to compute absolute class log probability """
children = list(self.graph.successors(node))
child_idxs = sorted(self.node_to_idx[node] for node in children)
if len(children) > 0:
# Ensure that all children are populated before the parents
for child in children:
populate1(child)
node_idx = self.node_to_idx[node]
fancy_node_index = tuple(fancy_prefix + [node_idx])
fancy_children_index = tuple(fancy_prefix + [child_idxs])
class_logprobs[fancy_node_index] = torch.logsumexp(
class_logprobs[fancy_children_index], dim=dim)
for node in self.graph.nodes():
populate1(node)
return class_logprobs
def sink_log_softmax_method2(self, class_energy, dim):
class_logprobs = torch.empty_like(class_energy)
leaf_idxs = sorted(self.node_to_idx[node]
for node in sink_nodes(self.graph))
leaf_idxs = torch.LongTensor(leaf_idxs).to(class_energy.device)
leaf_energy = torch.index_select(class_energy,
dim=dim,
index=leaf_idxs)
leaf_logprobs = F.log_softmax(leaf_energy, dim=dim)
class_logprobs.index_copy_(dim, leaf_idxs, leaf_logprobs)
@ub.memoize
def populate2(node):
""" dynamic program to compute absolute class log probability """
children = list(self.graph.successors(node))
if len(children) > 0:
# Ensure that all children are populated before the parents
for child in children:
populate2(child)
child_idxs = sorted(self.node_to_idx[node]
for node in children)
child_idxs = torch.LongTensor(child_idxs).to(
class_energy.device)
node_idx = self.node_to_idx[node]
selected = torch.index_select(class_logprobs,
dim=dim,
index=child_idxs)
total = torch.logsumexp(selected, dim=dim)
class_logprobs.select(dim, node_idx)[:] = total
for node in self.graph.nodes():
populate2(node)
return class_logprobs
from ovharn import category_tree
graph = nx.generators.gnr_graph(30, 0.3, seed=321).reverse()
self = category_tree.CategoryTree(graph)
class_energy = torch.randn(16, len(self.idx_to_node), 15, 15)
class_energy = class_energy.to(nh.XPU.cast('auto').main_device)
dim = 1
ti = CudaTimerit(500, bestof=50, verbose=1, unit='us')
outputs = ub.odict()
for timer in ti.reset('method1'):
with timer:
cond_logprobs = sink_log_softmax_method1(self, class_energy, dim)
outputs[ti.label] = cond_logprobs.clone()
for timer in ti.reset('method2'):
with timer:
cond_logprobs = sink_log_softmax_method2(self, class_energy, dim)
outputs[ti.label] = cond_logprobs.clone()
for k1, k2 in ub.iter_window(outputs, 2):
if torch.all(outputs[k1] == outputs[k2]):
print('MATCH: k1={} matches k2={}'.format(k1, k2))
else:
print('DIFF: k1={} DIFFERS k2={}'.format(k1, k2))
print((torch.abs(outputs[k1] - outputs[k2])).sum())
def _bench_catgraph_conditional_log_softmax_solution():
from ovharn import category_tree
graph = nx.generators.gnr_graph(30, 0.3, seed=321).reverse()
self = category_tree.CategoryTree(graph)
class_energy = torch.randn(16, len(self.idx_to_node), 15, 15)
class_energy = class_energy.to(nh.XPU.cast('auto').main_device)
dim = 1
def method1(self, class_energy, dim):
cond_logprobs = torch.empty_like(class_energy)
# Move indexes onto the class_energy device (perhaps precache this)
index_groups = [
torch.LongTensor(idxs).to(class_energy.device)
for idxs in self.idx_groups
]
for index in index_groups:
# Take each subset of classes that are mutually exclusive
energy_group = torch.index_select(class_energy,
dim=dim,
index=index)
# Then apply the log_softmax to those sets
logprob_group = F.log_softmax(energy_group, dim=dim)
cond_logprobs.index_copy_(dim, index, logprob_group)
return cond_logprobs
def method2(self, class_energy, dim):
cond_logprobs = torch.empty_like(class_energy)
fancy_prefix = [slice(None)] * dim
for idxs in self.idx_groups:
fancy_index = tuple(fancy_prefix + [idxs])
cond_logprobs[fancy_index] = F.log_softmax(
class_energy[fancy_index], dim=dim)
return cond_logprobs
ti = CudaTimerit(500, bestof=50, verbose=1, unit='us')
outputs = ub.odict()
for timer in ti.reset('method1'):
with timer:
cond_logprobs = method1(self, class_energy, dim)
outputs[ti.label] = cond_logprobs.clone()
for timer in ti.reset('method2'):
with timer:
cond_logprobs = method2(self, class_energy, dim)
outputs[ti.label] = cond_logprobs.clone()
# ------
for k1, k2 in ub.iter_window(outputs, 2):
if torch.all(outputs[k1] == outputs[k2]):
print('MATCH: k1={} matches k2={}'.format(k1, k2))
else:
print('DIFF: k1={} DIFFERS k2={}'.format(k1, k2))
print((torch.abs(outputs[k1] - outputs[k2])).sum())
# Meausre how much overhead creating the LongTensor takes.
# Is it worth precaching? Not really, its like 1% of the time.
def method1_overhead(self, class_energy, dim):
# Move indexes onto the class_energy device (perhaps precache this)
index_groups = [
torch.LongTensor(idxs).to(class_energy.device)
for idxs in self.idx_groups
]
for index in index_groups:
pass
for timer in ti.reset('method1-overhead'):
with timer:
method1_overhead(self, class_energy, dim)
def render_facts():
"""
Render facts to a latex document
"""
import pylatex
from pylatex.base_classes.command import Options # NOQA
import pyqrcode
fact_data = load_facts()
class MDFramed(pylatex.base_classes.Environment):
_latex_name = 'mdframed'
packages = [pylatex.Package('mdframed')]
class SamePage(pylatex.base_classes.Environment):
_latex_name = 'samepage'
class ComposeContexts:
def __init__(self, *contexts):
self.contexts = contexts
def __enter__(self):
return [c.__enter__() for c in self.contexts]
def __exit__(self, a, b, c):
return [c.__exit__(a, b, c) for c in self.contexts[::-1]]
# class NewUnicodeChar(pylatex.base_classes.CommandBase):
# pass
# Dont use fontenc, lmodern, or textcomp
# https://tex.stackexchange.com/questions/179778/xelatex-under-ubuntu
doc = pylatex.Document('fact_document',
inputenc=None,
page_numbers=False,
indent=False,
fontenc=None,
lmodern=False,
textcomp=False)
doc.preamble.append(pylatex.Package('graphicx')) # For PNG images
# doc.preamble.append(pylatex.Package('svg', options=dict(inkscapearea='page')))
# doc.preamble.append(pylatex.Command('title', 'Facts'))
# doc.preamble.append(pylatex.Command('author', 'Anonymous author'))
# doc.preamble.append(pylatex.Command('date', pylatex.NoEscape(r'\today')))
# doc.append(pylatex.NoEscape(r'\maketitle'))
# doc.preamble.append(pylatex.Package('newunicodechar'))
# doc.preamble.append(pylatex.NoEscape(r'\newunicodechar{±}{$\pm$}'))
# doc.append(pylatex.NoEscape('13.787±0.020'))
# print(doc.dumps())
# doc.generate_pdf(clean_tex=False, compiler='xelatex')
# return
QR_REFERENCE = True
stop_flag = 0
image_dpath = ub.Path('~/misc/facts/images').expand().ensuredir()
# image_dpath =
for fact in ub.ProgIter(fact_data['facts']):
contexts = ComposeContexts(
# doc.create(SamePage()),
doc.create(MDFramed()),
doc.create(pylatex.MiniPage(width=r'0.99\textwidth')))
# with doc.create(pylatex.MiniPage(width=r'\textwidth')):
with contexts:
doc.append(pylatex.NoEscape(r'\paragraph{Fact:}'))
text = ub.paragraph(fact['text'])
if r'\[' in text:
found = list(
re.finditer(
'(' + re.escape(r'\[') + '|' + re.escape(r'\]') + ')',
text))
prev_x = 0
for a, b in ub.iter_window(found, step=2):
part = text[prev_x:a.span()[0]]
doc.append(part)
ax, bx = a.span()[1], b.span()[0]
part = pylatex.NoEscape(r'$' + text[ax:bx] + r'$ ')
doc.append(part)
prev_x = b.span()[1]
part = text[prev_x:]
doc.append(part)
else:
# if '$' in text:
# parts = text.split('$')
# for idx, p in enumerate(parts):
# if idx % 2 == 1:
# doc.append(pylatex.NoEscape('$' + p + '$ '))
# else:
# doc.append(p)
# else:
doc.append(text)
if QR_REFERENCE:
doc.append('\n')
num_refs = 0
for refline in fact['references'].split('\n'):
if refline.startswith('http'):
found = refline
image_fname = ub.hash_data(found,
base='abc')[0:16] + '.png'
image_fpath = image_dpath / image_fname
if not image_fpath.exists():
# pyqrcode.create(found).svg(fpath, scale=6)
pyqrcode.create(found).png(str(image_fpath),
scale=2)
doc.append(
pylatex.NoEscape(r'\includegraphics[width=90px]{' +
str(image_fpath) + '}'))
# doc.append(pylatex.NoEscape(r'\includesvg[width=120px]{' + fpath + '}'))
num_refs += 1
if num_refs > 3:
break
else:
doc.append(pylatex.NoEscape(r'\paragraph{References:}'))
with doc.create(pylatex.Itemize()) as itemize:
for refline in fact['references'].split('\n'):
if refline:
refline = refline.strip()
itemize.add_item(refline)
doc.append(pylatex.NoEscape(r'\bigskip'))
if stop_flag:
break
print(doc.dumps())
print('generate pdf')
doc.generate_pdf(str(ub.Path('~/misc/facts/fact_document').expand()),
clean_tex=True)
label_to_nodes = ub.group_items(node_to_label.keys(), node_to_label.values())
aug_graph = graph.copy()
# remove cut edges from augmented graph
edge_to_iscut = nx.get_edge_attributes(aug_graph, 'is_cut')
cut_edges = [
(u, v) for (u, v, d) in aug_graph.edges(data=True)
if not (d.get('is_cut') or d.get('decision', 'unreviewed') in ['nomatch'])
]
cut_edges = [edge for edge, flag in edge_to_iscut.items() if flag]
aug_graph.remove_edges_from(cut_edges)
# Enumerate cliques inside labels
unflat_edges = [
list(ub.iter_window(nodes, 2)) for nodes in label_to_nodes.values()
]
node_pairs = [tup for tup in ub.flatten(unflat_edges) if tup[0] != tup[1]]
# Remove candidate MST edges that exist in the original graph
orig_edges = list(aug_graph.edges())
candidate_mst_edges = [
edge for edge in node_pairs if not aug_graph.has_edge(*edge)
]
# randomness prevents chains and visually looks better
rng = np.random.RandomState(42)
def _randint():
return 0
return rng.randint(0, 100)
def run_pvpoke_ultra_experiment():
"""
https://pvpoke.com/battle/matrix/
!pip install selenium
"""
"""
Relevant page items:
<button class="add-poke-btn button">+ Add Pokemon</button>
'//*[@id="main"]/div[3]/div[3]/div/div[1]/button[1]'
'/html/body/div[1]/div/div[3]/div[3]/div/div[1]/button[1]'
<input class="poke-search" type="text" placeholder="Search name">
/html/body/div[5]/div/div[3]/div[1]/input
/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/a/span[1]
Level Cap
/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[2]/div[2]/div[5]
# IV GROUP
ivs-group
save-poke
import sys, ubelt
sys.path.append(ubelt.expandpath('~/code/pypogo'))
from pypogo.pvpoke_experiment import * # NOQA
from pypogo.pvpoke_experiment import _oldstuff
"""
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
import ubelt as ub
import os
import pathlib
import time
import pandas as pd
import pypogo
# Requires the driver be in the PATH
fpath = ensure_selenium_chromedriver()
os.environ['PATH'] = os.pathsep.join(
ub.oset(os.environ['PATH'].split(os.pathsep))
| ub.oset([str(fpath.parent)]))
url = 'https://pvpoke.com/battle/matrix/'
# chrome_exe = ub.find_exe("google-chrome")
driver = webdriver.Chrome()
driver.get(url)
league = 'Great'
# league = 'Master40'
if league == 'Great':
league_box_target = 'Great League (CP 1500)'
have_ivs = list(
ub.oset([
tuple([int(x) for x in p.strip().split(',') if x])
for p in ub.codeblock('''
10, 10, 12,
10, 12, 14,
10, 12, 14,
10, 13, 10,
10, 13, 12,
10, 14, 14,
11, 12, 14,
11, 14, 12,
11, 14, 15,
11, 15, 11,
11, 15, 11,
11, 15, 12,
11, 15, 12,
12, 10, 12,
12, 11, 12,
12, 12, 15,
12, 14, 11,
12, 14, 15,
12, 15, 11,
12, 15, 12
12, 15, 12,
13, 11, 13
13, 12, 10
13, 12, 13,
13, 13, 10,
13, 13, 11,
13, 15, 10,
13, 15, 11,
13, 15, 11,
14, 10, 12,
14, 11, 10,
14, 11, 10,
14, 13, 11
14, 13, 14,
15, 10, 12
15, 11, 10,
15, 11, 11,
15, 12, 11
''').split('\n')
]))
to_check_mons = [
pypogo.Pokemon('Deoxys',
form='defense',
ivs=ivs,
moves=['Counter', 'Rock Slide',
'Psycho Boost']).maximize(1500)
for ivs in have_ivs
]
meta_text = 'Great League Meta'
elif league == 'Master40':
league_box_target = 'Master League (Level 40)'
meta_text = 'Master League Meta'
# Test the effect of best buddies vs the master league
to_check_mons = [
pypogo.Pokemon('Mewtwo', ivs=[15, 15, 15], level=40),
pypogo.Pokemon('Mewtwo', ivs=[15, 15, 15], level=41),
pypogo.Pokemon('Garchomp', ivs=[15, 15, 15], level=40),
pypogo.Pokemon('Garchomp', ivs=[15, 15, 15], level=41),
pypogo.Pokemon('Dragonite', ivs=[15, 14, 15], level=40),
pypogo.Pokemon('Dragonite', ivs=[15, 14, 15], level=41),
pypogo.Pokemon('Giratina',
form='origin',
ivs=[15, 14, 15],
level=40),
pypogo.Pokemon('Giratina',
form='origin',
ivs=[15, 14, 15],
level=41),
pypogo.Pokemon('Kyogre', ivs=[15, 15, 14], level=40),
pypogo.Pokemon('Kyogre', ivs=[15, 15, 14], level=41),
pypogo.Pokemon('Groudon', ivs=[14, 14, 13], level=40),
pypogo.Pokemon('Groudon', ivs=[14, 14, 13], level=41),
pypogo.Pokemon('Togekiss', ivs=[15, 15, 14], level=40),
pypogo.Pokemon('Togekiss', ivs=[15, 15, 14], level=41),
]
for mon in to_check_mons:
mon.populate_all()
else:
pass
leage_select = driver.find_elements_by_class_name('league-select')[0]
leage_select.click()
leage_select.send_keys(league_box_target)
leage_select.click()
leage_select.text.split('\n')
leage_select.send_keys('\n')
leage_select.send_keys('\n')
def add_pokemon(mon):
add_poke1_button = driver.find_elements_by_class_name(
'add-poke-btn')[0]
add_poke1_button.click()
select_drop = driver.find_element_by_xpath(
'/html/body/div[5]/div/div[3]/div[1]/select')
if 1:
import xdev
all_names = select_drop.text.split('\n')
distances = xdev.edit_distance(mon.display_name(), all_names)
chosen_name = all_names[ub.argmin(distances)]
else:
chosen_name = mon.name
search_box = driver.find_element_by_xpath(
'/html/body/div[5]/div/div[3]/div[1]/input')
search_box.send_keys(chosen_name)
advanced_ivs_arrow = driver.find_element_by_xpath(
'/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/a/span[1]')
advanced_ivs_arrow.click()
level40_cap = driver.find_element_by_xpath(
'/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[2]/div[2]/div[2]'
)
level41_cap = driver.find_element_by_xpath(
'/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[2]/div[2]/div[3]'
)
level50_cap = driver.find_element_by_xpath(
'/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[2]/div[2]/div[4]'
)
level51_cap = driver.find_element_by_xpath(
'/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[2]/div[2]/div[5]'
)
if mon.level >= 51:
level51_cap.click()
elif mon.level >= 50:
level50_cap.click()
elif mon.level >= 41:
level41_cap.click()
elif mon.level >= 40:
level40_cap.click()
level_box = driver.find_element_by_xpath(
'/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[1]/input'
)
level_box.click()
level_box.clear()
level_box.clear()
level_box.send_keys(str(mon.level))
iv_a = driver.find_element_by_xpath(
'/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[1]/div/input[1]'
)
iv_d = driver.find_element_by_xpath(
'/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[1]/div/input[2]'
)
iv_s = driver.find_element_by_xpath(
'/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[1]/div/input[3]'
)
# TODO
# driver.find_elements_by_class_name('move-select')
iv_a.clear()
iv_a.send_keys(str(mon.ivs[0]))
iv_d.clear()
iv_d.send_keys(str(mon.ivs[1]))
iv_s.clear()
iv_s.send_keys(str(mon.ivs[2]))
# USE_MOVES = 1
if mon.moves is not None:
# mon.populate_all()
fast_select = driver.find_element_by_xpath(
'/html/body/div[5]/div/div[3]/div[1]/div[2]/div[10]/select[1]')
fast_select.click()
fast_select.send_keys(mon.pvp_fast_move['name'])
fast_select.send_keys(Keys.ENTER)
charge1_select = driver.find_element_by_xpath(
'/html/body/div[5]/div/div[3]/div[1]/div[2]/div[10]/select[2]')
charge1_select.click()
charge1_select.send_keys(mon.pvp_charge_moves[0]['name'])
charge1_select.send_keys(Keys.ENTER)
charge2_select = driver.find_element_by_xpath(
'/html/body/div[5]/div/div[3]/div[1]/div[2]/div[10]/select[3]')
charge2_select.click()
charge2_select.send_keys(mon.pvp_charge_moves[1]['name'])
charge2_select.send_keys(Keys.ENTER)
save_button = driver.find_elements_by_class_name('save-poke')[0]
save_button.click()
quickfills = driver.find_elements_by_class_name('quick-fill-select')
quickfill = quickfills[1]
quickfill.text.split('\n')
quickfill.click()
quickfill.send_keys(meta_text)
quickfill.click()
import pypogo
# mon1 = pypogo.Pokemon('Mewtwo', ivs=[15, 15, 15], level=40)
# mon2 = pypogo.Pokemon('Mewtwo', ivs=[15, 15, 15], level=41)
if 1:
for mon in to_check_mons:
pass
add_pokemon(mon)
shield_selectors = driver.find_elements_by_class_name('shield-select')
shield_selectors[2].click()
shield_selectors[2].send_keys('No shields')
shield_selectors[2].send_keys(Keys.ENTER)
shield_selectors[3].click()
shield_selectors[3].send_keys('No shields')
shield_selectors[3].send_keys(Keys.ENTER)
shield_selectors[0].click()
battle_btn = driver.find_elements_by_class_name('battle-btn')[0]
battle_btn.click()
# Clear previous downloaded files
dlfolder = pathlib.Path(ub.expandpath('$HOME/Downloads'))
for old_fpath in list(dlfolder.glob('_vs*.csv')):
old_fpath.unlink()
time.sleep(2.0)
# Download new data
dl_btn = driver.find_element_by_xpath(
'//*[@id="main"]/div[4]/div[9]/div/a')
dl_btn.click()
while len(list(dlfolder.glob('_vs*.csv'))) < 1:
pass
new_fpaths = list(dlfolder.glob('_vs*.csv'))
assert len(new_fpaths) == 1
fpath = new_fpaths[0]
data = pd.read_csv(fpath, header=0, index_col=0)
if 1:
# GROUP ANALYSIS
data.sum(axis=1).sort_values()
(data > 500).sum(axis=1).sort_values()
flipped = []
for key, col in data.T.iterrows():
if not ub.allsame(col > 500):
flipped.append(key)
flip_df = data.loc[:, flipped]
def color(x):
if x > 500:
return ub.color_text(str(x), 'green')
else:
return ub.color_text(str(x), 'red')
print(flip_df.applymap(color))
print(flip_df.columns.tolist())
(data > 500)
else:
# PAIR ANALYSIS
pairs = list(ub.iter_window(range(len(data)), step=2))
for i, j in pairs:
print('-----')
matchup0 = data.iloc[i]
matchup1 = data.iloc[j]
delta = matchup1 - matchup0
print(delta[delta != 0])
wins0 = matchup0 > 500
wins1 = matchup1 > 500
flips = (wins0 != wins1)
flipped_vs = matchup0.index[flips]
num_flips = sum(flips)
print('flipped_vs = {!r}'.format(flipped_vs))
print('num_flips = {!r}'.format(num_flips))
print(matchup0.mean())
print(matchup1.mean())
print(matchup1.mean() / matchup0.mean())
def main():
import kwplot
plt = kwplot.autoplt()
sns = kwplot.autosns()
alias = {
'3090': 'nvctrl GeForce GTX 1080 Ti 1 temp',
'1080ti': 'nvctrl GeForce RTX 3090 0 temp',
# 'cpu': 'lmsensor coretemp-isa-0000 Package id 0',
}
all_df = read_psensor_log()
unique_rawdevs = all_df.device.unique()
for rawdev in unique_rawdevs:
cpu_prefix = 'lmsensor coretemp-isa'
if rawdev.startswith(cpu_prefix):
suffix = rawdev[len(cpu_prefix):].split(' ', 1)[1].strip()
alias['CPU ' + suffix] = rawdev
if 'nvctrl' in rawdev and 'temp' in rawdev:
alias['GPU ' + rawdev[7:-5]] = rawdev
mapper = ub.invert_dict(alias)
all_df['device'] = all_df['device'].apply(lambda x: mapper.get(x, None))
all_df = all_df[all_df['device'].apply(lambda x: x is not None)]
hours = int(ub.argval('--hours', default=48))
delta = datetime.timedelta(hours=hours)
min_time = datetime.datetime.now() - delta
is_recent = all_df.datetime > min_time
recent_df = all_df[is_recent]
chosen = recent_df
# chosen = all_df
if 0:
pivtbl = recent_df.pivot('unix_timestamp', 'device', 'temp')
pivtbl = pivtbl.sort_index()
smoothed_rows = []
for window_idxs in ub.iter_window(list(range(len(pivtbl))), size=10):
window = pivtbl.iloc[list(window_idxs)]
max_val = window.max(axis=0, skipna=True)
for k, v in max_val.to_dict().items():
smoothed_rows.append({
'unix_timestamp': window.index[1],
'device': k,
'temp': v,
})
max_extra = pd.DataFrame(smoothed_rows)
sns.lineplot(data=max_extra,
x='unix_timestamp',
y='temp',
hue='device')
df = recent_df.copy()
df['device'] = df['device'].apply(lambda x: 'Core'
if x.startswith('Core') else x)
df['time'] = df['unix_timestamp'].apply(
datetime.datetime.fromtimestamp)
plt.gcf().clf()
# sns.lineplot(data=chosen, x='unix_timestamp', y='temp', hue='device')
for xx, (sess, group) in enumerate(chosen.groupby('session_x')):
# ax.cla()
ax = plt.gca()
sns.lineplot(data=group,
x='unix_timestamp',
y='temp',
hue='device',
legend=xx == 0)
label_xaxis_dates(ax)
ax.figure.subplots_adjust(bottom=0.2)
ax.set_ylim(0, 100)
plt.locator_params(axis='y', nbins=10)
# import matplotlib as mpl
# Draw shutdown time as black lines
end_times = []
for sx, group in chosen.groupby('session_x'):
shutdown_time = group['unix_timestamp'].max()
end_times.append(shutdown_time)
for shutdown_time in sorted(end_times)[:-1]:
ax.plot((shutdown_time, shutdown_time), [0, 100], color='k')
# ci_df = pd.concat([max_extra, recent_df])
# ci_df['device'] = ci_df['device'].apply(lambda x: 'Core' if x.startswith('Core') else x)
# sns.lineplot(data=ci_df, x='unix_timestamp', y='temp', hue='device')
# from matplotlib.dates import date2num
# all_df['date_ord'] = all_df['datetime'].map(lambda a: date2num(a))
# sns.lineplot(data=pt)
# sns.lineplot(data=recent_df, x='unix_timestamp', y='temp', hue='device')
# sns.regplot(data=recent_df, x='unix_timestamp', y='temp', hue='device')
plt.show()
def do_tags(verbose=True, inplace=False, dry=True, auto_rollback=False):
if verbose:
if dry:
print('squashing streaks (DRY RUN)')
else:
print('squashing streaks')
# print('authors = {!r}'.format(authors))
# If you are in a repo subdirectory, find the repo root
cwd = os.getcwd()
repodir = cwd
while True:
if os.path.exists(os.path.join(repodir, '.git')):
break
newpath = os.path.dirname(repodir)
if newpath == repodir:
raise git.exc.InvalidGitRepositoryError(cwd)
repodir = newpath
repo = git.Repo(repodir)
orig_branch_name = repo.active_branch.name
# head = repo.commit('HEAD')
info = ub.cmd('git tag -l --sort=v:refname', verbose=3)
info2 = ub.cmd('git show-ref --tags', verbose=3)
tag_to_hash = {}
for line in info2['out'].splitlines():
if line:
hashtext, tags = line.split(' ')
tag = tags.replace('refs/tags/', '')
tag_to_hash[tag] = hashtext
print('tag_to_hash = {!r}'.format(tag_to_hash))
tag_order = [line for line in info['out'].splitlines() if line]
custom_streaks = list(ub.iter_window(tag_order, 2))
print('Forcing hacked steaks')
print('custom_streaks = {!r}'.format(custom_streaks))
streaks = []
for custom_streak in custom_streaks:
print('custom_streak = {!r}'.format(custom_streak))
assert len(custom_streak) == 2
hash_a = tag_to_hash[custom_streak[0]]
hash_b = tag_to_hash[custom_streak[1]]
a = repo.commit(hash_a)
b = repo.commit(hash_b)
if repo.is_ancestor(ancestor_rev=a, rev=b):
a, b = b, a
# assert repo.is_ancestor(ancestor_rev=b, rev=a)
streak = Streak(a, _streak=[a, b])
if len(streak.start.parents) != 1:
print('WARNING: cannot include streak = {!r}'.format(streak))
continue
# assert start.authored_datetime < stop.authored_datetime
if not repo.is_ancestor(ancestor_rev=streak.start, rev=streak.stop):
print('WARNING: cannot include streak = {!r}'.format(streak))
continue
# raise AssertionError('cant handle')
streaks.append(streak)
if verbose:
print('Found {!r} streaks'.format(len(streaks)))
# Switch to a temp branch before we start working
if not dry:
temp_branchname = checkout_temporary_branch(repo, '-squash-temp')
else:
temp_branchname = None
try:
for streak in ub.ProgIter(streaks, 'squashing', verbose=3 * verbose):
if verbose:
print('Squashing streak = %r' % (str(streak),))
# Start is the commit further back in time
_squash_between(repo, streak.start, streak.stop, dry=dry,
verbose=verbose)
except Exception as ex:
print_exc(sys.exc_info())
print('ERROR: squash_streaks failed.')
if not dry and auto_rollback:
print('ROLLING BACK')
repo.git.checkout(orig_branch_name)
# repo.git.branch(D=temp_branchname)
print('You can debug the difference with:')
print(' gitk {} {}'.format(orig_branch_name, temp_branchname))
return
if dry:
if verbose:
print('Finished. did nothing')
elif inplace:
# Copy temp branch back over original
repo.git.checkout(orig_branch_name)
repo.git.reset(temp_branchname, hard=True)
repo.git.branch(D=temp_branchname)
if verbose:
print('Finished. Now you should force push the branch back to the server')
else:
# Go back to the original branch
repo.git.checkout(orig_branch_name)
if verbose:
print('Finished')
print('The squashed branch is: {}'.format(temp_branchname))
print('You can inspect the difference with:')
print(' gitk {} {}'.format(orig_branch_name, temp_branchname))
print('Finished. Now you must manually clean this branch up.')
print('Or, to automatically accept changes run with --inplace')
