def demodata_infr2(defaultdb='PZ_MTEST'): import ibeis from graphid.core.annot_inference import AnnotInference defaultdb = 'PZ_MTEST' ibs = ibeis.opendb(defaultdb=defaultdb) annots = ibs.annots() names = list(annots.group_items(annots.nids).values())[0:20] def dummy_phi(c, n): x = np.arange(n) phi = c * x / (c * x + 1) phi = phi / phi.sum() phi = np.diff(phi) return phi phis = {c: dummy_phi(c, 30) for c in range(1, 4)} aids = list(ub.flatten(names)) infr = AnnotInference(ibs, aids, autoinit=True) infr.init_termination_criteria(phis) infr.init_refresh_criteria() # Partially review n1, n2, n3, n4 = names[0:4] for name in names[4:]: for a, b in ub.iter_window(name.aids, 2): infr.add_feedback((a, b), POSTV) for name1, name2 in it.combinations(names[4:], 2): infr.add_feedback((name1.aids[0], name2.aids[0]), NEGTV) return infr
def _warp_imgaug(self, augmenter, input_dims, inplace=False): """ Warps by applying an augmenter from the imgaug library Args: augmenter (imgaug.augmenters.Augmenter): input_dims (Tuple): h/w of the input image inplace (bool, default=False): if True, modifies data inplace Example: >>> # xdoctest: +REQUIRES(module:imgaug) >>> from kwimage.structs.polygon import * # NOQA >>> import imgaug >>> input_dims = np.array((10, 10)) >>> self = Polygon.random(10, n_holes=1, rng=0).scale(input_dims) >>> augmenter = imgaug.augmenters.Fliplr(p=1) >>> new = self._warp_imgaug(augmenter, input_dims) >>> assert np.allclose(self.data['exterior'].data[:, 1], new.data['exterior'].data[:, 1]) >>> assert np.allclose(input_dims[0] - self.data['exterior'].data[:, 0], new.data['exterior'].data[:, 0]) >>> # xdoc: +REQUIRES(--show) >>> import kwplot >>> kwplot.autompl() >>> kwplot.figure(fnum=1, doclf=True) >>> from matplotlib import pyplot as pl >>> ax = plt.gca() >>> ax.set_xlim(0, 10) >>> ax.set_ylim(0, 10) >>> self.draw(color='red', alpha=.4) >>> new.draw(color='blue', alpha=.4) """ import kwimage new = self if inplace else self.__class__(self.data.copy()) # current version of imgaug doesnt fully support polygons # coerce to and from points instead dtype = self.data['exterior'].data.dtype parts = [self.data['exterior']] + self.data.get('interiors', []) parts = [p.data for p in parts] cs = [0] + np.cumsum(np.array(list(map(len, parts)))).tolist() flat_kps = np.concatenate(parts, axis=0) flat_coords = kwimage.Coords(flat_kps) flat_coords = flat_coords._warp_imgaug(augmenter, input_dims, inplace=True) flat_parts = flat_coords.data new_parts = [] for a, b in ub.iter_window(cs, 2): new_part = np.array(flat_parts[a:b], dtype=dtype) new_parts.append(new_part) new_exterior = kwimage.Coords(new_parts[0]) new_interiors = [kwimage.Coords(p) for p in new_parts[1:]] new.data['exterior'] = new_exterior new.data['interiors'] = new_interiors return new
def paths_to_otree(paths): tree = nx.OrderedDiGraph() for path in sorted(paths): parts = tuple(path.split(sep)) node_path = [] for i in range(1, len(parts) + 1): node = parts[0:i] tree.add_node(node) tree.nodes[node]['label'] = node[-1] node_path.append(node) for u, v in ub.iter_window(node_path, 2): tree.add_edge(u, v) return tree
def demodata_tarjan_bridge(): """ Example: >>> from graphid import util >>> G = demodata_tarjan_bridge() >>> # xdoc: +REQUIRES(--show) >>> util.show_nx(G) >>> util.show_if_requested() """ # define 2-connected compoments and bridges cc2 = [(1, 2, 4, 3, 1, 4), (5, 6, 7, 5), (8, 9, 10, 8), (17, 18, 16, 15, 17), (11, 12, 14, 13, 11, 14)] bridges = [(4, 8), (3, 5), (3, 17)] G = nx.Graph(ub.flatten(ub.iter_window(path, 2) for path in cc2 + bridges)) return G
def test_convT_rf(): """ CommandLine: xdoctest -m ~/code/netharn/tests/test_receptive_feild.py test_convT_rf """ # Test that we always invert whatever weird crazy thing we do import netharn as nh rng = np.random.RandomState(3668028386) ntrials = 100 M = 9 for _ in ub.ProgIter(range(ntrials), desc='testing rand convT instances'): depth = rng.randint(1, M) params = [] for i in range(depth): k = rng.randint(0, 1 + M // 2) * 2 + 1 s = rng.randint(1, 1 + M) d = rng.randint(1, 1 + M) p = rng.randint(0, 1 + M) params.append((i, (k, s, d))) # Construct a series of forward convolutions and the tranpose # convolutions that should "invert" them. Assert that the strides and # crop of the RF are the same on every layer. Furthremote the RF size # should strictly increase. layers = ub.odict() for i, (k, s, d) in params: key = 'c{}'.format(i) conv = nn.Conv2d(1, 1, kernel_size=k, stride=s, padding=p, dilation=d) layers[key] = conv for i, (k, s, d) in reversed(params): key = 'c{}T'.format(i) convT = nn.ConvTranspose2d(1, 1, kernel_size=k, stride=s, padding=p, dilation=d) layers[key] = convT module = nn.Sequential(layers) field = nh.ReceptiveFieldFor(module)() input_rf = nh.ReceptiveFieldFor.input() symmetric = [('input', input_rf)] + list(field.hidden.items()) for a, b, in ub.iter_window(symmetric, 2): k1, v1 = a k2, v2 = b assert np.all(v1['shape'] <= v2['shape']), 'v1={} v2={}'.format( v1, v2) for a, b in zip(symmetric, symmetric[::-1]): k1, v1 = a k2, v2 = b assert np.all(v1['stride'] == v2['stride']), 'v1={} v2={}'.format( v1, v2) assert np.all(v1['crop'] == v2['crop']), 'v1={} v2={}'.format( v1, v2)
def _fix_keys(model_state_dict): """ Hack around DataParallel wrapper. If there is nothing in common between the two models check to see if prepending 'module.' to other keys fixes it. """ other_keys = set(model_state_dict) self_keys = set(self_state) if 0: # Automatic way to reduce nodes in the trees? # If node b always follows node a, can we contract it? nodes1 = [n for p in other_keys for n in p.split('.')] nodes2 = [n for p in self_keys for n in p.split('.')] tups1 = list(tup for key in other_keys for tup in ub.iter_window(key.split('.'), 2)) tups2 = list(tup for key in self_keys for tup in ub.iter_window(key.split('.'), 2)) x = ub.ddict(list) for a, b in tups1: x[a].append(b) for a, b in tups2: x[a].append(b) nodehist = ub.dict_hist(nodes1 + nodes2) for k, v in x.items(): print('----') print(k) print(nodehist[k]) follow_hist = ub.dict_hist(v) print(follow_hist) total = sum(follow_hist.values()) if ub.allsame(follow_hist.values()) and total == nodehist[k]: print('CONTRACT') # pair_freq = ub.dict_hist(ub.flatten([tups1, tups2])) # print(forest_str(paths_to_otree(other_keys, '.'))) # common_keys = other_keys.intersection(self_keys) # if not common_keys: if not other_keys.issubset(self_keys): if association == 'strict': pass elif association == 'module-hack': # If there are no common keys try a hack prefix = 'module.' def smap(f, ss): return set(map(f, ss)) def fix1(k): return prefix + k def fix2(k): if k.startswith(prefix): return k[len(prefix):] if smap(fix1, other_keys).intersection(self_keys): model_state_dict = ub.map_keys(fix1, model_state_dict) elif smap(fix2, other_keys).intersection(self_keys): model_state_dict = ub.map_keys(fix2, model_state_dict) elif association == 'prefix-hack': import functools def add_prefix(k, prefix): return prefix + k def remove_prefix(k, prefix): if k.startswith(prefix): return k[len(prefix):] # set1 = other_keys # target_set2 = self_keys found = _best_prefix_transform(other_keys, self_keys) if found is not None: for action, prefix in found['transform']: if action == 'add': func = functools.partial(add_prefix, prefix=prefix) elif action == 'remove': func = functools.partial(remove_prefix, prefix=prefix) else: raise AssertionError model_state_dict = ub.map_keys(func, model_state_dict) elif association in {'embedding', 'isomorphism'}: if verbose > 1: print('Using subpath {} association, may take some time'. format(association)) # I believe this is the correct way to solve the problem paths1 = sorted(other_keys) paths2 = sorted(self_state) if 1: # hack to filter to reduce tree size in embedding problem def shrink_paths(paths): new_paths = [] for p in paths: p = p.replace('.0', ':0') p = p.replace('.1', ':1') p = p.replace('.2', ':2') p = p.replace('.3', ':3') p = p.replace('.4', ':4') p = p.replace('.5', ':5') p = p.replace('.6', ':6') p = p.replace('.7', ':7') p = p.replace('.8', ':8') p = p.replace('.9', ':9') p = p.replace('.weight', ':weight') p = p.replace('.bias', ':bias') p = p.replace('.num_batches_tracked', ':num_batches_tracked') p = p.replace('.running_mean', ':running_mean') p = p.replace('.running_var', ':running_var') # p = p.replace('.conv1', ':conv1') # p = p.replace('.conv2', ':conv2') # p = p.replace('.conv3', ':conv3') # p = p.replace('.bn1', ':bn1') # p = p.replace('.bn2', ':bn2') # p = p.replace('.bn3', ':bn3') new_paths.append(p) return new_paths # Reducing the depth saves a lot of time paths1_ = shrink_paths(paths1) paths2_ = shrink_paths(paths2) subpaths1, subpaths2 = maximum_common_ordered_subpaths( paths1_, paths2_, sep='.', mode=association) subpaths1 = [p.replace(':', '.') for p in subpaths1] subpaths2 = [p.replace(':', '.') for p in subpaths2] mapping = ub.dzip(subpaths1, subpaths2) if verbose > 1: other_unmapped = sorted(other_keys - set(mapping.keys())) self_unmapped = sorted(self_keys - set(mapping.values())) print('-- embed association (other -> self) --') print('mapping = {}'.format(ub.repr2(mapping, nl=1))) print('self_unmapped = {}'.format( ub.repr2(self_unmapped, nl=1))) print('other_unmapped = {}'.format( ub.repr2(other_unmapped, nl=1))) print('len(mapping) = {}'.format( ub.repr2(len(mapping), nl=1))) print('len(self_unmapped) = {}'.format( ub.repr2(len(self_unmapped), nl=1))) print('len(other_unmapped) = {}'.format( ub.repr2(len(other_unmapped), nl=1))) print('-- end embed association --') # HACK: something might be wrong, there was an instance with # HRNet_w32 where multiple keys mapped to the same key # bad keys were incre_modules.3.0.conv1.weight and conv1.weight # # This will not error, but may produce bad output try: model_state_dict = ub.map_keys(lambda k: mapping.get(k, k), model_state_dict) except Exception as ex: HACK = 1 if HACK: new_state_dict_ = {} for k, v in model_state_dict.items(): new_state_dict_[mapping.get(k, k)] = v model_state_dict = new_state_dict_ warnings.warn('ex = {!r}'.format(ex)) else: raise else: raise KeyError(association) return model_state_dict
def demodata_bridge(): # define 2-connected compoments and bridges cc2 = [(1, 2, 4, 3, 1, 4), (8, 9, 10, 8), (11, 12, 13, 11)] bridges = [(4, 8), (3, 5), (20, 21), (22, 23, 24)] G = nx.Graph(ub.flatten(ub.iter_window(path, 2) for path in cc2 + bridges)) return G
def _bench_put(): B, C, H, W = (32, 100, 32, 32) idxs = [2, 3, 5, 7, 11, 13, 17, 21] B, C, H, W = (32, 100, 64, 64) idxs = [2, 3, 5, 7, 11, 13, 17, 21] # B, C, H, W = (1, 7, 3, 3) # idxs = [2, 3, 5] dim = 1 class_energy = torch.rand(B, C, H, W) class_energy = class_energy.to(nh.XPU.cast('auto').main_device) ti = CudaTimerit(1000, bestof=100, verbose=1) outputs = ub.odict() for timer in ti.reset('put multi-index (fancy_index)'): class_logprobs = torch.zeros_like(class_energy) with timer: fancy_prefix = [slice(None)] * dim fancy_index = tuple(fancy_prefix + [idxs]) index = torch.LongTensor(idxs).to(class_energy.device) selected = torch.index_select(class_energy, dim=dim, index=index) class_logprobs[fancy_index] = selected outputs[ti.label] = class_logprobs.clone() assert not torch.all(class_logprobs[fancy_index] == 0) for timer in ti.reset('put multi-index (loop, select)'): class_logprobs = torch.zeros_like(class_energy) with timer: for idx in idxs: class_logprobs.select(dim, idx)[:] = class_energy.select(dim, idx) outputs[ti.label] = class_logprobs.clone() assert torch.all( class_logprobs.select(dim, idx) == class_energy.select(dim, idx)) for timer in ti.reset('index-copy multi-index'): class_logprobs = torch.zeros_like(class_energy) with timer: index = torch.LongTensor(idxs).to(class_energy.device) selected = torch.index_select(class_energy, dim=dim, index=index) class_logprobs.index_copy_(dim, index, selected) outputs[ti.label] = class_logprobs.clone() for k1, k2 in ub.iter_window(outputs, 2): if torch.all(outputs[k1] == outputs[k2]): print('MATCH: k1={} matches k2={}'.format(k1, k2)) else: print('DIFF: k1={} DIFFERS k2={}'.format(k1, k2)) print((torch.abs(outputs[k1] - outputs[k2])).sum()) # for timer in ti.reset('put multi-index 1 (fancy_index)'): # class_logprobs = torch.zeros_like(class_energy) # with timer: # fancy_prefix = [slice(None)] * dim # fancy_index = tuple(fancy_prefix + [idxs]) # class_logprobs[fancy_index] = class_logprobs[fancy_index] + 1 # outputs[ti.label] = class_logprobs.clone() # assert not torch.all(class_logprobs[fancy_index] == 0) # for timer in ti.reset('put multi-index 1 (loop, select)'): # class_logprobs = torch.zeros_like(class_energy) # with timer: # for idx in idxs: # class_logprobs.select(dim, idx)[:] = class_logprobs.select(dim, idx) + 1 # assert torch.all(class_logprobs.select(dim, idx) == 1) # outputs[ti.label] = class_logprobs.clone() for timer in ti.reset('index-copy multi-index (just-copy)'): class_logprobs = torch.zeros_like(class_energy) index = torch.LongTensor(idxs).to(class_energy.device) selected = torch.index_select(class_energy, dim=dim, index=index) with timer: class_logprobs.index_copy_(dim, index, selected) outputs[ti.label] = class_logprobs.clone() for timer in ti.reset('put multi-index (fancy_index) (just-copy)'): class_logprobs = torch.zeros_like(class_energy) fancy_prefix = [slice(None)] * dim fancy_index = tuple(fancy_prefix + [idxs]) index = torch.LongTensor(idxs).to(class_energy.device) selected = torch.index_select(class_energy, dim=dim, index=index) with timer: class_logprobs[fancy_index] = selected outputs[ti.label] = class_logprobs.clone() assert not torch.all(class_logprobs[fancy_index] == 0)
def _bench_catgraph_sink_log_softmax(): from ovharn import category_tree sink_nodes = category_tree.sink_nodes def sink_log_softmax_method1(self, class_energy, dim): leaf_idxs = sorted(self.node_to_idx[node] for node in sink_nodes(self.graph)) class_logprobs = torch.empty_like(class_energy) fancy_prefix = [slice(None)] * dim fancy_index = tuple(fancy_prefix + [leaf_idxs]) class_logprobs[fancy_index] = F.log_softmax(class_energy[fancy_index], dim=dim) @ub.memoize def populate1(node): """ dynamic program to compute absolute class log probability """ children = list(self.graph.successors(node)) child_idxs = sorted(self.node_to_idx[node] for node in children) if len(children) > 0: # Ensure that all children are populated before the parents for child in children: populate1(child) node_idx = self.node_to_idx[node] fancy_node_index = tuple(fancy_prefix + [node_idx]) fancy_children_index = tuple(fancy_prefix + [child_idxs]) class_logprobs[fancy_node_index] = torch.logsumexp( class_logprobs[fancy_children_index], dim=dim) for node in self.graph.nodes(): populate1(node) return class_logprobs def sink_log_softmax_method2(self, class_energy, dim): class_logprobs = torch.empty_like(class_energy) leaf_idxs = sorted(self.node_to_idx[node] for node in sink_nodes(self.graph)) leaf_idxs = torch.LongTensor(leaf_idxs).to(class_energy.device) leaf_energy = torch.index_select(class_energy, dim=dim, index=leaf_idxs) leaf_logprobs = F.log_softmax(leaf_energy, dim=dim) class_logprobs.index_copy_(dim, leaf_idxs, leaf_logprobs) @ub.memoize def populate2(node): """ dynamic program to compute absolute class log probability """ children = list(self.graph.successors(node)) if len(children) > 0: # Ensure that all children are populated before the parents for child in children: populate2(child) child_idxs = sorted(self.node_to_idx[node] for node in children) child_idxs = torch.LongTensor(child_idxs).to( class_energy.device) node_idx = self.node_to_idx[node] selected = torch.index_select(class_logprobs, dim=dim, index=child_idxs) total = torch.logsumexp(selected, dim=dim) class_logprobs.select(dim, node_idx)[:] = total for node in self.graph.nodes(): populate2(node) return class_logprobs from ovharn import category_tree graph = nx.generators.gnr_graph(30, 0.3, seed=321).reverse() self = category_tree.CategoryTree(graph) class_energy = torch.randn(16, len(self.idx_to_node), 15, 15) class_energy = class_energy.to(nh.XPU.cast('auto').main_device) dim = 1 ti = CudaTimerit(500, bestof=50, verbose=1, unit='us') outputs = ub.odict() for timer in ti.reset('method1'): with timer: cond_logprobs = sink_log_softmax_method1(self, class_energy, dim) outputs[ti.label] = cond_logprobs.clone() for timer in ti.reset('method2'): with timer: cond_logprobs = sink_log_softmax_method2(self, class_energy, dim) outputs[ti.label] = cond_logprobs.clone() for k1, k2 in ub.iter_window(outputs, 2): if torch.all(outputs[k1] == outputs[k2]): print('MATCH: k1={} matches k2={}'.format(k1, k2)) else: print('DIFF: k1={} DIFFERS k2={}'.format(k1, k2)) print((torch.abs(outputs[k1] - outputs[k2])).sum())
def _bench_catgraph_conditional_log_softmax_solution(): from ovharn import category_tree graph = nx.generators.gnr_graph(30, 0.3, seed=321).reverse() self = category_tree.CategoryTree(graph) class_energy = torch.randn(16, len(self.idx_to_node), 15, 15) class_energy = class_energy.to(nh.XPU.cast('auto').main_device) dim = 1 def method1(self, class_energy, dim): cond_logprobs = torch.empty_like(class_energy) # Move indexes onto the class_energy device (perhaps precache this) index_groups = [ torch.LongTensor(idxs).to(class_energy.device) for idxs in self.idx_groups ] for index in index_groups: # Take each subset of classes that are mutually exclusive energy_group = torch.index_select(class_energy, dim=dim, index=index) # Then apply the log_softmax to those sets logprob_group = F.log_softmax(energy_group, dim=dim) cond_logprobs.index_copy_(dim, index, logprob_group) return cond_logprobs def method2(self, class_energy, dim): cond_logprobs = torch.empty_like(class_energy) fancy_prefix = [slice(None)] * dim for idxs in self.idx_groups: fancy_index = tuple(fancy_prefix + [idxs]) cond_logprobs[fancy_index] = F.log_softmax( class_energy[fancy_index], dim=dim) return cond_logprobs ti = CudaTimerit(500, bestof=50, verbose=1, unit='us') outputs = ub.odict() for timer in ti.reset('method1'): with timer: cond_logprobs = method1(self, class_energy, dim) outputs[ti.label] = cond_logprobs.clone() for timer in ti.reset('method2'): with timer: cond_logprobs = method2(self, class_energy, dim) outputs[ti.label] = cond_logprobs.clone() # ------ for k1, k2 in ub.iter_window(outputs, 2): if torch.all(outputs[k1] == outputs[k2]): print('MATCH: k1={} matches k2={}'.format(k1, k2)) else: print('DIFF: k1={} DIFFERS k2={}'.format(k1, k2)) print((torch.abs(outputs[k1] - outputs[k2])).sum()) # Meausre how much overhead creating the LongTensor takes. # Is it worth precaching? Not really, its like 1% of the time. def method1_overhead(self, class_energy, dim): # Move indexes onto the class_energy device (perhaps precache this) index_groups = [ torch.LongTensor(idxs).to(class_energy.device) for idxs in self.idx_groups ] for index in index_groups: pass for timer in ti.reset('method1-overhead'): with timer: method1_overhead(self, class_energy, dim)
def render_facts(): """ Render facts to a latex document """ import pylatex from pylatex.base_classes.command import Options # NOQA import pyqrcode fact_data = load_facts() class MDFramed(pylatex.base_classes.Environment): _latex_name = 'mdframed' packages = [pylatex.Package('mdframed')] class SamePage(pylatex.base_classes.Environment): _latex_name = 'samepage' class ComposeContexts: def __init__(self, *contexts): self.contexts = contexts def __enter__(self): return [c.__enter__() for c in self.contexts] def __exit__(self, a, b, c): return [c.__exit__(a, b, c) for c in self.contexts[::-1]] # class NewUnicodeChar(pylatex.base_classes.CommandBase): # pass # Dont use fontenc, lmodern, or textcomp # https://tex.stackexchange.com/questions/179778/xelatex-under-ubuntu doc = pylatex.Document('fact_document', inputenc=None, page_numbers=False, indent=False, fontenc=None, lmodern=False, textcomp=False) doc.preamble.append(pylatex.Package('graphicx')) # For PNG images # doc.preamble.append(pylatex.Package('svg', options=dict(inkscapearea='page'))) # doc.preamble.append(pylatex.Command('title', 'Facts')) # doc.preamble.append(pylatex.Command('author', 'Anonymous author')) # doc.preamble.append(pylatex.Command('date', pylatex.NoEscape(r'\today'))) # doc.append(pylatex.NoEscape(r'\maketitle')) # doc.preamble.append(pylatex.Package('newunicodechar')) # doc.preamble.append(pylatex.NoEscape(r'\newunicodechar{±}{$\pm$}')) # doc.append(pylatex.NoEscape('13.787±0.020')) # print(doc.dumps()) # doc.generate_pdf(clean_tex=False, compiler='xelatex') # return QR_REFERENCE = True stop_flag = 0 image_dpath = ub.Path('~/misc/facts/images').expand().ensuredir() # image_dpath = for fact in ub.ProgIter(fact_data['facts']): contexts = ComposeContexts( # doc.create(SamePage()), doc.create(MDFramed()), doc.create(pylatex.MiniPage(width=r'0.99\textwidth'))) # with doc.create(pylatex.MiniPage(width=r'\textwidth')): with contexts: doc.append(pylatex.NoEscape(r'\paragraph{Fact:}')) text = ub.paragraph(fact['text']) if r'\[' in text: found = list( re.finditer( '(' + re.escape(r'\[') + '|' + re.escape(r'\]') + ')', text)) prev_x = 0 for a, b in ub.iter_window(found, step=2): part = text[prev_x:a.span()[0]] doc.append(part) ax, bx = a.span()[1], b.span()[0] part = pylatex.NoEscape(r'$' + text[ax:bx] + r'$ ') doc.append(part) prev_x = b.span()[1] part = text[prev_x:] doc.append(part) else: # if '$' in text: # parts = text.split('$') # for idx, p in enumerate(parts): # if idx % 2 == 1: # doc.append(pylatex.NoEscape('$' + p + '$ ')) # else: # doc.append(p) # else: doc.append(text) if QR_REFERENCE: doc.append('\n') num_refs = 0 for refline in fact['references'].split('\n'): if refline.startswith('http'): found = refline image_fname = ub.hash_data(found, base='abc')[0:16] + '.png' image_fpath = image_dpath / image_fname if not image_fpath.exists(): # pyqrcode.create(found).svg(fpath, scale=6) pyqrcode.create(found).png(str(image_fpath), scale=2) doc.append( pylatex.NoEscape(r'\includegraphics[width=90px]{' + str(image_fpath) + '}')) # doc.append(pylatex.NoEscape(r'\includesvg[width=120px]{' + fpath + '}')) num_refs += 1 if num_refs > 3: break else: doc.append(pylatex.NoEscape(r'\paragraph{References:}')) with doc.create(pylatex.Itemize()) as itemize: for refline in fact['references'].split('\n'): if refline: refline = refline.strip() itemize.add_item(refline) doc.append(pylatex.NoEscape(r'\bigskip')) if stop_flag: break print(doc.dumps()) print('generate pdf') doc.generate_pdf(str(ub.Path('~/misc/facts/fact_document').expand()), clean_tex=True)
label_to_nodes = ub.group_items(node_to_label.keys(), node_to_label.values()) aug_graph = graph.copy() # remove cut edges from augmented graph edge_to_iscut = nx.get_edge_attributes(aug_graph, 'is_cut') cut_edges = [ (u, v) for (u, v, d) in aug_graph.edges(data=True) if not (d.get('is_cut') or d.get('decision', 'unreviewed') in ['nomatch']) ] cut_edges = [edge for edge, flag in edge_to_iscut.items() if flag] aug_graph.remove_edges_from(cut_edges) # Enumerate cliques inside labels unflat_edges = [ list(ub.iter_window(nodes, 2)) for nodes in label_to_nodes.values() ] node_pairs = [tup for tup in ub.flatten(unflat_edges) if tup[0] != tup[1]] # Remove candidate MST edges that exist in the original graph orig_edges = list(aug_graph.edges()) candidate_mst_edges = [ edge for edge in node_pairs if not aug_graph.has_edge(*edge) ] # randomness prevents chains and visually looks better rng = np.random.RandomState(42) def _randint(): return 0 return rng.randint(0, 100)
def run_pvpoke_ultra_experiment(): """ https://pvpoke.com/battle/matrix/ !pip install selenium """ """ Relevant page items: <button class="add-poke-btn button">+ Add Pokemon</button> '//*[@id="main"]/div[3]/div[3]/div/div[1]/button[1]' '/html/body/div[1]/div/div[3]/div[3]/div/div[1]/button[1]' <input class="poke-search" type="text" placeholder="Search name"> /html/body/div[5]/div/div[3]/div[1]/input /html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/a/span[1] Level Cap /html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[2]/div[2]/div[5] # IV GROUP ivs-group save-poke import sys, ubelt sys.path.append(ubelt.expandpath('~/code/pypogo')) from pypogo.pvpoke_experiment import * # NOQA from pypogo.pvpoke_experiment import _oldstuff """ from selenium import webdriver from selenium.webdriver.common.keys import Keys from selenium.webdriver.support.ui import Select import ubelt as ub import os import pathlib import time import pandas as pd import pypogo # Requires the driver be in the PATH fpath = ensure_selenium_chromedriver() os.environ['PATH'] = os.pathsep.join( ub.oset(os.environ['PATH'].split(os.pathsep)) | ub.oset([str(fpath.parent)])) url = 'https://pvpoke.com/battle/matrix/' # chrome_exe = ub.find_exe("google-chrome") driver = webdriver.Chrome() driver.get(url) league = 'Great' # league = 'Master40' if league == 'Great': league_box_target = 'Great League (CP 1500)' have_ivs = list( ub.oset([ tuple([int(x) for x in p.strip().split(',') if x]) for p in ub.codeblock(''' 10, 10, 12, 10, 12, 14, 10, 12, 14, 10, 13, 10, 10, 13, 12, 10, 14, 14, 11, 12, 14, 11, 14, 12, 11, 14, 15, 11, 15, 11, 11, 15, 11, 11, 15, 12, 11, 15, 12, 12, 10, 12, 12, 11, 12, 12, 12, 15, 12, 14, 11, 12, 14, 15, 12, 15, 11, 12, 15, 12 12, 15, 12, 13, 11, 13 13, 12, 10 13, 12, 13, 13, 13, 10, 13, 13, 11, 13, 15, 10, 13, 15, 11, 13, 15, 11, 14, 10, 12, 14, 11, 10, 14, 11, 10, 14, 13, 11 14, 13, 14, 15, 10, 12 15, 11, 10, 15, 11, 11, 15, 12, 11 ''').split('\n') ])) to_check_mons = [ pypogo.Pokemon('Deoxys', form='defense', ivs=ivs, moves=['Counter', 'Rock Slide', 'Psycho Boost']).maximize(1500) for ivs in have_ivs ] meta_text = 'Great League Meta' elif league == 'Master40': league_box_target = 'Master League (Level 40)' meta_text = 'Master League Meta' # Test the effect of best buddies vs the master league to_check_mons = [ pypogo.Pokemon('Mewtwo', ivs=[15, 15, 15], level=40), pypogo.Pokemon('Mewtwo', ivs=[15, 15, 15], level=41), pypogo.Pokemon('Garchomp', ivs=[15, 15, 15], level=40), pypogo.Pokemon('Garchomp', ivs=[15, 15, 15], level=41), pypogo.Pokemon('Dragonite', ivs=[15, 14, 15], level=40), pypogo.Pokemon('Dragonite', ivs=[15, 14, 15], level=41), pypogo.Pokemon('Giratina', form='origin', ivs=[15, 14, 15], level=40), pypogo.Pokemon('Giratina', form='origin', ivs=[15, 14, 15], level=41), pypogo.Pokemon('Kyogre', ivs=[15, 15, 14], level=40), pypogo.Pokemon('Kyogre', ivs=[15, 15, 14], level=41), pypogo.Pokemon('Groudon', ivs=[14, 14, 13], level=40), pypogo.Pokemon('Groudon', ivs=[14, 14, 13], level=41), pypogo.Pokemon('Togekiss', ivs=[15, 15, 14], level=40), pypogo.Pokemon('Togekiss', ivs=[15, 15, 14], level=41), ] for mon in to_check_mons: mon.populate_all() else: pass leage_select = driver.find_elements_by_class_name('league-select')[0] leage_select.click() leage_select.send_keys(league_box_target) leage_select.click() leage_select.text.split('\n') leage_select.send_keys('\n') leage_select.send_keys('\n') def add_pokemon(mon): add_poke1_button = driver.find_elements_by_class_name( 'add-poke-btn')[0] add_poke1_button.click() select_drop = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/select') if 1: import xdev all_names = select_drop.text.split('\n') distances = xdev.edit_distance(mon.display_name(), all_names) chosen_name = all_names[ub.argmin(distances)] else: chosen_name = mon.name search_box = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/input') search_box.send_keys(chosen_name) advanced_ivs_arrow = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/a/span[1]') advanced_ivs_arrow.click() level40_cap = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[2]/div[2]/div[2]' ) level41_cap = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[2]/div[2]/div[3]' ) level50_cap = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[2]/div[2]/div[4]' ) level51_cap = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[2]/div[2]/div[5]' ) if mon.level >= 51: level51_cap.click() elif mon.level >= 50: level50_cap.click() elif mon.level >= 41: level41_cap.click() elif mon.level >= 40: level40_cap.click() level_box = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[1]/input' ) level_box.click() level_box.clear() level_box.clear() level_box.send_keys(str(mon.level)) iv_a = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[1]/div/input[1]' ) iv_d = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[1]/div/input[2]' ) iv_s = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[9]/div/div[1]/div/input[3]' ) # TODO # driver.find_elements_by_class_name('move-select') iv_a.clear() iv_a.send_keys(str(mon.ivs[0])) iv_d.clear() iv_d.send_keys(str(mon.ivs[1])) iv_s.clear() iv_s.send_keys(str(mon.ivs[2])) # USE_MOVES = 1 if mon.moves is not None: # mon.populate_all() fast_select = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[10]/select[1]') fast_select.click() fast_select.send_keys(mon.pvp_fast_move['name']) fast_select.send_keys(Keys.ENTER) charge1_select = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[10]/select[2]') charge1_select.click() charge1_select.send_keys(mon.pvp_charge_moves[0]['name']) charge1_select.send_keys(Keys.ENTER) charge2_select = driver.find_element_by_xpath( '/html/body/div[5]/div/div[3]/div[1]/div[2]/div[10]/select[3]') charge2_select.click() charge2_select.send_keys(mon.pvp_charge_moves[1]['name']) charge2_select.send_keys(Keys.ENTER) save_button = driver.find_elements_by_class_name('save-poke')[0] save_button.click() quickfills = driver.find_elements_by_class_name('quick-fill-select') quickfill = quickfills[1] quickfill.text.split('\n') quickfill.click() quickfill.send_keys(meta_text) quickfill.click() import pypogo # mon1 = pypogo.Pokemon('Mewtwo', ivs=[15, 15, 15], level=40) # mon2 = pypogo.Pokemon('Mewtwo', ivs=[15, 15, 15], level=41) if 1: for mon in to_check_mons: pass add_pokemon(mon) shield_selectors = driver.find_elements_by_class_name('shield-select') shield_selectors[2].click() shield_selectors[2].send_keys('No shields') shield_selectors[2].send_keys(Keys.ENTER) shield_selectors[3].click() shield_selectors[3].send_keys('No shields') shield_selectors[3].send_keys(Keys.ENTER) shield_selectors[0].click() battle_btn = driver.find_elements_by_class_name('battle-btn')[0] battle_btn.click() # Clear previous downloaded files dlfolder = pathlib.Path(ub.expandpath('$HOME/Downloads')) for old_fpath in list(dlfolder.glob('_vs*.csv')): old_fpath.unlink() time.sleep(2.0) # Download new data dl_btn = driver.find_element_by_xpath( '//*[@id="main"]/div[4]/div[9]/div/a') dl_btn.click() while len(list(dlfolder.glob('_vs*.csv'))) < 1: pass new_fpaths = list(dlfolder.glob('_vs*.csv')) assert len(new_fpaths) == 1 fpath = new_fpaths[0] data = pd.read_csv(fpath, header=0, index_col=0) if 1: # GROUP ANALYSIS data.sum(axis=1).sort_values() (data > 500).sum(axis=1).sort_values() flipped = [] for key, col in data.T.iterrows(): if not ub.allsame(col > 500): flipped.append(key) flip_df = data.loc[:, flipped] def color(x): if x > 500: return ub.color_text(str(x), 'green') else: return ub.color_text(str(x), 'red') print(flip_df.applymap(color)) print(flip_df.columns.tolist()) (data > 500) else: # PAIR ANALYSIS pairs = list(ub.iter_window(range(len(data)), step=2)) for i, j in pairs: print('-----') matchup0 = data.iloc[i] matchup1 = data.iloc[j] delta = matchup1 - matchup0 print(delta[delta != 0]) wins0 = matchup0 > 500 wins1 = matchup1 > 500 flips = (wins0 != wins1) flipped_vs = matchup0.index[flips] num_flips = sum(flips) print('flipped_vs = {!r}'.format(flipped_vs)) print('num_flips = {!r}'.format(num_flips)) print(matchup0.mean()) print(matchup1.mean()) print(matchup1.mean() / matchup0.mean())
def main(): import kwplot plt = kwplot.autoplt() sns = kwplot.autosns() alias = { '3090': 'nvctrl GeForce GTX 1080 Ti 1 temp', '1080ti': 'nvctrl GeForce RTX 3090 0 temp', # 'cpu': 'lmsensor coretemp-isa-0000 Package id 0', } all_df = read_psensor_log() unique_rawdevs = all_df.device.unique() for rawdev in unique_rawdevs: cpu_prefix = 'lmsensor coretemp-isa' if rawdev.startswith(cpu_prefix): suffix = rawdev[len(cpu_prefix):].split(' ', 1)[1].strip() alias['CPU ' + suffix] = rawdev if 'nvctrl' in rawdev and 'temp' in rawdev: alias['GPU ' + rawdev[7:-5]] = rawdev mapper = ub.invert_dict(alias) all_df['device'] = all_df['device'].apply(lambda x: mapper.get(x, None)) all_df = all_df[all_df['device'].apply(lambda x: x is not None)] hours = int(ub.argval('--hours', default=48)) delta = datetime.timedelta(hours=hours) min_time = datetime.datetime.now() - delta is_recent = all_df.datetime > min_time recent_df = all_df[is_recent] chosen = recent_df # chosen = all_df if 0: pivtbl = recent_df.pivot('unix_timestamp', 'device', 'temp') pivtbl = pivtbl.sort_index() smoothed_rows = [] for window_idxs in ub.iter_window(list(range(len(pivtbl))), size=10): window = pivtbl.iloc[list(window_idxs)] max_val = window.max(axis=0, skipna=True) for k, v in max_val.to_dict().items(): smoothed_rows.append({ 'unix_timestamp': window.index[1], 'device': k, 'temp': v, }) max_extra = pd.DataFrame(smoothed_rows) sns.lineplot(data=max_extra, x='unix_timestamp', y='temp', hue='device') df = recent_df.copy() df['device'] = df['device'].apply(lambda x: 'Core' if x.startswith('Core') else x) df['time'] = df['unix_timestamp'].apply( datetime.datetime.fromtimestamp) plt.gcf().clf() # sns.lineplot(data=chosen, x='unix_timestamp', y='temp', hue='device') for xx, (sess, group) in enumerate(chosen.groupby('session_x')): # ax.cla() ax = plt.gca() sns.lineplot(data=group, x='unix_timestamp', y='temp', hue='device', legend=xx == 0) label_xaxis_dates(ax) ax.figure.subplots_adjust(bottom=0.2) ax.set_ylim(0, 100) plt.locator_params(axis='y', nbins=10) # import matplotlib as mpl # Draw shutdown time as black lines end_times = [] for sx, group in chosen.groupby('session_x'): shutdown_time = group['unix_timestamp'].max() end_times.append(shutdown_time) for shutdown_time in sorted(end_times)[:-1]: ax.plot((shutdown_time, shutdown_time), [0, 100], color='k') # ci_df = pd.concat([max_extra, recent_df]) # ci_df['device'] = ci_df['device'].apply(lambda x: 'Core' if x.startswith('Core') else x) # sns.lineplot(data=ci_df, x='unix_timestamp', y='temp', hue='device') # from matplotlib.dates import date2num # all_df['date_ord'] = all_df['datetime'].map(lambda a: date2num(a)) # sns.lineplot(data=pt) # sns.lineplot(data=recent_df, x='unix_timestamp', y='temp', hue='device') # sns.regplot(data=recent_df, x='unix_timestamp', y='temp', hue='device') plt.show()
def do_tags(verbose=True, inplace=False, dry=True, auto_rollback=False): if verbose: if dry: print('squashing streaks (DRY RUN)') else: print('squashing streaks') # print('authors = {!r}'.format(authors)) # If you are in a repo subdirectory, find the repo root cwd = os.getcwd() repodir = cwd while True: if os.path.exists(os.path.join(repodir, '.git')): break newpath = os.path.dirname(repodir) if newpath == repodir: raise git.exc.InvalidGitRepositoryError(cwd) repodir = newpath repo = git.Repo(repodir) orig_branch_name = repo.active_branch.name # head = repo.commit('HEAD') info = ub.cmd('git tag -l --sort=v:refname', verbose=3) info2 = ub.cmd('git show-ref --tags', verbose=3) tag_to_hash = {} for line in info2['out'].splitlines(): if line: hashtext, tags = line.split(' ') tag = tags.replace('refs/tags/', '') tag_to_hash[tag] = hashtext print('tag_to_hash = {!r}'.format(tag_to_hash)) tag_order = [line for line in info['out'].splitlines() if line] custom_streaks = list(ub.iter_window(tag_order, 2)) print('Forcing hacked steaks') print('custom_streaks = {!r}'.format(custom_streaks)) streaks = [] for custom_streak in custom_streaks: print('custom_streak = {!r}'.format(custom_streak)) assert len(custom_streak) == 2 hash_a = tag_to_hash[custom_streak[0]] hash_b = tag_to_hash[custom_streak[1]] a = repo.commit(hash_a) b = repo.commit(hash_b) if repo.is_ancestor(ancestor_rev=a, rev=b): a, b = b, a # assert repo.is_ancestor(ancestor_rev=b, rev=a) streak = Streak(a, _streak=[a, b]) if len(streak.start.parents) != 1: print('WARNING: cannot include streak = {!r}'.format(streak)) continue # assert start.authored_datetime < stop.authored_datetime if not repo.is_ancestor(ancestor_rev=streak.start, rev=streak.stop): print('WARNING: cannot include streak = {!r}'.format(streak)) continue # raise AssertionError('cant handle') streaks.append(streak) if verbose: print('Found {!r} streaks'.format(len(streaks))) # Switch to a temp branch before we start working if not dry: temp_branchname = checkout_temporary_branch(repo, '-squash-temp') else: temp_branchname = None try: for streak in ub.ProgIter(streaks, 'squashing', verbose=3 * verbose): if verbose: print('Squashing streak = %r' % (str(streak),)) # Start is the commit further back in time _squash_between(repo, streak.start, streak.stop, dry=dry, verbose=verbose) except Exception as ex: print_exc(sys.exc_info()) print('ERROR: squash_streaks failed.') if not dry and auto_rollback: print('ROLLING BACK') repo.git.checkout(orig_branch_name) # repo.git.branch(D=temp_branchname) print('You can debug the difference with:') print(' gitk {} {}'.format(orig_branch_name, temp_branchname)) return if dry: if verbose: print('Finished. did nothing') elif inplace: # Copy temp branch back over original repo.git.checkout(orig_branch_name) repo.git.reset(temp_branchname, hard=True) repo.git.branch(D=temp_branchname) if verbose: print('Finished. Now you should force push the branch back to the server') else: # Go back to the original branch repo.git.checkout(orig_branch_name) if verbose: print('Finished') print('The squashed branch is: {}'.format(temp_branchname)) print('You can inspect the difference with:') print(' gitk {} {}'.format(orig_branch_name, temp_branchname)) print('Finished. Now you must manually clean this branch up.') print('Or, to automatically accept changes run with --inplace')