def plot_counts(self, counts: Mapping[str, Sequence[LangDataPoint]]): output_file( str(self.output_file), title=self.title, root_dir=self.output_file.parent, mode="relative", ) for lang, lang_counts in counts.items(): non_zero = [elem for elem in lang_counts if elem.code > 0] subjects = lmap(op.attrgetter("commit.subject"), non_zero) dates = lmap(op.attrgetter("commit.dt"), non_zero) code = lmap(op.attrgetter("code"), non_zero) refs = lmap(op.itemgetter(slice(6)), map(op.attrgetter("commit.ref"), non_zero)) data = { "x": dates, "y": code, "date": dates, "count": code, "subject": subjects, "ref": refs, } source = ColumnDataSource(data=data) self.plot_lang(self.fig, lang, source) self.configure_fig() save(self.fig)
def get_groups(ip): def _get_infs(record): name = re_find(r'(Smartgroup:\d+)', record) if name: name = name.lower().replace(':', '') infs = re_all(r'(x?gei_\d+/\d+/\d+)\s?selected', record) return dict(name=name, infs=infs) def _get_desc_mode(child, group): rslt = do_some(child, 'show run int {name}'.format(name=group['name'])) desc = re_find(r'description\s+(\S+)', rslt) group['desc'] = desc rslt = do_some(child, 'show run int {inf}'.format( inf=group['infs'][0])) mode = re_find(r'smartgroup\s\d+\smode\s(\S+)', rslt) group['mode'] = mode return group try: child = telnet(ip) rslt = re.split(r'\r\n\s*\r\n', do_some(child, 'show lacp internal')) groups = thread_last(rslt, (lmap, _get_infs), (select, lambda x: x['name'] and x['infs'])) lmap(partial(_get_desc_mode, child), groups) close(child) except (pexpect.EOF, pexpect.TIMEOUT) as e: return ('fail', None, ip) return ('success', groups, ip)
def get_groups(ip): def _get_infs(record): name = re_find(r'(Smartgroup:\d+)', record) if name: name = name.lower().replace(':', '') infs = re_all(r'(x?gei_\d+/\d+/\d+)\s?selected', record) return dict(name=name, infs=infs) def _get_desc_mode(child, group): rslt = do_some(child, 'show run int {name}'.format(name=group['name'])) desc = re_find(r'description\s+(\S+)', rslt) group['desc'] = desc rslt = do_some(child, 'show run int {inf}'.format(inf=group['infs'][0])) mode = re_find(r'smartgroup\s\d+\smode\s(\S+)', rslt) group['mode'] = mode return group try: child = telnet(ip) rslt = re.split(r'\r\n\s*\r\n', do_some(child, 'show lacp internal')) groups = thread_last(rslt, (lmap, _get_infs), (select, lambda x: x['name'] and x['infs'])) lmap(partial(_get_desc_mode, child), groups) close(child) except (pexpect.EOF, pexpect.TIMEOUT) as e: return ('fail', None, ip) return ('success', groups, ip)
def _dnf(where): """ Constructs DNF of where tree consisting of terms in form: (alias, attribute, value, negation) meaning `alias.attribute = value` or `not alias.attribute = value` if negation is False Any conditions other then eq are dropped. """ if isinstance(where, Lookup): # If where.lhs don't refer to a field then don't bother if not hasattr(where.lhs, 'target'): return SOME_TREE # Don't bother with complex right hand side either if isinstance(where.rhs, (QuerySet, Query, BaseExpression)): return SOME_TREE # Skip conditions on non-serialized fields if where.lhs.target not in serializable_fields( where.lhs.target.model): return SOME_TREE attname = where.lhs.target.attname if isinstance(where, Exact): return [[(where.lhs.alias, attname, where.rhs, True)]] elif isinstance(where, IsNull): return [[(where.lhs.alias, attname, None, where.rhs)]] elif isinstance(where, In) and len( where.rhs) < settings.CACHEOPS_LONG_DISJUNCTION: return [[(where.lhs.alias, attname, v, True)] for v in where.rhs] else: return SOME_TREE elif isinstance(where, NothingNode): return [] elif isinstance(where, (ExtraWhere, SubqueryConstraint, Exists)): return SOME_TREE elif len(where) == 0: return [[]] else: chilren_dnfs = lmap(_dnf, where.children) if len(chilren_dnfs) == 0: return [[]] elif len(chilren_dnfs) == 1: result = chilren_dnfs[0] else: # Just unite children joined with OR if where.connector == OR: result = lcat(chilren_dnfs) # Use Cartesian product to AND children else: result = lmap(lcat, product(*chilren_dnfs)) # Negating and expanding brackets if where.negated: result = [lmap(negate, p) for p in product(*result)] return result
def add_bingfa(): funcs = {'ME60': ME60.get_bingfa, 'ME60-X16': ME60.get_bingfa, 'M6000': M6k.get_bingfa} _get_bf = partial(_model, funcs) clear() nodes = graph.find('Bras') bras = [(x['ip'], x['model']) for x in nodes] lmap(compose(_add_bingfa, _get_bf), bras)
def high_bounded_range(self, children): are_floats = any(["." in x for x in children]) if are_floats: children = lmap(float, children) else: children = lmap(int, children) high, = children res = {"type": "number", "maximum": high} if not are_floats: res.update({"multipleOf": 1.0}) return res
def _add_traffics(lock, record): mark, rslt, ip = record cmd = """ match (s:Switch {ip:{ip}})-->(i:Inf{name:{name}}) set i.state={state},i.bw={bw},i.inTraffic={inTraffic},i.outTraffic={outTraffic},i.updated=timestamp() """ with lock: if mark == 'success': tx = graph.cypher.begin() lmap(lambda x: tx.append(cmd, ip=ip, **x), rslt) tx.process() tx.commit()
def add_bingfa(): funcs = { 'ME60': ME60.get_bingfa, 'ME60-X16': ME60.get_bingfa, 'M6000-S': M6k.get_bingfa, 'M6000': M6k.get_bingfa } _get_bf = partial(_model, funcs) clear() nodes = graph.find('Bras') bras = [(x['ip'], x['model']) for x in nodes] lmap(compose(_add_bingfa, _get_bf), bras)
def test_pqtree_after_reduce_chars_rand_examples(): ITERATIONS = 1000 merged = 0 rts = [] for i in range(ITERATIONS): iter_stats = {} # ------------------------- Generation of permutations ------------------------- id_perm = list(range(1, 10)) duplication_mutations(id_perm, 2) other_perms = [list(id_perm), list(id_perm)] for p in other_perms: mutate_collection(p, 2) ps = tmap(tuple, (id_perm, *other_perms)) # ------------------------- Try to merge same adjacent chars ------------------------- start_time = time.time() pq = PQTreeDup.from_perms_wth_multi(ps) iter_stats["merged"] = time.time() - start_time if not pq: continue else: merged += 1 # ------------------------- find all the trees with minimal size ------------------------- start_time = time.time() all_possibilities = list(PQTreeDup.from_perms(ps)) iter_stats["no_merge"] = time.time() - start_time iter_stats["perms"] = ps best_size = all_possibilities[0].approx_frontier_size() only_best_sized = lfilter(lambda t: t.approx_frontier_size() == best_size, all_possibilities) # verify tree with multi chars contains in its frontier one of the best trees try: front = set(pq.frontier()) assert any(front.issuperset(t.frontier()) for t in only_best_sized) except: print(ps) # PQTreeVisualizer.show_all(pq, *only_best_sized) raise else: rts.append(iter_stats) print(f"multi merged: {merged} / {ITERATIONS}") lmap(print, rts)
def _add_bingfa(rslt): cmd = """ match (b:Bras {ip:{ip}}) merge (b)-[:HAS]->(c:Card {slot:{slot}}) set c.peakUsers={peakUsers},c.peakTime={peakTime},c.updated=timestamp() """ mark, record, ip = rslt with open(logFile, 'a') as flog: flog.write('{ip}:{mark}\n'.format(ip=ip, mark=mark)) if mark == 'success': tx = graph.cypher.begin() lmap(lambda x: tx.append(cmd, ip=ip, slot=x[0], peakUsers=x[1], peakTime=x[2]), record) tx.process() tx.commit()
def calculate_running_statistics(in_data: Tuple[Sequence[int], List[float], int]) \ -> Tuple[List[float], List[float]]: steps, list_avg, i = in_data sized_pot = SizedPot(steps[i]) group_sum = GroupSum() group_sum.counts_needed(list(steps)) group_sum.set_data(list_avg) average = lmap( lambda s: s[0] / max(min(steps[i], s[1]), 1), zip(map(group_sum.sum_getter(steps[i]), range(len(list_avg))), range(len(list_avg)))) std_dev = lmap(lambda s: sized_pot.running_std_dev(*s), zip(list_avg, average)) return average, std_dev
def _add_groups(lock, record): mark, groups, ip = record statement = """match (s:Switch {ip:{ip}}) merge (s)-[r:HAS]->(g:Group {name:{name}}) on create set g.mode={mode},g.desc={desc},g.updated=timestamp() on match set g.mode={mode},g.desc={desc},g.updated=timestamp() with s,g match (s)-->(g)-[r:OWNED]->(i:Inf) delete r""" with lock: if mark == 'success': tx = graph.cypher.begin() lmap(lambda x: tx.append(statement, ip=ip, **x), groups) tx.process() tx.commit()
def traverse_perm_space(cls, perms): """ example for 2 perms [(1, 1, 2), (1, 2, 1)] the first perm is translated to (0, 1, 2) translation: {1: [0,1], 2: [2]} yield w.l.o.g: a) 0 1 2 ; 0 2 1 b) 0 1 2 ; 1 2 0 Note: as seen in the example - perms[0] is static - to avoid isomorphic results """ p1 = perms[0] norm_p1 = range(len(p1)) translated_p1 = tuple(TranslatedChar(val, org) for val, org in zip(norm_p1, p1)) translations = defaultdict(list) for i, val in enumerate(p1): translations[val].append(norm_p1[i]) iters = lmap(lambda p: cls.perm_variations(p, translations), perms[1:]) for perm_set in IterProduct.iproduct([translated_p1], *iters): yield perm_set
def load(data): tiles = {} for lines in map(str.splitlines, data.split("\n\n")): no = parse("Tile {:d}:", lines[0])[0] tile = lmap(lambda line: [sym == "#" for sym in line], lines[1:]) tiles[no] = np.array(tile, dtype=int) return tiles
def pretty_print_fund_data(fund_data: List[dict]): def parse_to_chinese_readable(_data): res: dict = { '估算增量': _data['gszzl'], '基金编码': _data['fundcode'], '基金名称': _data['name'], '单位净值': _data['dwjz'], '净值日期': _data['jzrq'], '估算值': _data['gsz'], '估值时间': _data['gztime'], } return res fund_data = lmap(parse_to_chinese_readable, fund_data) table_decrease = PrettyTable() table_increase = PrettyTable() if fund_data: for _data in fund_data: if _data['估算增量'].startswith('-'): if not table_decrease.field_names: table_decrease.field_names = list(_data.keys()) table_decrease.add_row(_data.values()) else: if not table_increase.field_names: table_increase.field_names = list(_data.keys()) table_increase.add_row(_data.values()) table_decrease.align = table_increase.align = 'l' if table_decrease.rowcount > 0: print(table_decrease) if table_decrease.rowcount > 0 and table_increase.rowcount > 0: print() # print a blank line as separator if table_increase.rowcount > 0: print(table_increase)
def get_processes(services) -> Iterable[Coroutine]: for name, service in services.items(): env: dict = dict(os.environ) if "env_file" in service: env_file = service["env_file"] if not isinstance(env_file, list): env_file = [env_file] env.update(merge(*[DotEnv(path) for path in env_file])) if "environment" in service: environment = service["environment"] if isinstance(environment, list): env.update({b.key: b.value for b in lmap(parse_binding, environment)}) else: env.update(environment) cmd = service.get("entrypoint", "") + " " + service.get("command") if not cmd: raise Exception("cannot run without commands on the config") build = service["build"] if isinstance(build, str): cwd = build else: cwd = build.get("context", ".") async def f(name, cmd, env, cwd): print("Attaching to " + name) color = random.choice(colors) log = lambda x: sys.stdout.write(getattr(Fore, color) + f"{name} | " + Fore.RESET + x) p = await exec(cmd, env=env, cwd=cwd, stdout=log, stderr=log) if p: log(f"{name} exited with code {p.returncode}" + "\n") yield f(name, cmd, env, cwd)
def iter_bottom_up( self, from_level=float('inf')) -> Iterable[CommonInterval]: keys_in_order = reversed( sorted( filter(lambda k: k <= from_level, self.nesting_levels.keys()))) lst_in_order = [self.nesting_levels[k] for k in keys_in_order] return chain(*lmap(iter, lst_in_order))
def main(args): with open(args.annotations, 'rt', encoding='UTF-8') as annotations: coco = json.load(annotations) #info = coco['info'] #licenses = coco['licenses'] images = coco['images'] annotations = coco['annotations'] categories = coco['categories'] number_of_images = len(images) images_with_annotations = funcy.lmap(lambda a: int(a['image_id']), annotations) if args.having_annotations: images = funcy.lremove( lambda i: i['id'] not in images_with_annotations, images) x, y = train_test_split(images, train_size=args.split) #save_coco(args.train, info, licenses, x, filter_annotations(annotations, x), categories) save_coco(args.train, x, filter_annotations(annotations, x), categories) #save_coco(args.test, info, licenses, y, filter_annotations(annotations, y), categories) save_coco(args.test, y, filter_annotations(annotations, y), categories) print("Saved {} entries in {} and {} in {}".format( len(x), args.train, len(y), args.test))
def infer(): trcs = [encode_trace(trc) for trc in TRACES] mdp = DYN2 best, spec2score = spec_mle( mdp, trcs, SPEC2MONITORS.values(), parallel=False, psat=0.9 ) def normalize(score): return int(round(score - spec2score[SPEC2MONITORS[CONST_TRUE]])) best_score = normalize(spec2score[best]) fig = tpl.figure() fig.barh( fn.lmap(normalize, spec2score.values()), labels=SPEC_NAMES, force_ascii=False, show_vals=True, ) print('\n' + "="*80) print(' (log likelihood(spec) - log_likelihood(True))'.rjust(40) + '\n') print('(higher is better)'.rjust(41)) print("="*80) fig.show() print(f"\n\nbest score: {abs(best_score)}") return best
def split_coco_annotation(annotations, split_ratio, train_json='train_anno.json', test_json='test_anno.json', is_having=True): with open(annotations, 'rt', encoding='UTF-8') as anno: coco = json.load(anno) images = coco['images'] annotations = coco['annotations'] categories = coco['categories'] images_with_annotations = funcy.lmap(lambda a: int(a['image_id']), annotations) if is_having: images = funcy.lremove( lambda i: i['id'] not in images_with_annotations, images) x, y = train_test_split(images, train_size=split_ratio) save_coco(train_json, x, filter_annotations(annotations, x), categories) save_coco(test_json, y, filter_annotations(annotations, y), categories) print("Saved {} entries in {} and {} in {}".format( len(x), train_json, len(y), test_json))
def get_infs(ip): def _get_info(child, inf): rslt = do_some(child, 'show int {inf}'.format(inf=inf)) desc = re_find(r'Description\sis\s(\S+)', rslt) state = re_find(r'{inf}\sis\s(\S+\s?\S+),'.format(inf=inf), rslt) bw = re_find(r'BW\s(\d+)\sKbits', rslt) bw = int(bw or 0) / 1000 inTraffic = re_find(r'seconds\sinput\srate\s?:\s+(\d+)\sBps', rslt) inTraffic = int(inTraffic or 0) * 8 / 1e6 outTraffic = re_find(r'seconds\soutput\srate:\s+(\d+)\sBps', rslt) outTraffic = int(outTraffic or 0) * 8 / 1e6 return dict(name=inf, desc=desc, state=state, bw=bw, inTraffic=inTraffic, outTraffic=outTraffic) try: child = telnet(ip) rslt = do_some(child, 'show run | in interface', timeout=180) rslt = re_all(r'interface\s+(x?gei_\d+/\d+/\d+)', rslt) infs = lmap(partial(_get_info, child), rslt) close(child) except (pexpect.EOF, pexpect.TIMEOUT) as e: return ('fail', None, ip) return ('success', infs, ip)
def get_infs_bySnmp(ip): def _get_infs(oid): index = oid.value desc = 'None' name = session.get(('ifDescr', index)).value if ' ' in name: name, desc = name.split(' ', 1) state = session.get(('ifOperStatus', index)).value if state == '1': state = 'up' else: state = 'down' bw = int(session.get(('ifSpeed', index)).value or 0) collTime = time.time() inCount = int(session.get(('ifInOctets', index)).value or 0) outCount = int(session.get(('ifOutOctets', index)).value or 0) return dict(name=name, desc=desc, state=state, bw=bw, inCount=inCount, outCount=outCount, collTime=collTime) try: session = easysnmp.Session( hostname=ip, community=community_read, version=1) indexs = session.walk('ifIndex')[1:] rslt = lmap(_get_infs, indexs) return ('success', rslt, ip) except (easysnmp.EasySNMPTimeoutError) as e: return ('fail', None, ip)
def get_ports(ip): def _get_info(record): name = re_find(r'(\S+) current state :', record) state = re_find(r'current state : ?(\S+ ?\S+)', record) desc = re_find(r'Description: (\S+ *\S+)', record) inTraffic = int( re_find(r'\d+ seconds input:\s+\d+\spackets/sec\s(\d+)\sbits/sec', record) or 0) / 1000000 outTraffic = int( re_find(r'\d+ seconds output:\s+\d+\spackets/sec\s(\d+)\sbits/sec', record) or 0) / 1000000 return dict(name=name, desc=desc, state=state, inTraffic=inTraffic, outTraffic=outTraffic) try: child = telnet(ip) rslt = do_some(child, 'disp interface') close(child) rslt = re.split(r'\r\n *\r\n', rslt) rslt = select(lambda x: bool(x['name']), lmap(_get_info, rslt)) except (pexpect.EOF, pexpect.TIMEOUT) as e: return ('fail', None, ip) return ('success', rslt, ip)
def _add_bingfa(rslt): cmd = """ match (b:Bras {ip:{ip}}) merge (b)-[:HAS]->(c:Card {slot:{slot}}) set c.peakUsers={peakUsers},c.peakTime={peakTime},c.updated=timestamp() """ mark, record, ip = rslt with open(logFile, 'a') as flog: flog.write('{ip}:{mark}\n'.format(ip=ip, mark=mark)) if mark == 'success': tx = graph.cypher.begin() lmap( lambda x: tx.append( cmd, ip=ip, slot=x[0], peakUsers=x[1], peakTime=x[2]), record) tx.process() tx.commit()
def visit_aag(self, _, children): header, ios1, lgs1, ios2, lgs2, symbols, comments = children ios, lgs = ios1 + ios2, lgs1 + lgs2 assert len(ios) == header.num_inputs + header.num_outputs inputs, outputs = ios[:header.num_inputs], ios[header.num_inputs:] assert len(lgs) == header.num_ands + header.num_latches latches, gates = lgs[:header.num_latches], lgs[header.num_latches:] # TODO: need to allow for inputs, outputs, latches not in # symbol table. inputs = { symbols.inputs.inv.get(idx, f'i{idx}'): i for idx, i in enumerate(inputs) } outputs = { symbols.outputs.inv.get(idx, f'o{idx}'): i for idx, i in enumerate(outputs) } latches = { symbols.latches.inv.get(idx, f'l{idx}'): tuple(i) for idx, i in enumerate(latches) } latches = fn.walk_values(lambda l: (l + (0, ))[:3], latches) if len(comments) > 0: assert comments[0].startswith('c\n') comments[0] = comments[0][2:] return AAG(inputs=inputs, outputs=outputs, latches=fn.walk_values(tuple, latches), gates=fn.lmap(tuple, gates), comments=tuple(comments))
def get_traffics(ip, infs): def _get_traffic(child, inf): rslt = do_some(child, 'disp int {inf}'.format(inf=inf)) state = re_find( r'{inf}\scurrent\sstate\s:\s?(\w+\s?\w+)'.format(inf=inf), rslt).lower() bw = re_find(r'(\d+[MG])bps-speed mode', rslt) if bw is None: bw = 0 elif 'M' in bw: bw = int(bw.replace('M', '')) else: bw = int(bw.replace('G', '')) * 1000 inTraffic = int( re_find(r'\d+ seconds input:\s+\d+\spackets/sec\s(\d+)\sbits/sec', rslt)) / 1000000 outTraffic = int( re_find(r'\d+ seconds output:\s+\d+\spackets/sec\s(\d+)\sbits/sec', rslt)) / 1000000 infDict = dict(name=inf, state=state, bw=bw, inTraffic=inTraffic, outTraffic=outTraffic) return infDict try: child = telnet(ip) rslt = lmap(partial(_get_traffic, child), infs) close(child) except (pexpect.EOF, pexpect.TIMEOUT) as e: return ('fail', None, ip) return ('success', rslt, ip)
def main(args): with open(args.annotations, "rt", encoding="UTF-8") as annotations: coco = json.load(annotations) info = coco["info"] licenses = coco["licenses"] images = coco["images"] annotations = coco["annotations"] categories = coco["categories"] number_of_images = len(images) images_with_annotations = funcy.lmap(lambda a: int(a["image_id"]), annotations) if args.having_annotations: images = funcy.lremove( lambda i: i["id"] not in images_with_annotations, images) x, y = train_test_split(images, train_size=args.split) save_coco( args.train, info, licenses, x, filter_annotations(annotations, x), categories, ) save_coco(args.test, info, licenses, y, filter_annotations(annotations, y), categories) print("Saved {} entries in {} and {} in {}".format( len(x), args.train, len(y), args.test))
def test_prop__range_search(gen): ixys = gen['ixys'] mode = gen['mode']; xORy = c_char(mode.encode()) min_key = gen['min_key'] max_key = gen['max_key'] includeds = gen['includeds'] n_node, ixy_arr, c_bst, n_inserted = bst_tree(ixys, xORy) tup_bst = tup_tree(c_bst[:n_inserted+4]) #pprint(tup_bst) ixy_idxes = (c_int * n_inserted)() stack = (c_int * MAX_LEN)() n_included = bst.includeds1d( c_bst, ixy_arr, xORy, min_key, max_key, ixy_idxes, stack) actual_idxes = [int(i) for i in ixy_idxes[:n_included]] expect_idxes = F.lmap(F.first, includeds) #actual_ixys = F.lmap(cobj2tuple, ixy_idxes) assert set(actual_idxes) == set(expect_idxes), \ f'{actual_idxes} != {expect_idxes}, {tup_bst}' key = prop(mode) for i1, i2 in F.pairwise(actual_idxes): assert key(ixy_arr[i1]) <= key(ixy_arr[i2]) assert n_included == len(includeds), \ f'{n_included} != {len(includeds)}, {tup_bst}'
def get_traffics(ip, infs): def _get_traffic(child, inf): rslt = do_some(child, 'disp int {inf}'.format(inf=inf)) state = re_find(r'{inf}\scurrent\sstate\s:\s?(\w+\s?\w+)' .format(inf=inf), rslt).lower() bw = re_find(r'(\d+[MG])bps-speed mode', rslt) if bw is None: bw = 0 elif 'M' in bw: bw = int(bw.replace('M', '')) else: bw = int(bw.replace('G', '')) * 1000 inTraffic = int(re_find( r'\d+ seconds input:\s+\d+\spackets/sec\s(\d+)\sbits/sec', rslt)) / 1000000 outTraffic = int(re_find( r'\d+ seconds output:\s+\d+\spackets/sec\s(\d+)\sbits/sec', rslt)) / 1000000 infDict = dict(name=inf, state=state, bw=bw, inTraffic=inTraffic, outTraffic=outTraffic) return infDict try: child = telnet(ip) rslt = lmap(partial(_get_traffic, child), infs) close(child) except (pexpect.EOF, pexpect.TIMEOUT) as e: return ('fail', None, ip) return ('success', rslt, ip)
def get_infs_bySnmp(ip): def _get_infs(oid): index = oid.value desc = 'None' name = session.get(('ifDescr', index)).value if ' ' in name: name, desc = name.split(' ', 1) state = session.get(('ifOperStatus', index)).value if state == '1': state = 'up' else: state = 'down' bw = int(session.get(('ifSpeed', index)).value or 0) collTime = time.time() inCount = int(session.get(('ifInOctets', index)).value or 0) outCount = int(session.get(('ifOutOctets', index)).value or 0) return dict(name=name, desc=desc, state=state, bw=bw, inCount=inCount, outCount=outCount, collTime=collTime) try: session = easysnmp.Session(hostname=ip, community=community_read, version=1) indexs = session.walk('ifIndex') rslt = lmap(_get_infs, indexs) return ('success', rslt, ip) except (easysnmp.EasySNMPTimeoutError) as e: return ('fail', None, ip)
def show(cls, pqtree: PQTree, figure_index=1, skip_show=False, title='PQTree'): g = nx.DiGraph() leaf_strs = cls.gen_leaf_strs(pqtree) def child_str(child): if isinstance(child, LeafNode): return leaf_strs[child] return str(child) def rec_construct_graph(node): children = getattr(node, 'children', []) children_strs = map(child_str, children) [g.add_edge(str(node), child) for child in children_strs] [rec_construct_graph(child) for child in children] rec_construct_graph(pqtree.root) plt.figure(figure_index, figsize=(cls.FIG_EDGE_SIZE, cls.FIG_EDGE_SIZE)) plt.title(title) pos = graphviz_layout(g, prog='dot') node_colors = lmap(lambda n: cls.NODE_COLORS[n[0]], g.nodes) nx.draw(g, pos, with_labels=True, font_size=cls.FONT_SIZE, font_weight='bold', node_size=cls.NODE_SIZE, node_shape=cls.NODE_SHAPE, node_color=node_colors) if not skip_show: plt.show()
def get_ports(ip): def _get_info(record): name = re_find(r'((?:xg|g|f)ei\S+) is \w+ ?\w+,', record) state = re_find(r'(?:xg|g|f)ei\S+ is (\w+ ?\w+),', record) desc = re_find(r'Description is (\S+ *\S+)', record) inTraffic = int( re_find(r'120 seconds input.*:\s+(\d+)\sBps', record) or 0) * 8 / 1000000 outTraffic = int( re_find(r'120 seconds output.*:\s+(\d+)\sBps', record) or 0) * 8 / 1000000 return dict(name=name, desc=desc, state=state, inTraffic=inTraffic, outTraffic=outTraffic) try: child = telnet(ip) rslt = do_some(child, 'show interface') close(child) except (pexpect.EOF, pexpect.TIMEOUT) as e: return ('fail', None, ip) rslt = re.split(r'\r\n *\r\n', rslt) rslt = select(lambda x: bool(x['name']), lmap(_get_info, rslt)) return ('success', rslt, ip)
def main(annotation_path, split_ratio, having_annotations, train_save_path, test_save_path, random_state=None): with open(annotation_path, 'rt', encoding='UTF-8') as annotations: coco = json.load(annotations) info = coco['info'] licenses = coco['licenses'] images = coco['images'] annotations = coco['annotations'] categories = coco['categories'] number_of_images = len(images) images_with_annotations = funcy.lmap(lambda a: int(a['image_id']), annotations) if having_annotations: images = funcy.lremove( lambda i: i['id'] not in images_with_annotations, images) x, y = train_test_split(images, train_size=split_ratio, random_state=random_state) save_coco(train_save_path, info, licenses, x, filter_annotations(annotations, x), categories) save_coco(test_save_path, info, licenses, y, filter_annotations(annotations, y), categories) print("Saved {} entries in {} and {} in {}".format( len(x), train_save_path, len(y), test_save_path))
def main(args): with open(args.annotations, 'rt', encoding='UTF-8') as annotations: coco = json.load(annotations) info = coco['info'] licenses = coco['licenses'] images = coco['images'] annotations = coco['annotations'] categories = coco['categories'] number_of_images = len(images) images_with_annotations = funcy.lmap(lambda a: int(a['image_id']), annotations) if not os.path.exists(args.traindir): pl.Path(args.traindir).mkdir(parents=True, exist_ok=True) if not os.path.exists(args.testdir): pl.Path(args.testdir).mkdir(parents=True, exist_ok=True) if args.having_annotations: images = funcy.lremove(lambda i: i['id'] not in images_with_annotations, images) x, y = train_test_split(images, train_size=args.split) save_coco(os.path.join(args.traindir,args.train), info, licenses, x, filter_annotations(annotations, x), categories) save_coco(os.path.join(args.testdir,args.test), info, licenses, y, filter_annotations(annotations, y), categories) for file in x: shutil.copy(os.path.join(args.inputdir, file["file_name"]), args.traindir) for file in y: shutil.copy(os.path.join(args.inputdir, file["file_name"]), args.testdir) print("Saved {} entries in {} and {} in {}".format(len(x), args.train, len(y), args.test))
def run_test(model, dataloader, criterion, device, threshold=None): if device is None: device = torch.device("cuda" if torch.cuda.is_available() else "cpu") if isinstance(device, str): device = torch.device(device) progress_bar = tqdm(dataloader, desc="Running test") model.eval() predictions = [] for i, it in enumerate(progress_bar): batch_data = it[0] inputs = batch_data["inputs"] inputs = inputs.to(device) with torch.set_grad_enabled(False): net_outputs = model(inputs) net_outputs = torch.softmax(net_outputs, dim=1) cls_outputs = funcy.lmap(lambda t: t.item(), net_outputs[:, 1]) if threshold is not None: cls_outputs = [int(out > threshold) for out in cls_outputs] predictions.extend(list(cls_outputs)) return predictions
def _add_infs(lock, record): mark, infs, ip = record statement = """ match(n:Olt {ip:{ip}}) merge (n)-[:HAS]->(i:Inf {name:{name}}) on create set i.desc={desc},i.state={state},i.bw={bw}, i.inTraffic={inTraffic},i.outTraffic={outTraffic},i.updated=timestamp() on match set i.desc={desc},i.state={state},i.bw={bw}, i.inTraffic={inTraffic},i.outTraffic={outTraffic},i.updated=timestamp()""" with lock: with open(log_file, 'a') as lf: lf.write('{ip}:{mark}\n'.format(ip=ip, mark=mark)) if mark == 'success' and infs: tx = graph.cypher.begin() lmap(lambda x: tx.append(statement, ip=ip, **x), infs) tx.process() tx.commit()
def _add_infs(lock, record): mark, infs, ip = record statement = """ match (s:Switch {ip:{ip}}) merge (s)-[:has]->(i:Inf {name:{name}}) set i.desc={desc},i.updated=timestamp() with s,i match (s)-->(g:Group {name:{group}}) merge (g)-[r:OWNED]->(i) set r.updated=timestamp() """ with lock: if mark == 'success': tx = graph.cypher.begin() lmap(lambda x: tx.append(statement, ip=ip, name=x[ 'name'], desc=x['desc'], group=x['group']), infs) tx.process() tx.commit()
def get_vlan_users(bras): def _get_vlan_users(bas): funcs = {'m6k': M6k.get_vlan_users, 'me60': ME60.get_vlan_users} _gvu = partial(_model, funcs) return _gvu(bas) bras = [dict(ip=x[0], model=x[1], inf=x[2]) for x in bras] rslt = lmap(_get_vlan_users, bras) return rslt
def get_onus(ip): mark, ports = get_pon_ports(ip)[:-1] if mark == 'fail': return ('fail', None, ip) try: child = telnet(ip) gpo = partial(get_port_onus, child) rslt = lmap(gpo, ports) child.sendline('exit') child.close() except (pexpect.EOF, pexpect.TIMEOUT) as e: return ('fail', None, ip) rslt1 = filter(lambda x: bool(x[1]), rslt) return ('success', rslt1, ip)
def get_vlans(ip): def _vlan(record): if re_test(r'(Ports:\snone.*Ports:\snone)', record, re.S): return 0 vlan = re_find(r'VLAN\sID:\s(\d+)', record) vlan = int(vlan or 0) return vlan try: child = telnet(ip) rslt = do_some(child, 'disp vlan all') close(child) rslt = re.split(r'\r\n *\r\n', rslt) vlans = select(lambda x: x > 1, lmap(_vlan, rslt)) except (pexpect.EOF, pexpect.TIMEOUT) as e: return ('fail', None, ip) return ('success', vlans, ip)
def get_groups(ip): def _get_group(child, group): rslt = do_some( child, 'disp link-aggregation {group}'.format(group=group)) desc = re_find(r'description:(\S+)', rslt) mode = re_find(r'work mode:\s+(\S+)', rslt) temp = re_all(r'(\d+/\d+)\s+(\d\S+)', rslt) temp1 = lmapcat(lambda x: ['{0}/{1}'.format(x[0], y) for y in x[1].split(',')], temp) return dict(name=group, desc=desc, mode=mode, infs=temp1) try: child = telnet(ip) temp = re_all(r'(\d+/\d+/\d+)', do_some(child, 'disp link-aggregation all')) groups = lmap(partial(_get_group, child), temp) close(child) except (pexpect.EOF, pexpect.TIMEOUT) as e: return ('fail', None, ip) return ('success', groups, ip)
def get_ports(ip): def _get_info(record): name = re_find(r'((?:xg|g|f)ei\S+) is \w+ ?\w*,', record) state = re_find(r'(?:xg|g|f)ei\S+ is (\w+ ?\w+),', record) desc = re_find(r'Description is (\S+ *\S+)', record) inTraffic = int( re_find(r'120s input.*:\s(\d+)Bps', record)) * 8 / 1000000 outTraffic = int( re_find(r'120s output.*:\s(\d+)Bps', record)) * 8 / 1000000 return dict(name=name, desc=desc, state=state, inTraffic=inTraffic, outTraffic=outTraffic) try: child = telnet(ip) rslt = do_some(child, 'show interface include ei- | in (is |put rate)') close(child) except (pexpect.EOF, pexpect.TIMEOUT) as e: return ('fail', None, ip) rslt = re.split(r'\r\n *(?=(?:xg|g|f)ei-)', rslt) rslt = lmap(_get_info, rslt) return ('success', rslt, ip)
def get_traffics(ip, infs): def _get_traffic(child, inf): rslt = do_some(child, 'disp int {inf}'.format(inf=inf)) state = re_find(r'{inf}\scurrent\sstate\s:\s?(\w+\s?\w+)' .format(inf=inf), rslt).lower() bw = int(re_find(r'Speed\s+:\s+(\d+),', rslt)) inTraffic = int( re_find(r'300 seconds input rate (\d+)\sbits/sec', rslt)) / 1000000 outTraffic = int( re_find(r'300 seconds output rate (\d+)\sbits/sec', rslt)) / 1000000 infDict = dict(name=inf, state=state, bw=bw, inTraffic=inTraffic, outTraffic=outTraffic) return infDict try: child = telnet(ip) rslt = lmap(partial(_get_traffic, child), infs) close(child) except (pexpect.EOF, pexpect.TIMEOUT) as e: return ('fail', None, ip) return ('success', rslt, ip)
def get_ports(ip): def _get_info(record): name = re_find(r'(\S+) current state :', record) state = re_find(r'current state : ?(\S+ ?\S+)', record) desc = re_find(r'Description:(\S+ *\S+)', record) inTraffic = int( re_find(r'300 seconds input rate (\d+)\sbits/sec', record) or 0) / 1000000 outTraffic = int( re_find(r'300 seconds output rate (\d+)\sbits/sec', record) or 0) / 1000000 return dict(name=name, desc=desc, state=state, inTraffic=inTraffic, outTraffic=outTraffic) try: child = telnet(ip) rslt = do_some(child, 'disp interface') close(child) except (pexpect.EOF, pexpect.TIMEOUT) as e: return ('fail', None, ip) rslt = select(lambda x: re_test(r'^x?gigabitethernet', x, re.I), re.split(r'\r\n *\r\n *', rslt)) rslt = lmap(_get_info, rslt) return ('success', rslt, ip)
def get_traffics(ip, infs): def _get_traffic(child, inf): rslt = do_some(child, 'show interface {inf}'.format(inf=inf)) state = re_find( r'{inf}\sis\s(\w+\s?\w+),'.format(inf=inf), rslt).lower() bw = int(re_find(r'BW\s(\d+)\sKbps', rslt)) / 1000 inTraffic = int( re_find(r'120s input.*:\s(\d+)Bps', rslt)) * 8 / 1000000 outTraffic = int( re_find(r'120s output.*:\s(\d+)Bps', rslt)) * 8 / 1000000 infDict = dict(name=inf, state=state, bw=bw, inTraffic=inTraffic, outTraffic=outTraffic) return infDict try: child = telnet(ip) rslt = lmap(partial(_get_traffic, child), infs) close(child) except (pexpect.EOF, pexpect.TIMEOUT) as e: return ('fail', None, ip) return ('success', rslt, ip)
def get_ports(ip): def _get_info(record): name = re_find(r'(\S+) current state :', record) state = re_find(r'current state : ?(\S+ ?\S+)', record) desc = re_find(r'Description: (\S+ *\S+)', record) inTraffic = int(re_find( r'\d+ seconds input:\s+\d+\spackets/sec\s(\d+)\sbits/sec', record) or 0) / 1000000 outTraffic = int(re_find( r'\d+ seconds output:\s+\d+\spackets/sec\s(\d+)\sbits/sec', record) or 0) / 1000000 return dict(name=name, desc=desc, state=state, inTraffic=inTraffic, outTraffic=outTraffic) try: child = telnet(ip) rslt = do_some(child, 'disp interface') close(child) rslt = re.split(r'\r\n *\r\n', rslt) rslt = select(lambda x: bool(x['name']), lmap(_get_info, rslt)) except (pexpect.EOF, pexpect.TIMEOUT) as e: return ('fail', None, ip) return ('success', rslt, ip)
def bounding_box(domain, oracle, eps=1e-5): """Compute Bounding box. TODO: clean up""" # TODO: remove r input and assume unit rec. edges = [mdts.binsearch(r2, oracle, eps=eps) for r2 in box_edges(domain)] rtypes = fn.pluck(0, edges) if all(t == mdts.SearchResultType.TRIVIALLY_FALSE for t in rtypes): return domain elif all(t == mdts.SearchResultType.TRIVIALLY_TRUE for t in rtypes): return mdtr.to_rec(domain.dim*[[0, 0]]) itvls = [r for t, r in edges if t == mdts.SearchResultType.NON_TRIVIAL] def box_to_include(r): return domain.backward_cone(r.top) & domain.forward_cone(r.bot) bbox, *recs = fn.lmap(box_to_include, itvls) for r in recs: bbox = bbox.sup(r) return bbox
def get_bingfa(ip): def _get_users(child, slot): record = do_some( child, 'disp max-online slot {s}'.format(s=slot)) users = re_find( r'Max online users since startup\s+:\s+(\d+)', record) users = int(users or 0) date = re_find( r'Time of max online users\s+:\s+(\d{4}-\d{2}-\d{2})', record) return (slot, users, date) try: child = telnet(ip) rslt = do_some(child, 'disp dev | in BSU') ff = compose(partial(select, bool), partial(map, r'(\d+)\s+BSU')) slots = ff(rslt.split('\r\n')) maxUsers = lmap(partial(_get_users, child), slots) close(child) except (pexpect.EOF, pexpect.TIMEOUT) as e: return ('fail', None, ip) return ('success', maxUsers, ip)
def get_infs(ip): def _get_desc(child, inf): name = inf['name'] rslt = do_some(child, 'show run interface {name}'.format(name=name)) desc = re_find(r'description\s(\S+ *\S*)', rslt) group = re_find(r'(smartgroup\s\d+)', rslt) if group: group = group.replace(' ', '') inf['desc'] = desc inf['group'] = group return inf try: child = telnet(ip) rslt = do_some(child, 'show run | in interface (xg|g|f)ei_') temp = [dict(name=x) for x in re_all('interface\s(\S+)', rslt)] get_desc = partial(_get_desc, child) infs = lmap(get_desc, temp) close(child) except (pexpect.EOF, pexpect.TIMEOUT) as e: return ('fail', None, ip) return ('success', infs, ip)
def get_groups(ip): def _get_desc(child, group): name = group['name'] rslt = do_some(child, 'show run interface {name}'.format(name=name)) desc = re_find(r'description\s(\S+ *\S*)', rslt) mode = re_find(r'smartgroup mode (\S+)', rslt) if mode == '802.3ad': mode = 'yes' group['desc'] = desc group['mode'] = mode return group try: child = telnet(ip) rslt = do_some(child, 'show lacp internal') temp = re_all(r'Smartgroup:(\d+)', rslt) temp1 = [dict(name='smartgroup' + x) for x in temp] groups = lmap(partial(_get_desc, child), temp1) close(child) except (pexpect.EOF, pexpect.TIMEOUT) as e: return ('fail', None, ip) return ('success', groups, ip)
def get_groups(ip): def _get_desc(child, group): name = group['name'] rslt = do_some( child, 'show running-config-interface {name} all'.format(name=name)) desc = re_find(r'description\s(\S+ *\S*)', rslt) mode = re_find(r'lacp\smode\s(\S+)', rslt) if mode == '802.3ad': mode = 'yes' group['desc'] = desc group['mode'] = mode return group try: child = telnet(ip) rslt = do_some(child, 'show interface | in smartgroup') temp = [dict(name=x) for x in re_all(r'(smartgroup\d+)', rslt)] groups = lmap(partial(_get_desc, child), temp) close(child) except (pexpect.EOF, pexpect.TIMEOUT) as e: return ('fail', None, ip) return ('success', groups, ip)
def get_groups(ip): def _get_desc(child, group): name = group['name'] rslt = do_some(child, 'show run interface {name}'.format(name=name)) desc = re_find(r'description\s(\S+ *\S*)', rslt) group['desc'] = desc if group['mode'] == 'active': group['mode'] = 'yes' return group try: child = telnet(ip) rslt = do_some(child, 'show run | in smartgroup [0-9]+') ff = rcompose(partial(map, lambda x: x.strip()), distinct, partial(map, r'(smartgroup\s\d+)\smode\s(\w+)'), partial(map, lambda x: dict(name=x[0].replace(' ', ''), mode=x[1]))) temp = ff(rslt.splitlines()[:-1]) get_desc = partial(_get_desc, child) groups = lmap(get_desc, temp) close(child) except (pexpect.EOF, pexpect.TIMEOUT) as e: return ('fail', None, ip) return ('success', groups, ip)
def power_check(): clear_log() nodes = graph.find('Olt', property_key='company', property_value='hw') olts = [(x['ip'], x['company'], x['area']) for x in nodes] funcy.lmap(funcy.compose(output_info, get_power_info), olts)
def import_sw(file): switchs = (x.strip() for x in open(file)) lmap(lambda x: graph.create(create_sw_node(x)), switchs)
def import_bras(file): bras = (x.strip().split(',') for x in open(file)) brasNode = lambda x: graph.create( Node('Bras', **dict(zip(('name', 'ip', 'model', 'area'), x)))) lmap(brasNode, bras)