def plot_counts(self, counts: Mapping[str, Sequence[LangDataPoint]]):
output_file(
str(self.output_file),
title=self.title,
root_dir=self.output_file.parent,
mode="relative",
)
for lang, lang_counts in counts.items():
non_zero = [elem for elem in lang_counts if elem.code > 0]
subjects = lmap(op.attrgetter("commit.subject"), non_zero)
dates = lmap(op.attrgetter("commit.dt"), non_zero)
code = lmap(op.attrgetter("code"), non_zero)
refs = lmap(op.itemgetter(slice(6)),
map(op.attrgetter("commit.ref"), non_zero))
data = {
"x": dates,
"y": code,
"date": dates,
"count": code,
"subject": subjects,
"ref": refs,
}
source = ColumnDataSource(data=data)
self.plot_lang(self.fig, lang, source)
self.configure_fig()
save(self.fig)
def get_groups(ip):
def _get_infs(record):
name = re_find(r'(Smartgroup:\d+)', record)
if name:
name = name.lower().replace(':', '')
infs = re_all(r'(x?gei_\d+/\d+/\d+)\s?selected', record)
return dict(name=name, infs=infs)
def _get_desc_mode(child, group):
rslt = do_some(child, 'show run int {name}'.format(name=group['name']))
desc = re_find(r'description\s+(\S+)', rslt)
group['desc'] = desc
rslt = do_some(child, 'show run int {inf}'.format(
inf=group['infs'][0]))
mode = re_find(r'smartgroup\s\d+\smode\s(\S+)', rslt)
group['mode'] = mode
return group
try:
child = telnet(ip)
rslt = re.split(r'\r\n\s*\r\n', do_some(child, 'show lacp internal'))
groups = thread_last(rslt,
(lmap, _get_infs),
(select, lambda x: x['name'] and x['infs']))
lmap(partial(_get_desc_mode, child), groups)
close(child)
except (pexpect.EOF, pexpect.TIMEOUT) as e:
return ('fail', None, ip)
return ('success', groups, ip)
def get_groups(ip):
def _get_infs(record):
name = re_find(r'(Smartgroup:\d+)', record)
if name:
name = name.lower().replace(':', '')
infs = re_all(r'(x?gei_\d+/\d+/\d+)\s?selected', record)
return dict(name=name, infs=infs)
def _get_desc_mode(child, group):
rslt = do_some(child, 'show run int {name}'.format(name=group['name']))
desc = re_find(r'description\s+(\S+)', rslt)
group['desc'] = desc
rslt = do_some(child,
'show run int {inf}'.format(inf=group['infs'][0]))
mode = re_find(r'smartgroup\s\d+\smode\s(\S+)', rslt)
group['mode'] = mode
return group
try:
child = telnet(ip)
rslt = re.split(r'\r\n\s*\r\n', do_some(child, 'show lacp internal'))
groups = thread_last(rslt, (lmap, _get_infs),
(select, lambda x: x['name'] and x['infs']))
lmap(partial(_get_desc_mode, child), groups)
close(child)
except (pexpect.EOF, pexpect.TIMEOUT) as e:
return ('fail', None, ip)
return ('success', groups, ip)
def _dnf(where):
"""
Constructs DNF of where tree consisting of terms in form:
(alias, attribute, value, negation)
meaning `alias.attribute = value`
or `not alias.attribute = value` if negation is False
Any conditions other then eq are dropped.
"""
if isinstance(where, Lookup):
# If where.lhs don't refer to a field then don't bother
if not hasattr(where.lhs, 'target'):
return SOME_TREE
# Don't bother with complex right hand side either
if isinstance(where.rhs, (QuerySet, Query, BaseExpression)):
return SOME_TREE
# Skip conditions on non-serialized fields
if where.lhs.target not in serializable_fields(
where.lhs.target.model):
return SOME_TREE
attname = where.lhs.target.attname
if isinstance(where, Exact):
return [[(where.lhs.alias, attname, where.rhs, True)]]
elif isinstance(where, IsNull):
return [[(where.lhs.alias, attname, None, where.rhs)]]
elif isinstance(where, In) and len(
where.rhs) < settings.CACHEOPS_LONG_DISJUNCTION:
return [[(where.lhs.alias, attname, v, True)]
for v in where.rhs]
else:
return SOME_TREE
elif isinstance(where, NothingNode):
return []
elif isinstance(where, (ExtraWhere, SubqueryConstraint, Exists)):
return SOME_TREE
elif len(where) == 0:
return [[]]
else:
chilren_dnfs = lmap(_dnf, where.children)
if len(chilren_dnfs) == 0:
return [[]]
elif len(chilren_dnfs) == 1:
result = chilren_dnfs[0]
else:
# Just unite children joined with OR
if where.connector == OR:
result = lcat(chilren_dnfs)
# Use Cartesian product to AND children
else:
result = lmap(lcat, product(*chilren_dnfs))
# Negating and expanding brackets
if where.negated:
result = [lmap(negate, p) for p in product(*result)]
return result
def add_bingfa():
funcs = {'ME60': ME60.get_bingfa,
'ME60-X16': ME60.get_bingfa,
'M6000': M6k.get_bingfa}
_get_bf = partial(_model, funcs)
clear()
nodes = graph.find('Bras')
bras = [(x['ip'], x['model']) for x in nodes]
lmap(compose(_add_bingfa, _get_bf), bras)
def high_bounded_range(self, children):
are_floats = any(["." in x for x in children])
if are_floats:
children = lmap(float, children)
else:
children = lmap(int, children)
high, = children
res = {"type": "number", "maximum": high}
if not are_floats:
res.update({"multipleOf": 1.0})
return res
def _add_traffics(lock, record):
mark, rslt, ip = record
cmd = """
match (s:Switch {ip:{ip}})-->(i:Inf{name:{name}})
set i.state={state},i.bw={bw},i.inTraffic={inTraffic},i.outTraffic={outTraffic},i.updated=timestamp()
"""
with lock:
if mark == 'success':
tx = graph.cypher.begin()
lmap(lambda x: tx.append(cmd, ip=ip, **x), rslt)
tx.process()
tx.commit()
def _add_traffics(lock, record):
mark, rslt, ip = record
cmd = """
match (s:Switch {ip:{ip}})-->(i:Inf{name:{name}})
set i.state={state},i.bw={bw},i.inTraffic={inTraffic},i.outTraffic={outTraffic},i.updated=timestamp()
"""
with lock:
if mark == 'success':
tx = graph.cypher.begin()
lmap(lambda x: tx.append(cmd, ip=ip, **x), rslt)
tx.process()
tx.commit()
def add_bingfa():
funcs = {
'ME60': ME60.get_bingfa,
'ME60-X16': ME60.get_bingfa,
'M6000-S': M6k.get_bingfa,
'M6000': M6k.get_bingfa
}
_get_bf = partial(_model, funcs)
clear()
nodes = graph.find('Bras')
bras = [(x['ip'], x['model']) for x in nodes]
lmap(compose(_add_bingfa, _get_bf), bras)
def test_pqtree_after_reduce_chars_rand_examples():
ITERATIONS = 1000
merged = 0
rts = []
for i in range(ITERATIONS):
iter_stats = {}
# ------------------------- Generation of permutations -------------------------
id_perm = list(range(1, 10))
duplication_mutations(id_perm, 2)
other_perms = [list(id_perm), list(id_perm)]
for p in other_perms:
mutate_collection(p, 2)
ps = tmap(tuple, (id_perm, *other_perms))
# ------------------------- Try to merge same adjacent chars -------------------------
start_time = time.time()
pq = PQTreeDup.from_perms_wth_multi(ps)
iter_stats["merged"] = time.time() - start_time
if not pq:
continue
else:
merged += 1
# ------------------------- find all the trees with minimal size -------------------------
start_time = time.time()
all_possibilities = list(PQTreeDup.from_perms(ps))
iter_stats["no_merge"] = time.time() - start_time
iter_stats["perms"] = ps
best_size = all_possibilities[0].approx_frontier_size()
only_best_sized = lfilter(lambda t: t.approx_frontier_size() == best_size, all_possibilities)
# verify tree with multi chars contains in its frontier one of the best trees
try:
front = set(pq.frontier())
assert any(front.issuperset(t.frontier()) for t in only_best_sized)
except:
print(ps)
# PQTreeVisualizer.show_all(pq, *only_best_sized)
raise
else:
rts.append(iter_stats)
print(f"multi merged: {merged} / {ITERATIONS}")
lmap(print, rts)
def _add_bingfa(rslt):
cmd = """
match (b:Bras {ip:{ip}})
merge (b)-[:HAS]->(c:Card {slot:{slot}})
set c.peakUsers={peakUsers},c.peakTime={peakTime},c.updated=timestamp()
"""
mark, record, ip = rslt
with open(logFile, 'a') as flog:
flog.write('{ip}:{mark}\n'.format(ip=ip, mark=mark))
if mark == 'success':
tx = graph.cypher.begin()
lmap(lambda x: tx.append(cmd, ip=ip, slot=x[0], peakUsers=x[1], peakTime=x[2]), record)
tx.process()
tx.commit()
def calculate_running_statistics(in_data: Tuple[Sequence[int], List[float], int]) \
-> Tuple[List[float], List[float]]:
steps, list_avg, i = in_data
sized_pot = SizedPot(steps[i])
group_sum = GroupSum()
group_sum.counts_needed(list(steps))
group_sum.set_data(list_avg)
average = lmap(
lambda s: s[0] / max(min(steps[i], s[1]), 1),
zip(map(group_sum.sum_getter(steps[i]), range(len(list_avg))),
range(len(list_avg))))
std_dev = lmap(lambda s: sized_pot.running_std_dev(*s),
zip(list_avg, average))
return average, std_dev
def _add_groups(lock, record):
mark, groups, ip = record
statement = """match (s:Switch {ip:{ip}})
merge (s)-[r:HAS]->(g:Group {name:{name}})
on create set g.mode={mode},g.desc={desc},g.updated=timestamp()
on match set g.mode={mode},g.desc={desc},g.updated=timestamp()
with s,g
match (s)-->(g)-[r:OWNED]->(i:Inf)
delete r"""
with lock:
if mark == 'success':
tx = graph.cypher.begin()
lmap(lambda x: tx.append(statement, ip=ip, **x), groups)
tx.process()
tx.commit()
def traverse_perm_space(cls, perms):
"""
example
for 2 perms [(1, 1, 2), (1, 2, 1)]
the first perm is translated to (0, 1, 2)
translation: {1: [0,1], 2: [2]}
yield w.l.o.g:
a) 0 1 2 ; 0 2 1
b) 0 1 2 ; 1 2 0
Note: as seen in the example - perms[0] is static - to avoid isomorphic results
"""
p1 = perms[0]
norm_p1 = range(len(p1))
translated_p1 = tuple(TranslatedChar(val, org) for val, org in zip(norm_p1, p1))
translations = defaultdict(list)
for i, val in enumerate(p1):
translations[val].append(norm_p1[i])
iters = lmap(lambda p: cls.perm_variations(p, translations), perms[1:])
for perm_set in IterProduct.iproduct([translated_p1], *iters):
yield perm_set
def load(data):
tiles = {}
for lines in map(str.splitlines, data.split("\n\n")):
no = parse("Tile {:d}:", lines[0])[0]
tile = lmap(lambda line: [sym == "#" for sym in line], lines[1:])
tiles[no] = np.array(tile, dtype=int)
return tiles
def pretty_print_fund_data(fund_data: List[dict]):
def parse_to_chinese_readable(_data):
res: dict = {
'估算增量': _data['gszzl'],
'基金编码': _data['fundcode'],
'基金名称': _data['name'],
'单位净值': _data['dwjz'],
'净值日期': _data['jzrq'],
'估算值': _data['gsz'],
'估值时间': _data['gztime'],
}
return res
fund_data = lmap(parse_to_chinese_readable, fund_data)
table_decrease = PrettyTable()
table_increase = PrettyTable()
if fund_data:
for _data in fund_data:
if _data['估算增量'].startswith('-'):
if not table_decrease.field_names:
table_decrease.field_names = list(_data.keys())
table_decrease.add_row(_data.values())
else:
if not table_increase.field_names:
table_increase.field_names = list(_data.keys())
table_increase.add_row(_data.values())
table_decrease.align = table_increase.align = 'l'
if table_decrease.rowcount > 0:
print(table_decrease)
if table_decrease.rowcount > 0 and table_increase.rowcount > 0:
print() # print a blank line as separator
if table_increase.rowcount > 0:
print(table_increase)
def get_processes(services) -> Iterable[Coroutine]:
for name, service in services.items():
env: dict = dict(os.environ)
if "env_file" in service:
env_file = service["env_file"]
if not isinstance(env_file, list):
env_file = [env_file]
env.update(merge(*[DotEnv(path) for path in env_file]))
if "environment" in service:
environment = service["environment"]
if isinstance(environment, list):
env.update({b.key: b.value for b in lmap(parse_binding, environment)})
else:
env.update(environment)
cmd = service.get("entrypoint", "") + " " + service.get("command")
if not cmd:
raise Exception("cannot run without commands on the config")
build = service["build"]
if isinstance(build, str):
cwd = build
else:
cwd = build.get("context", ".")
async def f(name, cmd, env, cwd):
print("Attaching to " + name)
color = random.choice(colors)
log = lambda x: sys.stdout.write(getattr(Fore, color) + f"{name} | " + Fore.RESET + x)
p = await exec(cmd, env=env, cwd=cwd, stdout=log, stderr=log)
if p:
log(f"{name} exited with code {p.returncode}" + "\n")
yield f(name, cmd, env, cwd)
def iter_bottom_up(
self, from_level=float('inf')) -> Iterable[CommonInterval]:
keys_in_order = reversed(
sorted(
filter(lambda k: k <= from_level, self.nesting_levels.keys())))
lst_in_order = [self.nesting_levels[k] for k in keys_in_order]
return chain(*lmap(iter, lst_in_order))
def main(args):
with open(args.annotations, 'rt', encoding='UTF-8') as annotations:
coco = json.load(annotations)
#info = coco['info']
#licenses = coco['licenses']
images = coco['images']
annotations = coco['annotations']
categories = coco['categories']
number_of_images = len(images)
images_with_annotations = funcy.lmap(lambda a: int(a['image_id']),
annotations)
if args.having_annotations:
images = funcy.lremove(
lambda i: i['id'] not in images_with_annotations, images)
x, y = train_test_split(images, train_size=args.split)
#save_coco(args.train, info, licenses, x, filter_annotations(annotations, x), categories)
save_coco(args.train, x, filter_annotations(annotations, x),
categories)
#save_coco(args.test, info, licenses, y, filter_annotations(annotations, y), categories)
save_coco(args.test, y, filter_annotations(annotations, y), categories)
print("Saved {} entries in {} and {} in {}".format(
len(x), args.train, len(y), args.test))
def infer():
trcs = [encode_trace(trc) for trc in TRACES]
mdp = DYN2
best, spec2score = spec_mle(
mdp, trcs, SPEC2MONITORS.values(), parallel=False, psat=0.9
)
def normalize(score):
return int(round(score - spec2score[SPEC2MONITORS[CONST_TRUE]]))
best_score = normalize(spec2score[best])
fig = tpl.figure()
fig.barh(
fn.lmap(normalize, spec2score.values()),
labels=SPEC_NAMES,
force_ascii=False,
show_vals=True,
)
print('\n' + "="*80)
print(' (log likelihood(spec) - log_likelihood(True))'.rjust(40) + '\n')
print('(higher is better)'.rjust(41))
print("="*80)
fig.show()
print(f"\n\nbest score: {abs(best_score)}")
return best
def _add_groups(lock, record):
mark, groups, ip = record
statement = """match (s:Switch {ip:{ip}})
merge (s)-[r:HAS]->(g:Group {name:{name}})
on create set g.mode={mode},g.desc={desc},g.updated=timestamp()
on match set g.mode={mode},g.desc={desc},g.updated=timestamp()
with s,g
match (s)-->(g)-[r:OWNED]->(i:Inf)
delete r"""
with lock:
if mark == 'success':
tx = graph.cypher.begin()
lmap(lambda x: tx.append(statement, ip=ip, **x), groups)
tx.process()
tx.commit()
def split_coco_annotation(annotations,
split_ratio,
train_json='train_anno.json',
test_json='test_anno.json',
is_having=True):
with open(annotations, 'rt', encoding='UTF-8') as anno:
coco = json.load(anno)
images = coco['images']
annotations = coco['annotations']
categories = coco['categories']
images_with_annotations = funcy.lmap(lambda a: int(a['image_id']),
annotations)
if is_having:
images = funcy.lremove(
lambda i: i['id'] not in images_with_annotations, images)
x, y = train_test_split(images, train_size=split_ratio)
save_coco(train_json, x, filter_annotations(annotations, x),
categories)
save_coco(test_json, y, filter_annotations(annotations, y), categories)
print("Saved {} entries in {} and {} in {}".format(
len(x), train_json, len(y), test_json))
def get_infs(ip):
def _get_info(child, inf):
rslt = do_some(child, 'show int {inf}'.format(inf=inf))
desc = re_find(r'Description\sis\s(\S+)', rslt)
state = re_find(r'{inf}\sis\s(\S+\s?\S+),'.format(inf=inf), rslt)
bw = re_find(r'BW\s(\d+)\sKbits', rslt)
bw = int(bw or 0) / 1000
inTraffic = re_find(r'seconds\sinput\srate\s?:\s+(\d+)\sBps', rslt)
inTraffic = int(inTraffic or 0) * 8 / 1e6
outTraffic = re_find(r'seconds\soutput\srate:\s+(\d+)\sBps', rslt)
outTraffic = int(outTraffic or 0) * 8 / 1e6
return dict(name=inf,
desc=desc,
state=state,
bw=bw,
inTraffic=inTraffic,
outTraffic=outTraffic)
try:
child = telnet(ip)
rslt = do_some(child, 'show run | in interface', timeout=180)
rslt = re_all(r'interface\s+(x?gei_\d+/\d+/\d+)', rslt)
infs = lmap(partial(_get_info, child), rslt)
close(child)
except (pexpect.EOF, pexpect.TIMEOUT) as e:
return ('fail', None, ip)
return ('success', infs, ip)
def get_infs_bySnmp(ip):
def _get_infs(oid):
index = oid.value
desc = 'None'
name = session.get(('ifDescr', index)).value
if ' ' in name:
name, desc = name.split(' ', 1)
state = session.get(('ifOperStatus', index)).value
if state == '1':
state = 'up'
else:
state = 'down'
bw = int(session.get(('ifSpeed', index)).value or 0)
collTime = time.time()
inCount = int(session.get(('ifInOctets', index)).value or 0)
outCount = int(session.get(('ifOutOctets', index)).value or 0)
return dict(name=name, desc=desc, state=state, bw=bw,
inCount=inCount, outCount=outCount, collTime=collTime)
try:
session = easysnmp.Session(
hostname=ip, community=community_read, version=1)
indexs = session.walk('ifIndex')[1:]
rslt = lmap(_get_infs, indexs)
return ('success', rslt, ip)
except (easysnmp.EasySNMPTimeoutError) as e:
return ('fail', None, ip)
def get_ports(ip):
def _get_info(record):
name = re_find(r'(\S+) current state :', record)
state = re_find(r'current state : ?(\S+ ?\S+)', record)
desc = re_find(r'Description: (\S+ *\S+)', record)
inTraffic = int(
re_find(r'\d+ seconds input:\s+\d+\spackets/sec\s(\d+)\sbits/sec',
record) or 0) / 1000000
outTraffic = int(
re_find(r'\d+ seconds output:\s+\d+\spackets/sec\s(\d+)\sbits/sec',
record) or 0) / 1000000
return dict(name=name,
desc=desc,
state=state,
inTraffic=inTraffic,
outTraffic=outTraffic)
try:
child = telnet(ip)
rslt = do_some(child, 'disp interface')
close(child)
rslt = re.split(r'\r\n *\r\n', rslt)
rslt = select(lambda x: bool(x['name']), lmap(_get_info, rslt))
except (pexpect.EOF, pexpect.TIMEOUT) as e:
return ('fail', None, ip)
return ('success', rslt, ip)
def _add_bingfa(rslt):
cmd = """
match (b:Bras {ip:{ip}})
merge (b)-[:HAS]->(c:Card {slot:{slot}})
set c.peakUsers={peakUsers},c.peakTime={peakTime},c.updated=timestamp()
"""
mark, record, ip = rslt
with open(logFile, 'a') as flog:
flog.write('{ip}:{mark}\n'.format(ip=ip, mark=mark))
if mark == 'success':
tx = graph.cypher.begin()
lmap(
lambda x: tx.append(
cmd, ip=ip, slot=x[0], peakUsers=x[1], peakTime=x[2]), record)
tx.process()
tx.commit()
def visit_aag(self, _, children):
header, ios1, lgs1, ios2, lgs2, symbols, comments = children
ios, lgs = ios1 + ios2, lgs1 + lgs2
assert len(ios) == header.num_inputs + header.num_outputs
inputs, outputs = ios[:header.num_inputs], ios[header.num_inputs:]
assert len(lgs) == header.num_ands + header.num_latches
latches, gates = lgs[:header.num_latches], lgs[header.num_latches:]
# TODO: need to allow for inputs, outputs, latches not in
# symbol table.
inputs = {
symbols.inputs.inv.get(idx, f'i{idx}'): i
for idx, i in enumerate(inputs)
}
outputs = {
symbols.outputs.inv.get(idx, f'o{idx}'): i
for idx, i in enumerate(outputs)
}
latches = {
symbols.latches.inv.get(idx, f'l{idx}'): tuple(i)
for idx, i in enumerate(latches)
}
latches = fn.walk_values(lambda l: (l + (0, ))[:3], latches)
if len(comments) > 0:
assert comments[0].startswith('c\n')
comments[0] = comments[0][2:]
return AAG(inputs=inputs,
outputs=outputs,
latches=fn.walk_values(tuple, latches),
gates=fn.lmap(tuple, gates),
comments=tuple(comments))
def get_traffics(ip, infs):
def _get_traffic(child, inf):
rslt = do_some(child, 'disp int {inf}'.format(inf=inf))
state = re_find(
r'{inf}\scurrent\sstate\s:\s?(\w+\s?\w+)'.format(inf=inf),
rslt).lower()
bw = re_find(r'(\d+[MG])bps-speed mode', rslt)
if bw is None:
bw = 0
elif 'M' in bw:
bw = int(bw.replace('M', ''))
else:
bw = int(bw.replace('G', '')) * 1000
inTraffic = int(
re_find(r'\d+ seconds input:\s+\d+\spackets/sec\s(\d+)\sbits/sec',
rslt)) / 1000000
outTraffic = int(
re_find(r'\d+ seconds output:\s+\d+\spackets/sec\s(\d+)\sbits/sec',
rslt)) / 1000000
infDict = dict(name=inf,
state=state,
bw=bw,
inTraffic=inTraffic,
outTraffic=outTraffic)
return infDict
try:
child = telnet(ip)
rslt = lmap(partial(_get_traffic, child), infs)
close(child)
except (pexpect.EOF, pexpect.TIMEOUT) as e:
return ('fail', None, ip)
return ('success', rslt, ip)
def main(args):
with open(args.annotations, "rt", encoding="UTF-8") as annotations:
coco = json.load(annotations)
info = coco["info"]
licenses = coco["licenses"]
images = coco["images"]
annotations = coco["annotations"]
categories = coco["categories"]
number_of_images = len(images)
images_with_annotations = funcy.lmap(lambda a: int(a["image_id"]),
annotations)
if args.having_annotations:
images = funcy.lremove(
lambda i: i["id"] not in images_with_annotations, images)
x, y = train_test_split(images, train_size=args.split)
save_coco(
args.train,
info,
licenses,
x,
filter_annotations(annotations, x),
categories,
)
save_coco(args.test, info, licenses, y,
filter_annotations(annotations, y), categories)
print("Saved {} entries in {} and {} in {}".format(
len(x), args.train, len(y), args.test))
def test_prop__range_search(gen):
ixys = gen['ixys']
mode = gen['mode']; xORy = c_char(mode.encode())
min_key = gen['min_key']
max_key = gen['max_key']
includeds = gen['includeds']
n_node, ixy_arr, c_bst, n_inserted = bst_tree(ixys, xORy)
tup_bst = tup_tree(c_bst[:n_inserted+4])
#pprint(tup_bst)
ixy_idxes = (c_int * n_inserted)()
stack = (c_int * MAX_LEN)()
n_included = bst.includeds1d(
c_bst, ixy_arr, xORy, min_key, max_key, ixy_idxes, stack)
actual_idxes = [int(i) for i in ixy_idxes[:n_included]]
expect_idxes = F.lmap(F.first, includeds)
#actual_ixys = F.lmap(cobj2tuple, ixy_idxes)
assert set(actual_idxes) == set(expect_idxes), \
f'{actual_idxes} != {expect_idxes}, {tup_bst}'
key = prop(mode)
for i1, i2 in F.pairwise(actual_idxes):
assert key(ixy_arr[i1]) <= key(ixy_arr[i2])
assert n_included == len(includeds), \
f'{n_included} != {len(includeds)}, {tup_bst}'
def get_traffics(ip, infs):
def _get_traffic(child, inf):
rslt = do_some(child, 'disp int {inf}'.format(inf=inf))
state = re_find(r'{inf}\scurrent\sstate\s:\s?(\w+\s?\w+)'
.format(inf=inf), rslt).lower()
bw = re_find(r'(\d+[MG])bps-speed mode', rslt)
if bw is None:
bw = 0
elif 'M' in bw:
bw = int(bw.replace('M', ''))
else:
bw = int(bw.replace('G', '')) * 1000
inTraffic = int(re_find(
r'\d+ seconds input:\s+\d+\spackets/sec\s(\d+)\sbits/sec', rslt)) / 1000000
outTraffic = int(re_find(
r'\d+ seconds output:\s+\d+\spackets/sec\s(\d+)\sbits/sec', rslt)) / 1000000
infDict = dict(name=inf, state=state, bw=bw,
inTraffic=inTraffic, outTraffic=outTraffic)
return infDict
try:
child = telnet(ip)
rslt = lmap(partial(_get_traffic, child), infs)
close(child)
except (pexpect.EOF, pexpect.TIMEOUT) as e:
return ('fail', None, ip)
return ('success', rslt, ip)
def get_infs_bySnmp(ip):
def _get_infs(oid):
index = oid.value
desc = 'None'
name = session.get(('ifDescr', index)).value
if ' ' in name:
name, desc = name.split(' ', 1)
state = session.get(('ifOperStatus', index)).value
if state == '1':
state = 'up'
else:
state = 'down'
bw = int(session.get(('ifSpeed', index)).value or 0)
collTime = time.time()
inCount = int(session.get(('ifInOctets', index)).value or 0)
outCount = int(session.get(('ifOutOctets', index)).value or 0)
return dict(name=name,
desc=desc,
state=state,
bw=bw,
inCount=inCount,
outCount=outCount,
collTime=collTime)
try:
session = easysnmp.Session(hostname=ip,
community=community_read,
version=1)
indexs = session.walk('ifIndex')
rslt = lmap(_get_infs, indexs)
return ('success', rslt, ip)
except (easysnmp.EasySNMPTimeoutError) as e:
return ('fail', None, ip)
def show(cls, pqtree: PQTree, figure_index=1, skip_show=False, title='PQTree'):
g = nx.DiGraph()
leaf_strs = cls.gen_leaf_strs(pqtree)
def child_str(child):
if isinstance(child, LeafNode):
return leaf_strs[child]
return str(child)
def rec_construct_graph(node):
children = getattr(node, 'children', [])
children_strs = map(child_str, children)
[g.add_edge(str(node), child) for child in children_strs]
[rec_construct_graph(child) for child in children]
rec_construct_graph(pqtree.root)
plt.figure(figure_index, figsize=(cls.FIG_EDGE_SIZE, cls.FIG_EDGE_SIZE))
plt.title(title)
pos = graphviz_layout(g, prog='dot')
node_colors = lmap(lambda n: cls.NODE_COLORS[n[0]], g.nodes)
nx.draw(g, pos, with_labels=True, font_size=cls.FONT_SIZE, font_weight='bold', node_size=cls.NODE_SIZE,
node_shape=cls.NODE_SHAPE, node_color=node_colors)
if not skip_show:
plt.show()
def get_ports(ip):
def _get_info(record):
name = re_find(r'((?:xg|g|f)ei\S+) is \w+ ?\w+,', record)
state = re_find(r'(?:xg|g|f)ei\S+ is (\w+ ?\w+),', record)
desc = re_find(r'Description is (\S+ *\S+)', record)
inTraffic = int(
re_find(r'120 seconds input.*:\s+(\d+)\sBps', record)
or 0) * 8 / 1000000
outTraffic = int(
re_find(r'120 seconds output.*:\s+(\d+)\sBps', record)
or 0) * 8 / 1000000
return dict(name=name,
desc=desc,
state=state,
inTraffic=inTraffic,
outTraffic=outTraffic)
try:
child = telnet(ip)
rslt = do_some(child, 'show interface')
close(child)
except (pexpect.EOF, pexpect.TIMEOUT) as e:
return ('fail', None, ip)
rslt = re.split(r'\r\n *\r\n', rslt)
rslt = select(lambda x: bool(x['name']), lmap(_get_info, rslt))
return ('success', rslt, ip)
def main(annotation_path,
split_ratio,
having_annotations,
train_save_path,
test_save_path,
random_state=None):
with open(annotation_path, 'rt', encoding='UTF-8') as annotations:
coco = json.load(annotations)
info = coco['info']
licenses = coco['licenses']
images = coco['images']
annotations = coco['annotations']
categories = coco['categories']
number_of_images = len(images)
images_with_annotations = funcy.lmap(lambda a: int(a['image_id']),
annotations)
if having_annotations:
images = funcy.lremove(
lambda i: i['id'] not in images_with_annotations, images)
x, y = train_test_split(images,
train_size=split_ratio,
random_state=random_state)
save_coco(train_save_path, info, licenses, x,
filter_annotations(annotations, x), categories)
save_coco(test_save_path, info, licenses, y,
filter_annotations(annotations, y), categories)
print("Saved {} entries in {} and {} in {}".format(
len(x), train_save_path, len(y), test_save_path))
def main(args):
with open(args.annotations, 'rt', encoding='UTF-8') as annotations:
coco = json.load(annotations)
info = coco['info']
licenses = coco['licenses']
images = coco['images']
annotations = coco['annotations']
categories = coco['categories']
number_of_images = len(images)
images_with_annotations = funcy.lmap(lambda a: int(a['image_id']), annotations)
if not os.path.exists(args.traindir):
pl.Path(args.traindir).mkdir(parents=True, exist_ok=True)
if not os.path.exists(args.testdir):
pl.Path(args.testdir).mkdir(parents=True, exist_ok=True)
if args.having_annotations:
images = funcy.lremove(lambda i: i['id'] not in images_with_annotations, images)
x, y = train_test_split(images, train_size=args.split)
save_coco(os.path.join(args.traindir,args.train), info, licenses, x, filter_annotations(annotations, x), categories)
save_coco(os.path.join(args.testdir,args.test), info, licenses, y, filter_annotations(annotations, y), categories)
for file in x:
shutil.copy(os.path.join(args.inputdir, file["file_name"]), args.traindir)
for file in y:
shutil.copy(os.path.join(args.inputdir, file["file_name"]), args.testdir)
print("Saved {} entries in {} and {} in {}".format(len(x), args.train, len(y), args.test))
def run_test(model, dataloader, criterion, device, threshold=None):
if device is None:
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
if isinstance(device, str):
device = torch.device(device)
progress_bar = tqdm(dataloader, desc="Running test")
model.eval()
predictions = []
for i, it in enumerate(progress_bar):
batch_data = it[0]
inputs = batch_data["inputs"]
inputs = inputs.to(device)
with torch.set_grad_enabled(False):
net_outputs = model(inputs)
net_outputs = torch.softmax(net_outputs, dim=1)
cls_outputs = funcy.lmap(lambda t: t.item(), net_outputs[:, 1])
if threshold is not None:
cls_outputs = [int(out > threshold) for out in cls_outputs]
predictions.extend(list(cls_outputs))
return predictions
def _add_infs(lock, record):
mark, infs, ip = record
statement = """
match(n:Olt {ip:{ip}})
merge (n)-[:HAS]->(i:Inf {name:{name}})
on create set i.desc={desc},i.state={state},i.bw={bw},
i.inTraffic={inTraffic},i.outTraffic={outTraffic},i.updated=timestamp()
on match set i.desc={desc},i.state={state},i.bw={bw},
i.inTraffic={inTraffic},i.outTraffic={outTraffic},i.updated=timestamp()"""
with lock:
with open(log_file, 'a') as lf:
lf.write('{ip}:{mark}\n'.format(ip=ip, mark=mark))
if mark == 'success' and infs:
tx = graph.cypher.begin()
lmap(lambda x: tx.append(statement, ip=ip, **x), infs)
tx.process()
tx.commit()
def _add_infs(lock, record):
mark, infs, ip = record
statement = """
match (s:Switch {ip:{ip}})
merge (s)-[:has]->(i:Inf {name:{name}})
set i.desc={desc},i.updated=timestamp()
with s,i
match (s)-->(g:Group {name:{group}})
merge (g)-[r:OWNED]->(i)
set r.updated=timestamp()
"""
with lock:
if mark == 'success':
tx = graph.cypher.begin()
lmap(lambda x: tx.append(statement, ip=ip, name=x[
'name'], desc=x['desc'], group=x['group']), infs)
tx.process()
tx.commit()
def get_vlan_users(bras):
def _get_vlan_users(bas):
funcs = {'m6k': M6k.get_vlan_users,
'me60': ME60.get_vlan_users}
_gvu = partial(_model, funcs)
return _gvu(bas)
bras = [dict(ip=x[0], model=x[1], inf=x[2])
for x in bras]
rslt = lmap(_get_vlan_users, bras)
return rslt
def get_onus(ip):
mark, ports = get_pon_ports(ip)[:-1]
if mark == 'fail':
return ('fail', None, ip)
try:
child = telnet(ip)
gpo = partial(get_port_onus, child)
rslt = lmap(gpo, ports)
child.sendline('exit')
child.close()
except (pexpect.EOF, pexpect.TIMEOUT) as e:
return ('fail', None, ip)
rslt1 = filter(lambda x: bool(x[1]), rslt)
return ('success', rslt1, ip)
def get_vlans(ip):
def _vlan(record):
if re_test(r'(Ports:\snone.*Ports:\snone)', record, re.S):
return 0
vlan = re_find(r'VLAN\sID:\s(\d+)', record)
vlan = int(vlan or 0)
return vlan
try:
child = telnet(ip)
rslt = do_some(child, 'disp vlan all')
close(child)
rslt = re.split(r'\r\n *\r\n', rslt)
vlans = select(lambda x: x > 1,
lmap(_vlan, rslt))
except (pexpect.EOF, pexpect.TIMEOUT) as e:
return ('fail', None, ip)
return ('success', vlans, ip)
def get_groups(ip):
def _get_group(child, group):
rslt = do_some(
child, 'disp link-aggregation {group}'.format(group=group))
desc = re_find(r'description:(\S+)', rslt)
mode = re_find(r'work mode:\s+(\S+)', rslt)
temp = re_all(r'(\d+/\d+)\s+(\d\S+)', rslt)
temp1 = lmapcat(lambda x: ['{0}/{1}'.format(x[0], y)
for y in x[1].split(',')], temp)
return dict(name=group, desc=desc, mode=mode, infs=temp1)
try:
child = telnet(ip)
temp = re_all(r'(\d+/\d+/\d+)', do_some(child,
'disp link-aggregation all'))
groups = lmap(partial(_get_group, child), temp)
close(child)
except (pexpect.EOF, pexpect.TIMEOUT) as e:
return ('fail', None, ip)
return ('success', groups, ip)
def get_ports(ip):
def _get_info(record):
name = re_find(r'((?:xg|g|f)ei\S+) is \w+ ?\w*,', record)
state = re_find(r'(?:xg|g|f)ei\S+ is (\w+ ?\w+),', record)
desc = re_find(r'Description is (\S+ *\S+)', record)
inTraffic = int(
re_find(r'120s input.*:\s(\d+)Bps', record)) * 8 / 1000000
outTraffic = int(
re_find(r'120s output.*:\s(\d+)Bps', record)) * 8 / 1000000
return dict(name=name, desc=desc, state=state, inTraffic=inTraffic, outTraffic=outTraffic)
try:
child = telnet(ip)
rslt = do_some(child, 'show interface include ei- | in (is |put rate)')
close(child)
except (pexpect.EOF, pexpect.TIMEOUT) as e:
return ('fail', None, ip)
rslt = re.split(r'\r\n *(?=(?:xg|g|f)ei-)', rslt)
rslt = lmap(_get_info, rslt)
return ('success', rslt, ip)
def get_traffics(ip, infs):
def _get_traffic(child, inf):
rslt = do_some(child, 'disp int {inf}'.format(inf=inf))
state = re_find(r'{inf}\scurrent\sstate\s:\s?(\w+\s?\w+)'
.format(inf=inf), rslt).lower()
bw = int(re_find(r'Speed\s+:\s+(\d+),', rslt))
inTraffic = int(
re_find(r'300 seconds input rate (\d+)\sbits/sec', rslt)) / 1000000
outTraffic = int(
re_find(r'300 seconds output rate (\d+)\sbits/sec', rslt)) / 1000000
infDict = dict(name=inf, state=state, bw=bw,
inTraffic=inTraffic, outTraffic=outTraffic)
return infDict
try:
child = telnet(ip)
rslt = lmap(partial(_get_traffic, child), infs)
close(child)
except (pexpect.EOF, pexpect.TIMEOUT) as e:
return ('fail', None, ip)
return ('success', rslt, ip)
def get_ports(ip):
def _get_info(record):
name = re_find(r'(\S+) current state :', record)
state = re_find(r'current state : ?(\S+ ?\S+)', record)
desc = re_find(r'Description:(\S+ *\S+)', record)
inTraffic = int(
re_find(r'300 seconds input rate (\d+)\sbits/sec', record) or 0) / 1000000
outTraffic = int(
re_find(r'300 seconds output rate (\d+)\sbits/sec', record) or 0) / 1000000
return dict(name=name, desc=desc, state=state, inTraffic=inTraffic, outTraffic=outTraffic)
try:
child = telnet(ip)
rslt = do_some(child, 'disp interface')
close(child)
except (pexpect.EOF, pexpect.TIMEOUT) as e:
return ('fail', None, ip)
rslt = select(lambda x: re_test(r'^x?gigabitethernet', x, re.I),
re.split(r'\r\n *\r\n *', rslt))
rslt = lmap(_get_info, rslt)
return ('success', rslt, ip)
def get_traffics(ip, infs):
def _get_traffic(child, inf):
rslt = do_some(child, 'show interface {inf}'.format(inf=inf))
state = re_find(
r'{inf}\sis\s(\w+\s?\w+),'.format(inf=inf), rslt).lower()
bw = int(re_find(r'BW\s(\d+)\sKbps', rslt)) / 1000
inTraffic = int(
re_find(r'120s input.*:\s(\d+)Bps', rslt)) * 8 / 1000000
outTraffic = int(
re_find(r'120s output.*:\s(\d+)Bps', rslt)) * 8 / 1000000
infDict = dict(name=inf, state=state, bw=bw,
inTraffic=inTraffic, outTraffic=outTraffic)
return infDict
try:
child = telnet(ip)
rslt = lmap(partial(_get_traffic, child), infs)
close(child)
except (pexpect.EOF, pexpect.TIMEOUT) as e:
return ('fail', None, ip)
return ('success', rslt, ip)
def get_ports(ip):
def _get_info(record):
name = re_find(r'((?:xg|g|f)ei\S+) is \w+ ?\w+,', record)
state = re_find(r'(?:xg|g|f)ei\S+ is (\w+ ?\w+),', record)
desc = re_find(r'Description is (\S+ *\S+)', record)
inTraffic = int(
re_find(r'120 seconds input.*:\s+(\d+)\sBps', record) or 0) * 8 / 1000000
outTraffic = int(
re_find(r'120 seconds output.*:\s+(\d+)\sBps', record) or 0) * 8 / 1000000
return dict(name=name, desc=desc, state=state, inTraffic=inTraffic, outTraffic=outTraffic)
try:
child = telnet(ip)
rslt = do_some(child, 'show interface')
close(child)
except (pexpect.EOF, pexpect.TIMEOUT) as e:
return ('fail', None, ip)
rslt = re.split(r'\r\n *\r\n', rslt)
rslt = select(lambda x: bool(x['name']),
lmap(_get_info, rslt))
return ('success', rslt, ip)
def get_ports(ip):
def _get_info(record):
name = re_find(r'(\S+) current state :', record)
state = re_find(r'current state : ?(\S+ ?\S+)', record)
desc = re_find(r'Description: (\S+ *\S+)', record)
inTraffic = int(re_find(
r'\d+ seconds input:\s+\d+\spackets/sec\s(\d+)\sbits/sec', record) or 0) / 1000000
outTraffic = int(re_find(
r'\d+ seconds output:\s+\d+\spackets/sec\s(\d+)\sbits/sec', record) or 0) / 1000000
return dict(name=name, desc=desc, state=state, inTraffic=inTraffic, outTraffic=outTraffic)
try:
child = telnet(ip)
rslt = do_some(child, 'disp interface')
close(child)
rslt = re.split(r'\r\n *\r\n', rslt)
rslt = select(lambda x: bool(x['name']),
lmap(_get_info, rslt))
except (pexpect.EOF, pexpect.TIMEOUT) as e:
return ('fail', None, ip)
return ('success', rslt, ip)
def bounding_box(domain, oracle, eps=1e-5):
"""Compute Bounding box. TODO: clean up"""
# TODO: remove r input and assume unit rec.
edges = [mdts.binsearch(r2, oracle, eps=eps) for r2 in box_edges(domain)]
rtypes = fn.pluck(0, edges)
if all(t == mdts.SearchResultType.TRIVIALLY_FALSE for t in rtypes):
return domain
elif all(t == mdts.SearchResultType.TRIVIALLY_TRUE for t in rtypes):
return mdtr.to_rec(domain.dim*[[0, 0]])
itvls = [r for t, r in edges if t == mdts.SearchResultType.NON_TRIVIAL]
def box_to_include(r):
return domain.backward_cone(r.top) & domain.forward_cone(r.bot)
bbox, *recs = fn.lmap(box_to_include, itvls)
for r in recs:
bbox = bbox.sup(r)
return bbox
def get_bingfa(ip):
def _get_users(child, slot):
record = do_some(
child, 'disp max-online slot {s}'.format(s=slot))
users = re_find(
r'Max online users since startup\s+:\s+(\d+)', record)
users = int(users or 0)
date = re_find(
r'Time of max online users\s+:\s+(\d{4}-\d{2}-\d{2})', record)
return (slot, users, date)
try:
child = telnet(ip)
rslt = do_some(child, 'disp dev | in BSU')
ff = compose(partial(select, bool),
partial(map, r'(\d+)\s+BSU'))
slots = ff(rslt.split('\r\n'))
maxUsers = lmap(partial(_get_users, child), slots)
close(child)
except (pexpect.EOF, pexpect.TIMEOUT) as e:
return ('fail', None, ip)
return ('success', maxUsers, ip)
def get_infs(ip):
def _get_desc(child, inf):
name = inf['name']
rslt = do_some(child, 'show run interface {name}'.format(name=name))
desc = re_find(r'description\s(\S+ *\S*)', rslt)
group = re_find(r'(smartgroup\s\d+)', rslt)
if group:
group = group.replace(' ', '')
inf['desc'] = desc
inf['group'] = group
return inf
try:
child = telnet(ip)
rslt = do_some(child, 'show run | in interface (xg|g|f)ei_')
temp = [dict(name=x) for x in re_all('interface\s(\S+)', rslt)]
get_desc = partial(_get_desc, child)
infs = lmap(get_desc, temp)
close(child)
except (pexpect.EOF, pexpect.TIMEOUT) as e:
return ('fail', None, ip)
return ('success', infs, ip)
def get_groups(ip):
def _get_desc(child, group):
name = group['name']
rslt = do_some(child, 'show run interface {name}'.format(name=name))
desc = re_find(r'description\s(\S+ *\S*)', rslt)
mode = re_find(r'smartgroup mode (\S+)', rslt)
if mode == '802.3ad':
mode = 'yes'
group['desc'] = desc
group['mode'] = mode
return group
try:
child = telnet(ip)
rslt = do_some(child, 'show lacp internal')
temp = re_all(r'Smartgroup:(\d+)', rslt)
temp1 = [dict(name='smartgroup' + x)
for x in temp]
groups = lmap(partial(_get_desc, child), temp1)
close(child)
except (pexpect.EOF, pexpect.TIMEOUT) as e:
return ('fail', None, ip)
return ('success', groups, ip)
def get_infs(ip):
def _get_info(child, inf):
rslt = do_some(child, 'show int {inf}'.format(inf=inf))
desc = re_find(r'Description\sis\s(\S+)', rslt)
state = re_find(r'{inf}\sis\s(\S+\s?\S+),'.format(inf=inf), rslt)
bw = re_find(r'BW\s(\d+)\sKbits', rslt)
bw = int(bw or 0) / 1000
inTraffic = re_find(r'seconds\sinput\srate\s?:\s+(\d+)\sBps', rslt)
inTraffic = int(inTraffic or 0) * 8 / 1e6
outTraffic = re_find(r'seconds\soutput\srate:\s+(\d+)\sBps', rslt)
outTraffic = int(outTraffic or 0) * 8 / 1e6
return dict(name=inf, desc=desc, state=state, bw=bw,
inTraffic=inTraffic, outTraffic=outTraffic)
try:
child = telnet(ip)
rslt = do_some(child, 'show run | in interface', timeout=180)
rslt = re_all(r'interface\s+(x?gei_\d+/\d+/\d+)', rslt)
infs = lmap(partial(_get_info, child), rslt)
close(child)
except (pexpect.EOF, pexpect.TIMEOUT) as e:
return ('fail', None, ip)
return ('success', infs, ip)
def get_groups(ip):
def _get_desc(child, group):
name = group['name']
rslt = do_some(
child, 'show running-config-interface {name} all'.format(name=name))
desc = re_find(r'description\s(\S+ *\S*)', rslt)
mode = re_find(r'lacp\smode\s(\S+)', rslt)
if mode == '802.3ad':
mode = 'yes'
group['desc'] = desc
group['mode'] = mode
return group
try:
child = telnet(ip)
rslt = do_some(child, 'show interface | in smartgroup')
temp = [dict(name=x) for x in
re_all(r'(smartgroup\d+)', rslt)]
groups = lmap(partial(_get_desc, child), temp)
close(child)
except (pexpect.EOF, pexpect.TIMEOUT) as e:
return ('fail', None, ip)
return ('success', groups, ip)
def get_groups(ip):
def _get_desc(child, group):
name = group['name']
rslt = do_some(child, 'show run interface {name}'.format(name=name))
desc = re_find(r'description\s(\S+ *\S*)', rslt)
group['desc'] = desc
if group['mode'] == 'active':
group['mode'] = 'yes'
return group
try:
child = telnet(ip)
rslt = do_some(child, 'show run | in smartgroup [0-9]+')
ff = rcompose(partial(map, lambda x: x.strip()),
distinct,
partial(map, r'(smartgroup\s\d+)\smode\s(\w+)'),
partial(map, lambda x: dict(name=x[0].replace(' ', ''), mode=x[1])))
temp = ff(rslt.splitlines()[:-1])
get_desc = partial(_get_desc, child)
groups = lmap(get_desc, temp)
close(child)
except (pexpect.EOF, pexpect.TIMEOUT) as e:
return ('fail', None, ip)
return ('success', groups, ip)
def power_check():
clear_log()
nodes = graph.find('Olt', property_key='company', property_value='hw')
olts = [(x['ip'], x['company'], x['area']) for x in nodes]
funcy.lmap(funcy.compose(output_info, get_power_info), olts)
def import_sw(file):
switchs = (x.strip() for x in open(file))
lmap(lambda x: graph.create(create_sw_node(x)), switchs)
def import_bras(file):
bras = (x.strip().split(',') for x in open(file))
brasNode = lambda x: graph.create(
Node('Bras', **dict(zip(('name', 'ip', 'model', 'area'), x))))
lmap(brasNode, bras)
