def flattened(t):
def fmt(k):
idxs = range(imap[k].size)
return [f"{k}##time_{t}[{i}]" for i in idxs]
actions = fn.lmapcat(fmt, inputs)
coin_flips = fn.lmapcat(fmt, env_inputs)
return actions + coin_flips
def get_values_at_codeable_paths(value: dict, keys: List[str]):
"""Extract values from FHIR records based on keys (useful for extracting codes)"""
def _get_value_at_codeable_path(value: Union[list, dict],
components: List[str], key: str):
if value is None:
return []
if isinstance(value, list):
return lmapcat(
lambda v: _get_value_at_codeable_path(v, components, key),
value)
if len(components) == 0:
return [Hashabledict({"field": key, **value})]
if not isinstance(value, dict) and not isinstance(value, pd.Series):
return []
return _get_value_at_codeable_path(value.get(components[0], None),
components[1:], key)
def get_value_at_codeable_path(value: dict, key: str):
return _get_value_at_codeable_path(value, key.split("."), key)
return lmapcat(lambda key: get_value_at_codeable_path(value, key), keys)
def all_ixy_idxes(tup_tree, beg_idx=1):
''' Gey ixy indexes in (tuple mapped) tree '''
return F.lmapcat(
tup(lambda v,p,l,r:
(([l] if l < 0 else [])
+([r] if r < 0 else []))),
all_inodes(tup_tree, beg_idx))
def _get_group(child, group):
rslt = do_some(
child, 'disp link-aggregation {group}'.format(group=group))
desc = re_find(r'description:(\S+)', rslt)
mode = re_find(r'work mode:\s+(\S+)', rslt)
temp = re_all(r'(\d+/\d+)\s+(\d\S+)', rslt)
temp1 = lmapcat(lambda x: ['{0}/{1}'.format(x[0], y)
for y in x[1].split(',')], temp)
return dict(name=group, desc=desc, mode=mode, infs=temp1)
def _get_group(child, group):
rslt = do_some(
child, 'disp link-aggregation {group}'.format(group=group))
desc = re_find(r'description:(\S+)', rslt)
mode = re_find(r'work mode:\s+(\S+)', rslt)
temp = re_all(r'(\d+/\d+)\s+(\d\S+)', rslt)
temp1 = lmapcat(
lambda x: ['{0}/{1}'.format(x[0], y) for y in x[1].split(',')],
temp)
return dict(name=group, desc=desc, mode=mode, infs=temp1)
def add_prefixes(values: List[str], prefixes: List[str]):
"""Add prefix to each value if not already present"""
if len(prefixes) == 0:
return values
return lmapcat(
lambda prefix: [(value
if value.startswith(prefix) else f"{prefix}{value}")
for value in values],
prefixes,
)
def saveOnus_f(ip):
mark, rslt = Zte.get_onus(ip)[:-1]
if mark == 'success' and rslt:
_ff = lambda x: walk(partial(merge, (ip, x[0])), x[1])
rslt1 = lmapcat(_ff, rslt)
with open(result_file, 'a') as frslt:
for record in rslt1:
ip, port, onuid, loid = record
frslt.write("{ip},{port},{onuid},{loid}\n"
.format(ip=ip, port=port, onuid=onuid, loid=loid))
with open(log_file, 'a') as flog:
flog.write("{ip}:{mark}\n".format(ip=ip, mark=mark))
def prob(self, guard, *, log=False):
assert all(self.is_random(i) for i in guard.inputs)
order = list(guard.inputs)
guard = guard.bundle_inputs('##coins', order=order)
coin_biases = fn.lmapcat(self.coin_biases, order)
pcirc = C.PCirc(
circ=guard.aigbv,
coin_biases=len(order) * coin_biases,
coins_id='##coins',
)
return C.infer.prob(pcirc, log=log)
def get_ip_pool(ip):
def _get_sections(child, name):
rslt = do_some(child, 'disp cu configuration ip-pool {name}'.format(name=name))
sections = re_all(r'section \d+ (\S+) (\S+)', rslt)
return sections
try:
child = telnet(ip)
rslt = do_some(child, 'disp domain 163.js | in pool-name')
poolNames = re_all(r'pool-name\s+:\s(\S+)', rslt)
ips = lmapcat(partial(_get_sections, child), poolNames)
close(child)
except(pexpect.EOF, pexpect.TIMEOUT) as e:
return ('fail', None, ip)
return ('success', ips, ip)
def saveOnus_f(ip):
mark, rslt = Zte.get_onus(ip)[:-1]
if mark == 'success' and rslt:
_ff = lambda x: walk(partial(merge, (ip, x[0])), x[1])
rslt1 = lmapcat(_ff, rslt)
with open(result_file, 'a') as frslt:
for record in rslt1:
ip, port, onuid, loid = record
frslt.write("{ip},{port},{onuid},{loid}\n".format(ip=ip,
port=port,
onuid=onuid,
loid=loid))
with open(log_file, 'a') as flog:
flog.write("{ip}:{mark}\n".format(ip=ip, mark=mark))
def get_vlan_users(ip, inf):
def _get_users(child, i):
rslt = do_some(child, 'disp access-user interface {i} | in /'.format(i=i))
users = re_all(r'(\d+)/', rslt)
return users
try:
child = telnet(ip)
infs = do_some(child, 'disp cu interface | in Eth-Trunk{inf}\.'.format(inf=inf))
infs = re_all(r'interface (\S+)', infs)
rslt = lmapcat(partial(_get_users, child), infs)
close(child)
rslt = count_by(int, rslt)
except (pexpect.EOF, pexpect.TIMEOUT) as e:
return ('fail', None, ip)
return ('success', rslt, ip)
def get_vlan_users(ip, inf):
def _get_users(child, i):
rslt = do_some(child, 'show subscriber interface {i} | in external-vlan'.format(i=i))
vlans = re_all(r'external-vlan\s+:(\d+)', rslt)
return vlans
try:
child = telnet(ip)
rslt = do_some(child, 'show running-config | in smartgroup{inf}\.'.format(inf=inf))
infs = distinct(re_all(r'(smartgroup\S+)', rslt))
vlans = lmapcat(partial(_get_users, child), infs)
close(child)
vlans = count_by(int, vlans)
except (pexpect.EOF, pexpect.TIMEOUT) as e:
return ('fail', None, ip)
return ('success', vlans, ip)
def get_ip_pool(ip):
def _get_sections(child, name):
rslt = do_some(
child, 'disp cu configuration ip-pool {name}'.format(name=name))
sections = re_all(r'section \d+ (\S+) (\S+)', rslt)
return sections
try:
child = telnet(ip)
rslt = do_some(child, 'disp domain 163.js | in pool-name')
poolNames = re_all(r'pool-name\s+:\s(\S+)', rslt)
ips = lmapcat(partial(_get_sections, child), poolNames)
close(child)
except (pexpect.EOF, pexpect.TIMEOUT) as e:
return ('fail', None, ip)
return ('success', ips, ip)
def get_infs(ip):
def _get_inf(child, board):
slot, boardName = board
rslt = do_some(child, 'disp board 0/{slot}'.format(slot=slot))
rslt = [
re_find(r'(\d+).*-\s+(?:auto_)?(\d+)', x)
for x in rslt.split('\r\n') if 'online' in x
]
if boardName.lower() == 'gic':
boardName = 'giu'
child.sendline('conf')
child.expect(prompter)
child.sendline('interface {boardName} 0/{slot}'.format(
boardName=boardName, slot=slot))
child.expect(prompter)
temp = []
for x, y in rslt:
traffic = do_some(child, 'disp port traffic {port}'.format(port=x))
inTraffic, outTraffic = re_all(r'\(octets/s\)\s+=(\d+)', traffic)
inTraffic = int(inTraffic) * 8 / 1e6
outTraffic = int(outTraffic) * 8 / 1e6
bw = int(y or 0)
temp.append(
dict(
name='0/{slot}/{port}'.format(slot=slot, port=x),
desc='cannot set',
bw=bw,
state='up',
inTraffic=inTraffic,
outTraffic=outTraffic))
child.sendline('quit')
child.expect(prompter)
child.sendline('quit')
child.expect(prompter)
return temp
try:
child = telnet(ip)
rslt = do_some(child, 'disp board 0')
boards = re_all(r'(\d+)\s+\w+(eth|gic)\w+\s+normal', rslt, flags=re.I)
infs = lmapcat(partial(_get_inf, child), boards)
close(child)
except (pexpect.EOF, pexpect.TIMEOUT):
return ('fail', None, ip)
return ('success', infs, ip)
def _get_value_at_codeable_path(value: Union[list, dict],
components: List[str], key: str):
if value is None:
return []
if isinstance(value, list):
return lmapcat(
lambda v: _get_value_at_codeable_path(v, components, key),
value)
if len(components) == 0:
return [Hashabledict({"field": key, **value})]
if not isinstance(value, dict) and not isinstance(value, pd.Series):
return []
return _get_value_at_codeable_path(value.get(components[0], None),
components[1:], key)
def get_vlan_users(ip, inf):
def _get_users(child, i):
rslt = do_some(child,
'disp access-user interface {i} | in /'.format(i=i))
users = re_all(r'(\d+)/', rslt)
return users
try:
child = telnet(ip)
infs = do_some(
child, 'disp cu interface | in Eth-Trunk{inf}\.'.format(inf=inf))
infs = re_all(r'interface (\S+)', infs)
rslt = lmapcat(partial(_get_users, child), infs)
close(child)
rslt = count_by(int, rslt)
except (pexpect.EOF, pexpect.TIMEOUT) as e:
return ('fail', None, ip)
return ('success', rslt, ip)
def fetch(pattern, dry_run, workers) -> None:
"""Fetch assets for datasets matching PATTERN."""
# Hack to work around issue with sphinx-click:
# https://github.com/click-contrib/sphinx-click/issues/86#issuecomment-991196764
from qgreenland.util.config.config import (
get_config,
init_config,
)
from qgreenland.util.luigi import fetch_tasks_from_dataset
init_config()
config = get_config()
dataset_matches = select(
lambda i: fnmatch(i[1].id, pattern),
config.datasets,
).values()
print('Fetching all assets for the following datasets:')
print(textwrap.indent(
'\n'.join([d.id for d in dataset_matches]),
' - ',
))
if dry_run:
print('DRY RUN enabled. Aborting fetch.')
return
fetch_tasks = lmapcat(
lambda i: fetch_tasks_from_dataset(i),
dataset_matches,
)
result = luigi.build(
fetch_tasks,
workers=workers,
# Unlike CLI, running tasks from Python does not feature an "identical
# process lock" by default.
no_lock=False,
detailed_summary=True,
)
if not result.scheduling_succeeded:
raise click.UsageError('Scheduling failed. See error log above.')
def to_bdd2(mdp, horizon, output=None, manager=None):
"""
Compute the BDD for `output`'s value after unrolling the dynamics
(`mdp`) `horizon` steps.
Returns a triplet of:
1. The BDD.
2. The BDD Manager.
3. The order object determining if a given bit is a decision or
chance bit.
"""
if output is None:
assert len(mdp.outputs) == 1
output = fn.first(mdp.outputs)
circ = cone(mdp.aigbv, output)
inputs, env_inputs = mdp.inputs, circ.inputs - mdp.inputs
imap = circ.imap
def flattened(t):
def fmt(k):
idxs = range(imap[k].size)
return [f"{k}##time_{t}[{i}]" for i in idxs]
actions = fn.lmapcat(fmt, inputs)
coin_flips = fn.lmapcat(fmt, env_inputs)
return actions + coin_flips
unrolled_inputs = fn.lmapcat(flattened, range(horizon))
levels = {k: i for i, k in enumerate(unrolled_inputs)}
circ2 = BV.AIGBV(circ.aig.lazy_aig).unroll(horizon, only_last_outputs=True)
bexpr, *_ = aiger_bdd.to_bdd(circ2, levels=levels, renamer=lambda _, x: x)
def count_bits(inputs):
return sum(imap[i].size for i in inputs)
order = BitOrder(count_bits(inputs), count_bits(env_inputs), horizon)
return bexpr, bexpr.bdd, order
def get_active_port(ip):
def _get_active_port(child, slot):
info = do_some(child, 'display board 0/{0}'.format(slot))
ports = [
re_find(r'\d+', x) for x in info.split('\r\n')
if re_test(r'ge\s+normal.*online(?i)', x)
]
return ['{0}/{1}'.format(slot, port) for port in ports]
try:
child = telnet(ip)
rslt = do_some(child, 'display board 0')
slots = [
re_find(r'\d+', x) for x in rslt.split('\r\n')
if re_test(r'normal(?i)', x)
]
ports = lmapcat(lambda slot: _get_active_port(child, slot), slots)
close(child)
except Exception:
return [[ip, 'HW', 'failed']]
ports = [[ip, 'successed', x] for x in ports]
return ports
def get_infs(ip):
def _get_inf(child, board):
slot, boardName = board
rslt = do_some(child, 'disp board 0/{slot}'.format(slot=slot))
rslt = [re_find(r'(\d+).*-\s+(?:auto_)?(\d+)', x)
for x in rslt.split('\r\n')
if 'online' in x]
if boardName.lower() == 'gic':
boardName = 'giu'
child.sendline('conf')
child.expect(prompter)
child.sendline(
'interface {boardName} 0/{slot}'.format(boardName=boardName, slot=slot))
child.expect(prompter)
temp = []
for x, y in rslt:
traffic = do_some(child, 'disp port traffic {port}'.format(port=x))
inTraffic, outTraffic = re_all(r'\(octets/s\)\s+=(\d+)', traffic)
inTraffic = int(inTraffic) * 8 / 1e6
outTraffic = int(outTraffic) * 8 / 1e6
bw = int(y or 0)
temp.append(dict(name='0/{slot}/{port}'.format(slot=slot, port=x),
desc='cannot set', bw=bw, state='up',
inTraffic=inTraffic, outTraffic=outTraffic))
child.sendline('quit')
child.expect(prompter)
child.sendline('quit')
child.expect(prompter)
return temp
try:
child = telnet(ip)
rslt = do_some(child, 'disp board 0')
boards = re_all(r'(\d+)\s+\w+(eth|gic)\w+\s+normal', rslt, flags=re.I)
infs = lmapcat(partial(_get_inf, child), boards)
close(child)
except (pexpect.EOF, pexpect.TIMEOUT) as e:
return ('fail', None, ip)
return ('success', infs, ip)
def blast(bmap, vals):
return fn.lmapcat(bmap.get, vals)
def class_tree(cls):
return [cls] + lmapcat(class_tree, cls.__subclasses__())
def sink(wordlen, inputs):
imap = BundleMap({i: wordlen for i in inputs})
return aigbv.AIGBV(imap=imap, aig=aiger.sink(fn.lmapcat(imap.get, inputs)))
def _list_mapcat(func):
return lambda els: lmapcat(func, els) if isinstance(els, list) else func(
els)
def lmapcat(f,*seq):
return F.lmapcat(f,*seq) if seq \
else lambda *xs: F.lmapcat(f,*xs)
