def _get_active_port(child, slot):
info = do_some(child, 'display board 0/{0}'.format(slot))
ports = [
re_find(r'\d+', x) for x in info.split('\r\n')
if re_test(r'ge\s+normal.*online(?i)', x)
]
return ['{0}/{1}'.format(slot, port) for port in ports]
def _extract_latest_from_search_triple(triple):
"""Try to extract latest version number from a triple of search results"""
description, installed, latest = triple
if re_test(r'\s*ballet \(.+\)\s*-\s*\w*', description):
if 'INSTALLED' in installed and 'LATEST' in latest:
return re_find(r'\s*LATEST:\s*(.+)', latest)
return None
def check(self, diff):
"""Check that the name of the subpackage within contrib is valid
The package name must match ``user_[a-zA-Z0-9_]+``.
"""
relative_path = relative_to_contrib(diff, self.project)
subpackage_name = relative_path.parts[0]
assert re_test(SUBPACKAGE_NAME_REGEX, subpackage_name)
def check(self, diff):
r"""Check that the new file introduced has a valid name
The module can either be an __init__.py file or must
match ``feature_[a-zA-Z0-9_]+\.\w+``.
"""
filename = pathlib.Path(diff.b_path).parts[-1]
is_valid_feature_module_name = re_test(
FEATURE_MODULE_NAME_REGEX, filename)
is_valid_init_module_name = filename == '__init__.py'
assert is_valid_feature_module_name or is_valid_init_module_name
def del_onu():
records = (x.strip().split(',') for x in open('e8c_diff.csv'))
for ip, port, onuid, loid in records:
child = Zte.telnet(ip)
rslt = Zte.do_some(child, 'show run {port}'.format(port=port))
if re_test(r'onu\s{0}\stype\sE8C[PG]24\sloid\s{1}'.format(onuid, loid),
rslt):
child.sendline('conf t')
child.expect('#')
child.sendline(port)
child.expect('#')
child.sendline('no onu {onuid}'.format(onuid=onuid))
child.expect('#')
Zte.close(child)
def del_onu():
records = (x.strip().split(',') for x in open('e8c_diff.csv'))
for ip, port, onuid, loid in records:
child = Zte.telnet(ip)
rslt = Zte.do_some(child, 'show run {port}'.format(port=port))
if re_test(r'onu\s{0}\stype\sE8C[PG]24\sloid\s{1}'.format(onuid, loid),
rslt):
child.sendline('conf t')
child.expect('#')
child.sendline(port)
child.expect('#')
child.sendline('no onu {onuid}'.format(onuid=onuid))
child.expect('#')
Zte.close(child)
def get_vlans_of_port(ip, port):
try:
child = telnet(ip)
rslt = do_some(child, f'disp cu interface {port}')
eth_trunk = re_find(r'eth-trunk \d+', rslt).replace(' ', '')
rslt = do_some(child, 'disp cu interface filter user-vlan')
close(child)
except Exception as e:
raise e
rslt = rcompose(methodcaller('split', '#'),
autocurry(filter)(lambda x: re_test(eth_trunk, x, re.I)),
autocurry(mapcat)(lambda x: x.split('\r\n')),
autocurry(filter)('user-vlan'),
autocurry(map)(lambda x: x.strip()),
autocurry(map)(lambda x: _item_to_vlans(x)))(rslt)
return merge(set(), *rslt)
def get_svlan(ip):
def _format(port):
temp = port.split('/')
temp = map(lambda x: x if len(x) > 1 else '0' + x, temp)
return '/'.join(temp)
try:
child = telnet(ip)
rslt = do_some(
child, 'display service-port all | include epon', timeout=300)
close(child)
except (pexpect.EOF, pexpect.TIMEOUT):
return [(ip, 'HW', 'fail')]
rslt = re.split(r'\r\n\s*', rslt)
rslt = (re.sub(r'(0/\d) (/\d)', r'\1\2', x) for x in rslt
if re_test(r'^\d+', x))
rslt = (re.split(r'\s+', x) for x in rslt)
rslt = [(ip, _format(x[4]), x[1]) for x in rslt
if 51 <= int(x[8].split('-')[0]) <= 1999]
return rslt
def get_ports(ip):
def _get_info(record):
name = re_find(r'(\S+) current state :', record)
state = re_find(r'current state : ?(\S+ ?\S+)', record)
desc = re_find(r'Description:(\S+ *\S+)', record)
inTraffic = int(
re_find(r'300 seconds input rate (\d+)\sbits/sec', record) or 0) / 1000000
outTraffic = int(
re_find(r'300 seconds output rate (\d+)\sbits/sec', record) or 0) / 1000000
return dict(name=name, desc=desc, state=state, inTraffic=inTraffic, outTraffic=outTraffic)
try:
child = telnet(ip)
rslt = do_some(child, 'disp interface')
close(child)
except (pexpect.EOF, pexpect.TIMEOUT) as e:
return ('fail', None, ip)
rslt = select(lambda x: re_test(r'^x?gigabitethernet', x, re.I),
re.split(r'\r\n *\r\n *', rslt))
rslt = lmap(_get_info, rslt)
return ('success', rslt, ip)
def get_active_port(ip):
def _get_active_port(child, slot):
info = do_some(child, 'display board 0/{0}'.format(slot))
ports = [
re_find(r'\d+', x) for x in info.split('\r\n')
if re_test(r'ge\s+normal.*online(?i)', x)
]
return ['{0}/{1}'.format(slot, port) for port in ports]
try:
child = telnet(ip)
rslt = do_some(child, 'display board 0')
slots = [
re_find(r'\d+', x) for x in rslt.split('\r\n')
if re_test(r'normal(?i)', x)
]
ports = lmapcat(lambda slot: _get_active_port(child, slot), slots)
close(child)
except Exception:
return [[ip, 'HW', 'failed']]
ports = [[ip, 'successed', x] for x in ports]
return ports
def get_active_port(ip):
def _get_active_port(child, inf):
info = do_some(child, 'show {0}'.format(inf))
if re_test(r'line\sprotocol\sis\sup', info):
return inf
else:
return ''
try:
child = telnet(ip)
rslt = do_some(child, 'show run | include interface', timeout=300)
infs = [
_get_active_port(child, inf) for inf in rslt.split('\r\n')
if re_test(r'interface (xg|g)ei(?i)', inf)
]
close(child)
except Exception:
return [[ip, 'ZTE', 'failed']]
infs = [x.split()[1] for x in infs if x]
infs = [[ip, 'successed', x] for x in infs]
return infs
def _fail_if_feature_exists(dst: pathlib.Path) -> None:
subpackage_name, feature_name = str(dst.parent), str(dst.name)
if (dst.is_file() and fy.re_test(SUBPACKAGE_NAME_REGEX, subpackage_name)
and fy.re_test(FEATURE_MODULE_NAME_REGEX, feature_name)):
raise FileExistsError(f'The feature already exists here: {dst}')
def _check_environment(self):
assert re_test(PR_REF_PATH_REGEX, self._pr_path)
def __new__(cls, content):
content = cls.from_unicode(content)
if not funcy.re_test(r'^[A-Ga-g][b#]?$', content):
raise ValueError('"{0}" is not a valid pyleadsheet note'.format(content))
content = content.capitalize()
return str.__new__(cls, content)
def _vlan(record):
if re_test(r'(Ports:\snone.*Ports:\snone)', record, re.S):
return 0
vlan = re_find(r'VLAN\sID:\s(\d+)', record)
vlan = int(vlan or 0)
return vlan
def _vlan(record):
if re_test(r'(Ports:\snone.*Ports:\snone)', record, re.S):
return 0
vlan = re_find(r'VLAN\sID:\s(\d+)', record)
vlan = int(vlan or 0)
return vlan
def tarantool_docker_image():
subprocess.check_call(["make", "-s", "docker"], shell=False)
image_name = (subprocess.check_output(["make", "-s", "docker-image-name"],
shell=False).decode().strip())
assert re_test(r"^\S+$", image_name), f"bad image name: {image_name!r}"
return image_name
def _item_to_vlans(item):
if re_test(r'qinq \d+ \d+', item, re.I):
start, end = re_find(r'qinq (\d+) (\d+)', item, re.I)
return range(int(start), int(end) + 1)
vlan = item.split()[-1]
return [int(vlan)]
def _get_active_port(child, inf):
info = do_some(child, 'show {0}'.format(inf))
if re_test(r'line\sprotocol\sis\sup', info):
return inf
else:
return ''
def fill_probes(platform_id):
platform = Platform.objects.get(pk=platform_id)
gpl_name = platform.gpl_name
cprint('%s %s %s' % (platform.pk, platform.gpl_name, platform.specie),
attrs=['bold'])
assert platform.specie
platform.verdict = ''
platform.probes_total = None
platform.probes_matched = None
platform.stats = {}
platform.last_filled = timezone.now()
annot_file = '/pub/geo/DATA/annotation/platforms/%s.annot.gz' % gpl_name
family_file = '/pub/geo/DATA/SOFT/by_platform/%s/%s_family.soft.gz' % (
gpl_name, gpl_name)
files = [annot_file, family_file]
tables = list(map(peek_platform, files))
# Skip empty
files = list(compress(files, tables))
tables = lkeep(tables)
# TODO: check other supplementary files formats
supplementary_dir = '/pub/geo/DATA/supplementary/platforms/%s/' % gpl_name
_, supplementary_files = listdir(supplementary_dir)
supplementary_files = [
f for f in supplementary_files
if f.endswith('.txt.gz') and not re_test('\.cdf\.', f, re.I)
]
files.extend(supplementary_files)
tables.extend(
decompress(download('%s%s' % (supplementary_dir, f)))
for f in supplementary_files)
platform.stats['files'] = lkeep(files)
if not any(tables):
cprint('No data for %s' % gpl_name, 'red')
platform.verdict = 'no data'
platform.save()
return
# Read tables in
df = pd.concat(
read_table(table, file) for table, file in zip(tables, files))
del tables # free memory
platform.probes_total = len(set(df.index))
cprint('Found %d probes to match' % platform.probes_total, 'yellow')
# import ipdb; ipdb.set_trace() # noqa
# Try to resolve probes starting from best scopes
mygene_probes = []
platform.stats['matches'] = []
platform.verdict = 'no clue'
for scopes, cols in SCOPE_COLUMNS:
cols = list(set(cols) & set(df.columns))
if not cols:
continue
cprint('> Looking into %s' % ', '.join(sorted(cols)), 'cyan')
platform.verdict = 'nothing matched'
probes = pd.concat(df[col].dropna() for col in cols)
new_matches = mygene_fetch(platform, probes, scopes)
mygene_probes.extend(new_matches)
# Drop matched probes
if new_matches:
platform.stats['matches'].append({
'scopes': scopes,
'cols': cols,
'found': len(new_matches),
})
df = df.drop(lpluck('probe', new_matches))
if df.empty:
break
# Update stats and history
platform.probes_matched = len(mygene_probes)
platform.history.append({
'time': timezone.now().strftime('%Y-%m-%d %T'),
'probes_total': platform.probes_total,
'probes_matched': platform.probes_matched,
})
# Insert found genes
if mygene_probes:
with transaction.atomic():
platform.verdict = 'ok'
platform.save()
platform.probes.all().delete()
PlatformProbe.objects.bulk_create([
PlatformProbe(platform=platform, **probe_info)
for probe_info in mygene_probes
])
cprint('Inserted %d probes for %s' % (len(mygene_probes), gpl_name),
'green')
else:
cprint('Nothing matched for %s' % gpl_name, 'red')
platform.save()
