def __init__(self, *args, **kwargs):
self._view = None
self._session = None
self._active = False
self._filters = AttrDict(
{n: f(provider=self, name=n)
for n, f in self.FILTERS.items()})
rules = AttrDict(self.config.rules.label,
**config.settings.profile.rules.label)
labels = AttrDict(self.config.labels, **config.settings.profile.labels)
self.rule_map = AttrDict([(re.compile(k, re.IGNORECASE), v)
for k, v in [(r, rules[r])
for r in rules.keys()]])
self.highlight_map = AttrDict([(re.compile(k,
re.IGNORECASE), labels[v])
for k, v in rules.items()])
self.highlight_re = re.compile(
"(" + "|".join([k.pattern
for k in self.highlight_map.keys()]) + ")",
re.IGNORECASE)
def get_port_to_mac(mac_address_table, fmt='std', port_prefix=' '):
"""Convert MAC address table to dictionary.
Args:
mac_address_table (string): MAC address table. Lines delimited
with line feed. Assumes a header row with "Port" as a column
header followed by a delimiter row composed of dashes ('-')
which delimit columns. Handles MAC addresses formatted
as 'cc:cc:cc:cc:cc:cc' or 'cccc.cccc.cccc'
Returns:
dictionary: Keys are string port numbers and values are a list
of MAC addresses both in native switch format.
"""
import lib.logger as logger
log = logger.getlogger()
pos = None
mac_dict = AttrDict()
_mac_iee802 = '([\dA-F]{2}[\.:-]){5}([\dA-F]{2})'
_mac_cisco = '([\dA-F]{4}\.){2}[\dA-F]{4}'
_mac_all = "%s|%s" % (_mac_iee802, _mac_cisco)
_mac_regex = re.compile(_mac_all, re.I)
mac_address_table = mac_address_table.splitlines()
p2 = re.compile('Port', re.IGNORECASE)
for line in mac_address_table:
# find row with 'Port' label
match = p2.search(line)
if match:
pos = match.start()
# find header seperator row
if re.search(r'--+', line):
log.debug('Found header seperator row: {}'.format(line))
iter = re.finditer(r'--+', line)
for i, match in enumerate(iter):
# find column aligned with 'Port'
if pos >= match.span()[0] and pos < match.span()[1]:
port_span = (match.span()[0], match.span()[1])
# find rows with MACs
match = _mac_regex.search(line)
if match:
mac = match.group()
log.debug('Found mac address: {}'.format(mac))
_mac = mac
if fmt == 'std':
_mac = mac[0:2]
mac = re.sub(r'\.|\:', '', mac)
for i in (2, 4, 6, 8, 10):
_mac = _mac + ':' + mac[i:i + 2]
# Extract port section of row
port = line[port_span[0] - 1:port_span[1]].strip(' ')
if fmt == 'std':
port = port.replace(port_prefix, '')
if port not in mac_dict.keys():
mac_dict[port] = [_mac]
else:
mac_dict[port].append(_mac)
return mac_dict
class VLANS:
'''
Manage a bunch of VLANs, as a dictionary
'''
def __init__(self, vlans=None, delemiter=",", range_delemiter="-"):
super().__init__()
self._delemiter = delemiter
self._range_delemiter = range_delemiter
self._vlans = AttrDict()
if vlans:
self.__iadd__(vlans)
def __add__(self, other):
"""
Add two VLANS to each other
"""
if not isinstance(other, VLANS):
raise TypeError("Error: Can only handle object of VLANS()")
tmp = self.copy()
for vlan in other._vlans.values():
tmp._vlans[vlan.id] = vlan
return tmp
def __iadd__(self, other):
if isinstance(other, VLANS):
for vlan in other._vlans.values():
self._vlans[vlan.id] = vlan
elif isinstance(other, VLAN):
self._vlans[other.id] = other
else:
raise TypeError(
"Error: Can only handle object of VLANS() or VLAN() got %s" %
type(other))
return self
def __str__(self):
return dict_to_vlan_str(self._vlans,
delemiter=self._delemiter,
range_delemiter=self._range_delemiter)
def __repr__(self):
s = ""
for vlan in self._vlans.values():
s += "(%s)" % vlan.to_str()
return "VLANS(%s)" % s
def __iter__(self):
return iter(self.__dict__)
def items(self):
for item in self._vlans.items():
yield item
def keys(self):
for item in self._vlans.keys():
yield item
def values(self):
for item in self._vlans.values():
yield item
def updateReport(_id):
data = request.form
data = AttrDict(data)
for key in data.keys():
print("key", key, "value", data[key] )
_id = db.conn.update("report", d=data, primary_key="_id")
return jsonify( id=_id )
class VLANS:
'''
Manage a bunch of VLANs, as a dictionary
'''
def __init__(self, vlans=None, delemiter=",", range_delemiter="-"):
super().__init__()
self._delemiter = delemiter
self._range_delemiter = range_delemiter
self._vlans = AttrDict()
if vlans:
self.__iadd__(vlans)
def __add__(self, other):
"""
Add two VLANS to each other
"""
if not isinstance(other, VLANS):
raise TypeError("Error: Can only handle object of VLANS()")
tmp = self.copy()
for vlan in other._vlans.values():
tmp._vlans[vlan.id] = vlan
return tmp
def __iadd__(self, other):
if isinstance(other, VLANS):
for vlan in other._vlans.values():
self._vlans[vlan.id] = vlan
elif isinstance(other, VLAN):
self._vlans[other.id] = other
else:
raise TypeError("Error: Can only handle object of VLANS() or VLAN() got %s" % type(other))
return self
def __str__(self):
return dict_to_vlan_str(self._vlans,
delemiter=self._delemiter,
range_delemiter=self._range_delemiter)
def __repr__(self):
s = ""
for vlan in self._vlans.values():
s += "(%s)" % vlan.to_str()
return "VLANS(%s)" % s
def __iter__(self):
return iter(self.__dict__)
def items(self):
for item in self._vlans.items():
yield item
def keys(self):
for item in self._vlans.keys():
yield item
def values(self):
for item in self._vlans.values():
yield item
def load():
global PROVIDERS
global DEFAULT_PROVIDER
mgr = extension.ExtensionManager(
namespace='streamglob.providers',
on_load_failure_callback=log_plugin_exception,
)
PROVIDERS = AttrDict((x.name, x.plugin()) for x in mgr)
load_config()
if len(config.settings.profile.providers):
# first listed in config
DEFAULT_PROVIDER = list(config.settings.profile.providers.keys())[0]
else:
# first loaded
DEFAULT_PROVIDER = list(PROVIDERS.keys())[0]
class ValidateClusterHardware(object):
"""Discover and validate cluster hardware
Args:
log (object): Log
"""
def __init__(self, config_file=None):
self.log = logger.getlogger()
try:
self.cfg = Config(config_file)
self.inv = Inventory(None, config_file)
except UserException as exc:
self.log.critical(exc)
raise UserException(exc)
# initialize ipmi list of access info
self.ran_ipmi = False
self.bmc_ai = {}
vlan_ipmi = self.cfg.get_depl_netw_client_vlan(if_type='ipmi')[0]
vlan_pxe = self.cfg.get_depl_netw_client_vlan(if_type='pxe')[0]
self.dhcp_pxe_leases_file = GEN_PATH + \
'logs/dnsmasq{}.leases'.format(vlan_pxe)
self.dhcp_ipmi_leases_file = GEN_PATH + \
'logs/dnsmasq{}.leases'.format(vlan_ipmi)
self.tcp_dump_file = GEN_PATH + \
'logs/tcpdump{}.out'.format(vlan_pxe)
self.node_table_ipmi = AttrDict()
self.node_table_pxe = AttrDict()
self.node_list = []
def _add_offset_to_address(self, addr, offset):
"""calculates an address with an offset added.
Args:
addr (str): ipv4 or cidr representation of address
offset (int): integer offset
Returns:
addr_.ip (str) address in ipv4 representation
"""
addr_ = IPNetwork(addr)
addr_.value += offset
return str(addr_.ip)
def _get_port_cnts(self):
labels = self.cfg.get_sw_mgmt_label()
ipmi_cnt = 0
pxe_cnt = 0
for label in labels:
ipmi_cnt += len(self.cfg.get_client_switch_ports(label, 'ipmi'))
pxe_cnt += len(self.cfg.get_client_switch_ports(label, 'pxe'))
return ipmi_cnt, pxe_cnt
def _get_credentials(self, node_addr_list, cred_list):
""" Attempts to discover bmc credentials and generate a list of all
discovered nodes. For each node try all available credentials. If no
credentials allow access, the node is not marked as succesful.
Args:
node_addr_list (list): list of ipv4 addresses for the discovered
nodes. (ie those that previously fetched an address from the DHCP
server.
cred_list (list of lists): Each list item is a list containing the
the userid, password, bmc_type and number of nodes for a node template.
return: bmc access info (dict) : Values hold tuple of userid, password,
bmc_type
"""
tot = [cred_list[x][3] for x in range(len(cred_list))]
tot = sum(tot)
left = tot
max_attempts = 20
delay = 5
attempt = 0
timeout = 4
print()
self.log.info("Discover BMC credentials and verify communications")
print()
nodes = {}
bmc_ai = {}
for node in node_addr_list:
nodes[node] = False
while not all([x for x in nodes.values()]) and attempt <= max_attempts:
print(f'\rAttempt count: {max_attempts - attempt} ', end='')
sys.stdout.flush()
attempt += 1
timeout += 1
node_list = [x for x in nodes if not nodes[x]]
for node in node_list:
# each time through re-sort cred_list based on nodes left with
# those credentials to maximize the probability
# of using the correct credentials with minimum attempts
cred_list.sort(key=lambda x: x[3], reverse=True)
for j, creds in enumerate(cred_list):
self.log.debug(
f'BMC {node} - Trying userid: {creds[0]} | '
f'password: {creds[1]} | bmc type: {creds[2]}')
bmc = _bmc.Bmc(node, *creds[:-1], timeout=timeout)
if bmc.is_connected():
r = bmc.chassis_power('status')
self.log.debug(f'Chassis power status: {r}')
if r:
nodes[node] = True
time.sleep(1)
self.log.debug(f'Node {node} is powered {r}')
bmc_ai[node] = tuple(cred_list[j][:-1])
cred_list[j][3] -= 1
left -= left
print(
f'\r{tot - left} of {tot} nodes communicating via IPMI',
end='')
sys.stdout.flush()
bmc.logout()
else:
self.log.debug(
f'No power status response from node {node}')
time.sleep(delay)
if left != 0:
self.log.error(
f'IPMI communication successful with only {tot - left} '
f'of {tot} nodes')
print('\n')
return bmc_ai
def _get_ipmi_ports(self, switch_lbl):
""" Get all of the ipmi ports for a given switch
Args:
switch_lbl (str): switch label
Returns:
ports (list of str): port name or number
"""
ports = []
for node_tmpl_idx in self.cfg.yield_ntmpl_ind():
for sw_idx in self.cfg.yield_ntmpl_phyintf_ipmi_ind(node_tmpl_idx):
if switch_lbl == self.cfg.get_ntmpl_phyintf_ipmi_switch(
node_tmpl_idx, sw_idx):
ports += self.cfg.get_ntmpl_phyintf_ipmi_ports(
node_tmpl_idx, sw_idx)
ports = [str(port) for port in ports]
return ports
def _get_pxe_ports(self, switch_lbl):
""" Get all of the pxe ports for a given switch
Args:
switch_lbl (str): switch label
Returns:
ports (list of str): port name or number
"""
ports = []
for node_tmpl_idx in self.cfg.yield_ntmpl_ind():
for sw_idx in self.cfg.yield_ntmpl_phyintf_pxe_ind(node_tmpl_idx):
if switch_lbl == self.cfg.get_ntmpl_phyintf_pxe_switch(
node_tmpl_idx, sw_idx):
ports += self.cfg.get_ntmpl_phyintf_pxe_ports(
node_tmpl_idx, sw_idx)
ports = [str(port) for port in ports]
return ports
def _get_port_table_ipmi(self, node_list):
""" Build table of discovered nodes. The responding IP addresses are
correlated to MAC addresses in the dnsmasq.leases file. The MAC
address is then used to correlate the IP address to a switch port.
Args:
node_list (list of str): IPV4 addresses
Returns:
table (AttrDict): switch, switch port, IPV4 address, MAC address
"""
dhcp_leases = GetDhcpLeases(self.dhcp_ipmi_leases_file)
dhcp_mac_ip = dhcp_leases.get_mac_ip()
dhcp_mac_table = AttrDict()
for ip in node_list:
for item in dhcp_mac_ip.items():
if ip in item:
dhcp_mac_table[item[0]] = item[1]
self.log.debug('ipmi mac-ip table')
self.log.debug(dhcp_mac_table)
for sw_ai in self.cfg.yield_sw_mgmt_access_info():
sw = SwitchFactory.factory(*sw_ai[1:])
label = sw_ai[0]
ipmi_ports = self._get_ipmi_ports(label)
mgmt_sw_cfg_mac_lists = \
sw.show_mac_address_table(format='std')
# Get switch ipmi port mac address table
# Logic below maintains same port order as config.yml
sw_ipmi_mac_table = AttrDict()
for port in ipmi_ports:
if port in mgmt_sw_cfg_mac_lists:
sw_ipmi_mac_table[port] = mgmt_sw_cfg_mac_lists[port]
self.log.debug('Switch ipmi port mac table')
self.log.debug(sw_ipmi_mac_table)
if label not in self.node_table_ipmi.keys():
self.node_table_ipmi[label] = []
for port in sw_ipmi_mac_table:
for mac in dhcp_mac_table:
if mac in sw_ipmi_mac_table[port]:
if not self._is_port_in_table(
self.node_table_ipmi[label], port):
self.node_table_ipmi[label].append(
[port, mac, dhcp_mac_table[mac]])
def _build_port_table_pxe(self, mac_list):
""" Build table of discovered nodes. The responding mac addresses
discovered by tcpdump are correlated to switch ports from cluster
switches. If nodes have taken an ip address (via dnsmasq) the ip
address is included in the table.
Args:
node_list (list of str): IPV4 addresses
Returns:
table (AttrDict): switch, switch port, IPV4 address, MAC address
"""
dhcp_leases = GetDhcpLeases(self.dhcp_pxe_leases_file)
dhcp_mac_ip = dhcp_leases.get_mac_ip()
dhcp_mac_table = AttrDict()
for mac in mac_list:
for item in dhcp_mac_ip.items():
if mac in item:
dhcp_mac_table[item[0]] = item[1]
self.log.debug('pxe dhcp mac table')
self.log.debug(dhcp_mac_table)
for sw_ai in self.cfg.yield_sw_mgmt_access_info():
sw = SwitchFactory.factory(*sw_ai[1:])
sw_label = sw_ai[0]
pxe_ports = self._get_pxe_ports(sw_label)
mgmt_sw_mac_lists = \
sw.show_mac_address_table(format='std')
# Get switch pxe port mac address table
# Logic below maintains same port order as config.yml
sw_pxe_mac_table = AttrDict()
for port in pxe_ports:
if port in mgmt_sw_mac_lists:
sw_pxe_mac_table[port] = mgmt_sw_mac_lists[port]
self.log.debug('Switch pxe port mac table')
self.log.debug(sw_pxe_mac_table)
# self.node_table_pxe is structured around switches
if sw_label not in self.node_table_pxe.keys():
self.node_table_pxe[sw_label] = []
for mac in mac_list:
_port = '-'
for port in sw_pxe_mac_table:
if mac in sw_pxe_mac_table[port]:
_port = port
break
if mac in dhcp_mac_table:
ip = dhcp_mac_table[mac]
else:
ip = '-'
if not self._is_val_in_table(self.node_table_pxe[sw_label],
mac):
self.node_table_pxe[sw_label].append([_port, mac, ip])
def _reset_existing_bmcs(self, node_addr_list, cred_list):
""" Attempts to reset any BMCs which have existing IP addresses since
we don't have control over their address lease time.
Args:
node_addr_list (list): list of ipv4 addresses for the discovered
nodes. (ie those that previously fetched an address from the DHCP
server.
cred_list (list of lists): Each list item is a list containing the
the userid, password and number of nodes for a node template.
"""
for node in node_addr_list:
for j, creds in enumerate(cred_list):
bmc = _bmc.Bmc(node, creds[0], creds[1], creds[2])
if bmc.is_connected():
self.log.info(
f'Resetting BMC with existing ip address: {node}')
if not bmc.bmc_reset('cold'):
self.log.error(
f'Failed attempting BMC reset on {node}')
bmc.logout()
break
def validate_ipmi(self):
self.log.info("Discover and validate cluster nodes")
# if self.inv.check_all_nodes_ipmi_macs() and self.inv.check_all_nodes_pxe_macs():
# self.log.info("Inventory exists with IPMI and PXE MACs populated.")
# print("\nPress Enter to continue cluster deployment without "
# "running IPMI hardware validation.")
# print("Type 'C' to validate cluster nodes defined in current "
# "'config.yml'")
# resp = input("Type 'T' to terminate Power-Up ")
# if resp == 'T':
# resp = input("Type 'y' to confirm ")
# if resp == 'y':
# self.log.info("'{}' entered. Terminating Power-Up at user "
# "request".format(resp))
# sys.exit(1)
# elif resp == 'C':
# self.log.info("'{}' entered. Continuing with hardware "
# "validation".format(resp))
# else:
# print()
# return
ipmi_cnt, pxe_cnt = self._get_port_cnts()
ipmi_addr, bridge_addr, ipmi_prefix, ipmi_vlan = self._get_network(
'ipmi')
ipmi_network = ipmi_addr + '/' + str(ipmi_prefix)
addr = IPNetwork(bridge_addr + '/' + str(ipmi_prefix))
netmask = str(addr.netmask)
ipmi_size = addr.size
addr.value += NAME_SPACE_OFFSET_ADDR
addr = str(addr)
cred_list = self._get_cred_list()
rc = False
dhcp_st = get_dhcp_pool_start()
self.ipmi_ns = NetNameSpace('ipmi-ns-', 'br-ipmi-' + str(ipmi_vlan),
addr)
# setup DHCP, unless already running in namesapce
# save start and end addr raw numeric values
self.log.debug('Installing DHCP server in network namespace')
addr_st = self._add_offset_to_address(ipmi_network, dhcp_st)
addr_end = self._add_offset_to_address(ipmi_network, ipmi_size - 2)
dhcp_end = self._add_offset_to_address(ipmi_network,
dhcp_st + ipmi_cnt + 2)
# scan ipmi network for nodes with pre-existing ip addresses
cmd = 'fping -r0 -a -g {} {}'.format(addr_st, addr_end)
node_list, stderr, rc = sub_proc_exec(cmd)
if rc not in (0, 1):
self.log.warning(f'Error scanning IPMI network. rc: {rc}')
self.log.debug('Pre-existing node list: \n{}'.format(node_list))
node_list = node_list.splitlines()
self._reset_existing_bmcs(node_list, cred_list)
if len(node_list) > 0:
print('Pause 60s for BMCs to begin reset')
time.sleep(60)
dns_list, stderr, rc = sub_proc_exec('pgrep dnsmasq')
if rc not in [0, 1]:
self.log.warning(f'Error looking for dnsmasq. rc: {rc}')
dns_list = dns_list.splitlines()
for pid in dns_list:
ns_name, stderr, rc = sub_proc_exec(
'ip netns identify {}'.format(pid))
if self.ipmi_ns._get_name_sp_name() in ns_name:
self.log.debug('DHCP already running in {}'.format(ns_name))
break
else:
cmd = (f'dnsmasq --dhcp-leasefile={self.dhcp_ipmi_leases_file} '
f'--interface={self.ipmi_ns._get_name_sp_ifc_name()} '
f'--dhcp-range={addr_st},{dhcp_end},{netmask},600')
stdout, stderr, rc = self.ipmi_ns._exec_cmd(cmd)
if rc != 0:
self.log.warning(f'Error setting up dnsmasq. rc: {rc}')
print(stderr)
# Scan up to 25 times. Delay 5 seconds between scans
# Allow infinite number of retries
self.log.info('Scanning BMC network on 5 s intervals')
cnt = 0
cnt_down = 25
while cnt < ipmi_cnt:
print()
for i in range(cnt_down):
print(
'\r{} of {} nodes requesting DHCP address. Scan count: {} '
.format(cnt, ipmi_cnt, cnt_down - i),
end="")
sys.stdout.flush()
time.sleep(5)
cmd = 'fping -r0 -a -g {} {}'.format(addr_st, dhcp_end)
stdout, stderr, rc = sub_proc_exec(cmd)
node_list = stdout.splitlines()
cnt = len(node_list)
if cnt >= ipmi_cnt:
rc = True
print(
'\r{} of {} nodes requesting DHCP address. Scan count: {} '
.format(cnt, ipmi_cnt, cnt_down - i),
end="")
break
self._get_port_table_ipmi(node_list)
self.log.debug('Table of found IPMI ports: {}'.format(
self.node_table_ipmi))
for switch in self.node_table_ipmi:
print('\n\nSwitch: {} '.format(switch))
print(
tabulate(self.node_table_ipmi[switch],
headers=('port', 'MAC address', 'IP address')))
print()
if cnt >= ipmi_cnt:
break
print(
'\n\nPress Enter to continue scanning for cluster nodes.\nOr')
print(
"Or enter 'C' to continue cluster deployment with a subset of nodes"
)
resp = input("Or Enter 'T' to terminate Power-Up ")
if resp == 'T':
resp = input("Enter 'y' to confirm ")
if resp == 'y':
self.log.info(
"'{}' entered. Terminating Power-Up at user request".
format(resp))
self._teardown_ns(self.ipmi_ns)
sys.exit(1)
elif resp == 'C':
print('\nNot all nodes have been discovered')
resp = input("Enter 'y' to confirm continuation of"
" deployment without all nodes ")
if resp == 'y':
self.log.info(
"'{}' entered. Continuing PowerUp".format(resp))
break
self.node_list = node_list
if cnt < ipmi_cnt:
self.log.warning('Failed to validate expected number of nodes')
if len(node_list) > 0 and len(cred_list) > 0:
# Verify and power off nodes
self.bmc_ai = self._get_credentials(node_list, cred_list)
if not self.bmc_ai:
self.log.error(
'Critical error. Unable to establish BMC communication '
'with any cluster nodes.\n.')
sys.exit('Exiting.')
# set_power_cients('off') has built in 60 s delay
self.log.info('\nPowering off cluster nodes')
set_power_clients('off', clients=self.bmc_ai)
set_power_clients('on', clients=self.bmc_ai)
self.log.debug('\nSetting "network" boot device on all nodes')
set_bootdev_clients('network', clients=self.bmc_ai)
self.log.debug('Cluster nodes IPMI validation complete')
self.ran_ipmi = True
if not rc:
raise UserException('Not all node IPMI ports validated')
def _get_cred_list(self):
"""Returns list of list. Each list has the credentials
for a node template(userid, password, bmc_type).
Note that there is no association to any ip address.
"""
cred_list = []
for idx in self.cfg.yield_ntmpl_ind():
for idx_ipmi in self.cfg.yield_ntmpl_phyintf_ipmi_ind(idx):
port_cnt = self.cfg.get_ntmpl_phyintf_ipmi_pt_cnt(
idx, idx_ipmi)
cred_list.append([
self.cfg.get_ntmpl_ipmi_userid(index=idx),
self.cfg.get_ntmpl_ipmi_password(index=idx),
self.cfg.get_ntmpl_bmc_type(index=idx), port_cnt
])
return cred_list
def _teardown_ns(self, ns):
# kill dnsmasq
dns_list, stderr, rc = sub_proc_exec('pgrep dnsmasq')
dns_list = dns_list.splitlines()
for pid in dns_list:
ns_name, stderr, rc = sub_proc_exec('ip netns identify ' + pid)
if ns._get_name_sp_name() in ns_name:
self.log.debug('Killing dnsmasq {}'.format(pid))
stdout, stderr, rc = sub_proc_exec('kill -15 ' + pid)
# kill tcpdump
tcpdump_list, stderr, rc = sub_proc_exec('pgrep tcpdump')
tcpdump_list = tcpdump_list.splitlines()
for pid in tcpdump_list:
ns_name, stderr, rc = sub_proc_exec('ip netns identify ' + pid)
if ns._get_name_sp_name() in ns_name:
self.log.debug('Killing tcpdump {}'.format(pid))
stdout, stderr, rc = sub_proc_exec('kill -15 ' + pid)
# reconnect the veth pair to the container
ns._reconnect_container()
# Destroy the namespace
self.log.debug('Destroying namespace')
ns._destroy_name_sp()
def _get_macs(self, mac_list, dump):
""" Parse the data returned by tcpdump looking for pxe boot
requests.
Args:
mac_list(list): list of already found mac addresses
dump(str): tcpdump output from the tcpdump file
"""
_mac_iee802 = r'([\dA-F]{2}[\.:-]){5}([\dA-F]{2})'
_mac_regex = re.compile(_mac_iee802, re.I)
dump = dump.split('BOOTP/DHCP, Request')
for item in dump:
# look first for 'magic cookie'
pos = item.find('6382 5363')
if pos >= 0:
bootp = item[pos:]
match = re.search(' ff|ff ', bootp, re.DOTALL)
if match is not None:
bootp = bootp[:2 + match.start()]
# look for pxe request info.
# 0x37 = 55 (parameter list request)
# 0x43 = 67 (boot filename request)
# 0xd1 = 209 (pxeconfig file request)
if ('37 ' in bootp or ' 37' in bootp):
if (' d1' in bootp or 'd1 ' in bootp) or \
('43 ' in bootp or ' 43' in bootp):
self.log.debug('bootp param request field: '
f'{bootp}')
mac = _mac_regex.search(item).group()
if mac not in mac_list:
mac_list.append(mac)
return mac_list
def validate_pxe(self, bootdev='default', persist=True):
# if self.inv.check_all_nodes_pxe_macs():
# self.log.info("Inventory exists with PXE MACs populated.")
# if not self.ran_ipmi:
# return
# print("\nPress Enter to continue cluster deployment without "
# "running PXE hardware validation.")
# print("Type 'C' to validate cluster nodes defined in current "
# "'config.yml'")
# resp = input("Type 'T' to terminate Power-Up ")
# if resp == 'T':
# resp = input("Type 'y' to confirm ")
# if resp == 'y':
# self.log.info("'{}' entered. Terminating Power-Up at user "
# "request".format(resp))
# sys.exit(1)
# elif resp == 'C':
# self.log.info("'{}' entered. Continuing with hardware "
# "validation".format(resp))
# else:
# print()
# return
# if not self.ran_ipmi:
# return
if not self.node_table_ipmi:
raise UserCriticalException('No BMCs discovered')
self.log.debug("Checking PXE networks and client PXE"
" ports ________\n")
self.log.debug('Boot device: {}'.format(bootdev))
ipmi_cnt, pxe_cnt = self._get_port_cnts()
pxe_addr, bridge_addr, pxe_prefix, pxe_vlan = self._get_network('pxe')
pxe_network = pxe_addr + '/' + str(pxe_prefix)
addr = IPNetwork(bridge_addr + '/' + str(pxe_prefix))
netmask = str(addr.netmask)
addr.value += NAME_SPACE_OFFSET_ADDR
addr = str(addr)
foundall = False
dhcp_st = get_dhcp_pool_start()
pxe_ns = NetNameSpace('pxe-ns-', 'br-pxe-' + str(pxe_vlan), addr)
# setup DHCP. save start and end addr raw numeric values
self.log.debug('Installing DHCP server in network namespace')
addr_st = self._add_offset_to_address(pxe_network, dhcp_st)
addr_end = self._add_offset_to_address(pxe_network,
dhcp_st + pxe_cnt + 2)
dns_list, stderr, rc = sub_proc_exec('pgrep dnsmasq')
dns_list = dns_list.splitlines()
if os.path.exists(self.dhcp_pxe_leases_file):
os.remove(self.dhcp_pxe_leases_file)
# delete any remnant dnsmasq processes
for pid in dns_list:
ns_name, stderr, rc = sub_proc_exec(
'ip netns identify {}'.format(pid))
if pxe_ns._get_name_sp_name() in ns_name:
self.log.debug('Killing dnsmasq. pid {}'.format(pid))
stdout, stderr, rc = sub_proc_exec('kill -15 ' + pid)
cmd = (f'dnsmasq --dhcp-leasefile={self.dhcp_pxe_leases_file} '
f'--interface={pxe_ns._get_name_sp_ifc_name()} '
f'--dhcp-range={addr_st},{addr_end},{netmask},3600')
stdout, stderr, rc = pxe_ns._exec_cmd(cmd)
if rc != 0:
self.log.warning(f'Error configuring dnsmasq. rc: {rc}')
if os.path.exists(self.tcp_dump_file):
os.remove(self.tcp_dump_file)
tcpdump_list, stderr, rc = sub_proc_exec('pgrep tcpdump')
tcpdump_list = tcpdump_list.splitlines()
# delete any remnant tcpdump processes
for pid in tcpdump_list:
ns_name, stderr, rc = sub_proc_exec('ip netns identify ' + pid)
if pxe_ns._get_name_sp_name() in ns_name:
self.log.debug('Killing tcpdump. pid {}'.format(pid))
stdout, stderr, rc = sub_proc_exec('kill -15 ' + pid)
cmd = (f'sudo tcpdump -X -U -i {pxe_ns._get_name_sp_ifc_name()} '
f'-w {self.tcp_dump_file} --immediate-mode port 67')
proc = pxe_ns._launch_cmd(cmd)
if not isinstance(proc, object):
self.log.error(
f'Failure to launch process of tcpdump monitor {proc}')
# Scan up to 25 times. Delay 10 seconds between scans
# Allow infinite number of retries
self.log.info('Scanning pxe network on 10 s intervals.')
cnt = 0
cnt_prev = 0
cnt_down = 25
mac_list = []
dump = ''
while cnt < pxe_cnt:
print()
cmd = 'sudo tcpdump -r {} -xx'.format(self.tcp_dump_file)
for i in range(cnt_down):
print('\r{} of {} nodes requesting PXE boot. Scan cnt: {} '.
format(cnt, pxe_cnt, cnt_down - i),
end="")
sys.stdout.flush()
time.sleep(10)
# read the tcpdump file if size is not 0
if os.path.exists(self.tcp_dump_file) and os.path.getsize(
self.tcp_dump_file):
dump, stderr, rc = sub_proc_exec(cmd)
if rc != 0:
self.log.warning(
f'Error reading tcpdump file. rc: {rc}')
if 'reading' not in stderr:
self.log.warning(
f'Failure reading tcpdump file - {stderr}')
mac_list = self._get_macs(mac_list, dump)
cnt = len(mac_list)
if cnt > cnt_prev:
cnt_prev = cnt
# Pause briefly for in flight DHCP to complete and lease file to update
time.sleep(5)
self._build_port_table_pxe(mac_list)
if cnt >= pxe_cnt:
foundall = True
print(
'\r{} of {} nodes requesting PXE boot. Scan count: {} '
.format(cnt, pxe_cnt, cnt_down - i),
end="")
break
self.log.debug('Table of found PXE ports: {}'.format(
self.node_table_pxe))
for switch in self.node_table_pxe:
print('\n\nSwitch: {}'.format(switch))
print(
tabulate(self.node_table_pxe[switch],
headers=('port', 'MAC address', 'IP address')))
print()
if cnt >= pxe_cnt:
break
print('\n\nPress Enter to continue scanning for cluster nodes.')
print(
"Or enter 'C' to continue cluster deployment with a subset of nodes"
)
print("Or enter 'R' to cycle power to missing nodes")
resp = input("Or enter 'T' to terminate Power-Up ")
if resp == 'T':
resp = input("Enter 'y' to confirm ")
if resp == 'y':
self.log.info("'{}' entered. Terminating Power-Up at user"
" request".format(resp))
self._teardown_ns(self.ipmi_ns)
self._teardown_ns(pxe_ns)
sys.exit(1)
elif resp == 'R':
self._reset_unfound_nodes()
elif resp == 'C':
print('\nNot all nodes have been discovered')
resp = input("Enter 'y' to confirm continuation of"
" deployment without all nodes ")
if resp == 'y':
self.log.info(
"'{}' entered. Continuing Power-Up".format(resp))
break
if cnt < pxe_cnt:
self.log.warning('Failed to validate expected number of nodes')
self._teardown_ns(pxe_ns)
# Cycle power on all discovered nodes if bootdev set to 'network'
if bootdev == 'network':
self.log.debug('\nCycling power to discovered nodes.\n')
set_power_clients('off', clients=self.bmc_ai)
set_power_clients('on', clients=self.bmc_ai)
set_bootdev_clients('network', clients=self.bmc_ai)
self._teardown_ns(self.ipmi_ns)
# Reset BMCs to insure they acquire a new address from container
# during inv_add_ports. Avoids conflicting addresses during redeploy
self._reset_existing_bmcs(self.node_list, self._get_cred_list())
self.log.info('Cluster nodes validation complete')
if not foundall:
raise UserException('Not all node PXE ports validated')
def _reset_unfound_nodes(self):
""" Power cycle the nodes who's PXE ports are not requesting pxe boot.
"""
ipmi_missing_list_ai = {}
for label in self.cfg.yield_sw_mgmt_label():
pxe_ports = self._get_pxe_ports(label)
ipmi_ports = self._get_ipmi_ports(label)
for node in self.node_table_ipmi[label]:
if node[0] in ipmi_ports:
idx = ipmi_ports.index(node[0])
if label not in self.node_table_pxe or not self._is_port_in_table(
self.node_table_pxe[label], pxe_ports[idx]):
ipmi_missing_list_ai[node[2]] = self.bmc_ai[node[2]]
self.log.debug(
f'Cycling power to missing nodes list: {ipmi_missing_list_ai}')
print('Cycling power to non responding nodes:')
for node in ipmi_missing_list_ai:
print(node)
t1 = time.time()
set_power_clients('off', clients=ipmi_missing_list_ai)
while time.time() < t1 + 10:
time.sleep(0.5)
set_bootdev_clients('network', clients=ipmi_missing_list_ai)
set_power_clients('on', clients=ipmi_missing_list_ai)
def _is_port_in_table(self, table, port):
for node in table:
if port == node[0]:
self.log.debug('Table port: {} port: {}'.format(node[0], port))
return True
return False
def _is_val_in_table(self, table, val):
for item in table:
if val == item[0] or val == item[1]:
self.log.debug('Found in table: {} item: {}'.format(val, item))
return True
return False
def _get_network(self, type_):
"""Returns details of a Power-Up network.
Args:
type_ (str): Either 'pxe' or 'ipmi'
Returns:
network_addr: (str) ipv4 addr
bridge_ipaddr: (str) ipv4 addr
netprefix: (str)
vlan: (str)
"""
types = self.cfg.get_depl_netw_client_type()
bridge_ipaddr = self.cfg.get_depl_netw_client_brg_ip()
vlan = self.cfg.get_depl_netw_client_vlan()
netprefix = self.cfg.get_depl_netw_client_prefix()
idx = types.index(type_)
network = IPNetwork(bridge_ipaddr[idx] + '/' + str(netprefix[idx]))
network_addr = str(network.network)
return network_addr, bridge_ipaddr[idx], netprefix[idx], vlan[idx]
def validate_data_switches(self):
self.log.info('Verifying data switches')
sw_cnt = self.cfg.get_sw_data_cnt()
self.log.debug(
'Number of data switches defined in config file: {}'.format(
sw_cnt))
for index, switch_label in enumerate(self.cfg.yield_sw_data_label()):
print('.', end="")
sys.stdout.flush()
label = self.cfg.get_sw_data_label(index)
self.log.debug('switch_label: {}'.format(switch_label))
switch_class = self.cfg.get_sw_data_class(index)
if not switch_class:
self.log.error('No switch class found')
return False
userid = None
password = None
rc = True
try:
userid = self.cfg.get_sw_data_userid(index)
except AttributeError:
self.log.info('Passive switch mode specified')
return True
try:
password = self.cfg.get_sw_data_password(index)
except AttributeError:
try:
self.cfg.get_sw_data_ssh_key(index)
except AttributeError:
return True
else:
self.log.error(
'Switch authentication via ssh keys not yet supported')
return False
# Verify communication on each defined interface
for ip in self.cfg.yield_sw_data_interfaces_ip(index):
self.log.debug('Verifying switch communication on ip'
' address: {}'.format(ip))
sw = SwitchFactory.factory(switch_class, ip, userid, password,
'active')
if sw.is_pingable():
self.log.debug(
'Successfully pinged data switch \"%s\" at %s' %
(label, ip))
else:
self.log.warning(
'Failed to ping data switch \"%s\" at %s' %
(label, ip))
rc = False
try:
vlans = sw.show_vlans()
if vlans and len(vlans) > 1:
self.log.debug(
'Successfully communicated with data switch \"%s\"'
' at %s' % (label, ip))
else:
self.log.warning(
'Failed to communicate with data switch \"%s\"'
'at %s' % (label, ip))
rc = False
except (SwitchException, SSH_Exception):
self.log.error('Failed communicating with data switch'
' at address {}'.format(ip))
rc = False
print()
if rc:
self.log.debug(' OK - All data switches verified')
else:
raise UserException('Failed verification of data switches')
def validate_mgmt_switches(self):
self.log.info('Verifying management switches')
sw_cnt = self.cfg.get_sw_mgmt_cnt()
self.log.debug(
'Number of management switches defined in config file: {}'.format(
sw_cnt))
for index, switch_label in enumerate(self.cfg.yield_sw_mgmt_label()):
print('.', end="")
sys.stdout.flush()
label = self.cfg.get_sw_mgmt_label(index)
self.log.debug('switch_label: {}'.format(switch_label))
switch_class = self.cfg.get_sw_mgmt_class(index)
if not switch_class:
self.log.error('No switch class found')
return False
userid = None
password = None
rc = True
try:
userid = self.cfg.get_sw_mgmt_userid(index)
except AttributeError:
self.log.debug('Passive switch mode specified')
return rc
try:
password = self.cfg.get_sw_mgmt_password(index)
except AttributeError:
try:
self.cfg.get_sw_mgmt_ssh_key(index)
except AttributeError:
return rc
else:
self.log.error(
'Switch authentication via ssh keys not yet supported')
rc = False
# Verify communication on each defined interface
for ip in self.cfg.yield_sw_mgmt_interfaces_ip(index):
self.log.debug('Verifying switch communication on ip address:'
' {}'.format(ip))
sw = SwitchFactory.factory(switch_class, ip, userid, password,
'active')
if sw.is_pingable():
self.log.debug(
'Successfully pinged management switch \"%s\" at %s' %
(label, ip))
else:
self.log.warning(
'Failed to ping management switch \"%s\" at %s' %
(label, ip))
rc = False
try:
vlans = sw.show_vlans()
except (SwitchException, SSH_Exception):
self.log.error(
'Failed communicating with management switch'
' at address {}'.format(ip))
rc = False
if vlans and len(vlans) > 1:
self.log.debug(
'Successfully communicated with management switch \"%s\"'
' at %s' % (label, ip))
else:
self.log.warning(
'Failed to communicate with data switch \"%s\"'
'at %s' % (label, ip))
rc = False
print()
if rc:
self.log.debug(' OK - All management switches verified')
else:
raise UserCriticalException(
'Failed verification of management switches')
class BaseProvider(abc.ABC):
"""
Abstract base class from which providers should inherit from
"""
SESSION_CLASS = StreamSession
ITEM_CLASS = model.MediaItem
# VIEW_CLASS = SimpleProviderView
FILTERS = AttrDict()
ATTRIBUTES = AttrDict(title={"width": ("weight", 1)})
MEDIA_TYPES = None
def __init__(self, *args, **kwargs):
self._view = None
self._session = None
self._active = False
self._filters = AttrDict(
{n: f(provider=self, name=n)
for n, f in self.FILTERS.items()})
rules = AttrDict(self.config.rules.label,
**config.settings.profile.rules.label)
labels = AttrDict(self.config.labels, **config.settings.profile.labels)
self.rule_map = AttrDict([(re.compile(k, re.IGNORECASE), v)
for k, v in [(r, rules[r])
for r in rules.keys()]])
self.highlight_map = AttrDict([(re.compile(k,
re.IGNORECASE), labels[v])
for k, v in rules.items()])
self.highlight_re = re.compile(
"(" + "|".join([k.pattern
for k in self.highlight_map.keys()]) + ")",
re.IGNORECASE)
def init_config(self):
pass
@property
def LISTING_CLASS(self):
for cls in [self.__class__] + list(self.__class__.__bases__):
pkg = sys.modules.get(cls.__module__)
pkgname = pkg.__name__.split(".")[-1]
try:
return next(
v for k, v in pkg.__dict__.items()
if pkgname in k.lower() and k.endswith("MediaListing"))
except StopIteration:
continue
return model.MediaListing
@property
def MEDIA_SOURCE_CLASS(self):
for cls in [self.__class__] + list(self.__class__.mro()):
pkg = sys.modules.get(cls.__module__)
pkgname = pkg.__name__.split(".")[-1]
try:
return next(
v for k, v in pkg.__dict__.items()
if pkgname in k.lower() and k.endswith("MediaSource"))
except (StopIteration, AttributeError):
continue
return model.MediaSource
@property
def session_params(self):
return {"proxies": config.settings.profile.get("proxies")}
@property
def session(self):
if self._session is None:
session_params = self.session_params
self._session = self.SESSION_CLASS.new(self.IDENTIFIER,
**session_params)
return self._session
@property
def gui(self):
return self._view is not None
@property
def filters(self):
return self._filters
@property
def view(self):
if not self._view:
self._view = self.make_view()
self._view.update()
return self._view
@property
def is_active(self):
return self._active
def activate(self):
if self.is_active:
return
self._active = True
self.on_activate()
def deactivate(self):
if not self.is_active:
return
self.on_deactivate()
self._active = False
def on_activate(self):
pass
def on_deactivate(self):
pass
@abc.abstractmethod
def make_view(self):
pass
@classproperty
def IDENTIFIER(cls):
return next(c.__module__ for c in cls.__mro__
if __package__ in c.__module__).split(".")[-1]
@classproperty
@abc.abstractmethod
def NAME(cls):
return cls.__name__.replace("Provider", "")
@property
def FILTERS_BROWSE(self):
return AttrDict()
@property
def FILTERS_OPTIONS(self):
return AttrDict()
@property
def FILTERS(self):
d = getattr(self, "FILTERS_BROWSE", AttrDict())
d.update(getattr(self, "FILTERS_OPTIONS", {}))
return d
def parse_identifier(self, identifier):
return
def new_media_source(self, *args, **kwargs):
return self.MEDIA_SOURCE_CLASS(self.IDENTIFIER, *args, **kwargs)
def new_listing(self, **kwargs):
return self.LISTING_CLASS(self.IDENTIFIER, **kwargs)
@abc.abstractmethod
def listings(self, filters=None):
pass
def should_download(self, listing):
return listing.label in (list(self.config.rules) +
list(config.settings.profile.rules.download))
def on_new_listing(self, listing):
try:
label = next(l for r, l in self.rule_map.items()
if r.search(listing.title))
listing.label = label
if self.should_download(listing):
self.download(listing)
except StopIteration:
pass
@property
def config(self):
return config.ConfigTree(
config.settings.profile.providers.get(self.IDENTIFIER, {}))
@property
def config_is_valid(self):
def check_config(required, cfg):
if isinstance(required, dict):
for k, v in required.items():
if not k in cfg:
return False
if not check_config(required[k], cfg[k]):
return False
else:
for k in required:
if not k in cfg:
return False
return True
# return all([ self.config.get(x, None) is not None
# for x in getattr(self, "REQUIRED_CONFIG", [])
return check_config(getattr(self, "REQUIRED_CONFIG", []), self.config)
def parse_options(self, options):
if not options:
return AttrDict()
return AttrDict(
[(list(self.FILTERS_OPTIONS.keys())[n], v) for n, v in enumerate(
[o for o in options.split(",") if "=" not in o])],
**dict(o.split("=") for o in options.split(",") if "=" in o))
def get_source(self, selection, **kwargs):
source = selection.content
if not isinstance(source, list):
source = [source]
return source
def play_args(self, selection, **kwargs):
source = self.get_source(selection, **kwargs)
kwargs = {
k: v
for k, v in list(kwargs.items()) +
[(f, self.filters[f].value)
for f in self.filters if f not in kwargs]
}
return (source, kwargs)
def play(self, selection, no_task_manager=False, **kwargs):
try:
sources, kwargs = self.play_args(selection, **kwargs)
except SGStreamNotFound as e:
logger.error(f"stream not found: {e}")
return
# media_type = kwargs.pop("media_type", None)
# FIXME: For now, we just throw playlists of media items at the default
# player program and hope it can handle all of them.
player_spec = None
helper_spec = None
if not isinstance(sources, list):
sources = [sources]
for s in sources:
if not s.media_type:
# Try to set the content types of the source(s) with a HTTP HEAD
# request if the provider didn't specify one.
s.media_type = self.session.head(
s.locator).headers.get("Content-Type").split("/")[0]
media_types = set([s.media_type for s in sources if s.media_type])
player_spec = {"media_types": media_types}
if media_types == {"image"}:
helper_spec = {None: None}
else:
helper_spec = getattr(self.config, "helpers",
None) or sources[0].helper
task = model.PlayMediaTask(provider=self.NAME,
title=selection.title,
sources=sources)
if not (no_task_manager or state.options.debug_console):
return state.task_manager.play(task, player_spec, helper_spec,
**kwargs)
else:
return state.asyncio_loop.create_task(
Player.play(task, player_spec, helper_spec, **kwargs))
def download(self, selection, no_task_manager=False, **kwargs):
sources, kwargs = self.play_args(selection, **kwargs)
# filename = selection.download_filename
if not isinstance(sources, list):
sources = [sources]
for i, s in enumerate(sources):
# filename = s.download_filename
# kwargs = {"ext": getattr(s, "ext", None)}
if len(sources):
kwargs["index"] = i
try:
filename = selection.download_filename(**kwargs)
except SGInvalidFilenameTemplate as e:
logger.warn(
f"filename template for provider {self.IDENTIFIER} is invalid: {e}"
)
helper_spec = getattr(self.config, "helpers") or s.download_helper
# logger.info(f"helper: {helper_spec}")
task = model.DownloadMediaTask(provider=self.NAME,
title=selection.title,
sources=[s],
dest=filename)
# s = AttrDict(dataclasses.asdict(s))
# s.provider = self.NAME
# s.title = selection.title
# s.dest = filename
if not (no_task_manager or state.options.debug_console):
return state.task_manager.download(task, filename, helper_spec,
**kwargs)
else:
return state.asyncio_loop.create_task(
Downloader.download(task, filename, helper_spec, **kwargs))
def on_select(self, widget, selection):
self.play(selection)
@property
def limit(self):
return None
def refresh(self):
self.view.refresh()
def reset(self):
self.view.reset()
def __str__(self):
return self.NAME
def __repr__(self):
return f"<{type(self)}: {self.NAME}>"
def get_ports(self):
dhcp_leases = GetDhcpLeases(self.dhcp_leases_file)
dhcp_mac_ip = dhcp_leases.get_mac_ip()
self.log.debug('DHCP leases: {}'.format(dhcp_mac_ip))
mgmt_sw_cfg_mac_lists = AttrDict()
if self.cfg.is_passive_mgmt_switches():
self.log.debug('passive mode')
for switch_label in self.cfg.yield_sw_mgmt_label():
file_path = os.path.join(GEN_PASSIVE_PATH, switch_label)
mac_info = {}
try:
with open(file_path, 'r') as f:
mac_info = f.read()
except IOError as error:
self.log.error(
'Passive switch MAC address table file not found {}'
.format(error))
raise
mgmt_sw_cfg_mac_lists[switch_label] = \
SwitchCommon.get_port_to_mac(mac_info, self.log)
else:
for switch in self.sw_dict:
self.log.debug('Switch: {}'.format(switch))
mgmt_sw_cfg_mac_lists[switch] = \
self.sw_dict[switch].show_mac_address_table(format='std')
self.log.debug('Management switches MAC address tables: {}'.format(
mgmt_sw_cfg_mac_lists))
# Remove all the mac address table entries which do not have a matching
# MAC address in the DHCP leases table, then remove any MAC addresses
# which do not have a DHCP table entry.
for switch in mgmt_sw_cfg_mac_lists.keys():
for port in list(mgmt_sw_cfg_mac_lists[switch].keys()):
port_macs = mgmt_sw_cfg_mac_lists[switch][port]
found_mac = False
for mac in dhcp_mac_ip.keys():
if mac in port_macs:
found_mac = True
# keep only the mac which has a dhcp address
mgmt_sw_cfg_mac_lists[switch][port] = [mac]
if not found_mac:
del mgmt_sw_cfg_mac_lists[switch][port]
self.log.debug('Management switches MAC address table of ports with'
'dhcp leases: {}'.format(mgmt_sw_cfg_mac_lists))
if self.port_type == "ipmi":
self.inv.add_macs_ipmi(mgmt_sw_cfg_mac_lists)
self.inv.add_ipaddrs_ipmi(dhcp_mac_ip)
elif self.port_type == "pxe":
self.inv.add_macs_pxe(mgmt_sw_cfg_mac_lists)
self.inv.add_ipaddrs_pxe(dhcp_mac_ip)
if self.port_type == 'ipmi':
self.node_table, self.ports_found, self.ports_total = \
self._build_node_table_ipmi(self.cfg, dhcp_mac_ip, mgmt_sw_cfg_mac_lists)
if self.port_type == 'pxe':
self.node_table, self.ports_found, self.ports_total = \
self._build_node_table_pxe(self.cfg, dhcp_mac_ip, mgmt_sw_cfg_mac_lists)
def loadConfig(mode):
"""
Build config objects for application launch instance(appConfig) and model arguments.
Configuration is built by merging:
Options specified at the command line(highest priority).
Domain specific defaults specified in the domain directory(next priority).
Global defaults defined above.
Returns:
Tuple of appConfig and modelArgs.
"""
def __basic_arguments_parser(add_help):
"""
Creates a command line parser with a minimal set of folder level arguments.
"""
parser = argparse.ArgumentParser(add_help=add_help)
# Basic load/store settings.
parser.add_argument('--domain',
action='store',
dest='domain',
default="toy2",
help='The app domain to use.')
parser.add_argument(
'--inputs_root_dir',
action='store',
dest='inputs_root_dir',
default="data/inputs/",
help=
'Path to folder containing dev, train and test input data folders.'
)
if mode in [AppMode.Train, AppMode.Evaluate]:
parser.add_argument('--training_dir',
action='store',
dest='training_dir',
default='./data/training/'
if mode != AppMode.Test else "./data/testing/",
help='Path to experiment directory.')
parser.add_argument(
"--run",
action='store',
dest='run',
default=None,
type=int,
help="Index of the run that should be operated upon.")
parser.add_argument("--schemaVersion",
type=int,
default=None,
help="Maximum number of nodes in an XML file.")
parser.add_argument(
'--resume',
default=False if mode == AppMode.Test else None,
type=str2bool3,
help=
'Indicates if training has to be resumed from the latest checkpoint'
)
# Testing data folders.
if mode == AppMode.Evaluate:
parser.add_argument(
'--test_dataset',
default="test",
type=str,
help='Dataset(test/dev/train) to use for evaluation.')
return parser
# Some config defaults depend on the appConfig. So, peeking into appConfig, before configuring the rest
basicAppConfig, _ = __basic_arguments_parser(False).parse_known_args()
postProcessAppConfig(basicAppConfig, mode)
# Get domain defaults and merge them.
domainModule = import_module("domains." + basicAppConfig.domain)
appConfigDefaults = AttrDict({
**appConfigGlobalDefaults,
**domainModule.appConfigDefaults
})
if mode == AppMode.Generate:
domainGeneratorModule = import_module("domains." +
basicAppConfig.domain +
".generate")
generatorArgsDefaults = AttrDict({
**generatorArgsGlobalDefaults,
**domainGeneratorModule.generatorArgsDefaults
})
else:
modelArgsDefaults = AttrDict({
**modelArgsGlobalDefaults,
**domainModule.modelArgsDefaults
})
# Create the parser which parses basic arguments and will also parse the entire kitchen sink, below.
parser = __basic_arguments_parser(True)
# Testing multiple times.
if mode == AppMode.Test:
parser.add_argument("--repetitionCount",
type=int,
default=5,
help="Number of times to repeat test.")
if mode in [AppMode.Train]:
# Domain customizable load/store settings.
parser.add_argument(
"--checkpoint_every",
type=int,
default=appConfigDefaults.checkpoint_every,
help="Number of epochs after which we take a checkpoint.")
parser.add_argument(
"--print_every",
type=int,
default=10,
help="Print progress information, after every so many batches.")
if mode in [AppMode.Train, AppMode.Evaluate]:
parser.add_argument(
"--input_select_percent",
type=float,
default=None,
help=
"Percentage of inputs actually to be selected for training. This helps in training"
+ " with smaller dataset that what all is available.")
# Randomizaion settings.
parser.add_argument(
'--random_seed',
dest='random_seed',
type=float,
default=None,
help='Random seed to use before start of the training.')
# Various logging and debug settings.
parser.add_argument('--log_level',
dest='log_level',
default='info',
help='Logging level.')
parser.add_argument(
"--tensorboard",
type=int,
default=0,
help="Frequency of logging data into tensorboard. Set to 0 to disable."
)
parser.add_argument(
"--profile",
type=str2bool,
default=False,
help="Set to true to enable profiling info printing mode.")
if mode in [AppMode.Train, AppMode.Evaluate]:
parser.add_argument(
"--runtests",
type=str2bool,
default=False,
help="Set to true to enable unit testing of components.")
parser.add_argument(
"--debugAttention",
dest="attention",
type=str2bool,
default=False,
help="Debug attention by loggnig it into tensorboard.")
parser.add_argument("--batch_size",
type=int,
default=1000,
help="Batch size for training.")
if mode in [AppMode.Train, AppMode.Evaluate]:
# Build args needed during training.
parser.add_argument(
"--epochs",
type=int,
default=random.randint(1, 10) if mode == AppMode.Test else 400,
help="Number of epochs to train for.")
# parser.add_argument("--num_samples", type = int, default = None,
# help="Number of samples to train on.")
if mode in [AppMode.Train, AppMode.Evaluate, AppMode.Test]:
# XML Schema params.
parser.add_argument("--max_node_count",
type=int,
default=modelArgsDefaults.max_node_count,
help="Maximum number of nodes in an XML file.")
parser.add_argument(
"--total_attrs_count",
type=int,
default=modelArgsDefaults.total_attrs_count,
help="Total number of known attributes in the schema.")
parser.add_argument(
"--value_symbols_count",
type=int,
default=modelArgsDefaults.value_symbols_count,
help="Total number of symbols used in attribute value strings.")
parser.add_argument("--max_node_fanout",
type=int,
default=modelArgsDefaults.max_node_fanout,
help="Maximum connectivity fanout of an XML node.")
parser.add_argument("--max_node_text_len",
type=int,
default=modelArgsDefaults.max_node_text_len,
help="Maximum length of text attribute in a node.")
parser.add_argument(
"--max_attrib_value_len",
type=int,
default=modelArgsDefaults.max_attrib_value_len,
help="Maximum length of any text attribute value in a node.")
parser.add_argument("--max_output_len",
type=int,
default=modelArgsDefaults.max_output_len,
help="Maximum length of the output file.")
# Size meta-parameters of the generated neural network.
parser.add_argument("--node_text_vec_len",
type=int,
default=modelArgsDefaults.node_text_vec_len,
help="Length of encoded vector for node text.")
parser.add_argument("--attrib_value_vec_len",
type=int,
default=modelArgsDefaults.attrib_value_vec_len,
help="Length of an encoded attribute value.")
parser.add_argument(
"--node_info_propagator_stack_depth",
type=int,
default=modelArgsDefaults.node_info_propagator_stack_depth,
help=
"Depth of the graph layer stack. This determines the number of " +
"hops that information would propagate in the graph inside nodeInfoPropagator."
)
parser.add_argument(
"--propagated_info_len",
type=int,
default=modelArgsDefaults.propagated_info_len,
help="Length of node information vector, when being propagated.")
parser.add_argument(
"--output_decoder_stack_depth",
type=int,
default=modelArgsDefaults.output_decoder_stack_depth,
help="Stack depth of node decoder.")
parser.add_argument(
"--output_decoder_state_width",
type=int,
default=modelArgsDefaults.output_decoder_state_width,
help="Width of GRU cell in output decoder.")
parser.add_argument(
"--attentionSubspaceVecLen",
type=int,
default=modelArgsDefaults.attentionSubspaceVecLen,
help=
"Vec length of subspace of attnReadyVecs and decoder hidden state"
+ "that is used to comptue attention factors.")
# Other meta-parameters for training the neural network.
parser.add_argument("--input_dropout_p",
type=float,
default=None if basicAppConfig.resume else
modelArgsDefaults.input_dropout_p,
help="Input dropout probability.")
parser.add_argument("--dropout_p",
type=float,
default=None if basicAppConfig.resume else
modelArgsDefaults.dropout_p,
help="Dropout probability.")
parser.add_argument(
"--use_attention",
type=int,
default=modelArgsDefaults.use_attention,
help="Use attention while selcting most appropriate.")
parser.add_argument(
"--teacher_forcing_ratio",
type=int,
default=modelArgsDefaults.teacher_forcing_ratio,
help="Teacher forcing ratio to using during decoder training.")
parser.add_argument("--learning_rate",
type=float,
default=modelArgsDefaults.learning_rate,
help="Learning rate to use during training.")
parser.add_argument('--clip_gradient',
type=float,
default=modelArgsDefaults.clip_gradient,
help='gradient clipping')
parser.add_argument(
"--disable_batch_norm",
type=str2bool,
default=modelArgsDefaults.disable_batch_norm,
help="Disable batch norm. Needed for running some tests.")
parser.add_argument(
"--enableSpotlight",
type=str2bool,
default=modelArgsDefaults.enableSpotlight,
help="Whether to enable spotlight, which is designed to optimize" +
" search.")
parser.add_argument(
"--spotlightThreshold",
type=float,
default=modelArgsDefaults.spotlightThreshold,
help="Threshold used to identify encoder positions to be considered"
+ "for evaluation.")
parser.add_argument(
"--useSrcPtr",
type=str2bool,
default=modelArgsDefaults.useSrcPtr,
help=
"When set to true, the model is enabled to directly copy symbols from source."
)
if mode == AppMode.Evaluate:
parser.add_argument(
"--beam_count",
type=int,
default=None,
help=
"Number of beams to use when decoding. Leave as None for not using beam decoding."
)
if mode == AppMode.Generate:
parser.add_argument('--count',
help="Total number of data entries to generate",
default=generatorArgsDefaults.count)
parser = domainGeneratorModule.addArguments(parser,
generatorArgsDefaults)
# Parse args to build app config dictionary.
appConfig = parser.parse_args()
# Apply random seed.
if appConfig.random_seed is not None:
random.seed(appConfig.random_seed)
torch.manual_seed(appConfig.random_seed)
appConfig.mode = int(mode)
if mode == AppMode.Generate:
domainGeneratorModule.postProcessArguments(appConfig)
generatorArgs = levelDown(appConfig, "generatorArgs",
generatorArgsDefaults.keys())
return appConfig, generatorArgs
else:
# Post process app config.
postProcessAppConfig(appConfig, mode)
# Spin out model arguments from app configuration.
modelArgs = levelDown(appConfig, "modelArgs", modelArgsDefaults.keys())
return appConfig, modelArgs
class Dropdown(urwid.PopUpLauncher):
# Based in part on SelectOne widget from
# https://github.com/tuffy/python-audio-tools
signals = ["change"]
label = None
empty_label = u"\N{EMPTY SET}"
margin = 0
def __init__(
self,
items=None,
label=None,
default=None,
border=False,
scrollbar=False,
margin=None,
left_chars=None,
right_chars=None,
left_chars_top=None,
right_chars_top=None,
auto_complete=False,
max_height=10,
# keymap = {}
):
if items is not None:
self._items = items
if label is not None:
self.label = label
self.default = default
self.border = border
self.scrollbar = scrollbar
self.auto_complete = auto_complete
# self.keymap = keymap
if margin:
self.margin = margin
if isinstance(self.items, list):
if len(self.items):
if isinstance(self.items[0], tuple):
self._items = AttrDict(self.items)
else:
logger.debug(self.items)
self._items = AttrDict(
((item, n) for n, item in enumerate(self.items)))
else:
self._items = AttrDict()
else:
self._items = self.items
self.button = DropdownItem(
u"",
None,
margin=self.margin,
left_chars=left_chars_top if left_chars_top else left_chars,
right_chars=right_chars_top if right_chars_top else right_chars)
self.pop_up = DropdownDialog(
self,
self._items,
self.default,
label=self.label,
border=self.border,
margin=self.margin,
left_chars=left_chars,
right_chars=right_chars,
auto_complete=self.auto_complete,
scrollbar=scrollbar,
max_height=max_height,
# keymap = self.KEYMAP
)
urwid.connect_signal(self.pop_up, "select",
lambda souce, selection: self.select(selection))
urwid.connect_signal(self.pop_up, "close",
lambda button: self.close_pop_up())
if self.default is not None:
try:
if isinstance(self.default, str):
self.select_label(self.default)
else:
raise StopIteration
except StopIteration:
try:
self.select_value(self.default)
except StopIteration:
self.focus_position = 0
if len(self):
self.select(self.selection)
else:
self.button.set_label(("dropdown_text", self.empty_label))
cols = [(self.button_width, self.button)]
if self.label:
cols[0:0] = [
("pack", urwid.Text([("dropdown_label", "%s: " % (self.label))
])),
]
self.columns = urwid.Columns(cols, dividechars=0)
w = self.columns
if self.border:
w = urwid.LineBox(self.columns)
w = urwid.Padding(w, width=self.width)
super(Dropdown, self).__init__(w)
urwid.connect_signal(self.button, 'click',
lambda button: self.open_pop_up())
@classmethod
def get_palette_entries(cls):
return {
"dropdown_text":
PaletteEntry(
foreground="light gray",
background="dark blue",
foreground_high="light gray",
background_high="#003",
),
"dropdown_focused":
PaletteEntry(
foreground="white",
background="light blue",
foreground_high="white",
background_high="#009",
),
"dropdown_highlight":
PaletteEntry(
foreground="yellow",
background="light blue",
foreground_high="yellow",
background_high="#009",
),
"dropdown_label":
PaletteEntry(foreground="white", background="black"),
"dropdown_prompt":
PaletteEntry(foreground="light blue", background="black")
}
@keymap_command()
def complete_prefix(self):
if not self.auto_complete:
return
self.open_pop_up()
self.pop_up.complete_prefix()
@keymap_command()
def complete_substring(self):
if not self.auto_complete:
return
self.open_pop_up()
self.pop_up.complete_substring()
def create_pop_up(self):
# print("create")
return self.pop_up
@property
def button_width(self):
return self.pop_up.max_item_width + self.button.decoration_width
@property
def pop_up_width(self):
w = self.button_width
if self.border:
w += 2
return w
@property
def contents_width(self):
# raise Exception(self.button.width)
w = self.button_width
if self.label:
w += len(self.label) + 2
return max(self.pop_up.width, w)
@property
def width(self):
width = max(self.contents_width, self.pop_up.width)
if self.border:
width += 2
return width
@property
def height(self):
height = self.pop_up.height + 1
return height
def pack(self, size, focus=False):
return (self.width, self.height)
@property
def page_size(self):
return self.pop_up.height
def open_pop_up(self):
# print("open")
super(Dropdown, self).open_pop_up()
def close_pop_up(self):
super(Dropdown, self).close_pop_up()
def get_pop_up_parameters(self):
return {
'left': (len(self.label) + 2 if self.label else 0),
'top': 0,
'overlay_width': self.pop_up_width,
'overlay_height': self.pop_up.height
}
@property
def focus_position(self):
return self.pop_up.focus_position
@focus_position.setter
def focus_position(self, pos):
# self.select_index(pos)
old_pos = self.focus_position
self.pop_up.selected_button = self.pop_up.focus_position = pos
self.select(self.selection)
@property
def items(self):
return self._items
@property
def selection(self):
return self.pop_up.selection
def select_label(self, label, case_sensitive=False):
old_value = self.value
def f(x):
return x
if not case_sensitive:
def f(x):
return x.lower()
index = next(
itertools.dropwhile(lambda x: f(x[1]) != f(label),
enumerate((self._items.keys()))))[0]
self.focus_position = index
@property
def items(self):
return self._items
@property
def selection(self):
return self.pop_up.selection
def select_label(self, label, case_sensitive=False):
old_value = self.value
def f(x):
return x
if not case_sensitive:
def f(x):
return x.lower()
index = next(
itertools.dropwhile(lambda x: f(x[1]) != f(label),
enumerate((self._items.keys()))))[0]
self.focus_position = index
def select_value(self, value):
index = next(
itertools.dropwhile(lambda x: x[1] != value,
enumerate((self._items.values()))))[0]
self.focus_position = index
@property
def labels(self):
return self._items.keys()
@property
def values(self):
return self._items.values()
@property
def selected_label(self):
return self.selection.label
@selected_label.setter
def selected_label(self, label):
return self.select_label(label)
@property
def selected_value(self):
if not self.selection:
return None
return self.selection.value
@selected_value.setter
def selected_value(self, value):
return self.select_value(value)
@property
def value(self):
return self.selected_value
@value.setter
def value(self, value):
old_value = self.value
# try to set by value. if not found, try to set by label
try:
self.selected_value = value
except StopIteration:
self.selected_label = value
def cycle_prev(self):
self.cycle(-1)
@keymap_command("cycle")
def cycle(self, n):
pos = self.focus_position + n
if pos > len(self) - 1:
pos = len(self) - 1
elif pos < 0:
pos = 0
# self.focus_position = pos
self.focus_position = pos
def select(self, button):
logger.debug("select: %s" % (button))
self.button.set_label(("dropdown_text", button.label))
self.pop_up.dropdown_buttons.listbox.set_focus_valign("top")
# if old_pos != pos:
self._emit("change", self.selected_label, self.selected_value)
# def set_items(self, items, selected_value):
# self._items = items
# self.make_selection([label for (label, value) in items if
# value is selected_value][0],
# selected_value)
def __len__(self):
return len(self.items)
class BaseProvider(abc.ABC, Observable):
"""
Abstract base class from which providers should inherit from
"""
SESSION_CLASS = StreamSession
LISTING_CLASS = model.TitledMediaListing
# VIEW_CLASS = SimpleProviderView
# FILTERS = AttrDict()
ATTRIBUTES = AttrDict(title={"width": ("weight", 1)})
MEDIA_TYPES = None
RPC_METHODS = []
def __init__(self, *args, **kwargs):
self._view = None
self._session = None
self._active = False
self._filters = AttrDict(
{n: f(provider=self, name=n)
for n, f in self.FILTERS.items()})
rules = AttrDict(self.config.rules.label or {},
**config.settings.profile.rules.label or {})
labels = AttrDict(self.config.labels, **config.settings.profile.labels)
self.rule_map = AttrDict([(re.compile(k, re.IGNORECASE), v)
for k, v in [(r, rules[r])
for r in rules.keys()]])
self.highlight_map = AttrDict([(re.compile(k,
re.IGNORECASE), labels[v])
for k, v in rules.items()])
self.highlight_re = re.compile(
"(" + "|".join([k.pattern
for k in self.highlight_map.keys()]) + ")",
re.IGNORECASE)
# print(self.filters)
self.filters["search"].connect("changed", self.on_search_change)
def init_config(self):
with db_session:
try:
self.provider_data = model.ProviderData.get(
name=self.IDENTIFIER).settings
except AttributeError:
self.provider_data = model.ProviderData(
name=self.IDENTIFIER).settings
for name, f in self.filters.items():
value = self.default_filter_values.get(name, None)
if value:
try:
f.value = value
except (ValueError, ):
# import ipdb; ipdb.set_trace()
pass
@property
def default_filter_values(self):
return AttrDict()
@db_session
def save_provider_data(self):
model.ProviderData.get(
name=self.IDENTIFIER).settings = self.provider_data
commit()
@property
def LISTING_CLASS(self):
for cls in [self.__class__] + list(self.__class__.__bases__):
pkg = sys.modules.get(cls.__module__)
pkgname = pkg.__name__.split(".")[-1]
try:
return next(
v for k, v in pkg.__dict__.items()
if pkgname in k.lower() and k.endswith("MediaListing"))
except StopIteration:
continue
return model.TitledMediaListing
@property
def MEDIA_SOURCE_CLASS(self):
for cls in [self.__class__] + list(self.__class__.mro()):
pkg = sys.modules.get(cls.__module__)
pkgname = pkg.__name__.split(".")[-1]
try:
return next(
v for k, v in pkg.__dict__.items()
if pkgname in k.lower() and k.endswith("MediaSource"))
except (StopIteration, AttributeError):
continue
return model.MediaSource
@property
def helper(self):
return None
@property
def session_params(self):
return {"proxies": config.settings.profile.get("proxies")}
@property
def PREVIEW_TYPES(self):
return ["default"]
@property
def session(self):
if self._session is None:
session_params = self.session_params
self._session = self.SESSION_CLASS.new(self.IDENTIFIER,
**session_params)
return self._session
@property
def gui(self):
return self._view is not None
@property
def filters(self):
return self._filters
@property
def view(self):
if not self._view:
self._view = self.make_view()
self._view.update()
return self._view
@property
def toolbar(self):
return self.view.toolbar
@property
def is_active(self):
return self._active
def activate(self):
if self.is_active:
return
self._active = True
self.on_activate()
def deactivate(self):
if not self.is_active:
return
self.on_deactivate()
self._active = False
def on_activate(self):
self.reset()
def on_deactivate(self):
self.view.on_deactivate()
@property
def VIEW(self):
return SimpleProviderView(self, ProviderDataTable(self))
# @abc.abstractmethod
def make_view(self):
if not self.config_is_valid:
return InvalidConfigView(self.NAME, self.REQUIRED_CONFIG)
return self.VIEW
@classproperty
def IDENTIFIER(cls):
return next(c.__module__ for c in cls.__mro__
if __package__ in c.__module__).split(".")[-1]
@classproperty
@abc.abstractmethod
def NAME(cls):
return cls.__name__.replace("Provider", "")
@property
def FILTERS_BROWSE(self):
return AttrDict()
@property
def FILTERS_OPTIONS(self):
return AttrDict([("search", TextFilter)])
@property
def FILTERS(self):
d = getattr(self, "FILTERS_BROWSE", AttrDict())
d.update(getattr(self, "FILTERS_OPTIONS", {}))
return d
def on_search_change(self, value, *args):
if getattr(self, "search_task", False):
self.search_task.cancel()
async def apply_search_async():
await asyncio.sleep(1)
await self.apply_search_query(value)
self.search_task = state.event_loop.create_task(apply_search_async())
async def apply_search_query(self, query):
self.view.apply_search_query(query)
def parse_spec(self, spec):
(identifier, options) = MEDIA_SPEC_RE.search(spec).groups()
try:
selection, filters, identifier_options = self.parse_identifier(
identifier)
self.apply_identifier(selection, filters, identifier_options)
except SGIncompleteIdentifier:
selection, identifier_options = None, {}
options = AttrDict(identifier_options, **self.parse_options(options))
self.apply_options(options)
return (selection, options)
def parse_identifier(self, identifier):
return (None, identifier, {})
def apply_identifier(self, selection, filters, options):
if filters:
selected_filters = zip(self.filters.keys(), filters)
for f, value in selected_filters:
if value is None or value in [
getattr(self.filters[f], "selected_label", None),
self.filters[f].value
]:
continue
try:
self.filters[f].selected_label = value
except ValueError:
self.filters[f].value = value
def parse_options(self, options):
if not options:
options = ""
d = AttrDict(
[(list(self.FILTERS_OPTIONS.keys())[n], v) for n, v in enumerate(
[o for o in options.split(",") if "=" not in o]) if v],
**dict(o.split("=") for o in options.split(",") if "=" in o))
return d
def apply_options(self, options):
for k, v in options.items():
if v is None:
continue
if k in self.filters:
logger.debug(f"option: {k}={v}")
try:
if self.filters[k].value != v:
self.filters[k].value = v
except StopIteration:
raise SGException("invalid value for %s: %s" % (k, v))
def new_media_source(self, *args, **kwargs):
return self.MEDIA_SOURCE_CLASS.attr_class(provider_id=self.IDENTIFIER,
*args,
**kwargs)
def new_listing(self, **kwargs):
return self.LISTING_CLASS.attr_class(provider_id=self.IDENTIFIER,
**kwargs)
async def play(self, listing, **kwargs):
# sources, kwargs = self.extract_sources(listing, **kwargs)
task = self.create_play_task(listing, **kwargs)
yield state.task_manager.play(task)
async def download(self,
listing,
index=None,
no_task_manager=False,
**kwargs):
for task in self.create_download_tasks(listing, index=index, **kwargs):
yield state.task_manager.download(task)
def translate_template(self, template):
return template
# def new_listing_attr(self, **kwargs):
# return self.LISTING_CLASS.attr_class(
# provider_id = self.IDENTIFIER,
# **kwargs
# )
def sort(self, field, reverse=False):
self.view.sort(field, reverse=reverse)
@abc.abstractmethod
def listings(self, filters=None):
pass
def should_download(self, listing):
return listing.label in (list(self.config.rules) +
list(config.settings.profile.rules.download))
def on_new_listing(self, listing):
try:
label = next(l for r, l in self.rule_map.items()
if r.search(listing.title))
listing.label = label
if self.should_download(listing):
self.download(listing)
except StopIteration:
pass
@property
def config(self):
return config.ConfigTree(
config.settings.profile.providers.get(self.IDENTIFIER, {}))
@property
def config_is_valid(self):
def check_config(required, cfg):
if isinstance(required, dict):
for k, v in required.items():
if not k in cfg:
return False
if not check_config(required[k], cfg[k]):
return False
else:
for k in required:
if not k in cfg:
return False
return True
# return all([ self.config.get(x, None) is not None
# for x in getattr(self, "REQUIRED_CONFIG", [])
return check_config(getattr(self, "REQUIRED_CONFIG", []), self.config)
def get_source(self, selection, **kwargs):
sources = sorted(selection.sources,
key=lambda s: getattr(s, "rank", 0))
if not isinstance(sources, list):
sources = [sources]
return sources
def play_args(self, selection, **kwargs):
source = self.get_source(selection, **kwargs)
return (source, kwargs)
def filter_args(self):
return {f: self.filters[f].value for f in self.filters}
def extract_sources(self, listing, **kwargs):
try:
sources, kwargs = self.play_args(listing, **kwargs)
kwargs.update({
k: v
for k, v in self.filter_args().items() if k not in kwargs
})
except SGStreamNotFound as e:
logger.error(f"stream not found: {e}")
return
# FIXME: For now, we just throw playlists of media items at the default
# player program and hope it can handle all of them.
player_spec = None
downloader_spec = None
if not isinstance(sources, list):
sources = [sources]
for s in sources:
if not s.media_type:
# Try to set the content types of the source(s) with a HTTP HEAD
# request if the provider didn't specify one.
s.media_type = self.session.head(
s.locator).headers.get("Content-Type").split("/")[0]
return sources, kwargs
def create_play_task(self, listing, **kwargs):
sources, kwargs = self.extract_sources(listing, **kwargs)
media_types = set([s.media_type for s in sources if s.media_type])
player_spec = {"media_types": media_types}
if media_types == {"image"}:
downloader_spec = {None: None}
else:
downloader_spec = (getattr(self.config, "helpers", None)
or getattr(sources[0], "helper", None)
or self.helper)
return ListingsPlayMediaTask.attr_class(provider=self.NAME,
title=listing.title,
listing=listing,
sources=sources,
args=(player_spec,
downloader_spec),
kwargs=kwargs)
@property
def limit(self):
return None
def refresh(self):
self.view.refresh()
def reset(self):
self.view.reset()
def __str__(self):
return self.NAME
def __repr__(self):
return f"<{type(self)}: {self.NAME}>"
@property
def playlist_title(self):
return f"[{self.IDENTIFIER}"
@property
def auto_preview_enabled(self):
return not self.config.auto_preview.disabled
@property
def auto_preview_default(self):
return self.config.auto_preview.default if self.auto_preview_enabled else "default"
@property
def strip_emoji(self):
return (self.config.get("strip_emoji")
or config.settings.profile.tables.get("strip_emoji") or False)
@property
def translate(self):
return (self.config.get("translate")
or config.settings.profile.tables.get("translate") or False)
@property
def translate_src(self):
return "auto"
@property
def translate_dest(self):
return (self.config.get("translate_dest")
or config.settings.profile.tables.get("translate_dest")
or "en")
@property
def output_path(self):
return (self.config.get_path("output.path")
or config.settings.profile.get_path("output.path"))
