def __getitem__(self, attr):
# if using star notation -> return dict of all keys that match
if "*" in attr:
return search(self, attr, separator=DEFAULT_STAR_SEPARATOR)
# otherwise -> basic dict.get
else:
return dict.get(self, attr)
async def show_keys(
client: influx.QueryClient,
database: Optional[str] = None,
measurement: Optional[str] = None,
**_ # allow, but discard all other kwargs
) -> dict:
"""Selects available keys (without data) from Influx."""
query = 'SHOW FIELD KEYS'
if measurement:
query += ' FROM "{measurement}"'
params = _prune(locals(), ['query', 'database', 'measurement'])
query_response = await client.query(**params)
response = dict()
for path, meas_name in dpath.search(query_response,
'results/*/series/*/name',
yielded=True,
dirs=False):
# results/[index]/series/[index]/values/*/0
values_glob = '/'.join(path.split('/')[:-1] + ['values', '*', '0'])
response[meas_name] = dpath.values(query_response, values_glob)
return response
def marketise_all_tickers_for_stock_cash(cls,
start_date: datetime,
end_date: datetime.date,
overwrite: bool = False) -> None:
"""
This function marketise all tickers for cash stocks
:parm start_date: the date from which we marketise
:parm end_date: the date will which we marketise (inclusive)
:parm overwrite: a flag that determines if we overwrite existing data or not
"""
mkt_cfg = mkt_classes.mkt_data_cfg()
xpath = '{0}/{1}'.format("equity", "single stock").upper()
search = dpath.search(mkt_cfg, xpath, yielded=True)
equity_tickers_to_marketise = [
i for i in search
].pop()[1] # let us dump all the tickers and their metadata
for ticker, metadata in equity_tickers_to_marketise.items():
cls.marketise_equity_index_ticker(ticker,
metadata['default_source'],
start_date,
end_date,
overwrite=overwrite)
print('finished the marketisiation process for {}'.format(xpath))
def _set_if_absent(d, path, value):
if '*' in path:
[pre, post] = path.split('*')
elem_count = len(du.values(d, f'{pre}*'))
for i in range(elem_count):
_set_if_absent(d, f'{pre}{i}{post}', value)
elif du.search(d, path) == {}:
du.new(d, path, value())
def get(self, path, default=None, separator='/'):
for found_path, value in dpath_util.search(self.source.obj,
path,
yielded=True,
separator=separator):
if path == '{}{}'.format(separator, found_path):
return value
return default
def __setitem__(self, attr, val):
# if using star notation -> search and set all keys matching
if "*" in attr:
for k, _ in search(self,
attr,
yielded=True,
separator=DEFAULT_STAR_SEPARATOR):
setattr(self, k, val)
# otherwise -> just __setitem__
else:
dict.__setitem__(self, attr, val)
def save_snp_500_tickers(tickers: list) -> None:
"""update the YAML market coordinates config with the SNP500 tickers"""
mkt_class = "equity".upper()
mkt_type = "single stock".upper()
market_coordinates = mkt_classes.mkt_data_cfg()
# lets load the defaults and then see if there is tsdb yaml to overwrite base defaults
defaults = mkt_coord_defaults.defaults.copy()
mkt_default_cfg_load = mkt_classes.mkt_defaults_cfg()
dp.merge(defaults, mkt_default_cfg_load)
equity_defaults = [
i for i in dp.search(
defaults, '{0}/{1}'.format(mkt_class, mkt_type), yielded=True)
].pop()[1]
for ticker in tickers:
mkt_asset = ticker
points_default = [
i for i in dp.search(market_coordinates,
'{0}/{1}/{2}/points'.format(
mkt_class, mkt_type, mkt_asset),
yielded=True)
]
points_default = points_default.pop()[1] if len(points_default) else []
points = list(set(points_default))
value = {'points': points}
value.update(equity_defaults)
xpath = '{0}/{1}/{2}'.format(mkt_class, mkt_type, mkt_asset)
dp.new(market_coordinates, xpath, value)
mkt_data_cfg = {
'market_coordinates': market_coordinates,
'defaults': defaults
}
with open(mkt_classes.tsdb_path() + 'market_coord_cfg.YAML', "w") as f:
yaml.dump(mkt_data_cfg, f)
"added snp500 tickers to the config"
def to_csv(self, tag, filename):
with open(filename, 'w') as f:
f.write('ct, tag')
for vname in du.get(self.memory, '/memory/var_names'):
f.write(f', {vname}')
f.write('\n')
data = du.search(self.memory, f'/data/*/{tag}/*')['data']
for ct in data.keys():
f.write(f'{ct}, {tag}')
for vname in du.get(self.memory, '/memory/var_names'):
vvalue = data[ct][tag].get(vname, 'n.a')
f.write(f', {vvalue}')
f.write('\n')
return
def __getitem__(self, attr):
# if using minus and star notation: split attribute -> search -> filter
if ("|" in attr) and ("*" in attr):
assert attr.count("|") == 1, f"Only one filter allowed: {attr}"
search_attr, filter_out = attr.split("|")
# recursive call __getitem__ without filtering substring
searched = self.__getitem__(search_attr)
# filter
return {k: v for k, v in searched.items() if filter_out not in k}
# if using star notation -> return dict of all keys that match
if "*" in attr:
return search(self, attr, separator=DEFAULT_STAR_SEPARATOR)
# otherwise -> basic dict.get
else:
return dict.get(self, attr)
def __setitem__(self, attr, val):
# if using minus and star notation: split attribute -> search -> filter
if ("|" in attr) and ("*" in attr):
assert attr.count("|") == 1, f"Only one filter allowed: {attr}"
search_attr, filter_out = attr.split("|")
attr = search_attr
else:
# if not, filter is long random string
filter_out = "".join(
random.choice(string.ascii_lowercase) for _ in range(30))
# if using star notation -> search and set all keys matching
if "*" in attr:
for k, _ in search(self,
attr,
yielded=True,
separator=DEFAULT_STAR_SEPARATOR):
if filter_out not in k:
setattr(self, k, val)
# otherwise -> just __setitem__
else:
dict.__setitem__(self, attr, val)
def get(self, path, default=None, separator='/'):
for found_path, value in dpath_util.search(self.source.obj, path, yielded=True, separator=separator):
if path == '{}{}'.format(separator, found_path):
return value
return default
def DoGetAll(dictionary, path):
vals = list(dpu.search(dictionary, path, yielded=True))
return vals
def dump_results_to_db(device_name, lldp_infos) -> None:
nodes_list: List[Tuple[Dict[str, str], Dict[str, str]]] = []
links_list: List[Tuple[Dict[str, str], Dict[str, str]]] = []
# Each item of the lists are composed by the "query" (so the DB knows which entry to update
# And the actual data
dev_name = device_name.lower()
query = {"device_name": dev_name}
# We add the device if it doesn't exist
nodes_list.append((query, query))
for lldp_nei in lldp_infos:
# Getting neigh node infos and adding it to nodes_list
_, neigh_name = next(
search(lldp_nei,
f"{NEEDED_MIBS['lldp_neigh_name']}*",
yielded=True))
if not neigh_name:
continue
neigh_name = neigh_name.lower()
# IP is a lil' special since it is written in the oid (yeah weird)
neigh_ip_oid, _ = next(
search(lldp_nei, f"{NEEDED_MIBS['lldp_neigh_ip']}*", yielded=True))
neigh_ip = ".".join(neigh_ip_oid.split(".")[-4:])
query_neigh = {"device_name": neigh_name}
nodes_list.append((query_neigh, {
"device_name": neigh_name,
"device_ip": neigh_ip
}))
# Getting neigh and local ifaces infos and adding them to link list
_, local_iface = next(
search(lldp_nei,
f"{NEEDED_MIBS['lldp_local_iface']}*",
yielded=True))
_, neigh_iface = next(
search(lldp_nei,
f"{NEEDED_MIBS['lldp_neigh_iface']}*",
yielded=True))
# Stripping "Et, Ethernet, E,... " which can be different per equipment
dev_iface = "/".join(
"".join(x)
for is_number, x in groupby(local_iface, key=str.isdigit)
if is_number is True)
neigh_iface = "/".join(
"".join(x)
for is_number, x in groupby(neigh_iface, key=str.isdigit)
if is_number is True)
query_link = {
"device_name": dev_name,
"iface_name": dev_iface,
"neighbor_name": neigh_name,
"neighbor_iface": neigh_iface,
}
links_list.append((query_link, query_link))
query_neigh_link = {
"device_name": neigh_name,
"iface_name": neigh_iface,
"neighbor_name": dev_name,
"neighbor_iface": dev_iface,
}
links_list.append((query_neigh_link, query_neigh_link))
try:
bulk_update_collection(NODES_COLLECTION, nodes_list)
bulk_update_collection(LINKS_COLLECTION, links_list)
except InvalidOperation:
print(
"Nothing to dump to db (wasn't able to scrap devices?), passing..")
def dump_results_to_db(device_name, ifaces_infos) -> None: # pylint: disable=too-many-locals
utilization_list: List[Tuple[Dict[str, str], Dict[str, str]]] = []
stats_list: List[Dict[str, str]] = []
for iface in ifaces_infos:
_, ifname = next(
search(iface, f"{NEEDED_MIBS['iface_name']}*", yielded=True))
ifname = ifname.lower()
if (ifname.startswith("se") or ifname.startswith("nu")
or ifname.startswith("lo") or ifname.startswith("mgm")
or ifname.startswith("mana") or ifname.startswith("po")
or ifname == "vlan1"):
# TODO: Mgmt ifaces/lo & po could actually be interesting... Need to think about this
continue
_, mtu = next(search(iface, f"{NEEDED_MIBS['mtu']}*", yielded=True))
_, mac = next(search(iface, f"{NEEDED_MIBS['mac']}*", yielded=True))
_, speed = next(search(iface, f"{NEEDED_MIBS['speed']}*",
yielded=True))
_, in_disc = next(
search(iface, f"{NEEDED_MIBS['in_disc']}*", yielded=True))
_, in_err = next(
search(iface, f"{NEEDED_MIBS['in_err']}*", yielded=True))
_, out_disc = next(
search(iface, f"{NEEDED_MIBS['out_disc']}*", yielded=True))
_, out_err = next(
search(iface, f"{NEEDED_MIBS['out_err']}*", yielded=True))
_, in_octets = next(
search(iface, f"{NEEDED_MIBS['in_octets']}*", yielded=True))
_, in_ucast_pkts = next(
search(iface, f"{NEEDED_MIBS['in_ucast_pkts']}*", yielded=True))
_, in_mcast_pkts = next(
search(iface, f"{NEEDED_MIBS['in_mcast_pkts']}*", yielded=True))
_, in_bcast_pkts = next(
search(iface, f"{NEEDED_MIBS['in_bcast_pkts']}*", yielded=True))
_, out_octets = next(
search(iface, f"{NEEDED_MIBS['out_octets']}*", yielded=True))
_, out_ucast_pkts = next(
search(iface, f"{NEEDED_MIBS['out_ucast_pkts']}*", yielded=True))
_, out_mcast_pkts = next(
search(iface, f"{NEEDED_MIBS['out_mcast_pkts']}*", yielded=True))
_, out_bcast_pkts = next(
search(iface, f"{NEEDED_MIBS['out_bcast_pkts']}*", yielded=True))
iface_infos_dict = {
"mtu": mtu,
"mac": hexlify(mac.encode()).decode(),
"speed": speed,
"in_discards": in_disc,
"in_errors": in_err,
"out_discards": out_disc,
"out_errors": out_err,
"in_bytes": in_octets,
"in_ucast_pkts": in_ucast_pkts,
"in_mcast_pkts": in_mcast_pkts,
"in_bcast_pkts": in_bcast_pkts,
"out_bytes": out_octets,
"out_ucast_pkts": out_ucast_pkts,
"out_mcast_pkts": out_mcast_pkts,
"out_bcast_pkts": out_bcast_pkts,
}
iface_name = "/".join(
"".join(x) for is_number, x in groupby(ifname, key=str.isdigit)
if is_number is True)
iface_stats_dict = {
"device_name": device_name,
"iface_name": iface_name,
"timestamp": int(time()),
}
iface_stats_dict.update(iface_infos_dict)
stats_list.append(iface_stats_dict)
# Each item of the lists are composed are the "query" (so the DB knows which entry to update
# And the actual data
query = {"device_name": device_name, "iface_name": iface_name}
highest = int(iface_infos_dict["in_bytes"])
lowest = int(iface_infos_dict["out_bytes"])
if lowest > highest:
highest = lowest
previous_utilization = get_latest_utilization(device_name, iface_name)
utilization = {
"device_name": device_name,
"iface_name": iface_name,
"prev_utilization": previous_utilization,
"last_utilization": highest,
}
utilization_list.append((query, utilization))
try:
bulk_update_collection(UTILIZATION_COLLECTION, utilization_list)
add_iface_stats(stats_list)
except InvalidOperation:
print(
"Nothing to dump to db (wasn't able to scrap devices?), passing..")
def modify_values_yaml(experiment_folder: str, script_location: str, script_parameters: Tuple[str, ...],
experiment_name: str, pack_type: str, username: str,
cluster_registry_port: int, pack_params: List[Tuple[str, str]] = None,
env_variables: List[str] = None):
log.debug("Modify values.yaml - start")
pack_params = pack_params if pack_params else []
values_yaml_filename = os.path.join(experiment_folder, f"charts/{pack_type}/values.yaml")
values_yaml_temp_filename = os.path.join(experiment_folder, f"charts/{pack_type}/values_temp.yaml")
with open(values_yaml_filename, "r") as values_yaml_file:
template = jinja2.Template(values_yaml_file.read())
rendered_values = template.render(NAUTA = {
'ExperimentName': experiment_name,
'CommandLine': common.prepare_script_paramaters(script_parameters, script_location),
'RegistryPort': str(cluster_registry_port),
'ExperimentImage': f'127.0.0.1:{cluster_registry_port}/{username}/{experiment_name}:latest',
'ImageRepository': f'127.0.0.1:{cluster_registry_port}/{username}/{experiment_name}:latest'
})
v = yaml.safe_load(rendered_values)
workersCount = None
pServersCount = None
regex = re.compile(r"^\[.*|^\{.*") # Regex used for detecting dicts/arrays in pack params
for key, value in pack_params:
if re.match(regex, value):
try:
value = ast.literal_eval(value)
except Exception as e:
raise AttributeError(Texts.CANT_PARSE_VALUE.format(value=value, error=e))
# Handle boolean params
elif value in {"true", "false"}:
value = str(_parse_yaml_boolean(value))
if key == WORK_CNT_PARAM:
workersCount = value
if key == P_SERV_CNT_PARAM:
pServersCount = value
dutil.new(v, key, value, '.')
# setting sum of replicas involved in multinode training if both pServersCount and workersCount are present in
# the pack or given in the cli
if (WORK_CNT_PARAM in v or workersCount) and (P_SERV_CNT_PARAM in v or pServersCount):
number_of_replicas = int(v.get(WORK_CNT_PARAM)) if not workersCount else int(workersCount)
number_of_replicas += int(v.get(P_SERV_CNT_PARAM)) if not pServersCount else int(pServersCount)
v[POD_COUNT_PARAM] = number_of_replicas
elif (WORK_CNT_PARAM in v or workersCount) and (POD_COUNT_PARAM not in v):
number_of_replicas = int(v.get(WORK_CNT_PARAM)) if not workersCount else int(workersCount)
v[POD_COUNT_PARAM] = number_of_replicas + 1
env_variables = env_variables if env_variables else []
parsed_envs = []
for variable in env_variables:
key, value = variable.split("=")
one_env_map = {"name": key, "value": value}
parsed_envs.append(one_env_map)
# Set OMP_NUM_THREADS to be equal to cpu limit if it was not explicitly passed
if "OMP_NUM_THREADS" not in (env["name"] for env in parsed_envs):
try:
cpu_limit = calculate_omp_num_threads(v)
if cpu_limit:
parsed_envs.append({"name": "OMP_NUM_THREADS", "value": str(cpu_limit)})
except (ValueError, TypeError, KeyError):
log.exception("Failed to infer OMP_NUM_THREADS value.")
envs_to_set = {'env', 'worker.env', 'master.env'} # Env placeholders in values.yaml that we expect
for env in envs_to_set:
if dutil.search(v, env, separator='.'):
dutil.get(v, env, separator='.').extend(parsed_envs)
with open(values_yaml_temp_filename, "w") as values_yaml_file:
yaml.safe_dump(v, values_yaml_file)
shutil.move(values_yaml_temp_filename, values_yaml_filename)
log.debug("Modify values.yaml - end")
