def load_agg_info(agg_result: list, j: json, keys: list):
if j.items().__len__() == 0:
return "...... *** NO DATA ***"
for key, value in j.items():
if key == 'dv':
continue
if keys[0] == '*':
agg_result.append({'key': key, 'value': value})
continue
for onekey in keys:
if key.startswith(onekey):
agg_result.append({'key': key, 'value': value})
continue
def clinc_json_to_df(
loaded_json: json,
portions: List = None
) -> Dict[str, pd.DataFrame]:
"""
Convert a particular CLINC JSON file to a dict of DFs.
:param loaded_json: JSON object - must be loaded using json.load.
:param portions: The portions of the dataset requested; train,
dev, etc.
:return dfs: A dict of DataFrames, containing data in the
provided JSON.
"""
# Load subset of portions of the dataset, else all of them.
if portions:
subset = portions
else:
subset = loaded_json.items()
# Submethod verifies the portion exists, etc.
dfs = {
key: clinc_json_portion_to_df(loaded_json, value)
for key, value
in subset
}
return dfs
def _cumulus_ipv4_converter(hostname: str(),
plateform: str(),
cmd_output: json,
*,
filters=dict()) -> ListIPV4Interface:
ipv4_addresses_lst = ListIPV4Interface(hostname=hostname,
ipv4_addresses_lst=list())
for interface_name, facts in cmd_output.items():
# v0 = vrrp / vrr / hsrp / HA ip address
# bridge is a Linux bridge
if _generic_interface_filter(
interface_name=_mapping_interface_name(interface_name),
plateform=plateform,
filters=filters):
for ip_address_in_interface in facts.get("iface_obj").get(
'ip_address').get('allentries'):
if "/128" not in ip_address_in_interface and "/64" not in ip_address_in_interface and \
"/48" not in ip_address_in_interface and "::" not in ip_address_in_interface and \
"127.0.0.1" not in ip_address_in_interface and "::1/128" not in ip_address_in_interface:
ipv4_obj = IPV4Interface(
interface_name=_mapping_interface_name(interface_name),
ip_address_with_mask=ip_address_in_interface)
ipv4_addresses_lst.ipv4_addresses_lst.append(ipv4_obj)
return ipv4_addresses_lst
def set_custom_config_options(cfg: json):
global CONFIG, CONFIG_OVERRIDES
for key, value in cfg.items():
CONFIG[key] = value
# Update record of what overrides have been applied.
CONFIG_OVERRIDES[key] = value
def debug_print(data: json):
try:
print("[+] ---Debug info---")
for i, v in data.items():
if i == "outline":
print("[+] -", i, " :", len(v), "characters")
continue
if i == "actor_photo" or i == "year":
continue
print("[+] -", "%-11s" % i, ":", v)
print("[+] ---Debug info---")
except:
pass
def debug_print(data: json):
try:
print("[+] ---Debug info---")
for i, v in data.items():
if i == 'outline':
print('[+] -', i, ' :', len(v), 'characters')
continue
if i == 'actor_photo' or i == 'year':
continue
print('[+] -', "%-11s" % i, ':', v)
print("[+] ---Debug info---")
except:
pass
def check_json_safey(json_data: json, max_json_length=250, include_fields=None,
current_depth=1, max_depth=3, replace_blank=True):
"""
Check Json Safey
"""
if current_depth > max_depth:
return False, json_data, '数据错误'
elif current_depth == 1:
json_length = len(json.dumps(json_data))
if json_length > max_json_length:
return False, json_data, '数据超出最大长度'
if isinstance(json_data, dict):
new_data = {}
for key, val in json_data.items():
if include_fields is not None and key not in include_fields:
continue
if isinstance(val, str) and replace_blank:
new_data[key] = val.replace(' ', '')
elif isinstance(val, dict) or isinstance(val, list):
ret = check_json_safey(
val, current_depth=current_depth+1,
max_depth=max_depth, replace_blank=replace_blank)
if not ret[0]:
return ret
_, new_data[key], _ = ret
else:
new_data[key] = val
return True, new_data, ""
elif isinstance(json_data, list):
new_data = []
for val in json_data:
if isinstance(val, str) and replace_blank:
new_data.append(val.replace(' ', ''))
elif isinstance(val, dict) or isinstance(val, list):
ret = check_json_safey(
val, current_depth=current_depth+1,
max_depth=max_depth, replace_blank=replace_blank)
if not ret[0]:
return ret
_, ret, _ = ret
new_data.append(ret)
else:
new_data.append(val)
return True, new_data, ""
return False, json_data, "参数错误"
def to_sql_update(self, json_data: json, keys: list, table_name=None):
val = []
cond = []
if not table_name:
table_name = self.tablename
for k, v in json_data.items():
if k in self.table_def.keys():
tmp = f"{k}={self.trans_value(v)}"
val.append(tmp)
value = ','.join(val)
for k in keys:
tmp = f" ({k}={self.trans_value(json_data[k])}) "
cond.append(tmp)
condition = 'AND'.join(cond)
sql = f"UPDATE {table_name} set {value} WHERE {condition}"
return sql
def debug_print(data: json):
try:
print("[+] ------- DEBUG INFO -------")
for i, v in data.items():
if i == 'outline':
print('[+] -', "%-19s" % i, ':', len(v), 'characters')
continue
if i == 'actor_photo' or i == 'year':
continue
if i == 'extrafanart':
print('[+] -', "%-19s" % i, ':', len(v), 'links')
continue
print(f'[+] - {i:<{cnspace(i,19)}} : {v}')
print("[+] ------- DEBUG INFO -------")
except:
pass
def schema_path_id_generator(json_schema: json,
nested_dict: NestedDict,
id_path=[]):
for key, value in json_schema.items():
if isinstance(value, dict):
id_path.append(key)
if key != 'properties':
nested_dict[id_path].update({'$id': f'#/{"/".join(id_path)}'})
if all(not isinstance(item, dict) for item in value.values()):
id_path.pop()
schema_path_id_generator(value, nested_dict)
if key in SCHEMA_OBJECT_VARIABLES:
id_path.pop(-1)
if id_path:
if key not in SCHEMA_VARIABLES and id_path[
-1] not in SCHEMA_OBJECT_VARIABLES:
id_path.pop(-1)
def _cumulus_ipv6_converter(hostname: str(),
plateform: str(),
cmd_output: json,
*,
filters=dict()) -> ListIPV6Interface:
if cmd_output is None or cmd_output == "":
return None
ipv6_addresses_lst = ListIPV6Interface(ipv6_addresses_lst=list())
for interface_name, facts in cmd_output.items():
# v0 = vrrp / vrr / hsrp / HA ip address
# bridge is a Linux bridge
if _generic_interface_filter(
plateform=plateform,
interface_name=_mapping_interface_name(interface_name),
filters=filters):
for ip_address_in_interface in facts.get("iface_obj").get(
'ip_address').get('allentries'):
if str(ip_address_in_interface).find("/") != -1:
index_slash = str(ip_address_in_interface).find("/")
try:
ipaddress.IPv6Address(
ip_address_in_interface[:index_slash])
is_valid = True
except ipaddress.AddressValueError as e:
is_valid = False
if is_valid:
if "::1/128" not in ip_address_in_interface:
ipv6_addresses_lst.ipv6_addresses_lst.append(
IPV6Interface(
interface_name=_mapping_interface_name(
interface_name),
ip_address_with_mask=ip_address_in_interface
))
return ipv6_addresses_lst
def to_sql_insert(self, json_data: json, table_name=None):
"""
Generate a sql like 'Replace into <table> (<cols>) values (<vals>)'
"""
# initial 2 list
col_section = []
value_section = []
# set table name
if not table_name:
table_name = self.tablename
# iter for each key in json.
for k, v in json_data.items():
if k in self.table_def.keys():
col_section.append(k)
# values should be tranlate into sql format
value_section.append(self.trans_value(v))
# combine into sql and returns.
col = ','.join(col_section)
val = ','.join(value_section)
sql = f"INSERT IGNORE into {table_name} ({col}) values ({val})"
return (sql)
def validate_metric_restrictions(value: json):
""" Check if metric restrictions JSON is in the correct format
and contains correct keys and values.
"""
# Allow empty value
if not value:
return {}
if isinstance(value, str):
value = json.loads(value)
if not isinstance(value, dict):
raise ValidationError(
f"Incorrect input format: {type(value)}. Correct format is {type({})}."
)
for key, restrictions in list(value.items()):
if not isinstance(restrictions, dict):
raise ValidationError(
f"Incorrect input format for dict value: {type(restrictions)}. Correct format is {type({})}."
)
if key not in choices.METRICS:
raise ValidationError(
f"Invalid key: {key}. Allowed metric keys are: {choices.METRICS}."
)
for restriction_key, restriction_value in list(restrictions.items()):
if restriction_key not in choices.METRIC_RESTRICTION_FIELDS:
raise ValidationError(
f"Invalid restriction key: {restriction_key}. Allowed restriction keys are: {choices.METRIC_RESTRICTION_FIELDS}."
)
if not isinstance(restriction_value, float) and not isinstance(
restriction_value, int):
raise ValidationError(
f"Invalid type for restriction '{key} - {restriction_key}': {type(restriction_value)}. Correct type is {type(1.0)}."
)
if not 0 <= restriction_value <= 1.0:
raise ValidationError(
f"Invalid value for restriction '{key} - {restriction_key}': {restriction_value}. The value should be in range [0.0, 1.0]."
)
return value
def convert_to_list(data: json) -> list:
my_return = list(data.items())
my_return.sort(key=lambda my_tuple: my_tuple[1])
return my_return