return list_group_prefix
if __name__ == "__main__":
args = get_args()
# get logger
logger = h.get_logger(args.file_id, 'missing_values')
# load proteomics data
data_df = pd.read_csv(args.input_file, header=0, index_col=None)
# get parameters
rule_params = h.load_json_parameter(args.file_id)
filename = h.filename(args.input_file)
data_structure = h.load_json_data(args.file_id, filename,
rule_params['all']['divide'])
values_cols_prefix = rule_params['all']['values_cols_prefix']
# NaN per protein and per group
group_prefix = get_groups(data_structure, values_cols_prefix)
result_df, stats_per_groups = fqc.na_per_group(data_df, group_prefix,
values_cols_prefix)
result_df = fqc.flag_row_supp(
result_df, stats_per_groups,
rule_params['missing_values']['max_na_percent_proteins'], 'na')
keep_specific = rule_params['all']['specific_proteins']['keep']
col_name = rule_params['all']['specific_proteins']['column_name']
# get data for samples in group
data_group = df.filter(regex=group)
# compute reduction on this data
reduced_data_group = compute_reduction(data_group, ddof)
# add result to result df
res = pd.concat([res, reduced_data_group], axis=1)
return res
if __name__ == "__main__":
args = get_args()
rule_params = h.load_json_parameter(args.file_id)
filename = h.filename(args.input_file)
data_structure = h.load_json_data(args.file_id, filename)
# get logger
logpath = os.path.join(paths.global_data_dir, args.file_id,
'log/reduce_line.log')
logger = h.get_logger(logpath)
# load data
data_df = pd.read_csv(args.input_file, header=0, index_col=None)
# get parameters
ddof = rule_params['reduction']['ddof']
values_cols_prefix = rule_params['all']['values_cols_prefix']
metadata_col = rule_params['all']['metadata_col']
depth = len(rule_params['reduction']['on']) + 1 # +1 for prefix
def compute_protein_rank_per_subset(data_df):
gmean_col = data_df.filter(regex='gmean_')
for col in gmean_col:
subset_name = '_'.join(col.split('_')[1:])
data_df['rank_abundance_{}'.format(subset_name)] = data_df[
'gmean_{}'.format(subset_name)].rank(ascending=False)
return data_df
if __name__ == "__main__":
args = get_args()
rule_params = h.load_json_parameter(args.project, args.version)
filename = h.filename(args.input_file)
data_structure = h.load_json_data(args.project, args.version, filename)
logpath = os.path.join(rule_params['all']['logpath'], 'abundance_rank.log')
logger = h.get_logger(logpath)
data_df = pd.read_csv(args.input_file, header=0, index_col=0)
print(data_df.columns.values)
print(data_df.head())
logger.info('{} proteins to analyse.'.format(len(data_df)))
ab_col = get_abundance_subset(data_structure, rule_params)
print(ab_col)
data_df = compute_protein_gmean_per_subset(data_df, ab_col)
result_df = compute_protein_rank_per_subset(data_df)
try:
def main() -> int:
"""
:param debug: Set to True to enable debug
:return:
"""
# Parsing arguments
arg_desc = sys.argv[0] + ' script taking file in JSON as input and apply configuration'
parser = argparse.ArgumentParser(description=arg_desc)
parser.add_argument('-n', help='JSON formatted file with node connectivity params', action='store',
required=True)
parser.add_argument('-r', help='directory for reports', action='store', required=False)
parser.add_argument('--log', help=f'set logging level'
f'Possible values: {log_levels}', action='store', required=False)
args = parser.parse_args()
if args.log.upper() in log_levels:
user_level = getattr(logging, args.log.upper())
else:
user_level = logging.WARNING
if args.r:
user_report_directory = args.r
else:
user_report_directory = REPORTS_PATH
# TODO: extend with capability to load several configuration params data files
# Loading node attributes
node_data = load_json_data(f"{NODE_DATA_PATH}/{args.n}")
logger = enable_logging(name="l3topo", log_file=f"{LOG_PATH}/l3topo.log", level=user_level)
logger.info(f"Logging level: {user_level}")
logger.info(f"Directory for reporting: {user_report_directory}")
logger.debug("%s", "=" * 20 + " Node connectivity data " + "=" * 20)
logger.debug(msg=json.dumps(node_data, indent=4, skipkeys=True))
# Check for proxy object presence
fabric_proxy = None
if node_data.get('fabric_proxy'):
fabric_proxy = node_data.pop('fabric_proxy')
# TODO: add checking for proxy attrs
fp_ip = sroslib.extract_ipaddr(fabric_proxy["ip_address"])
if fp_ip:
fabric_proxy["ip_address"] = sroslib.extract_ipaddr(fabric_proxy["ip_address"])
logger.debug("Fabric proxy configuration set: %s", fabric_proxy)
else:
fabric_proxy = None
sros_node_obj = {}
for k, v in node_data.items():
ip = sroslib.extract_ipaddr(node_data[k]["ip_address"])
if ip:
node_data[k]["ip_address"] = ip
if node_data[k].get("proxy"):
if not fabric_proxy:
msg = "Proxy enabled, but proxy object is not correct or not specified."
raise ValueError(msg)
node_data[k].pop("proxy")
node_data[k]["fabric_proxy"] = fabric_proxy
sros_node_obj[k] = sroslib.SROSNode(k, **node_data[k])
else:
msg = f"Node {k} IP@ is not correct. Please provide correct one."
raise ValueError(msg)
sros_ordered_list = list(sros_node_obj.keys())
# Availability checking common DataFrame creation
ping_res = []
for node in sros_ordered_list:
logger.info(f"Checking {node} availability ....")
# Collecting results
ping_res.append([sros_node_obj[node].ping_ones for i in range(3)])
# Checking ping results and adding alert, if needed
for n, ping in enumerate(ping_res):
if any(ping):
ping.append(NO_ALERT)
else:
ping.append(ALERT)
# Removing not available elements for further processing
sros_ordered_list.pop(n)
data = np.array(ping_res)
if not sros_ordered_list:
logger.error("No nodes are available for processing.")
exit(1)
df_ping = pd.DataFrame(data, columns=["TRY#1", "TRY#2", "TRY#3", "IssueFound"], index=sros_ordered_list)
st_ping = df_ping.style.applymap(color_up_down)
# Template handling
env = load_j2_env(path_to_templ=f"./{J2_PATH}/")
# TODO: l2 templates directory to be taken from params
template = env.get_template('node_availability.html')
logger.info(f"Rendering report for nodes availability ....")
# Rendering
html = template.render(ping_table=st_ping.render())
# Write the HTML file
with open(f'./{user_report_directory}/node_availability.html', mode='w') as fh:
fh.write(html)
# Processing nodes one by one and generating report for each
for node in sros_ordered_list:
html = ip_fabric_l3topo(sros_node_obj[node], logger)
# Write the HTML file
with open(f'./{user_report_directory}/{node}.html', mode='w') as fh:
fh.write(html)
return 0