def sensor_analysis_main(sensor_df, switch_params_aggregated_df,
report_columns_usage_dct, report_data_lst):
"""Main function to analyze zoning configuration"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = ['sensor_aggregated', 'Датчики']
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# unpacking DataFrames from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
sensor_aggregated_df, sensor_report_df = data_lst
# list of data to analyze from report_info table
analyzed_data_names = [
'switch_params_aggregated', 'fabric_labels', 'sensor'
]
# force run when any data from data_lst was not saved (file not found) or
# procedure execution explicitly requested for output data or data used during fn execution
force_run = verify_force_run(data_names, data_lst, report_steps_dct,
max_title, analyzed_data_names)
if force_run:
# current operation information string
info = f'Generating sensor readings table'
print(info, end=" ")
# aggregated DataFrames
sensor_aggregated_df = sensor_aggregation(sensor_df,
switch_params_aggregated_df)
# after finish display status
status_info('ok', max_title, len(info))
# report tables
sensor_report_df = sensor_report(sensor_aggregated_df, data_names,
report_columns_usage_dct, max_title)
# create list with partitioned DataFrames
data_lst = [sensor_aggregated_df, sensor_report_df]
# saving data to json or csv file
save_data(report_data_lst, data_names, *data_lst)
# verify if loaded data is empty and replace information string with empty DataFrame
else:
sensor_aggregated_df, sensor_report_df = verify_data(
report_data_lst, data_names, *data_lst)
data_lst = [sensor_aggregated_df, sensor_report_df]
# save data to service file if it's required
for data_name, data_frame in zip(data_names, data_lst):
save_xlsx_file(data_frame, data_name, report_data_lst)
return sensor_aggregated_df
def blade_system_extract(blade_folder, report_data_lst):
"""Function to extract blade systems information"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = ['blade_interconnect', 'blade_servers', 'blade_vc']
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# unpacking from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
module_comprehensive_lst, blades_comprehensive_lst, blade_vc_comprehensive_lst = data_lst
# data force extract check
# list of keys for each data from data_lst representing if it is required
# to re-collect or re-analyze data even they were obtained on previous iterations
force_extract_keys_lst = [
report_steps_dct[data_name][1] for data_name in data_names
]
# print data which were loaded but for which force extract flag is on
force_extract_check(data_names, data_lst, force_extract_keys_lst,
max_title)
# when any of data_lst was not saved or
# force extract flag is on then re-extract data from configueation files
if not all(data_lst) or any(force_extract_keys_lst):
# lists to store only REQUIRED infromation
# collecting data for all blades during looping
# list containing enclosure, blade and hba information for all blade systems
blades_comprehensive_lst = []
# list containing enclosure and interconnect modules information for all blade systems
module_comprehensive_lst = []
# list containing virtual connect ports information for all blade systems
blade_vc_comprehensive_lst = []
if blade_folder:
print('\nEXTRACTING BLADES SYSTEM INFORMATION ...\n')
# collects files in folder with txt extension
txt_files = find_files(blade_folder,
max_title,
filename_extension='txt')
log_files = find_files(blade_folder,
max_title,
filename_extension='log')
blade_configs_lst = txt_files + log_files
# number of files to check
configs_num = len(blade_configs_lst)
if configs_num:
# data imported from init file to extract values from config file
enclosure_params, _, comp_keys, match_keys, comp_dct = data_extract_objects(
'blades', max_title)
module_params = columns_import('blades', max_title,
'module_params')
blade_params = columns_import('blades', max_title,
'blade_params')
for i, blade_config in enumerate(blade_configs_lst):
# file name with extension
configname_wext = os.path.basename(blade_config)
# remove extension from filename
configname, _ = os.path.splitext(configname_wext)
# Active Onboard Administrator IP address
oa_ip = None
# interconnect modules number
module_num = 0
# current operation information string
info = f'[{i+1} of {configs_num}]: {configname} system.'
print(info, end=" ")
# search control dictionary. continue to check file until all parameters groups are found
collected = {
'enclosure': False,
'oa_ip': False,
'module': False,
'servers': False,
'vc': False
}
# if blade_lst remains empty after file checkimng than status_info shows NO_DATA for current file
blade_lst = []
enclosure_vc_lst = []
with open(blade_config, encoding='utf-8',
errors='ignore') as file:
# check file until all groups of parameters extracted
while not all(collected.values()):
line = file.readline()
if not line:
break
# enclosure section start
if re.search(
r'>SHOW ENCLOSURE INFO|^ +ENCLOSURE INFORMATION$',
line):
enclosure_dct = {}
collected['enclosure'] = True
# while not reach empty line
while not re.search(r'Serial Number', line):
line = file.readline()
# dictionary with match names as keys and match result of current line with all imported regular expressions as values
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# name_value_pair_match
if match_dct[match_keys[0]]:
result = match_dct[match_keys[0]]
enclosure_dct[result.group(1).strip(
)] = result.group(2).strip()
if not line:
break
# rename Description key to Enclosure Type key for VC
if enclosure_dct.get('Description'):
enclosure_dct[
'Enclosure Type'] = enclosure_dct.pop(
'Description')
# creating list with REQUIRED enclosure information only
enclosure_lst = [
enclosure_dct.get(param)
for param in enclosure_params
]
# enclosure section end
# vc fabric connection section start
elif re.search(r'FABRIC INFORMATION', line):
info_type = 'Type VC'
print(info_type, end=" ")
info = info + " " + info_type
line = file.readline()
collected['vc'] = True
while not re.search(
r'FC-CONNECTION INFORMATION', line):
# dictionary with match names as keys and match result of current line with all imported regular expressions as values
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# vc_port_match
if match_dct[match_keys[14]]:
vc_port = line_to_list(
comp_dct[comp_keys[14]], line,
*enclosure_lst)
enclosure_vc_lst.append(vc_port)
blade_vc_comprehensive_lst.append(
vc_port)
line = file.readline()
else:
line = file.readline()
if not line:
break
# vc fabric connection section end
# active onboard administrator ip section start
elif re.search(r'>SHOW TOPOLOGY *$', line):
info_type = 'Type Blade Enclosure'
print(info_type, end=" ")
info = info + " " + info_type
line = file.readline()
collected['oa_ip'] = True
while not re.search(r'^>SHOW', line):
# dictionary with match names as keys and match result of current line with all imported regular expressions as values
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# oa_ip_match
if match_dct[match_keys[1]]:
oa_ip = match_dct[match_keys[1]].group(
1)
line = file.readline()
break
else:
line = file.readline()
if not line:
break
# active onboard administrator ip section end
# interconnect modules section start
elif re.search(r'>SHOW INTERCONNECT INFO ALL',
line):
line = file.readline()
collected['modules'] = True
while not re.search(r'^>SHOW', line):
# dictionary with match names as keys and match result of current line with all imported regular expressions as values
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# module_type_num_match
if match_dct[match_keys[2]]:
module_dct = {}
module_lst = []
module = match_dct[match_keys[2]]
# interconnect module slot number
module_slot = module.group(1)
# interconnect module type (Ethernet, FC)
module_type = module.group(2).rstrip()
line = file.readline()
# module_section_end_comp
while not re.search(
comp_dct[comp_keys[3]], line):
# dictionary with match names as keys and match result of current line with all imported regular expressions as values
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# name_value_pair_match
if match_dct[match_keys[0]]:
result = match_dct[
match_keys[0]]
name = result.group(1).strip()
value = result.group(2).strip()
# if value is empty string use None
if value == '':
value = None
module_dct[name] = value
line = file.readline()
else:
line = file.readline()
if not line:
break
# creating list with REQUIRED interconnect module information only
module_lst = [
module_dct.get(param)
for param in module_params
]
# add current module information to list containing all modules infromation
# oa_ip added as None and extracted later in the file
module_comprehensive_lst.append([
*enclosure_lst, oa_ip, module_slot,
module_type, *module_lst
])
# based on module's number oa_ip is added to module_comprehensive_lst after extraction
module_num += 1
else:
line = file.readline()
if not line:
break
# interconnect modules section end
# blade server, hba and flb section start
elif re.search(r'>SHOW SERVER INFO ALL', line):
line = file.readline()
collected['servers'] = True
while not re.search(r'^>SHOW', line):
# dictionary with match names as keys and match result of current line with all imported regular expressions as values
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# blade_server_num_match
if match_dct[match_keys[4]]:
blade_dct = {}
blade_lst = []
hba_lst = []
result = match_dct[match_keys[4]]
blade_dct[result.group(
1)] = result.group(2)
# blade_num = result.group(2)
# print("Blade number:", blade_num)
line = file.readline()
# server_section_end_comp
while not re.search(
comp_dct[comp_keys[11]], line):
# dictionary with match names as keys and match result of current line with all imported regular expressions as values
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# mezzanin hba section start
# mezzanine_model_match
if match_dct[match_keys[6]]:
result = match_dct[
match_keys[6]]
hba_description = result.group(
1)
hba_model = result.group(2)
line = file.readline()
# mezzanine_wwn_comp
while re.search(
comp_dct[comp_keys[7]],
line):
# dictionary with match names as keys and match result of current line with all imported regular expressions as values
match_dct = {
match_key:
comp_dct[comp_key].
match(line)
for comp_key, match_key
in zip(
comp_keys,
match_keys)
}
# mezzanine_wwn_match
result = match_dct[
match_keys[7]]
wwnp = result.group(1)
hba_lst.append([
hba_description,
hba_model, wwnp
])
line = file.readline()
# mezzanin hba section end
# flex flb hba section start
# flb_model_match and flex_ethernet_match
elif match_dct[match_keys[
8]] or match_dct[
match_keys[15]]:
if match_dct[match_keys[8]]:
result = match_dct[
match_keys[8]]
flex_description = result.group(
1)
if re.search(
comp_dct[
comp_keys[13]],
line):
flex_model = re.search(
comp_dct[
comp_keys[13]],
line).group(1)
else:
flex_model = None
elif match_dct[match_keys[15]]:
result = match_dct[
match_keys[15]]
flex_description = result.group(
1)
flex_model = result.group(
1)
line = file.readline()
# wwn_mac_line_comp
while re.search(
comp_dct[comp_keys[9]],
line):
# dictionary with match names as keys and match result of current line with all imported regular expressions as values
match_dct = {
match_key:
comp_dct[comp_key].
match(line)
for comp_key, match_key
in zip(
comp_keys,
match_keys)
}
# flb_wwn_match
if match_dct[
match_keys[10]]:
result = match_dct[
match_keys[10]]
wwnp = result.group(1)
hba_lst.append([
flex_description,
flex_model, wwnp
])
line = file.readline()
# flex flb hba section end
# blade server section start
# blade_server_info_match
elif match_dct[match_keys[5]]:
result = match_dct[
match_keys[5]]
# name = result.group(1) + result.group(2) if result.group(2) else result.group(1)
name = result.group(1).rstrip()
value = result.group(
3).rstrip()
# to avoid Type parameter overwrire
# add parameter only if parameter has not been added to blade dictionary before
if not blade_dct.get(name):
blade_dct[name] = value
line = file.readline()
# blade server section end
# if none of matches found for current blade server than next line
else:
line = file.readline()
if not line:
break
# unpopulated blade slots have 'Server Blade Type' line but populated have 'Type' line
# add 'Server Blade Type' parameter for populated slots for consistency
if blade_dct.get('Type'):
blade_dct[
'Server Blade Type'] = blade_dct.pop(
'Type')
# creating list with REQUIRED blade server information only
blade_lst = [
blade_dct.get(param)
for param in blade_params
]
# if hba or flex cards installed in blade server
if len(hba_lst):
# add line for each hba to blades_comprehensive_lst
for hba in hba_lst:
blades_comprehensive_lst.append(
[
*enclosure_lst,
*blade_lst, *hba
])
# if no nba add one line with enclosure and blade info only
else:
blades_comprehensive_lst.append([
*enclosure_lst, *blade_lst,
None, None
])
# if no blade_server_num_match found in >SHOW SERVER INFO ALL section than next line
else:
line = file.readline()
if not line:
break
# blade server, hba and flb section end
# adding OA IP to module_comprehensive_lst based on interconnect modules number
for num in range(-1, -module_num - 1, -1):
module_comprehensive_lst[num][3] = oa_ip
# show status blades information extraction from file
if blade_lst or enclosure_vc_lst:
status_info('ok', max_title, len(info))
else:
status_info('no data', max_title, len(info))
# save extracted data to json file
save_data(report_data_lst, data_names,
module_comprehensive_lst, blades_comprehensive_lst,
blade_vc_comprehensive_lst)
else:
# current operation information string
info = f'Collecting enclosure, interconnect modules, blade servers, hba'
print(info, end=" ")
status_info('skip', max_title, len(info))
# save empty data to json file
save_data(report_data_lst, data_names, module_comprehensive_lst,
blades_comprehensive_lst, blade_vc_comprehensive_lst)
# verify if loaded data is empty after first iteration and replace information string with empty list
else:
module_comprehensive_lst, blades_comprehensive_lst, blade_vc_comprehensive_lst = verify_data(
report_data_lst, data_names, *data_lst)
return module_comprehensive_lst, blades_comprehensive_lst, blade_vc_comprehensive_lst
def interswitch_connection_extract(switch_params_lst, report_data_lst):
"""Function to extract interswitch connection information"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = ['isl', 'trunk', 'porttrunkarea']
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# unpacking from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
isl_lst, trunk_lst, porttrunkarea_lst = data_lst
# data force extract check
# list of keys for each data from data_lst representing if it is required
# to re-collect or re-analyze data even they were obtained on previous iterations
force_extract_keys_lst = [
report_steps_dct[data_name][1] for data_name in data_names
]
# print data which were loaded but for which force extract flag is on
force_extract_check(data_names, data_lst, force_extract_keys_lst,
max_title)
# when any of data_lst was not saved or
# force extract flag is on then re-extract data from configueation files
if not all(data_lst) or any(force_extract_keys_lst):
print(
'\nEXTRACTING INTERSWITCH CONNECTION INFORMATION (ISL, TRUNK, TRUNKAREA) ...\n'
)
# extract chassis parameters names from init file
switch_columns = columns_import('switch', max_title, 'columns')
# number of switches to check
switch_num = len(switch_params_lst)
# data imported from init file to extract values from config file
*_, comp_keys, match_keys, comp_dct = data_extract_objects(
'isl', max_title)
# lists to store only REQUIRED infromation
# collecting data for all switches ports during looping
isl_lst = []
trunk_lst = []
porttrunkarea_lst = []
# switch_params_lst [[switch_params_sw1], [switch_params_sw1]]
# checking each switch for switch level parameters
for i, switch_params_data in enumerate(switch_params_lst):
# data unpacking from iter param
# dictionary with parameters for the current switch
switch_params_data_dct = dict(
zip(switch_columns, switch_params_data))
switch_info_keys = [
'configname', 'chassis_name', 'chassis_wwn', 'switch_index',
'SwitchName', 'switchWwn', 'switchRole', 'Fabric_ID',
'FC_Router', 'switchMode'
]
switch_info_lst = [
switch_params_data_dct.get(key) for key in switch_info_keys
]
ls_mode_on = True if switch_params_data_dct[
'LS_mode'] == 'ON' else False
sshow_file, _, _, switch_index, switch_name, *_, switch_mode = switch_info_lst
# current operation information string
info = f'[{i+1} of {switch_num}]: {switch_name} isl, trunk and trunk area ports. Switch mode: {switch_mode}'
print(info, end=" ")
# search control dictionary. continue to check sshow_file until all parameters groups are found
collected = {'isl': False, 'trunk': False, 'trunkarea': False}
if switch_mode == 'Native':
with open(sshow_file, encoding='utf-8',
errors='ignore') as file:
# check file until all groups of parameters extracted
while not all(collected.values()):
line = file.readline()
if not line:
break
# isl section start
# switchcmd_islshow_comp
if re.search(comp_dct[comp_keys[0]],
line) and not collected['isl']:
collected['isl'] = True
if ls_mode_on:
while not re.search(
fr'^CURRENT CONTEXT -- {switch_index} *, \d+$',
line):
line = file.readline()
if not line:
break
# switchcmd_end_comp
while not re.search(comp_dct[comp_keys[2]], line):
line = file.readline()
match_dct = {
match_key: comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# islshow_match
if match_dct[match_keys[1]]:
isl_port = line_to_list(
comp_dct[comp_keys[1]], line,
*switch_info_lst[:-1])
# portcfg parameters
if isl_port[-1]:
isl_port[-1] = isl_port[-1].replace(
' ', ', ')
# appending list with only REQUIRED port info for the current loop iteration
# to the list with all ISL port info
isl_lst.append(isl_port)
if not line:
break
# isl section end
# trunk section start
# switchcmd_trunkshow_comp
if re.search(comp_dct[comp_keys[3]],
line) and not collected['trunk']:
collected['trunk'] = True
if ls_mode_on:
while not re.search(
fr'^CURRENT CONTEXT -- {switch_index} *, \d+$',
line):
line = file.readline()
if not line:
break
# switchcmd_end_comp
while not re.search(comp_dct[comp_keys[2]], line):
match_dct = {
match_key: comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# trunkshow_match
if match_dct[match_keys[4]]:
trunk_port = line_to_list(
comp_dct[comp_keys[4]], line,
*switch_info_lst[:-1])
# if trunk line has trunk number then remove ":" from trunk number
if trunk_port[9]:
trunk_port[9] = trunk_port[9].strip(
':')
trunk_num = trunk_port[9]
# if trunk line has no number then use number from previous line
else:
trunk_port[9] = trunk_num
# appending list with only REQUIRED trunk info for the current loop iteration
# to the list with all trunk port info
trunk_lst.append(trunk_port)
line = file.readline()
if not line:
break
# trunk section end
# porttrunkarea section start
# switchcmd_trunkarea_comp
if re.search(comp_dct[comp_keys[5]],
line) and not collected['trunkarea']:
collected['trunkarea'] = True
if ls_mode_on:
while not re.search(
fr'^CURRENT CONTEXT -- {switch_index} *, \d+$',
line):
line = file.readline()
if not line:
break
# switchcmd_end_comp
while not re.search(comp_dct[comp_keys[2]], line):
line = file.readline()
match_dct = {
match_key: comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# 'porttrunkarea_match'
if match_dct[match_keys[6]]:
porttrunkarea_port_lst = line_to_list(
comp_dct[comp_keys[6]], line,
*switch_info_lst[:6])
# for No_light ports port and slot numbers are '--'
if porttrunkarea_port_lst[11] == '--':
porttrunkarea_port_lst[10] = '--'
# if switch has no slots than slot number is 0
for idx in [6, 10]:
if not porttrunkarea_port_lst[idx]:
porttrunkarea_port_lst[idx] = str(
0)
porttrunkarea_lst.append(
porttrunkarea_port_lst)
if not line:
break
# porttrunkarea section end
status_info('ok', max_title, len(info))
# if switch in Access Gateway mode then skip
else:
status_info('skip', max_title, len(info))
# save extracted data to json file
save_data(report_data_lst, data_names, isl_lst, trunk_lst,
porttrunkarea_lst)
# verify if loaded data is empty after first iteration and replace information string with empty list
else:
isl_lst, trunk_lst, porttrunkarea_lst = verify_data(
report_data_lst, data_names, *data_lst)
return isl_lst, trunk_lst, porttrunkarea_lst
def err_sfp_cfg_analysis_main(portshow_aggregated_df, sfpshow_df,
portcfgshow_df, report_columns_usage_dct,
report_data_lst):
"""Main function to add porterr, transceiver and portcfg information to portshow DataFrame"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
portshow_sfp_force_flag = False
portshow_sfp_export_flag, *_ = report_steps_dct['portshow_sfp_aggregated']
# names to save data obtained after current module execution
data_names = [
'portshow_sfp_aggregated', 'Ошибки', 'Параметры_SFP',
'Параметры_портов'
]
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# unpacking DataFrames from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
portshow_sfp_aggregated_df, error_report_df, sfp_report_df, portcfg_report_df = data_lst
# list of data to analyze from report_info table
analyzed_data_names = [
'portshow_aggregated', 'sfpshow', 'portcfgshow', 'portcmd',
'switchshow_ports', 'switch_params_aggregated', 'fdmi',
'device_rename', 'report_columns_usage_upd', 'nscamshow', 'nsshow',
'alias', 'blade_servers', 'fabric_labels'
]
# force run when any data from data_lst was not saved (file not found) or
# procedure execution explicitly requested for output data or data used during fn execution
force_run = verify_force_run(data_names, data_lst, report_steps_dct,
max_title, analyzed_data_names)
if force_run:
# import transeivers information from file
sfp_model_df = dataframe_import('sfp_models', max_title)
# current operation information string
info = f'Updating connected devices table'
print(info, end=" ")
# add sfpshow, transceiver information and portcfg to aggregated portcmd DataFrame
portshow_sfp_aggregated_df = port_complete(portshow_aggregated_df,
sfpshow_df, sfp_model_df,
portcfgshow_df)
# after finish display status
status_info('ok', max_title, len(info))
# warning if UKNOWN SFP present
if (portshow_sfp_aggregated_df['Transceiver_Supported'] ==
'Unknown SFP').any():
info_columns = [
'Fabric_name', 'Fabric_label', 'configname', 'chassis_name',
'chassis_wwn', 'slot', 'port', 'Transceiver_Supported'
]
portshow_sfp_info_df = portshow_sfp_aggregated_df.drop_duplicates(
subset=info_columns).copy()
unknown_count = len(portshow_sfp_info_df[
portshow_sfp_info_df['Transceiver_Supported'] ==
'Unknown SFP'])
info = f'{unknown_count} {"port" if unknown_count == 1 else "ports"} with UNKNOWN supported SFP tag found'
print(info, end=" ")
status_info('warning', max_title, len(info))
# ask if save portshow_aggregated_df
if not portshow_sfp_export_flag:
reply = reply_request(
"Do you want to save 'portshow_sfp_aggregated'? (y)es/(n)o: "
)
if reply == 'y':
portshow_sfp_force_flag = True
# create reaport tables from port_complete_df DataFrtame
error_report_df, sfp_report_df, portcfg_report_df = \
create_report_tables(portshow_sfp_aggregated_df, data_names[1:], report_columns_usage_dct, max_title)
# saving data to json or csv file
data_lst = [
portshow_sfp_aggregated_df, error_report_df, sfp_report_df,
portcfg_report_df
]
save_data(report_data_lst, data_names, *data_lst)
# verify if loaded data is empty and reset DataFrame if yes
else:
portshow_sfp_aggregated_df, error_report_df, sfp_report_df, portcfg_report_df \
= verify_data(report_data_lst, data_names, *data_lst)
data_lst = [
portshow_sfp_aggregated_df, error_report_df, sfp_report_df,
portcfg_report_df
]
# save data to excel file if it's required
for data_name, data_frame in zip(data_names, data_lst):
force_flag = False
if data_name == 'portshow_sfp_aggregated':
force_flag = portshow_sfp_force_flag
save_xlsx_file(data_frame,
data_name,
report_data_lst,
force_flag=force_flag)
return portshow_sfp_aggregated_df
def fabricstatistics_main(portshow_aggregated_df, switchshow_ports_df,
fabricshow_ag_labels_df, nscamshow_df, portshow_df,
report_columns_usage_dct, report_data_lst):
"""Main function to count Fabrics statistics"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = ['fabric_statistics', 'Статистика_фабрики']
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were obtained on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# unpacking DataFrames from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
fabric_statistics_df, fabric_statistics_report_df = data_lst
# list of data to analyze from report_info table
analyzed_data_names = [
'portcmd', 'switchshow_ports', 'switch_params_aggregated',
'portshow_aggregated', 'switch_parameters', 'chassis_parameters',
'fdmi', 'nscamshow', 'nsshow', 'alias', 'blade_servers',
'fabric_labels'
]
chassis_column_usage = report_columns_usage_dct['chassis_info_usage']
force_run = verify_force_run(data_names, data_lst, report_steps_dct,
max_title, analyzed_data_names)
if force_run:
# current operation information string
info = f'Counting up Fabrics statistics'
print(info, end=" ")
fabric_statistics_df = statisctics_aggregated(portshow_aggregated_df,
switchshow_ports_df,
fabricshow_ag_labels_df,
nscamshow_df,
portshow_df,
report_data_lst)
# after finish display status
status_info('ok', max_title, len(info))
# get report DataFrame
fabric_statistics_report_df = statistics_report(
fabric_statistics_df, chassis_column_usage, max_title)
# create list with partitioned DataFrames
data_lst = [fabric_statistics_df, fabric_statistics_report_df]
# saving data to json or csv file
save_data(report_data_lst, data_names, *data_lst)
# verify if loaded data is empty and replace information string with empty DataFrame
else:
fabric_statistics_df, fabric_statistics_report_df = \
verify_data(report_data_lst, data_names, *data_lst)
data_lst = [fabric_statistics_df, fabric_statistics_report_df]
# save data to service file if it's required
for data_name, data_frame in zip(data_names, data_lst):
save_xlsx_file(data_frame, data_name, report_data_lst)
return fabric_statistics_df
def fabricshow_extract(switch_params_lst, report_data_lst):
"""
Function to extract from principal switch configuration
list of switches in fabric including AG switches
"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = ['fabricshow', 'ag_principal']
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# unpacking from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
fabricshow_lst, ag_principal_lst = data_lst
# data force extract check
# list of keys for each data from data_lst representing if it is required
# to re-collect or re-analyze data even they were obtained on previous iterations
force_extract_keys_lst = [
report_steps_dct[data_name][1] for data_name in data_names
]
# print data which were loaded but for which force extract flag is on
force_extract_check(data_names, data_lst, force_extract_keys_lst,
max_title)
# when any of data_lst was not saved or
# force extract flag is on then re-extract data from configueation files
if not all(data_lst) or any(force_extract_keys_lst):
print(
'\nEXTRACTING FABRICS INFORMATION FROM SUPPORTSHOW CONFIGURATION FILES ...\n'
)
# extract switch parameters names from init file
switch_columns = columns_import('switch', max_title, 'columns')
# number of switches to check
switch_num = len(switch_params_lst)
# list to store only REQUIRED switch parameters
# collecting data for all switches during looping
fabricshow_lst = []
ag_principal_lst = []
# data imported from init file to extract values from config file
*_, comp_keys, match_keys, comp_dct = data_extract_objects(
'fabricshow', max_title)
ag_params = columns_import('fabricshow', max_title, 'ag_params')
# switch_params_lst [[switch_params_sw1], [switch_params_sw1]]
# checking each switch for switch level parameters
for i, switch_params_data in enumerate(switch_params_lst):
# data unpacking from iter param
# dictionary with parameters for the current switch
switch_params_data_dct = dict(
zip(switch_columns, switch_params_data))
switch_info_keys = [
'configname', 'chassis_name', 'chassis_wwn', 'switch_index',
'SwitchName', 'switchWwn', 'switchRole', 'Fabric_ID',
'FC_Router', 'switchMode'
]
switch_info_lst = [
switch_params_data_dct[key] for key in switch_info_keys
]
ls_mode_on = True if switch_params_data_dct[
'LS_mode'] == 'ON' else False
sshow_file, _, _, switch_index, switch_name, _, switch_role = switch_info_lst[:
7]
# current operation information string
info = f'[{i+1} of {switch_num}]: {switch_name} fabric environment. Switch role: {switch_role}'
print(info, end=" ")
collected = {'fabricshow': False, 'ag_principal': False}
# switch_params_data_dct.get('FC_Router') == 'ON'
# print('collected', collected)
# check config of Principal switch only
if switch_role == 'Principal':
# principal_switch_lst contains sshow_file, chassis_name, chassis_wwn, switch_index, switch_name, switch_fid
principal_switch_lst = [
*switch_info_lst[:6], *switch_info_lst[7:9]
]
# search control dictionary. continue to check sshow_file until all parameters groups are found
with open(sshow_file, encoding='utf-8',
errors='ignore') as file:
# check file until all groups of parameters extracted
while not all(collected.values()):
line = file.readline()
if not line:
break
# fabricshow section start
if re.search(
r'^(SWITCHCMD /fabos/cliexec/)?fabricshow\s*:$',
line):
# when section is found corresponding collected dict values changed to True
collected['fabricshow'] = True
if ls_mode_on:
while not re.search(
fr'^CURRENT CONTEXT -- {switch_index} *, \d+$',
line):
line = file.readline()
if not line:
break
while not re.search(
r'^(real [\w.]+)|(\*\* SS CMD END \*\*)$',
line):
line = file.readline()
# dictionary with match names as keys and match result of current line with all imported regular expressions as values
match_dct = {
match_key: comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# match_keys ['fabricshow_match']
# 'fabricshow_match'
if match_dct[match_keys[0]]:
fabricshow_lst.append(
line_to_list(comp_dct[comp_keys[0]],
line,
*principal_switch_lst))
if not line:
break
# ag_principal section start
# switchcmd_agshow_comp
if re.search(comp_dct[comp_keys[4]], line):
collected['ag_principal'] = True
# if switch in LS mode switch to required LS number
if ls_mode_on:
while not re.search(
fr'^CURRENT CONTEXT -- {switch_index} *, \d+$',
line):
line = file.readline()
if not line:
break
# switchcmd_end_comp
while not re.search(comp_dct[comp_keys[10]], line):
match_dct = {
match_key: comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# ag_num_match
if match_dct[match_keys[5]]:
# dictionary to store all DISCOVERED switch ports information
# collecting data only for the logical switch in current loop
# Access Gateway common information dictionary
ag_info_dct = {}
# Attached F-Port information dictionary
ag_attach_dct = {}
# Access Gateway F-Port information dictionary
ag_fport_dct = {}
# Domaid ID, port_ID, port_index dictionary
did_port_dct = {}
# move cursor to one line down to get inside while loop
line = file.readline()
# ag_switchcmd_end_comp
while not re.search(
comp_dct[comp_keys[9]], line):
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# ag_info_match
if match_dct[match_keys[6]]:
ag_info_dct[match_dct[
match_keys[6]].group(
1).rstrip()] = match_dct[
match_keys[6]].group(
2).rstrip()
# ag_attached_match
elif match_dct[match_keys[7]]:
# if Attached F-Port information dictionary is empty than create dictionary with N-Port ID(s) as keys and empty lists as values
# if ag_attach_dct has been already created (not empty) then it's preserved
ag_attach_dct = ag_attach_dct or dict(
(n_portid, []) for n_portid in
ag_info_dct['N-Port ID(s)'].
split(','))
# extracting attached F-port data from line to list
ag_attach_lst = line_to_list(
comp_dct[comp_keys[7]], line)
# getting port_ID of N-port from port_id of F-port
n_portid = ag_attach_lst[
0][:-2] + '00'
# adding current line F-port information to Attached F-Port information dictionary
if n_portid in ag_attach_dct.keys(
):
ag_attach_dct[n_portid].append(
ag_attach_lst)
# ag_fport_match
elif match_dct[match_keys[8]]:
# create Access Gateway F-Port information dictionary
ag_fport_dct = ag_fport_dct or dict(
(n_portid, []) for n_portid in
ag_info_dct['N-Port ID(s)'].
split(','))
# extracting access gateway F-port data from line to list
ag_fport_lst = line_to_list(
comp_dct[comp_keys[8]], line)
# getting port_ID of N-port from port_id of F-port
n_portid = ag_fport_lst[
1][:-2] + '00'
# adding current line F-port information to Access Gateway F-Port information dictionary
if n_portid in ag_fport_dct.keys():
ag_fport_dct[n_portid].append(
ag_fport_lst)
line = file.readline()
if not line:
break
# list of N-ports extracted from N-Port ID(s) line
n_portids_lst = ag_info_dct[
'N-Port ID(s)'].split(',')
# (domain_id, n_portid)
did_port_lst = [
(int(n_portid[:4], 0), n_portid)
for n_portid in n_portids_lst
]
# creating dictionary with n_portid as keys and (domain_id, n_portid) as values
did_port_dct = {
port[1]: list(port)
for port in did_port_lst
}
# change values representation in dictionaries
# before {n_portid: [(port_id_1, port_wwn_1, f-port_num_1)], [(port_id_2, port_wwn_2, f-port_num_2)]}
# after {n_portid: [(port_id_1, port_id_2), (port_wwn_1, port_wwn_2), (f-port_num_1, f-port_num_1)]
ag_attach_dct = {
n_portid:
list(zip(*ag_attach_dct[n_portid]))
for n_portid in n_portids_lst
if ag_attach_dct.get(n_portid)
}
ag_fport_dct = {
n_portid:
list(zip(*ag_fport_dct[n_portid]))
for n_portid in n_portids_lst
if ag_fport_dct.get(n_portid)
}
# add connected switch port_index to did_port_dct extracted from ag_attach_dct
# (domain_id, n_portid, n_port_index)
# if no port_index then add None
for n_portid in n_portids_lst:
if ag_attach_dct.get(n_portid):
did_port_dct[n_portid].append(
ag_attach_dct[n_portid][2][0])
else:
did_port_dct[n_portid].append(None)
# for each element of list convert tuples to strings
# if no data extracted for the n_portid then add None for each parameter
for n_portid in n_portids_lst:
if ag_attach_dct.get(n_portid):
ag_attach_dct[n_portid] = [
', '.join(v) for v in
ag_attach_dct[n_portid]
]
else:
ag_attach_dct[n_portid] = [None
] * 3
for n_portid in n_portids_lst:
if ag_fport_dct.get(n_portid):
ag_fport_dct[n_portid] = [
', '.join(v)
for v in ag_fport_dct[n_portid]
]
else:
ag_fport_dct[n_portid] = [None] * 3
# getting data from ag_info_dct in required order
ag_info_lst = [
ag_info_dct.get(param, None)
for param in ag_params
]
# appending list with only REQUIRED ag info for the current loop iteration to the list with all ag switch info
for n_portid in n_portids_lst:
ag_principal_lst.append([
*principal_switch_lst[:-1],
*ag_info_lst,
*did_port_dct[n_portid],
*ag_attach_dct[n_portid],
*ag_fport_dct[n_portid]
])
else:
line = file.readline()
if not line:
break
# ag_principal section end
status_info('ok', max_title, len(info))
else:
status_info('skip', max_title, len(info))
# save extracted data to json file
save_data(report_data_lst, data_names, fabricshow_lst,
ag_principal_lst)
else:
fabricshow_lst, ag_principal_lst = verify_data(report_data_lst,
data_names, *data_lst)
return fabricshow_lst, ag_principal_lst
def sensor_extract(chassis_params_lst, report_data_lst):
"""Function to extract sensor information"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = ['sensor']
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# unpacking from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
sensor_lst, = data_lst
# data force extract check
# list of keys for each data from data_lst representing if it is required
# to re-collect or re-analyze data even they were obtained on previous iterations
force_extract_keys_lst = [
report_steps_dct[data_name][1] for data_name in data_names
]
# print data which were loaded but for which force extract flag is on
force_extract_check(data_names, data_lst, force_extract_keys_lst,
max_title)
# when any of data_lst was not saved or
# force extract flag is on then re-extract data from configueation files
if not all(data_lst) or any(force_extract_keys_lst):
print('\nEXTRACTING ENVIRONMENT DATA ...\n')
# # extract chassis parameters names from init file
# switch_columns = columns_import('switch', max_title, 'columns')
# extract chassis parameters names from init file
chassis_columns = columns_import('chassis', max_title, 'columns')
# number of switches to check
switch_num = len(chassis_params_lst)
# data imported from init file to extract values from config file
*_, comp_keys, match_keys, comp_dct = data_extract_objects(
'sensor', max_title)
# lists to store only REQUIRED infromation
# collecting data for all switches ports during looping
sensor_lst = []
# chassis_params_fabric_lst [[chassis_params_sw1], [chassis_params_sw1]]
# checking each chassis for switch level parameters
for i, chassis_params_data in enumerate(chassis_params_lst):
# data unpacking from iter param
# dictionary with parameters for the current chassis
chassis_params_data_dct = dict(
zip(chassis_columns, chassis_params_data))
chassis_info_keys = ['configname', 'chassis_name', 'chassis_wwn']
chassis_info_lst = [
chassis_params_data_dct.get(key) for key in chassis_info_keys
]
sshow_file, chassis_name, _ = chassis_info_lst
# current operation information string
info = f'[{i+1} of {switch_num}]: {chassis_name} sensor readings'
print(info, end=" ")
# search control dictionary. continue to check sshow_file until all parameters groups are found
collected = {'sensor': False}
with open(sshow_file, encoding='utf-8', errors='ignore') as file:
# check file until all groups of parameters extracted
while not all(collected.values()):
line = file.readline()
if not line:
break
# sensor section start
# switchcmd_sensorshow_comp
if re.search(comp_dct[comp_keys[0]],
line) and not collected['sensor']:
collected['sensor'] = True
# switchcmd_end_comp
while not re.search(comp_dct[comp_keys[2]], line):
line = file.readline()
match_dct = {
match_key: comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# islshow_match
if match_dct[match_keys[1]]:
sensor_reading = line_to_list(
comp_dct[comp_keys[1]], line,
*chassis_info_lst)
# appending list with only REQUIRED port info for the current loop iteration
# to the list with all ISL port info
sensor_lst.append(sensor_reading)
if not line:
break
# sensor section end
status_info('ok', max_title, len(info))
# save extracted data to json file
save_data(report_data_lst, data_names, sensor_lst)
# verify if loaded data is empty after first iteration and replace information string with empty list
else:
sensor_lst = verify_data(report_data_lst, data_names, *data_lst)
return sensor_lst
def switch_params_analysis_main(fabricshow_ag_labels_df, chassis_params_df,
switch_params_df, maps_params_df,
blade_module_loc_df, ag_principal_df,
report_data_lst):
"""Main function to create aggregated switch parameters table and report tables"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = [
'report_columns_usage', 'switch_params_aggregated', 'Коммутаторы',
'Фабрика', 'Параметры_коммутаторов', 'Лицензии',
'Глобальные_параметры_фабрики'
]
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# unpacking DataFrames from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
report_columns_usage_dct, switch_params_aggregated_df, switches_report_df, fabric_report_df, \
switches_parameters_report_df, licenses_report_df, global_fabric_parameters_report_df = data_lst
# list of data to analyze from report_info table
analyzed_data_names = [
'chassis_parameters', 'switch_parameters', 'switchshow_ports',
'maps_parameters', 'blade_interconnect', 'fabric_labels'
]
# clean fabricshow DataFrame from unneccessary data
fabric_clean_df = fabric_clean(fabricshow_ag_labels_df)
# force run when any data from data_lst was not saved (file not found) or
# procedure execution explicitly requested for output data or data used during fn execution
force_run = verify_force_run(data_names, data_lst, report_steps_dct,
max_title, analyzed_data_names)
if force_run:
# import data with switch models, firmware and etc
switch_models_df = dataframe_import('switch_models', max_title)
# current operation information string
info = f'Generating aggregated switch parameters table'
print(info, end=" ")
# create aggregated table by joining DataFrames
switch_params_aggregated_df, report_columns_usage_dct = \
fabric_aggregation(fabric_clean_df, chassis_params_df, \
switch_params_df, maps_params_df, switch_models_df, ag_principal_df)
# add 'Device_Location for Blade chassis switches
switch_params_aggregated_df = fill_device_location(
switch_params_aggregated_df, blade_module_loc_df)
# after finish display status
status_info('ok', max_title, len(info))
# check if switch config files missing
mask_fabric = switch_params_aggregated_df[[
'Fabric_name', 'Fabric_label'
]].notna().all(axis=1)
mask_no_config = switch_params_aggregated_df['chassis_name'].isna()
missing_configs_num = switch_params_aggregated_df.loc[mask_no_config][
'Fabric_name'].count()
if missing_configs_num:
info = f'{missing_configs_num} switch configuration{"s" if missing_configs_num > 1 else ""} MISSING'
print(info, end=" ")
status_info('warning', max_title, len(info))
switches_report_df, fabric_report_df, switches_parameters_report_df, \
licenses_report_df, global_fabric_parameters_report_df = \
switchs_params_report(switch_params_aggregated_df, data_names, report_columns_usage_dct, max_title)
# # partition aggregated DataFrame to required tables
# switches_report_df, fabric_report_df, \
# switches_parameters_report_df, licenses_report_df = \
# dataframe_segmentation(switch_params_aggregated_df, data_names[2:-1], \
# report_columns_usage_dct, max_title)
# # global parameters are equal for all switches in one fabric thus checking Principal switches only
# mask_principal = switch_params_aggregated_df['switchRole'] == 'Principal'
# switch_params_principal_df = switch_params_aggregated_df.loc[mask_principal].copy()
# global_fabric_parameters_report_df, = dataframe_segmentation(switch_params_principal_df, data_names[-1], \
# report_columns_usage_dct, max_title)
# # drop rows with empty switch names columns
# fabric_report_df.dropna(subset = ['Имя коммутатора'], inplace = True)
# switches_parameters_report_df.dropna(subset = ['Имя коммутатора'], inplace = True)
# licenses_report_df.dropna(subset = ['Имя коммутатора'], inplace = True)
# # drop fabric_id if all have same value
# if fabric_report_df['Fabric ID'].dropna().nunique() == 1:
# fabric_report_df.drop(columns=['Fabric ID'], inplace=True)
# # TO_REMOVE No need to drop duplicates coz Principal switches only used before
# # # parameters are equal for all switches in one fabric
# # if report_columns_usage_dct['fabric_name_usage']:
# # global_fabric_parameters_report_df.drop_duplicates(subset=['Фабрика', 'Подсеть'], inplace=True)
# # else:
# # global_fabric_parameters_report_df.drop_duplicates(subset=['Подсеть'], inplace=True)
# global_fabric_parameters_report_df.reset_index(inplace=True, drop=True)
# create list with partitioned DataFrames
data_lst = [
report_columns_usage_dct, switch_params_aggregated_df,
switches_report_df, fabric_report_df,
switches_parameters_report_df, licenses_report_df,
global_fabric_parameters_report_df
]
# saving data to json or csv file
save_data(report_data_lst, data_names, *data_lst)
# verify if loaded data is empty and replace information string with empty DataFrame
else:
report_columns_usage_dct, switch_params_aggregated_df, switches_report_df, fabric_report_df, \
switches_parameters_report_df, licenses_report_df, global_fabric_parameters_report_df = verify_data(report_data_lst, data_names, *data_lst)
data_lst = [
report_columns_usage_dct, switch_params_aggregated_df,
switches_report_df, fabric_report_df,
switches_parameters_report_df, licenses_report_df,
global_fabric_parameters_report_df
]
# save data to service file if it's required
for data_name, data_frame in zip(data_names[1:], data_lst[1:]):
save_xlsx_file(data_frame, data_name, report_data_lst)
return report_columns_usage_dct, switch_params_aggregated_df, fabric_clean_df
def switch_params_configshow_extract(chassis_params_fabric_lst, report_data_lst):
"""Function to extract switch parameters"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = ['switch_parameters', 'switchshow_ports']
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# unpacking from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
switch_params_lst, switchshow_ports_lst = data_lst
# data force extract check
# list of keys for each data from data_lst representing if it is required
# to re-collect or re-analyze data even they were obtained on previous iterations
force_extract_keys_lst = [report_steps_dct[data_name][1] for data_name in data_names]
# print data which were loaded but for which force extract flag is on
force_extract_check(data_names, data_lst, force_extract_keys_lst, max_title)
# when any of data_lst was not saved or
# force extract flag is on then re-extract data from configueation files
if not all(data_lst) or any(force_extract_keys_lst):
print('\nEXTRACTING SWITCH PARAMETERS FROM SUPPORTSHOW CONFIGURATION FILES ...\n')
# extract chassis parameters names from init file
chassis_columns = columns_import('chassis', max_title, 'columns')
# number of switches to check
switch_num = len(chassis_params_fabric_lst)
# list to store only REQUIRED switch parameters
# collecting data for all switches during looping
switch_params_lst = []
# list to store switch ports details
switchshow_ports_lst = []
# data imported from init file to extract values from config file
switch_params, params_add, comp_keys, match_keys, comp_dct = data_extract_objects('switch', max_title)
# chassis_params_fabric_lst [[chassis_params_sw1], [chassis_params_sw1]]
# checking each chassis for switch level parameters
for i, chassis_params_data in enumerate(chassis_params_fabric_lst):
# data unpacking from iter param
# dictionary with parameters for the current chassis
chassis_params_data_dct = dict(zip(chassis_columns, chassis_params_data))
sshow_file = chassis_params_data_dct['configname']
chassis_name = chassis_params_data_dct['chassis_name']
chassis_wwn = chassis_params_data_dct['chassis_wwn']
# num_ls = int(chassis_params_data_dct["Number_of_LS"]) if not chassis_params_data_dct["Number_of_LS"] in ['0', None] else 1
# when num of logical switches is 0 or None than mode is Non-VF otherwise VF
ls_mode_on = (True if not chassis_params_data_dct["Number_of_LS"] in ['0', None] else False)
ls_mode = ('ON' if not chassis_params_data_dct["Number_of_LS"] in ['0', None] else 'OFF')
# logical switches indexes. if switch is in Non-VF mode then ls_id is 0
ls_ids = chassis_params_data_dct['LS_IDs'].split(', ') if chassis_params_data_dct['LS_IDs'] else ['0']
# current operation information string
info = f'[{i+1} of {switch_num}]: {chassis_params_data_dct["chassis_name"]} switch parameters. Number of LS: {chassis_params_data_dct["Number_of_LS"]}'
print(info, end =" ")
# check each logical switch in chassis
for i in ls_ids:
# search control dictionary. continue to check sshow_file until all parameters groups are found
collected = {'configshow': False, 'switchshow': False}
# dictionary to store all DISCOVERED switch parameters
# collecting data only for the logical switch in current loop
switch_params_dct = {}
with open(sshow_file, encoding='utf-8', errors='ignore') as file:
# check file until all groups of parameters extracted
while not all(collected.values()):
line = file.readline()
if not line:
break
# configshow section start
if re.search(fr'^\[Switch Configuration Begin *: *{i}\]$', line) and not collected['configshow']:
# when section is found corresponding collected dict values changed to True
collected['configshow'] = True
while not re.search(fr'^\[Switch Configuration End : {i}\]$',line):
line = file.readline()
# dictionary with match names as keys and match result of current line with all imported regular expressions as values
match_dct ={match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip(comp_keys, match_keys)}
# match_keys ['switch_configall_match', 'switch_switchshow_match']
if match_dct[match_keys[0]]:
switch_params_dct[match_dct[match_keys[0]].group(1).rstrip()] = match_dct[match_keys[0]].group(3).rstrip()
if not line:
break
# config section end
# switchshow section start
if re.search(r'^(SWITCHCMD /fabos/bin/)?switchshow *:$', line) and not collected['switchshow']:
collected['switchshow'] = True
if ls_mode_on:
while not re.search(fr'^CURRENT CONTEXT -- {i} *, \d+$',line):
line = file.readline()
if not line:
break
while not re.search(r'^real [\w.]+$',line):
line = file.readline()
match_dct ={match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip(comp_keys, match_keys)}
# 'switch_switchshow_match'
if match_dct[match_keys[1]]:
switch_params_dct[match_dct[match_keys[1]].group(1).rstrip()] = match_dct[match_keys[1]].group(2).rstrip()
# 'ls_attr_match'
if match_dct[match_keys[2]]:
ls_attr = comp_dct[comp_keys[2]].findall(line)[0]
for k, v in zip(ls_attr[::2], ls_attr[1::2]):
switch_params_dct[k] = v
# 'switchshow_portinfo_match'
if match_dct[match_keys[3]]:
switchinfo_lst = [sshow_file, chassis_name, chassis_wwn, str(i),
switch_params_dct.get('switchName', None),
switch_params_dct.get('switchWwn', None),
switch_params_dct.get('switchState', None),
switch_params_dct.get('switchMode', None)
]
switchshow_port_lst = line_to_list(comp_dct[comp_keys[3]], line, *switchinfo_lst)
# if switch has no slots than slot number is 0
if not switchshow_port_lst[9]:
switchshow_port_lst[9] = str(0)
switchshow_ports_lst.append(switchshow_port_lst)
if not line:
break
# switchshow section end
# additional values which need to be added to the switch params dictionary
# switch_params_add order ('configname', 'chassis_name', 'switch_index', 'ls_mode')
# values axtracted in manual mode. if change values order change keys order in init.xlsx switch tab "params_add" column
switch_params_values = (sshow_file, chassis_name, chassis_wwn, str(i), ls_mode)
if switch_params_dct:
# adding additional parameters and values to the switch_params_switch_dct
update_dct(params_add, switch_params_values, switch_params_dct)
# creating list with REQUIRED chassis parameters for the current switch.
# if no value in the switch_params_dct for the parameter then None is added
# and appending this list to the list of all switches switch_params_fabric_lst
switch_params_lst.append([switch_params_dct.get(switch_param, None) for switch_param in switch_params])
status_info('ok', max_title, len(info))
# save extracted data to json file
save_data(report_data_lst, data_names, switch_params_lst, switchshow_ports_lst)
# verify if loaded data is empty after first iteration and replace information string with empty list
else:
switch_params_lst, switchshow_ports_lst = verify_data(report_data_lst, data_names, *data_lst)
return switch_params_lst, switchshow_ports_lst
def synergy_system_extract(synergy_folder, report_data_lst):
"""Function to extract blade systems information"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = ['synergy_interconnect', 'synergy_servers']
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# # unpacking from the loaded list with data
# # pylint: disable=unbalanced-tuple-unpacking
# module_comprehensive_lst, blades_comprehensive_lst, blade_vc_comprehensive_lst = data_lst
# force run when any data from data_lst was not saved (file not found) or
# procedure execution explicitly requested for output data or data used during fn execution
force_run = verify_force_run(data_names, data_lst, report_steps_dct,
max_title)
if force_run:
# lists to store only REQUIRED infromation
# collecting data for all blades during looping
# list containing enclosure, blade and hba information for all blade systems
# list containing enclosure and interconnect modules information for all blade systems
synergy_module_columns = [
'Enclosure_Name', 'Enclosure_SN', 'Enclosure_Type',
'Interconnect_Bay', 'Interconnect_Model', 'Interconnect_SN',
'Interconnect_Firmware', 'Interconnect_Name', 'NodeName',
'Device_Location'
]
synergy_server_columns = [
'Enclosure_Name', 'Enclosure_Slot', 'Host_Name', 'name',
'serverprofilename', 'Device_Model', 'Device_SN', 'Host_OS',
'HBA_Description', 'Mezz_type', 'Connected_portWwn',
'Mezz_location', 'Device_Location', 'HBA_Firmware'
]
synergy_module_aggregated_df = pd.DataFrame(
columns=synergy_module_columns)
synergy_servers_aggregated_df = pd.DataFrame(
columns=synergy_server_columns)
if synergy_folder:
print('\nEXTRACTING SYNERGY SYSTEM INFORMATION ...\n')
# collects files in folder with xlsm extension
synergy_config_lst = find_files(synergy_folder,
max_title,
filename_extension='xlsm')
# number of files to check
configs_num = len(synergy_config_lst)
if configs_num:
# # data imported from init file to extract values from config file
# enclosure_params, _, comp_keys, match_keys, comp_dct = data_extract_objects('blades', max_title)
# module_params = columns_import('blades', max_title, 'module_params')
# blade_params = columns_import('blades', max_title, 'blade_params')
for i, synergy_config in enumerate(synergy_config_lst):
# file name with extension
configname_wext = os.path.basename(synergy_config)
# remove extension from filename
configname, _ = os.path.splitext(configname_wext)
# current operation information string
info = f'[{i+1} of {configs_num}]: {configname} system.'
print(info, end=" ")
syn_enclosure_df = pd.read_excel(synergy_config,
sheet_name='enclosures')
syn_module_df = pd.read_excel(
synergy_config, sheet_name='interconnectbays')
syn_server_hw_df = pd.read_excel(
synergy_config, sheet_name='server-hardware')
syn_server_fw_sw_df = pd.read_excel(
synergy_config, sheet_name='server-fw-sw')
syn_server_profile_connection_df = pd.read_excel(
synergy_config, sheet_name='server-prof-conn-details')
synergy_module_df = synergy_module(syn_enclosure_df,
syn_module_df)
if synergy_module_aggregated_df.empty:
synergy_module_aggregated_df = synergy_module_df
else:
synergy_module_aggregated_df = pd.concat(
[synergy_module_aggregated_df, synergy_module_df],
ignore_index=True)
synergy_server_wwn_df = synergy_server_wwn(
syn_server_hw_df)
synergy_profile_wwn_df = synergy_profile_wwn(
syn_server_profile_connection_df,
synergy_server_wwn_df)
# conctenate connection profile and server hardware
synergy_servers_df = pd.concat(
[synergy_server_wwn_df, synergy_profile_wwn_df],
ignore_index=True)
synergy_servers_df.drop_duplicates(inplace=True)
# add mezzanine firmware details
synergy_servers_df = synergy_mezz_fw(
syn_server_fw_sw_df, synergy_servers_df)
synergy_servers_df.sort_values(
by=['enclosurename', 'position', 'Mezz_WWPN'],
ignore_index=True,
inplace=True)
if synergy_servers_aggregated_df.empty:
synergy_servers_aggregated_df = synergy_servers_df
else:
synergy_servers_aggregated_df = pd.concat(
[
synergy_servers_aggregated_df,
synergy_servers_df
],
ignore_index=True)
if synergy_module_aggregated_df['switchbasewwn'].notna(
).any():
synergy_module_aggregated_df[
'switchbasewwn'] = synergy_module_aggregated_df[
'switchbasewwn'].str.lower()
if synergy_servers_aggregated_df['Mezz_WWPN'].notna().any(
):
synergy_servers_aggregated_df[
'Mezz_WWPN'] = synergy_servers_aggregated_df[
'Mezz_WWPN'].str.lower()
if not synergy_servers_df.empty or not synergy_module_df.empty:
status_info('ok', max_title, len(info))
else:
status_info('no data', max_title, len(info))
module_columns_dct = {
'enclosurename': 'Enclosure_Name',
'enclosure_serialnumber': 'Enclosure_SN',
'enclosuretype': 'Enclosure_Type',
'baynumber': 'Interconnect_Bay',
'interconnectmodel': 'Interconnect_Model',
'serialnumber': 'Interconnect_SN',
'switchfwversion': 'Interconnect_Firmware',
'hostname': 'Interconnect_Name',
'switchbasewwn': 'NodeName',
'device_location': 'Device_Location'
}
synergy_module_aggregated_df.rename(columns=module_columns_dct,
inplace=True)
synergy_module_aggregated_df.replace(r'^None$|^none$|^ *$',
value=np.nan,
regex=True,
inplace=True)
server_columns_dct = {
'enclosurename': 'Enclosure_Name',
'position': 'Enclosure_Slot',
'servername': 'Host_Name',
'model': 'Device_Model',
'serialnumber': 'Device_SN',
'oshint': 'Host_OS',
'Mezz': 'HBA_Description',
'Mezz_WWPN': 'Connected_portWwn',
'device_location': 'Device_Location',
'componentversion': 'HBA_Firmware'
}
synergy_servers_aggregated_df.rename(
columns=server_columns_dct, inplace=True)
synergy_servers_aggregated_df.replace(r'^None$|^none$|^ *$',
value=np.nan,
regex=True,
inplace=True)
data_lst = [
synergy_module_aggregated_df, synergy_servers_aggregated_df
]
# save extracted data to json file
save_data(report_data_lst, data_names, *data_lst)
else:
# current operation information string
info = f'Collecting synergy details'
print(info, end=" ")
status_info('skip', max_title, len(info))
data_lst = [
synergy_module_aggregated_df, synergy_servers_aggregated_df
]
# save empty data to json file
save_data(report_data_lst, data_names, *data_lst)
# verify if loaded data is empty after first iteration and replace information string with empty list
else:
synergy_module_aggregated_df, synergy_servers_aggregated_df = verify_data(
report_data_lst, data_names, *data_lst)
data_lst = [
synergy_module_aggregated_df, synergy_servers_aggregated_df
]
# save data to service file if it's required
for data_name, data_frame in zip(data_names, data_lst):
save_xlsx_file(data_frame, data_name, report_data_lst)
return synergy_module_aggregated_df, synergy_servers_aggregated_df
def fabric_main(fabricshow_ag_labels_df, chassis_params_df, \
switch_params_df, maps_params_df, report_data_lst):
"""Main function to create tables"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = [
'Коммутаторы', 'Фабрика', 'Глобальные_параметры_фабрики',
'Параметры_коммутаторов', 'Лицензии'
]
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# unpacking DataFrames from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
switches_report_df, fabric_report_df, global_fabric_parameters_report_df, \
switches_parameters_report_df, licenses_report_df = data_lst
# data force extract check
# list of keys for each data from data_lst representing if it is required
# to re-collect or re-analyze data even they were obtained on previous iterations
force_extract_keys_lst = [
report_steps_dct[data_name][1] for data_name in data_names
]
# list with True (if data loaded) and/or False (if data was not found and None returned)
data_check = force_extract_check(data_names, data_lst,
force_extract_keys_lst, max_title)
# flag if fabrics labels was forced to be changed
fabric_labels_change = True if report_steps_dct['fabric_labels'][
1] else False
# initialization chassis information and farbric name columns usage
report_columns_usage_dct = {
'fabric_name_usage': True,
'chassis_info_usage': True
}
# import data with switch models, firmware and etc
switch_models_df = dataframe_import('switch_models', max_title)
# clean fabricshow DataFrame from unneccessary data
fabric_clean_df = fabric_clean(fabricshow_ag_labels_df)
# create aggregated table by joining DataFrames
switch_params_aggregated_df, report_columns_usage_dct = \
fabric_aggregation(fabric_clean_df, chassis_params_df, \
switch_params_df, maps_params_df, switch_models_df)
save_xlsx_file(switch_params_aggregated_df, 'switch_params_aggregated', \
report_data_lst, report_type = 'analysis')
# when no data saved or force extract flag is on or fabric labels have been changed than
# analyze extracted config data
if not all(data_check) or any(
force_extract_keys_lst) or fabric_labels_change:
# information string if fabric labels force changed was initiated
# and statistics recounting required
if fabric_labels_change and not any(force_extract_keys_lst) and all(
data_check):
info = f'Switch information force extract due to change in Fabrics labeling'
print(info, end=" ")
status_info('ok', max_title, len(info))
# partition aggregated DataFrame to required tables
switches_report_df, fabric_report_df, global_fabric_parameters_report_df, \
switches_parameters_report_df, licenses_report_df = \
dataframe_segmentation(switch_params_aggregated_df, data_names, \
report_columns_usage_dct, max_title)
# drop rows with empty switch names columns
fabric_report_df.dropna(subset=['Имя коммутатора'], inplace=True)
switches_parameters_report_df.dropna(subset=['Имя коммутатора'],
inplace=True)
licenses_report_df.dropna(subset=['Имя коммутатора'], inplace=True)
# parameters are equal for all switches in one fabric
if report_columns_usage_dct['fabric_name_usage']:
global_fabric_parameters_report_df.drop_duplicates(
subset=['Фабрика', 'Подсеть'], inplace=True)
else:
global_fabric_parameters_report_df.drop_duplicates(
subset=['Подсеть'], inplace=True)
global_fabric_parameters_report_df.reset_index(inplace=True, drop=True)
# create list with partitioned DataFrames
data_lst = [switches_report_df, fabric_report_df, global_fabric_parameters_report_df, \
switches_parameters_report_df, licenses_report_df]
# current operation information string
info = f'Generating Fabric and Switches tables'
print(info, end=" ")
# after finish display status
status_info('ok', max_title, len(info))
# saving DataFrames to csv file
save_data(report_data_lst, data_names, *data_lst)
# save_data(report_data_lst, data_auxillary_names, *data_auxillary_lst)
# save data to service file if it's required
for data_name, data_frame in zip(data_names, data_lst):
save_xlsx_file(data_frame, data_name, report_data_lst)
return switch_params_aggregated_df, report_columns_usage_dct, fabric_clean_df
def storage_3par_extract(nsshow_df, nscamshow_df, local_3par_folder,
project_folder, report_data_lst):
"""Function to extract 3PAR storage information"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = ['system_3par', 'port_3par', 'host_3par']
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# unpacking from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
system_3par_comprehensive_lst, port_3par_comprehensive_lst, host_3par_comprehensive_lst = data_lst
# data force extract check
# list of keys for each data from data_lst representing if it is required
# to re-collect or re-analyze data even they were obtained on previous iterations
force_extract_keys_lst = [
report_steps_dct[data_name][1] for data_name in data_names
]
# print data which were loaded but for which force extract flag is on
force_extract_check(data_names, data_lst, force_extract_keys_lst,
max_title)
# when any of data_lst was not saved or
# force extract flag is on then re-extract data from configueation files
if not all(data_lst) or any(force_extract_keys_lst):
# lists to store only REQUIRED infromation
# collecting data for all systems during looping
# list containing system parameters
system_3par_comprehensive_lst = []
# list containing 3par FC port information
port_3par_comprehensive_lst = []
# list containing hosts defined on 3par ports
host_3par_comprehensive_lst = []
# data imported from init file to extract values from config file
params, params_add, comp_keys, match_keys, comp_dct = data_extract_objects(
'3par', max_title)
# verify if 3par systems registered in fabric NameServer
ns_3par_df = verify_ns_3par(nsshow_df, nscamshow_df, comp_dct)
if not ns_3par_df.empty:
print('\n')
print('3PAR Storage Systems detected in SAN')
print(ns_3par_df)
print('\n')
# find configuration files to parse (download from STATs, local folder or use configurations
# downloaded on previous iterations)
configs_3par_lst = configs_download(ns_3par_df, project_folder,
local_3par_folder, comp_keys,
match_keys, comp_dct,
report_data_lst)
if configs_3par_lst:
print('\nEXTRACTING 3PAR STORAGE INFORMATION ...\n')
# number of files to check
configs_num = len(configs_3par_lst)
for i, config_3par in enumerate(configs_3par_lst):
# file name
configname = os.path.basename(config_3par)
# current operation information string
info = f'[{i+1} of {configs_num}]: {configname} system'
print(info, end=" ")
showsys_lst, port_lst, host_lst = parse_config(
config_3par, params, params_add, comp_keys, match_keys,
comp_dct)
system_3par_comprehensive_lst.extend(showsys_lst)
port_3par_comprehensive_lst.extend(port_lst)
host_3par_comprehensive_lst.extend(host_lst)
if port_lst or host_lst:
status_info('ok', max_title, len(info))
else:
status_info('no data', max_title, len(info))
# save extracted data to json file
save_data(report_data_lst, data_names,
system_3par_comprehensive_lst,
port_3par_comprehensive_lst,
host_3par_comprehensive_lst)
else:
# current operation information string
info = f'Collecting 3PAR storage systems information'
print(info, end=" ")
status_info('skip', max_title, len(info))
# save empty data to json file
save_data(report_data_lst, data_names,
system_3par_comprehensive_lst,
port_3par_comprehensive_lst, host_3par_comprehensive_lst)
# verify if loaded data is empty after first iteration and replace information string with empty list
else:
system_3par_comprehensive_lst, port_3par_comprehensive_lst, host_3par_comprehensive_lst = verify_data(
report_data_lst, data_names, *data_lst)
return system_3par_comprehensive_lst, port_3par_comprehensive_lst, host_3par_comprehensive_lst
def isl_main(fabricshow_ag_labels_df, switch_params_aggregated_df, report_columns_usage_dct,
isl_df, trunk_df, fcredge_df, portshow_df, sfpshow_df, portcfgshow_df, switchshow_ports_df, report_data_lst):
"""Main function to create ISL and IFR report tables"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = ['isl_aggregated', 'isl_statistics', 'Межкоммутаторные_соединения', 'Межфабричные_соединения', 'Статистика_ISL']
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# loading data if were saved on previous iterations
data_lst = load_data(report_data_lst, *data_names)
# unpacking DataFrames from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
isl_aggregated_df, isl_statistics_df, isl_report_df, ifl_report_df, isl_statistics_report_df = data_lst
# list of data to analyze from report_info table
analyzed_data_names = ['isl', 'trunk', 'fcredge', 'sfpshow', 'portcfgshow',
'chassis_parameters', 'switch_parameters', 'switchshow_ports',
'maps_parameters', 'blade_interconnect', 'fabric_labels']
# force run when any data from data_lst was not saved (file not found) or
# procedure execution explicitly requested for output data or data used during fn execution
force_run = verify_force_run(data_names, data_lst, report_steps_dct,
max_title, analyzed_data_names)
if force_run:
# data imported from init file (regular expression patterns) to extract values from data columns
# re_pattern list contains comp_keys, match_keys, comp_dct
_, _, *re_pattern_lst = data_extract_objects('common_regex', max_title)
# current operation information string
info = f'Generating ISL and IFL tables'
print(info, end =" ")
# get aggregated DataFrames
isl_aggregated_df, fcredge_df = \
isl_aggregated(fabricshow_ag_labels_df, switch_params_aggregated_df,
isl_df, trunk_df, fcredge_df, portshow_df, sfpshow_df, portcfgshow_df, switchshow_ports_df, re_pattern_lst)
isl_statistics_df = isl_statistics(isl_aggregated_df, re_pattern_lst, report_data_lst)
# after finish display status
status_info('ok', max_title, len(info))
# partition aggregated DataFrame to required tables
isl_report_df, = dataframe_segmentation(isl_aggregated_df, [data_names[2]], report_columns_usage_dct, max_title)
isl_report_df = translate_values(isl_report_df, translate_dct={'Yes': 'Да', 'No': 'Нет'})
# if no trunks in fabric drop trunk columns
if trunk_df.empty:
isl_report_df.drop(columns = ['Идентификатор транка', 'Deskew', 'Master'], inplace = True)
# check if IFL table required
if not fcredge_df.empty:
ifl_report_df, = dataframe_segmentation(fcredge_df, [data_names[3]], report_columns_usage_dct, max_title)
else:
ifl_report_df = fcredge_df.copy()
isl_statistics_report_df = isl_statistics_report(isl_statistics_df, report_columns_usage_dct, max_title)
# create list with partitioned DataFrames
data_lst = [isl_aggregated_df, isl_statistics_df, isl_report_df, ifl_report_df, isl_statistics_report_df]
# saving fabric_statistics and fabric_statistics_summary DataFrames to csv file
save_data(report_data_lst, data_names, *data_lst)
# verify if loaded data is empty and replace information string with empty DataFrame
else:
isl_aggregated_df, isl_statistics_df, isl_report_df, ifl_report_df, isl_statistics_report_df = \
verify_data(report_data_lst, data_names, *data_lst)
data_lst = [isl_aggregated_df, isl_statistics_df, isl_report_df, ifl_report_df, isl_statistics_report_df]
# save data to service file if it's required
for data_name, data_frame in zip(data_names, data_lst):
save_xlsx_file(data_frame, data_name, report_data_lst)
return isl_aggregated_df, isl_statistics_df
def portcmdshow_extract(chassis_params_fabric_lst, report_data_lst):
"""Function to extract portshow, portloginshow, portstatsshow information"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = ['portcmd']
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# unpacking from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
portshow_lst, = data_lst
# data force extract check
# list of keys for each data from data_lst representing if it is required
# to re-collect or re-analyze data even they were obtained on previous iterations
force_extract_keys_lst = [
report_steps_dct[data_name][1] for data_name in data_names
]
# print data which were loaded but for which force extract flag is on
force_extract_check(data_names, data_lst, force_extract_keys_lst,
max_title)
# when any of data_lst was not saved or
# force extract flag is on then re-extract data from configueation files
if not all(data_lst) or any(force_extract_keys_lst):
print(
'\nEXTRACTING PORTSHOW, PORTLOGINSHOW, PORTSTATSSHOW INFORMATION FROM SUPPORTSHOW CONFIGURATION FILES ...\n'
)
# extract chassis parameters names from init file
chassis_columns = columns_import('chassis', max_title, 'columns')
# number of switches to check
switch_num = len(chassis_params_fabric_lst)
# list to store only REQUIRED switch parameters
# collecting data for all switches during looping
portshow_lst = []
# data imported from init file to extract values from config file
portcmd_params, params_add, comp_keys, match_keys, comp_dct = data_extract_objects(
'portcmd', max_title)
# chassis_params_fabric_lst [[chassis_params_sw1], [chassis_params_sw1]]
# checking each chassis for switch level parameters
for i, chassis_params_data in enumerate(chassis_params_fabric_lst):
# data unpacking from iter param
# dictionary with parameters for the current chassis
chassis_params_data_dct = dict(
zip(chassis_columns, chassis_params_data))
sshow_file = chassis_params_data_dct['configname']
chassis_name = chassis_params_data_dct['chassis_name']
chassis_wwn = chassis_params_data_dct['chassis_wwn']
# current operation information string
info = f'[{i+1} of {switch_num}]: {chassis_params_data_dct["chassis_name"]} switch portshow, portloginshow and statsshow'
print(info, end=" ")
# search control dictionary. continue to check sshow_file until all parameters groups are found
collected = {'portshow': False}
with open(sshow_file, encoding='utf-8', errors='ignore') as file:
# check file until all groups of parameters extracted
while not all(collected.values()):
line = file.readline()
if not line:
break
# sshow_port section start
if re.search(r'^\| Section: SSHOW_PORT \|$', line):
# when section is found corresponding collected dict values changed to True
collected['portshow'] = True
while not re.search(r'^\| ... rebuilt finished *\|$',
line):
line = file.readline()
if not line:
break
# dictionary with match names as keys and match result of current line with all imported regular expressions as values
match_dct = {
match_key: comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# portFcPortCmdShow section start
if match_dct[match_keys[0]]:
# dictionary to store all DISCOVERED parameters
# collecting data only for the chassis in current loop
portcmd_dct = {}
# connected devices wwns in portshow section
connected_wwn_lst = []
# list to store connected devices port_id and wwn pairs in portlogin section
portid_wwn_lst = []
port_index = None
slot_port_lst = line_to_list(
comp_dct[comp_keys[0]], line)
while not re.search(r'^portshow +(\d{1,4})$',
line):
line = file.readline()
if not line:
break
match_dct = {
match_key: comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# portshow section start
if match_dct[match_keys[1]]:
port_index = match_dct[
match_keys[1]].group(1)
while not re.search(
fr'^portloginshow +{int(port_index)}$',
line):
line = file.readline()
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# portshow_params_match
if match_dct[match_keys[2]]:
portshow_attr, = comp_dct[
comp_keys[2]].findall(line)
# portstate parameter has special processing rule
if portshow_attr[0] == 'portState':
portcmd_dct[
'portState'] = portshow_attr[
2]
# portshow_attr can contain up to 3 parameter name and value pairs
# parameters name on even positions in the portshow_attr and values on odd
else:
for k, v in zip(
portshow_attr[::2],
portshow_attr[1::2]):
portcmd_dct[k] = v
# portscn_match has two parameter name and value pairs
if match_dct[match_keys[6]]:
portscn_line = line_to_list(
comp_dct[comp_keys[6]], line)
for k, v in zip(
portscn_line[::2],
portscn_line[1::2]):
portcmd_dct[k] = v
# portdistance_match
if match_dct[match_keys[7]]:
portdistance_line = line_to_list(
comp_dct[comp_keys[7]], line)
portcmd_dct[portdistance_line[
0]] = portdistance_line[1]
# device_connected_wwn_match
if match_dct[match_keys[8]]:
connected_wwn = line_to_list(
comp_dct[comp_keys[8]], line)
connected_wwn_lst.append(
(portcmd_dct.get('portId'),
connected_wwn))
if not line:
break
# portshow section end
# portlogin section start
if re.match(
fr'^portloginshow +{int(port_index)}$',
line):
while not re.search(
fr'^portregshow +{int(port_index)}$',
line):
line = file.readline()
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# connected_wwn_match
if match_dct[match_keys[3]]:
# first value in tuple unpacking is fe or fd and not required
_, port_id, wwn = line_to_list(
comp_dct[comp_keys[3]], line)
# port_id = '0x' + port_id
portid_wwn_lst.append(
(port_id, wwn))
if not line:
break
# sorting connected devices list by port_ids
if len(portid_wwn_lst) != 0:
portid_wwn_lst = sorted(portid_wwn_lst)
# if portlogin empty then use connected devices from portshow section
# applied for E-ports
elif len(connected_wwn_lst) != 0:
portid_wwn_lst = connected_wwn_lst.copy(
)
# adding port_id and None wwn if no device is connected or slave trunk link
else:
portid_wwn_lst.append(
[portcmd_dct.get('portId'), None])
# portlogin section end
while not re.match(
fr'^portstatsshow +{int(port_index)}$',
line):
line = file.readline()
if not line:
break
# portstatsshow section start
if re.match(
fr'^portstatsshow +{int(port_index)}$',
line):
while not re.search(
fr'^(portstats64show|portcamshow) +{int(port_index)}$',
line):
line = file.readline()
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# port information without virtual channel numbers
if match_dct[match_keys[4]]:
portcmd_dct[match_dct[
match_keys[4]].group(
1).rstrip()] = match_dct[
match_keys[4]].group(2)
# port information with virtual channel numbers
elif match_dct[match_keys[5]]:
line_values = line_to_list(
comp_dct[comp_keys[5]], line)
param_name, start_vc = line_values[
0:2]
for i, value in enumerate(
line_values[3:]):
param_name_vc = param_name + '_' + str(
int(start_vc) + i)
portcmd_dct[
param_name_vc] = value
if not line:
break
# portstatsshow section end
# portFcPortCmdShow section end
# additional values which need to be added to the dictionary with all DISCOVERED parameters during current loop iteration
# chassis_slot_port_values order (configname, chassis_name, port_index, slot_num, port_num, port_ids and wwns of connected devices)
# values axtracted in manual mode. if change values order change keys order in init.xlsx "chassis_params_add" column
for port_id, connected_wwn in portid_wwn_lst:
chassis_slot_port_values = [
sshow_file, chassis_name, chassis_wwn,
port_index, *slot_port_lst, port_id,
connected_wwn
]
# print('chassis_slot_port_values', chassis_slot_port_values)
# adding or changing data from chassis_slot_port_values to the DISCOVERED dictionary
update_dct(params_add,
chassis_slot_port_values,
portcmd_dct)
# adding data to the REQUIRED list for each device connected to the port
portshow_lst.append([
portcmd_dct.get(portcmd_param, None)
for portcmd_param in portcmd_params
])
# print('portshow_lst', portshow_lst)
# sshow_port section end
status_info('ok', max_title, len(info))
# save extracted data to json file
save_data(report_data_lst, data_names, portshow_lst)
# verify if loaded data is empty after first iteration and replace information string with empty list
else:
portshow_lst = verify_data(report_data_lst, data_names, *data_lst)
return portshow_lst
def fcr_extract(switch_params_lst, report_data_lst):
"""Function to extract fabrics routing information"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = [
'fcrfabric', 'fcrproxydev', 'fcrphydev', 'lsan', 'fcredge',
'fcrresource'
]
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# unpacking from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
fcrfabric_lst, fcrproxydev_lst, fcrphydev_lst, lsan_lst, \
fcredge_lst, fcrresource_lst = data_lst
# data force extract check
# list of keys for each data from data_lst representing if it is required
# to re-collect or re-analyze data even they were obtained on previous iterations
force_extract_keys_lst = [
report_steps_dct[data_name][1] for data_name in data_names
]
# print data which were loaded but for which force extract flag is on
force_extract_check(data_names, data_lst, force_extract_keys_lst,
max_title)
# when any of data_lst was not saved or
# force extract flag is on then re-extract data from configueation files
if not all(data_lst) or any(force_extract_keys_lst):
print(
'\nEXTRACTING FABRICS ROUTING INFORMATION FROM SUPPORTSHOW CONFIGURATION FILES ...\n'
)
# extract switch parameters names from init file
switch_columns = columns_import('switch', max_title, 'columns')
# number of switches to check
switch_num = len(switch_params_lst)
# lists to store only REQUIRED switch parameters
# collecting data for all switches during looping
fcrfabric_lst = []
fcrproxydev_lst = []
fcrphydev_lst = []
lsan_lst = []
fcredge_lst = []
fcrresource_lst = []
# dictionary to collect fcr device data
# first element of list is regular expression pattern number,
# second - list to collect data, third is index in line to which slice extracted list
fcrdev_dct = {
'fcrproxydev': [5, fcrproxydev_lst, None],
'fcrphydev': [6, fcrphydev_lst, -3]
}
# data imported from init file to extract values from config file
params, _, comp_keys, match_keys, comp_dct = data_extract_objects(
'fcr', max_title)
# switch_params_lst [[switch_params_sw1], [switch_params_sw1]]
# checking each switch for switch level parameters
for i, switch_params_data in enumerate(switch_params_lst):
# dictionary with parameters for the current switch
switch_params_data_dct = dict(
zip(switch_columns, switch_params_data))
switch_info_keys = [
'configname', 'chassis_name', 'chassis_wwn', 'switch_index',
'SwitchName', 'switchWwn', 'switchRole', 'Fabric_ID',
'FC_Router'
]
switch_info_lst = [
switch_params_data_dct.get(key) for key in switch_info_keys
]
ls_mode_on = True if switch_params_data_dct[
'LS_mode'] == 'ON' else False
# data unpacking from iter param
sshow_file, *_, switch_name, _, switch_role, fid, fc_router = switch_info_lst
# current operation information string
info = f'[{i+1} of {switch_num}]: {switch_name} fabric routing. FC Routing: {fc_router}'
print(info, end=" ")
# search control dictionary. continue to check sshow_file until all parameters groups are found
collected = {'fcrfabric': False, 'fcrproxydev': False, 'fcrphydev': False,
'lsanzone': False, 'fcredge': False, 'fcrresource': False} \
if switch_role == 'Principal' else {'fcredge': False, 'fcrresource': False}
# check config of FC routers only
if fc_router == 'ON':
# fcrouter_info_lst contains sshow_file, chassis_name, switch_index, switch_name, switch_fid
fcrouter_info_lst = [*switch_info_lst[:6], switch_info_lst[7]]
with open(sshow_file, encoding='utf-8',
errors='ignore') as file:
# check file until all groups of parameters extracted
while not all(collected.values()):
line = file.readline()
if not line:
break
# check configs of Principal switches only
if switch_role == 'Principal':
# fcrfabricshow section start
# switchcmd_fcrfabricshow
if re.search(comp_dct[comp_keys[0]],
line) and not collected['fcrfabric']:
collected['fcrfabric'] = True
if ls_mode_on:
while not re.search(
fr'^BASE +SWITCH +CONTEXT *-- *FID: *{fid}$',
line):
line = file.readline()
if not line:
break
# switchcmd_end_comp
while not re.search(comp_dct[comp_keys[4]],
line):
line = file.readline()
# dictionary with match names as keys and match result of current line with all imported regular expressions as values
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# fc_router_match'
if match_dct[match_keys[1]]:
fcrouter_params_lst = line_to_list(
comp_dct[comp_keys[1]], line)
# check if line is empty
while not re.match('\r?\n', line):
line = file.readline()
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# fcr_info_match
if match_dct[match_keys[2]]:
fcrouter_name = match_dct[
match_keys[2]].group(1)
# fcr_exports_match
if match_dct[match_keys[3]]:
fcrfabric_lst.append(
line_to_list(
comp_dct[comp_keys[3]],
line,
*fcrouter_info_lst,
fcrouter_name,
*fcrouter_params_lst))
if not line:
break
if not line:
break
# fcrfabricshow section end
# fcrproxydev and fcrphydev checked in a loop over dictionary keys coz data representation is similar
# fcrdevshow section start
for fcrdev_type in fcrdev_dct.keys():
re_num, fcrdev_lst, slice_index = fcrdev_dct[
fcrdev_type]
# switchcmd_fcrproxydevshow_comp
if re.search(
comp_dct[comp_keys[re_num]],
line) and not collected[fcrdev_type]:
collected[fcrdev_type] = True
if ls_mode_on:
while not re.search(
fr'^BASE +SWITCH +CONTEXT *-- *FID: *{fid} *$',
line):
line = file.readline()
if not line:
break
# switchcmd_end_comp
while not re.search(
comp_dct[comp_keys[4]], line):
line = file.readline()
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# fcrdevshow_match
if match_dct[match_keys[7]]:
fcrdev_lst.append(
line_to_list(
comp_dct[comp_keys[7]],
line, *fcrouter_info_lst)
[:slice_index])
if not line:
break
# fcrdevshow section end
# lsanzoneshow section start
if re.search(comp_dct[comp_keys[8]],
line) and not collected['lsanzone']:
collected['lsanzone'] = True
if ls_mode_on:
while not re.search(
fr'^BASE +SWITCH +CONTEXT *-- *FID: *{fid} *$',
line):
line = file.readline()
if not line:
break
# switchcmd_end_comp
while not re.search(comp_dct[comp_keys[4]],
line):
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# lsan_name_match
if match_dct[match_keys[9]]:
# switch_info and current connected device wwnp
lsan_name = line_to_list(
comp_dct[comp_keys[9]], line)
# move cursor to one line down to get inside while loop
line = file.readline()
# lsan_switchcmd_end_comp
while not re.search(
comp_dct[comp_keys[11]], line):
# line = file.readline()
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# lsan_members_match
if match_dct[match_keys[10]]:
lsan_member = line_to_list(
comp_dct[comp_keys[10]],
line)
lsan_lst.append([
*fcrouter_info_lst,
*lsan_name, *lsan_member
])
# line = file.readline()
# else:
# line = file.readline()
line = file.readline()
if not line:
break
else:
line = file.readline()
if not line:
break
# lsanzoneshow section end
# fcredge and fcrresource checked for Principal and Subordinate routers
# fcredgeshow section start
if re.search(comp_dct[comp_keys[12]],
line) and not collected['fcredge']:
collected['fcredge'] = True
if ls_mode_on:
while not re.search(
fr'^BASE +SWITCH +CONTEXT *-- *FID: *{fid} *$',
line):
line = file.readline()
if not line:
break
# switchcmd_end_comp
while not re.search(comp_dct[comp_keys[4]], line):
line = file.readline()
match_dct = {
match_key: comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# fcredgeshow_match
if match_dct[match_keys[13]]:
# fcredge_lst.append(line_to_list(comp_dct[comp_keys[13]], line, *fcrouter_info_lst, switch_wwn))
fcredge_lst.append(
line_to_list(comp_dct[comp_keys[13]],
line, *fcrouter_info_lst))
if not line:
break
# fcredgeshow section end
# fcrresourceshow section start
if re.search(comp_dct[comp_keys[14]],
line) and not collected['fcrresource']:
collected['fcrresource'] = True
fcrresource_dct = {}
if ls_mode_on:
while not re.search(
fr'^BASE +SWITCH +CONTEXT *-- *FID: *{fid} *$',
line):
line = file.readline()
if not line:
break
# switchcmd_end_comp
while not re.search(comp_dct[comp_keys[4]], line):
line = file.readline()
match_dct = {
match_key: comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# fcredgeshow_match
if match_dct[match_keys[15]]:
fcrresource_dct[match_dct[match_keys[15]].group(1).rstrip()] = \
[match_dct[match_keys[15]].group(2), match_dct[match_keys[15]].group(3)]
if not line:
break
# each value of dictionary is list of two elements
# itertools.chain makes flat tmp_lst list from all lists in dictionary
tmp_lst = list(
itertools.chain(*[
fcrresource_dct.get(param
) if fcrresource_dct.
get(param) else [None, None]
for param in params
]))
fcrresource_lst.append(
[*fcrouter_info_lst, *tmp_lst])
# fcrresourceshow section end
status_info('ok', max_title, len(info))
else:
status_info('skip', max_title, len(info))
# save extracted data to json file
save_data(report_data_lst, data_names, fcrfabric_lst, fcrproxydev_lst,
fcrphydev_lst, lsan_lst, fcredge_lst, fcrresource_lst)
# verify if loaded data is empty after first iteration and replace information string with empty list
else:
fcrfabric_lst, fcrproxydev_lst, fcrphydev_lst, lsan_lst, fcredge_lst, fcrresource_lst = verify_data(
report_data_lst, data_names, *data_lst)
return fcrfabric_lst, fcrproxydev_lst, fcrphydev_lst, lsan_lst, fcredge_lst, fcrresource_lst
def errdump_main(errdump_df, switchshow_df, switch_params_aggregated_df,
portshow_aggregated_df, report_columns_usage_dct,
report_data_lst):
"""Main function to get most frequently appeared log messages"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = ['errdump_aggregated', 'raslog_counter', 'Журнал']
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# loading data if were saved on previous iterations
data_lst = load_data(report_data_lst, *data_names)
# unpacking DataFrames from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
errdump_aggregated_df, raslog_counter_df, raslog_report_df = data_lst
# list of data to analyze from report_info table
analyzed_data_names = [
'chassis_parameters', 'switch_parameters', 'switchshow_ports',
'maps_parameters', 'portshow_aggregated', 'fabric_labels'
]
# force run when any data from data_lst was not saved (file not found) or
# procedure execution explicitly requested for output data or data used during fn execution
force_run = verify_force_run(data_names, data_lst, report_steps_dct,
max_title, analyzed_data_names)
if force_run:
# data imported from init file (regular expression patterns) to extract values from data columns
# re_pattern list contains comp_keys, match_keys, comp_dct
_, _, *re_pattern_lst = data_extract_objects('raslog', max_title)
# current operation information string
info = f'Counting RASLog messages'
print(info, end=" ")
# get aggregated DataFrames
errdump_aggregated_df, raslog_counter_df, raslog_frequent_df = \
errdump_aggregated(errdump_df, switchshow_df, switch_params_aggregated_df, portshow_aggregated_df, re_pattern_lst)
# after finish display status
status_info('ok', max_title, len(info))
# partition aggregated DataFrame to required tables
raslog_report_df = raslog_report(raslog_frequent_df, data_names,
report_columns_usage_dct, max_title)
# create list with partitioned DataFrames
data_lst = [errdump_aggregated_df, raslog_counter_df, raslog_report_df]
# saving fabric_statistics and fabric_statistics_summary DataFrames to csv file
save_data(report_data_lst, data_names, *data_lst)
# verify if loaded data is empty and replace information string with empty DataFrame
else:
errdump_aggregated_df, raslog_counter_df, raslog_report_df = \
verify_data(report_data_lst, data_names, *data_lst)
data_lst = [errdump_aggregated_df, raslog_counter_df, raslog_report_df]
# save data to service file if it's required
for data_name, data_frame in zip(data_names, data_lst):
save_xlsx_file(data_frame, data_name, report_data_lst)
return errdump_aggregated_df, raslog_counter_df
def connected_devices_extract(switch_params_lst, report_data_lst):
"""Function to extract connected devices information
(fdmi, nsshow, nscamshow)
"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = ['fdmi', 'nsshow', 'nscamshow']
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# unpacking from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
fdmi_lst, nsshow_lst, nscamshow_lst = data_lst
# data force extract check
# list of keys for each data from data_lst representing if it is required
# to re-collect or re-analyze data even they were obtained on previous iterations
force_extract_keys_lst = [
report_steps_dct[data_name][1] for data_name in data_names
]
# print data which were loaded but for which force extract flag is on
force_extract_check(data_names, data_lst, force_extract_keys_lst,
max_title)
# when any of data_lst was not saved or
# force extract flag is on then re-extract data from configueation files
if not all(data_lst) or any(force_extract_keys_lst):
print(
'\nEXTRACTING INFORMATION ABOUT CONNECTED DEVICES (FDMI, NSSHOW, NSCAMSHOW) ...\n'
)
# extract chassis parameters names from init file
switch_columns = columns_import('switch', max_title, 'columns')
# number of switches to check
switch_num = len(switch_params_lst)
# data imported from init file to extract values from config file
params, params_add, comp_keys, match_keys, comp_dct = data_extract_objects(
'connected_dev', max_title)
nsshow_params, nsshow_params_add = columns_import(
'connected_dev', max_title, 'nsshow_params', 'nsshow_params_add')
# lists to store only REQUIRED infromation
# collecting data for all switches ports during looping
fdmi_lst = []
# lists with local Name Server (NS) information
nsshow_lst = []
nscamshow_lst = []
# dictionary with required to collect nsshow data
# first element of list is regular expression pattern number, second - list to collect data
nsshow_dct = {
'nsshow': [5, nsshow_lst],
'nscamshow': [6, nscamshow_lst]
}
# switch_params_lst [[switch_params_sw1], [switch_params_sw1]]
# checking each switch for switch level parameters
for i, switch_params_data in enumerate(switch_params_lst):
# data unpacking from iter param
# dictionary with parameters for the current chassis
switch_params_data_dct = dict(
zip(switch_columns, switch_params_data))
switch_info_keys = [
'configname', 'chassis_name', 'chassis_wwn', 'switch_index',
'SwitchName', 'switchWwn', 'switchMode'
]
switch_info_lst = [
switch_params_data_dct.get(key) for key in switch_info_keys
]
ls_mode_on = True if switch_params_data_dct[
'LS_mode'] == 'ON' else False
sshow_file, *_, switch_index, switch_name, _, switch_mode = switch_info_lst
# current operation information string
info = f'[{i+1} of {switch_num}]: {switch_name} connected devices'
print(info, end=" ")
# search control dictionary. continue to check sshow_file until all parameters groups are found
# Name Server service started only in Native mode
collected = {'fdmi': False, 'nsshow': False, 'nscamshow': False} \
if switch_params_data_dct.get('switchMode') == 'Native' else {'fdmi': False}
with open(sshow_file, encoding='utf-8', errors='ignore') as file:
# check file until all groups of parameters extracted
while not all(collected.values()):
line = file.readline()
if not line:
break
# fdmi section start
# switchcmd_fdmishow_comp
if re.search(comp_dct[comp_keys[0]], line):
collected['fdmi'] = True
if ls_mode_on:
while not re.search(
fr'^CURRENT CONTEXT -- {switch_index} *, \d+$',
line):
line = file.readline()
if not line:
break
# local_database_comp
while not re.search(comp_dct[comp_keys[4]], line):
match_dct = {
match_key: comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# wwnp_match
if match_dct[match_keys[1]]:
# dictionary to store all DISCOVERED switch ports information
# collecting data only for the logical switch in current loop
fdmi_dct = {}
# switch_info and current connected device wwnp
switch_wwnp = line_to_list(
comp_dct[comp_keys[1]], line,
*switch_info_lst[:6])
# move cursor to one line down to get inside while loop
line = file.readline()
# wwnp_local_comp
while not re.search(comp_dct[comp_keys[3]],
line):
line = file.readline()
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# fdmi_port_match
if match_dct[match_keys[2]]:
fdmi_dct[match_dct[match_keys[
2]].group(1).rstrip()] = match_dct[
match_keys[2]].group(
2).rstrip()
if not line:
break
# adding additional parameters and values to the fdmi_dct
update_dct(params_add, switch_wwnp, fdmi_dct)
# appending list with only REQUIRED port info for the current loop iteration to the list with all fabrics port info
fdmi_lst.append([
fdmi_dct.get(param, None)
for param in params
])
else:
line = file.readline()
if not line:
break
# fdmi section end
# only switches in Native mode have Name Server service started
if switch_mode == 'Native':
# nsshow section start
for nsshow_type in nsshow_dct.keys():
# unpacking re number and list to save REQUIRED params
re_num, ns_lst = nsshow_dct[nsshow_type]
# switchcmd_nsshow_comp, switchcmd_nscamshow_comp
if re.search(comp_dct[comp_keys[re_num]], line):
collected[nsshow_type] = True
if ls_mode_on:
while not re.search(
fr'^CURRENT CONTEXT -- {switch_index} *, \d+$',
line):
line = file.readline()
if not line:
break
# switchcmd_end_comp
while not re.search(comp_dct[comp_keys[9]],
line):
# line = file.readline()
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# port_pid__match
if match_dct[match_keys[7]]:
# dictionary to store all DISCOVERED switch ports information
# collecting data only for the logical switch in current loop
nsshow_port_dct = {}
# switch_info and current connected device wwnp
switch_pid = line_to_list(
comp_dct[comp_keys[7]], line,
*switch_info_lst[:6])
# move cursor to one line down to get inside while loop
line = file.readline()
# pid_switchcmd_end_comp
while not re.search(
comp_dct[comp_keys[8]], line):
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# nsshow_port_match
if match_dct[match_keys[2]]:
nsshow_port_dct[match_dct[
match_keys[2]].group(
1
).rstrip()] = match_dct[
match_keys[2]].group(
2).rstrip()
line = file.readline()
if not line:
break
# adding additional parameters and values to the fdmi_dct
update_dct(nsshow_params_add,
switch_pid, nsshow_port_dct)
# appending list with only REQUIRED port info for the current loop iteration to the list with all fabrics port info
ns_lst.append([
nsshow_port_dct.get(
nsshow_param, None)
for nsshow_param in nsshow_params
])
else:
line = file.readline()
if not line:
break
# nsshow section end
status_info('ok', max_title, len(info))
# save extracted data to json file
save_data(report_data_lst, data_names, fdmi_lst, nsshow_lst,
nscamshow_lst)
# verify if loaded data is empty after first iteration and replace information string with empty list
else:
fdmi_lst, nsshow_lst, nscamshow_lst = verify_data(
report_data_lst, data_names, *data_lst)
return fdmi_lst, nsshow_lst, nscamshow_lst
def portinfo_extract(switch_params_lst, report_data_lst):
"""Function to extract switch port information"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = ['sfpshow', 'portcfgshow']
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# unpacking from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
sfpshow_lst, portcfgshow_lst = data_lst
# when any data from data_lst was not saved (file not found) or
# force extract flag is on then re-extract data from configuration files
force_run = verify_force_run(data_names, data_lst, report_steps_dct, max_title)
# # data force extract check
# # list of keys for each data from data_lst representing if it is required
# # to re-collect or re-analyze data even they were obtained on previous iterations
# force_extract_keys_lst = [report_steps_dct[data_name][1] for data_name in data_names]
# # print data which were loaded but for which force extract flag is on
# force_extract_check(data_names, data_lst, force_extract_keys_lst, max_title)
# # when any of data_lst was not saved or
# # force extract flag is on then re-extract data from configueation files
# if not all(data_lst) or any(force_extract_keys_lst):
if force_run:
print('\nEXTRACTING SWITCH PORTS SFP, PORTCFG INFORMATION FROM SUPPORTSHOW CONFIGURATION FILES ...\n')
# extract chassis parameters names from init file
switch_columns = columns_import('switch', max_title, 'columns')
# number of switches to check
switch_num = len(switch_params_lst)
# data imported from init file to extract values from config file
params, params_add, comp_keys, match_keys, comp_dct = data_extract_objects('portinfo', max_title)
portcfg_params = columns_import('portinfo', max_title, 'portcfg_params')
# dictionary to save portcfg ALL information for all ports in fabric
portcfgshow_dct = dict((key, []) for key in portcfg_params)
# list to store only REQUIRED switch parameters
# collecting sfpshow data for all switches ports during looping
sfpshow_lst = []
# list to save portcfg information for all ports in fabric
portcfgshow_lst = []
# switch_params_lst [[switch_params_sw1], [switch_params_sw1]]
# checking each switch for switch level parameters
for i, switch_params_data in enumerate(switch_params_lst):
# data unpacking from iter param
# dictionary with parameters for the current switch
switch_params_data_dct = dict(zip(switch_columns, switch_params_data))
switch_info_keys = ['configname', 'chassis_name', 'chassis_wwn', 'switch_index',
'SwitchName', 'switchWwn']
switch_info_lst = [switch_params_data_dct.get(key) for key in switch_info_keys]
ls_mode_on = True if switch_params_data_dct['LS_mode'] == 'ON' else False
sshow_file, _, _, switch_index, switch_name, *_ = switch_info_lst
# current operation information string
info = f'[{i+1} of {switch_num}]: {switch_name} ports sfp and cfg'
print(info, end =" ")
# search control dictionary. continue to check sshow_file until all parameters groups are found
collected = {'sfpshow': False, 'portcfgshow': False}
with open(sshow_file, encoding='utf-8', errors='ignore') as file:
# check file until all groups of parameters extracted
while not all(collected.values()):
line = file.readline()
if not line:
break
# sfpshow section start
if re.search(r'^(SWITCHCMD )?(/fabos/cliexec/)?sfpshow +-all *: *$', line) and not collected['sfpshow']:
collected['sfpshow'] = True
if ls_mode_on:
while not re.search(fr'^CURRENT CONTEXT -- {switch_index} *, \d+$',line):
line = file.readline()
if not line:
break
while not re.search(r'^(real [\w.]+)|(\*\* SS CMD END \*\*)$',line):
line = file.readline()
match_dct ={match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip(comp_keys, match_keys)}
if match_dct[match_keys[0]]:
# dictionary to store all DISCOVERED switch ports information
# collecting data only for the logical switch in current loop
sfpshow_dct = {}
_, slot_num, port_num = line_to_list(comp_dct[comp_keys[0]], line)
# if switch has no slots then all ports have slot 0
slot_num = '0' if not slot_num else slot_num
while not re.match('\r?\n', line):
line = file.readline()
match_dct ={match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip(comp_keys, match_keys)}
# power_match
if match_dct[match_keys[1]]:
sfp_power_lst = line_to_list(comp_dct[comp_keys[1]], line)
# cut off RX or TX Power
sfp_power_value_unit = sfp_power_lst[1:]
for v, k in zip(sfp_power_value_unit[::2], sfp_power_value_unit[1::2]):
if k == 'uWatts':
k = 'uW'
key = sfp_power_lst[0] + '_' + k
sfpshow_dct[key] = v
# transceiver_match
elif match_dct[match_keys[2]]:
sfpshow_dct[match_dct[match_keys[2]].group(1).rstrip()] = match_dct[match_keys[2]].group(2).rstrip()
# no_sfp_match
elif match_dct[match_keys[3]]:
sfpshow_dct['Vendor Name'] = 'No SFP module'
# not_available_match
elif match_dct[match_keys[4]]:
sfpshow_dct[match_dct[match_keys[4]].group(1).rstrip()] = match_dct[match_keys[4]].group(2).rstrip()
# sfp_info_match
elif match_dct[match_keys[5]]:
sfpshow_dct[match_dct[match_keys[5]].group(1).rstrip()] = match_dct[match_keys[5]].group(2).rstrip()
if not line:
break
# additional values which need to be added to the dictionary with all DISCOVERED parameters during current loop iteration
# values axtracted in manual mode. if change values order change keys order in init.xlsx "chassis_params_add" column
sfpshow_port_values = [*switch_info_lst, slot_num, port_num]
# adding additional parameters and values to the sfpshow_dct
update_dct(params_add, sfpshow_port_values, sfpshow_dct)
# appending list with only REQUIRED port info for the current loop iteration to the list with all fabrics port info
sfpshow_lst.append([sfpshow_dct.get(param, None) for param in params])
# sfpshow section end
# portcfgshow section start
if re.search(r'^(SWITCHCMD )?(/fabos/cliexec/)?portcfgshow *: *$', line) and not collected['portcfgshow']:
collected['portcfgshow'] = True
if ls_mode_on:
while not re.search(fr'^CURRENT CONTEXT -- {switch_index} *, \d+$',line):
line = file.readline()
if not line:
break
while not re.search(r'^(real [\w.]+)|(\*\* SS CMD END \*\*)$|No ports found in switch',line):
line = file.readline()
match_dct ={match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip(comp_keys, match_keys)}
# 'slot_port_line_match'
if match_dct[match_keys[6]]:
# dictionary to store all DISCOVERED switch ports information
portcfgshow_tmp_dct = {}
# extract slot and port numbers
slot_num, port_nums_str = line_to_list(comp_dct[comp_keys[6]], line)
port_nums_lst = port_nums_str.split()
port_nums = len(port_nums_lst)
# list with switch and slot information
switch_info_slot_lst = switch_info_lst.copy()
switch_info_slot_lst.append(slot_num)
# adding switch and slot information for each port to dictionary
for portcfg_param, switch_info_value in zip(portcfg_params[:7], switch_info_slot_lst):
portcfgshow_tmp_dct[portcfg_param] = [switch_info_value for i in range(port_nums)]
# adding port numbers to dictionary
portcfgshow_tmp_dct[portcfg_params[7]] = port_nums_lst
while not re.match('\r?\n', line):
line = file.readline()
match_dct ={match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip(comp_keys, match_keys)}
# portcfg_match
if match_dct[match_keys[7]]:
# extract param name and values for each port and adding to dictionary
param_name, param_values_str = line_to_list(comp_dct[comp_keys[7]], line)
portcfgshow_tmp_dct[param_name] = param_values_str.split()
if not line:
break
# saving portcfg information of REQUIRED parameters from dictionary with DISCOVERED parameters
for portcfg_param in portcfg_params:
portcfgshow_dct[portcfg_param].extend(portcfgshow_tmp_dct.get(portcfg_param, [None for i in range(port_nums)]))
# portcfgshow section end
status_info('ok', max_title, len(info))
# after check all config files create list of lists from dictionary. each nested list contains portcfg information for one port
for portcfg_param in portcfg_params:
portcfgshow_lst.append(portcfgshow_dct.get(portcfg_param))
portcfgshow_lst = list(zip(*portcfgshow_lst))
# save extracted data to json file
save_data(report_data_lst, data_names, sfpshow_lst, portcfgshow_lst)
# verify if loaded data is empty after first iteration and replace information string with empty list
else:
sfpshow_lst, portcfgshow_lst = verify_data(report_data_lst, data_names, *data_lst)
return sfpshow_lst, portcfgshow_lst
def zoning_analysis_main(switch_params_aggregated_df, portshow_aggregated_df,
cfg_df, zone_df, alias_df, cfg_effective_df,
fcrfabric_df, lsan_df, peerzone_df,
report_columns_usage_dct, report_data_lst):
"""Main function to analyze zoning configuration"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = [
'zoning_aggregated', 'alias_aggregated', 'zonemember_statistics',
'portshow_zoned_aggregated', 'alias_statistics',
'effective_cfg_statistics', 'Зонирование', 'Псевдонимы',
'Зонирование_A&B', 'Порты_не_в_зонах', 'Порты_без_псевдономов',
'Отсутсвуют_в_сети', 'Статистика_зон', 'Статистика_псевдонимов',
'Статистика_конфигурации'
]
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# unpacking DataFrames from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
zoning_aggregated_df, alias_aggregated_df, zonemember_statistics_df, \
portshow_zoned_aggregated_df, alias_statistics_df, effective_cfg_statistics_df, zoning_report_df, alias_report_df, \
zoning_compare_report_df, unzoned_device_report_df, no_alias_device_report_df, zoning_absent_device_report_df,\
zonemember_statistics_report_df, alias_statistics_report_df, effective_cfg_statistics_report_df = data_lst
# list of data to analyze from report_info table
analyzed_data_names = [
'cfg', 'cfg_effective', 'zone', 'alias', 'switch_params_aggregated',
'switch_parameters', 'switchshow_ports', 'chassis_parameters',
'portshow_aggregated', 'device_rename', 'report_columns_usage_upd',
'portcmd', 'fdmi', 'nscamshow', 'nsshow', 'blade_servers',
'fabric_labels'
]
# force run when any data from data_lst was not saved (file not found) or
# procedure execution explicitly requested for output data or data used during fn execution
force_run = verify_force_run(data_names, data_lst, report_steps_dct,
max_title, analyzed_data_names)
if force_run:
# current operation information string
info = f'Generating zoning table'
print(info, end=" ")
# aggregated DataFrames
zoning_aggregated_df, alias_aggregated_df \
= zoning_aggregated(switch_params_aggregated_df, portshow_aggregated_df,
cfg_df, zone_df, alias_df, cfg_effective_df, fcrfabric_df, lsan_df, peerzone_df, report_data_lst)
# create comprehesive statistics DataFrame with Fabric summaries and
# zones statistics DataFrame without summaries
zonemember_statistics_df, zonemember_zonelevel_stat_df = zonemember_statistics(
zoning_aggregated_df, report_data_lst)
# add zoning statistics notes, zone duplicates and zone pairs to zoning aggregated DataFrame
zoning_aggregated_df = statistics_to_aggregated_zoning(
zoning_aggregated_df, zonemember_zonelevel_stat_df)
# check all fabric devices (Wwnp) for usage in zoning configuration
portshow_zoned_aggregated_df = verify_cfg_type(portshow_aggregated_df,
zoning_aggregated_df,
['PortName'])
# create alias configuration statistics
alias_statistics_df = alias_dashboard(alias_aggregated_df,
portshow_zoned_aggregated_df)
# create Effective zoning configuration summary statistics
effective_cfg_statistics_df = cfg_dashborad(
zonemember_statistics_df, portshow_zoned_aggregated_df,
zoning_aggregated_df, alias_aggregated_df)
# after finish display status
status_info('ok', max_title, len(info))
# report tables
zoning_report_df, alias_report_df, zoning_compare_report_df, \
unzoned_device_report_df, no_alias_device_report_df, zoning_absent_device_report_df, \
zonemember_statistics_report_df, alias_statistics_report_df, effective_cfg_statistics_report_df = \
zoning_report_main(zoning_aggregated_df, alias_aggregated_df, portshow_zoned_aggregated_df,
zonemember_statistics_df, alias_statistics_df, effective_cfg_statistics_df,
data_names, report_columns_usage_dct, max_title)
# create list with partitioned DataFrames
data_lst = [
zoning_aggregated_df, alias_aggregated_df,
zonemember_statistics_df, portshow_zoned_aggregated_df,
alias_statistics_df, effective_cfg_statistics_df, zoning_report_df,
alias_report_df, zoning_compare_report_df,
unzoned_device_report_df, no_alias_device_report_df,
zoning_absent_device_report_df, zonemember_statistics_report_df,
alias_statistics_report_df, effective_cfg_statistics_report_df
]
# saving data to json or csv file
save_data(report_data_lst, data_names, *data_lst)
# verify if loaded data is empty and replace information string with empty DataFrame
else:
zoning_aggregated_df, alias_aggregated_df, zonemember_statistics_df, \
portshow_zoned_aggregated_df, alias_statistics_df, effective_cfg_statistics_df, zoning_report_df, alias_report_df, \
zoning_compare_report_df, unzoned_device_report_df, no_alias_device_report_df, zoning_absent_device_report_df, \
zonemember_statistics_report_df, alias_statistics_report_df, effective_cfg_statistics_report_df \
= verify_data(report_data_lst, data_names, *data_lst)
data_lst = [
zoning_aggregated_df, alias_aggregated_df,
zonemember_statistics_df, portshow_zoned_aggregated_df,
alias_statistics_df, effective_cfg_statistics_df, zoning_report_df,
alias_report_df, zoning_compare_report_df,
unzoned_device_report_df, no_alias_device_report_df,
zoning_absent_device_report_df, zonemember_statistics_report_df,
alias_statistics_report_df, effective_cfg_statistics_report_df
]
# save data to service file if it's required
for data_name, data_frame in zip(data_names, data_lst):
save_xlsx_file(data_frame, data_name, report_data_lst)
return zoning_aggregated_df, alias_aggregated_df, portshow_zoned_aggregated_df
def chassis_params_extract(all_config_data, report_data_lst):
"""Function to extract chassis parameters"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = ['chassis_parameters']
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# unpacking from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
chassis_params_fabric_lst, = data_lst
# data force extract check
# list of keys for each data from data_lst representing if it is required
# to re-collect or re-analyze data even they were obtained on previous iterations
force_extract_keys_lst = [
report_steps_dct[data_name][1] for data_name in data_names
]
# print data which were loaded but for which force extract flag is on
force_extract_check(data_names, data_lst, force_extract_keys_lst,
max_title)
# when any of data_lst was not saved or
# force extract flag is on then re-extract data from configueation files
if not all(data_lst) or any(force_extract_keys_lst):
print(
'\nEXTRACTING CHASSIS PARAMETERS FROM SUPPORTSHOW CONFIGURATION FILES ...\n'
)
# number of switches to check
switch_num = len(all_config_data)
# list to store only REQUIRED chassis parameters
# collecting data for all chassis during looping
chassis_params_fabric_lst = []
# data imported from init file to extract values from config file
chassis_params, chassis_params_add, comp_keys, match_keys, comp_dct = data_extract_objects(
'chassis', max_title)
# all_confg_data format ([swtch_name, supportshow file, (ams_maps_log files, ...)])
# checking each config set(supportshow file) for chassis level parameters
for i, switch_config_data in enumerate(all_config_data):
# data unpacking from iter param
switch_name, sshow_file, ams_maps_file = switch_config_data
# search control dictionary. continue to check sshow_file until all parameters groups are found
collected = {
'configshow': False,
'uptime_cpu': False,
'flash': False,
'memory': False,
'dhcp': False,
'licenses': False,
'vf_id': False
}
# dictionary to store all DISCOVERED chassis parameters
# collecting data only for the chassis in current loop
chassis_params_dct = {}
# sets and list to store parameters which could be joined in one group
snmp_target_set = set()
syslog_set = set()
tz_lst = []
licenses = []
vf_id_set = set()
# current operation information string
info = f'[{i+1} of {switch_num}]: {switch_name} chassis parameters'
print(info, end=" ")
with open(sshow_file, encoding='utf-8', errors='ignore') as file:
# check file until all groups of parameters extracted
while not all(collected.values()):
line = file.readline()
if not line:
break
# configshow section start
if re.search(
r'^(/fabos/cliexec/|/bin/cat /var/log/)?configshow *-?(all)? *:$',
line):
# when section is found corresponding collected dict values changed to True
collected['configshow'] = True
while not re.search(
r'^(\[Chassis Configuration End\])|(real [\w.]+)|(\*\* SS CMD END \*\*)$',
line):
line = file.readline()
# dictionary with match names as keys and match result of current line with all imported regular expressions as values
match_dct = {
match_key: comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# match_keys ['chassis_param_match', 'snmp_target_match', 'syslog_match', 'tz_match', 'uptime_cpu_match', 'memory_match', 'flash_match']
# 'chassis_param_match'
if match_dct[match_keys[0]]:
chassis_params_dct[match_dct[
match_keys[0]].group(1).rstrip(
)] = match_dct[match_keys[0]].group(3)
# for snmp and syslog data addresses are added to the coreesponding sets to avoid duplicates
# 'snmp_target_match'
if match_dct[match_keys[1]] and match_dct[
match_keys[1]].group(2) != '0.0.0.0':
snmp_target_set.add(
match_dct[match_keys[1]].group(2))
# 'syslog_match'
if match_dct[match_keys[2]]:
syslog_set.add(
match_dct[match_keys[2]].group(2))
# for timezone extracted data added to the list for later concatenation
# 'tz_match'
if match_dct[match_keys[3]]:
tz_lst.append(
match_dct[match_keys[3]].group(2))
if not line:
break
# config section end
# uptime section start
elif re.search(r'^(/fabos/cliexec/)?uptime *:$', line):
collected['uptime_cpu'] = True
while not re.search(r'^real [\w.]+$', line):
line = file.readline()
match_dct = {
match_key: comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# 'uptime_cpu_match'
if match_dct[match_keys[4]]:
uptime = match_dct[match_keys[4]].group(1)
cpu_load = match_dct[match_keys[4]].group(2)
if not line:
break
# uptime section end
# memory section start
elif re.search(r'^(/bin/)?(cat\s+)?/proc/meminfo\s*:$',
line):
collected['memory'] = True
memory_dct = {}
while not re.search(r'^real [\w.]+$', line):
line = file.readline()
match_dct = {
match_key: comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# 'memory_match'
if match_dct[match_keys[5]]:
memory_dct[match_dct[match_keys[5]].group(
1)] = match_dct[match_keys[5]].group(2)
if not line:
break
# free_mem + buffers > 5% from total memory
# memory usage < 95%
memory = round((1 - (int(memory_dct['MemFree']) +
int(memory_dct['Buffers'])) /
int(memory_dct['MemTotal'])) * 100)
memory = str(memory)
# memory section end
# flash section start
elif re.search(r'^(/bin/)?df\s*:$', line):
collected['flash'] = True
while not re.search(r'^real [\w.]+$', line):
line = file.readline()
match_dct = {
match_key: comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# 'flash_match'
if match_dct[match_keys[6]]:
flash = match_dct[match_keys[6]].group(1)
if not line:
break
# flash section end
# ipaddrshow section start
if re.search(r'^(/fabos/link_bin/)?ipaddrshow *:$', line):
collected['dhcp'] = True
while not re.search(
r'^(real [\w.]+)|(\*\* SS CMD END \*\*)$',
line):
line = file.readline()
match_dct = {
match_key: comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# 'dhcp_match'
if match_dct[match_keys[7]]:
chassis_params_dct[match_dct[
match_keys[7]].group(1).rstrip(
)] = match_dct[match_keys[7]].group(2)
if not line:
break
# ipaddrshow section end
# licenses section start
if re.search(r'^(/fabos/cliexec/)?licenseshow *:$', line):
collected['licenses'] = True
while not re.search(
r'^(real [\w.]+)|(\*\* SS CMD END \*\*)$',
line):
line = file.readline()
match_dct = {
match_key: comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# 'licenses_match'
if match_dct[match_keys[8]]:
licenses.append(
match_dct[match_keys[8]].group(1))
elif re.match('^No licenses installed.$', line):
licenses = 'No licenses installed'
if not line:
break
# licenses section end
# LS indexes identification start
if re.search(r'Section *: +SSHOW_FABRIC', line):
collected['vf_id'] = True
while not re.search(
r'^(SWITCHCMD /fabos/cliexec/)?dom *:$|Non-VF',
line):
if re.search(r'CURRENT +CONTEXT +-- +(\d+) *, \d+',
line):
id = re.match(
r'CURRENT +CONTEXT +-- +(\d+) *, \d+',
line).group(1)
vf_id_set.add(id)
line = file.readline()
else:
line = file.readline()
if not line:
break
# LS indexes identification end
# additional values which need to be added to the chassis params dictionary
# chassis_params_add order (configname, ams_maps_log, chassis_name, snmp_server, syslog_server, timezone_h:m, uptime, cpu_average_load, memory_usage, flash_usage, licenses)
# values axtracted in manual mode. if change values order change keys order in init.xlsx "chassis_params_add" column
vf_id_lst = list(vf_id_set)
vf_id_lst.sort()
chassis_params_values = (sshow_file, ams_maps_file, switch_name,
vf_id_lst, snmp_target_set, syslog_set,
tz_lst, uptime, cpu_load, memory, flash,
licenses)
# adding additional parameters and values to the chassis_params_switch_dct
for chassis_param_add, chassis_param_value in zip(
chassis_params_add, chassis_params_values):
if chassis_param_value:
if not isinstance(chassis_param_value, str):
s = ':' if chassis_param_add == 'timezone_h:m' else ', '
chassis_param_value = f'{s}'.join(chassis_param_value)
chassis_params_dct[chassis_param_add] = chassis_param_value
# creating list with REQUIRED chassis parameters for the current switch
# if no value in the chassis_params_dct for the parameter then None is added
# and appending this list to the list of all switches chassis_params_fabric_lst
chassis_params_fabric_lst.append([
chassis_params_dct.get(chassis_param, None)
for chassis_param in chassis_params
])
status_info('ok', max_title, len(info))
# save extracted data to json file
save_data(report_data_lst, data_names, chassis_params_fabric_lst)
# verify if loaded data is empty after first iteration and replace information string with empty list
else:
chassis_params_fabric_lst = verify_data(report_data_lst, data_names,
*data_lst)
return chassis_params_fabric_lst
def portcmd_analysis_main(portshow_df, switchshow_ports_df, switch_params_df,
switch_params_aggregated_df, isl_aggregated_df,
nsshow_df, nscamshow_df, ag_principal_df,
porttrunkarea_df, alias_df, fdmi_df, blade_module_df,
blade_servers_df, blade_vc_df, synergy_module_df,
synergy_servers_df, system_3par_df, port_3par_df,
report_columns_usage_dct, report_data_lst):
"""Main function to add connected devices information to portshow DataFrame"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = [
'portshow_aggregated', 'storage_connection_statistics',
'device_connection_statistics', 'device_rename',
'report_columns_usage_upd', 'Серверы', 'Массивы', 'Библиотеки',
'Микрокоды_HBA', 'Подключение_массивов', 'Подключение_библиотек',
'Подключение_серверов', 'NPIV', 'Статистика_массивов',
'Статистика_устройств'
]
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
report_columns_usage_bckp = report_columns_usage_dct
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# flag to forcible save portshow_aggregated_df if required
portshow_force_flag = False
# unpacking DataFrames from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
portshow_aggregated_df, storage_connection_statistics_df, device_connection_statistics_df, \
device_rename_df, report_columns_usage_dct, \
servers_report_df, storage_report_df, library_report_df, hba_report_df, \
storage_connection_df, library_connection_df, server_connection_df, npiv_report_df, \
storage_connection_statistics_report_df, device_connection_statistics_report_df = data_lst
nsshow_unsplit_df = pd.DataFrame()
if not report_columns_usage_dct:
report_columns_usage_dct = report_columns_usage_bckp
# list of data to analyze from report_info table
analyzed_data_names = [
'portcmd', 'switchshow_ports', 'switch_params_aggregated',
'switch_parameters', 'chassis_parameters', 'fdmi', 'nscamshow',
'nsshow', 'alias', 'blade_servers', 'fabric_labels', 'isl', 'trunk',
'isl_aggregated', 'Параметры_SFP', 'portshow_sfp_aggregated'
]
# force run when any data from data_lst was not saved (file not found) or
# procedure execution explicitly requested for output data or data used during fn execution
force_run = verify_force_run(data_names, data_lst, report_steps_dct,
max_title, analyzed_data_names)
if force_run:
# import data with switch models, firmware and etc
switch_models_df = dataframe_import('switch_models', max_title)
# data imported from init file (regular expression patterns) to extract values from data columns
# re_pattern list contains comp_keys, match_keys, comp_dct
_, _, *re_pattern_lst = data_extract_objects('nameserver', max_title)
oui_df = dataframe_import('oui',
max_title,
columns=['Connected_oui', 'type', 'subtype'])
# current operation information string
info = f'Generating connected devices table'
print(info, end=" ")
portshow_aggregated_df, alias_wwnn_wwnp_df, nsshow_unsplit_df, expected_ag_links_df = \
portshow_aggregated(portshow_df, switchshow_ports_df, switch_params_df,
switch_params_aggregated_df, isl_aggregated_df, nsshow_df,
nscamshow_df, ag_principal_df, porttrunkarea_df, switch_models_df, alias_df,
oui_df, fdmi_df, blade_module_df, blade_servers_df, blade_vc_df,
synergy_module_df, synergy_servers_df, system_3par_df, port_3par_df,
re_pattern_lst, report_data_lst)
# after finish display status
status_info('ok', max_title, len(info))
# show warning if any UNKNOWN device class founded, if any PortSymb or NodeSymb is not parsed,
# if new switch founded
portshow_force_flag, nsshow_unsplit_force_flag, expected_ag_links_force_flag = \
warning_notification(portshow_aggregated_df, switch_params_aggregated_df,
nsshow_unsplit_df, expected_ag_links_df, report_data_lst)
# correct device names manually
portshow_aggregated_df, device_rename_df = \
devicename_correction_main(portshow_aggregated_df, device_rename_df,
report_columns_usage_dct, report_data_lst)
# count Device_Host_Name instances for fabric_label, label and total in fabric
portshow_aggregated_df = device_ports_per_group(portshow_aggregated_df)
# count device connection statistics
info = f'Counting device connection statistics'
print(info, end=" ")
storage_connection_statistics_df = storage_connection_statistics(
portshow_aggregated_df, re_pattern_lst)
device_connection_statistics_df = device_connection_statistics(
portshow_aggregated_df)
status_info('ok', max_title, len(info))
servers_report_df, storage_report_df, library_report_df, hba_report_df, \
storage_connection_df, library_connection_df, server_connection_df, npiv_report_df, \
storage_connection_statistics_report_df, device_connection_statistics_report_df = \
create_report_tables(portshow_aggregated_df, storage_connection_statistics_df,
device_connection_statistics_df, data_names[5:-2],
report_columns_usage_dct, max_title)
# create list with partitioned DataFrames
data_lst = [
portshow_aggregated_df, storage_connection_statistics_df,
device_connection_statistics_df, device_rename_df,
report_columns_usage_dct, servers_report_df, storage_report_df,
library_report_df, hba_report_df, storage_connection_df,
library_connection_df, server_connection_df, npiv_report_df,
storage_connection_statistics_report_df,
device_connection_statistics_report_df
]
# saving data to json or csv file
save_data(report_data_lst, data_names, *data_lst)
save_xlsx_file(nsshow_unsplit_df,
'nsshow_unsplit',
report_data_lst,
force_flag=nsshow_unsplit_force_flag)
save_xlsx_file(expected_ag_links_df,
'expected_ag_links',
report_data_lst,
force_flag=expected_ag_links_force_flag)
# verify if loaded data is empty and replace information string with empty DataFrame
else:
portshow_aggregated_df, storage_connection_statistics_df, device_connection_statistics_df, \
device_rename_df, report_columns_usage_dct, \
servers_report_df, storage_report_df, library_report_df, hba_report_df, \
storage_connection_df, library_connection_df, server_connection_df, npiv_report_df, \
storage_connection_statistics_report_df, device_connection_statistics_report_df \
= verify_data(report_data_lst, data_names, *data_lst)
data_lst = [
portshow_aggregated_df, storage_connection_statistics_df,
device_connection_statistics_df, device_rename_df,
report_columns_usage_dct, servers_report_df, storage_report_df,
library_report_df, hba_report_df, storage_connection_df,
library_connection_df, server_connection_df, npiv_report_df,
storage_connection_statistics_report_df,
device_connection_statistics_report_df
]
# save data to service file if it's required
for data_name, data_frame in zip(data_names, data_lst):
force_flag = False
if data_name == 'portshow_aggregated':
force_flag = portshow_force_flag
save_xlsx_file(data_frame,
data_name,
report_data_lst,
force_flag=force_flag)
return portshow_aggregated_df
def fabriclabels_main(switchshow_ports_df, switch_params_df, fabricshow_df,
ag_principal_df, report_data_lst):
"""Function to set Fabric labels"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
customer_name, report_path, _, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = ['fabric_labels']
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# unpacking DataFrames from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
fabricshow_ag_labels_df, = data_lst
# list of data to analyze from report_info table
analyzed_data_names = []
# force run when any data from data_lst was not saved (file not found) or
# procedure execution explicitly requested for output data or data used during fn execution
force_run = verify_force_run(data_names, data_lst, report_steps_dct,
max_title, analyzed_data_names)
if force_run:
print('\nSETTING UP FABRICS NAMES AND LABELS ...\n')
fabricshow_summary_df = auto_fabrics_labeling(switchshow_ports_df,
switch_params_df,
fabricshow_df,
report_data_lst)
# display automatic fabric labeling
info_labels = [
'Fabric_name', 'Fabric_label', 'chassis_name',
'Principal_switch_name', 'Fabric_ID', 'FC_Route', 'Total_switch',
'Domain_IDs', 'Switch_names', 'Device_ports', 'Online_ports',
'LS_type', 'Fabric_Name'
]
# service file name for detailed information
current_date = str(date.today())
file_name = customer_name + '_' + report_steps_dct[
'fabricshow_summary'][2] + '_' + current_date + '.xlsx'
# file_name = customer_name + '_analysis_report_' + current_date + '.xlsx'
print('\nAutomatic fabrics labeling\n')
# set option to show all columns
with pd.option_context('display.max_columns', None,
'display.expand_frame_repr', False):
# pd.set_option('max_columns', None)
# pd.set_option('expand_frame_repr', False)
print(fabricshow_summary_df.loc[:, info_labels])
print(
f"\nFor detailed switch port types and numbers statistic in each fabric check '{file_name}' file 'fabricshow_statistics' sheet in"
)
print(f'{report_path} directory')
print('ATTN! CLOSE file after check\n')
# ask user if Automatic Fabric labeling need to be corrected
query = 'Do you want to change Fabrics Names or Labels? (y)es/(n)o: '
reply = reply_request(query)
if reply == 'y':
# saving DataFrame to Excel to check during manual labeling if required
save_xlsx_file(fabricshow_summary_df,
'fabricshow_summary',
report_data_lst,
force_flag=True)
fabricshow_summary_df = manual_fabrics_labeling(
fabricshow_summary_df, info_labels)
# takes all switches working in Native and AG switches
# merge the in one DataFrame and identify which Fabrics they belong too with fabricshow_summary DataFrame
fabricshow_ag_labels_df = native_ag_labeling(fabricshow_df,
ag_principal_df,
fabricshow_summary_df)
# # disable option to show all columns
# pd.reset_option('max_columns')
# pd.reset_option('expand_frame_repr')
# create list with partitioned DataFrames
data_lst = [fabricshow_ag_labels_df]
# saving data to json or csv file
save_data(report_data_lst, data_names, *data_lst)
# verify if loaded data is empty and replace information string with empty DataFrame
else:
fabricshow_ag_labels_df = verify_data(report_data_lst, data_names,
*data_lst)
data_lst = [fabricshow_ag_labels_df]
# save data to excel file if it's required
for data_name, data_frame in zip(data_names, data_lst):
save_xlsx_file(data_frame, data_name, report_data_lst)
return fabricshow_ag_labels_df
def maps_params_extract(all_config_data, report_data_lst):
"""Function to extract maps parameters
"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = ['maps_parameters']
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# unpacking from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
maps_params_fabric_lst, = data_lst
# data force extract check
# list of keys for each data from data_lst representing if it is required
# to re-collect or re-analyze data even they were obtained on previous iterations
force_extract_keys_lst = [
report_steps_dct[data_name][1] for data_name in data_names
]
# print data which were loaded but for which force extract flag is on
force_extract_check(data_names, data_lst, force_extract_keys_lst,
max_title)
# when any of data_lst was not saved or
# force extract flag is on then re-extract data from configueation files
if not all(data_lst) or any(force_extract_keys_lst):
print(
'\nEXTRACTING MAPS DATA FROM AMS_MAPS_LOG CONFIGURATION FILES ...\n'
)
# number of switches to check
switch_num = len(all_config_data)
# list to store only REQUIRED parameters
# collecting data for all switches during looping
maps_params_fabric_lst = []
# data imported from init file to extract values from config file
maps_params, maps_params_add, comp_keys, match_keys, comp_dct = data_extract_objects(
'maps', max_title)
# all_confg_data format ([swtch_name, supportshow file, (ams_maps_log files, ...)])
# checking each config set(supportshow file) for chassis level parameters
for i, switch_config_data in enumerate(all_config_data):
# data unpacking from iter param
switch_name, sshow_file, ams_maps_files = switch_config_data
# number of ams_maps configs
num_maps = len(ams_maps_files) if ams_maps_files else 0
# current operation information string
info = f'[{i+1} of {switch_num}]: {switch_name} MAPS parameters. Number of AMS_MAPS configs: {num_maps} ...'
print(info)
# checking ams_maps log file for each logical switch
if ams_maps_files:
for ams_maps_file in ams_maps_files:
# search control dictionary. continue to check sshow_file until all parameters groups are found
collected = {'switch_index': False, 'global_dash': False}
# dictionary to store all DISCOVERED switch parameters
# collecting data only for the logical switch in current loop
maps_params_dct = {}
info = ' ' * 16 + f'{os.path.basename(ams_maps_file)} processing'
print(info, end=" ")
with open(ams_maps_file, encoding='utf-8',
errors='ignore') as file:
# check file until all groups of parameters extracted
while not all(collected.values()):
line = file.readline()
if not line:
break
# logical switch index section start
if re.search(
r'^[= ]*AMS/MAPS *Data *Switch *(\d+)[= ]*$',
line):
# when section is found corresponding collected dict values changed to True
collected['switch_index'] = True
match_dct = {
match_key: comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
switch_index = match_dct[match_keys[0]].group(
1)
# logical switch index section end
# global dashboard section start
if re.search(
r'^[- ]*MAPS +Global +Monitoring +Configuration[ -]*$',
line):
collected['global_dash'] = True
while not re.search(r'^[- ]*NM +Data[- ]*$',
line):
line = file.readline()
# dictionary with match names as keys and match result of current line with all imported regular expressions as values
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# match_keys ['switch_index_match', 'dashboard_match', 'report_match', 'no_lic_match']
# 'dashboard_match'
if match_dct[match_keys[1]]:
maps_params_dct[match_dct[match_keys[
1]].group(1).rstrip()] = match_dct[
match_keys[1]].group(2)
# 'report_match'
if match_dct[match_keys[2]]:
maps_params_dct[match_dct[match_keys[
2]].group(1).rstrip()] = match_dct[
match_keys[2]].group(2)
# 'no Fabric lic match'
if match_dct[match_keys[3]]:
for maps_param in maps_params[6:23]:
maps_params_dct[
maps_param] = 'No FV lic'
if not line:
break
# global dashboard section end
# additional values which need to be added to the chassis params dictionary
# chassis_params_add order (configname, ams_maps_config, chassis_name, switch_index)
# values axtracted in manual mode. if change values order change keys order in init.xlsx "maps_params_add" column
maps_params_values = (sshow_file, ams_maps_file,
switch_name, switch_index)
# adding additional parameters and values to the chassis_params_switch_dct
for maps_param_add, maps_param_value in zip(
maps_params_add, maps_params_values):
maps_params_dct[maps_param_add] = maps_param_value
# creating list with REQUIRED maps parameters for the current switch
# if no value in the maps_params_dct for the parameter then None is added
# and appending this list to the list of all switches maps_params_fabric_lst
maps_params_fabric_lst.append([
maps_params_dct.get(maps_param, None)
for maps_param in maps_params
])
status_info('ok', max_title, len(info))
else:
info = ' ' * 16 + 'No AMS_MAPS configuration found.'
print(info, end=" ")
status_info('skip', max_title, len(info))
# save extracted data to json file
save_data(report_data_lst, data_names, maps_params_fabric_lst)
# verify if loaded data is empty after first iteration and replace information string with empty list
else:
maps_params_fabric_lst = verify_data(report_data_lst, data_names,
*data_lst)
return maps_params_fabric_lst
