def sensor_analysis_main(sensor_df, switch_params_aggregated_df,
report_columns_usage_dct, report_data_lst):
"""Main function to analyze zoning configuration"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = ['sensor_aggregated', 'Датчики']
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# unpacking DataFrames from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
sensor_aggregated_df, sensor_report_df = data_lst
# list of data to analyze from report_info table
analyzed_data_names = [
'switch_params_aggregated', 'fabric_labels', 'sensor'
]
# force run when any data from data_lst was not saved (file not found) or
# procedure execution explicitly requested for output data or data used during fn execution
force_run = verify_force_run(data_names, data_lst, report_steps_dct,
max_title, analyzed_data_names)
if force_run:
# current operation information string
info = f'Generating sensor readings table'
print(info, end=" ")
# aggregated DataFrames
sensor_aggregated_df = sensor_aggregation(sensor_df,
switch_params_aggregated_df)
# after finish display status
status_info('ok', max_title, len(info))
# report tables
sensor_report_df = sensor_report(sensor_aggregated_df, data_names,
report_columns_usage_dct, max_title)
# create list with partitioned DataFrames
data_lst = [sensor_aggregated_df, sensor_report_df]
# saving data to json or csv file
save_data(report_data_lst, data_names, *data_lst)
# verify if loaded data is empty and replace information string with empty DataFrame
else:
sensor_aggregated_df, sensor_report_df = verify_data(
report_data_lst, data_names, *data_lst)
data_lst = [sensor_aggregated_df, sensor_report_df]
# save data to service file if it's required
for data_name, data_frame in zip(data_names, data_lst):
save_xlsx_file(data_frame, data_name, report_data_lst)
return sensor_aggregated_df
def find_files(folder, max_title, filename_contains='', filename_extension=''):
"""
Function to create list with files. Takes directory, regex_pattern to verify if filename
contains that pattern (default empty string) and filename extension (default is empty string)
as parameters. Returns list of files with the extension deteceted in root folder defined as
folder parameter and it's nested folders. If both parameters are default functions returns
list of all files in directory
"""
info = f'Checking {os.path.basename(folder)} folder for configuration files'
print(info, end=" ")
# check if ssave_path folder exist
check_valid_path(folder)
# list to save configuration data files
files_lst = []
# going through all directories inside ssave folder to find configuration data
for root, _, files in os.walk(folder):
for file in files:
if file.endswith(filename_extension) and re.search(
filename_contains, file):
file_path = os.path.normpath(os.path.join(root, file))
files_lst.append(file_path)
if len(files_lst) == 0:
status_info('no data', max_title, len(info))
else:
status_info('ok', max_title, len(info))
return files_lst
def local_download(ns_3par_df, configs_local_lst, download_folder, comp_keys,
match_keys, comp_dct, max_title):
"""Function to copy 3PAR configuration files from local folder
to download folder"""
# verifu if download folder exist and create one if not (default behaviour)
verify_download_folder(download_folder, max_title)
if not 'STATs_status' in ns_3par_df.columns:
ns_3par_df['STATs_status'] = np.nan
drop_stats_column = True
else:
drop_stats_column = False
ns_3par_df[['filename', 'Local_status']] = np.nan
sn_lst = ns_3par_df['Serial_Number'].tolist()
print('\n')
# copy files from configs_local_lst to download folder
for i, source_file in enumerate(configs_local_lst):
filename = os.path.basename(source_file)
if os.path.isfile(source_file) or os.path.islink(source_file):
# extract model and seral number from config file
model, sn = parse_serial(source_file, comp_keys, match_keys,
comp_dct)
info = ' ' * 16 + f'[{i+1} of {len(configs_local_lst)}]: Copying {filename} for {model} {sn}'
print(info, end=' ')
# verify if config for current sn was downloaded from STATs or local folder before
mask_sn = ns_3par_df['Serial_Number'] == sn
status_ok = (ns_3par_df.loc[mask_sn,
['STATs_status', 'Local_status']] ==
'ok').any().any()
# copy config if 3par is in NameServer and was not downloaded from STATs or local folder before
if sn in sn_lst and not status_ok:
# destination_file = os.path.join(download_folder, filename)
status = 'unknown'
try:
shutil.copy2(source_file, download_folder)
status = status_info('ok', max_title, len(info))
except shutil.Error as e:
status = status_info('failed', max_title, len(info))
print('\n')
print(e)
finally:
ns_3par_df.loc[mask_sn, ['filename', 'Local_status']] = [
filename, status.lower()
]
else:
status = status_info('skip', max_title, len(info))
else:
info = ' ' * 16 + f'[{i+1} of {len(configs_local_lst)}]: copying {filename}'
print(info, end=' ')
status_info('skip', max_title, len(info))
if drop_stats_column:
ns_3par_df.drop(columns=['STATs_status'], inplace=True)
return ns_3par_df
def devicename_correction_main(portshow_aggregated_df, device_rename_df,
report_columns_usage_dct, report_data_lst):
"""Main function to rename devices"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
analyzed_data_names = [
'device_rename', 'portshow_aggregated', 'portcmd', 'switchshow_ports',
'switch_params_aggregated', 'switch_parameters', 'chassis_parameters',
'fdmi', 'nscamshow', 'nsshow', 'alias', 'blade_servers',
'fabric_labels'
]
# check if portshow_aggregated DataFrame is changed or 'device_rename' force flag is on
force_form_update_flag = any(
[report_steps_dct[data_name][1] for data_name in analyzed_data_names])
# list of related DataFrame names requested to change
force_change_data_lst = [
data_name for data_name in analyzed_data_names[1:]
if report_steps_dct[data_name][1]
]
# flag to force change group name usage mode
force_group_name_usage_update_flag = report_steps_dct[
'report_columns_usage_upd'][1]
# create DataFrame with devices required to change names
device_rename_df = define_device_to_rename(portshow_aggregated_df,
device_rename_df, max_title,
force_form_update_flag,
force_change_data_lst,
report_data_lst)
# current operation information string
info = f'Applying device rename schema'
print(info, end=" ")
# if at least one name is changed apply rename schema
if device_rename_df['Device_Host_Name_rename'].notna().any():
portshow_aggregated_df = device_rename(portshow_aggregated_df,
device_rename_df)
status_info('ok', max_title, len(info))
else:
# to ask if group usage required
force_group_name_usage_update_flag = 1
status_info('skip', max_title, len(info))
# check if device Group_Name should be used in report tables (alias group name)
group_name_usage(report_columns_usage_dct, device_rename_df, max_title,
force_group_name_usage_update_flag)
return portshow_aggregated_df, device_rename_df
def load_data(report_data_list, *args):
"""Function to load data from JSON or CSV file to data object
Detects wich type of data required to be loaded automaticaly
Returns list with imported data
"""
customer_name, _, json_data_dir, max_title, _ = report_data_list
# list to store loaded data
data_imported = []
# check real file path
for data_name in args:
# constructing filenames for json and csv files
file_name_json = customer_name + '_' + data_name + '.json'
file_path_json = os.path.join(json_data_dir, file_name_json)
file_name_csv = customer_name + '_' + data_name + '.csv'
file_path_csv = os.path.join(json_data_dir, file_name_csv)
# flags file existance
file_json = False
file_csv = False
# check if json file exist
if os.path.isfile(file_path_json):
file_path = file_path_json
file_name = file_name_json
file_json = True
# check if csv file exist
elif os.path.isfile(file_path_csv):
file_name = file_name_csv
file_path = file_path_csv
file_csv = True
# if no files exist then dislay info about no data availabilty
# and add None to data_imported list
else:
info = f'Loading {data_name}'
print(info, end=" ")
status_info('no file', max_title, len(info))
data_imported.append(None)
# when saved file founded
if any([file_json, file_csv]):
info = f'Loading {data_name} from {file_name} file'
print(info, end=" ")
# open file
try:
with open(file_path, 'r', encoding="utf-8") as file:
# for json file use load method
if file_json:
data_imported.append(json.load(file))
# for csv file use read_csv method
elif file_csv:
data_imported.append(pd.read_csv(file,
dtype='unicode'))
# display file load status
except:
status_info('no data', max_title, len(info))
data_imported.append(None)
else:
status_info('ok', max_title, len(info))
return data_imported
def dataframe_import(sheet_title, max_title, init_file = 'san_automation_info.xlsx',
columns = None, index_name = None, header = 0, display_status=True):
"""Function to import dataframe from exel file"""
warnings.filterwarnings('ignore', category=UserWarning, module="openpyxl")
init_file_base = os.path.basename(init_file)
# file to store all required data to process configuratin files
# init_file = 'san_automation_info.xlsx'
if display_status:
info = f'Importing {sheet_title} dataframe from {init_file_base} file'
print(info, end = ' ')
# try read data in excel
try:
dataframe = pd.read_excel(init_file, sheet_name=sheet_title, usecols=columns, index_col=index_name, header=header)
# if file is not found
except FileNotFoundError:
if display_status:
status_info('fail', max_title, len(info))
print(f'File not found. Check if file {init_file} exists.')
sys.exit()
# if sheet is not found
except xlrd.biffh.XLRDError:
if display_status:
status_info('fail', max_title, len(info))
print(f'Sheet {sheet_title} not found in {init_file}. Check if it exists.')
sys.exit()
else:
if display_status:
status_info('ok', max_title, len(info))
return dataframe
def save_data(report_data_list, data_names, *args):
"""
Function to export extracted configuration data to JSON or CSV file
depending on data passed
"""
customer_name, _, json_data_dir, max_title, _ = report_data_list
# data_names it's a list of names for data passed as args
for data_name, data_exported in zip(data_names, args):
empty_data = False
file_name = customer_name + '_' + data_name
# adding file extenson depending from type of data saved
# csv for DataFrame
if isinstance(data_exported, pd.DataFrame):
file_name += '.csv'
# for all other types is json
else:
# create file name and path for json file
file_name += '.json'
# full file path
file_path = os.path.join(json_data_dir, file_name)
info = f'Saving {data_name} to {file_name} file'
try:
print(info, end=" ")
with open(file_path, 'w', encoding="utf-8") as file:
# savng data for DataFrame
if isinstance(data_exported, pd.DataFrame):
# check if DataFrame have MultiIndex
# reset index if True due to MultiIndex is not saved
if isinstance(data_exported.index, pd.MultiIndex):
data_exported_flat = data_exported.reset_index()
# keep indexing if False
else:
data_exported_flat = data_exported.copy()
# when DataFrame is empty fill first row values
# with information string
if data_exported_flat.empty:
empty_data = True
if len(data_exported_flat.columns) == 0:
data_exported_flat['EMPTY'] = np.nan
data_exported_flat.loc[0] = 'NO DATA FOUND'
# save single level Index DataFrame to csv
data_exported_flat.to_csv(file, index=False)
# for all other types
else:
# if data list is not empty
if not len(data_exported):
empty_data = True
data_exported = 'NO DATA FOUND'
json.dump(data_exported, file)
# display writing data to file status
except FileNotFoundError:
status_info('fail', max_title, len(info))
else:
if not empty_data:
status_info('ok', max_title, len(info))
else:
status_info('empty', max_title, len(info))
def group_name_usage(report_columns_usage_dct, device_rename_df, max_title,
force_group_name_usage_update_flag):
"""
Function to determine if device alias Group_Name column required in report tables.
Value (True or False) written to the dictonary so no need to return value.
"""
info = f'Device Group Name usage'
# if all devices were renamed and force flag is off column is dropped from report tables
if device_rename_df['Device_Host_Name_rename'].notna().all(
) and not force_group_name_usage_update_flag:
report_columns_usage_dct['group_name_usage'] = False
print(info, end=" ")
status_info('off', max_title, len(info))
# if no information in report_columns_usage_dct or force flag is on or any device keep old name
# ask user input
elif report_columns_usage_dct.get('group_name_usage') is None or \
force_group_name_usage_update_flag or \
device_rename_df['Device_Host_Name_rename'].isna().any():
print('\n')
reply = reply_request(
'Do you want to use Alias Group Device names? (y)es/(n)o: ')
print('\n')
if reply == 'y':
report_columns_usage_dct['group_name_usage'] = True
print(info, end=" ")
status_info('on', max_title, len(info))
else:
report_columns_usage_dct['group_name_usage'] = False
print(info, end=" ")
status_info('off', max_title, len(info))
def create_folder(path, max_title):
"""Function to create any folder with path"""
info = f'Make directory {os.path.basename(path)}'
print(info, end=' ')
# if folder not exist create
if not os.path.exists(path):
try:
os.makedirs(path)
except OSError:
status_info('fail', max_title, len(info))
print(f'Not possible create directory {path}.')
print('Code execution finished')
sys.exit()
else:
status_info('ok', max_title, len(info))
# otherwise print 'SKIP' status
else:
status_info('skip', max_title, len(info))
def remove_files(files_lst, max_title):
"""Function to remove files from the list"""
# print('\n')
for i, file in enumerate(files_lst):
filename = os.path.basename(file)
info = ' ' * 16 + f'[{i+1} of {len(files_lst)}]: Removing file {filename}'
print(info, end=' ')
# verify if file exist
if os.path.isfile(file) or os.path.islink(file):
try:
os.remove(file)
status_info('ok', max_title, len(info))
except OSError as e:
status_info('failed', max_title, len(info))
print('\n')
print(e.strerror)
else:
status_info('skip', max_title, len(info))
print('\n')
def columns_import(sheet_title, max_title, *args,
init_file = 'san_automation_info.xlsx', display_status=True):
"""Function to import corresponding columns from init file.
Can import several columns.
"""
# file to store all required data to process configuratin files
# default init_file is 'san_automation_info.xlsx'
# columns titles string to display imported columns list without parenthesis
columns_str = ""
for arg in args:
columns_str += "'" + arg + "', "
columns_str = columns_str.rstrip(", ")
if display_status:
info = f'Importing {columns_str} from {sheet_title} tab'
print(info, end = ' ')
# try read data in excel
try:
columns = pd.read_excel(init_file, sheet_name = sheet_title, usecols = args, squeeze=True)
except FileNotFoundError:
if display_status:
status_info('fail', max_title, len(info))
print(f'File not found. Check if file {init_file} exist.')
sys.exit()
except ValueError:
if display_status:
status_info('fail', max_title, len(info))
print(f'Column(s) {columns_str} not found. Check if column exist in {sheet_title}.')
sys.exit()
else:
# if number of columns to read > 1 than returns corresponding number of lists
if len(args)>1:
columns_names = [columns[arg].dropna().tolist() if not columns[arg].empty else None for arg in args]
else:
columns_names = columns.dropna().tolist()
if display_status:
status_info('ok', max_title, len(info))
return columns_names
def sensor_extract(chassis_params_lst, report_data_lst):
"""Function to extract sensor information"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = ['sensor']
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# unpacking from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
sensor_lst, = data_lst
# data force extract check
# list of keys for each data from data_lst representing if it is required
# to re-collect or re-analyze data even they were obtained on previous iterations
force_extract_keys_lst = [
report_steps_dct[data_name][1] for data_name in data_names
]
# print data which were loaded but for which force extract flag is on
force_extract_check(data_names, data_lst, force_extract_keys_lst,
max_title)
# when any of data_lst was not saved or
# force extract flag is on then re-extract data from configueation files
if not all(data_lst) or any(force_extract_keys_lst):
print('\nEXTRACTING ENVIRONMENT DATA ...\n')
# # extract chassis parameters names from init file
# switch_columns = columns_import('switch', max_title, 'columns')
# extract chassis parameters names from init file
chassis_columns = columns_import('chassis', max_title, 'columns')
# number of switches to check
switch_num = len(chassis_params_lst)
# data imported from init file to extract values from config file
*_, comp_keys, match_keys, comp_dct = data_extract_objects(
'sensor', max_title)
# lists to store only REQUIRED infromation
# collecting data for all switches ports during looping
sensor_lst = []
# chassis_params_fabric_lst [[chassis_params_sw1], [chassis_params_sw1]]
# checking each chassis for switch level parameters
for i, chassis_params_data in enumerate(chassis_params_lst):
# data unpacking from iter param
# dictionary with parameters for the current chassis
chassis_params_data_dct = dict(
zip(chassis_columns, chassis_params_data))
chassis_info_keys = ['configname', 'chassis_name', 'chassis_wwn']
chassis_info_lst = [
chassis_params_data_dct.get(key) for key in chassis_info_keys
]
sshow_file, chassis_name, _ = chassis_info_lst
# current operation information string
info = f'[{i+1} of {switch_num}]: {chassis_name} sensor readings'
print(info, end=" ")
# search control dictionary. continue to check sshow_file until all parameters groups are found
collected = {'sensor': False}
with open(sshow_file, encoding='utf-8', errors='ignore') as file:
# check file until all groups of parameters extracted
while not all(collected.values()):
line = file.readline()
if not line:
break
# sensor section start
# switchcmd_sensorshow_comp
if re.search(comp_dct[comp_keys[0]],
line) and not collected['sensor']:
collected['sensor'] = True
# switchcmd_end_comp
while not re.search(comp_dct[comp_keys[2]], line):
line = file.readline()
match_dct = {
match_key: comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# islshow_match
if match_dct[match_keys[1]]:
sensor_reading = line_to_list(
comp_dct[comp_keys[1]], line,
*chassis_info_lst)
# appending list with only REQUIRED port info for the current loop iteration
# to the list with all ISL port info
sensor_lst.append(sensor_reading)
if not line:
break
# sensor section end
status_info('ok', max_title, len(info))
# save extracted data to json file
save_data(report_data_lst, data_names, sensor_lst)
# verify if loaded data is empty after first iteration and replace information string with empty list
else:
sensor_lst = verify_data(report_data_lst, data_names, *data_lst)
return sensor_lst
def portinfo_extract(switch_params_lst, report_data_lst):
"""Function to extract switch port information"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = ['sfpshow', 'portcfgshow']
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# unpacking from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
sfpshow_lst, portcfgshow_lst = data_lst
# when any data from data_lst was not saved (file not found) or
# force extract flag is on then re-extract data from configuration files
force_run = verify_force_run(data_names, data_lst, report_steps_dct, max_title)
# # data force extract check
# # list of keys for each data from data_lst representing if it is required
# # to re-collect or re-analyze data even they were obtained on previous iterations
# force_extract_keys_lst = [report_steps_dct[data_name][1] for data_name in data_names]
# # print data which were loaded but for which force extract flag is on
# force_extract_check(data_names, data_lst, force_extract_keys_lst, max_title)
# # when any of data_lst was not saved or
# # force extract flag is on then re-extract data from configueation files
# if not all(data_lst) or any(force_extract_keys_lst):
if force_run:
print('\nEXTRACTING SWITCH PORTS SFP, PORTCFG INFORMATION FROM SUPPORTSHOW CONFIGURATION FILES ...\n')
# extract chassis parameters names from init file
switch_columns = columns_import('switch', max_title, 'columns')
# number of switches to check
switch_num = len(switch_params_lst)
# data imported from init file to extract values from config file
params, params_add, comp_keys, match_keys, comp_dct = data_extract_objects('portinfo', max_title)
portcfg_params = columns_import('portinfo', max_title, 'portcfg_params')
# dictionary to save portcfg ALL information for all ports in fabric
portcfgshow_dct = dict((key, []) for key in portcfg_params)
# list to store only REQUIRED switch parameters
# collecting sfpshow data for all switches ports during looping
sfpshow_lst = []
# list to save portcfg information for all ports in fabric
portcfgshow_lst = []
# switch_params_lst [[switch_params_sw1], [switch_params_sw1]]
# checking each switch for switch level parameters
for i, switch_params_data in enumerate(switch_params_lst):
# data unpacking from iter param
# dictionary with parameters for the current switch
switch_params_data_dct = dict(zip(switch_columns, switch_params_data))
switch_info_keys = ['configname', 'chassis_name', 'chassis_wwn', 'switch_index',
'SwitchName', 'switchWwn']
switch_info_lst = [switch_params_data_dct.get(key) for key in switch_info_keys]
ls_mode_on = True if switch_params_data_dct['LS_mode'] == 'ON' else False
sshow_file, _, _, switch_index, switch_name, *_ = switch_info_lst
# current operation information string
info = f'[{i+1} of {switch_num}]: {switch_name} ports sfp and cfg'
print(info, end =" ")
# search control dictionary. continue to check sshow_file until all parameters groups are found
collected = {'sfpshow': False, 'portcfgshow': False}
with open(sshow_file, encoding='utf-8', errors='ignore') as file:
# check file until all groups of parameters extracted
while not all(collected.values()):
line = file.readline()
if not line:
break
# sfpshow section start
if re.search(r'^(SWITCHCMD )?(/fabos/cliexec/)?sfpshow +-all *: *$', line) and not collected['sfpshow']:
collected['sfpshow'] = True
if ls_mode_on:
while not re.search(fr'^CURRENT CONTEXT -- {switch_index} *, \d+$',line):
line = file.readline()
if not line:
break
while not re.search(r'^(real [\w.]+)|(\*\* SS CMD END \*\*)$',line):
line = file.readline()
match_dct ={match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip(comp_keys, match_keys)}
if match_dct[match_keys[0]]:
# dictionary to store all DISCOVERED switch ports information
# collecting data only for the logical switch in current loop
sfpshow_dct = {}
_, slot_num, port_num = line_to_list(comp_dct[comp_keys[0]], line)
# if switch has no slots then all ports have slot 0
slot_num = '0' if not slot_num else slot_num
while not re.match('\r?\n', line):
line = file.readline()
match_dct ={match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip(comp_keys, match_keys)}
# power_match
if match_dct[match_keys[1]]:
sfp_power_lst = line_to_list(comp_dct[comp_keys[1]], line)
# cut off RX or TX Power
sfp_power_value_unit = sfp_power_lst[1:]
for v, k in zip(sfp_power_value_unit[::2], sfp_power_value_unit[1::2]):
if k == 'uWatts':
k = 'uW'
key = sfp_power_lst[0] + '_' + k
sfpshow_dct[key] = v
# transceiver_match
elif match_dct[match_keys[2]]:
sfpshow_dct[match_dct[match_keys[2]].group(1).rstrip()] = match_dct[match_keys[2]].group(2).rstrip()
# no_sfp_match
elif match_dct[match_keys[3]]:
sfpshow_dct['Vendor Name'] = 'No SFP module'
# not_available_match
elif match_dct[match_keys[4]]:
sfpshow_dct[match_dct[match_keys[4]].group(1).rstrip()] = match_dct[match_keys[4]].group(2).rstrip()
# sfp_info_match
elif match_dct[match_keys[5]]:
sfpshow_dct[match_dct[match_keys[5]].group(1).rstrip()] = match_dct[match_keys[5]].group(2).rstrip()
if not line:
break
# additional values which need to be added to the dictionary with all DISCOVERED parameters during current loop iteration
# values axtracted in manual mode. if change values order change keys order in init.xlsx "chassis_params_add" column
sfpshow_port_values = [*switch_info_lst, slot_num, port_num]
# adding additional parameters and values to the sfpshow_dct
update_dct(params_add, sfpshow_port_values, sfpshow_dct)
# appending list with only REQUIRED port info for the current loop iteration to the list with all fabrics port info
sfpshow_lst.append([sfpshow_dct.get(param, None) for param in params])
# sfpshow section end
# portcfgshow section start
if re.search(r'^(SWITCHCMD )?(/fabos/cliexec/)?portcfgshow *: *$', line) and not collected['portcfgshow']:
collected['portcfgshow'] = True
if ls_mode_on:
while not re.search(fr'^CURRENT CONTEXT -- {switch_index} *, \d+$',line):
line = file.readline()
if not line:
break
while not re.search(r'^(real [\w.]+)|(\*\* SS CMD END \*\*)$|No ports found in switch',line):
line = file.readline()
match_dct ={match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip(comp_keys, match_keys)}
# 'slot_port_line_match'
if match_dct[match_keys[6]]:
# dictionary to store all DISCOVERED switch ports information
portcfgshow_tmp_dct = {}
# extract slot and port numbers
slot_num, port_nums_str = line_to_list(comp_dct[comp_keys[6]], line)
port_nums_lst = port_nums_str.split()
port_nums = len(port_nums_lst)
# list with switch and slot information
switch_info_slot_lst = switch_info_lst.copy()
switch_info_slot_lst.append(slot_num)
# adding switch and slot information for each port to dictionary
for portcfg_param, switch_info_value in zip(portcfg_params[:7], switch_info_slot_lst):
portcfgshow_tmp_dct[portcfg_param] = [switch_info_value for i in range(port_nums)]
# adding port numbers to dictionary
portcfgshow_tmp_dct[portcfg_params[7]] = port_nums_lst
while not re.match('\r?\n', line):
line = file.readline()
match_dct ={match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip(comp_keys, match_keys)}
# portcfg_match
if match_dct[match_keys[7]]:
# extract param name and values for each port and adding to dictionary
param_name, param_values_str = line_to_list(comp_dct[comp_keys[7]], line)
portcfgshow_tmp_dct[param_name] = param_values_str.split()
if not line:
break
# saving portcfg information of REQUIRED parameters from dictionary with DISCOVERED parameters
for portcfg_param in portcfg_params:
portcfgshow_dct[portcfg_param].extend(portcfgshow_tmp_dct.get(portcfg_param, [None for i in range(port_nums)]))
# portcfgshow section end
status_info('ok', max_title, len(info))
# after check all config files create list of lists from dictionary. each nested list contains portcfg information for one port
for portcfg_param in portcfg_params:
portcfgshow_lst.append(portcfgshow_dct.get(portcfg_param))
portcfgshow_lst = list(zip(*portcfgshow_lst))
# save extracted data to json file
save_data(report_data_lst, data_names, sfpshow_lst, portcfgshow_lst)
# verify if loaded data is empty after first iteration and replace information string with empty list
else:
sfpshow_lst, portcfgshow_lst = verify_data(report_data_lst, data_names, *data_lst)
return sfpshow_lst, portcfgshow_lst
def switch_params_analysis_main(fabricshow_ag_labels_df, chassis_params_df,
switch_params_df, maps_params_df,
blade_module_loc_df, ag_principal_df,
report_data_lst):
"""Main function to create aggregated switch parameters table and report tables"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = [
'report_columns_usage', 'switch_params_aggregated', 'Коммутаторы',
'Фабрика', 'Параметры_коммутаторов', 'Лицензии',
'Глобальные_параметры_фабрики'
]
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# unpacking DataFrames from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
report_columns_usage_dct, switch_params_aggregated_df, switches_report_df, fabric_report_df, \
switches_parameters_report_df, licenses_report_df, global_fabric_parameters_report_df = data_lst
# list of data to analyze from report_info table
analyzed_data_names = [
'chassis_parameters', 'switch_parameters', 'switchshow_ports',
'maps_parameters', 'blade_interconnect', 'fabric_labels'
]
# clean fabricshow DataFrame from unneccessary data
fabric_clean_df = fabric_clean(fabricshow_ag_labels_df)
# force run when any data from data_lst was not saved (file not found) or
# procedure execution explicitly requested for output data or data used during fn execution
force_run = verify_force_run(data_names, data_lst, report_steps_dct,
max_title, analyzed_data_names)
if force_run:
# import data with switch models, firmware and etc
switch_models_df = dataframe_import('switch_models', max_title)
# current operation information string
info = f'Generating aggregated switch parameters table'
print(info, end=" ")
# create aggregated table by joining DataFrames
switch_params_aggregated_df, report_columns_usage_dct = \
fabric_aggregation(fabric_clean_df, chassis_params_df, \
switch_params_df, maps_params_df, switch_models_df, ag_principal_df)
# add 'Device_Location for Blade chassis switches
switch_params_aggregated_df = fill_device_location(
switch_params_aggregated_df, blade_module_loc_df)
# after finish display status
status_info('ok', max_title, len(info))
# check if switch config files missing
mask_fabric = switch_params_aggregated_df[[
'Fabric_name', 'Fabric_label'
]].notna().all(axis=1)
mask_no_config = switch_params_aggregated_df['chassis_name'].isna()
missing_configs_num = switch_params_aggregated_df.loc[mask_no_config][
'Fabric_name'].count()
if missing_configs_num:
info = f'{missing_configs_num} switch configuration{"s" if missing_configs_num > 1 else ""} MISSING'
print(info, end=" ")
status_info('warning', max_title, len(info))
switches_report_df, fabric_report_df, switches_parameters_report_df, \
licenses_report_df, global_fabric_parameters_report_df = \
switchs_params_report(switch_params_aggregated_df, data_names, report_columns_usage_dct, max_title)
# # partition aggregated DataFrame to required tables
# switches_report_df, fabric_report_df, \
# switches_parameters_report_df, licenses_report_df = \
# dataframe_segmentation(switch_params_aggregated_df, data_names[2:-1], \
# report_columns_usage_dct, max_title)
# # global parameters are equal for all switches in one fabric thus checking Principal switches only
# mask_principal = switch_params_aggregated_df['switchRole'] == 'Principal'
# switch_params_principal_df = switch_params_aggregated_df.loc[mask_principal].copy()
# global_fabric_parameters_report_df, = dataframe_segmentation(switch_params_principal_df, data_names[-1], \
# report_columns_usage_dct, max_title)
# # drop rows with empty switch names columns
# fabric_report_df.dropna(subset = ['Имя коммутатора'], inplace = True)
# switches_parameters_report_df.dropna(subset = ['Имя коммутатора'], inplace = True)
# licenses_report_df.dropna(subset = ['Имя коммутатора'], inplace = True)
# # drop fabric_id if all have same value
# if fabric_report_df['Fabric ID'].dropna().nunique() == 1:
# fabric_report_df.drop(columns=['Fabric ID'], inplace=True)
# # TO_REMOVE No need to drop duplicates coz Principal switches only used before
# # # parameters are equal for all switches in one fabric
# # if report_columns_usage_dct['fabric_name_usage']:
# # global_fabric_parameters_report_df.drop_duplicates(subset=['Фабрика', 'Подсеть'], inplace=True)
# # else:
# # global_fabric_parameters_report_df.drop_duplicates(subset=['Подсеть'], inplace=True)
# global_fabric_parameters_report_df.reset_index(inplace=True, drop=True)
# create list with partitioned DataFrames
data_lst = [
report_columns_usage_dct, switch_params_aggregated_df,
switches_report_df, fabric_report_df,
switches_parameters_report_df, licenses_report_df,
global_fabric_parameters_report_df
]
# saving data to json or csv file
save_data(report_data_lst, data_names, *data_lst)
# verify if loaded data is empty and replace information string with empty DataFrame
else:
report_columns_usage_dct, switch_params_aggregated_df, switches_report_df, fabric_report_df, \
switches_parameters_report_df, licenses_report_df, global_fabric_parameters_report_df = verify_data(report_data_lst, data_names, *data_lst)
data_lst = [
report_columns_usage_dct, switch_params_aggregated_df,
switches_report_df, fabric_report_df,
switches_parameters_report_df, licenses_report_df,
global_fabric_parameters_report_df
]
# save data to service file if it's required
for data_name, data_frame in zip(data_names[1:], data_lst[1:]):
save_xlsx_file(data_frame, data_name, report_data_lst)
return report_columns_usage_dct, switch_params_aggregated_df, fabric_clean_df
def errdump_main(errdump_df, switchshow_df, switch_params_aggregated_df,
portshow_aggregated_df, report_columns_usage_dct,
report_data_lst):
"""Main function to get most frequently appeared log messages"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = ['errdump_aggregated', 'raslog_counter', 'Журнал']
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# loading data if were saved on previous iterations
data_lst = load_data(report_data_lst, *data_names)
# unpacking DataFrames from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
errdump_aggregated_df, raslog_counter_df, raslog_report_df = data_lst
# list of data to analyze from report_info table
analyzed_data_names = [
'chassis_parameters', 'switch_parameters', 'switchshow_ports',
'maps_parameters', 'portshow_aggregated', 'fabric_labels'
]
# force run when any data from data_lst was not saved (file not found) or
# procedure execution explicitly requested for output data or data used during fn execution
force_run = verify_force_run(data_names, data_lst, report_steps_dct,
max_title, analyzed_data_names)
if force_run:
# data imported from init file (regular expression patterns) to extract values from data columns
# re_pattern list contains comp_keys, match_keys, comp_dct
_, _, *re_pattern_lst = data_extract_objects('raslog', max_title)
# current operation information string
info = f'Counting RASLog messages'
print(info, end=" ")
# get aggregated DataFrames
errdump_aggregated_df, raslog_counter_df, raslog_frequent_df = \
errdump_aggregated(errdump_df, switchshow_df, switch_params_aggregated_df, portshow_aggregated_df, re_pattern_lst)
# after finish display status
status_info('ok', max_title, len(info))
# partition aggregated DataFrame to required tables
raslog_report_df = raslog_report(raslog_frequent_df, data_names,
report_columns_usage_dct, max_title)
# create list with partitioned DataFrames
data_lst = [errdump_aggregated_df, raslog_counter_df, raslog_report_df]
# saving fabric_statistics and fabric_statistics_summary DataFrames to csv file
save_data(report_data_lst, data_names, *data_lst)
# verify if loaded data is empty and replace information string with empty DataFrame
else:
errdump_aggregated_df, raslog_counter_df, raslog_report_df = \
verify_data(report_data_lst, data_names, *data_lst)
data_lst = [errdump_aggregated_df, raslog_counter_df, raslog_report_df]
# save data to service file if it's required
for data_name, data_frame in zip(data_names, data_lst):
save_xlsx_file(data_frame, data_name, report_data_lst)
return errdump_aggregated_df, raslog_counter_df
def fcr_extract(switch_params_lst, report_data_lst):
"""Function to extract fabrics routing information"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = [
'fcrfabric', 'fcrproxydev', 'fcrphydev', 'lsan', 'fcredge',
'fcrresource'
]
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# unpacking from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
fcrfabric_lst, fcrproxydev_lst, fcrphydev_lst, lsan_lst, \
fcredge_lst, fcrresource_lst = data_lst
# data force extract check
# list of keys for each data from data_lst representing if it is required
# to re-collect or re-analyze data even they were obtained on previous iterations
force_extract_keys_lst = [
report_steps_dct[data_name][1] for data_name in data_names
]
# print data which were loaded but for which force extract flag is on
force_extract_check(data_names, data_lst, force_extract_keys_lst,
max_title)
# when any of data_lst was not saved or
# force extract flag is on then re-extract data from configueation files
if not all(data_lst) or any(force_extract_keys_lst):
print(
'\nEXTRACTING FABRICS ROUTING INFORMATION FROM SUPPORTSHOW CONFIGURATION FILES ...\n'
)
# extract switch parameters names from init file
switch_columns = columns_import('switch', max_title, 'columns')
# number of switches to check
switch_num = len(switch_params_lst)
# lists to store only REQUIRED switch parameters
# collecting data for all switches during looping
fcrfabric_lst = []
fcrproxydev_lst = []
fcrphydev_lst = []
lsan_lst = []
fcredge_lst = []
fcrresource_lst = []
# dictionary to collect fcr device data
# first element of list is regular expression pattern number,
# second - list to collect data, third is index in line to which slice extracted list
fcrdev_dct = {
'fcrproxydev': [5, fcrproxydev_lst, None],
'fcrphydev': [6, fcrphydev_lst, -3]
}
# data imported from init file to extract values from config file
params, _, comp_keys, match_keys, comp_dct = data_extract_objects(
'fcr', max_title)
# switch_params_lst [[switch_params_sw1], [switch_params_sw1]]
# checking each switch for switch level parameters
for i, switch_params_data in enumerate(switch_params_lst):
# dictionary with parameters for the current switch
switch_params_data_dct = dict(
zip(switch_columns, switch_params_data))
switch_info_keys = [
'configname', 'chassis_name', 'chassis_wwn', 'switch_index',
'SwitchName', 'switchWwn', 'switchRole', 'Fabric_ID',
'FC_Router'
]
switch_info_lst = [
switch_params_data_dct.get(key) for key in switch_info_keys
]
ls_mode_on = True if switch_params_data_dct[
'LS_mode'] == 'ON' else False
# data unpacking from iter param
sshow_file, *_, switch_name, _, switch_role, fid, fc_router = switch_info_lst
# current operation information string
info = f'[{i+1} of {switch_num}]: {switch_name} fabric routing. FC Routing: {fc_router}'
print(info, end=" ")
# search control dictionary. continue to check sshow_file until all parameters groups are found
collected = {'fcrfabric': False, 'fcrproxydev': False, 'fcrphydev': False,
'lsanzone': False, 'fcredge': False, 'fcrresource': False} \
if switch_role == 'Principal' else {'fcredge': False, 'fcrresource': False}
# check config of FC routers only
if fc_router == 'ON':
# fcrouter_info_lst contains sshow_file, chassis_name, switch_index, switch_name, switch_fid
fcrouter_info_lst = [*switch_info_lst[:6], switch_info_lst[7]]
with open(sshow_file, encoding='utf-8',
errors='ignore') as file:
# check file until all groups of parameters extracted
while not all(collected.values()):
line = file.readline()
if not line:
break
# check configs of Principal switches only
if switch_role == 'Principal':
# fcrfabricshow section start
# switchcmd_fcrfabricshow
if re.search(comp_dct[comp_keys[0]],
line) and not collected['fcrfabric']:
collected['fcrfabric'] = True
if ls_mode_on:
while not re.search(
fr'^BASE +SWITCH +CONTEXT *-- *FID: *{fid}$',
line):
line = file.readline()
if not line:
break
# switchcmd_end_comp
while not re.search(comp_dct[comp_keys[4]],
line):
line = file.readline()
# dictionary with match names as keys and match result of current line with all imported regular expressions as values
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# fc_router_match'
if match_dct[match_keys[1]]:
fcrouter_params_lst = line_to_list(
comp_dct[comp_keys[1]], line)
# check if line is empty
while not re.match('\r?\n', line):
line = file.readline()
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# fcr_info_match
if match_dct[match_keys[2]]:
fcrouter_name = match_dct[
match_keys[2]].group(1)
# fcr_exports_match
if match_dct[match_keys[3]]:
fcrfabric_lst.append(
line_to_list(
comp_dct[comp_keys[3]],
line,
*fcrouter_info_lst,
fcrouter_name,
*fcrouter_params_lst))
if not line:
break
if not line:
break
# fcrfabricshow section end
# fcrproxydev and fcrphydev checked in a loop over dictionary keys coz data representation is similar
# fcrdevshow section start
for fcrdev_type in fcrdev_dct.keys():
re_num, fcrdev_lst, slice_index = fcrdev_dct[
fcrdev_type]
# switchcmd_fcrproxydevshow_comp
if re.search(
comp_dct[comp_keys[re_num]],
line) and not collected[fcrdev_type]:
collected[fcrdev_type] = True
if ls_mode_on:
while not re.search(
fr'^BASE +SWITCH +CONTEXT *-- *FID: *{fid} *$',
line):
line = file.readline()
if not line:
break
# switchcmd_end_comp
while not re.search(
comp_dct[comp_keys[4]], line):
line = file.readline()
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# fcrdevshow_match
if match_dct[match_keys[7]]:
fcrdev_lst.append(
line_to_list(
comp_dct[comp_keys[7]],
line, *fcrouter_info_lst)
[:slice_index])
if not line:
break
# fcrdevshow section end
# lsanzoneshow section start
if re.search(comp_dct[comp_keys[8]],
line) and not collected['lsanzone']:
collected['lsanzone'] = True
if ls_mode_on:
while not re.search(
fr'^BASE +SWITCH +CONTEXT *-- *FID: *{fid} *$',
line):
line = file.readline()
if not line:
break
# switchcmd_end_comp
while not re.search(comp_dct[comp_keys[4]],
line):
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# lsan_name_match
if match_dct[match_keys[9]]:
# switch_info and current connected device wwnp
lsan_name = line_to_list(
comp_dct[comp_keys[9]], line)
# move cursor to one line down to get inside while loop
line = file.readline()
# lsan_switchcmd_end_comp
while not re.search(
comp_dct[comp_keys[11]], line):
# line = file.readline()
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# lsan_members_match
if match_dct[match_keys[10]]:
lsan_member = line_to_list(
comp_dct[comp_keys[10]],
line)
lsan_lst.append([
*fcrouter_info_lst,
*lsan_name, *lsan_member
])
# line = file.readline()
# else:
# line = file.readline()
line = file.readline()
if not line:
break
else:
line = file.readline()
if not line:
break
# lsanzoneshow section end
# fcredge and fcrresource checked for Principal and Subordinate routers
# fcredgeshow section start
if re.search(comp_dct[comp_keys[12]],
line) and not collected['fcredge']:
collected['fcredge'] = True
if ls_mode_on:
while not re.search(
fr'^BASE +SWITCH +CONTEXT *-- *FID: *{fid} *$',
line):
line = file.readline()
if not line:
break
# switchcmd_end_comp
while not re.search(comp_dct[comp_keys[4]], line):
line = file.readline()
match_dct = {
match_key: comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# fcredgeshow_match
if match_dct[match_keys[13]]:
# fcredge_lst.append(line_to_list(comp_dct[comp_keys[13]], line, *fcrouter_info_lst, switch_wwn))
fcredge_lst.append(
line_to_list(comp_dct[comp_keys[13]],
line, *fcrouter_info_lst))
if not line:
break
# fcredgeshow section end
# fcrresourceshow section start
if re.search(comp_dct[comp_keys[14]],
line) and not collected['fcrresource']:
collected['fcrresource'] = True
fcrresource_dct = {}
if ls_mode_on:
while not re.search(
fr'^BASE +SWITCH +CONTEXT *-- *FID: *{fid} *$',
line):
line = file.readline()
if not line:
break
# switchcmd_end_comp
while not re.search(comp_dct[comp_keys[4]], line):
line = file.readline()
match_dct = {
match_key: comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# fcredgeshow_match
if match_dct[match_keys[15]]:
fcrresource_dct[match_dct[match_keys[15]].group(1).rstrip()] = \
[match_dct[match_keys[15]].group(2), match_dct[match_keys[15]].group(3)]
if not line:
break
# each value of dictionary is list of two elements
# itertools.chain makes flat tmp_lst list from all lists in dictionary
tmp_lst = list(
itertools.chain(*[
fcrresource_dct.get(param
) if fcrresource_dct.
get(param) else [None, None]
for param in params
]))
fcrresource_lst.append(
[*fcrouter_info_lst, *tmp_lst])
# fcrresourceshow section end
status_info('ok', max_title, len(info))
else:
status_info('skip', max_title, len(info))
# save extracted data to json file
save_data(report_data_lst, data_names, fcrfabric_lst, fcrproxydev_lst,
fcrphydev_lst, lsan_lst, fcredge_lst, fcrresource_lst)
# verify if loaded data is empty after first iteration and replace information string with empty list
else:
fcrfabric_lst, fcrproxydev_lst, fcrphydev_lst, lsan_lst, fcredge_lst, fcrresource_lst = verify_data(
report_data_lst, data_names, *data_lst)
return fcrfabric_lst, fcrproxydev_lst, fcrphydev_lst, lsan_lst, fcredge_lst, fcrresource_lst
def synergy_system_extract(synergy_folder, report_data_lst):
"""Function to extract blade systems information"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = ['synergy_interconnect', 'synergy_servers']
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# # unpacking from the loaded list with data
# # pylint: disable=unbalanced-tuple-unpacking
# module_comprehensive_lst, blades_comprehensive_lst, blade_vc_comprehensive_lst = data_lst
# force run when any data from data_lst was not saved (file not found) or
# procedure execution explicitly requested for output data or data used during fn execution
force_run = verify_force_run(data_names, data_lst, report_steps_dct,
max_title)
if force_run:
# lists to store only REQUIRED infromation
# collecting data for all blades during looping
# list containing enclosure, blade and hba information for all blade systems
# list containing enclosure and interconnect modules information for all blade systems
synergy_module_columns = [
'Enclosure_Name', 'Enclosure_SN', 'Enclosure_Type',
'Interconnect_Bay', 'Interconnect_Model', 'Interconnect_SN',
'Interconnect_Firmware', 'Interconnect_Name', 'NodeName',
'Device_Location'
]
synergy_server_columns = [
'Enclosure_Name', 'Enclosure_Slot', 'Host_Name', 'name',
'serverprofilename', 'Device_Model', 'Device_SN', 'Host_OS',
'HBA_Description', 'Mezz_type', 'Connected_portWwn',
'Mezz_location', 'Device_Location', 'HBA_Firmware'
]
synergy_module_aggregated_df = pd.DataFrame(
columns=synergy_module_columns)
synergy_servers_aggregated_df = pd.DataFrame(
columns=synergy_server_columns)
if synergy_folder:
print('\nEXTRACTING SYNERGY SYSTEM INFORMATION ...\n')
# collects files in folder with xlsm extension
synergy_config_lst = find_files(synergy_folder,
max_title,
filename_extension='xlsm')
# number of files to check
configs_num = len(synergy_config_lst)
if configs_num:
# # data imported from init file to extract values from config file
# enclosure_params, _, comp_keys, match_keys, comp_dct = data_extract_objects('blades', max_title)
# module_params = columns_import('blades', max_title, 'module_params')
# blade_params = columns_import('blades', max_title, 'blade_params')
for i, synergy_config in enumerate(synergy_config_lst):
# file name with extension
configname_wext = os.path.basename(synergy_config)
# remove extension from filename
configname, _ = os.path.splitext(configname_wext)
# current operation information string
info = f'[{i+1} of {configs_num}]: {configname} system.'
print(info, end=" ")
syn_enclosure_df = pd.read_excel(synergy_config,
sheet_name='enclosures')
syn_module_df = pd.read_excel(
synergy_config, sheet_name='interconnectbays')
syn_server_hw_df = pd.read_excel(
synergy_config, sheet_name='server-hardware')
syn_server_fw_sw_df = pd.read_excel(
synergy_config, sheet_name='server-fw-sw')
syn_server_profile_connection_df = pd.read_excel(
synergy_config, sheet_name='server-prof-conn-details')
synergy_module_df = synergy_module(syn_enclosure_df,
syn_module_df)
if synergy_module_aggregated_df.empty:
synergy_module_aggregated_df = synergy_module_df
else:
synergy_module_aggregated_df = pd.concat(
[synergy_module_aggregated_df, synergy_module_df],
ignore_index=True)
synergy_server_wwn_df = synergy_server_wwn(
syn_server_hw_df)
synergy_profile_wwn_df = synergy_profile_wwn(
syn_server_profile_connection_df,
synergy_server_wwn_df)
# conctenate connection profile and server hardware
synergy_servers_df = pd.concat(
[synergy_server_wwn_df, synergy_profile_wwn_df],
ignore_index=True)
synergy_servers_df.drop_duplicates(inplace=True)
# add mezzanine firmware details
synergy_servers_df = synergy_mezz_fw(
syn_server_fw_sw_df, synergy_servers_df)
synergy_servers_df.sort_values(
by=['enclosurename', 'position', 'Mezz_WWPN'],
ignore_index=True,
inplace=True)
if synergy_servers_aggregated_df.empty:
synergy_servers_aggregated_df = synergy_servers_df
else:
synergy_servers_aggregated_df = pd.concat(
[
synergy_servers_aggregated_df,
synergy_servers_df
],
ignore_index=True)
if synergy_module_aggregated_df['switchbasewwn'].notna(
).any():
synergy_module_aggregated_df[
'switchbasewwn'] = synergy_module_aggregated_df[
'switchbasewwn'].str.lower()
if synergy_servers_aggregated_df['Mezz_WWPN'].notna().any(
):
synergy_servers_aggregated_df[
'Mezz_WWPN'] = synergy_servers_aggregated_df[
'Mezz_WWPN'].str.lower()
if not synergy_servers_df.empty or not synergy_module_df.empty:
status_info('ok', max_title, len(info))
else:
status_info('no data', max_title, len(info))
module_columns_dct = {
'enclosurename': 'Enclosure_Name',
'enclosure_serialnumber': 'Enclosure_SN',
'enclosuretype': 'Enclosure_Type',
'baynumber': 'Interconnect_Bay',
'interconnectmodel': 'Interconnect_Model',
'serialnumber': 'Interconnect_SN',
'switchfwversion': 'Interconnect_Firmware',
'hostname': 'Interconnect_Name',
'switchbasewwn': 'NodeName',
'device_location': 'Device_Location'
}
synergy_module_aggregated_df.rename(columns=module_columns_dct,
inplace=True)
synergy_module_aggregated_df.replace(r'^None$|^none$|^ *$',
value=np.nan,
regex=True,
inplace=True)
server_columns_dct = {
'enclosurename': 'Enclosure_Name',
'position': 'Enclosure_Slot',
'servername': 'Host_Name',
'model': 'Device_Model',
'serialnumber': 'Device_SN',
'oshint': 'Host_OS',
'Mezz': 'HBA_Description',
'Mezz_WWPN': 'Connected_portWwn',
'device_location': 'Device_Location',
'componentversion': 'HBA_Firmware'
}
synergy_servers_aggregated_df.rename(
columns=server_columns_dct, inplace=True)
synergy_servers_aggregated_df.replace(r'^None$|^none$|^ *$',
value=np.nan,
regex=True,
inplace=True)
data_lst = [
synergy_module_aggregated_df, synergy_servers_aggregated_df
]
# save extracted data to json file
save_data(report_data_lst, data_names, *data_lst)
else:
# current operation information string
info = f'Collecting synergy details'
print(info, end=" ")
status_info('skip', max_title, len(info))
data_lst = [
synergy_module_aggregated_df, synergy_servers_aggregated_df
]
# save empty data to json file
save_data(report_data_lst, data_names, *data_lst)
# verify if loaded data is empty after first iteration and replace information string with empty list
else:
synergy_module_aggregated_df, synergy_servers_aggregated_df = verify_data(
report_data_lst, data_names, *data_lst)
data_lst = [
synergy_module_aggregated_df, synergy_servers_aggregated_df
]
# save data to service file if it's required
for data_name, data_frame in zip(data_names, data_lst):
save_xlsx_file(data_frame, data_name, report_data_lst)
return synergy_module_aggregated_df, synergy_servers_aggregated_df
def warning_notification(portshow_aggregated_df, switch_params_aggregated_df,
nsshow_unsplit_df, expected_ag_links_df,
report_data_lst):
"""Function to show WARNING notification if any deviceType is UNKNOWN,
if any PortSymb or NodeSymb was not parsed or if new switch founded which was
not previously discovered"""
*_, max_title, report_steps_dct = report_data_lst
portshow_force_flag = False
nsshow_unsplit_force_flag = False
expected_ag_links_force_flag = False
portshow_export_flag, *_ = report_steps_dct['portshow_aggregated']
nsshow_unsplit_export_flag, *_ = report_steps_dct['nsshow_unsplit']
expected_ag_links_export_flag, *_ = report_steps_dct['expected_ag_links']
# warning if UKNOWN device class present
if (portshow_aggregated_df['deviceType'] == 'UNKNOWN').any():
unknown_count = len(portshow_aggregated_df[
portshow_aggregated_df['deviceType'] == 'UNKNOWN'])
info = f'{unknown_count} {"port" if unknown_count == 1 else "ports"} with UNKNOWN device Class found'
print(info, end=" ")
status_info('warning', max_title, len(info))
# ask if save portshow_aggregated_df
if not portshow_export_flag:
reply = reply_request(
"Do you want to save 'portshow_aggregated'? (y)es/(n)o: ")
if reply == 'y':
portshow_force_flag = True
# warning if any values in PortSymb or NodeSymb were not parsed
if not nsshow_unsplit_df.empty:
portsymb_unsplit_count = nsshow_unsplit_df['PortSymb'].notna().sum()
nodesymb_unsplit_count = nsshow_unsplit_df['NodeSymb'].notna().sum()
unsplit_lst = [[portsymb_unsplit_count, 'PortSymb'],
[nodesymb_unsplit_count, 'NodeSymb']]
unsplit_str = ' and '.join(
[str(num) + " " + name for num, name in unsplit_lst if num])
info = f'{unsplit_str} {"is" if portsymb_unsplit_count + nodesymb_unsplit_count == 1 else "are"} UNPARSED'
print(info, end=" ")
status_info('warning', max_title, len(info))
# ask if save nsshow_unsplit
if not nsshow_unsplit_export_flag:
reply = reply_request(
"Do you want to save 'nsshow_unsplit'? (y)es/(n)o: ")
if reply == 'y':
nsshow_unsplit_force_flag = True
# warning if unknown switches was found
switch_name_set = set(switch_params_aggregated_df['switchName'])
# all founded switches in portshow_aggregated_df
mask_switch = portshow_aggregated_df['deviceType'] == 'SWITCH'
portshow_switch_name_set = set(
portshow_aggregated_df.loc[mask_switch, 'Device_Host_Name'])
# if unknown switches found
if not portshow_switch_name_set.issubset(switch_name_set):
unknown_count = len(
portshow_switch_name_set.difference(switch_name_set))
info = f'{unknown_count} NEW {"switch" if unknown_count == 1 else "switches"} detected'
print(info, end=" ")
status_info('warning', max_title, len(info))
# ask if save portshow_aggregated_df
if not portshow_export_flag and not portshow_force_flag:
reply = reply_request(
"Do you want to save 'portshow_aggregated'? (y)es/(n)o: ")
if reply == 'y':
portshow_force_flag = True
# if any unconfirmed AG links found
if not expected_ag_links_df.empty:
unknown_count = expected_ag_links_df['chassis_name'].notna().sum()
info = f'{unknown_count} AG {"link" if unknown_count == 1 else "links"} detected'
print(info, end=" ")
status_info('warning', max_title, len(info))
# ask if save expected_ag_links_df
if not expected_ag_links_export_flag:
reply = reply_request(
"Do you want to save 'expected_ag_link'? (y)es/(n)o: ")
if reply == 'y':
expected_ag_links_force_flag = True
return portshow_force_flag, nsshow_unsplit_force_flag, expected_ag_links_force_flag
def err_sfp_cfg_analysis_main(portshow_aggregated_df, sfpshow_df,
portcfgshow_df, report_columns_usage_dct,
report_data_lst):
"""Main function to add porterr, transceiver and portcfg information to portshow DataFrame"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
portshow_sfp_force_flag = False
portshow_sfp_export_flag, *_ = report_steps_dct['portshow_sfp_aggregated']
# names to save data obtained after current module execution
data_names = [
'portshow_sfp_aggregated', 'Ошибки', 'Параметры_SFP',
'Параметры_портов'
]
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# unpacking DataFrames from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
portshow_sfp_aggregated_df, error_report_df, sfp_report_df, portcfg_report_df = data_lst
# list of data to analyze from report_info table
analyzed_data_names = [
'portshow_aggregated', 'sfpshow', 'portcfgshow', 'portcmd',
'switchshow_ports', 'switch_params_aggregated', 'fdmi',
'device_rename', 'report_columns_usage_upd', 'nscamshow', 'nsshow',
'alias', 'blade_servers', 'fabric_labels'
]
# force run when any data from data_lst was not saved (file not found) or
# procedure execution explicitly requested for output data or data used during fn execution
force_run = verify_force_run(data_names, data_lst, report_steps_dct,
max_title, analyzed_data_names)
if force_run:
# import transeivers information from file
sfp_model_df = dataframe_import('sfp_models', max_title)
# current operation information string
info = f'Updating connected devices table'
print(info, end=" ")
# add sfpshow, transceiver information and portcfg to aggregated portcmd DataFrame
portshow_sfp_aggregated_df = port_complete(portshow_aggregated_df,
sfpshow_df, sfp_model_df,
portcfgshow_df)
# after finish display status
status_info('ok', max_title, len(info))
# warning if UKNOWN SFP present
if (portshow_sfp_aggregated_df['Transceiver_Supported'] ==
'Unknown SFP').any():
info_columns = [
'Fabric_name', 'Fabric_label', 'configname', 'chassis_name',
'chassis_wwn', 'slot', 'port', 'Transceiver_Supported'
]
portshow_sfp_info_df = portshow_sfp_aggregated_df.drop_duplicates(
subset=info_columns).copy()
unknown_count = len(portshow_sfp_info_df[
portshow_sfp_info_df['Transceiver_Supported'] ==
'Unknown SFP'])
info = f'{unknown_count} {"port" if unknown_count == 1 else "ports"} with UNKNOWN supported SFP tag found'
print(info, end=" ")
status_info('warning', max_title, len(info))
# ask if save portshow_aggregated_df
if not portshow_sfp_export_flag:
reply = reply_request(
"Do you want to save 'portshow_sfp_aggregated'? (y)es/(n)o: "
)
if reply == 'y':
portshow_sfp_force_flag = True
# create reaport tables from port_complete_df DataFrtame
error_report_df, sfp_report_df, portcfg_report_df = \
create_report_tables(portshow_sfp_aggregated_df, data_names[1:], report_columns_usage_dct, max_title)
# saving data to json or csv file
data_lst = [
portshow_sfp_aggregated_df, error_report_df, sfp_report_df,
portcfg_report_df
]
save_data(report_data_lst, data_names, *data_lst)
# verify if loaded data is empty and reset DataFrame if yes
else:
portshow_sfp_aggregated_df, error_report_df, sfp_report_df, portcfg_report_df \
= verify_data(report_data_lst, data_names, *data_lst)
data_lst = [
portshow_sfp_aggregated_df, error_report_df, sfp_report_df,
portcfg_report_df
]
# save data to excel file if it's required
for data_name, data_frame in zip(data_names, data_lst):
force_flag = False
if data_name == 'portshow_sfp_aggregated':
force_flag = portshow_sfp_force_flag
save_xlsx_file(data_frame,
data_name,
report_data_lst,
force_flag=force_flag)
return portshow_sfp_aggregated_df
def storage_3par_extract(nsshow_df, nscamshow_df, local_3par_folder,
project_folder, report_data_lst):
"""Function to extract 3PAR storage information"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = ['system_3par', 'port_3par', 'host_3par']
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# unpacking from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
system_3par_comprehensive_lst, port_3par_comprehensive_lst, host_3par_comprehensive_lst = data_lst
# data force extract check
# list of keys for each data from data_lst representing if it is required
# to re-collect or re-analyze data even they were obtained on previous iterations
force_extract_keys_lst = [
report_steps_dct[data_name][1] for data_name in data_names
]
# print data which were loaded but for which force extract flag is on
force_extract_check(data_names, data_lst, force_extract_keys_lst,
max_title)
# when any of data_lst was not saved or
# force extract flag is on then re-extract data from configueation files
if not all(data_lst) or any(force_extract_keys_lst):
# lists to store only REQUIRED infromation
# collecting data for all systems during looping
# list containing system parameters
system_3par_comprehensive_lst = []
# list containing 3par FC port information
port_3par_comprehensive_lst = []
# list containing hosts defined on 3par ports
host_3par_comprehensive_lst = []
# data imported from init file to extract values from config file
params, params_add, comp_keys, match_keys, comp_dct = data_extract_objects(
'3par', max_title)
# verify if 3par systems registered in fabric NameServer
ns_3par_df = verify_ns_3par(nsshow_df, nscamshow_df, comp_dct)
if not ns_3par_df.empty:
print('\n')
print('3PAR Storage Systems detected in SAN')
print(ns_3par_df)
print('\n')
# find configuration files to parse (download from STATs, local folder or use configurations
# downloaded on previous iterations)
configs_3par_lst = configs_download(ns_3par_df, project_folder,
local_3par_folder, comp_keys,
match_keys, comp_dct,
report_data_lst)
if configs_3par_lst:
print('\nEXTRACTING 3PAR STORAGE INFORMATION ...\n')
# number of files to check
configs_num = len(configs_3par_lst)
for i, config_3par in enumerate(configs_3par_lst):
# file name
configname = os.path.basename(config_3par)
# current operation information string
info = f'[{i+1} of {configs_num}]: {configname} system'
print(info, end=" ")
showsys_lst, port_lst, host_lst = parse_config(
config_3par, params, params_add, comp_keys, match_keys,
comp_dct)
system_3par_comprehensive_lst.extend(showsys_lst)
port_3par_comprehensive_lst.extend(port_lst)
host_3par_comprehensive_lst.extend(host_lst)
if port_lst or host_lst:
status_info('ok', max_title, len(info))
else:
status_info('no data', max_title, len(info))
# save extracted data to json file
save_data(report_data_lst, data_names,
system_3par_comprehensive_lst,
port_3par_comprehensive_lst,
host_3par_comprehensive_lst)
else:
# current operation information string
info = f'Collecting 3PAR storage systems information'
print(info, end=" ")
status_info('skip', max_title, len(info))
# save empty data to json file
save_data(report_data_lst, data_names,
system_3par_comprehensive_lst,
port_3par_comprehensive_lst, host_3par_comprehensive_lst)
# verify if loaded data is empty after first iteration and replace information string with empty list
else:
system_3par_comprehensive_lst, port_3par_comprehensive_lst, host_3par_comprehensive_lst = verify_data(
report_data_lst, data_names, *data_lst)
return system_3par_comprehensive_lst, port_3par_comprehensive_lst, host_3par_comprehensive_lst
def portcmd_analysis_main(portshow_df, switchshow_ports_df, switch_params_df,
switch_params_aggregated_df, isl_aggregated_df,
nsshow_df, nscamshow_df, ag_principal_df,
porttrunkarea_df, alias_df, fdmi_df, blade_module_df,
blade_servers_df, blade_vc_df, synergy_module_df,
synergy_servers_df, system_3par_df, port_3par_df,
report_columns_usage_dct, report_data_lst):
"""Main function to add connected devices information to portshow DataFrame"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = [
'portshow_aggregated', 'storage_connection_statistics',
'device_connection_statistics', 'device_rename',
'report_columns_usage_upd', 'Серверы', 'Массивы', 'Библиотеки',
'Микрокоды_HBA', 'Подключение_массивов', 'Подключение_библиотек',
'Подключение_серверов', 'NPIV', 'Статистика_массивов',
'Статистика_устройств'
]
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
report_columns_usage_bckp = report_columns_usage_dct
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# flag to forcible save portshow_aggregated_df if required
portshow_force_flag = False
# unpacking DataFrames from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
portshow_aggregated_df, storage_connection_statistics_df, device_connection_statistics_df, \
device_rename_df, report_columns_usage_dct, \
servers_report_df, storage_report_df, library_report_df, hba_report_df, \
storage_connection_df, library_connection_df, server_connection_df, npiv_report_df, \
storage_connection_statistics_report_df, device_connection_statistics_report_df = data_lst
nsshow_unsplit_df = pd.DataFrame()
if not report_columns_usage_dct:
report_columns_usage_dct = report_columns_usage_bckp
# list of data to analyze from report_info table
analyzed_data_names = [
'portcmd', 'switchshow_ports', 'switch_params_aggregated',
'switch_parameters', 'chassis_parameters', 'fdmi', 'nscamshow',
'nsshow', 'alias', 'blade_servers', 'fabric_labels', 'isl', 'trunk',
'isl_aggregated', 'Параметры_SFP', 'portshow_sfp_aggregated'
]
# force run when any data from data_lst was not saved (file not found) or
# procedure execution explicitly requested for output data or data used during fn execution
force_run = verify_force_run(data_names, data_lst, report_steps_dct,
max_title, analyzed_data_names)
if force_run:
# import data with switch models, firmware and etc
switch_models_df = dataframe_import('switch_models', max_title)
# data imported from init file (regular expression patterns) to extract values from data columns
# re_pattern list contains comp_keys, match_keys, comp_dct
_, _, *re_pattern_lst = data_extract_objects('nameserver', max_title)
oui_df = dataframe_import('oui',
max_title,
columns=['Connected_oui', 'type', 'subtype'])
# current operation information string
info = f'Generating connected devices table'
print(info, end=" ")
portshow_aggregated_df, alias_wwnn_wwnp_df, nsshow_unsplit_df, expected_ag_links_df = \
portshow_aggregated(portshow_df, switchshow_ports_df, switch_params_df,
switch_params_aggregated_df, isl_aggregated_df, nsshow_df,
nscamshow_df, ag_principal_df, porttrunkarea_df, switch_models_df, alias_df,
oui_df, fdmi_df, blade_module_df, blade_servers_df, blade_vc_df,
synergy_module_df, synergy_servers_df, system_3par_df, port_3par_df,
re_pattern_lst, report_data_lst)
# after finish display status
status_info('ok', max_title, len(info))
# show warning if any UNKNOWN device class founded, if any PortSymb or NodeSymb is not parsed,
# if new switch founded
portshow_force_flag, nsshow_unsplit_force_flag, expected_ag_links_force_flag = \
warning_notification(portshow_aggregated_df, switch_params_aggregated_df,
nsshow_unsplit_df, expected_ag_links_df, report_data_lst)
# correct device names manually
portshow_aggregated_df, device_rename_df = \
devicename_correction_main(portshow_aggregated_df, device_rename_df,
report_columns_usage_dct, report_data_lst)
# count Device_Host_Name instances for fabric_label, label and total in fabric
portshow_aggregated_df = device_ports_per_group(portshow_aggregated_df)
# count device connection statistics
info = f'Counting device connection statistics'
print(info, end=" ")
storage_connection_statistics_df = storage_connection_statistics(
portshow_aggregated_df, re_pattern_lst)
device_connection_statistics_df = device_connection_statistics(
portshow_aggregated_df)
status_info('ok', max_title, len(info))
servers_report_df, storage_report_df, library_report_df, hba_report_df, \
storage_connection_df, library_connection_df, server_connection_df, npiv_report_df, \
storage_connection_statistics_report_df, device_connection_statistics_report_df = \
create_report_tables(portshow_aggregated_df, storage_connection_statistics_df,
device_connection_statistics_df, data_names[5:-2],
report_columns_usage_dct, max_title)
# create list with partitioned DataFrames
data_lst = [
portshow_aggregated_df, storage_connection_statistics_df,
device_connection_statistics_df, device_rename_df,
report_columns_usage_dct, servers_report_df, storage_report_df,
library_report_df, hba_report_df, storage_connection_df,
library_connection_df, server_connection_df, npiv_report_df,
storage_connection_statistics_report_df,
device_connection_statistics_report_df
]
# saving data to json or csv file
save_data(report_data_lst, data_names, *data_lst)
save_xlsx_file(nsshow_unsplit_df,
'nsshow_unsplit',
report_data_lst,
force_flag=nsshow_unsplit_force_flag)
save_xlsx_file(expected_ag_links_df,
'expected_ag_links',
report_data_lst,
force_flag=expected_ag_links_force_flag)
# verify if loaded data is empty and replace information string with empty DataFrame
else:
portshow_aggregated_df, storage_connection_statistics_df, device_connection_statistics_df, \
device_rename_df, report_columns_usage_dct, \
servers_report_df, storage_report_df, library_report_df, hba_report_df, \
storage_connection_df, library_connection_df, server_connection_df, npiv_report_df, \
storage_connection_statistics_report_df, device_connection_statistics_report_df \
= verify_data(report_data_lst, data_names, *data_lst)
data_lst = [
portshow_aggregated_df, storage_connection_statistics_df,
device_connection_statistics_df, device_rename_df,
report_columns_usage_dct, servers_report_df, storage_report_df,
library_report_df, hba_report_df, storage_connection_df,
library_connection_df, server_connection_df, npiv_report_df,
storage_connection_statistics_report_df,
device_connection_statistics_report_df
]
# save data to service file if it's required
for data_name, data_frame in zip(data_names, data_lst):
force_flag = False
if data_name == 'portshow_aggregated':
force_flag = portshow_force_flag
save_xlsx_file(data_frame,
data_name,
report_data_lst,
force_flag=force_flag)
return portshow_aggregated_df
def fabric_main(fabricshow_ag_labels_df, chassis_params_df, \
switch_params_df, maps_params_df, report_data_lst):
"""Main function to create tables"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = [
'Коммутаторы', 'Фабрика', 'Глобальные_параметры_фабрики',
'Параметры_коммутаторов', 'Лицензии'
]
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# unpacking DataFrames from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
switches_report_df, fabric_report_df, global_fabric_parameters_report_df, \
switches_parameters_report_df, licenses_report_df = data_lst
# data force extract check
# list of keys for each data from data_lst representing if it is required
# to re-collect or re-analyze data even they were obtained on previous iterations
force_extract_keys_lst = [
report_steps_dct[data_name][1] for data_name in data_names
]
# list with True (if data loaded) and/or False (if data was not found and None returned)
data_check = force_extract_check(data_names, data_lst,
force_extract_keys_lst, max_title)
# flag if fabrics labels was forced to be changed
fabric_labels_change = True if report_steps_dct['fabric_labels'][
1] else False
# initialization chassis information and farbric name columns usage
report_columns_usage_dct = {
'fabric_name_usage': True,
'chassis_info_usage': True
}
# import data with switch models, firmware and etc
switch_models_df = dataframe_import('switch_models', max_title)
# clean fabricshow DataFrame from unneccessary data
fabric_clean_df = fabric_clean(fabricshow_ag_labels_df)
# create aggregated table by joining DataFrames
switch_params_aggregated_df, report_columns_usage_dct = \
fabric_aggregation(fabric_clean_df, chassis_params_df, \
switch_params_df, maps_params_df, switch_models_df)
save_xlsx_file(switch_params_aggregated_df, 'switch_params_aggregated', \
report_data_lst, report_type = 'analysis')
# when no data saved or force extract flag is on or fabric labels have been changed than
# analyze extracted config data
if not all(data_check) or any(
force_extract_keys_lst) or fabric_labels_change:
# information string if fabric labels force changed was initiated
# and statistics recounting required
if fabric_labels_change and not any(force_extract_keys_lst) and all(
data_check):
info = f'Switch information force extract due to change in Fabrics labeling'
print(info, end=" ")
status_info('ok', max_title, len(info))
# partition aggregated DataFrame to required tables
switches_report_df, fabric_report_df, global_fabric_parameters_report_df, \
switches_parameters_report_df, licenses_report_df = \
dataframe_segmentation(switch_params_aggregated_df, data_names, \
report_columns_usage_dct, max_title)
# drop rows with empty switch names columns
fabric_report_df.dropna(subset=['Имя коммутатора'], inplace=True)
switches_parameters_report_df.dropna(subset=['Имя коммутатора'],
inplace=True)
licenses_report_df.dropna(subset=['Имя коммутатора'], inplace=True)
# parameters are equal for all switches in one fabric
if report_columns_usage_dct['fabric_name_usage']:
global_fabric_parameters_report_df.drop_duplicates(
subset=['Фабрика', 'Подсеть'], inplace=True)
else:
global_fabric_parameters_report_df.drop_duplicates(
subset=['Подсеть'], inplace=True)
global_fabric_parameters_report_df.reset_index(inplace=True, drop=True)
# create list with partitioned DataFrames
data_lst = [switches_report_df, fabric_report_df, global_fabric_parameters_report_df, \
switches_parameters_report_df, licenses_report_df]
# current operation information string
info = f'Generating Fabric and Switches tables'
print(info, end=" ")
# after finish display status
status_info('ok', max_title, len(info))
# saving DataFrames to csv file
save_data(report_data_lst, data_names, *data_lst)
# save_data(report_data_lst, data_auxillary_names, *data_auxillary_lst)
# save data to service file if it's required
for data_name, data_frame in zip(data_names, data_lst):
save_xlsx_file(data_frame, data_name, report_data_lst)
return switch_params_aggregated_df, report_columns_usage_dct, fabric_clean_df
def connected_devices_extract(switch_params_lst, report_data_lst):
"""Function to extract connected devices information
(fdmi, nsshow, nscamshow)
"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = ['fdmi', 'nsshow', 'nscamshow']
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# unpacking from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
fdmi_lst, nsshow_lst, nscamshow_lst = data_lst
# data force extract check
# list of keys for each data from data_lst representing if it is required
# to re-collect or re-analyze data even they were obtained on previous iterations
force_extract_keys_lst = [
report_steps_dct[data_name][1] for data_name in data_names
]
# print data which were loaded but for which force extract flag is on
force_extract_check(data_names, data_lst, force_extract_keys_lst,
max_title)
# when any of data_lst was not saved or
# force extract flag is on then re-extract data from configueation files
if not all(data_lst) or any(force_extract_keys_lst):
print(
'\nEXTRACTING INFORMATION ABOUT CONNECTED DEVICES (FDMI, NSSHOW, NSCAMSHOW) ...\n'
)
# extract chassis parameters names from init file
switch_columns = columns_import('switch', max_title, 'columns')
# number of switches to check
switch_num = len(switch_params_lst)
# data imported from init file to extract values from config file
params, params_add, comp_keys, match_keys, comp_dct = data_extract_objects(
'connected_dev', max_title)
nsshow_params, nsshow_params_add = columns_import(
'connected_dev', max_title, 'nsshow_params', 'nsshow_params_add')
# lists to store only REQUIRED infromation
# collecting data for all switches ports during looping
fdmi_lst = []
# lists with local Name Server (NS) information
nsshow_lst = []
nscamshow_lst = []
# dictionary with required to collect nsshow data
# first element of list is regular expression pattern number, second - list to collect data
nsshow_dct = {
'nsshow': [5, nsshow_lst],
'nscamshow': [6, nscamshow_lst]
}
# switch_params_lst [[switch_params_sw1], [switch_params_sw1]]
# checking each switch for switch level parameters
for i, switch_params_data in enumerate(switch_params_lst):
# data unpacking from iter param
# dictionary with parameters for the current chassis
switch_params_data_dct = dict(
zip(switch_columns, switch_params_data))
switch_info_keys = [
'configname', 'chassis_name', 'chassis_wwn', 'switch_index',
'SwitchName', 'switchWwn', 'switchMode'
]
switch_info_lst = [
switch_params_data_dct.get(key) for key in switch_info_keys
]
ls_mode_on = True if switch_params_data_dct[
'LS_mode'] == 'ON' else False
sshow_file, *_, switch_index, switch_name, _, switch_mode = switch_info_lst
# current operation information string
info = f'[{i+1} of {switch_num}]: {switch_name} connected devices'
print(info, end=" ")
# search control dictionary. continue to check sshow_file until all parameters groups are found
# Name Server service started only in Native mode
collected = {'fdmi': False, 'nsshow': False, 'nscamshow': False} \
if switch_params_data_dct.get('switchMode') == 'Native' else {'fdmi': False}
with open(sshow_file, encoding='utf-8', errors='ignore') as file:
# check file until all groups of parameters extracted
while not all(collected.values()):
line = file.readline()
if not line:
break
# fdmi section start
# switchcmd_fdmishow_comp
if re.search(comp_dct[comp_keys[0]], line):
collected['fdmi'] = True
if ls_mode_on:
while not re.search(
fr'^CURRENT CONTEXT -- {switch_index} *, \d+$',
line):
line = file.readline()
if not line:
break
# local_database_comp
while not re.search(comp_dct[comp_keys[4]], line):
match_dct = {
match_key: comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# wwnp_match
if match_dct[match_keys[1]]:
# dictionary to store all DISCOVERED switch ports information
# collecting data only for the logical switch in current loop
fdmi_dct = {}
# switch_info and current connected device wwnp
switch_wwnp = line_to_list(
comp_dct[comp_keys[1]], line,
*switch_info_lst[:6])
# move cursor to one line down to get inside while loop
line = file.readline()
# wwnp_local_comp
while not re.search(comp_dct[comp_keys[3]],
line):
line = file.readline()
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# fdmi_port_match
if match_dct[match_keys[2]]:
fdmi_dct[match_dct[match_keys[
2]].group(1).rstrip()] = match_dct[
match_keys[2]].group(
2).rstrip()
if not line:
break
# adding additional parameters and values to the fdmi_dct
update_dct(params_add, switch_wwnp, fdmi_dct)
# appending list with only REQUIRED port info for the current loop iteration to the list with all fabrics port info
fdmi_lst.append([
fdmi_dct.get(param, None)
for param in params
])
else:
line = file.readline()
if not line:
break
# fdmi section end
# only switches in Native mode have Name Server service started
if switch_mode == 'Native':
# nsshow section start
for nsshow_type in nsshow_dct.keys():
# unpacking re number and list to save REQUIRED params
re_num, ns_lst = nsshow_dct[nsshow_type]
# switchcmd_nsshow_comp, switchcmd_nscamshow_comp
if re.search(comp_dct[comp_keys[re_num]], line):
collected[nsshow_type] = True
if ls_mode_on:
while not re.search(
fr'^CURRENT CONTEXT -- {switch_index} *, \d+$',
line):
line = file.readline()
if not line:
break
# switchcmd_end_comp
while not re.search(comp_dct[comp_keys[9]],
line):
# line = file.readline()
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# port_pid__match
if match_dct[match_keys[7]]:
# dictionary to store all DISCOVERED switch ports information
# collecting data only for the logical switch in current loop
nsshow_port_dct = {}
# switch_info and current connected device wwnp
switch_pid = line_to_list(
comp_dct[comp_keys[7]], line,
*switch_info_lst[:6])
# move cursor to one line down to get inside while loop
line = file.readline()
# pid_switchcmd_end_comp
while not re.search(
comp_dct[comp_keys[8]], line):
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# nsshow_port_match
if match_dct[match_keys[2]]:
nsshow_port_dct[match_dct[
match_keys[2]].group(
1
).rstrip()] = match_dct[
match_keys[2]].group(
2).rstrip()
line = file.readline()
if not line:
break
# adding additional parameters and values to the fdmi_dct
update_dct(nsshow_params_add,
switch_pid, nsshow_port_dct)
# appending list with only REQUIRED port info for the current loop iteration to the list with all fabrics port info
ns_lst.append([
nsshow_port_dct.get(
nsshow_param, None)
for nsshow_param in nsshow_params
])
else:
line = file.readline()
if not line:
break
# nsshow section end
status_info('ok', max_title, len(info))
# save extracted data to json file
save_data(report_data_lst, data_names, fdmi_lst, nsshow_lst,
nscamshow_lst)
# verify if loaded data is empty after first iteration and replace information string with empty list
else:
fdmi_lst, nsshow_lst, nscamshow_lst = verify_data(
report_data_lst, data_names, *data_lst)
return fdmi_lst, nsshow_lst, nscamshow_lst
def save_xlsx_file(data_frame,
sheet_title,
report_data_lst,
current_date=str(date.today()),
force_flag=False):
"""
Check if excel file exists, check if dataframe sheet is in file,
delete sheet with stored dataframe (if sheet tabs number > 1)
and save new dataframe
"""
customer_name, report_path, _, max_title, report_steps_dct = report_data_lst
options = {}
options['strings_to_formulas'] = False
options['strings_to_urls'] = False
# report_steps_dct for each data_name contains: export_to_excel flag,
# force_extract flag, report_type, step_info, data_description
if report_steps_dct.get(sheet_title):
export_flag, _, report_type, _, df_decription = report_steps_dct.get(
sheet_title)
else:
info = f'DataFrame {sheet_title}'
print(info, end=" ")
status_info('unknown', max_title, len(info))
export_flag, report_type = 1, 'unknown'
df_decription = '-'
# check DataFrame report type to save
if report_type == 'report':
report_mark = 'SAN_Assessment_Tables'
else:
report_mark = report_type
# current_date = str(date.today())
# construct excel filename
file_name = customer_name + '_' + report_mark + '_' + current_date + '.xlsx'
# information string
info = f'Exporting {sheet_title} table to {report_mark} file'
print(info, end=" ")
file_path = os.path.join(report_path, file_name)
# save DataFrame to excel file if export_to_excel trigger is ON
# and DataFrame is not empty
if (force_flag or export_flag) and not data_frame.empty:
# pd.ExcelWriter has only two file open modes
# if file doesn't exist it has be opened in "w" mode otherwise in "a"
if os.path.isfile(file_path):
file_mode = 'a'
# open existing excel file
workbook = openpyxl.load_workbook(file_path)
if sheet_title in workbook.sheetnames:
# after sheet removal excel file must contain at least one sheet
if len(workbook.sheetnames) != 1:
del workbook[sheet_title]
try:
workbook.save(file_path)
except PermissionError:
status_info('fail', max_title, len(info))
print('\nPermission denied. Close the file.\n')
sys.exit()
else:
file_mode = 'w'
else:
file_mode = 'w'
# if required export DataFrame to the new sheet
if file_mode:
try:
with pd.ExcelWriter(file_path, mode=file_mode,
options=options) as writer: # pylint: disable=abstract-class-instantiated
# table of content item generation start
# if file is new then create table of content
if file_mode == 'w':
# save current item with header
content_df = pd.DataFrame(
[[sheet_title, df_decription]],
columns=['Закладка', 'Название таблицы'])
content_df.to_excel(writer,
sheet_name='Содержание',
index=False)
writer.save()
# create hyperlink to wotksheet
workbook = openpyxl.load_workbook(file_path)
writer.book = workbook
writer.sheets = {
ws.title: ws
for ws in workbook.worksheets
}
startrow = workbook['Содержание'].max_row
cell_address = 'A' + str(startrow)
create_hyperlink(workbook['Содержание'],
cell_address,
sheet_name=sheet_title,
cell_ref='A1',
display_name=None)
# if file exist new table of content item is appended if it's not exist
elif file_mode == 'a':
workbook = openpyxl.load_workbook(file_path)
# check if item exist in menu
item_exist = []
for row in workbook.get_sheet_by_name(
'Содержание').values:
if any(value == sheet_title for value in row):
item_exist.append(True)
else:
item_exist.append(False)
# if not exist add item without header and create hyperlink
if not any(item_exist):
# it makes append item to the same worksheet instead of making new one
writer.book = workbook
writer.sheets = {
ws.title: ws
for ws in workbook.worksheets
}
# content item to add
content_df = pd.DataFrame(
[[sheet_title, df_decription]],
columns=['Закладка', 'Название таблицы'])
# find row and cell to add new item and hyperlink
startrow = writer.sheets['Содержание'].max_row
cell_address = 'A' + str(startrow + 1)
# add content item
content_df.to_excel(writer,
sheet_name='Содержание',
startrow=startrow,
index=False,
header=False)
# create hyperlink
create_hyperlink(workbook['Содержание'],
cell_address,
sheet_name=sheet_title,
cell_ref='A1',
display_name=None)
# table of content item generation finish
# DataFrame save to worksheet start
# check if DataFrame have MultiIndex
# reset index if True
if isinstance(data_frame.index, pd.MultiIndex):
data_frame_flat = data_frame.reset_index()
# keep index if False
else:
data_frame_flat = data_frame.copy()
# saving DataFrame with single Index
data_frame_flat.to_excel(writer,
sheet_name=sheet_title,
startrow=2,
index=False)
writer.save()
workbook = openpyxl.load_workbook(file_path)
writer.book = workbook
writer.sheets = {
ws.title: ws
for ws in workbook.worksheets
}
# add table title
workbook[sheet_title]['A1'] = df_decription
workbook[sheet_title][
'A1'].font = openpyxl.styles.fonts.Font(bold=True)
# add hyperlink to the Table of contents
create_hyperlink(workbook[sheet_title],
'A2',
'Содержание',
cell_ref='A2',
display_name='К содержанию')
workbook.save(file_path)
# DataFrame save to worksheet finish
except PermissionError:
status_info('fail', max_title, len(info))
print('\nPermission denied. Close the file.\n')
sys.exit()
else:
status_info('ok', max_title, len(info))
return file_path
else:
# if save key is on but DataFrame empty
if report_steps_dct[sheet_title][0] and data_frame.empty:
status_info('no data', max_title, len(info))
else:
status_info('skip', max_title, len(info))
return None
def chassis_params_extract(all_config_data, report_data_lst):
"""Function to extract chassis parameters"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = ['chassis_parameters']
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# unpacking from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
chassis_params_fabric_lst, = data_lst
# data force extract check
# list of keys for each data from data_lst representing if it is required
# to re-collect or re-analyze data even they were obtained on previous iterations
force_extract_keys_lst = [
report_steps_dct[data_name][1] for data_name in data_names
]
# print data which were loaded but for which force extract flag is on
force_extract_check(data_names, data_lst, force_extract_keys_lst,
max_title)
# when any of data_lst was not saved or
# force extract flag is on then re-extract data from configueation files
if not all(data_lst) or any(force_extract_keys_lst):
print(
'\nEXTRACTING CHASSIS PARAMETERS FROM SUPPORTSHOW CONFIGURATION FILES ...\n'
)
# number of switches to check
switch_num = len(all_config_data)
# list to store only REQUIRED chassis parameters
# collecting data for all chassis during looping
chassis_params_fabric_lst = []
# data imported from init file to extract values from config file
chassis_params, chassis_params_add, comp_keys, match_keys, comp_dct = data_extract_objects(
'chassis', max_title)
# all_confg_data format ([swtch_name, supportshow file, (ams_maps_log files, ...)])
# checking each config set(supportshow file) for chassis level parameters
for i, switch_config_data in enumerate(all_config_data):
# data unpacking from iter param
switch_name, sshow_file, ams_maps_file = switch_config_data
# search control dictionary. continue to check sshow_file until all parameters groups are found
collected = {
'configshow': False,
'uptime_cpu': False,
'flash': False,
'memory': False,
'dhcp': False,
'licenses': False,
'vf_id': False
}
# dictionary to store all DISCOVERED chassis parameters
# collecting data only for the chassis in current loop
chassis_params_dct = {}
# sets and list to store parameters which could be joined in one group
snmp_target_set = set()
syslog_set = set()
tz_lst = []
licenses = []
vf_id_set = set()
# current operation information string
info = f'[{i+1} of {switch_num}]: {switch_name} chassis parameters'
print(info, end=" ")
with open(sshow_file, encoding='utf-8', errors='ignore') as file:
# check file until all groups of parameters extracted
while not all(collected.values()):
line = file.readline()
if not line:
break
# configshow section start
if re.search(
r'^(/fabos/cliexec/|/bin/cat /var/log/)?configshow *-?(all)? *:$',
line):
# when section is found corresponding collected dict values changed to True
collected['configshow'] = True
while not re.search(
r'^(\[Chassis Configuration End\])|(real [\w.]+)|(\*\* SS CMD END \*\*)$',
line):
line = file.readline()
# dictionary with match names as keys and match result of current line with all imported regular expressions as values
match_dct = {
match_key: comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# match_keys ['chassis_param_match', 'snmp_target_match', 'syslog_match', 'tz_match', 'uptime_cpu_match', 'memory_match', 'flash_match']
# 'chassis_param_match'
if match_dct[match_keys[0]]:
chassis_params_dct[match_dct[
match_keys[0]].group(1).rstrip(
)] = match_dct[match_keys[0]].group(3)
# for snmp and syslog data addresses are added to the coreesponding sets to avoid duplicates
# 'snmp_target_match'
if match_dct[match_keys[1]] and match_dct[
match_keys[1]].group(2) != '0.0.0.0':
snmp_target_set.add(
match_dct[match_keys[1]].group(2))
# 'syslog_match'
if match_dct[match_keys[2]]:
syslog_set.add(
match_dct[match_keys[2]].group(2))
# for timezone extracted data added to the list for later concatenation
# 'tz_match'
if match_dct[match_keys[3]]:
tz_lst.append(
match_dct[match_keys[3]].group(2))
if not line:
break
# config section end
# uptime section start
elif re.search(r'^(/fabos/cliexec/)?uptime *:$', line):
collected['uptime_cpu'] = True
while not re.search(r'^real [\w.]+$', line):
line = file.readline()
match_dct = {
match_key: comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# 'uptime_cpu_match'
if match_dct[match_keys[4]]:
uptime = match_dct[match_keys[4]].group(1)
cpu_load = match_dct[match_keys[4]].group(2)
if not line:
break
# uptime section end
# memory section start
elif re.search(r'^(/bin/)?(cat\s+)?/proc/meminfo\s*:$',
line):
collected['memory'] = True
memory_dct = {}
while not re.search(r'^real [\w.]+$', line):
line = file.readline()
match_dct = {
match_key: comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# 'memory_match'
if match_dct[match_keys[5]]:
memory_dct[match_dct[match_keys[5]].group(
1)] = match_dct[match_keys[5]].group(2)
if not line:
break
# free_mem + buffers > 5% from total memory
# memory usage < 95%
memory = round((1 - (int(memory_dct['MemFree']) +
int(memory_dct['Buffers'])) /
int(memory_dct['MemTotal'])) * 100)
memory = str(memory)
# memory section end
# flash section start
elif re.search(r'^(/bin/)?df\s*:$', line):
collected['flash'] = True
while not re.search(r'^real [\w.]+$', line):
line = file.readline()
match_dct = {
match_key: comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# 'flash_match'
if match_dct[match_keys[6]]:
flash = match_dct[match_keys[6]].group(1)
if not line:
break
# flash section end
# ipaddrshow section start
if re.search(r'^(/fabos/link_bin/)?ipaddrshow *:$', line):
collected['dhcp'] = True
while not re.search(
r'^(real [\w.]+)|(\*\* SS CMD END \*\*)$',
line):
line = file.readline()
match_dct = {
match_key: comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# 'dhcp_match'
if match_dct[match_keys[7]]:
chassis_params_dct[match_dct[
match_keys[7]].group(1).rstrip(
)] = match_dct[match_keys[7]].group(2)
if not line:
break
# ipaddrshow section end
# licenses section start
if re.search(r'^(/fabos/cliexec/)?licenseshow *:$', line):
collected['licenses'] = True
while not re.search(
r'^(real [\w.]+)|(\*\* SS CMD END \*\*)$',
line):
line = file.readline()
match_dct = {
match_key: comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# 'licenses_match'
if match_dct[match_keys[8]]:
licenses.append(
match_dct[match_keys[8]].group(1))
elif re.match('^No licenses installed.$', line):
licenses = 'No licenses installed'
if not line:
break
# licenses section end
# LS indexes identification start
if re.search(r'Section *: +SSHOW_FABRIC', line):
collected['vf_id'] = True
while not re.search(
r'^(SWITCHCMD /fabos/cliexec/)?dom *:$|Non-VF',
line):
if re.search(r'CURRENT +CONTEXT +-- +(\d+) *, \d+',
line):
id = re.match(
r'CURRENT +CONTEXT +-- +(\d+) *, \d+',
line).group(1)
vf_id_set.add(id)
line = file.readline()
else:
line = file.readline()
if not line:
break
# LS indexes identification end
# additional values which need to be added to the chassis params dictionary
# chassis_params_add order (configname, ams_maps_log, chassis_name, snmp_server, syslog_server, timezone_h:m, uptime, cpu_average_load, memory_usage, flash_usage, licenses)
# values axtracted in manual mode. if change values order change keys order in init.xlsx "chassis_params_add" column
vf_id_lst = list(vf_id_set)
vf_id_lst.sort()
chassis_params_values = (sshow_file, ams_maps_file, switch_name,
vf_id_lst, snmp_target_set, syslog_set,
tz_lst, uptime, cpu_load, memory, flash,
licenses)
# adding additional parameters and values to the chassis_params_switch_dct
for chassis_param_add, chassis_param_value in zip(
chassis_params_add, chassis_params_values):
if chassis_param_value:
if not isinstance(chassis_param_value, str):
s = ':' if chassis_param_add == 'timezone_h:m' else ', '
chassis_param_value = f'{s}'.join(chassis_param_value)
chassis_params_dct[chassis_param_add] = chassis_param_value
# creating list with REQUIRED chassis parameters for the current switch
# if no value in the chassis_params_dct for the parameter then None is added
# and appending this list to the list of all switches chassis_params_fabric_lst
chassis_params_fabric_lst.append([
chassis_params_dct.get(chassis_param, None)
for chassis_param in chassis_params
])
status_info('ok', max_title, len(info))
# save extracted data to json file
save_data(report_data_lst, data_names, chassis_params_fabric_lst)
# verify if loaded data is empty after first iteration and replace information string with empty list
else:
chassis_params_fabric_lst = verify_data(report_data_lst, data_names,
*data_lst)
return chassis_params_fabric_lst
def interswitch_connection_extract(switch_params_lst, report_data_lst):
"""Function to extract interswitch connection information"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = ['isl', 'trunk', 'porttrunkarea']
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# unpacking from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
isl_lst, trunk_lst, porttrunkarea_lst = data_lst
# data force extract check
# list of keys for each data from data_lst representing if it is required
# to re-collect or re-analyze data even they were obtained on previous iterations
force_extract_keys_lst = [
report_steps_dct[data_name][1] for data_name in data_names
]
# print data which were loaded but for which force extract flag is on
force_extract_check(data_names, data_lst, force_extract_keys_lst,
max_title)
# when any of data_lst was not saved or
# force extract flag is on then re-extract data from configueation files
if not all(data_lst) or any(force_extract_keys_lst):
print(
'\nEXTRACTING INTERSWITCH CONNECTION INFORMATION (ISL, TRUNK, TRUNKAREA) ...\n'
)
# extract chassis parameters names from init file
switch_columns = columns_import('switch', max_title, 'columns')
# number of switches to check
switch_num = len(switch_params_lst)
# data imported from init file to extract values from config file
*_, comp_keys, match_keys, comp_dct = data_extract_objects(
'isl', max_title)
# lists to store only REQUIRED infromation
# collecting data for all switches ports during looping
isl_lst = []
trunk_lst = []
porttrunkarea_lst = []
# switch_params_lst [[switch_params_sw1], [switch_params_sw1]]
# checking each switch for switch level parameters
for i, switch_params_data in enumerate(switch_params_lst):
# data unpacking from iter param
# dictionary with parameters for the current switch
switch_params_data_dct = dict(
zip(switch_columns, switch_params_data))
switch_info_keys = [
'configname', 'chassis_name', 'chassis_wwn', 'switch_index',
'SwitchName', 'switchWwn', 'switchRole', 'Fabric_ID',
'FC_Router', 'switchMode'
]
switch_info_lst = [
switch_params_data_dct.get(key) for key in switch_info_keys
]
ls_mode_on = True if switch_params_data_dct[
'LS_mode'] == 'ON' else False
sshow_file, _, _, switch_index, switch_name, *_, switch_mode = switch_info_lst
# current operation information string
info = f'[{i+1} of {switch_num}]: {switch_name} isl, trunk and trunk area ports. Switch mode: {switch_mode}'
print(info, end=" ")
# search control dictionary. continue to check sshow_file until all parameters groups are found
collected = {'isl': False, 'trunk': False, 'trunkarea': False}
if switch_mode == 'Native':
with open(sshow_file, encoding='utf-8',
errors='ignore') as file:
# check file until all groups of parameters extracted
while not all(collected.values()):
line = file.readline()
if not line:
break
# isl section start
# switchcmd_islshow_comp
if re.search(comp_dct[comp_keys[0]],
line) and not collected['isl']:
collected['isl'] = True
if ls_mode_on:
while not re.search(
fr'^CURRENT CONTEXT -- {switch_index} *, \d+$',
line):
line = file.readline()
if not line:
break
# switchcmd_end_comp
while not re.search(comp_dct[comp_keys[2]], line):
line = file.readline()
match_dct = {
match_key: comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# islshow_match
if match_dct[match_keys[1]]:
isl_port = line_to_list(
comp_dct[comp_keys[1]], line,
*switch_info_lst[:-1])
# portcfg parameters
if isl_port[-1]:
isl_port[-1] = isl_port[-1].replace(
' ', ', ')
# appending list with only REQUIRED port info for the current loop iteration
# to the list with all ISL port info
isl_lst.append(isl_port)
if not line:
break
# isl section end
# trunk section start
# switchcmd_trunkshow_comp
if re.search(comp_dct[comp_keys[3]],
line) and not collected['trunk']:
collected['trunk'] = True
if ls_mode_on:
while not re.search(
fr'^CURRENT CONTEXT -- {switch_index} *, \d+$',
line):
line = file.readline()
if not line:
break
# switchcmd_end_comp
while not re.search(comp_dct[comp_keys[2]], line):
match_dct = {
match_key: comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# trunkshow_match
if match_dct[match_keys[4]]:
trunk_port = line_to_list(
comp_dct[comp_keys[4]], line,
*switch_info_lst[:-1])
# if trunk line has trunk number then remove ":" from trunk number
if trunk_port[9]:
trunk_port[9] = trunk_port[9].strip(
':')
trunk_num = trunk_port[9]
# if trunk line has no number then use number from previous line
else:
trunk_port[9] = trunk_num
# appending list with only REQUIRED trunk info for the current loop iteration
# to the list with all trunk port info
trunk_lst.append(trunk_port)
line = file.readline()
if not line:
break
# trunk section end
# porttrunkarea section start
# switchcmd_trunkarea_comp
if re.search(comp_dct[comp_keys[5]],
line) and not collected['trunkarea']:
collected['trunkarea'] = True
if ls_mode_on:
while not re.search(
fr'^CURRENT CONTEXT -- {switch_index} *, \d+$',
line):
line = file.readline()
if not line:
break
# switchcmd_end_comp
while not re.search(comp_dct[comp_keys[2]], line):
line = file.readline()
match_dct = {
match_key: comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# 'porttrunkarea_match'
if match_dct[match_keys[6]]:
porttrunkarea_port_lst = line_to_list(
comp_dct[comp_keys[6]], line,
*switch_info_lst[:6])
# for No_light ports port and slot numbers are '--'
if porttrunkarea_port_lst[11] == '--':
porttrunkarea_port_lst[10] = '--'
# if switch has no slots than slot number is 0
for idx in [6, 10]:
if not porttrunkarea_port_lst[idx]:
porttrunkarea_port_lst[idx] = str(
0)
porttrunkarea_lst.append(
porttrunkarea_port_lst)
if not line:
break
# porttrunkarea section end
status_info('ok', max_title, len(info))
# if switch in Access Gateway mode then skip
else:
status_info('skip', max_title, len(info))
# save extracted data to json file
save_data(report_data_lst, data_names, isl_lst, trunk_lst,
porttrunkarea_lst)
# verify if loaded data is empty after first iteration and replace information string with empty list
else:
isl_lst, trunk_lst, porttrunkarea_lst = verify_data(
report_data_lst, data_names, *data_lst)
return isl_lst, trunk_lst, porttrunkarea_lst
def fabricshow_extract(switch_params_lst, report_data_lst):
"""
Function to extract from principal switch configuration
list of switches in fabric including AG switches
"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = ['fabricshow', 'ag_principal']
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# unpacking from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
fabricshow_lst, ag_principal_lst = data_lst
# data force extract check
# list of keys for each data from data_lst representing if it is required
# to re-collect or re-analyze data even they were obtained on previous iterations
force_extract_keys_lst = [
report_steps_dct[data_name][1] for data_name in data_names
]
# print data which were loaded but for which force extract flag is on
force_extract_check(data_names, data_lst, force_extract_keys_lst,
max_title)
# when any of data_lst was not saved or
# force extract flag is on then re-extract data from configueation files
if not all(data_lst) or any(force_extract_keys_lst):
print(
'\nEXTRACTING FABRICS INFORMATION FROM SUPPORTSHOW CONFIGURATION FILES ...\n'
)
# extract switch parameters names from init file
switch_columns = columns_import('switch', max_title, 'columns')
# number of switches to check
switch_num = len(switch_params_lst)
# list to store only REQUIRED switch parameters
# collecting data for all switches during looping
fabricshow_lst = []
ag_principal_lst = []
# data imported from init file to extract values from config file
*_, comp_keys, match_keys, comp_dct = data_extract_objects(
'fabricshow', max_title)
ag_params = columns_import('fabricshow', max_title, 'ag_params')
# switch_params_lst [[switch_params_sw1], [switch_params_sw1]]
# checking each switch for switch level parameters
for i, switch_params_data in enumerate(switch_params_lst):
# data unpacking from iter param
# dictionary with parameters for the current switch
switch_params_data_dct = dict(
zip(switch_columns, switch_params_data))
switch_info_keys = [
'configname', 'chassis_name', 'chassis_wwn', 'switch_index',
'SwitchName', 'switchWwn', 'switchRole', 'Fabric_ID',
'FC_Router', 'switchMode'
]
switch_info_lst = [
switch_params_data_dct[key] for key in switch_info_keys
]
ls_mode_on = True if switch_params_data_dct[
'LS_mode'] == 'ON' else False
sshow_file, _, _, switch_index, switch_name, _, switch_role = switch_info_lst[:
7]
# current operation information string
info = f'[{i+1} of {switch_num}]: {switch_name} fabric environment. Switch role: {switch_role}'
print(info, end=" ")
collected = {'fabricshow': False, 'ag_principal': False}
# switch_params_data_dct.get('FC_Router') == 'ON'
# print('collected', collected)
# check config of Principal switch only
if switch_role == 'Principal':
# principal_switch_lst contains sshow_file, chassis_name, chassis_wwn, switch_index, switch_name, switch_fid
principal_switch_lst = [
*switch_info_lst[:6], *switch_info_lst[7:9]
]
# search control dictionary. continue to check sshow_file until all parameters groups are found
with open(sshow_file, encoding='utf-8',
errors='ignore') as file:
# check file until all groups of parameters extracted
while not all(collected.values()):
line = file.readline()
if not line:
break
# fabricshow section start
if re.search(
r'^(SWITCHCMD /fabos/cliexec/)?fabricshow\s*:$',
line):
# when section is found corresponding collected dict values changed to True
collected['fabricshow'] = True
if ls_mode_on:
while not re.search(
fr'^CURRENT CONTEXT -- {switch_index} *, \d+$',
line):
line = file.readline()
if not line:
break
while not re.search(
r'^(real [\w.]+)|(\*\* SS CMD END \*\*)$',
line):
line = file.readline()
# dictionary with match names as keys and match result of current line with all imported regular expressions as values
match_dct = {
match_key: comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# match_keys ['fabricshow_match']
# 'fabricshow_match'
if match_dct[match_keys[0]]:
fabricshow_lst.append(
line_to_list(comp_dct[comp_keys[0]],
line,
*principal_switch_lst))
if not line:
break
# ag_principal section start
# switchcmd_agshow_comp
if re.search(comp_dct[comp_keys[4]], line):
collected['ag_principal'] = True
# if switch in LS mode switch to required LS number
if ls_mode_on:
while not re.search(
fr'^CURRENT CONTEXT -- {switch_index} *, \d+$',
line):
line = file.readline()
if not line:
break
# switchcmd_end_comp
while not re.search(comp_dct[comp_keys[10]], line):
match_dct = {
match_key: comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# ag_num_match
if match_dct[match_keys[5]]:
# dictionary to store all DISCOVERED switch ports information
# collecting data only for the logical switch in current loop
# Access Gateway common information dictionary
ag_info_dct = {}
# Attached F-Port information dictionary
ag_attach_dct = {}
# Access Gateway F-Port information dictionary
ag_fport_dct = {}
# Domaid ID, port_ID, port_index dictionary
did_port_dct = {}
# move cursor to one line down to get inside while loop
line = file.readline()
# ag_switchcmd_end_comp
while not re.search(
comp_dct[comp_keys[9]], line):
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# ag_info_match
if match_dct[match_keys[6]]:
ag_info_dct[match_dct[
match_keys[6]].group(
1).rstrip()] = match_dct[
match_keys[6]].group(
2).rstrip()
# ag_attached_match
elif match_dct[match_keys[7]]:
# if Attached F-Port information dictionary is empty than create dictionary with N-Port ID(s) as keys and empty lists as values
# if ag_attach_dct has been already created (not empty) then it's preserved
ag_attach_dct = ag_attach_dct or dict(
(n_portid, []) for n_portid in
ag_info_dct['N-Port ID(s)'].
split(','))
# extracting attached F-port data from line to list
ag_attach_lst = line_to_list(
comp_dct[comp_keys[7]], line)
# getting port_ID of N-port from port_id of F-port
n_portid = ag_attach_lst[
0][:-2] + '00'
# adding current line F-port information to Attached F-Port information dictionary
if n_portid in ag_attach_dct.keys(
):
ag_attach_dct[n_portid].append(
ag_attach_lst)
# ag_fport_match
elif match_dct[match_keys[8]]:
# create Access Gateway F-Port information dictionary
ag_fport_dct = ag_fport_dct or dict(
(n_portid, []) for n_portid in
ag_info_dct['N-Port ID(s)'].
split(','))
# extracting access gateway F-port data from line to list
ag_fport_lst = line_to_list(
comp_dct[comp_keys[8]], line)
# getting port_ID of N-port from port_id of F-port
n_portid = ag_fport_lst[
1][:-2] + '00'
# adding current line F-port information to Access Gateway F-Port information dictionary
if n_portid in ag_fport_dct.keys():
ag_fport_dct[n_portid].append(
ag_fport_lst)
line = file.readline()
if not line:
break
# list of N-ports extracted from N-Port ID(s) line
n_portids_lst = ag_info_dct[
'N-Port ID(s)'].split(',')
# (domain_id, n_portid)
did_port_lst = [
(int(n_portid[:4], 0), n_portid)
for n_portid in n_portids_lst
]
# creating dictionary with n_portid as keys and (domain_id, n_portid) as values
did_port_dct = {
port[1]: list(port)
for port in did_port_lst
}
# change values representation in dictionaries
# before {n_portid: [(port_id_1, port_wwn_1, f-port_num_1)], [(port_id_2, port_wwn_2, f-port_num_2)]}
# after {n_portid: [(port_id_1, port_id_2), (port_wwn_1, port_wwn_2), (f-port_num_1, f-port_num_1)]
ag_attach_dct = {
n_portid:
list(zip(*ag_attach_dct[n_portid]))
for n_portid in n_portids_lst
if ag_attach_dct.get(n_portid)
}
ag_fport_dct = {
n_portid:
list(zip(*ag_fport_dct[n_portid]))
for n_portid in n_portids_lst
if ag_fport_dct.get(n_portid)
}
# add connected switch port_index to did_port_dct extracted from ag_attach_dct
# (domain_id, n_portid, n_port_index)
# if no port_index then add None
for n_portid in n_portids_lst:
if ag_attach_dct.get(n_portid):
did_port_dct[n_portid].append(
ag_attach_dct[n_portid][2][0])
else:
did_port_dct[n_portid].append(None)
# for each element of list convert tuples to strings
# if no data extracted for the n_portid then add None for each parameter
for n_portid in n_portids_lst:
if ag_attach_dct.get(n_portid):
ag_attach_dct[n_portid] = [
', '.join(v) for v in
ag_attach_dct[n_portid]
]
else:
ag_attach_dct[n_portid] = [None
] * 3
for n_portid in n_portids_lst:
if ag_fport_dct.get(n_portid):
ag_fport_dct[n_portid] = [
', '.join(v)
for v in ag_fport_dct[n_portid]
]
else:
ag_fport_dct[n_portid] = [None] * 3
# getting data from ag_info_dct in required order
ag_info_lst = [
ag_info_dct.get(param, None)
for param in ag_params
]
# appending list with only REQUIRED ag info for the current loop iteration to the list with all ag switch info
for n_portid in n_portids_lst:
ag_principal_lst.append([
*principal_switch_lst[:-1],
*ag_info_lst,
*did_port_dct[n_portid],
*ag_attach_dct[n_portid],
*ag_fport_dct[n_portid]
])
else:
line = file.readline()
if not line:
break
# ag_principal section end
status_info('ok', max_title, len(info))
else:
status_info('skip', max_title, len(info))
# save extracted data to json file
save_data(report_data_lst, data_names, fabricshow_lst,
ag_principal_lst)
else:
fabricshow_lst, ag_principal_lst = verify_data(report_data_lst,
data_names, *data_lst)
return fabricshow_lst, ag_principal_lst
def blade_system_extract(blade_folder, report_data_lst):
"""Function to extract blade systems information"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = ['blade_interconnect', 'blade_servers', 'blade_vc']
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# unpacking from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
module_comprehensive_lst, blades_comprehensive_lst, blade_vc_comprehensive_lst = data_lst
# data force extract check
# list of keys for each data from data_lst representing if it is required
# to re-collect or re-analyze data even they were obtained on previous iterations
force_extract_keys_lst = [
report_steps_dct[data_name][1] for data_name in data_names
]
# print data which were loaded but for which force extract flag is on
force_extract_check(data_names, data_lst, force_extract_keys_lst,
max_title)
# when any of data_lst was not saved or
# force extract flag is on then re-extract data from configueation files
if not all(data_lst) or any(force_extract_keys_lst):
# lists to store only REQUIRED infromation
# collecting data for all blades during looping
# list containing enclosure, blade and hba information for all blade systems
blades_comprehensive_lst = []
# list containing enclosure and interconnect modules information for all blade systems
module_comprehensive_lst = []
# list containing virtual connect ports information for all blade systems
blade_vc_comprehensive_lst = []
if blade_folder:
print('\nEXTRACTING BLADES SYSTEM INFORMATION ...\n')
# collects files in folder with txt extension
txt_files = find_files(blade_folder,
max_title,
filename_extension='txt')
log_files = find_files(blade_folder,
max_title,
filename_extension='log')
blade_configs_lst = txt_files + log_files
# number of files to check
configs_num = len(blade_configs_lst)
if configs_num:
# data imported from init file to extract values from config file
enclosure_params, _, comp_keys, match_keys, comp_dct = data_extract_objects(
'blades', max_title)
module_params = columns_import('blades', max_title,
'module_params')
blade_params = columns_import('blades', max_title,
'blade_params')
for i, blade_config in enumerate(blade_configs_lst):
# file name with extension
configname_wext = os.path.basename(blade_config)
# remove extension from filename
configname, _ = os.path.splitext(configname_wext)
# Active Onboard Administrator IP address
oa_ip = None
# interconnect modules number
module_num = 0
# current operation information string
info = f'[{i+1} of {configs_num}]: {configname} system.'
print(info, end=" ")
# search control dictionary. continue to check file until all parameters groups are found
collected = {
'enclosure': False,
'oa_ip': False,
'module': False,
'servers': False,
'vc': False
}
# if blade_lst remains empty after file checkimng than status_info shows NO_DATA for current file
blade_lst = []
enclosure_vc_lst = []
with open(blade_config, encoding='utf-8',
errors='ignore') as file:
# check file until all groups of parameters extracted
while not all(collected.values()):
line = file.readline()
if not line:
break
# enclosure section start
if re.search(
r'>SHOW ENCLOSURE INFO|^ +ENCLOSURE INFORMATION$',
line):
enclosure_dct = {}
collected['enclosure'] = True
# while not reach empty line
while not re.search(r'Serial Number', line):
line = file.readline()
# dictionary with match names as keys and match result of current line with all imported regular expressions as values
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# name_value_pair_match
if match_dct[match_keys[0]]:
result = match_dct[match_keys[0]]
enclosure_dct[result.group(1).strip(
)] = result.group(2).strip()
if not line:
break
# rename Description key to Enclosure Type key for VC
if enclosure_dct.get('Description'):
enclosure_dct[
'Enclosure Type'] = enclosure_dct.pop(
'Description')
# creating list with REQUIRED enclosure information only
enclosure_lst = [
enclosure_dct.get(param)
for param in enclosure_params
]
# enclosure section end
# vc fabric connection section start
elif re.search(r'FABRIC INFORMATION', line):
info_type = 'Type VC'
print(info_type, end=" ")
info = info + " " + info_type
line = file.readline()
collected['vc'] = True
while not re.search(
r'FC-CONNECTION INFORMATION', line):
# dictionary with match names as keys and match result of current line with all imported regular expressions as values
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# vc_port_match
if match_dct[match_keys[14]]:
vc_port = line_to_list(
comp_dct[comp_keys[14]], line,
*enclosure_lst)
enclosure_vc_lst.append(vc_port)
blade_vc_comprehensive_lst.append(
vc_port)
line = file.readline()
else:
line = file.readline()
if not line:
break
# vc fabric connection section end
# active onboard administrator ip section start
elif re.search(r'>SHOW TOPOLOGY *$', line):
info_type = 'Type Blade Enclosure'
print(info_type, end=" ")
info = info + " " + info_type
line = file.readline()
collected['oa_ip'] = True
while not re.search(r'^>SHOW', line):
# dictionary with match names as keys and match result of current line with all imported regular expressions as values
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# oa_ip_match
if match_dct[match_keys[1]]:
oa_ip = match_dct[match_keys[1]].group(
1)
line = file.readline()
break
else:
line = file.readline()
if not line:
break
# active onboard administrator ip section end
# interconnect modules section start
elif re.search(r'>SHOW INTERCONNECT INFO ALL',
line):
line = file.readline()
collected['modules'] = True
while not re.search(r'^>SHOW', line):
# dictionary with match names as keys and match result of current line with all imported regular expressions as values
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# module_type_num_match
if match_dct[match_keys[2]]:
module_dct = {}
module_lst = []
module = match_dct[match_keys[2]]
# interconnect module slot number
module_slot = module.group(1)
# interconnect module type (Ethernet, FC)
module_type = module.group(2).rstrip()
line = file.readline()
# module_section_end_comp
while not re.search(
comp_dct[comp_keys[3]], line):
# dictionary with match names as keys and match result of current line with all imported regular expressions as values
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# name_value_pair_match
if match_dct[match_keys[0]]:
result = match_dct[
match_keys[0]]
name = result.group(1).strip()
value = result.group(2).strip()
# if value is empty string use None
if value == '':
value = None
module_dct[name] = value
line = file.readline()
else:
line = file.readline()
if not line:
break
# creating list with REQUIRED interconnect module information only
module_lst = [
module_dct.get(param)
for param in module_params
]
# add current module information to list containing all modules infromation
# oa_ip added as None and extracted later in the file
module_comprehensive_lst.append([
*enclosure_lst, oa_ip, module_slot,
module_type, *module_lst
])
# based on module's number oa_ip is added to module_comprehensive_lst after extraction
module_num += 1
else:
line = file.readline()
if not line:
break
# interconnect modules section end
# blade server, hba and flb section start
elif re.search(r'>SHOW SERVER INFO ALL', line):
line = file.readline()
collected['servers'] = True
while not re.search(r'^>SHOW', line):
# dictionary with match names as keys and match result of current line with all imported regular expressions as values
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# blade_server_num_match
if match_dct[match_keys[4]]:
blade_dct = {}
blade_lst = []
hba_lst = []
result = match_dct[match_keys[4]]
blade_dct[result.group(
1)] = result.group(2)
# blade_num = result.group(2)
# print("Blade number:", blade_num)
line = file.readline()
# server_section_end_comp
while not re.search(
comp_dct[comp_keys[11]], line):
# dictionary with match names as keys and match result of current line with all imported regular expressions as values
match_dct = {
match_key:
comp_dct[comp_key].match(line)
for comp_key, match_key in zip(
comp_keys, match_keys)
}
# mezzanin hba section start
# mezzanine_model_match
if match_dct[match_keys[6]]:
result = match_dct[
match_keys[6]]
hba_description = result.group(
1)
hba_model = result.group(2)
line = file.readline()
# mezzanine_wwn_comp
while re.search(
comp_dct[comp_keys[7]],
line):
# dictionary with match names as keys and match result of current line with all imported regular expressions as values
match_dct = {
match_key:
comp_dct[comp_key].
match(line)
for comp_key, match_key
in zip(
comp_keys,
match_keys)
}
# mezzanine_wwn_match
result = match_dct[
match_keys[7]]
wwnp = result.group(1)
hba_lst.append([
hba_description,
hba_model, wwnp
])
line = file.readline()
# mezzanin hba section end
# flex flb hba section start
# flb_model_match and flex_ethernet_match
elif match_dct[match_keys[
8]] or match_dct[
match_keys[15]]:
if match_dct[match_keys[8]]:
result = match_dct[
match_keys[8]]
flex_description = result.group(
1)
if re.search(
comp_dct[
comp_keys[13]],
line):
flex_model = re.search(
comp_dct[
comp_keys[13]],
line).group(1)
else:
flex_model = None
elif match_dct[match_keys[15]]:
result = match_dct[
match_keys[15]]
flex_description = result.group(
1)
flex_model = result.group(
1)
line = file.readline()
# wwn_mac_line_comp
while re.search(
comp_dct[comp_keys[9]],
line):
# dictionary with match names as keys and match result of current line with all imported regular expressions as values
match_dct = {
match_key:
comp_dct[comp_key].
match(line)
for comp_key, match_key
in zip(
comp_keys,
match_keys)
}
# flb_wwn_match
if match_dct[
match_keys[10]]:
result = match_dct[
match_keys[10]]
wwnp = result.group(1)
hba_lst.append([
flex_description,
flex_model, wwnp
])
line = file.readline()
# flex flb hba section end
# blade server section start
# blade_server_info_match
elif match_dct[match_keys[5]]:
result = match_dct[
match_keys[5]]
# name = result.group(1) + result.group(2) if result.group(2) else result.group(1)
name = result.group(1).rstrip()
value = result.group(
3).rstrip()
# to avoid Type parameter overwrire
# add parameter only if parameter has not been added to blade dictionary before
if not blade_dct.get(name):
blade_dct[name] = value
line = file.readline()
# blade server section end
# if none of matches found for current blade server than next line
else:
line = file.readline()
if not line:
break
# unpopulated blade slots have 'Server Blade Type' line but populated have 'Type' line
# add 'Server Blade Type' parameter for populated slots for consistency
if blade_dct.get('Type'):
blade_dct[
'Server Blade Type'] = blade_dct.pop(
'Type')
# creating list with REQUIRED blade server information only
blade_lst = [
blade_dct.get(param)
for param in blade_params
]
# if hba or flex cards installed in blade server
if len(hba_lst):
# add line for each hba to blades_comprehensive_lst
for hba in hba_lst:
blades_comprehensive_lst.append(
[
*enclosure_lst,
*blade_lst, *hba
])
# if no nba add one line with enclosure and blade info only
else:
blades_comprehensive_lst.append([
*enclosure_lst, *blade_lst,
None, None
])
# if no blade_server_num_match found in >SHOW SERVER INFO ALL section than next line
else:
line = file.readline()
if not line:
break
# blade server, hba and flb section end
# adding OA IP to module_comprehensive_lst based on interconnect modules number
for num in range(-1, -module_num - 1, -1):
module_comprehensive_lst[num][3] = oa_ip
# show status blades information extraction from file
if blade_lst or enclosure_vc_lst:
status_info('ok', max_title, len(info))
else:
status_info('no data', max_title, len(info))
# save extracted data to json file
save_data(report_data_lst, data_names,
module_comprehensive_lst, blades_comprehensive_lst,
blade_vc_comprehensive_lst)
else:
# current operation information string
info = f'Collecting enclosure, interconnect modules, blade servers, hba'
print(info, end=" ")
status_info('skip', max_title, len(info))
# save empty data to json file
save_data(report_data_lst, data_names, module_comprehensive_lst,
blades_comprehensive_lst, blade_vc_comprehensive_lst)
# verify if loaded data is empty after first iteration and replace information string with empty list
else:
module_comprehensive_lst, blades_comprehensive_lst, blade_vc_comprehensive_lst = verify_data(
report_data_lst, data_names, *data_lst)
return module_comprehensive_lst, blades_comprehensive_lst, blade_vc_comprehensive_lst
def zoning_analysis_main(switch_params_aggregated_df, portshow_aggregated_df,
cfg_df, zone_df, alias_df, cfg_effective_df,
fcrfabric_df, lsan_df, peerzone_df,
report_columns_usage_dct, report_data_lst):
"""Main function to analyze zoning configuration"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = [
'zoning_aggregated', 'alias_aggregated', 'zonemember_statistics',
'portshow_zoned_aggregated', 'alias_statistics',
'effective_cfg_statistics', 'Зонирование', 'Псевдонимы',
'Зонирование_A&B', 'Порты_не_в_зонах', 'Порты_без_псевдономов',
'Отсутсвуют_в_сети', 'Статистика_зон', 'Статистика_псевдонимов',
'Статистика_конфигурации'
]
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# unpacking DataFrames from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
zoning_aggregated_df, alias_aggregated_df, zonemember_statistics_df, \
portshow_zoned_aggregated_df, alias_statistics_df, effective_cfg_statistics_df, zoning_report_df, alias_report_df, \
zoning_compare_report_df, unzoned_device_report_df, no_alias_device_report_df, zoning_absent_device_report_df,\
zonemember_statistics_report_df, alias_statistics_report_df, effective_cfg_statistics_report_df = data_lst
# list of data to analyze from report_info table
analyzed_data_names = [
'cfg', 'cfg_effective', 'zone', 'alias', 'switch_params_aggregated',
'switch_parameters', 'switchshow_ports', 'chassis_parameters',
'portshow_aggregated', 'device_rename', 'report_columns_usage_upd',
'portcmd', 'fdmi', 'nscamshow', 'nsshow', 'blade_servers',
'fabric_labels'
]
# force run when any data from data_lst was not saved (file not found) or
# procedure execution explicitly requested for output data or data used during fn execution
force_run = verify_force_run(data_names, data_lst, report_steps_dct,
max_title, analyzed_data_names)
if force_run:
# current operation information string
info = f'Generating zoning table'
print(info, end=" ")
# aggregated DataFrames
zoning_aggregated_df, alias_aggregated_df \
= zoning_aggregated(switch_params_aggregated_df, portshow_aggregated_df,
cfg_df, zone_df, alias_df, cfg_effective_df, fcrfabric_df, lsan_df, peerzone_df, report_data_lst)
# create comprehesive statistics DataFrame with Fabric summaries and
# zones statistics DataFrame without summaries
zonemember_statistics_df, zonemember_zonelevel_stat_df = zonemember_statistics(
zoning_aggregated_df, report_data_lst)
# add zoning statistics notes, zone duplicates and zone pairs to zoning aggregated DataFrame
zoning_aggregated_df = statistics_to_aggregated_zoning(
zoning_aggregated_df, zonemember_zonelevel_stat_df)
# check all fabric devices (Wwnp) for usage in zoning configuration
portshow_zoned_aggregated_df = verify_cfg_type(portshow_aggregated_df,
zoning_aggregated_df,
['PortName'])
# create alias configuration statistics
alias_statistics_df = alias_dashboard(alias_aggregated_df,
portshow_zoned_aggregated_df)
# create Effective zoning configuration summary statistics
effective_cfg_statistics_df = cfg_dashborad(
zonemember_statistics_df, portshow_zoned_aggregated_df,
zoning_aggregated_df, alias_aggregated_df)
# after finish display status
status_info('ok', max_title, len(info))
# report tables
zoning_report_df, alias_report_df, zoning_compare_report_df, \
unzoned_device_report_df, no_alias_device_report_df, zoning_absent_device_report_df, \
zonemember_statistics_report_df, alias_statistics_report_df, effective_cfg_statistics_report_df = \
zoning_report_main(zoning_aggregated_df, alias_aggregated_df, portshow_zoned_aggregated_df,
zonemember_statistics_df, alias_statistics_df, effective_cfg_statistics_df,
data_names, report_columns_usage_dct, max_title)
# create list with partitioned DataFrames
data_lst = [
zoning_aggregated_df, alias_aggregated_df,
zonemember_statistics_df, portshow_zoned_aggregated_df,
alias_statistics_df, effective_cfg_statistics_df, zoning_report_df,
alias_report_df, zoning_compare_report_df,
unzoned_device_report_df, no_alias_device_report_df,
zoning_absent_device_report_df, zonemember_statistics_report_df,
alias_statistics_report_df, effective_cfg_statistics_report_df
]
# saving data to json or csv file
save_data(report_data_lst, data_names, *data_lst)
# verify if loaded data is empty and replace information string with empty DataFrame
else:
zoning_aggregated_df, alias_aggregated_df, zonemember_statistics_df, \
portshow_zoned_aggregated_df, alias_statistics_df, effective_cfg_statistics_df, zoning_report_df, alias_report_df, \
zoning_compare_report_df, unzoned_device_report_df, no_alias_device_report_df, zoning_absent_device_report_df, \
zonemember_statistics_report_df, alias_statistics_report_df, effective_cfg_statistics_report_df \
= verify_data(report_data_lst, data_names, *data_lst)
data_lst = [
zoning_aggregated_df, alias_aggregated_df,
zonemember_statistics_df, portshow_zoned_aggregated_df,
alias_statistics_df, effective_cfg_statistics_df, zoning_report_df,
alias_report_df, zoning_compare_report_df,
unzoned_device_report_df, no_alias_device_report_df,
zoning_absent_device_report_df, zonemember_statistics_report_df,
alias_statistics_report_df, effective_cfg_statistics_report_df
]
# save data to service file if it's required
for data_name, data_frame in zip(data_names, data_lst):
save_xlsx_file(data_frame, data_name, report_data_lst)
return zoning_aggregated_df, alias_aggregated_df, portshow_zoned_aggregated_df
def switch_params_configshow_extract(chassis_params_fabric_lst, report_data_lst):
"""Function to extract switch parameters"""
# report_data_lst contains information:
# customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct
*_, max_title, report_steps_dct = report_data_lst
# names to save data obtained after current module execution
data_names = ['switch_parameters', 'switchshow_ports']
# service step information
print(f'\n\n{report_steps_dct[data_names[0]][3]}\n')
# load data if they were saved on previos program execution iteration
data_lst = load_data(report_data_lst, *data_names)
# unpacking from the loaded list with data
# pylint: disable=unbalanced-tuple-unpacking
switch_params_lst, switchshow_ports_lst = data_lst
# data force extract check
# list of keys for each data from data_lst representing if it is required
# to re-collect or re-analyze data even they were obtained on previous iterations
force_extract_keys_lst = [report_steps_dct[data_name][1] for data_name in data_names]
# print data which were loaded but for which force extract flag is on
force_extract_check(data_names, data_lst, force_extract_keys_lst, max_title)
# when any of data_lst was not saved or
# force extract flag is on then re-extract data from configueation files
if not all(data_lst) or any(force_extract_keys_lst):
print('\nEXTRACTING SWITCH PARAMETERS FROM SUPPORTSHOW CONFIGURATION FILES ...\n')
# extract chassis parameters names from init file
chassis_columns = columns_import('chassis', max_title, 'columns')
# number of switches to check
switch_num = len(chassis_params_fabric_lst)
# list to store only REQUIRED switch parameters
# collecting data for all switches during looping
switch_params_lst = []
# list to store switch ports details
switchshow_ports_lst = []
# data imported from init file to extract values from config file
switch_params, params_add, comp_keys, match_keys, comp_dct = data_extract_objects('switch', max_title)
# chassis_params_fabric_lst [[chassis_params_sw1], [chassis_params_sw1]]
# checking each chassis for switch level parameters
for i, chassis_params_data in enumerate(chassis_params_fabric_lst):
# data unpacking from iter param
# dictionary with parameters for the current chassis
chassis_params_data_dct = dict(zip(chassis_columns, chassis_params_data))
sshow_file = chassis_params_data_dct['configname']
chassis_name = chassis_params_data_dct['chassis_name']
chassis_wwn = chassis_params_data_dct['chassis_wwn']
# num_ls = int(chassis_params_data_dct["Number_of_LS"]) if not chassis_params_data_dct["Number_of_LS"] in ['0', None] else 1
# when num of logical switches is 0 or None than mode is Non-VF otherwise VF
ls_mode_on = (True if not chassis_params_data_dct["Number_of_LS"] in ['0', None] else False)
ls_mode = ('ON' if not chassis_params_data_dct["Number_of_LS"] in ['0', None] else 'OFF')
# logical switches indexes. if switch is in Non-VF mode then ls_id is 0
ls_ids = chassis_params_data_dct['LS_IDs'].split(', ') if chassis_params_data_dct['LS_IDs'] else ['0']
# current operation information string
info = f'[{i+1} of {switch_num}]: {chassis_params_data_dct["chassis_name"]} switch parameters. Number of LS: {chassis_params_data_dct["Number_of_LS"]}'
print(info, end =" ")
# check each logical switch in chassis
for i in ls_ids:
# search control dictionary. continue to check sshow_file until all parameters groups are found
collected = {'configshow': False, 'switchshow': False}
# dictionary to store all DISCOVERED switch parameters
# collecting data only for the logical switch in current loop
switch_params_dct = {}
with open(sshow_file, encoding='utf-8', errors='ignore') as file:
# check file until all groups of parameters extracted
while not all(collected.values()):
line = file.readline()
if not line:
break
# configshow section start
if re.search(fr'^\[Switch Configuration Begin *: *{i}\]$', line) and not collected['configshow']:
# when section is found corresponding collected dict values changed to True
collected['configshow'] = True
while not re.search(fr'^\[Switch Configuration End : {i}\]$',line):
line = file.readline()
# dictionary with match names as keys and match result of current line with all imported regular expressions as values
match_dct ={match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip(comp_keys, match_keys)}
# match_keys ['switch_configall_match', 'switch_switchshow_match']
if match_dct[match_keys[0]]:
switch_params_dct[match_dct[match_keys[0]].group(1).rstrip()] = match_dct[match_keys[0]].group(3).rstrip()
if not line:
break
# config section end
# switchshow section start
if re.search(r'^(SWITCHCMD /fabos/bin/)?switchshow *:$', line) and not collected['switchshow']:
collected['switchshow'] = True
if ls_mode_on:
while not re.search(fr'^CURRENT CONTEXT -- {i} *, \d+$',line):
line = file.readline()
if not line:
break
while not re.search(r'^real [\w.]+$',line):
line = file.readline()
match_dct ={match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip(comp_keys, match_keys)}
# 'switch_switchshow_match'
if match_dct[match_keys[1]]:
switch_params_dct[match_dct[match_keys[1]].group(1).rstrip()] = match_dct[match_keys[1]].group(2).rstrip()
# 'ls_attr_match'
if match_dct[match_keys[2]]:
ls_attr = comp_dct[comp_keys[2]].findall(line)[0]
for k, v in zip(ls_attr[::2], ls_attr[1::2]):
switch_params_dct[k] = v
# 'switchshow_portinfo_match'
if match_dct[match_keys[3]]:
switchinfo_lst = [sshow_file, chassis_name, chassis_wwn, str(i),
switch_params_dct.get('switchName', None),
switch_params_dct.get('switchWwn', None),
switch_params_dct.get('switchState', None),
switch_params_dct.get('switchMode', None)
]
switchshow_port_lst = line_to_list(comp_dct[comp_keys[3]], line, *switchinfo_lst)
# if switch has no slots than slot number is 0
if not switchshow_port_lst[9]:
switchshow_port_lst[9] = str(0)
switchshow_ports_lst.append(switchshow_port_lst)
if not line:
break
# switchshow section end
# additional values which need to be added to the switch params dictionary
# switch_params_add order ('configname', 'chassis_name', 'switch_index', 'ls_mode')
# values axtracted in manual mode. if change values order change keys order in init.xlsx switch tab "params_add" column
switch_params_values = (sshow_file, chassis_name, chassis_wwn, str(i), ls_mode)
if switch_params_dct:
# adding additional parameters and values to the switch_params_switch_dct
update_dct(params_add, switch_params_values, switch_params_dct)
# creating list with REQUIRED chassis parameters for the current switch.
# if no value in the switch_params_dct for the parameter then None is added
# and appending this list to the list of all switches switch_params_fabric_lst
switch_params_lst.append([switch_params_dct.get(switch_param, None) for switch_param in switch_params])
status_info('ok', max_title, len(info))
# save extracted data to json file
save_data(report_data_lst, data_names, switch_params_lst, switchshow_ports_lst)
# verify if loaded data is empty after first iteration and replace information string with empty list
else:
switch_params_lst, switchshow_ports_lst = verify_data(report_data_lst, data_names, *data_lst)
return switch_params_lst, switchshow_ports_lst
