def sensor_analysis_main(sensor_df, switch_params_aggregated_df, report_columns_usage_dct, report_data_lst): """Main function to analyze zoning configuration""" # report_data_lst contains information: # customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct *_, max_title, report_steps_dct = report_data_lst # names to save data obtained after current module execution data_names = ['sensor_aggregated', 'Датчики'] # service step information print(f'\n\n{report_steps_dct[data_names[0]][3]}\n') # load data if they were saved on previos program execution iteration data_lst = load_data(report_data_lst, *data_names) # unpacking DataFrames from the loaded list with data # pylint: disable=unbalanced-tuple-unpacking sensor_aggregated_df, sensor_report_df = data_lst # list of data to analyze from report_info table analyzed_data_names = [ 'switch_params_aggregated', 'fabric_labels', 'sensor' ] # force run when any data from data_lst was not saved (file not found) or # procedure execution explicitly requested for output data or data used during fn execution force_run = verify_force_run(data_names, data_lst, report_steps_dct, max_title, analyzed_data_names) if force_run: # current operation information string info = f'Generating sensor readings table' print(info, end=" ") # aggregated DataFrames sensor_aggregated_df = sensor_aggregation(sensor_df, switch_params_aggregated_df) # after finish display status status_info('ok', max_title, len(info)) # report tables sensor_report_df = sensor_report(sensor_aggregated_df, data_names, report_columns_usage_dct, max_title) # create list with partitioned DataFrames data_lst = [sensor_aggregated_df, sensor_report_df] # saving data to json or csv file save_data(report_data_lst, data_names, *data_lst) # verify if loaded data is empty and replace information string with empty DataFrame else: sensor_aggregated_df, sensor_report_df = verify_data( report_data_lst, data_names, *data_lst) data_lst = [sensor_aggregated_df, sensor_report_df] # save data to service file if it's required for data_name, data_frame in zip(data_names, data_lst): save_xlsx_file(data_frame, data_name, report_data_lst) return sensor_aggregated_df
def find_files(folder, max_title, filename_contains='', filename_extension=''): """ Function to create list with files. Takes directory, regex_pattern to verify if filename contains that pattern (default empty string) and filename extension (default is empty string) as parameters. Returns list of files with the extension deteceted in root folder defined as folder parameter and it's nested folders. If both parameters are default functions returns list of all files in directory """ info = f'Checking {os.path.basename(folder)} folder for configuration files' print(info, end=" ") # check if ssave_path folder exist check_valid_path(folder) # list to save configuration data files files_lst = [] # going through all directories inside ssave folder to find configuration data for root, _, files in os.walk(folder): for file in files: if file.endswith(filename_extension) and re.search( filename_contains, file): file_path = os.path.normpath(os.path.join(root, file)) files_lst.append(file_path) if len(files_lst) == 0: status_info('no data', max_title, len(info)) else: status_info('ok', max_title, len(info)) return files_lst
def local_download(ns_3par_df, configs_local_lst, download_folder, comp_keys, match_keys, comp_dct, max_title): """Function to copy 3PAR configuration files from local folder to download folder""" # verifu if download folder exist and create one if not (default behaviour) verify_download_folder(download_folder, max_title) if not 'STATs_status' in ns_3par_df.columns: ns_3par_df['STATs_status'] = np.nan drop_stats_column = True else: drop_stats_column = False ns_3par_df[['filename', 'Local_status']] = np.nan sn_lst = ns_3par_df['Serial_Number'].tolist() print('\n') # copy files from configs_local_lst to download folder for i, source_file in enumerate(configs_local_lst): filename = os.path.basename(source_file) if os.path.isfile(source_file) or os.path.islink(source_file): # extract model and seral number from config file model, sn = parse_serial(source_file, comp_keys, match_keys, comp_dct) info = ' ' * 16 + f'[{i+1} of {len(configs_local_lst)}]: Copying {filename} for {model} {sn}' print(info, end=' ') # verify if config for current sn was downloaded from STATs or local folder before mask_sn = ns_3par_df['Serial_Number'] == sn status_ok = (ns_3par_df.loc[mask_sn, ['STATs_status', 'Local_status']] == 'ok').any().any() # copy config if 3par is in NameServer and was not downloaded from STATs or local folder before if sn in sn_lst and not status_ok: # destination_file = os.path.join(download_folder, filename) status = 'unknown' try: shutil.copy2(source_file, download_folder) status = status_info('ok', max_title, len(info)) except shutil.Error as e: status = status_info('failed', max_title, len(info)) print('\n') print(e) finally: ns_3par_df.loc[mask_sn, ['filename', 'Local_status']] = [ filename, status.lower() ] else: status = status_info('skip', max_title, len(info)) else: info = ' ' * 16 + f'[{i+1} of {len(configs_local_lst)}]: copying {filename}' print(info, end=' ') status_info('skip', max_title, len(info)) if drop_stats_column: ns_3par_df.drop(columns=['STATs_status'], inplace=True) return ns_3par_df
def devicename_correction_main(portshow_aggregated_df, device_rename_df, report_columns_usage_dct, report_data_lst): """Main function to rename devices""" # report_data_lst contains information: # customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct *_, max_title, report_steps_dct = report_data_lst analyzed_data_names = [ 'device_rename', 'portshow_aggregated', 'portcmd', 'switchshow_ports', 'switch_params_aggregated', 'switch_parameters', 'chassis_parameters', 'fdmi', 'nscamshow', 'nsshow', 'alias', 'blade_servers', 'fabric_labels' ] # check if portshow_aggregated DataFrame is changed or 'device_rename' force flag is on force_form_update_flag = any( [report_steps_dct[data_name][1] for data_name in analyzed_data_names]) # list of related DataFrame names requested to change force_change_data_lst = [ data_name for data_name in analyzed_data_names[1:] if report_steps_dct[data_name][1] ] # flag to force change group name usage mode force_group_name_usage_update_flag = report_steps_dct[ 'report_columns_usage_upd'][1] # create DataFrame with devices required to change names device_rename_df = define_device_to_rename(portshow_aggregated_df, device_rename_df, max_title, force_form_update_flag, force_change_data_lst, report_data_lst) # current operation information string info = f'Applying device rename schema' print(info, end=" ") # if at least one name is changed apply rename schema if device_rename_df['Device_Host_Name_rename'].notna().any(): portshow_aggregated_df = device_rename(portshow_aggregated_df, device_rename_df) status_info('ok', max_title, len(info)) else: # to ask if group usage required force_group_name_usage_update_flag = 1 status_info('skip', max_title, len(info)) # check if device Group_Name should be used in report tables (alias group name) group_name_usage(report_columns_usage_dct, device_rename_df, max_title, force_group_name_usage_update_flag) return portshow_aggregated_df, device_rename_df
def load_data(report_data_list, *args): """Function to load data from JSON or CSV file to data object Detects wich type of data required to be loaded automaticaly Returns list with imported data """ customer_name, _, json_data_dir, max_title, _ = report_data_list # list to store loaded data data_imported = [] # check real file path for data_name in args: # constructing filenames for json and csv files file_name_json = customer_name + '_' + data_name + '.json' file_path_json = os.path.join(json_data_dir, file_name_json) file_name_csv = customer_name + '_' + data_name + '.csv' file_path_csv = os.path.join(json_data_dir, file_name_csv) # flags file existance file_json = False file_csv = False # check if json file exist if os.path.isfile(file_path_json): file_path = file_path_json file_name = file_name_json file_json = True # check if csv file exist elif os.path.isfile(file_path_csv): file_name = file_name_csv file_path = file_path_csv file_csv = True # if no files exist then dislay info about no data availabilty # and add None to data_imported list else: info = f'Loading {data_name}' print(info, end=" ") status_info('no file', max_title, len(info)) data_imported.append(None) # when saved file founded if any([file_json, file_csv]): info = f'Loading {data_name} from {file_name} file' print(info, end=" ") # open file try: with open(file_path, 'r', encoding="utf-8") as file: # for json file use load method if file_json: data_imported.append(json.load(file)) # for csv file use read_csv method elif file_csv: data_imported.append(pd.read_csv(file, dtype='unicode')) # display file load status except: status_info('no data', max_title, len(info)) data_imported.append(None) else: status_info('ok', max_title, len(info)) return data_imported
def dataframe_import(sheet_title, max_title, init_file = 'san_automation_info.xlsx', columns = None, index_name = None, header = 0, display_status=True): """Function to import dataframe from exel file""" warnings.filterwarnings('ignore', category=UserWarning, module="openpyxl") init_file_base = os.path.basename(init_file) # file to store all required data to process configuratin files # init_file = 'san_automation_info.xlsx' if display_status: info = f'Importing {sheet_title} dataframe from {init_file_base} file' print(info, end = ' ') # try read data in excel try: dataframe = pd.read_excel(init_file, sheet_name=sheet_title, usecols=columns, index_col=index_name, header=header) # if file is not found except FileNotFoundError: if display_status: status_info('fail', max_title, len(info)) print(f'File not found. Check if file {init_file} exists.') sys.exit() # if sheet is not found except xlrd.biffh.XLRDError: if display_status: status_info('fail', max_title, len(info)) print(f'Sheet {sheet_title} not found in {init_file}. Check if it exists.') sys.exit() else: if display_status: status_info('ok', max_title, len(info)) return dataframe
def save_data(report_data_list, data_names, *args): """ Function to export extracted configuration data to JSON or CSV file depending on data passed """ customer_name, _, json_data_dir, max_title, _ = report_data_list # data_names it's a list of names for data passed as args for data_name, data_exported in zip(data_names, args): empty_data = False file_name = customer_name + '_' + data_name # adding file extenson depending from type of data saved # csv for DataFrame if isinstance(data_exported, pd.DataFrame): file_name += '.csv' # for all other types is json else: # create file name and path for json file file_name += '.json' # full file path file_path = os.path.join(json_data_dir, file_name) info = f'Saving {data_name} to {file_name} file' try: print(info, end=" ") with open(file_path, 'w', encoding="utf-8") as file: # savng data for DataFrame if isinstance(data_exported, pd.DataFrame): # check if DataFrame have MultiIndex # reset index if True due to MultiIndex is not saved if isinstance(data_exported.index, pd.MultiIndex): data_exported_flat = data_exported.reset_index() # keep indexing if False else: data_exported_flat = data_exported.copy() # when DataFrame is empty fill first row values # with information string if data_exported_flat.empty: empty_data = True if len(data_exported_flat.columns) == 0: data_exported_flat['EMPTY'] = np.nan data_exported_flat.loc[0] = 'NO DATA FOUND' # save single level Index DataFrame to csv data_exported_flat.to_csv(file, index=False) # for all other types else: # if data list is not empty if not len(data_exported): empty_data = True data_exported = 'NO DATA FOUND' json.dump(data_exported, file) # display writing data to file status except FileNotFoundError: status_info('fail', max_title, len(info)) else: if not empty_data: status_info('ok', max_title, len(info)) else: status_info('empty', max_title, len(info))
def group_name_usage(report_columns_usage_dct, device_rename_df, max_title, force_group_name_usage_update_flag): """ Function to determine if device alias Group_Name column required in report tables. Value (True or False) written to the dictonary so no need to return value. """ info = f'Device Group Name usage' # if all devices were renamed and force flag is off column is dropped from report tables if device_rename_df['Device_Host_Name_rename'].notna().all( ) and not force_group_name_usage_update_flag: report_columns_usage_dct['group_name_usage'] = False print(info, end=" ") status_info('off', max_title, len(info)) # if no information in report_columns_usage_dct or force flag is on or any device keep old name # ask user input elif report_columns_usage_dct.get('group_name_usage') is None or \ force_group_name_usage_update_flag or \ device_rename_df['Device_Host_Name_rename'].isna().any(): print('\n') reply = reply_request( 'Do you want to use Alias Group Device names? (y)es/(n)o: ') print('\n') if reply == 'y': report_columns_usage_dct['group_name_usage'] = True print(info, end=" ") status_info('on', max_title, len(info)) else: report_columns_usage_dct['group_name_usage'] = False print(info, end=" ") status_info('off', max_title, len(info))
def create_folder(path, max_title): """Function to create any folder with path""" info = f'Make directory {os.path.basename(path)}' print(info, end=' ') # if folder not exist create if not os.path.exists(path): try: os.makedirs(path) except OSError: status_info('fail', max_title, len(info)) print(f'Not possible create directory {path}.') print('Code execution finished') sys.exit() else: status_info('ok', max_title, len(info)) # otherwise print 'SKIP' status else: status_info('skip', max_title, len(info))
def remove_files(files_lst, max_title): """Function to remove files from the list""" # print('\n') for i, file in enumerate(files_lst): filename = os.path.basename(file) info = ' ' * 16 + f'[{i+1} of {len(files_lst)}]: Removing file {filename}' print(info, end=' ') # verify if file exist if os.path.isfile(file) or os.path.islink(file): try: os.remove(file) status_info('ok', max_title, len(info)) except OSError as e: status_info('failed', max_title, len(info)) print('\n') print(e.strerror) else: status_info('skip', max_title, len(info)) print('\n')
def columns_import(sheet_title, max_title, *args, init_file = 'san_automation_info.xlsx', display_status=True): """Function to import corresponding columns from init file. Can import several columns. """ # file to store all required data to process configuratin files # default init_file is 'san_automation_info.xlsx' # columns titles string to display imported columns list without parenthesis columns_str = "" for arg in args: columns_str += "'" + arg + "', " columns_str = columns_str.rstrip(", ") if display_status: info = f'Importing {columns_str} from {sheet_title} tab' print(info, end = ' ') # try read data in excel try: columns = pd.read_excel(init_file, sheet_name = sheet_title, usecols = args, squeeze=True) except FileNotFoundError: if display_status: status_info('fail', max_title, len(info)) print(f'File not found. Check if file {init_file} exist.') sys.exit() except ValueError: if display_status: status_info('fail', max_title, len(info)) print(f'Column(s) {columns_str} not found. Check if column exist in {sheet_title}.') sys.exit() else: # if number of columns to read > 1 than returns corresponding number of lists if len(args)>1: columns_names = [columns[arg].dropna().tolist() if not columns[arg].empty else None for arg in args] else: columns_names = columns.dropna().tolist() if display_status: status_info('ok', max_title, len(info)) return columns_names
def sensor_extract(chassis_params_lst, report_data_lst): """Function to extract sensor information""" # report_data_lst contains information: # customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct *_, max_title, report_steps_dct = report_data_lst # names to save data obtained after current module execution data_names = ['sensor'] # service step information print(f'\n\n{report_steps_dct[data_names[0]][3]}\n') # load data if they were saved on previos program execution iteration data_lst = load_data(report_data_lst, *data_names) # unpacking from the loaded list with data # pylint: disable=unbalanced-tuple-unpacking sensor_lst, = data_lst # data force extract check # list of keys for each data from data_lst representing if it is required # to re-collect or re-analyze data even they were obtained on previous iterations force_extract_keys_lst = [ report_steps_dct[data_name][1] for data_name in data_names ] # print data which were loaded but for which force extract flag is on force_extract_check(data_names, data_lst, force_extract_keys_lst, max_title) # when any of data_lst was not saved or # force extract flag is on then re-extract data from configueation files if not all(data_lst) or any(force_extract_keys_lst): print('\nEXTRACTING ENVIRONMENT DATA ...\n') # # extract chassis parameters names from init file # switch_columns = columns_import('switch', max_title, 'columns') # extract chassis parameters names from init file chassis_columns = columns_import('chassis', max_title, 'columns') # number of switches to check switch_num = len(chassis_params_lst) # data imported from init file to extract values from config file *_, comp_keys, match_keys, comp_dct = data_extract_objects( 'sensor', max_title) # lists to store only REQUIRED infromation # collecting data for all switches ports during looping sensor_lst = [] # chassis_params_fabric_lst [[chassis_params_sw1], [chassis_params_sw1]] # checking each chassis for switch level parameters for i, chassis_params_data in enumerate(chassis_params_lst): # data unpacking from iter param # dictionary with parameters for the current chassis chassis_params_data_dct = dict( zip(chassis_columns, chassis_params_data)) chassis_info_keys = ['configname', 'chassis_name', 'chassis_wwn'] chassis_info_lst = [ chassis_params_data_dct.get(key) for key in chassis_info_keys ] sshow_file, chassis_name, _ = chassis_info_lst # current operation information string info = f'[{i+1} of {switch_num}]: {chassis_name} sensor readings' print(info, end=" ") # search control dictionary. continue to check sshow_file until all parameters groups are found collected = {'sensor': False} with open(sshow_file, encoding='utf-8', errors='ignore') as file: # check file until all groups of parameters extracted while not all(collected.values()): line = file.readline() if not line: break # sensor section start # switchcmd_sensorshow_comp if re.search(comp_dct[comp_keys[0]], line) and not collected['sensor']: collected['sensor'] = True # switchcmd_end_comp while not re.search(comp_dct[comp_keys[2]], line): line = file.readline() match_dct = { match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip( comp_keys, match_keys) } # islshow_match if match_dct[match_keys[1]]: sensor_reading = line_to_list( comp_dct[comp_keys[1]], line, *chassis_info_lst) # appending list with only REQUIRED port info for the current loop iteration # to the list with all ISL port info sensor_lst.append(sensor_reading) if not line: break # sensor section end status_info('ok', max_title, len(info)) # save extracted data to json file save_data(report_data_lst, data_names, sensor_lst) # verify if loaded data is empty after first iteration and replace information string with empty list else: sensor_lst = verify_data(report_data_lst, data_names, *data_lst) return sensor_lst
def portinfo_extract(switch_params_lst, report_data_lst): """Function to extract switch port information""" # report_data_lst contains information: # customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct *_, max_title, report_steps_dct = report_data_lst # names to save data obtained after current module execution data_names = ['sfpshow', 'portcfgshow'] # service step information print(f'\n\n{report_steps_dct[data_names[0]][3]}\n') # load data if they were saved on previos program execution iteration data_lst = load_data(report_data_lst, *data_names) # unpacking from the loaded list with data # pylint: disable=unbalanced-tuple-unpacking sfpshow_lst, portcfgshow_lst = data_lst # when any data from data_lst was not saved (file not found) or # force extract flag is on then re-extract data from configuration files force_run = verify_force_run(data_names, data_lst, report_steps_dct, max_title) # # data force extract check # # list of keys for each data from data_lst representing if it is required # # to re-collect or re-analyze data even they were obtained on previous iterations # force_extract_keys_lst = [report_steps_dct[data_name][1] for data_name in data_names] # # print data which were loaded but for which force extract flag is on # force_extract_check(data_names, data_lst, force_extract_keys_lst, max_title) # # when any of data_lst was not saved or # # force extract flag is on then re-extract data from configueation files # if not all(data_lst) or any(force_extract_keys_lst): if force_run: print('\nEXTRACTING SWITCH PORTS SFP, PORTCFG INFORMATION FROM SUPPORTSHOW CONFIGURATION FILES ...\n') # extract chassis parameters names from init file switch_columns = columns_import('switch', max_title, 'columns') # number of switches to check switch_num = len(switch_params_lst) # data imported from init file to extract values from config file params, params_add, comp_keys, match_keys, comp_dct = data_extract_objects('portinfo', max_title) portcfg_params = columns_import('portinfo', max_title, 'portcfg_params') # dictionary to save portcfg ALL information for all ports in fabric portcfgshow_dct = dict((key, []) for key in portcfg_params) # list to store only REQUIRED switch parameters # collecting sfpshow data for all switches ports during looping sfpshow_lst = [] # list to save portcfg information for all ports in fabric portcfgshow_lst = [] # switch_params_lst [[switch_params_sw1], [switch_params_sw1]] # checking each switch for switch level parameters for i, switch_params_data in enumerate(switch_params_lst): # data unpacking from iter param # dictionary with parameters for the current switch switch_params_data_dct = dict(zip(switch_columns, switch_params_data)) switch_info_keys = ['configname', 'chassis_name', 'chassis_wwn', 'switch_index', 'SwitchName', 'switchWwn'] switch_info_lst = [switch_params_data_dct.get(key) for key in switch_info_keys] ls_mode_on = True if switch_params_data_dct['LS_mode'] == 'ON' else False sshow_file, _, _, switch_index, switch_name, *_ = switch_info_lst # current operation information string info = f'[{i+1} of {switch_num}]: {switch_name} ports sfp and cfg' print(info, end =" ") # search control dictionary. continue to check sshow_file until all parameters groups are found collected = {'sfpshow': False, 'portcfgshow': False} with open(sshow_file, encoding='utf-8', errors='ignore') as file: # check file until all groups of parameters extracted while not all(collected.values()): line = file.readline() if not line: break # sfpshow section start if re.search(r'^(SWITCHCMD )?(/fabos/cliexec/)?sfpshow +-all *: *$', line) and not collected['sfpshow']: collected['sfpshow'] = True if ls_mode_on: while not re.search(fr'^CURRENT CONTEXT -- {switch_index} *, \d+$',line): line = file.readline() if not line: break while not re.search(r'^(real [\w.]+)|(\*\* SS CMD END \*\*)$',line): line = file.readline() match_dct ={match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip(comp_keys, match_keys)} if match_dct[match_keys[0]]: # dictionary to store all DISCOVERED switch ports information # collecting data only for the logical switch in current loop sfpshow_dct = {} _, slot_num, port_num = line_to_list(comp_dct[comp_keys[0]], line) # if switch has no slots then all ports have slot 0 slot_num = '0' if not slot_num else slot_num while not re.match('\r?\n', line): line = file.readline() match_dct ={match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip(comp_keys, match_keys)} # power_match if match_dct[match_keys[1]]: sfp_power_lst = line_to_list(comp_dct[comp_keys[1]], line) # cut off RX or TX Power sfp_power_value_unit = sfp_power_lst[1:] for v, k in zip(sfp_power_value_unit[::2], sfp_power_value_unit[1::2]): if k == 'uWatts': k = 'uW' key = sfp_power_lst[0] + '_' + k sfpshow_dct[key] = v # transceiver_match elif match_dct[match_keys[2]]: sfpshow_dct[match_dct[match_keys[2]].group(1).rstrip()] = match_dct[match_keys[2]].group(2).rstrip() # no_sfp_match elif match_dct[match_keys[3]]: sfpshow_dct['Vendor Name'] = 'No SFP module' # not_available_match elif match_dct[match_keys[4]]: sfpshow_dct[match_dct[match_keys[4]].group(1).rstrip()] = match_dct[match_keys[4]].group(2).rstrip() # sfp_info_match elif match_dct[match_keys[5]]: sfpshow_dct[match_dct[match_keys[5]].group(1).rstrip()] = match_dct[match_keys[5]].group(2).rstrip() if not line: break # additional values which need to be added to the dictionary with all DISCOVERED parameters during current loop iteration # values axtracted in manual mode. if change values order change keys order in init.xlsx "chassis_params_add" column sfpshow_port_values = [*switch_info_lst, slot_num, port_num] # adding additional parameters and values to the sfpshow_dct update_dct(params_add, sfpshow_port_values, sfpshow_dct) # appending list with only REQUIRED port info for the current loop iteration to the list with all fabrics port info sfpshow_lst.append([sfpshow_dct.get(param, None) for param in params]) # sfpshow section end # portcfgshow section start if re.search(r'^(SWITCHCMD )?(/fabos/cliexec/)?portcfgshow *: *$', line) and not collected['portcfgshow']: collected['portcfgshow'] = True if ls_mode_on: while not re.search(fr'^CURRENT CONTEXT -- {switch_index} *, \d+$',line): line = file.readline() if not line: break while not re.search(r'^(real [\w.]+)|(\*\* SS CMD END \*\*)$|No ports found in switch',line): line = file.readline() match_dct ={match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip(comp_keys, match_keys)} # 'slot_port_line_match' if match_dct[match_keys[6]]: # dictionary to store all DISCOVERED switch ports information portcfgshow_tmp_dct = {} # extract slot and port numbers slot_num, port_nums_str = line_to_list(comp_dct[comp_keys[6]], line) port_nums_lst = port_nums_str.split() port_nums = len(port_nums_lst) # list with switch and slot information switch_info_slot_lst = switch_info_lst.copy() switch_info_slot_lst.append(slot_num) # adding switch and slot information for each port to dictionary for portcfg_param, switch_info_value in zip(portcfg_params[:7], switch_info_slot_lst): portcfgshow_tmp_dct[portcfg_param] = [switch_info_value for i in range(port_nums)] # adding port numbers to dictionary portcfgshow_tmp_dct[portcfg_params[7]] = port_nums_lst while not re.match('\r?\n', line): line = file.readline() match_dct ={match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip(comp_keys, match_keys)} # portcfg_match if match_dct[match_keys[7]]: # extract param name and values for each port and adding to dictionary param_name, param_values_str = line_to_list(comp_dct[comp_keys[7]], line) portcfgshow_tmp_dct[param_name] = param_values_str.split() if not line: break # saving portcfg information of REQUIRED parameters from dictionary with DISCOVERED parameters for portcfg_param in portcfg_params: portcfgshow_dct[portcfg_param].extend(portcfgshow_tmp_dct.get(portcfg_param, [None for i in range(port_nums)])) # portcfgshow section end status_info('ok', max_title, len(info)) # after check all config files create list of lists from dictionary. each nested list contains portcfg information for one port for portcfg_param in portcfg_params: portcfgshow_lst.append(portcfgshow_dct.get(portcfg_param)) portcfgshow_lst = list(zip(*portcfgshow_lst)) # save extracted data to json file save_data(report_data_lst, data_names, sfpshow_lst, portcfgshow_lst) # verify if loaded data is empty after first iteration and replace information string with empty list else: sfpshow_lst, portcfgshow_lst = verify_data(report_data_lst, data_names, *data_lst) return sfpshow_lst, portcfgshow_lst
def switch_params_analysis_main(fabricshow_ag_labels_df, chassis_params_df, switch_params_df, maps_params_df, blade_module_loc_df, ag_principal_df, report_data_lst): """Main function to create aggregated switch parameters table and report tables""" # report_data_lst contains information: # customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct *_, max_title, report_steps_dct = report_data_lst # names to save data obtained after current module execution data_names = [ 'report_columns_usage', 'switch_params_aggregated', 'Коммутаторы', 'Фабрика', 'Параметры_коммутаторов', 'Лицензии', 'Глобальные_параметры_фабрики' ] # service step information print(f'\n\n{report_steps_dct[data_names[0]][3]}\n') # load data if they were saved on previos program execution iteration data_lst = load_data(report_data_lst, *data_names) # unpacking DataFrames from the loaded list with data # pylint: disable=unbalanced-tuple-unpacking report_columns_usage_dct, switch_params_aggregated_df, switches_report_df, fabric_report_df, \ switches_parameters_report_df, licenses_report_df, global_fabric_parameters_report_df = data_lst # list of data to analyze from report_info table analyzed_data_names = [ 'chassis_parameters', 'switch_parameters', 'switchshow_ports', 'maps_parameters', 'blade_interconnect', 'fabric_labels' ] # clean fabricshow DataFrame from unneccessary data fabric_clean_df = fabric_clean(fabricshow_ag_labels_df) # force run when any data from data_lst was not saved (file not found) or # procedure execution explicitly requested for output data or data used during fn execution force_run = verify_force_run(data_names, data_lst, report_steps_dct, max_title, analyzed_data_names) if force_run: # import data with switch models, firmware and etc switch_models_df = dataframe_import('switch_models', max_title) # current operation information string info = f'Generating aggregated switch parameters table' print(info, end=" ") # create aggregated table by joining DataFrames switch_params_aggregated_df, report_columns_usage_dct = \ fabric_aggregation(fabric_clean_df, chassis_params_df, \ switch_params_df, maps_params_df, switch_models_df, ag_principal_df) # add 'Device_Location for Blade chassis switches switch_params_aggregated_df = fill_device_location( switch_params_aggregated_df, blade_module_loc_df) # after finish display status status_info('ok', max_title, len(info)) # check if switch config files missing mask_fabric = switch_params_aggregated_df[[ 'Fabric_name', 'Fabric_label' ]].notna().all(axis=1) mask_no_config = switch_params_aggregated_df['chassis_name'].isna() missing_configs_num = switch_params_aggregated_df.loc[mask_no_config][ 'Fabric_name'].count() if missing_configs_num: info = f'{missing_configs_num} switch configuration{"s" if missing_configs_num > 1 else ""} MISSING' print(info, end=" ") status_info('warning', max_title, len(info)) switches_report_df, fabric_report_df, switches_parameters_report_df, \ licenses_report_df, global_fabric_parameters_report_df = \ switchs_params_report(switch_params_aggregated_df, data_names, report_columns_usage_dct, max_title) # # partition aggregated DataFrame to required tables # switches_report_df, fabric_report_df, \ # switches_parameters_report_df, licenses_report_df = \ # dataframe_segmentation(switch_params_aggregated_df, data_names[2:-1], \ # report_columns_usage_dct, max_title) # # global parameters are equal for all switches in one fabric thus checking Principal switches only # mask_principal = switch_params_aggregated_df['switchRole'] == 'Principal' # switch_params_principal_df = switch_params_aggregated_df.loc[mask_principal].copy() # global_fabric_parameters_report_df, = dataframe_segmentation(switch_params_principal_df, data_names[-1], \ # report_columns_usage_dct, max_title) # # drop rows with empty switch names columns # fabric_report_df.dropna(subset = ['Имя коммутатора'], inplace = True) # switches_parameters_report_df.dropna(subset = ['Имя коммутатора'], inplace = True) # licenses_report_df.dropna(subset = ['Имя коммутатора'], inplace = True) # # drop fabric_id if all have same value # if fabric_report_df['Fabric ID'].dropna().nunique() == 1: # fabric_report_df.drop(columns=['Fabric ID'], inplace=True) # # TO_REMOVE No need to drop duplicates coz Principal switches only used before # # # parameters are equal for all switches in one fabric # # if report_columns_usage_dct['fabric_name_usage']: # # global_fabric_parameters_report_df.drop_duplicates(subset=['Фабрика', 'Подсеть'], inplace=True) # # else: # # global_fabric_parameters_report_df.drop_duplicates(subset=['Подсеть'], inplace=True) # global_fabric_parameters_report_df.reset_index(inplace=True, drop=True) # create list with partitioned DataFrames data_lst = [ report_columns_usage_dct, switch_params_aggregated_df, switches_report_df, fabric_report_df, switches_parameters_report_df, licenses_report_df, global_fabric_parameters_report_df ] # saving data to json or csv file save_data(report_data_lst, data_names, *data_lst) # verify if loaded data is empty and replace information string with empty DataFrame else: report_columns_usage_dct, switch_params_aggregated_df, switches_report_df, fabric_report_df, \ switches_parameters_report_df, licenses_report_df, global_fabric_parameters_report_df = verify_data(report_data_lst, data_names, *data_lst) data_lst = [ report_columns_usage_dct, switch_params_aggregated_df, switches_report_df, fabric_report_df, switches_parameters_report_df, licenses_report_df, global_fabric_parameters_report_df ] # save data to service file if it's required for data_name, data_frame in zip(data_names[1:], data_lst[1:]): save_xlsx_file(data_frame, data_name, report_data_lst) return report_columns_usage_dct, switch_params_aggregated_df, fabric_clean_df
def errdump_main(errdump_df, switchshow_df, switch_params_aggregated_df, portshow_aggregated_df, report_columns_usage_dct, report_data_lst): """Main function to get most frequently appeared log messages""" # report_data_lst contains information: # customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct *_, max_title, report_steps_dct = report_data_lst # names to save data obtained after current module execution data_names = ['errdump_aggregated', 'raslog_counter', 'Журнал'] # service step information print(f'\n\n{report_steps_dct[data_names[0]][3]}\n') # loading data if were saved on previous iterations data_lst = load_data(report_data_lst, *data_names) # unpacking DataFrames from the loaded list with data # pylint: disable=unbalanced-tuple-unpacking errdump_aggregated_df, raslog_counter_df, raslog_report_df = data_lst # list of data to analyze from report_info table analyzed_data_names = [ 'chassis_parameters', 'switch_parameters', 'switchshow_ports', 'maps_parameters', 'portshow_aggregated', 'fabric_labels' ] # force run when any data from data_lst was not saved (file not found) or # procedure execution explicitly requested for output data or data used during fn execution force_run = verify_force_run(data_names, data_lst, report_steps_dct, max_title, analyzed_data_names) if force_run: # data imported from init file (regular expression patterns) to extract values from data columns # re_pattern list contains comp_keys, match_keys, comp_dct _, _, *re_pattern_lst = data_extract_objects('raslog', max_title) # current operation information string info = f'Counting RASLog messages' print(info, end=" ") # get aggregated DataFrames errdump_aggregated_df, raslog_counter_df, raslog_frequent_df = \ errdump_aggregated(errdump_df, switchshow_df, switch_params_aggregated_df, portshow_aggregated_df, re_pattern_lst) # after finish display status status_info('ok', max_title, len(info)) # partition aggregated DataFrame to required tables raslog_report_df = raslog_report(raslog_frequent_df, data_names, report_columns_usage_dct, max_title) # create list with partitioned DataFrames data_lst = [errdump_aggregated_df, raslog_counter_df, raslog_report_df] # saving fabric_statistics and fabric_statistics_summary DataFrames to csv file save_data(report_data_lst, data_names, *data_lst) # verify if loaded data is empty and replace information string with empty DataFrame else: errdump_aggregated_df, raslog_counter_df, raslog_report_df = \ verify_data(report_data_lst, data_names, *data_lst) data_lst = [errdump_aggregated_df, raslog_counter_df, raslog_report_df] # save data to service file if it's required for data_name, data_frame in zip(data_names, data_lst): save_xlsx_file(data_frame, data_name, report_data_lst) return errdump_aggregated_df, raslog_counter_df
def fcr_extract(switch_params_lst, report_data_lst): """Function to extract fabrics routing information""" # report_data_lst contains information: # customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct *_, max_title, report_steps_dct = report_data_lst # names to save data obtained after current module execution data_names = [ 'fcrfabric', 'fcrproxydev', 'fcrphydev', 'lsan', 'fcredge', 'fcrresource' ] # service step information print(f'\n\n{report_steps_dct[data_names[0]][3]}\n') # load data if they were saved on previos program execution iteration data_lst = load_data(report_data_lst, *data_names) # unpacking from the loaded list with data # pylint: disable=unbalanced-tuple-unpacking fcrfabric_lst, fcrproxydev_lst, fcrphydev_lst, lsan_lst, \ fcredge_lst, fcrresource_lst = data_lst # data force extract check # list of keys for each data from data_lst representing if it is required # to re-collect or re-analyze data even they were obtained on previous iterations force_extract_keys_lst = [ report_steps_dct[data_name][1] for data_name in data_names ] # print data which were loaded but for which force extract flag is on force_extract_check(data_names, data_lst, force_extract_keys_lst, max_title) # when any of data_lst was not saved or # force extract flag is on then re-extract data from configueation files if not all(data_lst) or any(force_extract_keys_lst): print( '\nEXTRACTING FABRICS ROUTING INFORMATION FROM SUPPORTSHOW CONFIGURATION FILES ...\n' ) # extract switch parameters names from init file switch_columns = columns_import('switch', max_title, 'columns') # number of switches to check switch_num = len(switch_params_lst) # lists to store only REQUIRED switch parameters # collecting data for all switches during looping fcrfabric_lst = [] fcrproxydev_lst = [] fcrphydev_lst = [] lsan_lst = [] fcredge_lst = [] fcrresource_lst = [] # dictionary to collect fcr device data # first element of list is regular expression pattern number, # second - list to collect data, third is index in line to which slice extracted list fcrdev_dct = { 'fcrproxydev': [5, fcrproxydev_lst, None], 'fcrphydev': [6, fcrphydev_lst, -3] } # data imported from init file to extract values from config file params, _, comp_keys, match_keys, comp_dct = data_extract_objects( 'fcr', max_title) # switch_params_lst [[switch_params_sw1], [switch_params_sw1]] # checking each switch for switch level parameters for i, switch_params_data in enumerate(switch_params_lst): # dictionary with parameters for the current switch switch_params_data_dct = dict( zip(switch_columns, switch_params_data)) switch_info_keys = [ 'configname', 'chassis_name', 'chassis_wwn', 'switch_index', 'SwitchName', 'switchWwn', 'switchRole', 'Fabric_ID', 'FC_Router' ] switch_info_lst = [ switch_params_data_dct.get(key) for key in switch_info_keys ] ls_mode_on = True if switch_params_data_dct[ 'LS_mode'] == 'ON' else False # data unpacking from iter param sshow_file, *_, switch_name, _, switch_role, fid, fc_router = switch_info_lst # current operation information string info = f'[{i+1} of {switch_num}]: {switch_name} fabric routing. FC Routing: {fc_router}' print(info, end=" ") # search control dictionary. continue to check sshow_file until all parameters groups are found collected = {'fcrfabric': False, 'fcrproxydev': False, 'fcrphydev': False, 'lsanzone': False, 'fcredge': False, 'fcrresource': False} \ if switch_role == 'Principal' else {'fcredge': False, 'fcrresource': False} # check config of FC routers only if fc_router == 'ON': # fcrouter_info_lst contains sshow_file, chassis_name, switch_index, switch_name, switch_fid fcrouter_info_lst = [*switch_info_lst[:6], switch_info_lst[7]] with open(sshow_file, encoding='utf-8', errors='ignore') as file: # check file until all groups of parameters extracted while not all(collected.values()): line = file.readline() if not line: break # check configs of Principal switches only if switch_role == 'Principal': # fcrfabricshow section start # switchcmd_fcrfabricshow if re.search(comp_dct[comp_keys[0]], line) and not collected['fcrfabric']: collected['fcrfabric'] = True if ls_mode_on: while not re.search( fr'^BASE +SWITCH +CONTEXT *-- *FID: *{fid}$', line): line = file.readline() if not line: break # switchcmd_end_comp while not re.search(comp_dct[comp_keys[4]], line): line = file.readline() # dictionary with match names as keys and match result of current line with all imported regular expressions as values match_dct = { match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip( comp_keys, match_keys) } # fc_router_match' if match_dct[match_keys[1]]: fcrouter_params_lst = line_to_list( comp_dct[comp_keys[1]], line) # check if line is empty while not re.match('\r?\n', line): line = file.readline() match_dct = { match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip( comp_keys, match_keys) } # fcr_info_match if match_dct[match_keys[2]]: fcrouter_name = match_dct[ match_keys[2]].group(1) # fcr_exports_match if match_dct[match_keys[3]]: fcrfabric_lst.append( line_to_list( comp_dct[comp_keys[3]], line, *fcrouter_info_lst, fcrouter_name, *fcrouter_params_lst)) if not line: break if not line: break # fcrfabricshow section end # fcrproxydev and fcrphydev checked in a loop over dictionary keys coz data representation is similar # fcrdevshow section start for fcrdev_type in fcrdev_dct.keys(): re_num, fcrdev_lst, slice_index = fcrdev_dct[ fcrdev_type] # switchcmd_fcrproxydevshow_comp if re.search( comp_dct[comp_keys[re_num]], line) and not collected[fcrdev_type]: collected[fcrdev_type] = True if ls_mode_on: while not re.search( fr'^BASE +SWITCH +CONTEXT *-- *FID: *{fid} *$', line): line = file.readline() if not line: break # switchcmd_end_comp while not re.search( comp_dct[comp_keys[4]], line): line = file.readline() match_dct = { match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip( comp_keys, match_keys) } # fcrdevshow_match if match_dct[match_keys[7]]: fcrdev_lst.append( line_to_list( comp_dct[comp_keys[7]], line, *fcrouter_info_lst) [:slice_index]) if not line: break # fcrdevshow section end # lsanzoneshow section start if re.search(comp_dct[comp_keys[8]], line) and not collected['lsanzone']: collected['lsanzone'] = True if ls_mode_on: while not re.search( fr'^BASE +SWITCH +CONTEXT *-- *FID: *{fid} *$', line): line = file.readline() if not line: break # switchcmd_end_comp while not re.search(comp_dct[comp_keys[4]], line): match_dct = { match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip( comp_keys, match_keys) } # lsan_name_match if match_dct[match_keys[9]]: # switch_info and current connected device wwnp lsan_name = line_to_list( comp_dct[comp_keys[9]], line) # move cursor to one line down to get inside while loop line = file.readline() # lsan_switchcmd_end_comp while not re.search( comp_dct[comp_keys[11]], line): # line = file.readline() match_dct = { match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip( comp_keys, match_keys) } # lsan_members_match if match_dct[match_keys[10]]: lsan_member = line_to_list( comp_dct[comp_keys[10]], line) lsan_lst.append([ *fcrouter_info_lst, *lsan_name, *lsan_member ]) # line = file.readline() # else: # line = file.readline() line = file.readline() if not line: break else: line = file.readline() if not line: break # lsanzoneshow section end # fcredge and fcrresource checked for Principal and Subordinate routers # fcredgeshow section start if re.search(comp_dct[comp_keys[12]], line) and not collected['fcredge']: collected['fcredge'] = True if ls_mode_on: while not re.search( fr'^BASE +SWITCH +CONTEXT *-- *FID: *{fid} *$', line): line = file.readline() if not line: break # switchcmd_end_comp while not re.search(comp_dct[comp_keys[4]], line): line = file.readline() match_dct = { match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip( comp_keys, match_keys) } # fcredgeshow_match if match_dct[match_keys[13]]: # fcredge_lst.append(line_to_list(comp_dct[comp_keys[13]], line, *fcrouter_info_lst, switch_wwn)) fcredge_lst.append( line_to_list(comp_dct[comp_keys[13]], line, *fcrouter_info_lst)) if not line: break # fcredgeshow section end # fcrresourceshow section start if re.search(comp_dct[comp_keys[14]], line) and not collected['fcrresource']: collected['fcrresource'] = True fcrresource_dct = {} if ls_mode_on: while not re.search( fr'^BASE +SWITCH +CONTEXT *-- *FID: *{fid} *$', line): line = file.readline() if not line: break # switchcmd_end_comp while not re.search(comp_dct[comp_keys[4]], line): line = file.readline() match_dct = { match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip( comp_keys, match_keys) } # fcredgeshow_match if match_dct[match_keys[15]]: fcrresource_dct[match_dct[match_keys[15]].group(1).rstrip()] = \ [match_dct[match_keys[15]].group(2), match_dct[match_keys[15]].group(3)] if not line: break # each value of dictionary is list of two elements # itertools.chain makes flat tmp_lst list from all lists in dictionary tmp_lst = list( itertools.chain(*[ fcrresource_dct.get(param ) if fcrresource_dct. get(param) else [None, None] for param in params ])) fcrresource_lst.append( [*fcrouter_info_lst, *tmp_lst]) # fcrresourceshow section end status_info('ok', max_title, len(info)) else: status_info('skip', max_title, len(info)) # save extracted data to json file save_data(report_data_lst, data_names, fcrfabric_lst, fcrproxydev_lst, fcrphydev_lst, lsan_lst, fcredge_lst, fcrresource_lst) # verify if loaded data is empty after first iteration and replace information string with empty list else: fcrfabric_lst, fcrproxydev_lst, fcrphydev_lst, lsan_lst, fcredge_lst, fcrresource_lst = verify_data( report_data_lst, data_names, *data_lst) return fcrfabric_lst, fcrproxydev_lst, fcrphydev_lst, lsan_lst, fcredge_lst, fcrresource_lst
def synergy_system_extract(synergy_folder, report_data_lst): """Function to extract blade systems information""" # report_data_lst contains information: # customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct *_, max_title, report_steps_dct = report_data_lst # names to save data obtained after current module execution data_names = ['synergy_interconnect', 'synergy_servers'] # service step information print(f'\n\n{report_steps_dct[data_names[0]][3]}\n') # load data if they were saved on previos program execution iteration data_lst = load_data(report_data_lst, *data_names) # # unpacking from the loaded list with data # # pylint: disable=unbalanced-tuple-unpacking # module_comprehensive_lst, blades_comprehensive_lst, blade_vc_comprehensive_lst = data_lst # force run when any data from data_lst was not saved (file not found) or # procedure execution explicitly requested for output data or data used during fn execution force_run = verify_force_run(data_names, data_lst, report_steps_dct, max_title) if force_run: # lists to store only REQUIRED infromation # collecting data for all blades during looping # list containing enclosure, blade and hba information for all blade systems # list containing enclosure and interconnect modules information for all blade systems synergy_module_columns = [ 'Enclosure_Name', 'Enclosure_SN', 'Enclosure_Type', 'Interconnect_Bay', 'Interconnect_Model', 'Interconnect_SN', 'Interconnect_Firmware', 'Interconnect_Name', 'NodeName', 'Device_Location' ] synergy_server_columns = [ 'Enclosure_Name', 'Enclosure_Slot', 'Host_Name', 'name', 'serverprofilename', 'Device_Model', 'Device_SN', 'Host_OS', 'HBA_Description', 'Mezz_type', 'Connected_portWwn', 'Mezz_location', 'Device_Location', 'HBA_Firmware' ] synergy_module_aggregated_df = pd.DataFrame( columns=synergy_module_columns) synergy_servers_aggregated_df = pd.DataFrame( columns=synergy_server_columns) if synergy_folder: print('\nEXTRACTING SYNERGY SYSTEM INFORMATION ...\n') # collects files in folder with xlsm extension synergy_config_lst = find_files(synergy_folder, max_title, filename_extension='xlsm') # number of files to check configs_num = len(synergy_config_lst) if configs_num: # # data imported from init file to extract values from config file # enclosure_params, _, comp_keys, match_keys, comp_dct = data_extract_objects('blades', max_title) # module_params = columns_import('blades', max_title, 'module_params') # blade_params = columns_import('blades', max_title, 'blade_params') for i, synergy_config in enumerate(synergy_config_lst): # file name with extension configname_wext = os.path.basename(synergy_config) # remove extension from filename configname, _ = os.path.splitext(configname_wext) # current operation information string info = f'[{i+1} of {configs_num}]: {configname} system.' print(info, end=" ") syn_enclosure_df = pd.read_excel(synergy_config, sheet_name='enclosures') syn_module_df = pd.read_excel( synergy_config, sheet_name='interconnectbays') syn_server_hw_df = pd.read_excel( synergy_config, sheet_name='server-hardware') syn_server_fw_sw_df = pd.read_excel( synergy_config, sheet_name='server-fw-sw') syn_server_profile_connection_df = pd.read_excel( synergy_config, sheet_name='server-prof-conn-details') synergy_module_df = synergy_module(syn_enclosure_df, syn_module_df) if synergy_module_aggregated_df.empty: synergy_module_aggregated_df = synergy_module_df else: synergy_module_aggregated_df = pd.concat( [synergy_module_aggregated_df, synergy_module_df], ignore_index=True) synergy_server_wwn_df = synergy_server_wwn( syn_server_hw_df) synergy_profile_wwn_df = synergy_profile_wwn( syn_server_profile_connection_df, synergy_server_wwn_df) # conctenate connection profile and server hardware synergy_servers_df = pd.concat( [synergy_server_wwn_df, synergy_profile_wwn_df], ignore_index=True) synergy_servers_df.drop_duplicates(inplace=True) # add mezzanine firmware details synergy_servers_df = synergy_mezz_fw( syn_server_fw_sw_df, synergy_servers_df) synergy_servers_df.sort_values( by=['enclosurename', 'position', 'Mezz_WWPN'], ignore_index=True, inplace=True) if synergy_servers_aggregated_df.empty: synergy_servers_aggregated_df = synergy_servers_df else: synergy_servers_aggregated_df = pd.concat( [ synergy_servers_aggregated_df, synergy_servers_df ], ignore_index=True) if synergy_module_aggregated_df['switchbasewwn'].notna( ).any(): synergy_module_aggregated_df[ 'switchbasewwn'] = synergy_module_aggregated_df[ 'switchbasewwn'].str.lower() if synergy_servers_aggregated_df['Mezz_WWPN'].notna().any( ): synergy_servers_aggregated_df[ 'Mezz_WWPN'] = synergy_servers_aggregated_df[ 'Mezz_WWPN'].str.lower() if not synergy_servers_df.empty or not synergy_module_df.empty: status_info('ok', max_title, len(info)) else: status_info('no data', max_title, len(info)) module_columns_dct = { 'enclosurename': 'Enclosure_Name', 'enclosure_serialnumber': 'Enclosure_SN', 'enclosuretype': 'Enclosure_Type', 'baynumber': 'Interconnect_Bay', 'interconnectmodel': 'Interconnect_Model', 'serialnumber': 'Interconnect_SN', 'switchfwversion': 'Interconnect_Firmware', 'hostname': 'Interconnect_Name', 'switchbasewwn': 'NodeName', 'device_location': 'Device_Location' } synergy_module_aggregated_df.rename(columns=module_columns_dct, inplace=True) synergy_module_aggregated_df.replace(r'^None$|^none$|^ *$', value=np.nan, regex=True, inplace=True) server_columns_dct = { 'enclosurename': 'Enclosure_Name', 'position': 'Enclosure_Slot', 'servername': 'Host_Name', 'model': 'Device_Model', 'serialnumber': 'Device_SN', 'oshint': 'Host_OS', 'Mezz': 'HBA_Description', 'Mezz_WWPN': 'Connected_portWwn', 'device_location': 'Device_Location', 'componentversion': 'HBA_Firmware' } synergy_servers_aggregated_df.rename( columns=server_columns_dct, inplace=True) synergy_servers_aggregated_df.replace(r'^None$|^none$|^ *$', value=np.nan, regex=True, inplace=True) data_lst = [ synergy_module_aggregated_df, synergy_servers_aggregated_df ] # save extracted data to json file save_data(report_data_lst, data_names, *data_lst) else: # current operation information string info = f'Collecting synergy details' print(info, end=" ") status_info('skip', max_title, len(info)) data_lst = [ synergy_module_aggregated_df, synergy_servers_aggregated_df ] # save empty data to json file save_data(report_data_lst, data_names, *data_lst) # verify if loaded data is empty after first iteration and replace information string with empty list else: synergy_module_aggregated_df, synergy_servers_aggregated_df = verify_data( report_data_lst, data_names, *data_lst) data_lst = [ synergy_module_aggregated_df, synergy_servers_aggregated_df ] # save data to service file if it's required for data_name, data_frame in zip(data_names, data_lst): save_xlsx_file(data_frame, data_name, report_data_lst) return synergy_module_aggregated_df, synergy_servers_aggregated_df
def warning_notification(portshow_aggregated_df, switch_params_aggregated_df, nsshow_unsplit_df, expected_ag_links_df, report_data_lst): """Function to show WARNING notification if any deviceType is UNKNOWN, if any PortSymb or NodeSymb was not parsed or if new switch founded which was not previously discovered""" *_, max_title, report_steps_dct = report_data_lst portshow_force_flag = False nsshow_unsplit_force_flag = False expected_ag_links_force_flag = False portshow_export_flag, *_ = report_steps_dct['portshow_aggregated'] nsshow_unsplit_export_flag, *_ = report_steps_dct['nsshow_unsplit'] expected_ag_links_export_flag, *_ = report_steps_dct['expected_ag_links'] # warning if UKNOWN device class present if (portshow_aggregated_df['deviceType'] == 'UNKNOWN').any(): unknown_count = len(portshow_aggregated_df[ portshow_aggregated_df['deviceType'] == 'UNKNOWN']) info = f'{unknown_count} {"port" if unknown_count == 1 else "ports"} with UNKNOWN device Class found' print(info, end=" ") status_info('warning', max_title, len(info)) # ask if save portshow_aggregated_df if not portshow_export_flag: reply = reply_request( "Do you want to save 'portshow_aggregated'? (y)es/(n)o: ") if reply == 'y': portshow_force_flag = True # warning if any values in PortSymb or NodeSymb were not parsed if not nsshow_unsplit_df.empty: portsymb_unsplit_count = nsshow_unsplit_df['PortSymb'].notna().sum() nodesymb_unsplit_count = nsshow_unsplit_df['NodeSymb'].notna().sum() unsplit_lst = [[portsymb_unsplit_count, 'PortSymb'], [nodesymb_unsplit_count, 'NodeSymb']] unsplit_str = ' and '.join( [str(num) + " " + name for num, name in unsplit_lst if num]) info = f'{unsplit_str} {"is" if portsymb_unsplit_count + nodesymb_unsplit_count == 1 else "are"} UNPARSED' print(info, end=" ") status_info('warning', max_title, len(info)) # ask if save nsshow_unsplit if not nsshow_unsplit_export_flag: reply = reply_request( "Do you want to save 'nsshow_unsplit'? (y)es/(n)o: ") if reply == 'y': nsshow_unsplit_force_flag = True # warning if unknown switches was found switch_name_set = set(switch_params_aggregated_df['switchName']) # all founded switches in portshow_aggregated_df mask_switch = portshow_aggregated_df['deviceType'] == 'SWITCH' portshow_switch_name_set = set( portshow_aggregated_df.loc[mask_switch, 'Device_Host_Name']) # if unknown switches found if not portshow_switch_name_set.issubset(switch_name_set): unknown_count = len( portshow_switch_name_set.difference(switch_name_set)) info = f'{unknown_count} NEW {"switch" if unknown_count == 1 else "switches"} detected' print(info, end=" ") status_info('warning', max_title, len(info)) # ask if save portshow_aggregated_df if not portshow_export_flag and not portshow_force_flag: reply = reply_request( "Do you want to save 'portshow_aggregated'? (y)es/(n)o: ") if reply == 'y': portshow_force_flag = True # if any unconfirmed AG links found if not expected_ag_links_df.empty: unknown_count = expected_ag_links_df['chassis_name'].notna().sum() info = f'{unknown_count} AG {"link" if unknown_count == 1 else "links"} detected' print(info, end=" ") status_info('warning', max_title, len(info)) # ask if save expected_ag_links_df if not expected_ag_links_export_flag: reply = reply_request( "Do you want to save 'expected_ag_link'? (y)es/(n)o: ") if reply == 'y': expected_ag_links_force_flag = True return portshow_force_flag, nsshow_unsplit_force_flag, expected_ag_links_force_flag
def err_sfp_cfg_analysis_main(portshow_aggregated_df, sfpshow_df, portcfgshow_df, report_columns_usage_dct, report_data_lst): """Main function to add porterr, transceiver and portcfg information to portshow DataFrame""" # report_data_lst contains information: # customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct *_, max_title, report_steps_dct = report_data_lst portshow_sfp_force_flag = False portshow_sfp_export_flag, *_ = report_steps_dct['portshow_sfp_aggregated'] # names to save data obtained after current module execution data_names = [ 'portshow_sfp_aggregated', 'Ошибки', 'Параметры_SFP', 'Параметры_портов' ] # service step information print(f'\n\n{report_steps_dct[data_names[0]][3]}\n') # load data if they were saved on previos program execution iteration data_lst = load_data(report_data_lst, *data_names) # unpacking DataFrames from the loaded list with data # pylint: disable=unbalanced-tuple-unpacking portshow_sfp_aggregated_df, error_report_df, sfp_report_df, portcfg_report_df = data_lst # list of data to analyze from report_info table analyzed_data_names = [ 'portshow_aggregated', 'sfpshow', 'portcfgshow', 'portcmd', 'switchshow_ports', 'switch_params_aggregated', 'fdmi', 'device_rename', 'report_columns_usage_upd', 'nscamshow', 'nsshow', 'alias', 'blade_servers', 'fabric_labels' ] # force run when any data from data_lst was not saved (file not found) or # procedure execution explicitly requested for output data or data used during fn execution force_run = verify_force_run(data_names, data_lst, report_steps_dct, max_title, analyzed_data_names) if force_run: # import transeivers information from file sfp_model_df = dataframe_import('sfp_models', max_title) # current operation information string info = f'Updating connected devices table' print(info, end=" ") # add sfpshow, transceiver information and portcfg to aggregated portcmd DataFrame portshow_sfp_aggregated_df = port_complete(portshow_aggregated_df, sfpshow_df, sfp_model_df, portcfgshow_df) # after finish display status status_info('ok', max_title, len(info)) # warning if UKNOWN SFP present if (portshow_sfp_aggregated_df['Transceiver_Supported'] == 'Unknown SFP').any(): info_columns = [ 'Fabric_name', 'Fabric_label', 'configname', 'chassis_name', 'chassis_wwn', 'slot', 'port', 'Transceiver_Supported' ] portshow_sfp_info_df = portshow_sfp_aggregated_df.drop_duplicates( subset=info_columns).copy() unknown_count = len(portshow_sfp_info_df[ portshow_sfp_info_df['Transceiver_Supported'] == 'Unknown SFP']) info = f'{unknown_count} {"port" if unknown_count == 1 else "ports"} with UNKNOWN supported SFP tag found' print(info, end=" ") status_info('warning', max_title, len(info)) # ask if save portshow_aggregated_df if not portshow_sfp_export_flag: reply = reply_request( "Do you want to save 'portshow_sfp_aggregated'? (y)es/(n)o: " ) if reply == 'y': portshow_sfp_force_flag = True # create reaport tables from port_complete_df DataFrtame error_report_df, sfp_report_df, portcfg_report_df = \ create_report_tables(portshow_sfp_aggregated_df, data_names[1:], report_columns_usage_dct, max_title) # saving data to json or csv file data_lst = [ portshow_sfp_aggregated_df, error_report_df, sfp_report_df, portcfg_report_df ] save_data(report_data_lst, data_names, *data_lst) # verify if loaded data is empty and reset DataFrame if yes else: portshow_sfp_aggregated_df, error_report_df, sfp_report_df, portcfg_report_df \ = verify_data(report_data_lst, data_names, *data_lst) data_lst = [ portshow_sfp_aggregated_df, error_report_df, sfp_report_df, portcfg_report_df ] # save data to excel file if it's required for data_name, data_frame in zip(data_names, data_lst): force_flag = False if data_name == 'portshow_sfp_aggregated': force_flag = portshow_sfp_force_flag save_xlsx_file(data_frame, data_name, report_data_lst, force_flag=force_flag) return portshow_sfp_aggregated_df
def storage_3par_extract(nsshow_df, nscamshow_df, local_3par_folder, project_folder, report_data_lst): """Function to extract 3PAR storage information""" # report_data_lst contains information: # customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct *_, max_title, report_steps_dct = report_data_lst # names to save data obtained after current module execution data_names = ['system_3par', 'port_3par', 'host_3par'] # service step information print(f'\n\n{report_steps_dct[data_names[0]][3]}\n') # load data if they were saved on previos program execution iteration data_lst = load_data(report_data_lst, *data_names) # unpacking from the loaded list with data # pylint: disable=unbalanced-tuple-unpacking system_3par_comprehensive_lst, port_3par_comprehensive_lst, host_3par_comprehensive_lst = data_lst # data force extract check # list of keys for each data from data_lst representing if it is required # to re-collect or re-analyze data even they were obtained on previous iterations force_extract_keys_lst = [ report_steps_dct[data_name][1] for data_name in data_names ] # print data which were loaded but for which force extract flag is on force_extract_check(data_names, data_lst, force_extract_keys_lst, max_title) # when any of data_lst was not saved or # force extract flag is on then re-extract data from configueation files if not all(data_lst) or any(force_extract_keys_lst): # lists to store only REQUIRED infromation # collecting data for all systems during looping # list containing system parameters system_3par_comprehensive_lst = [] # list containing 3par FC port information port_3par_comprehensive_lst = [] # list containing hosts defined on 3par ports host_3par_comprehensive_lst = [] # data imported from init file to extract values from config file params, params_add, comp_keys, match_keys, comp_dct = data_extract_objects( '3par', max_title) # verify if 3par systems registered in fabric NameServer ns_3par_df = verify_ns_3par(nsshow_df, nscamshow_df, comp_dct) if not ns_3par_df.empty: print('\n') print('3PAR Storage Systems detected in SAN') print(ns_3par_df) print('\n') # find configuration files to parse (download from STATs, local folder or use configurations # downloaded on previous iterations) configs_3par_lst = configs_download(ns_3par_df, project_folder, local_3par_folder, comp_keys, match_keys, comp_dct, report_data_lst) if configs_3par_lst: print('\nEXTRACTING 3PAR STORAGE INFORMATION ...\n') # number of files to check configs_num = len(configs_3par_lst) for i, config_3par in enumerate(configs_3par_lst): # file name configname = os.path.basename(config_3par) # current operation information string info = f'[{i+1} of {configs_num}]: {configname} system' print(info, end=" ") showsys_lst, port_lst, host_lst = parse_config( config_3par, params, params_add, comp_keys, match_keys, comp_dct) system_3par_comprehensive_lst.extend(showsys_lst) port_3par_comprehensive_lst.extend(port_lst) host_3par_comprehensive_lst.extend(host_lst) if port_lst or host_lst: status_info('ok', max_title, len(info)) else: status_info('no data', max_title, len(info)) # save extracted data to json file save_data(report_data_lst, data_names, system_3par_comprehensive_lst, port_3par_comprehensive_lst, host_3par_comprehensive_lst) else: # current operation information string info = f'Collecting 3PAR storage systems information' print(info, end=" ") status_info('skip', max_title, len(info)) # save empty data to json file save_data(report_data_lst, data_names, system_3par_comprehensive_lst, port_3par_comprehensive_lst, host_3par_comprehensive_lst) # verify if loaded data is empty after first iteration and replace information string with empty list else: system_3par_comprehensive_lst, port_3par_comprehensive_lst, host_3par_comprehensive_lst = verify_data( report_data_lst, data_names, *data_lst) return system_3par_comprehensive_lst, port_3par_comprehensive_lst, host_3par_comprehensive_lst
def portcmd_analysis_main(portshow_df, switchshow_ports_df, switch_params_df, switch_params_aggregated_df, isl_aggregated_df, nsshow_df, nscamshow_df, ag_principal_df, porttrunkarea_df, alias_df, fdmi_df, blade_module_df, blade_servers_df, blade_vc_df, synergy_module_df, synergy_servers_df, system_3par_df, port_3par_df, report_columns_usage_dct, report_data_lst): """Main function to add connected devices information to portshow DataFrame""" # report_data_lst contains information: # customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct *_, max_title, report_steps_dct = report_data_lst # names to save data obtained after current module execution data_names = [ 'portshow_aggregated', 'storage_connection_statistics', 'device_connection_statistics', 'device_rename', 'report_columns_usage_upd', 'Серверы', 'Массивы', 'Библиотеки', 'Микрокоды_HBA', 'Подключение_массивов', 'Подключение_библиотек', 'Подключение_серверов', 'NPIV', 'Статистика_массивов', 'Статистика_устройств' ] # service step information print(f'\n\n{report_steps_dct[data_names[0]][3]}\n') report_columns_usage_bckp = report_columns_usage_dct # load data if they were saved on previos program execution iteration data_lst = load_data(report_data_lst, *data_names) # flag to forcible save portshow_aggregated_df if required portshow_force_flag = False # unpacking DataFrames from the loaded list with data # pylint: disable=unbalanced-tuple-unpacking portshow_aggregated_df, storage_connection_statistics_df, device_connection_statistics_df, \ device_rename_df, report_columns_usage_dct, \ servers_report_df, storage_report_df, library_report_df, hba_report_df, \ storage_connection_df, library_connection_df, server_connection_df, npiv_report_df, \ storage_connection_statistics_report_df, device_connection_statistics_report_df = data_lst nsshow_unsplit_df = pd.DataFrame() if not report_columns_usage_dct: report_columns_usage_dct = report_columns_usage_bckp # list of data to analyze from report_info table analyzed_data_names = [ 'portcmd', 'switchshow_ports', 'switch_params_aggregated', 'switch_parameters', 'chassis_parameters', 'fdmi', 'nscamshow', 'nsshow', 'alias', 'blade_servers', 'fabric_labels', 'isl', 'trunk', 'isl_aggregated', 'Параметры_SFP', 'portshow_sfp_aggregated' ] # force run when any data from data_lst was not saved (file not found) or # procedure execution explicitly requested for output data or data used during fn execution force_run = verify_force_run(data_names, data_lst, report_steps_dct, max_title, analyzed_data_names) if force_run: # import data with switch models, firmware and etc switch_models_df = dataframe_import('switch_models', max_title) # data imported from init file (regular expression patterns) to extract values from data columns # re_pattern list contains comp_keys, match_keys, comp_dct _, _, *re_pattern_lst = data_extract_objects('nameserver', max_title) oui_df = dataframe_import('oui', max_title, columns=['Connected_oui', 'type', 'subtype']) # current operation information string info = f'Generating connected devices table' print(info, end=" ") portshow_aggregated_df, alias_wwnn_wwnp_df, nsshow_unsplit_df, expected_ag_links_df = \ portshow_aggregated(portshow_df, switchshow_ports_df, switch_params_df, switch_params_aggregated_df, isl_aggregated_df, nsshow_df, nscamshow_df, ag_principal_df, porttrunkarea_df, switch_models_df, alias_df, oui_df, fdmi_df, blade_module_df, blade_servers_df, blade_vc_df, synergy_module_df, synergy_servers_df, system_3par_df, port_3par_df, re_pattern_lst, report_data_lst) # after finish display status status_info('ok', max_title, len(info)) # show warning if any UNKNOWN device class founded, if any PortSymb or NodeSymb is not parsed, # if new switch founded portshow_force_flag, nsshow_unsplit_force_flag, expected_ag_links_force_flag = \ warning_notification(portshow_aggregated_df, switch_params_aggregated_df, nsshow_unsplit_df, expected_ag_links_df, report_data_lst) # correct device names manually portshow_aggregated_df, device_rename_df = \ devicename_correction_main(portshow_aggregated_df, device_rename_df, report_columns_usage_dct, report_data_lst) # count Device_Host_Name instances for fabric_label, label and total in fabric portshow_aggregated_df = device_ports_per_group(portshow_aggregated_df) # count device connection statistics info = f'Counting device connection statistics' print(info, end=" ") storage_connection_statistics_df = storage_connection_statistics( portshow_aggregated_df, re_pattern_lst) device_connection_statistics_df = device_connection_statistics( portshow_aggregated_df) status_info('ok', max_title, len(info)) servers_report_df, storage_report_df, library_report_df, hba_report_df, \ storage_connection_df, library_connection_df, server_connection_df, npiv_report_df, \ storage_connection_statistics_report_df, device_connection_statistics_report_df = \ create_report_tables(portshow_aggregated_df, storage_connection_statistics_df, device_connection_statistics_df, data_names[5:-2], report_columns_usage_dct, max_title) # create list with partitioned DataFrames data_lst = [ portshow_aggregated_df, storage_connection_statistics_df, device_connection_statistics_df, device_rename_df, report_columns_usage_dct, servers_report_df, storage_report_df, library_report_df, hba_report_df, storage_connection_df, library_connection_df, server_connection_df, npiv_report_df, storage_connection_statistics_report_df, device_connection_statistics_report_df ] # saving data to json or csv file save_data(report_data_lst, data_names, *data_lst) save_xlsx_file(nsshow_unsplit_df, 'nsshow_unsplit', report_data_lst, force_flag=nsshow_unsplit_force_flag) save_xlsx_file(expected_ag_links_df, 'expected_ag_links', report_data_lst, force_flag=expected_ag_links_force_flag) # verify if loaded data is empty and replace information string with empty DataFrame else: portshow_aggregated_df, storage_connection_statistics_df, device_connection_statistics_df, \ device_rename_df, report_columns_usage_dct, \ servers_report_df, storage_report_df, library_report_df, hba_report_df, \ storage_connection_df, library_connection_df, server_connection_df, npiv_report_df, \ storage_connection_statistics_report_df, device_connection_statistics_report_df \ = verify_data(report_data_lst, data_names, *data_lst) data_lst = [ portshow_aggregated_df, storage_connection_statistics_df, device_connection_statistics_df, device_rename_df, report_columns_usage_dct, servers_report_df, storage_report_df, library_report_df, hba_report_df, storage_connection_df, library_connection_df, server_connection_df, npiv_report_df, storage_connection_statistics_report_df, device_connection_statistics_report_df ] # save data to service file if it's required for data_name, data_frame in zip(data_names, data_lst): force_flag = False if data_name == 'portshow_aggregated': force_flag = portshow_force_flag save_xlsx_file(data_frame, data_name, report_data_lst, force_flag=force_flag) return portshow_aggregated_df
def fabric_main(fabricshow_ag_labels_df, chassis_params_df, \ switch_params_df, maps_params_df, report_data_lst): """Main function to create tables""" # report_data_lst contains information: # customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct *_, max_title, report_steps_dct = report_data_lst # names to save data obtained after current module execution data_names = [ 'Коммутаторы', 'Фабрика', 'Глобальные_параметры_фабрики', 'Параметры_коммутаторов', 'Лицензии' ] # service step information print(f'\n\n{report_steps_dct[data_names[0]][3]}\n') # load data if they were saved on previos program execution iteration data_lst = load_data(report_data_lst, *data_names) # unpacking DataFrames from the loaded list with data # pylint: disable=unbalanced-tuple-unpacking switches_report_df, fabric_report_df, global_fabric_parameters_report_df, \ switches_parameters_report_df, licenses_report_df = data_lst # data force extract check # list of keys for each data from data_lst representing if it is required # to re-collect or re-analyze data even they were obtained on previous iterations force_extract_keys_lst = [ report_steps_dct[data_name][1] for data_name in data_names ] # list with True (if data loaded) and/or False (if data was not found and None returned) data_check = force_extract_check(data_names, data_lst, force_extract_keys_lst, max_title) # flag if fabrics labels was forced to be changed fabric_labels_change = True if report_steps_dct['fabric_labels'][ 1] else False # initialization chassis information and farbric name columns usage report_columns_usage_dct = { 'fabric_name_usage': True, 'chassis_info_usage': True } # import data with switch models, firmware and etc switch_models_df = dataframe_import('switch_models', max_title) # clean fabricshow DataFrame from unneccessary data fabric_clean_df = fabric_clean(fabricshow_ag_labels_df) # create aggregated table by joining DataFrames switch_params_aggregated_df, report_columns_usage_dct = \ fabric_aggregation(fabric_clean_df, chassis_params_df, \ switch_params_df, maps_params_df, switch_models_df) save_xlsx_file(switch_params_aggregated_df, 'switch_params_aggregated', \ report_data_lst, report_type = 'analysis') # when no data saved or force extract flag is on or fabric labels have been changed than # analyze extracted config data if not all(data_check) or any( force_extract_keys_lst) or fabric_labels_change: # information string if fabric labels force changed was initiated # and statistics recounting required if fabric_labels_change and not any(force_extract_keys_lst) and all( data_check): info = f'Switch information force extract due to change in Fabrics labeling' print(info, end=" ") status_info('ok', max_title, len(info)) # partition aggregated DataFrame to required tables switches_report_df, fabric_report_df, global_fabric_parameters_report_df, \ switches_parameters_report_df, licenses_report_df = \ dataframe_segmentation(switch_params_aggregated_df, data_names, \ report_columns_usage_dct, max_title) # drop rows with empty switch names columns fabric_report_df.dropna(subset=['Имя коммутатора'], inplace=True) switches_parameters_report_df.dropna(subset=['Имя коммутатора'], inplace=True) licenses_report_df.dropna(subset=['Имя коммутатора'], inplace=True) # parameters are equal for all switches in one fabric if report_columns_usage_dct['fabric_name_usage']: global_fabric_parameters_report_df.drop_duplicates( subset=['Фабрика', 'Подсеть'], inplace=True) else: global_fabric_parameters_report_df.drop_duplicates( subset=['Подсеть'], inplace=True) global_fabric_parameters_report_df.reset_index(inplace=True, drop=True) # create list with partitioned DataFrames data_lst = [switches_report_df, fabric_report_df, global_fabric_parameters_report_df, \ switches_parameters_report_df, licenses_report_df] # current operation information string info = f'Generating Fabric and Switches tables' print(info, end=" ") # after finish display status status_info('ok', max_title, len(info)) # saving DataFrames to csv file save_data(report_data_lst, data_names, *data_lst) # save_data(report_data_lst, data_auxillary_names, *data_auxillary_lst) # save data to service file if it's required for data_name, data_frame in zip(data_names, data_lst): save_xlsx_file(data_frame, data_name, report_data_lst) return switch_params_aggregated_df, report_columns_usage_dct, fabric_clean_df
def connected_devices_extract(switch_params_lst, report_data_lst): """Function to extract connected devices information (fdmi, nsshow, nscamshow) """ # report_data_lst contains information: # customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct *_, max_title, report_steps_dct = report_data_lst # names to save data obtained after current module execution data_names = ['fdmi', 'nsshow', 'nscamshow'] # service step information print(f'\n\n{report_steps_dct[data_names[0]][3]}\n') # load data if they were saved on previos program execution iteration data_lst = load_data(report_data_lst, *data_names) # unpacking from the loaded list with data # pylint: disable=unbalanced-tuple-unpacking fdmi_lst, nsshow_lst, nscamshow_lst = data_lst # data force extract check # list of keys for each data from data_lst representing if it is required # to re-collect or re-analyze data even they were obtained on previous iterations force_extract_keys_lst = [ report_steps_dct[data_name][1] for data_name in data_names ] # print data which were loaded but for which force extract flag is on force_extract_check(data_names, data_lst, force_extract_keys_lst, max_title) # when any of data_lst was not saved or # force extract flag is on then re-extract data from configueation files if not all(data_lst) or any(force_extract_keys_lst): print( '\nEXTRACTING INFORMATION ABOUT CONNECTED DEVICES (FDMI, NSSHOW, NSCAMSHOW) ...\n' ) # extract chassis parameters names from init file switch_columns = columns_import('switch', max_title, 'columns') # number of switches to check switch_num = len(switch_params_lst) # data imported from init file to extract values from config file params, params_add, comp_keys, match_keys, comp_dct = data_extract_objects( 'connected_dev', max_title) nsshow_params, nsshow_params_add = columns_import( 'connected_dev', max_title, 'nsshow_params', 'nsshow_params_add') # lists to store only REQUIRED infromation # collecting data for all switches ports during looping fdmi_lst = [] # lists with local Name Server (NS) information nsshow_lst = [] nscamshow_lst = [] # dictionary with required to collect nsshow data # first element of list is regular expression pattern number, second - list to collect data nsshow_dct = { 'nsshow': [5, nsshow_lst], 'nscamshow': [6, nscamshow_lst] } # switch_params_lst [[switch_params_sw1], [switch_params_sw1]] # checking each switch for switch level parameters for i, switch_params_data in enumerate(switch_params_lst): # data unpacking from iter param # dictionary with parameters for the current chassis switch_params_data_dct = dict( zip(switch_columns, switch_params_data)) switch_info_keys = [ 'configname', 'chassis_name', 'chassis_wwn', 'switch_index', 'SwitchName', 'switchWwn', 'switchMode' ] switch_info_lst = [ switch_params_data_dct.get(key) for key in switch_info_keys ] ls_mode_on = True if switch_params_data_dct[ 'LS_mode'] == 'ON' else False sshow_file, *_, switch_index, switch_name, _, switch_mode = switch_info_lst # current operation information string info = f'[{i+1} of {switch_num}]: {switch_name} connected devices' print(info, end=" ") # search control dictionary. continue to check sshow_file until all parameters groups are found # Name Server service started only in Native mode collected = {'fdmi': False, 'nsshow': False, 'nscamshow': False} \ if switch_params_data_dct.get('switchMode') == 'Native' else {'fdmi': False} with open(sshow_file, encoding='utf-8', errors='ignore') as file: # check file until all groups of parameters extracted while not all(collected.values()): line = file.readline() if not line: break # fdmi section start # switchcmd_fdmishow_comp if re.search(comp_dct[comp_keys[0]], line): collected['fdmi'] = True if ls_mode_on: while not re.search( fr'^CURRENT CONTEXT -- {switch_index} *, \d+$', line): line = file.readline() if not line: break # local_database_comp while not re.search(comp_dct[comp_keys[4]], line): match_dct = { match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip( comp_keys, match_keys) } # wwnp_match if match_dct[match_keys[1]]: # dictionary to store all DISCOVERED switch ports information # collecting data only for the logical switch in current loop fdmi_dct = {} # switch_info and current connected device wwnp switch_wwnp = line_to_list( comp_dct[comp_keys[1]], line, *switch_info_lst[:6]) # move cursor to one line down to get inside while loop line = file.readline() # wwnp_local_comp while not re.search(comp_dct[comp_keys[3]], line): line = file.readline() match_dct = { match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip( comp_keys, match_keys) } # fdmi_port_match if match_dct[match_keys[2]]: fdmi_dct[match_dct[match_keys[ 2]].group(1).rstrip()] = match_dct[ match_keys[2]].group( 2).rstrip() if not line: break # adding additional parameters and values to the fdmi_dct update_dct(params_add, switch_wwnp, fdmi_dct) # appending list with only REQUIRED port info for the current loop iteration to the list with all fabrics port info fdmi_lst.append([ fdmi_dct.get(param, None) for param in params ]) else: line = file.readline() if not line: break # fdmi section end # only switches in Native mode have Name Server service started if switch_mode == 'Native': # nsshow section start for nsshow_type in nsshow_dct.keys(): # unpacking re number and list to save REQUIRED params re_num, ns_lst = nsshow_dct[nsshow_type] # switchcmd_nsshow_comp, switchcmd_nscamshow_comp if re.search(comp_dct[comp_keys[re_num]], line): collected[nsshow_type] = True if ls_mode_on: while not re.search( fr'^CURRENT CONTEXT -- {switch_index} *, \d+$', line): line = file.readline() if not line: break # switchcmd_end_comp while not re.search(comp_dct[comp_keys[9]], line): # line = file.readline() match_dct = { match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip( comp_keys, match_keys) } # port_pid__match if match_dct[match_keys[7]]: # dictionary to store all DISCOVERED switch ports information # collecting data only for the logical switch in current loop nsshow_port_dct = {} # switch_info and current connected device wwnp switch_pid = line_to_list( comp_dct[comp_keys[7]], line, *switch_info_lst[:6]) # move cursor to one line down to get inside while loop line = file.readline() # pid_switchcmd_end_comp while not re.search( comp_dct[comp_keys[8]], line): match_dct = { match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip( comp_keys, match_keys) } # nsshow_port_match if match_dct[match_keys[2]]: nsshow_port_dct[match_dct[ match_keys[2]].group( 1 ).rstrip()] = match_dct[ match_keys[2]].group( 2).rstrip() line = file.readline() if not line: break # adding additional parameters and values to the fdmi_dct update_dct(nsshow_params_add, switch_pid, nsshow_port_dct) # appending list with only REQUIRED port info for the current loop iteration to the list with all fabrics port info ns_lst.append([ nsshow_port_dct.get( nsshow_param, None) for nsshow_param in nsshow_params ]) else: line = file.readline() if not line: break # nsshow section end status_info('ok', max_title, len(info)) # save extracted data to json file save_data(report_data_lst, data_names, fdmi_lst, nsshow_lst, nscamshow_lst) # verify if loaded data is empty after first iteration and replace information string with empty list else: fdmi_lst, nsshow_lst, nscamshow_lst = verify_data( report_data_lst, data_names, *data_lst) return fdmi_lst, nsshow_lst, nscamshow_lst
def save_xlsx_file(data_frame, sheet_title, report_data_lst, current_date=str(date.today()), force_flag=False): """ Check if excel file exists, check if dataframe sheet is in file, delete sheet with stored dataframe (if sheet tabs number > 1) and save new dataframe """ customer_name, report_path, _, max_title, report_steps_dct = report_data_lst options = {} options['strings_to_formulas'] = False options['strings_to_urls'] = False # report_steps_dct for each data_name contains: export_to_excel flag, # force_extract flag, report_type, step_info, data_description if report_steps_dct.get(sheet_title): export_flag, _, report_type, _, df_decription = report_steps_dct.get( sheet_title) else: info = f'DataFrame {sheet_title}' print(info, end=" ") status_info('unknown', max_title, len(info)) export_flag, report_type = 1, 'unknown' df_decription = '-' # check DataFrame report type to save if report_type == 'report': report_mark = 'SAN_Assessment_Tables' else: report_mark = report_type # current_date = str(date.today()) # construct excel filename file_name = customer_name + '_' + report_mark + '_' + current_date + '.xlsx' # information string info = f'Exporting {sheet_title} table to {report_mark} file' print(info, end=" ") file_path = os.path.join(report_path, file_name) # save DataFrame to excel file if export_to_excel trigger is ON # and DataFrame is not empty if (force_flag or export_flag) and not data_frame.empty: # pd.ExcelWriter has only two file open modes # if file doesn't exist it has be opened in "w" mode otherwise in "a" if os.path.isfile(file_path): file_mode = 'a' # open existing excel file workbook = openpyxl.load_workbook(file_path) if sheet_title in workbook.sheetnames: # after sheet removal excel file must contain at least one sheet if len(workbook.sheetnames) != 1: del workbook[sheet_title] try: workbook.save(file_path) except PermissionError: status_info('fail', max_title, len(info)) print('\nPermission denied. Close the file.\n') sys.exit() else: file_mode = 'w' else: file_mode = 'w' # if required export DataFrame to the new sheet if file_mode: try: with pd.ExcelWriter(file_path, mode=file_mode, options=options) as writer: # pylint: disable=abstract-class-instantiated # table of content item generation start # if file is new then create table of content if file_mode == 'w': # save current item with header content_df = pd.DataFrame( [[sheet_title, df_decription]], columns=['Закладка', 'Название таблицы']) content_df.to_excel(writer, sheet_name='Содержание', index=False) writer.save() # create hyperlink to wotksheet workbook = openpyxl.load_workbook(file_path) writer.book = workbook writer.sheets = { ws.title: ws for ws in workbook.worksheets } startrow = workbook['Содержание'].max_row cell_address = 'A' + str(startrow) create_hyperlink(workbook['Содержание'], cell_address, sheet_name=sheet_title, cell_ref='A1', display_name=None) # if file exist new table of content item is appended if it's not exist elif file_mode == 'a': workbook = openpyxl.load_workbook(file_path) # check if item exist in menu item_exist = [] for row in workbook.get_sheet_by_name( 'Содержание').values: if any(value == sheet_title for value in row): item_exist.append(True) else: item_exist.append(False) # if not exist add item without header and create hyperlink if not any(item_exist): # it makes append item to the same worksheet instead of making new one writer.book = workbook writer.sheets = { ws.title: ws for ws in workbook.worksheets } # content item to add content_df = pd.DataFrame( [[sheet_title, df_decription]], columns=['Закладка', 'Название таблицы']) # find row and cell to add new item and hyperlink startrow = writer.sheets['Содержание'].max_row cell_address = 'A' + str(startrow + 1) # add content item content_df.to_excel(writer, sheet_name='Содержание', startrow=startrow, index=False, header=False) # create hyperlink create_hyperlink(workbook['Содержание'], cell_address, sheet_name=sheet_title, cell_ref='A1', display_name=None) # table of content item generation finish # DataFrame save to worksheet start # check if DataFrame have MultiIndex # reset index if True if isinstance(data_frame.index, pd.MultiIndex): data_frame_flat = data_frame.reset_index() # keep index if False else: data_frame_flat = data_frame.copy() # saving DataFrame with single Index data_frame_flat.to_excel(writer, sheet_name=sheet_title, startrow=2, index=False) writer.save() workbook = openpyxl.load_workbook(file_path) writer.book = workbook writer.sheets = { ws.title: ws for ws in workbook.worksheets } # add table title workbook[sheet_title]['A1'] = df_decription workbook[sheet_title][ 'A1'].font = openpyxl.styles.fonts.Font(bold=True) # add hyperlink to the Table of contents create_hyperlink(workbook[sheet_title], 'A2', 'Содержание', cell_ref='A2', display_name='К содержанию') workbook.save(file_path) # DataFrame save to worksheet finish except PermissionError: status_info('fail', max_title, len(info)) print('\nPermission denied. Close the file.\n') sys.exit() else: status_info('ok', max_title, len(info)) return file_path else: # if save key is on but DataFrame empty if report_steps_dct[sheet_title][0] and data_frame.empty: status_info('no data', max_title, len(info)) else: status_info('skip', max_title, len(info)) return None
def chassis_params_extract(all_config_data, report_data_lst): """Function to extract chassis parameters""" # report_data_lst contains information: # customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct *_, max_title, report_steps_dct = report_data_lst # names to save data obtained after current module execution data_names = ['chassis_parameters'] # service step information print(f'\n\n{report_steps_dct[data_names[0]][3]}\n') # load data if they were saved on previos program execution iteration data_lst = load_data(report_data_lst, *data_names) # unpacking from the loaded list with data # pylint: disable=unbalanced-tuple-unpacking chassis_params_fabric_lst, = data_lst # data force extract check # list of keys for each data from data_lst representing if it is required # to re-collect or re-analyze data even they were obtained on previous iterations force_extract_keys_lst = [ report_steps_dct[data_name][1] for data_name in data_names ] # print data which were loaded but for which force extract flag is on force_extract_check(data_names, data_lst, force_extract_keys_lst, max_title) # when any of data_lst was not saved or # force extract flag is on then re-extract data from configueation files if not all(data_lst) or any(force_extract_keys_lst): print( '\nEXTRACTING CHASSIS PARAMETERS FROM SUPPORTSHOW CONFIGURATION FILES ...\n' ) # number of switches to check switch_num = len(all_config_data) # list to store only REQUIRED chassis parameters # collecting data for all chassis during looping chassis_params_fabric_lst = [] # data imported from init file to extract values from config file chassis_params, chassis_params_add, comp_keys, match_keys, comp_dct = data_extract_objects( 'chassis', max_title) # all_confg_data format ([swtch_name, supportshow file, (ams_maps_log files, ...)]) # checking each config set(supportshow file) for chassis level parameters for i, switch_config_data in enumerate(all_config_data): # data unpacking from iter param switch_name, sshow_file, ams_maps_file = switch_config_data # search control dictionary. continue to check sshow_file until all parameters groups are found collected = { 'configshow': False, 'uptime_cpu': False, 'flash': False, 'memory': False, 'dhcp': False, 'licenses': False, 'vf_id': False } # dictionary to store all DISCOVERED chassis parameters # collecting data only for the chassis in current loop chassis_params_dct = {} # sets and list to store parameters which could be joined in one group snmp_target_set = set() syslog_set = set() tz_lst = [] licenses = [] vf_id_set = set() # current operation information string info = f'[{i+1} of {switch_num}]: {switch_name} chassis parameters' print(info, end=" ") with open(sshow_file, encoding='utf-8', errors='ignore') as file: # check file until all groups of parameters extracted while not all(collected.values()): line = file.readline() if not line: break # configshow section start if re.search( r'^(/fabos/cliexec/|/bin/cat /var/log/)?configshow *-?(all)? *:$', line): # when section is found corresponding collected dict values changed to True collected['configshow'] = True while not re.search( r'^(\[Chassis Configuration End\])|(real [\w.]+)|(\*\* SS CMD END \*\*)$', line): line = file.readline() # dictionary with match names as keys and match result of current line with all imported regular expressions as values match_dct = { match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip( comp_keys, match_keys) } # match_keys ['chassis_param_match', 'snmp_target_match', 'syslog_match', 'tz_match', 'uptime_cpu_match', 'memory_match', 'flash_match'] # 'chassis_param_match' if match_dct[match_keys[0]]: chassis_params_dct[match_dct[ match_keys[0]].group(1).rstrip( )] = match_dct[match_keys[0]].group(3) # for snmp and syslog data addresses are added to the coreesponding sets to avoid duplicates # 'snmp_target_match' if match_dct[match_keys[1]] and match_dct[ match_keys[1]].group(2) != '0.0.0.0': snmp_target_set.add( match_dct[match_keys[1]].group(2)) # 'syslog_match' if match_dct[match_keys[2]]: syslog_set.add( match_dct[match_keys[2]].group(2)) # for timezone extracted data added to the list for later concatenation # 'tz_match' if match_dct[match_keys[3]]: tz_lst.append( match_dct[match_keys[3]].group(2)) if not line: break # config section end # uptime section start elif re.search(r'^(/fabos/cliexec/)?uptime *:$', line): collected['uptime_cpu'] = True while not re.search(r'^real [\w.]+$', line): line = file.readline() match_dct = { match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip( comp_keys, match_keys) } # 'uptime_cpu_match' if match_dct[match_keys[4]]: uptime = match_dct[match_keys[4]].group(1) cpu_load = match_dct[match_keys[4]].group(2) if not line: break # uptime section end # memory section start elif re.search(r'^(/bin/)?(cat\s+)?/proc/meminfo\s*:$', line): collected['memory'] = True memory_dct = {} while not re.search(r'^real [\w.]+$', line): line = file.readline() match_dct = { match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip( comp_keys, match_keys) } # 'memory_match' if match_dct[match_keys[5]]: memory_dct[match_dct[match_keys[5]].group( 1)] = match_dct[match_keys[5]].group(2) if not line: break # free_mem + buffers > 5% from total memory # memory usage < 95% memory = round((1 - (int(memory_dct['MemFree']) + int(memory_dct['Buffers'])) / int(memory_dct['MemTotal'])) * 100) memory = str(memory) # memory section end # flash section start elif re.search(r'^(/bin/)?df\s*:$', line): collected['flash'] = True while not re.search(r'^real [\w.]+$', line): line = file.readline() match_dct = { match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip( comp_keys, match_keys) } # 'flash_match' if match_dct[match_keys[6]]: flash = match_dct[match_keys[6]].group(1) if not line: break # flash section end # ipaddrshow section start if re.search(r'^(/fabos/link_bin/)?ipaddrshow *:$', line): collected['dhcp'] = True while not re.search( r'^(real [\w.]+)|(\*\* SS CMD END \*\*)$', line): line = file.readline() match_dct = { match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip( comp_keys, match_keys) } # 'dhcp_match' if match_dct[match_keys[7]]: chassis_params_dct[match_dct[ match_keys[7]].group(1).rstrip( )] = match_dct[match_keys[7]].group(2) if not line: break # ipaddrshow section end # licenses section start if re.search(r'^(/fabos/cliexec/)?licenseshow *:$', line): collected['licenses'] = True while not re.search( r'^(real [\w.]+)|(\*\* SS CMD END \*\*)$', line): line = file.readline() match_dct = { match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip( comp_keys, match_keys) } # 'licenses_match' if match_dct[match_keys[8]]: licenses.append( match_dct[match_keys[8]].group(1)) elif re.match('^No licenses installed.$', line): licenses = 'No licenses installed' if not line: break # licenses section end # LS indexes identification start if re.search(r'Section *: +SSHOW_FABRIC', line): collected['vf_id'] = True while not re.search( r'^(SWITCHCMD /fabos/cliexec/)?dom *:$|Non-VF', line): if re.search(r'CURRENT +CONTEXT +-- +(\d+) *, \d+', line): id = re.match( r'CURRENT +CONTEXT +-- +(\d+) *, \d+', line).group(1) vf_id_set.add(id) line = file.readline() else: line = file.readline() if not line: break # LS indexes identification end # additional values which need to be added to the chassis params dictionary # chassis_params_add order (configname, ams_maps_log, chassis_name, snmp_server, syslog_server, timezone_h:m, uptime, cpu_average_load, memory_usage, flash_usage, licenses) # values axtracted in manual mode. if change values order change keys order in init.xlsx "chassis_params_add" column vf_id_lst = list(vf_id_set) vf_id_lst.sort() chassis_params_values = (sshow_file, ams_maps_file, switch_name, vf_id_lst, snmp_target_set, syslog_set, tz_lst, uptime, cpu_load, memory, flash, licenses) # adding additional parameters and values to the chassis_params_switch_dct for chassis_param_add, chassis_param_value in zip( chassis_params_add, chassis_params_values): if chassis_param_value: if not isinstance(chassis_param_value, str): s = ':' if chassis_param_add == 'timezone_h:m' else ', ' chassis_param_value = f'{s}'.join(chassis_param_value) chassis_params_dct[chassis_param_add] = chassis_param_value # creating list with REQUIRED chassis parameters for the current switch # if no value in the chassis_params_dct for the parameter then None is added # and appending this list to the list of all switches chassis_params_fabric_lst chassis_params_fabric_lst.append([ chassis_params_dct.get(chassis_param, None) for chassis_param in chassis_params ]) status_info('ok', max_title, len(info)) # save extracted data to json file save_data(report_data_lst, data_names, chassis_params_fabric_lst) # verify if loaded data is empty after first iteration and replace information string with empty list else: chassis_params_fabric_lst = verify_data(report_data_lst, data_names, *data_lst) return chassis_params_fabric_lst
def interswitch_connection_extract(switch_params_lst, report_data_lst): """Function to extract interswitch connection information""" # report_data_lst contains information: # customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct *_, max_title, report_steps_dct = report_data_lst # names to save data obtained after current module execution data_names = ['isl', 'trunk', 'porttrunkarea'] # service step information print(f'\n\n{report_steps_dct[data_names[0]][3]}\n') # load data if they were saved on previos program execution iteration data_lst = load_data(report_data_lst, *data_names) # unpacking from the loaded list with data # pylint: disable=unbalanced-tuple-unpacking isl_lst, trunk_lst, porttrunkarea_lst = data_lst # data force extract check # list of keys for each data from data_lst representing if it is required # to re-collect or re-analyze data even they were obtained on previous iterations force_extract_keys_lst = [ report_steps_dct[data_name][1] for data_name in data_names ] # print data which were loaded but for which force extract flag is on force_extract_check(data_names, data_lst, force_extract_keys_lst, max_title) # when any of data_lst was not saved or # force extract flag is on then re-extract data from configueation files if not all(data_lst) or any(force_extract_keys_lst): print( '\nEXTRACTING INTERSWITCH CONNECTION INFORMATION (ISL, TRUNK, TRUNKAREA) ...\n' ) # extract chassis parameters names from init file switch_columns = columns_import('switch', max_title, 'columns') # number of switches to check switch_num = len(switch_params_lst) # data imported from init file to extract values from config file *_, comp_keys, match_keys, comp_dct = data_extract_objects( 'isl', max_title) # lists to store only REQUIRED infromation # collecting data for all switches ports during looping isl_lst = [] trunk_lst = [] porttrunkarea_lst = [] # switch_params_lst [[switch_params_sw1], [switch_params_sw1]] # checking each switch for switch level parameters for i, switch_params_data in enumerate(switch_params_lst): # data unpacking from iter param # dictionary with parameters for the current switch switch_params_data_dct = dict( zip(switch_columns, switch_params_data)) switch_info_keys = [ 'configname', 'chassis_name', 'chassis_wwn', 'switch_index', 'SwitchName', 'switchWwn', 'switchRole', 'Fabric_ID', 'FC_Router', 'switchMode' ] switch_info_lst = [ switch_params_data_dct.get(key) for key in switch_info_keys ] ls_mode_on = True if switch_params_data_dct[ 'LS_mode'] == 'ON' else False sshow_file, _, _, switch_index, switch_name, *_, switch_mode = switch_info_lst # current operation information string info = f'[{i+1} of {switch_num}]: {switch_name} isl, trunk and trunk area ports. Switch mode: {switch_mode}' print(info, end=" ") # search control dictionary. continue to check sshow_file until all parameters groups are found collected = {'isl': False, 'trunk': False, 'trunkarea': False} if switch_mode == 'Native': with open(sshow_file, encoding='utf-8', errors='ignore') as file: # check file until all groups of parameters extracted while not all(collected.values()): line = file.readline() if not line: break # isl section start # switchcmd_islshow_comp if re.search(comp_dct[comp_keys[0]], line) and not collected['isl']: collected['isl'] = True if ls_mode_on: while not re.search( fr'^CURRENT CONTEXT -- {switch_index} *, \d+$', line): line = file.readline() if not line: break # switchcmd_end_comp while not re.search(comp_dct[comp_keys[2]], line): line = file.readline() match_dct = { match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip( comp_keys, match_keys) } # islshow_match if match_dct[match_keys[1]]: isl_port = line_to_list( comp_dct[comp_keys[1]], line, *switch_info_lst[:-1]) # portcfg parameters if isl_port[-1]: isl_port[-1] = isl_port[-1].replace( ' ', ', ') # appending list with only REQUIRED port info for the current loop iteration # to the list with all ISL port info isl_lst.append(isl_port) if not line: break # isl section end # trunk section start # switchcmd_trunkshow_comp if re.search(comp_dct[comp_keys[3]], line) and not collected['trunk']: collected['trunk'] = True if ls_mode_on: while not re.search( fr'^CURRENT CONTEXT -- {switch_index} *, \d+$', line): line = file.readline() if not line: break # switchcmd_end_comp while not re.search(comp_dct[comp_keys[2]], line): match_dct = { match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip( comp_keys, match_keys) } # trunkshow_match if match_dct[match_keys[4]]: trunk_port = line_to_list( comp_dct[comp_keys[4]], line, *switch_info_lst[:-1]) # if trunk line has trunk number then remove ":" from trunk number if trunk_port[9]: trunk_port[9] = trunk_port[9].strip( ':') trunk_num = trunk_port[9] # if trunk line has no number then use number from previous line else: trunk_port[9] = trunk_num # appending list with only REQUIRED trunk info for the current loop iteration # to the list with all trunk port info trunk_lst.append(trunk_port) line = file.readline() if not line: break # trunk section end # porttrunkarea section start # switchcmd_trunkarea_comp if re.search(comp_dct[comp_keys[5]], line) and not collected['trunkarea']: collected['trunkarea'] = True if ls_mode_on: while not re.search( fr'^CURRENT CONTEXT -- {switch_index} *, \d+$', line): line = file.readline() if not line: break # switchcmd_end_comp while not re.search(comp_dct[comp_keys[2]], line): line = file.readline() match_dct = { match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip( comp_keys, match_keys) } # 'porttrunkarea_match' if match_dct[match_keys[6]]: porttrunkarea_port_lst = line_to_list( comp_dct[comp_keys[6]], line, *switch_info_lst[:6]) # for No_light ports port and slot numbers are '--' if porttrunkarea_port_lst[11] == '--': porttrunkarea_port_lst[10] = '--' # if switch has no slots than slot number is 0 for idx in [6, 10]: if not porttrunkarea_port_lst[idx]: porttrunkarea_port_lst[idx] = str( 0) porttrunkarea_lst.append( porttrunkarea_port_lst) if not line: break # porttrunkarea section end status_info('ok', max_title, len(info)) # if switch in Access Gateway mode then skip else: status_info('skip', max_title, len(info)) # save extracted data to json file save_data(report_data_lst, data_names, isl_lst, trunk_lst, porttrunkarea_lst) # verify if loaded data is empty after first iteration and replace information string with empty list else: isl_lst, trunk_lst, porttrunkarea_lst = verify_data( report_data_lst, data_names, *data_lst) return isl_lst, trunk_lst, porttrunkarea_lst
def fabricshow_extract(switch_params_lst, report_data_lst): """ Function to extract from principal switch configuration list of switches in fabric including AG switches """ # report_data_lst contains information: # customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct *_, max_title, report_steps_dct = report_data_lst # names to save data obtained after current module execution data_names = ['fabricshow', 'ag_principal'] # service step information print(f'\n\n{report_steps_dct[data_names[0]][3]}\n') # load data if they were saved on previos program execution iteration data_lst = load_data(report_data_lst, *data_names) # unpacking from the loaded list with data # pylint: disable=unbalanced-tuple-unpacking fabricshow_lst, ag_principal_lst = data_lst # data force extract check # list of keys for each data from data_lst representing if it is required # to re-collect or re-analyze data even they were obtained on previous iterations force_extract_keys_lst = [ report_steps_dct[data_name][1] for data_name in data_names ] # print data which were loaded but for which force extract flag is on force_extract_check(data_names, data_lst, force_extract_keys_lst, max_title) # when any of data_lst was not saved or # force extract flag is on then re-extract data from configueation files if not all(data_lst) or any(force_extract_keys_lst): print( '\nEXTRACTING FABRICS INFORMATION FROM SUPPORTSHOW CONFIGURATION FILES ...\n' ) # extract switch parameters names from init file switch_columns = columns_import('switch', max_title, 'columns') # number of switches to check switch_num = len(switch_params_lst) # list to store only REQUIRED switch parameters # collecting data for all switches during looping fabricshow_lst = [] ag_principal_lst = [] # data imported from init file to extract values from config file *_, comp_keys, match_keys, comp_dct = data_extract_objects( 'fabricshow', max_title) ag_params = columns_import('fabricshow', max_title, 'ag_params') # switch_params_lst [[switch_params_sw1], [switch_params_sw1]] # checking each switch for switch level parameters for i, switch_params_data in enumerate(switch_params_lst): # data unpacking from iter param # dictionary with parameters for the current switch switch_params_data_dct = dict( zip(switch_columns, switch_params_data)) switch_info_keys = [ 'configname', 'chassis_name', 'chassis_wwn', 'switch_index', 'SwitchName', 'switchWwn', 'switchRole', 'Fabric_ID', 'FC_Router', 'switchMode' ] switch_info_lst = [ switch_params_data_dct[key] for key in switch_info_keys ] ls_mode_on = True if switch_params_data_dct[ 'LS_mode'] == 'ON' else False sshow_file, _, _, switch_index, switch_name, _, switch_role = switch_info_lst[: 7] # current operation information string info = f'[{i+1} of {switch_num}]: {switch_name} fabric environment. Switch role: {switch_role}' print(info, end=" ") collected = {'fabricshow': False, 'ag_principal': False} # switch_params_data_dct.get('FC_Router') == 'ON' # print('collected', collected) # check config of Principal switch only if switch_role == 'Principal': # principal_switch_lst contains sshow_file, chassis_name, chassis_wwn, switch_index, switch_name, switch_fid principal_switch_lst = [ *switch_info_lst[:6], *switch_info_lst[7:9] ] # search control dictionary. continue to check sshow_file until all parameters groups are found with open(sshow_file, encoding='utf-8', errors='ignore') as file: # check file until all groups of parameters extracted while not all(collected.values()): line = file.readline() if not line: break # fabricshow section start if re.search( r'^(SWITCHCMD /fabos/cliexec/)?fabricshow\s*:$', line): # when section is found corresponding collected dict values changed to True collected['fabricshow'] = True if ls_mode_on: while not re.search( fr'^CURRENT CONTEXT -- {switch_index} *, \d+$', line): line = file.readline() if not line: break while not re.search( r'^(real [\w.]+)|(\*\* SS CMD END \*\*)$', line): line = file.readline() # dictionary with match names as keys and match result of current line with all imported regular expressions as values match_dct = { match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip( comp_keys, match_keys) } # match_keys ['fabricshow_match'] # 'fabricshow_match' if match_dct[match_keys[0]]: fabricshow_lst.append( line_to_list(comp_dct[comp_keys[0]], line, *principal_switch_lst)) if not line: break # ag_principal section start # switchcmd_agshow_comp if re.search(comp_dct[comp_keys[4]], line): collected['ag_principal'] = True # if switch in LS mode switch to required LS number if ls_mode_on: while not re.search( fr'^CURRENT CONTEXT -- {switch_index} *, \d+$', line): line = file.readline() if not line: break # switchcmd_end_comp while not re.search(comp_dct[comp_keys[10]], line): match_dct = { match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip( comp_keys, match_keys) } # ag_num_match if match_dct[match_keys[5]]: # dictionary to store all DISCOVERED switch ports information # collecting data only for the logical switch in current loop # Access Gateway common information dictionary ag_info_dct = {} # Attached F-Port information dictionary ag_attach_dct = {} # Access Gateway F-Port information dictionary ag_fport_dct = {} # Domaid ID, port_ID, port_index dictionary did_port_dct = {} # move cursor to one line down to get inside while loop line = file.readline() # ag_switchcmd_end_comp while not re.search( comp_dct[comp_keys[9]], line): match_dct = { match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip( comp_keys, match_keys) } # ag_info_match if match_dct[match_keys[6]]: ag_info_dct[match_dct[ match_keys[6]].group( 1).rstrip()] = match_dct[ match_keys[6]].group( 2).rstrip() # ag_attached_match elif match_dct[match_keys[7]]: # if Attached F-Port information dictionary is empty than create dictionary with N-Port ID(s) as keys and empty lists as values # if ag_attach_dct has been already created (not empty) then it's preserved ag_attach_dct = ag_attach_dct or dict( (n_portid, []) for n_portid in ag_info_dct['N-Port ID(s)']. split(',')) # extracting attached F-port data from line to list ag_attach_lst = line_to_list( comp_dct[comp_keys[7]], line) # getting port_ID of N-port from port_id of F-port n_portid = ag_attach_lst[ 0][:-2] + '00' # adding current line F-port information to Attached F-Port information dictionary if n_portid in ag_attach_dct.keys( ): ag_attach_dct[n_portid].append( ag_attach_lst) # ag_fport_match elif match_dct[match_keys[8]]: # create Access Gateway F-Port information dictionary ag_fport_dct = ag_fport_dct or dict( (n_portid, []) for n_portid in ag_info_dct['N-Port ID(s)']. split(',')) # extracting access gateway F-port data from line to list ag_fport_lst = line_to_list( comp_dct[comp_keys[8]], line) # getting port_ID of N-port from port_id of F-port n_portid = ag_fport_lst[ 1][:-2] + '00' # adding current line F-port information to Access Gateway F-Port information dictionary if n_portid in ag_fport_dct.keys(): ag_fport_dct[n_portid].append( ag_fport_lst) line = file.readline() if not line: break # list of N-ports extracted from N-Port ID(s) line n_portids_lst = ag_info_dct[ 'N-Port ID(s)'].split(',') # (domain_id, n_portid) did_port_lst = [ (int(n_portid[:4], 0), n_portid) for n_portid in n_portids_lst ] # creating dictionary with n_portid as keys and (domain_id, n_portid) as values did_port_dct = { port[1]: list(port) for port in did_port_lst } # change values representation in dictionaries # before {n_portid: [(port_id_1, port_wwn_1, f-port_num_1)], [(port_id_2, port_wwn_2, f-port_num_2)]} # after {n_portid: [(port_id_1, port_id_2), (port_wwn_1, port_wwn_2), (f-port_num_1, f-port_num_1)] ag_attach_dct = { n_portid: list(zip(*ag_attach_dct[n_portid])) for n_portid in n_portids_lst if ag_attach_dct.get(n_portid) } ag_fport_dct = { n_portid: list(zip(*ag_fport_dct[n_portid])) for n_portid in n_portids_lst if ag_fport_dct.get(n_portid) } # add connected switch port_index to did_port_dct extracted from ag_attach_dct # (domain_id, n_portid, n_port_index) # if no port_index then add None for n_portid in n_portids_lst: if ag_attach_dct.get(n_portid): did_port_dct[n_portid].append( ag_attach_dct[n_portid][2][0]) else: did_port_dct[n_portid].append(None) # for each element of list convert tuples to strings # if no data extracted for the n_portid then add None for each parameter for n_portid in n_portids_lst: if ag_attach_dct.get(n_portid): ag_attach_dct[n_portid] = [ ', '.join(v) for v in ag_attach_dct[n_portid] ] else: ag_attach_dct[n_portid] = [None ] * 3 for n_portid in n_portids_lst: if ag_fport_dct.get(n_portid): ag_fport_dct[n_portid] = [ ', '.join(v) for v in ag_fport_dct[n_portid] ] else: ag_fport_dct[n_portid] = [None] * 3 # getting data from ag_info_dct in required order ag_info_lst = [ ag_info_dct.get(param, None) for param in ag_params ] # appending list with only REQUIRED ag info for the current loop iteration to the list with all ag switch info for n_portid in n_portids_lst: ag_principal_lst.append([ *principal_switch_lst[:-1], *ag_info_lst, *did_port_dct[n_portid], *ag_attach_dct[n_portid], *ag_fport_dct[n_portid] ]) else: line = file.readline() if not line: break # ag_principal section end status_info('ok', max_title, len(info)) else: status_info('skip', max_title, len(info)) # save extracted data to json file save_data(report_data_lst, data_names, fabricshow_lst, ag_principal_lst) else: fabricshow_lst, ag_principal_lst = verify_data(report_data_lst, data_names, *data_lst) return fabricshow_lst, ag_principal_lst
def blade_system_extract(blade_folder, report_data_lst): """Function to extract blade systems information""" # report_data_lst contains information: # customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct *_, max_title, report_steps_dct = report_data_lst # names to save data obtained after current module execution data_names = ['blade_interconnect', 'blade_servers', 'blade_vc'] # service step information print(f'\n\n{report_steps_dct[data_names[0]][3]}\n') # load data if they were saved on previos program execution iteration data_lst = load_data(report_data_lst, *data_names) # unpacking from the loaded list with data # pylint: disable=unbalanced-tuple-unpacking module_comprehensive_lst, blades_comprehensive_lst, blade_vc_comprehensive_lst = data_lst # data force extract check # list of keys for each data from data_lst representing if it is required # to re-collect or re-analyze data even they were obtained on previous iterations force_extract_keys_lst = [ report_steps_dct[data_name][1] for data_name in data_names ] # print data which were loaded but for which force extract flag is on force_extract_check(data_names, data_lst, force_extract_keys_lst, max_title) # when any of data_lst was not saved or # force extract flag is on then re-extract data from configueation files if not all(data_lst) or any(force_extract_keys_lst): # lists to store only REQUIRED infromation # collecting data for all blades during looping # list containing enclosure, blade and hba information for all blade systems blades_comprehensive_lst = [] # list containing enclosure and interconnect modules information for all blade systems module_comprehensive_lst = [] # list containing virtual connect ports information for all blade systems blade_vc_comprehensive_lst = [] if blade_folder: print('\nEXTRACTING BLADES SYSTEM INFORMATION ...\n') # collects files in folder with txt extension txt_files = find_files(blade_folder, max_title, filename_extension='txt') log_files = find_files(blade_folder, max_title, filename_extension='log') blade_configs_lst = txt_files + log_files # number of files to check configs_num = len(blade_configs_lst) if configs_num: # data imported from init file to extract values from config file enclosure_params, _, comp_keys, match_keys, comp_dct = data_extract_objects( 'blades', max_title) module_params = columns_import('blades', max_title, 'module_params') blade_params = columns_import('blades', max_title, 'blade_params') for i, blade_config in enumerate(blade_configs_lst): # file name with extension configname_wext = os.path.basename(blade_config) # remove extension from filename configname, _ = os.path.splitext(configname_wext) # Active Onboard Administrator IP address oa_ip = None # interconnect modules number module_num = 0 # current operation information string info = f'[{i+1} of {configs_num}]: {configname} system.' print(info, end=" ") # search control dictionary. continue to check file until all parameters groups are found collected = { 'enclosure': False, 'oa_ip': False, 'module': False, 'servers': False, 'vc': False } # if blade_lst remains empty after file checkimng than status_info shows NO_DATA for current file blade_lst = [] enclosure_vc_lst = [] with open(blade_config, encoding='utf-8', errors='ignore') as file: # check file until all groups of parameters extracted while not all(collected.values()): line = file.readline() if not line: break # enclosure section start if re.search( r'>SHOW ENCLOSURE INFO|^ +ENCLOSURE INFORMATION$', line): enclosure_dct = {} collected['enclosure'] = True # while not reach empty line while not re.search(r'Serial Number', line): line = file.readline() # dictionary with match names as keys and match result of current line with all imported regular expressions as values match_dct = { match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip( comp_keys, match_keys) } # name_value_pair_match if match_dct[match_keys[0]]: result = match_dct[match_keys[0]] enclosure_dct[result.group(1).strip( )] = result.group(2).strip() if not line: break # rename Description key to Enclosure Type key for VC if enclosure_dct.get('Description'): enclosure_dct[ 'Enclosure Type'] = enclosure_dct.pop( 'Description') # creating list with REQUIRED enclosure information only enclosure_lst = [ enclosure_dct.get(param) for param in enclosure_params ] # enclosure section end # vc fabric connection section start elif re.search(r'FABRIC INFORMATION', line): info_type = 'Type VC' print(info_type, end=" ") info = info + " " + info_type line = file.readline() collected['vc'] = True while not re.search( r'FC-CONNECTION INFORMATION', line): # dictionary with match names as keys and match result of current line with all imported regular expressions as values match_dct = { match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip( comp_keys, match_keys) } # vc_port_match if match_dct[match_keys[14]]: vc_port = line_to_list( comp_dct[comp_keys[14]], line, *enclosure_lst) enclosure_vc_lst.append(vc_port) blade_vc_comprehensive_lst.append( vc_port) line = file.readline() else: line = file.readline() if not line: break # vc fabric connection section end # active onboard administrator ip section start elif re.search(r'>SHOW TOPOLOGY *$', line): info_type = 'Type Blade Enclosure' print(info_type, end=" ") info = info + " " + info_type line = file.readline() collected['oa_ip'] = True while not re.search(r'^>SHOW', line): # dictionary with match names as keys and match result of current line with all imported regular expressions as values match_dct = { match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip( comp_keys, match_keys) } # oa_ip_match if match_dct[match_keys[1]]: oa_ip = match_dct[match_keys[1]].group( 1) line = file.readline() break else: line = file.readline() if not line: break # active onboard administrator ip section end # interconnect modules section start elif re.search(r'>SHOW INTERCONNECT INFO ALL', line): line = file.readline() collected['modules'] = True while not re.search(r'^>SHOW', line): # dictionary with match names as keys and match result of current line with all imported regular expressions as values match_dct = { match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip( comp_keys, match_keys) } # module_type_num_match if match_dct[match_keys[2]]: module_dct = {} module_lst = [] module = match_dct[match_keys[2]] # interconnect module slot number module_slot = module.group(1) # interconnect module type (Ethernet, FC) module_type = module.group(2).rstrip() line = file.readline() # module_section_end_comp while not re.search( comp_dct[comp_keys[3]], line): # dictionary with match names as keys and match result of current line with all imported regular expressions as values match_dct = { match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip( comp_keys, match_keys) } # name_value_pair_match if match_dct[match_keys[0]]: result = match_dct[ match_keys[0]] name = result.group(1).strip() value = result.group(2).strip() # if value is empty string use None if value == '': value = None module_dct[name] = value line = file.readline() else: line = file.readline() if not line: break # creating list with REQUIRED interconnect module information only module_lst = [ module_dct.get(param) for param in module_params ] # add current module information to list containing all modules infromation # oa_ip added as None and extracted later in the file module_comprehensive_lst.append([ *enclosure_lst, oa_ip, module_slot, module_type, *module_lst ]) # based on module's number oa_ip is added to module_comprehensive_lst after extraction module_num += 1 else: line = file.readline() if not line: break # interconnect modules section end # blade server, hba and flb section start elif re.search(r'>SHOW SERVER INFO ALL', line): line = file.readline() collected['servers'] = True while not re.search(r'^>SHOW', line): # dictionary with match names as keys and match result of current line with all imported regular expressions as values match_dct = { match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip( comp_keys, match_keys) } # blade_server_num_match if match_dct[match_keys[4]]: blade_dct = {} blade_lst = [] hba_lst = [] result = match_dct[match_keys[4]] blade_dct[result.group( 1)] = result.group(2) # blade_num = result.group(2) # print("Blade number:", blade_num) line = file.readline() # server_section_end_comp while not re.search( comp_dct[comp_keys[11]], line): # dictionary with match names as keys and match result of current line with all imported regular expressions as values match_dct = { match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip( comp_keys, match_keys) } # mezzanin hba section start # mezzanine_model_match if match_dct[match_keys[6]]: result = match_dct[ match_keys[6]] hba_description = result.group( 1) hba_model = result.group(2) line = file.readline() # mezzanine_wwn_comp while re.search( comp_dct[comp_keys[7]], line): # dictionary with match names as keys and match result of current line with all imported regular expressions as values match_dct = { match_key: comp_dct[comp_key]. match(line) for comp_key, match_key in zip( comp_keys, match_keys) } # mezzanine_wwn_match result = match_dct[ match_keys[7]] wwnp = result.group(1) hba_lst.append([ hba_description, hba_model, wwnp ]) line = file.readline() # mezzanin hba section end # flex flb hba section start # flb_model_match and flex_ethernet_match elif match_dct[match_keys[ 8]] or match_dct[ match_keys[15]]: if match_dct[match_keys[8]]: result = match_dct[ match_keys[8]] flex_description = result.group( 1) if re.search( comp_dct[ comp_keys[13]], line): flex_model = re.search( comp_dct[ comp_keys[13]], line).group(1) else: flex_model = None elif match_dct[match_keys[15]]: result = match_dct[ match_keys[15]] flex_description = result.group( 1) flex_model = result.group( 1) line = file.readline() # wwn_mac_line_comp while re.search( comp_dct[comp_keys[9]], line): # dictionary with match names as keys and match result of current line with all imported regular expressions as values match_dct = { match_key: comp_dct[comp_key]. match(line) for comp_key, match_key in zip( comp_keys, match_keys) } # flb_wwn_match if match_dct[ match_keys[10]]: result = match_dct[ match_keys[10]] wwnp = result.group(1) hba_lst.append([ flex_description, flex_model, wwnp ]) line = file.readline() # flex flb hba section end # blade server section start # blade_server_info_match elif match_dct[match_keys[5]]: result = match_dct[ match_keys[5]] # name = result.group(1) + result.group(2) if result.group(2) else result.group(1) name = result.group(1).rstrip() value = result.group( 3).rstrip() # to avoid Type parameter overwrire # add parameter only if parameter has not been added to blade dictionary before if not blade_dct.get(name): blade_dct[name] = value line = file.readline() # blade server section end # if none of matches found for current blade server than next line else: line = file.readline() if not line: break # unpopulated blade slots have 'Server Blade Type' line but populated have 'Type' line # add 'Server Blade Type' parameter for populated slots for consistency if blade_dct.get('Type'): blade_dct[ 'Server Blade Type'] = blade_dct.pop( 'Type') # creating list with REQUIRED blade server information only blade_lst = [ blade_dct.get(param) for param in blade_params ] # if hba or flex cards installed in blade server if len(hba_lst): # add line for each hba to blades_comprehensive_lst for hba in hba_lst: blades_comprehensive_lst.append( [ *enclosure_lst, *blade_lst, *hba ]) # if no nba add one line with enclosure and blade info only else: blades_comprehensive_lst.append([ *enclosure_lst, *blade_lst, None, None ]) # if no blade_server_num_match found in >SHOW SERVER INFO ALL section than next line else: line = file.readline() if not line: break # blade server, hba and flb section end # adding OA IP to module_comprehensive_lst based on interconnect modules number for num in range(-1, -module_num - 1, -1): module_comprehensive_lst[num][3] = oa_ip # show status blades information extraction from file if blade_lst or enclosure_vc_lst: status_info('ok', max_title, len(info)) else: status_info('no data', max_title, len(info)) # save extracted data to json file save_data(report_data_lst, data_names, module_comprehensive_lst, blades_comprehensive_lst, blade_vc_comprehensive_lst) else: # current operation information string info = f'Collecting enclosure, interconnect modules, blade servers, hba' print(info, end=" ") status_info('skip', max_title, len(info)) # save empty data to json file save_data(report_data_lst, data_names, module_comprehensive_lst, blades_comprehensive_lst, blade_vc_comprehensive_lst) # verify if loaded data is empty after first iteration and replace information string with empty list else: module_comprehensive_lst, blades_comprehensive_lst, blade_vc_comprehensive_lst = verify_data( report_data_lst, data_names, *data_lst) return module_comprehensive_lst, blades_comprehensive_lst, blade_vc_comprehensive_lst
def zoning_analysis_main(switch_params_aggregated_df, portshow_aggregated_df, cfg_df, zone_df, alias_df, cfg_effective_df, fcrfabric_df, lsan_df, peerzone_df, report_columns_usage_dct, report_data_lst): """Main function to analyze zoning configuration""" # report_data_lst contains information: # customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct *_, max_title, report_steps_dct = report_data_lst # names to save data obtained after current module execution data_names = [ 'zoning_aggregated', 'alias_aggregated', 'zonemember_statistics', 'portshow_zoned_aggregated', 'alias_statistics', 'effective_cfg_statistics', 'Зонирование', 'Псевдонимы', 'Зонирование_A&B', 'Порты_не_в_зонах', 'Порты_без_псевдономов', 'Отсутсвуют_в_сети', 'Статистика_зон', 'Статистика_псевдонимов', 'Статистика_конфигурации' ] # service step information print(f'\n\n{report_steps_dct[data_names[0]][3]}\n') # load data if they were saved on previos program execution iteration data_lst = load_data(report_data_lst, *data_names) # unpacking DataFrames from the loaded list with data # pylint: disable=unbalanced-tuple-unpacking zoning_aggregated_df, alias_aggregated_df, zonemember_statistics_df, \ portshow_zoned_aggregated_df, alias_statistics_df, effective_cfg_statistics_df, zoning_report_df, alias_report_df, \ zoning_compare_report_df, unzoned_device_report_df, no_alias_device_report_df, zoning_absent_device_report_df,\ zonemember_statistics_report_df, alias_statistics_report_df, effective_cfg_statistics_report_df = data_lst # list of data to analyze from report_info table analyzed_data_names = [ 'cfg', 'cfg_effective', 'zone', 'alias', 'switch_params_aggregated', 'switch_parameters', 'switchshow_ports', 'chassis_parameters', 'portshow_aggregated', 'device_rename', 'report_columns_usage_upd', 'portcmd', 'fdmi', 'nscamshow', 'nsshow', 'blade_servers', 'fabric_labels' ] # force run when any data from data_lst was not saved (file not found) or # procedure execution explicitly requested for output data or data used during fn execution force_run = verify_force_run(data_names, data_lst, report_steps_dct, max_title, analyzed_data_names) if force_run: # current operation information string info = f'Generating zoning table' print(info, end=" ") # aggregated DataFrames zoning_aggregated_df, alias_aggregated_df \ = zoning_aggregated(switch_params_aggregated_df, portshow_aggregated_df, cfg_df, zone_df, alias_df, cfg_effective_df, fcrfabric_df, lsan_df, peerzone_df, report_data_lst) # create comprehesive statistics DataFrame with Fabric summaries and # zones statistics DataFrame without summaries zonemember_statistics_df, zonemember_zonelevel_stat_df = zonemember_statistics( zoning_aggregated_df, report_data_lst) # add zoning statistics notes, zone duplicates and zone pairs to zoning aggregated DataFrame zoning_aggregated_df = statistics_to_aggregated_zoning( zoning_aggregated_df, zonemember_zonelevel_stat_df) # check all fabric devices (Wwnp) for usage in zoning configuration portshow_zoned_aggregated_df = verify_cfg_type(portshow_aggregated_df, zoning_aggregated_df, ['PortName']) # create alias configuration statistics alias_statistics_df = alias_dashboard(alias_aggregated_df, portshow_zoned_aggregated_df) # create Effective zoning configuration summary statistics effective_cfg_statistics_df = cfg_dashborad( zonemember_statistics_df, portshow_zoned_aggregated_df, zoning_aggregated_df, alias_aggregated_df) # after finish display status status_info('ok', max_title, len(info)) # report tables zoning_report_df, alias_report_df, zoning_compare_report_df, \ unzoned_device_report_df, no_alias_device_report_df, zoning_absent_device_report_df, \ zonemember_statistics_report_df, alias_statistics_report_df, effective_cfg_statistics_report_df = \ zoning_report_main(zoning_aggregated_df, alias_aggregated_df, portshow_zoned_aggregated_df, zonemember_statistics_df, alias_statistics_df, effective_cfg_statistics_df, data_names, report_columns_usage_dct, max_title) # create list with partitioned DataFrames data_lst = [ zoning_aggregated_df, alias_aggregated_df, zonemember_statistics_df, portshow_zoned_aggregated_df, alias_statistics_df, effective_cfg_statistics_df, zoning_report_df, alias_report_df, zoning_compare_report_df, unzoned_device_report_df, no_alias_device_report_df, zoning_absent_device_report_df, zonemember_statistics_report_df, alias_statistics_report_df, effective_cfg_statistics_report_df ] # saving data to json or csv file save_data(report_data_lst, data_names, *data_lst) # verify if loaded data is empty and replace information string with empty DataFrame else: zoning_aggregated_df, alias_aggregated_df, zonemember_statistics_df, \ portshow_zoned_aggregated_df, alias_statistics_df, effective_cfg_statistics_df, zoning_report_df, alias_report_df, \ zoning_compare_report_df, unzoned_device_report_df, no_alias_device_report_df, zoning_absent_device_report_df, \ zonemember_statistics_report_df, alias_statistics_report_df, effective_cfg_statistics_report_df \ = verify_data(report_data_lst, data_names, *data_lst) data_lst = [ zoning_aggregated_df, alias_aggregated_df, zonemember_statistics_df, portshow_zoned_aggregated_df, alias_statistics_df, effective_cfg_statistics_df, zoning_report_df, alias_report_df, zoning_compare_report_df, unzoned_device_report_df, no_alias_device_report_df, zoning_absent_device_report_df, zonemember_statistics_report_df, alias_statistics_report_df, effective_cfg_statistics_report_df ] # save data to service file if it's required for data_name, data_frame in zip(data_names, data_lst): save_xlsx_file(data_frame, data_name, report_data_lst) return zoning_aggregated_df, alias_aggregated_df, portshow_zoned_aggregated_df
def switch_params_configshow_extract(chassis_params_fabric_lst, report_data_lst): """Function to extract switch parameters""" # report_data_lst contains information: # customer_name, dir_report, dir to save obtained data, max_title, report_steps_dct *_, max_title, report_steps_dct = report_data_lst # names to save data obtained after current module execution data_names = ['switch_parameters', 'switchshow_ports'] # service step information print(f'\n\n{report_steps_dct[data_names[0]][3]}\n') # load data if they were saved on previos program execution iteration data_lst = load_data(report_data_lst, *data_names) # unpacking from the loaded list with data # pylint: disable=unbalanced-tuple-unpacking switch_params_lst, switchshow_ports_lst = data_lst # data force extract check # list of keys for each data from data_lst representing if it is required # to re-collect or re-analyze data even they were obtained on previous iterations force_extract_keys_lst = [report_steps_dct[data_name][1] for data_name in data_names] # print data which were loaded but for which force extract flag is on force_extract_check(data_names, data_lst, force_extract_keys_lst, max_title) # when any of data_lst was not saved or # force extract flag is on then re-extract data from configueation files if not all(data_lst) or any(force_extract_keys_lst): print('\nEXTRACTING SWITCH PARAMETERS FROM SUPPORTSHOW CONFIGURATION FILES ...\n') # extract chassis parameters names from init file chassis_columns = columns_import('chassis', max_title, 'columns') # number of switches to check switch_num = len(chassis_params_fabric_lst) # list to store only REQUIRED switch parameters # collecting data for all switches during looping switch_params_lst = [] # list to store switch ports details switchshow_ports_lst = [] # data imported from init file to extract values from config file switch_params, params_add, comp_keys, match_keys, comp_dct = data_extract_objects('switch', max_title) # chassis_params_fabric_lst [[chassis_params_sw1], [chassis_params_sw1]] # checking each chassis for switch level parameters for i, chassis_params_data in enumerate(chassis_params_fabric_lst): # data unpacking from iter param # dictionary with parameters for the current chassis chassis_params_data_dct = dict(zip(chassis_columns, chassis_params_data)) sshow_file = chassis_params_data_dct['configname'] chassis_name = chassis_params_data_dct['chassis_name'] chassis_wwn = chassis_params_data_dct['chassis_wwn'] # num_ls = int(chassis_params_data_dct["Number_of_LS"]) if not chassis_params_data_dct["Number_of_LS"] in ['0', None] else 1 # when num of logical switches is 0 or None than mode is Non-VF otherwise VF ls_mode_on = (True if not chassis_params_data_dct["Number_of_LS"] in ['0', None] else False) ls_mode = ('ON' if not chassis_params_data_dct["Number_of_LS"] in ['0', None] else 'OFF') # logical switches indexes. if switch is in Non-VF mode then ls_id is 0 ls_ids = chassis_params_data_dct['LS_IDs'].split(', ') if chassis_params_data_dct['LS_IDs'] else ['0'] # current operation information string info = f'[{i+1} of {switch_num}]: {chassis_params_data_dct["chassis_name"]} switch parameters. Number of LS: {chassis_params_data_dct["Number_of_LS"]}' print(info, end =" ") # check each logical switch in chassis for i in ls_ids: # search control dictionary. continue to check sshow_file until all parameters groups are found collected = {'configshow': False, 'switchshow': False} # dictionary to store all DISCOVERED switch parameters # collecting data only for the logical switch in current loop switch_params_dct = {} with open(sshow_file, encoding='utf-8', errors='ignore') as file: # check file until all groups of parameters extracted while not all(collected.values()): line = file.readline() if not line: break # configshow section start if re.search(fr'^\[Switch Configuration Begin *: *{i}\]$', line) and not collected['configshow']: # when section is found corresponding collected dict values changed to True collected['configshow'] = True while not re.search(fr'^\[Switch Configuration End : {i}\]$',line): line = file.readline() # dictionary with match names as keys and match result of current line with all imported regular expressions as values match_dct ={match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip(comp_keys, match_keys)} # match_keys ['switch_configall_match', 'switch_switchshow_match'] if match_dct[match_keys[0]]: switch_params_dct[match_dct[match_keys[0]].group(1).rstrip()] = match_dct[match_keys[0]].group(3).rstrip() if not line: break # config section end # switchshow section start if re.search(r'^(SWITCHCMD /fabos/bin/)?switchshow *:$', line) and not collected['switchshow']: collected['switchshow'] = True if ls_mode_on: while not re.search(fr'^CURRENT CONTEXT -- {i} *, \d+$',line): line = file.readline() if not line: break while not re.search(r'^real [\w.]+$',line): line = file.readline() match_dct ={match_key: comp_dct[comp_key].match(line) for comp_key, match_key in zip(comp_keys, match_keys)} # 'switch_switchshow_match' if match_dct[match_keys[1]]: switch_params_dct[match_dct[match_keys[1]].group(1).rstrip()] = match_dct[match_keys[1]].group(2).rstrip() # 'ls_attr_match' if match_dct[match_keys[2]]: ls_attr = comp_dct[comp_keys[2]].findall(line)[0] for k, v in zip(ls_attr[::2], ls_attr[1::2]): switch_params_dct[k] = v # 'switchshow_portinfo_match' if match_dct[match_keys[3]]: switchinfo_lst = [sshow_file, chassis_name, chassis_wwn, str(i), switch_params_dct.get('switchName', None), switch_params_dct.get('switchWwn', None), switch_params_dct.get('switchState', None), switch_params_dct.get('switchMode', None) ] switchshow_port_lst = line_to_list(comp_dct[comp_keys[3]], line, *switchinfo_lst) # if switch has no slots than slot number is 0 if not switchshow_port_lst[9]: switchshow_port_lst[9] = str(0) switchshow_ports_lst.append(switchshow_port_lst) if not line: break # switchshow section end # additional values which need to be added to the switch params dictionary # switch_params_add order ('configname', 'chassis_name', 'switch_index', 'ls_mode') # values axtracted in manual mode. if change values order change keys order in init.xlsx switch tab "params_add" column switch_params_values = (sshow_file, chassis_name, chassis_wwn, str(i), ls_mode) if switch_params_dct: # adding additional parameters and values to the switch_params_switch_dct update_dct(params_add, switch_params_values, switch_params_dct) # creating list with REQUIRED chassis parameters for the current switch. # if no value in the switch_params_dct for the parameter then None is added # and appending this list to the list of all switches switch_params_fabric_lst switch_params_lst.append([switch_params_dct.get(switch_param, None) for switch_param in switch_params]) status_info('ok', max_title, len(info)) # save extracted data to json file save_data(report_data_lst, data_names, switch_params_lst, switchshow_ports_lst) # verify if loaded data is empty after first iteration and replace information string with empty list else: switch_params_lst, switchshow_ports_lst = verify_data(report_data_lst, data_names, *data_lst) return switch_params_lst, switchshow_ports_lst