print( 'file "{product}" succeeded to generate "final test" report.' .format(product=logName)) except Exception as e: print(logName + ' fail: Read file fail') print(e) except: print('file "{product}" fail to generate "final test" report.'.format( product=file)) # Export Final Test report targetDir = "finalTestReport" fileName = "FT-2" finalReport.generate_report(product, targetDir, fileName, 'xlsx', 'DUT SN') # Export shipping SN Report targetDir = "shippingSNReport" fileName = "serial" snReport.generate_report(product, targetDir, fileName, 'csv', 'SN') product['sndf'] = snReport.generate_df() # Function test data fields, and generate function test report (funcData) print('*' * 50 + '\nstart generating {product} function test rerport\n'.format( product=product['name']) + '*' * 2) funcData = { 'Application Version_FT1': [], 'DUT SN': [], 'DUT MAC': [],
def gen_and_save_histograms(self, dirs_and_files, dir_priorities, indicator_data, bins, row, rownum, program, missing_data): """Generates and saves histograms for all files passed in Each file passed to gen_and_save_histograms will have a histogram generated for it and saved. Args: dirs_and_files(dict(list(string))): The grade files that were found. See the Examples section for some sample inputs dir_priorities(list(string)): The directory priority to which files should be saved. In cases where data is not found at the highest priority directory, but data is found at a lower priority directory, the highest priority directory will get a copy of the histogram(s) using the data sets in that low priority directory. indicator_data(dict(string)): The indicator data for generating Report objects. The method will pull other required items from here such as Program, Course Number, etc. bins(list(int or float)): The bins for generating Report objects row(pd.Series): The row that the program got all of this indicator data from rownum(int): The row number that all of this indicator data is coming from program(string): The program that all of this indicator data belongs to missing_data: The missing data file Examples: dirs_and_files comes in like this (and dir_priorities is in the same order as the keys):: { 'ENCM': ['MATH 1001 Course Grade - ENCM.xlsx'], 'Core': [ 'MATH 1001 Course Grade - Core.xlsx', 'MATH 1001 Course Grade - Core - By Semester.xlsx' ], 'ECE': ['MATH 1001 Course Grade - ECE.xlsx'] } Histograms get saved like so: * ENCM gets a histogram of the one ENCM file * Core gets 2 histograms of the Core files * ECE gets 1 histogram of the ECE file dirs_and_files comes in like this (and dir_priorities is in the same order as the keys):: { 'ENCM': [], 'Core': ['MATH 1001 Course Grade - Core.xlsx'], 'ECE': [] } Histograms get saved like so: * ENCM gets a copy of a histogram using the data from Core * Core gets a copy of a histogram using their data, but ENCM's indicator information * ECE gets nothing saved to it dirs_and files comes in like this (and dir_priorities is in the same order as the keys):: { 'ENCM': [], 'Core': [ 'MATH 1001 Course Grade - Core.xlsx', 'MATH 1001 Course Grade - Core - Program Comparison.xlsx' ], 'ECE': ['MATH 1001 Course Grade - ECE.xlsx'] } Histograms get saved like so: * ENCM gets two histograms, both copies of the ones using data from Core * Core gets two histograms using their data, but with ENCM's indicator information * ECE gets one histogram using their data, but with ENCM's indicator information dirs_and_files comes in like this (and dir_priorities is in the same order as the keys):: { 'ENCM': [], 'Core': [], 'ECE': [] } Histograms get saved like so: * Only ENCM gets a histogram. That histogram indicates that no data was found """ logging.info("Preparing to save and generate a list of histograms") logging.debug("dirs_and_files is %s", str(dirs_and_files)) logging.debug("dir_priorities is %s", str(dir_priorities)) # Keep track of the number of locations that come up empty empty_locations = 0 # Indicates if an extra copy of the histogram needs to be saved to the top priority dir save_extra_to = "" if dirs_and_files[ dir_priorities[0]] == []: # If the top priority dir is empty logging.debug( "First priority dir %s is empty. Will save a histogram there", dir_priorities[0]) save_extra_to = dir_priorities[0] # Check the unique course table to see if the course has a unique term offering term_offered = None for _, unique_course in self.ds.unique_courses.iterrows(): if row['Course #'] == unique_course['Course #']: term_offered = unique_course['Term Offered'] logging.info("Course %s came up with unique term_offered %s", row['Course #'], str(term_offered)) break for dir in dir_priorities: logging.debug("Generating histograms from dir %s", dir) if dirs_and_files[dir] == []: logging.debug("Dir %s is empty. Moving on...", dir) empty_locations += 1 logging.debug("empty_locations is %i", empty_locations) else: # If the dir does not match the program that the indicator belongs to, indicate that if dir != program: indicator_data['Notes'] = "Using data from {}".format(dir) for file in dirs_and_files[ dir]: # For each grades file in the specified dir # Open the grades file logging.debug( "Reading file %s", "{}/{}/{}".format(self.config.grades_loc, dir, file)) grades = pd.read_excel("{}/{}/{}".format( self.config.grades_loc, dir, file)) # Ready the DataFrame for histogramming grades = self.ready_DataFrame(grades, course=row['Course #'], term_offered=term_offered) # Generate a Report object rprt = Report.generate_report(indicator_data, bins, self.config, grades) # File name format if the list of files is only one long if len(dirs_and_files[dir]) == 1: histogram_name = "{program} {row} {ind}-{lvl} {cfg}" # File name format if the list of files is longer else: histogram_name = "{program} {row} {ind}-{lvl}_{i} {cfg}" # Save the Report to the current directory save_name = histogram_name.format( program=program, ind=row["Indicator #"], lvl=row["Level"][0], i=dirs_and_files[dir].index(file), cfg=self.config.name, # row is rownumber + 2 because Pandas indexes from 0 starting from row 2 in # the original spreadsheet. zfill adds padding zeros row=str(rownum + 2).zfill(3)) # Make sure the save directory exists, then save the file os.makedirs("{path}/{dir}".format( path=self.config.histograms_loc, dir=dir), exist_ok=True) rprt.save("{path}/{dir}/{name}".format( path=self.config.histograms_loc, dir=dir, name=save_name)) # If the extra save location was specified, save a copy there if save_extra_to: os.makedirs("{path}/{dir}".format( path=self.config.histograms_loc, dir=program), exist_ok=True) rprt.save("{path}/{dir}/{name}".format( path=self.config.histograms_loc, dir=save_extra_to, name=save_name)) # Set the extra save location back to an empty string to prevent further saving # Happens outside of the "files in directory" for-loop so that the extra location # gets copies of every file that matches the criteria. To prevent that behaviour, # indent by one. save_extra_to = "" # If all locations came up empty, generate a blank histogram in the Core directory if empty_locations == len(dir_priorities): logging.debug("empty_locations = %i while len(dir_priorities = %i", empty_locations, len(dir_priorities)) # Write the missing data entry to the missing data file missing_thing = "{a} {b} ({c}-{d})".format( a=row["Course #"], b=row["Method of Assessment"], c=row['Indicator #'], d=row['Level']) logging.warning("No data found for %s", missing_thing) missing_data.write("Missing data for {}\n".format(missing_thing)) # Set up blank histogram Report grades = pd.DataFrame({'NO DATA FOUND': [-1]}) indicator_data[ 'Notes'] = 'No file containing any of this assessment data was found' rprt = Report.generate_report(indicator_data, bins, self.config, grades) # Set up file name histogram_name = "{program} {row} {ind}-{lvl} {cfg} NDA" save_name = histogram_name.format( program=program, ind=row["Indicator #"], lvl=row["Level"][0], cfg=self.config.name, # row is rownumber + 2 because Pandas indexes from 0 starting from row 2 in # the original spreadsheet. zfill adds padding zeros row=str(rownum + 2).zfill(3)) # Check to see that the directory exists os.makedirs("{path}/{dir}".format(path=self.config.histograms_loc, dir=program), exist_ok=True) rprt.save("{path}/{dir}/{name}".format( path=self.config.histograms_loc, dir=program, name=save_name)) # Save histogram to program directory rprt.save("{path}/{dir}/{name}".format( path=self.config.histograms_loc, dir=program, name=save_name)) logging.info( "Finished generating histograms for row %s of %s indicators", str(rownum), str(program))