print(
'file "{product}" succeeded to generate "final test" report.'
.format(product=logName))
except Exception as e:
print(logName + ' fail: Read file fail')
print(e)
except:
print('file "{product}" fail to generate "final test" report.'.format(
product=file))
# Export Final Test report
targetDir = "finalTestReport"
fileName = "FT-2"
finalReport.generate_report(product, targetDir, fileName, 'xlsx', 'DUT SN')
# Export shipping SN Report
targetDir = "shippingSNReport"
fileName = "serial"
snReport.generate_report(product, targetDir, fileName, 'csv', 'SN')
product['sndf'] = snReport.generate_df()
# Function test data fields, and generate function test report (funcData)
print('*' * 50 +
'\nstart generating {product} function test rerport\n'.format(
product=product['name']) + '*' * 2)
funcData = {
'Application Version_FT1': [],
'DUT SN': [],
'DUT MAC': [],
def gen_and_save_histograms(self, dirs_and_files, dir_priorities,
indicator_data, bins, row, rownum, program,
missing_data):
"""Generates and saves histograms for all files passed in
Each file passed to gen_and_save_histograms will have a histogram generated for it
and saved.
Args:
dirs_and_files(dict(list(string))): The grade files that were found. See the
Examples section for some sample inputs
dir_priorities(list(string)): The directory priority to which files should be
saved. In cases where data is not found at the highest priority directory,
but data is found at a lower priority directory, the highest priority
directory will get a copy of the histogram(s) using the data sets in that
low priority directory.
indicator_data(dict(string)): The indicator data for generating Report objects.
The method will pull other required items from here such as Program,
Course Number, etc.
bins(list(int or float)): The bins for generating Report objects
row(pd.Series): The row that the program got all of this indicator data from
rownum(int): The row number that all of this indicator data is coming from
program(string): The program that all of this indicator data belongs to
missing_data: The missing data file
Examples:
dirs_and_files comes in like this (and dir_priorities is in the same order
as the keys)::
{
'ENCM': ['MATH 1001 Course Grade - ENCM.xlsx'],
'Core': [
'MATH 1001 Course Grade - Core.xlsx',
'MATH 1001 Course Grade - Core - By Semester.xlsx'
],
'ECE': ['MATH 1001 Course Grade - ECE.xlsx']
}
Histograms get saved like so:
* ENCM gets a histogram of the one ENCM file
* Core gets 2 histograms of the Core files
* ECE gets 1 histogram of the ECE file
dirs_and_files comes in like this (and dir_priorities is in the same order
as the keys)::
{
'ENCM': [],
'Core': ['MATH 1001 Course Grade - Core.xlsx'],
'ECE': []
}
Histograms get saved like so:
* ENCM gets a copy of a histogram using the data from Core
* Core gets a copy of a histogram using their data, but ENCM's
indicator information
* ECE gets nothing saved to it
dirs_and files comes in like this (and dir_priorities is in the same order
as the keys)::
{
'ENCM': [],
'Core': [
'MATH 1001 Course Grade - Core.xlsx',
'MATH 1001 Course Grade - Core - Program Comparison.xlsx'
],
'ECE': ['MATH 1001 Course Grade - ECE.xlsx']
}
Histograms get saved like so:
* ENCM gets two histograms, both copies of the ones using data from Core
* Core gets two histograms using their data, but with ENCM's indicator
information
* ECE gets one histogram using their data, but with ENCM's indicator
information
dirs_and_files comes in like this (and dir_priorities is in the same order
as the keys)::
{
'ENCM': [],
'Core': [],
'ECE': []
}
Histograms get saved like so:
* Only ENCM gets a histogram. That histogram indicates that no data was
found
"""
logging.info("Preparing to save and generate a list of histograms")
logging.debug("dirs_and_files is %s", str(dirs_and_files))
logging.debug("dir_priorities is %s", str(dir_priorities))
# Keep track of the number of locations that come up empty
empty_locations = 0
# Indicates if an extra copy of the histogram needs to be saved to the top priority dir
save_extra_to = ""
if dirs_and_files[
dir_priorities[0]] == []: # If the top priority dir is empty
logging.debug(
"First priority dir %s is empty. Will save a histogram there",
dir_priorities[0])
save_extra_to = dir_priorities[0]
# Check the unique course table to see if the course has a unique term offering
term_offered = None
for _, unique_course in self.ds.unique_courses.iterrows():
if row['Course #'] == unique_course['Course #']:
term_offered = unique_course['Term Offered']
logging.info("Course %s came up with unique term_offered %s",
row['Course #'], str(term_offered))
break
for dir in dir_priorities:
logging.debug("Generating histograms from dir %s", dir)
if dirs_and_files[dir] == []:
logging.debug("Dir %s is empty. Moving on...", dir)
empty_locations += 1
logging.debug("empty_locations is %i", empty_locations)
else:
# If the dir does not match the program that the indicator belongs to, indicate that
if dir != program:
indicator_data['Notes'] = "Using data from {}".format(dir)
for file in dirs_and_files[
dir]: # For each grades file in the specified dir
# Open the grades file
logging.debug(
"Reading file %s",
"{}/{}/{}".format(self.config.grades_loc, dir, file))
grades = pd.read_excel("{}/{}/{}".format(
self.config.grades_loc, dir, file))
# Ready the DataFrame for histogramming
grades = self.ready_DataFrame(grades,
course=row['Course #'],
term_offered=term_offered)
# Generate a Report object
rprt = Report.generate_report(indicator_data, bins,
self.config, grades)
# File name format if the list of files is only one long
if len(dirs_and_files[dir]) == 1:
histogram_name = "{program} {row} {ind}-{lvl} {cfg}"
# File name format if the list of files is longer
else:
histogram_name = "{program} {row} {ind}-{lvl}_{i} {cfg}"
# Save the Report to the current directory
save_name = histogram_name.format(
program=program,
ind=row["Indicator #"],
lvl=row["Level"][0],
i=dirs_and_files[dir].index(file),
cfg=self.config.name,
# row is rownumber + 2 because Pandas indexes from 0 starting from row 2 in
# the original spreadsheet. zfill adds padding zeros
row=str(rownum + 2).zfill(3))
# Make sure the save directory exists, then save the file
os.makedirs("{path}/{dir}".format(
path=self.config.histograms_loc, dir=dir),
exist_ok=True)
rprt.save("{path}/{dir}/{name}".format(
path=self.config.histograms_loc,
dir=dir,
name=save_name))
# If the extra save location was specified, save a copy there
if save_extra_to:
os.makedirs("{path}/{dir}".format(
path=self.config.histograms_loc, dir=program),
exist_ok=True)
rprt.save("{path}/{dir}/{name}".format(
path=self.config.histograms_loc,
dir=save_extra_to,
name=save_name))
# Set the extra save location back to an empty string to prevent further saving
# Happens outside of the "files in directory" for-loop so that the extra location
# gets copies of every file that matches the criteria. To prevent that behaviour,
# indent by one.
save_extra_to = ""
# If all locations came up empty, generate a blank histogram in the Core directory
if empty_locations == len(dir_priorities):
logging.debug("empty_locations = %i while len(dir_priorities = %i",
empty_locations, len(dir_priorities))
# Write the missing data entry to the missing data file
missing_thing = "{a} {b} ({c}-{d})".format(
a=row["Course #"],
b=row["Method of Assessment"],
c=row['Indicator #'],
d=row['Level'])
logging.warning("No data found for %s", missing_thing)
missing_data.write("Missing data for {}\n".format(missing_thing))
# Set up blank histogram Report
grades = pd.DataFrame({'NO DATA FOUND': [-1]})
indicator_data[
'Notes'] = 'No file containing any of this assessment data was found'
rprt = Report.generate_report(indicator_data, bins, self.config,
grades)
# Set up file name
histogram_name = "{program} {row} {ind}-{lvl} {cfg} NDA"
save_name = histogram_name.format(
program=program,
ind=row["Indicator #"],
lvl=row["Level"][0],
cfg=self.config.name,
# row is rownumber + 2 because Pandas indexes from 0 starting from row 2 in
# the original spreadsheet. zfill adds padding zeros
row=str(rownum + 2).zfill(3))
# Check to see that the directory exists
os.makedirs("{path}/{dir}".format(path=self.config.histograms_loc,
dir=program),
exist_ok=True)
rprt.save("{path}/{dir}/{name}".format(
path=self.config.histograms_loc, dir=program, name=save_name))
# Save histogram to program directory
rprt.save("{path}/{dir}/{name}".format(
path=self.config.histograms_loc, dir=program, name=save_name))
logging.info(
"Finished generating histograms for row %s of %s indicators",
str(rownum), str(program))
