def write_scores_html(score_dir, auto_refresh=True, append=False):
filename = 'detailed_results.html'
image_paths = sorted(ls(os.path.join(score_dir, '*.png')))
if auto_refresh:
html_head = '<html><head> <meta http-equiv="refresh" content="5"> ' +\
'</head><body><pre>'
else:
html_head = """<html><body><pre>"""
html_end = '</pre></body></html>'
if append:
mode = 'a'
else:
mode = 'w'
filepath = os.path.join(score_dir, filename)
with open(filepath, mode) as html_file:
html_file.write(html_head)
for image_path in image_paths:
with open(image_path, "rb") as image_file:
encoded_string = base64.b64encode(image_file.read())
encoded_string = encoded_string.decode('utf-8')
s = '<img src="data:image/png;charset=utf-8;base64,{}"/>'\
.format(encoded_string)
html_file.write(s + '<br>')
html_file.write(html_end)
logger.debug("Wrote learning curve page to {}".format(filepath))
def get_prediction_files(prediction_dir, basename): """Return prediction files for the task <basename>. Examples of prediction file name: mini.predict_0, mini.predict_1 """ prediction_files = ls(os.path.join(prediction_dir, basename + '*.predict_*')) return prediction_files
def get_task_name(solution_dir):
"""Get the task name from solution directory."""
solution_names = sorted(ls(os.path.join(solution_dir, '*.solution')))
if len(solution_names) != 1: # Assert only one file is found
logger.warning("{} solution files found: {}! "\
.format(len(solution_names), solution_names) +
"Return `None` as task name.")
return None
solution_file = solution_names[0]
task_name = solution_file.split(os.sep)[-1].split('.')[0]
return task_name
def get_solution(solution_dir):
"""Get the solution array from solution directory."""
solution_names = sorted(ls(os.path.join(solution_dir, '*.solution')))
if len(solution_names) != 1: # Assert only one file is found
logger.warning("{} solution files found: {}! "\
.format(len(solution_names), solution_names) +
"Return `None` as solution.")
return None
solution_file = solution_names[0]
solution = read_array(solution_file)
return solution
def get_prediction_files(prediction_dir, basename, start):
"""Return prediction files for the task <basename>.
Examples of prediction file name: mini.predict_0, mini.predict_1
"""
prediction_files = ls(
os.path.join(prediction_dir, basename + '*.predict_*'))
# Exclude all files (if any) generated before start
prediction_files = [
f for f in prediction_files if os.path.getmtime(f) > start
]
return prediction_files
def get_prediction_files(prediction_dir):
"""Return prediction files in prediction directory.
Examples of prediction file name: mini.predict_0, mini.predict_1
"""
prediction_files = ls(os.path.join(prediction_dir, '*.predict_*'))
task_names = set(
[os.path.basename(f.split('.')[-2]) for f in prediction_files])
if len(task_names) > 1:
raise ValueError("Predictions of multiple tasks are found: {}!" \
.format(prediction_files))
order_key = lambda filename: int(filename.split('_')[-1])
prediction_files = sorted(prediction_files, key=order_key)
return prediction_files
def get_new_prediction_files(self):
"""Fetch new prediction file(name)s found in prediction directory and update
corresponding attributes.
Examples of prediction file name: mini.predict_0, mini.predict_1
Returns:
List of new prediction files found.
"""
prediction_files = ls(os.path.join(self.prediction_dir,
self.prediction_filename_pattern()))
logger.debug("Prediction files: {}".format(prediction_files))
new_prediction_files = [p for p in prediction_files
if p not in self.prediction_files_so_far]
order_key = lambda filename: int(filename.split('_')[-1])
self.new_prediction_files = sorted(new_prediction_files, key=order_key)
return self.new_prediction_files
def write_scores_html(score_dir, auto_refresh=True):
filename = 'detailed_results.html'
image_paths = sorted(ls(os.path.join(score_dir, '*.png')))
if auto_refresh:
html_head = """<html><head> <meta http-equiv="refresh" content="5"> </head><body><pre>"""
else:
html_head = """<html><body><pre>"""
html_end = '</pre></body></html>'
with open(os.path.join(score_dir, filename), 'w') as html_file:
# Automatic refreshing the page on file change using Live.js
html_file.write(html_head)
for image_path in image_paths:
with open(image_path, "rb") as image_file:
encoded_string = base64.b64encode(image_file.read())
encoded_string = encoded_string.decode('utf-8')
s = '<img src="data:image/png;charset=utf-8;base64,%s"/>'%encoded_string
html_file.write(s + '<br>')
html_file.write(html_end)
) == 1: # Use the default input and output directories if no arguments are provided
input_dir = default_input_dir
output_dir = default_output_dir
else:
input_dir = argv[1]
output_dir = argv[2]
# Create the output directory, if it does not already exist and open output files
mkdir(output_dir)
score_file = open(os.path.join(output_dir, 'scores.txt'), 'wb')
html_file = open(os.path.join(output_dir, 'scores.html'), 'wb')
# Get the metric
metric_name, scoring_function = _load_scoring_function()
# Get all the solution files from the solution directory
solution_names = sorted(ls(os.path.join(input_dir, 'ref', '*.solution')))
# Loop over files in solution directory and search for predictions with extension .predict having the same basename
for i, solution_file in enumerate(solution_names):
set_num = i + 1 # 1-indexed
score_name = 'set%s_score' % set_num
# Extract the dataset name from the file name
basename = solution_file[-solution_file[::-1].index(filesep):
-solution_file[::-1].index('.') - 1]
try:
# Get the last prediction from the res subdirectory (must end with '.predict')
predict_file = ls(
os.path.join(input_dir, 'res', basename + '*.predict'))[-1]
if (predict_file == []):
# print_log("Scoring datetime:", the_date)
# Create the output directory, if it does not already exist and open output files
os.makedirs(score_dir, exist_ok=True)
score_file = open(os.path.join(score_dir, 'scores.txt'), 'w')
detailed_results_filepath = os.path.join(score_dir, 'detailed_results.html')
# Initialize detailed_results.html
init_scores_html(detailed_results_filepath)
# Get the metric
scoring_function = autodl_bac
metric_name = "Area under Learning Curve"
# Get all the solution files from the solution directory
solution_names = sorted(ls(os.path.join(solution_dir, '*.solution')))
if not len(solution_names) == 1: # Assert only one file is found
raise ValueError("Multiple solution file found!")
solution_file = solution_names[0]
# Extract the dataset name from the file name
basename = get_basename(solution_file)
nb_preds = {x:0 for x in solution_names}
scores = {x:0 for x in solution_names}
# Use 'duration.txt' file to detect if ingestion program exits early
duration_filepath = os.path.join(prediction_dir, 'duration.txt')
# Begin scoring process, along with ingestion program
# Moniter training processes while time budget is not attained
while(time.time() < start + TIME_BUDGET):
time.sleep(0.5)
