def get_last_input(self):
assert self._config['mode'] == DEV_MODE
if os.path.exists(LAST_INPUT_PATH):
user_input = util.load_user_input(LAST_INPUT_PATH)
else:
user_input = scene_if.UserInput()
logging.warning('No last user input found')
return user_input
def get_task_from_id(self, task_id):
template_id, _ = task_id.split(':')
if self._config['mode'] == DEMO_MODE:
# In demo mode use cached tasks.
task = self.task_cache[task_id]
else:
# Try to load the task instance directly.
_, _, module = loader.load_task_script(template_id)
task = module.build_task.get_specific_task(task_id)
solution_path = util.get_solution_path(task_id)
if os.path.exists(solution_path):
task.solutions = [util.load_user_input(solution_path)]
meta_task = task_if.TaskWithMeta(task=task)
if hasattr(task, 'template_params'):
meta_task.template_params = ' '.join(
f'{k}={v}' for k, v in task.template_params.items())
eval_stats = eval_task_complexity.maybe_load_evaluation(template_id)
if eval_stats is not None and eval_stats_has_task(eval_stats, task_id):
meta_task.eval_data = eval_stats_to_thrift(eval_stats, task_id)
meta_task.eval_data.known_solutions = filter_known_solutions(
meta_task.eval_data.known_solutions, self._config['mode'],
task.tier)
status_counts = {
tier: stats[task_id]
for tier, stats in eval_stats['status_counts'].items() if
tier.upper() in PROD_TIERS or self._config['mode'] == DEV_MODE
}
# Create a text description of the eval stats.
chunks = ['attempts to solution:']
for tier, stats in sorted(status_counts.items()):
if stats[SOLVED] == 0:
chunks.append(f'{tier}=inf')
else:
cnt = int(
(stats[SOLVED] + stats[NOT_SOLVED]) / stats[SOLVED])
chunks.append(f'{tier}={cnt}')
chunks.append('valid share:')
for tier, stats in sorted(status_counts.items()):
if sum(stats.values()):
share = 1.0 - (stats[INVALID_INPUT] / sum(stats.values()))
chunks.append(f'{tier}={share * 100:.1f}%')
else:
chunks.append(f'{tier}=nan')
chunks.append('flags:')
for tier, stats in sorted(eval_stats['flags'].items()):
chunks.append('%s={%s}' %
(tier, ','.join(flag.name.lower()
for flag in stats[task_id])))
meta_task.text_eval_info = ' '.join(chunks)
if self._config['mode'] != DEMO_MODE:
meta_task.rendered_img = get_task_as_base64_image(task)
return meta_task