def make_input(assume=None): n = 0 try: os.makedirs(input_dir) except OSError: pass for (is_copy, line, st) in testcases: print( "Generating", add_color_to_string("input # %d" % n, colors.BLACK, stream=sys.stderr, bold=True), file=sys.stderr ) new_input = os.path.join(input_dir, 'input%d.txt' % (n)) if is_copy: # Copy the file print("> Copy input file from:", line) copy_input = os.path.join(base_dir, line) shutil.copyfile(copy_input, new_input) else: # Call the generator with open(new_input, 'wb') as fout: call(base_dir, [gen_exe] + line.split(), stdout=fout) command = [validator_exe, new_input] if st != 0: command.append("%s" % st) call(base_dir, command) n += 1 for _ in range(3): move_cursor(directions.UP, erase=True, stream=sys.stderr)
def make_input(assume=None): n = 0 try: os.makedirs(input_dir) except OSError: pass for (is_copy, line, st) in testcases: print("Generating", add_color_to_string("input # %d" % n, colors.BLACK, stream=sys.stderr, bold=True), file=sys.stderr) new_input = os.path.join(input_dir, 'input%d.txt' % (n)) if is_copy: # Copy the file print("> Copy input file from:", line) copy_input = os.path.join(base_dir, line) shutil.copyfile(copy_input, new_input) else: # Call the generator with open(new_input, 'wb') as fout: call(base_dir, [gen_exe] + line.split(), stdout=fout) command = [validator_exe, new_input] if st != 0: command.append("%s" % st) call(base_dir, command) n += 1 for _ in range(3): move_cursor(directions.UP, erase=True, stream=sys.stderr)
def compile_src(srcs, exe, for_evaluation, lang, assume=None): if lang != 'pas' or len(srcs) == 1: compilation_commands = get_compilation_commands( lang, srcs, exe, for_evaluation=for_evaluation) for command in compilation_commands: call(base_dir, command) move_cursor(directions.UP, erase=True, stream=sys.stderr) # When using Pascal with graders, file naming conventions # require us to do a bit of trickery, i.e., performing the # compilation in a separate temporary directory else: tempdir = tempfile.mkdtemp() task_name = detect_task_name(base_dir) new_srcs = [os.path.split(srcs[0])[1], '%s.pas' % (task_name)] new_exe = os.path.split(srcs[1])[1][:-4] shutil.copyfile(os.path.join(base_dir, srcs[0]), os.path.join(tempdir, new_srcs[0])) shutil.copyfile(os.path.join(base_dir, srcs[1]), os.path.join(tempdir, new_srcs[1])) lib_filename = '%slib.pas' % (task_name) if os.path.exists(os.path.join(SOL_DIRNAME, lib_filename)): shutil.copyfile(os.path.join(SOL_DIRNAME, lib_filename), os.path.join(tempdir, lib_filename)) compilation_commands = get_compilation_commands( lang, new_srcs, new_exe, for_evaluation=for_evaluation) for command in compilation_commands: call(tempdir, command) move_cursor(directions.UP, erase=True, stream=sys.stderr) shutil.copyfile(os.path.join(tempdir, new_exe), os.path.join(base_dir, exe)) shutil.copymode(os.path.join(tempdir, new_exe), os.path.join(base_dir, exe)) shutil.rmtree(tempdir)
def compile_src(srcs, exe, for_evaluation, lang, assume=None): # We put everything in a temporary directory to reproduce # the same conditions that we have when compiling a # submission. tempdir = tempfile.mkdtemp() try: task_name = detect_task_name(base_dir) grader_num = 1 if len(srcs) > 1 else 0 new_srcs = [] for grader in srcs[:grader_num]: grader_name = os.path.basename(grader) shutil.copyfile(os.path.join(base_dir, grader), os.path.join(tempdir, grader_name)) new_srcs.append(os.path.join(tempdir, grader_name)) # For now, we assume we only have one non-grader source. source_name = task_name + LANGUAGE_TO_SOURCE_EXT_MAP[lang] shutil.copyfile(os.path.join(base_dir, srcs[grader_num]), os.path.join(tempdir, source_name)) new_srcs.append(source_name) # Libraries are needed/used only for C/C++ and Pascal if lang in LANGUAGE_TO_HEADER_EXT_MAP: lib_template = "%s" + LANGUAGE_TO_HEADER_EXT_MAP[lang] lib_filename = lib_template % (task_name) lib_path = os.path.join(base_dir, SOL_DIRNAME, lib_filename) if os.path.exists(lib_path): shutil.copyfile(lib_path, os.path.join(tempdir, lib_filename)) new_exe = os.path.join(tempdir, task_name) compilation_commands = get_compilation_commands(lang, new_srcs, new_exe, for_evaluation=for_evaluation) for command in compilation_commands: call(tempdir, command) move_cursor(directions.UP, erase=True, stream=sys.stderr) shutil.copyfile(os.path.join(tempdir, new_exe), os.path.join(base_dir, exe)) shutil.copymode(os.path.join(tempdir, new_exe), os.path.join(base_dir, exe)) finally: shutil.rmtree(tempdir)
def make_output(n, assume=None): try: os.makedirs(output_dir) except OSError: pass print( "Generating", add_color_to_string("output # %d" % n, colors.BLACK, stream=sys.stderr, bold=True), file=sys.stderr ) temp_dir = tempfile.mkdtemp(prefix=os.path.join(base_dir, "tmp")) use_stdin = yaml_conf.get("infile") in {None, ""} use_stdout = yaml_conf.get("outfile") in {None, ""} # Names of the actual source and destination. infile = os.path.join(input_dir, 'input%d.txt' % (n)) outfile = os.path.join(output_dir, 'output%d.txt' % (n)) # Names of the input and output in temp directory. copied_infile = os.path.join( temp_dir, "input.txt" if use_stdin else yaml_conf.get("infile")) copied_outfile = os.path.join( temp_dir, "output.txt" if use_stdout else yaml_conf.get("outfile")) os.symlink(infile, copied_infile) fin = None fout = None try: if use_stdin: fin = open(copied_infile, "rb") if use_stdout: fout = open(copied_outfile, 'wb') shutil.copy(sol_exe, temp_dir) # If the task of of type Communication, then there is # nothing to put in the output files if task_type != ['Communication', '']: call(temp_dir, [os.path.join(temp_dir, SOL_FILENAME)], stdin=fin, stdout=fout) move_cursor(directions.UP, erase=True, stream=sys.stderr) finally: if fin is not None: fin.close() if fout is not None: fout.close() os.rename(copied_outfile, outfile) shutil.rmtree(temp_dir) move_cursor(directions.UP, erase=True, stream=sys.stderr)
def make_output(n, assume=None): try: os.makedirs(output_dir) except OSError: pass print("Generating", add_color_to_string("output # %d" % n, colors.BLACK, stream=sys.stderr, bold=True), file=sys.stderr) temp_dir = tempfile.mkdtemp(prefix=os.path.join(base_dir, "tmp")) use_stdin = yaml_conf.get("infile") in {None, ""} use_stdout = yaml_conf.get("outfile") in {None, ""} # Names of the actual source and destination. infile = os.path.join(input_dir, 'input%d.txt' % (n)) outfile = os.path.join(output_dir, 'output%d.txt' % (n)) # Names of the input and output in temp directory. copied_infile = os.path.join( temp_dir, "input.txt" if use_stdin else yaml_conf.get("infile")) copied_outfile = os.path.join( temp_dir, "output.txt" if use_stdout else yaml_conf.get("outfile")) os.symlink(infile, copied_infile) fin = None fout = None try: if use_stdin: fin = open(copied_infile, "rb") if use_stdout: fout = open(copied_outfile, 'wb') shutil.copy(sol_exe, temp_dir) # If the task of of type Communication, then there is # nothing to put in the output files if task_type != ['Communication', '']: call(temp_dir, [os.path.join(temp_dir, SOL_FILENAME)], stdin=fin, stdout=fout) move_cursor(directions.UP, erase=True, stream=sys.stderr) finally: if fin is not None: fin.close() if fout is not None: fout.close() os.rename(copied_outfile, outfile) shutil.rmtree(temp_dir) move_cursor(directions.UP, erase=True, stream=sys.stderr)
def compile_src(srcs, exe, for_evaluation, lang, assume=None): # We put everything in a temporary directory to reproduce # the same conditions that we have when compiling a # submission. tempdir = tempfile.mkdtemp() try: task_name = detect_task_name(base_dir) grader_num = 1 if len(srcs) > 1 else 0 new_srcs = [] for grader in srcs[:grader_num]: grader_name = os.path.basename(grader) shutil.copyfile(os.path.join(base_dir, grader), os.path.join(tempdir, grader_name)) new_srcs.append(os.path.join(tempdir, grader_name)) # For now, we assume we only have one non-grader source. source_name = task_name + LANGUAGE_TO_SOURCE_EXT_MAP[lang] shutil.copyfile(os.path.join(base_dir, srcs[grader_num]), os.path.join(tempdir, source_name)) new_srcs.append(source_name) # Libraries are needed/used only for C/C++ and Pascal if lang in LANGUAGE_TO_HEADER_EXT_MAP: lib_template = "%s" + LANGUAGE_TO_HEADER_EXT_MAP[lang] lib_filename = lib_template % (task_name) lib_path = os.path.join(base_dir, SOL_DIRNAME, lib_filename) if os.path.exists(lib_path): shutil.copyfile(lib_path, os.path.join(tempdir, lib_filename)) new_exe = os.path.join(tempdir, task_name) compilation_commands = get_compilation_commands( lang, new_srcs, new_exe, for_evaluation=for_evaluation) for command in compilation_commands: call(tempdir, command) move_cursor(directions.UP, erase=True, stream=sys.stderr) shutil.copyfile(os.path.join(tempdir, new_exe), os.path.join(base_dir, exe)) shutil.copymode(os.path.join(tempdir, new_exe), os.path.join(base_dir, exe)) finally: shutil.rmtree(tempdir)
def compile_src(srcs, exe, for_evaluation, lang, assume=None): if lang != 'pas' or len(srcs) == 1: compilation_commands = get_compilation_commands( lang, srcs, exe, for_evaluation=for_evaluation) for command in compilation_commands: call(base_dir, command) move_cursor(directions.UP, erase=True, stream=sys.stderr) # When using Pascal with graders, file naming conventions # require us to do a bit of trickery, i.e., performing the # compilation in a separate temporary directory else: tempdir = tempfile.mkdtemp() task_name = detect_task_name(base_dir) new_srcs = [os.path.split(srcs[0])[1], '%s.pas' % (task_name)] new_exe = os.path.split(srcs[1])[1][:-4] shutil.copyfile(os.path.join(base_dir, srcs[0]), os.path.join(tempdir, new_srcs[0])) shutil.copyfile(os.path.join(base_dir, srcs[1]), os.path.join(tempdir, new_srcs[1])) lib_filename = '%slib.pas' % (task_name) if os.path.exists(os.path.join(SOL_DIRNAME, lib_filename)): shutil.copyfile(os.path.join(SOL_DIRNAME, lib_filename), os.path.join(tempdir, lib_filename)) compilation_commands = get_compilation_commands( lang, new_srcs, new_exe, for_evaluation=for_evaluation) for command in compilation_commands: call(tempdir, command) move_cursor(directions.UP, erase=True, stream=sys.stderr) shutil.copyfile(os.path.join(tempdir, new_exe), os.path.join(base_dir, exe)) shutil.copymode(os.path.join(tempdir, new_exe), os.path.join(base_dir, exe)) shutil.rmtree(tempdir)
def make_output(n, assume=None): try: os.makedirs(output_dir) except OSError: pass print( "Generating", add_color_to_string("output # %d" % n, colors.BLACK, stream=sys.stderr, bold=True), file=sys.stderr ) with io.open(os.path.join(input_dir, 'input%d.txt' % (n)), 'rb') as fin: with io.open(os.path.join(output_dir, 'output%d.txt' % (n)), 'wb') as fout: if task_type != ['Communication', '']: call(base_dir, [sol_exe], stdin=fin, stdout=fout) move_cursor(directions.UP, erase=True, stream=sys.stderr) # If the task of of type Communication, then there is # nothing to put in the output files else: pass move_cursor(directions.UP, erase=True, stream=sys.stderr)
def test_testcases(base_dir, solution, language, assume=None): global task, file_cacher # Use a FileCacher with a NullBackend in order to avoid to fill # the database with junk if file_cacher is None: file_cacher = FileCacher(null=True) cmscontrib.loaders.italy_yaml.logger = NullLogger() # Load the task # TODO - This implies copying a lot of data to the FileCacher, # which is annoying if you have to do it continuously; it would be # better to use a persistent cache (although local, possibly # filesystem-based instead of database-based) and somehow detect # when the task has already been loaded if task is None: loader = cmscontrib.loaders.italy_yaml.YamlLoader(base_dir, file_cacher) task = loader.get_task(get_statement=False) # Prepare the EvaluationJob dataset = task.active_dataset digest = file_cacher.put_file_from_path( os.path.join(base_dir, solution), "Solution %s for task %s" % (solution, task.name)) executables = {task.name: Executable(filename=task.name, digest=digest)} jobs = [(t, EvaluationJob( language=language, task_type=dataset.task_type, task_type_parameters=json.loads(dataset.task_type_parameters), managers=dict(dataset.managers), executables=executables, input=dataset.testcases[t].input, output=dataset.testcases[t].output, time_limit=dataset.time_limit, memory_limit=dataset.memory_limit)) for t in dataset.testcases] tasktype = get_task_type(dataset=dataset) ask_again = True last_status = "ok" status = "ok" stop = False info = [] points = [] comments = [] tcnames = [] for jobinfo in sorted(jobs): print(jobinfo[0]) sys.stdout.flush() job = jobinfo[1] # Skip the testcase if we decide to consider everything to # timeout if stop: info.append("Time limit exceeded") points.append(0.0) comments.append("Timeout.") move_cursor(directions.UP, erase=True) continue # Evaluate testcase last_status = status tasktype.evaluate(job, file_cacher) status = job.plus.get("exit_status") info.append((job.plus.get("execution_time"), job.plus.get("execution_memory"))) points.append(float(job.outcome)) comments.append(format_status_text(job.text)) tcnames.append(jobinfo[0]) # If we saw two consecutive timeouts, ask wether we want to # consider everything to timeout if ask_again and status == "timeout" and last_status == "timeout": print("Want to stop and consider everything to timeout? [y/N]", end='') if assume is not None: print(assume) tmp = assume else: tmp = raw_input().lower() if tmp in ['y', 'yes']: stop = True else: ask_again = False print() move_cursor(directions.UP, erase=True) # Subtasks scoring try: subtasks = json.loads(dataset.score_type_parameters) subtasks[0] except: subtasks = [[100, len(info)]] if dataset.score_type == 'GroupMin': scoreFun = min else: if dataset.score_type != 'Sum': logger.warning("Score type %s not yet supported! Using Sum" % dataset.score_type) def scoreFun(x): return sum(x) / len(x) pos = 0 sts = [] # For each subtask generate a list of testcase it owns, the score gained # and the highest time and memory usage. for i in subtasks: stscores = [] stsdata = [] worst = [0, 0] try: for _ in xrange(i[1]): stscores.append(points[pos]) stsdata.append((tcnames[pos], points[pos], comments[pos], info[pos])) if info[pos][0] > worst[0]: worst[0] = info[pos][0] if info[pos][1] > worst[1]: worst[1] = info[pos][1] pos += 1 sts.append((scoreFun(stscores) * i[0], i[0], stsdata, worst)) except: sts.append((0, i[0], stsdata, [0, 0])) # Result pretty printing # Strips sol/ and _EVAL from the solution's name solution = solution[4:-5] print() clen = max(len(c) for c in comments) for st, d in enumerate(sts): print( "Subtask %d:" % st, add_color_to_string( "%5.2f/%d" % (d[0], d[1]), colors.RED if abs(d[0] - d[1]) > 0.01 else colors.GREEN, bold=True ) ) for (i, p, c, w) in d[2]: print( "%s)" % i, add_color_to_string( "%5.2lf" % p, colors.RED if abs(p - 1) > 0.01 else colors.BLACK ), "--- %s [Time:" % c.ljust(clen), add_color_to_string( ("%5.3f" % w[0]) if w[0] is not None else "N/A", colors.BLUE if w[0] is not None and w[0] >= 0.95 * d[3][0] else colors.BLACK ), "Memory:", add_color_to_string( "%5s" % mem_human(w[1]) if w[1] is not None else "N/A", colors.BLUE if w[1] is not None and w[1] >= 0.95 * d[3][1] else colors.BLACK, ), end="]" ) move_cursor(directions.RIGHT, 1000) move_cursor(directions.LEFT, len(solution) - 1) print(add_color_to_string(solution, colors.BLACK, bold=True)) print() sols.append((solution, sum([st[0] for st in sts]))) global tested_something if not tested_something: tested_something = True atexit.register(print_at_exit) return zip(points, comments, info)
def test_testcases(base_dir, solution, language, assume=None): global task, file_cacher # Use a FileCacher with a NullBackend in order to avoid to fill # the database with junk if file_cacher is None: file_cacher = FileCacher(null=True) cmscontrib.loaders.italy_yaml.logger = NullLogger() # Load the task # TODO - This implies copying a lot of data to the FileCacher, # which is annoying if you have to do it continuously; it would be # better to use a persistent cache (although local, possibly # filesystem-based instead of database-based) and somehow detect # when the task has already been loaded if task is None: loader = cmscontrib.loaders.italy_yaml.YamlLoader( base_dir, file_cacher) task = loader.get_task(get_statement=False) # Prepare the EvaluationJob dataset = task.active_dataset digest = file_cacher.put_file_from_path( os.path.join(base_dir, solution), "Solution %s for task %s" % (solution, task.name)) executables = {task.name: Executable(filename=task.name, digest=digest)} jobs = [ (t, EvaluationJob( operation=ESOperation(ESOperation.EVALUATION, None, dataset.id, dataset.testcases[t].codename).to_dict(), language=language, task_type=dataset.task_type, task_type_parameters=json.loads(dataset.task_type_parameters), managers=dict(dataset.managers), executables=executables, input=dataset.testcases[t].input, output=dataset.testcases[t].output, time_limit=dataset.time_limit, memory_limit=dataset.memory_limit)) for t in dataset.testcases ] tasktype = get_task_type(dataset=dataset) ask_again = True last_status = "ok" status = "ok" stop = False info = [] points = [] comments = [] tcnames = [] for jobinfo in sorted(jobs): print(jobinfo[0]) sys.stdout.flush() job = jobinfo[1] # Skip the testcase if we decide to consider everything to # timeout if stop: info.append("Time limit exceeded") points.append(0.0) comments.append("Timeout.") move_cursor(directions.UP, erase=True) continue # Evaluate testcase last_status = status tasktype.evaluate(job, file_cacher) status = job.plus.get("exit_status") info.append( (job.plus.get("execution_time"), job.plus.get("execution_memory"))) points.append(float(job.outcome)) # Avoid printing unneeded newline job.text = [t.rstrip() for t in job.text] comments.append(format_status_text(job.text)) tcnames.append(jobinfo[0]) # If we saw two consecutive timeouts, ask wether we want to # consider everything to timeout if ask_again and status == "timeout" and last_status == "timeout": print("Want to stop and consider everything to timeout? [y/N] ", end='') sys.stdout.flush() if assume is not None: tmp = assume print(tmp) else: # User input with a timeout of 5 seconds, at the end of which # we automatically say "n". ready will be a list of input ready # for reading, or an empty list if the timeout expired. # See: http://stackoverflow.com/a/2904057 ready, _, _ = select.select([sys.stdin], [], [], 5) if ready: tmp = sys.stdin.readline().strip().lower() else: tmp = 'n' print(tmp) if tmp in ['y', 'yes']: stop = True else: ask_again = False print() move_cursor(directions.UP, erase=True) # Subtasks scoring subtasks = json.loads(dataset.score_type_parameters) if not isinstance(subtasks, list) or len(subtasks) == 0: subtasks = [[100, len(info)]] if dataset.score_type == 'GroupMin': scoreFun = min else: if dataset.score_type != 'Sum': logger.warning("Score type %s not yet supported! Using Sum" % dataset.score_type) def scoreFun(x): return sum(x) / len(x) pos = 0 sts = [] # For each subtask generate a list of testcase it owns, the score gained # and the highest time and memory usage. for i in subtasks: stscores = [] stsdata = [] worst = [0, 0] try: for _ in xrange(i[1]): stscores.append(points[pos]) stsdata.append( (tcnames[pos], points[pos], comments[pos], info[pos])) if info[pos][0] > worst[0]: worst[0] = info[pos][0] if info[pos][1] > worst[1]: worst[1] = info[pos][1] pos += 1 sts.append((scoreFun(stscores) * i[0], i[0], stsdata, worst)) except: sts.append((0, i[0], stsdata, [0, 0])) # Result pretty printing # Strips sol/ and _EVAL from the solution's name solution = solution[4:-5] print() clen = max(len(c) for c in comments) for st, d in enumerate(sts): print( "Subtask %d:" % st, add_color_to_string( "%5.2f/%d" % (d[0], d[1]), colors.RED if abs(d[0] - d[1]) > 0.01 else colors.GREEN, bold=True)) for (i, p, c, w) in d[2]: print("%s)" % i, add_color_to_string( "%5.2lf" % p, colors.RED if abs(p - 1) > 0.01 else colors.BLACK), "--- %s [Time:" % c.ljust(clen), add_color_to_string( ("%5.3f" % w[0]) if w[0] is not None else "N/A", colors.BLUE if w[0] is not None and w[0] >= 0.95 * d[3][0] else colors.BLACK), "Memory:", add_color_to_string( "%5s" % mem_human(w[1]) if w[1] is not None else "N/A", colors.BLUE if w[1] is not None and w[1] >= 0.95 * d[3][1] else colors.BLACK, ), end="]") move_cursor(directions.RIGHT, 1000) move_cursor(directions.LEFT, len(solution) - 1) print(add_color_to_string(solution, colors.BLACK, bold=True)) print() sols.append((solution, sum([st[0] for st in sts]))) global tested_something if not tested_something: tested_something = True atexit.register(print_at_exit) return zip(points, comments, info)