def do_format(self, record): """Produce a human-readable message from the given record. This is the "core" of the format method, but it has been split out to allow the code to focus only on formatting, rather than bookkeeping (putting args into their placeholders in msg and formatting time and exception). record (LogRecord): the data for the log message. return (string): the formatted log message. """ # Determine the first part (time and severity) and its color. severity_str = record.asctime + " - " + record.levelname severity_col = self.SEVERITY_COLORS[record.levelno] # Determine the second part (service coords) and its color. if hasattr(record, "service_name") and \ hasattr(record, "service_shard"): coord_str = "%s,%d" % (record.service_name, record.service_shard) else: coord_str = "None" coord_col = get_color_hash(coord_str) # Determine the third part (operation) and its color. if hasattr(record, "operation"): operation_str = record.operation else: operation_str = "" operation_col = get_color_hash(operation_str) # Colorize the strings. if self.colors: severity_str = add_color_to_string(severity_str, severity_col, bold=True, force=True) coord_str = add_color_to_string(coord_str, coord_col, bold=True, force=True) operation_str = add_color_to_string(operation_str, operation_col, bold=True, force=True) # Put them all together. fmt = severity_str fmt += " [" + coord_str if hasattr(record, "operation"): fmt += "/" + operation_str fmt += "] " + record.message return fmt
def make_input(assume=None): n = 0 try: os.makedirs(input_dir) except OSError: pass for (is_copy, line, st) in testcases: print( "Generating", add_color_to_string("input # %d" % n, colors.BLACK, stream=sys.stderr, bold=True), file=sys.stderr ) new_input = os.path.join(input_dir, 'input%d.txt' % (n)) if is_copy: # Copy the file print("> Copy input file from:", line) copy_input = os.path.join(base_dir, line) shutil.copyfile(copy_input, new_input) else: # Call the generator with open(new_input, 'wb') as fout: call(base_dir, [gen_exe] + line.split(), stdout=fout) command = [validator_exe, new_input] if st != 0: command.append("%s" % st) call(base_dir, command) n += 1 for _ in range(3): move_cursor(directions.UP, erase=True, stream=sys.stderr)
def make_input(assume=None): n = 0 try: os.makedirs(input_dir) except OSError: pass for (is_copy, line, st) in testcases: print("Generating", add_color_to_string("input # %d" % n, colors.BLACK, stream=sys.stderr, bold=True), file=sys.stderr) new_input = os.path.join(input_dir, 'input%d.txt' % (n)) if is_copy: # Copy the file print("> Copy input file from:", line) copy_input = os.path.join(base_dir, line) shutil.copyfile(copy_input, new_input) else: # Call the generator with open(new_input, 'wb') as fout: call(base_dir, [gen_exe] + line.split(), stdout=fout) command = [validator_exe, new_input] if st != 0: command.append("%s" % st) call(base_dir, command) n += 1 for _ in range(3): move_cursor(directions.UP, erase=True, stream=sys.stderr)
def make_output(n, assume=None): try: os.makedirs(output_dir) except OSError: pass print( "Generating", add_color_to_string("output # %d" % n, colors.BLACK, stream=sys.stderr, bold=True), file=sys.stderr ) temp_dir = tempfile.mkdtemp(prefix=os.path.join(base_dir, "tmp")) use_stdin = yaml_conf.get("infile") in {None, ""} use_stdout = yaml_conf.get("outfile") in {None, ""} # Names of the actual source and destination. infile = os.path.join(input_dir, 'input%d.txt' % (n)) outfile = os.path.join(output_dir, 'output%d.txt' % (n)) # Names of the input and output in temp directory. copied_infile = os.path.join( temp_dir, "input.txt" if use_stdin else yaml_conf.get("infile")) copied_outfile = os.path.join( temp_dir, "output.txt" if use_stdout else yaml_conf.get("outfile")) os.symlink(infile, copied_infile) fin = None fout = None try: if use_stdin: fin = open(copied_infile, "rb") if use_stdout: fout = open(copied_outfile, 'wb') shutil.copy(sol_exe, temp_dir) # If the task of of type Communication, then there is # nothing to put in the output files if task_type != ['Communication', '']: call(temp_dir, [os.path.join(temp_dir, SOL_FILENAME)], stdin=fin, stdout=fout) move_cursor(directions.UP, erase=True, stream=sys.stderr) finally: if fin is not None: fin.close() if fout is not None: fout.close() os.rename(copied_outfile, outfile) shutil.rmtree(temp_dir) move_cursor(directions.UP, erase=True, stream=sys.stderr)
def print_at_exit(): print() print() for s in sols: print("%s: %3d" % ( add_color_to_string("%30s" % s[0], colors.BLACK, bold=True), s[1]) )
def make_output(n, assume=None): try: os.makedirs(output_dir) except OSError: pass print("Generating", add_color_to_string("output # %d" % n, colors.BLACK, stream=sys.stderr, bold=True), file=sys.stderr) temp_dir = tempfile.mkdtemp(prefix=os.path.join(base_dir, "tmp")) use_stdin = yaml_conf.get("infile") in {None, ""} use_stdout = yaml_conf.get("outfile") in {None, ""} # Names of the actual source and destination. infile = os.path.join(input_dir, 'input%d.txt' % (n)) outfile = os.path.join(output_dir, 'output%d.txt' % (n)) # Names of the input and output in temp directory. copied_infile = os.path.join( temp_dir, "input.txt" if use_stdin else yaml_conf.get("infile")) copied_outfile = os.path.join( temp_dir, "output.txt" if use_stdout else yaml_conf.get("outfile")) os.symlink(infile, copied_infile) fin = None fout = None try: if use_stdin: fin = open(copied_infile, "rb") if use_stdout: fout = open(copied_outfile, 'wb') shutil.copy(sol_exe, temp_dir) # If the task of of type Communication, then there is # nothing to put in the output files if task_type != ['Communication', '']: call(temp_dir, [os.path.join(temp_dir, SOL_FILENAME)], stdin=fin, stdout=fout) move_cursor(directions.UP, erase=True, stream=sys.stderr) finally: if fin is not None: fin.close() if fout is not None: fout.close() os.rename(copied_outfile, outfile) shutil.rmtree(temp_dir) move_cursor(directions.UP, erase=True, stream=sys.stderr)
def do_format(self, record): """Produce a human-readable message from the given record. This is the "core" of the format method, but it has been split out to allow the code to focus only on formatting, rather than bookkeeping (putting args into their placeholders in msg and formatting time and exception). record (LogRecord): the data for the log message. return (string): the formatted log message. """ severity = self.get_severity(record) coordinates = self.get_coordinates(record) operation = self.get_operation(record) message = record.message if self.colors: severity_col = self.SEVERITY_COLORS[record.levelno] severity = add_color_to_string(severity, severity_col, bold=True, force=True) coordinates_col = get_color_hash(coordinates) if len(coordinates) > 0: coordinates = add_color_to_string(coordinates, coordinates_col, bold=True, force=True) operation_col = get_color_hash(operation) if len(operation) > 0: operation = add_color_to_string(operation, operation_col, bold=True, force=True) fmt = severity if coordinates.strip() != "": fmt += " [%s]" % (coordinates.strip()) if operation.strip() != "": fmt += " [%s]" % (operation.strip()) fmt += " %s" % message return fmt
def test_src(exe, lang, assume=None): # Solution names begin with sol/ and end with _EVAL, we strip that print( "Testing solution", add_color_to_string(exe[4:-5], colors.BLACK, bold=True) ) test_testcases( base_dir, exe, language=lang, assume=assume)
def do_format(self, record): """Produce a human-readable message from the given record. This is the "core" of the format method, but it has been split out to allow the code to focus only on formatting, rather than bookkeeping (putting args into their placeholders in msg and formatting time and exception). record (LogRecord): the data for the log message. return (string): the formatted log message. """ severity = self.get_severity(record) coordinates = self.get_coordinates(record) operation = self.get_operation(record) message = record.message if self.colors: severity_col = self.SEVERITY_COLORS[record.levelno] severity = add_color_to_string(severity, severity_col, bold=True, force=True) coordinates_col = get_color_hash(coordinates) if coordinates != "": coordinates = add_color_to_string(coordinates, coordinates_col, bold=True, force=True) operation_col = get_color_hash(operation) if operation != "": operation = add_color_to_string(operation, operation_col, bold=True, force=True) fmt = severity if coordinates.strip() != "": fmt += " [%s]" % (coordinates.strip()) if operation.strip() != "": fmt += " [%s]" % (operation.strip()) fmt += " %s" % message return fmt
def make_output(n, assume=None): try: os.makedirs(output_dir) except OSError: pass print( "Generating", add_color_to_string("output # %d" % n, colors.BLACK, stream=sys.stderr, bold=True), file=sys.stderr ) with io.open(os.path.join(input_dir, 'input%d.txt' % (n)), 'rb') as fin: with io.open(os.path.join(output_dir, 'output%d.txt' % (n)), 'wb') as fout: if task_type != ['Communication', '']: call(base_dir, [sol_exe], stdin=fin, stdout=fout) move_cursor(directions.UP, erase=True, stream=sys.stderr) # If the task of of type Communication, then there is # nothing to put in the output files else: pass move_cursor(directions.UP, erase=True, stream=sys.stderr)
def test_testcases(base_dir, solution, language, assume=None): global task, file_cacher # Use a FileCacher with a NullBackend in order to avoid to fill # the database with junk if file_cacher is None: file_cacher = FileCacher(null=True) cmscontrib.loaders.italy_yaml.logger = NullLogger() # Load the task # TODO - This implies copying a lot of data to the FileCacher, # which is annoying if you have to do it continuously; it would be # better to use a persistent cache (although local, possibly # filesystem-based instead of database-based) and somehow detect # when the task has already been loaded if task is None: loader = cmscontrib.loaders.italy_yaml.YamlLoader(base_dir, file_cacher) task = loader.get_task(get_statement=False) # Prepare the EvaluationJob dataset = task.active_dataset digest = file_cacher.put_file_from_path( os.path.join(base_dir, solution), "Solution %s for task %s" % (solution, task.name)) executables = {task.name: Executable(filename=task.name, digest=digest)} jobs = [(t, EvaluationJob( language=language, task_type=dataset.task_type, task_type_parameters=json.loads(dataset.task_type_parameters), managers=dict(dataset.managers), executables=executables, input=dataset.testcases[t].input, output=dataset.testcases[t].output, time_limit=dataset.time_limit, memory_limit=dataset.memory_limit)) for t in dataset.testcases] tasktype = get_task_type(dataset=dataset) ask_again = True last_status = "ok" status = "ok" stop = False info = [] points = [] comments = [] tcnames = [] for jobinfo in sorted(jobs): print(jobinfo[0]) sys.stdout.flush() job = jobinfo[1] # Skip the testcase if we decide to consider everything to # timeout if stop: info.append("Time limit exceeded") points.append(0.0) comments.append("Timeout.") move_cursor(directions.UP, erase=True) continue # Evaluate testcase last_status = status tasktype.evaluate(job, file_cacher) status = job.plus.get("exit_status") info.append((job.plus.get("execution_time"), job.plus.get("execution_memory"))) points.append(float(job.outcome)) comments.append(format_status_text(job.text)) tcnames.append(jobinfo[0]) # If we saw two consecutive timeouts, ask wether we want to # consider everything to timeout if ask_again and status == "timeout" and last_status == "timeout": print("Want to stop and consider everything to timeout? [y/N]", end='') if assume is not None: print(assume) tmp = assume else: tmp = raw_input().lower() if tmp in ['y', 'yes']: stop = True else: ask_again = False print() move_cursor(directions.UP, erase=True) # Subtasks scoring try: subtasks = json.loads(dataset.score_type_parameters) subtasks[0] except: subtasks = [[100, len(info)]] if dataset.score_type == 'GroupMin': scoreFun = min else: if dataset.score_type != 'Sum': logger.warning("Score type %s not yet supported! Using Sum" % dataset.score_type) def scoreFun(x): return sum(x) / len(x) pos = 0 sts = [] # For each subtask generate a list of testcase it owns, the score gained # and the highest time and memory usage. for i in subtasks: stscores = [] stsdata = [] worst = [0, 0] try: for _ in xrange(i[1]): stscores.append(points[pos]) stsdata.append((tcnames[pos], points[pos], comments[pos], info[pos])) if info[pos][0] > worst[0]: worst[0] = info[pos][0] if info[pos][1] > worst[1]: worst[1] = info[pos][1] pos += 1 sts.append((scoreFun(stscores) * i[0], i[0], stsdata, worst)) except: sts.append((0, i[0], stsdata, [0, 0])) # Result pretty printing # Strips sol/ and _EVAL from the solution's name solution = solution[4:-5] print() clen = max(len(c) for c in comments) for st, d in enumerate(sts): print( "Subtask %d:" % st, add_color_to_string( "%5.2f/%d" % (d[0], d[1]), colors.RED if abs(d[0] - d[1]) > 0.01 else colors.GREEN, bold=True ) ) for (i, p, c, w) in d[2]: print( "%s)" % i, add_color_to_string( "%5.2lf" % p, colors.RED if abs(p - 1) > 0.01 else colors.BLACK ), "--- %s [Time:" % c.ljust(clen), add_color_to_string( ("%5.3f" % w[0]) if w[0] is not None else "N/A", colors.BLUE if w[0] is not None and w[0] >= 0.95 * d[3][0] else colors.BLACK ), "Memory:", add_color_to_string( "%5s" % mem_human(w[1]) if w[1] is not None else "N/A", colors.BLUE if w[1] is not None and w[1] >= 0.95 * d[3][1] else colors.BLACK, ), end="]" ) move_cursor(directions.RIGHT, 1000) move_cursor(directions.LEFT, len(solution) - 1) print(add_color_to_string(solution, colors.BLACK, bold=True)) print() sols.append((solution, sum([st[0] for st in sts]))) global tested_something if not tested_something: tested_something = True atexit.register(print_at_exit) return zip(points, comments, info)
def test_testcases(base_dir, solution, language, assume=None): global task, file_cacher # Use a FileCacher with a NullBackend in order to avoid to fill # the database with junk if file_cacher is None: file_cacher = FileCacher(null=True) cmscontrib.loaders.italy_yaml.logger = NullLogger() # Load the task # TODO - This implies copying a lot of data to the FileCacher, # which is annoying if you have to do it continuously; it would be # better to use a persistent cache (although local, possibly # filesystem-based instead of database-based) and somehow detect # when the task has already been loaded if task is None: loader = cmscontrib.loaders.italy_yaml.YamlLoader( base_dir, file_cacher) task = loader.get_task(get_statement=False) # Prepare the EvaluationJob dataset = task.active_dataset digest = file_cacher.put_file_from_path( os.path.join(base_dir, solution), "Solution %s for task %s" % (solution, task.name)) executables = {task.name: Executable(filename=task.name, digest=digest)} jobs = [ (t, EvaluationJob( operation=ESOperation(ESOperation.EVALUATION, None, dataset.id, dataset.testcases[t].codename).to_dict(), language=language, task_type=dataset.task_type, task_type_parameters=json.loads(dataset.task_type_parameters), managers=dict(dataset.managers), executables=executables, input=dataset.testcases[t].input, output=dataset.testcases[t].output, time_limit=dataset.time_limit, memory_limit=dataset.memory_limit)) for t in dataset.testcases ] tasktype = get_task_type(dataset=dataset) ask_again = True last_status = "ok" status = "ok" stop = False info = [] points = [] comments = [] tcnames = [] for jobinfo in sorted(jobs): print(jobinfo[0]) sys.stdout.flush() job = jobinfo[1] # Skip the testcase if we decide to consider everything to # timeout if stop: info.append("Time limit exceeded") points.append(0.0) comments.append("Timeout.") move_cursor(directions.UP, erase=True) continue # Evaluate testcase last_status = status tasktype.evaluate(job, file_cacher) status = job.plus.get("exit_status") info.append( (job.plus.get("execution_time"), job.plus.get("execution_memory"))) points.append(float(job.outcome)) # Avoid printing unneeded newline job.text = [t.rstrip() for t in job.text] comments.append(format_status_text(job.text)) tcnames.append(jobinfo[0]) # If we saw two consecutive timeouts, ask wether we want to # consider everything to timeout if ask_again and status == "timeout" and last_status == "timeout": print("Want to stop and consider everything to timeout? [y/N] ", end='') sys.stdout.flush() if assume is not None: tmp = assume print(tmp) else: # User input with a timeout of 5 seconds, at the end of which # we automatically say "n". ready will be a list of input ready # for reading, or an empty list if the timeout expired. # See: http://stackoverflow.com/a/2904057 ready, _, _ = select.select([sys.stdin], [], [], 5) if ready: tmp = sys.stdin.readline().strip().lower() else: tmp = 'n' print(tmp) if tmp in ['y', 'yes']: stop = True else: ask_again = False print() move_cursor(directions.UP, erase=True) # Subtasks scoring subtasks = json.loads(dataset.score_type_parameters) if not isinstance(subtasks, list) or len(subtasks) == 0: subtasks = [[100, len(info)]] if dataset.score_type == 'GroupMin': scoreFun = min else: if dataset.score_type != 'Sum': logger.warning("Score type %s not yet supported! Using Sum" % dataset.score_type) def scoreFun(x): return sum(x) / len(x) pos = 0 sts = [] # For each subtask generate a list of testcase it owns, the score gained # and the highest time and memory usage. for i in subtasks: stscores = [] stsdata = [] worst = [0, 0] try: for _ in xrange(i[1]): stscores.append(points[pos]) stsdata.append( (tcnames[pos], points[pos], comments[pos], info[pos])) if info[pos][0] > worst[0]: worst[0] = info[pos][0] if info[pos][1] > worst[1]: worst[1] = info[pos][1] pos += 1 sts.append((scoreFun(stscores) * i[0], i[0], stsdata, worst)) except: sts.append((0, i[0], stsdata, [0, 0])) # Result pretty printing # Strips sol/ and _EVAL from the solution's name solution = solution[4:-5] print() clen = max(len(c) for c in comments) for st, d in enumerate(sts): print( "Subtask %d:" % st, add_color_to_string( "%5.2f/%d" % (d[0], d[1]), colors.RED if abs(d[0] - d[1]) > 0.01 else colors.GREEN, bold=True)) for (i, p, c, w) in d[2]: print("%s)" % i, add_color_to_string( "%5.2lf" % p, colors.RED if abs(p - 1) > 0.01 else colors.BLACK), "--- %s [Time:" % c.ljust(clen), add_color_to_string( ("%5.3f" % w[0]) if w[0] is not None else "N/A", colors.BLUE if w[0] is not None and w[0] >= 0.95 * d[3][0] else colors.BLACK), "Memory:", add_color_to_string( "%5s" % mem_human(w[1]) if w[1] is not None else "N/A", colors.BLUE if w[1] is not None and w[1] >= 0.95 * d[3][1] else colors.BLACK, ), end="]") move_cursor(directions.RIGHT, 1000) move_cursor(directions.LEFT, len(solution) - 1) print(add_color_to_string(solution, colors.BLACK, bold=True)) print() sols.append((solution, sum([st[0] for st in sts]))) global tested_something if not tested_something: tested_something = True atexit.register(print_at_exit) return zip(points, comments, info)