def _make_outs(self, outs_to_make): """Compiles the model solution and executes it in order to generate test outputs. :return: Result from workers. """ env = self._find_and_compile('', command='outgen') if not env: return {} jobs = {} for outname, test in outs_to_make: job = env.copy() job['job_type'] = 'exec' if self.use_sandboxes else 'unsafe-exec' job['task_priority'] = TASK_PRIORITY job['exe_file'] = env['compiled_file'] job['upload_out'] = True job['in_file'] = django_to_filetracker_path(test.input_file) job['out_file'] = outname if test.memory_limit: job['exec_mem_limit'] = test.memory_limit jobs[test.name] = job jobs = run_sioworkers_jobs(jobs) get_client().delete_file(env['compiled_file']) return jobs
def _verify_inputs(self, tests): """Check if correct solution exits with code 0 on all tests. :raises: :class:`~oioioi.problems.package.ProblemPackageError` otherwise. """ env = self._find_and_compile('inwer') if env and not self.use_make: jobs = {} for test in tests: job = env.copy() job['job_type'] = 'inwer' job['task_priority'] = TASK_PRIORITY job['exe_file'] = env['compiled_file'] job['in_file'] = django_to_filetracker_path(test.input_file) job['use_sandboxes'] = self.use_sandboxes jobs[test.name] = job jobs = run_sioworkers_jobs(jobs) get_client().delete_file(env['compiled_file']) for test_name, job in jobs.iteritems(): if job['result_code'] != 'OK': raise ProblemPackageError( _("Inwer failed on test " "%(test)s. Inwer output %(output)s") % { 'test': test_name, 'output': '\n'.join(job['stdout']) }) logger.info("%s: inwer success", self.filename)
def _verify_inputs(self, tests): """Check if correct solution exits with code 0 on all tests. :raises: :class:`~oioioi.problems.package.ProblemPackageError` otherwise. """ env = self._find_and_compile('inwer') if env and not self.use_make: jobs = {} for test in tests: job = env.copy() job['job_type'] = 'inwer' job['task_priority'] = TASK_PRIORITY job['exe_file'] = env['compiled_file'] job['in_file'] = django_to_filetracker_path(test.input_file) job['use_sandboxes'] = self.use_sandboxes jobs[test.name] = job jobs = run_sioworkers_jobs(jobs) get_client().delete_file(env['compiled_file']) for test_name, job in six.iteritems(jobs): if job['result_code'] != 'OK': raise ProblemPackageError(_("Inwer failed on test " "%(test)s. Inwer output %(output)s") % { 'test': test_name, 'output': '\n'.join(job['stdout'])} ) logger.info("%s: inwer success", self.filename)
def _extract_and_process_package(self): tmpdir = tempfile.mkdtemp() logger.info("%s: tmpdir is %s", self.filename, tmpdir) try: self.archive.extract(to_path=tmpdir) self.rootdir = os.path.join(tmpdir, self.short_name) self._process_package() return self.problem finally: shutil.rmtree(tmpdir) if self.prog_archive: get_client().delete_file(self.prog_archive)
def _make_ins(self, re_string): env = self._find_and_compile('ingen') if env and not self.use_make: env['job_type'] = 'ingen' env['exe_file'] = env['compiled_file'] env['re_string'] = re_string env['use_sandboxes'] = self.use_sandboxes env['collected_files_path'] = _make_filename(self.env, 'in') renv = run_sioworkers_job(env) get_client().delete_file(env['compiled_file']) return renv['collected_files'] else: return {}
def handle(self, *args, **options): needed_files = self._get_needed_files() all_files = get_client().list_local_files() max_date_to_delete = datetime.datetime.now() - datetime.timedelta( days=options['days']) diff = set([f[0] for f in all_files]) - set(needed_files) to_delete = [ f[0] for f in all_files if f[0] in diff and datetime.datetime.fromtimestamp(f[1]) < max_date_to_delete ] files_count = len(to_delete) if files_count == 0 and int(options['verbosity']) > 0: print(_("No files to delete.")) elif options['pretend']: if int(options['verbosity']) > 1: print( ungettext( "The following %d file is scheduled for " "deletion:", "The following %d files are scheduled for " "deletion:", files_count, ) % files_count) for file in to_delete: print(" ", file) elif int(options['verbosity']) == 1: print( ungettext( "%d file scheduled for deletion.", "%d files scheduled for deletion.", files_count, ) % files_count) else: if int(options['verbosity']) > 1: print( ungettext( "Deleting the following %d file:", "Deleting the following %d files:", files_count, ) % files_count) if int(options['verbosity']) == 1: print( ungettext("Deleting %d file", "Deleting %d files", files_count) % files_count) for file in to_delete: if int(options['verbosity']) > 1: print(" ", file) get_client().delete_file('/' + file)
def test_handlers(self): submission = TestRunProgramSubmission.objects.get(pk=1) environ = create_environ() environ['job_id'] = 'job_id' environ['submission_id'] = submission.id environ['submission_kind'] = submission.kind environ['problem_instance_id'] = submission.problem_instance.id environ['problem_id'] = submission.problem.id environ['round_id'] = submission.problem_instance.round.id environ['contest_id'] = submission.problem_instance.id # Simulate successful compilation environ['compilation_result'] = 'OK' environ['compilation_message'] = '' environ = handlers.make_test(environ) self.assertIn('tests', environ) self.assertIn('test', environ['tests']) self.assertIn('in_file', environ['tests']['test']) # Simulate running tests FiletrackerStorage().save('output', ContentFile('o')) try: environ['test_results'] = {} environ['test_results']['test'] = { 'result_cpode': 'OK', 'result_string': 'OK', 'time_used': 111, 'out_file': '/output' } environ = handlers.grade_submission(environ) self.assertEqual(None, environ['score']) self.assertEqual('OK', environ['status']) environ = handlers.make_report(environ) self.assertIn('report_id', environ) report = TestRunReport.objects.get( submission_report=environ['report_id']) self.assertEqual(111, report.time_used) self.assertEqual('', report.comment) self.assertEqual('o', report.output_file.read()) handlers.delete_output(environ) except Exception: get_client().delete_file('/output')
def unpack(self, env, package): self.short_name = self._find_main_folder() self.env = env self.package = package existing_problem = self.package.problem if existing_problem: self.problem = existing_problem self.main_problem_instance = self.problem.main_problem_instance if existing_problem.short_name != self.short_name: raise ProblemPackageError(_("Tried to replace problem " "'%(oldname)s' with '%(newname)s'. For safety, changing " "problem short name is not possible.") % dict(oldname=existing_problem.short_name, newname=self.short_name)) else: author_username = env.get('author') if author_username: author = User.objects.get(username=author_username) else: author = None self.problem = Problem.create( name=self.short_name, short_name=self.short_name, controller_name=self.controller_name, contest=self.package.contest, is_public=(author is None), author=author) problem_site = ProblemSite(problem=self.problem, url_key=generate_key()) problem_site.save() self.problem.problem_site = problem_site self.main_problem_instance = self.problem.main_problem_instance self.problem.package_backend_name = self.package_backend_name self.problem.save() tmpdir = tempfile.mkdtemp() logger.info("%s: tmpdir is %s", self.filename, tmpdir) try: self.archive.extract(to_path=tmpdir) self.rootdir = os.path.join(tmpdir, self.short_name) self.process_package() return self.problem finally: shutil.rmtree(tmpdir) if self.prog_archive: get_client().delete_file(self.prog_archive)
def _make_ins(self, re_string): env = self._find_and_compile('ingen') if env and not self.use_make: env['job_type'] = 'ingen' env['task_priority'] = TASK_PRIORITY env['exe_file'] = env['compiled_file'] env['re_string'] = re_string env['use_sandboxes'] = self.use_sandboxes env['collected_files_path'] = \ _make_filename_in_job_dir(self.env, 'in') renv = run_sioworkers_job(env) get_client().delete_file(env['compiled_file']) return renv['collected_files'] return {}
def __init__(self, prefix='/', client=None): if client is None: client = get_client() assert prefix.startswith('/'), \ 'FiletrackerStorage.__init__ prefix must start with /' self.client = client self.prefix = prefix
def tearDown(self): fc = get_client() for filename in (self.remote_source_file, self.remote_wrong_source_file, self.remote_in_file, self.remote_out_file): fc.delete_file(filename)
def upload_inout(env, **kwargs): fc = get_client() env.update({ 'in_file': env['remote_in_file'], 'hint_file': env['remote_out_file']}) fc.put_file(env['remote_in_file'], env['local_in_file']) fc.put_file(env['remote_out_file'], env['local_out_file']) return env
def fill_outfile_in_existing_test_reports(env, **kwargs): """Fill output files into existing test reports that are not directly related to present submission. Also change status of UserOutGenStatus object to finished. Used ``environ`` keys: * ``extra_args`` dictionary with ``submission_report`` object * ``test_results`` """ if 'submission_report_id' not in env['extra_args']: logger.info('No submission_report given to fill tests outputs') return env submission_report_id = env['extra_args']['submission_report_id'] submission_report = SubmissionReport.objects.get(id=submission_report_id) test_reports = TestReport.objects.filter( submission_report=submission_report) test_results = env.get('test_results', {}) for test_name, result in six.iteritems(test_results): try: testreport = test_reports.get(test_name=test_name) except (TestReport.DoesNotExist, TestReport.MultipleObjectsReturned): logger.warning('Test report for test: %s can not be determined', test_name) continue if testreport.output_file: logger.warning( 'Output for test report %s exists. Deleting old one.', testreport.id) get_client().delete_file(testreport.output_file) testreport.output_file = filetracker_to_django_file(result['out_file']) testreport.save() try: download_controller = UserOutGenStatus.objects.get( testreport=testreport) except UserOutGenStatus.DoesNotExist: download_controller = UserOutGenStatus(testreport=testreport) download_controller.status = 'OK' download_controller.save() return env
def fill_outfile_in_existing_test_reports(env, **kwargs): """Fill output files into existing test reports that are not directly related to present submission. Also change status of UserOutGenStatus object to finished. Used ``environ`` keys: * ``extra_args`` dictionary with ``submission_report`` object * ``test_results`` """ if 'submission_report_id' not in env['extra_args']: logger.info('No submission_report given to fill tests outputs') return env submission_report_id = env['extra_args']['submission_report_id'] submission_report = SubmissionReport.objects.get(id=submission_report_id) test_reports = TestReport.objects.filter( submission_report=submission_report) test_results = env.get('test_results', {}) for test_name, result in test_results.iteritems(): try: testreport = test_reports.get(test_name=test_name) except (TestReport.DoesNotExist, TestReport.MultipleObjectsReturned): logger.warn('Test report for test: %s can not be determined', test_name) continue if testreport.output_file: logger.warn('Output for test report %s exists. Deleting old one.', testreport.id) get_client().delete_file(testreport.output_file) testreport.output_file = filetracker_to_django_file(result['out_file']) testreport.save() try: download_controller = UserOutGenStatus.objects.get( testreport=testreport) except UserOutGenStatus.DoesNotExist: download_controller = UserOutGenStatus(testreport=testreport) download_controller.status = 'OK' download_controller.save() return env
def _make_outs(self, outs_to_make): env = self._find_and_compile('', command='outgen') if not env: return {} jobs = {} for outname, test in outs_to_make: job = env.copy() job['job_type'] = 'exec' if self.use_sandboxes else 'unsafe-exec' job['exe_file'] = env['compiled_file'] job['upload_out'] = True job['in_file'] = django_to_filetracker_path(test.input_file) job['out_file'] = outname jobs[test.name] = job jobs = run_sioworkers_jobs(jobs) get_client().delete_file(env['compiled_file']) return jobs
def _save_prog_dir(self): prog_dir = os.path.join(self.rootdir, 'prog') if not os.path.isdir(prog_dir): return archive_name = 'compilation-dir-archive' archive = shutil.make_archive( os.path.join(self.rootdir, archive_name), format="zip", root_dir=prog_dir) self.prog_archive = get_client().put_file( _make_filename(self.env, archive), archive)
def _save_prog_dir(self): """Creates an archive to store compiled programs.""" prog_dir = os.path.join(self.rootdir, 'prog') if not os.path.isdir(prog_dir): return archive_name = 'compilation-dir-archive' archive = shutil.make_archive(os.path.join(self.rootdir, archive_name), format="zip", root_dir=prog_dir) self.prog_archive = get_client().put_file( _make_filename_in_job_dir(self.env, archive), archive)
def _save_prog_dir(self): """Creates archive with programs directory. """ prog_dir = os.path.join(self.rootdir, 'prog') if not os.path.isdir(prog_dir): return archive_name = 'compilation-dir-archive' archive = shutil.make_archive( os.path.join(self.rootdir, archive_name), format="zip", root_dir=prog_dir) self.prog_archive = get_client().put_file( _make_filename_in_job_dir(self.env, archive), archive)
def handle(self, *args, **options): needed_files = self._get_needed_files() all_files = get_client().list_local_files() max_date_to_delete = datetime.datetime.now() - datetime. \ timedelta(days=options['days']) diff = set([f[0] for f in all_files]) - set(needed_files) to_delete = [f[0] for f in all_files if f[0] in diff and datetime. datetime.fromtimestamp(f[1]) < max_date_to_delete] files_count = len(to_delete) if files_count == 0 and int(options['verbosity']) > 0: print(_("No files to delete.")) elif options['pretend']: if int(options['verbosity']) > 1: print(ungettext("The following %d file is scheduled for " "deletion:", "The following %d files are scheduled for " "deletion:", files_count) % files_count) for file in to_delete: print(" ", file) elif int(options['verbosity']) == 1: print(ungettext("%d file scheduled for deletion.", "%d files scheduled for deletion.", files_count) % files_count) else: if int(options['verbosity']) > 1: print(ungettext("Deleting the following %d file:", "Deleting the following %d files:", files_count) % files_count) if int(options['verbosity']) == 1: print(ungettext("Deleting %d file", "Deleting %d files", files_count) % files_count) for file in to_delete: if int(options['verbosity']) > 1: print(" ", file) get_client().delete_file('/' + file)
def _verify_ins(self, tests): env = self._find_and_compile('inwer') if env and not self.use_make: jobs = {} for test in tests: job = env.copy() job['job_type'] = 'inwer' job['exe_file'] = env['compiled_file'] job['in_file'] = django_to_filetracker_path(test.input_file) job['use_sandboxes'] = self.use_sandboxes jobs[test.name] = job jobs = run_sioworkers_jobs(jobs) get_client().delete_file(env['compiled_file']) for test_name, job in jobs.iteritems(): if job['result_code'] != 'OK': raise ProblemPackageError(_("Inwer failed on test " "%(test)s. Inwer output %(output)s") % {'test': test_name, 'output': '\n'.join(job['stdout'])} ) logger.info("%s: inwer success", self.filename)
def _make_outs(self, outs_to_make): """Run jobs to generate test outputs. :return: Result from workers. """ env = self._find_and_compile('', command='outgen') if not env: return {} jobs = {} for outname, test in outs_to_make: job = env.copy() job['job_type'] = 'exec' if self.use_sandboxes else 'unsafe-exec' job['task_priority'] = TASK_PRIORITY job['exe_file'] = env['compiled_file'] job['upload_out'] = True job['in_file'] = django_to_filetracker_path(test.input_file) job['out_file'] = outname if test.memory_limit: job['exec_mem_limit'] = test.memory_limit jobs[test.name] = job jobs = run_sioworkers_jobs(jobs) get_client().delete_file(env['compiled_file']) return jobs
def _compile(self, filename, prog_name, ext, out_name=None): client = get_client() source_name = '%s.%s' % (prog_name, ext) ft_source_name = client.put_file( _make_filename_in_job_dir(self.env, source_name), filename) if not out_name: out_name = _make_filename_in_job_dir(self.env, '%s.e' % prog_name) new_env = self._run_compilation_job(ext, ft_source_name, out_name) client.delete_file(ft_source_name) self._ensure_compilation_success(filename, new_env) # TODO Remeber about 'exec_info' when Java support is introduced. new_env['compiled_file'] = new_env['out_file'] return new_env
def _compile(self, filename, prog_name, ext, out_name=None): client = get_client() source_name = '%s.%s' % (prog_name, ext) ft_source_name = client.put_file(_make_filename(self.env, source_name), filename) if not out_name: out_name = _make_filename(self.env, '%s.e' % prog_name) compilation_job = self.env.copy() compilation_job['job_type'] = 'compile' compilation_job['source_file'] = ft_source_name compilation_job['out_file'] = out_name lang = ext compilation_job['language'] = lang if self.use_sandboxes: prefix = 'default' else: prefix = 'system' compilation_job['compiler'] = prefix + '-' + lang if not self.use_make and self.prog_archive: compilation_job['additional_archive'] = self.prog_archive add_extra_files(compilation_job, self.problem, additional_args=self.extra_compilation_args) new_env = run_sioworkers_job(compilation_job) client.delete_file(ft_source_name) compilation_message = new_env.get('compiler_output', '') compilation_result = new_env.get('result_code', 'CE') if compilation_result != 'OK': logger.warning("%s: compilation of file %s failed with code %s", self.filename, filename, compilation_result) logger.warning("%s: compiler output: %r", self.filename, compilation_message) raise ProblemPackageError(_("Compilation of file %(filename)s " "failed. Compiler output: %(output)s") % { 'filename': filename, 'output': compilation_message}) # TODO Remeber about 'exec_info' when Java support is introduced. new_env['compiled_file'] = new_env['out_file'] return new_env
def delete_executable(env, **kwargs): if 'compiled_file' in env: get_client().delete_file(env['compiled_file']) return env
def _save_to_field(self, field, file): basename = os.path.basename(filetracker_to_django_file(file).name) filename = os.path.join(self.rootdir, basename) get_client().get_file(file, filename) field.save(os.path.basename(filename), File(open(filename, 'rb'))) get_client().delete_file(file)
def delete_output(env, **kwargs): if 'out_file' in env.get('test_results', {}).get('test', {}): get_client().delete_file(env['test_results']['test']['out_file']) return env
except urllib2.HTTPError, e: if e.code != 404: raise if not args: args = manifest print >>self.stdout, "--- Preparing ..." urls = [] for arg in args: if arg not in manifest: raise CommandError("Sandbox '%s' not available (not in " "Manifest)" % (arg,)) urls.append(urlparse.urljoin(manifest_url, arg + '.tar.gz')) filetracker = get_client() dir = options['download_dir'] if not os.path.exists(dir): os.makedirs(dir) try: execute([options['wget'], '--version']) except ExecuteError: raise CommandError("Wget not working. Please specify a working " "Wget binary using --wget option.") print >>self.stdout, "--- Downloading sandboxes ..." execute([options['wget'], '-N', '-i', '-'], stdin='\n'.join(urls), capture_output=False, cwd=dir)
def upload_source(env, **kwargs): fc = get_client() fc.put_file(env['remote_source_file'], env['local_source_file']) return env
def handle(self, *args, **options): print("--- Downloading Manifest ...", file=self.stdout) try: manifest_url = options['manifest_url'] manifest = (six.moves.urllib.request.urlopen( manifest_url).read().decode('utf-8')) manifest = manifest.strip().splitlines() except Exception as e: raise CommandError("Error downloading manifest: %s" % (e, )) print("--- Looking for license ...", file=self.stdout) try: license_url = six.moves.urllib.parse.urljoin( manifest_url, 'LICENSE') license = (six.moves.urllib.request.urlopen( license_url).read().decode('utf-8')) if not options['license_agreement']: self.display_license(license) except six.moves.urllib.error.HTTPError as e: if e.code != 404: raise args = options['sandboxes'] if not args: args = manifest print("--- Preparing ...", file=self.stdout) urls = [] cached_args = [] for arg in args: basename = arg + '.tar.gz' if options['cache_dir']: path = os.path.join(options['cache_dir'], basename) if os.path.isfile(path): cached_args.append(arg) continue if arg not in manifest: raise CommandError("Sandbox '%s' not available (not in " "Manifest)" % (arg, )) urls.append(six.moves.urllib.parse.urljoin(manifest_url, basename)) filetracker = get_client() download_dir = options['download_dir'] if not os.path.exists(download_dir): os.makedirs(download_dir) try: execute([options['wget'], '--version']) except ExecuteError: raise CommandError("Wget not working. Please specify a working " "Wget binary using --wget option.") if len(urls) > 0: print("--- Downloading sandboxes ...", file=self.stdout) quiet_flag = ['-nv'] if options['quiet'] else [] execute( [options['wget'], '-N', '-i', '-'] + quiet_flag, stdin='\n'.join(urls).encode('utf-8'), capture_output=False, cwd=download_dir, ) print("--- Saving sandboxes to the Filetracker ...", file=self.stdout) for arg in args: basename = arg + '.tar.gz' if arg in cached_args: local_file = os.path.join(options['cache_dir'], basename) else: local_file = os.path.join(download_dir, basename) print(" ", basename, file=self.stdout) filetracker.put_file('/sandboxes/' + basename, local_file) if arg not in cached_args: os.unlink(local_file) try: os.rmdir(download_dir) except OSError: print( "--- Done, but couldn't remove the " "downloads directory.", file=self.stdout, ) else: print("--- Done.", file=self.stdout)
def __init__(self, contest, round=None, only_final=True): self.contest = contest self.round = round self.only_final = only_final self.filetracker = get_client()
def on_worker_init(self): super(OioioiLoader, self).on_worker_init() # This initializes sioworker's filetracker client as well. get_client()
def delete_executable(env, **kwargs): if "compiled_file" in env: get_client().delete_file(env["compiled_file"]) return env
except urllib2.HTTPError, e: if e.code != 404: raise if not args: args = manifest print >> self.stdout, "--- Preparing ..." urls = [] for arg in args: if arg not in manifest: raise CommandError("Sandbox '%s' not available (not in " "Manifest)" % (arg, )) urls.append(urlparse.urljoin(manifest_url, arg + '.tar.gz')) filetracker = get_client() dir = options['download_dir'] if not os.path.exists(dir): os.makedirs(dir) try: execute([options['wget'], '--version']) except ExecuteError: raise CommandError("Wget not working. Please specify a working " "Wget binary using --wget option.") print >> self.stdout, "--- Downloading sandboxes ..." execute([options['wget'], '-N', '-i', '-'], stdin='\n'.join(urls), capture_output=False,
def __init__(self, prefix="/", client=None): if client is None: client = get_client() assert prefix.startswith("/"), "FiletrackerStorage.__init__ prefix must start with /" self.client = client self.prefix = prefix