def _make_outs(self, outs_to_make):
"""Compiles the model solution and executes it in order to generate
test outputs.
:return: Result from workers.
"""
env = self._find_and_compile('', command='outgen')
if not env:
return {}
jobs = {}
for outname, test in outs_to_make:
job = env.copy()
job['job_type'] = 'exec' if self.use_sandboxes else 'unsafe-exec'
job['task_priority'] = TASK_PRIORITY
job['exe_file'] = env['compiled_file']
job['upload_out'] = True
job['in_file'] = django_to_filetracker_path(test.input_file)
job['out_file'] = outname
if test.memory_limit:
job['exec_mem_limit'] = test.memory_limit
jobs[test.name] = job
jobs = run_sioworkers_jobs(jobs)
get_client().delete_file(env['compiled_file'])
return jobs
def _verify_inputs(self, tests):
"""Check if correct solution exits with code 0 on all tests.
:raises: :class:`~oioioi.problems.package.ProblemPackageError`
otherwise.
"""
env = self._find_and_compile('inwer')
if env and not self.use_make:
jobs = {}
for test in tests:
job = env.copy()
job['job_type'] = 'inwer'
job['task_priority'] = TASK_PRIORITY
job['exe_file'] = env['compiled_file']
job['in_file'] = django_to_filetracker_path(test.input_file)
job['use_sandboxes'] = self.use_sandboxes
jobs[test.name] = job
jobs = run_sioworkers_jobs(jobs)
get_client().delete_file(env['compiled_file'])
for test_name, job in jobs.iteritems():
if job['result_code'] != 'OK':
raise ProblemPackageError(
_("Inwer failed on test "
"%(test)s. Inwer output %(output)s") % {
'test': test_name,
'output': '\n'.join(job['stdout'])
})
logger.info("%s: inwer success", self.filename)
def _verify_inputs(self, tests):
"""Check if correct solution exits with code 0 on all tests.
:raises: :class:`~oioioi.problems.package.ProblemPackageError`
otherwise.
"""
env = self._find_and_compile('inwer')
if env and not self.use_make:
jobs = {}
for test in tests:
job = env.copy()
job['job_type'] = 'inwer'
job['task_priority'] = TASK_PRIORITY
job['exe_file'] = env['compiled_file']
job['in_file'] = django_to_filetracker_path(test.input_file)
job['use_sandboxes'] = self.use_sandboxes
jobs[test.name] = job
jobs = run_sioworkers_jobs(jobs)
get_client().delete_file(env['compiled_file'])
for test_name, job in six.iteritems(jobs):
if job['result_code'] != 'OK':
raise ProblemPackageError(_("Inwer failed on test "
"%(test)s. Inwer output %(output)s") %
{
'test': test_name,
'output': '\n'.join(job['stdout'])}
)
logger.info("%s: inwer success", self.filename)
def _extract_and_process_package(self):
tmpdir = tempfile.mkdtemp()
logger.info("%s: tmpdir is %s", self.filename, tmpdir)
try:
self.archive.extract(to_path=tmpdir)
self.rootdir = os.path.join(tmpdir, self.short_name)
self._process_package()
return self.problem
finally:
shutil.rmtree(tmpdir)
if self.prog_archive:
get_client().delete_file(self.prog_archive)
def _extract_and_process_package(self):
tmpdir = tempfile.mkdtemp()
logger.info("%s: tmpdir is %s", self.filename, tmpdir)
try:
self.archive.extract(to_path=tmpdir)
self.rootdir = os.path.join(tmpdir, self.short_name)
self._process_package()
return self.problem
finally:
shutil.rmtree(tmpdir)
if self.prog_archive:
get_client().delete_file(self.prog_archive)
def _make_ins(self, re_string):
env = self._find_and_compile('ingen')
if env and not self.use_make:
env['job_type'] = 'ingen'
env['exe_file'] = env['compiled_file']
env['re_string'] = re_string
env['use_sandboxes'] = self.use_sandboxes
env['collected_files_path'] = _make_filename(self.env, 'in')
renv = run_sioworkers_job(env)
get_client().delete_file(env['compiled_file'])
return renv['collected_files']
else:
return {}
def handle(self, *args, **options):
needed_files = self._get_needed_files()
all_files = get_client().list_local_files()
max_date_to_delete = datetime.datetime.now() - datetime.timedelta(
days=options['days'])
diff = set([f[0] for f in all_files]) - set(needed_files)
to_delete = [
f[0] for f in all_files if f[0] in diff
and datetime.datetime.fromtimestamp(f[1]) < max_date_to_delete
]
files_count = len(to_delete)
if files_count == 0 and int(options['verbosity']) > 0:
print(_("No files to delete."))
elif options['pretend']:
if int(options['verbosity']) > 1:
print(
ungettext(
"The following %d file is scheduled for "
"deletion:",
"The following %d files are scheduled for "
"deletion:",
files_count,
) % files_count)
for file in to_delete:
print(" ", file)
elif int(options['verbosity']) == 1:
print(
ungettext(
"%d file scheduled for deletion.",
"%d files scheduled for deletion.",
files_count,
) % files_count)
else:
if int(options['verbosity']) > 1:
print(
ungettext(
"Deleting the following %d file:",
"Deleting the following %d files:",
files_count,
) % files_count)
if int(options['verbosity']) == 1:
print(
ungettext("Deleting %d file", "Deleting %d files",
files_count) % files_count)
for file in to_delete:
if int(options['verbosity']) > 1:
print(" ", file)
get_client().delete_file('/' + file)
def test_handlers(self):
submission = TestRunProgramSubmission.objects.get(pk=1)
environ = create_environ()
environ['job_id'] = 'job_id'
environ['submission_id'] = submission.id
environ['submission_kind'] = submission.kind
environ['problem_instance_id'] = submission.problem_instance.id
environ['problem_id'] = submission.problem.id
environ['round_id'] = submission.problem_instance.round.id
environ['contest_id'] = submission.problem_instance.id
# Simulate successful compilation
environ['compilation_result'] = 'OK'
environ['compilation_message'] = ''
environ = handlers.make_test(environ)
self.assertIn('tests', environ)
self.assertIn('test', environ['tests'])
self.assertIn('in_file', environ['tests']['test'])
# Simulate running tests
FiletrackerStorage().save('output', ContentFile('o'))
try:
environ['test_results'] = {}
environ['test_results']['test'] = {
'result_cpode': 'OK',
'result_string': 'OK',
'time_used': 111,
'out_file': '/output'
}
environ = handlers.grade_submission(environ)
self.assertEqual(None, environ['score'])
self.assertEqual('OK', environ['status'])
environ = handlers.make_report(environ)
self.assertIn('report_id', environ)
report = TestRunReport.objects.get(
submission_report=environ['report_id'])
self.assertEqual(111, report.time_used)
self.assertEqual('', report.comment)
self.assertEqual('o', report.output_file.read())
handlers.delete_output(environ)
except Exception:
get_client().delete_file('/output')
def test_handlers(self):
submission = TestRunProgramSubmission.objects.get(pk=1)
environ = create_environ()
environ['job_id'] = 'job_id'
environ['submission_id'] = submission.id
environ['submission_kind'] = submission.kind
environ['problem_instance_id'] = submission.problem_instance.id
environ['problem_id'] = submission.problem.id
environ['round_id'] = submission.problem_instance.round.id
environ['contest_id'] = submission.problem_instance.id
# Simulate successful compilation
environ['compilation_result'] = 'OK'
environ['compilation_message'] = ''
environ = handlers.make_test(environ)
self.assertIn('tests', environ)
self.assertIn('test', environ['tests'])
self.assertIn('in_file', environ['tests']['test'])
# Simulate running tests
FiletrackerStorage().save('output', ContentFile('o'))
try:
environ['test_results'] = {}
environ['test_results']['test'] = {
'result_cpode': 'OK',
'result_string': 'OK',
'time_used': 111,
'out_file': '/output'
}
environ = handlers.grade_submission(environ)
self.assertEqual(None, environ['score'])
self.assertEqual('OK', environ['status'])
environ = handlers.make_report(environ)
self.assertIn('report_id', environ)
report = TestRunReport.objects.get(
submission_report=environ['report_id'])
self.assertEqual(111, report.time_used)
self.assertEqual('', report.comment)
self.assertEqual('o', report.output_file.read())
handlers.delete_output(environ)
except Exception:
get_client().delete_file('/output')
def unpack(self, env, package):
self.short_name = self._find_main_folder()
self.env = env
self.package = package
existing_problem = self.package.problem
if existing_problem:
self.problem = existing_problem
self.main_problem_instance = self.problem.main_problem_instance
if existing_problem.short_name != self.short_name:
raise ProblemPackageError(_("Tried to replace problem "
"'%(oldname)s' with '%(newname)s'. For safety, changing "
"problem short name is not possible.") %
dict(oldname=existing_problem.short_name,
newname=self.short_name))
else:
author_username = env.get('author')
if author_username:
author = User.objects.get(username=author_username)
else:
author = None
self.problem = Problem.create(
name=self.short_name,
short_name=self.short_name,
controller_name=self.controller_name,
contest=self.package.contest,
is_public=(author is None),
author=author)
problem_site = ProblemSite(problem=self.problem,
url_key=generate_key())
problem_site.save()
self.problem.problem_site = problem_site
self.main_problem_instance = self.problem.main_problem_instance
self.problem.package_backend_name = self.package_backend_name
self.problem.save()
tmpdir = tempfile.mkdtemp()
logger.info("%s: tmpdir is %s", self.filename, tmpdir)
try:
self.archive.extract(to_path=tmpdir)
self.rootdir = os.path.join(tmpdir, self.short_name)
self.process_package()
return self.problem
finally:
shutil.rmtree(tmpdir)
if self.prog_archive:
get_client().delete_file(self.prog_archive)
def _make_ins(self, re_string):
env = self._find_and_compile('ingen')
if env and not self.use_make:
env['job_type'] = 'ingen'
env['task_priority'] = TASK_PRIORITY
env['exe_file'] = env['compiled_file']
env['re_string'] = re_string
env['use_sandboxes'] = self.use_sandboxes
env['collected_files_path'] = \
_make_filename_in_job_dir(self.env, 'in')
renv = run_sioworkers_job(env)
get_client().delete_file(env['compiled_file'])
return renv['collected_files']
return {}
def __init__(self, prefix='/', client=None):
if client is None:
client = get_client()
assert prefix.startswith('/'), \
'FiletrackerStorage.__init__ prefix must start with /'
self.client = client
self.prefix = prefix
def _make_ins(self, re_string):
env = self._find_and_compile('ingen')
if env and not self.use_make:
env['job_type'] = 'ingen'
env['task_priority'] = TASK_PRIORITY
env['exe_file'] = env['compiled_file']
env['re_string'] = re_string
env['use_sandboxes'] = self.use_sandboxes
env['collected_files_path'] = \
_make_filename_in_job_dir(self.env, 'in')
renv = run_sioworkers_job(env)
get_client().delete_file(env['compiled_file'])
return renv['collected_files']
return {}
def tearDown(self):
fc = get_client()
for filename in (self.remote_source_file,
self.remote_wrong_source_file,
self.remote_in_file,
self.remote_out_file):
fc.delete_file(filename)
def tearDown(self):
fc = get_client()
for filename in (self.remote_source_file,
self.remote_wrong_source_file,
self.remote_in_file,
self.remote_out_file):
fc.delete_file(filename)
def __init__(self, prefix='/', client=None):
if client is None:
client = get_client()
assert prefix.startswith('/'), \
'FiletrackerStorage.__init__ prefix must start with /'
self.client = client
self.prefix = prefix
def upload_inout(env, **kwargs):
fc = get_client()
env.update({
'in_file': env['remote_in_file'],
'hint_file': env['remote_out_file']})
fc.put_file(env['remote_in_file'], env['local_in_file'])
fc.put_file(env['remote_out_file'], env['local_out_file'])
return env
def upload_inout(env, **kwargs):
fc = get_client()
env.update({
'in_file': env['remote_in_file'],
'hint_file': env['remote_out_file']})
fc.put_file(env['remote_in_file'], env['local_in_file'])
fc.put_file(env['remote_out_file'], env['local_out_file'])
return env
def fill_outfile_in_existing_test_reports(env, **kwargs):
"""Fill output files into existing test reports that are not directly
related to present submission. Also change status of UserOutGenStatus
object to finished.
Used ``environ`` keys:
* ``extra_args`` dictionary with ``submission_report`` object
* ``test_results``
"""
if 'submission_report_id' not in env['extra_args']:
logger.info('No submission_report given to fill tests outputs')
return env
submission_report_id = env['extra_args']['submission_report_id']
submission_report = SubmissionReport.objects.get(id=submission_report_id)
test_reports = TestReport.objects.filter(
submission_report=submission_report)
test_results = env.get('test_results', {})
for test_name, result in six.iteritems(test_results):
try:
testreport = test_reports.get(test_name=test_name)
except (TestReport.DoesNotExist, TestReport.MultipleObjectsReturned):
logger.warning('Test report for test: %s can not be determined',
test_name)
continue
if testreport.output_file:
logger.warning(
'Output for test report %s exists. Deleting old one.',
testreport.id)
get_client().delete_file(testreport.output_file)
testreport.output_file = filetracker_to_django_file(result['out_file'])
testreport.save()
try:
download_controller = UserOutGenStatus.objects.get(
testreport=testreport)
except UserOutGenStatus.DoesNotExist:
download_controller = UserOutGenStatus(testreport=testreport)
download_controller.status = 'OK'
download_controller.save()
return env
def fill_outfile_in_existing_test_reports(env, **kwargs):
"""Fill output files into existing test reports that are not directly
related to present submission. Also change status of UserOutGenStatus
object to finished.
Used ``environ`` keys:
* ``extra_args`` dictionary with ``submission_report`` object
* ``test_results``
"""
if 'submission_report_id' not in env['extra_args']:
logger.info('No submission_report given to fill tests outputs')
return env
submission_report_id = env['extra_args']['submission_report_id']
submission_report = SubmissionReport.objects.get(id=submission_report_id)
test_reports = TestReport.objects.filter(
submission_report=submission_report)
test_results = env.get('test_results', {})
for test_name, result in test_results.iteritems():
try:
testreport = test_reports.get(test_name=test_name)
except (TestReport.DoesNotExist, TestReport.MultipleObjectsReturned):
logger.warn('Test report for test: %s can not be determined',
test_name)
continue
if testreport.output_file:
logger.warn('Output for test report %s exists. Deleting old one.',
testreport.id)
get_client().delete_file(testreport.output_file)
testreport.output_file = filetracker_to_django_file(result['out_file'])
testreport.save()
try:
download_controller = UserOutGenStatus.objects.get(
testreport=testreport)
except UserOutGenStatus.DoesNotExist:
download_controller = UserOutGenStatus(testreport=testreport)
download_controller.status = 'OK'
download_controller.save()
return env
def _make_outs(self, outs_to_make):
env = self._find_and_compile('', command='outgen')
if not env:
return {}
jobs = {}
for outname, test in outs_to_make:
job = env.copy()
job['job_type'] = 'exec' if self.use_sandboxes else 'unsafe-exec'
job['exe_file'] = env['compiled_file']
job['upload_out'] = True
job['in_file'] = django_to_filetracker_path(test.input_file)
job['out_file'] = outname
jobs[test.name] = job
jobs = run_sioworkers_jobs(jobs)
get_client().delete_file(env['compiled_file'])
return jobs
def _save_prog_dir(self):
prog_dir = os.path.join(self.rootdir, 'prog')
if not os.path.isdir(prog_dir):
return
archive_name = 'compilation-dir-archive'
archive = shutil.make_archive(
os.path.join(self.rootdir, archive_name), format="zip",
root_dir=prog_dir)
self.prog_archive = get_client().put_file(
_make_filename(self.env, archive), archive)
def _save_prog_dir(self):
"""Creates an archive to store compiled programs."""
prog_dir = os.path.join(self.rootdir, 'prog')
if not os.path.isdir(prog_dir):
return
archive_name = 'compilation-dir-archive'
archive = shutil.make_archive(os.path.join(self.rootdir, archive_name),
format="zip",
root_dir=prog_dir)
self.prog_archive = get_client().put_file(
_make_filename_in_job_dir(self.env, archive), archive)
def _save_prog_dir(self):
"""Creates archive with programs directory.
"""
prog_dir = os.path.join(self.rootdir, 'prog')
if not os.path.isdir(prog_dir):
return
archive_name = 'compilation-dir-archive'
archive = shutil.make_archive(
os.path.join(self.rootdir, archive_name), format="zip",
root_dir=prog_dir)
self.prog_archive = get_client().put_file(
_make_filename_in_job_dir(self.env, archive), archive)
def handle(self, *args, **options):
needed_files = self._get_needed_files()
all_files = get_client().list_local_files()
max_date_to_delete = datetime.datetime.now() - datetime. \
timedelta(days=options['days'])
diff = set([f[0] for f in all_files]) - set(needed_files)
to_delete = [f[0] for f in all_files if f[0] in diff and datetime.
datetime.fromtimestamp(f[1]) < max_date_to_delete]
files_count = len(to_delete)
if files_count == 0 and int(options['verbosity']) > 0:
print(_("No files to delete."))
elif options['pretend']:
if int(options['verbosity']) > 1:
print(ungettext("The following %d file is scheduled for "
"deletion:",
"The following %d files are scheduled for "
"deletion:",
files_count) % files_count)
for file in to_delete:
print(" ", file)
elif int(options['verbosity']) == 1:
print(ungettext("%d file scheduled for deletion.",
"%d files scheduled for deletion.",
files_count) % files_count)
else:
if int(options['verbosity']) > 1:
print(ungettext("Deleting the following %d file:",
"Deleting the following %d files:",
files_count) % files_count)
if int(options['verbosity']) == 1:
print(ungettext("Deleting %d file",
"Deleting %d files",
files_count) % files_count)
for file in to_delete:
if int(options['verbosity']) > 1:
print(" ", file)
get_client().delete_file('/' + file)
def _verify_ins(self, tests):
env = self._find_and_compile('inwer')
if env and not self.use_make:
jobs = {}
for test in tests:
job = env.copy()
job['job_type'] = 'inwer'
job['exe_file'] = env['compiled_file']
job['in_file'] = django_to_filetracker_path(test.input_file)
job['use_sandboxes'] = self.use_sandboxes
jobs[test.name] = job
jobs = run_sioworkers_jobs(jobs)
get_client().delete_file(env['compiled_file'])
for test_name, job in jobs.iteritems():
if job['result_code'] != 'OK':
raise ProblemPackageError(_("Inwer failed on test "
"%(test)s. Inwer output %(output)s") %
{'test': test_name, 'output': '\n'.join(job['stdout'])}
)
logger.info("%s: inwer success", self.filename)
def _make_outs(self, outs_to_make):
"""Run jobs to generate test outputs.
:return: Result from workers.
"""
env = self._find_and_compile('', command='outgen')
if not env:
return {}
jobs = {}
for outname, test in outs_to_make:
job = env.copy()
job['job_type'] = 'exec' if self.use_sandboxes else 'unsafe-exec'
job['task_priority'] = TASK_PRIORITY
job['exe_file'] = env['compiled_file']
job['upload_out'] = True
job['in_file'] = django_to_filetracker_path(test.input_file)
job['out_file'] = outname
if test.memory_limit:
job['exec_mem_limit'] = test.memory_limit
jobs[test.name] = job
jobs = run_sioworkers_jobs(jobs)
get_client().delete_file(env['compiled_file'])
return jobs
def _compile(self, filename, prog_name, ext, out_name=None):
client = get_client()
source_name = '%s.%s' % (prog_name, ext)
ft_source_name = client.put_file(
_make_filename_in_job_dir(self.env, source_name), filename)
if not out_name:
out_name = _make_filename_in_job_dir(self.env, '%s.e' % prog_name)
new_env = self._run_compilation_job(ext, ft_source_name, out_name)
client.delete_file(ft_source_name)
self._ensure_compilation_success(filename, new_env)
# TODO Remeber about 'exec_info' when Java support is introduced.
new_env['compiled_file'] = new_env['out_file']
return new_env
def _compile(self, filename, prog_name, ext, out_name=None):
client = get_client()
source_name = '%s.%s' % (prog_name, ext)
ft_source_name = client.put_file(
_make_filename_in_job_dir(self.env, source_name),
filename)
if not out_name:
out_name = _make_filename_in_job_dir(self.env, '%s.e' % prog_name)
new_env = self._run_compilation_job(ext, ft_source_name, out_name)
client.delete_file(ft_source_name)
self._ensure_compilation_success(filename, new_env)
# TODO Remeber about 'exec_info' when Java support is introduced.
new_env['compiled_file'] = new_env['out_file']
return new_env
def _compile(self, filename, prog_name, ext, out_name=None):
client = get_client()
source_name = '%s.%s' % (prog_name, ext)
ft_source_name = client.put_file(_make_filename(self.env, source_name),
filename)
if not out_name:
out_name = _make_filename(self.env, '%s.e' % prog_name)
compilation_job = self.env.copy()
compilation_job['job_type'] = 'compile'
compilation_job['source_file'] = ft_source_name
compilation_job['out_file'] = out_name
lang = ext
compilation_job['language'] = lang
if self.use_sandboxes:
prefix = 'default'
else:
prefix = 'system'
compilation_job['compiler'] = prefix + '-' + lang
if not self.use_make and self.prog_archive:
compilation_job['additional_archive'] = self.prog_archive
add_extra_files(compilation_job, self.problem,
additional_args=self.extra_compilation_args)
new_env = run_sioworkers_job(compilation_job)
client.delete_file(ft_source_name)
compilation_message = new_env.get('compiler_output', '')
compilation_result = new_env.get('result_code', 'CE')
if compilation_result != 'OK':
logger.warning("%s: compilation of file %s failed with code %s",
self.filename, filename, compilation_result)
logger.warning("%s: compiler output: %r", self.filename,
compilation_message)
raise ProblemPackageError(_("Compilation of file %(filename)s "
"failed. Compiler output: %(output)s") % {
'filename': filename, 'output': compilation_message})
# TODO Remeber about 'exec_info' when Java support is introduced.
new_env['compiled_file'] = new_env['out_file']
return new_env
def delete_executable(env, **kwargs):
if 'compiled_file' in env:
get_client().delete_file(env['compiled_file'])
return env
def _save_to_field(self, field, file):
basename = os.path.basename(filetracker_to_django_file(file).name)
filename = os.path.join(self.rootdir, basename)
get_client().get_file(file, filename)
field.save(os.path.basename(filename), File(open(filename, 'rb')))
get_client().delete_file(file)
def delete_executable(env, **kwargs):
if 'compiled_file' in env:
get_client().delete_file(env['compiled_file'])
return env
def _save_to_field(self, field, file):
basename = os.path.basename(filetracker_to_django_file(file).name)
filename = os.path.join(self.rootdir, basename)
get_client().get_file(file, filename)
field.save(os.path.basename(filename), File(open(filename, 'rb')))
get_client().delete_file(file)
def delete_output(env, **kwargs):
if 'out_file' in env.get('test_results', {}).get('test', {}):
get_client().delete_file(env['test_results']['test']['out_file'])
return env
except urllib2.HTTPError, e:
if e.code != 404:
raise
if not args:
args = manifest
print >>self.stdout, "--- Preparing ..."
urls = []
for arg in args:
if arg not in manifest:
raise CommandError("Sandbox '%s' not available (not in "
"Manifest)" % (arg,))
urls.append(urlparse.urljoin(manifest_url, arg + '.tar.gz'))
filetracker = get_client()
dir = options['download_dir']
if not os.path.exists(dir):
os.makedirs(dir)
try:
execute([options['wget'], '--version'])
except ExecuteError:
raise CommandError("Wget not working. Please specify a working "
"Wget binary using --wget option.")
print >>self.stdout, "--- Downloading sandboxes ..."
execute([options['wget'], '-N', '-i', '-'], stdin='\n'.join(urls),
capture_output=False, cwd=dir)
def upload_source(env, **kwargs):
fc = get_client()
fc.put_file(env['remote_source_file'], env['local_source_file'])
return env
def delete_output(env, **kwargs):
if 'out_file' in env.get('test_results', {}).get('test', {}):
get_client().delete_file(env['test_results']['test']['out_file'])
return env
def handle(self, *args, **options):
print("--- Downloading Manifest ...", file=self.stdout)
try:
manifest_url = options['manifest_url']
manifest = (six.moves.urllib.request.urlopen(
manifest_url).read().decode('utf-8'))
manifest = manifest.strip().splitlines()
except Exception as e:
raise CommandError("Error downloading manifest: %s" % (e, ))
print("--- Looking for license ...", file=self.stdout)
try:
license_url = six.moves.urllib.parse.urljoin(
manifest_url, 'LICENSE')
license = (six.moves.urllib.request.urlopen(
license_url).read().decode('utf-8'))
if not options['license_agreement']:
self.display_license(license)
except six.moves.urllib.error.HTTPError as e:
if e.code != 404:
raise
args = options['sandboxes']
if not args:
args = manifest
print("--- Preparing ...", file=self.stdout)
urls = []
cached_args = []
for arg in args:
basename = arg + '.tar.gz'
if options['cache_dir']:
path = os.path.join(options['cache_dir'], basename)
if os.path.isfile(path):
cached_args.append(arg)
continue
if arg not in manifest:
raise CommandError("Sandbox '%s' not available (not in "
"Manifest)" % (arg, ))
urls.append(six.moves.urllib.parse.urljoin(manifest_url, basename))
filetracker = get_client()
download_dir = options['download_dir']
if not os.path.exists(download_dir):
os.makedirs(download_dir)
try:
execute([options['wget'], '--version'])
except ExecuteError:
raise CommandError("Wget not working. Please specify a working "
"Wget binary using --wget option.")
if len(urls) > 0:
print("--- Downloading sandboxes ...", file=self.stdout)
quiet_flag = ['-nv'] if options['quiet'] else []
execute(
[options['wget'], '-N', '-i', '-'] + quiet_flag,
stdin='\n'.join(urls).encode('utf-8'),
capture_output=False,
cwd=download_dir,
)
print("--- Saving sandboxes to the Filetracker ...", file=self.stdout)
for arg in args:
basename = arg + '.tar.gz'
if arg in cached_args:
local_file = os.path.join(options['cache_dir'], basename)
else:
local_file = os.path.join(download_dir, basename)
print(" ", basename, file=self.stdout)
filetracker.put_file('/sandboxes/' + basename, local_file)
if arg not in cached_args:
os.unlink(local_file)
try:
os.rmdir(download_dir)
except OSError:
print(
"--- Done, but couldn't remove the "
"downloads directory.",
file=self.stdout,
)
else:
print("--- Done.", file=self.stdout)
def __init__(self, contest, round=None, only_final=True):
self.contest = contest
self.round = round
self.only_final = only_final
self.filetracker = get_client()
def __init__(self, contest, round=None, only_final=True):
self.contest = contest
self.round = round
self.only_final = only_final
self.filetracker = get_client()
def on_worker_init(self):
super(OioioiLoader, self).on_worker_init()
# This initializes sioworker's filetracker client as well.
get_client()
def delete_executable(env, **kwargs):
if "compiled_file" in env:
get_client().delete_file(env["compiled_file"])
return env
def on_worker_init(self):
super(OioioiLoader, self).on_worker_init()
# This initializes sioworker's filetracker client as well.
get_client()
def upload_source(env, **kwargs):
fc = get_client()
fc.put_file(env['remote_source_file'], env['local_source_file'])
return env
except urllib2.HTTPError, e:
if e.code != 404:
raise
if not args:
args = manifest
print >> self.stdout, "--- Preparing ..."
urls = []
for arg in args:
if arg not in manifest:
raise CommandError("Sandbox '%s' not available (not in "
"Manifest)" % (arg, ))
urls.append(urlparse.urljoin(manifest_url, arg + '.tar.gz'))
filetracker = get_client()
dir = options['download_dir']
if not os.path.exists(dir):
os.makedirs(dir)
try:
execute([options['wget'], '--version'])
except ExecuteError:
raise CommandError("Wget not working. Please specify a working "
"Wget binary using --wget option.")
print >> self.stdout, "--- Downloading sandboxes ..."
execute([options['wget'], '-N', '-i', '-'],
stdin='\n'.join(urls),
capture_output=False,
def __init__(self, prefix="/", client=None):
if client is None:
client = get_client()
assert prefix.startswith("/"), "FiletrackerStorage.__init__ prefix must start with /"
self.client = client
self.prefix = prefix
