def compress(self, in_ext=None, sol_ext=None):
"""Compress all test cases in this dataset in a single zip file.
The basename of the corresponding subtask subdirectory is prepended
to each file.
"""
in_ext = in_ext or self._in_ext
sol_ext = sol_ext or self._sol_ext
dst_file = FilePath(self._directory, 'data.zip')
if dst_file.exists() and dst_file.mtime() >= self.mtime():
return True
tmpdir = Directory.tmpdir()
try:
copied = 0
for subtask in self._subtasks:
copied += subtask.copy_to(tmpdir)
if not copied:
# ui.show_message("Warning", "no files in dataset", ui.WARNING)
return True
cmd = 'cd %s && zip data.zip *%s *%s' % (tmpdir, in_ext, sol_ext)
st = subprocess.call(cmd, stdout=subprocess.DEVNULL, shell=True)
FilePath(tmpdir, 'data.zip').copy(dst_file)
finally:
tmpdir.rmtree()
return st == 0
def compress(self, in_ext=None, sol_ext=None, random_sort=False):
"""Compress all test cases in this dataset in a single zip file.
The basename of the corresponding subtask subdirectory is prepended
to each file.
"""
in_ext = in_ext or self._in_ext
sol_ext = sol_ext or self._sol_ext
dst_file = FilePath(self._directory, 'data.zip')
if dst_file.exists() and dst_file.mtime() >= self.mtime():
return True
tmpdir = Directory.tmpdir()
try:
copied = 0
for subtask in self._subtasks:
copied += subtask.copy_to(tmpdir, random_sort=random_sort)
if not copied:
# ui.show_message("Warning", "no files in dataset", ui.WARNING)
return True
cmd = 'cd %s && zip data.zip *%s *%s' % (tmpdir, in_ext, sol_ext)
st = subprocess.call(cmd, stdout=subprocess.DEVNULL, shell=True)
FilePath(tmpdir, 'data.zip').copy(dst_file)
finally:
tmpdir.rmtree()
return st == 0
def copy(self, src, dst):
fp = FilePath(self._task_directory, src)
if not fp.exists():
return (False, 'No such file')
try:
fp.copy(dst)
(st, msg) = (True, 'OK')
return (st, msg)
except Exception: # pylint: disable=broad-except
return (False, 'Error when copying file')
def copy(self, src, dst):
fp = FilePath(self._task_directory, src)
if not fp.exists():
return (False, 'No such file')
try:
fp.copy(dst)
(st, msg) = (True, 'OK')
return (st, msg)
except Exception: # pylint: disable=broad-except
return (False, 'Error when copying file')
def build_validator(self, source):
fp = FilePath(self._directory, source)
if not fp.exists():
return (None, 'File does not exists.')
if fp.ext == '.cpp':
binary = fp.chext('.bin')
if binary.mtime() < fp.mtime() and not self._cpp_compiler(fp, binary):
return (None, 'Failed to build validator.')
return (Runnable(binary), 'OK')
if fp.ext in ['.py', '.py3']:
return (Runnable('python3', [str(source)]), 'OK')
if fp.ext == '.py2':
return (Runnable('python2', [str(source)]), 'OK')
return (None, 'Not supported source file.')
def copy_to(self, directory):
new_dir = directory.mkdir(str(self))
(st, _) = self.compress_dataset()
if st:
dataset = FilePath(self._directory.chdir('dataset'), 'data.zip')
dataset_dst = FilePath(new_dir, 'data.zip')
if dataset.exists():
dataset.copy(dataset_dst)
(st, _) = self.build_statement()
if st:
statement = FilePath(new_dir, 'statement.pdf')
FilePath(self._directory.chdir('statement'),
'statement.pdf').copy(statement)
def package(self):
"""Compress statement and dataset of all tasks in a single file"""
tmpdir = Directory.tmpdir()
try:
for task in self._tasks:
task.copy_to(tmpdir)
self.build_problemset_twoside()
self.build_problemset_oneside()
oneside = FilePath(self._directory, 'oneside.pdf')
if oneside.exists():
oneside.copy(FilePath(tmpdir, 'oneside.pdf'))
twoside = FilePath(self._directory, 'twoside.pdf')
if twoside.exists():
twoside.copy(FilePath(tmpdir, 'twoside.pdf'))
cmd = 'cd %s && zip -r contest.zip .' % tmpdir
st = subprocess.call(cmd, stdout=subprocess.DEVNULL, shell=True)
contest = FilePath(self._directory, '%s.zip' % self.name)
FilePath(tmpdir, 'contest.zip').copy(contest)
finally:
tmpdir.rmtree()
return st == 0
def build_validator(self, source):
fp = FilePath(self._directory, source)
if not fp.exists():
return (None, 'File does not exists.')
if fp.ext == '.cpp':
binary = fp.chext('.bin')
if binary.mtime() < fp.mtime() and not self._cpp_compiler(
fp, binary):
return (None, 'Failed to build validator.')
return (Runnable(binary), 'OK')
if fp.ext in ['.py', '.py3']:
return (Runnable('python3', [str(source)]), 'OK')
if fp.ext == '.py2':
return (Runnable('python2', [str(source)]), 'OK')
return (None, 'Not supported source file.')
def merge_pdfs(self, filename):
"""Merges statements and title page in a single file """
if not shutil.which('gs'):
return (False, 'Cannot find gs')
pdfs = ' '.join('"%s"' % t.statement.pdf for t in self._tasks
if t.statement.pdf)
titlepage = FilePath(self._directory, 'titlepage.pdf')
if titlepage.exists():
pdfs = '"%s" %s' % (titlepage, pdfs)
cmd = ('gs -dBATCH -dNOPAUSE -q -sDEVICE=pdfwrite'
' -dPDFSETTINGS=/prepress -sOutputFile=%s %s') % (FilePath(
self._directory, filename), pdfs)
complete = subprocess.run(cmd,
shell=True,
timeout=20,
stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
st = complete.returncode == 0
return (st, 'OK' if st else 'FAILED')
class DatasetPlan:
"""Functionality to read and run a plan for generating dataset."""
def __init__(self, directory, task_directory, dataset_directory, filename='testplan.txt'):
self._directory = directory
self._testplan_path = FilePath(directory, filename)
if not self._testplan_path.exists():
ui.fatal_error('No such file plan for creating dataset: "%s"' % self._testplan_path)
self._task_directory = task_directory
self._dataset_directory = dataset_directory
self._cpp_compiler = CppCompiler()
self._java_compiler = JavaCompiler()
def test_filepath(self, stn, group, i):
st_dir = FilePath(self._dataset_directory, 'st%d' % stn).get_or_create_dir()
return FilePath(st_dir, '%s-%d.in' % (group, i))
def validate_input(self):
(_, cmds) = self.parse_file()
for (st, subtask) in sorted(cmds.items()):
self.validate_subtask(st, subtask)
@ui.workgroup('Subtask {1}')
def validate_subtask(self, stn, subtask):
validator = None
if subtask['validator']:
(validator, msg) = self.build_validator(subtask['validator'])
if validator is None:
ui.show_message('Warning', 'Failed to build validator: %s' % msg, ui.WARNING)
else:
ui.show_message('Info', 'No validator specified', ui.INFO)
if validator:
for (group, tests) in sorted(subtask['groups'].items()):
for (i, _) in enumerate(tests, 1):
test_file = self.test_filepath(stn, group, i)
self.validate_test_input(test_file, validator)
@ui.work('Validating', '{1}')
def validate_test_input(self, test_file, validator):
if not test_file.exists():
return False, 'Test file does not exist'
(st, _time, msg) = validator.run(test_file, None)
return st, msg
def build_validator(self, source):
fp = FilePath(self._directory, source)
if not fp.exists():
return (None, 'File does not exists.')
if fp.ext == '.cpp':
binary = fp.chext('.bin')
if binary.mtime() < fp.mtime() and not self._cpp_compiler(fp, binary):
return (None, 'Failed to build validator.')
return (Runnable(binary), 'OK')
if fp.ext in ['.py', '.py3']:
return (Runnable('python3', [str(source)]), 'OK')
if fp.ext == '.py2':
return (Runnable('python2', [str(source)]), 'OK')
return (None, 'Not supported source file.')
def run(self):
(subtasks, cmds) = self.parse_file()
for stn in range(1, subtasks + 1):
dire = FilePath(self._dataset_directory, 'st%d' % stn).get_or_create_dir()
dire.clear()
if not cmds:
ui.show_message("Warning", 'no commands were executed for the plan.', ui.WARNING)
for (stn, subtask) in sorted(cmds.items()):
self.run_subtask(stn, subtask)
@ui.workgroup('Subtask {1}')
def run_subtask(self, stn, subtask):
groups = subtask['groups']
for (group, tests) in sorted(groups.items()):
for (i, test) in enumerate(tests, 1):
cmd = test['cmd']
test_file = self.test_filepath(stn, group, i)
if cmd == 'copy':
self.copy(test['file'], test_file)
elif cmd == 'echo':
self.echo(test['args'], test_file)
else:
test['args'].insert(0, '%s-%s-%s' % (stn, group, i))
source = FilePath(self._directory, test['source'])
if cmd == 'cpp':
self.run_cpp_generator(source, test['args'], test_file)
elif cmd in ['py', 'py2', 'py3']:
self.run_py_generator(source, test['args'], test_file, cmd)
elif cmd == 'java':
self.run_java_generator(source, test['args'], test_file)
elif cmd == 'run':
bin_path = FilePath(self._directory, test['bin'])
self.run_bin_generator(bin_path, test['args'], test_file)
else:
ui.fatal_error('unexpected command when running plan: %s ' % cmd)
@ui.work('Copy', '{1}')
def copy(self, src, dst):
fp = FilePath(self._task_directory, src)
if not fp.exists():
return (False, 'No such file')
try:
fp.copy(dst)
(st, msg) = (True, 'OK')
return (st, msg)
except Exception: # pylint: disable=broad-except
return (False, 'Error when copying file')
@ui.work('Echo', '{1}')
def echo(self, args, dst):
with dst.open('w') as test_file:
test_file.write(' '.join(args) + '\n')
(st, msg) = (True, 'Ok')
return (st, msg)
@ui.work('Gen', '{1}')
def run_cpp_generator(self, source, args, dst):
if not source.exists():
return (False, 'No such file')
binary = source.chext('.bin')
if binary.mtime() < source.mtime():
st = self._cpp_compiler(source, binary)
if not st:
return (st, 'Failed to build generator')
(st, _time, msg) = Runnable(binary).run(None, dst, args)
return (st, msg)
@ui.work('Gen', '{1}')
def run_py_generator(self, source, args, dst, cmd):
if not source.exists():
return (False, 'No such file')
python = 'python2' if cmd == 'py2' else 'python3'
(st, _time, msg) = Runnable(python, [str(source)]).run(None, dst, args)
return (st, msg)
@ui.work('Gen', '{1}')
def run_java_generator(self, source, args, dst):
if not source.exists():
return (False, 'No such file')
bytecode = source.chext('.class')
if bytecode.mtime() < source.mtime():
st = self._java_compiler(source)
if not st:
return (st, 'Failed to build generator')
classname = bytecode.rootname()
classpath = str(bytecode.directory().path())
(st, _time, msg) = Runnable('java', ['-cp', classpath, classname]).run(None, dst, args)
return (st, msg)
@ui.work('Gen', '{1}')
def run_bin_generator(self, bin_path, args, dst):
if not bin_path.exists():
return (False, 'No such file')
if not Runnable.is_callable(bin_path):
return (False, 'Cannot run file, it may not have correct permissions')
(st, _time, msg) = Runnable(bin_path).run(None, dst, args)
return (st, msg)
def parse_file(self): # pylint: disable=too-many-locals,too-many-branches
"""
Args:
path (FilePath)
"""
cmds = {}
st = 0
for (lineno, line) in enumerate(self._testplan_path.open('r').readlines(), 1):
line = line.strip()
subtask_header = re.compile(r'\s*\[\s*Subtask\s*(\d+)\s*(?:-\s*([^\]\s]+))?\s*\]\s*')
cmd_line = re.compile(r'\s*([^;\s]+)\s*;\s*(\S+)(:?\s+(.*))?')
comment = re.compile(r'\s*#.*')
if not line:
continue
if not comment.fullmatch(line):
header_match = subtask_header.fullmatch(line)
cmd_match = cmd_line.fullmatch(line)
if header_match:
found_st = int(header_match.group(1))
validator = header_match.group(2)
if st + 1 != found_st:
ui.fatal_error('line %d: found subtask %d, but subtask %d was expected' %
(lineno, found_st, st + 1))
st += 1
cmds[st] = {'validator': validator, 'groups': {}}
elif cmd_match:
if st == 0:
ui.fatal_error(
'line %d: found command before declaring a subtask.' % lineno)
group = cmd_match.group(1)
cmd = cmd_match.group(2)
args = (cmd_match.group(3) or '').split()
if group not in cmds[st]['groups']:
cmds[st]['groups'][group] = []
if cmd == 'copy':
if len(args) > 2:
ui.fatal_error(
'line %d: command copy expects exactly one argument.' % lineno)
cmds[st]['groups'][group].append({
'cmd': 'copy',
'file': args[0],
})
elif cmd == 'echo':
cmds[st]['groups'][group].append({'cmd': 'echo', 'args': args})
else:
f = FilePath(self._directory, cmd)
if f.ext in ['.cpp', '.java', '.py', '.py2', '.py3']:
cmds[st]['groups'][group].append({
'cmd': f.ext[1:],
'source': cmd,
'args': args
})
else:
cmds[st]['groups'][group].append({
'cmd': 'run',
'bin': cmd,
'args': args
})
else:
ui.fatal_error('line %d: error while parsing line `%s`\n' % (lineno, line))
return (st, cmds)
class DatasetPlan:
"""Functionality to read and run a plan for generating dataset."""
def __init__(self,
directory,
task_directory,
dataset_directory,
filename='testplan.txt'):
self._directory = directory
self._testplan_path = FilePath(directory, filename)
if not self._testplan_path.exists():
ui.fatal_error('No such file plan for creating dataset: "%s"' %
self._testplan_path)
self._task_directory = task_directory
self._dataset_directory = dataset_directory
self._cpp_compiler = CppCompiler()
self._java_compiler = JavaCompiler()
def test_filepath(self, stn, group, i):
st_dir = FilePath(self._dataset_directory,
'st%d' % stn).get_or_create_dir()
return FilePath(st_dir, '%s-%d.in' % (group, i))
def validate_input(self):
(_, cmds) = self.parse_file()
for (st, subtask) in sorted(cmds.items()):
self.validate_subtask(st, subtask)
@ui.workgroup('Subtask {1}')
def validate_subtask(self, stn, subtask):
validator = None
if subtask['validator']:
(validator, msg) = self.build_validator(subtask['validator'])
if validator is None:
ui.show_message('Warning',
'Failed to build validator: %s' % msg,
ui.WARNING)
else:
ui.show_message('Info', 'No validator specified', ui.INFO)
if validator:
for (group, tests) in sorted(subtask['groups'].items()):
for (i, _) in enumerate(tests, 1):
test_file = self.test_filepath(stn, group, i)
self.validate_test_input(test_file, validator)
@ui.work('Validating', '{1}')
def validate_test_input(self, test_file, validator):
if not test_file.exists():
return False, 'Test file does not exist'
(st, _time, msg) = validator.run(test_file, None)
return st, msg
def build_validator(self, source):
fp = FilePath(self._directory, source)
if not fp.exists():
return (None, 'File does not exists.')
if fp.ext == '.cpp':
binary = fp.chext('.bin')
if binary.mtime() < fp.mtime() and not self._cpp_compiler(
fp, binary):
return (None, 'Failed to build validator.')
return (Runnable(binary), 'OK')
if fp.ext in ['.py', '.py3']:
return (Runnable('python3', [str(source)]), 'OK')
if fp.ext == '.py2':
return (Runnable('python2', [str(source)]), 'OK')
return (None, 'Not supported source file.')
def run(self):
(subtasks, cmds) = self.parse_file()
for stn in range(1, subtasks + 1):
dire = FilePath(self._dataset_directory,
'st%d' % stn).get_or_create_dir()
dire.clear()
if not cmds:
ui.show_message("Warning",
'no commands were executed for the plan.',
ui.WARNING)
for (stn, subtask) in sorted(cmds.items()):
self.run_subtask(stn, subtask)
@ui.workgroup('Subtask {1}')
def run_subtask(self, stn, subtask):
groups = subtask['groups']
for (group, tests) in sorted(groups.items()):
for (i, test) in enumerate(tests, 1):
cmd = test['cmd']
test_file = self.test_filepath(stn, group, i)
if cmd == 'copy':
self.copy(test['file'], test_file)
elif cmd == 'echo':
self.echo(test['args'], test_file)
else:
test['args'].insert(0, '%s-%s-%s' % (stn, group, i))
source = FilePath(self._directory, test['source'])
if cmd == 'cpp':
self.run_cpp_generator(source, test['args'], test_file)
elif cmd in ['py', 'py2', 'py3']:
self.run_py_generator(source, test['args'], test_file,
cmd)
elif cmd == 'java':
self.run_java_generator(source, test['args'],
test_file)
elif cmd == 'run':
bin_path = FilePath(self._directory, test['bin'])
self.run_bin_generator(bin_path, test['args'],
test_file)
else:
ui.fatal_error(
'unexpected command when running plan: %s ' % cmd)
@ui.work('Copy', '{1}')
def copy(self, src, dst):
fp = FilePath(self._task_directory, src)
if not fp.exists():
return (False, 'No such file')
try:
fp.copy(dst)
(st, msg) = (True, 'OK')
return (st, msg)
except Exception: # pylint: disable=broad-except
return (False, 'Error when copying file')
@ui.work('Echo', '{1}')
def echo(self, args, dst):
with dst.open('w') as test_file:
test_file.write(' '.join(args) + '\n')
(st, msg) = (True, 'Ok')
return (st, msg)
@ui.work('Gen', '{1}')
def run_cpp_generator(self, source, args, dst):
if not source.exists():
return (False, 'No such file')
binary = source.chext('.bin')
if binary.mtime() < source.mtime():
st = self._cpp_compiler(source, binary)
if not st:
return (st, 'Failed to build generator')
(st, _time, msg) = Runnable(binary).run(None, dst, args)
return (st, msg)
@ui.work('Gen', '{1}')
def run_py_generator(self, source, args, dst, cmd):
if not source.exists():
return (False, 'No such file')
python = 'python2' if cmd == 'py2' else 'python3'
(st, _time, msg) = Runnable(python, [str(source)]).run(None, dst, args)
return (st, msg)
@ui.work('Gen', '{1}')
def run_java_generator(self, source, args, dst):
if not source.exists():
return (False, 'No such file')
bytecode = source.chext('.class')
if bytecode.mtime() < source.mtime():
st = self._java_compiler(source)
if not st:
return (st, 'Failed to build generator')
classname = bytecode.rootname()
classpath = str(bytecode.directory().path())
(st, _time,
msg) = Runnable('java',
['-cp', classpath, classname]).run(None, dst, args)
return (st, msg)
@ui.work('Gen', '{1}')
def run_bin_generator(self, bin_path, args, dst):
if not bin_path.exists():
return (False, 'No such file')
if not Runnable.is_callable(bin_path):
return (False,
'Cannot run file, it may not have correct permissions')
(st, _time, msg) = Runnable(bin_path).run(None, dst, args)
return (st, msg)
def parse_file(self): # pylint: disable=too-many-locals,too-many-branches
"""
Args:
path (FilePath)
"""
cmds = {}
st = 0
for (lineno,
line) in enumerate(self._testplan_path.open('r').readlines(), 1):
line = line.strip()
subtask_header = re.compile(
r'\s*\[\s*Subtask\s*(\d+)\s*(?:-\s*([^\]\s]+))?\s*\]\s*')
cmd_line = re.compile(r'\s*([^;\s]+)\s*;\s*(\S+)(:?\s+(.*))?')
comment = re.compile(r'\s*#.*')
if not line:
continue
if not comment.fullmatch(line):
header_match = subtask_header.fullmatch(line)
cmd_match = cmd_line.fullmatch(line)
if header_match:
found_st = int(header_match.group(1))
validator = header_match.group(2)
if st + 1 != found_st:
ui.fatal_error(
'line %d: found subtask %d, but subtask %d was expected'
% (lineno, found_st, st + 1))
st += 1
cmds[st] = {'validator': validator, 'groups': {}}
elif cmd_match:
if st == 0:
ui.fatal_error(
'line %d: found command before declaring a subtask.'
% lineno)
group = cmd_match.group(1)
cmd = cmd_match.group(2)
args = _parse_args(cmd_match.group(3) or '')
if group not in cmds[st]['groups']:
cmds[st]['groups'][group] = []
if cmd == 'copy':
if len(args) > 2:
ui.fatal_error(
'line %d: command copy expects exactly one argument.'
% lineno)
cmds[st]['groups'][group].append({
'cmd': 'copy',
'file': args[0],
})
elif cmd == 'echo':
cmds[st]['groups'][group].append({
'cmd': 'echo',
'args': args
})
else:
f = FilePath(self._directory, cmd)
if f.ext in ['.cpp', '.java', '.py', '.py2', '.py3']:
cmds[st]['groups'][group].append({
'cmd': f.ext[1:],
'source': cmd,
'args': args
})
else:
cmds[st]['groups'][group].append({
'cmd': 'run',
'bin': cmd,
'args': args
})
else:
ui.fatal_error('line %d: error while parsing line `%s`\n' %
(lineno, line))
return (st, cmds)
