def create_pypy(self, arg_rest):
executor = BinExecutor(self.get_command(arg_rest))
pypy = PyPy(executor)
pypy.case = self.case
pypy.limit_monitor.set_limits(self.case)
pypy.end_monitor.deactivate()
pypy.start_monitor.format = 'Running: {}'.format(self.case)
pypy.progress = self.progress
pypy.executor.output = OutputMode.file_write(self.case.fs.job_output)
pypy.full_output = pypy.executor.output.filename
return pypy
def create_pypy(self, arg_rest):
executor = BinExecutor(self.get_command(arg_rest))
pypy = PyPy(executor)
pypy.case = self.case
pypy.limit_monitor.set_limits(self.case)
pypy.end_monitor.deactivate()
pypy.start_monitor.format = 'Running: {}'.format(self.case)
pypy.progress = self.progress
pypy.executor.output = OutputMode.file_write(self.case.fs.job_output)
pypy.full_output = pypy.executor.output.filename
return pypy
def create_pypy(self, arg_rest):
executor = BinExecutor(self.get_command(arg_rest))
pypy = PyPy(executor)
pypy.case = self.case
pypy.limit_monitor.set_limits(self.case)
pypy.info_monitor.end_fmt = ''
pypy.info_monitor.start_fmt = 'Running: {}'.format(self.case)
pypy.progress = self.progress
# pypy.executor.output = OutputMode.file_write(Paths.temp_file('runtest-{datetime}.log'))
pypy.executor.output = OutputMode.file_write(self.case.fs.job_output)
pypy.full_output = pypy.executor.output.filename
return pypy
def _run(self):
"""
Run method for this module
"""
# prepare executor
progress = not self.arg_options.batch
executor = BinExecutor(self.arg_options.rest)
pypy = PyPy(executor, progress=progress)
n_lines = 0 if self.arg_options.batch else 10
# set up streams
log_file = Paths.temp_file('exec-limit-{date}-{time}-{rnd}.log')
pypy.executor.output = OutputMode.variable_output()
pypy.full_output = log_file
# set limits
pypy.limit_monitor.time_limit = self.arg_options.time_limit
pypy.limit_monitor.memory_limit = self.arg_options.memory_limit
# save output to file
pypy.output_monitor.log_file = log_file
# start process
pypy.start()
pypy.join()
return pypy
def run_local_mode_one(self, proc):
"""
Method runs single job with specified number of CPU
:param proc:
"""
if int(proc) == 0:
command = self.arg_options.rest[1:]
else:
command = [self.arg_options.rest[0], '-np', proc
] + self.arg_options.rest[1:]
n_lines = 0 if self.arg_options.batch else 10
pypy = PyPy(BinExecutor(command))
# set limits
pypy.limit_monitor.time_limit = self.time_limit
pypy.limit_monitor.memory_limit = self.memory_limit
# catch output to variable
# in batched mode we will keep the files
# otherwise we will keep logs only on error
log_file = Paths.temp_file('exec-parallel-{date}-{time}-{rnd}.log')
pypy.executor.output = OutputMode.variable_output()
pypy.full_output = log_file
# save output to file
pypy.output_monitor.log_file = log_file
# start and wait for exit
pypy.start()
pypy.join()
return pypy
def create_comparisons(self):
comparisons = ComparisonMultiThread(self.case.fs.ndiff_log)
comparisons.thread_name_property = True
for check_rule in self.case.check_rules:
method = str(check_rule.keys()[0])
module = getattr(file_comparison, 'Compare{}'.format(method.capitalize()), None)
comp_data = check_rule[method]
if not module:
Printer.all.err('Warning! No module for check_rule method "{}"', method)
continue
pairs = self._get_ref_output_files(comp_data)
if pairs:
for pair in pairs:
command = module.get_command(*pair, **comp_data)
pm = PyPy(BinExecutor(command), progress=True)
# if we fail, set error to 13
pm.custom_error = 13
pm.start_monitor.deactivate()
pm.end_monitor.deactivate()
pm.progress_monitor.deactivate()
pm.limit_monitor.deactivate() # TODO: maybe some time limit would be useful
pm.output_monitor.policy = pm.output_monitor.POLICY_ERROR_ONLY
pm.error_monitor.message = 'Comparison using method {} failed!'.format(method)
pm.error_monitor.indent = 1
# catch output
pm.executor.output = OutputMode.variable_output()
pm.full_output = self.case.fs.ndiff_log
path = Paths.path_end_until(pair[0], REF_OUTPUT_DIR)
test_name = Paths.basename(Paths.dirname(Paths.dirname(self.case.fs.ref_output)))
size = Paths.filesize(pair[0], True)
pm.name = '{}: {} ({})'.format(test_name, path, size)
comparisons.add(pm)
return comparisons
def run_local_mode_one(self, proc):
"""
Method runs single job with specified number of CPU
:param proc:
"""
if int(proc) == 0:
command = self.rest[1:]
else:
command = [self.rest[0], '-np', proc] + self.rest[1:]
n_lines = 0 if self.arg_options.batch else 10
pypy = PyPy(BinExecutor(command))
# set limits
pypy.limit_monitor.time_limit = self.time_limit
pypy.limit_monitor.memory_limit = self.memory_limit
# catch output to variable
# in batched mode we will keep the files
# otherwise we will keep logs only on error
log_file = Paths.temp_file('exec-parallel-{date}-{time}-{rnd}.log')
pypy.executor.output = OutputMode.variable_output()
pypy.full_output = log_file
# save output to file
pypy.output_monitor.log_file = log_file
# start and wait for exit
pypy.start()
pypy.join()
return pypy
def _run(self):
"""
Run method for this module
"""
# prepare executor
progress = not self.arg_options.batch
executor = BinExecutor(self.rest)
pypy = PyPy(executor, progress=progress)
n_lines = 0 if self.arg_options.batch else 10
# set up streams
log_file = Paths.temp_file('exec-limit-{date}-{time}-{rnd}.log')
pypy.executor.output = OutputMode.variable_output()
pypy.full_output = log_file
# set limits
pypy.limit_monitor.time_limit = self.arg_options.time_limit
pypy.limit_monitor.memory_limit = self.arg_options.memory_limit
# save output to file
pypy.output_monitor.log_file = log_file
# start process
pypy.start()
pypy.join()
return pypy
def do_work(parser, args=None):
"""
:type args: list
:type parser: utils.argparser.ArgParser
"""
# parse arguments
options, others, rest = parser.parse(args)
# check commands
if not rest:
parser.exit_usage('No command specified!', exit_code=1)
# check limits (at least one limit must be set)
if (options.time_limit, options.memory_limit) == (None, None):
parser.exit_usage('No limits specified!', exit_code=2)
# prepare executor
executor = BinExecutor(rest)
pypy = PyPy(executor, progress=not options.batch)
# set limits
pypy.error_monitor.message = None
pypy.limit_monitor.time_limit = options.time_limit
pypy.limit_monitor.memory_limit = options.memory_limit
# turn on output
if options.batch:
pypy.info_monitor.stdout_stderr = None
else:
pypy.info_monitor.stdout_stderr = Paths.temp_file('exec-limit.log')
# start process
Printer.separator()
pypy.start()
pypy.join()
return pypy.returncode
def create_comparisons(self):
comparisons = ComparisonMultiThread(self.case.fs.ndiff_log)
comparisons.thread_name_property = True
for check_rule in self.case.check_rules:
method = str(check_rule.keys()[0])
module = self.get_module(method)
comp_data = check_rule[method]
if not module:
Printer.all.err(
'Warning! No module for check_rule method "{}"', method)
continue
pairs = self._get_ref_output_files(comp_data)
if pairs:
for pair in pairs:
# load module and determine whether we are dealing with
# exec comparison or inplace comparison
if issubclass(module.__class__, modules.ExecComparison):
command = module.get_command(*pair, **comp_data)
pm = PyPy(BinExecutor(command), progress=True)
pm.executor.output = OutputMode.variable_output()
else:
module = self.get_module(method)
module.prepare(*pair, **comp_data)
pm = PyPy(module, progress=True)
pm.executor.output = OutputMode.dummy_output()
pm.error_monitor.deactivate()
# if we fail, set error to 13
pm.custom_error = 13
pm.start_monitor.deactivate()
pm.end_monitor.deactivate()
pm.progress_monitor.deactivate()
pm.limit_monitor.deactivate(
) # TODO: maybe some time limit would be useful
pm.output_monitor.policy = pm.output_monitor.POLICY_ERROR_ONLY
pm.error_monitor.message = 'Comparison using method {} failed!'.format(
method)
pm.error_monitor.indent = 1
pm.full_output = self.case.fs.ndiff_log
path = Paths.path_end_until(pair[0], REF_OUTPUT_DIR)
test_name = Paths.basename(
Paths.dirname(Paths.dirname(self.case.fs.ref_output)))
size = Paths.filesize(pair[0], True)
pm.name = '{}: {} ({})'.format(test_name, path, size)
comparisons.add(pm)
return comparisons
def run_local_mode_one(proc, time_limit, memory_limit):
if proc == 0:
command = arg_rest[1:]
else:
command = [arg_rest[0], '-np', proc] + arg_rest[1:]
n_lines = 0 if arg_options.batch else 10
pypy = PyPy(BinExecutor(command))
# set limits
pypy.limit_monitor.time_limit = time_limit
pypy.limit_monitor.memory_limit = memory_limit
pypy.progress = not arg_options.batch
pypy.info_monitor.deactivate()
pypy.error_monitor.deactivate()
# catch output to variable
# in batch mode we will keep the files
# otherwise we will keep logs only on error
log_file = Paths.temp_file('exec-parallel-{date}-{time}-{rnd}.log')
pypy.executor.output = OutputMode.variable_output()
pypy.full_output = log_file
# start and wait for exit
pypy.start()
pypy.join()
# add result to global json result
GlobalResult.add(pypy)
# in batch mode or on error
if not pypy.with_success() or arg_options.batch:
content = pypy.executor.output.read()
IO.write(log_file, content)
Printer.close()
Printer.out(format_n_lines(content, indent=' ', n_lines=-n_lines))
Printer.open()
return pypy