def remakefile_info(remakefile, short, long, display):
if display == 'print_status':
remake = load_remake(remakefile).finalize()
if short:
remake.short_status(mode='print')
else:
remake.tasks.status(long, long)
elif display == 'task_dag':
remake = load_remake(remakefile).finalize()
remake.display_task_dag()
else:
raise Exception(f'Unrecognized display: {display}')
def worker(proc_id, remakefile_name, task_queue, task_complete_queue):
remake = load_remake(remakefile_name)
task_ctrl = remake.task_ctrl
logger = getLogger(__name__ + '.worker')
add_file_logging(f'.remake/worker.{proc_id}.log', 'DEBUG')
logger.debug('starting')
task = None
while True:
try:
item = task_queue.get()
if item is None:
break
task_type, task_key, force = item
logger.debug(f'{task_type}: {task_key} (force={force})')
if task_type == 'rescan':
task = task_ctrl.gen_rescan_task(task_key)
else:
task = task_ctrl.task_from_path_hash_key[task_key]
logger.debug(f'worker {current_process().name} running {task}')
task.run(use_task_control=False)
logger.debug(f'worker {current_process().name} complete {task}')
task_complete_queue.put((task_key, True, None))
except Exception as e:
logger.error(e)
if task:
logger.error(str(task))
task_complete_queue.put((task_key, False, e))
item = task_queue.get()
if item is None:
break
logger.debug('stopping')
remove_file_logging(f'.remake/worker.{proc_id}.log')
def run_job(remakefile, remakefile_hash, task_type, task_key):
setup_stdout_logging('DEBUG', colour=False, detailed=True)
remakefile = Path(remakefile).absolute()
curr_remakefile_hash = sha1(remakefile.read_bytes()).hexdigest()
if remakefile_hash != curr_remakefile_hash:
raise Exception(f'config file {remakefile} has changed -- cannot run task.')
remake = load_remake(remakefile)
task_ctrl = remake.task_ctrl
assert not task_ctrl.finalized, f'task control {task_ctrl} already finalized'
# Note, task_ctrl is not finalized.
# This is because another task could be finishing, and writing its output's metadata
# when this is called, and finalize can be trying to read it at the same time.
# Can perhaps fix if instead Task is responsible for working out if rerun needed,
# and removing finalize here.
# But the task DAG needs to be build.
task_ctrl.build_task_DAG()
if task_type == 'task':
task = task_ctrl.task_from_path_hash_key[task_key]
elif task_type == 'rescan':
task = task_ctrl.gen_rescan_task(task_key)
force = False
# Task might not be required anymore -- find out.
requires_rerun = task_ctrl.task_requires_rerun(task, print_reasons=True)
if force or task.force or requires_rerun & task_ctrl.remake_on:
logger.info(f'Running task: {task}')
# Can't run this; not finalized.
# task_ctrl.run_requested([task])
task.run(use_task_control=False)
task.update_status('COMPLETED')
else:
print(f'Run task not required: {task}')
logger.info(f'Run task not required: {task}')
def test_slurm_executor_run_job_rescan(self):
remakefile = Path('ex1.py')
remake = load_remake(remakefile)
remake.finalize()
remakefile_hash = sha1(remakefile.read_bytes()).hexdigest()
rescan_task = remake.task_ctrl.rescan_tasks[0]
run_job(remakefile, remakefile_hash, 'rescan', rescan_task.filepath)
def rm_files(remakefile, force, filetype, produced_by_rule, used_by_rule,
produced_by_task, used_by_task):
remake = load_remake(remakefile)
filelist = remake.list_files(filetype, True, produced_by_rule,
used_by_rule, produced_by_task, used_by_task)
if not filelist:
logger.info('No files to delete')
return
if force:
r = 'yes'
else:
r = input(
bcolors.BOLD + bcolors.WARNING +
f'This will delete {len(filelist)} files, do you want to proceed? (yes/[no]): '
+ bcolors.ENDC)
if r != 'yes':
print('Not deleting files (yes not entered)')
return
for file, ftype, exists in filelist:
if ftype == 'input-only':
if force:
r = 'yes'
else:
r = input(
bcolors.BOLD + bcolors.FAIL +
f'Are you sure you want to delete input-only file: {file}? (yes/[no]): '
+ bcolors.ENDC)
if r != 'yes':
print('Not deleting files (yes not entered)')
continue
logger.info(f'Deleting file: {file}')
file.unlink()
def file_info(remakefile, filenames):
remake = load_remake(remakefile).finalize()
info = remake.file_info(filenames)
for path, (path_md, produced_by_task, used_by_tasks) in info.items():
if path.exists():
print(f'exists: {path}')
else:
print(f'does not exist: {path}')
if not path_md:
print(f'Path not found in {remake.name}')
print()
continue
if produced_by_task:
print('Produced by:')
print(' ' + str(produced_by_task))
if used_by_tasks:
print('Used by:')
for task in used_by_tasks:
print(' ' + str(task))
if path.exists():
metadata_has_changed = path_md.compare_path_with_previous()
if metadata_has_changed:
print('Path metadata has changed since last use')
else:
print('Path metadata unchanged')
print()
def ls_tasks(remakefile, long, tfilter, rule, requires_rerun, uses_file,
produces_file, ancestor_of, descendant_of):
remake = load_remake(remakefile).finalize()
if tfilter:
tfilter = dict([kv.split('=') for kv in tfilter.split(',')])
tasks = remake.list_tasks(tfilter, rule, requires_rerun, uses_file,
produces_file, ancestor_of, descendant_of)
tasks.status(long, long)
def test_bug2():
"""Absolute paths should end up with metadata under .remake"""
orig_cwd = os.getcwd()
os.chdir(bugs_remakefiles_dir)
remake = load_remake('absolute_paths.py')
task_md = remake.tasks[0].task_md
path_md = list(task_md.inputs_metadata_map.values())[0]
assert is_relative_to(path_md.metadata_path, Path('.remake'))
os.chdir(orig_cwd)
def test_slurm_executor_run_job_task1(self):
remakefile = Path('ex1.py')
remake = load_remake(remakefile)
remake.finalize()
remakefile_hash = sha1(remakefile.read_bytes()).hexdigest()
rescan_task = remake.task_ctrl.rescan_tasks[0]
run_job(remakefile, remakefile_hash, 'rescan', rescan_task.filepath)
task = list(remake.task_ctrl.sorted_tasks.keys())[0]
run_job(remakefile, remakefile_hash, 'task', task.path_hash_key())
def test_bug4():
"""remake run --one not working #29"""
orig_cwd = os.getcwd()
os.chdir(examples_dir)
sysrun('make clean')
ex1 = load_remake('ex1.py')
ex1.finalize()
ex1.run_one()
sysrun('make reset')
os.chdir(orig_cwd)
def test1_worker_run_no_jobs(self):
remakefile = Path('ex1.py')
remake = load_remake(remakefile)
remake.finalize()
task_queue = mock.MagicMock()
task_queue.get.side_effect = [None]
task_complete_queue = mock.MagicMock()
worker(1, remakefile, task_queue, task_complete_queue)
def ls_files(remakefile, long, filetype, exists, produced_by_rule,
used_by_rule, produced_by_task, used_by_task):
remake = load_remake(remakefile)
filelist = remake.list_files(filetype, exists, produced_by_rule,
used_by_rule, produced_by_task, used_by_task)
if long:
print(tabulate(filelist, headers=('path', 'filetype', 'exists')))
else:
for file, ftype, exists in filelist:
print(file)
def setUpClass(cls) -> None:
cls.orig_cwd = os.getcwd()
os.chdir(examples_dir)
sysrun('make clean')
cls.remake = load_remake('ex3.py')
cls.remake.finalize()
cls.monitor = RemakeMonitor(cls.remake)
# Using the real curses package causes problems for PyCharm and github CI.
# Mock out all the important parts.
cls.stdscr = mock.MagicMock()
cls.stdscr.getmaxyx.return_value = (100, 50)
# Generate dummy keypresses to feed into RemakeMonitorCurses.
commands = [
'r',
'f',
't',
':task 0',
':task 1',
':task 2',
':show tasks',
'w'
'j',
'k',
'g',
'G',
'F',
'R',
':q' # Note, end by quiting application.
]
clist = []
for command in commands:
clist_command = [-1] * 100 + [ord(c) for c in command]
if len(command) > 1:
clist_command += [13]
clist.extend(clist_command)
cls.stdscr.getch.side_effect = clist
# Create patches for all curses functions called.
curses_patch_fns = [
'init_pair',
'curs_set',
'color_pair',
'napms',
'is_term_resized',
'resizeterm',
]
cls.patchers = []
for fn in curses_patch_fns:
patcher = mock.patch(f'curses.{fn}')
setattr(cls, fn, patcher.start())
cls.patchers.append(patcher)
cls.is_term_resized.return_value = False
def rule_info(remakefile, long, rule_names):
remake = load_remake(remakefile).finalize()
rules = remake.list_rules()
for rule_name in rule_names:
found = False
for rule in rules:
if rule.__name__ == rule_name:
print(rule)
found = True
break
if not found:
logger.error(f'No rule {rule_name} in {remake.name} found')
def test3_worker_run_task(self):
remakefile = Path('ex1.py')
remake = load_remake(remakefile)
remake.finalize()
task = remake.tasks[0]
task_queue = mock.MagicMock()
task_queue.get.side_effect = [('task', task.path_hash_key(), False),
None]
task_complete_queue = mock.MagicMock()
worker(1, remakefile, task_queue, task_complete_queue)
def test_all_examples():
example_runner = load_remake('test_all_examples.py').finalize()
for task in example_runner.tasks.in_rule('RunAllRemakes').filter(
executor='singleproc'):
yield run_task, example_runner, task
for task in example_runner.tasks.in_rule('RunAllRemakes').filter(
executor='multiproc'):
yield run_task, example_runner, task
for task in example_runner.tasks.in_rule('TestCLI'):
yield run_task, example_runner, task
for task in example_runner.tasks.in_rule('TestCLI2'):
yield run_task, example_runner, task
for task in example_runner.tasks.in_rule('TestEx1'):
yield run_task, example_runner, task
def test4_worker_exception(self, mock_run):
remakefile = Path('ex1.py')
remake = load_remake(remakefile)
remake.finalize()
task = remake.tasks[0]
task_queue = mock.MagicMock()
task_queue.get.side_effect = [('rescan', task.path_hash_key(), False),
None]
task_complete_queue = mock.MagicMock()
mock_run.return_value = Exception('Boom!')
self.assertRaises(
Exception, worker(1, remakefile, task_queue, task_complete_queue))
def task_info(remakefile, long, task_path_hash_keys):
remake = load_remake(remakefile).finalize()
info = remake.task_info(task_path_hash_keys)
for task_path_hash_key, (task, task_md, status) in info.items():
print(str(task))
print(status)
print(task_md.task_requires_rerun())
if long:
print('Uses files:')
for key, path in task.inputs.items():
print(f' {key}: {path}')
print('Produces files:')
for key, path in task.outputs.items():
print(f' {key}: {path}')
def remake_run(remakefile, rescan_only, force, one, random, print_reasons,
executor, display):
if force and (one or random):
raise ValueError('--force cannot be used with --one or --random')
remake = load_remake(remakefile).finalize()
remake.configure(print_reasons, executor, display)
remake.short_status()
if rescan_only:
remake.task_ctrl.run_rescan_only()
elif one:
remake.run_one()
elif random:
remake.run_random()
else:
remake.run_all(force=force)
if display == 'task_dag':
# Give user time to see final task_dag state.
sleep(3)
remake.short_status()
def remake_run_tasks(remakefile, task_path_hash_keys, handle_dependencies,
force, print_reasons, executor, display, tfilter, rule,
requires_rerun, uses_file, produces_file, ancestor_of,
descendant_of):
remake = load_remake(remakefile).finalize()
remake.configure(print_reasons, executor, display)
remake.short_status()
if task_path_hash_keys and (tfilter or rule):
raise RemakeError(
'Can only use one of --tasks and (--filter or --rule)')
if task_path_hash_keys:
tasks = remake.find_tasks(task_path_hash_keys)
else:
if tfilter:
tfilter = dict([kv.split('=') for kv in tfilter.split(',')])
tasks = remake.list_tasks(tfilter, rule, requires_rerun, uses_file,
produces_file, ancestor_of, descendant_of)
remake.run_requested(tasks,
force=force,
handle_dependencies=handle_dependencies)
if display == 'task_dag':
# Give user time to see final task_dag state.
sleep(3)
remake.short_status()
def setUp(self) -> None:
self.orig_cwd = os.getcwd()
os.chdir(examples_dir)
sysrun('make clean')
self.remake = load_remake('ex3.py')
self.remake.finalize()
def ls_rules(remakefile, long, tfilter, uses_file, produces_file):
# TODO: implement all args.
remake = load_remake(remakefile)
rules = remake.list_rules()
for rule in rules:
print(f'{rule.__name__}')
def setUpClass(cls) -> None:
cls.orig_cwd = os.getcwd()
os.chdir(examples_dir)
sysrun('make clean')
cls.remake = load_remake('demo.py')
def input_loop(self, mode, command, keypresses, show, i_offset):
stdscr = self.stdscr
for i in range(self.num_input_loops):
# Input loop.
curses.napms(self.input_loop_timeout)
# Action in loop if resize is True:
# Not working!
if curses.is_term_resized(self.rows, self.cols):
self.rows, self.cols = self.stdscr.getmaxyx()
curses.resizeterm(self.rows, self.cols)
break
try:
c = self.getch()
# stdscr.addstr(rows - 1, cols - 10, str(c))
if c == -1:
continue
if c in (curses.KEY_ENTER, 10, 13):
command = ''.join(keypresses[1:]).split(' ')
keypresses = []
break
elif c == 127:
# Backspace
keypresses = keypresses[:-1]
break
else:
if chr(c) == ':':
mode = 'command'
if mode == 'command':
keypresses.append(chr(c))
else:
if chr(c) == 't':
show = 'tasks'
i_offset = 0
break
elif chr(c) == 'r':
show = 'rules'
i_offset = 0
break
elif chr(c) == 'f':
show = 'files'
i_offset = 0
break
elif chr(c) == 'j':
i_offset -= 1
break
elif chr(c) == 'k':
if i_offset <= -1:
i_offset += 1
break
elif chr(c) == 'g':
i_offset = 0
break
elif chr(c) == 'G':
i_offset = -10000
break
elif chr(c) == 'w':
self.wrap = not self.wrap
break
elif chr(c) == 'R':
self.remake = load_remake(self.remake.name)
self.remake.task_ctrl.build_task_DAG()
self.monitor = RemakeMonitor(self.remake)
self.remake_sha1sum = sha1sum(Path(self.remake.name + '.py'))
break
elif chr(c) == 'F':
self.remake = load_remake(self.remake.name)
self.remake.finalize()
self.monitor = RemakeMonitor(self.remake)
self.remake_sha1sum = sha1sum(Path(self.remake.name + '.py'))
break
stdscr.addstr(self.rows - 1, 0, ''.join(keypresses))
except curses.error:
pass
stdscr.refresh()
if command:
if command[0] == 'q':
raise Quit()
elif command[0] == 'show':
show = command[1]
i_offset = 0
elif command[0] == 'task':
show = 'task'
self.task_i = int(command[1])
i_offset = 0
return mode, command, keypresses, show, i_offset
def monitor(remakefile, timeout):
from curses import wrapper
remake = load_remake(remakefile)
remake.task_ctrl.build_task_DAG()
wrapper(remake_curses_monitor, remake, timeout)
