def remakefile_info(remakefile, short, long, display): if display == 'print_status': remake = load_remake(remakefile).finalize() if short: remake.short_status(mode='print') else: remake.tasks.status(long, long) elif display == 'task_dag': remake = load_remake(remakefile).finalize() remake.display_task_dag() else: raise Exception(f'Unrecognized display: {display}')
def worker(proc_id, remakefile_name, task_queue, task_complete_queue): remake = load_remake(remakefile_name) task_ctrl = remake.task_ctrl logger = getLogger(__name__ + '.worker') add_file_logging(f'.remake/worker.{proc_id}.log', 'DEBUG') logger.debug('starting') task = None while True: try: item = task_queue.get() if item is None: break task_type, task_key, force = item logger.debug(f'{task_type}: {task_key} (force={force})') if task_type == 'rescan': task = task_ctrl.gen_rescan_task(task_key) else: task = task_ctrl.task_from_path_hash_key[task_key] logger.debug(f'worker {current_process().name} running {task}') task.run(use_task_control=False) logger.debug(f'worker {current_process().name} complete {task}') task_complete_queue.put((task_key, True, None)) except Exception as e: logger.error(e) if task: logger.error(str(task)) task_complete_queue.put((task_key, False, e)) item = task_queue.get() if item is None: break logger.debug('stopping') remove_file_logging(f'.remake/worker.{proc_id}.log')
def run_job(remakefile, remakefile_hash, task_type, task_key): setup_stdout_logging('DEBUG', colour=False, detailed=True) remakefile = Path(remakefile).absolute() curr_remakefile_hash = sha1(remakefile.read_bytes()).hexdigest() if remakefile_hash != curr_remakefile_hash: raise Exception(f'config file {remakefile} has changed -- cannot run task.') remake = load_remake(remakefile) task_ctrl = remake.task_ctrl assert not task_ctrl.finalized, f'task control {task_ctrl} already finalized' # Note, task_ctrl is not finalized. # This is because another task could be finishing, and writing its output's metadata # when this is called, and finalize can be trying to read it at the same time. # Can perhaps fix if instead Task is responsible for working out if rerun needed, # and removing finalize here. # But the task DAG needs to be build. task_ctrl.build_task_DAG() if task_type == 'task': task = task_ctrl.task_from_path_hash_key[task_key] elif task_type == 'rescan': task = task_ctrl.gen_rescan_task(task_key) force = False # Task might not be required anymore -- find out. requires_rerun = task_ctrl.task_requires_rerun(task, print_reasons=True) if force or task.force or requires_rerun & task_ctrl.remake_on: logger.info(f'Running task: {task}') # Can't run this; not finalized. # task_ctrl.run_requested([task]) task.run(use_task_control=False) task.update_status('COMPLETED') else: print(f'Run task not required: {task}') logger.info(f'Run task not required: {task}')
def test_slurm_executor_run_job_rescan(self): remakefile = Path('ex1.py') remake = load_remake(remakefile) remake.finalize() remakefile_hash = sha1(remakefile.read_bytes()).hexdigest() rescan_task = remake.task_ctrl.rescan_tasks[0] run_job(remakefile, remakefile_hash, 'rescan', rescan_task.filepath)
def rm_files(remakefile, force, filetype, produced_by_rule, used_by_rule, produced_by_task, used_by_task): remake = load_remake(remakefile) filelist = remake.list_files(filetype, True, produced_by_rule, used_by_rule, produced_by_task, used_by_task) if not filelist: logger.info('No files to delete') return if force: r = 'yes' else: r = input( bcolors.BOLD + bcolors.WARNING + f'This will delete {len(filelist)} files, do you want to proceed? (yes/[no]): ' + bcolors.ENDC) if r != 'yes': print('Not deleting files (yes not entered)') return for file, ftype, exists in filelist: if ftype == 'input-only': if force: r = 'yes' else: r = input( bcolors.BOLD + bcolors.FAIL + f'Are you sure you want to delete input-only file: {file}? (yes/[no]): ' + bcolors.ENDC) if r != 'yes': print('Not deleting files (yes not entered)') continue logger.info(f'Deleting file: {file}') file.unlink()
def file_info(remakefile, filenames): remake = load_remake(remakefile).finalize() info = remake.file_info(filenames) for path, (path_md, produced_by_task, used_by_tasks) in info.items(): if path.exists(): print(f'exists: {path}') else: print(f'does not exist: {path}') if not path_md: print(f'Path not found in {remake.name}') print() continue if produced_by_task: print('Produced by:') print(' ' + str(produced_by_task)) if used_by_tasks: print('Used by:') for task in used_by_tasks: print(' ' + str(task)) if path.exists(): metadata_has_changed = path_md.compare_path_with_previous() if metadata_has_changed: print('Path metadata has changed since last use') else: print('Path metadata unchanged') print()
def ls_tasks(remakefile, long, tfilter, rule, requires_rerun, uses_file, produces_file, ancestor_of, descendant_of): remake = load_remake(remakefile).finalize() if tfilter: tfilter = dict([kv.split('=') for kv in tfilter.split(',')]) tasks = remake.list_tasks(tfilter, rule, requires_rerun, uses_file, produces_file, ancestor_of, descendant_of) tasks.status(long, long)
def test_bug2(): """Absolute paths should end up with metadata under .remake""" orig_cwd = os.getcwd() os.chdir(bugs_remakefiles_dir) remake = load_remake('absolute_paths.py') task_md = remake.tasks[0].task_md path_md = list(task_md.inputs_metadata_map.values())[0] assert is_relative_to(path_md.metadata_path, Path('.remake')) os.chdir(orig_cwd)
def test_slurm_executor_run_job_task1(self): remakefile = Path('ex1.py') remake = load_remake(remakefile) remake.finalize() remakefile_hash = sha1(remakefile.read_bytes()).hexdigest() rescan_task = remake.task_ctrl.rescan_tasks[0] run_job(remakefile, remakefile_hash, 'rescan', rescan_task.filepath) task = list(remake.task_ctrl.sorted_tasks.keys())[0] run_job(remakefile, remakefile_hash, 'task', task.path_hash_key())
def test_bug4(): """remake run --one not working #29""" orig_cwd = os.getcwd() os.chdir(examples_dir) sysrun('make clean') ex1 = load_remake('ex1.py') ex1.finalize() ex1.run_one() sysrun('make reset') os.chdir(orig_cwd)
def test1_worker_run_no_jobs(self): remakefile = Path('ex1.py') remake = load_remake(remakefile) remake.finalize() task_queue = mock.MagicMock() task_queue.get.side_effect = [None] task_complete_queue = mock.MagicMock() worker(1, remakefile, task_queue, task_complete_queue)
def ls_files(remakefile, long, filetype, exists, produced_by_rule, used_by_rule, produced_by_task, used_by_task): remake = load_remake(remakefile) filelist = remake.list_files(filetype, exists, produced_by_rule, used_by_rule, produced_by_task, used_by_task) if long: print(tabulate(filelist, headers=('path', 'filetype', 'exists'))) else: for file, ftype, exists in filelist: print(file)
def setUpClass(cls) -> None: cls.orig_cwd = os.getcwd() os.chdir(examples_dir) sysrun('make clean') cls.remake = load_remake('ex3.py') cls.remake.finalize() cls.monitor = RemakeMonitor(cls.remake) # Using the real curses package causes problems for PyCharm and github CI. # Mock out all the important parts. cls.stdscr = mock.MagicMock() cls.stdscr.getmaxyx.return_value = (100, 50) # Generate dummy keypresses to feed into RemakeMonitorCurses. commands = [ 'r', 'f', 't', ':task 0', ':task 1', ':task 2', ':show tasks', 'w' 'j', 'k', 'g', 'G', 'F', 'R', ':q' # Note, end by quiting application. ] clist = [] for command in commands: clist_command = [-1] * 100 + [ord(c) for c in command] if len(command) > 1: clist_command += [13] clist.extend(clist_command) cls.stdscr.getch.side_effect = clist # Create patches for all curses functions called. curses_patch_fns = [ 'init_pair', 'curs_set', 'color_pair', 'napms', 'is_term_resized', 'resizeterm', ] cls.patchers = [] for fn in curses_patch_fns: patcher = mock.patch(f'curses.{fn}') setattr(cls, fn, patcher.start()) cls.patchers.append(patcher) cls.is_term_resized.return_value = False
def rule_info(remakefile, long, rule_names): remake = load_remake(remakefile).finalize() rules = remake.list_rules() for rule_name in rule_names: found = False for rule in rules: if rule.__name__ == rule_name: print(rule) found = True break if not found: logger.error(f'No rule {rule_name} in {remake.name} found')
def test3_worker_run_task(self): remakefile = Path('ex1.py') remake = load_remake(remakefile) remake.finalize() task = remake.tasks[0] task_queue = mock.MagicMock() task_queue.get.side_effect = [('task', task.path_hash_key(), False), None] task_complete_queue = mock.MagicMock() worker(1, remakefile, task_queue, task_complete_queue)
def test_all_examples(): example_runner = load_remake('test_all_examples.py').finalize() for task in example_runner.tasks.in_rule('RunAllRemakes').filter( executor='singleproc'): yield run_task, example_runner, task for task in example_runner.tasks.in_rule('RunAllRemakes').filter( executor='multiproc'): yield run_task, example_runner, task for task in example_runner.tasks.in_rule('TestCLI'): yield run_task, example_runner, task for task in example_runner.tasks.in_rule('TestCLI2'): yield run_task, example_runner, task for task in example_runner.tasks.in_rule('TestEx1'): yield run_task, example_runner, task
def test4_worker_exception(self, mock_run): remakefile = Path('ex1.py') remake = load_remake(remakefile) remake.finalize() task = remake.tasks[0] task_queue = mock.MagicMock() task_queue.get.side_effect = [('rescan', task.path_hash_key(), False), None] task_complete_queue = mock.MagicMock() mock_run.return_value = Exception('Boom!') self.assertRaises( Exception, worker(1, remakefile, task_queue, task_complete_queue))
def task_info(remakefile, long, task_path_hash_keys): remake = load_remake(remakefile).finalize() info = remake.task_info(task_path_hash_keys) for task_path_hash_key, (task, task_md, status) in info.items(): print(str(task)) print(status) print(task_md.task_requires_rerun()) if long: print('Uses files:') for key, path in task.inputs.items(): print(f' {key}: {path}') print('Produces files:') for key, path in task.outputs.items(): print(f' {key}: {path}')
def remake_run(remakefile, rescan_only, force, one, random, print_reasons, executor, display): if force and (one or random): raise ValueError('--force cannot be used with --one or --random') remake = load_remake(remakefile).finalize() remake.configure(print_reasons, executor, display) remake.short_status() if rescan_only: remake.task_ctrl.run_rescan_only() elif one: remake.run_one() elif random: remake.run_random() else: remake.run_all(force=force) if display == 'task_dag': # Give user time to see final task_dag state. sleep(3) remake.short_status()
def remake_run_tasks(remakefile, task_path_hash_keys, handle_dependencies, force, print_reasons, executor, display, tfilter, rule, requires_rerun, uses_file, produces_file, ancestor_of, descendant_of): remake = load_remake(remakefile).finalize() remake.configure(print_reasons, executor, display) remake.short_status() if task_path_hash_keys and (tfilter or rule): raise RemakeError( 'Can only use one of --tasks and (--filter or --rule)') if task_path_hash_keys: tasks = remake.find_tasks(task_path_hash_keys) else: if tfilter: tfilter = dict([kv.split('=') for kv in tfilter.split(',')]) tasks = remake.list_tasks(tfilter, rule, requires_rerun, uses_file, produces_file, ancestor_of, descendant_of) remake.run_requested(tasks, force=force, handle_dependencies=handle_dependencies) if display == 'task_dag': # Give user time to see final task_dag state. sleep(3) remake.short_status()
def setUp(self) -> None: self.orig_cwd = os.getcwd() os.chdir(examples_dir) sysrun('make clean') self.remake = load_remake('ex3.py') self.remake.finalize()
def ls_rules(remakefile, long, tfilter, uses_file, produces_file): # TODO: implement all args. remake = load_remake(remakefile) rules = remake.list_rules() for rule in rules: print(f'{rule.__name__}')
def setUpClass(cls) -> None: cls.orig_cwd = os.getcwd() os.chdir(examples_dir) sysrun('make clean') cls.remake = load_remake('demo.py')
def input_loop(self, mode, command, keypresses, show, i_offset): stdscr = self.stdscr for i in range(self.num_input_loops): # Input loop. curses.napms(self.input_loop_timeout) # Action in loop if resize is True: # Not working! if curses.is_term_resized(self.rows, self.cols): self.rows, self.cols = self.stdscr.getmaxyx() curses.resizeterm(self.rows, self.cols) break try: c = self.getch() # stdscr.addstr(rows - 1, cols - 10, str(c)) if c == -1: continue if c in (curses.KEY_ENTER, 10, 13): command = ''.join(keypresses[1:]).split(' ') keypresses = [] break elif c == 127: # Backspace keypresses = keypresses[:-1] break else: if chr(c) == ':': mode = 'command' if mode == 'command': keypresses.append(chr(c)) else: if chr(c) == 't': show = 'tasks' i_offset = 0 break elif chr(c) == 'r': show = 'rules' i_offset = 0 break elif chr(c) == 'f': show = 'files' i_offset = 0 break elif chr(c) == 'j': i_offset -= 1 break elif chr(c) == 'k': if i_offset <= -1: i_offset += 1 break elif chr(c) == 'g': i_offset = 0 break elif chr(c) == 'G': i_offset = -10000 break elif chr(c) == 'w': self.wrap = not self.wrap break elif chr(c) == 'R': self.remake = load_remake(self.remake.name) self.remake.task_ctrl.build_task_DAG() self.monitor = RemakeMonitor(self.remake) self.remake_sha1sum = sha1sum(Path(self.remake.name + '.py')) break elif chr(c) == 'F': self.remake = load_remake(self.remake.name) self.remake.finalize() self.monitor = RemakeMonitor(self.remake) self.remake_sha1sum = sha1sum(Path(self.remake.name + '.py')) break stdscr.addstr(self.rows - 1, 0, ''.join(keypresses)) except curses.error: pass stdscr.refresh() if command: if command[0] == 'q': raise Quit() elif command[0] == 'show': show = command[1] i_offset = 0 elif command[0] == 'task': show = 'task' self.task_i = int(command[1]) i_offset = 0 return mode, command, keypresses, show, i_offset
def monitor(remakefile, timeout): from curses import wrapper remake = load_remake(remakefile) remake.task_ctrl.build_task_DAG() wrapper(remake_curses_monitor, remake, timeout)