"""Basic definition which takes in1.txt -> out1.txt -> out2.txt """ from remake import Remake, TaskRule ex1 = Remake() class Basic1(TaskRule): rule_inputs = {'in': 'data/inputs/in1.txt'} rule_outputs = {'out': 'data/outputs/ex1/out1.txt'} def rule_run(self): # Changed! print('changed') assert len(self.inputs) == len(self.outputs) for i, o in zip(self.inputs.values(), self.outputs.values()): o.write_text('\n'.join([ f'changed output {line}' for line in i.read_text().split('\n')[:-1] ]) + '\n') class Basic2(TaskRule): rule_inputs = Basic1.rule_outputs rule_outputs = {'out': 'data/outputs/ex1/out2.txt'} def rule_run(self): assert len(self.inputs) == len(self.outputs) for i, o in zip(self.inputs.values(), self.outputs.values()): o.write_text('\n'.join(
curr_date = start_date while curr_date < end_date: next_date = curr_date + dt.timedelta(minutes=30) filename = filename_tpl.format( datestr=curr_date.fmt_date(), start_time=curr_date.fmt_time(), end_time=(next_date - dt.timedelta(seconds=1)).fmt_time(), minutes=curr_date.fmt_minutes()) url = url_tpl.format(year=curr_date.fmt_year(), doy=curr_date.fmt_doy(), filename=filename) yield curr_date, url, filename curr_date = next_date downloader = Remake() class GpmImerg30MinDownload(TaskRule): #@staticmethod #def rule_inputs(year): # # Each year (bar 2000) depends on output from prev year. # # Ensures years run sequentially. # if year == 2000: # return {} # else: # return {'prev_year': (IMERG_FINAL_30MIN_DIR / f'{year - 1}' / # f'download.{year - 1}.done')} rule_inputs = {} rule_outputs = { 'output_filenames':
"""Simple remake file: in.txt -> fan_out1.txt -> out.txt `> fan_out2.txt / """ from remake import Remake, TaskRule # A remake file is defined by creating a Remake object. demo = Remake() class FanOut(TaskRule): """Takes one input file and uses two tasks to generate two output files""" rule_inputs = {'in': 'data/in.txt'} rule_outputs = {'fan_out_{i}': 'data/fan_out_{i}.txt'} # This defines the output files, and the number of tasks this TaskRule will create. var_matrix = {'i': [1, 2]} def rule_run(self): # self.inputs and self.outputs are dictionaries created from rule_inputs # Each value is a pathlib.Path. input_value = self.inputs['in'].read_text() self.outputs[f'fan_out_{self.i}'].write_text( f'FanOut {self.i}: {input_value}') class Out(TaskRule): """Takes the two output files produced by FanOut and combines them into one output file""" rule_inputs = {f'fan_out_{i}': f'data/fan_out_{i}.txt' for i in [1, 2]} rule_outputs = {'out': 'data/out.txt'} def rule_run(self): input_values = []
from remake import Remake, TaskRule cyclic_dependency = Remake() class T1(TaskRule): rule_inputs = {'in1': 'data/inputs/in1.txt'} rule_outputs = {'out1': 'data/out1.txt'} def rule_run(self): self.rule_outputs['out1'].touch() class T2(TaskRule): rule_inputs = {**T1.rule_outputs, **{'in2': 'data/inputs/in2.txt'}} rule_outputs = {'out2': 'data/out2.txt'} def rule_run(self): self.outputs['out2'].touch() class T3(TaskRule): rule_inputs = T2.rule_inputs rule_outputs = {'out3': 'data/inputs/in2.txt'} def rule_run(self): self.outputs['out3'].touch()
"""Examples using TaskRule""" import random from time import sleep from remake import Remake, TaskRule from remake.formatter import remake_dict_expand as dict_exp ex2 = Remake() VAR_MATRIX = {'i': range(4), 'j': range(4)} class Init(TaskRule): rule_inputs = {} rule_outputs = {'out': 'data/outputs/ex2/out1.out'} def rule_run(self): self.outputs['out'].touch() class FanOut(TaskRule): rule_inputs = Init.rule_outputs rule_outputs = {'a{i},{j}': 'data/outputs/ex2/fan_out.{i}.{j}.out'} var_matrix = VAR_MATRIX def rule_run(self): sleep(random.randint(0, 10) / 10) payload = f'{self.__class__.__name__} ({self.i}, {self.j})' for o in self.outputs.values(): o.write_text(payload)
"""Demonstrates partial run when some input data not there. """ from remake import Remake, TaskRule ex8 = Remake() class CannotRun(TaskRule): rule_inputs = {'in1': 'data/inputs/input_not_there.txt'} rule_outputs = {'out': 'data/inputs/ex8_in1.txt'} def rule_run(self): input_text = self.inputs['in1'].read_text() self.outputs['out'].write_text(input_text + '\n') class CanRun1(TaskRule): rule_inputs = CannotRun.rule_outputs rule_outputs = { 'out1': 'data/outputs/ex8/out1.txt', 'out2': 'data/outputs/ex8/out2.txt' } def rule_run(self): for o in self.outputs.values(): o.write_text('out') class CanRun2(TaskRule): rule_inputs = {'in': 'data/outputs/ex8/out{i}.txt'} rule_outputs = {'out1': 'data/outputs/ex8/out2.{i}.txt'}
from pathlib import Path from remake import Remake, TaskRule remake = Remake() class AbsolutePathsTasks(TaskRule): # Doesn't matter if these don't exist. rule_inputs = {'in': Path('/tmp/data/in.txt')} rule_outputs = {'out': Path('/tmp/data/out.txt')} def rule_run(self): self.outputs['out'].write_text('done')
import calendar from remake import Remake, TaskRule ex3 = Remake() YEARS_MONTHS = { 'year': range(2015, 2021), 'month': range(1, 13), } class VariableNumberOutput(TaskRule): rule_inputs = {} @staticmethod def rule_outputs(year, month): days_in_month = calendar.monthrange(year, month)[1] return { f'day{d}': f'data/outputs/ex3/{year}.{month:02}.{d:02}' for d in range(1, days_in_month + 1) } var_matrix = YEARS_MONTHS def rule_run(self): payload = f'{self.__class__.__name__} ({self.year}, {self.month})' for key, o in self.outputs.items(): o.write_text(f'{key} {payload}')
"""Put remake through its paces by calling remake on all examples. Best run with `remake -W run test_all_examples`, which disables info logging from this. Dogfooding by using this instead of Makefile (previous), although it's a bit meta. Allows easy broadcast on e.g. all examples remakefiles, and using different executors. """ from pathlib import Path import subprocess from remake import TaskRule, Remake, remake_cmd remake_all = Remake() def sysrun(command): """Streams output from command to stdout""" return subprocess.run(command, check=True, shell=True, encoding='utf8') def run_commands(commands): for command in commands: print(command) system = command.split()[0] != 'remake' if system: output = sysrun(command) assert output.returncode == 0 else: remake_cmd.remake_cmd(['examples/test_all_examples.py'] + command.split()[1:]) VAR_MATRIX = {
from remake import Remake, TaskRule rmk = Remake() class Init(TaskRule): rule_inputs = {} rule_outputs = { 'out0': 'data/outputs/ex6/rule1.0.in', 'out1': 'data/outputs/ex6/chain.0.in' } def rule_run(self): self.outputs['out0'].touch() self.outputs['out1'].touch() class Rule1(TaskRule): """Links to Rule2 and back""" rule_inputs = {'in': 'data/outputs/ex6/rule1.{i}.in'} rule_outputs = {'out': 'data/outputs/ex6/rule1.{i}.out'} var_matrix = {'i': range(4)} def rule_run(self): self.outputs['out'].touch() class Rule2(TaskRule): """Links to Rule1 and back""" rule_inputs = {'in': 'data/outputs/ex6/rule1.{i}.out'}
"""Basic definition which takes in1.txt -> out1.txt -> out2.txt """ from remake import Remake, TaskRule ex7 = Remake() class Basic1(TaskRule): rule_inputs = {'in1': 'data/inputs/in1.txt', 'in2': 'data/inputs/in2.txt'} rule_outputs = {'out': 'data/outputs/ex7/out1.txt'} def rule_run(self): input_text = self.inputs['in1'].read_text( ) + '\n' + self.inputs['in2'].read_text() self.outputs['out'].write_text(input_text + '\n') class Basic2(TaskRule): rule_inputs = Basic1.rule_outputs rule_outputs = {'out': 'data/outputs/ex7/out2.txt'} def rule_run(self): assert len(self.inputs) == len(self.outputs) for i, o in zip(self.inputs.values(), self.outputs.values()): o.write_text('\n'.join( [f'f1 {line}' for line in i.read_text().split('\n')[:-1]]) + '\n')
"""Basic definition which takes in1.txt -> out1.txt -> out2.txt -> out3.txt """ from remake import Remake, TaskRule ex4 = Remake() def join_lines(path, prepend_text): return '\n'.join([ f'{prepend_text} {line}' for line in path.read_text().split('\n')[:-1] ]) + '\n' class DependsOn1(TaskRule): rule_inputs = {'in': 'data/inputs/in1.txt'} rule_outputs = {'out1': 'data/outputs/ex4/out1.txt'} depends_on = [join_lines] def rule_run(self): self.outputs['out1'].write_text( join_lines(self.inputs['in'], 'DependsOn1')) class DependsOn2(TaskRule): rule_inputs = DependsOn1.rule_outputs rule_outputs = {'out2': 'data/outputs/ex4/out2.txt'} depends_on = [join_lines] def rule_run(self): self.outputs['out2'].write_text( join_lines(self.inputs['out1'], 'DependsOn2'))
"""Examples using TaskRule""" import random from time import sleep from remake import Remake, TaskRule from remake.formatter import remake_dict_expand as dict_exp slurm_config = {'queue': 'test', 'mem': 64000} ex2 = Remake(config=dict(slurm=slurm_config)) VAR_MATRIX = {'i': range(4), 'j': range(4)} class Init(TaskRule): rule_inputs = {} rule_outputs = {'out': 'data/outputs/ex2/out1.out'} def rule_run(self): self.outputs['out'].touch() class FanOut(TaskRule): rule_inputs = Init.rule_outputs rule_outputs = {'a{i},{j}': 'data/outputs/ex2/fan_out.{i}.{j}.out'} var_matrix = VAR_MATRIX def rule_run(self): sleep(random.randint(0, 2)) payload = f'{self.__class__.__name__} ({self.i}, {self.j})' for o in self.outputs.values(): o.write_text(payload)