"""Basic definition which takes in1.txt -> out1.txt -> out2.txt
"""
from remake import Remake, TaskRule
ex1 = Remake()
class Basic1(TaskRule):
rule_inputs = {'in': 'data/inputs/in1.txt'}
rule_outputs = {'out': 'data/outputs/ex1/out1.txt'}
def rule_run(self):
# Changed!
print('changed')
assert len(self.inputs) == len(self.outputs)
for i, o in zip(self.inputs.values(), self.outputs.values()):
o.write_text('\n'.join([
f'changed output {line}'
for line in i.read_text().split('\n')[:-1]
]) + '\n')
class Basic2(TaskRule):
rule_inputs = Basic1.rule_outputs
rule_outputs = {'out': 'data/outputs/ex1/out2.txt'}
def rule_run(self):
assert len(self.inputs) == len(self.outputs)
for i, o in zip(self.inputs.values(), self.outputs.values()):
o.write_text('\n'.join(
curr_date = start_date
while curr_date < end_date:
next_date = curr_date + dt.timedelta(minutes=30)
filename = filename_tpl.format(
datestr=curr_date.fmt_date(),
start_time=curr_date.fmt_time(),
end_time=(next_date - dt.timedelta(seconds=1)).fmt_time(),
minutes=curr_date.fmt_minutes())
url = url_tpl.format(year=curr_date.fmt_year(),
doy=curr_date.fmt_doy(),
filename=filename)
yield curr_date, url, filename
curr_date = next_date
downloader = Remake()
class GpmImerg30MinDownload(TaskRule):
#@staticmethod
#def rule_inputs(year):
# # Each year (bar 2000) depends on output from prev year.
# # Ensures years run sequentially.
# if year == 2000:
# return {}
# else:
# return {'prev_year': (IMERG_FINAL_30MIN_DIR / f'{year - 1}' /
# f'download.{year - 1}.done')}
rule_inputs = {}
rule_outputs = {
'output_filenames':
"""Simple remake file: in.txt -> fan_out1.txt -> out.txt
`> fan_out2.txt /
"""
from remake import Remake, TaskRule
# A remake file is defined by creating a Remake object.
demo = Remake()
class FanOut(TaskRule):
"""Takes one input file and uses two tasks to generate two output files"""
rule_inputs = {'in': 'data/in.txt'}
rule_outputs = {'fan_out_{i}': 'data/fan_out_{i}.txt'}
# This defines the output files, and the number of tasks this TaskRule will create.
var_matrix = {'i': [1, 2]}
def rule_run(self):
# self.inputs and self.outputs are dictionaries created from rule_inputs
# Each value is a pathlib.Path.
input_value = self.inputs['in'].read_text()
self.outputs[f'fan_out_{self.i}'].write_text(
f'FanOut {self.i}: {input_value}')
class Out(TaskRule):
"""Takes the two output files produced by FanOut and combines them into one output file"""
rule_inputs = {f'fan_out_{i}': f'data/fan_out_{i}.txt' for i in [1, 2]}
rule_outputs = {'out': 'data/out.txt'}
def rule_run(self):
input_values = []
from remake import Remake, TaskRule
cyclic_dependency = Remake()
class T1(TaskRule):
rule_inputs = {'in1': 'data/inputs/in1.txt'}
rule_outputs = {'out1': 'data/out1.txt'}
def rule_run(self):
self.rule_outputs['out1'].touch()
class T2(TaskRule):
rule_inputs = {**T1.rule_outputs, **{'in2': 'data/inputs/in2.txt'}}
rule_outputs = {'out2': 'data/out2.txt'}
def rule_run(self):
self.outputs['out2'].touch()
class T3(TaskRule):
rule_inputs = T2.rule_inputs
rule_outputs = {'out3': 'data/inputs/in2.txt'}
def rule_run(self):
self.outputs['out3'].touch()
"""Examples using TaskRule"""
import random
from time import sleep
from remake import Remake, TaskRule
from remake.formatter import remake_dict_expand as dict_exp
ex2 = Remake()
VAR_MATRIX = {'i': range(4), 'j': range(4)}
class Init(TaskRule):
rule_inputs = {}
rule_outputs = {'out': 'data/outputs/ex2/out1.out'}
def rule_run(self):
self.outputs['out'].touch()
class FanOut(TaskRule):
rule_inputs = Init.rule_outputs
rule_outputs = {'a{i},{j}': 'data/outputs/ex2/fan_out.{i}.{j}.out'}
var_matrix = VAR_MATRIX
def rule_run(self):
sleep(random.randint(0, 10) / 10)
payload = f'{self.__class__.__name__} ({self.i}, {self.j})'
for o in self.outputs.values():
o.write_text(payload)
"""Demonstrates partial run when some input data not there.
"""
from remake import Remake, TaskRule
ex8 = Remake()
class CannotRun(TaskRule):
rule_inputs = {'in1': 'data/inputs/input_not_there.txt'}
rule_outputs = {'out': 'data/inputs/ex8_in1.txt'}
def rule_run(self):
input_text = self.inputs['in1'].read_text()
self.outputs['out'].write_text(input_text + '\n')
class CanRun1(TaskRule):
rule_inputs = CannotRun.rule_outputs
rule_outputs = {
'out1': 'data/outputs/ex8/out1.txt',
'out2': 'data/outputs/ex8/out2.txt'
}
def rule_run(self):
for o in self.outputs.values():
o.write_text('out')
class CanRun2(TaskRule):
rule_inputs = {'in': 'data/outputs/ex8/out{i}.txt'}
rule_outputs = {'out1': 'data/outputs/ex8/out2.{i}.txt'}
from pathlib import Path
from remake import Remake, TaskRule
remake = Remake()
class AbsolutePathsTasks(TaskRule):
# Doesn't matter if these don't exist.
rule_inputs = {'in': Path('/tmp/data/in.txt')}
rule_outputs = {'out': Path('/tmp/data/out.txt')}
def rule_run(self):
self.outputs['out'].write_text('done')
import calendar
from remake import Remake, TaskRule
ex3 = Remake()
YEARS_MONTHS = {
'year': range(2015, 2021),
'month': range(1, 13),
}
class VariableNumberOutput(TaskRule):
rule_inputs = {}
@staticmethod
def rule_outputs(year, month):
days_in_month = calendar.monthrange(year, month)[1]
return {
f'day{d}': f'data/outputs/ex3/{year}.{month:02}.{d:02}'
for d in range(1, days_in_month + 1)
}
var_matrix = YEARS_MONTHS
def rule_run(self):
payload = f'{self.__class__.__name__} ({self.year}, {self.month})'
for key, o in self.outputs.items():
o.write_text(f'{key} {payload}')
"""Put remake through its paces by calling remake on all examples.
Best run with `remake -W run test_all_examples`, which disables info logging from this.
Dogfooding by using this instead of Makefile (previous), although it's a bit meta.
Allows easy broadcast on e.g. all examples remakefiles, and using different executors.
"""
from pathlib import Path
import subprocess
from remake import TaskRule, Remake, remake_cmd
remake_all = Remake()
def sysrun(command):
"""Streams output from command to stdout"""
return subprocess.run(command, check=True, shell=True, encoding='utf8')
def run_commands(commands):
for command in commands:
print(command)
system = command.split()[0] != 'remake'
if system:
output = sysrun(command)
assert output.returncode == 0
else:
remake_cmd.remake_cmd(['examples/test_all_examples.py'] +
command.split()[1:])
VAR_MATRIX = {
from remake import Remake, TaskRule
rmk = Remake()
class Init(TaskRule):
rule_inputs = {}
rule_outputs = {
'out0': 'data/outputs/ex6/rule1.0.in',
'out1': 'data/outputs/ex6/chain.0.in'
}
def rule_run(self):
self.outputs['out0'].touch()
self.outputs['out1'].touch()
class Rule1(TaskRule):
"""Links to Rule2 and back"""
rule_inputs = {'in': 'data/outputs/ex6/rule1.{i}.in'}
rule_outputs = {'out': 'data/outputs/ex6/rule1.{i}.out'}
var_matrix = {'i': range(4)}
def rule_run(self):
self.outputs['out'].touch()
class Rule2(TaskRule):
"""Links to Rule1 and back"""
rule_inputs = {'in': 'data/outputs/ex6/rule1.{i}.out'}
"""Basic definition which takes in1.txt -> out1.txt -> out2.txt
"""
from remake import Remake, TaskRule
ex7 = Remake()
class Basic1(TaskRule):
rule_inputs = {'in1': 'data/inputs/in1.txt', 'in2': 'data/inputs/in2.txt'}
rule_outputs = {'out': 'data/outputs/ex7/out1.txt'}
def rule_run(self):
input_text = self.inputs['in1'].read_text(
) + '\n' + self.inputs['in2'].read_text()
self.outputs['out'].write_text(input_text + '\n')
class Basic2(TaskRule):
rule_inputs = Basic1.rule_outputs
rule_outputs = {'out': 'data/outputs/ex7/out2.txt'}
def rule_run(self):
assert len(self.inputs) == len(self.outputs)
for i, o in zip(self.inputs.values(), self.outputs.values()):
o.write_text('\n'.join(
[f'f1 {line}'
for line in i.read_text().split('\n')[:-1]]) + '\n')
"""Basic definition which takes in1.txt -> out1.txt -> out2.txt -> out3.txt
"""
from remake import Remake, TaskRule
ex4 = Remake()
def join_lines(path, prepend_text):
return '\n'.join([
f'{prepend_text} {line}' for line in path.read_text().split('\n')[:-1]
]) + '\n'
class DependsOn1(TaskRule):
rule_inputs = {'in': 'data/inputs/in1.txt'}
rule_outputs = {'out1': 'data/outputs/ex4/out1.txt'}
depends_on = [join_lines]
def rule_run(self):
self.outputs['out1'].write_text(
join_lines(self.inputs['in'], 'DependsOn1'))
class DependsOn2(TaskRule):
rule_inputs = DependsOn1.rule_outputs
rule_outputs = {'out2': 'data/outputs/ex4/out2.txt'}
depends_on = [join_lines]
def rule_run(self):
self.outputs['out2'].write_text(
join_lines(self.inputs['out1'], 'DependsOn2'))
"""Examples using TaskRule"""
import random
from time import sleep
from remake import Remake, TaskRule
from remake.formatter import remake_dict_expand as dict_exp
slurm_config = {'queue': 'test', 'mem': 64000}
ex2 = Remake(config=dict(slurm=slurm_config))
VAR_MATRIX = {'i': range(4), 'j': range(4)}
class Init(TaskRule):
rule_inputs = {}
rule_outputs = {'out': 'data/outputs/ex2/out1.out'}
def rule_run(self):
self.outputs['out'].touch()
class FanOut(TaskRule):
rule_inputs = Init.rule_outputs
rule_outputs = {'a{i},{j}': 'data/outputs/ex2/fan_out.{i}.{j}.out'}
var_matrix = VAR_MATRIX
def rule_run(self):
sleep(random.randint(0, 2))
payload = f'{self.__class__.__name__} ({self.i}, {self.j})'
for o in self.outputs.values():
o.write_text(payload)