def test_multiple_input_tracking(self, task_c):
with dbnd_config({CoreConfig.tracker: ["file", "console"]}):
dbnd_task = wrap_luigi_task(task_c)
assert dbnd_task
# Output1 and 2 are actually inputs from TaskB, just badly named
assert dbnd_task.output1
assert dbnd_task.output10
assert dbnd_task.output2
assert dbnd_task.output20
output1 = [
x for x in dbnd_task.task_meta.task_params if x.name == "output1"
][0]
output10 = [
x for x in dbnd_task.task_meta.task_params if x.name == "output10"
][0]
output2 = [
x for x in dbnd_task.task_meta.task_params if x.name == "output2"
][0]
output20 = [
x for x in dbnd_task.task_meta.task_params if x.name == "output20"
][0]
assert output1.parameter.kind == _ParameterKind.task_input
assert output10.parameter.kind == _ParameterKind.task_input
assert output2.parameter.kind == _ParameterKind.task_input
assert output20.parameter.kind == _ParameterKind.task_input
def get_dbnd_task(self, luigi_task):
self.init_context()
if luigi_task.task_id in self.task_cache:
dbnd_task = self.task_cache[luigi_task.task_id][1]
else:
dbnd_task = wrap_luigi_task(luigi_task)
self.task_cache[luigi_task.task_id] = (luigi_task, dbnd_task)
return dbnd_task
def test_multiple_output_tracking(self, task_b):
dbnd_task = wrap_luigi_task(task_b)
assert dbnd_task
assert len(dbnd_task.task_outputs) == 3
assert dbnd_task.output1
assert dbnd_task.output2
output1 = _find_param(dbnd_task, "output1")
output2 = _find_param(dbnd_task, "output2")
assert output1.parameter.kind == _ParameterKind.task_output
assert output2.parameter.kind == _ParameterKind.task_output
def test_luigi_sanity_input_target_tracking(self, top10_artists):
dbnd_task = wrap_luigi_task(top10_artists)
assert dbnd_task
dbnd_input_target_p = dbnd_task.task_params.get_param_value(
"artist_streams")
assert dbnd_input_target_p
dbnd_input_target = dbnd_input_target_p.value
luigi_target = top10_artists.input()
assert luigi_target
assert luigi_target.path in dbnd_input_target.path
def test_luigi_sanity_input_target_tracking(self, top10_artists):
with dbnd_config({CoreConfig.tracker: ["file", "console"]}):
dbnd_task = wrap_luigi_task(top10_artists)
assert dbnd_task
dbnd_input_target = [
x for x in dbnd_task.task_meta.task_params if "artist_streams" in x.name
][0].value
assert dbnd_input_target
luigi_target = top10_artists.input()
assert luigi_target
assert luigi_target.path in dbnd_input_target.path
def test_luigi_sanity_input_target_tracking(self, top10_artists):
dbnd_task = wrap_luigi_task(top10_artists)
assert dbnd_task
dbnd_input_target = [
x for x in dbnd_task.task_meta.task_params.values()
if "artist_streams" in x.name
][0].value
assert dbnd_input_target
luigi_target = top10_artists.input()
assert luigi_target
assert luigi_target.path in dbnd_input_target.path
def test_luigi_sanity_output_target_tracking(self, top10_artists):
dbnd_task = wrap_luigi_task(top10_artists)
assert dbnd_task
assert dbnd_task.task_outputs
# 'result' is our added output target
assert len(dbnd_task.task_outputs) == 2
dbnd_output = [
v for k, v in dbnd_task.task_outputs.items() if k != "task_band"
][0]
assert dbnd_output
luigi_output = top10_artists.output()
assert luigi_output
# Assert we preserve filename and directory tree format
assert luigi_output.path in dbnd_output.path
def test_multiple_output_tracking(self, task_b):
with dbnd_config({CoreConfig.tracker: ["file", "console"]}):
dbnd_task = wrap_luigi_task(task_b)
assert dbnd_task
assert len(dbnd_task.task_outputs) == 3
assert dbnd_task.output1
assert dbnd_task.output2
output1 = [
x for x in dbnd_task.task_meta.task_params if x.name == "output1"
][0]
output2 = [
x for x in dbnd_task.task_meta.task_params if x.name == "output2"
][0]
assert output1.parameter.kind == _ParameterKind.task_output
assert output2.parameter.kind == _ParameterKind.task_output
def test_multiple_input_tracking(self, task_c):
dbnd_task = wrap_luigi_task(task_c)
assert dbnd_task
# Output1 and 2 are actually inputs from TaskB, just badly named
assert dbnd_task.output1
assert dbnd_task.output10
assert dbnd_task.output2
assert dbnd_task.output20
output1 = _find_param(dbnd_task, "output1")
output10 = _find_param(dbnd_task, "output10")
output2 = _find_param(dbnd_task, "output2")
output20 = _find_param(dbnd_task, "output20")
assert output1.parameter.kind == _ParameterKind.task_input
assert output10.parameter.kind == _ParameterKind.task_input
assert output2.parameter.kind == _ParameterKind.task_input
assert output20.parameter.kind == _ParameterKind.task_input
