def test_get_uri():
class TargetWithURI(LocalTarget):
def uri(self):
return 'i://have/a/uri'
class TargetWithPath(LocalTarget):
pass
class NotATarget:
pass
assert get_uri(TargetWithURI('fake/path')) == 'i://have/a/uri'
assert get_uri(TargetWithPath('a/path')) == 'a/path'
try:
get_uri(NotATarget())
assert False
except ValueError as e:
assert "Unknown input target type" in e.message
def _get_output_args(self):
if not isinstance(self._output, dict):
raise ValueError("Output must be dict type")
output_args = []
self._output_uris = {}
for (name, target) in self._output.items():
uri = target.generate_uri() if hasattr(target, "generate_uri") else get_uri(target)
uri = uri.rstrip("/")
output_args.append("--%s=%s" % (name, uri))
self._output_uris[name] = uri
return output_args
def _get_input_args(self):
# TODO(brianm): this doesn't work when subclass yields from `requires`
job_input = self.input()
if isinstance(job_input, luigi.Target):
job_input = {"input": job_input}
if len(job_input) == 0: # default requires()
return []
if not isinstance(job_input, dict):
raise ValueError("Input (requires()) must be dict type")
input_args = []
for (name, targets) in job_input.items():
uris = [get_uri(target) for target in luigi.task.flatten(targets)]
if isinstance(targets, dict):
# If targets is a dict that means it had multiple outputs. In this case make the
# input args "<input key>-<task output key>"
names = ["%s-%s" % (name, key) for key in targets.keys()]
else:
names = [name] * len(uris)
for (arg_name, uri) in zip(names, uris):
input_args.append("--%s=%s" % (arg_name, uri))
return input_args
def _get_input_uri(self, file_pattern, target):
uri = get_uri(target)
uri = uri.rstrip("/") + "/" + file_pattern
return uri