def output(self):
"""
The only output of this Task is
the target path. Luigi will
check to be sure it exists.
"""
return [target_factory.get_target(self.path)]
def get_metadata(self):
"""
Get the metadata dictionary for this input data.
"""
metadata_file_target = target_factory.get_target(self.metadata_file_path())
with metadata_file_target.open('r') as metadata_file:
return json.load(metadata_file)
def output_token(self):
"""
Luigi Target providing path to a token that indicates
completion of this Task.
:rtype: Target:
:returns: Target for Task completion token
"""
return target_factory.get_target('%s/%s' % (self.token_path, self.__class__.__name__))
def success_token(self):
"""
The MortarProjectTask writes out several "tokens" as it executes to ensure
idempotence. This method provides the token file that indicates that the job
has finished successfully. If this token exists, the Task will not be rerun.
By default, it is stored underneath the path provided by the `token_path` method,
and is named after your class name. So, if your `token_path` is set to
`s3://my-bucket/my-folder` and your Task is named FooTask, the token will be:
`s3://my-bucket/my-folder/FooTask`
If you want this Task to be rerun, you should delete that token.
:rtype: Target:
:returns: Target for the token that indicates that this Task has succeeded.
"""
return target_factory.get_target('%s/%s' % (self.token_path(), self.__class__.__name__))
def running_token(self):
"""
The MortarProjectTask writes out several "tokens" as it executes to ensure
idempotence. This method provides the token file that indicates that the job
is running.
By default, it is stored underneath the path provided by the `token_path` method,
and is named after your class name. So, if your `token_path` is set to
`s3://my-bucket/my-folder` and your Task is named FooTask, the token will be:
`s3://my-bucket/my-folder/FooTask-Running`
This token will contain the Mortar job_id of the job that is running.
:rtype: Target:
:returns: Target for the token that indicates job is running.
"""
return target_factory.get_target('%s/%s-%s' % (self.token_path(), self.__class__.__name__, 'Running'))
def success_token(self):
"""
The MortarProjectTask writes out several "tokens" as it executes to ensure
idempotence. This method provides the token file that indicates that the job
has finished successfully. If this token exists, the Task will not be rerun.
By default, it is stored underneath the path provided by the `token_path` method,
and is named after your class name. So, if your `token_path` is set to
`s3://my-bucket/my-folder` and your Task is named FooTask, the token will be:
`s3://my-bucket/my-folder/FooTask`
If you want this Task to be rerun, you should delete that token.
:rtype: Target:
:returns: Target for the token that indicates that this Task has succeeded.
"""
return target_factory.get_target(
'%s/%s' % (self.token_path(), self.__class__.__name__))
def running_token(self):
"""
The MortarProjectTask writes out several "tokens" as it executes to ensure
idempotence. This method provides the token file that indicates that the job
is running.
By default, it is stored underneath the path provided by the `token_path` method,
and is named after your class name. So, if your `token_path` is set to
`s3://my-bucket/my-folder` and your Task is named FooTask, the token will be:
`s3://my-bucket/my-folder/FooTask-Running`
This token will contain the Mortar job_id of the job that is running.
:rtype: Target:
:returns: Target for the token that indicates job is running.
"""
return target_factory.get_target(
'%s/%s-%s' %
(self.token_path(), self.__class__.__name__, 'Running'))
def output(self):
"""
Tell Luigi about the output that this Task produces.
If that output already exists, Luigi will not rerun it.
"""
return [target_factory.get_target(self.output_path)]
def output_token(self):
"""
Token written out to indicate finished shell script
"""
return target_factory.get_target('%s/%s' % (self.token_path, self.__class__.__name__))
def test_get_target_file(self):
file_target = target_factory.get_target('file://%s' % self.tmp.name)
reread_data = file_target.open().read()
self.assertEquals(self.data, reread_data)
def test_get_target_local(self):
local_target = target_factory.get_target(self.tmp.name)
reread_data = local_target.open().read()
self.assertEquals(self.data, reread_data)
def test_get_target_s3(self, mock_s3_target_cls):
mock_s3_target = mock.Mock()
mock_s3_target_cls.return_value = mock_s3_target
s3_target = target_factory.get_target('s3://mybucket/mypath')
self.assertEquals(mock_s3_target, s3_target)
def success_token(self):
"""
Token written out to indicate the Pigscript has finished
"""
return target_factory.get_target('%s/%s' % (self.token_path(), self.__class__.__name__))
def running_token(self):
"""
Token written out to indicate a running Pigscript
"""
return target_factory.get_target('%s/%s-%s' % (self.token_path(), self.__class__.__name__, 'Running'))
def output(self):
"""
Tell Luigi about the output that this Task produces.
If that output already exists, Luigi will not rerun it.
"""
return [target_factory.get_target(self.output_path)]
