def configure(self):
'''Set the output to the opposite of the input.
This process will upload or download a file from/to s3.
'''
super(S3Process, self).configure()
if not fs.is_s3url(self.main_src.output):
self.output = self.prop.sub('${s3_root}/%s' % self.output)
def add_dep(self, dep):
'''Check sources, inject extrace process to upload files.
XXX Refactor this function with RemoteProcess
'''
if self.prop.hadoop.runner == 'emr' and isinstance(dep, Node) and (not dep.output is None) and not fs.is_s3url(dep.output):
s3p = dep | process.S3Process()
dep = s3p.check_configure()
return super(HadoopProcess, self).add_dep(dep)
def add_dep(self, dep):
'''Check sources, inject extra process to download files.'''
if isinstance(dep, Node) and (not dep.output is None) and fs.is_s3url(dep.output):
s3p = dep | S3Process()
dep = s3p.check_configure()
return super(LocalProcess, self).add_dep(dep)
