def configure(self): '''Set the output to the opposite of the input. This process will upload or download a file from/to s3. ''' super(S3Process, self).configure() if not fs.is_s3url(self.main_src.output): self.output = self.prop.sub('${s3_root}/%s' % self.output)
def add_dep(self, dep): '''Check sources, inject extrace process to upload files. XXX Refactor this function with RemoteProcess ''' if self.prop.hadoop.runner == 'emr' and isinstance(dep, Node) and (not dep.output is None) and not fs.is_s3url(dep.output): s3p = dep | process.S3Process() dep = s3p.check_configure() return super(HadoopProcess, self).add_dep(dep)
def add_dep(self, dep): '''Check sources, inject extra process to download files.''' if isinstance(dep, Node) and (not dep.output is None) and fs.is_s3url(dep.output): s3p = dep | S3Process() dep = s3p.check_configure() return super(LocalProcess, self).add_dep(dep)