def move_output(self, working_dir):
for out_file in self.output:
relpath = os.path.relpath(out_file, working_dir)
# figure out where this task was run and its path at that host
src_host = self.host
src_path = os.path.join(self.local_wd, relpath)
# make the directory which we're going to copy to
local('mkdir -p {0}'.format(os.path.dirname(out_file)))
src = ''.join([src_host, ':', src_path])
self.transfer_started(task=self.name,
source=src,
destination=out_file)
# now copy the file to the workflow working directory
if remote('cp {0} {1}'.format(src_path, out_file), src_host) == 0:
self.transfer_success(task=self.name,
source=src,
destination=out_file)
else:
expl = 'could not copy file {0} to {1}'.format(src, out_file)
self.transfer_failed(task=self.name,
source=src,
destination=out_file,
explanation=expl)
def get_input(self):
for in_file, dependency in self.dependencies:
# build path to file on remote source and destination
relpath = os.path.relpath(in_file, self.working_dir)
src_path = os.path.join(dependency.local_wd, relpath)
dst_path = os.path.join(self.local_wd, relpath)
# figure out source and destination hosts
src_host = dependency.host
dst_host = self.host
# if the dependency was checkpointed, we check if the file exists
# on the node at which it was generated. If not, we should fetch
# the file from remote storage.
if dependency.checkpoint and not dependency.host:
src_host = dst_host
src_path = in_file
if local('stat {0} &> /dev/null'.format(src_path)) < 0:
logging.error('''output of checkpointed
dependency does not exist at %s --
halting''' %
src_path)
os.exit(1)
remote('mkdir -p {0}'.format(os.path.dirname(dst_path)), dst_host)
# if the source host is the same as the destination host, we won't
# copy any files, but just make a hardlink to the source file.
if src_host == dst_host and not dependency.checkpoint:
self.transfer_started(task=self.name,
source=src_path,
destination=dst_path)
errorcode = remote('ln {0} {1}'.format(src_path, dst_path),
src_host)
if errorcode > 0:
expl = 'could not hard link {0} to {1}'.format(src_path,
dst_path)
self.transfer_failed(task=self.name,
source=src_path,
destination=dst_path,
explanation=expl)
else:
self.transfer_success(task=self.name,
source=src_path,
destination=dst_path)
elif dependency.checkpoint:
self.transfer_started(task=self.name,
source=src_path,
destination=dst_path)
errorcode = remote('cp {0} {1}'.format(src_path, dst_path),
src_host)
if errorcode > 0:
expl = 'could not copy file {0} to {1}'.format(src_path,
dst_path)
self.transfer_failed(task=self.name,
source=src_path,
destination=dst_path,
explanation=expl)
else:
self.transfer_success(task=self.name,
source=src_path,
destination=dst_path)
else:
src = ''.join([src_host, ':', src_path])
dst = ''.join([dst_host, ':', dst_path])
self.transfer_started(task=self.name,
source=src,
destination=dst)
if local('scp -c arcfour {0} {1}'.format(src, dst)) == 0:
self.transfer_success(task=self.name,
source=src,
destination=dst)
else:
expl = 'could not copy file {0} to {1}'.format(src, dst)
self.transfer_failed(task=self.name,
source=src,
destination=dst,
explanation=expl)