def transfer_files(self):
JobDeploymentBase.transfer_files(self)
# Here we transfer any input files to the relevant directory on the
# target platform.
# Use SAGA-Python to handle the file transfer.
LOG.debug('Transfer files...')
job_dir = self.platform_config.storage_job_directory
host = self.platform_config.platform_service_host
try:
directory = Directory('sftp://%s%s' % (host, job_dir),
session=self.session)
except saga.BadParameter as e:
LOG.error('The specified job directory does not exist on PBS '
'submission node <%s> (%s).' % (host, str(e)))
raise JobError('The specified job directory does not exist on PBS'
'submission node <%s> (%s)' % (host, str(e)))
try:
# directory.make_dir() does not return a handle to the new directory
# so need to create the directory URL manually.
directory.make_dir(self.job_config.job_id)
job_data_dir = os.path.join(str(directory.url),
self.job_config.job_id)
except saga.NoSuccess as e:
LOG.error('The specified job data directory already exists on '
'PBS submission node <%s> (%s).' % (host, str(e)))
raise JobError('The specified job directory already exists on PBS'
'submission node <%s> (%s)' % (host, str(e)))
# Now upload the file(s) to the job data directory
# and create an input file list containing the resulting locations
# of the files.
# There are some cases where jobs may not have input files (they may,
# for example pull the input files from a remote location as part of
# the job process) so we first check whether there are any input files
# to process, if not, then return from this function
if not self.job_config.input_files:
LOG.debug('There are no input files to transfer for this job...')
return
self.transferred_input_files = []
for f in self.job_config.input_files:
try:
f_obj = File('file://%s' % f, session=self.session)
f_obj.copy(job_data_dir)
dest_dir = os.path.join(directory.url.path,
self.job_config.job_id)
self.transferred_input_files.append(
os.path.join(dest_dir, os.path.basename(f_obj.url.path)))
except:
LOG.error('Error copying the input file <%s> to the remote '
'platform.' % f)
raise JobError('Error copying the input file <%s> to the '
'remote platform.' % f)
def transfer_files(self):
JobDeploymentBase.transfer_files(self)
def transfer_files(self):
JobDeploymentBase.transfer_files(self)
LOG.debug('SSH Deployer: Transfer files...')
# Here we transfer any input files to the relevant directory on the
# target platform.
# Use SAGA-Python to handle the file transfer.
job_dir = self.platform_config.storage_job_directory
# Check that the job storage directory exists and then create a
# sub-directory specifically for this job.
try:
LOG.debug('URL for file transfer: <sftp://%s:%s%s>' %
(self.host, self.port, job_dir))
directory = Directory('sftp://%s:%s%s' %
(self.host, self.port, job_dir),
session=self.session)
except saga.BadParameter as e:
LOG.error('Error setting up connection to resource directory.')
if 'connection refused' in str(e).lower():
raise ConnectionError('Unable to connect to remote resource '
'to set up connection to directory.')
raise StorageDirectoryNotFoundError(
'The specified job data base '
'directory does not exist on resource <%s> (%s)' %
(self.host, str(e)))
try:
# directory.make_dir() does not return a handle to the new directory
# so need to create the directory URL manually.
directory.make_dir(self.job_config.job_id)
job_data_dir = os.path.join(str(directory.url),
self.job_config.job_id)
except saga.NoSuccess as e:
LOG.error('The specified job data directory already exists on '
'resource <%s> (%s).' % (self.host, str(e)))
raise DirectoryExistsError('The specified job directory already '
'exists on resource <%s> (%s)' %
(self.host, str(e)))
# Now upload the file(s) to the job data directory
# and create an input file list containing the resulting locations
# of the files.
# There are some cases where jobs may not have input files (they may,
# for example pull the input files from a remote location as part of
# the job process) so we first check whether there are any input files
# to process, if not, then return from this function
if not self.job_config.input_files:
LOG.debug('There are no input files to transfer for this job...')
return
self.transferred_input_files = []
for f in self.job_config.input_files:
try:
f_obj = File('file://%s' % f, session=self.session)
f_obj.copy(job_data_dir)
dest_dir = os.path.join(directory.url.path,
self.job_config.job_id)
self.transferred_input_files.append(
os.path.join(dest_dir, os.path.basename(f_obj.url.path)))
except:
LOG.error('Error copying the input file <%s> to the remote '
'platform.' % f)
raise JobError('Error copying the input file <%s> to the '
'remote platform.' % f)
def transfer_files(self):
JobDeploymentBase.transfer_files(self)
# Here we transfer any input files to the relevant directory on the
# target platform.
# Use SAGA-Python to handle the file transfer.
LOG.debug('Transfer files...')
job_dir = self.platform_config.storage_job_directory
# At this point we need to switch back to using the job secruity
# context. If we were using unconfigured resources, these will have
# been configured using an admin context by now.
self.session = saga.Session(default=False)
self.session.add_context(self.job_ctx)
# Begin by checking if we're working with more than one instance, if
# so we have a master and one or more slave nodes. We'll push the data
# to the master and then direct the master to distribute it to the
# slave nodes.
master_node = self.running_nodes[0][0]
slave_nodes = []
if len(self.running_nodes) > 1:
slave_nodes = [node[0] for node in self.running_nodes[1:]]
# On the master node: Check that the job storage directory exists and
# then create a sub-directory specifically for this job.
# Node is a tuple consisting of two items, the node object and an
# IP list. For now we work with the node object directly.
node_ip = master_node.public_ips[0]
try:
directory = Directory('sftp://%s%s' % (node_ip, job_dir),
session=self.session)
except saga.BadParameter as e:
LOG.error('The specified job directory does not exist on node '
'<%s> (%s).' % (node_ip, str(e)))
#raise JobError('The specified job directory does not exist '
# 'on node <%s> (%s)' % (node_ip, str(e)))
try:
# directory.make_dir() does not return a handle to the new directory
# so need to create the directory URL manually.
directory.make_dir(self.job_config.job_id)
except saga.NoSuccess as e:
LOG.warning('The specified job data directory already exists on '
'node <%s> (%s).' % (node_ip, str(e)))
#raise JobError('The specified job directory already exists on '
# 'on node <%s> (%s)' % (node_ip, str(e)))
job_data_dir = os.path.join(str(directory.url), self.job_config.job_id)
# Now upload the file(s) to the job data directory
# and create an input file list containing the resulting locations
# of the files.
# There are some cases where jobs may not have input files (they may,
# for example pull the input files from a remote location as part of
# the job process) so we first check whether there are any input files
# to process, if not, then return from this function
if not self.job_config.input_files:
LOG.debug('There are no input files to transfer for this job...')
return
self.transferred_input_files = []
for f in self.job_config.input_files:
try:
f_obj = File('file://%s' % f, session=self.session)
f_obj.copy(job_data_dir)
dest_dir = os.path.join(directory.url.path,
self.job_config.job_id)
self.transferred_input_files.append(
os.path.join(dest_dir, os.path.basename(f_obj.url.path)))
except:
LOG.error('Error copying the input file <%s> to the remote '
'platform.' % f)
raise JobError('Error copying the input file <%s> to the '
'remote platform.' % f)
# At this point input files have been successfully transferred to
# the master node. We now direct the master node to send the files
# to each of the slave nodes:
if slave_nodes:
slave_private_ips = [node.private_ips[0] for node in slave_nodes]
self._distribute_job_data(master_node.public_ips[0],
slave_private_ips,
self.platform_config.user_id,
self.platform_config.user_key_file,
job_dir, self.job_config.job_id)