def shutdown_resources(self): JobDeploymentBase.shutdown_resources(self) # Here we terminate the running resources for this job and # wait until they have been shut down. res_ids = [node.id for node in self.nodes] LOG.debug('About to shut down the following nodes: %s' % res_ids) LOG.debug('Shutdown resources...') for node in self.nodes: self.driver.destroy_node(node) while res_ids: nodes_to_wait_for = self.driver.list_nodes(res_ids) still_running = [node.id for node in nodes_to_wait_for] new_res_ids = [] # Now go through res_ids and delete the nodes that don't appear # in still_running. for res_id in res_ids: if res_id not in still_running: LOG.debug('Resource <%s> has terminated...' % res_id) else: new_res_ids.append(res_id) res_ids = new_res_ids if res_ids: LOG.debug('Still waiting for termination of resources %s...' % res_ids) time.sleep(2) LOG.debug('All resources terminated.')
def deploy_software(self): JobDeploymentBase.deploy_software(self) # Here we undertake transfer of the code to the remote platform if this # is required. In many cases, the software is likely to already be # deployed on the target platform or may have been configured via a # tool such as cloud-init, puppet, etc at resource initialisation time. LOG.debug('SSH Deployer: Deploy software...')
def run_job(self, job_details=None): JobDeploymentBase.run_job(self) # This function uses the SAGA-Python library to run the job. Separate # functionality in the library is used for monitoring the job process. # TODO: Add modules to PBS job confiuguration # Here we extract the job details from the previously stored job details # object into a SAGA Python job description object so that we can run # the job. job_arguments = getattr(self.job_config, 'args', []) input_files = getattr(self, 'transferred_input_files', []) job_arguments += input_files jd = saga.job.Description() jd.environment = getattr(self.job_config, 'environment', {}) jd.executable = getattr(self.job_config, 'executable', None) jd.arguments = job_arguments jd.working_directory = getattr(self.job_config, 'working_dir', None) jd.output = getattr(self.job_config, 'stdout', None) jd.error = getattr(self.job_config, 'stderr', None) jd.wall_time_limit = getattr(self.job_config, 'time_limit_mins', 0) jd.total_cpu_count = getattr(self.job_config, 'num_processes', 1) #jd.processes_per_host = 1 #jd.total_physical_memory = "2400" self.job = self.svc.create_job(jd) self.job.run()
def wait_for_job_completion(self): JobDeploymentBase.wait_for_job_completion(self) # Wait for job to complete self.job.wait() return (self.job.state, self.job.exit_code)
def run_job(self): JobDeploymentBase.run_job(self) # This function uses the libhpc resource daemon client to talk to the # resource daemon that is installed on cloud resources. It uses this # interface to run jobs and monitor their state to see when they are # complete. # TODO: Should this be running/managing the job remotely via a SAGA # SSH session or should we be expecting to communicate with a remote # resource management service to handle this? LOG.debug('Run job...') job_arguments = getattr(self.job_config, 'args', []) input_files = getattr(self, 'transferred_input_files', []) job_arguments += input_files # Check if we have a JOB_ID variable in the arguments or input files. # If so, replace this variable with the actual job ID. job_arguments_tmp = job_arguments job_arguments = [] for item in job_arguments_tmp: # Can't do a replace on items that are not string types! if isinstance(item, basestring): job_arguments.append( item.replace('$JOB_ID', self.job_config.job_id)) else: job_arguments.append(item) LOG.debug('Modified job arguments: %s' % job_arguments) jd = saga.job.Description() jd.environment = getattr(self.job_config, 'environment', {}) if self.job_config.num_processes > 1: jd.executable = ('mpirun -np %s -machinefile /tmp/machinefile' % (self.job_config.num_processes)) executable = getattr(self.job_config, 'executable', None) if executable: job_arguments.insert(0, executable) else: jd.executable = getattr(self.job_config, 'executable', None) jd.arguments = job_arguments jd.working_directory = getattr(self.job_config, 'working_dir', None) jd.output = getattr(self.job_config, 'stdout', None) jd.error = getattr(self.job_config, 'stderr', None) jd.wall_time_limit = getattr(self.job_config, 'time_limit_mins', 0) #jd.number_of_processes = 4 #jd.processes_per_host = 1 #jd.total_physical_memory = "2400" if not jd.output: jd.output = 'std.out' if not jd.error: jd.error = 'std.err' self.svc = saga.job.Service('ssh://%s/' % self.running_nodes[0][0].public_ips[0], session=self.session) self.job = self.svc.create_job(jd) self.job.run()
def initialise_resources(self, *args, **kwargs): JobDeploymentBase.initialise_resources(self) # Resource initialisation is not required directly but we use this # function to intiialise the connection with the PBS platform via the # SAGA-Python library. self.svc = saga.job.Service('pbs+ssh://%s/' % self.platform_config.platform_service_host, session=self.session) return None
def transfer_files(self): JobDeploymentBase.transfer_files(self) # Here we transfer any input files to the relevant directory on the # target platform. # Use SAGA-Python to handle the file transfer. LOG.debug('Transfer files...') job_dir = self.platform_config.storage_job_directory host = self.platform_config.platform_service_host try: directory = Directory('sftp://%s%s' % (host, job_dir), session=self.session) except saga.BadParameter as e: LOG.error('The specified job directory does not exist on PBS ' 'submission node <%s> (%s).' % (host, str(e))) raise JobError('The specified job directory does not exist on PBS' 'submission node <%s> (%s)' % (host, str(e))) try: # directory.make_dir() does not return a handle to the new directory # so need to create the directory URL manually. directory.make_dir(self.job_config.job_id) job_data_dir = os.path.join(str(directory.url), self.job_config.job_id) except saga.NoSuccess as e: LOG.error('The specified job data directory already exists on ' 'PBS submission node <%s> (%s).' % (host, str(e))) raise JobError('The specified job directory already exists on PBS' 'submission node <%s> (%s)' % (host, str(e))) # Now upload the file(s) to the job data directory # and create an input file list containing the resulting locations # of the files. # There are some cases where jobs may not have input files (they may, # for example pull the input files from a remote location as part of # the job process) so we first check whether there are any input files # to process, if not, then return from this function if not self.job_config.input_files: LOG.debug('There are no input files to transfer for this job...') return self.transferred_input_files = [] for f in self.job_config.input_files: try: f_obj = File('file://%s' % f, session=self.session) f_obj.copy(job_data_dir) dest_dir = os.path.join(directory.url.path, self.job_config.job_id) self.transferred_input_files.append( os.path.join(dest_dir, os.path.basename(f_obj.url.path))) except: LOG.error('Error copying the input file <%s> to the remote ' 'platform.' % f) raise JobError('Error copying the input file <%s> to the ' 'remote platform.' % f)
def run_job(self): JobDeploymentBase.run_job(self) # TODO: Should this be running/managing the job remotely via a SAGA # SSH session or should we be expecting to communicate with a remote # resource management service to handle this? # This function could use the libhpc resource daemon client to talk to # a resource daemon that is installed on the target resource, however, # at present we simply use SSH (via SAGA Python) to trigger job # execution and handle compressing and returning the output files. LOG.debug('SSH Deployer: Run job...') job_arguments = getattr(self.job_config, 'args', []) input_files = getattr(self, 'transferred_input_files', []) job_arguments += input_files # Check if we have a JOB_ID variable in the arguments or input files. # If so, replace this variable with the actual job ID. job_arguments_tmp = job_arguments job_arguments = [] for item in job_arguments_tmp: # Can't do a replace on items that are not string types! if isinstance(item, basestring): job_arguments.append( item.replace('$JOB_ID', self.job_config.job_id)) else: job_arguments.append(item) LOG.debug('Modified job arguments: %s' % job_arguments) jd = saga.job.Description() jd.environment = getattr(self.job_config, 'environment', {}) jd.executable = getattr(self.job_config, 'executable', None) jd.arguments = job_arguments jd.working_directory = getattr(self.job_config, 'working_dir', None) jd.output = getattr(self.job_config, 'stdout', None) jd.error = getattr(self.job_config, 'stderr', None) jd.wall_time_limit = getattr(self.job_config, 'time_limit_mins', 0) if not jd.output: jd.output = 'std.out' if not jd.error: jd.error = 'std.err' self.svc = saga.job.Service('ssh://%s/' % self.host, session=self.session) self.job = self.svc.create_job(jd) self.job.run()
def shutdown_resources(self): JobDeploymentBase.shutdown_resources(self) # Number of seconds between chceking for shutdown of resources. SHUTDOWN_POLL_DELAY = 4 # Here we terminate the running resources for this job and # wait until they have been shut down. res_ids = [node.id for node in self.nodes] LOG.debug('About to shut down the following nodes: %s' % res_ids) LOG.debug('Shutdown resources...') for node in self.nodes: self.driver.destroy_node(node) while res_ids: # TODO: Find a better approach to remove nodes that have vanished # from the system, at present we need to manually go through each # node to identify individual nodes that are no longer accessible. try: nodes_to_wait_for = self._get_node_list(res_ids) except Exception as e: LOG.debug('Exception <%s> getting node list, getting node info' ' individually.' % str(e)) nodes_to_wait_for = self._get_node_list(res_ids, manual=True) still_running = [] for node_info in nodes_to_wait_for: if node_info.state != NodeState.TERMINATED: still_running.append(node_info.id) new_res_ids = [] # Now go through res_ids and delete the nodes that don't appear # in still_running. for res_id in res_ids: if res_id not in still_running: LOG.debug('Resource <%s> has terminated...' % res_id) else: new_res_ids.append(res_id) res_ids = new_res_ids if res_ids: LOG.debug('Still waiting for termination of resources %s...' % res_ids) time.sleep(SHUTDOWN_POLL_DELAY) LOG.debug('All resources terminated.')
def run_job(self): JobDeploymentBase.run_job(self) # This function uses the libhpc resource daemon client to talk to the # resource daemon that is installed on cloud resources. It uses this # interface to run jobs and monitor their state to see when they are # complete. # TODO: Should this be running/managing the job remotely via a SAGA # SSH session or should we be expecting to communicate with a remote # resource management service to handle this? LOG.debug('Run job...') job_arguments = getattr(self.job_config, 'args', []) input_files = getattr(self, 'transferred_input_files', []) job_arguments += input_files jd = Description() jd.environment = getattr(self.job_config, 'environment', {}) if self.job_config.num_processes > 1: jd.executable = ('mpirun -np %s -machinefile /tmp/machinefile' % (self.job_config.num_processes)) executable = getattr(self.job_config, 'executable', None) if executable: job_arguments.insert(0, executable) else: jd.executable = getattr(self.job_config, 'executable', None) jd.arguments = job_arguments jd.working_directory = getattr(self.job_config, 'working_dir', None) jd.output = getattr(self.job_config, 'stdout', None) jd.error = getattr(self.job_config, 'stderr', None) jd.wall_time_limit = getattr(self.job_config, 'time_limit_mins', 0) #jd.number_of_processes = 4 #jd.processes_per_host = 1 #jd.total_physical_memory = "2400" if not jd.output: jd.output = 'std.out' if not jd.error: jd.error = 'std.err' self.svc = Service('ssh://%s/' % self.running_nodes[0][0].public_ips[0], session=self.session) self.job = self.svc.create_job(jd) self.job.run()
def collect_output(self, destination): # We're using the default implementation of the file transfer code # This doesn't take into account a different port for the remote host # connection. To work around this, we temporarily set the host property # to include the port and the revert to the original value after the # file transfer is complete. host_tmp = self.host self.host = ('%s:%s' % (self.host, self.port)) # Using the base implementation of job output file collection... JobDeploymentBase.collect_output(self, destination) # If job_config delete_job_files is True, we can now delete the job # files on the remote platform if self.job_config.delete_job_files: jobs_dir = self.platform_config.storage_job_directory # Check that the job storage directory exists and then create a # sub-directory specifically for this job. try: LOG.debug('URL for file job directory: sftp://%s%s' % (self.host, jobs_dir)) directory = Directory('sftp://%s%s' % (self.host, jobs_dir), session=self.session) except saga.BadParameter as e: LOG.error('The specified job directory does not exist on ' 'resource <%s> (%s).' % (self.host, str(e))) raise JobError('The specified job directory does not exist ' 'on resource <%s> (%s)' % (self.host, str(e))) try: LOG.debug('Deleting job directory after job completion ' '<sftp://%s%s/%s>' % (self.host, jobs_dir, self.job_config.job_id)) directory.remove(self.job_config.job_id, RECURSIVE) except saga.NoSuccess as e: LOG.error('The specified job data directory couldn\'t be ' 'removed <%s> (%s).' % (self.job_config.job_id, str(e))) raise JobError('The specified job data directory couldn\'t be ' 'removed <%s> (%s)' % (self.job_config.job_id, str(e))) # Set the host value back to its original value self.host = host_tmp
def collect_output(self, destination): # Before calling the base implementation of output file collection to # pull files back from the master node, we first need to gather output # from each of the slave nodes onto the master node LOG.debug('Gather files from slave nodes to master...') job_dir = self.platform_config.storage_job_directory # If we have only one node then we can skip this stage... master_node = self.running_nodes[0][0] slave_nodes = [] if len(self.running_nodes) > 1: slave_nodes = [node[0] for node in self.running_nodes[1:]] if slave_nodes: slave_private_ips = [node.private_ips[0] for node in slave_nodes] self._gather_results_data(master_node.public_ips[0], slave_private_ips, self.platform_config.user_id, self.platform_config.user_key_file, job_dir, self.job_config.job_id) # Using the base implementation of job output file collection... JobDeploymentBase.collect_output(self, destination)
def transfer_files(self): JobDeploymentBase.transfer_files(self)
def deploy_software(self): JobDeploymentBase.deploy_software(self)
def initialise_resources(self, resource_config=None, num_resources=1, resource_type='m1.small', job_id=None): JobDeploymentBase.initialise_resources(self) # Start up the cloud resources here and wait for them to reach the # running state. Need to know the image ID that we're starting. The # image ID is available from the job configuration image_id = None image_id_configured = self.job_config.image_id_pre_configured image_id_unconfigured = self.job_config.image_id_unconfigured if image_id_configured and not image_id_unconfigured: image_id = image_id_configured LOG.debug('Only a configured image identifier has been provided, ' 'using image ID <%s>.' % image_id) elif (not image_id_configured) and image_id_unconfigured: if not resource_config: LOG.error('Only an unconfigured image ID provided but ' 'no resource configuration has been provided.') raise ResourceInitialisationError('ERROR: Only an unconfigured ' 'image type is available but no image ' 'configuration has been provided.') image_id = image_id_unconfigured LOG.debug('Only an unconfigured image identifier has been ' 'provided, using image ID <%s>.' % image_id) elif image_id_configured and image_id_unconfigured: image_id = image_id_unconfigured if resource_config else image_id_configured LOG.debug('Both configured and unconfigured images provided, ' 'using image ID <%s>.' % image_id) else: raise ResourceInitialisationError('ERROR: No image information ' 'available in the platform configuration, unable ' 'to initialise resources.') # Check that the image is present and then use the libcloud driver to # start the resources and return once they're running. # TODO: This is currently synchronous but could also be done # asynchronously using a callback to notify the caller when the nodes # are ready. #images = self.driver.list_images() #img = next((i for i in images if i.id == image_id), None) #if not img: try: img = self.driver.get_image(image_id) except socket.error as e: img = None raise ResourceInitialisationError('ERROR contacting the remote ' 'cloud platform. Do you have an active network ' 'connection? - <%s>' % str(e)) except: img = None raise ResourceInitialisationError('ERROR: The specified image <%s> ' 'is not present on the target platform, unable ' 'to start resources.' % image_id) sizes = self.driver.list_sizes() size = next((s for s in sizes if s.name == resource_type), None) if not size: raise ResourceInitialisationError('ERROR: The specified resource ' 'size <%s> is not present on the target platform. ' 'Unable to start resources.' % resource_type) # Get the keypair name from the configuration keypair_name = self.job_config.key_name # At this point we know that the image is available and the specified # resource type is valid so we can request to start the instance(s) LOG.debug('About to start <%s> resources of type <%s> based on image ' '<%s (%s)> with keypair <%s>.' % (num_resources, size.name, img.id, img.name, keypair_name)) # When starting a resource we need the name, image, type, keypair, # configuration data and details of the number of resources to start. name = job_id if not name: name = generate_instance_id() self.driver.create_node(name=name, image=img, size=size, ex_keyname=keypair_name) return
def initialise_resources(self, prefer_unconfigured=True, num_processes=1, processes_per_node=1, node_type='m1.small', job_id=None, retries=3, software_config=None): JobDeploymentBase.initialise_resources(self) # Start up the cloud resources here and wait for them to reach the # running state. Need to know the image ID that we're starting. The # image ID is available from the job configuration image_id = None image_preconfigured_id = self.platform_config.image_preconfigured_id image_unconfigured_id = self.platform_config.image_unconfigured_id # Store whether or not we're using an unconfigured image - this # determines whether we end up running the deploy software function # or not. self.use_unconfigured = False if image_preconfigured_id and not image_unconfigured_id: image_id = image_preconfigured_id LOG.debug('Only a configured image identifier has been provided, ' 'using image ID <%s>.' % image_id) elif (not image_preconfigured_id) and image_unconfigured_id: image_id = image_unconfigured_id self.use_unconfigured = True LOG.debug('Only an unconfigured image identifier has been ' 'provided, using image ID <%s>.' % image_id) if not software_config: raise JobError( 'Only an unconfigured image identifier has been ' 'provided but no software config has been specified. ' 'Unable to continue...') elif image_preconfigured_id and image_unconfigured_id: LOG.debug('Both configured and unconfigured images provided...') if prefer_unconfigured: image_id = image_unconfigured_id self.use_unconfigured = True LOG.debug('Using unconfigured image ID <%s>.' % image_id) if not software_config: raise JobError( 'An unconfigured image identifier has been ' 'chosen but no software config has been specified. ' 'Unable to continue...') else: image_id = image_preconfigured_id LOG.debug('Using pre-configured image ID <%s>.' % image_id) else: raise ResourceInitialisationError( 'ERROR: No image information ' 'available in the platform configuration, unable ' 'to initialise resources.') # If we're using an unconfigured image, we need to prepare the admin # security context based on the information that should be provided # in the YAML file with the unconfigured image details. if self.use_unconfigured: self.admin_ctx = saga.Context("ssh") self.admin_ctx.user_id = self.platform_config.image_unconfigured_admin_key_user self.admin_ctx.user_key = self.platform_config.image_unconfigured_admin_key_file # Check that the image is present and then use the libcloud driver to # start the resources and return once they're running. # TODO: This is currently synchronous but could also be done # asynchronously using a callback to notify the caller when the nodes # are ready. #images = self.driver.list_images() #img = next((i for i in images if i.id == image_id), None) #if not img: img = None try: #img = self.driver.get_image(image_id) images = self.driver.list_images() for image in images: if image.id == image_id: img = image break if img == None: raise ResourceInitialisationError('The specified image <%s> ' 'could not be found' % image_id) except socket.error as e: img = None raise ResourceInitialisationError( 'ERROR contacting the remote ' 'cloud platform. Do you have an active network ' 'connection? - <%s>' % str(e)) except Exception as e: LOG.debug('ERROR STRING: %s' % str(e)) img = None if str(e).startswith('Unauthorized:'): raise InvalidCredentialsError( 'ERROR: Access to the cloud ' 'platform at <%s> was not authorised. Are your ' 'credentials correct?' % (self.platform_config.platform_service_host + ':' + str(self.platform_config.platform_service_port))) else: raise ResourceInitialisationError( 'ERROR: The specified image <%s> ' 'is not present on the target platform, unable ' 'to start resources.' % image_id) sizes = self.driver.list_sizes() size = next((s for s in sizes if s.id == node_type), None) if not size: raise ResourceInitialisationError( 'ERROR: The specified resource ' 'size (node_type) <%s> is not present on the ' 'target platform. Unable to start resources. Have ' 'you set the node_type parameter in your job spec?' % node_type) # Get the keypair name from the configuration # If we're using an unconfigured resource, we use the admin key pair # name if provided. if self.use_unconfigured and self.platform_config.image_unconfigured_admin_key_name: keypair_name = self.platform_config.image_unconfigured_admin_key_name else: keypair_name = self.platform_config.user_key_name # Get the number of resources from the job configuration # TODO: Fix this to obtain number of cores per node from the cloud # cloud platform. For now use the specified processes_per_node in the # job specification. cores_per_node = processes_per_node #cores_per_node = self.RESOURCE_TYPE_CORES[node_type] #if cores_per_node < processes_per_node: # LOG.debug('A processes_per_node value <%s> greater than the number ' # 'of cores in a node <%s> has been specified. Altering ' # 'processes per node to the maximum available on this ' # 'node type <%s>.' % (processes_per_node, cores_per_node, # node_type)) # processes_per_node = cores_per_node num_nodes = int(ceil(float(num_processes) / float(processes_per_node))) # At this point we know that the image is available and the specified # resource type is valid so we can request to start the instance(s) LOG.debug('About to start <%s> resources of type <%s> based on image ' '<%s (%s)> with keypair <%s>.' % (num_nodes, size.name, img.id, img.name, keypair_name)) # When starting a resource we need the name, image, type, keypair, # configuration data and details of the number of resources to start. name = job_id if not name: name = generate_instance_id() self.nodes = self.driver.create_node(name=name, image=img, size=size, ex_keyname=keypair_name, ex_mincount=num_nodes, ex_maxcount=num_nodes) if type(self.nodes) != type([]): self.nodes = [self.nodes] self.running_nodes = self.driver.wait_until_running(self.nodes) # Before we return details of the running nodes, we need to check # that they're accessible - it takes some time for the nodes to boot # and become available. We do this by setting up a handle to a # directory - we assume all nodes have a '/' directory - and then # trying to list that directory. If an exception is thrown, we assume # that the nodes are not yet available. # TODO: Need to replace this wait with a reliable check as to whether # the server is up and running. Looks like, for now, this will need to # use Paramiko while awaiting updates on saga-python. #LOG.debug('Waiting 60 seconds for node to boot...') #time.sleep(60) # Replaced 60 second wait with check using Paramiko to see if # resource is accessible... LOG.debug('Checking node is available...') nodes_to_check = [] for node in self.running_nodes: nodes_to_check.append(node[0].public_ips[0]) res = self._wait_for_node_accessbility( nodes_to_check, self.platform_config.user_id, self.platform_config.user_key_file, retries=retries) if not res: # We still have nodes that are not avialable so assume there's a # problem and throw a job error. raise JobError('After <%s> retries, the following nodes are ' 'still not accessible <%s>. Cancelling job.' % (retries, nodes_to_check)) # If we have multiple nodes, now is the time to create the machinefile # for MPI job runs # For the machinefile we need the private IP of each node and the # number of cores. machinefile = tempfile.NamedTemporaryFile('w', delete=True) machinefile.write("# Machine file for MPI job runs\n") for node in self.running_nodes: machinefile.write( '%s slots=%s max_slots=%s\n' % (node[0].private_ips[0], cores_per_node, cores_per_node)) machinefile.flush() LOG.debug('The following machinefile has been created:\n\n%s\n' % machinefile.name) # The master node is always considered to be node 0 in # the self.running_nodes list. LOG.debug('Copying machinefile to master node...') saga_machinefile = File('file://%s' % machinefile.name, session=self.session) saga_machinefile.copy('sftp://%s/tmp/machinefile' % self.running_nodes[0][0].public_ips[0]) machinefile.close() LOG.debug('machinefile copied to master node...') conn = PTYShell('ssh://%s' % self.running_nodes[0][0].public_ips[0], session=self.session) conn.run_sync('chmod 644 /tmp/machinefile') LOG.debug('Set permissions on /tmp/machinefile on master node to 644.') return self.running_nodes
def collect_output(self, destination): JobDeploymentBase.collect_output(self, destination)
def deploy_software(self, software_config=None): JobDeploymentBase.deploy_software(self) # Here we undertake transfer of the code to the remote platform if this # is required. In many cases, the software is likely to already be # deployed on the target platform or may have been configured via a # tool such as cloud-init, puppet, etc at resource initialisation time. LOG.debug('Deploy software...') # If we're not using an unconfigured image, we don't need to run the # deploy software function if not self.use_unconfigured: LOG.info('Using a pre-configured image so running software ' 'deployment process...') return # Software deployment requires root access to the target node(s). This # should be possible using the key that has been passed to start the # reosurce(s). # If no software configuration is provided, we ignore this function # call and return. If a configuration is provided, we check that the # configuration is for the right type of platform and then deploy # the software. if not software_config: return if type(software_config) != type([]): software_config = [software_config] LOG.debug('Received a request to deploy the following software ' 'configuration IDs to the target platforms: <%s>...' % software_config) # Check that we have an admin security context available. If we don't # we can't connect to the remote resource(s) to do the required # configuration if not self.admin_ctx: raise JobError( 'deploy_software: There is no admin context ' 'available so it will not be possible to connect ' 'to remote resources to configure them. Ensure that ' '') # Check that we can get each of the software configurations and that # each one supports the target deployment platform. scm = SoftwareConfigManager.get_instance() scm.init_configuration() os_name = self.platform_config.image_unconfigured_os flavour = self.platform_config.image_unconfigured_flavour admin_key_user = self.platform_config.image_unconfigured_admin_key_user admin_key_file = self.platform_config.image_unconfigured_admin_key_file sc_dict = {} for sc in software_config: try: conf = scm.get_software_configuration(sc) sc_dict[sc] = conf except ValueError as e: raise JobError('Job error - no software could be found for ' 'the configuration id <%s>: %s' % (sc, str(e))) if not ((os_name == conf.software_os_type) and (flavour == conf.software_os_flavour)): LOG.error( 'The OS <%s> and flavour <%s> in the provided software ' 'configuration don\'t match the target platform with ' 'OS <%s> and flavour <%s>.' % (conf.software_os_type, conf.software_os_flavour, os_name, flavour)) raise JobError( 'The OS <%s> and flavour <%s> in the provided ' 'software configuration don\'t match the target ' 'platform with OS <%s> and flavour <%s>.' % (conf.software_os_type, conf.software_os_flavour, os_name, flavour)) # If we reach this point we assume that each of the software # configurations has been found and they are for the right target # platform. for sc_key in sc_dict.keys(): sc_obj = sc_dict[sc_key] install_commands = sc_obj.get_install_commands() # Now run each of the install commands synchronously on all of the # target machines to get the software installed. node_ips = [node[0].public_ips[0] for node in self.running_nodes] LOG.debug('Deploying to the following list of nodes: %s' % node_ips) # Set up a new session using the admin user and key provided for # the unconfigured image. adm_session = saga.Session(default=False) adm_ctx = saga.Context("ssh") adm_ctx.user_id = admin_key_user adm_ctx.user_key = admin_key_file adm_session.add_context(adm_ctx) # Open shell connections to each of the machines shell_conns = [] opts = {} opts['ssh_options'] = {'StrictHostKeyChecking': 'no'} for node_ip in node_ips: conn = PTYShell('ssh://%s' % node_ip, session=adm_session, opts=opts) shell_conns.append(conn) if conf.software_os_type == 'linux': self._setup_job_account(conn, self.platform_config) else: LOG.warning( 'Support for creation of job accounts on ' 'platforms other than linux is not yet supported...') # Copy the job account key to the master node job_session = saga.Session(default=False) job_session.add_context(self.job_ctx) keyfile = File('file://%s' % self.platform_config.user_key_file, session=job_session) keyfile_target = shell_conns[0].url + os.path.join( self.platform_config.user_home, '.ssh', 'id_rsa') LOG.debug('Copying job key to target directory <%s>' % keyfile_target) keyfile.copy(keyfile_target) for cmd in install_commands: for shell_connection in shell_conns: if isinstance(cmd, SoftwareConfigFile): LOG.debug( 'Software deployment: About to write data to ' 'remote file <%s> on node <%s>' % (cmd.filename, shell_connection.url)) shell_connection.write_to_remote( cmd.data, cmd.filename) else: LOG.debug('Software deployment: About to run command ' '<%s> on resource <%s>...' % (cmd, shell_connection.url)) if admin_key_user != 'root': cmd = 'sudo ' + cmd result, out, err = shell_connection.run_sync(cmd) LOG.debug('Command completed - Exit code: <%s>, ' 'StdOut: <%s>, StdErr:\n<%s>' % (result, out, err))
def transfer_files(self): JobDeploymentBase.transfer_files(self) # Here we transfer any input files to the relevant directory on the # target platform. # Use SAGA-Python to handle the file transfer. LOG.debug('Transfer files...') job_dir = self.platform_config.storage_job_directory # At this point we need to switch back to using the job secruity # context. If we were using unconfigured resources, these will have # been configured using an admin context by now. self.session = saga.Session(default=False) self.session.add_context(self.job_ctx) # Begin by checking if we're working with more than one instance, if # so we have a master and one or more slave nodes. We'll push the data # to the master and then direct the master to distribute it to the # slave nodes. master_node = self.running_nodes[0][0] slave_nodes = [] if len(self.running_nodes) > 1: slave_nodes = [node[0] for node in self.running_nodes[1:]] # On the master node: Check that the job storage directory exists and # then create a sub-directory specifically for this job. # Node is a tuple consisting of two items, the node object and an # IP list. For now we work with the node object directly. node_ip = master_node.public_ips[0] try: directory = Directory('sftp://%s%s' % (node_ip, job_dir), session=self.session) except saga.BadParameter as e: LOG.error('The specified job directory does not exist on node ' '<%s> (%s).' % (node_ip, str(e))) #raise JobError('The specified job directory does not exist ' # 'on node <%s> (%s)' % (node_ip, str(e))) try: # directory.make_dir() does not return a handle to the new directory # so need to create the directory URL manually. directory.make_dir(self.job_config.job_id) except saga.NoSuccess as e: LOG.warning('The specified job data directory already exists on ' 'node <%s> (%s).' % (node_ip, str(e))) #raise JobError('The specified job directory already exists on ' # 'on node <%s> (%s)' % (node_ip, str(e))) job_data_dir = os.path.join(str(directory.url), self.job_config.job_id) # Now upload the file(s) to the job data directory # and create an input file list containing the resulting locations # of the files. # There are some cases where jobs may not have input files (they may, # for example pull the input files from a remote location as part of # the job process) so we first check whether there are any input files # to process, if not, then return from this function if not self.job_config.input_files: LOG.debug('There are no input files to transfer for this job...') return self.transferred_input_files = [] for f in self.job_config.input_files: try: f_obj = File('file://%s' % f, session=self.session) f_obj.copy(job_data_dir) dest_dir = os.path.join(directory.url.path, self.job_config.job_id) self.transferred_input_files.append( os.path.join(dest_dir, os.path.basename(f_obj.url.path))) except: LOG.error('Error copying the input file <%s> to the remote ' 'platform.' % f) raise JobError('Error copying the input file <%s> to the ' 'remote platform.' % f) # At this point input files have been successfully transferred to # the master node. We now direct the master node to send the files # to each of the slave nodes: if slave_nodes: slave_private_ips = [node.private_ips[0] for node in slave_nodes] self._distribute_job_data(master_node.public_ips[0], slave_private_ips, self.platform_config.user_id, self.platform_config.user_key_file, job_dir, self.job_config.job_id)
def deploy_software(self, *args, **kwargs): JobDeploymentBase.deploy_software(self)
def initialise_resources(self, *args, **kwargs): JobDeploymentBase.initialise_resources(self) LOG.debug('SSH Deployer: Initialise resources - Nothing to do here...') return None
def transfer_files(self): JobDeploymentBase.transfer_files(self) LOG.debug('SSH Deployer: Transfer files...') # Here we transfer any input files to the relevant directory on the # target platform. # Use SAGA-Python to handle the file transfer. job_dir = self.platform_config.storage_job_directory # Check that the job storage directory exists and then create a # sub-directory specifically for this job. try: LOG.debug('URL for file transfer: <sftp://%s:%s%s>' % (self.host, self.port, job_dir)) directory = Directory('sftp://%s:%s%s' % (self.host, self.port, job_dir), session=self.session) except saga.BadParameter as e: LOG.error('Error setting up connection to resource directory.') if 'connection refused' in str(e).lower(): raise ConnectionError('Unable to connect to remote resource ' 'to set up connection to directory.') raise StorageDirectoryNotFoundError( 'The specified job data base ' 'directory does not exist on resource <%s> (%s)' % (self.host, str(e))) try: # directory.make_dir() does not return a handle to the new directory # so need to create the directory URL manually. directory.make_dir(self.job_config.job_id) job_data_dir = os.path.join(str(directory.url), self.job_config.job_id) except saga.NoSuccess as e: LOG.error('The specified job data directory already exists on ' 'resource <%s> (%s).' % (self.host, str(e))) raise DirectoryExistsError('The specified job directory already ' 'exists on resource <%s> (%s)' % (self.host, str(e))) # Now upload the file(s) to the job data directory # and create an input file list containing the resulting locations # of the files. # There are some cases where jobs may not have input files (they may, # for example pull the input files from a remote location as part of # the job process) so we first check whether there are any input files # to process, if not, then return from this function if not self.job_config.input_files: LOG.debug('There are no input files to transfer for this job...') return self.transferred_input_files = [] for f in self.job_config.input_files: try: f_obj = File('file://%s' % f, session=self.session) f_obj.copy(job_data_dir) dest_dir = os.path.join(directory.url.path, self.job_config.job_id) self.transferred_input_files.append( os.path.join(dest_dir, os.path.basename(f_obj.url.path))) except: LOG.error('Error copying the input file <%s> to the remote ' 'platform.' % f) raise JobError('Error copying the input file <%s> to the ' 'remote platform.' % f)
def run_job(self, job_details=None): JobDeploymentBase.run_job(self)
def collect_output(self): JobDeploymentBase.collect_output(self)
def shutdown_resources(self): JobDeploymentBase.shutdown_resources(self) # Here we collect terminate the running resources for this job and # wait until they have been shut down. LOG.debug('SSH Deployer: Shutdown resources - nothing to do here.')