def __init__(self,
resource_id=None,
login=None,
password=None,
config=None,
rsa_key_pass=None):
'''
Sets up the connection to the computing resource.
Looks for a soma-workflow configuration file (if not specified in the
*config* argument).
* resource_id *string*
Identifier of the computing resource to connect to.
If None, the number of cpu of the current machine is detected and the basic scheduler is lauched.
* login *string*
Required if the computing resource is remote.
* password *string*
Required if the computing resource is remote and not RSA key where
configured to log on the remote machine with ssh.
* config *configuration.Configuration*
Optional configuration.
* rsa_key_pass *string*
Required if the RSA key is protected with a password.
.. note::
The login and password are only required for a remote computing resource.
'''
if config == None:
self.config = configuration.Configuration.load_from_file(resource_id)
else:
self.config = config
if password == '':
password =None
self.scheduler_config = None
mode = self.config.get_mode()
# print mode + " mode"
self._resource_id = resource_id
# LOCAL MODE
if mode == configuration.LOCAL_MODE:
self._connection = connection.LocalConnection(resource_id, "")
self._engine_proxy = self._connection.get_workflow_engine()
self.engine_config_proxy = self._connection.get_configuration()
self._transfer = TransferLocal(self._engine_proxy)
self._transfer_stdouterr = TransferLocal(self._engine_proxy)
# REMOTE MODE
elif mode == configuration.REMOTE_MODE:
submitting_machines = self.config.get_submitting_machines()
sub_machine = submitting_machines[random.randint(0,
len(submitting_machines)-1)]
cluster_address = self.config.get_cluster_address()
print 'cluster address: ' + cluster_address + ', submission machine: ' + sub_machine
self._connection = connection.RemoteConnection(login,
password,
cluster_address,
sub_machine,
resource_id,
"",
rsa_key_pass)
self._engine_proxy = self._connection.get_workflow_engine()
self.engine_config_proxy = self._connection.get_configuration()
if not password and not rsa_key_pass:
self._transfer = TransferSCP(self._engine_proxy,
username=login,
hostname=sub_machine)
else:
self._transfer = PortableRemoteTransfer(self._engine_proxy)
self._transfer_stdouterr = PortableRemoteTransfer(self._engine_proxy)
# LIGHT MODE
elif mode == configuration.LIGHT_MODE:
local_scdl_cfg_path = configuration.LocalSchedulerCfg.search_config_path()
if local_scdl_cfg_path == None:
cpu_count = Helper.cpu_count()
self.scheduler_config = configuration.LocalSchedulerCfg(proc_nb=cpu_count)
else:
self.scheduler_config = configuration.LocalSchedulerCfg.load_from_file(local_scdl_cfg_path)
self._engine_proxy = _embedded_engine_and_server(self.config,
self.scheduler_config)
self.engine_config_proxy = self.config
self._connection = None
self._transfer = TransferLocal(self._engine_proxy)
self._transfer_stdouterr = TransferLocal(self._engine_proxy)
self._transfer_monitoring = TransferMonitoring(self._engine_proxy)
class WorkflowController(object):
'''
Submission, control and monitoring of Job, FileTransfer and Workflow
objects.
'''
_connection = None
_engine_proxy = None
_transfer = None
_transfer_stdouterr = None
config = None
engine_config_proxy = None
_resource_id = None
scheduler_config = None
def __init__(self,
resource_id=None,
login=None,
password=None,
config=None,
rsa_key_pass=None):
'''
Sets up the connection to the computing resource.
Looks for a soma-workflow configuration file (if not specified in the
*config* argument).
* resource_id *string*
Identifier of the computing resource to connect to.
If None, the number of cpu of the current machine is detected and the basic scheduler is lauched.
* login *string*
Required if the computing resource is remote.
* password *string*
Required if the computing resource is remote and not RSA key where
configured to log on the remote machine with ssh.
* config *configuration.Configuration*
Optional configuration.
* rsa_key_pass *string*
Required if the RSA key is protected with a password.
.. note::
The login and password are only required for a remote computing resource.
'''
if config == None:
self.config = configuration.Configuration.load_from_file(resource_id)
else:
self.config = config
if password == '':
password =None
self.scheduler_config = None
mode = self.config.get_mode()
# print mode + " mode"
self._resource_id = resource_id
# LOCAL MODE
if mode == configuration.LOCAL_MODE:
self._connection = connection.LocalConnection(resource_id, "")
self._engine_proxy = self._connection.get_workflow_engine()
self.engine_config_proxy = self._connection.get_configuration()
self._transfer = TransferLocal(self._engine_proxy)
self._transfer_stdouterr = TransferLocal(self._engine_proxy)
# REMOTE MODE
elif mode == configuration.REMOTE_MODE:
submitting_machines = self.config.get_submitting_machines()
sub_machine = submitting_machines[random.randint(0,
len(submitting_machines)-1)]
cluster_address = self.config.get_cluster_address()
print 'cluster address: ' + cluster_address + ', submission machine: ' + sub_machine
self._connection = connection.RemoteConnection(login,
password,
cluster_address,
sub_machine,
resource_id,
"",
rsa_key_pass)
self._engine_proxy = self._connection.get_workflow_engine()
self.engine_config_proxy = self._connection.get_configuration()
if not password and not rsa_key_pass:
self._transfer = TransferSCP(self._engine_proxy,
username=login,
hostname=sub_machine)
else:
self._transfer = PortableRemoteTransfer(self._engine_proxy)
self._transfer_stdouterr = PortableRemoteTransfer(self._engine_proxy)
# LIGHT MODE
elif mode == configuration.LIGHT_MODE:
local_scdl_cfg_path = configuration.LocalSchedulerCfg.search_config_path()
if local_scdl_cfg_path == None:
cpu_count = Helper.cpu_count()
self.scheduler_config = configuration.LocalSchedulerCfg(proc_nb=cpu_count)
else:
self.scheduler_config = configuration.LocalSchedulerCfg.load_from_file(local_scdl_cfg_path)
self._engine_proxy = _embedded_engine_and_server(self.config,
self.scheduler_config)
self.engine_config_proxy = self.config
self._connection = None
self._transfer = TransferLocal(self._engine_proxy)
self._transfer_stdouterr = TransferLocal(self._engine_proxy)
self._transfer_monitoring = TransferMonitoring(self._engine_proxy)
def disconnect(self):
'''
Simulates a disconnection for TEST PURPOSE ONLY.
!!! The current instance will not be usable anymore after this call !!!!
'''
if self._connection:
self._connection.stop()
########## SUBMISSION / REGISTRATION ####################################
def submit_workflow(self,
workflow,
expiration_date=None,
name=None,
queue=None):
'''
Submits a workflow and returns a workflow identifier.
* workflow *client.Workflow*
Workflow description.
* expiration_date *datetime.datetime*
After this date the workflow will be deleted.
* name *string*
Optional workflow name.
* queue *string*
Optional name of the queue where to submit jobs. If it is not specified
the jobs will be submitted to the default queue.
* returns: *int*
Workflow identifier.
Raises *WorkflowError* or *JobError* if the workflow is not correct.
'''
if self.engine_config_proxy.get_scheduler_type() == configuration.MPI_SCHEDULER:
raise SomaWorkflowError("The MPI scheduler is configured for this resource. "
"Use soma_workflow.MPI_workflow_runner to submit a workflow using the MPI scheduler.")
#cProfile.runctx("wf_id = self._engine_proxy.submit_workflow(workflow, expiration_date, name, queue)", globals(), locals(), "/home/soizic/profile/profile_submit_workflow")
wf_id = self._engine_proxy.submit_workflow(workflow,
expiration_date,
name,
queue)
return wf_id
def submit_job(self, job, queue=None):
'''
**deprecated since version 2.4:** Use submit_workflow instead.
Submits a job which is not part of a workflow.
Returns a job identifier.
If the job used transfered files the list of involved file transfer **must
be** specified setting the arguments: *referenced_input_files* and
*referenced_output_files*.
Each path must be reachable from the computing resource.
* job *client.Job*
* queue *string*
Name of the queue where to submit the jobs. If it is not
specified the job will be submitted to the default queue.
* returns: int
Job identifier.
Raises *JobError* if the job is not correct.
'''
print "The method submit_job is deprecated since version 2.4. Use submit_workflow instead."
if self.engine_config_proxy.get_scheduler_type() == configuration.MPI_SCHEDULER:
raise SomaWorkflowError("The MPI scheduler is configured for this resource. "
"Use soma_workflow.MPI_workflow_runner to submit a workflow using the MPI scheduler.")
job_id = self._engine_proxy.submit_job(job, queue)
return job_id
def register_transfer(self, file_transfer):
'''
Registers a file transfer which is not part of a workflow and returns a
file transfer identifier.
* file_transfer *client.FileTransfer*
* returns *EngineTransfer*
'''
engine_transfer = self._engine_proxy.register_transfer(file_transfer)
return engine_transfer
########## WORKFLOWS, JOBS and FILE TRANSFERS RETRIEVAL ###################
def workflow(self, workflow_id):
'''
* workflow_id *workflow_identifier*
* returns: *Workflow*
Raises *UnknownObjectError* if the workflow_id is not valid
'''
return self._engine_proxy.workflow(workflow_id)
def workflows(self, workflow_ids=None):
'''
Lists the identifiers and general information about all the workflows
submitted by the user, or about the workflows specified in the
*workflow_ids* argument.
* workflow_ids *sequence of workflow identifiers*
* returns: *dictionary: workflow identifier -> tuple(date, string)*
workflow_id -> (workflow_name, expiration_date)
'''
return self._engine_proxy.workflows(workflow_ids)
def jobs(self, job_ids=None):
'''
Lists the identifiers and general information about all the jobs submitted
by the user and which are not part of a workflow, or about the jobs
specified in the *job_ids* argument.
* job_ids *sequence of job identifiers*
* returns: *dictionary: job identifiers -> tuple(string, string, date)*
job_id -> (name, command, submission date)
'''
return self._engine_proxy.jobs(job_ids)
def transfers(self, transfer_ids=None):
'''
Lists the identifiers and information about all the user's file transfers
which are not part of a workflow or about the file transfers specified in
the *transfer_ids* argument.
* transfer_ids *sequence of FileTransfer identifiers*
* returns: *dictionary: string -> tuple(string, date, None or sequence of string)*
transfer_id -> (
* client_path: client file or directory path
* expiration_date: after this date the file copied
on the computing resource and all the transfer
information will be deleted, unless an existing
job has declared this file as output or input.
* client_paths: sequence of file or directory path or None)
'''
return self._engine_proxy.transfers(transfer_ids)
########## WORKFLOW MONITORING #########################################
def workflow_status(self, workflow_id):
'''
* workflow_id *workflow identifier*
* returns: *string or None*
Status of the workflow: see :ref:`workflow-status` or the
constants.WORKFLOW_STATUS list.
Raises *UnknownObjectError* if the workflow_id is not valid
'''
return self._engine_proxy.workflow_status(workflow_id)
def workflow_elements_status(self, workflow_id):
'''
Gets back the status of all the workflow elements at once, minimizing the
communication with the server and request to the database.
TO DO => make it more user friendly.
* workflow_id *workflow identifier*
* returns: tuple (sequence of tuple (job_id, status, queue, exit_info,
(submission_date, execution_date, ending_date)), sequence of tuple
(transfer_id, (status, progression_info)), workflow_status, workflow_queue)
Raises *UnknownObjectError* if the workflow_id is not valid
'''
wf_status = self._engine_proxy.workflow_elements_status(workflow_id)
# special processing for transfer status:
new_transfer_status = []
for engine_path, client_path, client_paths, status, transfer_type in wf_status[1]:
progression = self._transfer_progression(status,
transfer_type,
client_path,
client_paths,
engine_path)
new_transfer_status.append((engine_path, (status, progression)))
new_wf_status = (wf_status[0], new_transfer_status, wf_status[2], wf_status[3], wf_status[4])
return new_wf_status
########## JOB MONITORING #############################################
def job_status( self, job_id ):
'''
* job_id *job identifier*
* returns: *string*
Status of the job: see :ref:`job-status` or the list
constants.JOB_STATUS.
Raises *UnknownObjectError* if the job_id is not valid
'''
return self._engine_proxy.job_status(job_id)
def job_termination_status(self, job_id ):
'''
Information related to the end of the job.
* job_id *job identifier*
* returns: *tuple(string, int or None, string or None, string) or None*
* exit status: status of the terminated job: see
:ref:`job-exit-status` or the constants.JOB_EXIT_STATUS list.
* exit value: operating system exit code of the job if the job
terminated normally.
* terminating signal: representation of the signal that caused the
termination of the job if the job terminated due to the receipt of
a signal.
* resource usage: resource usage information provided as an array of
strings where each string complies with the format <name>=<value>.
The information provided depends on the DRMS and DRMAA implementation.
Raises *UnknownObjectError* if the job_id is not valid
'''
return self._engine_proxy.job_termination_status(job_id)
def retrieve_job_stdouterr(self,
job_id,
stdout_file_path,
stderr_file_path = None,
buffer_size = 512**2):
'''
Copies the job standard output and error to specified file.
* job_id *job identifier*
* stdout_file_path *string*
Path of the file where to copy the standard output.
* stderr_file_path *string*
Path of the file where to copy the standard error.
* buffer_size *int*
The file is transfered piece by piece of size buffer_size.
Raises *UnknownObjectError* if the job_id is not valid
'''
stdout_file_path = os.path.abspath(stdout_file_path)
stderr_file_path = os.path.abspath(stderr_file_path)
(engine_stdout_file, engine_stderr_file) = self._engine_proxy.stdouterr_file_path(job_id)
self._transfer_stdouterr.transfer_from_remote(engine_stdout_file,
stdout_file_path)
self._transfer_stdouterr.transfer_from_remote(engine_stderr_file,
stderr_file_path)
########## FILE TRANSFER MONITORING ###################################
def transfer_status(self, transfer_id):
'''
File transfer status and information related to the transfer progress.
* transfer_id *transfer identifier*
* returns: *tuple(transfer_status or None, tuple or None)*
* Status of the file transfer : see :ref:`file-transfer-status` or the
constants.FILE_TRANSFER_STATUS list.
* None if the transfer status in not
constants.TRANSFERING_FROM_CLIENT_TO_CR or
constants.TRANSFERING_FROM_CR_TO_CLIENT.
tuple (file size, size already transfered) if it is a file transfer.
tuple (cumulated size, sequence of tuple (relative_path, file_size, size already transfered) if it is a directory transfer.
Raises *UnknownObjectError* if the transfer_id is not valid
'''
(transfer_id,
client_path,
expiration_date,
workflow_id,
client_paths,
transfer_type,
status) = self._engine_proxy.transfer_information(transfer_id)
progression = self._transfer_progression(status,
transfer_type,
client_path,
client_paths,
transfer_id)
return (status, progression)
########## WORKFLOW CONTROL ############################################
def restart_workflow(self, workflow_id, queue=None):
'''
Restarts the jobs of the workflow which failed. The jobs will be submitted
again.
The workflow status has to be constants.WORKFLOW_DONE.
* workflow_id *workflow identifier*
* queue *string*
Optional name of the queue where to submit jobs. If it is not specified
the jobs will be submitted to the default queue.
* returns: *boolean*
True if some jobs were restarted.
Raises *UnknownObjectError* if the workflow_id is not valid
'''
if self.engine_config_proxy.get_scheduler_type() == configuration.MPI_SCHEDULER:
raise SomaWorkflowError("The MPI scheduler is configured for this resource. "
"Use soma_workflow.MPI_workflow_runner to restart a workflow using the MPI scheduler.")
return self._engine_proxy.restart_workflow(workflow_id, queue)
def delete_workflow(self, workflow_id, force=True):
'''
Deletes the workflow and all its associated elements (FileTransfers and
Jobs). The worklfow_id will become invalid and can not be used anymore.
The workflow jobs which are running will be killed.
If force is set to True: the call will block until the workflow is
deleted. With force set to True, if the workflow can not be deleted properly
it is deleted from Soma-workflow database. However, if some jobs are still
running they are not be killed. In this case the return value is False.
* workflow_id *workflow_identifier*
* force *boolean*
* returns: *boolean*
Raises *UnknownObjectError* if the workflow_id is not valid
'''
#cProfile.runctx("self._engine_proxy.delete_workflow(workflow_id)", globals(), locals(), "/home/soizic/profile/profile_delete_workflow")
return self._engine_proxy.delete_workflow(workflow_id, force)
def stop_workflow(self, workflow_id):
'''
Stops a workflow.
The running jobs will be killed.
The jobs in queues will be removed from queues.
It will be possible to restart the workflow afterwards.
* returns: *boolean*
return True if the running jobs were killed and False
if some jobs are possibly still running on the computing resource
despite the workflow was stopped.
'''
if self.engine_config_proxy.get_scheduler_type() == configuration.MPI_SCHEDULER:
raise SomaWorkflowError("The MPI scheduler is configured for this resource. "
"Kill the soma_workflow.MPI_workflow_runner job to stop the workflow.")
return self._engine_proxy.stop_workflow(workflow_id)
def change_workflow_expiration_date(self, workflow_id, new_expiration_date):
'''
Sets a new expiration date for the workflow.
* workflow_id *workflow identifier*
* new_expiration_date *datetime.datetime*
* returns: *boolean*
True if the expiration date was changed.
Raises *UnknownObjectError* if the workflow_id is not valid
'''
return self._engine_proxy.change_workflow_expiration_date(workflow_id, new_expiration_date)
########## JOB CONTROL #################################################
def wait_job( self, job_ids, timeout = -1):
'''
Waits for all the specified jobs to finish.
* job_ids *sequence of job identifier*
Jobs to wait for.
* timeout *int*
The call to wait_job exits before timeout seconds.
A negative value means that the method will wait indefinetely.
Raises *UnknownObjectError* if the job_id is not valid
'''
self._engine_proxy.wait_job(job_ids, timeout)
def kill_job(self, job_id ):
'''
**deprecated since version 2.4:** Use stop_workflow instead.
Kills a running job. The job will not be deleted from the system (the job
identifier remains valid).
Use the restart_job method to restart the job.
Raises *UnknownObjectError* if the job_id is not valid
'''
print "The method kill_job is deprecated since version 2.4. Use stop_workflow instead."
self._engine_proxy.kill_job(job_id)
def restart_job(self, job_id):
'''
**deprecated since version 2.4:** Use restart_workflow instead.
Restarts a job which status is constants.FAILED or constants.WARNING.
* job_id *job identifier*
* returns: *boolean*
True if the job was restarted.
Raises *UnknownObjectError* if the job_id is not valid
'''
print "The method restart_job is deprecated since version 2.4. Use submit_workflow instead."
self._engine_proxy.restart_job(job_id)
def delete_job(self, job_id, force=True):
'''
**deprecated since version 2.4:** Use delete_workflow instead.
Deletes a job which is not part of a workflow.
The job_id will become invalid and can not be used anymore.
The job is killed if it is running.
Raises *UnknownObjectError* if the job_id is not valid
* returns: *boolean*
If force is True: return True if the running jobs were killed and False
if some jobs are possibly still running on the computing resource despite
the workflow doesn't exist.
'''
print "The method delete_job is deprecated since version 2.4. Use delete_workflow instead."
return self._engine_proxy.delete_job(job_id, force)
########## FILE TRANSFER CONTROL #######################################
def transfer_files(self, transfer_ids, buffer_size = 512**2):
'''
Transfer file(s) associated to the transfer_id.
If the files are only located on the client side (that is the transfer
status is constants.FILES_ON_CLIENT) the file(s) will be transfered from the
client to the computing resource.
If the files are located on the computing resource side (that is the
transfer status is constants.FILES_ON_CR or
constants.FILES_ON_CLIENT_AND_CR)
the files will be transfered from the computing resource to the client.
* transfer_id *FileTransfer identifier*
* buffer_size *int*
Depending on the transfer method, the files can be transfered piece by
piece. The size of each piece can be tuned using the buffer_size
argument.
* returns: *boolean*
The transfer was done. (TBI right error management)
Raises *UnknownObjectError* if the transfer_id is not valid
#Raises *TransferError*
'''
if not type(transfer_ids) in types.StringTypes:
for transfer_id in transfer_ids:
self._transfer_file(transfer_id, buffer_size)
else:
self._transfer_file(transfer_ids, buffer_size)
def delete_transfer(self, transfer_id):
'''
Deletes the FileTransfer and the associated files and directories on the
computing resource side. The transfer_id will become invalid and can not be
used anymore. If some jobs reference the FileTransfer as an input or an
output the FileTransfer will not be deleted immediately but as soon as these
jobs will be deleted.
Raises *UnknownObjectError* if the transfer_id is not valid
'''
self._engine_proxy.delete_transfer(transfer_id)
########## PRIVATE #############################################
def _initialize_transfer(self, transfer_id):
'''
Initializes the transfer and returns the transfer action information.
* transfer_id *FileTransfer identifier*
* returns: *tuple*
transfer_type
* (file_size, md5_hash) in the case of a file transfer
* (cumulated_size, dictionary relative path -> (file_size, md5_hash)) in
case of a directory transfer.
Raises *UnknownObjectError* if the transfer_id is not valid
'''
(transfer_id,
client_path,
expiration_date,
workflow_id,
client_paths,
transfer_type,
status) = self._engine_proxy.transfer_information(transfer_id)
if status == constants.FILES_ON_CLIENT:
if not client_paths:
if os.path.isfile(client_path):
transfer_type = constants.TR_FILE_C_TO_CR
self._engine_proxy.set_transfer_status(transfer_id,
constants.TRANSFERING_FROM_CLIENT_TO_CR)
self._engine_proxy.set_transfer_type(transfer_id,
transfer_type)
elif os.path.isdir(client_path):
transfer_type = constants.TR_DIR_C_TO_CR
self._engine_proxy.set_transfer_status(transfer_id,
constants.TRANSFERING_FROM_CLIENT_TO_CR)
self._engine_proxy.set_transfer_type(transfer_id,
constants.TR_DIR_C_TO_CR)
else:
print("WARNING: The file or directory %s doesn't exist "
"on the client machine." %(client_path))
else: #client_paths
for path in client_paths:
if not os.path.isfile(path) and not os.path.isdir(path):
print("WARNING: The file or directory %s doesn't exist "
"on the client machine." %(path))
transfer_type = constants.TR_MFF_C_TO_CR
self._engine_proxy.set_transfer_status(transfer_id,
constants.TRANSFERING_FROM_CLIENT_TO_CR)
self._engine_proxy.set_transfer_type(transfer_id,
transfer_type)
return transfer_type
elif status == constants.FILES_ON_CR or status == constants.FILES_ON_CLIENT_AND_CR:
#transfer_type = self._engine_proxy.init_transfer_from_cr(transfer_id,
#client_path,
#expiration_date,
#workflow_id,
#client_paths,
#status)
if not client_paths:
if self._engine_proxy.is_file(transfer_id):
transfer_type = constants.TR_FILE_CR_TO_C
elif self._engine_proxy.is_dir(transfer_id):
transfer_type = constants.TR_DIR_CR_TO_C
else:
print("WARNING: The file or directory %s doesn't exist "
"on the computing resource side." %(transfer_id))
else: #client_paths
for path in client_paths:
relative_path = os.path.basename(path)
r_path = posixpath.join(transfer_id, relative_path)
if not self._engine_proxy.is_file(r_path) and \
not self._engine_proxy.is_dir(r_path):
print("WARNING: The file or directory %s doesn't exist "
"on the computing resource side." %(r_path))
transfer_type = constants.TR_MFF_CR_TO_C
self._engine_proxy.set_transfer_status(transfer_id,
constants.TRANSFERING_FROM_CR_TO_CLIENT)
self._engine_proxy.set_transfer_type(transfer_id,
transfer_type)
return transfer_type
def _transfer_file(self, transfer_id, buffer_size):
(transfer_id,
client_path,
expiration_date,
workflow_id,
client_paths,
transfer_type,
status) = self._engine_proxy.transfer_information(transfer_id)
if status == constants.FILES_ON_CLIENT or \
status == constants.TRANSFERING_FROM_CLIENT_TO_CR:
# transfer from client to computing resource
#overwrite = False
#if not transfer_type or \
#transfer_type == constants.TR_FILE_CR_TO_C or \
#transfer_type == constants.TR_DIR_CR_TO_C or \
#transfer_type == constants.TR_MFF_CR_TO_C:
## transfer reset
#overwrite = True
transfer_type = self._initialize_transfer(transfer_id)
remote_path = transfer_id
if transfer_type == constants.TR_FILE_C_TO_CR or \
transfer_type == constants.TR_DIR_C_TO_CR:
self._transfer.transfer_to_remote(client_path,
remote_path)
self._engine_proxy.set_transfer_status(transfer_id,
constants.FILES_ON_CLIENT_AND_CR)
self._engine_proxy.signalTransferEnded(transfer_id, workflow_id)
return True
if transfer_type == constants.TR_MFF_C_TO_CR:
for path in client_paths:
relative_path = os.path.basename(path)
r_path = posixpath.join(remote_path, relative_path)
self._transfer.transfer_to_remote(path,
r_path)
self._engine_proxy.set_transfer_status(transfer_id,
constants.FILES_ON_CLIENT_AND_CR)
self._engine_proxy.signalTransferEnded(transfer_id, workflow_id)
return True
if status == constants.FILES_ON_CR or \
status == constants.TRANSFERING_FROM_CR_TO_CLIENT or \
status == constants.FILES_ON_CLIENT_AND_CR:
# transfer from computing resource to client
#overwrite = False
#if not transfer_type or \
#transfer_type == constants.TR_FILE_C_TO_CR or \
#transfer_type == constants.TR_DIR_C_TO_CR or \
#transfer_type == constants.TR_MFF_C_TO_CR :
## TBI remove existing files
#overwrite = True
transfer_type = self._initialize_transfer(transfer_id)
remote_path = transfer_id
if transfer_type == constants.TR_FILE_CR_TO_C or \
transfer_type == constants.TR_DIR_CR_TO_C:
# file case
self._transfer.transfer_from_remote(remote_path,
client_path)
self._engine_proxy.set_transfer_status(transfer_id,
constants.FILES_ON_CLIENT_AND_CR)
self._engine_proxy.signalTransferEnded(transfer_id, workflow_id)
return True
if transfer_type == constants.TR_MFF_CR_TO_C:
for path in client_paths:
relative_path = os.path.basename(path)
r_path = posixpath.join(remote_path, relative_path)
self._transfer.transfer_from_remote(r_path,
path)
self._engine_proxy.set_transfer_status(transfer_id,
constants.FILES_ON_CLIENT_AND_CR)
self._engine_proxy.signalTransferEnded(transfer_id, workflow_id)
return True
return False
def _transfer_progression(self,
status,
transfer_type,
client_path,
client_paths,
engine_path):
if status == constants.TRANSFERING_FROM_CLIENT_TO_CR:
if transfer_type == constants.TR_MFF_C_TO_CR:
data_size = 0
data_transfered = 0
for path in client_paths:
relative_path = os.path.basename(path)
r_path = posixpath.join(engine_path, relative_path)
(ds,
dt) = self._transfer_monitoring.transfer_to_remote_progression(path,
r_path)
data_size = data_size + ds
data_transfered = data_transfered + dt
progression = (data_size, data_transfered)
else:
progression = self._transfer_monitoring.transfer_to_remote_progression(
client_path,
engine_path)
elif status == constants.TRANSFERING_FROM_CR_TO_CLIENT:
if transfer_type == constants.TR_MFF_CR_TO_C:
data_size = 0
data_transfered = 0
for path in client_paths:
relative_path = os.path.basename(path)
r_path = posixpath.join(engine_path, relative_path)
(ds,
dt) = self._transfer_monitoring.transfer_from_remote_progression(r_path,
path)
data_size = data_size + ds
data_transfered = data_transfered + dt
progression = (data_size, data_transfered)
else:
progression = self._transfer_monitoring.transfer_from_remote_progression(engine_path,
client_path)
else:
progression = (100, 100)
return progression