def _do_submit(self, context):
if not self.options.access_token:
delegator = Delegator(context)
delegator.delegate(
timedelta(minutes=self.options.proxy_lifetime),
delegate_when_lifetime_lt=timedelta(minutes=self.options.delegate_when_lifetime_lt)
)
submitter = Submitter(context)
job_id = submitter.submit(
transfers=self.transfers,
params=self.params
)
if self.options.json:
self.logger.info(json.dumps(job_id))
else:
self.logger.info("Job successfully submitted.")
self.logger.info("Job id: %s" % job_id)
if job_id and self.options.blocking:
inquirer = Inquirer(context)
job = inquirer.get_job_status(job_id)
while job['job_state'] in ['SUBMITTED', 'READY', 'STAGING', 'ACTIVE', 'ARCHIVING', 'QOS_TRANSITION', 'QOS_REQUEST_SUBMITTED']:
self.logger.info("Job in state %s" % job['job_state'])
time.sleep(self.options.poll_interval)
job = inquirer.get_job_status(job_id)
self.logger.info("Job finished with state %s" % job['job_state'])
if job['reason']:
self.logger.info("Reason: %s" % job['reason'])
return job_id
def _do_submit(self, context):
delegator = Delegator(context)
delegator.delegate(timedelta(minutes=self.options.proxy_lifetime))
submitter = Submitter(context)
job_id = submitter.submit(
delete=self._build_delete(),
spacetoken=self.options.spacetoken,
job_metadata=_metadata(self.options.job_metadata),
retry=self.options.retry,
credential=self.options.cloud_cred
)
if self.options.json:
self.logger.info(json.dumps(job_id))
else:
self.logger.info("Job successfully submitted.")
self.logger.info("Job id: %s" % job_id)
if job_id and self.options.blocking:
inquirer = Inquirer(context)
job = inquirer.get_job_status(job_id)
while job['job_state'] in ['SUBMITTED', 'READY', 'STAGING', 'ACTIVE', 'DELETE']:
self.logger.info("Job in state %s" % job['job_state'])
time.sleep(self.options.poll_interval)
job = inquirer.get_job_status(job_id)
self.logger.info("Job finished with state %s" % job['job_state'])
if job['reason']:
self.logger.info("Reason: %s" % job['reason'])
return job_id
def _do_submit(self, context):
delegator = Delegator(context)
delegator.delegate(timedelta(minutes=self.options.proxy_lifetime))
submitter = Submitter(context)
job_id = submitter.submit(delete=self._build_delete(),
spacetoken=self.options.spacetoken,
job_metadata=_metadata(
self.options.job_metadata),
retry=self.options.retry,
credential=self.options.cloud_cred)
if self.options.json:
self.logger.info(json.dumps(job_id))
else:
self.logger.info("Job successfully submitted.")
self.logger.info("Job id: %s" % job_id)
if job_id and self.options.blocking:
inquirer = Inquirer(context)
job = inquirer.get_job_status(job_id)
while job['job_state'] in [
'SUBMITTED', 'READY', 'STAGING', 'ACTIVE', 'DELETE'
]:
self.logger.info("Job in state %s" % job['job_state'])
time.sleep(self.options.poll_interval)
job = inquirer.get_job_status(job_id)
self.logger.info("Job finished with state %s" % job['job_state'])
if job['reason']:
self.logger.info("Reason: %s" % job['reason'])
return job_id
def run(self):
context = self._create_context()
inquirer = Inquirer(context)
snapshot = inquirer.get_snapshot(self.options.vo, self.options.source, self.options.destination)
if self.options.json:
self.logger.info(json.dumps(snapshot, indent=2))
else:
_human_readable_snapshot(self.logger, snapshot)
def __call__(self):
context = Context(self.options.endpoint)
inquirer = Inquirer(context)
job = inquirer.getJobStatus(self.jobId)
if not self.options.json:
self.logger.info(job2HumanReadable(job))
else:
self.logger.info(job2Json(job))
def run(self):
context = self._create_context()
inquirer = Inquirer(context)
job_list = inquirer.get_job_list(
self.options.user_dn, self.options.vo_name, self.options.source_se, self.options.dest_se
)
if not self.options.json:
self.logger.info(job_list_human_readable(job_list))
else:
self.logger.info(job_list_as_json(job_list))
def __call__(self): context = Context(self.options.endpoint) inquirer = Inquirer(context) jobList = inquirer.getJobList(self.options.user_dn, self.options.vo_name) if not self.options.json: self.logger.info(jobList2HumanReadable(jobList)) else: self.logger.info(jobList2Json(jobList))
def run(self):
context = self._create_context()
inquirer = Inquirer(context)
job_list = inquirer.get_job_list(self.options.user_dn,
self.options.vo_name,
self.options.source_se,
self.options.dest_se)
if not self.options.json:
self.logger.info(job_list_human_readable(job_list))
else:
self.logger.info(job_list_as_json(job_list))
def run(self):
job_id = self.args[0]
context = self._create_context()
inquirer = Inquirer(context)
job = inquirer.get_job_status(job_id, list_files=self.options.json)
if not self.options.json:
self.logger.info(job_human_readable(job))
else:
self.logger.info(job_as_json(job))
def __call__(self):
context = Context(self.options.endpoint)
inquirer = Inquirer(context)
jobList = inquirer.getJobList(self.options.user_dn,
self.options.vo_name)
if not self.options.json:
self.logger.info(jobList2HumanReadable(jobList))
else:
self.logger.info(jobList2Json(jobList))
def run(self):
context = self._create_context()
inquirer = Inquirer(context)
whoami = inquirer.whoami()
if self.options.json:
print json.dumps(whoami, indent=2)
else:
self.logger.info("User DN: %s" % whoami['dn'][0])
for vo in whoami['vos']:
self.logger.info("VO: %s" % vo)
self.logger.info("Delegation id: %s" % whoami['delegation_id'])
def whoami(context):
"""
Queries the server to see how does it see us
Args:
context: fts3.rest.client.context.Context instance
Returns:
Decoded JSON message returned by the server with a representation of
the user credentials (as set in context)
"""
inquirer = Inquirer(context)
return inquirer.whoami()
def get_jobs_statuses(context, job_ids, list_files=False):
"""
Get status for a list of jobs
Args:
context: fts3.rest.client.context.Context instance
job_ids: The job list
list_files: If True, the status of each individual file will be queried
Returns:
Decoded JSON message returned by the server (job status plus, optionally, list of files)
"""
inquirer = Inquirer(context)
return inquirer.get_jobs_statuses(job_ids, list_files)
def _do_submit(self, context):
verify_checksum = None
if self.options.compare_checksum:
verify_checksum = True
delegator = Delegator(context)
delegator.delegate(timedelta(minutes=self.options.proxy_lifetime))
submitter = Submitter(context)
job_id = submitter.submit(
self._build_transfers(),
checksum=self.checksum,
bring_online=self.options.bring_online,
verify_checksum=verify_checksum,
spacetoken=self.options.destination_token,
source_spacetoken=self.options.source_token,
fail_nearline=self.options.fail_nearline,
file_metadata=_metadata(self.options.file_metadata),
filesize=self.options.file_size,
gridftp=self.options.gridftp_params,
job_metadata=_metadata(self.options.job_metadata),
overwrite=self.options.overwrite,
copy_pin_lifetime=self.options.pin_lifetime,
reuse=self.options.reuse,
retry=self.options.retry,
multihop=self.options.multihop,
credential=self.options.cloud_cred
)
if self.options.json:
self.logger.info(json.dumps(job_id))
else:
self.logger.info("Job successfully submitted.")
self.logger.info("Job id: %s" % job_id)
if job_id and self.options.blocking:
inquirer = Inquirer(context)
job = inquirer.get_job_status(job_id)
while job['job_state'] in ['SUBMITTED', 'READY', 'STAGING', 'ACTIVE']:
self.logger.info("Job in state %s" % job['job_state'])
time.sleep(self.options.poll_interval)
job = inquirer.get_job_status(job_id)
self.logger.info("Job finished with state %s" % job['job_state'])
if job['reason']:
self.logger.info("Reason: %s" % job['reason'])
return job_id
def get_snapshot(context, vo=None, source=None, destination=None):
"""
Get a snapshot of the server
Args:
context: fts3.rest.client.context.Context instance
vo: Filter by vo. Can be left empty.
source: Filter by source SE. Can be left empty
destination: Filter by destination SE. Can be left empty.
Returns:
Decoded JSON message returned by the server (server snapshot)
"""
inquirer = Inquirer(context)
return inquirer.get_snapshot(vo, source, destination)
def run(self):
context = self._create_context()
inquirer = Inquirer(context)
whoami = inquirer.whoami()
if self.options.json:
print json.dumps(whoami, indent=2)
else:
self.logger.info("User DN: %s" % whoami['dn'][0])
for vo in whoami['vos']:
self.logger.info("VO: %s" % vo)
for vo_id in whoami['vos_id']:
self.logger.info("VO id: %s" % vo_id)
self.logger.info("Delegation id: %s" % whoami['delegation_id'])
self.logger.info("Base id: %s" % whoami['base_id'])
def _doSubmit(self):
verify_checksum = None
if self.options.compare_checksum:
verify_checksum = True
delegator = Delegator(self.context)
delegationId = delegator.delegate(
timedelta(minutes=self.options.proxy_lifetime))
submitter = Submitter(self.context)
jobId = submitter.submit(self.source,
self.destination,
checksum=self.checksum,
bring_online=self.options.bring_online,
verify_checksum=verify_checksum,
spacetoken=self.options.destination_token,
source_spacetoken=self.options.source_token,
fail_nearline=self.options.fail_nearline,
file_metadata=self.options.file_metadata,
filesize=self.options.file_size,
gridftp=self.options.gridftp_params,
job_metadata=self.options.job_metadata,
overwrite=self.options.overwrite,
copy_pin_lifetime=self.options.pin_lifetime,
reuse=self.options.reuse)
if self.options.json:
self.logger.info(jobId)
else:
self.logger.info("Job successfully submitted.")
self.logger.info("Job id: %s" % jobId)
if jobId and self.options.blocking:
inquirer = Inquirer(self.context)
while True:
time.sleep(self.options.poll_interval)
job = inquirer.getJobStatus(jobId)
if job['job_state'] not in [
'SUBMITTED', 'READY', 'STAGING', 'ACTIVE'
]:
break
self.logger.info("Job in state %s" % job['job_state'])
self.logger.info("Job finished with state %s" % job['job_state'])
if job['reason']:
self.logger.info("Reason: %s" % job['reason'])
return jobId
def list_jobs(context, user_dn=None, vo=None, source_se=None, dest_se=None, delegation_id=None, state_in=None):
"""
List active jobs. Can filter by user_dn and vo
Args:
context: fts3.rest.client.context.Context instance
user_dn: Filter by user dn. Can be left empty
vo: Filter by vo. Can be left empty
delegation_id: Filter by delegation ID. Mandatory for state_in
state_in: Filter by job state. An iterable is expected (i.e. ['SUBMITTED', 'ACTIVE']
Returns:
Decoded JSON message returned by the server (list of jobs)
"""
inquirer = Inquirer(context)
return inquirer.get_job_list(user_dn, vo, source_se, dest_se, delegation_id, state_in)
def _doSubmit(self):
verify_checksum = None
if self.options.compare_checksum:
verify_checksum = True
delegator = Delegator(self.context)
delegationId = delegator.delegate(timedelta(minutes = self.options.proxy_lifetime))
submitter = Submitter(self.context)
jobId = submitter.submit(self.source, self.destination,
checksum = self.checksum,
bring_online = self.options.bring_online,
verify_checksum = verify_checksum,
spacetoken = self.options.destination_token,
source_spacetoken = self.options.source_token,
fail_nearline = self.options.fail_nearline,
file_metadata = self.options.file_metadata,
filesize = self.options.file_size,
gridftp = self.options.gridftp_params,
job_metadata = self.options.job_metadata,
overwrite = self.options.overwrite,
copy_pin_lifetime = self.options.pin_lifetime,
reuse = self.options.reuse
)
if self.options.json:
self.logger.info(jobId)
else:
self.logger.info("Job successfully submitted.")
self.logger.info("Job id: %s" % jobId)
if jobId and self.options.blocking:
inquirer = Inquirer(self.context)
while True:
time.sleep(self.options.poll_interval)
job = inquirer.getJobStatus(jobId)
if job['job_state'] not in ['SUBMITTED', 'READY', 'STAGING', 'ACTIVE']:
break
self.logger.info("Job in state %s" % job['job_state'])
self.logger.info("Job finished with state %s" % job['job_state'])
if job['reason']:
self.logger.info("Reason: %s" % job['reason'])
return jobId
def _do_submit(self, context):
#Backwards compatibility: compare_checksum parameter
if self.options.compare_checksum:
checksum_mode = 'both'
else:
if self.checksum:
checksum_mode = 'target'
else:
checksum = 'none'
#Compare checksum has major priority than checksum_mode
if not self.options.compare_checksum:
if len(self.options.checksum_mode) > 0:
checksum_mode = self.options.checksum_mode
else:
checksum_mode = 'none'
if not self.checksum:
self.checksum = DEFAULT_CHECKSUM
delegator = Delegator(context)
delegator.delegate(timedelta(minutes=self.options.proxy_lifetime),
delegate_when_lifetime_lt=timedelta(
minutes=self.options.delegate_when_lifetime_lt))
submitter = Submitter(context)
job_id = submitter.submit(
self._build_transfers(),
checksum=self.checksum,
bring_online=self.options.bring_online,
timeout=self.options.timeout,
verify_checksum=checksum_mode[0],
spacetoken=self.options.destination_token,
source_spacetoken=self.options.source_token,
fail_nearline=self.options.fail_nearline,
file_metadata=_metadata(self.options.file_metadata),
filesize=self.options.file_size,
gridftp=self.options.gridftp_params,
job_metadata=_metadata(self.options.job_metadata),
overwrite=self.options.overwrite,
copy_pin_lifetime=self.options.pin_lifetime,
reuse=self.options.reuse,
retry=self.options.retry,
multihop=self.options.multihop,
credential=self.options.cloud_cred,
nostreams=self.options.nostreams,
ipv4=self.options.ipv4,
ipv6=self.options.ipv6)
if self.options.json:
self.logger.info(json.dumps(job_id))
else:
self.logger.info("Job successfully submitted.")
self.logger.info("Job id: %s" % job_id)
if job_id and self.options.blocking:
inquirer = Inquirer(context)
job = inquirer.get_job_status(job_id)
while job['job_state'] in [
'SUBMITTED', 'READY', 'STAGING', 'ACTIVE'
]:
self.logger.info("Job in state %s" % job['job_state'])
time.sleep(self.options.poll_interval)
job = inquirer.get_job_status(job_id)
self.logger.info("Job finished with state %s" % job['job_state'])
if job['reason']:
self.logger.info("Reason: %s" % job['reason'])
return job_id