def download_job(opts, work_dn, status_mon, job_db, job_obj, jobnum):
if job_obj.get('download') == 'True' and not opts.mark_ignore_dl:
return status_mon.register_job_result(jobnum, 'All files already downloaded',
JobDownloadStatus.JOB_ALREADY)
# Read the file hash entries from job info file
fi_list = FileInfoProcessor().process(os.path.join(work_dn, 'output', 'job_%d' % jobnum)) or []
is_download_failed = False
if not fi_list:
if opts.mark_empty_fail:
is_download_failed = True
else:
return status_mon.register_job_result(jobnum, 'Job has no output files',
JobDownloadStatus.JOB_NO_OUTPUT)
download_result_list = []
progress = ProgressActivity('Processing output files', len(fi_list))
for (fi_idx, fi) in enumerate(fi_list):
progress.update_progress(fi_idx, msg='Processing output file %r' % fi[FileInfo.NameDest])
download_result_list.append(download_single_file(opts, jobnum, fi_idx, fi, status_mon))
progress.finish()
is_download_failed = is_download_failed or any(imap(download_result_list.__contains__, [
FileDownloadStatus.FILE_TIMEOUT, FileDownloadStatus.FILE_HASH_FAILED,
FileDownloadStatus.FILE_TRANSFER_FAILED, FileDownloadStatus.FILE_MKDIR_FAILED]))
is_download_success = all(imap([FileDownloadStatus.FILE_OK,
FileDownloadStatus.FILE_EXISTS].__contains__, download_result_list))
# Ignore the first opts.retry number of failed jobs
retry_count = int(job_obj.get('download attempt', 0))
if fi_list and is_download_failed and opts.retry and (retry_count < int(opts.retry)):
set_job_prop(job_db, jobnum, job_obj, 'download attempt', str(retry_count + 1))
return status_mon.register_job_result(jobnum, 'Download attempt #%d failed' % retry_count + 1,
JobDownloadStatus.RETRY)
delete_files(opts, jobnum, fi_list, is_download_failed)
if is_download_failed:
if opts.mark_fail:
# Mark job as failed to trigger resubmission
job_obj.state = Job.FAILED
job_db.commit(jobnum, job_obj)
status_mon.register_job_result(jobnum, 'Download failed', JobDownloadStatus.JOB_FAILED)
elif is_download_success:
if opts.mark_dl:
# Mark as downloaded
set_job_prop(job_db, jobnum, job_obj, 'download', 'True')
status_mon.register_job_result(jobnum, 'Download successful', JobDownloadStatus.JOB_OK)
else:
# eg. because of SE blacklist
status_mon.register_job_result(jobnum, 'Download incomplete', JobDownloadStatus.JOB_INCOMPLETE)
def _iter_datasource_items(self, item, metadata_dict, entries, location_list, obj_dict): if all(imap(metadata_dict.__contains__, ['CMSSW_EVENTS_WRITE', 'CMSSW_CONFIG_FILE'])): yield (item, metadata_dict, entries, location_list, obj_dict)
def getEntries(self, path, metadata, events, seList, objStore): if all(imap(lambda x: x in metadata, ['CMSSW_EVENTS_WRITE', 'CMSSW_CONFIG_FILE'])): yield (path, metadata, events, seList, objStore)
def getEntries(self, path, metadata, events, seList, objStore):
if all(
imap(lambda x: x in metadata,
['CMSSW_EVENTS_WRITE', 'CMSSW_CONFIG_FILE'])):
yield (path, metadata, events, seList, objStore)
def canFinish(self): return all(imap(lambda p: p.canFinish(), self._psourceList))
def download_job(opts, work_dn, status_mon, job_db, job_obj, jobnum):
if job_obj.get('download') == 'True' and not opts.mark_ignore_dl:
return status_mon.register_job_result(jobnum,
'All files already downloaded',
JobDownloadStatus.JOB_ALREADY)
# Read the file hash entries from job info file
fi_list = FileInfoProcessor().process(
os.path.join(work_dn, 'output', 'job_%d' % jobnum)) or []
is_download_failed = False
if not fi_list:
if opts.mark_empty_fail:
is_download_failed = True
else:
return status_mon.register_job_result(
jobnum, 'Job has no output files',
JobDownloadStatus.JOB_NO_OUTPUT)
download_result_list = []
progress = ProgressActivity('Processing output files', len(fi_list))
for (fi_idx, fi) in enumerate(fi_list):
progress.update_progress(fi_idx,
msg='Processing output file %r' %
fi[FileInfo.NameDest])
download_result_list.append(
download_single_file(opts, jobnum, fi_idx, fi, status_mon))
progress.finish()
is_download_failed = is_download_failed or any(
imap(download_result_list.__contains__, [
FileDownloadStatus.FILE_TIMEOUT,
FileDownloadStatus.FILE_HASH_FAILED,
FileDownloadStatus.FILE_TRANSFER_FAILED,
FileDownloadStatus.FILE_MKDIR_FAILED
]))
is_download_success = all(
imap([FileDownloadStatus.FILE_OK,
FileDownloadStatus.FILE_EXISTS].__contains__,
download_result_list))
# Ignore the first opts.retry number of failed jobs
retry_count = int(job_obj.get('download attempt', 0))
if fi_list and is_download_failed and opts.retry and (retry_count < int(
opts.retry)):
set_job_prop(job_db, jobnum, job_obj, 'download attempt',
str(retry_count + 1))
return status_mon.register_job_result(
jobnum, 'Download attempt #%d failed' % retry_count + 1,
JobDownloadStatus.RETRY)
delete_files(opts, jobnum, fi_list, is_download_failed)
if is_download_failed:
if opts.mark_fail:
# Mark job as failed to trigger resubmission
job_obj.state = Job.FAILED
job_db.commit(jobnum, job_obj)
status_mon.register_job_result(jobnum, 'Download failed',
JobDownloadStatus.JOB_FAILED)
elif is_download_success:
if opts.mark_dl:
# Mark as downloaded
set_job_prop(job_db, jobnum, job_obj, 'download', 'True')
status_mon.register_job_result(jobnum, 'Download successful',
JobDownloadStatus.JOB_OK)
else:
# eg. because of SE blacklist
status_mon.register_job_result(jobnum, 'Download incomplete',
JobDownloadStatus.JOB_INCOMPLETE)
def can_finish(self): return all(imap(lambda psrc: psrc.can_finish(), self._psrc_list))
def can_finish(self):
return all(imap(lambda psrc: psrc.can_finish(), self._psrc_list))
def canFinish(self):
return all(imap(lambda p: p.canFinish(), self._psourceList))