def blocking_transfer(self, filelist):
""" Do a blocking transfer
Parameters
----------
filelist : dict
Dictionary of the files to transfer
Returns
-------
tuple
Transfer results
"""
miscutils.fwdebug_print("\tNumber files to transfer: %d" %
len(filelist))
if miscutils.fwdebug_check(1, "ARCHIVETRANSFER_DEBUG"):
miscutils.fwdebug_print("\tfilelist: %s" % filelist)
srcroot = self.src_archive_info['root']
dstroot = self.dst_archive_info['root']
files2copy = copy.deepcopy(filelist)
for _, finfo in files2copy.items():
finfo['src'] = '%s/%s' % (srcroot, finfo['src'])
finfo['dst'] = '%s/%s' % (dstroot, finfo['dst'])
transresults = disk_utils_local.copyfiles(files2copy, None)
return transresults
def job2home(self, filelist, verify=False):
""" Transfer files from the job to the home archive
Parameters
----------
filelist : dict
Dictionary containing the file names and path information
Returns
-------
dict
The results
"""
if miscutils.fwdebug_check(3, "JOBFILEMVMT_DEBUG"):
miscutils.fwdebug_print("len(filelist)=%s" % len(filelist))
if miscutils.fwdebug_check(6, "JOBFILEMVMT_DEBUG"):
miscutils.fwdebug_print("filelist=%s" % filelist)
# if staging outside job, this function shouldn't be called
if self.home is None:
raise Exception(
"Home archive info is None. Should not be calling this function"
)
absfilelist = copy.deepcopy(filelist)
for finfo in absfilelist.values():
finfo['dst'] = self.home['root'] + '/' + finfo['dst']
if self.tstats is not None:
self.tstats.stat_beg_batch(
'job2home', 'job_scratch', self.home['name'],
self.__module__ + '.' + self.__class__.__name__)
(status, results) = disk_utils_local.copyfiles(absfilelist,
self.tstats, verify)
if self.tstats is not None:
self.tstats.stat_end_batch(status)
return results
def target2job(self, filelist):
""" Transfer files from the target archive
Parameters
----------
filelist : dict
Dictionary containing the file names and path information
Returns
-------
dict
The results
"""
if miscutils.fwdebug_check(3, "JOBFILEMVMT_DEBUG"):
miscutils.fwdebug_print("len(filelist)=%s" % len(filelist))
if miscutils.fwdebug_check(6, "JOBFILEMVMT_DEBUG"):
miscutils.fwdebug_print("filelist=%s" % filelist)
if self.target is None:
raise Exception(
"Target archive info is None. Should not be calling this function"
)
absfilelist = copy.deepcopy(filelist)
for finfo in absfilelist.values():
finfo['src'] = self.target['root'] + '/' + finfo['src']
if self.tstats is not None:
self.tstats.stat_beg_batch(
'target2job', self.target['name'], 'job_scratch',
self.__module__ + '.' + self.__class__.__name__)
(status, results) = disk_utils_local.copyfiles(absfilelist,
self.tstats)
if self.tstats is not None:
self.tstats.stat_end_batch(status)
return results
def blocking_transfer(self, filelist):
miscutils.fwdebug_print(f"\tNumber files to transfer: {len(filelist)}")
if miscutils.fwdebug_check(1, "ARCHIVETRANSFER_DEBUG"):
miscutils.fwdebug_print(f"\tfilelist: {filelist}")
srcroot = self.src_archive_info['root']
dstroot = self.dst_archive_info['root']
files2copy = copy.deepcopy(filelist)
for _, finfo in files2copy.items():
finfo['src'] = f"{srcroot}/{finfo['src']}"
finfo['dst'] = f"{dstroot}/{finfo['dst']}"
transresults = disk_utils_local.copyfiles(files2copy, None)
return transresults
def job2target(self, filelist):
""" From inside job, push files to target archive from job scratch directory """
if miscutils.fwdebug_check(3, "JOBFILEMVMT_DEBUG"):
miscutils.fwdebug_print(f"len(filelist)={len(filelist)}")
if miscutils.fwdebug_check(6, "JOBFILEMVMT_DEBUG"):
miscutils.fwdebug_print(f"filelist={filelist}")
if self.target is None:
raise Exception("Target archive info is None. Should not be calling this function")
absfilelist = copy.deepcopy(filelist)
for finfo in absfilelist.values():
finfo['dst'] = os.path.join(self.target['root'], finfo['dst'])
if self.tstats is not None:
self.tstats.stat_beg_batch('job2target', 'job_scratch', self.home['name'],
self.__module__ + '.' + self.__class__.__name__)
(status, results) = disk_utils_local.copyfiles(absfilelist, self.tstats)
if self.tstats is not None:
self.tstats.stat_end_batch(status)
return results