def resolve_input_paths(self, jobspec):
# get input files
inFiles = jobspec.get_input_file_attributes()
# set path to each file
for inLFN, inFile in iteritems(inFiles):
inFile['path'] = mover_utils.construct_file_path(self.basePath, inFile['scope'], inLFN)
# set
jobspec.set_input_file_paths(inFiles)
return True, ''
def resolve_input_paths(self, jobspec):
# get input files
inFiles = jobspec.get_input_file_attributes()
# set path to each file
for inLFN, inFile in iteritems(inFiles):
inFile['path'] = mover_utils.construct_file_path(self.basePath, inFile['scope'], inLFN)
# set
jobspec.set_input_file_paths(inFiles)
return True, ''
def trigger_preparation(self, jobspec):
# make logger
tmpLog = self.make_logger(baseLogger,
'PandaID={0}'.format(jobspec.PandaID),
method_name='trigger_preparation')
tmpLog.debug('start')
# check that jobspec.computingSite is defined
if jobspec.computingSite is None:
# not found
tmpLog.error('jobspec.computingSite is not defined')
return False, 'jobspec.computingSite is not defined'
else:
tmpLog.debug('jobspec.computingSite : {0}'.format(
jobspec.computingSite))
# get input files
files = []
inFiles = jobspec.get_input_file_attributes(skip_ready=True)
# set path to each file
for inLFN, inFile in iteritems(inFiles):
inFile['path'] = mover_utils.construct_file_path(
self.basePath, inFile['scope'], inLFN)
dstpath = os.path.dirname(inFile['path'])
# check if path exists if not create it.
if not os.access(dstpath, os.F_OK):
os.makedirs(dstpath)
files.append({
'scope': inFile['scope'],
'name': inLFN,
'destination': dstpath
})
tmpLog.debug('files[] {0}'.format(files))
data_client = data.StageInClient(site=jobspec.computingSite)
allChecked = True
ErrMsg = 'These files failed to download : '
if len(files) > 0:
result = data_client.transfer(files)
tmpLog.debug(
'pilot.api data.StageInClient.transfer(files) result: {0}'.
format(result))
# loop over each file check result all must be true for entire result to be true
if result:
for answer in result:
if answer['errno'] != 0:
allChecked = False
ErrMsg = ErrMsg + (" %s " % answer['name'])
else:
tmpLog.info(
'Looks like all files already inplace: {0}'.format(files))
# return
tmpLog.debug('stop')
if allChecked:
return True, ''
else:
return False, ErrMsg
def resolve_input_paths(self, jobspec):
# input files
inFileInfo = jobspec.get_input_file_attributes()
pathInfo = dict()
for tmpFileSpec in jobspec.inFiles:
accPath = mover_utils.construct_file_path(self.localBasePath, inFileInfo[tmpFileSpec.lfn]['scope'],
tmpFileSpec.lfn)
pathInfo[tmpFileSpec.lfn] = {'path': accPath}
jobspec.set_input_file_paths(pathInfo)
return True, ''
def trigger_preparation(self, jobspec):
# make logger
tmpLog = self.make_logger(baseLogger, 'PandaID={0}'.format(jobspec.PandaID),
method_name='trigger_preparation')
tmpLog.debug('Start. Trigger data transfer for job: {0}'.format(jobspec.PandaID))
# check that jobspec.computingSite is defined
if jobspec.computingSite is None:
# not found
tmpLog.error('jobspec.computingSite is not defined')
return False, 'jobspec.computingSite is not defined'
else:
tmpLog.debug('jobspec.computingSite : {0}'.format(jobspec.computingSite))
# get input files
files = []
inFiles = jobspec.get_input_file_attributes(skip_ready=True)
# set path to each file
tmpLog.info("Prepare files to download (construct path and verifiy existing files)")
for inLFN, inFile in iteritems(inFiles):
inFile['path'] = mover_utils.construct_file_path(self.basePath, inFile['scope'], inLFN)
# check if file exist. Skip alrady downoladed files
if os.path.exists(inFile['path']):
checksum = core_utils.calc_adler32(inFile['path'])
checksum = 'ad:%s' % checksum
#tmpLog.debug('checksum for file %s is %s' % (inFile['path'], checksum))
if 'checksum' in inFile and inFile['checksum'] and inFile['checksum'] == checksum:
#tmpLog.debug('File %s already exists at %s' % (inLFN, inFile['path']))
continue
dstpath = os.path.dirname(inFile['path'])
# check if path exists if not create it.
if not os.access(dstpath, os.F_OK):
os.makedirs(dstpath)
files.append({'scope': inFile['scope'],
'name': inLFN,
'destination': dstpath})
tmpLog.info('Number of files to dowload: {0} for job: {1}'.format(len(files), jobspec.PandaID))
#tmpLog.debug('files {0}'.format(files))
tmpLog.info('Setup of Pilot2 API client')
data_client = data.StageInClient(site=jobspec.computingSite)
allChecked = True
ErrMsg = 'These files failed to download : '
if len(files) > 0:
tmpLog.info("Going to transfer {0} of files with one call to Pilot2 Data API".format(len(files)))
try:
result = data_client.transfer(files)
except Exception as e:
tmpLog.error("Pilot2 Data API rise error: {0}".format(e.message))
tmpLog.debug('data_client.transfer(files) result:\n{0}'.format(result))
tmpLog.info("Transfer call to Pilot2 Data API completed")
# loop over each file check result all must be true for entire result to be true
if result:
for answer in result:
if answer['errno'] != 0:
allChecked = False
ErrMsg = ErrMsg + (" %s " % answer['name'])
else:
tmpLog.info('Looks like all files in place. Number of files: {0}'.format(len(files)))
# return
tmpLog.debug('Finished data transfer with {0} files for job {1}'.format(len(files), jobspec.PandaID))
if allChecked:
return True, ''
else:
return False, ErrMsg
def trigger_preparation(self, jobspec):
# make logger
tmpLog = self.make_logger(baseLogger,
'PandaID={0}'.format(jobspec.PandaID),
method_name='trigger_preparation')
tmpLog.debug('Start. Trigger data transfer for job: {0}'.format(
jobspec.PandaID))
# check that jobspec.computingSite is defined
if jobspec.computingSite is None:
# not found
tmpLog.error('jobspec.computingSite is not defined')
return False, 'jobspec.computingSite is not defined'
else:
tmpLog.debug('jobspec.computingSite : {0}'.format(
jobspec.computingSite))
# get input files
files = []
inFiles = jobspec.get_input_file_attributes(skip_ready=True)
# set path to each file
tmpLog.info(
"Prepare files to download (construct path and verifiy existing files)"
)
for inLFN, inFile in iteritems(inFiles):
inFile['path'] = mover_utils.construct_file_path(
self.basePath, inFile['scope'], inLFN)
# check if file exist. Skip alrady downoladed files
if os.path.exists(inFile['path']):
checksum = core_utils.calc_adler32(inFile['path'])
checksum = 'ad:%s' % checksum
#tmpLog.debug('checksum for file %s is %s' % (inFile['path'], checksum))
if 'checksum' in inFile and inFile['checksum'] and inFile[
'checksum'] == checksum:
#tmpLog.debug('File %s already exists at %s' % (inLFN, inFile['path']))
continue
dstpath = os.path.dirname(inFile['path'])
# check if path exists if not create it.
if not os.access(dstpath, os.F_OK):
os.makedirs(dstpath)
files.append({
'scope': inFile['scope'],
'name': inLFN,
'destination': dstpath
})
tmpLog.info('Number of files to dowload: {0} for job: {1}'.format(
len(files), jobspec.PandaID))
#tmpLog.debug('files {0}'.format(files))
tmpLog.info('Setup of Pilot2 API client')
data_client = data.StageInClient(site=jobspec.computingSite)
allChecked = True
ErrMsg = 'These files failed to download : '
if len(files) > 0:
tmpLog.info(
"Going to transfer {0} of files with one call to Pilot2 Data API"
.format(len(files)))
try:
result = data_client.transfer(files)
except Exception as e:
tmpLog.error("Pilot2 Data API rise error: {0}".format(
e.message))
tmpLog.debug(
'data_client.transfer(files) result:\n{0}'.format(result))
tmpLog.info("Transfer call to Pilot2 Data API completed")
# loop over each file check result all must be true for entire result to be true
if result:
for answer in result:
if answer['errno'] != 0:
allChecked = False
ErrMsg = ErrMsg + (" %s " % answer['name'])
else:
tmpLog.info(
'Looks like all files in place. Number of files: {0}'.
format(len(files)))
# return
tmpLog.debug(
'Finished data transfer with {0} files for job {1}'.format(
len(files), jobspec.PandaID))
if allChecked:
return True, ''
else:
return False, ErrMsg
def trigger_preparation(self, jobspec):
# make logger
tmpLog = self.make_logger(baseLogger, 'PandaID={0}'.format(jobspec.PandaID),
method_name='trigger_preparation')
tmpLog.debug('start')
# check that jobspec.computingSite is defined
if jobspec.computingSite is None:
# not found
tmpLog.error('jobspec.computingSite is not defined')
return False, 'jobspec.computingSite is not defined'
else:
tmpLog.debug('jobspec.computingSite : {0}'.format(jobspec.computingSite))
# get input files
files = []
inFiles = jobspec.get_input_file_attributes(skip_ready=True)
# set path to each file
for inLFN, inFile in iteritems(inFiles):
inFile['path'] = mover_utils.construct_file_path(self.basePath, inFile['scope'], inLFN)
tmpLog.debug('To check file: %s' % inFile)
if os.path.exists(inFile['path']):
checksum = core_utils.calc_adler32(inFile['path'])
checksum = 'ad:%s' % checksum
tmpLog.debug('checksum for file %s is %s' % (inFile['path'], checksum))
if 'checksum' in inFile and inFile['checksum'] and inFile['checksum'] == checksum:
tmpLog.debug('File %s already exists at %s' % (inLFN, inFile['path']))
continue
dstpath = os.path.dirname(inFile['path'])
# check if path exists if not create it.
if not os.access(dstpath, os.F_OK):
os.makedirs(dstpath)
files.append({'scope': inFile['scope'],
'name': inLFN,
'destination': dstpath})
tmpLog.debug('files[] {0}'.format(files))
allChecked = True
ErrMsg = 'These files failed to download : '
if files:
threads = []
n_files_per_thread = (len(files) + self.n_threads - 1) / self.n_threads
tmpLog.debug('num files per thread: %s' % n_files_per_thread)
for i in range(0, len(files), n_files_per_thread):
sub_files = files[i:i + n_files_per_thread]
thread = threading.Thread(target=self.stage_in, kwargs={'tmpLog': tmpLog, 'jobspec': jobspec, 'files': sub_files})
threads.append(thread)
[t.start() for t in threads]
while len(threads) > 0:
time.sleep(1)
threads = [t for t in threads if t and t.isAlive()]
tmpLog.info('Checking all files: {0}'.format(files))
for file in files:
if file['errno'] != 0:
allChecked = False
ErrMsg = ErrMsg + (" %s " % file['name'])
# return
tmpLog.debug('stop')
if allChecked:
tmpLog.info('Looks like all files are successfully downloaded.')
return True, ''
else:
return False, ErrMsg
def trigger_stage_out(self, jobspec):
# make logger
tmpLog = self.make_logger(baseLogger,
'PandaID={0}'.format(jobspec.PandaID),
method_name='trigger_stage_out')
tmpLog.debug('start')
# loop over all files
files = dict()
transferIDs = dict()
transferDatasets = dict()
fileAttrs = jobspec.get_output_file_attributes()
for fileSpec in jobspec.outFiles:
# skip zipped files
if fileSpec.zipFileID is not None:
continue
# skip if already processed
if 'transferDataset' in fileSpec.fileAttributes:
if fileSpec.fileType not in transferDatasets:
transferDatasets[
fileSpec.
fileType] = fileSpec.fileAttributes['transferDataset']
if fileSpec.fileType not in transferIDs:
transferIDs[fileSpec.fileType] = fileSpec.fileAttributes[
'transferID']
continue
# set OS ID
if fileSpec.fileType == ['es_output', 'zip_output']:
fileSpec.objstoreID = self.objStoreID_ES
# make path where file is copied for transfer
if fileSpec.fileType != 'zip_output':
scope = fileAttrs[fileSpec.lfn]['scope']
datasetName = fileAttrs[fileSpec.lfn]['dataset']
else:
# use panda scope for zipped files
scope = self.scopeForTmp
datasetName = 'dummy'
srcPath = fileSpec.path
dstPath = mover_utils.construct_file_path(self.srcBasePath, scope,
fileSpec.lfn)
# remove
if os.path.exists(dstPath):
os.remove(dstPath)
# copy
tmpLog.debug('copy src={srcPath} dst={dstPath}'.format(
srcPath=srcPath, dstPath=dstPath))
dstDir = os.path.dirname(dstPath)
if not os.path.exists(dstDir):
os.makedirs(dstDir)
shutil.copyfile(srcPath, dstPath)
# collect files
tmpFile = dict()
tmpFile['scope'] = scope
tmpFile['name'] = fileSpec.lfn
tmpFile['bytes'] = fileSpec.fsize
if fileSpec.fileType not in files:
files[fileSpec.fileType] = []
files[fileSpec.fileType].append(tmpFile)
# loop over all file types to be registered to rucio
rucioAPI = RucioClient()
for fileType, fileList in iteritems(files):
# set destination RSE
if fileType in ['es_output', 'zip_output']:
dstRSE = self.dstRSE_ES
elif fileType == 'output':
dstRSE = self.dstRSE_Out
elif fileType == 'log':
dstRSE = self.dstRSE_Log
else:
errMsg = 'unsupported file type {0}'.format(fileType)
tmpLog.error(errMsg)
return (False, errMsg)
# skip if destination is None
if dstRSE is None:
continue
# make datasets if missing
if fileType not in transferDatasets:
try:
tmpScope = self.scopeForTmp
tmpDS = 'panda.harvester_stage_out.{0}'.format(
str(uuid.uuid4()))
rucioAPI.add_dataset(tmpScope,
tmpDS,
meta={'hidden': True},
lifetime=7 * 24 * 60 * 60,
files=fileList,
rse=self.srcRSE)
transferDatasets[fileType] = tmpDS
# add rule
tmpDID = dict()
tmpDID['scope'] = tmpScope
tmpDID['name'] = tmpDS
tmpRet = rucioAPI.add_replication_rule([tmpDID],
1,
dstRSE,
lifetime=7 * 24 *
60 * 60)
tmpTransferIDs = tmpRet[0]
transferIDs[fileType] = tmpTransferIDs
tmpLog.debug('register dataset {0} with rule {1}'.format(
tmpDS, str(tmpTransferIDs)))
except:
errMsg = core_utils.dump_error_message(tmpLog)
return (False, errMsg)
else:
# add files to existing dataset
try:
tmpScope = self.scopeForTmp
tmpDS = transferDatasets[fileType]
rucioAPI.add_files_to_dataset(tmpScope, tmpDS, fileList,
self.srcRSE)
tmpLog.debug('added files to {0}'.format(tmpDS))
except:
errMsg = core_utils.dump_error_message(tmpLog)
return (False, errMsg)
# set transfer datasets and rules
for fileSpec in jobspec.outFiles:
# skip zipped files
if fileSpec.zipFileID is not None:
continue
# skip already done
if fileSpec.status in ['finished', 'failed']:
continue
# skip if already processed
if 'transferDataset' in fileSpec.fileAttributes:
continue
# no destination
if fileSpec.fileType not in transferDatasets:
fileSpec.status = 'finished'
continue
# set dataset
fileSpec.fileAttributes['transferDataset'] = transferDatasets[
fileSpec.fileType]
# set rule
fileSpec.fileAttributes['transferID'] = transferIDs[
fileSpec.fileType]
# force update
fileSpec.force_update('fileAttributes')
# return
tmpLog.debug('done')
return (True, '')
def trigger_stage_out(self, jobspec):
"""Trigger the stage-out procedure for the job.
Output files are available through jobspec.get_outfile_specs(skip_done=False) which gives
a list of FileSpecs not yet done.
FileSpec.attemptNr shows how many times transfer was tried for the file so far.
:param jobspec: job specifications
:type jobspec: JobSpec
:return: A tuple of return code (True: success, False: fatal failure, None: temporary failure)
and error dialog
:rtype: (bool, string)
"""
# let gc clean up memory
gc.collect()
# make logger
tmpLog = self.make_logger(_logger,
'PandaID={0}'.format(jobspec.PandaID),
method_name='trigger_stage_out')
tmpLog.debug('start')
# get the environment
harvester_env = os.environ.copy()
#tmpLog.debug('Harvester environment : {}'.format(harvester_env))
xrdcpOutput = None
allfiles_transfered = True
overall_errMsg = ""
fileAttrs = jobspec.get_output_file_attributes()
# loop over all output files
for fileSpec in jobspec.get_output_file_specs(skip_done=True):
# fileSpec.objstoreID = 123
# fileSpec.fileAttributes['guid']
# construct source and destination paths
dstPath = mover_utils.construct_file_path(
self.dstBasePath, fileAttrs[fileSpec.lfn]['scope'],
fileSpec.lfn)
# local path
localPath = mover_utils.construct_file_path(
self.localBasePath, fileAttrs[fileSpec.lfn]['scope'],
fileSpec.lfn)
tmpLog.debug('fileSpec.path - {0} fileSpec.lfn = {1}'.format(
fileSpec.path, fileSpec.lfn))
localPath = fileSpec.path
if self.checkLocalPath:
# check if already exits
if os.path.exists(localPath):
# calculate checksum
checksum = core_utils.calc_adler32(localPath)
checksum = 'ad:{0}'.format(checksum)
if checksum == fileAttrs[fileSpec.lfn]['checksum']:
continue
# collect list of output files
if xrdcpOutput is None:
xrdcpOutput = [dstPath]
else:
if dstPath not in xrdcpOutput:
xrdcpOutput.append(dstPath)
# transfer using xrdcp one file at a time
tmpLog.debug('execute xrdcp')
args = ['xrdcp', '--nopbar', '--force']
args_files = [localPath, dstPath]
if self.xrdcpOpts is not None:
args += self.xrdcpOpts.split()
args += args_files
fileSpec.attemptNr += 1
try:
xrdcp_cmd = ' '.join(args)
tmpLog.debug('execute: {0}'.format(xrdcp_cmd))
process = subprocess.Popen(xrdcp_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=harvester_env,
shell=True)
try:
stdout, stderr = process.communicate(timeout=self.timeout)
except subprocess.TimeoutExpired:
process.kill()
stdout, stderr = process.communicate()
tmpLog.warning('command timeout')
return_code = process.returncode
if stdout is not None:
if not isinstance(stdout, str):
stdout = stdout.decode()
stdout = stdout.replace('\n', ' ')
if stderr is not None:
if not isinstance(stderr, str):
stderr = stderr.decode()
stderr = stderr.replace('\n', ' ')
tmpLog.debug("stdout: %s" % stdout)
tmpLog.debug("stderr: %s" % stderr)
except Exception:
core_utils.dump_error_message(tmpLog)
return_code = 1
if return_code == 0:
fileSpec.status = 'finished'
else:
overall_errMsg += "file - {0} did not transfer error code {1} ".format(
localPath, return_code)
allfiles_transfered = False
errMsg = 'failed with {0}'.format(return_code)
tmpLog.error(errMsg)
# check attemptNr
if fileSpec.attemptNr >= self.maxAttempts:
tmpLog.error(
'reached maxattempts: {0}, marked it as failed'.format(
self.maxAttempts))
fileSpec.status = 'failed'
# force update
fileSpec.force_update('status')
tmpLog.debug('file: {0} status: {1}'.format(
fileSpec.lfn, fileSpec.status))
del process, stdout, stderr
# end loop over output files
# nothing to transfer
if xrdcpOutput is None:
tmpLog.debug('done with no transfers')
return True, ''
# check if all files were transfered
tmpLog.debug('done')
if allfiles_transfered:
return True, ''
else:
return None, overall_errMsg
def make_local_access_path(self, scope, lfn):
return mover_utils.construct_file_path(self.localBasePath, scope, lfn)
def trigger_preparation(self, jobspec):
# make logger
tmpLog = self.make_logger(baseLogger,
'PandaID={0}'.format(jobspec.PandaID),
method_name='trigger_preparation')
tmpLog.debug('start')
# check that jobspec.computingSite is defined
if jobspec.computingSite is None:
# not found
tmpLog.error('jobspec.computingSite is not defined')
return False, 'jobspec.computingSite is not defined'
else:
tmpLog.debug('jobspec.computingSite : {0}'.format(
jobspec.computingSite))
# get input files
files = []
inFiles = jobspec.get_input_file_attributes(skip_ready=True)
# set path to each file
for inLFN, inFile in iteritems(inFiles):
inFile['path'] = mover_utils.construct_file_path(
self.basePath, inFile['scope'], inLFN)
tmpLog.debug('To check file: %s' % inFile)
if os.path.exists(inFile['path']):
# checksum = core_utils.calc_adler32(inFile['path'])
# checksum = 'ad:%s' % checksum
# tmpLog.debug('checksum for file %s is %s' % (inFile['path'], checksum))
# if 'checksum' in inFile and inFile['checksum'] and inFile['checksum'] == checksum:
# tmpLog.debug('File %s already exists at %s' % (inLFN, inFile['path']))
# continue
# lazy but unsafe check to be faster...
file_size = os.stat(inFile['path']).st_size
tmpLog.debug('file size for file %s is %s' %
(inFile['path'], file_size))
if 'fsize' in inFile and inFile['fsize'] and inFile[
'fsize'] == file_size:
tmpLog.debug('File %s already exists at %s' %
(inLFN, inFile['path']))
continue
dstpath = os.path.dirname(inFile['path'])
# check if path exists if not create it.
if not os.access(dstpath, os.F_OK):
os.makedirs(dstpath)
file_data = {
'scope': inFile['scope'],
'dataset': inFile.get('dataset'),
'lfn': inLFN,
'ddmendpoint': inFile.get('endpoint'),
'guid': inFile.get('guid'),
'workdir': dstpath,
}
pilotfilespec = PilotFileSpec(type='input', **file_data)
files.append(pilotfilespec)
# tmpLog.debug('files[] {0}'.format(files))
tmpLog.debug('path set')
allChecked = True
ErrMsg = 'These files failed to download : '
if files:
threads = []
n_files_per_thread = (len(files) + self.n_threads -
1) // self.n_threads
tmpLog.debug('num files per thread: %s' % n_files_per_thread)
for i in range(0, len(files), n_files_per_thread):
sub_files = files[i:i + n_files_per_thread]
thread = threading.Thread(target=self.stage_in,
kwargs={
'tmpLog': tmpLog,
'jobspec': jobspec,
'files': sub_files,
})
threads.append(thread)
[t.start() for t in threads]
while len(threads) > 0:
time.sleep(1)
threads = [t for t in threads if t and t.isAlive()]
tmpLog.info('Checking all files: {0}'.format(files))
for file in files:
if file.status_code != 0:
allChecked = False
ErrMsg = ErrMsg + (" %s " % file.lfn)
for inLFN, inFile in iteritems(inFiles):
if not os.path.isfile(inFile['path']):
allChecked = False
ErrMsg = ErrMsg + (" %s " % file.lfn)
# return
tmpLog.debug('stop')
if allChecked:
tmpLog.info('Looks like all files are successfully downloaded.')
return True, ''
else:
# keep retrying
return None, ErrMsg
def trigger_preparation(self, jobspec):
# make logger
tmpLog = self.make_logger(baseLogger,
'PandaID={0}'.format(jobspec.PandaID),
method_name='trigger_preparation')
tmpLog.debug('start')
# loop over all inputs
inFileInfo = jobspec.get_input_file_attributes()
gucInput = None
for tmpFileSpec in jobspec.inFiles:
# construct source and destination paths
srcPath = mover_utils.construct_file_path(
self.srcBasePath, inFileInfo[tmpFileSpec.lfn]['scope'],
tmpFileSpec.lfn)
dstPath = mover_utils.construct_file_path(
self.dstBasePath, inFileInfo[tmpFileSpec.lfn]['scope'],
tmpFileSpec.lfn)
# local access path
accPath = mover_utils.construct_file_path(
self.localBasePath, inFileInfo[tmpFileSpec.lfn]['scope'],
tmpFileSpec.lfn)
if self.checkLocalPath:
# check if already exits
if os.path.exists(accPath):
# calculate checksum
checksum = core_utils.calc_adler32(accPath)
checksum = 'ad:{0}'.format(checksum)
if checksum == inFileInfo[tmpFileSpec.lfn]['checksum']:
continue
# make directories if needed
if not os.path.isdir(os.path.dirname(accPath)):
os.makedirs(os.path.dirname(accPath))
# make input for globus-url-copy
if gucInput is None:
gucInput = tempfile.NamedTemporaryFile(mode='w',
delete=False,
suffix='_guc_in.tmp')
gucInput.write("{0} {1}\n".format(srcPath, dstPath))
tmpFileSpec.attemptNr += 1
# nothing to transfer
if gucInput is None:
tmpLog.debug('done with no transfers')
return True, ''
# transfer
tmpLog.debug('execute globus-url-copy')
gucInput.close()
args = ['globus-url-copy', '-f', gucInput.name, '-cd']
if self.gulOpts is not None:
args += self.gulOpts.split()
try:
tmpLog.debug('execute: ' + ' '.join(args))
p = subprocess.Popen(args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
try:
stdout, stderr = p.communicate(timeout=self.timeout)
except subprocess.TimeoutExpired:
p.kill()
stdout, stderr = p.communicate()
tmpLog.warning('command timeout')
return_code = p.returncode
if stdout is not None:
if not isinstance(stdout, str):
stdout = stdout.decode()
stdout = stdout.replace('\n', ' ')
if stderr is not None:
if not isinstance(stderr, str):
stderr = stderr.decode()
stderr = stderr.replace('\n', ' ')
tmpLog.debug("stdout: %s" % stdout)
tmpLog.debug("stderr: %s" % stderr)
except Exception:
core_utils.dump_error_message(tmpLog)
return_code = 1
os.remove(gucInput.name)
if return_code == 0:
tmpLog.debug('succeeded')
return True, ''
else:
errMsg = 'failed with {0}'.format(return_code)
tmpLog.error(errMsg)
# check attemptNr
for tmpFileSpec in jobspec.inFiles:
if tmpFileSpec.attemptNr >= self.maxAttempts:
errMsg = 'gave up due to max attempts'
tmpLog.error(errMsg)
return (False, errMsg)
return None, errMsg
def trigger_preparation(self, jobspec):
# get logger
tmpLog = self.make_logger(_logger, 'PandaID={0}'.format(jobspec.PandaID),
method_name='trigger_preparation')
tmpLog.debug('start')
# check that jobspec.computingSite is defined
if jobspec.computingSite is None:
# not found
tmpLog.error('jobspec.computingSite is not defined')
return False, 'jobspec.computingSite is not defined'
else:
tmpLog.debug('jobspec.computingSite : {0}'.format(jobspec.computingSite))
# test we have a Globus Transfer Client
if not self.tc :
errStr = 'failed to get Globus Transfer Client'
tmpLog.error(errStr)
return False, errStr
# get label
label = self.make_label(jobspec)
tmpLog.debug('label={0}'.format(label))
# get transfer tasks
tmpStat, transferTasks = globus_utils.get_transfer_tasks(tmpLog,self.tc,label)
if not tmpStat:
errStr = 'failed to get transfer tasks'
tmpLog.error(errStr)
return False, errStr
# check if already queued
if label in transferTasks:
tmpLog.debug('skip since already queued with {0}'.format(str(transferTasks[label])))
return True, ''
# set the Globus destination Endpoint id and path will get them from Agis eventually
from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper
queueConfigMapper = QueueConfigMapper()
queueConfig = queueConfigMapper.get_queue(jobspec.computingSite)
self.Globus_srcPath = queueConfig.preparator['Globus_srcPath']
self.srcEndpoint = queueConfig.preparator['srcEndpoint']
self.Globus_dstPath = self.basePath
#self.Globus_dstPath = queueConfig.preparator['Globus_dstPath']
self.dstEndpoint = queueConfig.preparator['dstEndpoint']
# get input files
files = []
lfns = []
inFiles = jobspec.get_input_file_attributes(skip_ready=True)
for inLFN, inFile in iteritems(inFiles):
# set path to each file
inFile['path'] = mover_utils.construct_file_path(self.basePath, inFile['scope'], inLFN)
dstpath = inFile['path']
# check if path exists if not create it.
if not os.access(self.basePath, os.F_OK):
os.makedirs(self.basePath)
# create the file paths for the Globus source and destination endpoints
Globus_srcpath = mover_utils.construct_file_path(self.Globus_srcPath, inFile['scope'], inLFN)
Globus_dstpath = mover_utils.construct_file_path(self.Globus_dstPath, inFile['scope'], inLFN)
files.append({'scope': inFile['scope'],
'name': inLFN,
'Globus_dstPath': Globus_dstpath,
'Globus_srcPath': Globus_srcpath})
lfns.append(inLFN)
tmpLog.debug('files[] {0}'.format(files))
try:
# Test endpoints for activation
tmpStatsrc, srcStr = globus_utils.check_endpoint_activation(tmpLog,self.tc,self.srcEndpoint)
tmpStatdst, dstStr = globus_utils.check_endpoint_activation(tmpLog,self.tc,self.dstEndpoint)
if tmpStatsrc and tmpStatdst:
errStr = 'source Endpoint and destination Endpoint activated'
tmpLog.debug(errStr)
else:
errStr = ''
if not tmpStatsrc :
errStr += ' source Endpoint not activated '
if not tmpStatdst :
errStr += ' destination Endpoint not activated '
tmpLog.error(errStr)
return False,errStr
# both endpoints activated now prepare to transfer data
if len(files) > 0:
tdata = TransferData(self.tc,
self.srcEndpoint,
self.dstEndpoint,
label=label,
sync_level="checksum")
# loop over all input files and add
for myfile in files:
tdata.add_item(myfile['Globus_srcPath'],myfile['Globus_dstPath'])
# submit
transfer_result = self.tc.submit_transfer(tdata)
# check status code and message
tmpLog.debug(str(transfer_result))
if transfer_result['code'] == "Accepted":
# succeeded
# set transfer ID which are used for later lookup
transferID = transfer_result['task_id']
jobspec.set_groups_to_files({transferID: {'lfns': lfns, 'groupStatus': 'active'}})
tmpLog.debug('done')
return True,''
else:
return False,transfer_result['message']
# if no files to transfer return True
return True, 'No files to transfer'
except:
errStat,errMsg = globus_utils.handle_globus_exception(tmpLog)
return errStat, {}
def trigger_preparation(self, jobspec):
# make logger
tmpLog = self.make_logger(baseLogger,
'PandaID={0}'.format(jobspec.PandaID),
method_name='trigger_preparation')
tmpLog.debug('start')
# get the environment
harvester_env = os.environ.copy()
#tmpLog.debug('Harvester environment : {}'.format(harvester_env))
# loop over all inputs
inFileInfo = jobspec.get_input_file_attributes()
xrdcpInput = None
allfiles_transfered = True
overall_errMsg = ""
for tmpFileSpec in jobspec.inFiles:
# construct source and destination paths
srcPath = mover_utils.construct_file_path(
self.srcBasePath, inFileInfo[tmpFileSpec.lfn]['scope'],
tmpFileSpec.lfn)
# local path
localPath = mover_utils.construct_file_path(
self.localBasePath, inFileInfo[tmpFileSpec.lfn]['scope'],
tmpFileSpec.lfn)
if self.checkLocalPath:
# check if already exits
if os.path.exists(localPath):
# calculate checksum
checksum = core_utils.calc_adler32(localPath)
checksum = 'ad:{0}'.format(checksum)
if checksum == inFileInfo[tmpFileSpec.lfn]['checksum']:
continue
# make directories if needed
if not os.path.isdir(os.path.dirname(localPath)):
os.makedirs(os.path.dirname(localPath))
tmpLog.debug('Make directory - {0}'.format(
os.path.dirname(localPath)))
# collect list of input files
if xrdcpInput is None:
xrdcpInput = [srcPath]
else:
xrdcpInput.append[srcPath]
# transfer using xrdcp one file at a time
tmpLog.debug('execute xrdcp')
args = ['xrdcp', '--nopbar', '--force']
args_files = [srcPath, localPath]
if self.xrdcpOpts is not None:
args += self.xrdcpOpts.split()
args += args_files
tmpFileSpec.attemptNr += 1
try:
xrdcp_cmd = ' '.join(args)
tmpLog.debug('execute: {0}'.format(xrdcp_cmd))
p = subprocess.Popen(xrdcp_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=harvester_env,
shell=True)
try:
stdout, stderr = p.communicate(timeout=self.timeout)
except subprocess.TimeoutExpired:
p.kill()
stdout, stderr = p.communicate()
tmpLog.warning('command timeout')
return_code = p.returncode
if stdout is not None:
if not isinstance(stdout, str):
stdout = stdout.decode()
stdout = stdout.replace('\n', ' ')
if stderr is not None:
if not isinstance(stderr, str):
stderr = stderr.decode()
stderr = stderr.replace('\n', ' ')
tmpLog.debug("stdout: %s" % stdout)
tmpLog.debug("stderr: %s" % stderr)
except Exception:
core_utils.dump_error_message(tmpLog)
return_code = 1
if return_code != 0:
overall_errMsg += "file - {0} did not transfer error code {1} ".format(
localPath, return_code)
allfiles_transfered = False
errMsg = 'failed with {0}'.format(return_code)
tmpLog.error(errMsg)
# check attemptNr
if tmpFileSpec.attemptNr >= self.maxAttempts:
errMsg = 'gave up due to max attempts'
tmpLog.error(errMsg)
return (False, errMsg)
# end loop over input files
# nothing to transfer
if xrdcpInput is None:
tmpLog.debug('done with no transfers')
return True, ''
# check if all files were transfered
if allfiles_transfered:
return True, ''
else:
return None, overall_errMsg
def trigger_stage_out(self, jobspec):
# make logger
tmpLog = self.make_logger(baseLogger, 'PandaID={0}'.format(jobspec.PandaID),
method_name='trigger_stage_out')
tmpLog.debug('start')
# loop over all files
files = dict()
transferIDs = dict()
transferDatasets = dict()
fileAttrs = jobspec.get_output_file_attributes()
for fileSpec in jobspec.outFiles:
# skip zipped files
if fileSpec.zipFileID is not None:
continue
# skip if already processed
if 'transferDataset' in fileSpec.fileAttributes:
if fileSpec.fileType not in transferDatasets:
transferDatasets[fileSpec.fileType] = fileSpec.fileAttributes['transferDataset']
if fileSpec.fileType not in transferIDs:
transferIDs[fileSpec.fileType] = fileSpec.fileAttributes['transferID']
continue
# set OS ID
if fileSpec.fileType == ['es_output', 'zip_output']:
fileSpec.objstoreID = self.objStoreID_ES
# make path where file is copied for transfer
if fileSpec.fileType != 'zip_output':
scope = fileAttrs[fileSpec.lfn]['scope']
datasetName = fileAttrs[fileSpec.lfn]['dataset']
else:
# use panda scope for zipped files
scope = self.scopeForTmp
datasetName = 'dummy'
srcPath = fileSpec.path
dstPath = mover_utils.construct_file_path(self.srcBasePath, scope, fileSpec.lfn)
# remove
if os.path.exists(dstPath):
os.remove(dstPath)
# copy
tmpLog.debug('copy src={srcPath} dst={dstPath}'.format(srcPath=srcPath, dstPath=dstPath))
dstDir = os.path.dirname(dstPath)
if not os.path.exists(dstDir):
os.makedirs(dstDir)
shutil.copyfile(srcPath, dstPath)
# collect files
tmpFile = dict()
tmpFile['scope'] = scope
tmpFile['name'] = fileSpec.lfn
tmpFile['bytes'] = fileSpec.fsize
if fileSpec.fileType not in files:
files[fileSpec.fileType] = []
files[fileSpec.fileType].append(tmpFile)
# loop over all file types to be registered to rucio
rucioAPI = RucioClient()
for fileType, fileList in iteritems(files):
# set destination RSE
if fileType in ['es_output', 'zip_output']:
dstRSE = self.dstRSE_ES
elif fileType == 'output':
dstRSE = self.dstRSE_Out
elif fileType == 'log':
dstRSE = self.dstRSE_Log
else:
errMsg = 'unsupported file type {0}'.format(fileType)
tmpLog.error(errMsg)
return (False, errMsg)
# skip if destination is None
if dstRSE is None:
continue
# make datasets if missing
if fileType not in transferDatasets:
try:
tmpScope = self.scopeForTmp
tmpDS = 'panda.harvester_stage_out.{0}'.format(str(uuid.uuid4()))
rucioAPI.add_dataset(tmpScope, tmpDS,
meta={'hidden': True},
lifetime=30*24*60*60,
files=fileList,
rse=self.srcRSE
)
transferDatasets[fileType] = tmpDS
# add rule
tmpDID = dict()
tmpDID['scope'] = tmpScope
tmpDID['name'] = tmpDS
tmpRet = rucioAPI.add_replication_rule([tmpDID], 1, dstRSE,
lifetime=30*24*60*60
)
tmpTransferIDs = tmpRet[0]
transferIDs[fileType] = tmpTransferIDs
tmpLog.debug('register dataset {0} with rule {1}'.format(tmpDS, str(tmpTransferIDs)))
except:
errMsg = core_utils.dump_error_message(tmpLog)
return (False, errMsg)
else:
# add files to existing dataset
try:
tmpScope = self.scopeForTmp
tmpDS = transferDatasets[fileType]
rucioAPI.add_files_to_dataset(tmpScope, tmpDS, fileList, self.srcRSE)
tmpLog.debug('added files to {0}'.format(tmpDS))
except:
errMsg = core_utils.dump_error_message(tmpLog)
return (False, errMsg)
# set transfer datasets and rules
for fileSpec in jobspec.outFiles:
# skip zipped files
if fileSpec.zipFileID is not None:
continue
# skip already done
if fileSpec.status in ['finished', 'failed']:
continue
# skip if already processed
if 'transferDataset' in fileSpec.fileAttributes:
continue
# no destination
if fileSpec.fileType not in transferDatasets:
fileSpec.status = 'finished'
continue
# set dataset
fileSpec.fileAttributes['transferDataset'] = transferDatasets[fileSpec.fileType]
# set rule
fileSpec.fileAttributes['transferID'] = transferIDs[fileSpec.fileType]
# force update
fileSpec.force_update('fileAttributes')
# return
tmpLog.debug('done')
return (True, '')
def trigger_stage_out(self, jobspec):
# make logger
tmpLog = self.make_logger(baseLogger, 'PandaID={0}'.format(jobspec.PandaID),
method_name='trigger_stage_out')
tmpLog.debug('start')
# loop over all files
gucInput = None
is_multistep = isinstance(self.intermediateBasePaths, list) and len(self.intermediateBasePaths) > 0
guc_inputs_list = [None] * (len(self.intermediateBasePaths) + 1) if is_multistep else []
for fileSpec in jobspec.outFiles:
# skip if already done
if fileSpec.status in ['finished', 'failed']:
continue
# scope
if fileSpec.fileType in ['es_output', 'zip_output']:
scope = self.scopeForTmp
else:
scope = fileSpec.fileAttributes.get('scope')
if scope is None:
scope = fileSpec.scope
# construct source and destination paths
srcPath = re.sub(self.srcOldBasePath, self.srcNewBasePath, fileSpec.path)
dstPath = mover_utils.construct_file_path(self.dstBasePath, scope, fileSpec.lfn)
# make tempfiles of paths to transfer
if is_multistep:
# multi-step transfer
for ibp_i in range(len(self.intermediateBasePaths) + 1):
base_paths_old = self.intermediateBasePaths[ibp_i - 1] if ibp_i > 0 else ''
base_paths_new = self.intermediateBasePaths[ibp_i] if ibp_i < len(self.intermediateBasePaths) else ''
src_base = base_paths_old[1] if isinstance(base_paths_old, list) else base_paths_old
dst_base = base_paths_new[0] if isinstance(base_paths_new, list) else base_paths_new
# construct temporary source and destination paths
tmp_src_path = re.sub(self.srcNewBasePath, src_base, srcPath)
tmp_dest_path = re.sub(self.srcNewBasePath, dst_base, srcPath)
# make input for globus-url-copy
if guc_inputs_list[ibp_i] is None:
guc_inputs_list[ibp_i] = tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='_guc_out_{0}.tmp'.format(ibp_i))
guc_input = guc_inputs_list[ibp_i]
if ibp_i == 0:
guc_input.write("{0} {1}\n".format(srcPath, tmp_dest_path))
tmpLog.debug("step {0}: {1} {2}".format(ibp_i + 1, srcPath, tmp_dest_path))
elif ibp_i == len(self.intermediateBasePaths):
guc_input.write("{0} {1}\n".format(tmp_src_path, dstPath))
tmpLog.debug("step {0}: {1} {2}".format(ibp_i + 1, tmp_src_path, dstPath))
else:
guc_input.write("{0} {1}\n".format(tmp_src_path, tmp_dest_path))
tmpLog.debug("step {0}: {1} {2}".format(ibp_i + 1, tmp_src_path, tmp_dest_path))
else:
# single-step transfer
# make input for globus-url-copy
if gucInput is None:
gucInput = tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='_guc_out.tmp')
gucInput.write("{0} {1}\n".format(srcPath, dstPath))
fileSpec.attemptNr += 1
# nothing to transfer
if is_multistep:
for guc_input in guc_inputs_list:
if guc_input is None:
tmpLog.debug('done with no transfers (multistep)')
return True, ''
else:
if gucInput is None:
tmpLog.debug('done with no transfers')
return True, ''
# transfer
if is_multistep:
[ guc_input.close() for guc_input in guc_inputs_list ]
tmpLog.debug('start multistep transfer')
guc_input_i = 1
for guc_input in guc_inputs_list:
args = ['globus-url-copy', '-f', guc_input.name, '-cd']
if self.gulOpts is not None:
args += self.gulOpts.split()
try:
tmpLog.debug('execute: ' + ' '.join(args))
p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
try:
stdout, stderr = p.communicate(timeout=self.timeout)
except subprocess.TimeoutExpired:
p.kill()
stdout, stderr = p.communicate()
tmpLog.warning('command timeout')
return_code = p.returncode
if stdout is not None:
if not isinstance(stdout, str):
stdout = stdout.decode()
stdout = stdout.replace('\n', ' ')
if stderr is not None:
if not isinstance(stderr, str):
stderr = stderr.decode()
stderr = stderr.replace('\n', ' ')
tmpLog.debug("stdout: %s" % stdout)
tmpLog.debug("stderr: %s" % stderr)
except Exception:
core_utils.dump_error_message(tmpLog)
return_code = 1
os.remove(guc_input.name)
if return_code == 0:
tmpLog.debug('step {0} succeeded'.format(guc_input_i))
guc_input_i += 1
else:
errMsg = 'step {0} failed with {1}'.format(guc_input_i, return_code)
tmpLog.error(errMsg)
# check attemptNr
for fileSpec in jobspec.inFiles:
if fileSpec.attemptNr >= self.maxAttempts:
errMsg = 'gave up due to max attempts'
tmpLog.error(errMsg)
return (False, errMsg)
return None, errMsg
tmpLog.debug('multistep transfer ({0} steps) succeeded'.format(len(guc_inputs_list)))
return True, ''
else:
gucInput.close()
args = ['globus-url-copy', '-f', gucInput.name, '-cd']
if self.gulOpts is not None:
args += self.gulOpts.split()
try:
tmpLog.debug('execute: ' + ' '.join(args))
p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
try:
stdout, stderr = p.communicate(timeout=self.timeout)
except subprocess.TimeoutExpired:
p.kill()
stdout, stderr = p.communicate()
tmpLog.warning('command timeout')
return_code = p.returncode
if stdout is not None:
if not isinstance(stdout, str):
stdout = stdout.decode()
stdout = stdout.replace('\n', ' ')
if stderr is not None:
if not isinstance(stderr, str):
stderr = stderr.decode()
stderr = stderr.replace('\n', ' ')
tmpLog.debug("stdout: %s" % stdout)
tmpLog.debug("stderr: %s" % stderr)
except Exception:
core_utils.dump_error_message(tmpLog)
return_code = 1
os.remove(gucInput.name)
if return_code == 0:
tmpLog.debug('succeeded')
return True, ''
else:
errMsg = 'failed with {0}'.format(return_code)
tmpLog.error(errMsg)
# check attemptNr
for fileSpec in jobspec.inFiles:
if fileSpec.attemptNr >= self.maxAttempts:
errMsg = 'gave up due to max attempts'
tmpLog.error(errMsg)
return (False, errMsg)
return None, errMsg
def trigger_stage_out(self, jobspec):
# make logger
tmpLog = self.make_logger(baseLogger, 'PandaID={0}'.format(jobspec.PandaID),
method_name='trigger_stage_out')
tmpLog.debug('start')
# loop over all files
gucInput = None
for fileSpec in jobspec.outFiles:
# skip if already done
if fileSpec.status in ['finished', 'failed']:
continue
# scope
if fileSpec.fileType in ['es_output', 'zip_output']:
scope = self.scopeForTmp
else:
scope = fileSpec.fileAttributes.get('scope')
if scope is None:
scope = fileSpec.scope
# construct source and destination paths
srcPath = re.sub(self.srcOldBasePath, self.srcNewBasePath, fileSpec.path)
dstPath = mover_utils.construct_file_path(self.dstBasePath, scope, fileSpec.lfn)
# make input for globus-url-copy
if gucInput is None:
gucInput = tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='_guc_out.tmp')
gucInput.write("{0} {1}\n".format(srcPath, dstPath))
fileSpec.attemptNr += 1
# nothing to transfer
if gucInput is None:
tmpLog.debug('done with no transfers')
return True, ''
# transfer
gucInput.close()
args = ['globus-url-copy', '-f', gucInput.name, '-cd']
if self.gulOpts is not None:
args += self.gulOpts.split()
try:
tmpLog.debug('execute globus-url-copy' + ' '.join(args))
p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
return_code = p.returncode
if stdout is not None:
stdout = stdout.replace('\n', ' ')
if stderr is not None:
stderr = stderr.replace('\n', ' ')
tmpLog.debug("stdout: %s" % stdout)
tmpLog.debug("stderr: %s" % stderr)
except Exception:
core_utils.dump_error_message(tmpLog)
return_code = 1
os.remove(gucInput.name)
if return_code == 0:
tmpLog.debug('succeeded')
return True, ''
else:
errMsg = 'failed with {0}'.format(return_code)
tmpLog.error(errMsg)
# check attemptNr
for fileSpec in jobspec.inFiles:
if fileSpec.attemptNr >= self.maxAttempts:
errMsg = 'gave up due to max attempts'
tmpLog.error(errMsg)
return (False, errMsg)
return None, errMsg
def trigger_preparation(self, jobspec):
# get logger
tmpLog = core_utils.make_logger(_logger,
'PandaID={0}'.format(jobspec.PandaID),
method_name='trigger_preparation')
tmpLog.debug('start')
# check that jobspec.computingSite is defined
if jobspec.computingSite is None:
# not found
tmpLog.error('jobspec.computingSite is not defined')
return False, 'jobspec.computingSite is not defined'
else:
tmpLog.debug('jobspec.computingSite : {0}'.format(
jobspec.computingSite))
# test we have a Globus Transfer Client
if not self.tc:
errStr = 'failed to get Globus Transfer Client'
tmpLog.error(errStr)
return False, errStr
# get label
label = self.make_label(jobspec)
tmpLog.debug('label={0}'.format(label))
# get transfer tasks
tmpStat, transferTasks = globus_utils.get_transfer_tasks(
tmpLog, self.tc, label)
if not tmpStat:
errStr = 'failed to get transfer tasks'
tmpLog.error(errStr)
return False, errStr
# check if already queued
if label in transferTasks:
tmpLog.debug('skip since already queued with {0}'.format(
str(transferTasks[label])))
return True, ''
# set the Globus destination Endpoint id and path will get them from Agis eventually
from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper
queueConfigMapper = QueueConfigMapper()
queueConfig = queueConfigMapper.get_queue(jobspec.computingSite)
self.Globus_srcPath = queueConfig.preparator['Globus_srcPath']
self.srcEndpoint = queueConfig.preparator['srcEndpoint']
self.Globus_dstPath = self.basePath
#self.Globus_dstPath = queueConfig.preparator['Globus_dstPath']
self.dstEndpoint = queueConfig.preparator['dstEndpoint']
# get input files
files = []
lfns = []
inFiles = jobspec.get_input_file_attributes(skip_ready=True)
for inLFN, inFile in iteritems(inFiles):
# set path to each file
inFile['path'] = mover_utils.construct_file_path(
self.basePath, inFile['scope'], inLFN)
dstpath = inFile['path']
# check if path exists if not create it.
if not os.access(self.basePath, os.F_OK):
os.makedirs(self.basePath)
# create the file paths for the Globus source and destination endpoints
Globus_srcpath = mover_utils.construct_file_path(
self.Globus_srcPath, inFile['scope'], inLFN)
Globus_dstpath = mover_utils.construct_file_path(
self.Globus_dstPath, inFile['scope'], inLFN)
files.append({
'scope': inFile['scope'],
'name': inLFN,
'Globus_dstPath': Globus_dstpath,
'Globus_srcPath': Globus_srcpath
})
lfns.append(inLFN)
tmpLog.debug('files[] {0}'.format(files))
try:
# Test endpoints for activation
tmpStatsrc, srcStr = globus_utils.check_endpoint_activation(
tmpLog, self.tc, self.srcEndpoint)
tmpStatdst, dstStr = globus_utils.check_endpoint_activation(
tmpLog, self.tc, self.dstEndpoint)
if tmpStatsrc and tmpStatdst:
errStr = 'source Endpoint and destination Endpoint activated'
tmpLog.debug(errStr)
else:
errStr = ''
if not tmpStatsrc:
errStr += ' source Endpoint not activated '
if not tmpStatdst:
errStr += ' destination Endpoint not activated '
tmpLog.error(errStr)
return False, errStr
# both endpoints activated now prepare to transfer data
if len(files) > 0:
tdata = TransferData(self.tc,
self.srcEndpoint,
self.dstEndpoint,
label=label,
sync_level="checksum")
# loop over all input files and add
for myfile in files:
tdata.add_item(myfile['Globus_srcPath'],
myfile['Globus_dstPath'])
# submit
transfer_result = self.tc.submit_transfer(tdata)
# check status code and message
tmpLog.debug(str(transfer_result))
if transfer_result['code'] == "Accepted":
# succeeded
# set transfer ID which are used for later lookup
transferID = transfer_result['task_id']
jobspec.set_groups_to_files(
{transferID: {
'lfns': lfns,
'groupStatus': 'active'
}})
tmpLog.debug('done')
return True, ''
else:
return False, transfer_result['message']
# if no files to transfer return True
return True, 'No files to transfer'
except:
errStat, errMsg = globus_utils.handle_globus_exception(tmpLog)
return errStat, {}
def trigger_preparation(self, jobspec):
# make logger
tmpLog = self.make_logger(baseLogger, 'PandaID={0}'.format(jobspec.PandaID),
method_name='trigger_preparation')
tmpLog.debug('start')
# check that jobspec.computingSite is defined
if jobspec.computingSite is None:
# not found
tmpLog.error('jobspec.computingSite is not defined')
return False, 'jobspec.computingSite is not defined'
else:
tmpLog.debug('jobspec.computingSite : {0}'.format(jobspec.computingSite))
# get input files
files = []
inFiles = jobspec.get_input_file_attributes(skip_ready=True)
# set path to each file
for inLFN, inFile in iteritems(inFiles):
inFile['path'] = mover_utils.construct_file_path(self.basePath, inFile['scope'], inLFN)
tmpLog.debug('To check file: %s' % inFile)
if os.path.exists(inFile['path']):
checksum = core_utils.calc_adler32(inFile['path'])
checksum = 'ad:%s' % checksum
tmpLog.debug('checksum for file %s is %s' % (inFile['path'], checksum))
if 'checksum' in inFile and inFile['checksum'] and inFile['checksum'] == checksum:
tmpLog.debug('File %s already exists at %s' % (inLFN, inFile['path']))
continue
dstpath = os.path.dirname(inFile['path'])
# check if path exists if not create it.
if not os.access(dstpath, os.F_OK):
os.makedirs(dstpath)
files.append({'scope': inFile['scope'],
'name': inLFN,
'destination': dstpath})
tmpLog.debug('files[] {0}'.format(files))
allChecked = True
ErrMsg = 'These files failed to download : '
if files:
threads = []
n_files_per_thread = (len(files) + self.n_threads - 1) / self.n_threads
tmpLog.debug('num files per thread: %s' % n_files_per_thread)
for i in range(0, len(files), n_files_per_thread):
sub_files = files[i:i + n_files_per_thread]
thread = threading.Thread(target=self.stage_in, kwargs={'tmpLog': tmpLog, 'jobspec': jobspec, 'files': sub_files})
threads.append(thread)
[t.start() for t in threads]
while len(threads) > 0:
time.sleep(1)
threads = [t for t in threads if t and t.isAlive()]
tmpLog.info('Checking all files: {0}'.format(files))
for file in files:
if file['errno'] != 0:
allChecked = False
ErrMsg = ErrMsg + (" %s " % file['name'])
# return
tmpLog.debug('stop')
if allChecked:
tmpLog.info('Looks like all files are successfully downloaded.')
return True, ''
else:
return False, ErrMsg
