def check_status(self, jobspec):
# get logger
tmpLog = core_utils.make_logger(_logger,
'PandaID={0}'.format(jobspec.PandaID),
method_name='check_status')
# get groups of input files except ones already in ready state
transferGroups = jobspec.get_groups_of_input_files(skip_ready=True)
#print type(transferGroups)," ",transferGroups
# update transfer status
# get label
label = self.make_label(jobspec)
tmpLog.debug('label={0}'.format(label))
# get transfer task
tmpStat, transferTasks = globus_utils.get_transfer_tasks(
tmpLog, self.tc, label)
# return a temporary error when failed to get task
if not tmpStat:
errStr = 'failed to get transfer task'
tmpLog.error(errStr)
return None, errStr
# return a fatal error when task is missing # FIXME retry instead?
if label not in transferTasks:
errStr = 'transfer task is missing'
tmpLog.error(errStr)
return False, errStr
# succeeded
if transferTasks[label]['status'] == 'SUCCEEDED':
transferID = transferTasks[label]['task_id']
jobspec.update_group_status_in_files(transferID, 'done')
tmpLog.debug('transfer task succeeded')
return True, ''
# failed
if transferTasks[label]['status'] == 'FAILED':
errStr = 'transfer task failed'
tmpLog.error(errStr)
return False, errStr
# another status
tmpStr = 'transfer task is in {0}'.format(
transferTasks[label]['status'])
tmpLog.debug(tmpStr)
return None, ''
def check_status(self, jobspec):
# get logger
tmpLog = self.make_logger(_logger, 'PandaID={0}'.format(jobspec.PandaID),
method_name='check_status')
# get groups of input files except ones already in ready state
transferGroups = jobspec.get_groups_of_input_files(skip_ready=True)
#print type(transferGroups)," ",transferGroups
# update transfer status
# get label
label = self.make_label(jobspec)
tmpLog.debug('label={0}'.format(label))
# get transfer task
tmpStat, transferTasks = globus_utils.get_transfer_tasks(tmpLog,self.tc,label)
# return a temporary error when failed to get task
if not tmpStat:
errStr = 'failed to get transfer task'
tmpLog.error(errStr)
return None, errStr
# return a fatal error when task is missing # FIXME retry instead?
if label not in transferTasks:
errStr = 'transfer task is missing'
tmpLog.error(errStr)
return False, errStr
# succeeded
if transferTasks[label]['status'] == 'SUCCEEDED':
transferID = transferTasks[label]['task_id']
jobspec.update_group_status_in_files(transferID, 'done')
tmpLog.debug('transfer task succeeded')
return True, ''
# failed
if transferTasks[label]['status'] == 'FAILED':
errStr = 'transfer task failed'
tmpLog.error(errStr)
return False, errStr
# another status
tmpStr = 'transfer task is in {0}'.format(transferTasks[label]['status'])
tmpLog.debug(tmpStr)
return None, ''
def check_stage_out_status(self, jobspec):
# make logger
tmpLog = self.make_logger(_logger,
'PandaID={0}'.format(jobspec.PandaID),
method_name='check_stage_out_status')
tmpLog.debug('start')
# get label
label = self.make_label(jobspec)
tmpLog.debug('label={0}'.format(label))
# get transfer task
tmpStat, transferTasks = globus_utils.get_transfer_tasks(
tmpLog, self.tc, label)
# return a temporary error when failed to get task
if not tmpStat:
errStr = 'failed to get transfer task'
tmpLog.error(errStr)
return None, errStr
# return a fatal error when task is missing # FIXME retry instead?
if label not in transferTasks:
errStr = 'transfer task is missing'
tmpLog.error(errStr)
return False, errStr
# succeeded
transferID = transferTasks[label]['task_id']
if transferTasks[label]['status'] == 'SUCCEEDED':
tmpLog.debug('transfer task {} succeeded'.format(transferID))
self.set_FileSpec_status(jobspec, 'finished')
return True, ''
# failed
if transferTasks[label]['status'] == 'FAILED':
errStr = 'transfer task {} failed'.format(transferID)
tmpLog.error(errStr)
self.set_FileSpec_status(jobspec, 'failed')
return False, errStr
# another status
tmpStr = 'transfer task {0} status: {1}'.format(
transferID, transferTasks[label]['status'])
tmpLog.debug(tmpStr)
return None, ''
def check_status(self, jobspec):
# make logger
tmpLog = self.make_logger(_logger, 'PandaID={0}'.format(jobspec.PandaID),
method_name='check_status')
tmpLog.debug('start')
# get label
label = self.make_label(jobspec)
tmpLog.debug('label={0}'.format(label))
# get transfer task
tmpStat, transferTasks = globus_utils.get_transfer_tasks(tmpLog,self.tc,label)
# return a temporary error when failed to get task
if not tmpStat:
errStr = 'failed to get transfer task'
tmpLog.error(errStr)
return None, errStr
# return a fatal error when task is missing # FIXME retry instead?
if label not in transferTasks:
errStr = 'transfer task is missing'
tmpLog.error(errStr)
return False, errStr
# succeeded
transferID = transferTasks[label]['task_id']
if transferTasks[label]['status'] == 'SUCCEEDED':
tmpLog.debug('transfer task {} succeeded'.format(transferID))
self.set_FileSpec_status(jobspec,'finished')
return True, ''
# failed
if transferTasks[label]['status'] == 'FAILED':
errStr = 'transfer task {} failed'.format(transferID)
tmpLog.error(errStr)
self.set_FileSpec_status(jobspec,'failed')
return False, errStr
# another status
tmpStr = 'transfer task {0} status: {1}'.format(transferID,transferTasks[label]['status'])
tmpLog.debug(tmpStr)
return None, ''
def trigger_preparation(self, jobspec):
# get logger
tmpLog = core_utils.make_logger(_logger,
'PandaID={0}'.format(jobspec.PandaID),
method_name='trigger_preparation')
tmpLog.debug('start')
# check that jobspec.computingSite is defined
if jobspec.computingSite is None:
# not found
tmpLog.error('jobspec.computingSite is not defined')
return False, 'jobspec.computingSite is not defined'
else:
tmpLog.debug('jobspec.computingSite : {0}'.format(
jobspec.computingSite))
# test we have a Globus Transfer Client
if not self.tc:
errStr = 'failed to get Globus Transfer Client'
tmpLog.error(errStr)
return False, errStr
# get label
label = self.make_label(jobspec)
tmpLog.debug('label={0}'.format(label))
# get transfer tasks
tmpStat, transferTasks = globus_utils.get_transfer_tasks(
tmpLog, self.tc, label)
if not tmpStat:
errStr = 'failed to get transfer tasks'
tmpLog.error(errStr)
return False, errStr
# check if already queued
if label in transferTasks:
tmpLog.debug('skip since already queued with {0}'.format(
str(transferTasks[label])))
return True, ''
# set the Globus destination Endpoint id and path will get them from Agis eventually
from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper
queueConfigMapper = QueueConfigMapper()
queueConfig = queueConfigMapper.get_queue(jobspec.computingSite)
self.Globus_srcPath = queueConfig.preparator['Globus_srcPath']
self.srcEndpoint = queueConfig.preparator['srcEndpoint']
self.Globus_dstPath = self.basePath
#self.Globus_dstPath = queueConfig.preparator['Globus_dstPath']
self.dstEndpoint = queueConfig.preparator['dstEndpoint']
# get input files
files = []
lfns = []
inFiles = jobspec.get_input_file_attributes(skip_ready=True)
for inLFN, inFile in iteritems(inFiles):
# set path to each file
inFile['path'] = mover_utils.construct_file_path(
self.basePath, inFile['scope'], inLFN)
dstpath = inFile['path']
# check if path exists if not create it.
if not os.access(self.basePath, os.F_OK):
os.makedirs(self.basePath)
# create the file paths for the Globus source and destination endpoints
Globus_srcpath = mover_utils.construct_file_path(
self.Globus_srcPath, inFile['scope'], inLFN)
Globus_dstpath = mover_utils.construct_file_path(
self.Globus_dstPath, inFile['scope'], inLFN)
files.append({
'scope': inFile['scope'],
'name': inLFN,
'Globus_dstPath': Globus_dstpath,
'Globus_srcPath': Globus_srcpath
})
lfns.append(inLFN)
tmpLog.debug('files[] {0}'.format(files))
try:
# Test endpoints for activation
tmpStatsrc, srcStr = globus_utils.check_endpoint_activation(
tmpLog, self.tc, self.srcEndpoint)
tmpStatdst, dstStr = globus_utils.check_endpoint_activation(
tmpLog, self.tc, self.dstEndpoint)
if tmpStatsrc and tmpStatdst:
errStr = 'source Endpoint and destination Endpoint activated'
tmpLog.debug(errStr)
else:
errStr = ''
if not tmpStatsrc:
errStr += ' source Endpoint not activated '
if not tmpStatdst:
errStr += ' destination Endpoint not activated '
tmpLog.error(errStr)
return False, errStr
# both endpoints activated now prepare to transfer data
if len(files) > 0:
tdata = TransferData(self.tc,
self.srcEndpoint,
self.dstEndpoint,
label=label,
sync_level="checksum")
# loop over all input files and add
for myfile in files:
tdata.add_item(myfile['Globus_srcPath'],
myfile['Globus_dstPath'])
# submit
transfer_result = self.tc.submit_transfer(tdata)
# check status code and message
tmpLog.debug(str(transfer_result))
if transfer_result['code'] == "Accepted":
# succeeded
# set transfer ID which are used for later lookup
transferID = transfer_result['task_id']
jobspec.set_groups_to_files(
{transferID: {
'lfns': lfns,
'groupStatus': 'active'
}})
tmpLog.debug('done')
return True, ''
else:
return False, transfer_result['message']
# if no files to transfer return True
return True, 'No files to transfer'
except:
errStat, errMsg = globus_utils.handle_globus_exception(tmpLog)
return errStat, {}
def trigger_preparation(self, jobspec):
# get logger
tmpLog = self.make_logger(_logger, 'PandaID={0}'.format(jobspec.PandaID),
method_name='trigger_preparation')
tmpLog.debug('start')
# check that jobspec.computingSite is defined
if jobspec.computingSite is None:
# not found
tmpLog.error('jobspec.computingSite is not defined')
return False, 'jobspec.computingSite is not defined'
else:
tmpLog.debug('jobspec.computingSite : {0}'.format(jobspec.computingSite))
# test we have a Globus Transfer Client
if not self.tc :
errStr = 'failed to get Globus Transfer Client'
tmpLog.error(errStr)
return False, errStr
# get label
label = self.make_label(jobspec)
tmpLog.debug('label={0}'.format(label))
# get transfer tasks
tmpStat, transferTasks = globus_utils.get_transfer_tasks(tmpLog,self.tc,label)
if not tmpStat:
errStr = 'failed to get transfer tasks'
tmpLog.error(errStr)
return False, errStr
# check if already queued
if label in transferTasks:
tmpLog.debug('skip since already queued with {0}'.format(str(transferTasks[label])))
return True, ''
# set the Globus destination Endpoint id and path will get them from Agis eventually
from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper
queueConfigMapper = QueueConfigMapper()
queueConfig = queueConfigMapper.get_queue(jobspec.computingSite)
self.Globus_srcPath = queueConfig.preparator['Globus_srcPath']
self.srcEndpoint = queueConfig.preparator['srcEndpoint']
self.Globus_dstPath = self.basePath
#self.Globus_dstPath = queueConfig.preparator['Globus_dstPath']
self.dstEndpoint = queueConfig.preparator['dstEndpoint']
# get input files
files = []
lfns = []
inFiles = jobspec.get_input_file_attributes(skip_ready=True)
for inLFN, inFile in iteritems(inFiles):
# set path to each file
inFile['path'] = mover_utils.construct_file_path(self.basePath, inFile['scope'], inLFN)
dstpath = inFile['path']
# check if path exists if not create it.
if not os.access(self.basePath, os.F_OK):
os.makedirs(self.basePath)
# create the file paths for the Globus source and destination endpoints
Globus_srcpath = mover_utils.construct_file_path(self.Globus_srcPath, inFile['scope'], inLFN)
Globus_dstpath = mover_utils.construct_file_path(self.Globus_dstPath, inFile['scope'], inLFN)
files.append({'scope': inFile['scope'],
'name': inLFN,
'Globus_dstPath': Globus_dstpath,
'Globus_srcPath': Globus_srcpath})
lfns.append(inLFN)
tmpLog.debug('files[] {0}'.format(files))
try:
# Test endpoints for activation
tmpStatsrc, srcStr = globus_utils.check_endpoint_activation(tmpLog,self.tc,self.srcEndpoint)
tmpStatdst, dstStr = globus_utils.check_endpoint_activation(tmpLog,self.tc,self.dstEndpoint)
if tmpStatsrc and tmpStatdst:
errStr = 'source Endpoint and destination Endpoint activated'
tmpLog.debug(errStr)
else:
errStr = ''
if not tmpStatsrc :
errStr += ' source Endpoint not activated '
if not tmpStatdst :
errStr += ' destination Endpoint not activated '
tmpLog.error(errStr)
return False,errStr
# both endpoints activated now prepare to transfer data
if len(files) > 0:
tdata = TransferData(self.tc,
self.srcEndpoint,
self.dstEndpoint,
label=label,
sync_level="checksum")
# loop over all input files and add
for myfile in files:
tdata.add_item(myfile['Globus_srcPath'],myfile['Globus_dstPath'])
# submit
transfer_result = self.tc.submit_transfer(tdata)
# check status code and message
tmpLog.debug(str(transfer_result))
if transfer_result['code'] == "Accepted":
# succeeded
# set transfer ID which are used for later lookup
transferID = transfer_result['task_id']
jobspec.set_groups_to_files({transferID: {'lfns': lfns, 'groupStatus': 'active'}})
tmpLog.debug('done')
return True,''
else:
return False,transfer_result['message']
# if no files to transfer return True
return True, 'No files to transfer'
except:
errStat,errMsg = globus_utils.handle_globus_exception(tmpLog)
return errStat, {}
def trigger_stage_out(self, jobspec):
# make logger
tmpLog = self.make_logger(_logger,
'PandaID={0}'.format(jobspec.PandaID),
method_name='trigger_stage_out')
tmpLog.debug('start')
# default return
tmpRetVal = (True, '')
# check that jobspec.computingSite is defined
if jobspec.computingSite is None:
# not found
tmpLog.error('jobspec.computingSite is not defined')
return False, 'jobspec.computingSite is not defined'
else:
tmpLog.debug('jobspec.computingSite : {0}'.format(
jobspec.computingSite))
# test we have a Globus Transfer Client
if not self.tc:
errStr = 'failed to get Globus Transfer Client'
tmpLog.error(errStr)
return False, errStr
# get label
label = self.make_label(jobspec)
tmpLog.debug('label={0}'.format(label))
# get transfer tasks
tmpStat, transferTasks = globus_utils.get_transfer_tasks(
tmpLog, self.tc, label)
if not tmpStat:
errStr = 'failed to get transfer tasks'
tmpLog.error(errStr)
return False, errStr
# check if already queued
if label in transferTasks:
tmpLog.debug('skip since already queued with {0}'.format(
str(transferTasks[label])))
return True, ''
# set the Globus destination Endpoint id and path will get them from Agis eventually
from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper
queueConfigMapper = QueueConfigMapper()
queueConfig = queueConfigMapper.get_queue(jobspec.computingSite)
#self.Globus_srcPath = queueConfig.stager['Globus_srcPath']
self.srcEndpoint = queueConfig.stager['srcEndpoint']
self.Globus_srcPath = self.basePath
self.Globus_dstPath = queueConfig.stager['Globus_dstPath']
self.dstEndpoint = queueConfig.stager['dstEndpoint']
# Test the endpoints and create the transfer data class
errMsg = None
try:
# Test endpoints for activation
tmpStatsrc, srcStr = globus_utils.check_endpoint_activation(
tmpLog, self.tc, self.srcEndpoint)
tmpStatdst, dstStr = globus_utils.check_endpoint_activation(
tmpLog, self.tc, self.dstEndpoint)
if tmpStatsrc and tmpStatdst:
errStr = 'source Endpoint and destination Endpoint activated'
tmpLog.debug(errStr)
else:
errMsg = ''
if not tmpStatsrc:
errMsg += ' source Endpoint not activated '
if not tmpStatdst:
errMsg += ' destination Endpoint not activated '
tmpLog.error(errMsg)
tmpRetVal = (False, errMsg)
return tmpRetVal
# both endpoints activated now prepare to transfer data
tdata = TransferData(self.tc,
self.srcEndpoint,
self.dstEndpoint,
label=label,
sync_level="checksum")
except:
errStat, errMsg = globus_utils.handle_globus_exception(tmpLog)
tmpRetVal = (errStat, errMsg)
return tmpRetVal
# loop over all files
fileAttrs = jobspec.get_output_file_attributes()
lfns = []
for fileSpec in jobspec.outFiles:
scope = fileAttrs[fileSpec.lfn]['scope']
hash = hashlib.md5()
hash.update('%s:%s' % (scope, fileSpec.lfn))
hash_hex = hash.hexdigest()
correctedscope = "/".join(scope.split('.'))
srcURL = fileSpec.path
dstURL = "{endPoint}/{scope}/{hash1}/{hash2}/{lfn}".format(
endPoint=self.Globus_dstPath,
scope=correctedscope,
hash1=hash_hex[0:2],
hash2=hash_hex[2:4],
lfn=fileSpec.lfn)
tmpLog.debug('src={srcURL} dst={dstURL}'.format(srcURL=srcURL,
dstURL=dstURL))
# add files to transfer object - tdata
if os.access(srcURL, os.R_OK):
tmpLog.debug("tdata.add_item({},{})".format(srcURL, dstURL))
tdata.add_item(srcURL, dstURL)
lfns.append(fileSpec.lfn)
else:
errMsg = "source file {} does not exist".format(srcURL)
tmpLog.error(errMsg)
tmpRetVal = (False, errMsg)
return tmpRetVal
# submit transfer
try:
transfer_result = self.tc.submit_transfer(tdata)
# check status code and message
tmpLog.debug(str(transfer_result))
if transfer_result['code'] == "Accepted":
# succeeded
# set transfer ID which are used for later lookup
transferID = transfer_result['task_id']
tmpLog.debug(
'successfully submitted id={0}'.format(transferID))
jobspec.set_groups_to_files(
{transferID: {
'lfns': lfns,
'groupStatus': 'active'
}})
# set
for fileSpec in jobspec.outFiles:
if fileSpec.fileAttributes == None:
fileSpec.fileAttributes = {}
fileSpec.fileAttributes['transferID'] = transferID
else:
tmpRetVal = (False, transfer_result['message'])
except Exception as e:
errStat, errMsg = globus_utils.handle_globus_exception(tmpLog)
if errMsg is None:
errtype, errvalue = sys.exc_info()[:2]
errMsg = "{0} {1}".format(errtype.__name__, errvalue)
tmpRetVal = (errStat, errMsg)
# return
tmpLog.debug('done')
return tmpRetVal
def trigger_stage_out(self, jobspec):
# make logger
tmpLog = self.make_logger(_logger, 'PandaID={0}'.format(jobspec.PandaID),
method_name='trigger_stage_out')
tmpLog.debug('start')
# default return
tmpRetVal = (True, '')
# check that jobspec.computingSite is defined
if jobspec.computingSite is None:
# not found
tmpLog.error('jobspec.computingSite is not defined')
return False, 'jobspec.computingSite is not defined'
else:
tmpLog.debug('jobspec.computingSite : {0}'.format(jobspec.computingSite))
# test we have a Globus Transfer Client
if not self.tc :
errStr = 'failed to get Globus Transfer Client'
tmpLog.error(errStr)
return False, errStr
# get label
label = self.make_label(jobspec)
tmpLog.debug('label={0}'.format(label))
# get transfer tasks
tmpStat, transferTasks = globus_utils.get_transfer_tasks(tmpLog,self.tc,label)
if not tmpStat:
errStr = 'failed to get transfer tasks'
tmpLog.error(errStr)
return False, errStr
# check if already queued
if label in transferTasks:
tmpLog.debug('skip since already queued with {0}'.format(str(transferTasks[label])))
return True, ''
# set the Globus destination Endpoint id and path will get them from Agis eventually
from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper
queueConfigMapper = QueueConfigMapper()
queueConfig = queueConfigMapper.get_queue(jobspec.computingSite)
#self.Globus_srcPath = queueConfig.stager['Globus_srcPath']
self.srcEndpoint = queueConfig.stager['srcEndpoint']
self.Globus_srcPath = self.basePath
self.Globus_dstPath = queueConfig.stager['Globus_dstPath']
self.dstEndpoint = queueConfig.stager['dstEndpoint']
# Test the endpoints and create the transfer data class
errMsg = None
try:
# Test endpoints for activation
tmpStatsrc, srcStr = globus_utils.check_endpoint_activation(tmpLog,self.tc,self.srcEndpoint)
tmpStatdst, dstStr = globus_utils.check_endpoint_activation(tmpLog,self.tc,self.dstEndpoint)
if tmpStatsrc and tmpStatdst:
errStr = 'source Endpoint and destination Endpoint activated'
tmpLog.debug(errStr)
else:
errMsg = ''
if not tmpStatsrc :
errMsg += ' source Endpoint not activated '
if not tmpStatdst :
errMsg += ' destination Endpoint not activated '
tmpLog.error(errMsg)
tmpRetVal = (False,errMsg)
return tmpRetVal
# both endpoints activated now prepare to transfer data
tdata = TransferData(self.tc,
self.srcEndpoint,
self.dstEndpoint,
label=label,
sync_level="checksum")
except:
errStat,errMsg = globus_utils.handle_globus_exception(tmpLog)
tmpRetVal = (errStat, errMsg)
return tmpRetVal
# loop over all files
fileAttrs = jobspec.get_output_file_attributes()
lfns = []
for fileSpec in jobspec.outFiles:
scope = fileAttrs[fileSpec.lfn]['scope']
hash = hashlib.md5()
hash.update('%s:%s' % (scope, fileSpec.lfn))
hash_hex = hash.hexdigest()
correctedscope = "/".join(scope.split('.'))
srcURL = fileSpec.path
dstURL = "{endPoint}/{scope}/{hash1}/{hash2}/{lfn}".format(endPoint=self.Globus_dstPath,
scope=correctedscope,
hash1=hash_hex[0:2],
hash2=hash_hex[2:4],
lfn=fileSpec.lfn)
tmpLog.debug('src={srcURL} dst={dstURL}'.format(srcURL=srcURL, dstURL=dstURL))
# add files to transfer object - tdata
if os.access(srcURL, os.R_OK):
tmpLog.debug("tdata.add_item({},{})".format(srcURL,dstURL))
tdata.add_item(srcURL,dstURL)
lfns.append(fileSpec.lfn)
else:
errMsg = "source file {} does not exist".format(srcURL)
tmpLog.error(errMsg)
tmpRetVal = (False,errMsg)
return tmpRetVal
# submit transfer
try:
transfer_result = self.tc.submit_transfer(tdata)
# check status code and message
tmpLog.debug(str(transfer_result))
if transfer_result['code'] == "Accepted":
# succeeded
# set transfer ID which are used for later lookup
transferID = transfer_result['task_id']
tmpLog.debug('successfully submitted id={0}'.format(transferID))
jobspec.set_groups_to_files({transferID: {'lfns': lfns, 'groupStatus': 'active'}})
# set
for fileSpec in jobspec.outFiles:
if fileSpec.fileAttributes == None:
fileSpec.fileAttributes = {}
fileSpec.fileAttributes['transferID'] = transferID
else:
tmpRetVal = (False, transfer_result['message'])
except Exception as e:
errStat,errMsg = globus_utils.handle_globus_exception(tmpLog)
if errMsg is None:
errtype, errvalue = sys.exc_info()[:2]
errMsg = "{0} {1}".format(errtype.__name__, errvalue)
tmpRetVal = (errStat,errMsg)
# return
tmpLog.debug('done')
return tmpRetVal