class CloserAtlasPlugin:
# constructor
def __init__(self,job,datasets,log):
self.jobSpec = job
self.datasets = datasets
self.tmpLog = LogWrapper(log,"{0} CloserAtlasPlugin".format(self.jobSpec.PandaID))
# execute
def execute(self):
try:
# only for production
if not self.jobSpec.prodSourceLabel in ['managed','test']:
return True
# only for urgent or high prio
if not self.jobSpec.processingType in ['urgent'] and self.jobSpec.currentPriority <= 1000:
return True
# close datasets
for datasetSpec in self.datasets:
if re.search('_sub\d+$',datasetSpec.name) == None:
continue
if datasetSpec.status != 'tobeclosed':
continue
try:
self.tmpLog.debug('immediate close {0}'.format(datasetSpec.name))
rucioAPI.closeDataset(datasetSpec.name)
except:
errtype,errvalue = sys.exc_info()[:2]
self.tmpLog.warning('failed to close : {0} {1}'.format(errtype,errvalue))
except:
errtype,errvalue = sys.exc_info()[:2]
self.tmpLog.warning('failed to execute : {0} {1}'.format(errtype,errvalue))
return True
def _getPFNFromLFC(lfns,
dq2url,
guids,
storageName,
scopeList=[],
tmpLog=None):
if tmpLog == None:
tmpLog = LogWrapper(_log, logPrefix)
tmpLog.debug('_getPFNFromLFC %s %s / %s LFNs:%s %s' %
(dq2url, str(storageName), len(lfns), str(
lfns[:3]), str(scopeList[:3])))
outStr = ''
# check paramter
if guids == [] or storageName == [] or (len(lfns) != len(guids)):
tmpLog.debug('_getPFNFromLFC done with empty list')
return outStr
# check scopeList
if not scopeList in [None, []] and len(lfns) != len(scopeList):
tmpLog.warning('_getPFNFromLFC wrong scopeList %s %s %s %s' %
(dq2url, str(storageName), str(lfns), str(scopeList)))
tmpLog.error('_getPFNFromLFC failed')
return outStr
# loop over all LFNs
iLFN = 0
nLFN = 1000
strFiles = ''
outStr = ''
for iLFN in range(len(lfns)):
if scopeList != []:
strFiles += '%s %s %s\n' % (lfns[iLFN], guids[iLFN],
scopeList[iLFN])
else:
strFiles += '%s %s\n' % (lfns[iLFN], guids[iLFN])
# bulk operation
if (iLFN + 1) % nLFN == 0 or (iLFN + 1) >= len(lfns):
# write to file
inFileName = '%s/lfcin.%s' % (panda_config.logdir,
commands.getoutput('uuidgen'))
ifile = open(inFileName, 'w')
ifile.write(strFiles)
ifile.close()
# construct commands
strStorage = ''
for storage in storageName:
strStorage += '%s,' % storage
strStorage = strStorage[:-1]
com = 'cd %s > /dev/null 2>&1; export HOME=%s; ' % (
panda_config.home_dir_cwd, panda_config.home_dir_cwd)
com += 'unset LD_LIBRARY_PATH; unset PYTHONPATH; export PATH=/usr/local/bin:/bin:/usr/bin; '
com+= 'source %s; %s/python -Wignore %s/LFCclient.py -f %s -l %s -s %s' % \
(panda_config.glite_source,panda_config.native_python32,panda_config.lfcClient_dir,
inFileName,dq2url,strStorage)
tmpLog.debug(com)
# exeute
status, output = commands.getstatusoutput(com)
tmpLog.debug(status)
if status == 0:
outStr += output
else:
tmpLog.error("_getPFNFromLFC : %s %s %s" %
(dq2url, status, output))
# send message to logger
try:
# make message
message = 'LFC access : %s %s %s' % (dq2url, status,
output)
# get logger
_pandaLogger = PandaLogger()
_pandaLogger.lock()
_pandaLogger.setParams({'Type': 'broker_util'})
logger = _pandaLogger.getHttpLogger(
panda_config.loggername)
# add message
logger.error(message)
# release HTTP handler
_pandaLogger.release()
except:
pass
tmpLog.error('_getPFNFromLFC failed')
return status
# reset
strFiles = ''
tmpLog.debug('_getPFNFromLFC done')
# return
return outStr
def _getPFNFromLFC(lfns,dq2url,guids,storageName,scopeList=[],tmpLog=None):
if tmpLog == None:
tmpLog = LogWrapper(_log,logPrefix)
tmpLog.debug('_getPFNFromLFC %s %s / %s LFNs:%s %s' % (dq2url,str(storageName),
len(lfns),str(lfns[:3]),str(scopeList[:3])))
outStr = ''
# check paramter
if guids == [] or storageName == [] or (len(lfns) != len(guids)):
tmpLog.debug('_getPFNFromLFC done with empty list')
return outStr
# check scopeList
if not scopeList in [None,[]] and len(lfns) != len(scopeList):
tmpLog.warning('_getPFNFromLFC wrong scopeList %s %s %s %s' % (dq2url,str(storageName),
str(lfns),str(scopeList)))
tmpLog.error('_getPFNFromLFC failed')
return outStr
# loop over all LFNs
iLFN = 0
nLFN = 1000
strFiles = ''
outStr = ''
for iLFN in range(len(lfns)):
if scopeList != []:
strFiles += '%s %s %s\n' % (lfns[iLFN],guids[iLFN],scopeList[iLFN])
else:
strFiles += '%s %s\n' % (lfns[iLFN],guids[iLFN])
# bulk operation
if (iLFN+1) % nLFN == 0 or (iLFN+1) >= len(lfns):
# write to file
inFileName = '%s/lfcin.%s' % (panda_config.logdir,commands.getoutput('uuidgen'))
ifile = open(inFileName,'w')
ifile.write(strFiles)
ifile.close()
# construct commands
strStorage = ''
for storage in storageName:
strStorage += '%s,' % storage
strStorage = strStorage[:-1]
com = 'cd %s > /dev/null 2>&1; export HOME=%s; ' % (panda_config.home_dir_cwd,panda_config.home_dir_cwd)
com+= 'unset LD_LIBRARY_PATH; unset PYTHONPATH; export PATH=/usr/local/bin:/bin:/usr/bin; '
com+= 'source %s; %s/python -Wignore %s/LFCclient.py -f %s -l %s -s %s' % \
(panda_config.glite_source,panda_config.native_python32,panda_config.lfcClient_dir,
inFileName,dq2url,strStorage)
tmpLog.debug(com)
# exeute
status,output = commands.getstatusoutput(com)
tmpLog.debug(status)
if status == 0:
outStr += output
else:
tmpLog.error("_getPFNFromLFC : %s %s %s" % (dq2url,status,output))
# send message to logger
try:
# make message
message = 'LFC access : %s %s %s' % (dq2url,status,output)
# get logger
_pandaLogger = PandaLogger()
_pandaLogger.lock()
_pandaLogger.setParams({'Type':'broker_util'})
logger = _pandaLogger.getHttpLogger(panda_config.loggername)
# add message
logger.error(message)
# release HTTP handler
_pandaLogger.release()
except:
pass
tmpLog.error('_getPFNFromLFC failed')
return status
# reset
strFiles = ''
tmpLog.debug('_getPFNFromLFC done')
# return
return outStr