def execute(self):
"""Given the arguments from the Job Wrapper, this function calls existing
utilities in DIRAC to resolve input data according to LHCb VO policy.
"""
result = super(InputDataResolution, self).execute()
if not result['OK'] or not result['Value'].get('Successful', {}):
return result
resolvedData = result['Value']['Successful']
resolvedData = self._addPfnType(resolvedData)
if not resolvedData['OK']:
return resolvedData
resolvedData = resolvedData['Value']
# TODO: Below is temporary behaviour to prepend mdf: to resolved TURL(s) for case when not a ROOT file
# This instructs the Gaudi applications to use root to access different file types e.g. for MDF.
# In the longer term this should be derived from file catalog metadata information.
# 24/08/2010 - updated hack to use "mdf:" after udpates from Markus
for lfn, mdataList in resolvedData.items():
if not isinstance(mdataList, list):
mdataList = [mdataList]
for mdata in mdataList:
if mdata['pfntype'] == 'MDF':
mdata['turl'] = 'mdf:' + mdata['turl']
self.log.info('Prepending mdf: to TURL for %s' % lfn)
catalogName = self.arguments['Configuration'].get(
'CatalogName', 'pool_xml_catalog.xml')
self.log.verbose('Catalog name will be: %s' % catalogName)
appCatalog = PoolXMLSlice(catalogName)
check = appCatalog.execute(resolvedData)
if not check['OK']:
return check
return result
def execute( self ):
"""Given the arguments from the Job Wrapper, this function calls existing
utilities in DIRAC to resolve input data according to LHCb VO policy.
"""
result = self.__resolveInputData()
if not result['OK']:
self.log.error( 'InputData resolution failed with result:\n%s' % ( result ) )
#For local running of this module we can expose an option to ignore missing files
ignoreMissing = False
if self.arguments.has_key( 'IgnoreMissing' ):
ignoreMissing = self.arguments['IgnoreMissing']
# Missing some of the input files is a fatal error unless ignoreMissing option is defined
if result.has_key( 'Failed' ):
failedReplicas = result['Failed']
if failedReplicas and not ignoreMissing:
self.log.error( 'Failed to obtain access to the following files:\n%s'
% ( '\n'.join( failedReplicas ) ) )
return S_ERROR( 'Failed to access all of requested input data' )
if not result.has_key( 'Successful' ):
return result
if not result['Successful']:
return S_ERROR( 'Could not access any requested input data' )
#TODO: Must define file types in order to pass to POOL XML catalogue. In the longer
#term this will be derived from file catalog metadata information but for now is based
#on the file extension types.
resolvedData = result['Successful']
tmpDict = {}
for lfn, mdata in resolvedData.items():
tmpDict[lfn] = mdata
tmpDict[lfn]['pfntype'] = 'ROOT_All'
self.log.verbose( 'Adding PFN file type %s for LFN:%s' % ( 'ROOT_All', lfn ) )
resolvedData = tmpDict
catalogName = 'pool_xml_catalog.xml'
if self.arguments['Configuration'].has_key( 'CatalogName' ):
catalogName = self.arguments['Configuration']['CatalogName']
self.log.verbose( 'Catalog name will be: %s' % catalogName )
resolvedData = tmpDict
appCatalog = PoolXMLSlice( catalogName )
check = appCatalog.execute( resolvedData )
if not check['OK']:
return check
return result
def _createCatalog(self, resolvedInputData, catalogName='pool_xml_catalog.xml', pfnType='ROOT_All'):
""" By default uses PoolXMLSlice, VO extensions can modify at will
"""
resolvedData = resolvedInputData['Successful']
tmpDict = {}
for lfn, mdata in resolvedData.items():
tmpDict[lfn] = mdata
tmpDict[lfn]['pfntype'] = pfnType
self.log.verbose('Adding PFN file type %s for LFN:%s' % (pfnType, lfn))
catalogName = self.arguments['Configuration'].get('CatalogName', catalogName)
self.log.verbose('Catalog name will be: %s' % catalogName)
resolvedData = tmpDict
appCatalog = PoolXMLSlice(catalogName)
return appCatalog.execute(resolvedData)
def _createCatalog(self, resolvedInputData, catalogName="pool_xml_catalog.xml", pfnType="ROOT_All"):
""" By default uses PoolXMLSlice, VO extensions can modify at will
"""
resolvedData = resolvedInputData["Successful"]
tmpDict = {}
for lfn, mdata in resolvedData.items():
tmpDict[lfn] = mdata
tmpDict[lfn]["pfntype"] = pfnType
self.log.verbose("Adding PFN file type %s for LFN:%s" % (pfnType, lfn))
catalogName = self.arguments["Configuration"].get("CatalogName", catalogName)
self.log.verbose("Catalog name will be: %s" % catalogName)
resolvedData = tmpDict
appCatalog = PoolXMLSlice(catalogName)
return appCatalog.execute(resolvedData)
def _createCatalog(self,
resolvedInputData,
catalogName="pool_xml_catalog.xml",
pfnType="ROOT_All"):
"""By default uses PoolXMLSlice, VO extensions can modify at will"""
resolvedData = resolvedInputData["Successful"]
tmpDict = {}
for lfn, mdata in resolvedData.items():
tmpDict[lfn] = mdata
tmpDict[lfn]["pfntype"] = pfnType
self.log.verbose("Adding PFN file type %s for LFN:%s" %
(pfnType, lfn))
catalogName = self.arguments["Configuration"].get(
"CatalogName", catalogName)
self.log.verbose("Catalog name will be: %s" % catalogName)
resolvedData = tmpDict
appCatalog = PoolXMLSlice(catalogName)
return appCatalog.execute(resolvedData)
def _createCatalog(self,
resolvedInputData,
catalogName='pool_xml_catalog.xml',
pfnType='ROOT_All'):
""" By default uses PoolXMLSlice, VO extensions can modify at will
"""
resolvedData = resolvedInputData['Successful']
tmpDict = {}
for lfn, mdata in resolvedData.items():
tmpDict[lfn] = mdata
tmpDict[lfn]['pfntype'] = pfnType
self.log.verbose('Adding PFN file type %s for LFN:%s' %
(pfnType, lfn))
catalogName = self.arguments['Configuration'].get(
'CatalogName', catalogName)
self.log.verbose('Catalog name will be: %s' % catalogName)
resolvedData = tmpDict
appCatalog = PoolXMLSlice(catalogName)
return appCatalog.execute(resolvedData)
