def execute(self): """Given the arguments from the Job Wrapper, this function calls existing utilities in DIRAC to resolve input data according to LHCb VO policy. """ result = super(InputDataResolution, self).execute() if not result['OK'] or not result['Value'].get('Successful', {}): return result resolvedData = result['Value']['Successful'] resolvedData = self._addPfnType(resolvedData) if not resolvedData['OK']: return resolvedData resolvedData = resolvedData['Value'] # TODO: Below is temporary behaviour to prepend mdf: to resolved TURL(s) for case when not a ROOT file # This instructs the Gaudi applications to use root to access different file types e.g. for MDF. # In the longer term this should be derived from file catalog metadata information. # 24/08/2010 - updated hack to use "mdf:" after udpates from Markus for lfn, mdataList in resolvedData.items(): if not isinstance(mdataList, list): mdataList = [mdataList] for mdata in mdataList: if mdata['pfntype'] == 'MDF': mdata['turl'] = 'mdf:' + mdata['turl'] self.log.info('Prepending mdf: to TURL for %s' % lfn) catalogName = self.arguments['Configuration'].get( 'CatalogName', 'pool_xml_catalog.xml') self.log.verbose('Catalog name will be: %s' % catalogName) appCatalog = PoolXMLSlice(catalogName) check = appCatalog.execute(resolvedData) if not check['OK']: return check return result
def execute( self ): """Given the arguments from the Job Wrapper, this function calls existing utilities in DIRAC to resolve input data according to LHCb VO policy. """ result = self.__resolveInputData() if not result['OK']: self.log.error( 'InputData resolution failed with result:\n%s' % ( result ) ) #For local running of this module we can expose an option to ignore missing files ignoreMissing = False if self.arguments.has_key( 'IgnoreMissing' ): ignoreMissing = self.arguments['IgnoreMissing'] # Missing some of the input files is a fatal error unless ignoreMissing option is defined if result.has_key( 'Failed' ): failedReplicas = result['Failed'] if failedReplicas and not ignoreMissing: self.log.error( 'Failed to obtain access to the following files:\n%s' % ( '\n'.join( failedReplicas ) ) ) return S_ERROR( 'Failed to access all of requested input data' ) if not result.has_key( 'Successful' ): return result if not result['Successful']: return S_ERROR( 'Could not access any requested input data' ) #TODO: Must define file types in order to pass to POOL XML catalogue. In the longer #term this will be derived from file catalog metadata information but for now is based #on the file extension types. resolvedData = result['Successful'] tmpDict = {} for lfn, mdata in resolvedData.items(): tmpDict[lfn] = mdata tmpDict[lfn]['pfntype'] = 'ROOT_All' self.log.verbose( 'Adding PFN file type %s for LFN:%s' % ( 'ROOT_All', lfn ) ) resolvedData = tmpDict catalogName = 'pool_xml_catalog.xml' if self.arguments['Configuration'].has_key( 'CatalogName' ): catalogName = self.arguments['Configuration']['CatalogName'] self.log.verbose( 'Catalog name will be: %s' % catalogName ) resolvedData = tmpDict appCatalog = PoolXMLSlice( catalogName ) check = appCatalog.execute( resolvedData ) if not check['OK']: return check return result
def _createCatalog(self, resolvedInputData, catalogName='pool_xml_catalog.xml', pfnType='ROOT_All'): """ By default uses PoolXMLSlice, VO extensions can modify at will """ resolvedData = resolvedInputData['Successful'] tmpDict = {} for lfn, mdata in resolvedData.items(): tmpDict[lfn] = mdata tmpDict[lfn]['pfntype'] = pfnType self.log.verbose('Adding PFN file type %s for LFN:%s' % (pfnType, lfn)) catalogName = self.arguments['Configuration'].get('CatalogName', catalogName) self.log.verbose('Catalog name will be: %s' % catalogName) resolvedData = tmpDict appCatalog = PoolXMLSlice(catalogName) return appCatalog.execute(resolvedData)
def _createCatalog(self, resolvedInputData, catalogName="pool_xml_catalog.xml", pfnType="ROOT_All"): """ By default uses PoolXMLSlice, VO extensions can modify at will """ resolvedData = resolvedInputData["Successful"] tmpDict = {} for lfn, mdata in resolvedData.items(): tmpDict[lfn] = mdata tmpDict[lfn]["pfntype"] = pfnType self.log.verbose("Adding PFN file type %s for LFN:%s" % (pfnType, lfn)) catalogName = self.arguments["Configuration"].get("CatalogName", catalogName) self.log.verbose("Catalog name will be: %s" % catalogName) resolvedData = tmpDict appCatalog = PoolXMLSlice(catalogName) return appCatalog.execute(resolvedData)
def _createCatalog(self, resolvedInputData, catalogName="pool_xml_catalog.xml", pfnType="ROOT_All"): """By default uses PoolXMLSlice, VO extensions can modify at will""" resolvedData = resolvedInputData["Successful"] tmpDict = {} for lfn, mdata in resolvedData.items(): tmpDict[lfn] = mdata tmpDict[lfn]["pfntype"] = pfnType self.log.verbose("Adding PFN file type %s for LFN:%s" % (pfnType, lfn)) catalogName = self.arguments["Configuration"].get( "CatalogName", catalogName) self.log.verbose("Catalog name will be: %s" % catalogName) resolvedData = tmpDict appCatalog = PoolXMLSlice(catalogName) return appCatalog.execute(resolvedData)
def _createCatalog(self, resolvedInputData, catalogName='pool_xml_catalog.xml', pfnType='ROOT_All'): """ By default uses PoolXMLSlice, VO extensions can modify at will """ resolvedData = resolvedInputData['Successful'] tmpDict = {} for lfn, mdata in resolvedData.items(): tmpDict[lfn] = mdata tmpDict[lfn]['pfntype'] = pfnType self.log.verbose('Adding PFN file type %s for LFN:%s' % (pfnType, lfn)) catalogName = self.arguments['Configuration'].get( 'CatalogName', catalogName) self.log.verbose('Catalog name will be: %s' % catalogName) resolvedData = tmpDict appCatalog = PoolXMLSlice(catalogName) return appCatalog.execute(resolvedData)