def setInputDataQuery(self, metadata): """ Define the input data query needed """ retMetaKey = self._checkMetaKeys( metadata.keys() ) if not retMetaKey['OK']: return retMetaKey if "ProdID" not in metadata: return self._reportError("Input metadata dictionary must contain at least a key 'ProdID' as reference") retDirs = self._checkFindDirectories( metadata ) if not retDirs['OK']: return retDirs dirs = retDirs['Value'].values() for mdir in dirs[:self.maxFCFoldersToCheck]: LOG.notice("Directory: %s" % mdir) res = self.fc.getDirectoryUserMetadata(mdir) if not res['OK']: return self._reportError("Error looking up the catalog for directory metadata") compatmeta = res['Value'] compatmeta.update(metadata) if 'EvtType' in compatmeta: self.evttype = JobHelpers.getValue( compatmeta['EvtType'], str, basestring ) else: return self._reportError("EvtType is not in the metadata, it has to be!") if 'NumberOfEvents' in compatmeta: self.nbevts = JobHelpers.getValue( compatmeta['NumberOfEvents'], int, None ) self.basename = self.evttype LOG.notice("MetaData: %s" % compatmeta) LOG.notice("MetaData: %s" % metadata) if "Energy" in compatmeta: self.energycat = JobHelpers.getValue( compatmeta["Energy"], str, (int, long, basestring) ) if self.energycat.count("tev"): self.energy = Decimal("1000.") * Decimal(self.energycat.split("tev")[0]) elif self.energycat.count("gev"): self.energy = Decimal("1.") * Decimal(self.energycat.split("gev")[0]) else: self.energy = Decimal("1.") * Decimal(self.energycat) gendata = False if 'Datatype' in compatmeta: self.datatype = JobHelpers.getValue( compatmeta['Datatype'], str, basestring ) if self.datatype == 'gen': gendata = True if "DetectorType" in compatmeta and not gendata: self.detector = JobHelpers.getValue( compatmeta["DetectorType"], str, basestring ) self.inputBKSelection = metadata self.inputdataquery = True self.prodparameters['nbevts'] = self.nbevts self.prodparameters["FCInputQuery"] = self.inputBKSelection return S_OK()
def setInputDataQuery(self, metadata): """ Define the input data query needed, also get from the data the meta info requested to build the path """ retMetaKey = self._checkMetaKeys( metadata.keys() ) if not retMetaKey['OK']: return retMetaKey retDirs = self._checkFindDirectories( metadata ) if not retDirs['OK']: return retDirs dirs = retDirs['Value'].values() for mdir in dirs: res = self.fc.getDirectoryUserMetadata(mdir) if not res['OK']: return self._reportError("Error looking up the catalog for directory metadata") compatmeta = res['Value'] compatmeta.update(metadata) if 'EvtType' in compatmeta: self.evttype = JobHelpers.getValue( compatmeta['EvtType'], str, basestring ) else: return self._reportError("EvtType is not in the metadata, it has to be!") if 'NumberOfEvents' in compatmeta: self.nbevts = JobHelpers.getValue( compatmeta['NumberOfEvents'], int, None ) if 'Energy' in compatmeta: self.energycat = JobHelpers.getValue( compatmeta['Energy'], str, basestring ) if 'Polarisation' in compatmeta: self.polarization = JobHelpers.getValue( compatmeta["Polarisation"], str, basestring ) if 'MachineParams' in compatmeta: self.machineparams = JobHelpers.getValue( compatmeta["MachineParams"], str, basestring ) gendata = False if 'Datatype' in compatmeta: self.datatype = JobHelpers.getValue( compatmeta['Datatype'], str, basestring ) if self.datatype == 'GEN': gendata = True if 'DetectorModel' in compatmeta and not gendata: self.detector = JobHelpers.getValue( compatmeta["DetectorModel"], str, basestring ) self.basename = self.evttype+"_"+self.polarization self.energy = Decimal(self.energycat) self.inputBKSelection = metadata self.prodparameters['nbevts'] = self.nbevts self.prodparameters["FCInputQuery"] = self.inputBKSelection self.inputdataquery = True return S_OK()
def setInputDataQuery(self, metadata): """ Define the input data query needed """ retMetaKey = self._checkMetaKeys(metadata.keys()) if not retMetaKey['OK']: return retMetaKey if "ProdID" not in metadata: return self._reportError( "Input metadata dictionary must contain at least a key 'ProdID' as reference" ) retDirs = self._checkFindDirectories(metadata) if not retDirs['OK']: return retDirs dirs = retDirs['Value'].values() for mdir in dirs: gLogger.notice("Directory: %s" % mdir) res = self.fc.getDirectoryUserMetadata(mdir) if not res['OK']: return self._reportError( "Error looking up the catalog for directory metadata") compatmeta = res['Value'] compatmeta.update(metadata) if 'EvtType' in compatmeta: self.evttype = JobHelpers.getValue(compatmeta['EvtType'], str, basestring) else: return self._reportError( "EvtType is not in the metadata, it has to be!") if 'NumberOfEvents' in compatmeta: self.nbevts = JobHelpers.getValue(compatmeta['NumberOfEvents'], int, None) self.basename = self.evttype gLogger.notice("MetaData: %s" % compatmeta) gLogger.notice("MetaData: %s" % metadata) if "Energy" in compatmeta: self.energycat = JobHelpers.getValue(compatmeta["Energy"], str, (int, long, basestring)) if self.energycat.count("tev"): self.energy = Decimal("1000.") * Decimal( self.energycat.split("tev")[0]) elif self.energycat.count("gev"): self.energy = Decimal("1.") * Decimal( self.energycat.split("gev")[0]) else: self.energy = Decimal("1.") * Decimal(self.energycat) gendata = False if 'Datatype' in compatmeta: self.datatype = JobHelpers.getValue(compatmeta['Datatype'], str, basestring) if self.datatype == 'gen': gendata = True if "DetectorType" in compatmeta and not gendata: self.detector = JobHelpers.getValue(compatmeta["DetectorType"], str, basestring) self.inputBKSelection = metadata self.inputdataquery = True self.prodparameters['nbevts'] = self.nbevts self.prodparameters["FCInputQuery"] = self.inputBKSelection return S_OK()
def setInputDataQuery( self, metadata ): """ Define the input data query needed, also get from the data the meta info requested to build the path """ for key,val in metadata.iteritems(): print("[0] meta[%s] %s" % (key, val)) retMetaKey = self._checkMetaKeys( metadata.keys(), extendFileMeta = True) if not retMetaKey['OK']: return retMetaKey # do i need this? tmp_metadata = {} tmp_metadata.update(metadata) # for kk in ['SoftwareTag', 'ILDConfig']: for kk in ['SoftwareTag', 'ILDConfig','ProcessID']: tmp_metadata.pop(kk, None) # why i was dropping ProdID? retDirs = self._checkFindDirectories( tmp_metadata ) if not retDirs['OK']: return retDirs dirs = retDirs['Value'].values() compatmeta = {} print('dirs found: %d' % len(dirs)) # for d in dirs: # print '%s'%d dir_found = False if self.matchToInput: print('Will try to match dir with %s' % self.matchToInput) for mdir in dirs: if self.matchToInput: if self.matchToInput not in mdir: continue if 'ProdID' in metadata: val = '/' + str(metadata['ProdID']).zfill(8) if val not in mdir: continue dir_found = True print('[debug tino] Found mdir %s' % mdir) res = self.fc.getDirectoryUserMetadata( mdir ) if not res['OK']: return self._reportError( "Error looking up the catalog for directory metadata: %s" % res['Message'] ) compatmeta = res['Value'] # this reset compatmeta for each iteration (do we want this?) compatmeta.update( metadata ) print('[tino debug] Updated compatmeta to: %s' % compatmeta) if not dir_found: if self.dryrun: print('We could not find our target dir: please try w/o dryrun (maybe target dir still not registered)') else: print('We could not find our target dir and this is not a dryrun: please check') print('self.fc.findFilesByMetadata( %s, "/ilc/prod/ilc" ) ' % metadata) # get all the files available, if any res = self.fc.findFilesByMetadata( metadata, '/ilc/prod/ilc' ) # res = self.fc.findFilesByMetadata( metadata, '/ilc/user/c/calanchac/stdhep' ) if not res['OK']: return self._reportError( "Could not find the files with this metadata" ) if len( res['Value'] ): my_lfn = res['Value'][0] # #Get the meta data of the first one as it should be enough is the registration was # # done right res = self.fc.getFileUserMetadata( my_lfn ) if not res['OK']: return self._reportError( 'Failed to get file metadata, cannot build filename: %s' % res['Message'] ) compatmeta.update( res['Value'] ) print('[tino debug] Updated compatmeta to: %s' % compatmeta) for key,val in self.compatmeta.iteritems(): print("my_lfn %s compatmeta[%s] %s" % (my_lfn, key, val)) else: if not self.dryrun: print(res) self._reportError( "No files matching the metadata: Metadata is wrong or files are not under " "/ilc/prod/ilc directory" ) if not len(compatmeta): print('ERROR, compatmeta is empty: this is expected when dryrun = True') # print 'compatmeta contains ProcessID? (%s) See below:'%compatmeta # pprint.pprint(self.compatmeta) self.log.verbose( "Using %s to build path" % str( compatmeta ) ) if 'EvtClass' in compatmeta and not self.evtclass: self.evtclass = JobHelpers.getValue( compatmeta['EvtClass'], str, basestring ) if 'EvtType' in compatmeta and not self.evttype: self.evttype = JobHelpers.getValue( compatmeta['EvtType'], str, basestring ) elif 'GenProcessType' in compatmeta and not self.evttype: self.evttype = JobHelpers.getValue( compatmeta['GenProcessType'], str, basestring ) elif not self.evttype: return self._reportError( "Neither EvtType nor GenProcessType are " "in the metadata: if you dont set app " "evttype with setEvtType at least one " "should be." ) if 'GenProcessName' in compatmeta: self.genprocname = compatmeta['GenProcessName'] # APS: No longer possible to use these fields at this point # if not self.genprocname: # return self._reportError( "GenProcessName is missing! It should appear in the basename") # if 'GenProcessID' in compatmeta: # self.processID = JobHelpers.getValue( compatmeta['GenProcessID'], int, (int, long) ) # elif 'ProcessID' in compatmeta: # self.processID = JobHelpers.getValue( compatmeta['ProcessID'], int, (int, long) ) # else: # return self._reportError( "Cannot find ProcessID, it's mandatory for path definition" ) if 'Energy' in compatmeta: self.energycat = JobHelpers.getValue( compatmeta['Energy'], str, (int, long, basestring) ) if 'MachineParams' in compatmeta: self.machineparams = JobHelpers.getValue( compatmeta['MachineParams'], str, basestring ) if not self.machineparams: return self._reportError( "MachineParams should part of the metadata" ) gendata = False if 'Datatype' in compatmeta: self.datatype = JobHelpers.getValue( compatmeta['Datatype'], str, basestring ) if self.datatype.lower() == 'gen': gendata = True if 'DetectorModel' in compatmeta and not gendata: self.detector = JobHelpers.getValue( compatmeta['DetectorModel'], str, basestring ) self.compatmeta = compatmeta self.basename = '' # if not self.energycat:# FIXME print("Printing metadata before exit:") pprint.pprint( self.compatmeta ) return self._reportError("ERROR::ILDProductionJobOpt2017.py: self.energycat is null") self.energy = Decimal( self.energycat ) self.inputBKSelection = metadata self.prodparameters["FCInputQuery"] = self.inputBKSelection self.inputdataquery = True return S_OK()
def setInputDataQuery(self, metadata): """ Define the input data query needed, also get from the data the meta info requested to build the path """ retMetaKey = self._checkMetaKeys(metadata.keys()) if not retMetaKey['OK']: return retMetaKey retDirs = self._checkFindDirectories(metadata) if not retDirs['OK']: return retDirs dirs = retDirs['Value'].values() for mdir in dirs: res = self.fc.getDirectoryUserMetadata(mdir) if not res['OK']: return self._reportError( "Error looking up the catalog for directory metadata") compatmeta = res['Value'] compatmeta.update(metadata) if 'EvtType' in compatmeta: self.evttype = JobHelpers.getValue(compatmeta['EvtType'], str, basestring) else: return self._reportError( "EvtType is not in the metadata, it has to be!") if 'NumberOfEvents' in compatmeta: self.nbevts = JobHelpers.getValue(compatmeta['NumberOfEvents'], int, None) if 'Energy' in compatmeta: self.energycat = JobHelpers.getValue(compatmeta['Energy'], str, basestring) if 'Polarisation' in compatmeta: self.polarization = JobHelpers.getValue(compatmeta["Polarisation"], str, basestring) if 'MachineParams' in compatmeta: self.machineparams = JobHelpers.getValue( compatmeta["MachineParams"], str, basestring) gendata = False if 'Datatype' in compatmeta: self.datatype = JobHelpers.getValue(compatmeta['Datatype'], str, basestring) if self.datatype == 'GEN': gendata = True if 'DetectorModel' in compatmeta and not gendata: self.detector = JobHelpers.getValue(compatmeta["DetectorModel"], str, basestring) self.basename = self.evttype + "_" + self.polarization self.energy = Decimal(self.energycat) self.inputBKSelection = metadata self.prodparameters['nbevts'] = self.nbevts self.prodparameters["FCInputQuery"] = self.inputBKSelection self.inputdataquery = True return S_OK()
def setInputDataQuery(self, metadata): """ Define the input data query needed, also get from the data the meta info requested to build the path """ for key, val in metadata.iteritems(): print "[0] meta[%s] %s" % (key, val) retMetaKey = self._checkMetaKeys(metadata.keys(), extendFileMeta=True) if not retMetaKey['OK']: return retMetaKey # do i need this? tmp_metadata = {} tmp_metadata.update(metadata) # for kk in ['SoftwareTag', 'ILDConfig']: for kk in ['SoftwareTag', 'ILDConfig', 'ProcessID']: tmp_metadata.pop(kk, None) # why i was dropping ProdID? retDirs = self._checkFindDirectories(tmp_metadata) if not retDirs['OK']: return retDirs dirs = retDirs['Value'].values() compatmeta = {} print 'dirs found: %d' % len(dirs) # for d in dirs: # print '%s'%d dir_found = False if self.matchToInput: print 'Will try to match dir with %s' % self.matchToInput for mdir in dirs: if self.matchToInput: if self.matchToInput not in mdir: continue if 'ProdID' in metadata: val = '/' + str(metadata['ProdID']).zfill(8) if val not in mdir: continue dir_found = True print '[debug tino] Found mdir %s' % mdir res = self.fc.getDirectoryUserMetadata(mdir) if not res['OK']: return self._reportError( "Error looking up the catalog for directory metadata: %s" % res['Message']) compatmeta = res[ 'Value'] # this reset compatmeta for each iteration (do we want this?) compatmeta.update(metadata) print '[tino debug] Updated compatmeta to: %s' % compatmeta if not dir_found: if self.dryrun: print 'We could not find our target dir: please try w/o dryrun (maybe target dir still not registered)' else: print 'We could not find our target dir and this is not a dryrun: please check' print 'self.fc.findFilesByMetadata( %s, "/ilc/prod/ilc" ) ' % metadata # get all the files available, if any res = self.fc.findFilesByMetadata(metadata, '/ilc/prod/ilc') # res = self.fc.findFilesByMetadata( metadata, '/ilc/user/c/calanchac/stdhep' ) if not res['OK']: return self._reportError( "Could not find the files with this metadata") if len(res['Value']): my_lfn = res['Value'][0] # #Get the meta data of the first one as it should be enough is the registration was # # done right res = self.fc.getFileUserMetadata(my_lfn) if not res['OK']: return self._reportError( 'Failed to get file metadata, cannot build filename: %s' % res['Message']) compatmeta.update(res['Value']) print '[tino debug] Updated compatmeta to: %s' % compatmeta for key, val in self.compatmeta.iteritems(): print "my_lfn %s compatmeta[%s] %s" % (my_lfn, key, val) else: if not self.dryrun: print res self._reportError( "No files matching the metadata: Metadata is wrong or files are not under " "/ilc/prod/ilc directory") if not len(compatmeta): print 'ERROR, compatmeta is empty: this is expected when dryrun = True' # print 'compatmeta contains ProcessID? (%s) See below:'%compatmeta # pprint.pprint(self.compatmeta) self.log.verbose("Using %s to build path" % str(compatmeta)) if 'EvtClass' in compatmeta and not self.evtclass: self.evtclass = JobHelpers.getValue(compatmeta['EvtClass'], str, basestring) if 'EvtType' in compatmeta and not self.evttype: self.evttype = JobHelpers.getValue(compatmeta['EvtType'], str, basestring) elif 'GenProcessType' in compatmeta and not self.evttype: self.evttype = JobHelpers.getValue(compatmeta['GenProcessType'], str, basestring) elif not self.evttype: return self._reportError("Neither EvtType nor GenProcessType are " "in the metadata: if you dont set app " "evttype with setEvtType at least one " "should be.") if 'GenProcessName' in compatmeta: self.genprocname = compatmeta['GenProcessName'] # APS: No longer possible to use these fields at this point # if not self.genprocname: # return self._reportError( "GenProcessName is missing! It should appear in the basename") # if 'GenProcessID' in compatmeta: # self.processID = JobHelpers.getValue( compatmeta['GenProcessID'], int, (int, long) ) # elif 'ProcessID' in compatmeta: # self.processID = JobHelpers.getValue( compatmeta['ProcessID'], int, (int, long) ) # else: # return self._reportError( "Cannot find ProcessID, it's mandatory for path definition" ) if 'Energy' in compatmeta: self.energycat = JobHelpers.getValue(compatmeta['Energy'], str, (int, long, basestring)) if 'MachineParams' in compatmeta: self.machineparams = JobHelpers.getValue( compatmeta['MachineParams'], str, basestring) if not self.machineparams: return self._reportError( "MachineParams should part of the metadata") gendata = False if 'Datatype' in compatmeta: self.datatype = JobHelpers.getValue(compatmeta['Datatype'], str, basestring) if self.datatype.lower() == 'gen': gendata = True if 'DetectorModel' in compatmeta and not gendata: self.detector = JobHelpers.getValue(compatmeta['DetectorModel'], str, basestring) self.compatmeta = compatmeta self.basename = '' # if not self.energycat: # FIXME print "Printing metadata before exit:" pprint.pprint(self.compatmeta) return self._reportError( "ERROR::ILDProductionJobOpt2017.py: self.energycat is null") self.energy = Decimal(self.energycat) self.inputBKSelection = metadata self.prodparameters["FCInputQuery"] = self.inputBKSelection self.inputdataquery = True return S_OK()
def test_getValue_string_none(self): value = ["someString", "someOther"] ret = JobHelpers.getValue(value, str, basestring) self.assertIsInstance(ret, basestring) self.assertEqual(ret, value[0])
def test_getValue_int_none(self): value = "2" ret = JobHelpers.getValue(value, int, None) self.assertIsInstance(ret, int) self.assertEqual(ret, int(value))
def test_getValue_list_int(self): value = ["2", 3] ret = JobHelpers.getValue(value, int, int) self.assertIsInstance(ret, int) self.assertEqual(ret, int(value[0]))