def setInputDataQuery(self, metadata):
""" Define the input data query needed
"""
retMetaKey = self._checkMetaKeys( metadata.keys() )
if not retMetaKey['OK']:
return retMetaKey
if "ProdID" not in metadata:
return self._reportError("Input metadata dictionary must contain at least a key 'ProdID' as reference")
retDirs = self._checkFindDirectories( metadata )
if not retDirs['OK']:
return retDirs
dirs = retDirs['Value'].values()
for mdir in dirs[:self.maxFCFoldersToCheck]:
LOG.notice("Directory: %s" % mdir)
res = self.fc.getDirectoryUserMetadata(mdir)
if not res['OK']:
return self._reportError("Error looking up the catalog for directory metadata")
compatmeta = res['Value']
compatmeta.update(metadata)
if 'EvtType' in compatmeta:
self.evttype = JobHelpers.getValue( compatmeta['EvtType'], str, basestring )
else:
return self._reportError("EvtType is not in the metadata, it has to be!")
if 'NumberOfEvents' in compatmeta:
self.nbevts = JobHelpers.getValue( compatmeta['NumberOfEvents'], int, None )
self.basename = self.evttype
LOG.notice("MetaData: %s" % compatmeta)
LOG.notice("MetaData: %s" % metadata)
if "Energy" in compatmeta:
self.energycat = JobHelpers.getValue( compatmeta["Energy"], str, (int, long, basestring) )
if self.energycat.count("tev"):
self.energy = Decimal("1000.") * Decimal(self.energycat.split("tev")[0])
elif self.energycat.count("gev"):
self.energy = Decimal("1.") * Decimal(self.energycat.split("gev")[0])
else:
self.energy = Decimal("1.") * Decimal(self.energycat)
gendata = False
if 'Datatype' in compatmeta:
self.datatype = JobHelpers.getValue( compatmeta['Datatype'], str, basestring )
if self.datatype == 'gen':
gendata = True
if "DetectorType" in compatmeta and not gendata:
self.detector = JobHelpers.getValue( compatmeta["DetectorType"], str, basestring )
self.inputBKSelection = metadata
self.inputdataquery = True
self.prodparameters['nbevts'] = self.nbevts
self.prodparameters["FCInputQuery"] = self.inputBKSelection
return S_OK()
def setInputDataQuery(self, metadata):
""" Define the input data query needed, also get from the data the meta info requested to build the path
"""
retMetaKey = self._checkMetaKeys( metadata.keys() )
if not retMetaKey['OK']:
return retMetaKey
retDirs = self._checkFindDirectories( metadata )
if not retDirs['OK']:
return retDirs
dirs = retDirs['Value'].values()
for mdir in dirs:
res = self.fc.getDirectoryUserMetadata(mdir)
if not res['OK']:
return self._reportError("Error looking up the catalog for directory metadata")
compatmeta = res['Value']
compatmeta.update(metadata)
if 'EvtType' in compatmeta:
self.evttype = JobHelpers.getValue( compatmeta['EvtType'], str, basestring )
else:
return self._reportError("EvtType is not in the metadata, it has to be!")
if 'NumberOfEvents' in compatmeta:
self.nbevts = JobHelpers.getValue( compatmeta['NumberOfEvents'], int, None )
if 'Energy' in compatmeta:
self.energycat = JobHelpers.getValue( compatmeta['Energy'], str, basestring )
if 'Polarisation' in compatmeta:
self.polarization = JobHelpers.getValue( compatmeta["Polarisation"], str, basestring )
if 'MachineParams' in compatmeta:
self.machineparams = JobHelpers.getValue( compatmeta["MachineParams"], str, basestring )
gendata = False
if 'Datatype' in compatmeta:
self.datatype = JobHelpers.getValue( compatmeta['Datatype'], str, basestring )
if self.datatype == 'GEN':
gendata = True
if 'DetectorModel' in compatmeta and not gendata:
self.detector = JobHelpers.getValue( compatmeta["DetectorModel"], str, basestring )
self.basename = self.evttype+"_"+self.polarization
self.energy = Decimal(self.energycat)
self.inputBKSelection = metadata
self.prodparameters['nbevts'] = self.nbevts
self.prodparameters["FCInputQuery"] = self.inputBKSelection
self.inputdataquery = True
return S_OK()
def setInputDataQuery(self, metadata):
""" Define the input data query needed
"""
retMetaKey = self._checkMetaKeys(metadata.keys())
if not retMetaKey['OK']:
return retMetaKey
if "ProdID" not in metadata:
return self._reportError(
"Input metadata dictionary must contain at least a key 'ProdID' as reference"
)
retDirs = self._checkFindDirectories(metadata)
if not retDirs['OK']:
return retDirs
dirs = retDirs['Value'].values()
for mdir in dirs:
gLogger.notice("Directory: %s" % mdir)
res = self.fc.getDirectoryUserMetadata(mdir)
if not res['OK']:
return self._reportError(
"Error looking up the catalog for directory metadata")
compatmeta = res['Value']
compatmeta.update(metadata)
if 'EvtType' in compatmeta:
self.evttype = JobHelpers.getValue(compatmeta['EvtType'], str,
basestring)
else:
return self._reportError(
"EvtType is not in the metadata, it has to be!")
if 'NumberOfEvents' in compatmeta:
self.nbevts = JobHelpers.getValue(compatmeta['NumberOfEvents'],
int, None)
self.basename = self.evttype
gLogger.notice("MetaData: %s" % compatmeta)
gLogger.notice("MetaData: %s" % metadata)
if "Energy" in compatmeta:
self.energycat = JobHelpers.getValue(compatmeta["Energy"], str,
(int, long, basestring))
if self.energycat.count("tev"):
self.energy = Decimal("1000.") * Decimal(
self.energycat.split("tev")[0])
elif self.energycat.count("gev"):
self.energy = Decimal("1.") * Decimal(
self.energycat.split("gev")[0])
else:
self.energy = Decimal("1.") * Decimal(self.energycat)
gendata = False
if 'Datatype' in compatmeta:
self.datatype = JobHelpers.getValue(compatmeta['Datatype'], str,
basestring)
if self.datatype == 'gen':
gendata = True
if "DetectorType" in compatmeta and not gendata:
self.detector = JobHelpers.getValue(compatmeta["DetectorType"],
str, basestring)
self.inputBKSelection = metadata
self.inputdataquery = True
self.prodparameters['nbevts'] = self.nbevts
self.prodparameters["FCInputQuery"] = self.inputBKSelection
return S_OK()
def setInputDataQuery( self, metadata ):
""" Define the input data query needed, also get from the data the meta info requested to build the path
"""
for key,val in metadata.iteritems():
print("[0] meta[%s] %s" % (key, val))
retMetaKey = self._checkMetaKeys( metadata.keys(), extendFileMeta = True)
if not retMetaKey['OK']:
return retMetaKey
# do i need this?
tmp_metadata = {}
tmp_metadata.update(metadata)
# for kk in ['SoftwareTag', 'ILDConfig']:
for kk in ['SoftwareTag', 'ILDConfig','ProcessID']:
tmp_metadata.pop(kk, None) # why i was dropping ProdID?
retDirs = self._checkFindDirectories( tmp_metadata )
if not retDirs['OK']:
return retDirs
dirs = retDirs['Value'].values()
compatmeta = {}
print('dirs found: %d' % len(dirs))
# for d in dirs:
# print '%s'%d
dir_found = False
if self.matchToInput:
print('Will try to match dir with %s' % self.matchToInput)
for mdir in dirs:
if self.matchToInput:
if self.matchToInput not in mdir:
continue
if 'ProdID' in metadata:
val = '/' + str(metadata['ProdID']).zfill(8)
if val not in mdir:
continue
dir_found = True
print('[debug tino] Found mdir %s' % mdir)
res = self.fc.getDirectoryUserMetadata( mdir )
if not res['OK']:
return self._reportError( "Error looking up the catalog for directory metadata: %s" % res['Message'] )
compatmeta = res['Value'] # this reset compatmeta for each iteration (do we want this?)
compatmeta.update( metadata )
print('[tino debug] Updated compatmeta to: %s' % compatmeta)
if not dir_found:
if self.dryrun:
print('We could not find our target dir: please try w/o dryrun (maybe target dir still not registered)')
else:
print('We could not find our target dir and this is not a dryrun: please check')
print('self.fc.findFilesByMetadata( %s, "/ilc/prod/ilc" ) ' % metadata)
# get all the files available, if any
res = self.fc.findFilesByMetadata( metadata, '/ilc/prod/ilc' )
# res = self.fc.findFilesByMetadata( metadata, '/ilc/user/c/calanchac/stdhep' )
if not res['OK']:
return self._reportError( "Could not find the files with this metadata" )
if len( res['Value'] ):
my_lfn = res['Value'][0]
# #Get the meta data of the first one as it should be enough is the registration was
# # done right
res = self.fc.getFileUserMetadata( my_lfn )
if not res['OK']:
return self._reportError( 'Failed to get file metadata, cannot build filename: %s' % res['Message'] )
compatmeta.update( res['Value'] )
print('[tino debug] Updated compatmeta to: %s' % compatmeta)
for key,val in self.compatmeta.iteritems():
print("my_lfn %s compatmeta[%s] %s" % (my_lfn, key, val))
else:
if not self.dryrun:
print(res)
self._reportError( "No files matching the metadata: Metadata is wrong or files are not under "
"/ilc/prod/ilc directory" )
if not len(compatmeta):
print('ERROR, compatmeta is empty: this is expected when dryrun = True')
# print 'compatmeta contains ProcessID? (%s) See below:'%compatmeta
# pprint.pprint(self.compatmeta)
self.log.verbose( "Using %s to build path" % str( compatmeta ) )
if 'EvtClass' in compatmeta and not self.evtclass:
self.evtclass = JobHelpers.getValue( compatmeta['EvtClass'], str, basestring )
if 'EvtType' in compatmeta and not self.evttype:
self.evttype = JobHelpers.getValue( compatmeta['EvtType'], str, basestring )
elif 'GenProcessType' in compatmeta and not self.evttype:
self.evttype = JobHelpers.getValue( compatmeta['GenProcessType'], str, basestring )
elif not self.evttype:
return self._reportError( "Neither EvtType nor GenProcessType are "
"in the metadata: if you dont set app "
"evttype with setEvtType at least one "
"should be." )
if 'GenProcessName' in compatmeta:
self.genprocname = compatmeta['GenProcessName']
# APS: No longer possible to use these fields at this point
# if not self.genprocname:
# return self._reportError( "GenProcessName is missing! It should appear in the basename")
# if 'GenProcessID' in compatmeta:
# self.processID = JobHelpers.getValue( compatmeta['GenProcessID'], int, (int, long) )
# elif 'ProcessID' in compatmeta:
# self.processID = JobHelpers.getValue( compatmeta['ProcessID'], int, (int, long) )
# else:
# return self._reportError( "Cannot find ProcessID, it's mandatory for path definition" )
if 'Energy' in compatmeta:
self.energycat = JobHelpers.getValue( compatmeta['Energy'], str, (int, long, basestring) )
if 'MachineParams' in compatmeta:
self.machineparams = JobHelpers.getValue( compatmeta['MachineParams'], str, basestring )
if not self.machineparams:
return self._reportError( "MachineParams should part of the metadata" )
gendata = False
if 'Datatype' in compatmeta:
self.datatype = JobHelpers.getValue( compatmeta['Datatype'], str, basestring )
if self.datatype.lower() == 'gen':
gendata = True
if 'DetectorModel' in compatmeta and not gendata:
self.detector = JobHelpers.getValue( compatmeta['DetectorModel'], str, basestring )
self.compatmeta = compatmeta
self.basename = ''
#
if not self.energycat:# FIXME
print("Printing metadata before exit:")
pprint.pprint( self.compatmeta )
return self._reportError("ERROR::ILDProductionJobOpt2017.py: self.energycat is null")
self.energy = Decimal( self.energycat )
self.inputBKSelection = metadata
self.prodparameters["FCInputQuery"] = self.inputBKSelection
self.inputdataquery = True
return S_OK()
def setInputDataQuery(self, metadata):
""" Define the input data query needed, also get from the data the meta info requested to build the path
"""
retMetaKey = self._checkMetaKeys(metadata.keys())
if not retMetaKey['OK']:
return retMetaKey
retDirs = self._checkFindDirectories(metadata)
if not retDirs['OK']:
return retDirs
dirs = retDirs['Value'].values()
for mdir in dirs:
res = self.fc.getDirectoryUserMetadata(mdir)
if not res['OK']:
return self._reportError(
"Error looking up the catalog for directory metadata")
compatmeta = res['Value']
compatmeta.update(metadata)
if 'EvtType' in compatmeta:
self.evttype = JobHelpers.getValue(compatmeta['EvtType'], str,
basestring)
else:
return self._reportError(
"EvtType is not in the metadata, it has to be!")
if 'NumberOfEvents' in compatmeta:
self.nbevts = JobHelpers.getValue(compatmeta['NumberOfEvents'],
int, None)
if 'Energy' in compatmeta:
self.energycat = JobHelpers.getValue(compatmeta['Energy'], str,
basestring)
if 'Polarisation' in compatmeta:
self.polarization = JobHelpers.getValue(compatmeta["Polarisation"],
str, basestring)
if 'MachineParams' in compatmeta:
self.machineparams = JobHelpers.getValue(
compatmeta["MachineParams"], str, basestring)
gendata = False
if 'Datatype' in compatmeta:
self.datatype = JobHelpers.getValue(compatmeta['Datatype'], str,
basestring)
if self.datatype == 'GEN':
gendata = True
if 'DetectorModel' in compatmeta and not gendata:
self.detector = JobHelpers.getValue(compatmeta["DetectorModel"],
str, basestring)
self.basename = self.evttype + "_" + self.polarization
self.energy = Decimal(self.energycat)
self.inputBKSelection = metadata
self.prodparameters['nbevts'] = self.nbevts
self.prodparameters["FCInputQuery"] = self.inputBKSelection
self.inputdataquery = True
return S_OK()
def setInputDataQuery(self, metadata):
""" Define the input data query needed, also get from the data the meta info requested to build the path
"""
for key, val in metadata.iteritems():
print "[0] meta[%s] %s" % (key, val)
retMetaKey = self._checkMetaKeys(metadata.keys(), extendFileMeta=True)
if not retMetaKey['OK']:
return retMetaKey
# do i need this?
tmp_metadata = {}
tmp_metadata.update(metadata)
# for kk in ['SoftwareTag', 'ILDConfig']:
for kk in ['SoftwareTag', 'ILDConfig', 'ProcessID']:
tmp_metadata.pop(kk, None) # why i was dropping ProdID?
retDirs = self._checkFindDirectories(tmp_metadata)
if not retDirs['OK']:
return retDirs
dirs = retDirs['Value'].values()
compatmeta = {}
print 'dirs found: %d' % len(dirs)
# for d in dirs:
# print '%s'%d
dir_found = False
if self.matchToInput:
print 'Will try to match dir with %s' % self.matchToInput
for mdir in dirs:
if self.matchToInput:
if self.matchToInput not in mdir:
continue
if 'ProdID' in metadata:
val = '/' + str(metadata['ProdID']).zfill(8)
if val not in mdir:
continue
dir_found = True
print '[debug tino] Found mdir %s' % mdir
res = self.fc.getDirectoryUserMetadata(mdir)
if not res['OK']:
return self._reportError(
"Error looking up the catalog for directory metadata: %s" %
res['Message'])
compatmeta = res[
'Value'] # this reset compatmeta for each iteration (do we want this?)
compatmeta.update(metadata)
print '[tino debug] Updated compatmeta to: %s' % compatmeta
if not dir_found:
if self.dryrun:
print 'We could not find our target dir: please try w/o dryrun (maybe target dir still not registered)'
else:
print 'We could not find our target dir and this is not a dryrun: please check'
print 'self.fc.findFilesByMetadata( %s, "/ilc/prod/ilc" ) ' % metadata
# get all the files available, if any
res = self.fc.findFilesByMetadata(metadata, '/ilc/prod/ilc')
# res = self.fc.findFilesByMetadata( metadata, '/ilc/user/c/calanchac/stdhep' )
if not res['OK']:
return self._reportError(
"Could not find the files with this metadata")
if len(res['Value']):
my_lfn = res['Value'][0]
# #Get the meta data of the first one as it should be enough is the registration was
# # done right
res = self.fc.getFileUserMetadata(my_lfn)
if not res['OK']:
return self._reportError(
'Failed to get file metadata, cannot build filename: %s' %
res['Message'])
compatmeta.update(res['Value'])
print '[tino debug] Updated compatmeta to: %s' % compatmeta
for key, val in self.compatmeta.iteritems():
print "my_lfn %s compatmeta[%s] %s" % (my_lfn, key, val)
else:
if not self.dryrun:
print res
self._reportError(
"No files matching the metadata: Metadata is wrong or files are not under "
"/ilc/prod/ilc directory")
if not len(compatmeta):
print 'ERROR, compatmeta is empty: this is expected when dryrun = True'
# print 'compatmeta contains ProcessID? (%s) See below:'%compatmeta
# pprint.pprint(self.compatmeta)
self.log.verbose("Using %s to build path" % str(compatmeta))
if 'EvtClass' in compatmeta and not self.evtclass:
self.evtclass = JobHelpers.getValue(compatmeta['EvtClass'], str,
basestring)
if 'EvtType' in compatmeta and not self.evttype:
self.evttype = JobHelpers.getValue(compatmeta['EvtType'], str,
basestring)
elif 'GenProcessType' in compatmeta and not self.evttype:
self.evttype = JobHelpers.getValue(compatmeta['GenProcessType'],
str, basestring)
elif not self.evttype:
return self._reportError("Neither EvtType nor GenProcessType are "
"in the metadata: if you dont set app "
"evttype with setEvtType at least one "
"should be.")
if 'GenProcessName' in compatmeta:
self.genprocname = compatmeta['GenProcessName']
# APS: No longer possible to use these fields at this point
# if not self.genprocname:
# return self._reportError( "GenProcessName is missing! It should appear in the basename")
# if 'GenProcessID' in compatmeta:
# self.processID = JobHelpers.getValue( compatmeta['GenProcessID'], int, (int, long) )
# elif 'ProcessID' in compatmeta:
# self.processID = JobHelpers.getValue( compatmeta['ProcessID'], int, (int, long) )
# else:
# return self._reportError( "Cannot find ProcessID, it's mandatory for path definition" )
if 'Energy' in compatmeta:
self.energycat = JobHelpers.getValue(compatmeta['Energy'], str,
(int, long, basestring))
if 'MachineParams' in compatmeta:
self.machineparams = JobHelpers.getValue(
compatmeta['MachineParams'], str, basestring)
if not self.machineparams:
return self._reportError(
"MachineParams should part of the metadata")
gendata = False
if 'Datatype' in compatmeta:
self.datatype = JobHelpers.getValue(compatmeta['Datatype'], str,
basestring)
if self.datatype.lower() == 'gen':
gendata = True
if 'DetectorModel' in compatmeta and not gendata:
self.detector = JobHelpers.getValue(compatmeta['DetectorModel'],
str, basestring)
self.compatmeta = compatmeta
self.basename = ''
#
if not self.energycat: # FIXME
print "Printing metadata before exit:"
pprint.pprint(self.compatmeta)
return self._reportError(
"ERROR::ILDProductionJobOpt2017.py: self.energycat is null")
self.energy = Decimal(self.energycat)
self.inputBKSelection = metadata
self.prodparameters["FCInputQuery"] = self.inputBKSelection
self.inputdataquery = True
return S_OK()
def test_getValue_string_none(self):
value = ["someString", "someOther"]
ret = JobHelpers.getValue(value, str, basestring)
self.assertIsInstance(ret, basestring)
self.assertEqual(ret, value[0])
def test_getValue_int_none(self):
value = "2"
ret = JobHelpers.getValue(value, int, None)
self.assertIsInstance(ret, int)
self.assertEqual(ret, int(value))
def test_getValue_list_int(self):
value = ["2", 3]
ret = JobHelpers.getValue(value, int, int)
self.assertIsInstance(ret, int)
self.assertEqual(ret, int(value[0]))