def getDataset(self):
'''Gets the dataset from the bookkeeping for current path, etc.'''
if not self.path:
return None
if not self.type in ['Path', 'RunsByDate', 'Run', 'Production']:
raise GangaException('Type="%s" is not valid.' % self.type)
if not self.type is 'RunsByDate':
if self.startDate:
msg = 'startDate not supported for type="%s".' % self.type
raise GangaException(msg)
if self.endDate:
msg = 'endDate not supported for type="%s".' % self.type
raise GangaException(msg)
if self.selection:
msg = 'selection not supported for type="%s".' % self.type
raise GangaException(msg)
cmd = "getDataset('%s','%s','%s','%s','%s','%s')" % (self.path, self.dqflag, self.type, self.startDate, self.endDate, self.selection)
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList
knownLists = [tuple, list, GangaList]
if isType(self.dqflag, knownLists):
cmd = "getDataset('%s',%s,'%s','%s','%s','%s')" % (self.path, self.dqflag, self.type, self.startDate,
self.endDate, self.selection)
result = get_result(cmd, 'BK query error.', 'BK query error.')
logger.debug("Finished Running Command")
files = []
value = result['Value']
if 'LFNs' in value:
files = value['LFNs']
if not type(files) is list: # i.e. a dict of LFN:Metadata
# if 'LFNs' in files: # i.e. a dict of LFN:Metadata
files = files.keys()
logger.debug("Creating DiracFile objects")
## Doesn't work not clear why
from GangaDirac.Lib.Files.DiracFile import DiracFile
#new_files = []
#def _createDiracLFN(this_file):
# return DiracFile(lfn = this_file)
#GangaObject.__createNewList(new_files, files, _createDiracLFN)
logger.debug("Creating new list")
new_files = [DiracFile(lfn=f) for f in files]
#new_files = [DiracFile(lfn=_file) for _file in files]
#for f in files:
# new_files.append(DiracFile(lfn=f))
#ds.extend([DiracFile(lfn = f)])
logger.info("Constructing LHCbDataset")
from GangaLHCb.Lib.LHCbDataset import LHCbDataset
logger.debug("Imported LHCbDataset")
ds = LHCbDataset(files=new_files, fromRef=True)
logger.debug("Returning Dataset")
return addProxy(ds)
def bkMetadata(self):
'Returns the bookkeeping metadata for all LFNs. '
logger.info(
"Using BKQuery(bkpath).getDatasetMetadata() with bkpath=the bookkeeping path, will yeild more metadata such as 'TCK' info...")
cmd = 'bkMetaData(%s)' % self.getLFNs()
b = get_result(cmd, 'Error removing replica', 'Replica rm error.')
return b
def getDatasetMetadata(self):
'''Gets the dataset from the bookkeeping for current path, etc.'''
if not self.path:
return None
if not self.type in ['Path', 'RunsByDate', 'Run', 'Production']:
raise GangaException('Type="%s" is not valid.' % self.type)
if not self.type is 'RunsByDate':
if self.startDate:
msg = 'startDate not supported for type="%s".' % self.type
raise GangaException(msg)
if self.endDate:
msg = 'endDate not supported for type="%s".' % self.type
raise GangaException(msg)
if self.selection:
msg = 'selection not supported for type="%s".' % self.type
raise GangaException(msg)
cmd = "getDataset('%s','%s','%s','%s','%s','%s')" % (self.path, self.dqflag, self.type, self.startDate, self.endDate, self.selection)
if type(self.dqflag) == type([]):
cmd = "getDataset('%s',%s,'%s','%s','%s','%s')" % (self.path, self.dqflag, self.type, self.startDate, self.endDate, self.selection)
result = get_result(cmd, 'BK query error.', 'BK query error.')
files = []
metadata = {}
value = result['Value']
if 'LFNs' in value:
files = value['LFNs']
if not type(files) is list: # i.e. a dict of LFN:Metadata
# if 'LFNs' in files: # i.e. a dict of LFN:Metadata
metadata = files.copy()
if metadata:
return {'OK': True, 'Value': metadata}
return {'OK': False, 'Value': metadata}
def getDataset(self):
'''Gets the dataset from the bookkeeping for current dict.'''
if not self.dict:
return None
cmd = 'bkQueryDict(%s)' % self.dict
result = get_result(cmd, 'BK query error.', 'BK query error.')
files = []
value = result['Value']
if 'LFNs' in value:
files = value['LFNs']
if not type(files) is list:
if 'LFNs' in files: # i.e. a dict of LFN:Metadata
files = files['LFNs'].keys()
from Ganga.GPI import DiracFile
this_list = [DiracFile(lfn=_file) for _file in files]
ds = LHCbDataset(this_list)
#new_files = []
#for f in files:
# #new_files.append('LFN:' + str(f))
# ds.append([DiracFile(lfn = f)])
#ds = LHCbDataset(new_files)
return GPIProxyObjectFactory(ds)
def bkMetadata(self):
'Returns the bookkeeping metadata for all LFNs. '
logger.info(
"Using BKQuery(bkpath).getDatasetMetadata() with bkpath=the bookkeeping path, will yeild more metadata such as 'TCK' info..."
)
cmd = 'bkMetaData(%s)' % self.getLFNs()
b = get_result(cmd, 'Error removing replica. Replica rm error.')
return b
def getDatasetMetadata(self):
"""Gets the dataset from the bookkeeping for current path, etc."""
if not self.path:
return None
if not self.type in ["Path", "RunsByDate", "Run", "Production"]:
raise GangaException('Type="%s" is not valid.' % self.type)
if not self.type is "RunsByDate":
if self.startDate:
msg = 'startDate not supported for type="%s".' % self.type
raise GangaException(msg)
if self.endDate:
msg = 'endDate not supported for type="%s".' % self.type
raise GangaException(msg)
if self.selection:
msg = 'selection not supported for type="%s".' % self.type
raise GangaException(msg)
cmd = "getDataset('%s','%s','%s','%s','%s','%s')" % (
self.path,
self.dqflag,
self.type,
self.startDate,
self.endDate,
self.selection,
)
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList
knownLists = [tuple, list, GangaList]
if isType(self.dqflag, knownLists):
cmd = "getDataset('%s',%s,'%s','%s','%s','%s')" % (
self.path,
self.dqflag,
self.type,
self.startDate,
self.endDate,
self.selection,
)
result = get_result(cmd, "BK query error.", "BK query error.")
files = []
metadata = {}
value = result["Value"]
if "LFNs" in value:
files = value["LFNs"]
if not type(files) is list: # i.e. a dict of LFN:Metadata
# if 'LFNs' in files: # i.e. a dict of LFN:Metadata
metadata = files.copy()
if metadata:
return {"OK": True, "Value": metadata}
return {"OK": False, "Value": metadata}
def getDatasetMetadata(self):
'''Gets the dataset from the bookkeeping for current path, etc.'''
if not self.path:
return None
if not self.type in ['Path', 'RunsByDate', 'Run', 'Production']:
raise GangaException('Type="%s" is not valid.' % self.type)
if not self.type is 'RunsByDate':
if self.startDate:
msg = 'startDate not supported for type="%s".' % self.type
raise GangaException(msg)
if self.endDate:
msg = 'endDate not supported for type="%s".' % self.type
raise GangaException(msg)
if self.selection:
msg = 'selection not supported for type="%s".' % self.type
raise GangaException(msg)
cmd = "getDataset('%s','%s','%s','%s','%s','%s')" % (
self.path, self.dqflag, self.type, self.startDate, self.endDate,
self.selection)
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList
knownLists = [tuple, list, GangaList]
if isType(self.dqflag, knownLists):
cmd = "getDataset('%s',%s,'%s','%s','%s','%s')" % (
self.path, self.dqflag, self.type, self.startDate,
self.endDate, self.selection)
try:
value = get_result(
cmd,
'BK query error.',
credential_requirements=self.credential_requirements)
except GangaDiracError as err:
return {'OK': False, 'Value': str(err)}
files = []
metadata = {}
if 'LFNs' in value:
files = value['LFNs']
if not type(files) is list: # i.e. a dict of LFN:Metadata
# if 'LFNs' in files: # i.e. a dict of LFN:Metadata
metadata = files.copy()
if metadata:
return {'OK': True, 'Value': metadata}
return {'OK': False, 'Value': metadata}
def getDatasetMetadata(self):
"""Gets the dataset from the bookkeeping for current dict."""
if not self.dict:
return None
cmd = "bkQueryDict(%s)" % self.dict
result = get_result(cmd, "BK query error.", "BK query error.")
files = []
value = result["Value"]
if "LFNs" in value:
files = value["LFNs"]
metadata = {}
if not type(files) is list:
if "LFNs" in files: # i.e. a dict of LFN:Metadata
metadata = files["LFNs"].copy()
if metadata:
return {"OK": True, "Value": metadata}
return {"OK": False, "Value": metadata}
def getDatasetMetadata(self):
'''Gets the dataset from the bookkeeping for current dict.'''
if not self.dict:
return None
cmd = 'bkQueryDict(%s)' % self.dict
result = get_result(cmd, 'BK query error.', 'BK query error.')
files = []
value = result['Value']
if 'LFNs' in value:
files = value['LFNs']
metadata = {}
if not type(files) is list:
if 'LFNs' in files: # i.e. a dict of LFN:Metadata
metadata = files['LFNs'].copy()
if metadata:
return {'OK': True, 'Value': metadata}
return {'OK': False, 'Value': metadata}
def getDatasetMetadata(self):
'''Gets the dataset from the bookkeeping for current dict.'''
if not self.dict:
return None
cmd = 'bkQueryDict(%s)' % self.dict
result = get_result(cmd, 'BK query error.', 'BK query error.')
files = []
value = result['Value']
if 'LFNs' in value:
files = value['LFNs']
metadata = {}
if not type(files) is list:
if 'LFNs' in files: # i.e. a dict of LFN:Metadata
metadata = files['LFNs'].copy()
if metadata:
return {'OK': True, 'Value': metadata}
return {'OK': False, 'Value': metadata}
def getDataset(self):
'''Gets the dataset from the bookkeeping for current dict.'''
if not self.dict:
return None
cmd = 'bkQueryDict(%s)' % self.dict
result = get_result(cmd, 'BK query error.', 'BK query error.')
files = []
value = result['Value']
if 'LFNs' in value:
files = value['LFNs']
if not type(files) is list:
if 'LFNs' in files: # i.e. a dict of LFN:Metadata
files = files['LFNs'].keys()
from GangaDirac.Lib.Files.DiracFile import DiracFile
this_list = [DiracFile(lfn=f) for f in files]
from GangaLHCb.Lib.LHCbDataset import LHCbDataset
ds = LHCbDataset(files=this_list, fromRef=True)
return addProxy(ds)
def getDatasetMetadata(self):
'''Gets the dataset from the bookkeeping for current dict.'''
if not self.dict:
return None
cmd = 'bkQueryDict(%s)' % self.dict
try:
value = get_result(cmd, 'BK query error.', credential_requirements=self.credential_requirements)
except GangaDiracError as err:
return {'OK':False, 'Value': {}}
files = []
if 'LFNs' in value:
files = value['LFNs']
metadata = {}
if not type(files) is list:
if 'LFNs' in files: # i.e. a dict of LFN:Metadata
metadata = files['LFNs'].copy()
if metadata:
return {'OK': True, 'Value': metadata}
return {'OK': False, 'Value': metadata}
def getDataset(self):
'''Gets the dataset from the bookkeeping for current dict.'''
if not self.dict:
return None
cmd = 'bkQueryDict(%s)' % self.dict
result = get_result(cmd, 'BK query error.', 'BK query error.')
files = []
value = result['Value']
if 'LFNs' in value:
files = value['LFNs']
if not type(files) is list:
if 'LFNs' in files: # i.e. a dict of LFN:Metadata
files = files['LFNs'].keys()
from GangaDirac.Lib.Files.DiracFile import DiracFile
this_list = [DiracFile(lfn=f) for f in files]
from GangaLHCb.Lib.LHCbDataset import LHCbDataset
ds = LHCbDataset(files=this_list, fromRef=True)
return addProxy(ds)
def getDataset(self):
'''Gets the dataset from the bookkeeping for current path, etc.'''
if not self.path:
return None
if not self.type in ['Path', 'RunsByDate', 'Run', 'Production']:
raise GangaException('Type="%s" is not valid.' % self.type)
if not self.type is 'RunsByDate':
if self.startDate:
msg = 'startDate not supported for type="%s".' % self.type
raise GangaException(msg)
if self.endDate:
msg = 'endDate not supported for type="%s".' % self.type
raise GangaException(msg)
if self.selection:
msg = 'selection not supported for type="%s".' % self.type
raise GangaException(msg)
cmd = "getDataset('%s','%s','%s','%s','%s','%s')" % (self.path, self.dqflag, self.type, self.startDate, self.endDate,
self.selection)
if type(self.dqflag) == type([]):
cmd = "getDataset('%s',%s,'%s','%s','%s','%s')" % (self.path, self.dqflag, self.type, self.startDate,
self.endDate, self.selection)
result = get_result(cmd, 'BK query error.', 'BK query error.')
files = []
value = result['Value']
if 'LFNs' in value:
files = value['LFNs']
if not type(files) is list: # i.e. a dict of LFN:Metadata
# if 'LFNs' in files: # i.e. a dict of LFN:Metadata
files = files.keys()
from Ganga.GPI import DiracFile
#ds = LHCbDataset()
new_files = []
for f in files:
new_files.append(DiracFile(lfn=f))
#ds.extend([DiracFile(lfn = f)])
ds = LHCbDataset(new_files)
return GPIProxyObjectFactory(ds)
def getCatalog(self, site=''):
'''Generates an XML catalog from the dataset (returns the XML string).
Note: site defaults to config.LHCb.LocalSite
Note: If the XMLCatalogueSlice attribute is set, then it returns
what is written there.'''
if hasattr(self.XMLCatalogueSlice, 'name'):
if self.XMLCatalogueSlice.name:
f = open(self.XMLCatalogueSlice.name)
xml_catalog = f.read()
f.close()
return xml_catalog
if not site:
site = getConfig('LHCb')['LocalSite']
lfns = self.getLFNs()
depth = self.depth
tmp_xml = tempfile.NamedTemporaryFile(suffix='.xml')
cmd = 'getLHCbInputDataCatalog(%s,%d,"%s","%s")' \
% (str(lfns), depth, site, tmp_xml.name)
result = get_result(cmd, 'LFN->PFN error. XML catalog error.')
xml_catalog = tmp_xml.read()
tmp_xml.close()
return xml_catalog
def getCatalog(self, site=''):
'''Generates an XML catalog from the dataset (returns the XML string).
Note: site defaults to config.LHCb.LocalSite
Note: If the XMLCatalogueSlice attribute is set, then it returns
what is written there.'''
if hasattr(self.XMLCatalogueSlice, 'name'):
if self.XMLCatalogueSlice.name:
f = open(self.XMLCatalogueSlice.name)
xml_catalog = f.read()
f.close()
return xml_catalog
if not site:
site = getConfig('LHCb')['LocalSite']
lfns = self.getLFNs()
depth = self.depth
tmp_xml = tempfile.NamedTemporaryFile(suffix='.xml')
cmd = 'getLHCbInputDataCatalog(%s,%d,"%s","%s")' \
% (str(lfns), depth, site, tmp_xml.name)
result = get_result(cmd, 'LFN->PFN error', 'XML catalog error.')
xml_catalog = tmp_xml.read()
tmp_xml.close()
return xml_catalog
def getDatasetMetadata(self):
'''Gets the dataset from the bookkeeping for current path, etc.'''
if not self.path:
return None
if not self.type in ['Path', 'RunsByDate', 'Run', 'Production']:
raise GangaException('Type="%s" is not valid.' % self.type)
if not self.type is 'RunsByDate':
if self.startDate:
msg = 'startDate not supported for type="%s".' % self.type
raise GangaException(msg)
if self.endDate:
msg = 'endDate not supported for type="%s".' % self.type
raise GangaException(msg)
if self.selection:
msg = 'selection not supported for type="%s".' % self.type
raise GangaException(msg)
cmd = "getDataset('%s','%s','%s','%s','%s','%s')" % (self.path, self.dqflag, self.type, self.startDate, self.endDate, self.selection)
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList
knownLists = [tuple, list, GangaList]
if isType(self.dqflag, knownLists):
cmd = "getDataset('%s',%s,'%s','%s','%s','%s')" % (self.path, self.dqflag, self.type, self.startDate, self.endDate, self.selection)
try:
value = get_result(cmd, 'BK query error.', credential_requirements=self.credential_requirements)
except GangaDiracError as err:
return {'OK': False, 'Value': str(err)}
files = []
metadata = {}
if 'LFNs' in value:
files = value['LFNs']
if not type(files) is list: # i.e. a dict of LFN:Metadata
# if 'LFNs' in files: # i.e. a dict of LFN:Metadata
metadata = files.copy()
if metadata:
return {'OK': True, 'Value': metadata}
return {'OK': False, 'Value': metadata}
def getDataset(self):
"""Gets the dataset from the bookkeeping for current dict."""
if not self.dict:
return None
cmd = "bkQueryDict(%s)" % self.dict
result = get_result(cmd, "BK query error.", "BK query error.")
files = []
value = result["Value"]
if "LFNs" in value:
files = value["LFNs"]
if not type(files) is list:
if "LFNs" in files: # i.e. a dict of LFN:Metadata
files = files["LFNs"].keys()
from GangaDirac.Lib.Files.DiracFile import DiracFile
this_list = [DiracFile(lfn=_file) for _file in files]
from GangaLHCb.Lib.LHCbDataset import LHCbDataset
ds = LHCbDataset(this_list)
return addProxy(ds)
def getDatasetMetadata(self):
'''Gets the dataset from the bookkeeping for current dict.'''
if not self.dict:
return None
cmd = 'bkQueryDict(%s)' % self.dict
try:
value = get_result(
cmd,
'BK query error.',
credential_requirements=self.credential_requirements)
except GangaDiracError as err:
return {'OK': False, 'Value': {}}
files = []
if 'LFNs' in value:
files = value['LFNs']
metadata = {}
if not type(files) is list:
if 'LFNs' in files: # i.e. a dict of LFN:Metadata
metadata = files['LFNs'].copy()
if metadata:
return {'OK': True, 'Value': metadata}
return {'OK': False, 'Value': metadata}
def getDataset(self):
'''Gets the dataset from the bookkeeping for current path, etc.'''
if not self.path:
return None
if not self.type in ['Path', 'RunsByDate', 'Run', 'Production']:
raise GangaException('Type="%s" is not valid.' % self.type)
if not self.type is 'RunsByDate':
if self.startDate:
msg = 'startDate not supported for type="%s".' % self.type
raise GangaException(msg)
if self.endDate:
msg = 'endDate not supported for type="%s".' % self.type
raise GangaException(msg)
if self.selection:
msg = 'selection not supported for type="%s".' % self.type
raise GangaException(msg)
cmd = "getDataset('%s','%s','%s','%s','%s','%s')" % (
self.path, self.dqflag, self.type, self.startDate, self.endDate,
self.selection)
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList
knownLists = [tuple, list, GangaList]
if isType(self.dqflag, knownLists):
cmd = "getDataset('%s',%s,'%s','%s','%s','%s')" % (
self.path, self.dqflag, self.type, self.startDate,
self.endDate, self.selection)
result = get_result(
cmd,
'BK query error.',
credential_requirements=self.credential_requirements)
logger.debug("Finished Running Command")
files = []
value = result
if 'LFNs' in value:
files = value['LFNs']
if not type(files) is list: # i.e. a dict of LFN:Metadata
# if 'LFNs' in files: # i.e. a dict of LFN:Metadata
files = files.keys()
logger.debug("Creating DiracFile objects")
## Doesn't work not clear why
from GangaDirac.Lib.Files.DiracFile import DiracFile
#new_files = []
#def _createDiracLFN(this_file):
# return DiracFile(lfn = this_file)
#GangaObject.__createNewList(new_files, files, _createDiracLFN)
logger.debug("Creating new list")
new_files = [DiracFile(lfn=f) for f in files]
#new_files = [DiracFile(lfn=_file) for _file in files]
#for f in files:
# new_files.append(DiracFile(lfn=f))
#ds.extend([DiracFile(lfn = f)])
logger.info("Constructing LHCbDataset")
from GangaLHCb.Lib.LHCbDataset import LHCbDataset
logger.debug("Imported LHCbDataset")
ds = LHCbDataset(files=new_files, fromRef=True)
logger.debug("Returning Dataset")
return addProxy(ds)
def getReplicas(self):
'Returns the replicas for all files in the dataset.'
lfns = self.getLFNs()
cmd = 'getReplicas(%s)' % str(lfns)
result = get_result(cmd, 'LFC query error', 'Could not get replicas.')
return result['Value']['Successful']
def getReplicas(self):
'Returns the replicas for all files in the dataset.'
lfns = self.getLFNs()
cmd = 'getReplicas(%s)' % str(lfns)
result = get_result(cmd, 'LFC query error. Could not get replicas.')
return result['Successful']
