def checkDatasetPath(self, pathName):
"""
_checkDatasetPath_
"""
if pathName in ("", None):
raise DBSReaderError("Invalid Dataset Path name: => %s <=" %
pathName)
def listFilesInBlockWithParents(self, fileBlockName):
"""
_listFilesInBlockWithParents_
Get a list of files in the named fileblock including
the parents of that file.
"""
try:
files = self.dbs.listFiles(
"", # path
"", #primary
"", # processed
[], #tier_list
"", #analysisDataset
fileBlockName,
details=None,
retriveList=['retrive_parent'])
except DbsException, ex:
msg = "Error in "
msg += "DBSReader.listFilesInBlockWithParents(%s)\n" % (
fileBlockName, )
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
class DBSReader:
"""
_DBSReader_
General API for reading data from DBS
"""
def __init__(self, url, **contact):
args = {"url": url, "level": 'ERROR'}
args.update(contact)
try:
self.dbs = DbsApi(args)
except DbsException, ex:
msg = "Error in DBSReader with DbsApi\n"
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
# setup DLS api - with either dbs or phedex depending on dbs instance
if url.count('cmsdbsprod.cern.ch/cms_dbs_prod_global') or \
self.dbs.getServerInfo()['InstanceName'] == 'GLOBAL':
dlsType = 'DLS_TYPE_PHEDEX'
dlsUrl = 'https://cmsweb.cern.ch/phedex/datasvc/xml/prod'
else:
dlsType = 'DLS_TYPE_DBS'
dlsUrl = url
try:
self.dls = dlsClient.getDlsApi(dls_type=dlsType,
dls_endpoint=dlsUrl)
except DlsApiError, ex:
msg = "Error in DBSReader with DlsApi\n"
msg += "%s\n" % str(ex)
raise DBSReaderError(msg)
def __init__(self, url, **contact):
args = {"url": url, "level": 'ERROR'}
args.update(contact)
try:
self.dbs = DbsApi(args)
except DbsException, ex:
msg = "Error in DBSReader with DbsApi\n"
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def getFileBlocksInfo(self, dataset, onlyClosedBlocks=False):
"""
"""
self.checkDatasetPath(dataset)
try:
blocks = self.dbs.listBlocks(dataset)
except DbsException, ex:
msg = "Error in DBSReader.listFileBlocks(%s)\n" % dataset
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def matchProcessedDatasets(self, primary, tier, process):
"""
_matchProcessedDatasets_
return a list of Processed datasets
"""
try:
result = self.dbs.listProcessedDatasets(primary, tier, process)
except DbsException, ex:
msg = "Error in DBSReader.listProcessedDatasets(%s)\n" % primary
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def listFileBlockLocation(self, fileBlockName):
"""
_listFileBlockLocation_
Get a list of fileblock locations
"""
self.checkBlockName(fileBlockName)
try:
entryList = self.dls.getLocations([fileBlockName], showProd=True)
except DlsApiError, ex:
msg = "DLS Error in listFileBlockLocation() for %s" % fileBlockName
msg += "\n%s\n" % str(ex)
raise DBSReaderError(msg)
def listOpenFileBlocks(self, dataset):
"""
_listOpenFileBlocks_
Retrieve a list of open fileblock names for a dataset
"""
self.checkDatasetPath(dataset)
try:
blocks = self.dbs.listBlocks(dataset)
except DbsException, ex:
msg = "Error in DBSReader.listFileBlocks(%s)\n" % dataset
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def listPrimaryDatasets(self, match=None):
"""
_listPrimaryDatasets_
return a list of primary datasets matching the glob expression.
If no expression is provided, all datasets are returned
"""
arg = "*"
if match != None:
arg = match
try:
result = self.dbs.listPrimaryDatasets(arg)
except DbsException, ex:
msg = "Error in DBSReader.listPrimaryDataset(%s)\n" % arg
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def listProcessedDatasets(self, primary, dataTier=None):
"""
_listProcessedDatasets_
return a list of Processed datasets for the primary and optional
data tier value
"""
tier = "*"
if dataTier != None:
tier = dataTier
try:
result = self.dbs.listProcessedDatasets(primary, tier)
except DbsException, ex:
msg = "Error in DBSReader.listProcessedDatasets(%s)\n" % primary
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def blockToDatasetPath(self, blockName):
"""
_blockToDatasetPath_
Given a block name, get the dataset Path associated with that
Block.
Returns the dataset path, or None if not found
"""
self.checkBlockName(blockName)
try:
blocks = self.dbs.listBlocks(block_name=blockName)
except DbsException, ex:
msg = "Error in "
msg += "DBSReader.blockToDataset(%s)\n" % blockName
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def blockExists(self, fileBlockName):
"""
_blockExists_
Check to see if block with name provided exists in the DBS
Instance.
Return True if exists, False if not
"""
self.checkBlockName(fileBlockName)
try:
blocks = self.dbs.listBlocks(block_name=fileBlockName)
except DbsException, ex:
msg = "Error in "
msg += "DBSReader.blockExists(%s)\n" % fileBlockName
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def lfnsInBlock(self, fileBlockName):
"""
_lfnsInBlock_
LFN list only for block, details = False => faster query
"""
try:
files = self.dbs.listFiles(
"", # path
"", #primary
"", # processed
[], #tier_list
"", #analysisDataset
fileBlockName,
details="False")
except DbsException, ex:
msg = "Error in "
msg += "DBSReader.lfnsInBlock(%s)\n" % fileBlockName
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def listFilesInBlock(self, fileBlockName):
"""
_listFilesInBlock_
Get a list of files in the named fileblock
"""
try:
files = self.dbs.listFiles(
"", # path
"", #primary
"", # processed
[], #tier_list
"", #analysisDataset
fileBlockName,
details="True")
except DbsException, ex:
msg = "Error in "
msg += "DBSReader.listFilesInBlock(%s)\n" % fileBlockName
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def checkBlockName(self, blockName):
"""
_checkBlockName_
"""
if blockName in ("", "*", None):
raise DBSReaderError("Invalid Block name: => %s <=" % blockName)
dlsType = 'DLS_TYPE_PHEDEX'
dlsUrl = 'https://cmsweb.cern.ch/phedex/datasvc/xml/prod'
else:
dlsType = 'DLS_TYPE_DBS'
dlsUrl = url
try:
self.dls = dlsClient.getDlsApi(dls_type=dlsType,
dls_endpoint=dlsUrl)
except DlsApiError, ex:
msg = "Error in DBSReader with DlsApi\n"
msg += "%s\n" % str(ex)
raise DBSReaderError(msg)
except DbsException, ex:
msg = "Error in DBSReader with DbsApi\n"
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def listPrimaryDatasets(self, match=None):
"""
_listPrimaryDatasets_
return a list of primary datasets matching the glob expression.
If no expression is provided, all datasets are returned
"""
arg = "*"
if match != None:
arg = match
try:
result = self.dbs.listPrimaryDatasets(arg)
except DbsException, ex:
msg = "Error in DBSReader.listPrimaryDataset(%s)\n" % arg