def importDatasetWithExistingParents(self, sourceDBS, sourceDatasetPath, targetDBS, onlyClosed=True):
"""
_importDataset_
Import a dataset into the local scope DBS.
It complains if the parent dataset ar not there!!
- *sourceDBS* : URL for input DBS instance
- *sourceDatasetPath* : Dataset Path to be imported
- *targetDBS* : URL for DBS to have dataset imported to
"""
reader = DBSReader(sourceDBS)
inputBlocks = reader.getFileBlocksInfo(sourceDatasetPath, onlyClosed)
for inputBlock in inputBlocks:
block = inputBlock["Name"]
# //
# // Test block does not exist in target
# //
if self.reader.blockExists(block):
# //
# // block exists
# // If block is closed dont attempt transfer
if not str(inputBlock["OpenForWriting"]) != "1":
msg = "Block already exists in target DBS and is closed:\n"
msg += " ==> %s\n" % block
msg += "Skipping Import of that block"
logging.warning(msg)
locations = reader.listFileBlockLocation(block)
# only empty file blocks can have no location
if not locations and str(inputBlock["NumberOfFiles"]) != "0":
msg = "Error in DBSWriter.importDatasetWithExistingParents\n"
msg += "Block has no locations defined: %s" % block
raise DBSWriterError(msg)
logging.info("Update block locations to:")
for sename in locations:
self.dbs.addReplicaToBlock(block, sename)
logging.info(sename)
continue
try:
xferData = reader.dbs.listDatasetContents(sourceDatasetPath, block)
except DbsException, ex:
msg = "Error in DBSWriter.importDatasetWithExistingParents\n"
msg += "Could not read content of dataset:\n ==> %s\n" % (sourceDatasetPath,)
msg += "Block name:\n ==> %s\n" % block
msg += "%s\n" % formatEx(ex)
raise DBSWriterError(msg)
try:
self.dbs.insertDatasetContents(xferData)
except DbsException, ex:
msg = "Error in DBSWriter.importDatasetWithExistingParents\n"
msg += "Could not write content of dataset:\n ==> %s\n" % (sourceDatasetPath,)
msg += "Block name:\n ==> %s\n" % block
msg += "%s\n" % formatEx(ex)
raise DBSWriterError(msg)
def listRunLumis(self, dataset = None, block = None):
"""
It gets a list of DBSRun objects and returns the number of lumisections per run
DbsRun (RunNumber,
NumberOfEvents,
NumberOfLumiSections,
TotalLuminosity,
StoreNumber,
StartOfRungetLong,
EndOfRun,
CreationDate,
CreatedBy,
LastModificationDate,
LastModifiedBy
)
"""
try:
if block:
results = self.dbs.listRuns(block_name = block)
else:
results = self.dbs.listRuns(dataset = dataset)
except DbsException, ex:
msg = "Error in DBSReader.listRuns(%s, %s)\n" % (dataset, block)
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def listFilesInBlockWithParents(self, fileBlockName):
"""
_listFilesInBlockWithParents_
Get a list of files in the named fileblock including
the parents of that file.
"""
try:
files = self.dbs.listFiles(
"", # path
"", #primary
"", # processed
[], #tier_list
"", #analysisDataset
fileBlockName,
details = None,
retriveList = ['retrive_parent' ])
except DbsException, ex:
msg = "Error in "
msg += "DBSReader.listFilesInBlockWithParents(%s)\n" % (
fileBlockName,)
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def listRuns(self, dataset = None, block = None):
"""
it gets list of DbsRun object but for our purpose
only list of number is collected.
DbsRun (RunNumber,
NumberOfEvents,
NumberOfLumiSections,
TotalLuminosity,
StoreNumber,
StartOfRungetLong,
EndOfRun,
CreationDate,
CreatedBy,
LastModificationDate,
LastModifiedBy
)
"""
runs = []
try:
if block:
results = self.dbs.listRuns(block_name = block)
else:
results = self.dbs.listRuns(dataset = dataset)
except DbsException, ex:
msg = "Error in DBSReader.listRuns(%s, %s)\n" % (dataset, block)
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def listRunLumis(self, dataset=None, block=None):
"""
It gets a list of DBSRun objects and returns the number of lumisections per run
DbsRun (RunNumber,
NumberOfEvents,
NumberOfLumiSections,
TotalLuminosity,
StoreNumber,
StartOfRungetLong,
EndOfRun,
CreationDate,
CreatedBy,
LastModificationDate,
LastModifiedBy
)
"""
try:
if block:
results = self.dbs.listRuns(block=block)
else:
results = self.dbs.listRuns(dataset=dataset)
except DbsException, ex:
msg = "Error in DBSReader.listRuns(%s, %s)\n" % (dataset, block)
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def listRuns(self, dataset = None, block = None):
"""
it gets list of DbsRun object but for our purpose
only list of number is collected.
DbsRun (RunNumber,
NumberOfEvents,
NumberOfLumiSections,
TotalLuminosity,
StoreNumber,
StartOfRungetLong,
EndOfRun,
CreationDate,
CreatedBy,
LastModificationDate,
LastModifiedBy
)
"""
runs = []
try:
if block:
results = self.dbs.listRuns(block_name = block)
else:
results = self.dbs.listRuns(dataset = dataset)
except DbsException, ex:
msg = "Error in DBSReader.listRuns(%s, %s)\n" % (dataset, block)
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def listFilesInBlockWithParents(self, fileBlockName):
"""
_listFilesInBlockWithParents_
Get a list of files in the named fileblock including
the parents of that file.
"""
try:
files = self.dbs.listFiles(
"", # path
"", #primary
"", # processed
[], #tier_list
"", #analysisDataset
fileBlockName,
details="True",
retriveList=['retrive_parent'])
except DbsException, ex:
msg = "Error in "
msg += "DBSReader.listFilesInBlockWithParents(%s)\n" % (
fileBlockName, )
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def __init__(self, url, **contact):
# instantiate dbs api object
try:
self.dbs = DbsApi(url, **contact)
except DbsException, ex:
msg = "Error in DBSReader with DbsApi\n"
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def __init__(self, url, **contact):
# instantiate dbs api object
try:
self.dbs = DbsApi(url, **contact)
except DbsException, ex:
msg = "Error in DBSReader with DbsApi\n"
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def getFileBlocksInfo(self, dataset, onlyClosedBlocks = False):
"""
"""
self.checkDatasetPath(dataset)
try:
blocks = self.dbs.listBlocks(dataset)
except DbsException, ex:
msg = "Error in DBSReader.listFileBlocks(%s)\n" % dataset
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def getFileBlocksInfo(self, dataset, onlyClosedBlocks=False):
"""
"""
self.checkDatasetPath(dataset)
try:
blocks = self.dbs.listBlocks(dataset)
except DbsException, ex:
msg = "Error in DBSReader.listFileBlocks(%s)\n" % dataset
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def matchProcessedDatasets(self, primary, tier, process):
"""
_matchProcessedDatasets_
return a list of Processed datasets
"""
try:
result = self.dbs.listProcessedDatasets(primary, tier, process)
except DbsException, ex:
msg = "Error in DBSReader.listProcessedDatasets(%s)\n" % primary
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def matchProcessedDatasets(self, primary, tier, process):
"""
_matchProcessedDatasets_
return a list of Processed datasets
"""
try:
result = self.dbs.listProcessedDatasets(primary, tier, process)
except DbsException, ex:
msg = "Error in DBSReader.listProcessedDatasets(%s)\n" % primary
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def getDatasetInfo(self, dataset):
"""
_getDatasetInfo_
Get dataset summary includes # of files, events, blocks and total size
"""
self.checkDatasetPath(dataset)
try:
summary = self.dbs.listDatasetSummary(dataset)
except DbsException, ex:
msg = "Error in DBSReader.listDatasetSummary(%s)\n" % dataset
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def matchProcessedDatasets(self, primary, tier, process):
"""
_matchProcessedDatasets_
return a list of Processed datasets
"""
result = []
try:
datasets = self.dbs.listDatasets(primary_ds_name = primary, data_tier_name = tier, detail = True)
except DbsException, ex:
msg = "Error in DBSReader.listProcessedDatasets(%s)\n" % primary
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def getDatasetInfo(self, dataset):
"""
_getDatasetInfo_
Get dataset summary includes # of files, events, blocks and total size
"""
self.checkDatasetPath(dataset)
try:
summary = self.dbs.listDatasetSummary(dataset)
except DbsException, ex:
msg = "Error in DBSReader.listDatasetSummary(%s)\n" % dataset
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def matchProcessedDatasets(self, primary, tier, process):
"""
_matchProcessedDatasets_
return a list of Processed datasets
"""
result = []
try:
datasets = self.dbs.listDatasets(primary_ds_name = primary, data_tier_name = tier, detail = True)
except DbsException, ex:
msg = "Error in DBSReader.listProcessedDatasets(%s)\n" % primary
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def listFileBlocks(self, dataset, onlyClosedBlocks=False):
"""
_listFileBlocks_
Retrieve a list of fileblock names for a dataset
"""
self.checkDatasetPath(dataset)
try:
blocks = self.dbs.listBlocks(dataset)
except DbsException, ex:
msg = "Error in DBSReader.listFileBlocks(%s)\n" % dataset
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def listDatasetLocation(self, dataset):
"""
_listDatasetLocation_
List the SEs where there is at least a block of the given
dataset.
"""
self.checkDatasetPath(dataset)
try:
blocks = self.dbs.listBlocks(dataset, '*', nosite=False)
except DbsException, ex:
msg = "Error in DBSReader.listFileBlocks(%s)\n" % dataset
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def listOpenFileBlocks(self, dataset):
"""
_listOpenFileBlocks_
Retrieve a list of open fileblock names for a dataset
"""
self.checkDatasetPath(dataset)
try:
blocks = self.dbs.listBlocks(dataset = dataset, detail = True)
except DbsException, ex:
msg = "Error in DBSReader.listFileBlocks(%s)\n" % dataset
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def listDatasetLocation(self, dataset):
"""
_listDatasetLocation_
List the SEs where there is at least a block of the given
dataset.
"""
self.checkDatasetPath(dataset)
try:
blocks = self.dbs.listBlocks(dataset, '*', nosite = False)
except DbsException, ex:
msg = "Error in DBSReader.listFileBlocks(%s)\n" % dataset
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def getFileBlocksInfo(self, dataset, onlyClosedBlocks = False,
blockName = None, locations = True):
"""
"""
self.checkDatasetPath(dataset)
args = {'dataset' : dataset, 'detail' : True}
if blockName:
args['block_name'] = blockName
try:
blocks = self.dbs.listBlocks(**args)
except DbsException, ex:
msg = "Error in DBSReader.getFileBlocksInfo(%s)\n" % dataset
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def getFileBlocksInfo(self, dataset, onlyClosedBlocks = False,
blockName = None, locations = True):
"""
"""
self.checkDatasetPath(dataset)
args = {'dataset' : dataset, 'detail' : True}
if blockName:
args['block_name'] = blockName
try:
blocks = self.dbs.listBlocks(**args)
except DbsException, ex:
msg = "Error in DBSReader.getFileBlocksInfo(%s)\n" % dataset
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def listPrimaryDatasets(self, match = '*'):
"""
_listPrimaryDatasets_
return a list of primary datasets, The full dataset name must be provided
pattern based mathcing is no longer supported.
If no expression is provided, all datasets are returned
"""
try:
result = self.dbs.listPrimaryDatasets(primary_ds_name = match)
except DbsException, ex:
msg = "Error in DBSReader.listPrimaryDataset(%s)\n" % match
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def listProcessedDatasets(self, primary, dataTier = '*'):
"""
_listProcessedDatasets_
return a list of Processed datasets for the primary and optional
data tier value
"""
try:
result = self.dbs.listDatasets(primary_ds_name = primary, data_tier_name = dataTier)
except DbsException, ex:
msg = "Error in DBSReader.listProcessedDatasets(%s)\n" % primary
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def listProcessedDatasets(self, primary, dataTier = '*'):
"""
_listProcessedDatasets_
return a list of Processed datasets for the primary and optional
data tier value
"""
try:
result = self.dbs.listDatasets(primary_ds_name = primary, data_tier_name = dataTier)
except DbsException, ex:
msg = "Error in DBSReader.listProcessedDatasets(%s)\n" % primary
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def listPrimaryDatasets(self, match = '*'):
"""
_listPrimaryDatasets_
return a list of primary datasets, The full dataset name must be provided
pattern based mathcing is no longer supported.
If no expression is provided, all datasets are returned
"""
try:
result = self.dbs.listPrimaryDatasets(primary_ds_name = match)
except DbsException, ex:
msg = "Error in DBSReader.listPrimaryDataset(%s)\n" % match
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def listFileBlocks(self, dataset, onlyClosedBlocks = False):
"""
_listFileBlocks_
Retrieve a list of fileblock names for a dataset
"""
self.checkDatasetPath(dataset)
try:
blocks = self.dbs.listBlocks(dataset)
except DbsException, ex:
msg = "Error in DBSReader.listFileBlocks(%s)\n" % dataset
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def listDatasetLocation(self, datasetName):
"""
_listDatasetLocation_
List the origin SEs where there is at least a block of the given
dataset.
"""
self.checkDatasetPath(datasetName)
try:
blocksInfo = self.dbs.listBlockOrigin(dataset = datasetName)
except DbsException, ex:
msg = "Error in DBSReader: dbsApi.listBlocks(dataset=%s)\n" % datasetName
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def listOpenFileBlocks(self, dataset):
"""
_listOpenFileBlocks_
Retrieve a list of open fileblock names for a dataset
"""
self.checkDatasetPath(dataset)
try:
blocks = self.dbs.listBlocks(dataset, nosite=True)
except DbsException, ex:
msg = "Error in DBSReader.listFileBlocks(%s)\n" % dataset
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def listFileBlockLocation(self, fileBlockName):
"""
_listFileBlockLocation_
Get origin_site_name of a block
"""
self.checkBlockName(fileBlockName)
try:
blockInfo = self.dbs.listBlockOrigin(block_name = fileBlockName)
except DbsException, ex:
msg = "Error in DBSReader: dbsApi.listBlocks(block_name=%s)\n" % fileBlockName
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def createDatasets(self, workflowSpec):
"""
_createDatasets_
Create All the output datasets found in the workflow spec instance
provided
"""
try:
workflowSpec.payload.operate(_CreateDatasetOperator(self.dbs, workflowSpec))
except DbsException, ex:
msg = "Error in DBSWriter.createDatasets\n"
msg += "For Workflow: %s\n" % workflowSpec.workflowName()
msg += "%s\n" % formatEx(ex)
raise DBSWriterError(msg)
def getDBSSummaryInfo(self, dataset = None, block = None):
"""
Get dataset summary includes # of files, events, blocks and total size
"""
if dataset:
self.checkDatasetPath(dataset)
try:
if block:
summary = self.dbs.getSummary(block = block)
else: # dataset case dataset shouldn't be None
summary = self.dbs.getSummary(dataset = dataset)
except DbsException, ex:
msg = "Error in DBSReader.listDatasetSummary(%s, %s)\n" % (dataset, block)
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def getDBSSummaryInfo(self, dataset=None, block=None):
"""
Get dataset summary includes # of files, events, blocks and total size
"""
if dataset:
self.checkDatasetPath(dataset)
try:
if block:
summary = self.dbs.getSummary(block=block)
else: # dataset case dataset shouldn't be None
summary = self.dbs.getSummary(dataset=dataset)
except DbsException, ex:
msg = "Error in DBSReader.listDatasetSummary(%s, %s)\n" % (dataset,
block)
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def createDatasets(self, workflowSpec):
"""
_createDatasets_
Create All the output datasets found in the workflow spec instance
provided
"""
try:
workflowSpec.payload.operate(
_CreateDatasetOperator(self.dbs, workflowSpec))
except DbsException, ex:
msg = "Error in DBSWriter.createDatasets\n"
msg += "For Workflow: %s\n" % workflowSpec.workflowName()
msg += "%s\n" % formatEx(ex)
raise DBSWriterError(msg)
def getDBSSummaryInfo(self, dataset = None, block = None):
"""
Get dataset summary includes # of files, events, blocks and total size
"""
#FIXME: Doesnt raise exceptions on missing data as old api did
if dataset:
self.checkDatasetPath(dataset)
try:
if block:
summary = self.dbs.listFileSummaries(block_name = block)
else: # dataset case dataset shouldn't be None
summary = self.dbs.listFileSummaries(dataset = dataset)
except DbsException, ex:
msg = "Error in DBSReader.listDatasetSummary(%s, %s)\n" % (dataset, block)
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def getDBSSummaryInfo(self, dataset = None, block = None):
"""
Get dataset summary includes # of files, events, blocks and total size
"""
#FIXME: Doesnt raise exceptions on missing data as old api did
if dataset:
self.checkDatasetPath(dataset)
try:
if block:
summary = self.dbs.listFileSummaries(block_name = block)
else: # dataset case dataset shouldn't be None
summary = self.dbs.listFileSummaries(dataset = dataset)
except DbsException, ex:
msg = "Error in DBSReader.listDatasetSummary(%s, %s)\n" % (dataset, block)
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def listPrimaryDatasets(self, match = None):
"""
_listPrimaryDatasets_
return a list of primary datasets matching the glob expression.
If no expression is provided, all datasets are returned
"""
arg = "*"
if match != None:
arg = match
try:
result = self.dbs.listPrimaryDatasets(arg)
except DbsException, ex:
msg = "Error in DBSReader.listPrimaryDataset(%s)\n" % arg
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def listPrimaryDatasets(self, match=None):
"""
_listPrimaryDatasets_
return a list of primary datasets matching the glob expression.
If no expression is provided, all datasets are returned
"""
arg = "*"
if match != None:
arg = match
try:
result = self.dbs.listPrimaryDatasets(arg)
except DbsException, ex:
msg = "Error in DBSReader.listPrimaryDataset(%s)\n" % arg
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def __init__(self, url, **contact):
args = {"url": url, "level": "ERROR"}
args.update(contact)
try:
self.dbs = DbsApi(args)
self.args = args
self.version = args.get("version", None)
self.globalDBSUrl = args.get("globalDBSUrl", None)
self.globalVersion = args.get("globalVersion", None)
if self.globalDBSUrl:
globalArgs = {"url": url, "level": "ERROR"}
globalArgs.update(contact)
self.globalDBS = DbsApi(globalArgs)
except DbsException, ex:
msg = "Error in DBSWriterError with DbsApi\n"
msg += "%s\n" % formatEx(ex)
raise DBSWriterError(msg)
def __init__(self, url, **contact):
args = {"url": url, "level": 'ERROR'}
args.update(contact)
try:
self.dbs = DbsApi(args)
self.args = args
self.version = args.get('version', None)
self.globalDBSUrl = args.get('globalDBSUrl', None)
self.globalVersion = args.get('globalVersion', None)
if self.globalDBSUrl:
globalArgs = {'url': url, 'level': 'ERROR'}
globalArgs.update(contact)
self.globalDBS = DbsApi(globalArgs)
except DbsException, ex:
msg = "Error in DBSWriterError with DbsApi\n"
msg += "%s\n" % formatEx(ex)
raise DBSWriterError(msg)
def lfnsInBlock(self, fileBlockName):
"""
_lfnsInBlock_
LFN list only for block, details = False => faster query
"""
if not self.blockExists(fileBlockName):
msg = "DBSReader.lfnsInBlock(%s): No matching data"
raise DBSReaderError(msg % fileBlockName)
try:
files = self.dbs.listFiles(block_name = fileBlockName, detail = False)
except DbsException, ex:
msg = "Error in "
msg += "DBSReader.listFilesInBlock(%s)\n" % fileBlockName
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def listFilesInBlock(self, fileBlockName):
"""
_listFilesInBlock_
Get a list of files in the named fileblock
"""
if not self.blockExists(fileBlockName):
msg = "DBSReader.listFilesInBlock(%s): No matching data"
raise DBSReaderError(msg % fileBlockName)
try:
files = self.dbs.listFiles(block_name = fileBlockName, detail = True)
except DbsException, ex:
msg = "Error in "
msg += "DBSReader.listFilesInBlock(%s)\n" % fileBlockName
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def blockToDatasetPath(self, blockName):
"""
_blockToDatasetPath_
Given a block name, get the dataset Path associated with that
Block.
Returns the dataset path, or None if not found
"""
self.checkBlockName(blockName)
try:
blocks = self.dbs.listBlocks(block_name = blockName, detail = True)
except DbsException, ex:
msg = "Error in "
msg += "DBSReader.blockToDataset(%s)\n" % blockName
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def lfnsInBlock(self, fileBlockName):
"""
_lfnsInBlock_
LFN list only for block, details = False => faster query
"""
if not self.blockExists(fileBlockName):
msg = "DBSReader.lfnsInBlock(%s): No matching data"
raise DBSReaderError(msg % fileBlockName)
try:
files = self.dbs.listFiles(block_name = fileBlockName, detail = False)
except DbsException, ex:
msg = "Error in "
msg += "DBSReader.listFilesInBlock(%s)\n" % fileBlockName
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def blockToDatasetPath(self, blockName):
"""
_blockToDatasetPath_
Given a block name, get the dataset Path associated with that
Block.
Returns the dataset path, or None if not found
"""
self.checkBlockName(blockName)
try:
blocks = self.dbs.listBlocks(block_name=blockName, nosite=True)
except DbsException, ex:
msg = "Error in "
msg += "DBSReader.blockToDataset(%s)\n" % blockName
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def listFileBlocks(self, dataset, onlyClosedBlocks=False, blockName=None):
"""
_listFileBlocks_
Retrieve a list of fileblock names for a dataset
"""
self.checkDatasetPath(dataset)
args = {'dataset': dataset, 'detail': False}
if blockName:
args['block_name'] = blockName
if onlyClosedBlocks:
args['detail'] = True
try:
blocks = self.dbs.listBlocks(**args)
except DbsException, ex:
msg = "Error in DBSReader.listFileBlocks(%s)\n" % dataset
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def blockExists(self, fileBlockName):
"""
_blockExists_
Check to see if block with name provided exists in the DBS
Instance.
Return True if exists, False if not
"""
self.checkBlockName(fileBlockName)
try:
blocks = self.dbs.listBlocks(block_name=fileBlockName, nosite=True)
except DbsException, ex:
msg = "Error in "
msg += "DBSReader.blockExists(%s)\n" % fileBlockName
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def blockExists(self, fileBlockName):
"""
_blockExists_
Check to see if block with name provided exists in the DBS
Instance.
Return True if exists, False if not
"""
self.checkBlockName(fileBlockName)
try:
blocks = self.dbs.listBlocks(block_name = fileBlockName)
except DbsException, ex:
msg = "Error in "
msg += "DBSReader.blockExists(%s)\n" % fileBlockName
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def __init__(self, url, **contact):
args = { "url" : url, "level" : 'ERROR'}
args.update(contact)
try:
self.dbs = DbsApi(args)
self.args = args
self.version = args.get('version', None)
self.globalDBSUrl = args.get('globalDBSUrl', None)
self.globalVersion = args.get('globalVersion', None)
if self.globalDBSUrl:
globalArgs = {'url': url, 'level': 'ERROR'}
globalArgs.update(contact)
self.globalDBS = DbsApi(globalArgs)
except DbsException as ex:
msg = "Error in DBSWriterError with DbsApi\n"
msg += "%s\n" % formatEx(ex)
raise DBSWriterError(msg)
self.reader = DBSReader(**args)
def listFileBlocks(self, dataset, onlyClosedBlocks = False,
blockName = None):
"""
_listFileBlocks_
Retrieve a list of fileblock names for a dataset
"""
self.checkDatasetPath(dataset)
args = {'dataset' : dataset, 'detail' : False}
if blockName:
args['block_name'] = blockName
if onlyClosedBlocks:
args['detail'] = True
try:
blocks = self.dbs.listBlocks(**args)
except DbsException, ex:
msg = "Error in DBSReader.listFileBlocks(%s)\n" % dataset
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def lfnsInBlock(self, fileBlockName):
"""
_lfnsInBlock_
LFN list only for block, details = False => faster query
"""
try:
files = self.dbs.listFiles(
"", # path
"", #primary
"", # processed
[], #tier_list
"", #analysisDataset
fileBlockName, details = "False")
except DbsException, ex:
msg = "Error in "
msg += "DBSReader.lfnsInBlock(%s)\n" % fileBlockName
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def lfnsInBlock(self, fileBlockName):
"""
_lfnsInBlock_
LFN list only for block, details = False => faster query
"""
try:
files = self.dbs.listFiles(
"", # path
"", #primary
"", # processed
[], #tier_list
"", #analysisDataset
fileBlockName,
details="False")
except DbsException, ex:
msg = "Error in "
msg += "DBSReader.lfnsInBlock(%s)\n" % fileBlockName
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def listFilesInBlockWithParents(self, fileBlockName):
"""
_listFilesInBlockWithParents_
Get a list of files in the named fileblock including
the parents of that file.
"""
if not self.blockExists(fileBlockName):
msg = "DBSReader.listFilesInBlockWithParents(%s): No matching data"
raise DBSReaderError(msg % fileBlockName)
try:
files = self.dbs.listFileParents(block_name = fileBlockName)
except DbsException, ex:
msg = "Error in "
msg += "DBSReader.listFilesInBlockWithParents(%s)\n" % (
fileBlockName,)
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def listFilesInBlockWithParents(self, fileBlockName):
"""
_listFilesInBlockWithParents_
Get a list of files in the named fileblock including
the parents of that file.
"""
if not self.blockExists(fileBlockName):
msg = "DBSReader.listFilesInBlockWithParents(%s): No matching data"
raise DBSReaderError(msg % fileBlockName)
try:
files = self.dbs.listFileParents(block_name = fileBlockName)
except DbsException, ex:
msg = "Error in "
msg += "DBSReader.listFilesInBlockWithParents(%s)\n" % (
fileBlockName,)
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def listFilesInBlock(self, fileBlockName, lumis = True):
"""
_listFilesInBlock_
Get a list of files in the named fileblock
TODO: lumis can be false when lumi splitting is not required
However WMBSHelper expect file['LumiList'] to get the run number
so for now it will be always true.
We need to clean code up when dbs2 is completely deprecated.
calling lumis for run number is expensive.
"""
if not self.blockExists(fileBlockName):
msg = "DBSReader.listFilesInBlock(%s): No matching data"
raise DBSReaderError(msg % fileBlockName)
try:
files = self.dbs.listFiles(block_name = fileBlockName, detail = True)
except DbsException, ex:
msg = "Error in "
msg += "DBSReader.listFilesInBlock(%s)\n" % fileBlockName
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def listFilesInBlock(self, fileBlockName):
"""
_listFilesInBlock_
Get a list of files in the named fileblock
"""
try:
files = self.dbs.listFiles(
"", # path
"", #primary
"", # processed
[], #tier_list
"", #analysisDataset
fileBlockName,
details="True")
except DbsException, ex:
msg = "Error in "
msg += "DBSReader.listFilesInBlock(%s)\n" % fileBlockName
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
class DBS2Reader:
"""
_DBSReader_
General API for reading data from DBS
"""
def __init__(self, url, **contact):
args = {"url": url, "level": 'ERROR', "version": ''}
args.update(contact)
#try:
self.dbs = DbsApi(args)
#except DbsException, ex:
# msg = "Error in DBSReader with DbsApi\n"
# msg += "%s\n" % formatEx(ex)
# raise DBSReaderError(msg)
# setup DLS api - with either dbs or phedex depending on dbs instance
if url.count('cmsdbsprod.cern.ch/cms_dbs_prod_global') or \
self.dbs.getServerInfo()['InstanceName'] == 'GLOBAL':
dlsType = 'DLS_TYPE_PHEDEX'
dlsUrl = 'https://cmsweb.cern.ch/phedex/datasvc/xml/prod'
else:
dlsType = 'DLS_TYPE_DBS'
dlsUrl = url
try:
self.dls = dlsClient.getDlsApi(dls_type=dlsType,
dls_endpoint=dlsUrl,
version=args['version'])
except DlsApiError, ex:
msg = "Error in DBSReader with DlsApi\n"
msg += "%s\n" % str(ex)
raise DBSReaderError(msg)
except DbsException, ex:
msg = "Error in DBSReader with DbsApi\n"
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
def importDataset(self,
sourceDBS,
sourceDatasetPath,
targetDBS,
onlyClosed=True):
"""
_importDataset_
Import a dataset into the local scope DBS with full parentage hirerarchy
(at least not slow because branches info is dropped)
- *sourceDBS* : URL for input DBS instance
- *sourceDatasetPath* : Dataset Path to be imported
- *targetDBS* : URL for DBS to have dataset imported to
"""
reader = DBSReader(sourceDBS)
inputBlocks = reader.getFileBlocksInfo(sourceDatasetPath,
onlyClosed,
locations=False)
blkCounter = 0
for inputBlock in inputBlocks:
block = inputBlock['Name']
# //
# // Test block does not exist in target
#//
blkCounter = blkCounter + 1
msg = "Importing block %s of %s: %s " % (blkCounter,
len(inputBlocks), block)
logging.debug(msg)
if self.reader.blockExists(block):
# //
# // block exists
#// If block is closed dont attempt transfer
if str(inputBlock['OpenForWriting']) != '1':
msg = "Block already exists in target DBS and is closed:\n"
msg += " ==> %s\n" % block
msg += "Skipping Import of that block"
logging.warning(msg)
locations = reader.listFileBlockLocation(block)
# only empty file blocks can have no location
if not locations and str(
inputBlock['NumberOfFiles']) != "0":
msg = "Error in DBSWriter.importDataset\n"
msg += "Block has no locations defined: %s" % block
raise DBSWriterError(msg)
logging.info("Update block locations to:")
for sename in locations:
self.dbs.addReplicaToBlock(block, sename)
logging.info(sename)
continue
try:
self.dbs.migrateDatasetContents(sourceDBS,
targetDBS,
sourceDatasetPath,
block_name=block,
noParentsReadOnly=False)
except DbsException as ex:
msg = "Error in DBSWriter.importDataset\n"
msg += "Could not write content of dataset:\n ==> %s\n" % (
sourceDatasetPath, )
msg += "Block name:\n ==> %s\n" % block
msg += "%s\n" % formatEx(ex)
raise DBSWriterError(msg)
locations = reader.listFileBlockLocation(block)
# only empty file blocks can have no location
if not locations and str(inputBlock['NumberOfFiles']) != "0":
msg = "Error in DBSWriter.importDataset\n"
msg += "Block has no locations defined: %s" % block
raise DBSWriterError(msg)
for sename in locations:
self.dbs.addReplicaToBlock(block, sename)
return
try:
files = self.dbs.listFiles(block_name = fileBlockName, detail = True)
except DbsException, ex:
msg = "Error in "
msg += "DBSReader.listFilesInBlock(%s)\n" % fileBlockName
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
if lumis:
try:
lumiLists = self.dbs.listFileLumis(block_name = fileBlockName)
except DbsException, ex:
msg = "Error in "
msg += "DBSReader.listFileLumis(%s)\n" % fileBlockName
msg += "%s\n" % formatEx(ex)
raise DBSReaderError(msg)
lumiDict = {}
for lumisItem in lumiLists:
lumiDict.setdefault(lumisItem['logical_file_name'], [])
item = {}
item["RunNumber"] = lumisItem['run_num']
item['LumiSectionNumber'] = lumisItem['lumi_section_num']
lumiDict[lumisItem['logical_file_name']].append(item)
result = []
for file in files:
if lumis:
file["LumiList"] = lumiDict[file['logical_file_name']]
result.append(remapDBS3Keys(file, stringify = True))
