def importDatasetWithExistingParents(self, sourceDBS, sourceDatasetPath, targetDBS, onlyClosed=True): """ _importDataset_ Import a dataset into the local scope DBS. It complains if the parent dataset ar not there!! - *sourceDBS* : URL for input DBS instance - *sourceDatasetPath* : Dataset Path to be imported - *targetDBS* : URL for DBS to have dataset imported to """ reader = DBSReader(sourceDBS) inputBlocks = reader.getFileBlocksInfo(sourceDatasetPath, onlyClosed) for inputBlock in inputBlocks: block = inputBlock["Name"] # // # // Test block does not exist in target # // if self.reader.blockExists(block): # // # // block exists # // If block is closed dont attempt transfer if not str(inputBlock["OpenForWriting"]) != "1": msg = "Block already exists in target DBS and is closed:\n" msg += " ==> %s\n" % block msg += "Skipping Import of that block" logging.warning(msg) locations = reader.listFileBlockLocation(block) # only empty file blocks can have no location if not locations and str(inputBlock["NumberOfFiles"]) != "0": msg = "Error in DBSWriter.importDatasetWithExistingParents\n" msg += "Block has no locations defined: %s" % block raise DBSWriterError(msg) logging.info("Update block locations to:") for sename in locations: self.dbs.addReplicaToBlock(block, sename) logging.info(sename) continue try: xferData = reader.dbs.listDatasetContents(sourceDatasetPath, block) except DbsException, ex: msg = "Error in DBSWriter.importDatasetWithExistingParents\n" msg += "Could not read content of dataset:\n ==> %s\n" % (sourceDatasetPath,) msg += "Block name:\n ==> %s\n" % block msg += "%s\n" % formatEx(ex) raise DBSWriterError(msg) try: self.dbs.insertDatasetContents(xferData) except DbsException, ex: msg = "Error in DBSWriter.importDatasetWithExistingParents\n" msg += "Could not write content of dataset:\n ==> %s\n" % (sourceDatasetPath,) msg += "Block name:\n ==> %s\n" % block msg += "%s\n" % formatEx(ex) raise DBSWriterError(msg)
def listRunLumis(self, dataset = None, block = None): """ It gets a list of DBSRun objects and returns the number of lumisections per run DbsRun (RunNumber, NumberOfEvents, NumberOfLumiSections, TotalLuminosity, StoreNumber, StartOfRungetLong, EndOfRun, CreationDate, CreatedBy, LastModificationDate, LastModifiedBy ) """ try: if block: results = self.dbs.listRuns(block_name = block) else: results = self.dbs.listRuns(dataset = dataset) except DbsException, ex: msg = "Error in DBSReader.listRuns(%s, %s)\n" % (dataset, block) msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def listFilesInBlockWithParents(self, fileBlockName): """ _listFilesInBlockWithParents_ Get a list of files in the named fileblock including the parents of that file. """ try: files = self.dbs.listFiles( "", # path "", #primary "", # processed [], #tier_list "", #analysisDataset fileBlockName, details = None, retriveList = ['retrive_parent' ]) except DbsException, ex: msg = "Error in " msg += "DBSReader.listFilesInBlockWithParents(%s)\n" % ( fileBlockName,) msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def listRuns(self, dataset = None, block = None): """ it gets list of DbsRun object but for our purpose only list of number is collected. DbsRun (RunNumber, NumberOfEvents, NumberOfLumiSections, TotalLuminosity, StoreNumber, StartOfRungetLong, EndOfRun, CreationDate, CreatedBy, LastModificationDate, LastModifiedBy ) """ runs = [] try: if block: results = self.dbs.listRuns(block_name = block) else: results = self.dbs.listRuns(dataset = dataset) except DbsException, ex: msg = "Error in DBSReader.listRuns(%s, %s)\n" % (dataset, block) msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def listRunLumis(self, dataset=None, block=None): """ It gets a list of DBSRun objects and returns the number of lumisections per run DbsRun (RunNumber, NumberOfEvents, NumberOfLumiSections, TotalLuminosity, StoreNumber, StartOfRungetLong, EndOfRun, CreationDate, CreatedBy, LastModificationDate, LastModifiedBy ) """ try: if block: results = self.dbs.listRuns(block=block) else: results = self.dbs.listRuns(dataset=dataset) except DbsException, ex: msg = "Error in DBSReader.listRuns(%s, %s)\n" % (dataset, block) msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def listFilesInBlockWithParents(self, fileBlockName): """ _listFilesInBlockWithParents_ Get a list of files in the named fileblock including the parents of that file. """ try: files = self.dbs.listFiles( "", # path "", #primary "", # processed [], #tier_list "", #analysisDataset fileBlockName, details="True", retriveList=['retrive_parent']) except DbsException, ex: msg = "Error in " msg += "DBSReader.listFilesInBlockWithParents(%s)\n" % ( fileBlockName, ) msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def __init__(self, url, **contact): # instantiate dbs api object try: self.dbs = DbsApi(url, **contact) except DbsException, ex: msg = "Error in DBSReader with DbsApi\n" msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def getFileBlocksInfo(self, dataset, onlyClosedBlocks = False): """ """ self.checkDatasetPath(dataset) try: blocks = self.dbs.listBlocks(dataset) except DbsException, ex: msg = "Error in DBSReader.listFileBlocks(%s)\n" % dataset msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def getFileBlocksInfo(self, dataset, onlyClosedBlocks=False): """ """ self.checkDatasetPath(dataset) try: blocks = self.dbs.listBlocks(dataset) except DbsException, ex: msg = "Error in DBSReader.listFileBlocks(%s)\n" % dataset msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def matchProcessedDatasets(self, primary, tier, process): """ _matchProcessedDatasets_ return a list of Processed datasets """ try: result = self.dbs.listProcessedDatasets(primary, tier, process) except DbsException, ex: msg = "Error in DBSReader.listProcessedDatasets(%s)\n" % primary msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def getDatasetInfo(self, dataset): """ _getDatasetInfo_ Get dataset summary includes # of files, events, blocks and total size """ self.checkDatasetPath(dataset) try: summary = self.dbs.listDatasetSummary(dataset) except DbsException, ex: msg = "Error in DBSReader.listDatasetSummary(%s)\n" % dataset msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def matchProcessedDatasets(self, primary, tier, process): """ _matchProcessedDatasets_ return a list of Processed datasets """ result = [] try: datasets = self.dbs.listDatasets(primary_ds_name = primary, data_tier_name = tier, detail = True) except DbsException, ex: msg = "Error in DBSReader.listProcessedDatasets(%s)\n" % primary msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def listFileBlocks(self, dataset, onlyClosedBlocks=False): """ _listFileBlocks_ Retrieve a list of fileblock names for a dataset """ self.checkDatasetPath(dataset) try: blocks = self.dbs.listBlocks(dataset) except DbsException, ex: msg = "Error in DBSReader.listFileBlocks(%s)\n" % dataset msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def listDatasetLocation(self, dataset): """ _listDatasetLocation_ List the SEs where there is at least a block of the given dataset. """ self.checkDatasetPath(dataset) try: blocks = self.dbs.listBlocks(dataset, '*', nosite=False) except DbsException, ex: msg = "Error in DBSReader.listFileBlocks(%s)\n" % dataset msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def listOpenFileBlocks(self, dataset): """ _listOpenFileBlocks_ Retrieve a list of open fileblock names for a dataset """ self.checkDatasetPath(dataset) try: blocks = self.dbs.listBlocks(dataset = dataset, detail = True) except DbsException, ex: msg = "Error in DBSReader.listFileBlocks(%s)\n" % dataset msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def listDatasetLocation(self, dataset): """ _listDatasetLocation_ List the SEs where there is at least a block of the given dataset. """ self.checkDatasetPath(dataset) try: blocks = self.dbs.listBlocks(dataset, '*', nosite = False) except DbsException, ex: msg = "Error in DBSReader.listFileBlocks(%s)\n" % dataset msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def getFileBlocksInfo(self, dataset, onlyClosedBlocks = False, blockName = None, locations = True): """ """ self.checkDatasetPath(dataset) args = {'dataset' : dataset, 'detail' : True} if blockName: args['block_name'] = blockName try: blocks = self.dbs.listBlocks(**args) except DbsException, ex: msg = "Error in DBSReader.getFileBlocksInfo(%s)\n" % dataset msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def listPrimaryDatasets(self, match = '*'): """ _listPrimaryDatasets_ return a list of primary datasets, The full dataset name must be provided pattern based mathcing is no longer supported. If no expression is provided, all datasets are returned """ try: result = self.dbs.listPrimaryDatasets(primary_ds_name = match) except DbsException, ex: msg = "Error in DBSReader.listPrimaryDataset(%s)\n" % match msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def listProcessedDatasets(self, primary, dataTier = '*'): """ _listProcessedDatasets_ return a list of Processed datasets for the primary and optional data tier value """ try: result = self.dbs.listDatasets(primary_ds_name = primary, data_tier_name = dataTier) except DbsException, ex: msg = "Error in DBSReader.listProcessedDatasets(%s)\n" % primary msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def listFileBlocks(self, dataset, onlyClosedBlocks = False): """ _listFileBlocks_ Retrieve a list of fileblock names for a dataset """ self.checkDatasetPath(dataset) try: blocks = self.dbs.listBlocks(dataset) except DbsException, ex: msg = "Error in DBSReader.listFileBlocks(%s)\n" % dataset msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def listDatasetLocation(self, datasetName): """ _listDatasetLocation_ List the origin SEs where there is at least a block of the given dataset. """ self.checkDatasetPath(datasetName) try: blocksInfo = self.dbs.listBlockOrigin(dataset = datasetName) except DbsException, ex: msg = "Error in DBSReader: dbsApi.listBlocks(dataset=%s)\n" % datasetName msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def listOpenFileBlocks(self, dataset): """ _listOpenFileBlocks_ Retrieve a list of open fileblock names for a dataset """ self.checkDatasetPath(dataset) try: blocks = self.dbs.listBlocks(dataset, nosite=True) except DbsException, ex: msg = "Error in DBSReader.listFileBlocks(%s)\n" % dataset msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def listFileBlockLocation(self, fileBlockName): """ _listFileBlockLocation_ Get origin_site_name of a block """ self.checkBlockName(fileBlockName) try: blockInfo = self.dbs.listBlockOrigin(block_name = fileBlockName) except DbsException, ex: msg = "Error in DBSReader: dbsApi.listBlocks(block_name=%s)\n" % fileBlockName msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def createDatasets(self, workflowSpec): """ _createDatasets_ Create All the output datasets found in the workflow spec instance provided """ try: workflowSpec.payload.operate(_CreateDatasetOperator(self.dbs, workflowSpec)) except DbsException, ex: msg = "Error in DBSWriter.createDatasets\n" msg += "For Workflow: %s\n" % workflowSpec.workflowName() msg += "%s\n" % formatEx(ex) raise DBSWriterError(msg)
def getDBSSummaryInfo(self, dataset = None, block = None): """ Get dataset summary includes # of files, events, blocks and total size """ if dataset: self.checkDatasetPath(dataset) try: if block: summary = self.dbs.getSummary(block = block) else: # dataset case dataset shouldn't be None summary = self.dbs.getSummary(dataset = dataset) except DbsException, ex: msg = "Error in DBSReader.listDatasetSummary(%s, %s)\n" % (dataset, block) msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def getDBSSummaryInfo(self, dataset=None, block=None): """ Get dataset summary includes # of files, events, blocks and total size """ if dataset: self.checkDatasetPath(dataset) try: if block: summary = self.dbs.getSummary(block=block) else: # dataset case dataset shouldn't be None summary = self.dbs.getSummary(dataset=dataset) except DbsException, ex: msg = "Error in DBSReader.listDatasetSummary(%s, %s)\n" % (dataset, block) msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def createDatasets(self, workflowSpec): """ _createDatasets_ Create All the output datasets found in the workflow spec instance provided """ try: workflowSpec.payload.operate( _CreateDatasetOperator(self.dbs, workflowSpec)) except DbsException, ex: msg = "Error in DBSWriter.createDatasets\n" msg += "For Workflow: %s\n" % workflowSpec.workflowName() msg += "%s\n" % formatEx(ex) raise DBSWriterError(msg)
def getDBSSummaryInfo(self, dataset = None, block = None): """ Get dataset summary includes # of files, events, blocks and total size """ #FIXME: Doesnt raise exceptions on missing data as old api did if dataset: self.checkDatasetPath(dataset) try: if block: summary = self.dbs.listFileSummaries(block_name = block) else: # dataset case dataset shouldn't be None summary = self.dbs.listFileSummaries(dataset = dataset) except DbsException, ex: msg = "Error in DBSReader.listDatasetSummary(%s, %s)\n" % (dataset, block) msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def listPrimaryDatasets(self, match = None): """ _listPrimaryDatasets_ return a list of primary datasets matching the glob expression. If no expression is provided, all datasets are returned """ arg = "*" if match != None: arg = match try: result = self.dbs.listPrimaryDatasets(arg) except DbsException, ex: msg = "Error in DBSReader.listPrimaryDataset(%s)\n" % arg msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def listPrimaryDatasets(self, match=None): """ _listPrimaryDatasets_ return a list of primary datasets matching the glob expression. If no expression is provided, all datasets are returned """ arg = "*" if match != None: arg = match try: result = self.dbs.listPrimaryDatasets(arg) except DbsException, ex: msg = "Error in DBSReader.listPrimaryDataset(%s)\n" % arg msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def __init__(self, url, **contact): args = {"url": url, "level": "ERROR"} args.update(contact) try: self.dbs = DbsApi(args) self.args = args self.version = args.get("version", None) self.globalDBSUrl = args.get("globalDBSUrl", None) self.globalVersion = args.get("globalVersion", None) if self.globalDBSUrl: globalArgs = {"url": url, "level": "ERROR"} globalArgs.update(contact) self.globalDBS = DbsApi(globalArgs) except DbsException, ex: msg = "Error in DBSWriterError with DbsApi\n" msg += "%s\n" % formatEx(ex) raise DBSWriterError(msg)
def __init__(self, url, **contact): args = {"url": url, "level": 'ERROR'} args.update(contact) try: self.dbs = DbsApi(args) self.args = args self.version = args.get('version', None) self.globalDBSUrl = args.get('globalDBSUrl', None) self.globalVersion = args.get('globalVersion', None) if self.globalDBSUrl: globalArgs = {'url': url, 'level': 'ERROR'} globalArgs.update(contact) self.globalDBS = DbsApi(globalArgs) except DbsException, ex: msg = "Error in DBSWriterError with DbsApi\n" msg += "%s\n" % formatEx(ex) raise DBSWriterError(msg)
def lfnsInBlock(self, fileBlockName): """ _lfnsInBlock_ LFN list only for block, details = False => faster query """ if not self.blockExists(fileBlockName): msg = "DBSReader.lfnsInBlock(%s): No matching data" raise DBSReaderError(msg % fileBlockName) try: files = self.dbs.listFiles(block_name = fileBlockName, detail = False) except DbsException, ex: msg = "Error in " msg += "DBSReader.listFilesInBlock(%s)\n" % fileBlockName msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def listFilesInBlock(self, fileBlockName): """ _listFilesInBlock_ Get a list of files in the named fileblock """ if not self.blockExists(fileBlockName): msg = "DBSReader.listFilesInBlock(%s): No matching data" raise DBSReaderError(msg % fileBlockName) try: files = self.dbs.listFiles(block_name = fileBlockName, detail = True) except DbsException, ex: msg = "Error in " msg += "DBSReader.listFilesInBlock(%s)\n" % fileBlockName msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def blockToDatasetPath(self, blockName): """ _blockToDatasetPath_ Given a block name, get the dataset Path associated with that Block. Returns the dataset path, or None if not found """ self.checkBlockName(blockName) try: blocks = self.dbs.listBlocks(block_name = blockName, detail = True) except DbsException, ex: msg = "Error in " msg += "DBSReader.blockToDataset(%s)\n" % blockName msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def blockToDatasetPath(self, blockName): """ _blockToDatasetPath_ Given a block name, get the dataset Path associated with that Block. Returns the dataset path, or None if not found """ self.checkBlockName(blockName) try: blocks = self.dbs.listBlocks(block_name=blockName, nosite=True) except DbsException, ex: msg = "Error in " msg += "DBSReader.blockToDataset(%s)\n" % blockName msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def listFileBlocks(self, dataset, onlyClosedBlocks=False, blockName=None): """ _listFileBlocks_ Retrieve a list of fileblock names for a dataset """ self.checkDatasetPath(dataset) args = {'dataset': dataset, 'detail': False} if blockName: args['block_name'] = blockName if onlyClosedBlocks: args['detail'] = True try: blocks = self.dbs.listBlocks(**args) except DbsException, ex: msg = "Error in DBSReader.listFileBlocks(%s)\n" % dataset msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def blockExists(self, fileBlockName): """ _blockExists_ Check to see if block with name provided exists in the DBS Instance. Return True if exists, False if not """ self.checkBlockName(fileBlockName) try: blocks = self.dbs.listBlocks(block_name=fileBlockName, nosite=True) except DbsException, ex: msg = "Error in " msg += "DBSReader.blockExists(%s)\n" % fileBlockName msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def blockExists(self, fileBlockName): """ _blockExists_ Check to see if block with name provided exists in the DBS Instance. Return True if exists, False if not """ self.checkBlockName(fileBlockName) try: blocks = self.dbs.listBlocks(block_name = fileBlockName) except DbsException, ex: msg = "Error in " msg += "DBSReader.blockExists(%s)\n" % fileBlockName msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def __init__(self, url, **contact): args = { "url" : url, "level" : 'ERROR'} args.update(contact) try: self.dbs = DbsApi(args) self.args = args self.version = args.get('version', None) self.globalDBSUrl = args.get('globalDBSUrl', None) self.globalVersion = args.get('globalVersion', None) if self.globalDBSUrl: globalArgs = {'url': url, 'level': 'ERROR'} globalArgs.update(contact) self.globalDBS = DbsApi(globalArgs) except DbsException as ex: msg = "Error in DBSWriterError with DbsApi\n" msg += "%s\n" % formatEx(ex) raise DBSWriterError(msg) self.reader = DBSReader(**args)
def listFileBlocks(self, dataset, onlyClosedBlocks = False, blockName = None): """ _listFileBlocks_ Retrieve a list of fileblock names for a dataset """ self.checkDatasetPath(dataset) args = {'dataset' : dataset, 'detail' : False} if blockName: args['block_name'] = blockName if onlyClosedBlocks: args['detail'] = True try: blocks = self.dbs.listBlocks(**args) except DbsException, ex: msg = "Error in DBSReader.listFileBlocks(%s)\n" % dataset msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def lfnsInBlock(self, fileBlockName): """ _lfnsInBlock_ LFN list only for block, details = False => faster query """ try: files = self.dbs.listFiles( "", # path "", #primary "", # processed [], #tier_list "", #analysisDataset fileBlockName, details = "False") except DbsException, ex: msg = "Error in " msg += "DBSReader.lfnsInBlock(%s)\n" % fileBlockName msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def lfnsInBlock(self, fileBlockName): """ _lfnsInBlock_ LFN list only for block, details = False => faster query """ try: files = self.dbs.listFiles( "", # path "", #primary "", # processed [], #tier_list "", #analysisDataset fileBlockName, details="False") except DbsException, ex: msg = "Error in " msg += "DBSReader.lfnsInBlock(%s)\n" % fileBlockName msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def listFilesInBlockWithParents(self, fileBlockName): """ _listFilesInBlockWithParents_ Get a list of files in the named fileblock including the parents of that file. """ if not self.blockExists(fileBlockName): msg = "DBSReader.listFilesInBlockWithParents(%s): No matching data" raise DBSReaderError(msg % fileBlockName) try: files = self.dbs.listFileParents(block_name = fileBlockName) except DbsException, ex: msg = "Error in " msg += "DBSReader.listFilesInBlockWithParents(%s)\n" % ( fileBlockName,) msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def listFilesInBlock(self, fileBlockName, lumis = True): """ _listFilesInBlock_ Get a list of files in the named fileblock TODO: lumis can be false when lumi splitting is not required However WMBSHelper expect file['LumiList'] to get the run number so for now it will be always true. We need to clean code up when dbs2 is completely deprecated. calling lumis for run number is expensive. """ if not self.blockExists(fileBlockName): msg = "DBSReader.listFilesInBlock(%s): No matching data" raise DBSReaderError(msg % fileBlockName) try: files = self.dbs.listFiles(block_name = fileBlockName, detail = True) except DbsException, ex: msg = "Error in " msg += "DBSReader.listFilesInBlock(%s)\n" % fileBlockName msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def listFilesInBlock(self, fileBlockName): """ _listFilesInBlock_ Get a list of files in the named fileblock """ try: files = self.dbs.listFiles( "", # path "", #primary "", # processed [], #tier_list "", #analysisDataset fileBlockName, details="True") except DbsException, ex: msg = "Error in " msg += "DBSReader.listFilesInBlock(%s)\n" % fileBlockName msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
class DBS2Reader: """ _DBSReader_ General API for reading data from DBS """ def __init__(self, url, **contact): args = {"url": url, "level": 'ERROR', "version": ''} args.update(contact) #try: self.dbs = DbsApi(args) #except DbsException, ex: # msg = "Error in DBSReader with DbsApi\n" # msg += "%s\n" % formatEx(ex) # raise DBSReaderError(msg) # setup DLS api - with either dbs or phedex depending on dbs instance if url.count('cmsdbsprod.cern.ch/cms_dbs_prod_global') or \ self.dbs.getServerInfo()['InstanceName'] == 'GLOBAL': dlsType = 'DLS_TYPE_PHEDEX' dlsUrl = 'https://cmsweb.cern.ch/phedex/datasvc/xml/prod' else: dlsType = 'DLS_TYPE_DBS' dlsUrl = url try: self.dls = dlsClient.getDlsApi(dls_type=dlsType, dls_endpoint=dlsUrl, version=args['version']) except DlsApiError, ex: msg = "Error in DBSReader with DlsApi\n" msg += "%s\n" % str(ex) raise DBSReaderError(msg) except DbsException, ex: msg = "Error in DBSReader with DbsApi\n" msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg)
def importDataset(self, sourceDBS, sourceDatasetPath, targetDBS, onlyClosed=True): """ _importDataset_ Import a dataset into the local scope DBS with full parentage hirerarchy (at least not slow because branches info is dropped) - *sourceDBS* : URL for input DBS instance - *sourceDatasetPath* : Dataset Path to be imported - *targetDBS* : URL for DBS to have dataset imported to """ reader = DBSReader(sourceDBS) inputBlocks = reader.getFileBlocksInfo(sourceDatasetPath, onlyClosed, locations=False) blkCounter = 0 for inputBlock in inputBlocks: block = inputBlock['Name'] # // # // Test block does not exist in target #// blkCounter = blkCounter + 1 msg = "Importing block %s of %s: %s " % (blkCounter, len(inputBlocks), block) logging.debug(msg) if self.reader.blockExists(block): # // # // block exists #// If block is closed dont attempt transfer if str(inputBlock['OpenForWriting']) != '1': msg = "Block already exists in target DBS and is closed:\n" msg += " ==> %s\n" % block msg += "Skipping Import of that block" logging.warning(msg) locations = reader.listFileBlockLocation(block) # only empty file blocks can have no location if not locations and str( inputBlock['NumberOfFiles']) != "0": msg = "Error in DBSWriter.importDataset\n" msg += "Block has no locations defined: %s" % block raise DBSWriterError(msg) logging.info("Update block locations to:") for sename in locations: self.dbs.addReplicaToBlock(block, sename) logging.info(sename) continue try: self.dbs.migrateDatasetContents(sourceDBS, targetDBS, sourceDatasetPath, block_name=block, noParentsReadOnly=False) except DbsException as ex: msg = "Error in DBSWriter.importDataset\n" msg += "Could not write content of dataset:\n ==> %s\n" % ( sourceDatasetPath, ) msg += "Block name:\n ==> %s\n" % block msg += "%s\n" % formatEx(ex) raise DBSWriterError(msg) locations = reader.listFileBlockLocation(block) # only empty file blocks can have no location if not locations and str(inputBlock['NumberOfFiles']) != "0": msg = "Error in DBSWriter.importDataset\n" msg += "Block has no locations defined: %s" % block raise DBSWriterError(msg) for sename in locations: self.dbs.addReplicaToBlock(block, sename) return
try: files = self.dbs.listFiles(block_name = fileBlockName, detail = True) except DbsException, ex: msg = "Error in " msg += "DBSReader.listFilesInBlock(%s)\n" % fileBlockName msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg) if lumis: try: lumiLists = self.dbs.listFileLumis(block_name = fileBlockName) except DbsException, ex: msg = "Error in " msg += "DBSReader.listFileLumis(%s)\n" % fileBlockName msg += "%s\n" % formatEx(ex) raise DBSReaderError(msg) lumiDict = {} for lumisItem in lumiLists: lumiDict.setdefault(lumisItem['logical_file_name'], []) item = {} item["RunNumber"] = lumisItem['run_num'] item['LumiSectionNumber'] = lumisItem['lumi_section_num'] lumiDict[lumisItem['logical_file_name']].append(item) result = [] for file in files: if lumis: file["LumiList"] = lumiDict[file['logical_file_name']] result.append(remapDBS3Keys(file, stringify = True))