try: self.dbs.insertFiles(procDataset, filesToCommit, fileBlock) filesToCommit = [] # logging.debug("Inserted files: %s to FileBlock: %s" \ # % ( ([ x['LogicalFileName'] for x in insertFiles ]),fileBlock['Name'])) except DbsException, ex: msg = "Error in DBSWriter.insertFiles\n" msg += "Cannot insert processed files:\n" # msg += " %s\n" % ([ x['LogicalFileName'] for x in insertFiles ],) msg += "%s\n" % formatEx(ex) raise DBSWriterError(msg) # // # // Close the block # // self.dbs.closeBlock(DBSWriterObjects.createDBSFileBlock(fileblockName)) if self.globalDBSUrl: self.dbs.dbsMigrateBlock( srcURL=self.args["url"], dstURL=self.globalDBSUrl, block_name=fileblockName, srcVersion=self.version, dstVersion=self.globalVersion, ) for algo in algos: pass # self.globalDBS.insertAlgoInPD(dataset = get_path(fileblockName.split('#')[0]), # algorithm = algo) logging.info("Migrated block %s to global" % (fileblockName)) else:
def manageFileBlock(self, fileBlock, maxFiles=100, maxSize=None, timeOut=None, algos=[], filesToCommit=[], procDataset=None): """ _manageFileBlock_ Check to see wether the fileblock with the provided name is closeable based on number of files or total size. If the block equals or exceeds wither the maxFiles or maxSize parameters, close the block and return True, else do nothing and return False """ # // # // Check that the block exists, and is open before we close it #// fileblockName = fileBlock['Name'] blockInstance = self.dbs.listBlocks(block_name=fileblockName) if len(blockInstance) > 1: msg = "Multiple Blocks matching name: %s\n" % fileblockName msg += "Unable to manage file block..." raise DBSWriterError(msg) if len(blockInstance) == 0: msg = "Block name %s not found\n" % fileblockName msg += "Cant manage a non-existent fileblock" raise DBSWriterError(msg) blockInstance = blockInstance[0] isClosed = blockInstance.get('OpenForWriting', '1') if isClosed != '1': msg = "Block %s already closed" % fileblockName logging.warning(msg) # Now we need to commit files if len(filesToCommit) > 0: try: self.dbs.insertFiles(procDataset, filesToCommit, fileBlock) filesToCommit = [] except DbsException as ex: msg = "Error in DBSWriter.insertFiles\n" msg += "Cannot insert processed files:\n" raise DBSWriterError(msg) # Attempting to migrate to global if self.globalDBSUrl: self.dbs.dbsMigrateBlock( srcURL=self.args['url'], dstURL=self.globalDBSUrl, block_name=fileblockName, srcVersion=self.version, dstVersion=self.globalVersion, ) #for algo in algos: # self.globalDBS.insertAlgoInPD(dataset = get_path(fileblockName.split('#')[0]), # algorithm = algo) logging.info( "Migrated block %s to global due to pre-closed status" % (fileblockName)) else: logging.error( "Should've migrated block %s because it was already closed, but didn't" % (fileblockName)) return True # // # // We have an open block, sum number of files and file sizes #// #fileCount = int(blockInstance.get('NumberOfFiles', 0)) fileCount = len(fileBlock['files']) totalSize = float(blockInstance.get('BlockSize', 0)) msg = "Fileblock: %s\n ==> Size: %s Files: %s\n" % ( fileblockName, totalSize, fileCount) logging.warning(msg) # // # // Test close block conditions #// closeBlock = False if timeOut: if int(time.time()) - int(blockInstance['CreationDate']) > timeOut: closeBlock = True msg = "Closing Block based on timeOut: %s" % fileblockName logging.debug(msg) if fileCount >= maxFiles: closeBlock = True msg = "Closing Block Based on files: %s" % fileblockName logging.debug(msg) if maxSize != None: if totalSize >= maxSize: closeBlock = True msg = "Closing Block Based on size: %s" % fileblockName logging.debug(msg) if closeBlock: # Now we need to commit files if len(filesToCommit) > 0: try: self.dbs.insertFiles(procDataset, filesToCommit, fileBlock) filesToCommit = [] #logging.debug("Inserted files: %s to FileBlock: %s" \ # % ( ([ x['LogicalFileName'] for x in insertFiles ]),fileBlock['Name'])) except DbsException as ex: msg = "Error in DBSWriter.insertFiles\n" msg += "Cannot insert processed files:\n" #msg += " %s\n" % ([ x['LogicalFileName'] for x in insertFiles ],) msg += "%s\n" % formatEx(ex) raise DBSWriterError(msg) # // # // Close the block #// self.dbs.closeBlock( DBSWriterObjects.createDBSFileBlock(fileblockName)) if self.globalDBSUrl: self.dbs.dbsMigrateBlock(srcURL=self.args['url'], dstURL=self.globalDBSUrl, block_name=fileblockName, srcVersion=self.version, dstVersion=self.globalVersion) for algo in algos: pass #self.globalDBS.insertAlgoInPD(dataset = get_path(fileblockName.split('#')[0]), # algorithm = algo) logging.info("Migrated block %s to global" % (fileblockName)) else: logging.error("Should've migrated block %s, but didn't" % (fileblockName)) return closeBlock
self.dbs.insertFiles(procDataset, filesToCommit, fileBlock) filesToCommit = [] #logging.debug("Inserted files: %s to FileBlock: %s" \ # % ( ([ x['LogicalFileName'] for x in insertFiles ]),fileBlock['Name'])) except DbsException, ex: msg = "Error in DBSWriter.insertFiles\n" msg += "Cannot insert processed files:\n" #msg += " %s\n" % ([ x['LogicalFileName'] for x in insertFiles ],) msg += "%s\n" % formatEx(ex) raise DBSWriterError(msg) # // # // Close the block #// self.dbs.closeBlock( DBSWriterObjects.createDBSFileBlock(fileblockName) ) if self.globalDBSUrl: self.dbs.dbsMigrateBlock(srcURL = self.args['url'], dstURL = self.globalDBSUrl, block_name = fileblockName, srcVersion = self.version, dstVersion = self.globalVersion ) for algo in algos: pass #self.globalDBS.insertAlgoInPD(dataset = get_path(fileblockName.split('#')[0]), # algorithm = algo) logging.info("Migrated block %s to global" %(fileblockName)) else:
self.dbs.insertFiles(procDataset, filesToCommit, fileBlock) filesToCommit = [] #logging.debug("Inserted files: %s to FileBlock: %s" \ # % ( ([ x['LogicalFileName'] for x in insertFiles ]),fileBlock['Name'])) except DbsException, ex: msg = "Error in DBSWriter.insertFiles\n" msg += "Cannot insert processed files:\n" #msg += " %s\n" % ([ x['LogicalFileName'] for x in insertFiles ],) msg += "%s\n" % formatEx(ex) raise DBSWriterError(msg) # // # // Close the block #// self.dbs.closeBlock( DBSWriterObjects.createDBSFileBlock(fileblockName)) if self.globalDBSUrl: self.dbs.dbsMigrateBlock(srcURL=self.args['url'], dstURL=self.globalDBSUrl, block_name=fileblockName, srcVersion=self.version, dstVersion=self.globalVersion) for algo in algos: pass #self.globalDBS.insertAlgoInPD(dataset = get_path(fileblockName.split('#')[0]), # algorithm = algo) logging.info("Migrated block %s to global" % (fileblockName)) else: logging.error("Should've migrated block %s, but didn't" % (fileblockName))
def manageFileBlock(self, fileBlock, maxFiles = 100, maxSize = None, timeOut = None, algos = [], filesToCommit = [], procDataset = None): """ _manageFileBlock_ Check to see wether the fileblock with the provided name is closeable based on number of files or total size. If the block equals or exceeds wither the maxFiles or maxSize parameters, close the block and return True, else do nothing and return False """ # // # // Check that the block exists, and is open before we close it #// fileblockName = fileBlock['Name'] blockInstance = self.dbs.listBlocks(block_name=fileblockName) if len(blockInstance) > 1: msg = "Multiple Blocks matching name: %s\n" % fileblockName msg += "Unable to manage file block..." raise DBSWriterError(msg) if len(blockInstance) == 0: msg = "Block name %s not found\n" % fileblockName msg += "Cant manage a non-existent fileblock" raise DBSWriterError(msg) blockInstance = blockInstance[0] isClosed = blockInstance.get('OpenForWriting', '1') if isClosed != '1': msg = "Block %s already closed" % fileblockName logging.warning(msg) # Now we need to commit files if len(filesToCommit) > 0: try: self.dbs.insertFiles(procDataset, filesToCommit, fileBlock) filesToCommit = [] except DbsException as ex: msg = "Error in DBSWriter.insertFiles\n" msg += "Cannot insert processed files:\n" raise DBSWriterError(msg) # Attempting to migrate to global if self.globalDBSUrl: self.dbs.dbsMigrateBlock(srcURL = self.args['url'], dstURL = self.globalDBSUrl, block_name = fileblockName, srcVersion = self.version, dstVersion = self.globalVersion, ) #for algo in algos: # self.globalDBS.insertAlgoInPD(dataset = get_path(fileblockName.split('#')[0]), # algorithm = algo) logging.info("Migrated block %s to global due to pre-closed status" %(fileblockName)) else: logging.error("Should've migrated block %s because it was already closed, but didn't" % (fileblockName)) return True # // # // We have an open block, sum number of files and file sizes #// #fileCount = int(blockInstance.get('NumberOfFiles', 0)) fileCount = len(fileBlock['files']) totalSize = float(blockInstance.get('BlockSize', 0)) msg = "Fileblock: %s\n ==> Size: %s Files: %s\n" % ( fileblockName, totalSize, fileCount) logging.warning(msg) # // # // Test close block conditions #// closeBlock = False if timeOut: if int(time.time()) - int(blockInstance['CreationDate']) > timeOut: closeBlock = True msg = "Closing Block based on timeOut: %s" % fileblockName logging.debug(msg) if fileCount >= maxFiles: closeBlock = True msg = "Closing Block Based on files: %s" % fileblockName logging.debug(msg) if maxSize != None: if totalSize >= maxSize: closeBlock = True msg = "Closing Block Based on size: %s" % fileblockName logging.debug(msg) if closeBlock: # Now we need to commit files if len(filesToCommit) > 0: try: self.dbs.insertFiles(procDataset, filesToCommit, fileBlock) filesToCommit = [] #logging.debug("Inserted files: %s to FileBlock: %s" \ # % ( ([ x['LogicalFileName'] for x in insertFiles ]),fileBlock['Name'])) except DbsException as ex: msg = "Error in DBSWriter.insertFiles\n" msg += "Cannot insert processed files:\n" #msg += " %s\n" % ([ x['LogicalFileName'] for x in insertFiles ],) msg += "%s\n" % formatEx(ex) raise DBSWriterError(msg) # // # // Close the block #// self.dbs.closeBlock( DBSWriterObjects.createDBSFileBlock(fileblockName) ) if self.globalDBSUrl: self.dbs.dbsMigrateBlock(srcURL = self.args['url'], dstURL = self.globalDBSUrl, block_name = fileblockName, srcVersion = self.version, dstVersion = self.globalVersion ) for algo in algos: pass #self.globalDBS.insertAlgoInPD(dataset = get_path(fileblockName.split('#')[0]), # algorithm = algo) logging.info("Migrated block %s to global" %(fileblockName)) else: logging.error("Should've migrated block %s, but didn't" % (fileblockName)) return closeBlock