try:
self.dbs.insertFiles(procDataset, filesToCommit, fileBlock)
filesToCommit = []
# logging.debug("Inserted files: %s to FileBlock: %s" \
# % ( ([ x['LogicalFileName'] for x in insertFiles ]),fileBlock['Name']))
except DbsException, ex:
msg = "Error in DBSWriter.insertFiles\n"
msg += "Cannot insert processed files:\n"
# msg += " %s\n" % ([ x['LogicalFileName'] for x in insertFiles ],)
msg += "%s\n" % formatEx(ex)
raise DBSWriterError(msg)
# //
# // Close the block
# //
self.dbs.closeBlock(DBSWriterObjects.createDBSFileBlock(fileblockName))
if self.globalDBSUrl:
self.dbs.dbsMigrateBlock(
srcURL=self.args["url"],
dstURL=self.globalDBSUrl,
block_name=fileblockName,
srcVersion=self.version,
dstVersion=self.globalVersion,
)
for algo in algos:
pass
# self.globalDBS.insertAlgoInPD(dataset = get_path(fileblockName.split('#')[0]),
# algorithm = algo)
logging.info("Migrated block %s to global" % (fileblockName))
else:
def manageFileBlock(self,
fileBlock,
maxFiles=100,
maxSize=None,
timeOut=None,
algos=[],
filesToCommit=[],
procDataset=None):
"""
_manageFileBlock_
Check to see wether the fileblock with the provided name
is closeable based on number of files or total size.
If the block equals or exceeds wither the maxFiles or maxSize
parameters, close the block and return True, else do nothing and
return False
"""
# //
# // Check that the block exists, and is open before we close it
#//
fileblockName = fileBlock['Name']
blockInstance = self.dbs.listBlocks(block_name=fileblockName)
if len(blockInstance) > 1:
msg = "Multiple Blocks matching name: %s\n" % fileblockName
msg += "Unable to manage file block..."
raise DBSWriterError(msg)
if len(blockInstance) == 0:
msg = "Block name %s not found\n" % fileblockName
msg += "Cant manage a non-existent fileblock"
raise DBSWriterError(msg)
blockInstance = blockInstance[0]
isClosed = blockInstance.get('OpenForWriting', '1')
if isClosed != '1':
msg = "Block %s already closed" % fileblockName
logging.warning(msg)
# Now we need to commit files
if len(filesToCommit) > 0:
try:
self.dbs.insertFiles(procDataset, filesToCommit, fileBlock)
filesToCommit = []
except DbsException as ex:
msg = "Error in DBSWriter.insertFiles\n"
msg += "Cannot insert processed files:\n"
raise DBSWriterError(msg)
# Attempting to migrate to global
if self.globalDBSUrl:
self.dbs.dbsMigrateBlock(
srcURL=self.args['url'],
dstURL=self.globalDBSUrl,
block_name=fileblockName,
srcVersion=self.version,
dstVersion=self.globalVersion,
)
#for algo in algos:
# self.globalDBS.insertAlgoInPD(dataset = get_path(fileblockName.split('#')[0]),
# algorithm = algo)
logging.info(
"Migrated block %s to global due to pre-closed status" %
(fileblockName))
else:
logging.error(
"Should've migrated block %s because it was already closed, but didn't"
% (fileblockName))
return True
# //
# // We have an open block, sum number of files and file sizes
#//
#fileCount = int(blockInstance.get('NumberOfFiles', 0))
fileCount = len(fileBlock['files'])
totalSize = float(blockInstance.get('BlockSize', 0))
msg = "Fileblock: %s\n ==> Size: %s Files: %s\n" % (
fileblockName, totalSize, fileCount)
logging.warning(msg)
# //
# // Test close block conditions
#//
closeBlock = False
if timeOut:
if int(time.time()) - int(blockInstance['CreationDate']) > timeOut:
closeBlock = True
msg = "Closing Block based on timeOut: %s" % fileblockName
logging.debug(msg)
if fileCount >= maxFiles:
closeBlock = True
msg = "Closing Block Based on files: %s" % fileblockName
logging.debug(msg)
if maxSize != None:
if totalSize >= maxSize:
closeBlock = True
msg = "Closing Block Based on size: %s" % fileblockName
logging.debug(msg)
if closeBlock:
# Now we need to commit files
if len(filesToCommit) > 0:
try:
self.dbs.insertFiles(procDataset, filesToCommit, fileBlock)
filesToCommit = []
#logging.debug("Inserted files: %s to FileBlock: %s" \
# % ( ([ x['LogicalFileName'] for x in insertFiles ]),fileBlock['Name']))
except DbsException as ex:
msg = "Error in DBSWriter.insertFiles\n"
msg += "Cannot insert processed files:\n"
#msg += " %s\n" % ([ x['LogicalFileName'] for x in insertFiles ],)
msg += "%s\n" % formatEx(ex)
raise DBSWriterError(msg)
# //
# // Close the block
#//
self.dbs.closeBlock(
DBSWriterObjects.createDBSFileBlock(fileblockName))
if self.globalDBSUrl:
self.dbs.dbsMigrateBlock(srcURL=self.args['url'],
dstURL=self.globalDBSUrl,
block_name=fileblockName,
srcVersion=self.version,
dstVersion=self.globalVersion)
for algo in algos:
pass
#self.globalDBS.insertAlgoInPD(dataset = get_path(fileblockName.split('#')[0]),
# algorithm = algo)
logging.info("Migrated block %s to global" % (fileblockName))
else:
logging.error("Should've migrated block %s, but didn't" %
(fileblockName))
return closeBlock
self.dbs.insertFiles(procDataset, filesToCommit, fileBlock)
filesToCommit = []
#logging.debug("Inserted files: %s to FileBlock: %s" \
# % ( ([ x['LogicalFileName'] for x in insertFiles ]),fileBlock['Name']))
except DbsException, ex:
msg = "Error in DBSWriter.insertFiles\n"
msg += "Cannot insert processed files:\n"
#msg += " %s\n" % ([ x['LogicalFileName'] for x in insertFiles ],)
msg += "%s\n" % formatEx(ex)
raise DBSWriterError(msg)
# //
# // Close the block
#//
self.dbs.closeBlock(
DBSWriterObjects.createDBSFileBlock(fileblockName)
)
if self.globalDBSUrl:
self.dbs.dbsMigrateBlock(srcURL = self.args['url'],
dstURL = self.globalDBSUrl,
block_name = fileblockName,
srcVersion = self.version,
dstVersion = self.globalVersion
)
for algo in algos:
pass
#self.globalDBS.insertAlgoInPD(dataset = get_path(fileblockName.split('#')[0]),
# algorithm = algo)
logging.info("Migrated block %s to global" %(fileblockName))
else:
self.dbs.insertFiles(procDataset, filesToCommit, fileBlock)
filesToCommit = []
#logging.debug("Inserted files: %s to FileBlock: %s" \
# % ( ([ x['LogicalFileName'] for x in insertFiles ]),fileBlock['Name']))
except DbsException, ex:
msg = "Error in DBSWriter.insertFiles\n"
msg += "Cannot insert processed files:\n"
#msg += " %s\n" % ([ x['LogicalFileName'] for x in insertFiles ],)
msg += "%s\n" % formatEx(ex)
raise DBSWriterError(msg)
# //
# // Close the block
#//
self.dbs.closeBlock(
DBSWriterObjects.createDBSFileBlock(fileblockName))
if self.globalDBSUrl:
self.dbs.dbsMigrateBlock(srcURL=self.args['url'],
dstURL=self.globalDBSUrl,
block_name=fileblockName,
srcVersion=self.version,
dstVersion=self.globalVersion)
for algo in algos:
pass
#self.globalDBS.insertAlgoInPD(dataset = get_path(fileblockName.split('#')[0]),
# algorithm = algo)
logging.info("Migrated block %s to global" % (fileblockName))
else:
logging.error("Should've migrated block %s, but didn't" %
(fileblockName))
def manageFileBlock(self, fileBlock, maxFiles = 100, maxSize = None,
timeOut = None, algos = [], filesToCommit = [],
procDataset = None):
"""
_manageFileBlock_
Check to see wether the fileblock with the provided name
is closeable based on number of files or total size.
If the block equals or exceeds wither the maxFiles or maxSize
parameters, close the block and return True, else do nothing and
return False
"""
# //
# // Check that the block exists, and is open before we close it
#//
fileblockName = fileBlock['Name']
blockInstance = self.dbs.listBlocks(block_name=fileblockName)
if len(blockInstance) > 1:
msg = "Multiple Blocks matching name: %s\n" % fileblockName
msg += "Unable to manage file block..."
raise DBSWriterError(msg)
if len(blockInstance) == 0:
msg = "Block name %s not found\n" % fileblockName
msg += "Cant manage a non-existent fileblock"
raise DBSWriterError(msg)
blockInstance = blockInstance[0]
isClosed = blockInstance.get('OpenForWriting', '1')
if isClosed != '1':
msg = "Block %s already closed" % fileblockName
logging.warning(msg)
# Now we need to commit files
if len(filesToCommit) > 0:
try:
self.dbs.insertFiles(procDataset, filesToCommit, fileBlock)
filesToCommit = []
except DbsException as ex:
msg = "Error in DBSWriter.insertFiles\n"
msg += "Cannot insert processed files:\n"
raise DBSWriterError(msg)
# Attempting to migrate to global
if self.globalDBSUrl:
self.dbs.dbsMigrateBlock(srcURL = self.args['url'],
dstURL = self.globalDBSUrl,
block_name = fileblockName,
srcVersion = self.version,
dstVersion = self.globalVersion,
)
#for algo in algos:
# self.globalDBS.insertAlgoInPD(dataset = get_path(fileblockName.split('#')[0]),
# algorithm = algo)
logging.info("Migrated block %s to global due to pre-closed status" %(fileblockName))
else:
logging.error("Should've migrated block %s because it was already closed, but didn't" % (fileblockName))
return True
# //
# // We have an open block, sum number of files and file sizes
#//
#fileCount = int(blockInstance.get('NumberOfFiles', 0))
fileCount = len(fileBlock['files'])
totalSize = float(blockInstance.get('BlockSize', 0))
msg = "Fileblock: %s\n ==> Size: %s Files: %s\n" % (
fileblockName, totalSize, fileCount)
logging.warning(msg)
# //
# // Test close block conditions
#//
closeBlock = False
if timeOut:
if int(time.time()) - int(blockInstance['CreationDate']) > timeOut:
closeBlock = True
msg = "Closing Block based on timeOut: %s" % fileblockName
logging.debug(msg)
if fileCount >= maxFiles:
closeBlock = True
msg = "Closing Block Based on files: %s" % fileblockName
logging.debug(msg)
if maxSize != None:
if totalSize >= maxSize:
closeBlock = True
msg = "Closing Block Based on size: %s" % fileblockName
logging.debug(msg)
if closeBlock:
# Now we need to commit files
if len(filesToCommit) > 0:
try:
self.dbs.insertFiles(procDataset, filesToCommit, fileBlock)
filesToCommit = []
#logging.debug("Inserted files: %s to FileBlock: %s" \
# % ( ([ x['LogicalFileName'] for x in insertFiles ]),fileBlock['Name']))
except DbsException as ex:
msg = "Error in DBSWriter.insertFiles\n"
msg += "Cannot insert processed files:\n"
#msg += " %s\n" % ([ x['LogicalFileName'] for x in insertFiles ],)
msg += "%s\n" % formatEx(ex)
raise DBSWriterError(msg)
# //
# // Close the block
#//
self.dbs.closeBlock(
DBSWriterObjects.createDBSFileBlock(fileblockName)
)
if self.globalDBSUrl:
self.dbs.dbsMigrateBlock(srcURL = self.args['url'],
dstURL = self.globalDBSUrl,
block_name = fileblockName,
srcVersion = self.version,
dstVersion = self.globalVersion
)
for algo in algos:
pass
#self.globalDBS.insertAlgoInPD(dataset = get_path(fileblockName.split('#')[0]),
# algorithm = algo)
logging.info("Migrated block %s to global" %(fileblockName))
else:
logging.error("Should've migrated block %s, but didn't" % (fileblockName))
return closeBlock
