class DCapTaskCreate(IoTasklet):
def __init__(self, fs, path, dest, is_directory):
IoTasklet.__init__(self, fs, path, dest, is_directory)
self.dcap = Dcap("dcap://dcache-lab000.desy.de:22125")
def process(self):
if not self.is_directory:
version = GENERATION_FORMAT % (self.dest, time.time())
self.dcap.rename(self.dest, version)
f = self.dcap.open_file(self.dest, 'w')
f.send_file(self.path)
f.close()
def __del__(self):
self.dcap._send_bye()
self.dcap._close()
def run(self):
global scriptId
global running
global dcapUrl
dcap = Dcap(dcapUrl)
try:
now = int(datetime.now().strftime("%s"))
ctime_threshold = (now - self.minAge*60)
self.logger.debug("Looking for files matching { path: %s, group: %s, store: %s, ctime: { $lt: %d } }" % (self.pathPattern.pattern, self.sGroup.pattern, self.storeName.pattern, ctime_threshold) )
with self.db.files.find( { 'state': 'new', 'path': self.pathPattern, 'group': self.sGroup, 'store': self.storeName, 'ctime': { '$lt': ctime_threshold } }, timeout=False).batch_size(512) as cursor:
cursor.sort('ctime', ASCENDING)
sumsize = 0
old_file_mode = False
ctime_oldfile_threshold = (now - self.maxAge*60)
for f in cursor:
if f['ctime'] < ctime_oldfile_threshold:
old_file_mode = True
sumsize += f['size']
filecount = cursor.count()
self.logger.info("found %d files with a combined size of %d bytes" % (filecount, sumsize))
if old_file_mode:
self.logger.debug("containing old files: ctime < %d" % ctime_oldfile_threshold)
else:
self.logger.debug("containing no old files: ctime < %d" % ctime_oldfile_threshold)
if old_file_mode:
if sumsize < self.archiveSize:
self.logger.info("combined size of old files not big enough for a regular archive, packing in old-file-mode")
else:
old_file_mode = False
self.logger.info("combined size of old files big enough for regular archive, packing in normal mode")
elif sumsize < self.archiveSize:
self.logger.info("no old files found and %d bytes missing to create regular archive of size %d, leaving packager" % (self.archiveSize-sumsize, self.archiveSize))
return
cursor.rewind()
container = None
containerChimeraPath = None
try:
for f in cursor:
if filecount <= 0 or sumsize <= 0:
self.logger.info("Actual number of files exceeds precalculated number, will collect new files in next run.")
break
self.logger.debug("Next file %s [%s], remaining %d [%d bytes]" % (f['path'], f['pnfsid'], filecount, sumsize) )
if not running:
if container:
raise UserInterruptException(container.localfilepath)
else:
raise UserInterruptException(None)
if container is None:
if sumsize >= self.archiveSize or old_file_mode:
container = Container(self.archivePath, dcap)
self.logger.info("Creating new container %s . %d files [%d bytes] remaining." % (container.pnfsfilepath, filecount, sumsize))
else:
self.logger.info("remaining combined size %d < %d, leaving packager" % (sumsize, self.archiveSize))
return
if old_file_mode:
self.logger.debug("%d bytes remaining for this archive" % sumsize)
self.writeStatus(container.pnfsfilepath, sumsize, "%s [%s]" % ( f['path'], f['pnfsid'] ))
else:
self.logger.debug("%d bytes remaining for this archive" % (self.archiveSize-container.size))
self.writeStatus(container.pnfsfilepath, self.archiveSize-container.size, "%s [%s]" % ( f['path'], f['pnfsid'] ))
try:
localfile = f['path'].replace(dataRoot, mountPoint, 1)
self.logger.debug("before container.add(%s[%s], %s)" % (f['path'], f['pnfsid'], f['size']))
container.add(f['pnfsid'], f['path'], localfile, f['size'])
self.logger.debug("before collection.save")
f['state'] = "added: %s" % container.pnfsfilepath
f['lock'] = scriptId
cursor.collection.save(f)
self.logger.debug("Added file %s [%s]" % (f['path'], f['pnfsid']))
except IOError as e:
self.logger.exception("IOError while adding file %s to archive %s [%s], %s" % (f['path'], container.pnfsfilepath, f['pnfsid'], e.message))
self.logger.debug("Removing entry for file %s" % f['pnfsid'])
self.db.files.remove( { 'pnfsid': f['pnfsid'] } )
except OSError as e:
self.logger.exception("OSError while adding file %s to archive %s [%s], %s" % (f['path'], f['pnfsid'], container.pnfsfilepath, e.message))
self.logger.debug("Removing entry for file %s" % f['pnfsid'])
self.db.files.remove( { 'pnfsid': f['pnfsid'] } )
except errors.OperationFailure as e:
self.logger.error("Removing container %s due to OperationalFailure. See below for details." % container.localfilepath)
container.close()
os.remove(container.localfilepath)
raise e
except errors.ConnectionFailure as e:
self.logger.error("Removing container %s due to ConnectionFailure. See below for details." % container.localfilepath)
container.close()
os.remove(container.localfilepath)
raise e
sumsize -= f['size']
filecount -= 1
if container.size >= self.archiveSize:
self.logger.debug("Closing full container %s" % container.pnfsfilepath)
containerChimeraPath = container.pnfsfilepath
container.close()
if self.verifyContainer(container):
self.logger.info("Container %s successfully stored" % container.pnfsfilepath)
self.db.files.update( { 'state': 'added: %s' % containerChimeraPath }, { '$set': { 'state': 'archived: %s' % containerChimeraPath }, '$unset': { 'lock': "" } }, multi=True )
self.createArchiveEntry(container)
else:
self.logger.warn("Removing container %s due to verification error" % container.localfilepath)
self.db.files.update( { 'state': 'added: %s' % containerChimeraPath }, { '$set': { 'state': 'new' }, '$unset': { 'lock': "" } }, multi=True )
os.remove(container.localfilepath)
container = None
if container:
if not old_file_mode:
self.logger.warn("Removing unful container %s . Maybe a file was deleted during packaging." % container.localfilepath)
container.close()
os.remove(container.localfilepath)
return
self.logger.debug("Closing container %s containing remaining old files", container.pnfsfilepath)
containerChimeraPath = container.pnfsfilepath
container.close()
if self.verifyContainer(container):
self.logger.info("Container %s with old files successfully stored" % container.pnfsfilepath)
self.db.files.update( { 'state': 'added: %s' % containerChimeraPath }, { '$set': { 'state': 'archived: %s' % containerChimeraPath }, '$unset': { 'lock': "" } }, multi=True )
self.createArchiveEntry(container)
else:
self.logger.warn("Removing container %s with old files due to verification error" % container.localfilepath)
self.db.files.update( { 'state': 'added: %s' % containerChimeraPath }, { '$set': { 'state': 'new' }, '$unset': { 'lock': "" } }, multi=True )
os.remove(container.localfilepath)
except IOError as e:
self.logger.error("%s closing file %s. Trying to clean up files in state: 'added'. This might need additional manual fixing!" % (e.strerror, containerChimeraPath))
self.db.files.update( { 'state': 'added: %s' % containerChimeraPath }, { '$set': { 'state': 'new' }, '$unset': { 'lock': "" } }, multi=True )
except errors.OperationFailure as e:
self.logger.error("Operation Exception in database communication while creating container %s . Please check!" % containerChimeraPath )
self.logger.error('%s' % e.message)
os.remove(container.localfilepath)
except errors.ConnectionFailure as e:
self.logger.error("Connection Exception in database communication. Removing incomplete container %s ." % containerChimeraPath)
self.logger.error('%s' % e.message)
os.remove(container.localfilepath)
finally:
dcap.close()
def __init__(self, fs, path, dest, is_directory):
IoTasklet.__init__(self, fs, path, dest, is_directory)
self.dcap = Dcap("dcap://dcache-lab000.desy.de:22125")
def run(self):
global script_id
global running
global dcap_url
dcap = Dcap(dcap_url)
try:
now = int(datetime.now().strftime("%s"))
ctime_threshold = (now - self.min_age * 60)
self.logger.debug(
f"Looking for files matching {{ path: {self.path_pattern.pattern}, group: {self.s_group.pattern}, "
f"store: {self.store_name.pattern}, ctime: {{ $lt: {ctime_threshold} }} }}"
)
with self.db.files.find(
{
'state': 'new',
'path': self.path_pattern,
'group': self.s_group,
'store': self.store_name,
'ctime': {
'$lt': ctime_threshold
}
},
timeout=False).batch_size(512) as cursor:
cursor.sort('ctime', ASCENDING)
sumsize = 0
old_file_mode = False
ctime_oldfile_threshold = (now - self.max_age * 60)
for f in cursor:
if f['ctime'] < ctime_oldfile_threshold:
old_file_mode = True
sumsize += f['size']
filecount = cursor.count()
self.logger.info(
f"found {filecount} files with a combined size of {sumsize} bytes"
)
if old_file_mode:
self.logger.debug(
f"containing old files: ctime < {ctime_oldfile_threshold}"
)
else:
self.logger.debug(
f"containing no old files: ctime < {ctime_oldfile_threshold}"
)
if old_file_mode:
if sumsize < self.archive_size:
self.logger.info(
"combined size of old files not big enough for a regular archive, packing in old-file-mode"
)
else:
old_file_mode = False
self.logger.info(
"combined size of old files big enough for regular archive, packing in normal mode"
)
elif sumsize < self.archive_size:
self.logger.info(
f"no old files found and {self.archive_size - sumsize} bytes missing to create regular "
f"archive of size {self.archive_size}, leaving packager"
)
return
cursor.rewind()
container = None
container_chimera_path = None
try:
for f in cursor:
if filecount <= 0 or sumsize <= 0:
self.logger.info(
"Actual number of files exceeds precalculated number, will collect new files in next "
"run.")
break
self.logger.debug(
f"Next file {f['path']} [{f['pnfsid']}], remaining {filecount} [{sumsize} bytes]"
)
if not running:
if container:
raise UserInterruptException(
container.localfilepath)
else:
raise UserInterruptException(None)
if container is None:
if sumsize >= self.archive_size or old_file_mode:
container = Container(self.archive_path, dcap)
self.logger.info(
f"Creating new container {container.pnfsfilepath} . {filecount} files "
f"[{sumsize} bytes] remaining.")
else:
self.logger.info(
f"remaining combined size {sumsize} < {self.archive_size}, leaving packager"
)
return
if old_file_mode:
self.logger.debug(
f"{sumsize} bytes remaining for this archive")
self.write_status(container.pnfsfilepath, sumsize,
f"{f['path']} [{f['pnfsid']}]")
else:
self.logger.debug(
f"{self.archive_size - container.size} bytes remaining for this archive"
)
self.write_status(
container.pnfsfilepath,
self.archive_size - container.size,
f"{f['path']} [{f['pnfsid']}]")
try:
localfile = f['path'].replace(
data_root, mount_point, 1)
self.logger.debug(
f"before container.add({f['path']}[{f['pnfsid']}], {f['size']})"
)
container.add(f['pnfsid'], f['path'], localfile,
f['size'])
self.logger.debug("before collection.save")
f['state'] = f"added: {container.pnfsfilepath}"
f['lock'] = script_id
cursor.collection.save(f)
self.logger.debug(
f"Added file {f['path']} [{f['pnfsid']}]")
except IOError as e:
self.logger.exception(
f"IOError while adding file {f['path']} to archive {container.pnfsfilepath} [{f['pnfsid']}], {str(e)}"
)
self.logger.debug(
f"Removing entry for file {f['pnfsid']}")
self.db.files.remove({'pnfsid': f['pnfsid']})
except OSError as e:
self.logger.exception(
f"OSError while adding file {f['path']} to archive {f['pnfsid']} [{container.pnfsfilepath}], {str(e)}"
)
self.logger.debug(
f"Removing entry for file {f['pnfsid']}")
self.db.files.remove({'pnfsid': f['pnfsid']})
except errors.OperationFailure as e:
self.logger.error(
f"Removing container {container.localfilepath} due to OperationalFailure. See below for details."
)
container.close()
os.remove(container.localfilepath)
raise e
except errors.ConnectionFailure as e:
self.logger.error(
f"Removing container {container.localfilepath} due to ConnectionFailure. See below for details."
)
container.close()
os.remove(container.localfilepath)
raise e
sumsize -= f['size']
filecount -= 1
if container.size >= self.archive_size:
self.logger.debug(
f"Closing full container {container.pnfsfilepath}"
)
container_chimera_path = container.pnfsfilepath
container.close()
if self.verify_container(container):
self.logger.info(
f"Container {container.pnfsfilepath} successfully stored"
)
self.db.files.update(
{
'state':
f'added: {container_chimera_path}'
}, {
'$set': {
'state':
f'archived: {container_chimera_path}'
},
'$unset': {
'lock': ""
}
},
multi=True)
self.create_archive_entry(container)
else:
self.logger.warning(
f"Removing container {container.localfilepath} due to verification error"
)
self.db.files.update(
{
'state':
f'added: {container_chimera_path}'
}, {
'$set': {
'state': 'new'
},
'$unset': {
'lock': ""
}
},
multi=True)
os.remove(container.localfilepath)
container = None
if container:
if not old_file_mode:
self.logger.warning(
f"Removing unful container {container.localfilepath} . Maybe a file was deleted "
f"during packaging.")
container.close()
os.remove(container.localfilepath)
return
self.logger.debug(
f"Closing container {container.pnfsfilepath} containing remaining old files"
)
container_chimera_path = container.pnfsfilepath
container.close()
if self.verify_container(container):
self.logger.info(
f"Container {container.pnfsfilepath} with old files successfully stored"
)
self.db.files.update(
{'state': f'added: {container_chimera_path}'},
{
'$set': {
'state':
f'archived: {container_chimera_path}'
},
'$unset': {
'lock': ""
}
},
multi=True)
self.create_archive_entry(container)
else:
self.logger.warning(
f"Removing container {container.localfilepath} with old files due to verification error"
)
self.db.files.update(
{'state': f'added: {container_chimera_path}'},
{
'$set': {
'state': 'new'
},
'$unset': {
'lock': ""
}
},
multi=True)
os.remove(container.localfilepath)
except InterruptedError:
self.logger.info(f"Caught interruption. Cleanup")
os.remove(container.localfilepath)
# if lock is script_id, the state is set to new and lock removed when while-loop is entered
# in main
dcap.close()
sys.exit("Interruption signal")
except IOError as e:
self.logger.error(
f"{e.strerror} closing file {container_chimera_path}. Trying to clean up files in state: "
f"'added'. This might need additional manual fixing!")
self.db.files.update(
{'state': f'added: {container_chimera_path}'}, {
'$set': {
'state': 'new'
},
'$unset': {
'lock': ""
}
},
multi=True)
except errors.OperationFailure as e:
self.logger.error(
f"Operation Exception in database communication while creating container "
f"{container_chimera_path} . Please check!")
self.logger.error(f'{str(e)}')
os.remove(container.localfilepath)
except errors.ConnectionFailure as e:
self.logger.error(
f"Connection Exception in database communication. Removing incomplete container "
f"{container_chimera_path} .")
self.logger.error(f'{str(e)}')
os.remove(container.localfilepath)
finally:
dcap.close()
