class DCapTaskCreate(IoTasklet): def __init__(self, fs, path, dest, is_directory): IoTasklet.__init__(self, fs, path, dest, is_directory) self.dcap = Dcap("dcap://dcache-lab000.desy.de:22125") def process(self): if not self.is_directory: version = GENERATION_FORMAT % (self.dest, time.time()) self.dcap.rename(self.dest, version) f = self.dcap.open_file(self.dest, 'w') f.send_file(self.path) f.close() def __del__(self): self.dcap._send_bye() self.dcap._close()
def run(self): global scriptId global running global dcapUrl dcap = Dcap(dcapUrl) try: now = int(datetime.now().strftime("%s")) ctime_threshold = (now - self.minAge*60) self.logger.debug("Looking for files matching { path: %s, group: %s, store: %s, ctime: { $lt: %d } }" % (self.pathPattern.pattern, self.sGroup.pattern, self.storeName.pattern, ctime_threshold) ) with self.db.files.find( { 'state': 'new', 'path': self.pathPattern, 'group': self.sGroup, 'store': self.storeName, 'ctime': { '$lt': ctime_threshold } }, timeout=False).batch_size(512) as cursor: cursor.sort('ctime', ASCENDING) sumsize = 0 old_file_mode = False ctime_oldfile_threshold = (now - self.maxAge*60) for f in cursor: if f['ctime'] < ctime_oldfile_threshold: old_file_mode = True sumsize += f['size'] filecount = cursor.count() self.logger.info("found %d files with a combined size of %d bytes" % (filecount, sumsize)) if old_file_mode: self.logger.debug("containing old files: ctime < %d" % ctime_oldfile_threshold) else: self.logger.debug("containing no old files: ctime < %d" % ctime_oldfile_threshold) if old_file_mode: if sumsize < self.archiveSize: self.logger.info("combined size of old files not big enough for a regular archive, packing in old-file-mode") else: old_file_mode = False self.logger.info("combined size of old files big enough for regular archive, packing in normal mode") elif sumsize < self.archiveSize: self.logger.info("no old files found and %d bytes missing to create regular archive of size %d, leaving packager" % (self.archiveSize-sumsize, self.archiveSize)) return cursor.rewind() container = None containerChimeraPath = None try: for f in cursor: if filecount <= 0 or sumsize <= 0: self.logger.info("Actual number of files exceeds precalculated number, will collect new files in next run.") break self.logger.debug("Next file %s [%s], remaining %d [%d bytes]" % (f['path'], f['pnfsid'], filecount, sumsize) ) if not running: if container: raise UserInterruptException(container.localfilepath) else: raise UserInterruptException(None) if container is None: if sumsize >= self.archiveSize or old_file_mode: container = Container(self.archivePath, dcap) self.logger.info("Creating new container %s . %d files [%d bytes] remaining." % (container.pnfsfilepath, filecount, sumsize)) else: self.logger.info("remaining combined size %d < %d, leaving packager" % (sumsize, self.archiveSize)) return if old_file_mode: self.logger.debug("%d bytes remaining for this archive" % sumsize) self.writeStatus(container.pnfsfilepath, sumsize, "%s [%s]" % ( f['path'], f['pnfsid'] )) else: self.logger.debug("%d bytes remaining for this archive" % (self.archiveSize-container.size)) self.writeStatus(container.pnfsfilepath, self.archiveSize-container.size, "%s [%s]" % ( f['path'], f['pnfsid'] )) try: localfile = f['path'].replace(dataRoot, mountPoint, 1) self.logger.debug("before container.add(%s[%s], %s)" % (f['path'], f['pnfsid'], f['size'])) container.add(f['pnfsid'], f['path'], localfile, f['size']) self.logger.debug("before collection.save") f['state'] = "added: %s" % container.pnfsfilepath f['lock'] = scriptId cursor.collection.save(f) self.logger.debug("Added file %s [%s]" % (f['path'], f['pnfsid'])) except IOError as e: self.logger.exception("IOError while adding file %s to archive %s [%s], %s" % (f['path'], container.pnfsfilepath, f['pnfsid'], e.message)) self.logger.debug("Removing entry for file %s" % f['pnfsid']) self.db.files.remove( { 'pnfsid': f['pnfsid'] } ) except OSError as e: self.logger.exception("OSError while adding file %s to archive %s [%s], %s" % (f['path'], f['pnfsid'], container.pnfsfilepath, e.message)) self.logger.debug("Removing entry for file %s" % f['pnfsid']) self.db.files.remove( { 'pnfsid': f['pnfsid'] } ) except errors.OperationFailure as e: self.logger.error("Removing container %s due to OperationalFailure. See below for details." % container.localfilepath) container.close() os.remove(container.localfilepath) raise e except errors.ConnectionFailure as e: self.logger.error("Removing container %s due to ConnectionFailure. See below for details." % container.localfilepath) container.close() os.remove(container.localfilepath) raise e sumsize -= f['size'] filecount -= 1 if container.size >= self.archiveSize: self.logger.debug("Closing full container %s" % container.pnfsfilepath) containerChimeraPath = container.pnfsfilepath container.close() if self.verifyContainer(container): self.logger.info("Container %s successfully stored" % container.pnfsfilepath) self.db.files.update( { 'state': 'added: %s' % containerChimeraPath }, { '$set': { 'state': 'archived: %s' % containerChimeraPath }, '$unset': { 'lock': "" } }, multi=True ) self.createArchiveEntry(container) else: self.logger.warn("Removing container %s due to verification error" % container.localfilepath) self.db.files.update( { 'state': 'added: %s' % containerChimeraPath }, { '$set': { 'state': 'new' }, '$unset': { 'lock': "" } }, multi=True ) os.remove(container.localfilepath) container = None if container: if not old_file_mode: self.logger.warn("Removing unful container %s . Maybe a file was deleted during packaging." % container.localfilepath) container.close() os.remove(container.localfilepath) return self.logger.debug("Closing container %s containing remaining old files", container.pnfsfilepath) containerChimeraPath = container.pnfsfilepath container.close() if self.verifyContainer(container): self.logger.info("Container %s with old files successfully stored" % container.pnfsfilepath) self.db.files.update( { 'state': 'added: %s' % containerChimeraPath }, { '$set': { 'state': 'archived: %s' % containerChimeraPath }, '$unset': { 'lock': "" } }, multi=True ) self.createArchiveEntry(container) else: self.logger.warn("Removing container %s with old files due to verification error" % container.localfilepath) self.db.files.update( { 'state': 'added: %s' % containerChimeraPath }, { '$set': { 'state': 'new' }, '$unset': { 'lock': "" } }, multi=True ) os.remove(container.localfilepath) except IOError as e: self.logger.error("%s closing file %s. Trying to clean up files in state: 'added'. This might need additional manual fixing!" % (e.strerror, containerChimeraPath)) self.db.files.update( { 'state': 'added: %s' % containerChimeraPath }, { '$set': { 'state': 'new' }, '$unset': { 'lock': "" } }, multi=True ) except errors.OperationFailure as e: self.logger.error("Operation Exception in database communication while creating container %s . Please check!" % containerChimeraPath ) self.logger.error('%s' % e.message) os.remove(container.localfilepath) except errors.ConnectionFailure as e: self.logger.error("Connection Exception in database communication. Removing incomplete container %s ." % containerChimeraPath) self.logger.error('%s' % e.message) os.remove(container.localfilepath) finally: dcap.close()
def __init__(self, fs, path, dest, is_directory): IoTasklet.__init__(self, fs, path, dest, is_directory) self.dcap = Dcap("dcap://dcache-lab000.desy.de:22125")
def run(self): global script_id global running global dcap_url dcap = Dcap(dcap_url) try: now = int(datetime.now().strftime("%s")) ctime_threshold = (now - self.min_age * 60) self.logger.debug( f"Looking for files matching {{ path: {self.path_pattern.pattern}, group: {self.s_group.pattern}, " f"store: {self.store_name.pattern}, ctime: {{ $lt: {ctime_threshold} }} }}" ) with self.db.files.find( { 'state': 'new', 'path': self.path_pattern, 'group': self.s_group, 'store': self.store_name, 'ctime': { '$lt': ctime_threshold } }, timeout=False).batch_size(512) as cursor: cursor.sort('ctime', ASCENDING) sumsize = 0 old_file_mode = False ctime_oldfile_threshold = (now - self.max_age * 60) for f in cursor: if f['ctime'] < ctime_oldfile_threshold: old_file_mode = True sumsize += f['size'] filecount = cursor.count() self.logger.info( f"found {filecount} files with a combined size of {sumsize} bytes" ) if old_file_mode: self.logger.debug( f"containing old files: ctime < {ctime_oldfile_threshold}" ) else: self.logger.debug( f"containing no old files: ctime < {ctime_oldfile_threshold}" ) if old_file_mode: if sumsize < self.archive_size: self.logger.info( "combined size of old files not big enough for a regular archive, packing in old-file-mode" ) else: old_file_mode = False self.logger.info( "combined size of old files big enough for regular archive, packing in normal mode" ) elif sumsize < self.archive_size: self.logger.info( f"no old files found and {self.archive_size - sumsize} bytes missing to create regular " f"archive of size {self.archive_size}, leaving packager" ) return cursor.rewind() container = None container_chimera_path = None try: for f in cursor: if filecount <= 0 or sumsize <= 0: self.logger.info( "Actual number of files exceeds precalculated number, will collect new files in next " "run.") break self.logger.debug( f"Next file {f['path']} [{f['pnfsid']}], remaining {filecount} [{sumsize} bytes]" ) if not running: if container: raise UserInterruptException( container.localfilepath) else: raise UserInterruptException(None) if container is None: if sumsize >= self.archive_size or old_file_mode: container = Container(self.archive_path, dcap) self.logger.info( f"Creating new container {container.pnfsfilepath} . {filecount} files " f"[{sumsize} bytes] remaining.") else: self.logger.info( f"remaining combined size {sumsize} < {self.archive_size}, leaving packager" ) return if old_file_mode: self.logger.debug( f"{sumsize} bytes remaining for this archive") self.write_status(container.pnfsfilepath, sumsize, f"{f['path']} [{f['pnfsid']}]") else: self.logger.debug( f"{self.archive_size - container.size} bytes remaining for this archive" ) self.write_status( container.pnfsfilepath, self.archive_size - container.size, f"{f['path']} [{f['pnfsid']}]") try: localfile = f['path'].replace( data_root, mount_point, 1) self.logger.debug( f"before container.add({f['path']}[{f['pnfsid']}], {f['size']})" ) container.add(f['pnfsid'], f['path'], localfile, f['size']) self.logger.debug("before collection.save") f['state'] = f"added: {container.pnfsfilepath}" f['lock'] = script_id cursor.collection.save(f) self.logger.debug( f"Added file {f['path']} [{f['pnfsid']}]") except IOError as e: self.logger.exception( f"IOError while adding file {f['path']} to archive {container.pnfsfilepath} [{f['pnfsid']}], {str(e)}" ) self.logger.debug( f"Removing entry for file {f['pnfsid']}") self.db.files.remove({'pnfsid': f['pnfsid']}) except OSError as e: self.logger.exception( f"OSError while adding file {f['path']} to archive {f['pnfsid']} [{container.pnfsfilepath}], {str(e)}" ) self.logger.debug( f"Removing entry for file {f['pnfsid']}") self.db.files.remove({'pnfsid': f['pnfsid']}) except errors.OperationFailure as e: self.logger.error( f"Removing container {container.localfilepath} due to OperationalFailure. See below for details." ) container.close() os.remove(container.localfilepath) raise e except errors.ConnectionFailure as e: self.logger.error( f"Removing container {container.localfilepath} due to ConnectionFailure. See below for details." ) container.close() os.remove(container.localfilepath) raise e sumsize -= f['size'] filecount -= 1 if container.size >= self.archive_size: self.logger.debug( f"Closing full container {container.pnfsfilepath}" ) container_chimera_path = container.pnfsfilepath container.close() if self.verify_container(container): self.logger.info( f"Container {container.pnfsfilepath} successfully stored" ) self.db.files.update( { 'state': f'added: {container_chimera_path}' }, { '$set': { 'state': f'archived: {container_chimera_path}' }, '$unset': { 'lock': "" } }, multi=True) self.create_archive_entry(container) else: self.logger.warning( f"Removing container {container.localfilepath} due to verification error" ) self.db.files.update( { 'state': f'added: {container_chimera_path}' }, { '$set': { 'state': 'new' }, '$unset': { 'lock': "" } }, multi=True) os.remove(container.localfilepath) container = None if container: if not old_file_mode: self.logger.warning( f"Removing unful container {container.localfilepath} . Maybe a file was deleted " f"during packaging.") container.close() os.remove(container.localfilepath) return self.logger.debug( f"Closing container {container.pnfsfilepath} containing remaining old files" ) container_chimera_path = container.pnfsfilepath container.close() if self.verify_container(container): self.logger.info( f"Container {container.pnfsfilepath} with old files successfully stored" ) self.db.files.update( {'state': f'added: {container_chimera_path}'}, { '$set': { 'state': f'archived: {container_chimera_path}' }, '$unset': { 'lock': "" } }, multi=True) self.create_archive_entry(container) else: self.logger.warning( f"Removing container {container.localfilepath} with old files due to verification error" ) self.db.files.update( {'state': f'added: {container_chimera_path}'}, { '$set': { 'state': 'new' }, '$unset': { 'lock': "" } }, multi=True) os.remove(container.localfilepath) except InterruptedError: self.logger.info(f"Caught interruption. Cleanup") os.remove(container.localfilepath) # if lock is script_id, the state is set to new and lock removed when while-loop is entered # in main dcap.close() sys.exit("Interruption signal") except IOError as e: self.logger.error( f"{e.strerror} closing file {container_chimera_path}. Trying to clean up files in state: " f"'added'. This might need additional manual fixing!") self.db.files.update( {'state': f'added: {container_chimera_path}'}, { '$set': { 'state': 'new' }, '$unset': { 'lock': "" } }, multi=True) except errors.OperationFailure as e: self.logger.error( f"Operation Exception in database communication while creating container " f"{container_chimera_path} . Please check!") self.logger.error(f'{str(e)}') os.remove(container.localfilepath) except errors.ConnectionFailure as e: self.logger.error( f"Connection Exception in database communication. Removing incomplete container " f"{container_chimera_path} .") self.logger.error(f'{str(e)}') os.remove(container.localfilepath) finally: dcap.close()