def __init__(self, application, output_queue): self.logger = logging.getLogger("FR.%s" % self.__class__.__name__) self.events = deque([]) # The data structure that actually holds the status of the pathnames self.map = EventsTodoStructure() self._last_event_for_pathname = {} self.access = RLock() self.application = application self._output_queue = output_queue
class EventsQueue(object): """ Central database that tracks the status of all pathnames in the warebox. This component holds a record for every known pathname in the warebox, remembering its status, if it's currently under synchronization, etc. Other threads can update it by the mean of PathnameEvent objects. After an update, a pathname needs to be synchronized. Other threads can query EventsQueue for pathnames to synchronize, which are returned as PathnameOperation objects. The operation type tells what to do for synchronizing the pathname. """ def __init__(self, application, output_queue): self.logger = logging.getLogger("FR.%s" % self.__class__.__name__) self.events = deque([]) # The data structure that actually holds the status of the pathnames self.map = EventsTodoStructure() self._last_event_for_pathname = {} self.access = RLock() self.application = application self._output_queue = output_queue def length(self): """ @return Length of the queue of the pathnames that need synchronization. """ with self.access: return len(self.events) def isEmpty(self): """ @return Boolean telling whether there are pathnames that need synchronization. """ return self.length() < 1 def clear(self): """ Clear all internal data structures, thus forgetting about any pathname. """ with self.access: self.events.clear() self.map.clear() self._last_event_for_pathname.clear() def terminate(self): """ Terminate this component, release any acquired resource. """ with self.access: self.map.terminate() self.clear() def put(self, event): """ Update a pathname status with a pathname event. Any ongoing synchronization activity on the pathname gets interrupted. A PathnameOperation is produced and sent to the output queue. @param event: Instance of PathnameEvent """ with self.access: self._digest(event) self._send_pathname_operation() def _digest(self, event): """ Update a pathname status with a pathname event. Any ongoing synchronization activity on the pathname gets interrupted. @param event: Instance of PathnameEvent """ self.logger.debug(u'Digesting event %s' % event) self._last_event_for_pathname[event.pathname] = event # Force copies to be creations. We aren't ready to handle copies yet. if event.action == 'COPY': event.action = 'CREATE' event.paired_pathname = None if event.action in ['CREATE', 'MODIFY', 'DELETE', 'UPDATE_FROM_REMOTE', 'REMOTELY_DELETED']: self._digest_single_pathname_event(event) else: # TODO: we don't support COPY yet raise Exception('EventsQueue, unsupported event: %s' % event) def _digest_single_pathname_event(self, event): """ Handle status transitions for those events which involve just one pathname (e.g. UPDATE, DELETE, etc). """ #self.logger.debug(u'Digesting single pathname event "%s"' % repr(event)) action, pathname = event.action, event.pathname if self.map.isLocked(pathname): self.logger.debug( u'Pathname "%s" seems worker-locked. Sending' ' termination request to current worker.' % pathname) # Note: self.on_file_operation_abort is called on abort file_operation = self.map.getLockingWorker(pathname) file_operation.abort() if action == 'CREATE' or action == 'MODIFY': self.map.update(pathname) self.application.notify_pathname_status_change( pathname, PStatuses.TOBEUPLOADED) elif action == 'DELETE': self.map.delete(pathname) self.application.notify_pathname_status_change( pathname, PStatuses.DELETETOBESENT) elif action == 'UPDATE_FROM_REMOTE': self.map.update_from_remote(pathname) self.application.notify_pathname_status_change(pathname, PStatuses.TOBEDOWNLOADED) elif action == 'REMOTELY_DELETED': self.map.remotely_deleted(pathname) else: self.logger.warning(u'Unknown action requested: "%s" for' ' pathname "%s"' % (action, pathname)) self.events.append(pathname) def on_file_operation_complete(self, file_operation): """ Handler called by PathnameOperation objects when they have been completed. Update the pathname status and release any constraint. @param file_operation: The PathnameOperation that has been completed. """ with self.access: if self.map.has_constraints_from(file_operation.pathname): self.map.dropConstraintFrom(file_operation.pathname) self.map.setStatus('OK', file_operation.pathname) self.map.unlock(file_operation.pathname) def on_file_operation_abort(self, file_operation): """ Handler called by PathnameOperation objects when they have been aborted. Update the pathname status and release any constraint. Note: actually only EventsQueue can abort operations, so this is a self-call event handler. @param file_operation: The PathnameOperation that has been aborted. """ with self.access: # Note: constraints release here is reduntant, since it's performed # also on the next status change. I haven't decided yet which place # between here and there is better for this task. # The same goes for on_file_operation_complete. if self.map.has_constraints_from(file_operation.pathname): self.map.dropConstraintFrom(file_operation.pathname) self.map.setStatus('OK', file_operation.pathname) self.map.unlock(file_operation.pathname) self.application.notify_pathname_status_change(file_operation, PStatuses.ALIGNED) def _create_pathname_operation(self, status, pathname, oldpath=None): """ Factory method for PathnameOperation objects """ status_to_verb = { 'LN': 'UPLOAD', 'LD': 'DELETE', 'LRto': 'REMOTE_COPY', 'RN': 'DOWNLOAD', 'RD': 'DELETE_LOCAL' } event = self._last_event_for_pathname[pathname] operation = PathnameOperation( self.application, self.access, status_to_verb[status], pathname, oldpath, event.etag, event.size, event.lmtime, event.conflicted) operation.register_abort_handler(self.on_file_operation_abort) operation.register_complete_handler(self.on_file_operation_complete) return operation def _send_pathname_operation(self): """ Produce a PathnameOperation object corresponding to the last received PathnameEvent and send it to the output queue. """ pathname = self.events.popleft() status = self.map.getStatus(pathname) oldpath = None operation = self._create_pathname_operation(status, pathname, oldpath) self.map.lock(pathname, operation) self._output_queue.put(operation, 'operation')
class EventsQueue(object): """ Central database that tracks the status of all pathnames in the warebox. This component holds a record for every known pathname in the warebox, remembering its status, if it's currently under synchronization, etc. Other threads can update it by the mean of PathnameEvent objects. After an update, a pathname needs to be synchronized. Other threads can query EventsQueue for pathnames to synchronize, which are returned as PathnameOperation objects. The operation type tells what to do for synchronizing the pathname. """ def __init__(self, application, output_queue): self.logger = logging.getLogger("FR.%s" % self.__class__.__name__) self.events = deque([]) # The data structure that actually holds the status of the pathnames self.map = EventsTodoStructure() self._last_event_for_pathname = {} self.access = RLock() self.application = application self._output_queue = output_queue def length(self): """ @return Length of the queue of the pathnames that need synchronization. """ with self.access: return len(self.events) def isEmpty(self): """ @return Boolean telling whether there are pathnames that need synchronization. """ return self.length() < 1 def clear(self): """ Clear all internal data structures, thus forgetting about any pathname. """ with self.access: self.events.clear() self.map.clear() self._last_event_for_pathname.clear() def terminate(self): """ Terminate this component, release any acquired resource. """ with self.access: self.map.terminate() self.clear() def put(self, event): """ Update a pathname status with a pathname event. Any ongoing synchronization activity on the pathname gets interrupted. A PathnameOperation is produced and sent to the output queue. @param event: Instance of PathnameEvent """ with self.access: self._digest(event) self._send_pathname_operation() def _digest(self, event): """ Update a pathname status with a pathname event. Any ongoing synchronization activity on the pathname gets interrupted. @param event: Instance of PathnameEvent """ #self.logger.debug(u'Digesting event %s' % event) self._last_event_for_pathname[event.pathname] = event # Force copies to be creations. We aren't ready to handle copies yet. if event.action == 'COPY': event.action = 'CREATE' event.paired_pathname = None if event.action in [ 'CREATE', 'MODIFY', 'DELETE', 'UPDATE_FROM_REMOTE', 'REMOTELY_DELETED' ]: self._digest_single_pathname_event(event) else: # TODO: we don't support COPY yet raise Exception('EventsQueue, unsupported event: %s' % event) def _digest_single_pathname_event(self, event): """ Handle status transitions for those events which involve just one pathname (e.g. UPDATE, DELETE, etc). """ #self.logger.debug(u'Digesting single pathname event "%s"' % repr(event)) action, pathname = event.action, event.pathname if self.map.isLocked(pathname): self.logger.debug(u'Pathname "%s" seems worker-locked. Sending' ' termination request to current worker.' % pathname) # Note: self.on_file_operation_abort is called on abort file_operation = self.map.getLockingWorker(pathname) file_operation.abort() if action == 'CREATE' or action == 'MODIFY': self.map.update(pathname) self.application.notify_pathname_status_change( pathname, PStatuses.TOBEUPLOADED) elif action == 'DELETE': self.map.delete(pathname) self.application.notify_pathname_status_change( pathname, PStatuses.DELETETOBESENT) elif action == 'UPDATE_FROM_REMOTE': self.map.update_from_remote(pathname) self.application.notify_pathname_status_change( pathname, PStatuses.TOBEDOWNLOADED) elif action == 'REMOTELY_DELETED': self.map.remotely_deleted(pathname) else: self.logger.warning(u'Unknown action requested: "%s" for' ' pathname "%s"' % (action, pathname)) self.events.append(pathname) def on_file_operation_complete(self, file_operation): """ Handler called by PathnameOperation objects when they have been completed. Update the pathname status and release any constraint. @param file_operation: The PathnameOperation that has been completed. """ with self.access: if self.map.has_constraints_from(file_operation.pathname): self.map.dropConstraintFrom(file_operation.pathname) self.map.setStatus('OK', file_operation.pathname) self.map.unlock(file_operation.pathname) def on_file_operation_abort(self, file_operation): """ Handler called by PathnameOperation objects when they have been aborted. Update the pathname status and release any constraint. Note: actually only EventsQueue can abort operations, so this is a self-call event handler. @param file_operation: The PathnameOperation that has been aborted. """ with self.access: # Note: constraints release here is reduntant, since it's performed # also on the next status change. I haven't decided yet which place # between here and there is better for this task. # The same goes for on_file_operation_complete. if self.map.has_constraints_from(file_operation.pathname): self.map.dropConstraintFrom(file_operation.pathname) self.map.setStatus('OK', file_operation.pathname) self.map.unlock(file_operation.pathname) self.application.notify_pathname_status_change( file_operation.pathname, PStatuses.ALIGNED) def _create_pathname_operation(self, status, pathname, oldpath=None): """ Factory method for PathnameOperation objects """ status_to_verb = { 'LN': 'UPLOAD', 'LD': 'DELETE', 'LRto': 'REMOTE_COPY', 'RN': 'DOWNLOAD', 'RD': 'DELETE_LOCAL' } event = self._last_event_for_pathname[pathname] operation = PathnameOperation(self.application, self.access, status_to_verb[status], pathname, oldpath, event.etag, event.size, event.lmtime, event.conflicted) operation.register_abort_handler(self.on_file_operation_abort) operation.register_complete_handler(self.on_file_operation_complete) return operation def _send_pathname_operation(self): """ Produce a PathnameOperation object corresponding to the last received PathnameEvent and send it to the output queue. """ pathname = self.events.popleft() status = self.map.getStatus(pathname) oldpath = None operation = self._create_pathname_operation(status, pathname, oldpath) self.map.lock(pathname, operation) self._output_queue.put(operation, 'operation')