def _init(self): # We use this double spawn method to be able to catch NotImplementedError immediately # in case subclasses do not implement self._init_impl. try: spawn_greenlet(self._init_impl, timeout=45) except Exception: logger.warn('Could not initialize %s `%s`, e:`%s`', self.wrapper_type, self.config.name, format_exc())
def invoke_topic_callbacks(self, topic_list, request): # type: (list, dict) -> None for item in topic_list: # type: str try: spawn_greenlet(self.server.invoke, item, request) except Exception: logger.warn(format_exc())
def __init__(self, name, pid): self.name = name self.pid = pid self.ctx = zmq.Context() spawn_greenlet(self.set_up_sockets) self.keep_running = True self.logger = get_logger_for_class(self.__class__) self.log_connected()
def __init__(self, pubsub): self.pubsub = pubsub # type: PubSub self.sub_key_to_msg_id = {} # Sub key -> Msg ID set --- What messages are available for a given subcriber self.msg_id_to_sub_key = {} # Msg ID -> Sub key set - What subscribers are interested in a given message self.msg_id_to_msg = {} # Msg ID -> Message data - What is the actual contents of each message self.topic_msg_id = {} # Topic ID -> Msg ID set --- What messages are available for each topic (no matter sub_key) self.lock = RLock() # Start in background a cleanup task that deletes all expired and removed messages spawn_greenlet(self.run_cleanup_task)
def __init__(self, config): # type: (Bunch) -> None self.config = config # By default, we are not connected anywhere self.is_connected = False # Initialize in a separate greenlet so as not to block the main one # if the remote server is slow to respond. spawn_greenlet(self._init, timeout=2)
def serve_forever(self): try: try: spawn_greenlet(self.sched.run) except Exception: logger.warn(format_exc()) while not self.sched.ready: sleep(0.1) except Exception: logger.warn(format_exc())
def _start(self, observer_start_args): for path in self.path_list: # type: str # Start only for paths that are valid - all invalid ones # are handled by a background path inspector. if self.is_path_valid(path): logger.info('Starting %s file observer `%s` for `%s` (%s)', self.observer_type_name, path, self.name, self.observer_type_impl) spawn_greenlet(self._observe_func, path, observer_start_args) else: logger.info('Skipping invalid path `%s` for `%s` (%s)', path, self.name, self.observer_type_impl)
def _run_snapshot_observer(self, observer, max_iters=maxsize): # type: (BaseObserver, int) -> None if not observer.is_active: return source_type = observer.channel_config.source_type # type: str snapshot_maker_class = source_type_to_snapshot_maker_class[source_type] snapshot_maker = snapshot_maker_class( self, observer.channel_config) # type: (BaseSnapshotMaker) snapshot_maker.connect() for item in observer.path_list: # type: (str) spawn_greenlet(observer.observe_with_snapshots, snapshot_maker, item, max_iters, False)
def run(self): self.subscriber = Subscriber(self.on_message_callback, self.name, self.pid) spawn_greenlet(self.subscriber.serve_forever)
def amqp_invoke_async(self, *args, **kwargs): spawn_greenlet(self._amqp_invoke_async, *args, **kwargs)
def on_created(self, transfer_event, observer, snapshot_maker=None): # type: (PathCreatedEvent, BaseObserver, BaseSnapshotMaker) -> None try: # Ignore the event if it points to the directory itself, # as inotify will send CLOSE_WRITE when it is not a creation of a file # but a fact that a directory has been deleted that the event is about. # Note that we issue a log entry only if the path is not one of what # we observe, i.e. when one of our own directories is deleted, we do not log it here. # The path must have existed since we are being called # and we need to check why it does not exist anymore .. if not observer.path_exists(transfer_event.src_path, snapshot_maker): # .. if this type of an observer does not wait for paths, we can return immediately .. if not observer.should_wait_for_deleted_paths: return # .. if it is one of the paths that we observe, it means that it has been just deleted, # so we need to run a background inspector which will wait until it is created once again .. if transfer_event.src_path in self.config.pickup_from_list: self.manager.wait_for_deleted_path(transfer_event.src_path) else: logger.info('Ignoring local file event; path not found `%s` (%r)', transfer_event.src_path, self.config.name) # .. in either case, there is nothing else we can do here. return # Get file name to check if we should handle it .. file_name = os.path.basename(transfer_event.src_path) # type: str # .. return if we should not. if not self.manager.should_handle(self.config.name, file_name): return event = FileTransferEvent() event.full_path = transfer_event.src_path event.base_dir = os.path.dirname(transfer_event.src_path) event.file_name = file_name event.channel_name = self.channel_name if self.config.is_hot_deploy: spawn_greenlet(hot_deploy, self.manager.server, event.file_name, event.full_path, self.config.should_delete_after_pickup) return if self.config.should_read_on_pickup: if snapshot_maker: raw_data = snapshot_maker.get_file_data(event.full_path) else: f = open(event.full_path, 'rb') raw_data = f.read() f.close event.raw_data = raw_data if isinstance(raw_data, str) else raw_data.decode(self.config.data_encoding) # type: str event.has_raw_data = True if self.config.should_parse_on_pickup: try: data_to_parse = StringIO(event.raw_data) if self.config.parser_needs_string_io else event.raw_data parser = self.manager.get_parser(self.config.parse_with) event.data = parser(data_to_parse) event.has_data = True except Exception: exception = format_exc() event.parse_error = exception logger.warn('File transfer parsing error (%s) e:`%s`', self.config.name, exception) # Invokes all callbacks for the event spawn_greenlet(self.manager.invoke_callbacks, event, self.config.service_list, self.config.topic_list, self.config.outconn_rest_list) # Performs cleanup actions self.manager.post_handle(event, self.config, observer, snapshot_maker) except Exception: logger.warn('Exception in pickup event handler `%s` (%s) `%s`', self.config.name, transfer_event.src_path, format_exc())
def start(self, observer_start_args): if self.is_active: spawn_greenlet(self._start, observer_start_args) else: logger.info('Skipping an inactive file transfer channel `%s` (%s)', self.name, self.path_list)
def start(self): if self.observer.is_active: spawn_greenlet(self.observer.wait_for_path, self.path, self.observer_start_args)
def _run(self, name=None, log_after_started=False): # Under Linux, for each observer, map each of its watched directories # to the actual observer object so that when an event is emitted # we will know, based on the event's full path, which observers to notify. self.inotify_path_to_observer_list = {} for observer in self.observer_list: # type: LocalObserver for path in observer.path_list: # type: str observer_list = self.inotify_path_to_observer_list.setdefault( path, []) # type: list observer_list.append(observer) # Maps missing paths to all the observers interested in it. missing_path_to_inspector = {} # Start the observer objects, creating inotify watch descriptors (wd) in background .. for observer in self.observer_list: # type: BaseObserver try: # Skip non-local observers if not observer.is_local: continue # Filter out unneeded names if name and name != observer.name: continue # Quickly check if any of the observer's path is missing and if it is, do not start it now. # Instead, we will run a background task that will wait until the path becomes available and when it is, # it will add start the observer itself. for path in observer.path_list: if not observer.is_path_valid(path): path_observer_list = missing_path_to_inspector.setdefault( path, []) # type: list path_observer_list.append( BackgroundPathInspector(path, observer, self.observer_start_args)) # Inotify-based observers are set up here but their main loop is in _run_linux_inotify_loop .. if prefer_inotify: observer.start(self.observer_start_args) # .. whereas snapshot observers are started here. else: self._run_snapshot_observer(observer) if log_after_started: logger.info('Started file observer `%s` path:`%s`', observer.name, observer.path_list) except Exception: logger.warn( 'File observer `%s` could not be started, path:`%s`, e:`%s`', observer.name, observer.path_list, format_exc()) # If there are any paths missing for any observer .. if missing_path_to_inspector: # .. wait for each such path in background. self.run_inspectors(missing_path_to_inspector) # Under Linux, run the inotify main loop for each watch descriptor created for paths that do exist. # Note that if we are not on Linux, each observer.start call above already ran a new greenlet with an observer # for a particular directory. if prefer_inotify: spawn_greenlet(self._run_linux_inotify_loop)
def invoke_rest_outconn_callbacks(self, outconn_rest_list, request): # type: (list, dict) -> None for item_id in outconn_rest_list: # type: int spawn_greenlet(self._invoke_rest_outconn_callback, item_id, request)
def _spawn(self, *args, **kwargs): """ As in the Job class, this is a thin wrapper so that it is easier to mock this method out in unit-tests. """ return spawn_greenlet(*args, **kwargs)