def _init(self):
# We use this double spawn method to be able to catch NotImplementedError immediately
# in case subclasses do not implement self._init_impl.
try:
spawn_greenlet(self._init_impl, timeout=45)
except Exception:
logger.warn('Could not initialize %s `%s`, e:`%s`', self.wrapper_type, self.config.name, format_exc())
def invoke_topic_callbacks(self, topic_list, request):
# type: (list, dict) -> None
for item in topic_list: # type: str
try:
spawn_greenlet(self.server.invoke, item, request)
except Exception:
logger.warn(format_exc())
def __init__(self, name, pid):
self.name = name
self.pid = pid
self.ctx = zmq.Context()
spawn_greenlet(self.set_up_sockets)
self.keep_running = True
self.logger = get_logger_for_class(self.__class__)
self.log_connected()
def __init__(self, pubsub):
self.pubsub = pubsub # type: PubSub
self.sub_key_to_msg_id = {} # Sub key -> Msg ID set --- What messages are available for a given subcriber
self.msg_id_to_sub_key = {} # Msg ID -> Sub key set - What subscribers are interested in a given message
self.msg_id_to_msg = {} # Msg ID -> Message data - What is the actual contents of each message
self.topic_msg_id = {} # Topic ID -> Msg ID set --- What messages are available for each topic (no matter sub_key)
self.lock = RLock()
# Start in background a cleanup task that deletes all expired and removed messages
spawn_greenlet(self.run_cleanup_task)
def __init__(self, config):
# type: (Bunch) -> None
self.config = config
# By default, we are not connected anywhere
self.is_connected = False
# Initialize in a separate greenlet so as not to block the main one
# if the remote server is slow to respond.
spawn_greenlet(self._init, timeout=2)
def serve_forever(self):
try:
try:
spawn_greenlet(self.sched.run)
except Exception:
logger.warn(format_exc())
while not self.sched.ready:
sleep(0.1)
except Exception:
logger.warn(format_exc())
def _start(self, observer_start_args):
for path in self.path_list: # type: str
# Start only for paths that are valid - all invalid ones
# are handled by a background path inspector.
if self.is_path_valid(path):
logger.info('Starting %s file observer `%s` for `%s` (%s)',
self.observer_type_name, path, self.name,
self.observer_type_impl)
spawn_greenlet(self._observe_func, path, observer_start_args)
else:
logger.info('Skipping invalid path `%s` for `%s` (%s)', path,
self.name, self.observer_type_impl)
def _run_snapshot_observer(self, observer, max_iters=maxsize):
# type: (BaseObserver, int) -> None
if not observer.is_active:
return
source_type = observer.channel_config.source_type # type: str
snapshot_maker_class = source_type_to_snapshot_maker_class[source_type]
snapshot_maker = snapshot_maker_class(
self, observer.channel_config) # type: (BaseSnapshotMaker)
snapshot_maker.connect()
for item in observer.path_list: # type: (str)
spawn_greenlet(observer.observe_with_snapshots, snapshot_maker,
item, max_iters, False)
def run(self):
self.subscriber = Subscriber(self.on_message_callback, self.name,
self.pid)
spawn_greenlet(self.subscriber.serve_forever)
def amqp_invoke_async(self, *args, **kwargs):
spawn_greenlet(self._amqp_invoke_async, *args, **kwargs)
def on_created(self, transfer_event, observer, snapshot_maker=None):
# type: (PathCreatedEvent, BaseObserver, BaseSnapshotMaker) -> None
try:
# Ignore the event if it points to the directory itself,
# as inotify will send CLOSE_WRITE when it is not a creation of a file
# but a fact that a directory has been deleted that the event is about.
# Note that we issue a log entry only if the path is not one of what
# we observe, i.e. when one of our own directories is deleted, we do not log it here.
# The path must have existed since we are being called
# and we need to check why it does not exist anymore ..
if not observer.path_exists(transfer_event.src_path, snapshot_maker):
# .. if this type of an observer does not wait for paths, we can return immediately ..
if not observer.should_wait_for_deleted_paths:
return
# .. if it is one of the paths that we observe, it means that it has been just deleted,
# so we need to run a background inspector which will wait until it is created once again ..
if transfer_event.src_path in self.config.pickup_from_list:
self.manager.wait_for_deleted_path(transfer_event.src_path)
else:
logger.info('Ignoring local file event; path not found `%s` (%r)', transfer_event.src_path, self.config.name)
# .. in either case, there is nothing else we can do here.
return
# Get file name to check if we should handle it ..
file_name = os.path.basename(transfer_event.src_path) # type: str
# .. return if we should not.
if not self.manager.should_handle(self.config.name, file_name):
return
event = FileTransferEvent()
event.full_path = transfer_event.src_path
event.base_dir = os.path.dirname(transfer_event.src_path)
event.file_name = file_name
event.channel_name = self.channel_name
if self.config.is_hot_deploy:
spawn_greenlet(hot_deploy, self.manager.server, event.file_name, event.full_path,
self.config.should_delete_after_pickup)
return
if self.config.should_read_on_pickup:
if snapshot_maker:
raw_data = snapshot_maker.get_file_data(event.full_path)
else:
f = open(event.full_path, 'rb')
raw_data = f.read()
f.close
event.raw_data = raw_data if isinstance(raw_data, str) else raw_data.decode(self.config.data_encoding) # type: str
event.has_raw_data = True
if self.config.should_parse_on_pickup:
try:
data_to_parse = StringIO(event.raw_data) if self.config.parser_needs_string_io else event.raw_data
parser = self.manager.get_parser(self.config.parse_with)
event.data = parser(data_to_parse)
event.has_data = True
except Exception:
exception = format_exc()
event.parse_error = exception
logger.warn('File transfer parsing error (%s) e:`%s`', self.config.name, exception)
# Invokes all callbacks for the event
spawn_greenlet(self.manager.invoke_callbacks, event, self.config.service_list, self.config.topic_list,
self.config.outconn_rest_list)
# Performs cleanup actions
self.manager.post_handle(event, self.config, observer, snapshot_maker)
except Exception:
logger.warn('Exception in pickup event handler `%s` (%s) `%s`',
self.config.name, transfer_event.src_path, format_exc())
def start(self, observer_start_args):
if self.is_active:
spawn_greenlet(self._start, observer_start_args)
else:
logger.info('Skipping an inactive file transfer channel `%s` (%s)',
self.name, self.path_list)
def start(self):
if self.observer.is_active:
spawn_greenlet(self.observer.wait_for_path, self.path,
self.observer_start_args)
def _run(self, name=None, log_after_started=False):
# Under Linux, for each observer, map each of its watched directories
# to the actual observer object so that when an event is emitted
# we will know, based on the event's full path, which observers to notify.
self.inotify_path_to_observer_list = {}
for observer in self.observer_list: # type: LocalObserver
for path in observer.path_list: # type: str
observer_list = self.inotify_path_to_observer_list.setdefault(
path, []) # type: list
observer_list.append(observer)
# Maps missing paths to all the observers interested in it.
missing_path_to_inspector = {}
# Start the observer objects, creating inotify watch descriptors (wd) in background ..
for observer in self.observer_list: # type: BaseObserver
try:
# Skip non-local observers
if not observer.is_local:
continue
# Filter out unneeded names
if name and name != observer.name:
continue
# Quickly check if any of the observer's path is missing and if it is, do not start it now.
# Instead, we will run a background task that will wait until the path becomes available and when it is,
# it will add start the observer itself.
for path in observer.path_list:
if not observer.is_path_valid(path):
path_observer_list = missing_path_to_inspector.setdefault(
path, []) # type: list
path_observer_list.append(
BackgroundPathInspector(path, observer,
self.observer_start_args))
# Inotify-based observers are set up here but their main loop is in _run_linux_inotify_loop ..
if prefer_inotify:
observer.start(self.observer_start_args)
# .. whereas snapshot observers are started here.
else:
self._run_snapshot_observer(observer)
if log_after_started:
logger.info('Started file observer `%s` path:`%s`',
observer.name, observer.path_list)
except Exception:
logger.warn(
'File observer `%s` could not be started, path:`%s`, e:`%s`',
observer.name, observer.path_list, format_exc())
# If there are any paths missing for any observer ..
if missing_path_to_inspector:
# .. wait for each such path in background.
self.run_inspectors(missing_path_to_inspector)
# Under Linux, run the inotify main loop for each watch descriptor created for paths that do exist.
# Note that if we are not on Linux, each observer.start call above already ran a new greenlet with an observer
# for a particular directory.
if prefer_inotify:
spawn_greenlet(self._run_linux_inotify_loop)
def invoke_rest_outconn_callbacks(self, outconn_rest_list, request):
# type: (list, dict) -> None
for item_id in outconn_rest_list: # type: int
spawn_greenlet(self._invoke_rest_outconn_callback, item_id,
request)
def _spawn(self, *args, **kwargs):
""" As in the Job class, this is a thin wrapper so that it is easier to mock this method out in unit-tests.
"""
return spawn_greenlet(*args, **kwargs)
