def test_is_directory(self): event1 = FileModifiedEvent(path_1) self.assertFalse(event1.is_directory)
def test_dispatch(self): # Utilities. regexes = [r".*\.py", r".*\.txt"] ignore_regexes = [r".*\.pyc"] def assert_regexes(handler, event): if has_attribute(event, 'dest_path'): paths = [event.src_path, event.dest_path] else: paths = [event.src_path] filtered_paths = set() for p in paths: if any(r.match(p) for r in handler.regexes): filtered_paths.add(p) self.assertTrue(filtered_paths) dir_del_event_match = DirDeletedEvent('/path/blah.py') dir_del_event_not_match = DirDeletedEvent('/path/foobar') dir_del_event_ignored = DirDeletedEvent('/path/foobar.pyc') file_del_event_match = FileDeletedEvent('/path/blah.txt') file_del_event_not_match = FileDeletedEvent('/path/foobar') file_del_event_ignored = FileDeletedEvent('/path/blah.pyc') dir_cre_event_match = DirCreatedEvent('/path/blah.py') dir_cre_event_not_match = DirCreatedEvent('/path/foobar') dir_cre_event_ignored = DirCreatedEvent('/path/foobar.pyc') file_cre_event_match = FileCreatedEvent('/path/blah.txt') file_cre_event_not_match = FileCreatedEvent('/path/foobar') file_cre_event_ignored = FileCreatedEvent('/path/blah.pyc') dir_mod_event_match = DirModifiedEvent('/path/blah.py') dir_mod_event_not_match = DirModifiedEvent('/path/foobar') dir_mod_event_ignored = DirModifiedEvent('/path/foobar.pyc') file_mod_event_match = FileModifiedEvent('/path/blah.txt') file_mod_event_not_match = FileModifiedEvent('/path/foobar') file_mod_event_ignored = FileModifiedEvent('/path/blah.pyc') dir_mov_event_match = DirMovedEvent('/path/blah.py', '/path/blah') dir_mov_event_not_match = DirMovedEvent('/path/foobar', '/path/blah') dir_mov_event_ignored = DirMovedEvent('/path/foobar.pyc', '/path/blah') file_mov_event_match = FileMovedEvent('/path/blah.txt', '/path/blah') file_mov_event_not_match = FileMovedEvent('/path/foobar', '/path/blah') file_mov_event_ignored = FileMovedEvent('/path/blah.pyc', '/path/blah') all_dir_events = [ dir_mod_event_match, dir_mod_event_not_match, dir_mod_event_ignored, dir_del_event_match, dir_del_event_not_match, dir_del_event_ignored, dir_cre_event_match, dir_cre_event_not_match, dir_cre_event_ignored, dir_mov_event_match, dir_mov_event_not_match, dir_mov_event_ignored, ] all_file_events = [ file_mod_event_match, file_mod_event_not_match, file_mod_event_ignored, file_del_event_match, file_del_event_not_match, file_del_event_ignored, file_cre_event_match, file_cre_event_not_match, file_cre_event_ignored, file_mov_event_match, file_mov_event_not_match, file_mov_event_ignored, ] all_events = all_file_events + all_dir_events def assert_check_directory(handler, event): self.assertFalse(handler.ignore_directories and event.is_directory) def assert_equal(a, b): self.assertEqual(a, b) class TestableEventHandler(RegexMatchingEventHandler): def on_any_event(self, event): assert_check_directory(self, event) def on_modified(self, event): assert_check_directory(self, event) assert_equal(event.event_type, EVENT_TYPE_MODIFIED) assert_regexes(self, event) def on_deleted(self, event): assert_check_directory(self, event) assert_equal(event.event_type, EVENT_TYPE_DELETED) assert_regexes(self, event) def on_moved(self, event): assert_check_directory(self, event) assert_equal(event.event_type, EVENT_TYPE_MOVED) assert_regexes(self, event) def on_created(self, event): assert_check_directory(self, event) assert_equal(event.event_type, EVENT_TYPE_CREATED) assert_regexes(self, event) no_dirs_handler = TestableEventHandler(regexes=regexes, ignore_regexes=ignore_regexes, ignore_directories=True) handler = TestableEventHandler(regexes=regexes, ignore_regexes=ignore_regexes, ignore_directories=False) for event in all_events: no_dirs_handler.dispatch(event) for event in all_events: handler.dispatch(event)
def test___init__(self): event = FileModifiedEvent(path_1) self.assertEqual(path_1, event.src_path) self.assertEqual(EVENT_TYPE_MODIFIED, event.event_type) self.assertFalse(event.is_directory)
def observe_with_snapshots(self, snapshot_maker, path, max_iters, log_stop_event=True, *args, **kwargs): """ An observer's main loop that uses snapshots. """ # type: (BaseSnapshotMaker, str, int) -> None try: # Local aliases to avoid namespace lookups in self timeout = self.sleep_time handler_func = self.event_handler.on_created is_recursive = self.is_recursive # How many times to run the loop - either given on input or, essentially, infinitely. current_iter = 0 # Take an initial snapshot snapshot = snapshot_maker.get_snapshot(path, is_recursive, True, True) while self.keep_running: if current_iter == max_iters: break try: # The latest snapshot .. new_snapshot = snapshot_maker.get_snapshot( path, is_recursive, False, False) # .. difference between the old and new will return, in particular, new or modified files .. diff = DirSnapshotDiff(snapshot, new_snapshot) for path_created in diff.files_created: full_event_path = os.path.join(path, path_created) handler_func(FileCreatedEvent(full_event_path), self, snapshot_maker) for path_modified in diff.files_modified: full_event_path = os.path.join(path, path_modified) handler_func(FileModifiedEvent(full_event_path), self, snapshot_maker) # .. a new snapshot which will be treated as the old one in the next iteration snapshot = snapshot_maker.get_snapshot( path, is_recursive, False, True) # Note that this will be caught only with local files not with FTP, SFTP etc. except FileNotFoundError: # Log the error .. logger.warn( 'Path not found caught in %s file observer main loop (%s) `%s` (%s t:%s)', self.observer_type_name, path, format_exc(), self.name, self.observer_type_impl) # .. start a background inspector which will wait for the path to become available .. self.manager.wait_for_deleted_path(path) # .. and end the main loop. return except Exception as e: logger.warn( 'Exception %s in %s file observer main loop `%s` e:`%s (%s t:%s)', type(e), self.observer_type_name, path, format_exc(), self.name, self.observer_type_impl) finally: # Update loop counter after we completed current iteration current_iter += 1 # Sleep for a while but only if we are a local observer because any other # will be triggered from the scheduler and we treat the scheduler job's interval # as the sleep time. if self.is_local: sleep(timeout) except Exception: logger.warn('Exception in %s file observer `%s` e:`%s (%s t:%s)', self.observer_type_name, path, format_exc(), self.name, self.observer_type_impl) if log_stop_event: logger.warn( 'Stopped %s file transfer observer `%s` for `%s` (snapshot:%s/%s)', self.observer_type_name, self.name, path, current_iter, max_iters)
def test_dispatch(): # Utilities. patterns = ['*.py', '*.txt'] ignore_patterns = ["*.pyc"] dir_del_event_match = DirDeletedEvent('/path/blah.py') dir_del_event_not_match = DirDeletedEvent('/path/foobar') dir_del_event_ignored = DirDeletedEvent('/path/foobar.pyc') file_del_event_match = FileDeletedEvent('/path/blah.txt') file_del_event_not_match = FileDeletedEvent('/path/foobar') file_del_event_ignored = FileDeletedEvent('/path/blah.pyc') dir_cre_event_match = DirCreatedEvent('/path/blah.py') dir_cre_event_not_match = DirCreatedEvent('/path/foobar') dir_cre_event_ignored = DirCreatedEvent('/path/foobar.pyc') file_cre_event_match = FileCreatedEvent('/path/blah.txt') file_cre_event_not_match = FileCreatedEvent('/path/foobar') file_cre_event_ignored = FileCreatedEvent('/path/blah.pyc') dir_mod_event_match = DirModifiedEvent('/path/blah.py') dir_mod_event_not_match = DirModifiedEvent('/path/foobar') dir_mod_event_ignored = DirModifiedEvent('/path/foobar.pyc') file_mod_event_match = FileModifiedEvent('/path/blah.txt') file_mod_event_not_match = FileModifiedEvent('/path/foobar') file_mod_event_ignored = FileModifiedEvent('/path/blah.pyc') dir_mov_event_match = DirMovedEvent('/path/blah.py', '/path/blah') dir_mov_event_not_match = DirMovedEvent('/path/foobar', '/path/blah') dir_mov_event_ignored = DirMovedEvent('/path/foobar.pyc', '/path/blah') file_mov_event_match = FileMovedEvent('/path/blah.txt', '/path/blah') file_mov_event_not_match = FileMovedEvent('/path/foobar', '/path/blah') file_mov_event_ignored = FileMovedEvent('/path/blah.pyc', '/path/blah') all_dir_events = [ dir_mod_event_match, dir_mod_event_not_match, dir_mod_event_ignored, dir_del_event_match, dir_del_event_not_match, dir_del_event_ignored, dir_cre_event_match, dir_cre_event_not_match, dir_cre_event_ignored, dir_mov_event_match, dir_mov_event_not_match, dir_mov_event_ignored, ] all_file_events = [ file_mod_event_match, file_mod_event_not_match, file_mod_event_ignored, file_del_event_match, file_del_event_not_match, file_del_event_ignored, file_cre_event_match, file_cre_event_not_match, file_cre_event_ignored, file_mov_event_match, file_mov_event_not_match, file_mov_event_ignored, ] all_events = all_file_events + all_dir_events def assert_check_directory(handler, event): assert not (handler.ignore_directories and event.is_directory) class TestableEventHandler(PatternMatchingEventHandler): def on_any_event(self, event): assert_check_directory(self, event) def on_modified(self, event): assert_check_directory(self, event) assert event.event_type == EVENT_TYPE_MODIFIED assert_patterns(event) def on_deleted(self, event): assert_check_directory(self, event) assert event.event_type == EVENT_TYPE_DELETED assert_patterns(event) def on_moved(self, event): assert_check_directory(self, event) assert event.event_type == EVENT_TYPE_MOVED assert_patterns(event) def on_created(self, event): assert_check_directory(self, event) assert event.event_type == EVENT_TYPE_CREATED assert_patterns(event) no_dirs_handler = TestableEventHandler(patterns=patterns, ignore_patterns=ignore_patterns, ignore_directories=True) handler = TestableEventHandler(patterns=patterns, ignore_patterns=ignore_patterns, ignore_directories=False) for event in all_events: no_dirs_handler.dispatch(event) for event in all_events: handler.dispatch(event)
def test_dispatch(self): # Utilities. patterns = ['*.py', '*.txt'] ignore_patterns = ["*.pyc"] def assert_patterns(event): if has_attribute(event, 'dest_path'): paths = [event.src_path, event.dest_path] else: paths = [event.src_path] filtered_paths = filter_paths(paths, included_patterns=patterns, excluded_patterns=ignore_patterns, case_sensitive=False) self.assertTrue(filtered_paths) dir_del_event_match = DirDeletedEvent('/path/blah.py') dir_del_event_not_match = DirDeletedEvent('/path/foobar') dir_del_event_ignored = DirDeletedEvent('/path/foobar.pyc') file_del_event_match = FileDeletedEvent('/path/blah.txt') file_del_event_not_match = FileDeletedEvent('/path/foobar') file_del_event_ignored = FileDeletedEvent('/path/blah.pyc') dir_cre_event_match = DirCreatedEvent('/path/blah.py') dir_cre_event_not_match = DirCreatedEvent('/path/foobar') dir_cre_event_ignored = DirCreatedEvent('/path/foobar.pyc') file_cre_event_match = FileCreatedEvent('/path/blah.txt') file_cre_event_not_match = FileCreatedEvent('/path/foobar') file_cre_event_ignored = FileCreatedEvent('/path/blah.pyc') dir_mod_event_match = DirModifiedEvent('/path/blah.py') dir_mod_event_not_match = DirModifiedEvent('/path/foobar') dir_mod_event_ignored = DirModifiedEvent('/path/foobar.pyc') file_mod_event_match = FileModifiedEvent('/path/blah.txt') file_mod_event_not_match = FileModifiedEvent('/path/foobar') file_mod_event_ignored = FileModifiedEvent('/path/blah.pyc') dir_mov_event_match = DirMovedEvent('/path/blah.py', '/path/blah') dir_mov_event_not_match = DirMovedEvent('/path/foobar', '/path/blah') dir_mov_event_ignored = DirMovedEvent('/path/foobar.pyc', '/path/blah') file_mov_event_match = FileMovedEvent('/path/blah.txt', '/path/blah') file_mov_event_not_match = FileMovedEvent('/path/foobar', '/path/blah') file_mov_event_ignored = FileMovedEvent('/path/blah.pyc', '/path/blah') all_dir_events = [ dir_mod_event_match, dir_mod_event_not_match, dir_mod_event_ignored, dir_del_event_match, dir_del_event_not_match, dir_del_event_ignored, dir_cre_event_match, dir_cre_event_not_match, dir_cre_event_ignored, dir_mov_event_match, dir_mov_event_not_match, dir_mov_event_ignored, ] all_file_events = [ file_mod_event_match, file_mod_event_not_match, file_mod_event_ignored, file_del_event_match, file_del_event_not_match, file_del_event_ignored, file_cre_event_match, file_cre_event_not_match, file_cre_event_ignored, file_mov_event_match, file_mov_event_not_match, file_mov_event_ignored, ] all_events = all_file_events + all_dir_events def assert_check_directory(handler, event): self.assertFalse(handler.ignore_directories and event.is_directory) def assert_equal(a, b): self.assertEqual(a, b) class TestableEventHandler(PatternMatchingEventHandler): def on_any_event(self, event): assert_check_directory(self, event) def on_modified(self, event): assert_check_directory(self, event) assert_equal(event.event_type, EVENT_TYPE_MODIFIED) assert_patterns(event) def on_deleted(self, event): assert_check_directory(self, event) assert_equal(event.event_type, EVENT_TYPE_DELETED) assert_patterns(event) def on_moved(self, event): assert_check_directory(self, event) assert_equal(event.event_type, EVENT_TYPE_MOVED) assert_patterns(event) def on_created(self, event): assert_check_directory(self, event) assert_equal(event.event_type, EVENT_TYPE_CREATED) assert_patterns(event) no_dirs_handler = TestableEventHandler(patterns=patterns, ignore_patterns=ignore_patterns, ignore_directories=True) handler = TestableEventHandler(patterns=patterns, ignore_patterns=ignore_patterns, ignore_directories=False) for event in all_events: no_dirs_handler.dispatch(event) for event in all_events: handler.dispatch(event)
def monitor(configuration): """Monitors the filesystem for crontab changes""" pid = multiprocessing.current_process().pid print('Starting global crontab monitor process') logger.info('Starting global crontab monitor process') # Set base_dir and base_dir_len shared_state['base_dir'] = os.path.join(configuration.user_settings) shared_state['base_dir_len'] = len(shared_state['base_dir']) # Allow e.g. logrotate to force log re-open after rotates register_hangup_handler(configuration) # Monitor crontab configurations crontab_monitor_home = shared_state['base_dir'] recursive_crontab_monitor = True crontab_monitor = Observer() crontab_pattern = os.path.join(crontab_monitor_home, '*', crontab_name) atjobs_pattern = os.path.join(crontab_monitor_home, '*', atjobs_name) shared_state['crontab_handler'] = MiGCrontabEventHandler( patterns=[crontab_pattern, atjobs_pattern], ignore_directories=False, case_sensitive=True) crontab_monitor.schedule(shared_state['crontab_handler'], configuration.user_settings, recursive=recursive_crontab_monitor) crontab_monitor.start() if len(crontab_monitor._emitters) != 1: logger.error('(%s) Number of crontab_monitor._emitters != 1' % pid) return 1 crontab_monitor_emitter = min(crontab_monitor._emitters) if not hasattr(crontab_monitor_emitter, '_inotify'): logger.error('(%s) crontab_monitor_emitter require inotify' % pid) return 1 shared_state['crontab_inotify'] = crontab_monitor_emitter._inotify._inotify logger.info('(%s) trigger crontab and atjobs refresh' % (pid, )) # Fake touch event on all crontab files to load initial crontabs # logger.info('(%s) trigger load on all files (greedy) matching %s or %s' \ # % (pid, crontab_pattern, atjobs_pattern)) # We manually walk and test to get the greedy "*" directory match behaviour # of the PatternMatchingEventHandler all_crontab_files, all_atjobs_files = [], [] for (root, _, files) in walk(crontab_monitor_home): if crontab_name in files: crontab_path = os.path.join(root, crontab_name) all_crontab_files.append(crontab_path) if atjobs_name in files: atjobs_path = os.path.join(root, atjobs_name) all_atjobs_files.append(atjobs_path) for target_path in all_crontab_files + all_atjobs_files: logger.debug('(%s) trigger load on cron/at file in %s' % (pid, target_path)) shared_state['crontab_handler'].dispatch( FileModifiedEvent(target_path)) # logger.debug('(%s) loaded initial crontabs:\n%s' % (pid, # all_crontab_files)) while not stop_running.is_set(): try: loop_start = datetime.datetime.now() loop_minute = loop_start.replace(second=0, microsecond=0) logger.debug('main loop started with %d crontabs and %d atjobs' % (len(all_crontabs), len(all_atjobs))) for crontab_path, user_crontab in all_crontabs.items(): client_dir = os.path.basename(os.path.dirname(crontab_path)) client_id = client_dir_id(client_dir) for entry in user_crontab: logger.debug('inspect cron entry for %s: %s' % (client_id, entry)) if cron_match(configuration, loop_minute, entry): logger.info('run matching cron entry: %s' % entry) run_handler(configuration, client_id, loop_minute, entry) for atjobs_path, user_atjobs in all_atjobs.items(): client_dir = os.path.basename(os.path.dirname(atjobs_path)) client_id = client_dir_id(client_dir) remaining = [] for entry in user_atjobs: logger.debug('inspect atjobs entry for %s: %s' % (client_id, entry)) remain_mins = at_remain(configuration, loop_minute, entry) if remain_mins == 0: logger.info('run matching at entry: %s' % entry) run_handler(configuration, client_id, loop_minute, entry) elif remain_mins > 0: remaining.append(entry) else: logger.info('removing expired at job: %s' % entry) # Update remaining jobs to clean up expired if remaining: all_atjobs[atjobs_path] = remaining else: del all_atjobs[atjobs_path] except KeyboardInterrupt: print('(%s) caught interrupt' % pid) stop_running.set() except Exception as exc: logger.error('unexpected exception in monitor: %s' % exc) import traceback print(traceback.format_exc()) # Throttle down until next minute loop_time = (datetime.datetime.now() - loop_start).seconds if loop_time > 60: logger.warning('(%s) loop did not finish before next tick: %s' % (os.getpid(), loop_time)) loop_time = 59 # Target sleep until start of next minute sleep_time = max(60 - (loop_time + loop_start.second), 1) # TODO: this debug log never shows up - conflict with user info log? # at least it does if changed to info. logger.debug('main loop sleeping %ds' % sleep_time) # print('main loop sleeping %ds' % sleep_time) time.sleep(sleep_time) print('(%s) Exiting crontab monitor' % pid) logger.info('(%s) Exiting crontab monitor' % pid) return 0
def __init__(self, file_folder, file_name, on_modified): FileModifiedEvent.__init__(self, file_folder) self.file_folder = file_folder self.file_name = file_name self.on_modified = on_modified
class TestConfigsManager(unittest.TestCase): def setUp(self) -> None: self.CONFIG_MANAGER_NAME = "Config Manager" self.config_manager_logger = logging.getLogger("test_config_manager") self.config_manager_logger.disabled = True self.rabbit_logger = logging.getLogger("test_rabbit") self.rabbit_logger.disabled = True self.config_directory = "config" file_patterns = ["*.ini"] rabbit_ip = env.RABBIT_IP self.test_config_manager = ConfigsManager(self.CONFIG_MANAGER_NAME, self.config_manager_logger, self.config_directory, rabbit_ip, file_patterns=file_patterns) self.rabbitmq = RabbitMQApi( self.rabbit_logger, rabbit_ip, connection_check_time_interval=timedelta(seconds=0)) def tearDown(self) -> None: # flush and consume all from rabbit queues and exchanges connect_to_rabbit(self.rabbitmq) queues = [CONFIG_PING_QUEUE] for queue in queues: delete_queue_if_exists(self.rabbitmq, queue) exchanges = [CONFIG_EXCHANGE, HEALTH_CHECK_EXCHANGE] for exchange in exchanges: delete_exchange_if_exists(self.rabbitmq, exchange) disconnect_from_rabbit(self.rabbitmq) self.rabbitmq = None self.test_config_manager._rabbitmq = None self.test_config_manager._heartbeat_rabbit = None self.test_config_manager = None def test_instance_created(self): self.assertIsNotNone(self.test_config_manager) def test_name_returns_component_name(self): self.assertEqual(self.CONFIG_MANAGER_NAME, self.test_config_manager.name) @parameterized.expand([ (CONFIG_PING_QUEUE, ), ]) @mock.patch.object(RabbitMQApi, "confirm_delivery") @mock.patch.object(RabbitMQApi, "basic_consume") @mock.patch.object(RabbitMQApi, "basic_qos") def test__initialise_rabbit_initialises_queues( self, queue_to_check: str, mock_basic_qos: MagicMock, mock_basic_consume: MagicMock, mock_confirm_delivery: MagicMock): mock_basic_consume.return_value = None mock_confirm_delivery.return_value = None try: connect_to_rabbit(self.rabbitmq) # Testing this separately since this is a critical function self.test_config_manager._initialise_rabbitmq() mock_basic_qos.assert_called_once() mock_basic_consume.assert_called_once() mock_confirm_delivery.assert_called() self.rabbitmq.queue_declare(queue_to_check, passive=True) except pika.exceptions.ConnectionClosedByBroker: self.fail("Queue {} was not declared".format(queue_to_check)) finally: disconnect_from_rabbit(self.test_config_manager._rabbitmq) disconnect_from_rabbit(self.test_config_manager._heartbeat_rabbit) @parameterized.expand([ (CONFIG_EXCHANGE, ), (HEALTH_CHECK_EXCHANGE, ), ]) @mock.patch.object(RabbitMQApi, "confirm_delivery") @mock.patch.object(RabbitMQApi, "basic_consume") @mock.patch.object(RabbitMQApi, "basic_qos") def test__initialise_rabbit_initialises_exchanges( self, exchange_to_check: str, mock_basic_qos: MagicMock, mock_basic_consume: MagicMock, mock_confirm_delivery: MagicMock): mock_basic_consume.return_value = None mock_confirm_delivery.return_value = None try: connect_to_rabbit(self.rabbitmq) # Testing this separately since this is a critical function self.test_config_manager._initialise_rabbitmq() mock_basic_qos.assert_called_once() mock_basic_consume.assert_called() mock_confirm_delivery.assert_called() self.rabbitmq.exchange_declare(exchange_to_check, passive=True) except pika.exceptions.ConnectionClosedByBroker: self.fail("Exchange {} was not declared".format(exchange_to_check)) finally: disconnect_from_rabbit(self.test_config_manager._rabbitmq) disconnect_from_rabbit(self.test_config_manager._heartbeat_rabbit) @mock.patch.object(RabbitMQApi, "connect_till_successful", autospec=True) def test__connect_to_rabbit(self, mock_connect: MagicMock): mock_connect.return_value = None self.test_config_manager._connect_to_rabbit() mock_connect.assert_called() self.assertEqual(2, mock_connect.call_count) self.assertTrue(self.test_config_manager.connected_to_rabbit) @mock.patch.object(RabbitMQApi, "disconnect_till_successful", autospec=True) def test_disconnect_from_rabbit(self, mock_disconnect: MagicMock): mock_disconnect.return_value = None self.test_config_manager._connected_to_rabbit = True self.test_config_manager.disconnect_from_rabbit() mock_disconnect.assert_called() self.assertEqual(2, mock_disconnect.call_count) self.assertFalse(self.test_config_manager.connected_to_rabbit) @freeze_time("1997-08-15T10:21:33.000030") @mock.patch.object(PollingObserver, "is_alive", autospec=True) def test__process_ping_sends_valid_hb(self, mock_is_alive: MagicMock): mock_is_alive.return_value = True expected_output = { 'component_name': self.CONFIG_MANAGER_NAME, 'is_alive': True, 'timestamp': datetime(year=1997, month=8, day=15, hour=10, minute=21, second=33, microsecond=30).timestamp() } HEARTBEAT_QUEUE = "hb_test" try: connect_to_rabbit(self.rabbitmq) self.rabbitmq.exchange_declare(HEALTH_CHECK_EXCHANGE, "topic", False, True, False, False) queue_res = self.rabbitmq.queue_declare(queue=HEARTBEAT_QUEUE, durable=True, exclusive=False, auto_delete=False, passive=False) self.assertEqual(0, queue_res.method.message_count) self.rabbitmq.queue_bind(HEARTBEAT_QUEUE, HEALTH_CHECK_EXCHANGE, "heartbeat.*") self.test_config_manager._initialise_rabbitmq() blocking_channel = self.test_config_manager._rabbitmq.channel method_chains = pika.spec.Basic.Deliver(routing_key="ping") properties = pika.spec.BasicProperties() self.test_config_manager._process_ping(blocking_channel, method_chains, properties, b"ping") # By re-declaring the queue again we can get the number of messages # in the queue. queue_res = self.rabbitmq.queue_declare(queue=HEARTBEAT_QUEUE, durable=True, exclusive=False, auto_delete=False, passive=True) self.assertEqual(1, queue_res.method.message_count) # Check that the message received is a valid HB _, _, body = self.rabbitmq.basic_get(HEARTBEAT_QUEUE) self.assertDictEqual(expected_output, json.loads(body)) finally: delete_queue_if_exists(self.rabbitmq, HEARTBEAT_QUEUE) delete_exchange_if_exists(self.rabbitmq, HEALTH_CHECK_EXCHANGE) disconnect_from_rabbit(self.test_config_manager._rabbitmq) disconnect_from_rabbit(self.test_config_manager._heartbeat_rabbit) @parameterized.expand([ (FileCreatedEvent("test_config"), """ [test_section_1] test_field_1=Hello test_field_2= test_field_3=10 test_field_4=true """, { "DEFAULT": {}, "test_section_1": { "test_field_1": "Hello", "test_field_2": "", "test_field_3": "10", "test_field_4": "true" } }), (FileModifiedEvent("test_config"), """ [test_section_1] test_field_1=Hello test_field_2= test_field_3=10 test_field_4=true [test_section_2] test_field_1=OK test_field_2=Bye test_field_3=4 test_field_4=off """, { "DEFAULT": {}, "test_section_1": { "test_field_1": "Hello", "test_field_2": "", "test_field_3": "10", "test_field_4": "true" }, "test_section_2": { "test_field_1": "OK", "test_field_2": "Bye", "test_field_3": "4", "test_field_4": "off" } }), (FileDeletedEvent("test_config"), "", {}), ]) @mock.patch.object(ConfigParser, "read", autospec=True) @mock.patch.object(ConfigsManager, "_send_data", autospec=True) @mock.patch("src.utils.routing_key.get_routing_key", autospec=True) def test__on_event_thrown(self, event_to_trigger: FileSystemEvent, config_file_input: str, expected_dict: Dict, mock_get_routing_key: MagicMock, mock_send_data: MagicMock, mock_config_parser: MagicMock): TEST_ROUTING_KEY = "test_config" def read_config_side_effect(cp: ConfigParser, *args, **kwargs) -> None: """ cp would be "self" in the context of this function being injected. """ cp.read_string(config_file_input) mock_get_routing_key.return_value = TEST_ROUTING_KEY mock_send_data.return_value = None mock_config_parser.side_effect = read_config_side_effect self.test_config_manager._on_event_thrown(event_to_trigger) mock_get_routing_key.assert_called_once() mock_send_data.assert_called_once_with(self.test_config_manager, expected_dict, TEST_ROUTING_KEY) @parameterized.expand([ ({}, ), ({ "DEFAULT": {}, "test_section_1": { "test_field_1": "Hello", "test_field_2": "", "test_field_3": "10", "test_field_4": "true" } }, ), ({ "DEFAULT": {}, "test_section_1": { "test_field_1": "Hello", "test_field_2": "", "test_field_3": "10", "test_field_4": "true" }, "test_section_2": { "test_field_1": "OK", "test_field_2": "Bye", "test_field_3": "4", "test_field_4": "off" } }, ), ]) def test_send_data(self, config: Dict[str, Any]): route_key = "test.route" CONFIG_QUEUE = "hb_test" try: self.test_config_manager._initialise_rabbitmq() connect_to_rabbit(self.rabbitmq) queue_res = self.rabbitmq.queue_declare(queue=CONFIG_QUEUE, durable=True, exclusive=False, auto_delete=False, passive=False) self.assertEqual(0, queue_res.method.message_count) self.rabbitmq.queue_bind(CONFIG_QUEUE, CONFIG_EXCHANGE, route_key) self.test_config_manager._send_data(copy.deepcopy(config), route_key) # By re-declaring the queue again we can get the number of messages # in the queue. queue_res = self.rabbitmq.queue_declare(queue=CONFIG_QUEUE, durable=True, exclusive=False, auto_delete=False, passive=True) self.assertEqual(1, queue_res.method.message_count) # Check that the message received is what's expected _, _, body = self.rabbitmq.basic_get(CONFIG_QUEUE) self.assertDictEqual(config, json.loads(body)) finally: delete_queue_if_exists(self.rabbitmq, CONFIG_QUEUE) disconnect_from_rabbit(self.test_config_manager._rabbitmq) disconnect_from_rabbit(self.test_config_manager._heartbeat_rabbit) @mock.patch.object(ConfigsManager, "_initialise_rabbitmq", autospec=True) @mock.patch.object(ConfigsManager, "foreach_config_file", autospec=True) @mock.patch.object(PollingObserver, "start", autospec=True) @mock.patch.object(RabbitMQApi, "start_consuming", autospec=True) def test_start_not_watching(self, mock_start_consuming: MagicMock, mock_observer_start: MagicMock, mock_foreach: MagicMock, mock_initialise_rabbit: MagicMock): self.test_config_manager._watching = False mock_foreach.return_value = None mock_initialise_rabbit.return_value = None mock_observer_start.return_value = None mock_start_consuming.return_value = None self.test_config_manager.start() mock_initialise_rabbit.assert_called_once() mock_foreach.assert_called_once() mock_observer_start.assert_called_once() mock_start_consuming.assert_called_once() disconnect_from_rabbit(self.test_config_manager._rabbitmq) disconnect_from_rabbit(self.test_config_manager._heartbeat_rabbit) @mock.patch.object(ConfigsManager, "_initialise_rabbitmq", autospec=True) @mock.patch.object(ConfigsManager, "foreach_config_file", autospec=True) @mock.patch.object(PollingObserver, "start", autospec=True) def test_start_after_watching(self, mock_observer_start: MagicMock, mock_foreach: MagicMock, mock_initialise_rabbit: MagicMock): self.test_config_manager._watching = True mock_foreach.return_value = None mock_initialise_rabbit.return_value = None mock_observer_start.return_value = None self.test_config_manager.start() mock_initialise_rabbit.assert_called_once() mock_foreach.assert_called_once() mock_observer_start.assert_not_called() disconnect_from_rabbit(self.test_config_manager._rabbitmq) disconnect_from_rabbit(self.test_config_manager._heartbeat_rabbit) @mock.patch('sys.exit', autospec=True) @mock.patch.object(ConfigsManager, "disconnect_from_rabbit", autospec=True) def test__on_terminate_when_not_observing(self, mock_disconnect: MagicMock, mock_sys_exit: MagicMock): mock_disconnect.return_value = None mock_sys_exit.return_value = None # We mock the stack frame since we don't need it. mock_signal = MagicMock() mock_stack_frame = MagicMock() self.test_config_manager._on_terminate(mock_signal, mock_stack_frame) self.assertFalse(self.test_config_manager._watching) mock_disconnect.assert_called_once() mock_sys_exit.assert_called_once() @mock.patch('sys.exit', autospec=True) @mock.patch.object(ConfigsManager, "disconnect_from_rabbit", autospec=True) @mock.patch.object(PollingObserver, "stop", autospec=True) @mock.patch.object(PollingObserver, "join", autospec=True) def test__on_terminate_when_observing(self, mock_join: MagicMock, mock_stop: MagicMock, mock_disconnect: MagicMock, mock_sys_exit: MagicMock): mock_join.return_value = None mock_stop.return_value = None mock_disconnect.return_value = None mock_sys_exit.return_value = None self.test_config_manager._watching = True # We mock the signal and stack frame since we don't need them. mock_signal = MagicMock() mock_stack_frame = MagicMock() self.test_config_manager._on_terminate(mock_signal, mock_stack_frame) self.assertFalse(self.test_config_manager._watching) mock_disconnect.assert_called_once() mock_stop.assert_called_once() mock_join.assert_called_once() mock_sys_exit.assert_called_once() @mock.patch("os.path.join", autospec=True) @mock.patch("os.walk", autospec=True) def test_foreach_config_file(self, mock_os_walk: MagicMock, mock_os_path_join: MagicMock): def os_walk_fn(directory: str): file_system = [ ('/foo', ('bar', ), ('baz.ini', )), ('/foo/bar', (), ('spam.ini', 'eggs.txt')), ] for root, dirs, files in file_system: yield root, dirs, files def test_callback(input: str) -> None: self.assertIn(input, ['/foo/baz.ini', '/foo/bar/spam.ini']) mock_os_walk.side_effect = os_walk_fn mock_os_path_join.side_effect = lambda x, y: x + "/" + y self.test_config_manager.foreach_config_file(test_callback)
def perform_register(self, file_path: str) -> None: if self.has_regex_match(file_path): event = FileModifiedEvent(file_path) self.on_any_event(event)
def check_from_snapshot(self, sub_folder=None, state_callback=(lambda status: None)): from pydio.utils import i18n _ = i18n.language.ugettext logging.info('Scanning for changes since last application launch') if (not sub_folder and os.path.exists(self.basepath)) or (sub_folder and os.path.exists(self.basepath + sub_folder)): previous_snapshot = SqlSnapshot(self.basepath, self.job_data_path, sub_folder) if sub_folder: local_path = self.basepath + os.path.normpath(sub_folder) else: local_path = self.basepath state_callback(status=_('Walking through your local folder, please wait...')) def listdir(dir_path): try: return os.listdir(dir_path) except OSError as o: logging.error(o) return [] snapshot = DirectorySnapshot(local_path, recursive=True, listdir=listdir) diff = SnapshotDiffStart(previous_snapshot, snapshot) state_callback(status=_('Detected %i local changes...') % (len(diff.dirs_created) + len(diff.files_created) + len(diff.dirs_moved) + len(diff.dirs_deleted) + len(diff.files_moved) + len(diff.files_modified) + len(diff.files_deleted))) self.event_handler.begin_transaction() for path in diff.dirs_created: if self.interrupt: return self.event_handler.on_created(DirCreatedEvent(path)) for path in diff.files_created: if self.interrupt: return self.event_handler.on_created(FileCreatedEvent(path)) for path in diff.dirs_moved: if self.interrupt: return self.event_handler.on_moved(DirMovedEvent(path[0], path[1])) for path in diff.files_moved: if self.interrupt: return self.event_handler.on_moved(FileMovedEvent(path[0], path[1])) for path in diff.files_modified: if self.interrupt: return self.event_handler.on_modified(FileModifiedEvent(path)) for path in diff.files_deleted: if self.interrupt: return self.event_handler.on_deleted(FileDeletedEvent(path)) for path in diff.dirs_deleted: if self.interrupt: return self.event_handler.on_deleted(DirDeletedEvent(path)) self.event_handler.end_transaction()
def test___init__(self): event_queue = EventQueue() watch = ObservedWatch("/foobar", True) event_emitter = EventEmitter(event_queue, watch, timeout=1) event_emitter.queue_event(FileModifiedEvent("/foobar/blah"))
def test_behavior_readonly_public_attributes(self): event = FileModifiedEvent(path_1) for prop in list_attributes(event): self.assertRaises(AttributeError, setattr, event, prop, None)
def test___repr__(self): event = FileModifiedEvent(path_1) self.assertEqual("<FileModifiedEvent: src_path=%s>" %\ path_1, event.__repr__())
def test___init__(event_queue, emitter): SLEEP_TIME = 0.4 sleep(SLEEP_TIME) mkdir(p('project')) sleep(SLEEP_TIME) mkdir(p('project', 'blah')) sleep(SLEEP_TIME) touch(p('afile')) sleep(SLEEP_TIME) touch(p('fromfile')) sleep(SLEEP_TIME) mv(p('fromfile'), p('project', 'tofile')) sleep(SLEEP_TIME) touch(p('afile')) sleep(SLEEP_TIME) mv(p('project', 'blah'), p('project', 'boo')) sleep(SLEEP_TIME) rm(p('project'), recursive=True) sleep(SLEEP_TIME) rm(p('afile')) sleep(SLEEP_TIME) msize(p('bfile')) sleep(SLEEP_TIME) rm(p('bfile')) sleep(SLEEP_TIME) emitter.stop() # What we need here for the tests to pass is a collection type # that is: # * unordered # * non-unique # A multiset! Python's collections.Counter class seems appropriate. expected = { DirModifiedEvent(p()), DirCreatedEvent(p('project')), DirModifiedEvent(p('project')), DirCreatedEvent(p('project', 'blah')), FileCreatedEvent(p('afile')), DirModifiedEvent(p()), FileCreatedEvent(p('fromfile')), DirModifiedEvent(p()), DirModifiedEvent(p()), FileModifiedEvent(p('afile')), DirModifiedEvent(p('project')), DirModifiedEvent(p()), FileDeletedEvent(p('project', 'tofile')), DirDeletedEvent(p('project', 'boo')), DirDeletedEvent(p('project')), DirModifiedEvent(p()), FileDeletedEvent(p('afile')), DirModifiedEvent(p()), FileCreatedEvent(p('bfile')), FileModifiedEvent(p('bfile')), DirModifiedEvent(p()), FileDeletedEvent(p('bfile')), } expected.add(FileMovedEvent(p('fromfile'), p('project', 'tofile'))) expected.add(DirMovedEvent(p('project', 'blah'), p('project', 'boo'))) got = set() while True: try: event, _ = event_queue.get_nowait() got.add(event) except Empty: break assert expected == got
def test_clean_local_events(): def path(i): return f'/test {i}' # Simple cases file_events_test0 = [ # created + deleted -> None FileCreatedEvent(path(1)), FileDeletedEvent(path(1)), # deleted + created -> modified FileDeletedEvent(path(2)), FileCreatedEvent(path(2)), ] res0 = [ # created + deleted -> None # deleted + created -> modified FileModifiedEvent(path(2)) ] # Single file events, keep as is file_events_test1 = [ FileModifiedEvent(path(1)), FileCreatedEvent(path(2)), FileDeletedEvent(path(3)), FileMovedEvent(path(4), path(5)), ] res1 = [ FileModifiedEvent(path(1)), FileCreatedEvent(path(2)), FileDeletedEvent(path(3)), FileMovedEvent(path(4), path(5)), ] # Difficult move cases file_events_test2 = [ # created + moved -> created FileCreatedEvent(path(1)), FileMovedEvent(path(1), path(2)), # moved + deleted -> deleted FileMovedEvent(path(1), path(4)), FileDeletedEvent(path(4)), # moved + moved back -> modified FileMovedEvent(path(5), path(6)), FileMovedEvent(path(6), path(5)), # moved + moved -> deleted + created (this is currently not handled as a single # moved) FileMovedEvent(path(7), path(8)), FileMovedEvent(path(8), path(9)), ] res2 = [ # created + moved -> created FileCreatedEvent(path(2)), # moved + deleted -> deleted FileDeletedEvent(path(1)), # moved + moved back -> modified FileModifiedEvent(path(5)), # moved + moved -> deleted + created (this is currently not handled as a single # moved) FileDeletedEvent(path(7)), FileCreatedEvent(path(9)), ] # Gedit save event file_events_test3 = [ FileCreatedEvent('.gedit-save-UR4EC0'), # save new version to tmp file FileModifiedEvent('.gedit-save-UR4EC0'), # modify tmp file FileMovedEvent(path(1), path(1) + '~'), # move old version to backup FileMovedEvent('.gedit-save-UR4EC0', path(1)), # replace old version with tmp file ] res3 = [ FileModifiedEvent(path(1)), # modified file FileCreatedEvent(path(1) + '~'), # backup ] # macOS safe-save event file_events_test4 = [ FileMovedEvent(path(1), path(1) + '.sb-b78ef837-dLht38'), # move to backup FileCreatedEvent(path(1)), # create new version FileDeletedEvent(path(1) + '.sb-b78ef837-dLht38'), # delete backup ] res4 = [ FileModifiedEvent(path(1)), # modified file ] # Word on macOS created event file_events_test5 = [ FileCreatedEvent(path(1)), FileDeletedEvent(path(1)), FileCreatedEvent(path(1)), FileCreatedEvent('~$' + path(1)), ] res5 = [ FileCreatedEvent(path(1)), # created file FileCreatedEvent( '~$' + path(1)), # backup (will be deleted when file is closed) ] # simple type changes file_events_test6 = [ # keep as is FileDeletedEvent(path(1)), DirCreatedEvent(path(1)), # keep as is DirDeletedEvent(path(2)), FileCreatedEvent(path(2)), ] res6 = [ # keep as is FileDeletedEvent(path(1)), DirCreatedEvent(path(1)), # keep as is DirDeletedEvent(path(2)), FileCreatedEvent(path(2)), ] # difficult type changes file_events_test7 = [ # convert to FileDeleted -> DirCreated FileModifiedEvent(path(1)), FileDeletedEvent(path(1)), FileCreatedEvent(path(1)), FileDeletedEvent(path(1)), DirCreatedEvent(path(1)), # convert to FileDeleted(path1) -> DirCreated(path2) FileModifiedEvent(path(2)), FileDeletedEvent(path(2)), FileCreatedEvent(path(2)), FileDeletedEvent(path(2)), DirCreatedEvent(path(2)), DirMovedEvent(path(2), path(3)), ] res7 = [ FileDeletedEvent(path(1)), DirCreatedEvent(path(1)), FileDeletedEvent(path(2)), DirCreatedEvent(path(3)), ] # event hierarchies file_events_test8 = [ # convert to a single DirDeleted DirDeletedEvent(path(1)), FileDeletedEvent(path(1) + '/file1.txt'), FileDeletedEvent(path(1) + '/file2.txt'), DirDeletedEvent(path(1) + '/sub'), FileDeletedEvent(path(1) + '/sub/file3.txt'), # convert to a single DirMoved DirMovedEvent(path(2), path(3)), FileMovedEvent(path(2) + '/file1.txt', path(3) + '/file1.txt'), FileMovedEvent(path(2) + '/file2.txt', path(3) + '/file2.txt'), DirMovedEvent(path(2) + '/sub', path(3) + '/sub'), FileMovedEvent(path(2) + '/sub/file3.txt', path(3) + '/sub/file3.txt'), ] res8 = [ DirDeletedEvent(path(1)), DirMovedEvent(path(2), path(3)), ] # performance test: # 15,000 nested deleted events (10,000 folders, 5,000 files) # 15,000 nested moved events (10,000 folders, 5,000 files) # 4,995 unrelated created events file_events_test9 = [DirDeletedEvent(n * path(1)) for n in range(1, 10000)] file_events_test9 += [ FileDeletedEvent(n * path(1) + '.txt') for n in range(1, 5000) ] file_events_test9 += [ DirMovedEvent(n * path(2), n * path(3)) for n in range(1, 10000) ] file_events_test9 += [ FileMovedEvent(n * path(2) + '.txt', n * path(3) + '.txt') for n in range(1, 5000) ] file_events_test9 += [FileCreatedEvent(path(n)) for n in range(5, 5000)] res9 = [ DirDeletedEvent(path(1)), DirMovedEvent(path(2), path(3)), FileDeletedEvent(path(1) + '.txt'), FileMovedEvent(path(2) + '.txt', path(3) + '.txt'), ] res9 += [FileCreatedEvent(path(n)) for n in range(5, 5000)] sync = DummyUpDownSync() cleaned_file_events_test0 = sync._clean_local_events(file_events_test0) cleaned_file_events_test1 = sync._clean_local_events(file_events_test1) cleaned_file_events_test2 = sync._clean_local_events(file_events_test2) cleaned_file_events_test3 = sync._clean_local_events(file_events_test3) cleaned_file_events_test4 = sync._clean_local_events(file_events_test4) cleaned_file_events_test5 = sync._clean_local_events(file_events_test5) cleaned_file_events_test6 = sync._clean_local_events(file_events_test6) cleaned_file_events_test7 = sync._clean_local_events(file_events_test7) cleaned_file_events_test8 = sync._clean_local_events(file_events_test8) cleaned_file_events_test9 = sync._clean_local_events(file_events_test9) assert set(cleaned_file_events_test0) == set(res0) assert set(cleaned_file_events_test1) == set(res1) assert set(cleaned_file_events_test2) == set(res2) assert set(cleaned_file_events_test3) == set(res3) assert set(cleaned_file_events_test4) == set(res4) assert set(cleaned_file_events_test5) == set(res5) assert set(cleaned_file_events_test6) == set(res6) assert set(cleaned_file_events_test7) == set(res7) assert set(cleaned_file_events_test8) == set(res8) assert set(cleaned_file_events_test9) == set(res9) n_loops = 4 duration = timeit.timeit( lambda: sync._clean_local_events(file_events_test9), number=n_loops) assert duration < 5 * n_loops # less than 5 sec per call
def test_event_emitter(): event_queue = EventQueue() watch = ObservedWatch('/foobar', True) event_emitter = EventEmitter(event_queue, watch, timeout=1) event_emitter.queue_event(FileModifiedEvent('/foobar/blah'))
def test___init__(self): SLEEP_TIME = 0.4 self.emitter.start() sleep(SLEEP_TIME) mkdir(p("project")) sleep(SLEEP_TIME) mkdir(p("project", "blah")) sleep(SLEEP_TIME) touch(p("afile")) sleep(SLEEP_TIME) touch(p("fromfile")) sleep(SLEEP_TIME) mv(p("fromfile"), p("project", "tofile")) sleep(SLEEP_TIME) touch(p("afile")) sleep(SLEEP_TIME) mv(p("project", "blah"), p("project", "boo")) sleep(SLEEP_TIME) rm(p("project"), recursive=True) sleep(SLEEP_TIME) rm(p("afile")) sleep(SLEEP_TIME) self.emitter.stop() # What we need here for the tests to pass is a collection type # that is: # * unordered # * non-unique # A multiset! Python's collections.Counter class seems appropriate. expected = set([ DirModifiedEvent(p()), DirCreatedEvent(p("project")), DirModifiedEvent(p("project")), DirCreatedEvent(p("project", "blah")), FileCreatedEvent(p("afile")), DirModifiedEvent(p()), FileCreatedEvent(p("fromfile")), DirModifiedEvent(p()), DirModifiedEvent(p()), FileModifiedEvent(p("afile")), DirModifiedEvent(p("project")), DirModifiedEvent(p()), FileDeletedEvent(p("project", "tofile")), DirDeletedEvent(p("project", "boo")), DirDeletedEvent(p("project")), DirModifiedEvent(p()), FileDeletedEvent(p("afile")), ]) expected.add(FileMovedEvent(p("fromfile"), p("project", "tofile"))) expected.add(DirMovedEvent(p("project", "blah"), p("project", "boo"))) got = set() while True: try: event, _ = self.event_queue.get_nowait() got.add(event) except Empty: break self.assertEqual(expected, got)
def test___init__(self): SLEEP_TIME = 0.4 self.emitter.start() sleep(SLEEP_TIME) mkdir(p('project')) sleep(SLEEP_TIME) mkdir(p('project', 'blah')) sleep(SLEEP_TIME) touch(p('afile')) sleep(SLEEP_TIME) touch(p('fromfile')) sleep(SLEEP_TIME) mv(p('fromfile'), p('project', 'tofile')) sleep(SLEEP_TIME) touch(p('afile')) sleep(SLEEP_TIME) mv(p('project', 'blah'), p('project', 'boo')) sleep(SLEEP_TIME) rm(p('project'), recursive=True) sleep(SLEEP_TIME) rm(p('afile')) sleep(SLEEP_TIME) self.emitter.stop() # What we need here for the tests to pass is a collection type # that is: # * unordered # * non-unique # A multiset! Python's collections.Counter class seems appropriate. expected = set([ DirModifiedEvent(p()), DirCreatedEvent(p('project')), DirModifiedEvent(p('project')), DirCreatedEvent(p('project', 'blah')), FileCreatedEvent(p('afile')), DirModifiedEvent(p()), FileCreatedEvent(p('fromfile')), DirModifiedEvent(p()), DirModifiedEvent(p()), FileModifiedEvent(p('afile')), DirModifiedEvent(p('project')), DirModifiedEvent(p()), FileDeletedEvent(p('project', 'tofile')), DirDeletedEvent(p('project', 'boo')), DirDeletedEvent(p('project')), DirModifiedEvent(p()), FileDeletedEvent(p('afile')), ]) if sys.platform.startswith("win"): # On Windows depending on circumstances, a rename may turn into a Delete/Create expected.add(FileDeletedEvent(p('fromfile'))) expected.add(FileCreatedEvent(p('project', 'tofile'))) expected.add(DirCreatedEvent(p('project', 'boo'))) expected.add(DirDeletedEvent(p('project', 'blah'))) else: expected.add(FileMovedEvent(p('fromfile'), p('project', 'tofile'))) expected.add(DirMovedEvent(p('project', 'blah'), p('project', 'boo'))) got = set() while True: try: event, _ = self.event_queue.get_nowait() got.add(event) except queue.Empty: break self.assertEqual(expected, got)