def test_publisher_call(NoisyPublisher, ListenerContainer, Message, publisher_config_file_name): """Test the publisher being called properly.""" pub = Mock() NoisyPublisher.return_value = pub try: try: dispatcher = Dispatcher(publisher_config_file_name, publish_port=40000, publish_nameservers=["asd"]) msg = Mock(data={ 'uri': 'original_path', 'platform_name': 'platform' }) destinations = [['url1', 'params1', 'target2'], ['url2', 'params2', 'target3']] success = {'target2': False, 'target3': True} dispatcher._publish(msg, destinations, success) dispatcher.publisher.send.assert_called_once() # The message topic has been composed and uri has been replaced msg_call = call('/topic/platform', 'file', { 'uri': 'url2', 'platform_name': 'platform' }) assert msg_call in Message.mock_calls finally: if dispatcher is not None: dispatcher.close() dispatcher.publisher.stop.assert_called() finally: os.remove(publisher_config_file_name)
def main(): """Start and run the dispatcher.""" parser = argparse.ArgumentParser() parser.add_argument("config_file", help="The configuration file to run on.") parser.add_argument("-l", "--log", help="The file to log to. stdout otherwise.") parser.add_argument( "-c", "--log-config", help="Log config file to use instead of the standard logging.") parser.add_argument( "-v", "--verbose", dest="verbosity", action="count", default=0, help="Verbosity (between 1 and 2 occurrences with more leading to more " "verbose logging). WARN=0, INFO=1, " "DEBUG=2. This is overridden by the log config file if specified.") parser.add_argument( "-p", "--publish-port", type=int, dest="pub_port", nargs='?', const=0, default=None, help="Publish messages for dispatched files on this port. " "Default: no publishing.") parser.add_argument("-n", "--publish-nameserver", nargs='*', dest="pub_nameservers", help="Nameserver for publisher to connect to") cmd_args = parser.parse_args() setup_logging(cmd_args) logger.info("Starting up.") try: dispatcher = Dispatcher(cmd_args.config_file, publish_port=cmd_args.pub_port, publish_nameservers=cmd_args.pub_nameservers) except Exception as err: logger.error('Dispatcher crashed: %s', str(err)) sys.exit(1) try: dispatcher.start() dispatcher.join() except KeyboardInterrupt: logger.debug("Interrupting") finally: dispatcher.close()
def test_dispatcher(): """Test the dispatcher class.""" dp = None try: dest_dir = os.path.join(gettempdir(), 'dptest') if os.path.exists(dest_dir): shutil.rmtree(dest_dir) with patch('trollmoves.dispatcher.ListenerContainer') as lc: queue = Queue() lc.return_value.output_queue = queue with NamedTemporaryFile('w', delete=False) as config_file: config_file_name = config_file.name config_file.write(test_local) config_file.flush() config_file.close() dp = Dispatcher(config_file_name) dp.start() dest_dir = os.path.join(gettempdir(), 'dptest') assert not os.path.exists(dest_dir) with NamedTemporaryFile('w') as test_file: msg = Mock() msg.type = 'file' msg.subject = '/level2/viirs' msg.data = { 'sensor': 'viirs', 'product': 'green_snow', 'platform_name': 'NOAA-20', 'start_time': datetime(2019, 9, 19, 9, 19), 'format': 'tif', 'area': 'euron1', 'uid': '201909190919_NOAA-20_viirs.tif', 'uri': test_file.name } expected_file = os.path.join(dest_dir, 'NOAA-20_201909190919.tif') queue.put(msg) time.sleep(.1) assert os.path.exists(expected_file) # Check that the listener config items are passed correctly lc.assert_called_once_with( addresses=['tcp://127.0.0.1:40000'], nameserver='127.0.0.1', services=['service_name_1', 'service_name_2'], topics={'/level3/cloudtype', '/level2/viirs', '/level2/avhrr'}) finally: if dp is not None: dp.close() os.remove(expected_file) os.rmdir(dest_dir) os.remove(config_file_name)
def test_publisher_init_no_port(NoisyPublisher, ListenerContainer, Message, publisher_config_file_name): """Test the publisher is initialized when no port is defined.""" pub = Mock() NoisyPublisher.return_value = pub try: try: dispatcher = Dispatcher(publisher_config_file_name) assert dispatcher.publisher is None NoisyPublisher.assert_not_called() finally: if dispatcher is not None: dispatcher.close() finally: os.remove(publisher_config_file_name)
def test_publisher_init_with_random_publish_port(NoisyPublisher, ListenerContainer, Message, publisher_config_file_name): """Test the publisher is initialized with randomly selected publish port.""" pub = Mock() NoisyPublisher.return_value = pub try: try: dispatcher = Dispatcher(publisher_config_file_name, publish_port=0) init_call = call("dispatcher", port=0, nameservers=None) assert init_call in NoisyPublisher.mock_calls finally: if dispatcher is not None: dispatcher.close() dispatcher.publisher.stop.assert_called_once() finally: os.remove(publisher_config_file_name)
def test_publisher_init_no_port_with_nameserver(NoisyPublisher, ListenerContainer, Message, publisher_config_file_name): """Test the publisher is initialized without port but with nameservers.""" pub = Mock() NoisyPublisher.return_value = pub try: try: dispatcher = Dispatcher(publisher_config_file_name, publish_nameservers=["asd"]) assert dispatcher.publisher is None NoisyPublisher.assert_not_called() finally: if dispatcher is not None: dispatcher.close() finally: os.remove(publisher_config_file_name)
def test_create_dest_url(): """Test creation of destination URL.""" dp = None try: with patch('trollmoves.dispatcher.ListenerContainer') as lc: queue = Queue() lc.return_value.output_queue = queue with NamedTemporaryFile('w', delete=False) as config_file: config_file_name = config_file.name config_file.write(test_yaml_ssh_scp) config_file.flush() config_file.close() config = yaml.safe_load(test_yaml_ssh_scp) dp = Dispatcher(config_file_name) msg = Mock() msg.subject = '/level2/viirs' msg.data = { 'sensor': 'viirs', 'product': 'green_snow', 'platform_name': 'NOAA-20', 'start_time': datetime(2019, 9, 19, 9, 19), 'format': 'tif' } # SSH protocol, no username url, params, client = dp.create_dest_url(msg, 'target2', config['target2']) expected_url = "ssh://server.target2.com/satellite/viirs/sat_201909190919_NOAA-20.tif" assert url == expected_url assert params == {'ssh_key_filename': '~/.ssh/rsa_id.pub'} assert client == "target2" # SCP protocolw with username url, params, client = dp.create_dest_url(msg, 'target3', config['target3']) expected_url = "scp://[email protected]/satellite/viirs/sat_201909190919_NOAA-20.tif" assert url == expected_url assert client == "target3" finally: if dp is not None: dp.close() os.remove(config_file_name)
def main(): """Start and run the dispatcher.""" cmd_args = parse_args() setup_logging(cmd_args) logger.info("Starting up.") try: dispatcher = Dispatcher(cmd_args.config_file, publish_port=cmd_args.pub_port, publish_nameservers=cmd_args.pub_nameservers) except Exception as err: logger.error('Dispatcher crashed: %s', str(err)) sys.exit(1) try: dispatcher.start() dispatcher.join() except KeyboardInterrupt: logger.debug("Interrupting") finally: dispatcher.close()
def test_publisher_init_port_and_nameservers(NoisyPublisher, ListenerContainer, Message, publisher_config_file_name): """Test the publisher is initialized with port and nameservers.""" pub = Mock() NoisyPublisher.return_value = pub try: try: dispatcher = Dispatcher(publisher_config_file_name, publish_port=40000, publish_nameservers=["asd"]) assert dispatcher.publisher is pub init_call = call("dispatcher", port=40000, nameservers=["asd"]) assert init_call in NoisyPublisher.mock_calls finally: if dispatcher is not None: dispatcher.close() dispatcher.publisher.stop.assert_called_once() finally: os.remove(publisher_config_file_name)
def main(): """Start and run the dispatcher.""" parser = argparse.ArgumentParser() parser.add_argument("config_file", help="The configuration file to run on.") parser.add_argument("-l", "--log", help="The file to log to. stdout otherwise.") parser.add_argument( "-c", "--log-config", help="Log config file to use instead of the standard logging.") parser.add_argument( "-v", "--verbose", dest="verbosity", action="count", default=0, help="Verbosity (between 1 and 2 occurrences with more leading to more " "verbose logging). WARN=0, INFO=1, " "DEBUG=2. This is overridden by the log config file if specified.") cmd_args = parser.parse_args() setup_logging(cmd_args) logger.info("Starting up.") try: dispatcher = Dispatcher(cmd_args.config_file) except Exception as err: logger.error('Dispatcher crashed: %s', str(err)) sys.exit(1) try: dispatcher.start() dispatcher.join() except KeyboardInterrupt: logger.debug("Interrupting") finally: dispatcher.close()
def test_dispatcher(get_destinations_message): """Test the dispatcher class.""" dp = None try: dest_dir = os.path.join(gettempdir(), 'dptest') if os.path.exists(dest_dir): shutil.rmtree(dest_dir) with patch('trollmoves.dispatcher.ListenerContainer') as lc: queue = Queue() lc.return_value.output_queue = queue with NamedTemporaryFile('w', delete=False) as config_file: config_file_name = config_file.name config_file.write(test_local) config_file.flush() config_file.close() dp = Dispatcher(config_file_name) dp.start() dest_dir = os.path.join(gettempdir(), 'dptest') assert not os.path.exists(dest_dir) with NamedTemporaryFile('w') as test_file: get_destinations_message.data['uri'] = test_file.name expected_file = os.path.join(dest_dir, 'NOAA-20_201909190919.tif') queue.put(get_destinations_message) time.sleep(.1) assert os.path.exists(expected_file) # Check that the listener config items are passed correctly lc.assert_called_once_with( addresses=['tcp://127.0.0.1:40000'], nameserver='127.0.0.1', services=['service_name_1', 'service_name_2'], topics={'/level3/cloudtype', '/level2/viirs', '/level2/avhrr'}) finally: if dp is not None: dp.close() os.remove(expected_file) os.rmdir(dest_dir) os.remove(config_file_name)
def test_publisher(NoisyPublisher, ListenerContainer, Message): """Test the publisher is initialized.""" pub = Mock() NoisyPublisher.return_value = pub try: with NamedTemporaryFile('w', delete=False) as config_file: config_file_name = config_file.name config_file.write(test_yaml_pub) config_file.flush() config_file.close() try: dp = Dispatcher(config_file_name) assert dp.publisher is None NoisyPublisher.assert_not_called() finally: if dp is not None: dp.close() try: dp = Dispatcher(config_file_name, publish_nameservers=["asd"]) assert dp.publisher is None NoisyPublisher.assert_not_called() finally: if dp is not None: dp.close() try: dp = Dispatcher(config_file_name, publish_port=0) init_call = call("dispatcher", port=0, nameservers=None) assert init_call in NoisyPublisher.mock_calls finally: if dp is not None: dp.close() dp.publisher.stop.assert_called() try: dp = Dispatcher(config_file_name, publish_port=40000) init_call = call("dispatcher", port=40000, nameservers=None) assert init_call in NoisyPublisher.mock_calls finally: if dp is not None: dp.close() assert len(dp.publisher.stop.mock_calls) == 2 try: dp = Dispatcher(config_file_name, publish_port=40000, publish_nameservers=["asd"]) assert dp.publisher is pub init_call = call("dispatcher", port=40000, nameservers=["asd"]) assert init_call in NoisyPublisher.mock_calls msg = Mock(data={ 'uri': 'original_path', 'platform_name': 'platform' }) destinations = [['url1', 'params1', 'target2'], ['url2', 'params2', 'target3']] success = {'target2': False, 'target3': True} dp._publish(msg, destinations, success) dp.publisher.send.assert_called_once() # The message topic has been composed and uri has been replaced msg_call = call('/topic/platform', 'file', { 'uri': 'url2', 'platform_name': 'platform' }) assert msg_call in Message.mock_calls finally: if dp is not None: dp.close() dp.publisher.stop.assert_called() finally: os.remove(config_file_name)