def _get_dispatcher(config):
with patch('trollmoves.dispatcher.DispatchConfig'):
with NamedTemporaryFile('w') as fid:
fname = fid.name
dispatcher = Dispatcher(fname)
dispatcher.config = yaml.safe_load(config)
return dispatcher
def test_publisher_init_no_port(NoisyPublisher, ListenerContainer, Message,
publisher_config_file_name):
"""Test the publisher is initialized when no port is defined."""
pub = Mock()
NoisyPublisher.return_value = pub
try:
try:
dispatcher = Dispatcher(publisher_config_file_name)
assert dispatcher.publisher is None
NoisyPublisher.assert_not_called()
finally:
if dispatcher is not None:
dispatcher.close()
finally:
os.remove(publisher_config_file_name)
def _create_dest_url_dispatcher():
with patch('trollmoves.dispatcher.ListenerContainer') as lc:
queue = Queue()
lc.return_value.output_queue = queue
config_file_name = _write_config_file(test_yaml_ssh_scp)
dispatcher = Dispatcher(config_file_name)
return dispatcher, config_file_name
def test_publisher_init_with_random_publish_port(NoisyPublisher,
ListenerContainer, Message,
publisher_config_file_name):
"""Test the publisher is initialized with randomly selected publish port."""
pub = Mock()
NoisyPublisher.return_value = pub
try:
try:
dispatcher = Dispatcher(publisher_config_file_name, publish_port=0)
init_call = call("dispatcher", port=0, nameservers=None)
assert init_call in NoisyPublisher.mock_calls
finally:
if dispatcher is not None:
dispatcher.close()
dispatcher.publisher.stop.assert_called_once()
finally:
os.remove(publisher_config_file_name)
def test_publisher_init_no_port_with_nameserver(NoisyPublisher,
ListenerContainer, Message,
publisher_config_file_name):
"""Test the publisher is initialized without port but with nameservers."""
pub = Mock()
NoisyPublisher.return_value = pub
try:
try:
dispatcher = Dispatcher(publisher_config_file_name,
publish_nameservers=["asd"])
assert dispatcher.publisher is None
NoisyPublisher.assert_not_called()
finally:
if dispatcher is not None:
dispatcher.close()
finally:
os.remove(publisher_config_file_name)
def main():
"""Start and run the dispatcher."""
parser = argparse.ArgumentParser()
parser.add_argument("config_file",
help="The configuration file to run on.")
parser.add_argument("-l",
"--log",
help="The file to log to. stdout otherwise.")
parser.add_argument(
"-c",
"--log-config",
help="Log config file to use instead of the standard logging.")
parser.add_argument(
"-v",
"--verbose",
dest="verbosity",
action="count",
default=0,
help="Verbosity (between 1 and 2 occurrences with more leading to more "
"verbose logging). WARN=0, INFO=1, "
"DEBUG=2. This is overridden by the log config file if specified.")
parser.add_argument(
"-p",
"--publish-port",
type=int,
dest="pub_port",
nargs='?',
const=0,
default=None,
help="Publish messages for dispatched files on this port. "
"Default: no publishing.")
parser.add_argument("-n",
"--publish-nameserver",
nargs='*',
dest="pub_nameservers",
help="Nameserver for publisher to connect to")
cmd_args = parser.parse_args()
setup_logging(cmd_args)
logger.info("Starting up.")
try:
dispatcher = Dispatcher(cmd_args.config_file,
publish_port=cmd_args.pub_port,
publish_nameservers=cmd_args.pub_nameservers)
except Exception as err:
logger.error('Dispatcher crashed: %s', str(err))
sys.exit(1)
try:
dispatcher.start()
dispatcher.join()
except KeyboardInterrupt:
logger.debug("Interrupting")
finally:
dispatcher.close()
def test_publisher_init_port_and_nameservers(NoisyPublisher, ListenerContainer,
Message,
publisher_config_file_name):
"""Test the publisher is initialized with port and nameservers."""
pub = Mock()
NoisyPublisher.return_value = pub
try:
try:
dispatcher = Dispatcher(publisher_config_file_name,
publish_port=40000,
publish_nameservers=["asd"])
assert dispatcher.publisher is pub
init_call = call("dispatcher", port=40000, nameservers=["asd"])
assert init_call in NoisyPublisher.mock_calls
finally:
if dispatcher is not None:
dispatcher.close()
dispatcher.publisher.stop.assert_called_once()
finally:
os.remove(publisher_config_file_name)
def test_get_destinations():
"""Check getting destination urls."""
with patch('trollmoves.dispatcher.DispatchConfig'):
with NamedTemporaryFile('w') as the_file:
fname = the_file.name
dp = Dispatcher(fname)
dp.config = yaml.safe_load(test_yaml1)
msg = Mock()
msg.subject = '/level2/viirs'
msg.data = {
'sensor': 'viirs',
'product': 'green_snow',
'platform_name': 'NOAA-20',
'start_time': datetime(2019, 9, 19, 9, 19),
'format': 'tif'
}
expected_url = 'ftp://ftp.target1.com/input_data/viirs/NOAA-20_201909190919.tif'
expected_attrs = {'connection_uptime': 20}
res = dp.get_destinations(msg)
assert len(res) == 1
url, attrs, client = res[0]
assert url == expected_url
assert attrs == expected_attrs
assert client == "target1"
dp.config = yaml.safe_load(test_yaml2)
res = dp.get_destinations(msg)
assert len(res) == 2
def test_publisher_call(NoisyPublisher, ListenerContainer, Message,
publisher_config_file_name):
"""Test the publisher being called properly."""
pub = Mock()
NoisyPublisher.return_value = pub
try:
try:
dispatcher = Dispatcher(publisher_config_file_name,
publish_port=40000,
publish_nameservers=["asd"])
msg = Mock(data={
'uri': 'original_path',
'platform_name': 'platform'
})
destinations = [['url1', 'params1', 'target2'],
['url2', 'params2', 'target3']]
success = {'target2': False, 'target3': True}
dispatcher._publish(msg, destinations, success)
dispatcher.publisher.send.assert_called_once()
# The message topic has been composed and uri has been replaced
msg_call = call('/topic/platform', 'file', {
'uri': 'url2',
'platform_name': 'platform'
})
assert msg_call in Message.mock_calls
finally:
if dispatcher is not None:
dispatcher.close()
dispatcher.publisher.stop.assert_called()
finally:
os.remove(publisher_config_file_name)
def main():
"""Start and run the dispatcher."""
cmd_args = parse_args()
setup_logging(cmd_args)
logger.info("Starting up.")
try:
dispatcher = Dispatcher(cmd_args.config_file,
publish_port=cmd_args.pub_port,
publish_nameservers=cmd_args.pub_nameservers)
except Exception as err:
logger.error('Dispatcher crashed: %s', str(err))
sys.exit(1)
try:
dispatcher.start()
dispatcher.join()
except KeyboardInterrupt:
logger.debug("Interrupting")
finally:
dispatcher.close()
def test_dispatcher():
"""Test the dispatcher class."""
dp = None
try:
dest_dir = os.path.join(gettempdir(), 'dptest')
if os.path.exists(dest_dir):
shutil.rmtree(dest_dir)
with patch('trollmoves.dispatcher.ListenerContainer') as lc:
queue = Queue()
lc.return_value.output_queue = queue
with NamedTemporaryFile('w', delete=False) as config_file:
config_file_name = config_file.name
config_file.write(test_local)
config_file.flush()
config_file.close()
dp = Dispatcher(config_file_name)
dp.start()
dest_dir = os.path.join(gettempdir(), 'dptest')
assert not os.path.exists(dest_dir)
with NamedTemporaryFile('w') as test_file:
msg = Mock()
msg.type = 'file'
msg.subject = '/level2/viirs'
msg.data = {
'sensor': 'viirs',
'product': 'green_snow',
'platform_name': 'NOAA-20',
'start_time': datetime(2019, 9, 19, 9, 19),
'format': 'tif',
'area': 'euron1',
'uid': '201909190919_NOAA-20_viirs.tif',
'uri': test_file.name
}
expected_file = os.path.join(dest_dir,
'NOAA-20_201909190919.tif')
queue.put(msg)
time.sleep(.1)
assert os.path.exists(expected_file)
# Check that the listener config items are passed correctly
lc.assert_called_once_with(
addresses=['tcp://127.0.0.1:40000'],
nameserver='127.0.0.1',
services=['service_name_1', 'service_name_2'],
topics={'/level3/cloudtype', '/level2/viirs', '/level2/avhrr'})
finally:
if dp is not None:
dp.close()
os.remove(expected_file)
os.rmdir(dest_dir)
os.remove(config_file_name)
def main():
"""Start and run the dispatcher."""
parser = argparse.ArgumentParser()
parser.add_argument("config_file",
help="The configuration file to run on.")
parser.add_argument("-l",
"--log",
help="The file to log to. stdout otherwise.")
parser.add_argument(
"-c",
"--log-config",
help="Log config file to use instead of the standard logging.")
parser.add_argument(
"-v",
"--verbose",
dest="verbosity",
action="count",
default=0,
help="Verbosity (between 1 and 2 occurrences with more leading to more "
"verbose logging). WARN=0, INFO=1, "
"DEBUG=2. This is overridden by the log config file if specified.")
cmd_args = parser.parse_args()
setup_logging(cmd_args)
logger.info("Starting up.")
try:
dispatcher = Dispatcher(cmd_args.config_file)
except Exception as err:
logger.error('Dispatcher crashed: %s', str(err))
sys.exit(1)
try:
dispatcher.start()
dispatcher.join()
except KeyboardInterrupt:
logger.debug("Interrupting")
finally:
dispatcher.close()
def test_create_dest_url():
"""Test creation of destination URL."""
dp = None
try:
with patch('trollmoves.dispatcher.ListenerContainer') as lc:
queue = Queue()
lc.return_value.output_queue = queue
with NamedTemporaryFile('w', delete=False) as config_file:
config_file_name = config_file.name
config_file.write(test_yaml_ssh_scp)
config_file.flush()
config_file.close()
config = yaml.safe_load(test_yaml_ssh_scp)
dp = Dispatcher(config_file_name)
msg = Mock()
msg.subject = '/level2/viirs'
msg.data = {
'sensor': 'viirs',
'product': 'green_snow',
'platform_name': 'NOAA-20',
'start_time': datetime(2019, 9, 19, 9, 19),
'format': 'tif'
}
# SSH protocol, no username
url, params, client = dp.create_dest_url(msg, 'target2',
config['target2'])
expected_url = "ssh://server.target2.com/satellite/viirs/sat_201909190919_NOAA-20.tif"
assert url == expected_url
assert params == {'ssh_key_filename': '~/.ssh/rsa_id.pub'}
assert client == "target2"
# SCP protocolw with username
url, params, client = dp.create_dest_url(msg, 'target3',
config['target3'])
expected_url = "scp://[email protected]/satellite/viirs/sat_201909190919_NOAA-20.tif"
assert url == expected_url
assert client == "target3"
finally:
if dp is not None:
dp.close()
os.remove(config_file_name)
def test_dispatcher(get_destinations_message):
"""Test the dispatcher class."""
dp = None
try:
dest_dir = os.path.join(gettempdir(), 'dptest')
if os.path.exists(dest_dir):
shutil.rmtree(dest_dir)
with patch('trollmoves.dispatcher.ListenerContainer') as lc:
queue = Queue()
lc.return_value.output_queue = queue
with NamedTemporaryFile('w', delete=False) as config_file:
config_file_name = config_file.name
config_file.write(test_local)
config_file.flush()
config_file.close()
dp = Dispatcher(config_file_name)
dp.start()
dest_dir = os.path.join(gettempdir(), 'dptest')
assert not os.path.exists(dest_dir)
with NamedTemporaryFile('w') as test_file:
get_destinations_message.data['uri'] = test_file.name
expected_file = os.path.join(dest_dir,
'NOAA-20_201909190919.tif')
queue.put(get_destinations_message)
time.sleep(.1)
assert os.path.exists(expected_file)
# Check that the listener config items are passed correctly
lc.assert_called_once_with(
addresses=['tcp://127.0.0.1:40000'],
nameserver='127.0.0.1',
services=['service_name_1', 'service_name_2'],
topics={'/level3/cloudtype', '/level2/viirs', '/level2/avhrr'})
finally:
if dp is not None:
dp.close()
os.remove(expected_file)
os.rmdir(dest_dir)
os.remove(config_file_name)
def test_publisher(NoisyPublisher, ListenerContainer, Message):
"""Test the publisher is initialized."""
pub = Mock()
NoisyPublisher.return_value = pub
try:
with NamedTemporaryFile('w', delete=False) as config_file:
config_file_name = config_file.name
config_file.write(test_yaml_pub)
config_file.flush()
config_file.close()
try:
dp = Dispatcher(config_file_name)
assert dp.publisher is None
NoisyPublisher.assert_not_called()
finally:
if dp is not None:
dp.close()
try:
dp = Dispatcher(config_file_name, publish_nameservers=["asd"])
assert dp.publisher is None
NoisyPublisher.assert_not_called()
finally:
if dp is not None:
dp.close()
try:
dp = Dispatcher(config_file_name, publish_port=0)
init_call = call("dispatcher", port=0, nameservers=None)
assert init_call in NoisyPublisher.mock_calls
finally:
if dp is not None:
dp.close()
dp.publisher.stop.assert_called()
try:
dp = Dispatcher(config_file_name, publish_port=40000)
init_call = call("dispatcher", port=40000, nameservers=None)
assert init_call in NoisyPublisher.mock_calls
finally:
if dp is not None:
dp.close()
assert len(dp.publisher.stop.mock_calls) == 2
try:
dp = Dispatcher(config_file_name,
publish_port=40000,
publish_nameservers=["asd"])
assert dp.publisher is pub
init_call = call("dispatcher", port=40000, nameservers=["asd"])
assert init_call in NoisyPublisher.mock_calls
msg = Mock(data={
'uri': 'original_path',
'platform_name': 'platform'
})
destinations = [['url1', 'params1', 'target2'],
['url2', 'params2', 'target3']]
success = {'target2': False, 'target3': True}
dp._publish(msg, destinations, success)
dp.publisher.send.assert_called_once()
# The message topic has been composed and uri has been replaced
msg_call = call('/topic/platform', 'file', {
'uri': 'url2',
'platform_name': 'platform'
})
assert msg_call in Message.mock_calls
finally:
if dp is not None:
dp.close()
dp.publisher.stop.assert_called()
finally:
os.remove(config_file_name)