def test_read_config():
"""Test config handling."""
from trollmoves.client import read_config
with NamedTemporaryFile('w', delete=False) as fid:
config_fname = fid.name
fid.write(CLIENT_CONFIG_1_ITEM)
try:
conf = read_config(config_fname)
finally:
os.remove(config_fname)
# Test that required things are present
section_name = "eumetcast_hrit_0deg_scp_hot_spare"
assert section_name in conf
section_keys = conf[section_name].keys()
for key in [
"delete",
"working_directory",
"compression",
"heartbeat",
"req_timeout",
"transfer_req_timeout",
"nameservers",
"providers",
"topic",
"publish_port",
]:
assert key in section_keys
assert isinstance(conf[section_name]["providers"], list)
def test_unpack_and_create_local_message(unpackers):
"""Test unpacking and updating the message with new filenames."""
from trollmoves.client import unpack_and_create_local_message as unp
local_dir = '/local'
kwargs = {'kwarg': 'value'}
# No compression defined
res = unp(copy.copy(MSG_FILE), local_dir, **kwargs)
assert res.subject == MSG_FILE.subject
assert res.data == MSG_FILE.data
assert res.type == MSG_FILE.type
# A new message is returned
assert res is not MSG_FILE
unpackers.__getitem__.assert_not_called()
# One file with 'tar' compression
kwargs['compression'] = 'tar'
unpackers['tar'].return_value = 'new_file1.png'
res = unp(copy.copy(MSG_FILE_TAR), local_dir, **kwargs)
assert res.data['uri'] == os.path.join(local_dir, 'new_file1.png')
assert res.data['uid'] == 'new_file1.png'
assert res.subject == MSG_FILE_TAR.subject
assert res.type == MSG_FILE_TAR.type
unpackers['tar'].assert_called_with('/local/file1.tar', **kwargs)
# The unpacker returns a full path for some reason
unpackers['tar'].return_value = os.path.join(local_dir, 'new_file1.png')
res = unp(copy.copy(MSG_FILE_TAR), local_dir, **kwargs)
assert res.data['uri'] == os.path.join(local_dir, 'new_file1.png')
assert res.data['uid'] == 'new_file1.png'
# One file with 'bz2' compression
kwargs['compression'] = 'bzip'
unpackers['bzip'].return_value = 'file1.png'
res = unp(copy.copy(MSG_FILE_BZ2), local_dir, **kwargs)
assert res.data['uri'] == os.path.join(local_dir, 'file1.png')
assert res.data['uid'] == 'file1.png'
assert res.subject == MSG_FILE_BZ2.subject
assert res.type == MSG_FILE_BZ2.type
unpackers['bzip'].assert_called_with('/local/file1.png.bz2', **kwargs)
# One file with 'xrit' compression
kwargs['compression'] = 'xrit'
unpackers['xrit'].return_value = 'new_file1.png'
with patch('os.remove') as remove:
res = unp(copy.copy(MSG_FILE_XRIT), local_dir, **kwargs)
# Delete has not been setup, so it shouldn't been done
remove.assert_not_called()
assert res.data['uri'] == os.path.join(local_dir, 'new_file1.png')
assert res.data['uid'] == 'new_file1.png'
assert res.subject == MSG_FILE_XRIT.subject
assert res.type == MSG_FILE_XRIT.type
unpackers['xrit'].assert_called_with('/local/file1-C_', **kwargs)
# One file with 'xrit' compression, delete the compressed file
kwargs['delete'] = True
unpackers['xrit'].return_value = 'new_file1.png'
with patch('os.remove') as remove:
res = unp(copy.copy(MSG_FILE_XRIT), local_dir, **kwargs)
assert remove.called_once_with('/local/file1-C_')
del kwargs['delete']
# Multiple files returned when decompression is applied. 'tar'
# compression, 'file' message
kwargs['compression'] = 'tar'
unpackers['tar'].return_value = ('new_file1.png', 'new_file2.png')
res = unp(copy.copy(MSG_FILE_TAR), local_dir, **kwargs)
assert res.data['dataset'][0]['uid'] == 'new_file1.png'
assert res.data['dataset'][0]['uri'] == '/local/new_file1.png'
assert res.data['dataset'][1]['uid'] == 'new_file2.png'
assert res.data['dataset'][1]['uri'] == '/local/new_file2.png'
assert res.subject == MSG_FILE_TAR.subject
assert res.type == "dataset"
unpackers['tar'].assert_called_with('/local/file1.tar', **kwargs)
# Dataset message, 'tar' compression
kwargs['compression'] = 'tar'
unpackers['tar'].return_value = None
unpackers['tar'].side_effect = ['new_file1.png', 'new_file2.png']
res = unp(copy.copy(MSG_DATASET_TAR), local_dir, **kwargs)
assert res.data['dataset'][0]['uid'] == 'new_file1.png'
assert res.data['dataset'][0]['uri'] == '/local/new_file1.png'
assert res.data['dataset'][1]['uid'] == 'new_file2.png'
assert res.data['dataset'][1]['uri'] == '/local/new_file2.png'
assert res.subject == MSG_DATASET_TAR.subject
assert res.type == MSG_DATASET_TAR.type
assert call('/local/file1.tgz', **kwargs) in unpackers['tar'].mock_calls
assert call('/local/file2.tar.gz', **kwargs) in unpackers['tar'].mock_calls
# Collection message, 'tar' compression
kwargs['compression'] = 'tar'
unpackers['tar'].return_value = None
unpackers['tar'].side_effect = ['new_file1.png']
res = unp(copy.copy(MSG_COLLECTION_TAR), local_dir, **kwargs)
assert res.data['collection'][0]['dataset'][0]['uid'] == 'new_file1.png'
assert res.data['collection'][0]['dataset'][0][
'uri'] == '/local/new_file1.png'
assert res.subject == MSG_COLLECTION_TAR.subject
assert res.type == MSG_COLLECTION_TAR.type
assert call('/local/file1.tar.bz2',
**kwargs) in unpackers['tar'].mock_calls
# Test with config file
with NamedTemporaryFile('w', delete=False) as fid:
config_fname = fid.name
fid.write(COMPRESSION_CONFIG)
try:
from trollmoves.client import read_config
config = read_config(config_fname)
# No compression
kwargs = config['empty_decompression']
res = unp(copy.copy(MSG_FILE), local_dir, **kwargs)
assert res.subject == MSG_FILE.subject
assert res.data == MSG_FILE.data
assert res.type == MSG_FILE.type
# A new message is returned
assert res is not MSG_FILE
# One file with 'xrit' compression
kwargs = config['xrit_decompression']
unpackers['xrit'].side_effect = None
unpackers['xrit'].return_value = 'new_file1.png'
res = unp(copy.copy(MSG_FILE_XRIT), local_dir, **kwargs)
assert res.data['uri'] == os.path.join(local_dir, 'new_file1.png')
assert res.data['uid'] == 'new_file1.png'
assert res.subject == MSG_FILE_XRIT.subject
assert res.type == MSG_FILE_XRIT.type
finally:
os.remove(config_fname)
def test_chain(Listener, NoisyPublisher, caplog):
"""Test the Chain object."""
from trollmoves.client import Chain, read_config
with NamedTemporaryFile('w', delete=False) as fid:
config_fname = fid.name
fid.write(CLIENT_CONFIG_1_ITEM)
try:
conf = read_config(config_fname)
finally:
os.remove(config_fname)
def restart():
return Listener()
side_effect = [
MagicMock(),
MagicMock(),
MagicMock(),
MagicMock(),
MagicMock(),
MagicMock(),
MagicMock(),
MagicMock(),
MagicMock(),
MagicMock(),
MagicMock(),
MagicMock()
]
for lis in side_effect:
lis.is_alive.return_value = True
lis.death_count = 0
lis.restart.side_effect = restart
Listener.side_effect = side_effect
# Init
name = 'eumetcast_hrit_0deg_scp_hot_spare'
chain = Chain(name, conf[name])
NoisyPublisher.assert_called_once()
assert chain.listeners == {}
assert not chain.listener_died_event.is_set()
# Setup listeners
callback = MagicMock()
sync_pub_instance = MagicMock()
chain.setup_listeners(callback, sync_pub_instance)
assert len(chain.listeners) == 4
# Check running with alive listeners
import trollmoves.client
with patch('trollmoves.client.LISTENER_CHECK_INTERVAL', new=.1):
trollmoves.client.LISTENER_CHECK_INTERVAL = .1
chain.start()
try:
with patch.object(chain, 'restart_dead_listeners') as rdl:
time.sleep(.2)
assert rdl.call_count == 0
chain.listener_died_event.set()
time.sleep(.2)
assert rdl.call_count == 1
assert not chain.listener_died_event.is_set()
chain.listener_died_event.set()
time.sleep(.2)
assert not chain.listener_died_event.is_set()
# Check with listener crashing once
listener = chain.listeners['tcp://satmottag2:9010']
listener.is_alive.return_value = False
listener.cause_of_death = RuntimeError(
'OMG, they killed the listener!')
chain.listener_died_event.set()
time.sleep(.2)
listener.restart.assert_called_once()
assert "Listener for tcp://satmottag2:9010 died 1 time: OMG, they killed the listener!" in caplog.text
time.sleep(.6)
# Check with listener crashing all the time
death_count = 0
for lis in side_effect[5:]:
death_count += 1
lis.is_alive.return_value = False
lis.cause_of_death = RuntimeError(
'OMG, they killed the listener!')
lis.death_count = death_count
listener = chain.listeners['tcp://satmottag2:9010']
listener.is_alive.return_value = False
listener.death_count = 0
listener.cause_of_death = RuntimeError(
'OMG, they killed the listener!')
listener.restart.side_effect = restart
chain.listener_died_event.set()
time.sleep(2)
assert "Listener for tcp://satmottag2:9010 switched off: OMG, they killed the listener!" in caplog.text
finally:
chain.stop()