def add_to_file_cache(msg):
"""Add files in the message to received file cache."""
with cache_lock:
for uid in gen_dict_extract(msg.data, 'uid'):
if uid not in file_cache:
LOGGER.debug("Add %s to file cache", str(uid))
file_cache.append(uid)
def ack(self, message):
"""Reply with ack to a publication
"""
for url in gen_dict_extract(message.data, 'uri'):
uri = urlparse(url)
pathname = uri.path
if 'origin' in self._attrs and not fnmatch.fnmatch(
os.path.basename(pathname),
os.path.basename(globify(self._attrs["origin"]))):
LOGGER.warning('Client trying to get invalid file: %s',
pathname)
return Message(message.subject,
"err",
data="{0:s} not reacheable".format(pathname))
if (self._attrs.get('compression') or self._attrs.get(
'delete', 'False').lower() in ["1", "yes", "true", "on"]):
self._deleter.add(pathname)
new_msg = Message(message.subject, "ack", data=message.data.copy())
try:
new_msg.data['destination'] = clean_url(
new_msg.data['destination'])
except KeyError:
pass
return new_msg
def get_msg_uid(msg):
"""Compute the uid of the message."""
filenames = sorted(gen_dict_extract(msg.data, 'uid'))
m = hashlib.md5()
for filename in filenames:
m.update(filename.encode('utf-8'))
return m.hexdigest()
def run(self):
with Subscribe('', topics=self.attrs['listen'],
addr_listener=True) as sub:
for msg in sub.recv(1):
if msg is None:
if not self.loop:
break
else:
continue
# check that files are local
for uri in gen_dict_extract(msg.data, 'uri'):
urlobj = urlparse(uri)
if (urlobj.scheme not in ['', 'file']
and not socket.gethostbyname(
urlobj.netloc) in get_local_ips()):
break
else:
LOGGER.debug('We have a match: %s', str(msg))
#pathname = unpack(orig_pathname, **attrs)
info = self.attrs.get("info", {})
if info:
info = dict((elt.strip().split('=')
for elt in info.split(";")))
for infokey, infoval in info.items():
if "," in infoval:
info[infokey] = infoval.split(",")
# info.update(parse(attrs["origin"], orig_pathname))
# info['uri'] = pathname
# info['uid'] = os.path.basename(pathname)
info.update(msg.data)
info['request_address'] = self.attrs.get(
"request_address",
get_own_ip()) + ":" + self.attrs["request_port"]
old_data = msg.data
msg = Message(self.attrs["topic"], msg.type, info)
self.publisher.send(str(msg))
with file_cache_lock:
for filename in gen_dict_extract(old_data, 'uid'):
file_cache.appendleft(self.attrs["topic"] + '/' +
filename)
LOGGER.debug("Message sent: " + str(msg))
if not self.loop:
break
def already_received(msg):
"""Check if the files from msg already are in the local cache."""
with cache_lock:
for filename in gen_dict_extract(msg.data, 'uid'):
if filename not in file_cache:
return False
else:
return True
def resend_if_local(msg, publisher):
"""Resend the message provided all uris point to local files."""
for uri in gen_dict_extract(msg.data, 'uri'):
urlobj = urlparse(uri)
if not publisher or not socket.gethostbyname(urlobj.netloc) in get_local_ips():
return
else:
LOGGER.debug('Sending: %s', str(msg))
publisher.send(str(msg))
def test_gen_dict_extract_message(self):
"""Test extracting info from message data dictionary."""
orig = (
'pytroll://tm1 file [email protected] 2018-10-25T01:15:54.752065 v1.01 application/json '
'{"sensor": "viirs", "format": "SDR", "variant": "DR", "uid": "bla.png", "uri": "/home/user/bla.png"}'
)
expected_uid = ['bla.png']
res = list(gen_dict_extract(Message(rawstr=orig).data, 'uid'))
self.assertListEqual(expected_uid, res)
def test_translate_dict_value(self):
"""Test message data dictionary value translation."""
msg = Message(rawstr=test_msg)
orig_data = copy.deepcopy(msg.data)
res = translate_dict_value(msg.data, 'uri',
dummy_callback_translate_dict_values)
for uri in gen_dict_extract(res, 'uri'):
self.assertTrue(uri.startswith(DEST_DIR))
self.assertDictEqual(orig_data, msg.data)
def request_push(msg,
destination,
login,
publisher=None,
unpack=None,
delete=False,
**kwargs):
if already_received(msg):
resend_if_local(msg, publisher)
mtype = 'ack'
req = Message(msg.subject, mtype, data=msg.data)
LOGGER.debug("Sending: %s", str(req))
timeout = float(kwargs["req_timeout"])
else:
mtype = 'push'
req, fake_req = create_push_req_message(msg, destination, login)
LOGGER.info("Requesting: %s", str(fake_req))
timeout = float(kwargs["transfer_req_timeout"])
local_dir = create_local_dir(destination, kwargs.get('ftp_root', '/'))
LOGGER.debug("Send and recv timeout is %.2f seconds", timeout)
hostname, port = msg.data["request_address"].split(":")
requester = PushRequester(hostname, int(port))
response = requester.send_and_recv(req, timeout=timeout)
if response and response.type in ['file', 'collection', 'dataset']:
LOGGER.debug("Server done sending file")
with cache_lock:
for uid in gen_dict_extract(msg.data, 'uid'):
file_cache.append(uid)
try:
lmsg = unpack_and_create_local_message(response, local_dir, unpack,
delete)
except IOError:
LOGGER.exception("Couldn't unpack %s", str(response))
return
if publisher:
lmsg = make_uris(lmsg, destination, login)
lmsg.data['origin'] = response.data['request_address']
lmsg.data.pop('request_address', None)
lmsg = replace_mda(lmsg, kwargs)
lmsg.data.pop('destination', None)
LOGGER.debug("publishing %s", str(lmsg))
publisher.send(str(lmsg))
elif response and response.type == "ack":
pass
else:
LOGGER.error("Failed to get valid response from server %s: %s",
str(hostname), str(response))
def ack(self, message):
"""Reply with ack to a publication."""
new_msg = None
for url in gen_dict_extract(message.data, 'uri'):
pathname = urlparse(url).path
new_msg = self._validate_requested_file(pathname, message)
if new_msg is not None:
break
self._add_to_deleter(pathname)
if new_msg is None:
new_msg = _get_cleaned_ack_message(message)
return new_msg
def test_real_translate(self):
msg = Message(rawstr=test_msg)
orig_data = copy.deepcopy(msg.data)
def dummy_cb(k, v):
dirname, filename = os.path.split(v)
return os.path.join(dest_dir, filename)
dest_dir = '/tmp'
res = translate_dict_value(msg.data, 'uri', dummy_cb)
for uri in gen_dict_extract(res, 'uri'):
self.assertTrue(uri.startswith(dest_dir))
self.assertDictEqual(orig_data, msg.data)
def test_gen_dict_extract_dictionary(self):
"""Test extracting info from dictionary."""
test_dict = {
'sensor':
'viirs',
'format':
'SDR',
'variant':
'DR',
'start_time':
datetime.datetime(2018, 10, 25, 1, 1, 46),
'orbit_number':
36230,
'dataset': [{
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/GMTCO_npp_d20181025_t0101464_e0103106_b36230_c20181025011335298494_cspp_dev.h5',
'uid':
'GMTCO_npp_d20181025_t0101464_e0103106_b36230_c20181025011335298494_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVM01_npp_d20181025_t0101464_e0103106_b36230_c20181025011354163052_cspp_dev.h5',
'uid':
'SVM01_npp_d20181025_t0101464_e0103106_b36230_c20181025011354163052_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVM02_npp_d20181025_t0101464_e0103106_b36230_c20181025011354178693_cspp_dev.h5',
'uid':
'SVM02_npp_d20181025_t0101464_e0103106_b36230_c20181025011354178693_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVM03_npp_d20181025_t0101464_e0103106_b36230_c20181025011354194042_cspp_dev.h5',
'uid':
'SVM03_npp_d20181025_t0101464_e0103106_b36230_c20181025011354194042_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVM04_npp_d20181025_t0101464_e0103106_b36230_c20181025011354209273_cspp_dev.h5',
'uid':
'SVM04_npp_d20181025_t0101464_e0103106_b36230_c20181025011354209273_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVM05_npp_d20181025_t0101464_e0103106_b36230_c20181025011354224550_cspp_dev.h5',
'uid':
'SVM05_npp_d20181025_t0101464_e0103106_b36230_c20181025011354224550_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVM06_npp_d20181025_t0101464_e0103106_b36230_c20181025011354240108_cspp_dev.h5',
'uid':
'SVM06_npp_d20181025_t0101464_e0103106_b36230_c20181025011354240108_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVM07_npp_d20181025_t0101464_e0103106_b36230_c20181025011354256470_cspp_dev.h5',
'uid':
'SVM07_npp_d20181025_t0101464_e0103106_b36230_c20181025011354256470_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVM08_npp_d20181025_t0101464_e0103106_b36230_c20181025011354291614_cspp_dev.h5',
'uid':
'SVM08_npp_d20181025_t0101464_e0103106_b36230_c20181025011354291614_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVM09_npp_d20181025_t0101464_e0103106_b36230_c20181025011354320585_cspp_dev.h5',
'uid':
'SVM09_npp_d20181025_t0101464_e0103106_b36230_c20181025011354320585_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVM10_npp_d20181025_t0101464_e0103106_b36230_c20181025011354337251_cspp_dev.h5',
'uid':
'SVM10_npp_d20181025_t0101464_e0103106_b36230_c20181025011354337251_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVM11_npp_d20181025_t0101464_e0103106_b36230_c20181025011354366238_cspp_dev.h5',
'uid':
'SVM11_npp_d20181025_t0101464_e0103106_b36230_c20181025011354366238_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVM12_npp_d20181025_t0101464_e0103106_b36230_c20181025011354382899_cspp_dev.h5',
'uid':
'SVM12_npp_d20181025_t0101464_e0103106_b36230_c20181025011354382899_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVM13_npp_d20181025_t0101464_e0103106_b36230_c20181025011354407042_cspp_dev.h5',
'uid':
'SVM13_npp_d20181025_t0101464_e0103106_b36230_c20181025011354407042_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVM14_npp_d20181025_t0101464_e0103106_b36230_c20181025011354448503_cspp_dev.h5',
'uid':
'SVM14_npp_d20181025_t0101464_e0103106_b36230_c20181025011354448503_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVM15_npp_d20181025_t0101464_e0103106_b36230_c20181025011354478025_cspp_dev.h5',
'uid':
'SVM15_npp_d20181025_t0101464_e0103106_b36230_c20181025011354478025_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVM16_npp_d20181025_t0101464_e0103106_b36230_c20181025011354506942_cspp_dev.h5',
'uid':
'SVM16_npp_d20181025_t0101464_e0103106_b36230_c20181025011354506942_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/GITCO_npp_d20181025_t0101464_e0103106_b36230_c20181025011333965280_cspp_dev.h5',
'uid':
'GITCO_npp_d20181025_t0101464_e0103106_b36230_c20181025011333965280_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVI01_npp_d20181025_t0101464_e0103106_b36230_c20181025011353975082_cspp_dev.h5',
'uid':
'SVI01_npp_d20181025_t0101464_e0103106_b36230_c20181025011353975082_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVI02_npp_d20181025_t0101464_e0103106_b36230_c20181025011353990747_cspp_dev.h5',
'uid':
'SVI02_npp_d20181025_t0101464_e0103106_b36230_c20181025011353990747_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVI03_npp_d20181025_t0101464_e0103106_b36230_c20181025011354006115_cspp_dev.h5',
'uid':
'SVI03_npp_d20181025_t0101464_e0103106_b36230_c20181025011354006115_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVI04_npp_d20181025_t0101464_e0103106_b36230_c20181025011354022377_cspp_dev.h5',
'uid':
'SVI04_npp_d20181025_t0101464_e0103106_b36230_c20181025011354022377_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVI05_npp_d20181025_t0101464_e0103106_b36230_c20181025011354093439_cspp_dev.h5',
'uid':
'SVI05_npp_d20181025_t0101464_e0103106_b36230_c20181025011354093439_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/GDNBO_npp_d20181025_t0101464_e0103106_b36230_c20181025011333695023_cspp_dev.h5',
'uid':
'GDNBO_npp_d20181025_t0101464_e0103106_b36230_c20181025011333695023_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVDNB_npp_d20181025_t0101464_e0103106_b36230_c20181025011353771298_cspp_dev.h5',
'uid':
'SVDNB_npp_d20181025_t0101464_e0103106_b36230_c20181025011353771298_cspp_dev.h5'
}],
'platform_name':
'Suomi-NPP',
'orig_orbit_number':
36230,
'end_time':
datetime.datetime(2018, 10, 25, 1, 3, 10),
'type':
'HDF5',
'data_processing_level':
'1B',
'request_address':
'10.120.1.40:9099'
} # noqa
expected = [
'GMTCO_npp_d20181025_t0101464_e0103106_b36230_c20181025011335298494_cspp_dev.h5',
'SVM01_npp_d20181025_t0101464_e0103106_b36230_c20181025011354163052_cspp_dev.h5',
'SVM02_npp_d20181025_t0101464_e0103106_b36230_c20181025011354178693_cspp_dev.h5',
'SVM03_npp_d20181025_t0101464_e0103106_b36230_c20181025011354194042_cspp_dev.h5',
'SVM04_npp_d20181025_t0101464_e0103106_b36230_c20181025011354209273_cspp_dev.h5',
'SVM05_npp_d20181025_t0101464_e0103106_b36230_c20181025011354224550_cspp_dev.h5',
'SVM06_npp_d20181025_t0101464_e0103106_b36230_c20181025011354240108_cspp_dev.h5',
'SVM07_npp_d20181025_t0101464_e0103106_b36230_c20181025011354256470_cspp_dev.h5',
'SVM08_npp_d20181025_t0101464_e0103106_b36230_c20181025011354291614_cspp_dev.h5',
'SVM09_npp_d20181025_t0101464_e0103106_b36230_c20181025011354320585_cspp_dev.h5',
'SVM10_npp_d20181025_t0101464_e0103106_b36230_c20181025011354337251_cspp_dev.h5',
'SVM11_npp_d20181025_t0101464_e0103106_b36230_c20181025011354366238_cspp_dev.h5',
'SVM12_npp_d20181025_t0101464_e0103106_b36230_c20181025011354382899_cspp_dev.h5',
'SVM13_npp_d20181025_t0101464_e0103106_b36230_c20181025011354407042_cspp_dev.h5',
'SVM14_npp_d20181025_t0101464_e0103106_b36230_c20181025011354448503_cspp_dev.h5',
'SVM15_npp_d20181025_t0101464_e0103106_b36230_c20181025011354478025_cspp_dev.h5',
'SVM16_npp_d20181025_t0101464_e0103106_b36230_c20181025011354506942_cspp_dev.h5',
'GITCO_npp_d20181025_t0101464_e0103106_b36230_c20181025011333965280_cspp_dev.h5',
'SVI01_npp_d20181025_t0101464_e0103106_b36230_c20181025011353975082_cspp_dev.h5',
'SVI02_npp_d20181025_t0101464_e0103106_b36230_c20181025011353990747_cspp_dev.h5',
'SVI03_npp_d20181025_t0101464_e0103106_b36230_c20181025011354006115_cspp_dev.h5',
'SVI04_npp_d20181025_t0101464_e0103106_b36230_c20181025011354022377_cspp_dev.h5',
'SVI05_npp_d20181025_t0101464_e0103106_b36230_c20181025011354093439_cspp_dev.h5',
'GDNBO_npp_d20181025_t0101464_e0103106_b36230_c20181025011333695023_cspp_dev.h5',
'SVDNB_npp_d20181025_t0101464_e0103106_b36230_c20181025011353771298_cspp_dev.h5'
]
self.assertListEqual(expected, list(gen_dict_extract(test_dict,
'uid')))
def _files_in_message_are_local(msg):
for uri in gen_dict_extract(msg.data, 'uri'):
urlobj = urlparse(uri)
if not is_file_local(urlobj):
return False
return True
def _add_files_to_cache(self, msg):
with file_cache_lock:
for filename in gen_dict_extract(msg.data, 'uid'):
file_cache.appendleft(self.attrs["topic"] + '/' + filename)
def test_extractor(self):
"""Test extracting info from dictionary."""
orig = (
'pytroll://tm1 file [email protected] 2018-10-25T01:15:54.752065 v1.01 application/json '
'{"sensor": "viirs", "format": "SDR", "variant": "DR", "uid": "bla.png", "uri": "/home/user/bla.png"}'
)
expected_uid = ['bla.png']
res = list(gen_dict_extract(Message(rawstr=orig).data, 'uid'))
self.assertListEqual(expected_uid, res)
test_dict = {
'sensor':
'viirs',
'format':
'SDR',
'variant':
'DR',
'start_time':
datetime.datetime(2018, 10, 25, 1, 1, 46),
'orbit_number':
36230,
'dataset': [{
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/GMTCO_npp_d20181025_t0101464_e0103106_b36230_c20181025011335298494_cspp_dev.h5',
'uid':
'GMTCO_npp_d20181025_t0101464_e0103106_b36230_c20181025011335298494_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVM01_npp_d20181025_t0101464_e0103106_b36230_c20181025011354163052_cspp_dev.h5',
'uid':
'SVM01_npp_d20181025_t0101464_e0103106_b36230_c20181025011354163052_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVM02_npp_d20181025_t0101464_e0103106_b36230_c20181025011354178693_cspp_dev.h5',
'uid':
'SVM02_npp_d20181025_t0101464_e0103106_b36230_c20181025011354178693_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVM03_npp_d20181025_t0101464_e0103106_b36230_c20181025011354194042_cspp_dev.h5',
'uid':
'SVM03_npp_d20181025_t0101464_e0103106_b36230_c20181025011354194042_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVM04_npp_d20181025_t0101464_e0103106_b36230_c20181025011354209273_cspp_dev.h5',
'uid':
'SVM04_npp_d20181025_t0101464_e0103106_b36230_c20181025011354209273_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVM05_npp_d20181025_t0101464_e0103106_b36230_c20181025011354224550_cspp_dev.h5',
'uid':
'SVM05_npp_d20181025_t0101464_e0103106_b36230_c20181025011354224550_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVM06_npp_d20181025_t0101464_e0103106_b36230_c20181025011354240108_cspp_dev.h5',
'uid':
'SVM06_npp_d20181025_t0101464_e0103106_b36230_c20181025011354240108_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVM07_npp_d20181025_t0101464_e0103106_b36230_c20181025011354256470_cspp_dev.h5',
'uid':
'SVM07_npp_d20181025_t0101464_e0103106_b36230_c20181025011354256470_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVM08_npp_d20181025_t0101464_e0103106_b36230_c20181025011354291614_cspp_dev.h5',
'uid':
'SVM08_npp_d20181025_t0101464_e0103106_b36230_c20181025011354291614_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVM09_npp_d20181025_t0101464_e0103106_b36230_c20181025011354320585_cspp_dev.h5',
'uid':
'SVM09_npp_d20181025_t0101464_e0103106_b36230_c20181025011354320585_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVM10_npp_d20181025_t0101464_e0103106_b36230_c20181025011354337251_cspp_dev.h5',
'uid':
'SVM10_npp_d20181025_t0101464_e0103106_b36230_c20181025011354337251_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVM11_npp_d20181025_t0101464_e0103106_b36230_c20181025011354366238_cspp_dev.h5',
'uid':
'SVM11_npp_d20181025_t0101464_e0103106_b36230_c20181025011354366238_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVM12_npp_d20181025_t0101464_e0103106_b36230_c20181025011354382899_cspp_dev.h5',
'uid':
'SVM12_npp_d20181025_t0101464_e0103106_b36230_c20181025011354382899_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVM13_npp_d20181025_t0101464_e0103106_b36230_c20181025011354407042_cspp_dev.h5',
'uid':
'SVM13_npp_d20181025_t0101464_e0103106_b36230_c20181025011354407042_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVM14_npp_d20181025_t0101464_e0103106_b36230_c20181025011354448503_cspp_dev.h5',
'uid':
'SVM14_npp_d20181025_t0101464_e0103106_b36230_c20181025011354448503_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVM15_npp_d20181025_t0101464_e0103106_b36230_c20181025011354478025_cspp_dev.h5',
'uid':
'SVM15_npp_d20181025_t0101464_e0103106_b36230_c20181025011354478025_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVM16_npp_d20181025_t0101464_e0103106_b36230_c20181025011354506942_cspp_dev.h5',
'uid':
'SVM16_npp_d20181025_t0101464_e0103106_b36230_c20181025011354506942_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/GITCO_npp_d20181025_t0101464_e0103106_b36230_c20181025011333965280_cspp_dev.h5',
'uid':
'GITCO_npp_d20181025_t0101464_e0103106_b36230_c20181025011333965280_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVI01_npp_d20181025_t0101464_e0103106_b36230_c20181025011353975082_cspp_dev.h5',
'uid':
'SVI01_npp_d20181025_t0101464_e0103106_b36230_c20181025011353975082_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVI02_npp_d20181025_t0101464_e0103106_b36230_c20181025011353990747_cspp_dev.h5',
'uid':
'SVI02_npp_d20181025_t0101464_e0103106_b36230_c20181025011353990747_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVI03_npp_d20181025_t0101464_e0103106_b36230_c20181025011354006115_cspp_dev.h5',
'uid':
'SVI03_npp_d20181025_t0101464_e0103106_b36230_c20181025011354006115_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVI04_npp_d20181025_t0101464_e0103106_b36230_c20181025011354022377_cspp_dev.h5',
'uid':
'SVI04_npp_d20181025_t0101464_e0103106_b36230_c20181025011354022377_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVI05_npp_d20181025_t0101464_e0103106_b36230_c20181025011354093439_cspp_dev.h5',
'uid':
'SVI05_npp_d20181025_t0101464_e0103106_b36230_c20181025011354093439_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/GDNBO_npp_d20181025_t0101464_e0103106_b36230_c20181025011333695023_cspp_dev.h5',
'uid':
'GDNBO_npp_d20181025_t0101464_e0103106_b36230_c20181025011333695023_cspp_dev.h5'
}, {
'uri':
'ssh://lxserv1131.smhi.se/san1/polar_in/direct_readout/npp/lvl1/npp_20181025_0048_36230/SVDNB_npp_d20181025_t0101464_e0103106_b36230_c20181025011353771298_cspp_dev.h5',
'uid':
'SVDNB_npp_d20181025_t0101464_e0103106_b36230_c20181025011353771298_cspp_dev.h5'
}],
'platform_name':
'Suomi-NPP',
'orig_orbit_number':
36230,
'end_time':
datetime.datetime(2018, 10, 25, 1, 3, 10),
'type':
'HDF5',
'data_processing_level':
'1B',
'request_address':
'10.120.1.40:9099'
} # noqa
expected = [
'GMTCO_npp_d20181025_t0101464_e0103106_b36230_c20181025011335298494_cspp_dev.h5',
'SVM01_npp_d20181025_t0101464_e0103106_b36230_c20181025011354163052_cspp_dev.h5',
'SVM02_npp_d20181025_t0101464_e0103106_b36230_c20181025011354178693_cspp_dev.h5',
'SVM03_npp_d20181025_t0101464_e0103106_b36230_c20181025011354194042_cspp_dev.h5',
'SVM04_npp_d20181025_t0101464_e0103106_b36230_c20181025011354209273_cspp_dev.h5',
'SVM05_npp_d20181025_t0101464_e0103106_b36230_c20181025011354224550_cspp_dev.h5',
'SVM06_npp_d20181025_t0101464_e0103106_b36230_c20181025011354240108_cspp_dev.h5',
'SVM07_npp_d20181025_t0101464_e0103106_b36230_c20181025011354256470_cspp_dev.h5',
'SVM08_npp_d20181025_t0101464_e0103106_b36230_c20181025011354291614_cspp_dev.h5',
'SVM09_npp_d20181025_t0101464_e0103106_b36230_c20181025011354320585_cspp_dev.h5',
'SVM10_npp_d20181025_t0101464_e0103106_b36230_c20181025011354337251_cspp_dev.h5',
'SVM11_npp_d20181025_t0101464_e0103106_b36230_c20181025011354366238_cspp_dev.h5',
'SVM12_npp_d20181025_t0101464_e0103106_b36230_c20181025011354382899_cspp_dev.h5',
'SVM13_npp_d20181025_t0101464_e0103106_b36230_c20181025011354407042_cspp_dev.h5',
'SVM14_npp_d20181025_t0101464_e0103106_b36230_c20181025011354448503_cspp_dev.h5',
'SVM15_npp_d20181025_t0101464_e0103106_b36230_c20181025011354478025_cspp_dev.h5',
'SVM16_npp_d20181025_t0101464_e0103106_b36230_c20181025011354506942_cspp_dev.h5',
'GITCO_npp_d20181025_t0101464_e0103106_b36230_c20181025011333965280_cspp_dev.h5',
'SVI01_npp_d20181025_t0101464_e0103106_b36230_c20181025011353975082_cspp_dev.h5',
'SVI02_npp_d20181025_t0101464_e0103106_b36230_c20181025011353990747_cspp_dev.h5',
'SVI03_npp_d20181025_t0101464_e0103106_b36230_c20181025011354006115_cspp_dev.h5',
'SVI04_npp_d20181025_t0101464_e0103106_b36230_c20181025011354022377_cspp_dev.h5',
'SVI05_npp_d20181025_t0101464_e0103106_b36230_c20181025011354093439_cspp_dev.h5',
'GDNBO_npp_d20181025_t0101464_e0103106_b36230_c20181025011333695023_cspp_dev.h5',
'SVDNB_npp_d20181025_t0101464_e0103106_b36230_c20181025011353771298_cspp_dev.h5'
]
self.assertListEqual(expected, list(gen_dict_extract(test_dict,
'uid')))
