def _publish(self, metadata):
if "dataset" in metadata:
msg = pmessage.Message(self._subject, "dataset", metadata)
else:
msg = pmessage.Message(self._subject, "collection", metadata)
logger.info("Sending: %s", str(msg))
self._publisher.send(str(msg))
def terminator(self, metadata):
"""Dummy terminator function.
"""
sorted_mda = sorted(metadata, key=lambda x: x["start_time"])
mda = metadata[0].copy()
mda['end_time'] = sorted_mda[-1]['end_time']
mda['collection_area_id'] = sorted_mda[-1]['collection_area_id']
mda['collection'] = []
is_correct = False
for meta in sorted_mda:
new_mda = {}
if "uri" in meta or 'dataset' in meta:
is_correct = True
for key in ['dataset', 'uri', 'uid']:
if key in meta:
new_mda[key] = meta[key]
new_mda['start_time'] = meta['start_time']
new_mda['end_time'] = meta['end_time']
mda['collection'].append(new_mda)
for key in ['dataset', 'uri', 'uid']:
if key in mda:
del mda[key]
if is_correct:
msg = message.Message("/placeholder", "collection",
mda)
self.logger.info("Collection ready %s", str(msg))
self.output_queue.put(msg)
else:
self.logger.warning("Malformed metadata, no key: %s", "uri")
def terminator(metadata):
"""Dummy terminator function.
"""
sorted_mda = sorted(metadata, key=lambda x: x["start_time"])
mda = metadata[0].copy()
subject = "/".join(("", mda["format"], mda["data_processing_level"], ''))
mda['end_time'] = sorted_mda[-1]['end_time']
mda['collection'] = []
for meta in sorted_mda:
new_mda = {}
for key in ['dataset', 'uri', 'uid']:
if key in meta:
new_mda[key] = meta[key]
new_mda['start_time'] = meta['start_time']
new_mda['end_time'] = meta['end_time']
mda['collection'].append(new_mda)
for key in ['dataset', 'uri', 'uid']:
if key in mda:
del mda[key]
msg = message.Message(subject, "collection", mda)
logger.info("sending %s", str(msg))
pub.send(str(msg))
def _publish(self, time_slot, missing_files_check=True):
"""Publish file dataset and reinitialize gatherer."""
data = self.slots[time_slot]
# Diagnostic logging about delayed ...
delayed_files = data['delayed_files']
if len(delayed_files) > 0:
file_str = ''
for key in delayed_files:
file_str += "%s %f seconds, " % (key, delayed_files[key])
self.logger.warning("Files received late: %s",
file_str.strip(', '))
if missing_files_check:
# and missing files
missing_files = \
data['all_files'].difference(data['received_files'])
if len(missing_files) > 0:
self.logger.warning("Missing files: %s",
', '.join(missing_files))
# Although we're not publishing a message, generate one anyway
# for compatibility
msg = message.Message("/placeholder", "dataset", data['metadata'])
self.logger.info("Forwarding: %s", str(msg))
self.output_queue.put(msg)
def _publish(self, time_slot, missing_files_check=True):
"""Publish file dataset and reinitialize gatherer."""
data = self.slots[time_slot]
# Diagnostic logging about delayed ...
delayed_files = data['delayed_files']
if len(delayed_files) > 0:
file_str = ''
for key in delayed_files:
file_str += "%s %f seconds, " % (key, delayed_files[key])
self.logger.warning("Files received late: %s",
file_str.strip(', '))
if missing_files_check:
# and missing files
missing_files = data['all_files'].difference(
data['received_files'])
if len(missing_files) > 0:
self.logger.warning("Missing files: %s",
', '.join(missing_files))
# Remove tags that are not necessary for datasets
for tag in REMOVE_TAGS:
try:
del data['metadata'][tag]
except KeyError:
pass
msg = message.Message(self._subject, "dataset", data['metadata'])
self.logger.info("Sending: %s", str(msg))
self._publisher.send(str(msg))
def __init__(self, name, address, data_type, interval=2, nameservers=None):
msg = message.Message("/address/%s" % name, "info", {
"URI": address,
"service": data_type
}).encode()
MessageBroadcaster.__init__(self, msg, broadcast_port, interval,
nameservers)
def _add_existing_files_to_slot(self, slot, fnames, message):
for fname in fnames:
meta = {
"uid": os.path.basename(fname),
"uri": fname,
"sensor": message._posttroll_message.data["sensor"]
}
msg = self.message_from_posttroll(
pmessage.Message(message._posttroll_message.subject, "file",
meta))
slot.add_file(msg)
def publish_collection(self, metadata):
"""Terminate the gathering."""
subject = self._get_topic(metadata[0])
mda = _merge_metadata(metadata)
if mda:
msg = message.Message(subject, "collection", mda)
logger.info("sending %s", str(msg))
self.publisher.send(str(msg))
else:
logger.warning("Malformed metadata, no key: %s", "uri")
def terminator(metadata, publish_topic=None):
"""Dummy terminator function.
"""
sorted_mda = sorted(metadata, key=lambda x: x["start_time"])
mda = metadata[0].copy()
if publish_topic is not None:
LOGGER.info("Composing topic.")
subject = compose(publish_topic, mda)
else:
LOGGER.info("Using default topic.")
subject = "/".join(
("", mda["format"], mda["data_processing_level"], ''))
mda['start_time'] = sorted_mda[0]['start_time']
mda['end_time'] = sorted_mda[-1]['end_time']
mda['collection_area_id'] = sorted_mda[-1]['collection_area_id']
mda['collection'] = []
is_correct = False
for meta in sorted_mda:
new_mda = {}
if "uri" in meta or 'dataset' in meta:
is_correct = True
for key in ['dataset', 'uri', 'uid']:
if key in meta:
new_mda[key] = meta[key]
new_mda['start_time'] = meta['start_time']
new_mda['end_time'] = meta['end_time']
mda['collection'].append(new_mda)
for key in ['dataset', 'uri', 'uid']:
if key in mda:
del mda[key]
if is_correct:
msg = message.Message(subject, "collection", mda)
LOGGER.info("sending %s", str(msg))
PUB.send(str(msg))
else:
LOGGER.warning("Malformed metadata, no key: %s", "uri")
def _publish(self):
"""Publish file dataset and reinitialize gatherer."""
# Diagnostic logging about delayed ...
if len(self.delayed_files) > 0:
file_str = ''
for key in self.delayed_files:
file_str += "%s %f seconds, " % (key, self.delayed_files[key])
self.logger.warning("Files received late: %s",
file_str.strip(', '))
# and missing files
missing_files = self.all_files.difference(self.received_files)
if len(missing_files) > 0:
self.logger.warning("Missing files: %s", ', '.join(missing_files))
msg = message.Message(self._subject, "dataset", self.metadata)
self.logger.info("Sending: %s", str(msg))
self._publisher.send(str(msg))
self._clear_data()
def __init__(self, name, address, interval, nameservers):
msg = message.Message("/address/%s" % name, "info",
{"URI": "%s:%d" % address}).encode()
MessageBroadcaster.__init__(self, msg, broadcast_port, interval,
nameservers)
def test_WorldCompositeDaemon(self):
"""Test WorldCompositeDaemon"""
# Test incoming message handling and saving
# Epoch: message sending time
config = {
"topics": ["/test"],
"area_def": ADEF,
"timeout_epoch": "message",
"timeout": 45,
"num_expected": 5,
"out_pattern": os.path.join(THIS_DIR, "data", "test_out.png")
}
comp = gm.WorldCompositeDaemon(config)
# There should be no slots
self.assertEqual(len(comp.slots), 0)
for i in range(len(self.sat_fnames)):
msg = message.Message(
"/test", "file", {
"uri": self.sat_fnames[i],
"nominal_time": self.tslot,
"productname": "wv"
})
epoch = msg.time
comp._handle_message(msg)
# Number of slots
self.assertEqual(len(comp.slots), 1)
# Number of composites
self.assertEqual(len(comp.slots[self.tslot]), 1)
# Number of files
self.assertEqual(comp.slots[self.tslot]["wv"]["num"], i + 1)
# Timeout
diff = (comp.slots[self.tslot]["wv"]["timeout"] -
(epoch + dt.timedelta(minutes=config["timeout"])))
self.assertAlmostEqual(diff.total_seconds(), 0.0, places=2)
comp._check_timeouts_and_save()
# Saving should not happen before all the images are received
if i < 4:
self.assertEqual(comp.slots[self.tslot]["wv"]["num"], i + 1)
else:
# After fifth image the composite should be saved and
# all composites and slots removed
self.assertEqual(len(comp.slots), 0)
self.assertTrue(os.path.exists(config["out_pattern"]))
# Remove the file
os.remove(config["out_pattern"])
comp.stop()
# Epoch: file nominal time
config = {
"topics": ["/test"],
"area_def": ADEF,
"timeout_epoch": "nominal_time",
"timeout": 45,
"num_expected": 5,
"out_pattern": os.path.join(THIS_DIR, "data", "test_out.png")
}
comp = gm.WorldCompositeDaemon(config)
for i in range(len(self.sat_fnames)):
msg = message.Message(
"/test", "file", {
"uri": self.sat_fnames[i],
"nominal_time": self.tslot,
"productname": "wv"
})
epoch = self.tslot
comp._handle_message(msg)
# Number of slots
self.assertEqual(len(comp.slots), 1)
# Number of files should be one every time
self.assertEqual(comp.slots[self.tslot]["wv"]["num"], 1)
# Timeout
self.assertEqual(comp.slots[self.tslot]["wv"]["timeout"],
(epoch + dt.timedelta(minutes=config["timeout"])))
# Output file should be present after the first run
if i > 0:
self.assertTrue(os.path.exists(config["out_pattern"]))
comp._check_timeouts_and_save()
# There shouldn't be any slots now
self.assertEqual(len(comp.slots), 0)
# Remove the file
os.remove(config["out_pattern"])
# Stop compositor daemon
comp.stop()
