def check_validity(is_valid):
if is_valid:
log.debug("Event not previously seen")
return True
else:
log.debug("Event HAS been previously seen")
raise Exception("Previously seen by this broker")
def stringReceived(self, data):
"""
Called when we receive a string.
The sender should only ever receive an ack or a nak, after which it
should disconnect.
"""
log.debug("Got response from %s" % str(self.transport.getPeer()))
try:
incoming = xml_document(data)
if incoming.get('role') == "ack":
log.info("Acknowledgement received from %s" % str(self.transport.getPeer()))
self.factory.ack = True
elif incoming.get('role') == "nak":
log.warn("Nak received: %s refused to accept VOEvent (%s)" %
(
str(self.transport.getPeer()),
incoming.findtext("Meta/Result", default="no reason given")
)
)
else:
log.warn(
"Incomprehensible data received from %s (role=%s)" %
(self.transport.getPeer(), incoming.get("role"))
)
except ParseError:
log.warn("Unparsable message received from %s" % str(self.transport.getPeer()))
finally:
# After receiving a message, we shut down the connection.
self.transport.loseConnection()
def stringReceived(self, data):
"""
Called when we receive a string.
The sender should only ever receive an ack or a nak, after which it
should disconnect.
"""
log.debug("Got response from %s" % str(self.transport.getPeer()))
try:
incoming = xml_document.infer_type(data)
except ParseError:
log.warn(f"Unparsable message received from "
f"{self.transport.getPeer()}")
return
if incoming.role not in ("ack", "nak"):
log.warn(f"Unexpected {incoming.role} received "
f"from {self.transport.getPeer()}")
return
try:
d = self._sent_ivoids.pop(incoming.origin)
except KeyError:
log.warn(f"Received a receipt for {incoming.origin}, "
f"which is unknown to us")
return
d.callback(incoming)
def stringReceived(self, data):
"""
Called when we receive a string.
The sender should only ever receive an ack or a nak, after which it
should disconnect.
"""
log.debug("Got response from %s" % str(self.transport.getPeer()))
try:
incoming = xml_document(data)
if incoming.element.get('role') == "ack":
log.info("Acknowledgement received from %s" %
str(self.transport.getPeer()))
self.factory.ack = True
elif incoming.element.get('role') == "nak":
log.warn("Nak received: %s refused to accept VOEvent (%s)" %
(str(self.transport.getPeer()),
incoming.element.findtext(
"Meta/Result", default="no reason given")))
else:
log.warn(
"Incomprehensible data received from %s (role=%s)" %
(self.transport.getPeer(), incoming.element.get("role")))
except ParseError:
log.warn("Unparsable message received from %s" %
str(self.transport.getPeer()))
finally:
# After receiving a message, we shut down the connection.
self.transport.loseConnection()
def stringReceived(self, data):
"""
Called when a complete new message is received.
"""
try:
incoming = xml_document(data)
except ParseError:
log.warn("Unparsable message received")
return
# Reset the timeout counter and wait another 120 seconds before
# disconnecting due to inactivity.
self.resetTimeout()
# The root element of both VOEvent and Transport packets has a
# "role" element which we use to identify the type of message we
# have received.
if incoming.element.get("role") == "iamalive":
log.debug("IAmAlive received from %s" % str(self.transport.getPeer()))
self.send_xml(iamaliveresponse(self.factory.local_ivo, incoming.element.find("Origin").text))
elif incoming.element.get("role") == "authenticate":
log.debug("Authenticate received from %s" % str(self.transport.getPeer()))
self.send_xml(
authenticateresponse(self.factory.local_ivo, incoming.element.find("Origin").text, self.filters)
)
elif incoming.element.get("role") in VOEVENT_ROLES:
log.info("VOEvent %s received from %s" % (incoming.element.attrib["ivorn"], str(self.transport.getPeer())))
# We don't send a NAK even if the event is invalid since we don't
# want to be removed from upstream's distribution list.
self.process_event(incoming, can_nak=False)
else:
log.warn(
"Incomprehensible data received from %s (role=%s)"
% (self.transport.getPeer(), incoming.element.get("role"))
)
def stringReceived(self, data):
"""
Called when we receive a string.
The sender should only ever receive an ack or a nak, after which it
should disconnect.
"""
log.debug("Got response from %s" % str(self.transport.getPeer()))
try:
incoming = xml_document.infer_type(data)
except ParseError:
log.warn(f"Unparsable message received from "
f"{self.transport.getPeer()}")
return
if incoming.role not in ("ack", "nak"):
log.warn(f"Unexpected {incoming.role} received "
f"from {self.transport.getPeer()}")
return
try:
d = self._sent_ivoids.pop(incoming.origin)
except KeyError:
log.warn(f"Received a receipt for {incoming.origin}, "
f"which is unknown to us")
return
d.callback(incoming)
def check_validity(is_valid):
if is_valid:
log.debug("Event not previously seen")
return True
else:
log.debug("Event HAS been previously seen")
raise Exception("Previously seen by this broker")
def parse_ivoid(ivoid):
"""
Takes an IVOID of the form
ivo://[authorityID][resourceKey]#[local_ID]
and returns (authorityID, resourceKey, local_ID). Raise if that isn't
possible.
Note that the resourceKey will normally start with a slash. This is part
of the key, and this function will not trim it.
Refer to the IVOA Identifiers Recommendation (2.0) for details.
"""
try:
groups = IVOID_RE.match(ivoid).groups()
# An empty resource name is valid.
rsrc = groups[1] if groups[1] is not None else ""
# These may not appear in the resource key per IVOA Identifiers
# Version 2.0 \S2.3.3.
for forbidden in ['//', '/../', '/./']:
assert (forbidden not in rsrc)
assert (not rsrc.endswith('/'))
return groups[0], rsrc, groups[2]
except (AttributeError, AssertionError) as e:
log.debug("Failed to parse as IVOID: ", str(e))
raise BadIvoidError("Invalid IVOID: %s" % (ivoid, ))
def parse_ivoid(ivoid):
"""
Takes an IVOID of the form
ivo://[authorityID][resourceKey]#[local_ID]
and returns (authorityID, resourceKey, local_ID). Raise if that isn't
possible.
Note that the resourceKey will normally start with a slash. This is part
of the key, and this function will not trim it.
Refer to the IVOA Identifiers Recommendation (2.0) for details.
"""
try:
groups = IVOID_RE.match(ivoid).groups()
# An empty resource name is valid.
rsrc = groups[1] if groups[1] is not None else ""
# These may not appear in the resource key per IVOA Identifiers
# Version 2.0 \S2.3.3.
for forbidden in ['//', '/../', '/./']:
assert(forbidden not in rsrc)
assert(not rsrc.endswith('/'))
return groups[0], rsrc, groups[2]
except (AttributeError, AssertionError) as e:
log.debug("Failed to parse as IVOID: ", str(e))
raise BadIvoidError("Invalid IVOID: %s" % (ivoid,))
def handle_valid(status):
log.debug("Event accepted; sending ACK to %s" %
(self.transport.getPeer()))
self.send_xml(
ack(self.factory.local_ivo, event.element.attrib['ivorn']))
self.handle_event(event).addCallbacks(
lambda x: log.debug("Event processed"),
lambda x: log.warn("Event handlers failed"))
def handle_valid(status):
log.debug("Event accepted; sending ACK to %s" % (self.transport.getPeer()))
self.send_xml(TransportMessage.ack(self.factory.local_ivo,
event.element.attrib['ivorn']))
self.handle_event(event).addCallbacks(
lambda x: log.debug("Event processed"),
lambda x: log.warn("Event handlers failed")
)
def __call__(self, event):
"""
Save an event to disk.
"""
if not os.path.exists(self.directory):
os.makedirs(self.directory)
with event_file(event.element.attrib['ivorn'], self.directory) as f:
log.debug("Writing to %s" % (f.name, ))
f.write(event.raw_bytes.decode(event.encoding))
def __call__(self, event):
"""
Save an event to disk.
"""
if not os.path.exists(self.directory):
os.makedirs(self.directory)
with event_file(event.element.attrib['ivorn'], self.directory) as f:
log.debug("Writing to %s" % (f.name,))
f.write(event.raw_bytes.decode(event.encoding))
def handle_invalid(failure):
log.info("Event rejected (%s); discarding" % (failure.value.subFailure.getErrorMessage(),))
if can_nak:
log.debug("Sending NAK to %s" % (self.transport.getPeer()))
self.send_xml(TransportMessage.nak(self.factory.local_ivo,
event.element.attrib['ivorn'],
"Event rejected: %s" % (failure.value.subFailure.getErrorMessage(),)))
else:
log.debug("Sending ACK to %s" % (self.transport.getPeer()))
self.send_xml(TransportMessage.ack(self.factory.local_ivo,
event.element.attrib['ivorn']))
def __call__(self, event):
"""
Add an event to the celery processing queue
"""
log.debug("Passing to celery...")
try:
process_voevent_celerytask.delay(event.raw_bytes)
ingest_voevent_celerytask.delay(event.raw_bytes)
except Exception as e:
self.deferred.errback(e)
log.debug("Celery jobs sent OK.")
def handle_invalid(failure):
log.info("Event rejected (%s); discarding" %
(failure.value.subFailure.getErrorMessage(), ))
if can_nak:
log.debug("Sending NAK to %s" % (self.transport.getPeer()))
self.send_xml(
nak(
self.factory.local_ivo, event.element.attrib['ivorn'],
"Event rejected: %s" %
(failure.value.subFailure.getErrorMessage(), )))
else:
log.debug("Sending ACK to %s" % (self.transport.getPeer()))
self.send_xml(
ack(self.factory.local_ivo, event.element.attrib['ivorn']))
def stringReceived(self, data):
"""
Called when a complete new message is received.
"""
try:
incoming = xml_document(data)
except ParseError:
log.warn("Unparsable message received")
return
# Reset the timeout counter and wait another 120 seconds before
# disconnecting due to inactivity.
self.resetTimeout()
# The root element of both VOEvent and Transport packets has a
# "role" element which we use to identify the type of message we
# have received.
if incoming.element.get('role') == "iamalive":
log.debug("IAmAlive received from %s" % str(self.transport.getPeer()))
self.send_xml(
iamaliveresponse(self.factory.local_ivo,
incoming.element.find('Origin').text)
)
elif incoming.element.get('role') == "authenticate":
log.debug("Authenticate received from %s" % str(self.transport.getPeer()))
self.send_xml(
authenticateresponse(
self.factory.local_ivo,
incoming.element.find('Origin').text,
self.filters
)
)
elif incoming.element.get('role') in VOEVENT_ROLES:
log.info(
"VOEvent %s received from %s" % (
incoming.element.attrib['ivorn'],
str(self.transport.getPeer())
)
)
# We don't send a NAK even if the event is invalid since we don't
# want to be removed from upstream's distribution list.
self.process_event(incoming, can_nak=False)
else:
log.warn(
"Incomprehensible data received from %s (role=%s)" %
(self.transport.getPeer(), incoming.element.get("role"))
)
def __call__(self, event):
"""
Add an event to the celery processing queue
"""
log.debug("Passing to voeventdb: %s" % (event.attrib['ivorn'],))
session = Session(bind=dbengine)
try:
v = voeventparse.loads(event.text)
dbconv.safe_insert_voevent(session, v)
session.commit()
except Exception as e:
log.warn("Could not insert packet with ivorn {} into database {}".format(
v.attrib['ivorn'], voeventdb_dbname))
self.deferred.errback(e)
log.info("Loaded {} into database {}".format(
v.attrib['ivorn'], voeventdb_dbname))
def parse_ivorn(ivorn):
"""
Takes an IVORN of the form
ivo://authorityID/resourceKey#local_ID
and returns (authorityID, resourceKey, local_ID). Raise if that isn't
possible.
Refer to the IVOA Identifiers Recommendation (1.12) for justification, but
note that document is not as clear as unambiguous as one might hope. We
have assumed that anything which is not explicitly permitted is forbitten
in the authority and the resource name, while anything which would be
permitted in an RFC-3986 URI is permitted in the fragment.
"""
try:
return IVORN_RE.match(ivorn).groups()
except AttributeError, e:
log.debug("Failed to parse as IVORN: ", str(e))
raise Exception("Invalid IVORN: %s" % (ivorn,))
def stringReceived(self, data):
try:
incoming = TransportMessage(data)
except ParseError:
log.warn("Unparsable message received")
return
if incoming.role == "iamalive":
log.debug("IAmAlive received from %s" %
str(self.transport.getPeer()))
self.alive_count -= 1
elif incoming.role == "ack":
log.debug("Ack received from %s" % str(self.transport.getPeer()))
self.outstanding_ack -= 1
elif incoming.role == "nak":
log.info("Nak received from %s; terminating" %
str(self.transport.getPeer()))
self.transport.loseConnection()
elif incoming.role == "authenticate":
log.debug("Authentication received from %s" %
str(self.transport.getPeer()))
self.filters = []
# Accept both "new-style" (<Param type="xpath-filter" />) and
# old-style (<filter type="xpath" />) filters.
for xpath in chain([
elem.get('value') for elem in incoming.element.findall(
"Meta/Param[@name=\"xpath-filter\"]")
], [
elem.text for elem in incoming.element.findall(
"Meta/filter[@type=\"xpath\"]")
]):
log.info("Installing filter %s for %s" %
(xpath, str(self.transport.getPeer())))
try:
self.filters.append(ElementTree.XPath(xpath))
except ElementTree.XPathSyntaxError:
log.info("Filter %s is not valid XPath" % (xpath, ))
else:
log.warn("Incomprehensible data received from %s (role=%s)" %
(self.transport.getPeer(), incoming.role))
def stringReceived(self, data):
try:
incoming = xml_document(data)
except ParseError:
log.warn("Unparsable message received")
return
if incoming.get('role') == "iamalive":
log.debug("IAmAlive received from %s" % str(self.transport.getPeer()))
self.alive_count -= 1
elif incoming.get('role') == "ack":
log.debug("Ack received from %s" % str(self.transport.getPeer()))
self.outstanding_ack -= 1
elif incoming.get('role') == "nak":
log.info("Nak received from %s; terminating" % str(self.transport.getPeer()))
self.transport.loseConnection()
elif incoming.get('role') == "authenticate":
log.debug("Authentication received from %s" % str(self.transport.getPeer()))
self.filters = []
# Accept both "new-style" (<Param type="xpath-filter" />) and
# old-style (<filter type="xpath" />) filters.
for xpath in chain(
[elem.get('value') for elem in incoming.findall("Meta/Param[@name=\"xpath-filter\"]")],
[elem.text for elem in incoming.findall("Meta/filter[@type=\"xpath\"]")]
):
log.info(
"Installing filter %s for %s" %
(xpath, str(self.transport.getPeer()))
)
try:
self.filters.append(ElementTree.XPath(xpath))
except ElementTree.XPathSyntaxError:
log.info("Filter %s is not valid XPath" % (xpath,))
else:
log.warn(
"Incomprehensible data received from %s (role=%s)" %
(self.transport.getPeer(), incoming.get("role"))
)
def sendTestEvent(self):
log.debug("Broadcasting test event")
test_event = broker_test_message(self.local_ivo)
for broadcaster in self.broadcasters:
broadcaster.send_event(test_event)
def outReceived(self, data):
log.debug("External process said: %s" %
(data.decode(self.STDOUT_ENCODING), ))
def test_log_debug_level_warning(self):
log.LEVEL = log.Levels.WARNING
self.assertFalse(self.observer.messages)
d = log.debug(DUMMY_MESSAGE)
self.assertIsInstance(d, defer.Deferred)
return d.addCallback(self._check_log_empty)
def test_log_debug_level_warning(self):
log.LEVEL = log.Levels.WARNING
self.assertFalse(self.observer.messages)
d = log.debug(DUMMY_MESSAGE)
self.assertIsInstance(d, defer.Deferred)
return d.addCallback(self._check_log_empty)
def sendIAmAlive(self):
log.debug("Broadcasting iamalive")
for broadcaster in self.broadcasters:
broadcaster.sendIAmAlive()
def sendTestEvent(self):
log.debug("Broadcasting test event")
test_event = VOEventMessage.broker_test(self.local_ivo)
for broadcaster in self.broadcasters:
broadcaster.send_event(test_event)
def outReceived(self, data):
log.debug("External process said: %s" %
(data.decode(self.STDOUT_ENCODING),))
def sendIAmAlive(self):
log.debug("Broadcasting iamalive")
for broadcaster in self.broadcasters:
broadcaster.sendIAmAlive()