Example #1
0
 def check_validity(is_valid):
     if is_valid:
         log.debug("Event not previously seen")
         return True
     else:
         log.debug("Event HAS been previously seen")
         raise Exception("Previously seen by this broker")
Example #2
0
    def stringReceived(self, data):
        """
        Called when we receive a string.

        The sender should only ever receive an ack or a nak, after which it
        should disconnect.
        """
        log.debug("Got response from %s" % str(self.transport.getPeer()))
        try:
            incoming = xml_document(data)

            if incoming.get('role') == "ack":
                log.info("Acknowledgement received from %s" % str(self.transport.getPeer()))
                self.factory.ack = True
            elif incoming.get('role') == "nak":
                log.warn("Nak received: %s refused to accept VOEvent (%s)" %
                    (
                        str(self.transport.getPeer()),
                        incoming.findtext("Meta/Result", default="no reason given")
                    )
                )
            else:
                log.warn(
                    "Incomprehensible data received from %s (role=%s)" %
                    (self.transport.getPeer(), incoming.get("role"))
                )

        except ParseError:
            log.warn("Unparsable message received from %s" % str(self.transport.getPeer()))

        finally:
            # After receiving a message, we shut down the connection.
            self.transport.loseConnection()
Example #3
0
    def stringReceived(self, data):
        """
        Called when we receive a string.

        The sender should only ever receive an ack or a nak, after which it
        should disconnect.
        """
        log.debug("Got response from %s" % str(self.transport.getPeer()))
        try:
            incoming = xml_document.infer_type(data)
        except ParseError:
            log.warn(f"Unparsable message received from "
                     f"{self.transport.getPeer()}")
            return

        if incoming.role not in ("ack", "nak"):
            log.warn(f"Unexpected {incoming.role} received "
                     f"from {self.transport.getPeer()}")
            return

        try:
            d = self._sent_ivoids.pop(incoming.origin)
        except KeyError:
            log.warn(f"Received a receipt for {incoming.origin}, "
                     f"which is unknown to us")
            return

        d.callback(incoming)
Example #4
0
    def stringReceived(self, data):
        """
        Called when we receive a string.

        The sender should only ever receive an ack or a nak, after which it
        should disconnect.
        """
        log.debug("Got response from %s" % str(self.transport.getPeer()))
        try:
            incoming = xml_document(data)

            if incoming.element.get('role') == "ack":
                log.info("Acknowledgement received from %s" %
                         str(self.transport.getPeer()))
                self.factory.ack = True
            elif incoming.element.get('role') == "nak":
                log.warn("Nak received: %s refused to accept VOEvent (%s)" %
                         (str(self.transport.getPeer()),
                          incoming.element.findtext(
                              "Meta/Result", default="no reason given")))
            else:
                log.warn(
                    "Incomprehensible data received from %s (role=%s)" %
                    (self.transport.getPeer(), incoming.element.get("role")))

        except ParseError:
            log.warn("Unparsable message received from %s" %
                     str(self.transport.getPeer()))

        finally:
            # After receiving a message, we shut down the connection.
            self.transport.loseConnection()
Example #5
0
    def stringReceived(self, data):
        """
        Called when a complete new message is received.
        """
        try:
            incoming = xml_document(data)
        except ParseError:
            log.warn("Unparsable message received")
            return

        # Reset the timeout counter and wait another 120 seconds before
        # disconnecting due to inactivity.
        self.resetTimeout()

        # The root element of both VOEvent and Transport packets has a
        # "role" element which we use to identify the type of message we
        # have received.
        if incoming.element.get("role") == "iamalive":
            log.debug("IAmAlive received from %s" % str(self.transport.getPeer()))
            self.send_xml(iamaliveresponse(self.factory.local_ivo, incoming.element.find("Origin").text))
        elif incoming.element.get("role") == "authenticate":
            log.debug("Authenticate received from %s" % str(self.transport.getPeer()))
            self.send_xml(
                authenticateresponse(self.factory.local_ivo, incoming.element.find("Origin").text, self.filters)
            )
        elif incoming.element.get("role") in VOEVENT_ROLES:
            log.info("VOEvent %s received from %s" % (incoming.element.attrib["ivorn"], str(self.transport.getPeer())))
            # We don't send a NAK even if the event is invalid since we don't
            # want to be removed from upstream's distribution list.
            self.process_event(incoming, can_nak=False)
        else:
            log.warn(
                "Incomprehensible data received from %s (role=%s)"
                % (self.transport.getPeer(), incoming.element.get("role"))
            )
Example #6
0
    def stringReceived(self, data):
        """
        Called when we receive a string.

        The sender should only ever receive an ack or a nak, after which it
        should disconnect.
        """
        log.debug("Got response from %s" % str(self.transport.getPeer()))
        try:
            incoming = xml_document.infer_type(data)
        except ParseError:
            log.warn(f"Unparsable message received from "
                     f"{self.transport.getPeer()}")
            return

        if incoming.role not in ("ack", "nak"):
            log.warn(f"Unexpected {incoming.role} received "
                     f"from {self.transport.getPeer()}")
            return

        try:
            d = self._sent_ivoids.pop(incoming.origin)
        except KeyError:
            log.warn(f"Received a receipt for {incoming.origin}, "
                     f"which is unknown to us")
            return

        d.callback(incoming)
Example #7
0
 def check_validity(is_valid):
     if is_valid:
         log.debug("Event not previously seen")
         return True
     else:
         log.debug("Event HAS been previously seen")
         raise Exception("Previously seen by this broker")
Example #8
0
def parse_ivoid(ivoid):
    """
    Takes an IVOID of the form

        ivo://[authorityID][resourceKey]#[local_ID]

    and returns (authorityID, resourceKey, local_ID). Raise if that isn't
    possible.

    Note that the resourceKey will normally start with a slash. This is part
    of the key, and this function will not trim it.

    Refer to the IVOA Identifiers Recommendation (2.0) for details.
    """
    try:
        groups = IVOID_RE.match(ivoid).groups()

        # An empty resource name is valid.
        rsrc = groups[1] if groups[1] is not None else ""

        # These may not appear in the resource key per IVOA Identifiers
        # Version 2.0 \S2.3.3.
        for forbidden in ['//', '/../', '/./']:
            assert (forbidden not in rsrc)
        assert (not rsrc.endswith('/'))

        return groups[0], rsrc, groups[2]
    except (AttributeError, AssertionError) as e:
        log.debug("Failed to parse as IVOID: ", str(e))
        raise BadIvoidError("Invalid IVOID: %s" % (ivoid, ))
Example #9
0
def parse_ivoid(ivoid):
    """
    Takes an IVOID of the form

        ivo://[authorityID][resourceKey]#[local_ID]

    and returns (authorityID, resourceKey, local_ID). Raise if that isn't
    possible.

    Note that the resourceKey will normally start with a slash. This is part
    of the key, and this function will not trim it.

    Refer to the IVOA Identifiers Recommendation (2.0) for details.
    """
    try:
        groups = IVOID_RE.match(ivoid).groups()

        # An empty resource name is valid.
        rsrc = groups[1] if groups[1] is not None else ""

        # These may not appear in the resource key per IVOA Identifiers
        # Version 2.0 \S2.3.3.
        for forbidden in ['//', '/../', '/./']:
            assert(forbidden not in rsrc)
        assert(not rsrc.endswith('/'))

        return groups[0], rsrc, groups[2]
    except (AttributeError, AssertionError) as e:
        log.debug("Failed to parse as IVOID: ", str(e))
        raise BadIvoidError("Invalid IVOID: %s" % (ivoid,))
Example #10
0
File: base.py Project: sipior/Comet
 def handle_valid(status):
     log.debug("Event accepted; sending ACK to %s" %
               (self.transport.getPeer()))
     self.send_xml(
         ack(self.factory.local_ivo, event.element.attrib['ivorn']))
     self.handle_event(event).addCallbacks(
         lambda x: log.debug("Event processed"),
         lambda x: log.warn("Event handlers failed"))
Example #11
0
 def handle_valid(status):
     log.debug("Event accepted; sending ACK to %s" % (self.transport.getPeer()))
     self.send_xml(TransportMessage.ack(self.factory.local_ivo,
                                        event.element.attrib['ivorn']))
     self.handle_event(event).addCallbacks(
         lambda x: log.debug("Event processed"),
         lambda x: log.warn("Event handlers failed")
     )
Example #12
0
 def __call__(self, event):
     """
     Save an event to disk.
     """
     if not os.path.exists(self.directory):
         os.makedirs(self.directory)
     with event_file(event.element.attrib['ivorn'], self.directory) as f:
         log.debug("Writing to %s" % (f.name, ))
         f.write(event.raw_bytes.decode(event.encoding))
Example #13
0
 def __call__(self, event):
     """
     Save an event to disk.
     """
     if not os.path.exists(self.directory):
         os.makedirs(self.directory)
     with event_file(event.element.attrib['ivorn'], self.directory) as f:
         log.debug("Writing to %s" % (f.name,))
         f.write(event.raw_bytes.decode(event.encoding))
Example #14
0
 def handle_invalid(failure):
     log.info("Event rejected (%s); discarding" % (failure.value.subFailure.getErrorMessage(),))
     if can_nak:
         log.debug("Sending NAK to %s" % (self.transport.getPeer()))
         self.send_xml(TransportMessage.nak(self.factory.local_ivo,
                                            event.element.attrib['ivorn'],
                                            "Event rejected: %s" % (failure.value.subFailure.getErrorMessage(),)))
     else:
         log.debug("Sending ACK to %s" % (self.transport.getPeer()))
         self.send_xml(TransportMessage.ack(self.factory.local_ivo,
                                            event.element.attrib['ivorn']))
Example #15
0
    def __call__(self, event):
        """
        Add an event to the celery processing queue
        """
        log.debug("Passing to celery...")
        try:
            process_voevent_celerytask.delay(event.raw_bytes)
            ingest_voevent_celerytask.delay(event.raw_bytes)
        except Exception as e:
            self.deferred.errback(e)

        log.debug("Celery jobs sent OK.")
Example #16
0
File: base.py Project: sipior/Comet
 def handle_invalid(failure):
     log.info("Event rejected (%s); discarding" %
              (failure.value.subFailure.getErrorMessage(), ))
     if can_nak:
         log.debug("Sending NAK to %s" % (self.transport.getPeer()))
         self.send_xml(
             nak(
                 self.factory.local_ivo, event.element.attrib['ivorn'],
                 "Event rejected: %s" %
                 (failure.value.subFailure.getErrorMessage(), )))
     else:
         log.debug("Sending ACK to %s" % (self.transport.getPeer()))
         self.send_xml(
             ack(self.factory.local_ivo, event.element.attrib['ivorn']))
Example #17
0
    def stringReceived(self, data):
        """
        Called when a complete new message is received.
        """
        try:
            incoming = xml_document(data)
        except ParseError:
            log.warn("Unparsable message received")
            return

        # Reset the timeout counter and wait another 120 seconds before
        # disconnecting due to inactivity.
        self.resetTimeout()

        # The root element of both VOEvent and Transport packets has a
        # "role" element which we use to identify the type of message we
        # have received.
        if incoming.element.get('role') == "iamalive":
            log.debug("IAmAlive received from %s" % str(self.transport.getPeer()))
            self.send_xml(
                iamaliveresponse(self.factory.local_ivo,
                incoming.element.find('Origin').text)
            )
        elif incoming.element.get('role') == "authenticate":
            log.debug("Authenticate received from %s" % str(self.transport.getPeer()))
            self.send_xml(
                authenticateresponse(
                    self.factory.local_ivo,
                    incoming.element.find('Origin').text,
                    self.filters
                )
            )
        elif incoming.element.get('role') in VOEVENT_ROLES:
            log.info(
                "VOEvent %s received from %s" % (
                    incoming.element.attrib['ivorn'],
                    str(self.transport.getPeer())
                )
            )
            # We don't send a NAK even if the event is invalid since we don't
            # want to be removed from upstream's distribution list.
            self.process_event(incoming, can_nak=False)
        else:
            log.warn(
                "Incomprehensible data received from %s (role=%s)" %
                (self.transport.getPeer(), incoming.element.get("role"))
            )
Example #18
0
    def __call__(self, event):
        """
        Add an event to the celery processing queue
        """
        log.debug("Passing to voeventdb: %s" % (event.attrib['ivorn'],))
        session = Session(bind=dbengine)
        try:
            v = voeventparse.loads(event.text)
            dbconv.safe_insert_voevent(session, v)
            session.commit()
        except Exception as e:
            log.warn("Could not insert packet with ivorn {} into database {}".format(
                v.attrib['ivorn'], voeventdb_dbname))
            self.deferred.errback(e)

        log.info("Loaded {} into database {}".format(
            v.attrib['ivorn'], voeventdb_dbname))
Example #19
0
def parse_ivorn(ivorn):
    """
    Takes an IVORN of the form

        ivo://authorityID/resourceKey#local_ID

    and returns (authorityID, resourceKey, local_ID). Raise if that isn't
    possible.

    Refer to the IVOA Identifiers Recommendation (1.12) for justification, but
    note that document is not as clear as unambiguous as one might hope. We
    have assumed that anything which is not explicitly permitted is forbitten
    in the authority and the resource name, while anything which would be
    permitted in an RFC-3986 URI is permitted in the fragment.
    """
    try:
        return IVORN_RE.match(ivorn).groups()
    except AttributeError, e:
        log.debug("Failed to parse as IVORN: ", str(e))
        raise Exception("Invalid IVORN: %s" % (ivorn,))
Example #20
0
    def stringReceived(self, data):
        try:
            incoming = TransportMessage(data)
        except ParseError:
            log.warn("Unparsable message received")
            return

        if incoming.role == "iamalive":
            log.debug("IAmAlive received from %s" %
                      str(self.transport.getPeer()))
            self.alive_count -= 1
        elif incoming.role == "ack":
            log.debug("Ack received from %s" % str(self.transport.getPeer()))
            self.outstanding_ack -= 1
        elif incoming.role == "nak":
            log.info("Nak received from %s; terminating" %
                     str(self.transport.getPeer()))
            self.transport.loseConnection()
        elif incoming.role == "authenticate":
            log.debug("Authentication received from %s" %
                      str(self.transport.getPeer()))
            self.filters = []
            # Accept both "new-style" (<Param type="xpath-filter" />) and
            # old-style (<filter type="xpath" />) filters.
            for xpath in chain([
                    elem.get('value') for elem in incoming.element.findall(
                        "Meta/Param[@name=\"xpath-filter\"]")
            ], [
                    elem.text for elem in incoming.element.findall(
                        "Meta/filter[@type=\"xpath\"]")
            ]):
                log.info("Installing filter %s for %s" %
                         (xpath, str(self.transport.getPeer())))
                try:
                    self.filters.append(ElementTree.XPath(xpath))
                except ElementTree.XPathSyntaxError:
                    log.info("Filter %s is not valid XPath" % (xpath, ))
        else:
            log.warn("Incomprehensible data received from %s (role=%s)" %
                     (self.transport.getPeer(), incoming.role))
Example #21
0
    def stringReceived(self, data):
        try:
            incoming = xml_document(data)
        except ParseError:
            log.warn("Unparsable message received")
            return

        if incoming.get('role') == "iamalive":
            log.debug("IAmAlive received from %s" % str(self.transport.getPeer()))
            self.alive_count -= 1
        elif incoming.get('role') == "ack":
            log.debug("Ack received from %s" % str(self.transport.getPeer()))
            self.outstanding_ack -= 1
        elif incoming.get('role') == "nak":
            log.info("Nak received from %s; terminating" % str(self.transport.getPeer()))
            self.transport.loseConnection()
        elif incoming.get('role') == "authenticate":
            log.debug("Authentication received from %s" % str(self.transport.getPeer()))
            self.filters = []
            # Accept both "new-style" (<Param type="xpath-filter" />) and
            # old-style (<filter type="xpath" />) filters.
            for xpath in chain(
                [elem.get('value') for elem in incoming.findall("Meta/Param[@name=\"xpath-filter\"]")],
                [elem.text for elem in incoming.findall("Meta/filter[@type=\"xpath\"]")]
            ):
                log.info(
                    "Installing filter %s for %s" %
                    (xpath, str(self.transport.getPeer()))
                )
                try:
                    self.filters.append(ElementTree.XPath(xpath))
                except ElementTree.XPathSyntaxError:
                    log.info("Filter %s is not valid XPath" % (xpath,))
        else:
            log.warn(
                "Incomprehensible data received from %s (role=%s)" %
                (self.transport.getPeer(), incoming.get("role"))
            )
Example #22
0
 def sendTestEvent(self):
     log.debug("Broadcasting test event")
     test_event = broker_test_message(self.local_ivo)
     for broadcaster in self.broadcasters:
         broadcaster.send_event(test_event)
Example #23
0
 def outReceived(self, data):
     log.debug("External process said: %s" %
               (data.decode(self.STDOUT_ENCODING), ))
Example #24
0
 def test_log_debug_level_warning(self):
     log.LEVEL = log.Levels.WARNING
     self.assertFalse(self.observer.messages)
     d = log.debug(DUMMY_MESSAGE)
     self.assertIsInstance(d, defer.Deferred)
     return d.addCallback(self._check_log_empty)
Example #25
0
 def test_log_debug_level_warning(self):
     log.LEVEL = log.Levels.WARNING
     self.assertFalse(self.observer.messages)
     d = log.debug(DUMMY_MESSAGE)
     self.assertIsInstance(d, defer.Deferred)
     return d.addCallback(self._check_log_empty)
Example #26
0
 def sendIAmAlive(self):
     log.debug("Broadcasting iamalive")
     for broadcaster in self.broadcasters:
         broadcaster.sendIAmAlive()
Example #27
0
 def sendTestEvent(self):
     log.debug("Broadcasting test event")
     test_event = VOEventMessage.broker_test(self.local_ivo)
     for broadcaster in self.broadcasters:
         broadcaster.send_event(test_event)
Example #28
0
 def outReceived(self, data):
     log.debug("External process said: %s" %
               (data.decode(self.STDOUT_ENCODING),))
Example #29
0
 def sendIAmAlive(self):
     log.debug("Broadcasting iamalive")
     for broadcaster in self.broadcasters:
         broadcaster.sendIAmAlive()