def stringReceived(self, data):
"""
Called when a complete new message is received.
"""
try:
incoming = xml_document(data)
except ParseError:
log.warn("Unparsable message received")
return
# Reset the timeout counter and wait another 120 seconds before
# disconnecting due to inactivity.
self.resetTimeout()
# The root element of both VOEvent and Transport packets has a
# "role" element which we use to identify the type of message we
# have received.
if incoming.element.get("role") == "iamalive":
log.debug("IAmAlive received from %s" % str(self.transport.getPeer()))
self.send_xml(iamaliveresponse(self.factory.local_ivo, incoming.element.find("Origin").text))
elif incoming.element.get("role") == "authenticate":
log.debug("Authenticate received from %s" % str(self.transport.getPeer()))
self.send_xml(
authenticateresponse(self.factory.local_ivo, incoming.element.find("Origin").text, self.filters)
)
elif incoming.element.get("role") in VOEVENT_ROLES:
log.info("VOEvent %s received from %s" % (incoming.element.attrib["ivorn"], str(self.transport.getPeer())))
# We don't send a NAK even if the event is invalid since we don't
# want to be removed from upstream's distribution list.
self.process_event(incoming, can_nak=False)
else:
log.warn(
"Incomprehensible data received from %s (role=%s)"
% (self.transport.getPeer(), incoming.element.get("role"))
)
def log_response(incoming):
"""Default action to take on receiving an acknowledgement.
The result is logged, the connection is closed (per the VTP spec)
and the incoming packet is passed on to the next callback in the
chain (if any).
Parameters
----------
incoming : `comet.utility.xml_document`
The acknowledgement received.
Returns
-------
incoming : `comet.utility.xml_document`
The acknowledgement received.
Notes
-----
A NAK is not considered a failure (we do not call an errback).
"""
if incoming.role == "ack":
log.info(f"ACK received: "
f"{self.transport.getPeer()} accepted VOEvent")
elif incoming.role == "nak":
reason = incoming.element.findtext("Meta/Result",
default="no reason given")
log.warn(
f"NAK received: "
f"{self.transport.getPeer()} refused to accept VOEvent "
f"({reason})")
self.transport.loseConnection()
return incoming
def stringReceived(self, data):
"""
Called when a complete new message is received.
"""
try:
incoming = xml_document(data)
except ParseError:
d = log.warn("Unparsable message received from %s" % str(self.transport.getPeer()))
else:
# The root element of both VOEvent and Transport packets has a
# "role" element which we use to identify the type of message we
# have received.
if incoming.element.get('role') in VOEVENT_ROLES:
log.info(
"VOEvent %s received from %s" % (
incoming.element.attrib['ivorn'],
str(self.transport.getPeer())
)
)
d = self.process_event(incoming)
else:
d = log.warn(
"Incomprehensible data received from %s (role=%s)" %
(self.transport.getPeer(), incoming.element.get("role"))
)
finally:
return d.addCallback(
lambda x: self.transport.loseConnection()
)
def log_response(incoming):
"""Default action to take on receiving an acknowledgement.
The result is logged, the connection is closed (per the VTP spec)
and the incoming packet is passed on to the next callback in the
chain (if any).
Parameters
----------
incoming : `comet.utility.xml_document`
The acknowledgement received.
Returns
-------
incoming : `comet.utility.xml_document`
The acknowledgement received.
Notes
-----
A NAK is not considered a failure (we do not call an errback).
"""
if incoming.role == "ack":
log.info(f"ACK received: "
f"{self.transport.getPeer()} accepted VOEvent")
elif incoming.role == "nak":
reason = incoming.element.findtext("Meta/Result",
default="no reason given")
log.warn(f"NAK received: "
f"{self.transport.getPeer()} refused to accept VOEvent "
f"({reason})")
self.transport.loseConnection()
return incoming
def __call__(self, event):
log.info("Running external command: %s" % (self.cmd, ))
d = defer.Deferred()
if not os.access(self.cmd, os.X_OK):
msg = "%s is not executable" % (self.cmd, )
log.warn(msg)
d.errback(Exception(msg))
else:
def log_reason(reason):
"""
Catch a Failure returned from an unsuccessful process execution
and log the return value, then re-raise the error.
"""
msg = "%s returned non-zero (%d)" % (self.cmd,
reason.value.exitCode)
log.warn(msg)
return reason
d.addErrback(log_reason)
reactor.spawnProcess(SpawnCommandProtocol(d, event.raw_bytes),
self.cmd,
args=self.args,
env=os.environ)
return d
def __call__(self, event):
log.info("Running external command: %s" % (self.cmd,))
d = defer.Deferred()
if not os.access(self.cmd, os.X_OK):
msg = "%s is not executable" % (self.cmd,)
log.warn(msg)
d.errback(Exception(msg))
else:
def log_reason(reason):
"""
Catch a Failure returned from an unsuccessful process execution
and log the return value, then re-raise the error.
"""
msg = "%s returned non-zero (%d)" % (self.cmd,
reason.value.exitCode)
log.warn(msg)
return reason
d.addErrback(log_reason)
reactor.spawnProcess(
SpawnCommandProtocol(d, event.raw_bytes),
self.cmd,
args=self.args,
env=os.environ
)
return d
def log_reason(reason):
"""
Catch a Failure returned from an unsuccessful process execution
and log the return value, then re-raise the error.
"""
msg = "%s returned non-zero (%d)" % (self.cmd,
reason.value.exitCode)
log.warn(msg)
return reason
def log_reason(reason):
"""
Catch a Failure returned from an unsuccessful process execution
and log the return value, then re-raise the error.
"""
msg = "%s returned non-zero (%d)" % (self.cmd,
reason.value.exitCode)
log.warn(msg)
return reason
def buildProtocol(self, addr):
try:
remote_ip = ip_address(addr.host)
if not any(remote_ip in network for network in self.whitelist):
log.info(f"Attempted {self.connection_type} from "
f"non-whitelisted {addr}")
return None
except AttributeError:
log.warn(f"Bypassing whitelist for {self.connection_type} "
f"from {addr}")
return WrappingFactory.buildProtocol(self, addr)
def buildProtocol(self, addr):
try:
remote_ip = ip_address(addr.host)
if not any(remote_ip in network for network in self.whitelist):
log.info(f"Attempted {self.connection_type} from "
f"non-whitelisted {addr}")
return None
except AttributeError:
log.warn(f"Bypassing whitelist for {self.connection_type} "
f"from {addr}")
return WrappingFactory.buildProtocol(self, addr)
def log_reason(reason):
"""
Catch a Failure returned from an unsuccessful process execution
and log the return value, then re-raise the error.
"""
if not os.access(self.cmd, os.X_OK):
msg = "%s is not an executable" % (self.cmd, )
else:
msg = "%s returned non-zero (%d)" % (self.cmd,
reason.value.exitCode)
log.warn(msg)
return reason
def check_event(self, event):
"""
Returns True if event is unseen (and hence good to forward), False
otherwise.
"""
try:
db_path, key = self._get_event_details(event)
except Exception as e:
log.warn("Unparseable IVOID; failing eventdb lookup");
else:
with self.databases[db_path]: # Acquire lock
with closing(anydbm.open(os.path.join(self.root, db_path), 'c')) as db:
if not key in db:
db[key] = str(time.time())
return True
return False
def check_event(self, event):
"""
Returns True if event is unseen (and hence good to forward), False
otherwise.
"""
try:
db_path, key = self._get_event_details(event)
except Exception as e:
log.warn("Unparseable IVORN; failing eventdb lookup");
else:
with self.databases[db_path]: # Acquire lock
with closing(anydbm.open(os.path.join(self.root, db_path), 'c')) as db:
if not key in db:
db[key] = str(time.time())
return True
return False
def stringReceived(self, data):
"""
Called when a complete new message is received.
"""
try:
incoming = xml_document(data)
except ParseError:
log.warn("Unparsable message received")
return
# Reset the timeout counter and wait another 120 seconds before
# disconnecting due to inactivity.
self.resetTimeout()
# The root element of both VOEvent and Transport packets has a
# "role" element which we use to identify the type of message we
# have received.
if incoming.element.get('role') == "iamalive":
log.debug("IAmAlive received from %s" % str(self.transport.getPeer()))
self.send_xml(
iamaliveresponse(self.factory.local_ivo,
incoming.element.find('Origin').text)
)
elif incoming.element.get('role') == "authenticate":
log.debug("Authenticate received from %s" % str(self.transport.getPeer()))
self.send_xml(
authenticateresponse(
self.factory.local_ivo,
incoming.element.find('Origin').text,
self.filters
)
)
elif incoming.element.get('role') in VOEVENT_ROLES:
log.info(
"VOEvent %s received from %s" % (
incoming.element.attrib['ivorn'],
str(self.transport.getPeer())
)
)
# We don't send a NAK even if the event is invalid since we don't
# want to be removed from upstream's distribution list.
self.process_event(incoming, can_nak=False)
else:
log.warn(
"Incomprehensible data received from %s (role=%s)" %
(self.transport.getPeer(), incoming.element.get("role"))
)
def __call__(self, event):
"""
Add an event to the celery processing queue
"""
log.debug("Passing to voeventdb: %s" % (event.attrib['ivorn'],))
session = Session(bind=dbengine)
try:
v = voeventparse.loads(event.text)
dbconv.safe_insert_voevent(session, v)
session.commit()
except Exception as e:
log.warn("Could not insert packet with ivorn {} into database {}".format(
v.attrib['ivorn'], voeventdb_dbname))
self.deferred.errback(e)
log.info("Loaded {} into database {}".format(
v.attrib['ivorn'], voeventdb_dbname))
def handle_valid(status):
log.debug("Event accepted; sending ACK to %s" % (self.transport.getPeer()))
self.send_xml(TransportMessage.ack(self.factory.local_ivo,
event.element.attrib['ivorn']))
self.handle_event(event).addCallbacks(
lambda x: log.debug("Event processed"),
lambda x: log.warn("Event handlers failed")
)
def handle_valid(status):
log.debug("Event accepted; sending ACK to %s" %
(self.transport.getPeer()))
self.send_xml(
ack(self.factory.local_ivo, event.element.attrib['ivorn']))
self.handle_event(event).addCallbacks(
lambda x: log.debug("Event processed"),
lambda x: log.warn("Event handlers failed"))
def __call__(self, event):
"""
Add an event to the celery processing queue
"""
v = None
try:
session = Session(bind=dbengine)
v = voeventparse.loads(event.raw_bytes)
dbconv.safe_insert_voevent(session, v)
session.commit()
except Exception as e:
if v is None:
log.warn("Could not parse event-bytes as voevent")
else:
log.warn(
"Could not insert packet with ivorn {} into database {}".
format(v.attrib['ivorn'], voeventdb_dbname))
self.deferred.errback(e)
log.info("Loaded {} into database {}".format(v.attrib['ivorn'],
voeventdb_dbname))
def stringReceived(self, data):
"""
Called when we receive a string.
The sender should only ever receive an ack or a nak, after which it
should disconnect.
"""
log.debug("Got response from %s" % str(self.transport.getPeer()))
try:
incoming = xml_document.infer_type(data)
except ParseError:
log.warn(f"Unparsable message received from "
f"{self.transport.getPeer()}")
return
if incoming.role not in ("ack", "nak"):
log.warn(f"Unexpected {incoming.role} received "
f"from {self.transport.getPeer()}")
return
try:
d = self._sent_ivoids.pop(incoming.origin)
except KeyError:
log.warn(f"Received a receipt for {incoming.origin}, "
f"which is unknown to us")
return
d.callback(incoming)
def stringReceived(self, data):
"""
Called when we receive a string.
The sender should only ever receive an ack or a nak, after which it
should disconnect.
"""
log.debug("Got response from %s" % str(self.transport.getPeer()))
try:
incoming = xml_document.infer_type(data)
except ParseError:
log.warn(f"Unparsable message received from "
f"{self.transport.getPeer()}")
return
if incoming.role not in ("ack", "nak"):
log.warn(f"Unexpected {incoming.role} received "
f"from {self.transport.getPeer()}")
return
try:
d = self._sent_ivoids.pop(incoming.origin)
except KeyError:
log.warn(f"Received a receipt for {incoming.origin}, "
f"which is unknown to us")
return
d.callback(incoming)
def stringReceived(self, data):
try:
incoming = TransportMessage(data)
except ParseError:
log.warn("Unparsable message received")
return
if incoming.role == "iamalive":
log.debug("IAmAlive received from %s" %
str(self.transport.getPeer()))
self.alive_count -= 1
elif incoming.role == "ack":
log.debug("Ack received from %s" % str(self.transport.getPeer()))
self.outstanding_ack -= 1
elif incoming.role == "nak":
log.info("Nak received from %s; terminating" %
str(self.transport.getPeer()))
self.transport.loseConnection()
elif incoming.role == "authenticate":
log.debug("Authentication received from %s" %
str(self.transport.getPeer()))
self.filters = []
# Accept both "new-style" (<Param type="xpath-filter" />) and
# old-style (<filter type="xpath" />) filters.
for xpath in chain([
elem.get('value') for elem in incoming.element.findall(
"Meta/Param[@name=\"xpath-filter\"]")
], [
elem.text for elem in incoming.element.findall(
"Meta/filter[@type=\"xpath\"]")
]):
log.info("Installing filter %s for %s" %
(xpath, str(self.transport.getPeer())))
try:
self.filters.append(ElementTree.XPath(xpath))
except ElementTree.XPathSyntaxError:
log.info("Filter %s is not valid XPath" % (xpath, ))
else:
log.warn("Incomprehensible data received from %s (role=%s)" %
(self.transport.getPeer(), incoming.role))
def stringReceived(self, data):
"""
Called when we receive a string.
The sender should only ever receive an ack or a nak, after which it
should disconnect.
"""
log.debug("Got response from %s" % str(self.transport.getPeer()))
try:
incoming = xml_document(data)
if incoming.get('role') == "ack":
log.info("Acknowledgement received from %s" % str(self.transport.getPeer()))
self.factory.ack = True
elif incoming.get('role') == "nak":
log.warn("Nak received: %s refused to accept VOEvent (%s)" %
(
str(self.transport.getPeer()),
incoming.findtext("Meta/Result", default="no reason given")
)
)
else:
log.warn(
"Incomprehensible data received from %s (role=%s)" %
(self.transport.getPeer(), incoming.get("role"))
)
except ParseError:
log.warn("Unparsable message received from %s" % str(self.transport.getPeer()))
finally:
# After receiving a message, we shut down the connection.
self.transport.loseConnection()
def stringReceived(self, data):
"""
Called when we receive a string.
The sender should only ever receive an ack or a nak, after which it
should disconnect.
"""
log.debug("Got response from %s" % str(self.transport.getPeer()))
try:
incoming = xml_document(data)
if incoming.element.get('role') == "ack":
log.info("Acknowledgement received from %s" %
str(self.transport.getPeer()))
self.factory.ack = True
elif incoming.element.get('role') == "nak":
log.warn("Nak received: %s refused to accept VOEvent (%s)" %
(str(self.transport.getPeer()),
incoming.element.findtext(
"Meta/Result", default="no reason given")))
else:
log.warn(
"Incomprehensible data received from %s (role=%s)" %
(self.transport.getPeer(), incoming.element.get("role")))
except ParseError:
log.warn("Unparsable message received from %s" %
str(self.transport.getPeer()))
finally:
# After receiving a message, we shut down the connection.
self.transport.loseConnection()
def stringReceived(self, data):
try:
incoming = xml_document(data)
except ParseError:
log.warn("Unparsable message received")
return
if incoming.get('role') == "iamalive":
log.debug("IAmAlive received from %s" % str(self.transport.getPeer()))
self.alive_count -= 1
elif incoming.get('role') == "ack":
log.debug("Ack received from %s" % str(self.transport.getPeer()))
self.outstanding_ack -= 1
elif incoming.get('role') == "nak":
log.info("Nak received from %s; terminating" % str(self.transport.getPeer()))
self.transport.loseConnection()
elif incoming.get('role') == "authenticate":
log.debug("Authentication received from %s" % str(self.transport.getPeer()))
self.filters = []
# Accept both "new-style" (<Param type="xpath-filter" />) and
# old-style (<filter type="xpath" />) filters.
for xpath in chain(
[elem.get('value') for elem in incoming.findall("Meta/Param[@name=\"xpath-filter\"]")],
[elem.text for elem in incoming.findall("Meta/filter[@type=\"xpath\"]")]
):
log.info(
"Installing filter %s for %s" %
(xpath, str(self.transport.getPeer()))
)
try:
self.filters.append(ElementTree.XPath(xpath))
except ElementTree.XPathSyntaxError:
log.info("Filter %s is not valid XPath" % (xpath,))
else:
log.warn(
"Incomprehensible data received from %s (role=%s)" %
(self.transport.getPeer(), incoming.get("role"))
)
def stringReceived(self, data):
"""
Called when a complete new message is received.
"""
try:
incoming = xml_document.infer_type(data)
except ParseError:
d = log.warn("Unparsable message received from %s" %
str(self.transport.getPeer()))
else:
# The root element of both VOEvent and Transport packets has a
# "role" element which we use to identify the type of message we
# have received.
if hasattr(incoming, "ivoid"):
log.info("VOEvent %s received from %s" %
(incoming.ivoid, str(self.transport.getPeer())))
d = self.process_event(incoming)
else:
d = log.warn(
"Incomprehensible data received from %s (role=%s)" %
(self.transport.getPeer(), incoming.role))
finally:
return d.addCallback(lambda x: self.transport.loseConnection())
def test_log_warning_level_warning(self):
log.LEVEL = log.Levels.WARNING
self.assertFalse(self.observer.messages)
d = log.warn(DUMMY_MESSAGE)
self.assertTrue(self.observer.messages)
return d.addCallback(self._check_log_full)
def db_failure(failure):
log.warn("Event DB lookup failed!")
log.warn(failure.getTraceback())
return failure
def db_failure(failure):
log.warn("Event DB lookup failed!")
log.warn(failure.getTraceback())
return failure
import comet.log as log
import os
import voeventparse
from sqlalchemy import create_engine
from sqlalchemy.orm import Session
import voeventdb.server.database.config as dbconfig
from voeventdb.server.database import db_utils
import voeventdb.server.database.convenience as dbconv
voeventdb_dbname = os.environ.get("VOEVENTDB_DBNAME",
dbconfig.testdb_corpus_url.database)
dburl = dbconfig.make_db_url(dbconfig.default_admin_db_params, voeventdb_dbname)
if not db_utils.check_database_exists(dburl):
log.warn("voeventdb database not found: {}".format(
voeventdb_dbname))
dbengine = create_engine(dburl)
@implementer(IPlugin, IHandler)
class VoeventdbInserter(object):
name = "voeventdb-insert"
# When the handler is called, it is passed an instance of
# comet.utility.xml.xml_document.
def __call__(self, event):
"""
Add an event to the celery processing queue
"""
log.debug("Passing to voeventdb: %s" % (event.attrib['ivorn'],))
session = Session(bind=dbengine)
try:
def test_log_warning_level_warning(self):
log.LEVEL = log.Levels.WARNING
self.assertFalse(self.observer.messages)
d = log.warn(DUMMY_MESSAGE)
self.assertTrue(self.observer.messages)
return d.addCallback(self._check_log_full)
def stopFactory(self):
if self.ack:
log.info("Event was sent successfully")
else:
log.warn("Event was NOT sent successfully")
def stopFactory(self):
if self.ack:
log.info("Event was sent successfully")
else:
log.warn("Event was NOT sent successfully")
import comet.log as log
import os
import voeventparse
from sqlalchemy import create_engine
from sqlalchemy.orm import Session
import voeventdb.server.database.config as dbconfig
from voeventdb.server.database import db_utils
import voeventdb.server.database.convenience as dbconv
voeventdb_dbname = os.environ.get("VOEVENTDB_DBNAME",
dbconfig.testdb_corpus_url.database)
dburl = dbconfig.make_db_url(dbconfig.default_admin_db_params,
voeventdb_dbname)
if not db_utils.check_database_exists(dburl):
log.warn("voeventdb database not found: {}".format(voeventdb_dbname))
dbengine = create_engine(dburl)
@implementer(IPlugin, IHandler)
class VoeventdbInserter(object):
name = "voeventdb-insert"
# When the handler is called, it is passed an instance of
# comet.utility.xml.xml_document.
def __call__(self, event):
"""
Add an event to the celery processing queue
"""
v = None
try: