def get_templates_for_assessment(self, img_dir):
"""
:return:
"""
ams = ArgoMessagingService(endpoint=self.host, token=self.token, project=self.project)
ackids = list()
niftyids = list()
msgids = list()
pull_subscription = ams.pull_sub(self.requestSubscription, num=1, return_immediately=True)
if pull_subscription:
for id, msg in pull_subscription:
attr = msg.get_attr()
data = msg.get_data()
msgid = msg.get_msgid()
image_list_file = tempfile.NamedTemporaryFile(prefix='tmp_', delete=False, suffix='.list', dir=img_dir)
os.chmod(image_list_file.name, 0o644)
image_list_file.write(data)
image_list_file.close()
niftyids.append(os.path.basename(image_list_file.name))
ackids.append(id)
msgids.append(msgid)
if ackids:
ams.ack_sub(self.requestSubscription, ackids)
return niftyids, msgids
def get_templates_for_assessment(self):
"""
:return:
"""
ams = ArgoMessagingService(endpoint=self.host,
token=self.token,
project=self.project)
ackids = list()
niftyids = list()
logging.debug('[%s] %s: Start pulling from the %s subscription',
'SECANT', 'DEBUG', self.requestSubscription)
pull_subscription = ams.pull_sub(self.requestSubscription,
self.nummsgs, True)
logging.debug('[%s] %s: Finish pulling from the %s subscription',
'SECANT', 'DEBUG', self.requestSubscription)
if pull_subscription:
for id, msg in pull_subscription:
attr = msg.get_attr()
data = msg.get_data()
image_list_file = tempfile.NamedTemporaryFile(prefix='tmp_',
delete=False,
suffix='.list')
image_list_file.write(data)
image_list_file.close()
niftyids.append(image_list_file.name)
ackids.append(id)
else:
logging.debug('[%s] %s: No new requests to pull', 'SECANT',
'DEBUG')
if ackids:
ams.ack_sub(self.requestSubscription, ackids)
return niftyids
class PullPublish():
def __init__(self,config):
self.pull_sub = config['pull_sub']
self.pub_topic = config['pub_topic']
self.pull_topic = config['pull_topic']
self.ams = ArgoMessagingService(endpoint=config['host'], token=config['token'], project=config['project'])
def pull(self,nummsgs):
messages = []
try:
if not self.ams.has_sub(self.pull_sub):
self.ams.create_sub(self.pull_sub,self.pull_topic)
except AmsException as e:
print(e)
raise SystemExit(1)
# try to pull number of messages from subscription. method will
# return (ackIds, AmsMessage) tuples from which ackIds and messages
# payload will be extracted.
ackids = list()
for id, msg in self.ams.pull_sub(self.pull_sub, nummsgs):
data = msg.get_data()
msgid = msg.get_msgid()
attr = msg.get_attr()
messages.append(json.loads(data))
#print('msgid={0}, data={1}, attr={2}'.format(msgid, data, attr))
ackids.append(id)
# pass list of extracted ackIds to AMS Service so that
# it can move the offset for the next subscription pull
# (basically acknowledging pulled messages)
if ackids:
self.ams.ack_sub(self.pull_sub, ackids)
return messages
def publish(self,messages):
# messages = [{data:[{id:1},{state:'deployed'}],attributes=''}]
try:
if not self.ams.has_topic(self.pub_topic):
self.ams.create_topic(self.pub_topic)
except AmsException as e:
print(e)
raise SystemExit(1)
# publish one message to given topic. message is constructed with
# help of AmsMessage which accepts data and attributes keys.
# data is Base64 encoded, attributes is dictionary of arbitrary
# key/value pairs
msg = AmsMessage()
msglist = []
for message in messages:
msglist.append(msg(data=json.dumps(message['data']),attributes={}))
try:
ret = self.ams.publish(self.pub_topic, msglist)
print(ret)
except AmsException as e:
print(e)
def main(args):
ams_endpoint = "{}:{}".format(args.host, args.port)
ams = ArgoMessagingService(endpoint=ams_endpoint,
token=args.token,
project=args.project)
while True:
try:
consumed_msgs = ams.pull_sub(sub=args.sub,
num=args.bulk_size,
return_immediately=True,
verify=args.verify)
print(consumed_msgs)
last_msg_id = "-1"
if len(consumed_msgs) > 0:
last_msg_id = consumed_msgs.pop()[0]
print(last_msg_id)
print("\n")
if last_msg_id != "-1":
print(ams.ack_sub(args.sub, [last_msg_id], verify=args.verify))
time.sleep(args.fire_rate)
except Exception as e:
print("Couldn't consume from sub {}, {}".format(args.sub, str(e)))
continue
def testPushAndPull(self):
settings = ConfigParser.ConfigParser()
settings.read('../conf/argo.conf')
host = settings.get('AMS-GENERAL', 'host')
project = settings.get('AMS-GENERAL', 'project')
token = settings.get('AMS-GENERAL', 'token')
subscription = settings.get('REQUESTS', 'subscription')
topic = settings.get('REQUESTS', 'topic')
log = logging.getLogger("TestArgoMessageService.testPushAndPull")
log.debug("Host: {host}, Project: {project}, Topic: {topic}".format(
host=host, project=project, topic=topic))
ams = ArgoMessagingService(endpoint=host, token=token, project=project)
pullAllMessages(ams, subscription, log)
response = urllib2.urlopen(
"https://vmcaster.appdb.egi.eu/store/vappliance/demo.va.public/image.list",
timeout=5)
content = response.read()
msg = AmsMessage(data=content, attributes={'index': '1'}).dict()
try:
ret = ams.publish(topic, msg)
log.debug("Successfully published with ID:{id}".format(
id=ret['messageIds'][0]))
except AmsException as e:
print e
# Wait 10 seconds between pull and push
log.debug("Waiting 10s between push and pull")
time.sleep(10)
ackids = list()
pull_result = ams.pull_sub(subscription, 1)
if pull_result:
log.debug("Successfully pulled a message with ID:{id}".format(
id=pull_result[0][1].get_msgid()))
self.assertEquals(content, pull_result[0][1].get_data())
ackids.append(pull_result[0][1].get_msgid())
# Send Acknowledgement
if ackids:
ams.ack_sub(subscription, ackids)
def get_assessment_results(self):
"""
:return:
"""
ams = ArgoMessagingService(endpoint=self.host, token=self.token, project=self.project)
ackids = list()
niftyids = list()
logging.debug('Start pulling from the %s subscription', self.resultSubscription)
pull_subscription = ams.pull_sub(self.resultSubscription, 10, True)
logging.debug('Finish pulling from the %s subscription', self.resultSubscription)
if pull_subscription:
for id, msg in pull_subscription:
attr = msg.get_attr()
data = msg.get_data()
ackids.append(id)
if ackids:
logging.debug("[%s] %s: Acknowledging %s" % ('SECANT', 'DEBUG', "'".join(ackids)))
ams.ack_sub(self.resultSubscription, ackids)
return niftyids
def main():
parser = ArgumentParser(description="Simple AMS example of subscription pull/consume")
parser.add_argument('--host', type=str, default='messaging-devel.argo.grnet.gr', help='FQDN of AMS Service')
parser.add_argument('--token', type=str, required=True, help='Given token')
parser.add_argument('--project', type=str, required=True, help='Project registered in AMS Service')
parser.add_argument('--subscription', type=str, required=True, help='Subscription name')
parser.add_argument('--topic', type=str, required=True, help='Given topic')
parser.add_argument('--nummsgs', type=int, default=3, help='Number of messages to pull and ack')
args = parser.parse_args()
# initialize service with given token and project
ams = ArgoMessagingService(endpoint=args.host, token=args.token, project=args.project)
# ensure that subscription is created in first run. messages can be
# pulled from the subscription only when subscription already exists
# for given topic prior messages being published to topic
try:
if not ams.has_sub(args.subscription):
ams.create_sub(args.subscription, args.topic)
except AmsException as e:
print e
raise SystemExit(1)
# try to pull number of messages from subscription. method will
# return (ackIds, AmsMessage) tuples from which ackIds and messages
# payload will be extracted.
ackids = list()
for id, msg in ams.pull_sub(args.subscription, args.nummsgs):
data = msg.get_data()
msgid = msg.get_msgid()
attr = msg.get_attr()
print 'msgid={0}, data={1}, attr={2}'.format(msgid, data, attr)
ackids.append(id)
# pass list of extracted ackIds to AMS Service so that
# it can move the offset for the next subscription pull
# (basically acknowledging pulled messages)
if ackids:
ams.ack_sub(args.subscription, ackids)
def main():
parser = ArgumentParser(description="Simple AMS message publish example")
parser.add_argument('--host',
type=str,
default='messaging-devel.argo.grnet.gr',
help='FQDN of AMS Service')
parser.add_argument('--token', type=str, required=True, help='Given token')
parser.add_argument('--project',
type=str,
required=True,
help='Project registered in AMS Service')
parser.add_argument('--topic', type=str, required=True, help='Given topic')
parser.add_argument('--subscription',
type=str,
required=True,
help='Subscription name')
parser.add_argument('--nummsgs',
type=int,
default=3,
help='Number of messages to pull and ack')
args = parser.parse_args()
ams = ArgoMessagingService(endpoint=args.host,
token=args.token,
project=args.project)
# static sleep between retry attempts
msg = AmsMessage(data='foo1', attributes={'bar1': 'baz1'}).dict()
try:
ret = ams.publish(args.topic, msg, retry=3, retrysleep=5, timeout=5)
print(ret)
except AmsException as e:
print(e)
# iptables -A OUTPUT -d messaging-devel.argo.grnet.gr -j DROP
ackids = list()
for id, msg in ams.pull_sub(args.subscription,
args.nummsgs,
retry=3,
retrysleep=5,
timeout=5):
data = msg.get_data()
msgid = msg.get_msgid()
attr = msg.get_attr()
print('msgid={0}, data={1}, attr={2}'.format(msgid, data, attr))
ackids.append(id)
if ackids:
ams.ack_sub(args.subscription,
ackids,
retry=3,
retrysleep=5,
timeout=5)
# backoff with each next retry attempt exponentially longer
msg = AmsMessage(data='foo2', attributes={'bar2': 'baz2'}).dict()
try:
ret = ams.publish(args.topic, msg, retry=3, retrybackoff=5, timeout=5)
print(ret)
except AmsException as e:
print(e)
# iptables -A OUTPUT -d messaging-devel.argo.grnet.gr -j DROP
ackids = list()
for id, msg in ams.pull_sub(args.subscription,
args.nummsgs,
retrybackoff=3,
retrysleep=5,
timeout=5):
data = msg.get_data()
msgid = msg.get_msgid()
attr = msg.get_attr()
print('msgid={0}, data={1}, attr={2}'.format(msgid, data, attr))
ackids.append(id)
if ackids:
ams.ack_sub(args.subscription, ackids)
# static sleep between retry attempts. this example uses consume context
# method that pull and acks msgs in one call.
msg = AmsMessage(data='foo3', attributes={'bar3': 'baz3'}).dict()
try:
ret = ams.publish(args.topic, msg, retry=3, retrysleep=5, timeout=5)
print(ret)
except AmsException as e:
print(e)
try:
msgs = ams.pullack_sub(args.subscription,
args.nummsgs,
retry=3,
retrysleep=5,
timeout=5)
for msg in msgs:
data = msg.get_data()
msgid = msg.get_msgid()
attr = msg.get_attr()
print('msgid={0}, data={1}, attr={2}'.format(msgid, data, attr))
except AmsException as e:
print(e)
def main():
MSG_NUM = 100
MSG_SIZE = 500
TIMEOUT = 180
parser = ArgumentParser(description="Nagios sensor for AMS")
parser.add_argument('-H',
dest='host',
type=str,
default='messaging-devel.argo.grnet.gr',
help='FQDN of AMS Service')
parser.add_argument('--token', type=str, required=True, help='Given token')
parser.add_argument('--project',
type=str,
required=True,
help='Project registered in AMS Service')
parser.add_argument('--topic',
type=str,
default='nagios_sensor_topic',
help='Given topic')
parser.add_argument('--subscription',
type=str,
default='nagios_sensor_sub',
help='Subscription name')
parser.add_argument('-t',
dest='timeout',
type=int,
default=TIMEOUT,
help='Timeout')
cmd_options = parser.parse_args()
nagios = NagiosResponse("All messages received correctly.")
ams = ArgoMessagingService(endpoint=cmd_options.host,
token=cmd_options.token,
project=cmd_options.project)
try:
if ams.has_topic(cmd_options.topic, timeout=cmd_options.timeout):
ams.delete_topic(cmd_options.topic, timeout=cmd_options.timeout)
if ams.has_sub(cmd_options.subscription, timeout=cmd_options.timeout):
ams.delete_sub(cmd_options.subscription,
timeout=cmd_options.timeout)
ams.create_topic(cmd_options.topic, timeout=cmd_options.timeout)
ams.create_sub(cmd_options.subscription,
cmd_options.topic,
timeout=cmd_options.timeout)
except AmsException as e:
nagios.writeCriticalMessage(e.msg)
nagios.setCode(nagios.CRITICAL)
print(nagios.getMsg())
raise SystemExit(nagios.getCode())
ams_msg = AmsMessage()
msg_orig = set()
msg_array = []
for i in range(1, MSG_NUM):
msg_txt = ''.join(
random.choice(string.ascii_letters + string.digits)
for i in range(MSG_SIZE))
attr_name = ''.join(
random.choice(string.ascii_letters + string.digits)
for i in range(4))
attr_value = ''.join(
random.choice(string.ascii_letters + string.digits)
for i in range(8))
msg_array.append(
ams_msg(data=msg_txt, attributes={attr_name: attr_value}))
hash_obj = hashlib.md5(msg_txt + attr_name + attr_value)
msg_orig.add(hash_obj.hexdigest())
try:
msgs = ams.publish(cmd_options.topic,
msg_array,
timeout=cmd_options.timeout)
ackids = []
rcv_msg = set()
for id, msg in ams.pull_sub(cmd_options.subscription,
MSG_NUM - 1,
True,
timeout=cmd_options.timeout):
attr = msg.get_attr()
hash_obj = hashlib.md5(msg.get_data() + attr.keys()[0] +
attr.values()[0])
rcv_msg.add(hash_obj.hexdigest())
if ackids:
ams.ack_sub(cmd_options.subscription,
ackids,
timeout=cmd_options.timeout)
ams.delete_topic(cmd_options.topic, timeout=cmd_options.timeout)
ams.delete_sub(cmd_options.subscription, timeout=cmd_options.timeout)
except AmsException as e:
nagios.writeCriticalMessage(e.msg)
nagios.setCode(nagios.CRITICAL)
print(nagios.getMsg())
raise SystemExit(nagios.getCode())
if msg_orig != rcv_msg:
nagios.writeCriticalMessage("Messages received incorrectly.")
nagios.setCode(nagios.CRITICAL)
print(nagios.getMsg())
raise SystemExit(nagios.getCode())
def main():
MSG_NUM = 100
MSG_SIZE = 500
TIMEOUT = 180
parser = ArgumentParser(description="Nagios sensor for AMS")
parser.add_argument('-H', dest='host', type=str, default='messaging-devel.argo.grnet.gr', help='FQDN of AMS Service')
parser.add_argument('--token', type=str, required=True, help='Given token')
parser.add_argument('--project', type=str, required=True, help='Project registered in AMS Service')
parser.add_argument('--topic', type=str, default='nagios_sensor_topic', help='Given topic')
parser.add_argument('--subscription', type=str, default='nagios_sensor_sub', help='Subscription name')
parser.add_argument('-t', dest='timeout', type=int, default=TIMEOUT, help='Timeout')
cmd_options = parser.parse_args()
nagios = NagiosResponse("All messages received correctly.")
ams = ArgoMessagingService(endpoint=cmd_options.host, token=cmd_options.token, project=cmd_options.project)
try:
if ams.has_topic(cmd_options.topic, timeout=cmd_options.timeout):
ams.delete_topic(cmd_options.topic, timeout=cmd_options.timeout)
if ams.has_sub(cmd_options.subscription, timeout=cmd_options.timeout):
ams.delete_sub(cmd_options.subscription, timeout=cmd_options.timeout)
ams.create_topic(cmd_options.topic, timeout=cmd_options.timeout)
ams.create_sub(cmd_options.subscription, cmd_options.topic, timeout=cmd_options.timeout)
except AmsException as e:
nagios.writeCriticalMessage(e.msg)
nagios.setCode(nagios.CRITICAL)
print(nagios.getMsg())
raise SystemExit(nagios.getCode())
ams_msg = AmsMessage()
msg_orig = set()
msg_array = []
for i in range(1, MSG_NUM):
msg_txt = ''.join(random.choice(string.ascii_letters + string.digits) for i in range(MSG_SIZE))
attr_name = ''.join(random.choice(string.ascii_letters + string.digits) for i in range(4))
attr_value = ''.join(random.choice(string.ascii_letters + string.digits) for i in range(8))
msg_array.append(ams_msg(data=msg_txt, attributes={attr_name: attr_value}))
hash_obj = hashlib.md5(msg_txt + attr_name + attr_value)
msg_orig.add(hash_obj.hexdigest())
try:
msgs = ams.publish(cmd_options.topic, msg_array, timeout=cmd_options.timeout)
ackids = []
rcv_msg = set()
for id, msg in ams.pull_sub(cmd_options.subscription, MSG_NUM - 1, True, timeout=cmd_options.timeout):
attr = msg.get_attr()
hash_obj = hashlib.md5(msg.get_data() + attr.keys()[0] + attr.values()[0])
rcv_msg.add(hash_obj.hexdigest())
if ackids:
ams.ack_sub(cmd_options.subscription, ackids, timeout=cmd_options.timeout)
ams.delete_topic(cmd_options.topic, timeout=cmd_options.timeout)
ams.delete_sub(cmd_options.subscription, timeout=cmd_options.timeout)
except AmsException as e:
nagios.writeCriticalMessage(e.msg)
nagios.setCode(nagios.CRITICAL)
print(nagios.getMsg())
raise SystemExit(nagios.getCode())
if msg_orig != rcv_msg:
nagios.writeCriticalMessage("Messages received incorrectly.")
nagios.setCode(nagios.CRITICAL)
print(nagios.getMsg())
raise SystemExit(nagios.getCode())
class Ssm2(stomp.ConnectionListener):
"""Minimal SSM implementation."""
# Schema for the dirq message queue.
QSCHEMA = {'body': 'string', 'signer': 'string', 'empaid': 'string?'}
REJECT_SCHEMA = {'body': 'string', 'signer': 'string?',
'empaid': 'string?', 'error': 'string'}
CONNECTION_TIMEOUT = 10
# Messaging protocols
STOMP_MESSAGING = 'STOMP'
AMS_MESSAGING = 'AMS'
def __init__(self, hosts_and_ports, qpath, cert, key, dest=None, listen=None,
capath=None, check_crls=False, use_ssl=False, username=None, password=None,
enc_cert=None, verify_enc_cert=True, pidfile=None, path_type='dirq',
protocol=STOMP_MESSAGING, project=None, token=''):
"""Create an SSM2 object.
If a listen value is supplied, this SSM2 will be a receiver.
"""
self._conn = None
self._last_msg = None
self._brokers = hosts_and_ports
self._cert = cert
self._key = key
self._enc_cert = enc_cert
self._capath = capath
self._check_crls = check_crls
self._user = username
self._pwd = password
self._use_ssl = use_ssl
# use pwd auth if we're supplied both user and pwd
self._use_pwd = username is not None and password is not None
self.connected = False
self._listen = listen
self._dest = dest
self._valid_dns = []
self._pidfile = pidfile
# Used to differentiate between STOMP and AMS methods
self._protocol = protocol
# Used when interacting with an Argo Messaging Service
self._project = project
self._token = token
if self._protocol == Ssm2.AMS_MESSAGING:
if ArgoMessagingService is None:
raise ImportError(
"The Python package argo_ams_library must be installed to "
"use AMS. Please install or use STOMP."
)
self._ams = ArgoMessagingService(endpoint=self._brokers[0],
token=self._token,
cert=self._cert,
key=self._key,
project=self._project)
# create the filesystem queues for accepted and rejected messages
if dest is not None and listen is None:
# Determine what sort of outgoing structure to make
if path_type == 'dirq':
if QueueSimple is None:
raise ImportError("dirq path_type requested but the dirq "
"module wasn't found.")
self._outq = QueueSimple(qpath)
elif path_type == 'directory':
self._outq = MessageDirectory(qpath)
else:
raise Ssm2Exception('Unsupported path_type variable.')
elif listen is not None:
inqpath = os.path.join(qpath, 'incoming')
rejectqpath = os.path.join(qpath, 'reject')
# Receivers must use the dirq module, so make a quick sanity check
# that dirq is installed.
if Queue is None:
raise ImportError("Receiving SSMs must use dirq, but the dirq "
"module wasn't found.")
self._inq = Queue(inqpath, schema=Ssm2.QSCHEMA)
self._rejectq = Queue(rejectqpath, schema=Ssm2.REJECT_SCHEMA)
else:
raise Ssm2Exception('SSM must be either producer or consumer.')
# check that the cert and key match
if not crypto.check_cert_key(self._cert, self._key):
raise Ssm2Exception('Cert and key don\'t match.')
# Check that the certificate has not expired.
if not crypto.verify_cert_date(self._cert):
raise Ssm2Exception('Certificate %s has expired or will expire '
'within a day.' % self._cert)
# check the server certificate provided
if enc_cert is not None:
log.info('Messages will be encrypted using %s', enc_cert)
if not os.path.isfile(self._enc_cert):
raise Ssm2Exception('Specified certificate file does not exist: %s.' % self._enc_cert)
# Check that the encyption certificate has not expired.
if not crypto.verify_cert_date(enc_cert):
raise Ssm2Exception(
'Encryption certificate %s has expired or will expire '
'within a day. Please obtain the new one from the final '
'server receiving your messages.' % enc_cert
)
if verify_enc_cert:
if not crypto.verify_cert_path(self._enc_cert, self._capath, self._check_crls):
raise Ssm2Exception('Failed to verify server certificate %s against CA path %s.'
% (self._enc_cert, self._capath))
# If the overall SSM log level is info, we want to only
# see entries from stomp.py and connectionpool at WARNING and above.
if logging.getLogger("ssm.ssm2").getEffectiveLevel() == logging.INFO:
logging.getLogger("stomp.py").setLevel(logging.WARNING)
logging.getLogger("requests.packages.urllib3.connectionpool"
).setLevel(logging.WARNING)
# If the overall SSM log level is debug, we want to only
# see entries from stomp.py and connectionpool at INFO above.
elif logging.getLogger("ssm.ssm2").getEffectiveLevel() == logging.DEBUG:
logging.getLogger("stomp.py").setLevel(logging.INFO)
logging.getLogger("requests.packages.urllib3.connectionpool"
).setLevel(logging.INFO)
def set_dns(self, dn_list):
"""Set the list of DNs which are allowed to sign incoming messages."""
self._valid_dns = dn_list
##########################################################################
# Methods called by stomppy
##########################################################################
def on_send(self, frame, unused_body=None):
"""Log sending of message with empaid if available.
Called by stomppy when a message is sent. 'unused_body' is only present
to have a backward compatible method signature for stomp.py v3.1.X
"""
try:
# Try the stomp.py v4 way first
empaid = frame.headers['empa-id']
except KeyError:
# Then you are most likely using stomp.py v4.
# on_send is now triggered on non message frames
# (such as 'CONNECT' frames) and as such without an empa-id.
empaid = 'no empa-id'
except AttributeError:
# Then you are likely using stomp.py v3
empaid = frame['empa-id']
log.debug('Sent message: %s', empaid)
def on_message(self, headers, body):
"""Handle the message according to its content and headers.
Called by stomppy when a message is received.
"""
try:
empaid = headers['empa-id']
if empaid == 'ping': # ignore ping message
log.info('Received ping message.')
return
except KeyError:
empaid = 'noid'
log.info("Received message. ID = %s", empaid)
# Save the message to either accept or reject queue.
self._save_msg_to_queue(body, empaid)
def on_error(self, headers, body):
"""Log error messages.
Called by stomppy when an error frame is received.
"""
if 'No user for client certificate: ' in headers['message']:
log.error('The following certificate is not authorised: %s',
headers['message'].split(':')[1])
else:
log.error('Error message received: %s', body)
def on_connected(self, unused_headers, unused_body):
"""Track the connection.
Called by stomppy when a connection is established.
"""
self.connected = True
log.info('Connected.')
def on_disconnected(self):
"""Log disconnection and set 'connected' to 'False'.
Called by stomppy when disconnected from the broker.
"""
log.info('Disconnected from broker.')
self.connected = False
def on_receipt(self, headers, unused_body):
"""Log receipt of message by broker and set '_last_msg'.
Called by stomppy when the broker acknowledges receipt of a message.
"""
log.info('Broker received message: %s', headers['receipt-id'])
self._last_msg = headers['receipt-id']
def on_receiver_loop_completed(self, _unused_headers, _unused_body):
"""Log receiver loop complete for debug only.
Called by stompy when the receiver loop ends. This is usually triggered
as part of a disconnect.
"""
log.debug('on_receiver_loop_completed called.')
##########################################################################
# Message handling methods
##########################################################################
def _handle_msg(self, text):
"""Deal with the raw message contents appropriately.
Namely:
- decrypt if necessary
- verify signature
- Return plain-text message, signer's DN and an error/None.
"""
if text is None or text == '':
warning = 'Empty text passed to _handle_msg.'
log.warn(warning)
return None, None, warning
# if not text.startswith('MIME-Version: 1.0'):
# raise Ssm2Exception('Not a valid message.')
# encrypted - this could be nicer
if 'application/pkcs7-mime' in text or 'application/x-pkcs7-mime' in text:
try:
text = crypto.decrypt(text, self._cert, self._key)
except crypto.CryptoException as e:
error = 'Failed to decrypt message: %s' % e
log.error(error)
return None, None, error
# always signed
try:
message, signer = crypto.verify(text, self._capath, self._check_crls)
except crypto.CryptoException as e:
error = 'Failed to verify message: %s' % e
log.error(error)
return None, None, error
if signer not in self._valid_dns:
warning = 'Signer not in valid DNs list: %s' % signer
log.warn(warning)
return None, signer, warning
else:
log.info('Valid signer: %s', signer)
return message, signer, None
def _save_msg_to_queue(self, body, empaid):
"""Extract message contents and add to the accept or reject queue."""
extracted_msg, signer, err_msg = self._handle_msg(body)
try:
# If the message is empty or the error message is not empty
# then reject the message.
if extracted_msg is None or err_msg is not None:
if signer is None: # crypto failed
signer = 'Not available.'
elif extracted_msg is not None:
# If there is a signer then it was rejected for not being
# in the DNs list, so we can use the extracted msg, which
# allows the msg to be reloaded if needed.
body = extracted_msg
log.warn("Message rejected: %s", err_msg)
name = self._rejectq.add({'body': body,
'signer': signer,
'empaid': empaid,
'error': err_msg})
log.info("Message saved to reject queue as %s", name)
else: # message verified ok
name = self._inq.add({'body': extracted_msg,
'signer': signer,
'empaid': empaid})
log.info("Message saved to incoming queue as %s", name)
except (IOError, OSError) as error:
log.error('Failed to read or write file: %s', error)
def _send_msg(self, message, msgid):
"""Send one message using stomppy.
The message will be signed using the host cert and key. If an
encryption certificate has been supplied, it will also be encrypted.
"""
log.info('Sending message: %s', msgid)
headers = {'destination': self._dest, 'receipt': msgid,
'empa-id': msgid}
if message is not None:
to_send = crypto.sign(message, self._cert, self._key)
if self._enc_cert is not None:
to_send = crypto.encrypt(to_send, self._enc_cert)
else:
to_send = ''
try:
# Try using the v4 method signiture
self._conn.send(self._dest, to_send, headers=headers)
except TypeError:
# If it fails, use the v3 metod signiture
self._conn.send(to_send, headers=headers)
def _send_msg_ams(self, text, msgid):
"""Send one message using AMS, returning the AMS ID of the mesage.
The message will be signed using the host cert and key. If an
encryption certificate has been supplied, the message will also be
encrypted.
"""
log.info('Sending message: %s', msgid)
if text is not None:
# First we sign the message
to_send = crypto.sign(text, self._cert, self._key)
# Possibly encrypt the message.
if self._enc_cert is not None:
to_send = crypto.encrypt(to_send, self._enc_cert)
# Then we need to wrap text up as an AMS Message.
message = AmsMessage(data=to_send,
attributes={'empaid': msgid}).dict()
argo_response = self._ams.publish(self._dest, message, retry=3)
return argo_response['messageIds'][0]
def pull_msg_ams(self):
"""Pull 1 message from the AMS and acknowledge it."""
if self._protocol != Ssm2.AMS_MESSAGING:
# Then this method should not be called,
# raise an exception if it is.
raise Ssm2Exception('pull_msg_ams called, '
'but protocol not set to AMS. '
'Protocol: %s' % self._protocol)
# This method is setup so that you could pull down and
# acknowledge more than one message at a time, but
# currently there is no use case for it.
messages_to_pull = 1
# ack id's will be stored in this list and then acknowledged
ackids = []
for msg_ack_id, msg in self._ams.pull_sub(self._listen,
messages_to_pull,
retry=3):
# Get the AMS message id
msgid = msg.get_msgid()
# Get the SSM dirq id
try:
empaid = msg.get_attr().get('empaid')
except AttributeError:
# A message without an empaid could be received if it wasn't
# sent via the SSM, we need to pull down that message
# to prevent it blocking the message queue.
log.debug("Message %s has no empaid.", msgid)
empaid = "N/A"
# get the message body
body = msg.get_data()
log.info('Received message. ID = %s, Argo ID = %s', empaid, msgid)
# Save the message to either accept or reject queue.
self._save_msg_to_queue(body, empaid)
# The message has either been saved or there's been a problem with
# writing it out, but either way we add the ack ID to the list
# of those to be acknowledged so that we don't get stuck reading
# the same message.
ackids.append(msg_ack_id)
# pass list of extracted ackIds to AMS Service so that
# it can move the offset for the next subscription pull
# (basically acknowledging pulled messages)
if ackids:
self._ams.ack_sub(self._listen, ackids, retry=3)
def send_ping(self):
"""Perform connection stay-alive steps.
If a STOMP connection is left open with no activity for an hour or
so, it stops responding. Stomppy 3.1.3 has two ways of handling
this, but stomppy 3.0.3 (EPEL 5 and 6) has neither.
To get around this, we begin and then abort a STOMP transaction to
keep the connection active.
"""
# Use time as transaction id to ensure uniqueness within each connection
transaction_id = str(time.time())
self._conn.begin({'transaction': transaction_id})
self._conn.abort({'transaction': transaction_id})
def has_msgs(self):
"""Return True if there are any messages in the outgoing queue."""
return self._outq.count() > 0
def send_all(self):
"""
Send all the messages in the outgoing queue.
Either via STOMP or HTTPS (to an Argo Message Broker).
"""
log.info('Found %s messages.', self._outq.count())
for msgid in self._outq:
if not self._outq.lock(msgid):
log.warn('Message was locked. %s will not be sent.', msgid)
continue
text = self._outq.get(msgid)
if self._protocol == Ssm2.STOMP_MESSAGING:
# Then we are sending to a STOMP message broker.
self._send_msg(text, msgid)
log.info('Waiting for broker to accept message.')
while self._last_msg is None:
if not self.connected:
raise Ssm2Exception('Lost connection.')
# Small sleep to avoid hammering the CPU
time.sleep(0.01)
log_string = "Sent %s" % msgid
elif self._protocol == Ssm2.AMS_MESSAGING:
# Then we are sending to an Argo Messaging Service instance.
argo_id = self._send_msg_ams(text, msgid)
log_string = "Sent %s, Argo ID: %s" % (msgid, argo_id)
else:
# The SSM has been improperly configured
raise Ssm2Exception('Unknown messaging protocol: %s' %
self._protocol)
# log that the message was sent
log.info(log_string)
self._last_msg = None
self._outq.remove(msgid)
log.info('Tidying message directory.')
try:
# Remove empty dirs and unlock msgs older than 5 min (default)
self._outq.purge()
except OSError as e:
log.warn('OSError raised while purging message queue: %s', e)
###########################################################################
# Connection handling methods
###########################################################################
def _initialise_connection(self, host, port):
"""Create the self._connection object with the appropriate properties.
This doesn't start the connection.
"""
log.info("Established connection to %s, port %i", host, port)
if self._use_ssl:
log.info('Connecting using SSL...')
else:
log.warning("SSL connection not requested, your messages may be "
"intercepted.")
# _conn will use the default SSL version specified by stomp.py
self._conn = stomp.Connection([(host, port)],
use_ssl=self._use_ssl,
ssl_key_file=self._key,
ssl_cert_file=self._cert,
timeout=Ssm2.CONNECTION_TIMEOUT)
self._conn.set_listener('SSM', self)
def handle_connect(self):
"""Connect to broker.
Assuming that the SSM has retrieved the details of the broker or
brokers it wants to connect to, connect to one.
If more than one is in the list self._network_brokers, try to
connect to each in turn until successful.
"""
if self._protocol == Ssm2.AMS_MESSAGING:
log.debug('handle_connect called for AMS, doing nothing.')
return
log.info("Using stomp.py version %s.%s.%s.", *stomp.__version__)
for host, port in self._brokers:
self._initialise_connection(host, port)
try:
self.start_connection()
break
except ConnectFailedException as e:
# ConnectFailedException doesn't provide a message.
log.warn('Failed to connect to %s:%s.', host, port)
except Ssm2Exception as e:
log.warn('Failed to connect to %s:%s: %s', host, port, e)
if not self.connected:
raise Ssm2Exception('Attempts to start the SSM failed. The system will exit.')
def handle_disconnect(self):
"""Attempt to reconnect using the same method as when starting up."""
if self._protocol == Ssm2.AMS_MESSAGING:
log.debug('handle_disconnect called for AMS, doing nothing.')
return
self.connected = False
# Shut down properly
self.close_connection()
# Sometimes the SSM will reconnect to the broker before it's properly
# shut down! This prevents that.
time.sleep(2)
# Try again according to the same logic as the initial startup
try:
self.handle_connect()
except Ssm2Exception:
self.connected = False
# If reconnection fails, admit defeat.
if not self.connected:
err_msg = 'Reconnection attempts failed and have been abandoned.'
raise Ssm2Exception(err_msg)
def start_connection(self):
"""Start existing connection and subscribe to the relevant topics.
If the timeout is reached without receiving confirmation of
connection, raise an exception.
"""
if self._protocol == Ssm2.AMS_MESSAGING:
log.debug('start_connection called for AMS, doing nothing.')
return
if self._conn is None:
raise Ssm2Exception('Called start_connection() before a \
connection object was initialised.')
self._conn.start()
self._conn.connect(wait=False)
i = 0
while not self.connected:
time.sleep(0.1)
if i > Ssm2.CONNECTION_TIMEOUT * 10:
err = 'Timed out while waiting for connection. '
err += 'Check the connection details.'
raise Ssm2Exception(err)
i += 1
if self._dest is not None:
log.info('Will send messages to: %s', self._dest)
if self._listen is not None:
# Use a static ID for the subscription ID because we only ever have
# one subscription within a connection and ID is only considered
# to differentiate subscriptions within a connection.
self._conn.subscribe(destination=self._listen, id=1, ack='auto')
log.info('Subscribing to: %s', self._listen)
def close_connection(self):
"""Close the connection.
This is important because it runs in a separate thread, so it can
outlive the main process if it is not ended.
"""
if self._protocol == Ssm2.AMS_MESSAGING:
log.debug('close_connection called for AMS, doing nothing.')
return
try:
self._conn.disconnect()
except (stomp.exception.NotConnectedException, socket.error):
self._conn = None
except AttributeError:
# AttributeError if self._connection is None already
pass
log.info('SSM connection ended.')
def startup(self):
"""Create the pidfile then start the connection."""
if self._pidfile is not None:
try:
f = open(self._pidfile, 'w')
f.write(str(os.getpid()))
f.write('\n')
f.close()
except IOError as e:
log.warn('Failed to create pidfile %s: %s', self._pidfile, e)
self.handle_connect()
def shutdown(self):
"""Close the connection then remove the pidfile."""
self.close_connection()
if self._pidfile is not None:
try:
if os.path.exists(self._pidfile):
os.remove(self._pidfile)
else:
log.warn('pidfile %s not found.', self._pidfile)
except IOError as e:
log.warn('Failed to remove pidfile %s: %e', self._pidfile, e)
log.warn('SSM may not start again until it is removed.')
def main(args):
# set up the configuration object
config = dict()
# default values
config["bulk_size"] = 100
config["interval"] = 10
with open(args.ConfigPath, 'r') as f:
config = json.load(f)
# stream(console) handler
console_handler = logging.StreamHandler()
console_handler.setFormatter(logging.Formatter('%(asctime)s %(name)s[%(process)d]: %(levelname)s %(message)s'))
LOGGER.addHandler(console_handler)
if args.debug:
LOGGER.setLevel(logging.DEBUG)
else:
LOGGER.setLevel(logging.INFO)
# sys log handler
syslog_handler = logging.handlers.SysLogHandler(config["syslog_socket"])
syslog_handler.setFormatter(logging.Formatter('%(asctime)s %(name)s[%(process)d]: %(levelname)s %(message)s'))
if args.debug:
syslog_handler.setLevel(logging.DEBUG)
else:
syslog_handler.setLevel(logging.INFO)
syslog_handler.setLevel(logging.INFO)
LOGGER.addHandler(syslog_handler)
# start the process of republishing messages
ams_endpoint = "{}:{}".format(config["ams_host"], config["ams_port"])
ams = ArgoMessagingService(endpoint=ams_endpoint, token=config["ams_token"], project=config["ams_project"])
schema = avro.schema.Parse(open(config["avro_schema"], "rb").read())
while True:
start_time = time.time()
try:
consumed_msgs, last_msg_id = extract_messages(ams, config["ingest_subscription"], config["bulk_size"], schema, args.verify)
if last_msg_id == "-1":
LOGGER.info("No new messages")
time.sleep(config["interval"])
continue
LOGGER.debug("Consumed messages \n {}".format(consumed_msgs))
filtered_msgs = filter_messages(consumed_msgs, config["sites"])
LOGGER.debug("Filtered messages \n {}".format(filtered_msgs))
republish_messages(filtered_msgs, ams, args.verify)
# make sure that the acknowledgment happens
try:
# try to acknowledge
ams.ack_sub(config["ingest_subscription"], [last_msg_id], verify=args.verify)
except Exception as e:
# if the acknowledgment fails
LOGGER.critical("Retrying to acknowledge message {} after error {}".format(last_msg_id, str(e)))
while True:
try:
# consume again in order to refresh the TTL
ams.pull_sub(config["ingest_subscription"], config["bulk_size"], True, verify=args.verify)
# try to ack again using the msg_id from the first consumption
ams.ack_sub(config["ingest_subscription"], [last_msg_id], verify=args.verify)
break
except Exception as e:
LOGGER.critical(
"Retrying to acknowledge message {} after error {}".format(last_msg_id, str(e)))
time.sleep(config["interval"])
end_time = time.time()
LOGGER.info("Consumed {} and Republished {} messages. in {}".format(
len(consumed_msgs),
len(filtered_msgs),
end_time - start_time))
except Exception as e:
LOGGER.critical("Could not republish, {}".format(str(e)))
time.sleep(config["interval"])