def main():
parser = ArgumentParser(description="Simple AMS message publish example")
parser.add_argument('--host',
type=str,
default='messaging-devel.argo.grnet.gr',
help='FQDN of AMS Service')
parser.add_argument('--token', type=str, required=True, help='Given token')
parser.add_argument('--project',
type=str,
required=True,
help='Project registered in AMS Service')
parser.add_argument('--topic', type=str, required=True, help='Given topic')
args = parser.parse_args()
# initialize service with given token and project
ams = ArgoMessagingService(endpoint=args.host,
token=args.token,
project=args.project)
# ensure that topic is created in first run
try:
if not ams.has_topic(args.topic):
ams.create_topic(args.topic)
topic = ams.get_topic(args.topic, retobj=True)
except AmsException as e:
print e
raise SystemExit(1)
# publish one message to given topic. message is constructed with
# help of AmsMessage which accepts data and attributes keys.
# data is Base64 encoded, attributes is dictionary of arbitrary
# key/value pairs
msg = AmsMessage(data='foo1', attributes={'bar1': 'baz1'}).dict()
try:
ret = topic.publish(msg)
print ret
except AmsException as e:
print e
# publish a list of two messages to given topic. AmsMessage can also be
# used as a callable. publish() method accepts either one messages or
# list of messages
msg = AmsMessage()
msglist = [
msg(data='foo2', attributes={'bar2': 'baz2'}),
msg(data='foo3', attributes={'bar3': 'baz3'})
]
try:
ret = topic.publish(msglist)
print ret
except AmsException as e:
print e
def post_template_for_assessment(self, niftyId, msgId):
ams = ArgoMessagingService(endpoint=self.host, token=self.token, project=self.project)
msg = AmsMessage(data="", attributes={'NIFTY_APPLIANCE_ID': niftyId}).dict()
try:
ret = ams.publish(self.requestTopic, msg)
except AmsException as e:
print e
def publish_message(service, action):
"""
Send a message using argo messaging service when an action upon a service
takes place.
"""
service_id = str(service.get('id'))
service_name = service.get('name')
service_data = service.get('data', {})
ams = ArgoMessagingService(endpoint=AMS_ENDPOINT,
project=AMS_PROJECT,
token=AMS_TOKEN)
endpoint = '{0}/api/v2/ext-services/{1}'.format(get_root_url(), service_id)
# The value of the data property must be unicode in order to be
# encoded in base64 format in AmsMessage
data = json.dumps(service_data)
try:
if not ams.has_topic(AMS_TOPIC):
ams.create_topic(AMS_TOPIC)
except AmsException as e:
print e
raise SystemExit(1)
msg = AmsMessage(data=data,
attributes={
"method": action,
"service_id": service_id,
"service_name": service_name,
"endpoint": endpoint
}).dict()
try:
ret = ams.publish(AMS_TOPIC, msg)
print ret
except AmsException as e:
print e
def send(self, schema, msgtype, date, msglist):
def _avro_serialize(msg):
opened_schema = load_schema(schema)
avro_writer = DatumWriter(opened_schema)
bytesio = BytesIO()
encoder = BinaryEncoder(bytesio)
if isinstance(msg, list):
for m in msg:
avro_writer.write(m, encoder)
else:
avro_writer.write(msg, encoder)
return bytesio.getvalue()
if self.packsingle:
self.bulk = 1
msg = AmsMessage(attributes={
'partition_date': date,
'report': self.report,
'type': msgtype
},
data=_avro_serialize(msglist))
msgs = [msg]
else:
msgs = map(
lambda m: AmsMessage(attributes={
'partition_date': date,
'report': self.report,
'type': msgtype
},
data=_avro_serialize(m)), msglist)
if self._send(
self.logger, module_class_name(self), {
'ConnectionRetry'.lower(): self.retry,
'ConnectionTimeout'.lower(): self.timeout,
'ConnectionSleepRetry'.lower(): self.sleepretry
}, msgs, self.bulk, self):
return True
def post_assessment_results(self, niftyId, file_path):
ams = ArgoMessagingService(endpoint=self.host,
token=self.token,
project=self.project)
contents = Path(file_path).read_text()
msg = AmsMessage(data=contents,
attributes={
'NIFTY_APPLIANCE_ID': niftyId
}).dict()
try:
ret = ams.publish(self.resultTopic, msg)
logging.debug('[%s] %s: Results has been successfully pushed.',
niftyId, 'DEBUG')
except AmsException as e:
print e
def main():
parser = ArgumentParser(description="AMS message publish")
parser.add_argument('--host', type=str, default='', help='FQDN of AMS Service')
parser.add_argument('--token', type=str, default='', help='Given token')
parser.add_argument('--project', type=str, default='appdb-sec-test', help='Project registered in AMS Service')
parser.add_argument('--topic', type=str, default='VMISECURITY-REQUESTS', help='Given topic')
parser.add_argument('--id', type=str, default='', help='Appliance id')
args = parser.parse_args()
ams = ArgoMessagingService(endpoint=args.host, token=args.token, project=args.project)
msg = AmsMessage(data=" ", attributes={'NIFTY_APPLIANCE_ID': args.id}).dict()
try:
ret = ams.publish(args.topic, msg)
print ret
except AmsException as e:
print e
def testPushAndPull(self):
settings = ConfigParser.ConfigParser()
settings.read('../conf/argo.conf')
host = settings.get('AMS-GENERAL', 'host')
project = settings.get('AMS-GENERAL', 'project')
token = settings.get('AMS-GENERAL', 'token')
subscription = settings.get('REQUESTS', 'subscription')
topic = settings.get('REQUESTS', 'topic')
log = logging.getLogger("TestArgoMessageService.testPushAndPull")
log.debug("Host: {host}, Project: {project}, Topic: {topic}".format(
host=host, project=project, topic=topic))
ams = ArgoMessagingService(endpoint=host, token=token, project=project)
pullAllMessages(ams, subscription, log)
response = urllib2.urlopen(
"https://vmcaster.appdb.egi.eu/store/vappliance/demo.va.public/image.list",
timeout=5)
content = response.read()
msg = AmsMessage(data=content, attributes={'index': '1'}).dict()
try:
ret = ams.publish(topic, msg)
log.debug("Successfully published with ID:{id}".format(
id=ret['messageIds'][0]))
except AmsException as e:
print e
# Wait 10 seconds between pull and push
log.debug("Waiting 10s between push and pull")
time.sleep(10)
ackids = list()
pull_result = ams.pull_sub(subscription, 1)
if pull_result:
log.debug("Successfully pulled a message with ID:{id}".format(
id=pull_result[0][1].get_msgid()))
self.assertEquals(content, pull_result[0][1].get_data())
ackids.append(pull_result[0][1].get_msgid())
# Send Acknowledgement
if ackids:
ams.ack_sub(subscription, ackids)
def publish(config):
token = config.get("AUTH", "token")
host = config.get("AMS", "ams_host")
project = config.get("AMS", "ams_project")
topic = config.get("AMS", "ams_topic")
cert_path = config.get("AUTH", "cert_path")
key_path = config.get("AUTH", "key_path")
msg_file_path = config.get("AMS", "msg_file_path")
info_provider_path = config.get("AMS", "info_provider_path")
# initialize service
if token:
ams = ArgoMessagingService(endpoint=host, project=project, token=token)
else:
ams = ArgoMessagingService(endpoint=host,
project=project,
cert=cert_path,
key=key_path)
data = ''
if info_provider_path:
try:
data = subprocess.check_output([info_provider_path], shell=True)
except subprocess.CalledProcessError as cpe:
logger.error(cpe)
return 1
else:
try:
with open(msg_file_path, 'r') as ldif:
data = ldif.read()
except IOError as ioe:
logger.error(ioe)
return 1
msg = AmsMessage(data=data).dict()
try:
ret = ams.publish(topic, msg)
logger.info("Successfully published message at: %s, ret: %s" %
(topic, ret))
return 0
except AmsException as e:
logger.error("Failed to publish message: %s" % e)
return 1
def _send_msg_ams(self, text, msgid):
"""Send one message using AMS, returning the AMS ID of the mesage.
The message will be signed using the host cert and key. If an
encryption certificate has been supplied, the message will also be
encrypted.
"""
log.info('Sending message: %s', msgid)
if text is not None:
# First we sign the message
to_send = crypto.sign(text, self._cert, self._key)
# Possibly encrypt the message.
if self._enc_cert is not None:
to_send = crypto.encrypt(to_send, self._enc_cert)
# Then we need to wrap text up as an AMS Message.
message = AmsMessage(data=to_send,
attributes={'empaid': msgid}).dict()
argo_response = self._ams.publish(self._dest, message, retry=3)
return argo_response['messageIds'][0]
def publish(self, messages):
# messages = [{data:[{id:1},{state:'deployed'}],attributes=''}]
try:
if not self.ams.has_topic(self.pub_topic):
self.ams.create_topic(self.pub_topic)
except AmsException as e:
print(e)
raise SystemExit(1)
# publish one message to given topic. message is constructed with
# help of AmsMessage which accepts data and attributes keys.
# data is Base64 encoded, attributes is dictionary of arbitrary
# key/value pairs
msg = AmsMessage()
msglist = []
for message in messages:
msglist.append(msg(data=json.dumps(message['data']),
attributes={}))
try:
ret = self.ams.publish(self.pub_topic, msglist)
print(ret)
except AmsException as e:
print(e)
def main():
parser = ArgumentParser(description="Simple AMS message publish example")
parser.add_argument('--host',
type=str,
default='messaging-devel.argo.grnet.gr',
help='FQDN of AMS Service')
parser.add_argument('--token', type=str, required=True, help='Given token')
parser.add_argument('--project',
type=str,
required=True,
help='Project registered in AMS Service')
parser.add_argument('--topic', type=str, required=True, help='Given topic')
parser.add_argument('--subscription',
type=str,
required=True,
help='Subscription name')
parser.add_argument('--nummsgs',
type=int,
default=3,
help='Number of messages to pull and ack')
args = parser.parse_args()
ams = ArgoMessagingService(endpoint=args.host,
token=args.token,
project=args.project)
# static sleep between retry attempts
msg = AmsMessage(data='foo1', attributes={'bar1': 'baz1'}).dict()
try:
ret = ams.publish(args.topic, msg, retry=3, retrysleep=5, timeout=5)
print(ret)
except AmsException as e:
print(e)
# iptables -A OUTPUT -d messaging-devel.argo.grnet.gr -j DROP
ackids = list()
for id, msg in ams.pull_sub(args.subscription,
args.nummsgs,
retry=3,
retrysleep=5,
timeout=5):
data = msg.get_data()
msgid = msg.get_msgid()
attr = msg.get_attr()
print('msgid={0}, data={1}, attr={2}'.format(msgid, data, attr))
ackids.append(id)
if ackids:
ams.ack_sub(args.subscription,
ackids,
retry=3,
retrysleep=5,
timeout=5)
# backoff with each next retry attempt exponentially longer
msg = AmsMessage(data='foo2', attributes={'bar2': 'baz2'}).dict()
try:
ret = ams.publish(args.topic, msg, retry=3, retrybackoff=5, timeout=5)
print(ret)
except AmsException as e:
print(e)
# iptables -A OUTPUT -d messaging-devel.argo.grnet.gr -j DROP
ackids = list()
for id, msg in ams.pull_sub(args.subscription,
args.nummsgs,
retrybackoff=3,
retrysleep=5,
timeout=5):
data = msg.get_data()
msgid = msg.get_msgid()
attr = msg.get_attr()
print('msgid={0}, data={1}, attr={2}'.format(msgid, data, attr))
ackids.append(id)
if ackids:
ams.ack_sub(args.subscription, ackids)
# static sleep between retry attempts. this example uses consume context
# method that pull and acks msgs in one call.
msg = AmsMessage(data='foo3', attributes={'bar3': 'baz3'}).dict()
try:
ret = ams.publish(args.topic, msg, retry=3, retrysleep=5, timeout=5)
print(ret)
except AmsException as e:
print(e)
try:
msgs = ams.pullack_sub(args.subscription,
args.nummsgs,
retry=3,
retrysleep=5,
timeout=5)
for msg in msgs:
data = msg.get_data()
msgid = msg.get_msgid()
attr = msg.get_attr()
print('msgid={0}, data={1}, attr={2}'.format(msgid, data, attr))
except AmsException as e:
print(e)
def send_all(self):
'''
Send all the messages in the outgoing queue.
Either via STOMP or HTTPS (to an Argo Message Broker).
'''
log.info('Found %s messages.', self._outq.count())
for msgid in self._outq:
if not self._outq.lock(msgid):
log.warn('Message was locked. %s will not be sent.', msgid)
continue
text = self._outq.get(msgid)
if self._protocol == Ssm2.STOMP_MESSAGING:
# Then we are sending to a STOMP message broker.
self._send_msg(text, msgid)
log.info('Waiting for broker to accept message.')
while self._last_msg is None:
if not self.connected:
raise Ssm2Exception('Lost connection.')
log_string = "Sent %s" % msgid
elif self._protocol == Ssm2.AMS_MESSAGING:
# Then we are sending to an Argo Messaging Service instance.
if text is not None:
# First we sign the message
to_send = crypto.sign(text, self._cert, self._key)
# Possibly encrypt the message.
if self._enc_cert is not None:
to_send = crypto.encrypt(to_send, self._enc_cert)
# Then we need to wrap text up as an AMS Message.
message = AmsMessage(data=to_send,
attributes={
'empaid': msgid
}).dict()
argo_response = self._ams.publish(self._dest, message)
argo_id = argo_response['messageIds'][0]
log_string = "Sent %s, Argo ID: %s" % (msgid, argo_id)
else:
# The SSM has been improperly configured
raise Ssm2Exception('Unknown messaging protocol: %s' %
self._protocol)
time.sleep(0.1)
# log that the message was sent
log.info(log_string)
self._last_msg = None
self._outq.remove(msgid)
log.info('Tidying message directory.')
try:
# Remove empty dirs and unlock msgs older than 5 min (default)
self._outq.purge()
except OSError, e:
log.warn('OSError raised while purging message queue: %s', e)
def main():
MSG_NUM = 100
MSG_SIZE = 500
TIMEOUT = 180
parser = ArgumentParser(description="Nagios sensor for AMS")
parser.add_argument('-H',
dest='host',
type=str,
default='messaging-devel.argo.grnet.gr',
help='FQDN of AMS Service')
parser.add_argument('--token', type=str, required=True, help='Given token')
parser.add_argument('--project',
type=str,
required=True,
help='Project registered in AMS Service')
parser.add_argument('--topic',
type=str,
default='nagios_sensor_topic',
help='Given topic')
parser.add_argument('--subscription',
type=str,
default='nagios_sensor_sub',
help='Subscription name')
parser.add_argument('-t',
dest='timeout',
type=int,
default=TIMEOUT,
help='Timeout')
cmd_options = parser.parse_args()
nagios = NagiosResponse("All messages received correctly.")
ams = ArgoMessagingService(endpoint=cmd_options.host,
token=cmd_options.token,
project=cmd_options.project)
try:
if ams.has_topic(cmd_options.topic, timeout=cmd_options.timeout):
ams.delete_topic(cmd_options.topic, timeout=cmd_options.timeout)
if ams.has_sub(cmd_options.subscription, timeout=cmd_options.timeout):
ams.delete_sub(cmd_options.subscription,
timeout=cmd_options.timeout)
ams.create_topic(cmd_options.topic, timeout=cmd_options.timeout)
ams.create_sub(cmd_options.subscription,
cmd_options.topic,
timeout=cmd_options.timeout)
except AmsException as e:
nagios.writeCriticalMessage(e.msg)
nagios.setCode(nagios.CRITICAL)
print(nagios.getMsg())
raise SystemExit(nagios.getCode())
ams_msg = AmsMessage()
msg_orig = set()
msg_array = []
for i in range(1, MSG_NUM):
msg_txt = ''.join(
random.choice(string.ascii_letters + string.digits)
for i in range(MSG_SIZE))
attr_name = ''.join(
random.choice(string.ascii_letters + string.digits)
for i in range(4))
attr_value = ''.join(
random.choice(string.ascii_letters + string.digits)
for i in range(8))
msg_array.append(
ams_msg(data=msg_txt, attributes={attr_name: attr_value}))
hash_obj = hashlib.md5(msg_txt + attr_name + attr_value)
msg_orig.add(hash_obj.hexdigest())
try:
msgs = ams.publish(cmd_options.topic,
msg_array,
timeout=cmd_options.timeout)
ackids = []
rcv_msg = set()
for id, msg in ams.pull_sub(cmd_options.subscription,
MSG_NUM - 1,
True,
timeout=cmd_options.timeout):
attr = msg.get_attr()
hash_obj = hashlib.md5(msg.get_data() + attr.keys()[0] +
attr.values()[0])
rcv_msg.add(hash_obj.hexdigest())
if ackids:
ams.ack_sub(cmd_options.subscription,
ackids,
timeout=cmd_options.timeout)
ams.delete_topic(cmd_options.topic, timeout=cmd_options.timeout)
ams.delete_sub(cmd_options.subscription, timeout=cmd_options.timeout)
except AmsException as e:
nagios.writeCriticalMessage(e.msg)
nagios.setCode(nagios.CRITICAL)
print(nagios.getMsg())
raise SystemExit(nagios.getCode())
if msg_orig != rcv_msg:
nagios.writeCriticalMessage("Messages received incorrectly.")
nagios.setCode(nagios.CRITICAL)
print(nagios.getMsg())
raise SystemExit(nagios.getCode())
def post_assessment_results(self, niftyId, msgId, file_path, base_mpuri):
ams = ArgoMessagingService(endpoint=self.host, token=self.token, project=self.project)
contents = Path(file_path).read_text()
msg = AmsMessage(data=contents, attributes={'NIFTY_APPLIANCE_ID': niftyId, 'REQUEST_MESSAGE_ID': msgId, 'BASE_MPURI': base_mpuri}).dict()
ret = ams.publish(self.resultTopic, msg)
logging.debug('[%s] %s: Results has been successfully pushed.', niftyId, 'DEBUG')
