def main(): parser = ArgumentParser(description="Simple AMS message publish example") parser.add_argument('--host', type=str, default='messaging-devel.argo.grnet.gr', help='FQDN of AMS Service') parser.add_argument('--token', type=str, required=True, help='Given token') parser.add_argument('--project', type=str, required=True, help='Project registered in AMS Service') parser.add_argument('--topic', type=str, required=True, help='Given topic') args = parser.parse_args() # initialize service with given token and project ams = ArgoMessagingService(endpoint=args.host, token=args.token, project=args.project) # ensure that topic is created in first run try: if not ams.has_topic(args.topic): ams.create_topic(args.topic) topic = ams.get_topic(args.topic, retobj=True) except AmsException as e: print e raise SystemExit(1) # publish one message to given topic. message is constructed with # help of AmsMessage which accepts data and attributes keys. # data is Base64 encoded, attributes is dictionary of arbitrary # key/value pairs msg = AmsMessage(data='foo1', attributes={'bar1': 'baz1'}).dict() try: ret = topic.publish(msg) print ret except AmsException as e: print e # publish a list of two messages to given topic. AmsMessage can also be # used as a callable. publish() method accepts either one messages or # list of messages msg = AmsMessage() msglist = [ msg(data='foo2', attributes={'bar2': 'baz2'}), msg(data='foo3', attributes={'bar3': 'baz3'}) ] try: ret = topic.publish(msglist) print ret except AmsException as e: print e
def post_template_for_assessment(self, niftyId, msgId): ams = ArgoMessagingService(endpoint=self.host, token=self.token, project=self.project) msg = AmsMessage(data="", attributes={'NIFTY_APPLIANCE_ID': niftyId}).dict() try: ret = ams.publish(self.requestTopic, msg) except AmsException as e: print e
def publish_message(service, action): """ Send a message using argo messaging service when an action upon a service takes place. """ service_id = str(service.get('id')) service_name = service.get('name') service_data = service.get('data', {}) ams = ArgoMessagingService(endpoint=AMS_ENDPOINT, project=AMS_PROJECT, token=AMS_TOKEN) endpoint = '{0}/api/v2/ext-services/{1}'.format(get_root_url(), service_id) # The value of the data property must be unicode in order to be # encoded in base64 format in AmsMessage data = json.dumps(service_data) try: if not ams.has_topic(AMS_TOPIC): ams.create_topic(AMS_TOPIC) except AmsException as e: print e raise SystemExit(1) msg = AmsMessage(data=data, attributes={ "method": action, "service_id": service_id, "service_name": service_name, "endpoint": endpoint }).dict() try: ret = ams.publish(AMS_TOPIC, msg) print ret except AmsException as e: print e
def send(self, schema, msgtype, date, msglist): def _avro_serialize(msg): opened_schema = load_schema(schema) avro_writer = DatumWriter(opened_schema) bytesio = BytesIO() encoder = BinaryEncoder(bytesio) if isinstance(msg, list): for m in msg: avro_writer.write(m, encoder) else: avro_writer.write(msg, encoder) return bytesio.getvalue() if self.packsingle: self.bulk = 1 msg = AmsMessage(attributes={ 'partition_date': date, 'report': self.report, 'type': msgtype }, data=_avro_serialize(msglist)) msgs = [msg] else: msgs = map( lambda m: AmsMessage(attributes={ 'partition_date': date, 'report': self.report, 'type': msgtype }, data=_avro_serialize(m)), msglist) if self._send( self.logger, module_class_name(self), { 'ConnectionRetry'.lower(): self.retry, 'ConnectionTimeout'.lower(): self.timeout, 'ConnectionSleepRetry'.lower(): self.sleepretry }, msgs, self.bulk, self): return True
def post_assessment_results(self, niftyId, file_path): ams = ArgoMessagingService(endpoint=self.host, token=self.token, project=self.project) contents = Path(file_path).read_text() msg = AmsMessage(data=contents, attributes={ 'NIFTY_APPLIANCE_ID': niftyId }).dict() try: ret = ams.publish(self.resultTopic, msg) logging.debug('[%s] %s: Results has been successfully pushed.', niftyId, 'DEBUG') except AmsException as e: print e
def main(): parser = ArgumentParser(description="AMS message publish") parser.add_argument('--host', type=str, default='', help='FQDN of AMS Service') parser.add_argument('--token', type=str, default='', help='Given token') parser.add_argument('--project', type=str, default='appdb-sec-test', help='Project registered in AMS Service') parser.add_argument('--topic', type=str, default='VMISECURITY-REQUESTS', help='Given topic') parser.add_argument('--id', type=str, default='', help='Appliance id') args = parser.parse_args() ams = ArgoMessagingService(endpoint=args.host, token=args.token, project=args.project) msg = AmsMessage(data=" ", attributes={'NIFTY_APPLIANCE_ID': args.id}).dict() try: ret = ams.publish(args.topic, msg) print ret except AmsException as e: print e
def testPushAndPull(self): settings = ConfigParser.ConfigParser() settings.read('../conf/argo.conf') host = settings.get('AMS-GENERAL', 'host') project = settings.get('AMS-GENERAL', 'project') token = settings.get('AMS-GENERAL', 'token') subscription = settings.get('REQUESTS', 'subscription') topic = settings.get('REQUESTS', 'topic') log = logging.getLogger("TestArgoMessageService.testPushAndPull") log.debug("Host: {host}, Project: {project}, Topic: {topic}".format( host=host, project=project, topic=topic)) ams = ArgoMessagingService(endpoint=host, token=token, project=project) pullAllMessages(ams, subscription, log) response = urllib2.urlopen( "https://vmcaster.appdb.egi.eu/store/vappliance/demo.va.public/image.list", timeout=5) content = response.read() msg = AmsMessage(data=content, attributes={'index': '1'}).dict() try: ret = ams.publish(topic, msg) log.debug("Successfully published with ID:{id}".format( id=ret['messageIds'][0])) except AmsException as e: print e # Wait 10 seconds between pull and push log.debug("Waiting 10s between push and pull") time.sleep(10) ackids = list() pull_result = ams.pull_sub(subscription, 1) if pull_result: log.debug("Successfully pulled a message with ID:{id}".format( id=pull_result[0][1].get_msgid())) self.assertEquals(content, pull_result[0][1].get_data()) ackids.append(pull_result[0][1].get_msgid()) # Send Acknowledgement if ackids: ams.ack_sub(subscription, ackids)
def publish(config): token = config.get("AUTH", "token") host = config.get("AMS", "ams_host") project = config.get("AMS", "ams_project") topic = config.get("AMS", "ams_topic") cert_path = config.get("AUTH", "cert_path") key_path = config.get("AUTH", "key_path") msg_file_path = config.get("AMS", "msg_file_path") info_provider_path = config.get("AMS", "info_provider_path") # initialize service if token: ams = ArgoMessagingService(endpoint=host, project=project, token=token) else: ams = ArgoMessagingService(endpoint=host, project=project, cert=cert_path, key=key_path) data = '' if info_provider_path: try: data = subprocess.check_output([info_provider_path], shell=True) except subprocess.CalledProcessError as cpe: logger.error(cpe) return 1 else: try: with open(msg_file_path, 'r') as ldif: data = ldif.read() except IOError as ioe: logger.error(ioe) return 1 msg = AmsMessage(data=data).dict() try: ret = ams.publish(topic, msg) logger.info("Successfully published message at: %s, ret: %s" % (topic, ret)) return 0 except AmsException as e: logger.error("Failed to publish message: %s" % e) return 1
def _send_msg_ams(self, text, msgid): """Send one message using AMS, returning the AMS ID of the mesage. The message will be signed using the host cert and key. If an encryption certificate has been supplied, the message will also be encrypted. """ log.info('Sending message: %s', msgid) if text is not None: # First we sign the message to_send = crypto.sign(text, self._cert, self._key) # Possibly encrypt the message. if self._enc_cert is not None: to_send = crypto.encrypt(to_send, self._enc_cert) # Then we need to wrap text up as an AMS Message. message = AmsMessage(data=to_send, attributes={'empaid': msgid}).dict() argo_response = self._ams.publish(self._dest, message, retry=3) return argo_response['messageIds'][0]
def publish(self, messages): # messages = [{data:[{id:1},{state:'deployed'}],attributes=''}] try: if not self.ams.has_topic(self.pub_topic): self.ams.create_topic(self.pub_topic) except AmsException as e: print(e) raise SystemExit(1) # publish one message to given topic. message is constructed with # help of AmsMessage which accepts data and attributes keys. # data is Base64 encoded, attributes is dictionary of arbitrary # key/value pairs msg = AmsMessage() msglist = [] for message in messages: msglist.append(msg(data=json.dumps(message['data']), attributes={})) try: ret = self.ams.publish(self.pub_topic, msglist) print(ret) except AmsException as e: print(e)
def main(): parser = ArgumentParser(description="Simple AMS message publish example") parser.add_argument('--host', type=str, default='messaging-devel.argo.grnet.gr', help='FQDN of AMS Service') parser.add_argument('--token', type=str, required=True, help='Given token') parser.add_argument('--project', type=str, required=True, help='Project registered in AMS Service') parser.add_argument('--topic', type=str, required=True, help='Given topic') parser.add_argument('--subscription', type=str, required=True, help='Subscription name') parser.add_argument('--nummsgs', type=int, default=3, help='Number of messages to pull and ack') args = parser.parse_args() ams = ArgoMessagingService(endpoint=args.host, token=args.token, project=args.project) # static sleep between retry attempts msg = AmsMessage(data='foo1', attributes={'bar1': 'baz1'}).dict() try: ret = ams.publish(args.topic, msg, retry=3, retrysleep=5, timeout=5) print(ret) except AmsException as e: print(e) # iptables -A OUTPUT -d messaging-devel.argo.grnet.gr -j DROP ackids = list() for id, msg in ams.pull_sub(args.subscription, args.nummsgs, retry=3, retrysleep=5, timeout=5): data = msg.get_data() msgid = msg.get_msgid() attr = msg.get_attr() print('msgid={0}, data={1}, attr={2}'.format(msgid, data, attr)) ackids.append(id) if ackids: ams.ack_sub(args.subscription, ackids, retry=3, retrysleep=5, timeout=5) # backoff with each next retry attempt exponentially longer msg = AmsMessage(data='foo2', attributes={'bar2': 'baz2'}).dict() try: ret = ams.publish(args.topic, msg, retry=3, retrybackoff=5, timeout=5) print(ret) except AmsException as e: print(e) # iptables -A OUTPUT -d messaging-devel.argo.grnet.gr -j DROP ackids = list() for id, msg in ams.pull_sub(args.subscription, args.nummsgs, retrybackoff=3, retrysleep=5, timeout=5): data = msg.get_data() msgid = msg.get_msgid() attr = msg.get_attr() print('msgid={0}, data={1}, attr={2}'.format(msgid, data, attr)) ackids.append(id) if ackids: ams.ack_sub(args.subscription, ackids) # static sleep between retry attempts. this example uses consume context # method that pull and acks msgs in one call. msg = AmsMessage(data='foo3', attributes={'bar3': 'baz3'}).dict() try: ret = ams.publish(args.topic, msg, retry=3, retrysleep=5, timeout=5) print(ret) except AmsException as e: print(e) try: msgs = ams.pullack_sub(args.subscription, args.nummsgs, retry=3, retrysleep=5, timeout=5) for msg in msgs: data = msg.get_data() msgid = msg.get_msgid() attr = msg.get_attr() print('msgid={0}, data={1}, attr={2}'.format(msgid, data, attr)) except AmsException as e: print(e)
def send_all(self): ''' Send all the messages in the outgoing queue. Either via STOMP or HTTPS (to an Argo Message Broker). ''' log.info('Found %s messages.', self._outq.count()) for msgid in self._outq: if not self._outq.lock(msgid): log.warn('Message was locked. %s will not be sent.', msgid) continue text = self._outq.get(msgid) if self._protocol == Ssm2.STOMP_MESSAGING: # Then we are sending to a STOMP message broker. self._send_msg(text, msgid) log.info('Waiting for broker to accept message.') while self._last_msg is None: if not self.connected: raise Ssm2Exception('Lost connection.') log_string = "Sent %s" % msgid elif self._protocol == Ssm2.AMS_MESSAGING: # Then we are sending to an Argo Messaging Service instance. if text is not None: # First we sign the message to_send = crypto.sign(text, self._cert, self._key) # Possibly encrypt the message. if self._enc_cert is not None: to_send = crypto.encrypt(to_send, self._enc_cert) # Then we need to wrap text up as an AMS Message. message = AmsMessage(data=to_send, attributes={ 'empaid': msgid }).dict() argo_response = self._ams.publish(self._dest, message) argo_id = argo_response['messageIds'][0] log_string = "Sent %s, Argo ID: %s" % (msgid, argo_id) else: # The SSM has been improperly configured raise Ssm2Exception('Unknown messaging protocol: %s' % self._protocol) time.sleep(0.1) # log that the message was sent log.info(log_string) self._last_msg = None self._outq.remove(msgid) log.info('Tidying message directory.') try: # Remove empty dirs and unlock msgs older than 5 min (default) self._outq.purge() except OSError, e: log.warn('OSError raised while purging message queue: %s', e)
def main(): MSG_NUM = 100 MSG_SIZE = 500 TIMEOUT = 180 parser = ArgumentParser(description="Nagios sensor for AMS") parser.add_argument('-H', dest='host', type=str, default='messaging-devel.argo.grnet.gr', help='FQDN of AMS Service') parser.add_argument('--token', type=str, required=True, help='Given token') parser.add_argument('--project', type=str, required=True, help='Project registered in AMS Service') parser.add_argument('--topic', type=str, default='nagios_sensor_topic', help='Given topic') parser.add_argument('--subscription', type=str, default='nagios_sensor_sub', help='Subscription name') parser.add_argument('-t', dest='timeout', type=int, default=TIMEOUT, help='Timeout') cmd_options = parser.parse_args() nagios = NagiosResponse("All messages received correctly.") ams = ArgoMessagingService(endpoint=cmd_options.host, token=cmd_options.token, project=cmd_options.project) try: if ams.has_topic(cmd_options.topic, timeout=cmd_options.timeout): ams.delete_topic(cmd_options.topic, timeout=cmd_options.timeout) if ams.has_sub(cmd_options.subscription, timeout=cmd_options.timeout): ams.delete_sub(cmd_options.subscription, timeout=cmd_options.timeout) ams.create_topic(cmd_options.topic, timeout=cmd_options.timeout) ams.create_sub(cmd_options.subscription, cmd_options.topic, timeout=cmd_options.timeout) except AmsException as e: nagios.writeCriticalMessage(e.msg) nagios.setCode(nagios.CRITICAL) print(nagios.getMsg()) raise SystemExit(nagios.getCode()) ams_msg = AmsMessage() msg_orig = set() msg_array = [] for i in range(1, MSG_NUM): msg_txt = ''.join( random.choice(string.ascii_letters + string.digits) for i in range(MSG_SIZE)) attr_name = ''.join( random.choice(string.ascii_letters + string.digits) for i in range(4)) attr_value = ''.join( random.choice(string.ascii_letters + string.digits) for i in range(8)) msg_array.append( ams_msg(data=msg_txt, attributes={attr_name: attr_value})) hash_obj = hashlib.md5(msg_txt + attr_name + attr_value) msg_orig.add(hash_obj.hexdigest()) try: msgs = ams.publish(cmd_options.topic, msg_array, timeout=cmd_options.timeout) ackids = [] rcv_msg = set() for id, msg in ams.pull_sub(cmd_options.subscription, MSG_NUM - 1, True, timeout=cmd_options.timeout): attr = msg.get_attr() hash_obj = hashlib.md5(msg.get_data() + attr.keys()[0] + attr.values()[0]) rcv_msg.add(hash_obj.hexdigest()) if ackids: ams.ack_sub(cmd_options.subscription, ackids, timeout=cmd_options.timeout) ams.delete_topic(cmd_options.topic, timeout=cmd_options.timeout) ams.delete_sub(cmd_options.subscription, timeout=cmd_options.timeout) except AmsException as e: nagios.writeCriticalMessage(e.msg) nagios.setCode(nagios.CRITICAL) print(nagios.getMsg()) raise SystemExit(nagios.getCode()) if msg_orig != rcv_msg: nagios.writeCriticalMessage("Messages received incorrectly.") nagios.setCode(nagios.CRITICAL) print(nagios.getMsg()) raise SystemExit(nagios.getCode())
def post_assessment_results(self, niftyId, msgId, file_path, base_mpuri): ams = ArgoMessagingService(endpoint=self.host, token=self.token, project=self.project) contents = Path(file_path).read_text() msg = AmsMessage(data=contents, attributes={'NIFTY_APPLIANCE_ID': niftyId, 'REQUEST_MESSAGE_ID': msgId, 'BASE_MPURI': base_mpuri}).dict() ret = ams.publish(self.resultTopic, msg) logging.debug('[%s] %s: Results has been successfully pushed.', niftyId, 'DEBUG')