class PullPublish(): def __init__(self,config): self.pull_sub = config['pull_sub'] self.pub_topic = config['pub_topic'] self.pull_topic = config['pull_topic'] self.ams = ArgoMessagingService(endpoint=config['host'], token=config['token'], project=config['project']) def pull(self,nummsgs): messages = [] try: if not self.ams.has_sub(self.pull_sub): self.ams.create_sub(self.pull_sub,self.pull_topic) except AmsException as e: print(e) raise SystemExit(1) # try to pull number of messages from subscription. method will # return (ackIds, AmsMessage) tuples from which ackIds and messages # payload will be extracted. ackids = list() for id, msg in self.ams.pull_sub(self.pull_sub, nummsgs): data = msg.get_data() msgid = msg.get_msgid() attr = msg.get_attr() messages.append(json.loads(data)) #print('msgid={0}, data={1}, attr={2}'.format(msgid, data, attr)) ackids.append(id) # pass list of extracted ackIds to AMS Service so that # it can move the offset for the next subscription pull # (basically acknowledging pulled messages) if ackids: self.ams.ack_sub(self.pull_sub, ackids) return messages def publish(self,messages): # messages = [{data:[{id:1},{state:'deployed'}],attributes=''}] try: if not self.ams.has_topic(self.pub_topic): self.ams.create_topic(self.pub_topic) except AmsException as e: print(e) raise SystemExit(1) # publish one message to given topic. message is constructed with # help of AmsMessage which accepts data and attributes keys. # data is Base64 encoded, attributes is dictionary of arbitrary # key/value pairs msg = AmsMessage() msglist = [] for message in messages: msglist.append(msg(data=json.dumps(message['data']),attributes={})) try: ret = self.ams.publish(self.pub_topic, msglist) print(ret) except AmsException as e: print(e)
def main(): parser = ArgumentParser(description="Simple AMS example of subscription pull/consume") parser.add_argument('--host', type=str, default='messaging-devel.argo.grnet.gr', help='FQDN of AMS Service') parser.add_argument('--token', type=str, required=True, help='Given token') parser.add_argument('--project', type=str, required=True, help='Project registered in AMS Service') parser.add_argument('--subscription', type=str, required=True, help='Subscription name') parser.add_argument('--topic', type=str, required=True, help='Given topic') parser.add_argument('--nummsgs', type=int, default=3, help='Number of messages to pull and ack') parser.add_argument('--schema', type=str, required=True, help='Avro schema') parser.add_argument('--outfile', type=str, required=True, help='Output avro file') args = parser.parse_args() # initialize service with given token and project ams = ArgoMessagingService(endpoint=args.host, token=args.token, project=args.project) # ensure that subscription is created in first run. messages can be # pulled from the subscription only when subscription already exists # for given topic prior messages being published to topic try: if not ams.has_sub(args.subscription): ams.create_sub(args.subscription, args.topic) subscription = ams.get_sub(args.subscription, retobj=True) except AmsException as e: print(e) raise SystemExit(1) # try to pull number of messages from subscription. method will # return (ackIds, AmsMessage) tuples from which ackIds and messages # payload will be extracted. avro_payloads = list() for msg in subscription.pullack(args.nummsgs, retry=5, retrysleep=15, return_immediately=True): data = msg.get_data() msgid = msg.get_msgid() print('msgid={0}'.format(msgid)) avro_payloads.append(data) try: schema = load_schema(args.schema) if os.path.exists(args.outfile): avroFile = open(args.outfile, 'a+') writer = DataFileWriter(avroFile, DatumWriter()) else: avroFile = open(args.outfile, 'w+') writer = DataFileWriter(avroFile, DatumWriter(), schema) for am in avro_payloads: msg = avro_deserialize(am, args.schema) writer.append(msg) writer.close() avroFile.close() except Exception as e: print(e) raise SystemExit(1)
def main(): parser = ArgumentParser( description="Simple AMS example of subscription pull/consume") parser.add_argument('--host', type=str, default='messaging-devel.argo.grnet.gr', help='FQDN of AMS Service') parser.add_argument('--token', type=str, required=True, help='Given token') parser.add_argument('--project', type=str, required=True, help='Project registered in AMS Service') parser.add_argument('--subscription', type=str, required=True, help='Subscription name') parser.add_argument('--topic', type=str, required=True, help='Given topic') parser.add_argument('--nummsgs', type=int, default=3, help='Number of messages to pull and ack') parser.add_argument('--advance', required=False, type=int, default=0, help='Number of messages to pull and ack') args = parser.parse_args() # initialize service with given token and project ams = ArgoMessagingService(endpoint=args.host, token=args.token, project=args.project) # ensure that subscription is created in first run. messages can be # pulled from the subscription only when subscription already exists # for given topic prior messages being published to topic try: if not ams.has_sub(args.subscription): ams.create_sub(args.subscription, args.topic) except AmsException as e: print e raise SystemExit(1) if args.advance: print ams.getoffsets_sub(args.subscription) print ams.modifyoffset_sub(args.subscription, args.advance)
def main(): parser = ArgumentParser(description="Simple AMS example of subscription pull/consume") parser.add_argument('--host', type=str, default='messaging-devel.argo.grnet.gr', help='FQDN of AMS Service') parser.add_argument('--token', type=str, required=True, help='Given token') parser.add_argument('--project', type=str, required=True, help='Project registered in AMS Service') parser.add_argument('--subscription', type=str, required=True, help='Subscription name') parser.add_argument('--topic', type=str, required=True, help='Given topic') parser.add_argument('--nummsgs', type=int, default=3, help='Number of messages to pull and ack') args = parser.parse_args() # initialize service with given token and project ams = ArgoMessagingService(endpoint=args.host, token=args.token, project=args.project) # ensure that subscription is created in first run. messages can be # pulled from the subscription only when subscription already exists # for given topic prior messages being published to topic try: if not ams.has_sub(args.subscription): ams.create_sub(args.subscription, args.topic) except AmsException as e: print e raise SystemExit(1) # try to pull number of messages from subscription. method will # return (ackIds, AmsMessage) tuples from which ackIds and messages # payload will be extracted. ackids = list() for id, msg in ams.pull_sub(args.subscription, args.nummsgs): data = msg.get_data() msgid = msg.get_msgid() attr = msg.get_attr() print 'msgid={0}, data={1}, attr={2}'.format(msgid, data, attr) ackids.append(id) # pass list of extracted ackIds to AMS Service so that # it can move the offset for the next subscription pull # (basically acknowledging pulled messages) if ackids: ams.ack_sub(args.subscription, ackids)
def main(): MSG_NUM = 100 MSG_SIZE = 500 TIMEOUT = 180 parser = ArgumentParser(description="Nagios sensor for AMS") parser.add_argument('-H', dest='host', type=str, default='messaging-devel.argo.grnet.gr', help='FQDN of AMS Service') parser.add_argument('--token', type=str, required=True, help='Given token') parser.add_argument('--project', type=str, required=True, help='Project registered in AMS Service') parser.add_argument('--topic', type=str, default='nagios_sensor_topic', help='Given topic') parser.add_argument('--subscription', type=str, default='nagios_sensor_sub', help='Subscription name') parser.add_argument('-t', dest='timeout', type=int, default=TIMEOUT, help='Timeout') cmd_options = parser.parse_args() nagios = NagiosResponse("All messages received correctly.") ams = ArgoMessagingService(endpoint=cmd_options.host, token=cmd_options.token, project=cmd_options.project) try: if ams.has_topic(cmd_options.topic, timeout=cmd_options.timeout): ams.delete_topic(cmd_options.topic, timeout=cmd_options.timeout) if ams.has_sub(cmd_options.subscription, timeout=cmd_options.timeout): ams.delete_sub(cmd_options.subscription, timeout=cmd_options.timeout) ams.create_topic(cmd_options.topic, timeout=cmd_options.timeout) ams.create_sub(cmd_options.subscription, cmd_options.topic, timeout=cmd_options.timeout) except AmsException as e: nagios.writeCriticalMessage(e.msg) nagios.setCode(nagios.CRITICAL) print(nagios.getMsg()) raise SystemExit(nagios.getCode()) ams_msg = AmsMessage() msg_orig = set() msg_array = [] for i in range(1, MSG_NUM): msg_txt = ''.join( random.choice(string.ascii_letters + string.digits) for i in range(MSG_SIZE)) attr_name = ''.join( random.choice(string.ascii_letters + string.digits) for i in range(4)) attr_value = ''.join( random.choice(string.ascii_letters + string.digits) for i in range(8)) msg_array.append( ams_msg(data=msg_txt, attributes={attr_name: attr_value})) hash_obj = hashlib.md5(msg_txt + attr_name + attr_value) msg_orig.add(hash_obj.hexdigest()) try: msgs = ams.publish(cmd_options.topic, msg_array, timeout=cmd_options.timeout) ackids = [] rcv_msg = set() for id, msg in ams.pull_sub(cmd_options.subscription, MSG_NUM - 1, True, timeout=cmd_options.timeout): attr = msg.get_attr() hash_obj = hashlib.md5(msg.get_data() + attr.keys()[0] + attr.values()[0]) rcv_msg.add(hash_obj.hexdigest()) if ackids: ams.ack_sub(cmd_options.subscription, ackids, timeout=cmd_options.timeout) ams.delete_topic(cmd_options.topic, timeout=cmd_options.timeout) ams.delete_sub(cmd_options.subscription, timeout=cmd_options.timeout) except AmsException as e: nagios.writeCriticalMessage(e.msg) nagios.setCode(nagios.CRITICAL) print(nagios.getMsg()) raise SystemExit(nagios.getCode()) if msg_orig != rcv_msg: nagios.writeCriticalMessage("Messages received incorrectly.") nagios.setCode(nagios.CRITICAL) print(nagios.getMsg()) raise SystemExit(nagios.getCode())
def main(): MSG_NUM = 100 MSG_SIZE = 500 TIMEOUT = 180 parser = ArgumentParser(description="Nagios sensor for AMS") parser.add_argument('-H', dest='host', type=str, default='messaging-devel.argo.grnet.gr', help='FQDN of AMS Service') parser.add_argument('--token', type=str, required=True, help='Given token') parser.add_argument('--project', type=str, required=True, help='Project registered in AMS Service') parser.add_argument('--topic', type=str, default='nagios_sensor_topic', help='Given topic') parser.add_argument('--subscription', type=str, default='nagios_sensor_sub', help='Subscription name') parser.add_argument('-t', dest='timeout', type=int, default=TIMEOUT, help='Timeout') cmd_options = parser.parse_args() nagios = NagiosResponse("All messages received correctly.") ams = ArgoMessagingService(endpoint=cmd_options.host, token=cmd_options.token, project=cmd_options.project) try: if ams.has_topic(cmd_options.topic, timeout=cmd_options.timeout): ams.delete_topic(cmd_options.topic, timeout=cmd_options.timeout) if ams.has_sub(cmd_options.subscription, timeout=cmd_options.timeout): ams.delete_sub(cmd_options.subscription, timeout=cmd_options.timeout) ams.create_topic(cmd_options.topic, timeout=cmd_options.timeout) ams.create_sub(cmd_options.subscription, cmd_options.topic, timeout=cmd_options.timeout) except AmsException as e: nagios.writeCriticalMessage(e.msg) nagios.setCode(nagios.CRITICAL) print(nagios.getMsg()) raise SystemExit(nagios.getCode()) ams_msg = AmsMessage() msg_orig = set() msg_array = [] for i in range(1, MSG_NUM): msg_txt = ''.join(random.choice(string.ascii_letters + string.digits) for i in range(MSG_SIZE)) attr_name = ''.join(random.choice(string.ascii_letters + string.digits) for i in range(4)) attr_value = ''.join(random.choice(string.ascii_letters + string.digits) for i in range(8)) msg_array.append(ams_msg(data=msg_txt, attributes={attr_name: attr_value})) hash_obj = hashlib.md5(msg_txt + attr_name + attr_value) msg_orig.add(hash_obj.hexdigest()) try: msgs = ams.publish(cmd_options.topic, msg_array, timeout=cmd_options.timeout) ackids = [] rcv_msg = set() for id, msg in ams.pull_sub(cmd_options.subscription, MSG_NUM - 1, True, timeout=cmd_options.timeout): attr = msg.get_attr() hash_obj = hashlib.md5(msg.get_data() + attr.keys()[0] + attr.values()[0]) rcv_msg.add(hash_obj.hexdigest()) if ackids: ams.ack_sub(cmd_options.subscription, ackids, timeout=cmd_options.timeout) ams.delete_topic(cmd_options.topic, timeout=cmd_options.timeout) ams.delete_sub(cmd_options.subscription, timeout=cmd_options.timeout) except AmsException as e: nagios.writeCriticalMessage(e.msg) nagios.setCode(nagios.CRITICAL) print(nagios.getMsg()) raise SystemExit(nagios.getCode()) if msg_orig != rcv_msg: nagios.writeCriticalMessage("Messages received incorrectly.") nagios.setCode(nagios.CRITICAL) print(nagios.getMsg()) raise SystemExit(nagios.getCode())