def main():
parser = ArgumentParser(
description="Simple AMS example of subscription pull/consume")
parser.add_argument('--host',
type=str,
default='messaging-devel.argo.grnet.gr',
help='FQDN of AMS Service')
parser.add_argument('--token', type=str, required=True, help='Given token')
parser.add_argument('--project',
type=str,
required=True,
help='Project registered in AMS Service')
parser.add_argument('--subscription',
type=str,
required=True,
help='Subscription name')
parser.add_argument('--topic', type=str, required=True, help='Topic name')
args = parser.parse_args()
# initialize service with given token and project
try:
ams = ArgoMessagingService(endpoint=args.host,
token=args.token,
project=args.project)
ams.create_sub(args.subscription, args.topic)
except AmsException as e:
print e
raise SystemExit(1)
class PullPublish():
def __init__(self,config):
self.pull_sub = config['pull_sub']
self.pub_topic = config['pub_topic']
self.pull_topic = config['pull_topic']
self.ams = ArgoMessagingService(endpoint=config['host'], token=config['token'], project=config['project'])
def pull(self,nummsgs):
messages = []
try:
if not self.ams.has_sub(self.pull_sub):
self.ams.create_sub(self.pull_sub,self.pull_topic)
except AmsException as e:
print(e)
raise SystemExit(1)
# try to pull number of messages from subscription. method will
# return (ackIds, AmsMessage) tuples from which ackIds and messages
# payload will be extracted.
ackids = list()
for id, msg in self.ams.pull_sub(self.pull_sub, nummsgs):
data = msg.get_data()
msgid = msg.get_msgid()
attr = msg.get_attr()
messages.append(json.loads(data))
#print('msgid={0}, data={1}, attr={2}'.format(msgid, data, attr))
ackids.append(id)
# pass list of extracted ackIds to AMS Service so that
# it can move the offset for the next subscription pull
# (basically acknowledging pulled messages)
if ackids:
self.ams.ack_sub(self.pull_sub, ackids)
return messages
def publish(self,messages):
# messages = [{data:[{id:1},{state:'deployed'}],attributes=''}]
try:
if not self.ams.has_topic(self.pub_topic):
self.ams.create_topic(self.pub_topic)
except AmsException as e:
print(e)
raise SystemExit(1)
# publish one message to given topic. message is constructed with
# help of AmsMessage which accepts data and attributes keys.
# data is Base64 encoded, attributes is dictionary of arbitrary
# key/value pairs
msg = AmsMessage()
msglist = []
for message in messages:
msglist.append(msg(data=json.dumps(message['data']),attributes={}))
try:
ret = self.ams.publish(self.pub_topic, msglist)
print(ret)
except AmsException as e:
print(e)
def main():
parser = ArgumentParser(description="Simple AMS example of subscription pull/consume")
parser.add_argument('--host', type=str, default='messaging-devel.argo.grnet.gr', help='FQDN of AMS Service')
parser.add_argument('--token', type=str, required=True, help='Given token')
parser.add_argument('--project', type=str, required=True, help='Project registered in AMS Service')
parser.add_argument('--subscription', type=str, required=True, help='Subscription name')
parser.add_argument('--topic', type=str, required=True, help='Given topic')
parser.add_argument('--nummsgs', type=int, default=3, help='Number of messages to pull and ack')
parser.add_argument('--schema', type=str, required=True, help='Avro schema')
parser.add_argument('--outfile', type=str, required=True, help='Output avro file')
args = parser.parse_args()
# initialize service with given token and project
ams = ArgoMessagingService(endpoint=args.host, token=args.token, project=args.project)
# ensure that subscription is created in first run. messages can be
# pulled from the subscription only when subscription already exists
# for given topic prior messages being published to topic
try:
if not ams.has_sub(args.subscription):
ams.create_sub(args.subscription, args.topic)
subscription = ams.get_sub(args.subscription, retobj=True)
except AmsException as e:
print(e)
raise SystemExit(1)
# try to pull number of messages from subscription. method will
# return (ackIds, AmsMessage) tuples from which ackIds and messages
# payload will be extracted.
avro_payloads = list()
for msg in subscription.pullack(args.nummsgs, retry=5, retrysleep=15, return_immediately=True):
data = msg.get_data()
msgid = msg.get_msgid()
print('msgid={0}'.format(msgid))
avro_payloads.append(data)
try:
schema = load_schema(args.schema)
if os.path.exists(args.outfile):
avroFile = open(args.outfile, 'a+')
writer = DataFileWriter(avroFile, DatumWriter())
else:
avroFile = open(args.outfile, 'w+')
writer = DataFileWriter(avroFile, DatumWriter(), schema)
for am in avro_payloads:
msg = avro_deserialize(am, args.schema)
writer.append(msg)
writer.close()
avroFile.close()
except Exception as e:
print(e)
raise SystemExit(1)
def main():
parser = ArgumentParser(
description="Simple AMS example of subscription pull/consume")
parser.add_argument('--host',
type=str,
default='messaging-devel.argo.grnet.gr',
help='FQDN of AMS Service')
parser.add_argument('--token', type=str, required=True, help='Given token')
parser.add_argument('--project',
type=str,
required=True,
help='Project registered in AMS Service')
parser.add_argument('--subscription',
type=str,
required=True,
help='Subscription name')
parser.add_argument('--topic', type=str, required=True, help='Given topic')
parser.add_argument('--nummsgs',
type=int,
default=3,
help='Number of messages to pull and ack')
parser.add_argument('--advance',
required=False,
type=int,
default=0,
help='Number of messages to pull and ack')
args = parser.parse_args()
# initialize service with given token and project
ams = ArgoMessagingService(endpoint=args.host,
token=args.token,
project=args.project)
# ensure that subscription is created in first run. messages can be
# pulled from the subscription only when subscription already exists
# for given topic prior messages being published to topic
try:
if not ams.has_sub(args.subscription):
ams.create_sub(args.subscription, args.topic)
except AmsException as e:
print e
raise SystemExit(1)
if args.advance:
print ams.getoffsets_sub(args.subscription)
print ams.modifyoffset_sub(args.subscription, args.advance)
def main():
parser = ArgumentParser(description="Simple AMS example of subscription pull/consume")
parser.add_argument('--host', type=str, default='messaging-devel.argo.grnet.gr', help='FQDN of AMS Service')
parser.add_argument('--token', type=str, required=True, help='Given token')
parser.add_argument('--project', type=str, required=True, help='Project registered in AMS Service')
parser.add_argument('--subscription', type=str, required=True, help='Subscription name')
parser.add_argument('--topic', type=str, required=True, help='Given topic')
parser.add_argument('--nummsgs', type=int, default=3, help='Number of messages to pull and ack')
args = parser.parse_args()
# initialize service with given token and project
ams = ArgoMessagingService(endpoint=args.host, token=args.token, project=args.project)
# ensure that subscription is created in first run. messages can be
# pulled from the subscription only when subscription already exists
# for given topic prior messages being published to topic
try:
if not ams.has_sub(args.subscription):
ams.create_sub(args.subscription, args.topic)
except AmsException as e:
print e
raise SystemExit(1)
# try to pull number of messages from subscription. method will
# return (ackIds, AmsMessage) tuples from which ackIds and messages
# payload will be extracted.
ackids = list()
for id, msg in ams.pull_sub(args.subscription, args.nummsgs):
data = msg.get_data()
msgid = msg.get_msgid()
attr = msg.get_attr()
print 'msgid={0}, data={1}, attr={2}'.format(msgid, data, attr)
ackids.append(id)
# pass list of extracted ackIds to AMS Service so that
# it can move the offset for the next subscription pull
# (basically acknowledging pulled messages)
if ackids:
ams.ack_sub(args.subscription, ackids)
def main():
MSG_NUM = 100
MSG_SIZE = 500
TIMEOUT = 180
parser = ArgumentParser(description="Nagios sensor for AMS")
parser.add_argument('-H',
dest='host',
type=str,
default='messaging-devel.argo.grnet.gr',
help='FQDN of AMS Service')
parser.add_argument('--token', type=str, required=True, help='Given token')
parser.add_argument('--project',
type=str,
required=True,
help='Project registered in AMS Service')
parser.add_argument('--topic',
type=str,
default='nagios_sensor_topic',
help='Given topic')
parser.add_argument('--subscription',
type=str,
default='nagios_sensor_sub',
help='Subscription name')
parser.add_argument('-t',
dest='timeout',
type=int,
default=TIMEOUT,
help='Timeout')
cmd_options = parser.parse_args()
nagios = NagiosResponse("All messages received correctly.")
ams = ArgoMessagingService(endpoint=cmd_options.host,
token=cmd_options.token,
project=cmd_options.project)
try:
if ams.has_topic(cmd_options.topic, timeout=cmd_options.timeout):
ams.delete_topic(cmd_options.topic, timeout=cmd_options.timeout)
if ams.has_sub(cmd_options.subscription, timeout=cmd_options.timeout):
ams.delete_sub(cmd_options.subscription,
timeout=cmd_options.timeout)
ams.create_topic(cmd_options.topic, timeout=cmd_options.timeout)
ams.create_sub(cmd_options.subscription,
cmd_options.topic,
timeout=cmd_options.timeout)
except AmsException as e:
nagios.writeCriticalMessage(e.msg)
nagios.setCode(nagios.CRITICAL)
print(nagios.getMsg())
raise SystemExit(nagios.getCode())
ams_msg = AmsMessage()
msg_orig = set()
msg_array = []
for i in range(1, MSG_NUM):
msg_txt = ''.join(
random.choice(string.ascii_letters + string.digits)
for i in range(MSG_SIZE))
attr_name = ''.join(
random.choice(string.ascii_letters + string.digits)
for i in range(4))
attr_value = ''.join(
random.choice(string.ascii_letters + string.digits)
for i in range(8))
msg_array.append(
ams_msg(data=msg_txt, attributes={attr_name: attr_value}))
hash_obj = hashlib.md5(msg_txt + attr_name + attr_value)
msg_orig.add(hash_obj.hexdigest())
try:
msgs = ams.publish(cmd_options.topic,
msg_array,
timeout=cmd_options.timeout)
ackids = []
rcv_msg = set()
for id, msg in ams.pull_sub(cmd_options.subscription,
MSG_NUM - 1,
True,
timeout=cmd_options.timeout):
attr = msg.get_attr()
hash_obj = hashlib.md5(msg.get_data() + attr.keys()[0] +
attr.values()[0])
rcv_msg.add(hash_obj.hexdigest())
if ackids:
ams.ack_sub(cmd_options.subscription,
ackids,
timeout=cmd_options.timeout)
ams.delete_topic(cmd_options.topic, timeout=cmd_options.timeout)
ams.delete_sub(cmd_options.subscription, timeout=cmd_options.timeout)
except AmsException as e:
nagios.writeCriticalMessage(e.msg)
nagios.setCode(nagios.CRITICAL)
print(nagios.getMsg())
raise SystemExit(nagios.getCode())
if msg_orig != rcv_msg:
nagios.writeCriticalMessage("Messages received incorrectly.")
nagios.setCode(nagios.CRITICAL)
print(nagios.getMsg())
raise SystemExit(nagios.getCode())
def main():
MSG_NUM = 100
MSG_SIZE = 500
TIMEOUT = 180
parser = ArgumentParser(description="Nagios sensor for AMS")
parser.add_argument('-H', dest='host', type=str, default='messaging-devel.argo.grnet.gr', help='FQDN of AMS Service')
parser.add_argument('--token', type=str, required=True, help='Given token')
parser.add_argument('--project', type=str, required=True, help='Project registered in AMS Service')
parser.add_argument('--topic', type=str, default='nagios_sensor_topic', help='Given topic')
parser.add_argument('--subscription', type=str, default='nagios_sensor_sub', help='Subscription name')
parser.add_argument('-t', dest='timeout', type=int, default=TIMEOUT, help='Timeout')
cmd_options = parser.parse_args()
nagios = NagiosResponse("All messages received correctly.")
ams = ArgoMessagingService(endpoint=cmd_options.host, token=cmd_options.token, project=cmd_options.project)
try:
if ams.has_topic(cmd_options.topic, timeout=cmd_options.timeout):
ams.delete_topic(cmd_options.topic, timeout=cmd_options.timeout)
if ams.has_sub(cmd_options.subscription, timeout=cmd_options.timeout):
ams.delete_sub(cmd_options.subscription, timeout=cmd_options.timeout)
ams.create_topic(cmd_options.topic, timeout=cmd_options.timeout)
ams.create_sub(cmd_options.subscription, cmd_options.topic, timeout=cmd_options.timeout)
except AmsException as e:
nagios.writeCriticalMessage(e.msg)
nagios.setCode(nagios.CRITICAL)
print(nagios.getMsg())
raise SystemExit(nagios.getCode())
ams_msg = AmsMessage()
msg_orig = set()
msg_array = []
for i in range(1, MSG_NUM):
msg_txt = ''.join(random.choice(string.ascii_letters + string.digits) for i in range(MSG_SIZE))
attr_name = ''.join(random.choice(string.ascii_letters + string.digits) for i in range(4))
attr_value = ''.join(random.choice(string.ascii_letters + string.digits) for i in range(8))
msg_array.append(ams_msg(data=msg_txt, attributes={attr_name: attr_value}))
hash_obj = hashlib.md5(msg_txt + attr_name + attr_value)
msg_orig.add(hash_obj.hexdigest())
try:
msgs = ams.publish(cmd_options.topic, msg_array, timeout=cmd_options.timeout)
ackids = []
rcv_msg = set()
for id, msg in ams.pull_sub(cmd_options.subscription, MSG_NUM - 1, True, timeout=cmd_options.timeout):
attr = msg.get_attr()
hash_obj = hashlib.md5(msg.get_data() + attr.keys()[0] + attr.values()[0])
rcv_msg.add(hash_obj.hexdigest())
if ackids:
ams.ack_sub(cmd_options.subscription, ackids, timeout=cmd_options.timeout)
ams.delete_topic(cmd_options.topic, timeout=cmd_options.timeout)
ams.delete_sub(cmd_options.subscription, timeout=cmd_options.timeout)
except AmsException as e:
nagios.writeCriticalMessage(e.msg)
nagios.setCode(nagios.CRITICAL)
print(nagios.getMsg())
raise SystemExit(nagios.getCode())
if msg_orig != rcv_msg:
nagios.writeCriticalMessage("Messages received incorrectly.")
nagios.setCode(nagios.CRITICAL)
print(nagios.getMsg())
raise SystemExit(nagios.getCode())
