def main():
parser = ArgumentParser(
description="Simple AMS example of subscription pull/consume")
parser.add_argument('--host',
type=str,
default='messaging-devel.argo.grnet.gr',
help='FQDN of AMS Service')
parser.add_argument('--token', type=str, required=True, help='Given token')
parser.add_argument('--project',
type=str,
required=True,
help='Project registered in AMS Service')
parser.add_argument('--subscription',
type=str,
required=True,
help='Subscription name')
parser.add_argument('--topic', type=str, required=True, help='Topic name')
args = parser.parse_args()
# initialize service with given token and project
try:
ams = ArgoMessagingService(endpoint=args.host,
token=args.token,
project=args.project)
if not ams.has_topic(args.topic):
ams.create_topic(args.topic)
topic = ams.get_topic(args.topic, retobj=True)
sub = topic.subscription(args.subscription)
msg = sub.pullack(3, timeout=5)
if msg:
print(msg[0].get_msgid())
except AmsException as e:
print e
raise SystemExit(1)
def main(args):
ams_endpoint = "{}:{}".format(args.host, args.port)
ams = ArgoMessagingService(endpoint=ams_endpoint,
token=args.token,
project=args.project)
while True:
try:
consumed_msgs = ams.pull_sub(sub=args.sub,
num=args.bulk_size,
return_immediately=True,
verify=args.verify)
print(consumed_msgs)
last_msg_id = "-1"
if len(consumed_msgs) > 0:
last_msg_id = consumed_msgs.pop()[0]
print(last_msg_id)
print("\n")
if last_msg_id != "-1":
print(ams.ack_sub(args.sub, [last_msg_id], verify=args.verify))
time.sleep(args.fire_rate)
except Exception as e:
print("Couldn't consume from sub {}, {}".format(args.sub, str(e)))
continue
def main():
parser = ArgumentParser(
description="Simple AMS example of subscription pull/consume")
parser.add_argument('--host',
type=str,
default='messaging-devel.argo.grnet.gr',
help='FQDN of AMS Service')
parser.add_argument('--token', type=str, required=True, help='Given token')
parser.add_argument('--project',
type=str,
required=True,
help='Project registered in AMS Service')
parser.add_argument('--subscription',
type=str,
required=True,
help='Subscription name')
args = parser.parse_args()
# initialize service with given token and project
ams = ArgoMessagingService(endpoint=args.host,
token=args.token,
project=args.project)
for msg in ams.pullack_sub(args.subscription,
retry=5,
retrysleep=15,
return_immediately=True):
try:
data = msg.get_data()
msgid = msg.get_msgid()
print('msgid={0}'.format(msgid))
print('msgdata={0}'.format(data))
except AmsException as e:
print(e)
raise SystemExit(1)
def post_template_for_assessment(self, niftyId, msgId):
ams = ArgoMessagingService(endpoint=self.host, token=self.token, project=self.project)
msg = AmsMessage(data="", attributes={'NIFTY_APPLIANCE_ID': niftyId}).dict()
try:
ret = ams.publish(self.requestTopic, msg)
except AmsException as e:
print e
def get_templates_for_assessment(self, img_dir):
"""
:return:
"""
ams = ArgoMessagingService(endpoint=self.host, token=self.token, project=self.project)
ackids = list()
niftyids = list()
msgids = list()
pull_subscription = ams.pull_sub(self.requestSubscription, num=1, return_immediately=True)
if pull_subscription:
for id, msg in pull_subscription:
attr = msg.get_attr()
data = msg.get_data()
msgid = msg.get_msgid()
image_list_file = tempfile.NamedTemporaryFile(prefix='tmp_', delete=False, suffix='.list', dir=img_dir)
os.chmod(image_list_file.name, 0o644)
image_list_file.write(data)
image_list_file.close()
niftyids.append(os.path.basename(image_list_file.name))
ackids.append(id)
msgids.append(msgid)
if ackids:
ams.ack_sub(self.requestSubscription, ackids)
return niftyids, msgids
def main():
parser = ArgumentParser(
description="Simple AMS example of subscription pull/consume")
parser.add_argument('--host',
type=str,
default='messaging-devel.argo.grnet.gr',
help='FQDN of AMS Service')
parser.add_argument('--token', type=str, required=True, help='Given token')
parser.add_argument('--project',
type=str,
required=True,
help='Project registered in AMS Service')
parser.add_argument('--topic', type=str, required=True, help='Given topic')
parser.add_argument('--subscription',
type=str,
required=True,
help='Given subscription')
args = parser.parse_args()
topic = None
# initialize service with given token and project
try:
ams = ArgoMessagingService(endpoint=args.host,
token=args.token,
project=args.project)
topic = ams.topic(args.topic)
sub = topic.subscription(args.subscription)
print sub.acl()
sub.acl(["test-publisher01"])
print sub.acl()
except AmsException as e:
print e
raise SystemExit(1)
def main():
parser = ArgumentParser(
description="Simple AMS example of subscription pull/consume")
parser.add_argument('--host',
type=str,
default='messaging-devel.argo.grnet.gr',
help='FQDN of AMS Service')
parser.add_argument('--token', type=str, required=True, help='Given token')
parser.add_argument('--project',
type=str,
required=True,
help='Project registered in AMS Service')
parser.add_argument('--topic',
type=str,
default=None,
required=False,
help='Given topic')
args = parser.parse_args()
# initialize service with given token and project
ams = ArgoMessagingService(endpoint=args.host,
token=args.token,
project=args.project)
try:
for s in ams.iter_subs(topic=args.topic, retry=5):
print(s.name, s.fullname, s.topic.name, s.push_endpoint)
except AmsException as e:
print(e)
raise SystemExit(1)
def main():
parser = ArgumentParser(
description="Simple AMS example of subscription pull/consume")
parser.add_argument('--host',
type=str,
default='messaging-devel.argo.grnet.gr',
help='FQDN of AMS Service')
parser.add_argument('--token', type=str, required=True, help='Given token')
parser.add_argument('--project',
type=str,
required=True,
help='Project registered in AMS Service')
args = parser.parse_args()
# initialize service with given token and project
try:
ams = ArgoMessagingService(endpoint=args.host,
token=args.token,
project=args.project)
for t in ams.iter_topics(timeout=5):
print(t.name, t.fullname)
except AmsException as e:
print('ERROR!')
print(e)
raise SystemExit(1)
print('Second\n')
for t in ams.iter_topics():
print(t.name, t.fullname)
print('Third\n')
for t in ams.iter_topics():
print(t.name, t.fullname)
def publish_message(service, action):
"""
Send a message using argo messaging service when an action upon a service
takes place.
"""
service_id = str(service.get('id'))
service_name = service.get('name')
service_data = service.get('data', {})
ams = ArgoMessagingService(endpoint=AMS_ENDPOINT,
project=AMS_PROJECT,
token=AMS_TOKEN)
endpoint = '{0}/api/v2/ext-services/{1}'.format(get_root_url(), service_id)
# The value of the data property must be unicode in order to be
# encoded in base64 format in AmsMessage
data = json.dumps(service_data)
try:
if not ams.has_topic(AMS_TOPIC):
ams.create_topic(AMS_TOPIC)
except AmsException as e:
print e
raise SystemExit(1)
msg = AmsMessage(data=data,
attributes={
"method": action,
"service_id": service_id,
"service_name": service_name,
"endpoint": endpoint
}).dict()
try:
ret = ams.publish(AMS_TOPIC, msg)
print ret
except AmsException as e:
print e
def main():
parser = ArgumentParser(
description="Simple AMS example of subscription pull/consume")
parser.add_argument('--host',
type=str,
default='messaging-devel.argo.grnet.gr',
help='FQDN of AMS Service')
parser.add_argument('--token', type=str, required=True, help='Given token')
parser.add_argument('--project',
type=str,
required=True,
help='Project registered in AMS Service')
parser.add_argument('--subscription',
type=str,
required=True,
help='Subscription name')
parser.add_argument('--topic', type=str, required=True, help='Topic name')
args = parser.parse_args()
# initialize service with given token and project
try:
ams = ArgoMessagingService(endpoint=args.host,
token=args.token,
project=args.project)
ams.create_sub(args.subscription, args.topic)
except AmsException as e:
print e
raise SystemExit(1)
def get_templates_for_assessment(self):
"""
:return:
"""
ams = ArgoMessagingService(endpoint=self.host,
token=self.token,
project=self.project)
ackids = list()
niftyids = list()
logging.debug('[%s] %s: Start pulling from the %s subscription',
'SECANT', 'DEBUG', self.requestSubscription)
pull_subscription = ams.pull_sub(self.requestSubscription,
self.nummsgs, True)
logging.debug('[%s] %s: Finish pulling from the %s subscription',
'SECANT', 'DEBUG', self.requestSubscription)
if pull_subscription:
for id, msg in pull_subscription:
attr = msg.get_attr()
data = msg.get_data()
image_list_file = tempfile.NamedTemporaryFile(prefix='tmp_',
delete=False,
suffix='.list')
image_list_file.write(data)
image_list_file.close()
niftyids.append(image_list_file.name)
ackids.append(id)
else:
logging.debug('[%s] %s: No new requests to pull', 'SECANT',
'DEBUG')
if ackids:
ams.ack_sub(self.requestSubscription, ackids)
return niftyids
def __init__(self, config):
with open("config.json") as json_data_file:
data = json.load(json_data_file)
self.pull_sub = config['pull_sub']
self.pub_topic = config['pub_topic']
self.pull_topic = config['pull_topic']
self.ams = ArgoMessagingService(endpoint=data['host'],
token=config['token'],
project=config['project'])
def publish(config):
token = config.get("AUTH", "token")
host = config.get("AMS", "ams_host")
project = config.get("AMS", "ams_project")
topic = config.get("AMS", "ams_topic")
cert_path = config.get("AUTH", "cert_path")
key_path = config.get("AUTH", "key_path")
msg_file_path = config.get("AMS", "msg_file_path")
info_provider_path = config.get("AMS", "info_provider_path")
# initialize service
if token:
ams = ArgoMessagingService(endpoint=host, project=project, token=token)
else:
ams = ArgoMessagingService(endpoint=host,
project=project,
cert=cert_path,
key=key_path)
data = ''
if info_provider_path:
try:
data = subprocess.check_output([info_provider_path], shell=True)
except subprocess.CalledProcessError as cpe:
logger.error(cpe)
return 1
else:
try:
with open(msg_file_path, 'r') as ldif:
data = ldif.read()
except IOError as ioe:
logger.error(ioe)
return 1
msg = AmsMessage(data=data).dict()
try:
ret = ams.publish(topic, msg)
logger.info("Successfully published message at: %s, ret: %s" %
(topic, ret))
return 0
except AmsException as e:
logger.error("Failed to publish message: %s" % e)
return 1
def main():
parser = ArgumentParser(description="Simple AMS example of subscription pull/consume")
parser.add_argument('--host', type=str, default='messaging-devel.argo.grnet.gr', help='FQDN of AMS Service')
parser.add_argument('--token', type=str, required=True, help='Given token')
parser.add_argument('--project', type=str, required=True, help='Project registered in AMS Service')
parser.add_argument('--subscription', type=str, required=True, help='Subscription name')
parser.add_argument('--topic', type=str, required=True, help='Given topic')
parser.add_argument('--nummsgs', type=int, default=3, help='Number of messages to pull and ack')
parser.add_argument('--schema', type=str, required=True, help='Avro schema')
parser.add_argument('--outfile', type=str, required=True, help='Output avro file')
args = parser.parse_args()
# initialize service with given token and project
ams = ArgoMessagingService(endpoint=args.host, token=args.token, project=args.project)
# ensure that subscription is created in first run. messages can be
# pulled from the subscription only when subscription already exists
# for given topic prior messages being published to topic
try:
if not ams.has_sub(args.subscription):
ams.create_sub(args.subscription, args.topic)
subscription = ams.get_sub(args.subscription, retobj=True)
except AmsException as e:
print(e)
raise SystemExit(1)
# try to pull number of messages from subscription. method will
# return (ackIds, AmsMessage) tuples from which ackIds and messages
# payload will be extracted.
avro_payloads = list()
for msg in subscription.pullack(args.nummsgs, retry=5, retrysleep=15, return_immediately=True):
data = msg.get_data()
msgid = msg.get_msgid()
print('msgid={0}'.format(msgid))
avro_payloads.append(data)
try:
schema = load_schema(args.schema)
if os.path.exists(args.outfile):
avroFile = open(args.outfile, 'a+')
writer = DataFileWriter(avroFile, DatumWriter())
else:
avroFile = open(args.outfile, 'w+')
writer = DataFileWriter(avroFile, DatumWriter(), schema)
for am in avro_payloads:
msg = avro_deserialize(am, args.schema)
writer.append(msg)
writer.close()
avroFile.close()
except Exception as e:
print(e)
raise SystemExit(1)
def main():
parser = ArgumentParser(description="Simple AMS message publish example")
parser.add_argument('--host',
type=str,
default='messaging-devel.argo.grnet.gr',
help='FQDN of AMS Service')
parser.add_argument('--token', type=str, required=True, help='Given token')
parser.add_argument('--project',
type=str,
required=True,
help='Project registered in AMS Service')
parser.add_argument('--topic', type=str, required=True, help='Given topic')
args = parser.parse_args()
# initialize service with given token and project
ams = ArgoMessagingService(endpoint=args.host,
token=args.token,
project=args.project)
# ensure that topic is created in first run
try:
if not ams.has_topic(args.topic):
ams.create_topic(args.topic)
topic = ams.get_topic(args.topic, retobj=True)
except AmsException as e:
print e
raise SystemExit(1)
# publish one message to given topic. message is constructed with
# help of AmsMessage which accepts data and attributes keys.
# data is Base64 encoded, attributes is dictionary of arbitrary
# key/value pairs
msg = AmsMessage(data='foo1', attributes={'bar1': 'baz1'}).dict()
try:
ret = topic.publish(msg)
print ret
except AmsException as e:
print e
# publish a list of two messages to given topic. AmsMessage can also be
# used as a callable. publish() method accepts either one messages or
# list of messages
msg = AmsMessage()
msglist = [
msg(data='foo2', attributes={'bar2': 'baz2'}),
msg(data='foo3', attributes={'bar3': 'baz3'})
]
try:
ret = topic.publish(msglist)
print ret
except AmsException as e:
print e
def post_assessment_results(self, niftyId, file_path):
ams = ArgoMessagingService(endpoint=self.host,
token=self.token,
project=self.project)
contents = Path(file_path).read_text()
msg = AmsMessage(data=contents,
attributes={
'NIFTY_APPLIANCE_ID': niftyId
}).dict()
try:
ret = ams.publish(self.resultTopic, msg)
logging.debug('[%s] %s: Results has been successfully pushed.',
niftyId, 'DEBUG')
except AmsException as e:
print e
def main():
parser = ArgumentParser(description="AMS message publish")
parser.add_argument('--host', type=str, default='', help='FQDN of AMS Service')
parser.add_argument('--token', type=str, default='', help='Given token')
parser.add_argument('--project', type=str, default='appdb-sec-test', help='Project registered in AMS Service')
parser.add_argument('--topic', type=str, default='VMISECURITY-REQUESTS', help='Given topic')
parser.add_argument('--id', type=str, default='', help='Appliance id')
args = parser.parse_args()
ams = ArgoMessagingService(endpoint=args.host, token=args.token, project=args.project)
msg = AmsMessage(data=" ", attributes={'NIFTY_APPLIANCE_ID': args.id}).dict()
try:
ret = ams.publish(args.topic, msg)
print ret
except AmsException as e:
print e
def main():
parser = ArgumentParser(
description="Simple AMS example of subscription pull/consume")
parser.add_argument('--host',
type=str,
default='messaging-devel.argo.grnet.gr',
help='FQDN of AMS Service')
parser.add_argument('--token', type=str, required=True, help='Given token')
parser.add_argument('--project',
type=str,
required=True,
help='Project registered in AMS Service')
parser.add_argument('--subscription',
type=str,
required=True,
help='Subscription name')
parser.add_argument('--topic', type=str, required=True, help='Given topic')
parser.add_argument('--nummsgs',
type=int,
default=3,
help='Number of messages to pull and ack')
parser.add_argument('--advance',
required=False,
type=int,
default=0,
help='Number of messages to pull and ack')
args = parser.parse_args()
# initialize service with given token and project
ams = ArgoMessagingService(endpoint=args.host,
token=args.token,
project=args.project)
# ensure that subscription is created in first run. messages can be
# pulled from the subscription only when subscription already exists
# for given topic prior messages being published to topic
try:
if not ams.has_sub(args.subscription):
ams.create_sub(args.subscription, args.topic)
except AmsException as e:
print e
raise SystemExit(1)
if args.advance:
print ams.getoffsets_sub(args.subscription)
print ams.modifyoffset_sub(args.subscription, args.advance)
def testPushAndPull(self):
settings = ConfigParser.ConfigParser()
settings.read('../conf/argo.conf')
host = settings.get('AMS-GENERAL', 'host')
project = settings.get('AMS-GENERAL', 'project')
token = settings.get('AMS-GENERAL', 'token')
subscription = settings.get('REQUESTS', 'subscription')
topic = settings.get('REQUESTS', 'topic')
log = logging.getLogger("TestArgoMessageService.testPushAndPull")
log.debug("Host: {host}, Project: {project}, Topic: {topic}".format(
host=host, project=project, topic=topic))
ams = ArgoMessagingService(endpoint=host, token=token, project=project)
pullAllMessages(ams, subscription, log)
response = urllib2.urlopen(
"https://vmcaster.appdb.egi.eu/store/vappliance/demo.va.public/image.list",
timeout=5)
content = response.read()
msg = AmsMessage(data=content, attributes={'index': '1'}).dict()
try:
ret = ams.publish(topic, msg)
log.debug("Successfully published with ID:{id}".format(
id=ret['messageIds'][0]))
except AmsException as e:
print e
# Wait 10 seconds between pull and push
log.debug("Waiting 10s between push and pull")
time.sleep(10)
ackids = list()
pull_result = ams.pull_sub(subscription, 1)
if pull_result:
log.debug("Successfully pulled a message with ID:{id}".format(
id=pull_result[0][1].get_msgid()))
self.assertEquals(content, pull_result[0][1].get_data())
ackids.append(pull_result[0][1].get_msgid())
# Send Acknowledgement
if ackids:
ams.ack_sub(subscription, ackids)
def __init__(self,
host,
project,
token,
topic,
report,
bulk,
packsingle,
logger,
retry,
timeout=180,
sleepretry=60):
self.ams = ArgoMessagingService(host, token, project)
self.topic = topic
self.bulk = int(bulk)
self.report = report
self.timeout = int(timeout)
self.retry = int(retry)
self.sleepretry = int(sleepretry)
self.logger = logger
self.packsingle = eval(packsingle)
def get_assessment_results(self):
"""
:return:
"""
ams = ArgoMessagingService(endpoint=self.host, token=self.token, project=self.project)
ackids = list()
niftyids = list()
logging.debug('Start pulling from the %s subscription', self.resultSubscription)
pull_subscription = ams.pull_sub(self.resultSubscription, 10, True)
logging.debug('Finish pulling from the %s subscription', self.resultSubscription)
if pull_subscription:
for id, msg in pull_subscription:
attr = msg.get_attr()
data = msg.get_data()
ackids.append(id)
if ackids:
logging.debug("[%s] %s: Acknowledging %s" % ('SECANT', 'DEBUG', "'".join(ackids)))
ams.ack_sub(self.resultSubscription, ackids)
return niftyids
def main():
parser = ArgumentParser(description="Simple AMS example of subscription pull/consume")
parser.add_argument('--host', type=str, default='messaging-devel.argo.grnet.gr', help='FQDN of AMS Service')
parser.add_argument('--token', type=str, required=True, help='Given token')
parser.add_argument('--project', type=str, required=True, help='Project registered in AMS Service')
parser.add_argument('--subscription', type=str, required=True, help='Subscription name')
parser.add_argument('--topic', type=str, required=True, help='Given topic')
parser.add_argument('--nummsgs', type=int, default=3, help='Number of messages to pull and ack')
args = parser.parse_args()
# initialize service with given token and project
ams = ArgoMessagingService(endpoint=args.host, token=args.token, project=args.project)
# ensure that subscription is created in first run. messages can be
# pulled from the subscription only when subscription already exists
# for given topic prior messages being published to topic
try:
if not ams.has_sub(args.subscription):
ams.create_sub(args.subscription, args.topic)
except AmsException as e:
print e
raise SystemExit(1)
# try to pull number of messages from subscription. method will
# return (ackIds, AmsMessage) tuples from which ackIds and messages
# payload will be extracted.
ackids = list()
for id, msg in ams.pull_sub(args.subscription, args.nummsgs):
data = msg.get_data()
msgid = msg.get_msgid()
attr = msg.get_attr()
print 'msgid={0}, data={1}, attr={2}'.format(msgid, data, attr)
ackids.append(id)
# pass list of extracted ackIds to AMS Service so that
# it can move the offset for the next subscription pull
# (basically acknowledging pulled messages)
if ackids:
ams.ack_sub(args.subscription, ackids)
def main():
TIMEOUT = 180
INTERVAL = 300
parser = ArgumentParser(
description="Nagios probe for monitoring the compute engine's flow.")
parser.add_argument('-H',
dest='host',
type=str,
default='msg-devel.argo.grnet.gr',
help='FQDN of AMS Service')
parser.add_argument('--token', type=str, required=True, help='Given token')
parser.add_argument('--project',
type=str,
required=True,
help='Project registered in AMS Service')
parser.add_argument('--push_topic',
type=str,
default='create_data',
help='Given topic')
parser.add_argument('--push_subscription',
type=str,
default='create_data_sub',
help='Push_Subscription name')
parser.add_argument('--pull_subscription',
type=str,
default='retrieve_data_sub',
help='Push_Subscription name')
parser.add_argument('-t',
dest='timeout',
type=int,
default=TIMEOUT,
help='Timeout for ams calls')
parser.add_argument(
'-i',
dest='interval',
type=int,
default=INTERVAL,
help=
'The amount of time the probe should try to read from ams, beforing exiting'
)
cmd_options = parser.parse_args()
run_timestamp = str(datetime.datetime.now())
nagios = NagiosResponse("System Dataflow at " + run_timestamp +
" completed successfully.")
ams = ArgoMessagingService(endpoint=cmd_options.host,
token=cmd_options.token,
project=cmd_options.project)
try:
# For both subscriptions move their offset to max
move_sub_offset_to_max(ams,
cmd_options.push_subscription,
timeout=cmd_options.timeout)
move_sub_offset_to_max(ams,
cmd_options.pull_subscription,
timeout=cmd_options.timeout)
# publish a message with the current timestamp as its content
req_data = {'message': run_timestamp, 'errors': []}
d1 = {'data': json.dumps(req_data), 'attributes': {}}
ams.publish(cmd_options.push_topic, d1, timeout=cmd_options.timeout)
start = time.time()
no_resp = True
while no_resp:
end = time.time()
# check if the systsem has written to the retrieve topic
resp = ams.pull_sub(cmd_options.pull_subscription,
timeout=cmd_options.timeout)
if len(resp) > 0:
no_resp = False
resp_data = json.loads(resp[0][1]._data)
# check if the submitted and retrieved data differ
if req_data != resp_data:
nagios_report(
nagios, 'critical', "System Dataflow at " +
run_timestamp + " completed with errors. Expected: " +
str(req_data) + ". Found: " + str(resp_data) + ".")
# check if data was retrieved within the expected timeout period, BUT had some kind of delay
elif req_data == resp_data and end - start > cmd_options.interval:
nagios_report(
nagios, 'warning',
"System Dataflow at " + run_timestamp +
" completed successfully using an extra time of: " +
str((end - start) - cmd_options.interval) + "s.")
if (end - start) > 2 * cmd_options.interval:
nagios_report(
nagios, 'critical', "System Dataflow at " + run_timestamp +
" returned with no message from the systsem after " +
str(2 * cmd_options.interval) + "s.")
# check for a response every 10 seconds
time.sleep(10)
print(nagios.getMsg())
raise SystemExit(nagios.getCode())
except AmsException as e:
nagios_report(nagios, 'critical', e.msg)
def main(args):
# set up the configuration object
config = dict()
# default values
config["bulk_size"] = 100
config["interval"] = 10
with open(args.ConfigPath, 'r') as f:
config = json.load(f)
# stream(console) handler
console_handler = logging.StreamHandler()
console_handler.setFormatter(logging.Formatter('%(asctime)s %(name)s[%(process)d]: %(levelname)s %(message)s'))
LOGGER.addHandler(console_handler)
if args.debug:
LOGGER.setLevel(logging.DEBUG)
else:
LOGGER.setLevel(logging.INFO)
# sys log handler
syslog_handler = logging.handlers.SysLogHandler(config["syslog_socket"])
syslog_handler.setFormatter(logging.Formatter('%(asctime)s %(name)s[%(process)d]: %(levelname)s %(message)s'))
if args.debug:
syslog_handler.setLevel(logging.DEBUG)
else:
syslog_handler.setLevel(logging.INFO)
syslog_handler.setLevel(logging.INFO)
LOGGER.addHandler(syslog_handler)
# start the process of republishing messages
ams_endpoint = "{}:{}".format(config["ams_host"], config["ams_port"])
ams = ArgoMessagingService(endpoint=ams_endpoint, token=config["ams_token"], project=config["ams_project"])
schema = avro.schema.Parse(open(config["avro_schema"], "rb").read())
while True:
start_time = time.time()
try:
consumed_msgs, last_msg_id = extract_messages(ams, config["ingest_subscription"], config["bulk_size"], schema, args.verify)
if last_msg_id == "-1":
LOGGER.info("No new messages")
time.sleep(config["interval"])
continue
LOGGER.debug("Consumed messages \n {}".format(consumed_msgs))
filtered_msgs = filter_messages(consumed_msgs, config["sites"])
LOGGER.debug("Filtered messages \n {}".format(filtered_msgs))
republish_messages(filtered_msgs, ams, args.verify)
# make sure that the acknowledgment happens
try:
# try to acknowledge
ams.ack_sub(config["ingest_subscription"], [last_msg_id], verify=args.verify)
except Exception as e:
# if the acknowledgment fails
LOGGER.critical("Retrying to acknowledge message {} after error {}".format(last_msg_id, str(e)))
while True:
try:
# consume again in order to refresh the TTL
ams.pull_sub(config["ingest_subscription"], config["bulk_size"], True, verify=args.verify)
# try to ack again using the msg_id from the first consumption
ams.ack_sub(config["ingest_subscription"], [last_msg_id], verify=args.verify)
break
except Exception as e:
LOGGER.critical(
"Retrying to acknowledge message {} after error {}".format(last_msg_id, str(e)))
time.sleep(config["interval"])
end_time = time.time()
LOGGER.info("Consumed {} and Republished {} messages. in {}".format(
len(consumed_msgs),
len(filtered_msgs),
end_time - start_time))
except Exception as e:
LOGGER.critical("Could not republish, {}".format(str(e)))
time.sleep(config["interval"])
def main():
parser = ArgumentParser(description="Simple AMS message publish example")
parser.add_argument('--host',
type=str,
default='messaging-devel.argo.grnet.gr',
help='FQDN of AMS Service')
parser.add_argument('--token', type=str, required=True, help='Given token')
parser.add_argument('--project',
type=str,
required=True,
help='Project registered in AMS Service')
parser.add_argument('--topic', type=str, required=True, help='Given topic')
parser.add_argument('--subscription',
type=str,
required=True,
help='Subscription name')
parser.add_argument('--nummsgs',
type=int,
default=3,
help='Number of messages to pull and ack')
args = parser.parse_args()
ams = ArgoMessagingService(endpoint=args.host,
token=args.token,
project=args.project)
# static sleep between retry attempts
msg = AmsMessage(data='foo1', attributes={'bar1': 'baz1'}).dict()
try:
ret = ams.publish(args.topic, msg, retry=3, retrysleep=5, timeout=5)
print(ret)
except AmsException as e:
print(e)
# iptables -A OUTPUT -d messaging-devel.argo.grnet.gr -j DROP
ackids = list()
for id, msg in ams.pull_sub(args.subscription,
args.nummsgs,
retry=3,
retrysleep=5,
timeout=5):
data = msg.get_data()
msgid = msg.get_msgid()
attr = msg.get_attr()
print('msgid={0}, data={1}, attr={2}'.format(msgid, data, attr))
ackids.append(id)
if ackids:
ams.ack_sub(args.subscription,
ackids,
retry=3,
retrysleep=5,
timeout=5)
# backoff with each next retry attempt exponentially longer
msg = AmsMessage(data='foo2', attributes={'bar2': 'baz2'}).dict()
try:
ret = ams.publish(args.topic, msg, retry=3, retrybackoff=5, timeout=5)
print(ret)
except AmsException as e:
print(e)
# iptables -A OUTPUT -d messaging-devel.argo.grnet.gr -j DROP
ackids = list()
for id, msg in ams.pull_sub(args.subscription,
args.nummsgs,
retrybackoff=3,
retrysleep=5,
timeout=5):
data = msg.get_data()
msgid = msg.get_msgid()
attr = msg.get_attr()
print('msgid={0}, data={1}, attr={2}'.format(msgid, data, attr))
ackids.append(id)
if ackids:
ams.ack_sub(args.subscription, ackids)
# static sleep between retry attempts. this example uses consume context
# method that pull and acks msgs in one call.
msg = AmsMessage(data='foo3', attributes={'bar3': 'baz3'}).dict()
try:
ret = ams.publish(args.topic, msg, retry=3, retrysleep=5, timeout=5)
print(ret)
except AmsException as e:
print(e)
try:
msgs = ams.pullack_sub(args.subscription,
args.nummsgs,
retry=3,
retrysleep=5,
timeout=5)
for msg in msgs:
data = msg.get_data()
msgid = msg.get_msgid()
attr = msg.get_attr()
print('msgid={0}, data={1}, attr={2}'.format(msgid, data, attr))
except AmsException as e:
print(e)
def __init__(self,config):
self.pull_sub = config['pull_sub']
self.pub_topic = config['pub_topic']
self.pull_topic = config['pull_topic']
self.ams = ArgoMessagingService(endpoint=config['host'], token=config['token'], project=config['project'])
def post_assessment_results(self, niftyId, msgId, file_path, base_mpuri):
ams = ArgoMessagingService(endpoint=self.host, token=self.token, project=self.project)
contents = Path(file_path).read_text()
msg = AmsMessage(data=contents, attributes={'NIFTY_APPLIANCE_ID': niftyId, 'REQUEST_MESSAGE_ID': msgId, 'BASE_MPURI': base_mpuri}).dict()
ret = ams.publish(self.resultTopic, msg)
logging.debug('[%s] %s: Results has been successfully pushed.', niftyId, 'DEBUG')
def __init__(self,
hosts_and_ports,
qpath,
cert,
key,
dest=None,
listen=None,
capath=None,
check_crls=False,
use_ssl=False,
username=None,
password=None,
enc_cert=None,
verify_enc_cert=True,
pidfile=None,
path_type='dirq',
protocol=STOMP_MESSAGING,
project=None,
token=''):
'''
Creates an SSM2 object. If a listen value is supplied,
this SSM2 will be a receiver.
'''
self._conn = None
self._last_msg = None
self._brokers = hosts_and_ports
self._cert = cert
self._key = key
self._enc_cert = enc_cert
self._capath = capath
self._check_crls = check_crls
self._user = username
self._pwd = password
self._use_ssl = use_ssl
# use pwd auth if we're supplied both user and pwd
self._use_pwd = username is not None and password is not None
self.connected = False
self._listen = listen
self._dest = dest
self._valid_dns = []
self._pidfile = pidfile
# Used to differentiate between STOMP and AMS methods
self._protocol = protocol
# Used when interacting with an Argo Messaging Service
self._project = project
self._token = token
if self._protocol == Ssm2.AMS_MESSAGING:
self._ams = ArgoMessagingService(endpoint=self._brokers[0],
token=self._token,
cert=self._cert,
key=self._key,
project=self._project)
# create the filesystem queues for accepted and rejected messages
if dest is not None and listen is None:
# Determine what sort of outgoing structure to make
if path_type == 'dirq':
if QueueSimple is None:
raise ImportError("dirq path_type requested but the dirq "
"module wasn't found.")
self._outq = QueueSimple(qpath)
elif path_type == 'directory':
self._outq = MessageDirectory(qpath)
else:
raise Ssm2Exception('Unsupported path_type variable.')
elif listen is not None:
inqpath = os.path.join(qpath, 'incoming')
rejectqpath = os.path.join(qpath, 'reject')
# Receivers must use the dirq module, so make a quick sanity check
# that dirq is installed.
if Queue is None:
raise ImportError("Receiving SSMs must use dirq, but the dirq "
"module wasn't found.")
self._inq = Queue(inqpath, schema=Ssm2.QSCHEMA)
self._rejectq = Queue(rejectqpath, schema=Ssm2.REJECT_SCHEMA)
else:
raise Ssm2Exception('SSM must be either producer or consumer.')
# check that the cert and key match
if not crypto.check_cert_key(self._cert, self._key):
raise Ssm2Exception('Cert and key don\'t match.')
# Check that the certificate has not expired.
if not crypto.verify_cert_date(self._cert):
raise Ssm2Exception('Certificate %s has expired or will expire '
'within a day.' % self._cert)
# check the server certificate provided
if enc_cert is not None:
log.info('Messages will be encrypted using %s', enc_cert)
if not os.path.isfile(self._enc_cert):
raise Ssm2Exception(
'Specified certificate file does not exist: %s.' %
self._enc_cert)
# Check that the encyption certificate has not expired.
if not crypto.verify_cert_date(enc_cert):
raise Ssm2Exception(
'Encryption certificate %s has expired or will expire '
'within a day. Please obtain the new one from the final '
'server receiving your messages.' % enc_cert)
if verify_enc_cert:
if not crypto.verify_cert_path(self._enc_cert, self._capath,
self._check_crls):
raise Ssm2Exception(
'Failed to verify server certificate %s against CA path %s.'
% (self._enc_cert, self._capath))
# If the overall SSM log level is info, we want to only
# see entries from stomp.py and connectionpool at WARNING and above.
if logging.getLogger("ssm.ssm2").getEffectiveLevel() == logging.INFO:
logging.getLogger("stomp.py").setLevel(logging.WARNING)
logging.getLogger(
"requests.packages.urllib3.connectionpool").setLevel(
logging.WARNING)
# If the overall SSM log level is debug, we want to only
# see entries from stomp.py and connectionpool at INFO above.
elif logging.getLogger(
"ssm.ssm2").getEffectiveLevel() == logging.DEBUG:
logging.getLogger("stomp.py").setLevel(logging.INFO)
logging.getLogger(
"requests.packages.urllib3.connectionpool").setLevel(
logging.INFO)
def main():
MSG_NUM = 100
MSG_SIZE = 500
TIMEOUT = 180
parser = ArgumentParser(description="Nagios sensor for AMS")
parser.add_argument('-H',
dest='host',
type=str,
default='messaging-devel.argo.grnet.gr',
help='FQDN of AMS Service')
parser.add_argument('--token', type=str, required=True, help='Given token')
parser.add_argument('--project',
type=str,
required=True,
help='Project registered in AMS Service')
parser.add_argument('--topic',
type=str,
default='nagios_sensor_topic',
help='Given topic')
parser.add_argument('--subscription',
type=str,
default='nagios_sensor_sub',
help='Subscription name')
parser.add_argument('-t',
dest='timeout',
type=int,
default=TIMEOUT,
help='Timeout')
cmd_options = parser.parse_args()
nagios = NagiosResponse("All messages received correctly.")
ams = ArgoMessagingService(endpoint=cmd_options.host,
token=cmd_options.token,
project=cmd_options.project)
try:
if ams.has_topic(cmd_options.topic, timeout=cmd_options.timeout):
ams.delete_topic(cmd_options.topic, timeout=cmd_options.timeout)
if ams.has_sub(cmd_options.subscription, timeout=cmd_options.timeout):
ams.delete_sub(cmd_options.subscription,
timeout=cmd_options.timeout)
ams.create_topic(cmd_options.topic, timeout=cmd_options.timeout)
ams.create_sub(cmd_options.subscription,
cmd_options.topic,
timeout=cmd_options.timeout)
except AmsException as e:
nagios.writeCriticalMessage(e.msg)
nagios.setCode(nagios.CRITICAL)
print(nagios.getMsg())
raise SystemExit(nagios.getCode())
ams_msg = AmsMessage()
msg_orig = set()
msg_array = []
for i in range(1, MSG_NUM):
msg_txt = ''.join(
random.choice(string.ascii_letters + string.digits)
for i in range(MSG_SIZE))
attr_name = ''.join(
random.choice(string.ascii_letters + string.digits)
for i in range(4))
attr_value = ''.join(
random.choice(string.ascii_letters + string.digits)
for i in range(8))
msg_array.append(
ams_msg(data=msg_txt, attributes={attr_name: attr_value}))
hash_obj = hashlib.md5(msg_txt + attr_name + attr_value)
msg_orig.add(hash_obj.hexdigest())
try:
msgs = ams.publish(cmd_options.topic,
msg_array,
timeout=cmd_options.timeout)
ackids = []
rcv_msg = set()
for id, msg in ams.pull_sub(cmd_options.subscription,
MSG_NUM - 1,
True,
timeout=cmd_options.timeout):
attr = msg.get_attr()
hash_obj = hashlib.md5(msg.get_data() + attr.keys()[0] +
attr.values()[0])
rcv_msg.add(hash_obj.hexdigest())
if ackids:
ams.ack_sub(cmd_options.subscription,
ackids,
timeout=cmd_options.timeout)
ams.delete_topic(cmd_options.topic, timeout=cmd_options.timeout)
ams.delete_sub(cmd_options.subscription, timeout=cmd_options.timeout)
except AmsException as e:
nagios.writeCriticalMessage(e.msg)
nagios.setCode(nagios.CRITICAL)
print(nagios.getMsg())
raise SystemExit(nagios.getCode())
if msg_orig != rcv_msg:
nagios.writeCriticalMessage("Messages received incorrectly.")
nagios.setCode(nagios.CRITICAL)
print(nagios.getMsg())
raise SystemExit(nagios.getCode())
