def hermes2(once=False, thread=0, bulk=1000, sleep_time=10):
"""
Creates a Hermes2 Worker that can submit messages to different services (InfluXDB, ElasticSearch, ActiveMQ)
The list of services need to be define in the config service in the hermes section.
The list of endpoints need to be defined in rucio.cfg in the hermes section.
:param once: Run only once.
:param thread: Thread number at startup.
:param bulk: The number of requests to process.
:param sleep_time: Time between two cycles.
"""
executable = 'hermes2'
hostname = socket.getfqdn()
pid = os.getpid()
hb_thread = threading.current_thread()
heartbeat.sanity_check(executable=executable,
hostname=hostname,
pid=pid,
thread=hb_thread)
heart_beat = heartbeat.live(executable, hostname, pid, hb_thread)
# Make an initial heartbeat so that all daemons have the correct worker number on the next try
GRACEFUL_STOP.wait(10)
heart_beat = heartbeat.live(executable,
hostname,
pid,
hb_thread,
older_than=3600)
prepend_str = 'hermes2[%i/%i] : ' % (heart_beat['assign_thread'],
heart_beat['nr_threads'])
logger = formatted_logger(logging.log, prepend_str + '%s')
try:
services_list = get('hermes', 'services_list')
services_list = services_list.split(',')
except ConfigNotFound:
logger(logging.DEBUG, 'No services found, exiting')
sys.exit(1)
if 'influx' in services_list:
try:
influx_endpoint = config_get('hermes', 'influxdb_endpoint', False,
None)
if not influx_endpoint:
logger(
logging.ERROR,
'InfluxDB defined in the services list, but no endpoint can be find. Exiting'
)
sys.exit(1)
except Exception as err:
logger(logging.ERROR, str(err))
if 'elastic' in services_list:
try:
elastic_endpoint = config_get('hermes', 'elastic_endpoint', False,
None)
if not elastic_endpoint:
logger(
logging.ERROR,
'Elastic defined in the services list, but no endpoint can be find. Exiting'
)
sys.exit(1)
except Exception as err:
logger(logging.ERROR, str(err))
if 'activemq' in services_list:
try:
# activemq_endpoint = config_get('hermes', 'activemq_endpoint', False, None)
conns, destination, username, password, use_ssl = setup_activemq(
logger)
if not conns:
logger(
logging.ERROR,
'ActiveMQ defined in the services list, cannot be setup')
sys.exit(1)
except Exception as err:
logger(logging.ERROR, str(err))
while not GRACEFUL_STOP.is_set():
message_status = copy.deepcopy(services_list)
message_statuses = {}
stime = time.time()
try:
start_time = time.time()
heart_beat = heartbeat.live(executable,
hostname,
pid,
hb_thread,
older_than=3600)
prepend_str = 'hermes2[%i/%i] : ' % (heart_beat['assign_thread'],
heart_beat['nr_threads'])
logger = formatted_logger(logging.log, prepend_str + '%s')
messages = retrieve_messages(
bulk=bulk,
thread=heart_beat['assign_thread'],
total_threads=heart_beat['nr_threads'])
if messages:
for message in messages:
message_statuses[message['id']] = copy.deepcopy(
services_list)
logger(logging.DEBUG,
'Retrieved %i messages retrieved in %s seconds',
len(messages),
time.time() - start_time)
if 'influx' in message_status:
t_time = time.time()
logger(logging.DEBUG, 'Will submit to influxDB')
try:
state = aggregate_to_influx(messages=messages,
bin_size='1m',
endpoint=influx_endpoint,
logger=logger)
except Exception as error:
logger(logging.ERROR, 'Error sending to InfluxDB : %s',
str(error))
state = 500
if state in [204, 200]:
logger(
logging.INFO,
'Messages successfully submitted to influxDB in %s seconds',
time.time() - t_time)
for message in messages:
message_statuses[message['id']].remove('influx')
else:
logger(logging.INFO, 'Failure to submit to influxDB')
if 'elastic' in message_status:
t_time = time.time()
try:
state = submit_to_elastic(messages=messages,
endpoint=elastic_endpoint,
logger=logger)
except Exception as error:
logger(logging.ERROR, 'Error sending to Elastic : %s',
str(error))
state = 500
if state in [200, 204]:
logger(
logging.INFO,
'Messages successfully submitted to elastic in %s seconds',
time.time() - t_time)
for message in messages:
message_statuses[message['id']].remove('elastic')
else:
logger(logging.INFO, 'Failure to submit to elastic')
if 'emails' in message_status:
t_time = time.time()
try:
to_delete = deliver_emails(messages=messages,
logger=logger)
logger(
logging.INFO,
'Messages successfully submitted by emails in %s seconds',
time.time() - t_time)
for message_id in to_delete:
message_statuses[message_id].remove('emails')
except Exception as error:
logger(logging.ERROR, 'Error sending email : %s',
str(error))
if 'activemq' in message_status:
t_time = time.time()
try:
to_delete = deliver_to_activemq(
messages=messages,
conns=conns,
destination=destination,
username=username,
password=password,
use_ssl=use_ssl,
logger=logger)
logger(
logging.INFO,
'Messages successfully submitted to ActiveMQ in %s seconds',
time.time() - t_time)
for message_id in to_delete:
message_statuses[message_id].remove('activemq')
except Exception as error:
logger(logging.ERROR, 'Error sending to ActiveMQ : %s',
str(error))
to_delete = []
to_update = {}
for message in messages:
status = message_statuses[message['id']]
if not status:
to_delete.append({
'id': message['id'],
'created_at': message['created_at'],
'updated_at': message['created_at'],
'payload': str(message['payload']),
'event_type': message['event_type']
})
else:
status = ",".join(status)
if status not in to_update:
to_update[status] = []
to_update[status].append({
'id':
message['id'],
'created_at':
message['created_at'],
'updated_at':
message['created_at'],
'payload':
str(message['payload']),
'event_type':
message['event_type']
})
logger(logging.INFO, 'Deleting %s messages', len(to_delete))
delete_messages(messages=to_delete)
for status in to_update:
logger(
logging.INFO,
'Failure to submit %s messages to %s. Will update the message status',
str(len(to_update[status])), status)
update_messages_services(messages=to_update[status],
services=status)
if once:
break
daemon_sleep(start_time=stime,
sleep_time=sleep_time,
graceful_stop=GRACEFUL_STOP,
logger=logger)
except Exception:
logger(logging.ERROR, "Failed to submit messages", exc_info=True)
def hermes2(once=False, thread=0, bulk=1000, sleep_time=10):
"""
Creates a Hermes2 Worker that can submit messages to different services (InfluXDB, ElasticSearch, ActiveMQ)
The list of services need to be define in the config service in the hermes section.
The list of endpoints need to be defined in rucio.cfg in the hermes section.
:param once: Run only once.
:param thread: Thread number at startup.
:param bulk: The number of requests to process.
:param sleep_time: Time between two cycles.
"""
try:
services_list = get('hermes', 'services_list')
services_list = services_list.split(',')
except ConfigNotFound as err:
logging.debug('No services found, exiting')
sys.exit(1)
if 'influx' in services_list:
try:
influx_endpoint = config_get('hermes', 'influxdb_endpoint', False,
None)
if not influx_endpoint:
logging.error(
'InfluxDB defined in the services list, but no endpoint can be find. Exiting'
)
sys.exit(1)
except Exception as err:
logging.error(err)
if 'elastic' in services_list:
try:
elastic_endpoint = config_get('hermes', 'elastic_endpoint', False,
None)
if not elastic_endpoint:
logging.error(
'Elastic defined in the services list, but no endpoint can be find. Exiting'
)
sys.exit(1)
except Exception as err:
logging.error(err)
if 'activemq' in services_list:
try:
activemq_endpoint = config_get('hermes', 'activemq_endpoint',
False, None)
if not activemq_endpoint:
logging.error(
'ActiveMQ defined in the services list, but no endpoint can be find. Exiting'
)
sys.exit(1)
except Exception as err:
logging.error(err)
executable = 'hermes2'
hostname = socket.getfqdn()
pid = os.getpid()
hb_thread = threading.current_thread()
heartbeat.sanity_check(executable=executable,
hostname=hostname,
pid=pid,
thread=hb_thread)
heart_beat = heartbeat.live(executable, hostname, pid, hb_thread)
prepend_str = 'Thread [%i/%i] : ' % (heart_beat['assign_thread'],
heart_beat['nr_threads'])
# Make an initial heartbeat so that all daemons have the correct worker number on the next try
GRACEFUL_STOP.wait(10)
heart_beat = heartbeat.live(executable,
hostname,
pid,
hb_thread,
older_than=3600)
while not GRACEFUL_STOP.is_set():
message_status = deepcopy(services_list)
stime = time.time()
try:
start_time = time.time()
heart_beat = heartbeat.live(executable,
hostname,
pid,
hb_thread,
older_than=3600)
prepend_str = 'Thread [%i/%i] : ' % (heart_beat['assign_thread'],
heart_beat['nr_threads'])
messages = retrieve_messages(
bulk=bulk,
thread=heart_beat['assign_thread'],
total_threads=heart_beat['nr_threads'])
if messages:
logging.debug(
'%s Retrieved %i messages retrieved in %s seconds',
prepend_str, len(messages),
time.time() - start_time)
if 'influx' in message_status:
logging.debug('%s Will submit to influxDB', prepend_str)
state = aggregate_to_influx(messages,
bin_size='1m',
endpoint=influx_endpoint,
prepend_str=prepend_str)
if state in [204, 200]:
logging.info(
'%s Messages successfully submitted to influxDB',
prepend_str)
message_status.remove('influx')
else:
logging.info('%s Failure to submit to influxDB',
prepend_str)
if 'elastic' in message_status:
state = submit_to_elastic(messages,
endpoint=elastic_endpoint,
prepend_str=prepend_str)
if state in [200, 204]:
logging.info(
'%s Messages successfully submitted to elastic',
prepend_str)
message_status.remove('elastic')
else:
logging.info('%s Failure to submit to elastic',
prepend_str)
to_delete_or_update = []
for message in messages:
to_delete_or_update.append({
'id':
message['id'],
'created_at':
message['created_at'],
'updated_at':
message['created_at'],
'payload':
str(message['payload']),
'event_type':
'email'
})
if message_status == []:
delete_messages(messages=to_delete_or_update)
else:
logging.info(
'%s Failure to submit to one service. Will update the message status',
prepend_str)
update_messages_services(messages=to_delete_or_update,
services=",".join(message_status))
if once:
break
tottime = time.time() - stime
if tottime < sleep_time:
logging.info('%s Will sleep for %s seconds', prepend_str,
sleep_time - tottime)
time.sleep(sleep_time - tottime)
except:
logging.critical(traceback.format_exc())