def main(argv=tuple(), tbuf=None, **kwargs):
# instantiate TB
if tbuf is None:
from pandaserver.taskbuffer.TaskBuffer import taskBuffer
taskBuffer.init(panda_config.dbhost,
panda_config.dbpasswd,
nDBConnection=1)
else:
taskBuffer = tbuf
# dbif session
session = dbif.get_session()
# If no argument, call the basic configurator
if len(argv) == 1:
_logger = logger_utils.make_logger(base_logger, 'Configurator')
t1 = time.time()
configurator = Configurator(session=session)
if not configurator.run():
_logger.critical('Configurator loop FAILED')
t2 = time.time()
_logger.debug('Configurator run took {0}s'.format(t2 - t1))
# If --network argument, call the network configurator
elif len(argv) == 2 and argv[1].lower() == '--network':
_logger = logger_utils.make_logger(base_logger, 'NetworkConfigurator')
t1 = time.time()
network_configurator = NetworkConfigurator(taskBuffer=taskBuffer,
session=session)
if not network_configurator.run():
_logger.critical('Configurator loop FAILED')
t2 = time.time()
_logger.debug(' run took {0}s'.format(t2 - t1))
# If --json_dump
elif len(argv) == 2 and argv[1].lower() == '--json_dump':
_logger = logger_utils.make_logger(base_logger, 'JsonDumper')
t1 = time.time()
json_dumper = JsonDumper(taskBuffer=taskBuffer, session=session)
out_msg = json_dumper.run()
_logger.debug('Json_dumper finished with {0}'.format(out_msg))
t2 = time.time()
_logger.debug(' run took {0}s'.format(t2 - t1))
else:
_logger.error(
'Configurator being called with wrong arguments. Use either no arguments or --network or --json_dump'
)
# dbif session close
session.close()
dbif.engine_dispose()
def _spawn_processors(self, processor_list):
"""
spawn processors threads
"""
tmp_logger = logger_utils.make_logger(base_logger,
token=self.get_pid(),
method_name='_spawn_processors')
tmp_logger.debug('start')
for processor_name in processor_list:
try:
attr_dict = self.processor_attr_map[processor_name]
self.processor_thread_map[
processor_name] = SimpleMsgProcThread(
attr_dict, sleep_time=self.process_sleep_time)
mc_thread = self.processor_thread_map[processor_name]
mc_thread.start()
tmp_logger.info(
'spawned processors thread {0} with plugin={1} , in_q={2}, out_q={3}'
.format(processor_name,
attr_dict['plugin'].__class__.__name__,
attr_dict['in_queue'], attr_dict['out_queue']))
except Exception as e:
tmp_logger.error(
'failed to spawn processor thread {0} with plugin={1} , in_q={2}, out_q={3} ; {4}: {5} '
.format(processor_name,
attr_dict['plugin'].__class__.__name__,
attr_dict['in_queue'], attr_dict['out_queue'],
e.__class__.__name__, e))
tmp_logger.debug('done')
def run(self):
tmp_log = logger_utils.make_logger(base_logger, method_name='TestingAgent.run')
while True:
# do something
tmp_log.debug('start')
# sleep
time.sleep(self.sleepPeriod)
def _kill_processors(self, processor_list, block=True):
"""
kill processor threads
"""
tmp_logger = logger_utils.make_logger(base_logger,
token=self.get_pid(),
method_name='_kill_processors')
tmp_logger.debug('start')
for processor_name in processor_list:
try:
mc_thread = self.processor_thread_map.get(processor_name)
if mc_thread is None:
tmp_logger.debug(
'processor thread {0} does not exist. Skipped...'.
format(processor_name))
elif not mc_thread.is_alive():
tmp_logger.debug(
'processor thread {0} already stopped. Skipped...'.
format(processor_name))
else:
mc_thread.stop()
tmp_logger.info(
'signaled stop to processor thread {0}, block={1}'.
format(processor_name, block))
if block:
while mc_thread.is_alive():
time.sleep(0.125)
tmp_logger.info('processor thread {0} stopped'.format(
processor_name))
except Exception as e:
tmp_logger.error(
'failed to stop processor thread {0} ; {1}: {2} '.format(
processor_name, e.__class__.__name__, e))
tmp_logger.debug('done')
def initialize(self):
"""
customized initialize method
this method can override attributes set from config file
"""
tmp_logger = logger_utils.make_logger(base_logger,
token=self.get_pid(),
method_name='initialize')
tmp_logger.debug('start')
pass
tmp_logger.debug('done')
def process(self, msg_obj, decoded_data=None):
# logger
tmp_log = logger_utils.make_logger(base_logger, method_name='process')
# start
tmp_log.info('start')
# parse
if decoded_data is None:
# json decode
try:
msg_dict = json.loads(msg_obj.data)
except Exception as e:
err_str = 'failed to parse message json {2} , skipped. {0} : {1}'.format(e.__class__.__name__, e,
msg_obj.data)
tmp_log.error(err_str)
raise
else:
msg_dict = decoded_data
# run
try:
tmp_log.debug('got message {0}'.format(msg_dict))
if msg_dict['msg_type'] == 'generate_job':
# get task to generate jobs
jediTaskID = int(msg_dict['taskid'])
s, taskSpec = self.tbIF.getTaskWithID_JEDI(jediTaskID)
if not taskSpec:
tmp_log.debug('unknown task {}'.format(jediTaskID))
else:
# get WQ
vo = taskSpec.vo
prodSourceLabel = taskSpec.prodSourceLabel
workQueue = self.tbIF.getWorkQueueMap().getQueueWithIDGshare(taskSpec.workQueue_ID, taskSpec.gshare)
# get inputs
tmpList = self.tbIF.getTasksToBeProcessed_JEDI(self.pid, None, workQueue, None, None, nFiles=1000,
target_tasks=[jediTaskID])
if tmpList:
inputList = ListWithLock(tmpList)
# create thread
threadPool = ThreadPool()
siteMapper = self.tbIF.getSiteMapper()
taskSetupper = TaskSetupper(vo, prodSourceLabel)
taskSetupper.initializeMods(self.tbIF, self.ddmIF)
gen = JobGeneratorThread(inputList, threadPool, self.tbIF, self.ddmIF, siteMapper,
True, taskSetupper, self.pid, workQueue, 'pjmsg',
None, None, None, False)
gen.start()
gen.join()
else:
tmp_log.debug('unknown message type : {}'.format(msg_dict['msg_type']))
except Exception as e:
err_str = 'failed to run, skipped. {0} : {1}'.format(e.__class__.__name__, e)
tmp_log.error(err_str)
raise
# done
tmp_log.info('done')
def launcher():
tmp_log = logger_utils.make_logger(base_logger, method_name='launcher')
tmp_log.debug('start')
try:
pass
except Exception as e:
# tmp_log.error('failed to read config json file; should not happen... {0}: {1}'.format(e.__class__.__name__, e))
raise e
else:
agent = TestingAgent()
agent.run()
def _spawn_senders(self, mb_sender_proxy_list):
"""
spawn connection/listener threads of certain message broker sender proxy
"""
tmp_logger = logger_utils.make_logger(base_logger,
token=self.get_pid(),
method_name='_spawn_senders')
tmp_logger.debug('start')
for mb_proxy in mb_sender_proxy_list:
mb_proxy.go()
tmp_logger.info('spawned listener {0}'.format(mb_proxy.name))
tmp_logger.debug('done')
def __init__(self, mb_proxy, conn_id, *args, **kwargs):
# logger
_token = '{0}-{1}'.format(mb_proxy.__class__.__name__, mb_proxy.name)
self.logger = logger_utils.make_logger(base_logger,
token=_token,
method_name='MsgListener')
# associated messgage broker proxy
self.mb_proxy = mb_proxy
# connection id
self.conn_id = conn_id
# whether log verbosely
self.verbose = kwargs.get('verbose', False)
def __init__(self, attr_dict, sleep_time):
GenericThread.__init__(self)
self.logger = logger_utils.make_logger(
base_logger,
token=self.get_pid(),
method_name='SimpleMsgProcThread')
self.__to_run = True
self.plugin = attr_dict['plugin']
self.in_queue = attr_dict.get('in_queue')
self.mb_sender_proxy = attr_dict.get('mb_sender_proxy')
self.sleep_time = sleep_time
self.verbose = attr_dict.get('verbose', False)
def _parse_config(self):
"""
parse message processor configuration json file
Typical example dict from config json:
mb_servers_dict = {
'Server_1': {
'host_port_list': ['192.168.0.1:777', '192.168.0.2:777'],
'use_ssl': True,
'cert_file': 'aaa.cert.pem',
'key_file': 'bbb.key.pem',
'username': '******',
'passcode': 'xxxxyyyyzzzz',
'vhost': '/somehost',
'verbose': True,
},
...
}
queues_dict = {
'Queue_1': {
'enable': True,
'server': 'Server_1',
'destination': '/queue/some_queue',
},
...
}
processors_dict = {
'Processor_1': {
'enable': True,
'module': 'plugin.module',
'name': 'PluginClassName',
'in_queue': 'Queue_1',
'out_queue': 'Queue_2',
'verbose': True,
},
...
}
"""
# logger
tmp_logger = logger_utils.make_logger(base_logger,
token=self.get_pid(),
method_name='_parse_config')
tmp_logger.debug('start')
# parse config json
with open(self.config_file, 'r') as _f:
raw_dict = json.load(_f)
self._mb_servers_dict = raw_dict['mb_servers']
self._queues_dict = raw_dict['queues']
self._processors_dict = raw_dict.get('processors', {})
# set self optional attributes
if raw_dict.get('guard_period') is not None:
self.guard_period = raw_dict['guard_period']
tmp_logger.debug('done')
def _kill_listeners(self, mb_listener_proxy_list):
"""
kill connection/listener threads of certain message broker listener proxy
"""
tmp_logger = logger_utils.make_logger(base_logger,
token=self.get_pid(),
method_name='_kill_listeners')
tmp_logger.debug('start')
for mb_proxy in mb_listener_proxy_list:
mb_proxy.stop()
tmp_logger.info('signaled stop to listener {0}'.format(
mb_proxy.name))
tmp_logger.debug('done')
def _get_connection_dict(host_port_list,
use_ssl=False,
cert_file=None,
key_file=None,
vhost=None,
force=False):
"""
get dict {conn_id: connection}
"""
tmp_logger = logger_utils.make_logger(base_logger,
method_name='_get_connection_dict')
conn_dict = dict()
# resolve all distinct hosts behind hostname
resolved_host_port_set = set()
for host_port in host_port_list:
host, port = host_port.split(':')
port = int(port)
addrinfos = socket.getaddrinfo(host, port)
for addrinfo in addrinfos:
resolved_host = socket.getfqdn(addrinfo[4][0])
resolved_host_port_set.add((resolved_host, port))
# make connections
for host, port in resolved_host_port_set:
host_port = '{0}:{1}'.format(host, port)
conn_id = host_port
if conn_id not in conn_dict:
try:
conn = stomp.Connection12(host_and_ports=[(host, port)],
vhost=vhost)
if use_ssl:
ssl_opts = {
'ssl_version': ssl.PROTOCOL_TLSv1,
'cert_file': cert_file,
'key_file': key_file
}
conn.set_ssl(for_hosts=[(host, port)], **ssl_opts)
except AttributeError:
# Older version of stomp.py
ssl_opts = {
'use_ssl': use_ssl,
'ssl_version': ssl.PROTOCOL_TLSv1,
'ssl_cert_file': cert_file,
'ssl_key_file': key_file
}
conn = stomp.Connection12(host_and_ports=[(host, port)],
vhost=vhost,
**ssl_opts)
conn_dict[conn_id] = conn
tmp_logger.debug('got {0} connections to {1}'.format(
len(conn_dict), ' , '.join(conn_dict.keys())))
return conn_dict
def __init__(self,
name,
host_port_list,
destination,
use_ssl=False,
cert_file=None,
key_file=None,
vhost=None,
username=None,
passcode=None,
wait=True,
verbose=False,
**kwargs):
# logger
self.logger = logger_utils.make_logger(base_logger,
token=name,
method_name='MBSenderProxy')
# name of message queue
self.name = name
# connection parameters
self.host_port_list = host_port_list
self.use_ssl = use_ssl
self.cert_file = cert_file
self.key_file = key_file
self.vhost = vhost
# destination queue to subscribe
self.destination = destination
# subscription ID
self.sub_id = 'panda-MBSenderProxy_{0}_r{1:06}'.format(
socket.getfqdn(), random.randrange(10**6))
# client ID
self.client_id = 'client_{0}_{1}'.format(self.sub_id, hex(id(self)))
# connect parameters
self.connect_params = {
'username': username,
'passcode': passcode,
'wait': wait,
'headers': {
'client-id': self.client_id
}
}
# number of attempts to restart
self.n_restart = 0
# whether got disconnected from on_disconnected
self.got_disconnected = False
# whether to disconnect intentionally
self.to_disconnect = False
# whether to log verbosely
self.verbose = verbose
# get connection
self._get_connection()
def stop(self, block=True):
"""
send stop signal to this thread
"""
tmp_logger = logger_utils.make_logger(base_logger,
token=self.get_pid(),
method_name='stop')
tmp_logger.debug('start')
self.__to_run = False
tmp_logger.info('signaled stop')
if block:
while self.is_alive():
time.sleep(0.01)
tmp_logger.debug('done')
def process(self, msg_obj):
# logger
tmp_log = logger_utils.make_logger(base_logger, method_name='process')
# start
# tmp_log.info('start')
# tmp_log.debug('sub_id={0} ; msg_id={1}'.format(msg_obj.sub_id, msg_obj.msg_id))
# parse yaml
try:
message_dict = yaml.safe_load(msg_obj.data)
except Exception as e:
err_str = 'failed to parse message yaml {2} , skipped. {0} : {1}'.format(e.__class__.__name__, e, msg_obj.data)
tmp_log.error(err_str)
raise
# run
try:
to_continue = True
dsn = 'UNKNOWN'
# check event type
event_type = message_dict['event_type']
if event_type not in ['datasetlock_ok']:
# tmp_log.debug('{0} skip'.format(event_type))
to_continue = False
if to_continue:
# tmp_log.debug('{0} start'.format(event_type))
message_payload = message_dict['payload']
# only for _dis or _sub
dsn = message_payload['name']
if (re.search('_dis\d+$', dsn) is None) and (re.search('_sub\d+$', dsn) is None):
# tmp_log.debug('{0} is not _dis or _sub dataset, skip'.format(dsn))
to_continue = False
if to_continue:
tmp_log.debug('sub_id={0} ; msg_id={1}'.format(msg_obj.sub_id, msg_obj.msg_id))
tmp_log.debug('{0} start'.format(event_type))
# take action
scope = message_payload['scope']
site = message_payload['rse']
tmp_log.debug('{dsn} site={site} type={type}'.format(dsn=dsn, site=site, type=event_type))
thr = DDMHandler(taskBuffer=self.tbIF, vuid=None, site=site, dataset=dsn, scope=scope)
# just call run rather than start+join, to run it in main thread instead of spawning new thread
thr.run()
del thr
tmp_log.debug('done {0}'.format(dsn))
except Exception as e:
err_str = 'failed to run, skipped. {0} : {1}'.format(e.__class__.__name__, e)
tmp_log.error(err_str)
raise
def process(self, msg_obj):
# logger
tmp_log = logger_utils.make_logger(base_logger, method_name='process')
# start
# tmp_log.info('start')
# tmp_log.debug('sub_id={0} ; msg_id={1}'.format(msg_obj.sub_id, msg_obj.msg_id))
# run
try:
msg = msg_obj.data
tmp_log.debug('forward message {0}'.format(msg))
except Exception as e:
err_str = 'failed to run, skipped. {0} : {1}'.format(
e.__class__.__name__, e)
tmp_log.error(err_str)
raise
# done
# tmp_log.info('done')
return msg
def process(self, msg_obj):
# logger
tmp_log = logger_utils.make_logger(base_logger, method_name='process')
# start
tmp_log.info('start')
tmp_log.debug('sub_id={0} ; msg_id={1}'.format(msg_obj.sub_id,
msg_obj.msg_id))
# parse json
try:
msg_dict = json.loads(msg_obj.data)
except Exception as e:
err_str = 'failed to parse message json {2} , skipped. {0} : {1}'.format(
e.__class__.__name__, e, msg_obj.data)
tmp_log.error(err_str)
raise
# sanity check
try:
msg_type = msg_dict['msg_type']
except Exception as e:
err_str = 'failed to parse message object dict {2} , skipped. {0} : {1}'.format(
e.__class__.__name__, e, msg_dict)
tmp_log.error(err_str)
raise
# run different plugins according to message type
try:
if msg_type in ('file_stagein', 'collection_stagein',
'work_stagein'):
self.plugin_TapeCarousel.process(msg_obj,
decoded_data=msg_dict)
elif msg_type in ('file_hyperparameteropt',
'collection_hyperparameteropt',
'work_hyperparameteropt'):
self.plugin_HPO.process(msg_obj, decoded_data=msg_dict)
elif msg_type in ('file_processing', 'collection_processing',
'work_processing'):
self.plugin_Processing.process(msg_obj, decoded_data=msg_dict)
else:
# Asked by iDDS and message broker guys, JEDI needs to consume unknown types of messages and do nothing...
warn_str = 'unknown msg_type : {0}'.format(msg_type)
tmp_log.warning(warn_str)
except Exception:
raise
# done
tmp_log.info('done')
def _guard_senders(self, mb_sender_proxy_list):
"""
guard connection/listener threads of certain message broker sender proxy, reconnect when disconnected
"""
tmp_logger = logger_utils.make_logger(base_logger,
token=self.get_pid(),
method_name='_guard_senders')
tmp_logger.debug('start')
for mb_proxy in mb_sender_proxy_list:
if mb_proxy.got_disconnected and not mb_proxy.to_disconnect:
tmp_logger.debug(
'found listener {0} disconnected unexpectedly; trigger restart...'
.format(mb_proxy.name))
mb_proxy.restart()
if mb_proxy.n_restart > 10:
tmp_logger.warning(
'found listener {0} keep getting disconnected; already restarted {1} times'
.format(mb_proxy.name, mb_proxy.n_restart))
tmp_logger.info('restarted listener {0}'.format(mb_proxy.name))
tmp_logger.debug('done')
def launcher(stop_event):
tmp_log = logger_utils.make_logger(msg_processor.base_logger,
method_name='launcher')
tmp_log.debug('start')
try:
config_file = jedi_config.msgprocessor.configFile
except Exception as e:
tmp_log.error(
'failed to read config json file; should not happen... {0}: {1}'.
format(e.__class__.__name__, e))
raise e
# start
agent = MsgProcAgent(config_file)
agent.start()
tmp_log.debug('started')
# wait for stop event
stop_event.wait()
# stop
tmp_log.debug('stopping')
agent.stop()
tmp_log.debug('stopped')
def __init__(self, config_file, process_sleep_time=0.0001, **kwargs):
GenericThread.__init__(self)
self.__to_run = True
self.config_file = config_file
self.process_sleep_time = process_sleep_time
self.init_mb_listener_proxy_list = []
self.init_mb_sender_proxy_list = []
self.init_processor_list = []
self.processor_attr_map = dict()
self.processor_thread_map = dict()
self.guard_period = 300
self._last_guard_timestamp = 0
self.prefetch_count = None
# log
tmp_logger = logger_utils.make_logger(base_logger,
token=self.get_pid(),
method_name='__init__')
# parse config
self._parse_config()
# done
tmp_logger.info('done')
def get_plugin(self, plugin_conf):
# logger
tmpLog = logger_utils.make_logger(base_logger,
method_name='get_plugin')
# use module + class as key
moduleName = plugin_conf['module']
className = plugin_conf['name']
if moduleName is None or className is None:
tmpLog.warning(
'Invalid plugin; either module or name is missing '.format(
moduleName))
return None
pluginKey = '{0}.{1}'.format(moduleName, className)
# get class
with self.__lock:
if pluginKey not in self.classMap:
# import module
tmpLog.debug('importing {0}'.format(moduleName))
mod = __import__(moduleName)
for subModuleName in moduleName.split('.')[1:]:
mod = getattr(mod, subModuleName)
# get class
tmpLog.debug('getting class {0}'.format(className))
cls = getattr(mod, className)
# add
self.classMap[pluginKey] = cls
tmpLog.debug('loaded class {0}'.format(pluginKey))
else:
tmpLog.debug(
'class {0} already loaded. Skipped'.format(pluginKey))
# instantiate
cls = self.classMap[pluginKey]
inst = cls()
for tmpKey, tmpVal in plugin_conf.items():
if tmpKey in ['module', 'name']:
continue
setattr(inst, tmpKey, tmpVal)
tmpLog.debug('created an instance of {0}'.format(pluginKey))
# return
return inst
def gshare_preference(self):
tmp_log = logger_utils.make_logger(main_logger, 'FetchData')
try:
# get share and hs info
if self.gshare_status is None:
self.gshare_status = self.tbuf.getGShareStatus()
# initialize
gshare_dict = dict()
# rank and data
for idx, leaf in enumerate(self.gshare_status):
rank = idx + 1
gshare = leaf['name']
gshare_dict[gshare] = {
'rank': rank,
'running_hs': leaf['running'],
'target_hs': leaf['target'],
}
tmp_log.debug('rank={rank}, gshare={gshare}'.format(
gshare=gshare, **gshare_dict[gshare]))
# return
return gshare_dict
except Exception:
tmp_log.error(traceback.format_exc())
def run(self):
"""
main thread
"""
tmp_logger = logger_utils.make_logger(base_logger,
token=self.get_pid(),
method_name='run')
tmp_logger.debug('start')
# set up instances from config
self._setup_instances()
# initialize
self.initialize()
# spawn all message broker listener proxy threads
self._spawn_listeners(self.init_mb_listener_proxy_list)
# spawn all message broker sender proxy threads
self._spawn_senders(self.init_mb_sender_proxy_list)
# spawn all processor threads according to config
self._spawn_processors(self.init_processor_list)
# main loop
tmp_logger.debug('looping')
while self.__to_run:
# guard listeners and senders
if time.time() >= self._last_guard_timestamp + self.guard_period:
self._guard_listeners(self.init_mb_listener_proxy_list)
self._guard_senders(self.init_mb_sender_proxy_list)
self._last_guard_timestamp = time.time()
# sleep
time.sleep(0.01)
# tear down
tmp_logger.debug('tearing down')
# kill all message broker listener proxy threads
self._kill_listeners(self.init_mb_listener_proxy_list)
# kill all message broker sender proxy threads
self._kill_senders(self.init_mb_sender_proxy_list)
# kill all processor threads according to config
self._kill_processors(self.init_processor_list)
tmp_logger.debug('done')
def launcher(taskBufferIF, ddmIF):
tmp_log = logger_utils.make_logger(base_logger, method_name='launcher')
tmp_log.debug('start')
try:
jedi_config.daemon.config
except Exception as e:
tmp_log.error(
'failed to read config json file; should not happen... {0}: {1}'.
format(e.__class__.__name__, e))
raise e
# whether to run daemons
if not getattr(jedi_config.daemon, 'enable', False):
tmp_log.debug('daemon disabled ; skipped')
return
# parameters
n_workers = getattr(jedi_config.daemon, 'n_proc', 1)
worker_lifetime = getattr(jedi_config.daemon, 'proc_lifetime', 28800)
# start
agent = DaemonMaster(logger=tmp_log,
n_workers=n_workers,
worker_lifetime=worker_lifetime,
tbuf=taskBufferIF,
ddmif=ddmIF)
agent.run()
def start_passive_mode(self,
in_q_list=None,
out_q_list=None,
prefetch_size=100):
"""
start passive mode: only spwan mb proxies (without spawning agent and plugin threads)
in_q_list: list of inward queue name
out_q_list: list of outward queue name
prefetch_size: prefetch size of the message broker (can control number of un-acknowledged messages stored in the local buffer)
returns dict of mb proxies
"""
tmp_logger = logger_utils.make_logger(base_logger,
token=self.get_pid(),
method_name='start_passive_mode')
tmp_logger.debug('start')
# initialize
# self.initialize()
all_queue_names = list(self._queues_dict.keys())
if in_q_list is None:
in_q_list = all_queue_names
if out_q_list is None:
out_q_list = all_queue_names
# mb_listener_proxy instances
mb_listener_proxy_dict = dict()
for in_queue in in_q_list:
if in_queue not in self._queues_dict:
continue
qconf = self._queues_dict[in_queue]
if not qconf.get('enable', True):
continue
sconf = self._mb_servers_dict[qconf['server']]
mb_listener_proxy = get_mb_proxy(name=in_queue,
sconf=sconf,
qconf=qconf,
mode='listener',
prefetch_size=prefetch_size)
mb_listener_proxy_dict[in_queue] = mb_listener_proxy
# mb_sender_proxy instances
mb_sender_proxy_dict = dict()
for out_queue in out_q_list:
if out_queue not in self._queues_dict:
continue
qconf = self._queues_dict[out_queue]
if not qconf.get('enable', True):
continue
sconf = self._mb_servers_dict[qconf['server']]
mb_sender_proxy = get_mb_proxy(name=out_queue,
sconf=sconf,
qconf=qconf,
mode='sender')
mb_sender_proxy_dict[out_queue] = mb_sender_proxy
# spawn message broker listener proxy connections
for queue_name, mb_proxy in mb_listener_proxy_dict.items():
mb_proxy.go()
tmp_logger.debug(
'spawned listener proxy for {0}'.format(queue_name))
# spawn message broker sender proxy connections
for queue_name, mb_proxy in mb_sender_proxy_dict.items():
mb_proxy.go()
tmp_logger.debug('spawned sender proxy for {0}'.format(queue_name))
tmp_logger.debug('done')
# return
return {
'in': mb_listener_proxy_dict,
'out': mb_sender_proxy_dict,
}
def __init__(self,
name,
host_port_list,
destination,
use_ssl=False,
cert_file=None,
key_file=None,
vhost=None,
username=None,
passcode=None,
wait=True,
ack_mode='client-individual',
skip_buffer=False,
conn_mode='all',
prefetch_size=None,
verbose=False,
**kwargs):
# logger
self.logger = logger_utils.make_logger(base_logger,
token=name,
method_name='MBListenerProxy')
# name of message queue
self.name = name
# connection parameters
self.host_port_list = host_port_list
self.use_ssl = use_ssl
self.cert_file = cert_file
self.key_file = key_file
self.vhost = vhost
# destination queue to subscribe
self.destination = destination
# subscription ID
self.sub_id = 'panda-MBListenerProxy_{0}_r{1:06}'.format(
socket.getfqdn(), random.randrange(10**6))
# client ID
self.client_id = 'client_{0}_{1}'.format(self.sub_id, hex(id(self)))
# connect parameters
self.connect_params = {
'username': username,
'passcode': passcode,
'wait': wait,
'headers': {
'client-id': self.client_id
}
}
# acknowledge mode
self.ack_mode = ack_mode
# associate message buffer
self.msg_buffer = MsgBuffer(queue_name=self.name)
# connection mode; "all" or "any"
self.conn_mode = conn_mode
# connection dict
self.connection_dict = {}
# message listener dict
self.listener_dict = {}
# whether to skip buffer and dump to self.dump_msgs; True only in testing
self.skip_buffer = skip_buffer
# dump messages
self.dump_msgs = []
# number of attempts to restart
self.n_restart = 0
# whether got disconnected from on_disconnected
self.got_disconnected = False
# whether to disconnect intentionally
self.to_disconnect = False
# whether to log verbosely
self.verbose = verbose
# prefetch count of the MB (max number of un-acknowledge messages allowed)
self.prefetch_size = prefetch_size
# evaluate subscription headers
self._evaluate_subscription_headers()
# get connections
self._get_connections()
def run(self):
# update logger thread id
self.logger = logger_utils.make_logger(
base_logger,
token=self.get_pid(),
method_name='SimpleMsgProcThread')
# start
self.logger.info('start run')
# initialization step of plugin
self.logger.info('plugin initialize')
self.plugin.initialize()
# message buffer
self.logger.info('message buffer is {0}'.format(self.in_queue))
msg_buffer = MsgBuffer(queue_name=self.in_queue)
# main loop
self.logger.info('start loop')
while self.__to_run:
is_processed = False
proc_ret = None
# as consumer
if self.in_queue:
# get from buffer
msg_obj = msg_buffer.get()
if msg_obj is not None:
if self.verbose:
self.logger.debug('received a new message')
self.logger.debug('plugin process start')
try:
with msg_obj as _msg_obj:
proc_ret = self.plugin.process(_msg_obj)
is_processed = True
if self.verbose:
self.logger.debug('successfully processed')
except Exception as e:
self.logger.error(
'error when process message msg_id={0} with {1}: {2} '
.format(msg_obj.msg_id, e.__class__.__name__, e))
if self.verbose:
self.logger.debug('plugin process end')
else:
if self.verbose:
self.logger.debug('plugin process start')
try:
proc_ret = self.plugin.process(None)
is_processed = True
if self.verbose:
self.logger.debug('successfully processed')
except Exception as e:
self.logger.error(
'error when process with {0}: {1} '.format(
msg_obj.msg_id, e.__class__.__name__, e))
if self.verbose:
self.logger.debug('plugin process end')
# as producer
if self.mb_sender_proxy and is_processed:
self.mb_sender_proxy.send(proc_ret)
if self.verbose:
self.logger.debug('sent a processed message')
# sleep
time.sleep(self.sleep_time)
# stop loop
self.logger.info('stopped loop')
# tear down
self.logger.info('stopped run')
def process(self, msg_obj, decoded_data=None):
# logger
tmp_log = logger_utils.make_logger(base_logger, method_name='process')
# start
tmp_log.info('start')
tmp_log.debug('sub_id={0} ; msg_id={1}'.format(msg_obj.sub_id,
msg_obj.msg_id))
# parse
if decoded_data is None:
# json decode
try:
msg_dict = json.loads(msg_obj.data)
except Exception as e:
err_str = 'failed to parse message json {2} , skipped. {0} : {1}'.format(
e.__class__.__name__, e, msg_obj.data)
tmp_log.error(err_str)
raise
else:
msg_dict = decoded_data
# sanity check
try:
jeditaskid = int(msg_dict['workload_id'])
# message type
msg_type = msg_dict['msg_type']
if msg_type == 'file_processing':
target_list = msg_dict['files']
elif msg_type == 'collection_processing':
target_list = msg_dict['collections']
elif msg_type == 'work_processing':
pass
else:
raise ValueError(
'invalid msg_type value: {0}'.format(msg_type))
# relation type
relation_type = msg_dict.get('relation_type')
except Exception as e:
err_str = 'failed to parse message object dict {2} , skipped. {0} : {1}'.format(
e.__class__.__name__, e, msg_dict)
tmp_log.error(err_str)
raise
# run
try:
# initialize to_proceed
to_proceed = False
# type filters
if msg_type in ['file_processing', 'collection_processing'] \
and relation_type in ['input']:
to_proceed = True
# whether to proceed the targets
if to_proceed:
# initialize
scope_name_list_map = {}
missing_files_list = []
# loop over targets
for target in target_list:
name = target['name']
scope = target['scope']
if (msg_type == 'file_processing' and target['status'] in ['Available']) \
or (msg_type == 'collection_processing' and target['status'] in ['Closed']):
scope_name_list_map.setdefault(scope, [])
scope_name_list_map[scope].append(name)
elif (msg_type == 'file_processing'
and target['status'] in ['Missing']):
# missing files
missing_files_list.append(name)
else:
# got target in bad attributes, do nothing
tmp_log.debug(
'jeditaskid={jeditaskid}, scope={scope}, msg_type={msg_type}, status={status}, did nothing for bad target'
.format(jeditaskid=jeditaskid,
scope=scope,
msg_type=msg_type,
status=target['status']))
pass
# run by each scope
for scope, name_list in scope_name_list_map.items():
# about files or datasets in good status
if msg_type == 'file_processing':
tmp_log.debug(
'jeditaskid={0}, scope={1}, update about files...'.
format(jeditaskid, scope))
res = self.tbIF.updateInputFilesStagedAboutIdds_JEDI(
jeditaskid, scope, name_list)
if res is None:
# got error and rollback in dbproxy
err_str = 'jeditaskid={0}, scope={1}, failed to update files'.format(
jeditaskid, scope)
raise RuntimeError(err_str)
tmp_log.info(
'jeditaskid={0}, scope={1}, updated {2} files'.
format(jeditaskid, scope, res))
elif msg_type == 'collection_processing':
tmp_log.debug(
'jeditaskid={0}, scope={1}, update about datasets...'
.format(jeditaskid, scope))
res = self.tbIF.updateInputDatasetsStagedAboutIdds_JEDI(
jeditaskid, scope, name_list)
if res is None:
# got error and rollback in dbproxy
err_str = 'jeditaskid={0}, scope={1}, failed to update datasets'.format(
jeditaskid, scope)
raise RuntimeError(err_str)
tmp_log.info(
'jeditaskid={0}, scope={1}, updated {2} datasets'.
format(jeditaskid, scope, res))
# check if all ok
if res == len(target_list):
tmp_log.debug(
'jeditaskid={0}, scope={1}, all OK'.format(
jeditaskid, scope))
elif res < len(target_list):
tmp_log.warning(
'jeditaskid={0}, scope={1}, only {2} out of {3} done...'
.format(jeditaskid, scope, res, len(target_list)))
elif res > len(target_list):
tmp_log.warning(
'jeditaskid={0}, scope={1}, strangely, {2} out of {3} done...'
.format(jeditaskid, scope, res, len(target_list)))
else:
tmp_log.warning(
'jeditaskid={0}, scope={1}, something unwanted happened...'
.format(jeditaskid, scope))
# handle missing files
n_missing = len(missing_files_list)
if n_missing > 0:
res = self.tbIF.setMissingFilesAboutIdds_JEDI(
jeditaskid=jeditaskid, filenames=missing_files_list)
if res == n_missing:
tmp_log.debug(
'jeditaskid={0}, marked all {1} files missing'.
format(jeditaskid, n_missing))
elif res < n_missing:
tmp_log.warning(
'jeditaskid={0}, only {1} out of {2} files marked missing...'
.format(jeditaskid, res, n_missing))
elif res > n_missing:
tmp_log.warning(
'jeditaskid={0}, strangely, {1} out of {2} files marked missing...'
.format(jeditaskid, res, n_missing))
else:
tmp_log.warning(
'jeditaskid={0}, res={1}, something unwanted happened about missing files...'
.format(jeditaskid, res))
else:
# do nothing
tmp_log.debug(
'jeditaskid={jeditaskid}, msg_type={msg_type}, relation_type={relation_type}, nothing done'
.format(jeditaskid=jeditaskid,
msg_type=msg_type,
relation_type=relation_type))
except Exception as e:
err_str = 'failed to process the message, skipped. {0} : {1}'.format(
e.__class__.__name__, e)
tmp_log.error(err_str)
raise
# done
tmp_log.info('done')
def _setup_instances(self):
"""
set up attributes and MBListenerProxy/plugin instances accordingly
"""
# logger
tmp_logger = logger_utils.make_logger(base_logger,
token=self.get_pid(),
method_name='_setup_instances')
tmp_logger.debug('start')
# processor thread attribute dict
processor_attr_map = dict()
# inward/outward queues and plugin instances
in_q_set = set()
out_q_set = set()
for proc, pconf in self._processors_dict.items():
# skip if not enabled
if not pconf.get('enable', True):
continue
# queues
in_queue = pconf.get('in_queue')
out_queue = pconf.get('out_queue')
if in_queue:
in_q_set.add(in_queue)
if out_queue:
out_q_set.add(out_queue)
# plugin
plugin_factory = PluginFactory()
plugin = plugin_factory.get_plugin(pconf)
# fill in thread attribute dict
processor_attr_map[proc] = dict()
processor_attr_map[proc]['in_queue'] = in_queue
processor_attr_map[proc]['out_queue'] = out_queue
processor_attr_map[proc]['plugin'] = plugin
# mb_listener_proxy instances
mb_listener_proxy_dict = dict()
for in_queue in in_q_set:
qconf = self._queues_dict[in_queue]
if not qconf.get('enable', True):
continue
sconf = self._mb_servers_dict[qconf['server']]
mb_listener_proxy = get_mb_proxy(name=in_queue,
sconf=sconf,
qconf=qconf,
mode='listener')
mb_listener_proxy_dict[in_queue] = mb_listener_proxy
# mb_sender_proxy instances
mb_sender_proxy_dict = dict()
for out_queue in out_q_set:
qconf = self._queues_dict[out_queue]
if not qconf.get('enable', True):
continue
sconf = self._mb_servers_dict[qconf['server']]
mb_sender_proxy = get_mb_proxy(name=out_queue,
sconf=sconf,
qconf=qconf,
mode='sender')
mb_sender_proxy_dict[out_queue] = mb_sender_proxy
# keep filling in thread attribute dict
for proc in processor_attr_map.keys():
in_queue = processor_attr_map[proc]['in_queue']
if in_queue:
processor_attr_map[proc][
'mb_listener_proxy'] = mb_listener_proxy_dict[in_queue]
out_queue = processor_attr_map[proc]['out_queue']
if out_queue:
processor_attr_map[proc][
'mb_sender_proxy'] = mb_sender_proxy_dict[out_queue]
# set self attributes
self.init_processor_list = list(processor_attr_map.keys())
self.init_mb_listener_proxy_list = list(
mb_listener_proxy_dict.values())
self.init_mb_sender_proxy_list = list(mb_sender_proxy_dict.values())
self.processor_attr_map = dict(processor_attr_map)
# tear down
del in_q_set, out_q_set, mb_listener_proxy_dict, mb_sender_proxy_dict, processor_attr_map
tmp_logger.debug('done')
