def init(host='127.0.0.1', port=5672, usr='******', pas='******', channel=None, destinations=None):
global channel_inst
if not channel:
# TODO should lazy init channel_inst
channel_inst = connect_and_declare(host=host, port=port, usr=usr, pas=pas, destinations=destinations)
else:
channel_inst = channel
bashtasks = BashTasks()
bashtasks.post_task = post_task
bashtasks.execute_task = execute_task
return bashtasks
def subscribe(self, callback, queue=TASK_RESPONSES_POOL):
def pika_event_to_bashtasks_msg(ch, method, properties, body):
msg = MessageAmqpPika(ch, method, properties, body)
callback(msg)
global channel_inst
if not channel_inst:
channel_inst = connect_and_declare(host=self.host, port=self.port, usr=self.usr, pas=self.pas)
channel_inst.basic_qos(prefetch_count=1) # consume msgs one at a time
channel_inst.basic_consume(pika_event_to_bashtasks_msg, queue=queue, no_ack=False)
channel_inst.start_consuming()
def start_executor(host='127.0.0.1', port=5672, usr='******', pas='******', queue=DEFAULT_DESTINATION,
tasks_nr=1, max_retries=0, verbose=False, custom_callback=None,
ok_returncodes=(0, )):
curr_th_name = threading.current_thread().name
logger = get_logger(name=curr_module_name())
logger.info(">> Starting executor %s connecting to rabbitmq: %s:%s@%s for executing %d tasks.",
curr_th_name, usr, pas, host, tasks_nr)
ch = connect_and_declare(host=host, port=port, usr=usr, pas=pas, destinations=queue)
ch.basic_qos(prefetch_count=1) # consume msgs one at a time
channels.append(ch)
def is_ok_returncode(code):
return code in ok_returncodes
def create_response_for(msg):
resp = {}
resp.update(msg)
resp['executor_name'] = get_executor_name()
resp['retries'] = msg.get('retries', 0)
resp['max_retries'] = msg.get('max_retries', 0)
return resp
def should_retry(response_msg):
is_error = response_msg['returncode'] != 0
returncode = response_msg['returncode']
is_retriable = returncode not in response_msg['non_retriable']
current_retries = response_msg.get('retries', 0)
msg_max_retries = response_msg.get('max_retries', max_retries)
retries_pending = current_retries < msg_max_retries
return is_error and is_retriable and retries_pending
def send_response(response_msg):
if should_retry(response_msg):
logger.debug('---- retrying msg correlation_id: %d current_retries: %d of %d',
response_msg['correlation_id'], response_msg['retries'],
response_msg['max_retries'])
tgt_exch = TASK_REQUESTS_POOL
response_msg['retries'] += 1
else:
tgt_exch = TASK_RESPONSES_POOL
response_str = json.dumps(response_msg)
props = pika.BasicProperties(
delivery_mode=2, # make message persistent
)
ch.basic_publish(exchange=tgt_exch, routing_key='', body=response_str, properties=props)
def tasks_nr_generator(tasks_nr):
tasks_nr_gen = tasks_nr
while True:
tasks_nr_gen = tasks_nr_gen - 1 if tasks_nr_gen > 0 else tasks_nr_gen
yield tasks_nr_gen
def trace_msg(msg, context_info=''):
logger.info(u'------------- MSG: %s', context_info)
for key, value in msg.items():
try:
logger.info(u'\t%s:-> %s', key, value)
except Exception as e:
logger.error('error in trace message::::', exc_info=True)
sys.exit(777)
logger.info('---------------------------------------------')
def execute_command(command):
p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
o, e = p.communicate()
return p.returncode, o.decode('utf-8'), e.decode('utf-8')
def handle_message(ch, method, properties, body):
msg = json.loads(body.decode('utf-8'))
logger.debug(">>>> msg received: %s from queue %s : correlation_id %d command: %s",
curr_th_name, queue, msg['correlation_id'], msg['command'][:50])
try:
response_msg = create_response_for(msg)
response_msg['pre_command_ts'] = currtimemillis()
returncode, out, err = command_callback(msg['command'])
response_msg['post_command_ts'] = currtimemillis()
response_msg['returncode'] = returncode
response_msg['stdout'] = out
response_msg['stderr'] = err
except Exception as exc:
logger.error('**** Command execution error. Exception for : correlation_id %d ',
response_msg['correlation_id'], exc_info=True)
response_msg['post_command_ts'] = currtimemillis()
response_msg['returncode'] = -3791
response_msg['stdout'] = 'Exception trying to execute command.'
response_msg['stderr'] = repr(exc)
finally:
if verbose or not is_ok_returncode(response_msg['returncode']):
trace_msg(msg, context_info='Request msg: '+str(msg['correlation_id']))
resp_header = 'Response msg: '+str(response_msg['correlation_id']) + \
' returncode: '+str(response_msg['returncode'])
trace_msg(response_msg, context_info=resp_header)
send_response(response_msg)
ch.basic_ack(method.delivery_tag)
tasks_nr_new_elem = next(tasks_nr_gen)
logger.debug("<<<< executed by: executor %s correlation_id: %d pending: %d",
curr_th_name, response_msg['correlation_id'], tasks_nr_new_elem)
if tasks_nr_new_elem == 0:
logger.info('==== no more tasks to execute. Exiting.')
stop_and_exit()
tasks_nr_gen = tasks_nr_generator(tasks_nr)
command_callback = custom_callback if custom_callback else execute_command
ch.basic_consume(handle_message, queue=queue, no_ack=False)
logger.info("<< Ready: executor %s connected to rabbitmq: %s:%s@%s",
curr_th_name, usr, pas, host)
ch.start_consuming()
