def process_each_command(self, command_id = 0, _dict_list_params = [], step=''):
'''process command contains params'''
try:
self._request.url = self.requestURL.MEGA_URL_COMMAND_DETAIL % (command_id)
self.data_command = self._request.get().json()
command = None
#-------------------- process delay command ---------------------------------------------------
try:
if int(self.data_command['type']) == 5:
sleep(int(self.data_command['delay']))
stringhelpers.info("[DELAY COMMAND %s %s]" % (command_id, self.data_command['delay']))
return None
except Exception as ex_error:
errror = ex_error
#----------------------------------------------------------------------------------------------
################### process args for command ##############################################
#self.thread_lock.acquire()
if len(_dict_list_params) > 0:
for item in _dict_list_params: #array contain dict param argument
for k, v in item.items():
if command is None:
command = self.data_command['command']
command = command.replace('@{%s}' % (k), v)
else:
command = command.replace('@{%s}' % (k), v)
else:
command = self.data_command['command']
#self.thread_lock.release()
###########################################################################################
if command is not None:
commands = [command]
#stringhelpers.info_green(command)
self.fang.execute_template_action_command(commands, blanks=2, error_reporting=True, timeout=-1, terminal=False)
#result_fang = self.fang.get_output()
result_fang = self.fang.get_action_output(self.log_output_file_name)
# processing parsing command follow output ###########################################
action_command_log = self.parsing(command_id ,result_fang, commands[0], step)
action_command_log = None
return action_command_log
######################################################################################
else:
return None
except Exception as e:
stringhelpers.err("[DISCOVERY] MEGA ACTION PROCESS EACH COMMAND ERROR %s | THREAD %s" % (e, self.name))
return None
except ConnectionError as errConn:
stringhelpers.err("[DISCOVERY] MEGA ACTION CONNECT API URL ERROR %s | THREAD %s" % (errConn, self.name))
return None
def run(self):
try:
key_list_command = self.vendor_os
key_rollback = 'rollback'
self.action_log['result']['outputs'][key_list_command] = {}
self.action_log['result']['outputs'][key_list_command][
'config'] = []
self.action_log['result']['outputs'][key_list_command][
'rollback'] = []
action_type = self.data_action['category_1']
_list_action_commands = self.data_action['commands'][
key_list_command]['config'] # list action_command config
_list_action_rollback = self.data_action['commands'][
key_list_command]['rollback'] # list action_command rollback
# --------------- list dict action command -----------------------------------------------------------------
_dict_list_command = dict()
_dict_list_params = self.params_action
_array_step = []
if len(_list_action_commands) > 0:
count = 0
for _command in _list_action_commands:
count = count + 1
_dict_list_command[str(count)] = _command
_array_step.append(str(count)) # save step command
else:
pass
# ----------------------------------------------------------------------------------------------------------
# --------------- list dict action command rollback---------------------------------------------------------
_dict_list_command_rollback = dict()
_dict_list_params_rollback = self.param_rollback_action
_array_step_rollback = []
if len(_list_action_rollback) > 0:
count = 0
for _command in _list_action_rollback:
count = count + 1
_dict_list_command_rollback[str(count)] = _command
_array_step_rollback.append(
str(count)) # save step command rollback
else:
pass
# ------------------------------------------------------------------------------------------------------
'''#############################process command by dependency############################################'''
if len(_array_step) > 0 and self.is_rollback == False:
compare_final_output = []
previous_final_output = []
for step in _array_step:
_command_running = _dict_list_command[step]
# if _command_running['dependency'] == '0':
command_id = _command_running.get('command_id', 0)
if command_id > 0: # command_id > 0
dependency = int(_command_running['dependency'])
if dependency > 0: # run need compare
dependStep = dependency
if (int(_command_running['condition']) == int(
previous_final_output[dependStep - 1])):
output_info = self.process_each_command(
command_id, _dict_list_params)
if output_info is not None:
previous_final_output.append(output_info[
str(command_id)]['final_output'])
self.action_log['result']['outputs'][
key_list_command]['config'].append(
output_info)
stringhelpers.info(
"\nAction: [%s]-- config step [%s]: filnal-output: %s"
% (self.action_id, step,
str(output_info[str(command_id)]
['final_output'])))
else:
#previous_final_output.append(False)
previous_final_output.append(True)
else:
stringhelpers.err(
"MEGA ACTIONS STEP: %s NOT AVAIABLE WITH FINAL_OUTPUT OF STEP %d| THREAD %s"
% (step, dependStep, self.name))
previous_final_output.append(False)
continue
else: # dependency == 0
output_info = self.process_each_command(
command_id, _dict_list_params)
if output_info is not None:
previous_final_output.append(output_info[str(
command_id)]['final_output'])
self.action_log['result']['outputs'][
key_list_command]['config'].append(
output_info)
stringhelpers.info(
"\nAction: [%s]-- config step [%s]: filnal-output: %s"
% (self.action_id, step,
str(output_info[str(command_id)]
['final_output'])))
if int(step) > 1:
if int(output_info[str(command_id)]
['final_output']) == int(
_command_running.get(
'condition', 0)):
compare_final_output.append(True)
else:
#self.action_log['final_output'] = False
self.action_log['final_output'] = True
compare_final_output = []
break
else:
#previous_final_output.append(False)
previous_final_output.append(True)
if int(step) > 1:
#self.action_log['final_output'] = False
self.action_log['final_output'] = True
compare_final_output = []
break
else: # last command in actions check point
try:
dependency = int(_command_running['dependency'])
if dependency > 0 and int(step) > 0:
continue
if (int(_command_running['condition']) == int(
previous_final_output[dependency - 1])):
compare_final_output.append(True)
else:
#compare_final_output.append(False)
compare_final_output.append(True)
except:
#compare_final_output.append(False)
compare_final_output.append(True)
# -------------- compare final_output for action ----------------------------------------------------
try:
if len(compare_final_output) > 0:
first_value = None
count = 0
for x in compare_final_output:
if count == 0:
first_value = x
else:
first_value = func_compare('=', first_value, x)
count = count + 1
self.action_log['final_output'] = first_value
self.final_result = first_value #final result run action
self.dict_state_result[
'final_result_action'] = self.final_result
else:
if len(previous_final_output) > 0:
for x in previous_final_output:
self.dict_state_result[
'final_result_action'] = x
else:
#self.dict_state_result['final_result_action'] = False
self.dict_state_result[
'final_result_action'] = True
except Exception as ex:
stringhelpers.err(
"MEGA ACTIONS THREAD ERROR COMAPRE ACTION FINAL-OUTPUT: %s | THREAD %s"
% (ex, self.name))
# ---------------------------------------------------------------------------------------------------
'''######################################################################################################'''
except Exception as e:
stringhelpers.err("MEGA ACTIONS THREAD ERROR %s | THREAD %s" %
(e, self.name))
except ConnectionError as errConn:
stringhelpers.err(
"MEGA ACTIONS CONNECT API URL ERROR %s | THREAD %s" %
(self._request.url, self.name))
def run(self):
try:
if self.subtemplate is not None:
threading_array = []
stringhelpers.info("\n[INFO]-RUN SUBTEMPLATE: %s" %
(self.name))
filnal_result = []
#------------------ chay ko song song ------------------------------------------------------------------
if self.mode == 1: #mode = 1, not parallel
for device in self.subtemplate['devices']:
device_id = device['device']['device_info'][
'device_id']
# -------------- check has run next subtemplate belong previous result subtemplate
if len(self.result_templates) > 0:
before_result = self.result_templates[
len(self.result_templates) - 1]
try:
final_sub_template = before_result['state'][
str(device_id)]['final_sub_template']
if final_sub_template == False:
print(
"SUB TEMPLATE: %s FOR DEVICE: %s DON'T CONTINIOUS RUN\n\n"
% (self.name, device_id))
break
except:
print(
"SUB TEMPLATE: %s FOR DEVICE: %s DON'T CONTINIOUS RUN\n\n"
% (self.name, device_id))
break
# ------------------------------------------------------------------------------------------
if len(filnal_result) == 0:
self.excecute(device)
else:
if filnal_result[len(filnal_result) - 1] == True:
self.excecute(device)
try:
filnal_result.append(self.dict_state_result[str(
device_id)]["final_sub_template"])
except:
pass
#---------------------------------------------------------------------------------------------------
else: # mode = 2, run parallel
#------------------- chay song song ----------------------------------------------------------------
for device in self.subtemplate['devices']:
device_id = device['device']['device_info'][
'device_id']
# -------------- check has run next subtemplate belong previous result subtemplate
if len(self.result_templates) > 0:
before_result = self.result_templates[
len(self.result_templates) - 1]
try:
final_sub_template = before_result['state'][
str(device_id)]['final_sub_template']
if final_sub_template == False:
print(
"SUB TEMPLATE: %s FOR DEVICE: %s DON'T CONTINIOUS RUN\n\n"
% (self.name, device_id))
break
except:
print(
"SUB TEMPLATE: %s FOR DEVICE: %s DON'T CONTINIOUS RUN\n\n"
% (self.name, device_id))
break
# ------------------------------------------------------------------------------------------
if len(filnal_result) == 0:
_thread = threading.Thread(target=self.excecute,
args=(device, ))
_thread.start()
threading_array.append(_thread)
else:
if filnal_result[len(filnal_result) - 1] == True:
_thread = threading.Thread(
target=self.excecute, args=(device, ))
_thread.start()
threading_array.append(_thread)
for x in threading_array:
x.join()
for device in self.subtemplate['devices']:
device_id = device['device']['device_info'][
'device_id']
try:
filnal_result.append(self.dict_state_result[str(
device_id)]["final_sub_template"])
except:
pass
#-------------------------------------------------------------------------------------------------------
except Exception as exError:
stringhelpers.err("[ERROR] RUN SUBTEMPLATE-[%s]: %s" %
(self.name, exError))
def run(self):
_request = RequestHelpers()
_request.url = self.requestURL.URL_GET_DEVICE_DETAIL % (
self.data_command["test_device"])
try:
device = _request.get().json()
try:
if device['status_code'] == 500: #device not exist
stringhelpers.err(
"DEVICE ID %s NOT EXIST | THREAD %s" %
(self.data_command["test_device"], self.name))
except: #process fang test device by command
host = device['ip_mgmt']
port = int(device['port_mgmt'])
username = device['username']
password = device['password']
device_type = device['os']
method = device['method'] #ssh, telnet
parameters = {
'device_type': device_type,
'host': host,
'protocol': method,
'username': username,
'password': password,
'port': port
}
'''process command contains params'''
command = None
test_args = self.data_command['test_args']
if len(test_args) > 0:
command = self.data_command['command']
for x in self.data_command['test_args']:
command = command.replace('@{%s}' % (x['name']),
x['value'])
else:
command = self.data_command['command']
'''####################################'''
commands = [command]
fac = FactoryConnector(**parameters)
print("FANG DEVICE: host=%s, port=%s, devicetype=%s \n\n" %
(host, device['port_mgmt'], device_type))
fang = fac.execute(commands)
result_fang = fang.get_output()
fang.remove_file_log(fac.file_log_command) #remove file.log
_request.url = self.requestURL.MEGA_URL_COMMANDLOG_GETBY_COMMANDID % (
self.data_command['command_id'])
command_log = _request.get().json()
if len(command_log) > 0:
cmd_log = {
'command_id': self.data_command['command_id'],
'device_id': self.data_command["test_device"],
'console_log': result_fang,
'result': dict(),
'command': command
}
# processing parsing command follow output ###########################################
command_type = self.data_command['type']
cmd_log = self.parsing(command_type, cmd_log)
######################################################################################
try:
_request.url = self.requestURL.MEGA_URL_COMMANDLOG_UPDATE % (
command_log[0]['log_id'])
_request.params = cmd_log
_request.put()
stringhelpers.info(
"MEGA THREAD INFO: %s | THREAD %s" %
("UPDATE COMMAND LOG SUCCESS", self.name))
#---------------update mega_status to command------------------------------------------------
_request.url = self.requestURL.MEGA_URL_COMMAND_UPDATE % (
self.data_command['command_id'])
_request.params = {'mega_status': 'tested'}
_request.put()
key_command = 'command_%d' % (
self.data_command['command_id'])
del self.dict_command[key_command]
#--------------------------------------------------------------------------------------------
except ConnectionError as _conErr:
stringhelpers.info(
"MEGA THREAD ERROR: %s | THREAD %s" %
(_conErr, self.name))
else:
cmd_log = {
'command_id': self.data_command['command_id'],
'device_id': self.data_command["test_device"],
'console_log': result_fang,
'result': dict(),
'command': command
}
# processing parsing command follow output ###########################################
command_type = self.data_command['type']
cmd_log = self.parsing(command_type, cmd_log)
######################################################################################
try:
_request.url = self.requestURL.MEGA_URL_COMMANDLOG_CREATE
_request.params = cmd_log
_request.post()
stringhelpers.info(
"MEGA THREAD INFO: %s | THREAD %s" %
("INSERT COMMAND LOG SUCCESS", self.name))
# ---------------update mega_status to command------------------------------------------------
_request.url = self.requestURL.MEGA_URL_COMMAND_UPDATE % (
self.data_command['command_id'])
_request.params = {'mega_status': 'tested'}
_request.put()
key_command = 'command_%d' % (
self.data_command['command_id'])
del self.dict_command[key_command]
# --------------------------------------------------------------------------------------------
except ConnectionError as _conErr:
stringhelpers.info(
"MEGA THREAD ERROR: %s | THREAD %s" %
(_conErr, self.name))
except Exception as e:
stringhelpers.err("MEGA THREAD ERROR %s | THREAD %s" %
(e, self.name))
except ConnectionError as errConn:
stringhelpers.err("MEGA CONNECT API URL ERROR %s | THREAD %s" %
(_request.url, self.name))
def run(self):
self.socketio_iron.on('oncommand', self.on_command_response)
try:
dict_version_container = dict()
#---------------------------- waiting time for time start ------------------------------------------------
while self.is_waiting:
run_time = self.mop_data['run_datetime'].split("-")[1].strip()
time_start = datetime.strptime(run_time, '%H:%M').time()
time_current = datetime.strptime("%d:%d"%(datetime.now().hour, datetime.now().minute), "%H:%M").time()
if time_current >= time_start:
self.is_waiting = False
else:
stringhelpers.info('\n[IRON][DISCOVERY][START RUNNING TIME][%s]' % (str(self.mop_data['run_datetime'])),
socket_namespace = self.socketio_iron, on_command_text = 'overall_terminal')
#-----------------------------------------------------------------------------------------------------------
while not self.is_stop:
# -------------------- run device from mop -------------------------------------------------------------
key_mop = 'main_schedule_%d' % (int(self.mop_data['mop_id']))
try:
table_id = int(self.mop_data['save_to_table'])
tableImpl = TABLEImpl()
table_name = tableImpl.get(table_id)['table_name']
key_merge = self.mop_data.get('key_merge', None)
count_number = 0
arr_manager_discovery = []
len_submops = len(self.sub_mops) - 1
for sub_mop_item in self.sub_mops:
sub_no = int(sub_mop_item.get('subNo', 0))
irondiscovery = Iron_Mop_Discovery("IRONMAN-Thread-Template-%s" % (str(self.mop_id)),
sub_mop_item, {}, self.mop_id, table_name, self.output_mapping[str(sub_no)],
key_merge, sub_no, dict_version_container,
len_submops, self.socketio, self.socketio_iron)
# insert to queue discovery
self.queue.put(irondiscovery)
arr_manager_discovery.append(irondiscovery)
count_number = count_number + 1
stringhelpers.info('\n[ENQUEUE] - > [IRON][DISCOVERY][MOP_ID: %s][SUB_MOP_NAME: %s]' % (str(self.mop_id), sub_mop_item['name']),
socket_namespace=self.socketio_iron, on_command_text='overall_terminal')
if self.mechanism.upper() == 'MANUAL':
self.is_stop = True
del self.dict_schedule[key_mop]
else:
weekday = datetime.now().strftime('%A')
if weekday not in list(self.mop_data['day_in_week']):
del self.dict_schedule[key_mop]
self.is_stop = True
else:
if not self.is_stop:
while True:
count = 0
for discovery_item in arr_manager_discovery:
if discovery_item.done == True:
count = count + 1
time.sleep(0.2)
if count == len(arr_manager_discovery):
#----------------------------- get detail sub mop --------------------------------------------------------------------
self._request.url = self.requestURL.IRONMAN_URL_GET_MOP_DETAIL % (str(self.mop_id))
_mop_details = self._request.get().json()
sub_mops = _mop_details.get('sub_mops', None)
if sub_mops is not None:
self.sub_mops = sub_mops
stringhelpers.info('\n[IRON][DISCOVERY][GET_MOP_DETAIL_FOR_LOOP][MOP_ID:%s][%s]' % (self.mop_id, self.name),
socket_namespace=self.socketio_iron, on_command_text='overall_terminal')
#--------------------------------------------------------------------------------------------------------------
stringhelpers.info('\n[IRON][DISCOVERY][WAITING][%d minutes][%s]' % (int(self.mop_data['return_after']), self.name),
socket_namespace=self.socketio_iron, on_command_text='overall_terminal')
#time_remaining = int(self.mop_data['return_after']) * 60
time_remaining = 1 * 60
for remaining in range(time_remaining, 0, -1):
str_time = "Còn <strong>{:2d}s</strong> để chạy lần kế tiếp".format(remaining)
dict_time_remaining = {
'mop_id': self.mop_id,
'text': str_time,
'time_count': "{:2d}".format(remaining).strip()
}
self.socketio_iron.emit('running_time_remain_mop', dict_time_remaining)
time.sleep(1)
if self.is_stop: break #STOP MOP
break
time.sleep(1)
except Exception as _exError:
stringhelpers.err("[ERROR] %s" % (_exError))
except Exception as error:
stringhelpers.err("[ERROR] %s %s" % (self.name, error))
def run(self):
try:
#---------------------------- waiting time for time start ------------------------------------------------
while self.is_waiting:
run_time = self.mop_data['run_datetime'].split("-")[1].strip()
time_start = datetime.strptime(run_time, '%H:%M').time()
time_current = datetime.strptime(
"%d:%d" % (datetime.now().hour, datetime.now().minute),
"%H:%M").time()
if time_current >= time_start:
self.is_waiting = False
#---------------------------------------------------------------------------------------------------------
while not self.is_stop:
# -------------------- run device from mop -------------------------------------------
array_device_mop = self.mop_data['devices']
run_devices = {}
for item in array_device_mop:
self._request.url = self.requestURL.URL_GET_DEVICE_DETAIL % (
int(item)) # get device detail
device = self._request.get().json()
run_devices[str(item)] = device['role']
self.template_data['run_devices'] = run_devices
key_mop = 'main_schedule_%d' % (self.mop_data['mop_id'])
try:
table_id = int(self.mop_data['save_to_table'])
tableImpl = TABLEImpl()
table_name = tableImpl.get(table_id)['table_name']
irondiscovery = IronDiscovery(
"IRONMAN-Thread-Template-%s" %
(self.template_data['template_id']),
self.template_data, {}, self.mop_data['mop_id'],
table_name, self.output_mapping)
# insert to queue discovery
self.queue.put(irondiscovery)
# irondiscovery.start()
# irondiscovery.join()
if self.mechanism.upper() == 'MANUAL':
self.is_stop = True
del self.dict_schedule[key_mop]
else:
weekday = datetime.now().strftime('%A')
if weekday not in list(self.mop_data['day_in_week']):
del self.dict_schedule[key_mop]
self.is_stop = True
else:
while True:
if irondiscovery.done == True:
stringhelpers.info(
'[IRON][DISCOVERY][WAITING][%d minutes][%s]'
% (int(self.mop_data['return_after']),
self.name))
time.sleep(
int(self.mop_data['return_after']) *
60)
break
except Exception as _exError:
stringhelpers.err("[ERROR] %s" % (_exError))
except Exception as error:
stringhelpers.err("[ERROR] %s %s" % (self.name, error))
def on_iron_command(*args):
stringhelpers.info(str(args))
def callback_login(*args):
stringhelpers.info(str(args))
def run(self):
self._request.url = self.requestURL.URL_GET_DEVICE_DETAIL % (
self.data_action["test_device"])
try:
device = self._request.get().json()
try:
if device['status_code'] == 500: #device not exist
stringhelpers.err(
"DEVICE ID %s NOT EXIST | THREAD %s" %
(self.data_action["test_device"], self.name))
except: #process fang test device by command
host = device['ip_mgmt']
port = int(device['port_mgmt'])
username = device['username']
password = device['password']
device_type = device['os']
method = device['method'] #ssh, telnet
parameters = {
'device_type': device_type,
'host': host,
'protocol': method.lower(),
'username': username,
'password': password,
'port': port,
'timeout': 300
}
key_list_command = "%s|%s" % (device['vendor'], device['os'])
key_rollback = 'rollback'
self.log_output_file_name = "action_%s.log" % (
self.data_action['action_id'])
self.action_log['result']['outputs'][key_list_command] = {}
self.action_log['result']['outputs'][key_list_command][
'config'] = []
self.action_log['result']['outputs'][key_list_command][
'rollback'] = []
_list_action_commands = self.data_action['commands'][
key_list_command]['config'] #list action_command config
_list_action_rollback = self.data_action['commands'][
key_list_command][
'rollback'] #list action_command rollback
#--------------- list dict action command --------------------------------------------------------------
_dict_list_command = dict()
_dict_list_params = self.data_action['test_args']
_array_step = []
if len(_list_action_commands) > 0:
count = 0
for _command in _list_action_commands:
count = count + 1
_dict_list_command[str(count)] = _command
_array_step.append(str(count)) # save step command
else:
pass
#-------------------------------------------------------------------------------------------------------
# --------------- list dict action command rollback---------------------------------------------------------------------
_dict_list_command_rollback = dict()
_dict_list_params_rollback = self.data_action['rollback_args']
_array_step_rollback = []
if len(_list_action_rollback) > 0:
count = 0
for _command in _list_action_rollback:
count = count + 1
_dict_list_command_rollback[str(count)] = _command
_array_step_rollback.append(
str(count)) # save step command rollback
else:
pass
# -------------------------------------------------------------------------------------------------------
fac = FactoryConnector(**parameters)
'''#############################process command by dependency########################################'''
if len(_array_step) > 0:
print(
"MEGA ACTION FANG DEVICE: host=%s, port=%s, devicetype=%s \n\n"
% (parameters['host'], parameters['port'],
parameters['device_type']))
self.fang = fac.execute_keep_alive(
loginfo=self.log_output_file_name)
compare_final_output = []
previous_final_output = []
for step in _array_step:
_command_running = _dict_list_command[step]
#if _command_running['dependency'] == '0':
command_id = _command_running.get('command_id', 0)
if command_id > 0: #command_id > 0
dependency = int(_command_running['dependency'])
if dependency > 0: # run need compare
dependStep = dependency
if (int(_command_running['condition']) == int(
previous_final_output[dependStep -
1])):
output_info = self.process_each_command(
command_id, _dict_list_params)
if output_info is not None:
previous_final_output.append(
output_info[str(
command_id)]['final_output'])
self.action_log['result']['outputs'][
key_list_command]['config'].append(
output_info)
stringhelpers.info(
"\nstep %s: %s" %
(step, str(output_info)))
else:
previous_final_output.append(False)
else:
stringhelpers.err(
"MEGA ACTIONS STEP: %s NOT AVAIABLE WITH FINAL_OUTPUT OF STEP %d| THREAD %s"
% (step, dependStep, self.name))
previous_final_output.append(False)
continue
else: # dependency == 0
output_info = self.process_each_command(
command_id, _dict_list_params)
if output_info is not None:
previous_final_output.append(output_info[
str(command_id)]['final_output'])
self.action_log['result']['outputs'][
key_list_command]['config'].append(
output_info)
stringhelpers.info(
"\nstep %s: %s" %
(step, str(output_info)))
if int(step) > 1:
if int(output_info[str(command_id)]
['final_output']) == int(
_command_running.get(
'condition', 0)):
compare_final_output.append(True)
else:
self.action_log[
'final_output'] = False
compare_final_output = []
break
else:
previous_final_output.append(False)
if int(step) > 1:
self.action_log['final_output'] = False
compare_final_output = []
break
else: #last command in actions check point
try:
dependency = int(
_command_running['dependency'])
if dependency > 0 and int(step) > 0:
continue
if (int(_command_running['condition']) == int(
previous_final_output[dependency -
1])):
compare_final_output.append(True)
else:
compare_final_output.append(False)
except:
pass
#-------------- compare final_output for action ----------------------------------------------------
try:
if len(compare_final_output) > 0:
first_value = None
count = 0
for x in compare_final_output:
if count == 0:
first_value = x
else:
first_value = func_compare(
'=', first_value, x)
count = count + 1
self.action_log['final_output'] = first_value
else:
if len(previous_final_output) > 0:
for x in previous_final_output:
self.dict_state_result[
'final_result_action'] = x
else:
self.dict_state_result[
'final_result_action'] = False
except Exception as ex:
stringhelpers.err(
"MEGA ACTIONS THREAD ERROR COMAPRE ACTION FINAL-OUTPUT: %s | THREAD %s"
% (ex, self.name))
#---------------------------------------------------------------------------------------------------
'''##################################################################################################'''
'''#############################process command by rollback dependency########################################'''
if len(_array_step_rollback) > 0:
print(
"MEGA ACTION ROLLBACK FANG DEVICE: host=%s, port=%s, devicetype=%s \n\n"
% (parameters['host'], parameters['port'],
parameters['device_type']))
#self.fang = fac.execute_keep_alive(loginfo=self.log_output_file_name)
compare_final_output = []
previous_final_output = []
for step in _array_step_rollback:
_command_running = _dict_list_command_rollback[step]
# if _command_running['dependency'] == '0':
command_id = _command_running.get('command_id', 0)
if command_id > 0: # command_id > 0
dependency = int(_command_running['dependency'])
if dependency > 0: # run need compare
dependStep = dependency
if (int(_command_running['condition']) == int(
previous_final_output[dependStep -
1])):
output_info = self.process_each_command(
command_id, _dict_list_params_rollback)
if output_info is not None:
previous_final_output.append(
output_info[str(
command_id)]['final_output'])
self.action_log['result']['outputs'][
key_list_command][
'rollback'].append(output_info)
stringhelpers.info(
"\nstep %s: %s" %
(step, str(output_info)))
else:
previous_final_output.append(False)
else:
stringhelpers.err(
"MEGA ACTIONS ROLLBACK STEP: %s NOT AVAIABLE WITH FINAL_OUTPUT OF STEP %d| THREAD %s"
% (step, dependStep, self.name))
previous_final_output.append(False)
continue
else: # dependency == 0
output_info = self.process_each_command(
command_id, _dict_list_params_rollback)
if output_info is not None:
previous_final_output.append(output_info[
str(command_id)]['final_output'])
self.action_log['result']['outputs'][
key_list_command]['rollback'].append(
output_info)
stringhelpers.info(
"\nstep %s: %s" %
(step, str(output_info)))
if int(step) > 1:
if int(output_info[str(command_id)]
['final_output']) == int(
_command_running.get(
'condition', 0)):
compare_final_output.append(True)
else:
self.action_log[
'final_output'] = False
compare_final_output = []
break
else:
previous_final_output.append(False)
if int(step) > 1:
self.action_log['final_output'] = False
compare_final_output = []
break
else: # last command in actions check point
try: #catch error not command rollback
dependency = int(
_command_running['dependency'])
if dependency > 0 and int(step) > 0:
continue
if (int(_command_running['condition']) == int(
previous_final_output[dependency -
1])):
compare_final_output.append(True)
else:
compare_final_output.append(False)
except:
print("DO NOT COMMAND ROLLBACK\n")
stringhelpers.err(
"MEGA ACTIONS THREAD ROLLBACK FINISHED: | THREAD %s" %
(self.name))
# -------------- compare final_output for action ----------------------------------------------------
try:
if len(compare_final_output) > 0:
first_value = None
count = 0
for x in compare_final_output:
if count == 0:
first_value = x
else:
first_value = func_compare(
'=', first_value, x)
count = count + 1
self.action_log['final_output'] = first_value
except Exception as ex:
stringhelpers.err(
"MEGA ACTIONS THREAD ERROR ROLLBACK COMAPRE ACTION FINAL-OUTPUT: %s | THREAD %s"
% (ex, self.name))
# ---------------------------------------------------------------------------------------------------
'''##################################################################################################'''
# ------------------------------------ process save log action --------------------------------------
self._request.url = self.requestURL.MEGA_URL_ACTIONLOG_GETBY_ACTIONID % (
self.data_action['action_id'])
_request_action_log = self._request.get().json()
if len(_request_action_log) > 0: # update action log
self.action_log['action_id'] = self.data_action[
'action_id']
self.action_log['device_id'] = self.data_action[
"test_device"]
try:
self._request.url = self.requestURL.MEGA_URL_ACTIONLOG_UPDATE % (
_request_action_log[0]['log_id'])
self._request.params = self.action_log
self._request.put()
stringhelpers.info(
"MEGA ACTIONS THREAD INFO: %s | THREAD %s" %
("UPDATE ACTIONS LOG SUCCESS", self.name))
# ---------------update mega_status to action------------------------------------------------
self._request.url = self.requestURL.MEGA_URL_ACTION_UPDATE % (
self.data_action['action_id'])
self._request.params = {'mega_status': 'tested'}
self._request.put()
key_action = 'action_%d' % (
self.data_action['action_id'])
del self.dict_action[key_action]
# --------------------------------------------------------------------------------------------
except ConnectionError as _conErr:
stringhelpers.info(
"MEGA ACTIONS THREAD ERROR: %s | THREAD %s" %
(_conErr, self.name))
else: # insert action log
self.action_log['action_id'] = self.data_action[
'action_id']
self.action_log['device_id'] = self.data_action[
"test_device"]
try:
self._request.url = self.requestURL.MEGA_URL_ACTIONLOG_CREATE
self._request.params = self.action_log
self._request.post()
stringhelpers.info(
"MEGA ACTIONS THREAD INFO: %s | THREAD %s" %
("INSERT ACTIONS LOG SUCCESS", self.name))
# ---------------update mega_status to action------------------------------------------------
self._request.url = self.requestURL.MEGA_URL_ACTION_UPDATE % (
self.data_action['action_id'])
self._request.params = {'mega_status': 'tested'}
self._request.put()
key_action = 'action_%d' % (
self.data_action['action_id'])
del self.dict_action[key_action]
# --------------------------------------------------------------------------------------------
except ConnectionError as _conErr:
stringhelpers.err(
"MEGA ACTIONS THREAD ERROR: %s | THREAD %s" %
(_conErr, self.name))
# ---------------------------------------------------------------------------------------------------
self.fang.remove_file_log(self.log_output_file_name)
# stringhelpers.warn(str(self.action_log))
self.fang.terminal() # finished fang command
except Exception as e:
stringhelpers.err("MEGA ACTIONS THREAD ERROR %s | THREAD %s" %
(e, self.name))
except ConnectionError as errConn:
stringhelpers.err(
"MEGA ACTIONS CONNECT API URL ERROR %s | THREAD %s" %
(self._request.url, self.name))
def run(self):
_request = RequestHelpers()
dict_schedule = dict()
dict_schedule_queue = dict()
list_time = list()
queue_discovery = queue.Queue()
# run queue listining discovery ------------------------------------------------------------------------------
_ironQueue = IronQueue(queue_discovery, self.socketio,
self.socketio_iron)
_ironQueue.start()
# ------------------------------------------------------------------------------------------------------------
while not self.is_stop:
try:
self.counter = self.counter + 1
arr_schedule_manage = []
# -------------- IRONMAN RUN SCHEDULE ----------------------------------------------------------------
#stringhelpers.print_bold("IRONMAN SCHEDULE RUN NUMBER: " + str(self.counter), "\n")
# get current day name
#weekday = datetime.now().strftime('%A')
#_request.url = self.requestURL.IRONMAN_URL_GET_SCHEDULE % (weekday)
_request.url = self.requestURL.IRONMAN_URL_GET_MOP_RUN_IRON
_list_schedules = _request.get().json()
if len(_list_schedules) > 0:
for x in _list_schedules:
key_mop = 'main_schedule_%d' % (int(x['mop_id']))
mop_id = int(x['mop_id'])
mechanism = x['run_type']
run_time = x['run_datetime'].split("-")[1].strip()
dict_schedule_queue[str(x['mop_id'])] = run_time
if dict_schedule.get(key_mop, None) is not None:
pass
else:
stringhelpers.info(
"IRONMAN RUNNING MOP: " + str(x['mop_id']),
"\n")
list_time.append(run_time)
_sub_mops = x.get('sub_mops', None)
dict_schedule[key_mop] = key_mop
schedule = Schedule(
"SCHEDULE-%d" % (mop_id), x, _sub_mops,
dict_schedule, False, mechanism, mop_id,
queue_discovery, x['output_mapping'],
self.socketio, self.socketio_iron)
#arr_schedule_manage.append(schedule)
#-----------------------process call api sockbot mop---------------------------------------
params_mops = dict()
params_mops['name'] = x['name']
params_mops['app_secret_id'] = os.environ.get(
'SOCKBOT_APPCLIENT_SECRET')
params_mops['mop_id'] = x['mop_id']
params_mops['status'] = 'RUNNING'
params_mops['belong'] = 'IRONMAN'
params_mops['command'] = 'STARTED'
submops = []
for x_sub in _sub_mops:
arr_devices = []
for k, v in x_sub['devices'].items():
arr_devices.append({
'mop_id':
x['mop_id'],
'device_id':
v['device_id'],
'vendor':
'%s|%s' % (v['vendor'], v['os']),
'port':
v['port']
})
submops.append({
'subNo': x_sub['subNo'],
'name': x_sub['name'],
'devices': arr_devices,
'num_devices': len(arr_devices),
'mop_id': x['mop_id']
})
params_mops['submops'] = submops
self._sockbotAPIHelpers.url = self._sockbotAPIURL.SOCKBOT_API_CREATE_MOP
self._sockbotAPIHelpers.params = params_mops
self._sockbotAPIHelpers.post_json()
schedule.start()
#------------------------------------------------------------------------------------------
if len(arr_schedule_manage) > 0:
#for schedule in arr_schedule_manage:
# schedule.start()
arr_schedule_manage.clear()
except Exception as e:
stringhelpers.print_bold("IRONMAN SCHEDULE [ERROR]: " + str(e),
"\n")
time.sleep(60)
# stop iron queue ----------------------------------------------------------------------------------------------
_ironQueue.stop()