def run(self): _request = RequestHelpers() dict_schedule = dict() dict_schedule_queue = dict() list_time = list() queue_discovery = queue.Queue() # run queue listining discovery ------------------------------------------------------------------------------ _ironQueue = IronQueue(queue_discovery) _ironQueue.start() # ------------------------------------------------------------------------------------------------------------ while not self.is_stop: try: self.counter = self.counter + 1 # -------------- IRONMAN RUN SCHEDULE ---------------------------------------------------------------- # get current day name #weekday = datetime.now().strftime('%A') #_request.url = self.requestURL.IRONMAN_URL_GET_SCHEDULE % (weekday) _request.url = self.requestURL.IRONMAN_URL_GET_SCHEDULE _list_schedules = _request.get().json() if len(_list_schedules) > 0: for x in _list_schedules: key_mop = 'main_schedule_%d' % (x['mop_id']) schedule_id = x['mop_id'] template_id = int(x['template_id']) mechanism = x['run_type'] run_time = x['run_datetime'].split("-")[1].strip() dict_schedule_queue[str(x['mop_id'])] = run_time if dict_schedule.get(key_mop, None) is not None: pass else: list_time.append(run_time) _request.url = self.requestURL.MEGA_URL_TEMPLATE_DETAIL % ( str(template_id)) _template = _request.get().json() dict_schedule[key_mop] = key_mop schedule = Schedule("SCHEDULE-%d" % (schedule_id), x, _template, dict_schedule, False, mechanism, schedule_id, queue_discovery, x['output_mapping']) schedule.start() time.sleep(2) stringhelpers.print_bold( "IRONMAN SCHEDULE RUN NUMBER: " + str(self.counter), "\n") except Exception as e: stringhelpers.print_bold("IRONMAN SCHEDULE [ERROR]: " + str(e), "\n") time.sleep(30) # stop iron queue ---------------------------------------------------------------------------------------------- _ironQueue.stop()
def run(self): _request = RequestHelpers() dict_template = dict() while not self.is_stop: try: self.counter = self.counter + 1 stringhelpers.print_bold("Archieving info FLASK number: %d" % self.counter) #-------------- FLASK RUN MOP ---------------------------------------------------------------- _request.url = self.requestURL.FLASK_URL_MOP #_list_templates = _request.get().json() _list_mops = _request.get().json() if len(_list_mops) > 0: for x in _list_mops: key_mop = 'template_%d' % (x['mop_id']) mop_id = x['mop_id'] if dict_template.get(key_mop, None) is not None: pass else: _request.url = self.requestURL.MEGA_URL_TEMPLATE_DETAIL % ( x['template_id']) _template = _request.get().json() #-------------------- run device from mop ------------------------------------------- array_device_mop = x['devices'] run_devices = {} for item in array_device_mop: run_devices[str(item)] = "MOP" _template['run_devices'] = run_devices #------------------------------------------------------------------------------------ try: _template['run_args'] = x['run_args'] _template['rollback_args'] = x['rollback_args'] except: pass dict_template[key_mop] = key_mop flask_template = FlaskTemplate( "FLASK-Thread-Template-%s" % (x['template_id']), _template, dict_template, mop_id) flask_template.start() # --------------------------------------------------------------------------------------------------------- time.sleep(30) except Exception as e: stringhelpers.err("FLASK MAIN THREAD ERROR %s" % (e)) except ConnectionError as errConn: stringhelpers.err("FLASK CONNECT API ERROR %s" % (errConn))
def run(self): _request = RequestHelpers() dict_template = dict() while not self.is_stop: try: self.counter = self.counter + 1 stringhelpers.print_bold("Archieving info FLASK number: %d" % self.counter) #-------------- FLASK RUN MOP ---------------------------------------------------------------- _request.url = self.requestURL.FLASK_URL_MOP #_list_templates = _request.get().json() _list_mops = _request.get().json() if len(_list_mops) > 0: for x in _list_mops: mop_id = int(x['mop_id']) key_mop = 'flask_mop_%d' % (mop_id) if dict_template.get(key_mop, None) is not None: pass else: sub_mops = x['sub_mops'] dict_template[key_mop] = key_mop status = 'running' self.update_mop_status(status, mop_id) start_time = time() #for item_mop in sub_mops: flask_sub_mop = FlaskSubMop( "FLASK-Thread-MOP-%d" % (mop_id), sub_mops, mop_id) flask_sub_mop.start() status = flask_sub_mop.join() self.update_mop_status(status, mop_id, start_time, time()) del dict_template[key_mop] # --------------------------------------------------------------------------------------------------------- sleep(30) except Exception as e: stringhelpers.err("FLASK MAIN THREAD ERROR %s" % (e)) except ConnectionError as errConn: stringhelpers.err("FLASK CONNECT API ERROR %s" % (errConn))
class Action(threading.Thread): """ Thread instance each process mega """ def __init__(self, name, data_action=None, action_id=None, params_action=None, param_rollback_action=None, vendor_os=None, session_fang=None, is_rolback=False, file_log=None, deviceid=None): threading.Thread.__init__(self) self.name = name self.data_action = data_action self.requestURL = RequestURL() self.action_id = action_id self._request = RequestHelpers() self.data_command = None self.action_log = {'result': {'outputs': dict()}} self.fang = session_fang self.log_output_file_name = file_log self.vendor_os = vendor_os self.params_action = params_action self.param_rollback_action = param_rollback_action self.is_rollback = is_rolback self.final_result = False self.dict_state_result = dict() self.deviceid = deviceid # sao the nay def run(self): try: key_list_command = self.vendor_os key_rollback = 'rollback' self.action_log['result']['outputs'][key_list_command] = {} self.action_log['result']['outputs'][key_list_command][ 'config'] = [] self.action_log['result']['outputs'][key_list_command][ 'rollback'] = [] action_type = self.data_action['category_1'] _list_action_commands = self.data_action['commands'][ key_list_command]['config'] # list action_command config _list_action_rollback = self.data_action['commands'][ key_list_command]['rollback'] # list action_command rollback # --------------- list dict action command ----------------------------------------------------------------- _dict_list_command = dict() _dict_list_params = self.params_action _array_step = [] if len(_list_action_commands) > 0: count = 0 for _command in _list_action_commands: count = count + 1 _dict_list_command[str(count)] = _command _array_step.append(str(count)) # save step command else: pass # ---------------------------------------------------------------------------------------------------------- # --------------- list dict action command rollback--------------------------------------------------------- _dict_list_command_rollback = dict() _dict_list_params_rollback = self.param_rollback_action _array_step_rollback = [] if len(_list_action_rollback) > 0: count = 0 for _command in _list_action_rollback: count = count + 1 _dict_list_command_rollback[str(count)] = _command _array_step_rollback.append( str(count)) # save step command rollback else: pass # ------------------------------------------------------------------------------------------------------ '''#############################process command by dependency############################################''' if len(_array_step) > 0 and self.is_rollback == False: compare_final_output = [] previous_final_output = [] for step in _array_step: _command_running = _dict_list_command[step] # if _command_running['dependency'] == '0': command_id = _command_running.get('command_id', 0) if command_id > 0: # command_id > 0 dependency = int(_command_running['dependency']) if dependency > 0: # run need compare dependStep = dependency if (int(_command_running['condition']) == int( previous_final_output[dependStep - 1])): output_info = self.process_each_command( command_id, _dict_list_params) if output_info is not None: previous_final_output.append(output_info[ str(command_id)]['final_output']) self.action_log['result']['outputs'][ key_list_command]['config'].append( output_info) stringhelpers.info( "\nAction: [%s]-- config step [%s]: filnal-output: %s" % (self.action_id, step, str(output_info[str(command_id)] ['final_output']))) else: #previous_final_output.append(False) previous_final_output.append(True) else: stringhelpers.err( "MEGA ACTIONS STEP: %s NOT AVAIABLE WITH FINAL_OUTPUT OF STEP %d| THREAD %s" % (step, dependStep, self.name)) previous_final_output.append(False) continue else: # dependency == 0 output_info = self.process_each_command( command_id, _dict_list_params) if output_info is not None: previous_final_output.append(output_info[str( command_id)]['final_output']) self.action_log['result']['outputs'][ key_list_command]['config'].append( output_info) stringhelpers.info( "\nAction: [%s]-- config step [%s]: filnal-output: %s" % (self.action_id, step, str(output_info[str(command_id)] ['final_output']))) if int(step) > 1: if int(output_info[str(command_id)] ['final_output']) == int( _command_running.get( 'condition', 0)): compare_final_output.append(True) else: #self.action_log['final_output'] = False self.action_log['final_output'] = True compare_final_output = [] break else: #previous_final_output.append(False) previous_final_output.append(True) if int(step) > 1: #self.action_log['final_output'] = False self.action_log['final_output'] = True compare_final_output = [] break else: # last command in actions check point try: dependency = int(_command_running['dependency']) if dependency > 0 and int(step) > 0: continue if (int(_command_running['condition']) == int( previous_final_output[dependency - 1])): compare_final_output.append(True) else: #compare_final_output.append(False) compare_final_output.append(True) except: #compare_final_output.append(False) compare_final_output.append(True) # -------------- compare final_output for action ---------------------------------------------------- try: if len(compare_final_output) > 0: first_value = None count = 0 for x in compare_final_output: if count == 0: first_value = x else: first_value = func_compare('=', first_value, x) count = count + 1 self.action_log['final_output'] = first_value self.final_result = first_value #final result run action self.dict_state_result[ 'final_result_action'] = self.final_result else: if len(previous_final_output) > 0: for x in previous_final_output: self.dict_state_result[ 'final_result_action'] = x else: #self.dict_state_result['final_result_action'] = False self.dict_state_result[ 'final_result_action'] = True except Exception as ex: stringhelpers.err( "MEGA ACTIONS THREAD ERROR COMAPRE ACTION FINAL-OUTPUT: %s | THREAD %s" % (ex, self.name)) # --------------------------------------------------------------------------------------------------- '''######################################################################################################''' except Exception as e: stringhelpers.err("MEGA ACTIONS THREAD ERROR %s | THREAD %s" % (e, self.name)) except ConnectionError as errConn: stringhelpers.err( "MEGA ACTIONS CONNECT API URL ERROR %s | THREAD %s" % (self._request.url, self.name)) def process_each_command(self, command_id=0, _dict_list_params=[]): '''process command contains params''' try: self._request.url = self.requestURL.MEGA_URL_COMMAND_DETAIL % ( command_id) self.data_command = self._request.get().json() command = None ################### process args for command ############################################## #self.thread_lock.acquire() if len(_dict_list_params) > 0: for item in _dict_list_params: #array contain dict param argument for k, v in item.items(): if command is None: command = self.data_command['command'] command = command.replace('@{%s}' % (k), v) else: command = command.replace('@{%s}' % (k), v) else: command = self.data_command['command'] #self.thread_lock.release() ########################################################################################### commands = [command] #stringhelpers.info_green(command) self.fang.execute_template_action_command(commands, blanks=2, error_reporting=True, timeout=-1, terminal=False) #result_fang = self.fang.get_output() result_fang = self.fang.get_action_output( self.log_output_file_name) # processing parsing command follow output ########################################### action_command_log = self.parsing(command_id, result_fang, commands[0]) action_command_log = None return action_command_log ###################################################################################### except Exception as e: stringhelpers.err( "[DISCOVERY] MEGA ACTION PROCESS EACH COMMAND ERROR %s | THREAD %s" % (e, self.name)) return None except ConnectionError as errConn: stringhelpers.err( "[DISCOVERY] MEGA ACTION CONNECT API URL ERROR %s | THREAD %s" % (errConn, self.name)) return None def parsing(self, command_id=0, result_fang=None, commandtext=None): final_result_output = [] output_result = dict(deviceid=self.deviceid) output_result['rows'] = [] key = str(command_id) output_result[key] = dict() output_result[key]['output'] = [] array_interface_id = [] try: arrayRow = stringhelpers.text_to_arrayrow(result_fang) if commandtext == "show interface description": array_header = [] arrayRow = list(filter(None, arrayRow)) is_next = False for row in arrayRow: if '#' not in row: if is_next: rows_dict = dict() array_value = row.split() if len(array_value) > 0: for index, name in enumerate(array_header): try: rows_dict[name] = array_value[index] except: rows_dict[name] = '' #------------------- process insert/update/check database tablet interfaces ---------------- try: if rows_dict[ 'Interface'] is None or rows_dict[ 'Interface'] == '': continue interfaceimpl = InterfaceImpl() intf = interfaceimpl.get_interface( self.deviceid, rows_dict['Interface']) if intf: #exist interfaces then update interface_dict = { 'device_id': self.deviceid, 'interface_name': rows_dict['Interface'], 'data': rows_dict } array_interface_id.append( intf.interface_id) interfaceimpl.update(**interface_dict) else: #not exist then insert interface_dict = { 'device_id': self.deviceid, 'interface_name': rows_dict['Interface'], 'data': rows_dict } intf = interfaceimpl.save( **interface_dict) array_interface_id.append( intf.interface_id) except Exception as ex: _strError = "[DISCOVERY][INSERT][UPDATE][INTERFACE]: %s | THREAD %s" % ( ex, self.name) stringhelpers.err(_strError) #------------------------------------------------------------------------------------------- output_result['rows'].append(rows_dict) if ('Interface' in row) and ('Protocol' in row): # phan loai row header array_header = row.split() array_header = list(filter(None, array_header)) is_next = True elif commandtext == "show lldp neighbor": array_process = [] array_header = [] arrayRow = list(filter(None, arrayRow)) is_next = False for row in arrayRow: if '#' not in row and 'Total entries displayed:' not in row: if is_next: rows_dict = dict() array_value = row.split() if len(array_value) > 0: for index, name in enumerate(array_header): if name is not None or name is not '': try: rows_dict[name.replace( ' ', '')] = array_value[index] except: rows_dict[name.replace(' ', '')] = '' # ------------------- process insert/update/check database tablet interfaces ---------------- try: if rows_dict[ 'DeviceID'] is None or rows_dict[ 'LocalIntf'] == '': continue interfaceimpl = InterfaceImpl() intf = interfaceimpl.get_interface( self.deviceid, rows_dict['LocalIntf']) if intf: # exist interfaces then process lldp interface_id = intf.interface_id lldpImpl = LLDPImpl() lldp = lldpImpl.get(interface_id) lldp_dict = { "interface_id": interface_id, "remote_interface": rows_dict['PortID'], "local_interface": rows_dict['LocalIntf'], "remote_device": rows_dict['DeviceID'], "data": rows_dict } if lldp: #update lldp lldpImpl.update(**lldp_dict) else: #insert lldp lldpImpl.save(**lldp_dict) except Exception as ex: _strError = "[DISCOVERY][INSERT][UPDATE][LLDP]: %s | THREAD %s" % ( ex, self.name) stringhelpers.err(_strError) # -------------------------------------------------------------------------------------- output_result['rows'].append(rows_dict) if ('Device ID' in row) and ('Port ID' in row): # phan loai row header array_header = row.split(' ') array_header = list(filter(None, array_header)) is_next = True else: _strError = "[DISCOVERY] MEGA COMMAND: %s NOT SUIABLE | THREAD %s" % ( commandtext, self.name) stringhelpers.err(_strError) #-------------------------------process delete interfaces & lldp if device not exist interface and lldp----- array_delete_interface = [] if len(array_interface_id) > 0: interfaceimpl = InterfaceImpl() list_interfaces = interfaceimpl.get_list_interface( self.deviceid) if len(list_interfaces) > 0: for x in list_interfaces: if x.interface_id not in array_interface_id: array_delete_interface.append(x.interface_id) # dang xu ly #----------------------------------------------------------------------------------------------------------- return output_result except Exception as _errorException: output_result[key]['parsing_status'] = 'ERROR' _strError = "[DISCOVERY] MEGA ACTION PARSING %d ERROR %s | THREAD %s" % ( _errorException, self.name) stringhelpers.err(_strError) return output_result def join(self): threading.Thread.join(self) return self.dict_state_result
class SubTemplate(threading.Thread): '''sub template''' def __init__(self, name, subtemplate=None, is_rollback=False, result_templates=None, mode=0): threading.Thread.__init__(self) self.subtemplate = subtemplate self.name = name self.requestURL = RequestURL() self._request = RequestHelpers() self.array_state_action = [] self.dict_state_result = {} self.is_rollback = is_rollback self.is_check_run_finished = False self.result_templates = result_templates self.mode = mode def excecute(self, data_fang): actions = sorted( data_fang['actions'].items(), reverse=False ) # get actions of each sub template # action contain linkedlist #--------------------- build info device fang ------------------------------------------------------------------ device = data_fang['device']['device_info'] self.dict_state_result[str(device['device_id'])] = {} vendor_ios = data_fang['device']['vendor_ios'] host = device['ip_mgmt'] port = int(device['port_mgmt']) username = device['username'] password = device['password'] device_type = device['os'] method = device['method'] # ssh, telnet parameters = { 'device_type': device_type, 'host': host, 'protocol': method.lower(), 'username': username, 'password': password, 'port': port, 'timeout': 300 } print( "\nMEGA SUBTEMPLATE FANG DEVICE: host=%s, port=%s, devicetype=%s \n" % (parameters['host'], parameters['port'], parameters['device_type'])) fac = FactoryConnector(**parameters) log_output_file_name = "%s.log" % ( stringhelpers.generate_random_keystring(10)) fac = fac.execute_keep_alive(loginfo=log_output_file_name) if not fac.is_alive(): print("CONNECT DEVICE: host=%s, port=%s, devicetype=%s FAIL\n\n" % (parameters['host'], parameters['port'], parameters['device_type'])) return None #--------------------------------------------------------------------------------------------------------------- # --------------- list dict action command --------------------------------------------------------------------- _dict_list_actions = dict() # _dict_list_action params = self.data_action['test_args'] _array_step = [] param_action = None param_rollback_action = None if len(actions) > 0: for step, action in actions: if str(step) != 'args' and (step) != 'rollback_args': _dict_list_actions[str(step)] = action _array_step.append(str(step)) # save step action else: if str(step) == 'args': #array contain dict argument param_action = action else: pass else: pass # -------------------------------------------------------------------------------------------------------------- if len(_array_step) > 0: # and self.is_rollback == False: compare_final_output = [] previous_final_output = [] for step in _array_step: # print(_dict_list_actions[step]) _action = _dict_list_actions[str(step)] action_id = _action.get('action_id', 0) if action_id > 0: # command_id > 0 self._request.url = self.requestURL.MEGA_URL_ACTION_DETAIL % ( action_id) try: thread_action_name = "Thread-Action_%s-In-%s" % ( action_id, self.name) action_data = self._request.get().json() dependency = int(_action['dependency']) if dependency > 0: # run need compare dependStep = dependency if (int(_action['condition']) == int( previous_final_output[dependStep - 1])): thread_action = Action( thread_action_name, action_data, action_id, param_action, param_rollback_action, vendor_ios, fac, self.is_rollback, log_output_file_name, deviceid=device['device_id']) thread_action.start() result = thread_action.join() result['action_id'] = action_id result['device_id'] = device['device_id'] result['device_vendor_ios'] = vendor_ios previous_final_output.append( result['final_result_action']) self.array_state_action.append(result) else: stringhelpers.err( "MEGA ACTIONS STEP: %s NOT AVAIABLE WITH FINAL_OUTPUT OF STEP %d| THREAD %s" % (step, dependStep, self.name)) previous_final_output.append(False) continue else: # dependency == 0 thread_action = Action( thread_action_name, action_data, action_id, param_action, param_rollback_action, vendor_ios, fac, self.is_rollback, log_output_file_name, deviceid=device['device_id']) thread_action.start() result = thread_action.join() result['action_id'] = action_id result['device_id'] = device['device_id'] result['device_vendor_ios'] = vendor_ios previous_final_output.append( result['final_result_action']) self.array_state_action.append(result) if int(step) > 1: if int(result['final_result_action']) == int( _action.get('condition', 0)): self.dict_state_result[str( device['device_id'] )]["final_sub_template"] = True else: self.dict_state_result[str( device['device_id'] )]["final_sub_template"] = False compare_final_output = [] break except: stringhelpers.warn( "[%s] MEGA TEMPLATE REQUEST DATA ACTION %s FAIL\r\n" % (self.name, action_id)) else: # last command in actions check point dependency = int(_action['dependency']) try: if (int(_action['condition']) == int( previous_final_output[dependency - 1])): compare_final_output.append(True) else: compare_final_output.append(False) except: pass #------------------------ calculate decide template is True or False --------------------------------------- if len(previous_final_output) > 0: for x in previous_final_output: if x: self.dict_state_result[str( device['device_id'])]["final_sub_template"] = True self.dict_state_result[str( device['device_id'] )]["actions"] = self.array_state_action else: self.dict_state_result[str( device['device_id'])]["final_sub_template"] = False self.dict_state_result[str( device['device_id'] )]["actions"] = self.array_state_action else: self.dict_state_result[str( device['device_id'])]["final_sub_template"] = False self.dict_state_result[str( device['device_id'])]["actions"] = self.array_state_action #----------------------------------------------------------------------------------------------------------- self.array_state_action = [] fac.remove_file_log(log_output_file_name) # stringhelpers.warn(str(self.action_log)) fac.terminal() # finished fang command def run(self): try: if self.subtemplate is not None: threading_array = [] stringhelpers.info("\n[INFO]-RUN SUBTEMPLATE: %s" % (self.name)) filnal_result = [] #------------------ chay ko song song ------------------------------------------------------------------ if self.mode == 1: #mode = 1, not parallel for device in self.subtemplate['devices']: device_id = device['device']['device_info'][ 'device_id'] # -------------- check has run next subtemplate belong previous result subtemplate if len(self.result_templates) > 0: before_result = self.result_templates[ len(self.result_templates) - 1] try: final_sub_template = before_result['state'][ str(device_id)]['final_sub_template'] if final_sub_template == False: print( "SUB TEMPLATE: %s FOR DEVICE: %s DON'T CONTINIOUS RUN\n\n" % (self.name, device_id)) break except: print( "SUB TEMPLATE: %s FOR DEVICE: %s DON'T CONTINIOUS RUN\n\n" % (self.name, device_id)) break # ------------------------------------------------------------------------------------------ if len(filnal_result) == 0: self.excecute(device) else: if filnal_result[len(filnal_result) - 1] == True: self.excecute(device) try: filnal_result.append(self.dict_state_result[str( device_id)]["final_sub_template"]) except: pass #--------------------------------------------------------------------------------------------------- else: # mode = 2, run parallel #------------------- chay song song ---------------------------------------------------------------- for device in self.subtemplate['devices']: device_id = device['device']['device_info'][ 'device_id'] # -------------- check has run next subtemplate belong previous result subtemplate if len(self.result_templates) > 0: before_result = self.result_templates[ len(self.result_templates) - 1] try: final_sub_template = before_result['state'][ str(device_id)]['final_sub_template'] if final_sub_template == False: print( "SUB TEMPLATE: %s FOR DEVICE: %s DON'T CONTINIOUS RUN\n\n" % (self.name, device_id)) break except: print( "SUB TEMPLATE: %s FOR DEVICE: %s DON'T CONTINIOUS RUN\n\n" % (self.name, device_id)) break # ------------------------------------------------------------------------------------------ if len(filnal_result) == 0: _thread = threading.Thread(target=self.excecute, args=(device, )) _thread.start() threading_array.append(_thread) else: if filnal_result[len(filnal_result) - 1] == True: _thread = threading.Thread( target=self.excecute, args=(device, )) _thread.start() threading_array.append(_thread) for x in threading_array: x.join() for device in self.subtemplate['devices']: device_id = device['device']['device_info'][ 'device_id'] try: filnal_result.append(self.dict_state_result[str( device_id)]["final_sub_template"]) except: pass #------------------------------------------------------------------------------------------------------- except Exception as exError: stringhelpers.err("[ERROR] RUN SUBTEMPLATE-[%s]: %s" % (self.name, exError)) def join(self): threading.Thread.join(self) return self.dict_state_result
class MegaDiscovery(threading.Thread): """ Thread instance each process template """ def __init__(self, name, data_template=None, dict_template={}): threading.Thread.__init__(self) self.name = name self.data_template = data_template self.dict_template = dict_template self.requestURL = RequestURL() self._request = RequestHelpers() self.info_fang = self.buildinfo_subtemplates() self.result_templates = [] def run(self): if self.info_fang is not None: #----------------------------------------------------------------------------------------------------------- for fang in self.info_fang['subtemplates']: # fang sub template sub_template_name = fang['sub_template_name'] subtemplate_thread = SubTemplate(sub_template_name, fang, False, self.result_templates, int(fang['mode'])) subtemplate_thread.start() dict_template = dict(sub_template_name=sub_template_name, state=subtemplate_thread.join(), fang=fang, mode=int(fang['mode'])) self.result_templates.append(dict_template) # ---------------------------------------------------------------------------------------------------------- else: stringhelpers.warn("[%s] MEGA TEMPLATE NOT DATA TO FANG\r\n" % (self.name)) def buildinfo_subtemplates(self): data_fang = dict(subtemplates=[]) # ----------------device list------------------------------------------------------------------------------------ run_devices = sorted(self.data_template['run_devices'].items(), reverse=False) key_maps = sorted( self.data_template['map'].keys()) #key number sub template run_mode = self.data_template["run_mode"] for _k in key_maps: # _k = number template, _v = dict role apply for sub template, sub template sub_template_number = _k subtemplate = dict(devices=[]) dict_map_template = self.data_template['map'][sub_template_number] subtemplate_data = self.data_template['sub_templates'][int( sub_template_number)] #data are list action for k, v in run_devices: # get list device id need fang and role of each device # k = deviceid, v = role of device device_role_name = v role_exist = dict_map_template.get(device_role_name, None) count_step = 0 if role_exist: # compare role of device == role of template info_fang = {} #clear each add info subtemplate['sub_template_name'] = self.data_template[ 'nameMap'][sub_template_number] # ------------ get data chay mode parallel --------------------------------------------------------- try: mode = int(run_mode.get(sub_template_number, 0)) if mode == 1: subtemplate['mode'] = 1 # not run parallel elif mode == 2: subtemplate['mode'] = 2 # run parallel except: pass # -------------------------------------------------------------------------------------------------- try: device_fang = dict(device_id=k, role=device_role_name) device_id = device_fang['device_id'] self._request.url = self.requestURL.URL_GET_DEVICE_DETAIL % ( device_id) print(self._request.url, end='\n\n') device = self._request.get().json() device_fang['device_info'] = dict( port_mgmt=device['port_mgmt'], method=device['method'], vendor=device['vendor'], os=device['os'], username=device['username'], password=device['password'], ip_mgmt=device['ip_mgmt'], device_id=device['device_id']) device_fang['vendor_ios'] = "%s|%s" % ( device['vendor'], device['os'] ) # vendor+os = e.x: Cisco|ios-xr info_fang['device'] = device_fang dict_action = dict(args=[], rollback_args=[]) #-----------------action in each template ---------------------------------------------------------- for action in subtemplate_data: # list actions count_step = count_step + 1 # step dict_action[str(count_step)] = action # process argument for action ------------------------------------------------------------------ try: dict_argument = self.data_template[ 'run_args'].get( sub_template_number, None) # level get by number template if dict_argument is not None: dict_argument = dict_argument.get( device_id, None) # level get by deviceid if dict_argument is not None: dict_action['args'].append( dict_argument) except: pass # ------------------------------------------------------------------------------------------- #ll_actions.append(dict_action) # can xem lai co nen dung double linked list ko info_fang['actions'] = dict_action subtemplate['devices'].append(info_fang) except Exception as _error: stringhelpers.err( "MEGA TEMPLATE BUILD buildinfo_subtemplates ERROR %s\n\r" % (_error)) if subtemplate is not None: data_fang['subtemplates'].append(subtemplate) return data_fang
def run(self): _request = RequestHelpers() dict_command = dict() dict_action = dict() dict_template = dict() dict_template_discovery = dict() while not self.is_stop: try: self.counter = self.counter + 1 stringhelpers.print_bold("Archieving info MEGA number: %d" % self.counter) #--------------- MEGA RUN COMMAND TEST ----------------------------------------------------------------- _request.url = self.requestURL.MEGA_URL_LIST_COMMAND_UNTESTED _list_commands = _request.get().json() if len(_list_commands) > 0: for x in _list_commands: key_command = 'command_%d' % (x['command_id']) if dict_command.get(key_command, None) is not None: pass else: dict_command[key_command] = key_command mega_command = MegaCommand( "Thread-Command-%d" % (x['command_id']), x, dict_command) mega_command.start() time.sleep(10) #------------------------------------------------------------------------------------------------------- #-------------- MEGA RUN ACTION TEST ------------------------------------------------------------------- _request.url = self.requestURL.MEGA_URL_LIST_ACTION_UNTESTED _list_actions = _request.get().json() if len(_list_actions) > 0: for x in _list_actions: key_action = 'action_%d' % (x['action_id']) if dict_action.get(key_action, None) is not None: pass else: dict_action[key_action] = key_action mega_action = MegaAction( "Thread-Action-%d" % (x['action_id']), x, dict_action) mega_action.start() time.sleep(10) #------------------------------------------------------------------------------------------------------- #-------------- MEGA RUN TEMPLATE TEST ---------------------------------------------------------------- _request.url = self.requestURL.MEGA_URL_LIST_TEMPLATE_UNTESTED _list_templates = _request.get().json() #_list_templates = [_request.get().json()] if len(_list_templates) > 0: for x in _list_templates: key_template = 'template_%d' % (x['template_id']) if dict_template.get(key_template, None) is not None: pass else: dict_template[key_template] = key_template mega_template = MegaTemplate( "Thread-Template-%d" % (x['template_id']), x, dict_template) mega_template.start() pass # --------------------------------------------------------------------------------------------------------- time.sleep(10) #-------------- MEGA RUN TEMPLATE TEST ---------------------------------------------------------------- '''_request.url = self.requestURL.MEGA_URL_TEMPLATE_DISCOVERY #_list_templates = _request.get().json() _list_templates_descovery = [_request.get().json()] if len(_list_templates_descovery) > 0: for x in _list_templates_descovery: key_template = 'template_discovery_%d' % (x['template_id']) #if dict_template_discovery.get(key_template, None) is not None: # pass #else: dict_template_discovery[key_template] = key_template mega_template_discovery = MegaDiscovery("Thread-Discovery-Template-%d" % (x['template_id']), x, dict_template_discovery) mega_template_discovery.start() pass''' # --------------------------------------------------------------------------------------------------------- #time.sleep(90) except Exception as e: stringhelpers.err("MEGA MAIN THREAD ERROR %s" % (e)) except ConnectionError as errConn: stringhelpers.err("MEGA CONNECT API ERROR %s" % (errConn))
def run(self): _request = RequestHelpers() _request.url = self.requestURL.URL_GET_DEVICE_DETAIL % ( self.data_command["test_device"]) try: device = _request.get().json() try: if device['status_code'] == 500: #device not exist stringhelpers.err( "DEVICE ID %s NOT EXIST | THREAD %s" % (self.data_command["test_device"], self.name)) except: #process fang test device by command host = device['ip_mgmt'] port = int(device['port_mgmt']) username = device['username'] password = device['password'] device_type = device['os'] method = device['method'] #ssh, telnet parameters = { 'device_type': device_type, 'host': host, 'protocol': method, 'username': username, 'password': password, 'port': port } '''process command contains params''' command = None test_args = self.data_command['test_args'] if len(test_args) > 0: command = self.data_command['command'] for x in self.data_command['test_args']: command = command.replace('@{%s}' % (x['name']), x['value']) else: command = self.data_command['command'] '''####################################''' commands = [command] fac = FactoryConnector(**parameters) print("FANG DEVICE: host=%s, port=%s, devicetype=%s \n\n" % (host, device['port_mgmt'], device_type)) fang = fac.execute(commands) result_fang = fang.get_output() fang.remove_file_log(fac.file_log_command) #remove file.log _request.url = self.requestURL.MEGA_URL_COMMANDLOG_GETBY_COMMANDID % ( self.data_command['command_id']) command_log = _request.get().json() if len(command_log) > 0: cmd_log = { 'command_id': self.data_command['command_id'], 'device_id': self.data_command["test_device"], 'console_log': result_fang, 'result': dict(), 'command': command } # processing parsing command follow output ########################################### command_type = self.data_command['type'] cmd_log = self.parsing(command_type, cmd_log) ###################################################################################### try: _request.url = self.requestURL.MEGA_URL_COMMANDLOG_UPDATE % ( command_log[0]['log_id']) _request.params = cmd_log _request.put() stringhelpers.info( "MEGA THREAD INFO: %s | THREAD %s" % ("UPDATE COMMAND LOG SUCCESS", self.name)) #---------------update mega_status to command------------------------------------------------ _request.url = self.requestURL.MEGA_URL_COMMAND_UPDATE % ( self.data_command['command_id']) _request.params = {'mega_status': 'tested'} _request.put() key_command = 'command_%d' % ( self.data_command['command_id']) del self.dict_command[key_command] #-------------------------------------------------------------------------------------------- except ConnectionError as _conErr: stringhelpers.info( "MEGA THREAD ERROR: %s | THREAD %s" % (_conErr, self.name)) else: cmd_log = { 'command_id': self.data_command['command_id'], 'device_id': self.data_command["test_device"], 'console_log': result_fang, 'result': dict(), 'command': command } # processing parsing command follow output ########################################### command_type = self.data_command['type'] cmd_log = self.parsing(command_type, cmd_log) ###################################################################################### try: _request.url = self.requestURL.MEGA_URL_COMMANDLOG_CREATE _request.params = cmd_log _request.post() stringhelpers.info( "MEGA THREAD INFO: %s | THREAD %s" % ("INSERT COMMAND LOG SUCCESS", self.name)) # ---------------update mega_status to command------------------------------------------------ _request.url = self.requestURL.MEGA_URL_COMMAND_UPDATE % ( self.data_command['command_id']) _request.params = {'mega_status': 'tested'} _request.put() key_command = 'command_%d' % ( self.data_command['command_id']) del self.dict_command[key_command] # -------------------------------------------------------------------------------------------- except ConnectionError as _conErr: stringhelpers.info( "MEGA THREAD ERROR: %s | THREAD %s" % (_conErr, self.name)) except Exception as e: stringhelpers.err("MEGA THREAD ERROR %s | THREAD %s" % (e, self.name)) except ConnectionError as errConn: stringhelpers.err("MEGA CONNECT API URL ERROR %s | THREAD %s" % (_request.url, self.name))
class Schedule(threading.Thread): ''' Schedule threading''' def __init__(self, name=None, mop_data=None, sub_mops=None, dict_schedule=None, is_stop=None, mechanism=None, mop_id = 0, queue=None, output_mapping=None, socketio=None, socketio_iron=None): threading.Thread.__init__(self) self.name = name self.mop_data = mop_data self.sub_mops = sub_mops self.dict_schedule = dict_schedule self.is_stop = is_stop self.mechanism = mechanism self.requestURL = RequestURL() self._request = RequestHelpers() self.is_waiting = True self.mop_id = mop_id self.queue = queue self.output_mapping = output_mapping self.database_mop = dict() self.trying_waiting_done = 1000 self.socketio = socketio self.socketio_iron = socketio_iron def on_command_response(self, args): print("on_command:" + args) self.is_stop = True def run(self): self.socketio_iron.on('oncommand', self.on_command_response) try: dict_version_container = dict() #---------------------------- waiting time for time start ------------------------------------------------ while self.is_waiting: run_time = self.mop_data['run_datetime'].split("-")[1].strip() time_start = datetime.strptime(run_time, '%H:%M').time() time_current = datetime.strptime("%d:%d"%(datetime.now().hour, datetime.now().minute), "%H:%M").time() if time_current >= time_start: self.is_waiting = False else: stringhelpers.info('\n[IRON][DISCOVERY][START RUNNING TIME][%s]' % (str(self.mop_data['run_datetime'])), socket_namespace = self.socketio_iron, on_command_text = 'overall_terminal') #----------------------------------------------------------------------------------------------------------- while not self.is_stop: # -------------------- run device from mop ------------------------------------------------------------- key_mop = 'main_schedule_%d' % (int(self.mop_data['mop_id'])) try: table_id = int(self.mop_data['save_to_table']) tableImpl = TABLEImpl() table_name = tableImpl.get(table_id)['table_name'] key_merge = self.mop_data.get('key_merge', None) count_number = 0 arr_manager_discovery = [] len_submops = len(self.sub_mops) - 1 for sub_mop_item in self.sub_mops: sub_no = int(sub_mop_item.get('subNo', 0)) irondiscovery = Iron_Mop_Discovery("IRONMAN-Thread-Template-%s" % (str(self.mop_id)), sub_mop_item, {}, self.mop_id, table_name, self.output_mapping[str(sub_no)], key_merge, sub_no, dict_version_container, len_submops, self.socketio, self.socketio_iron) # insert to queue discovery self.queue.put(irondiscovery) arr_manager_discovery.append(irondiscovery) count_number = count_number + 1 stringhelpers.info('\n[ENQUEUE] - > [IRON][DISCOVERY][MOP_ID: %s][SUB_MOP_NAME: %s]' % (str(self.mop_id), sub_mop_item['name']), socket_namespace=self.socketio_iron, on_command_text='overall_terminal') if self.mechanism.upper() == 'MANUAL': self.is_stop = True del self.dict_schedule[key_mop] else: weekday = datetime.now().strftime('%A') if weekday not in list(self.mop_data['day_in_week']): del self.dict_schedule[key_mop] self.is_stop = True else: if not self.is_stop: while True: count = 0 for discovery_item in arr_manager_discovery: if discovery_item.done == True: count = count + 1 time.sleep(0.2) if count == len(arr_manager_discovery): #----------------------------- get detail sub mop -------------------------------------------------------------------- self._request.url = self.requestURL.IRONMAN_URL_GET_MOP_DETAIL % (str(self.mop_id)) _mop_details = self._request.get().json() sub_mops = _mop_details.get('sub_mops', None) if sub_mops is not None: self.sub_mops = sub_mops stringhelpers.info('\n[IRON][DISCOVERY][GET_MOP_DETAIL_FOR_LOOP][MOP_ID:%s][%s]' % (self.mop_id, self.name), socket_namespace=self.socketio_iron, on_command_text='overall_terminal') #-------------------------------------------------------------------------------------------------------------- stringhelpers.info('\n[IRON][DISCOVERY][WAITING][%d minutes][%s]' % (int(self.mop_data['return_after']), self.name), socket_namespace=self.socketio_iron, on_command_text='overall_terminal') #time_remaining = int(self.mop_data['return_after']) * 60 time_remaining = 1 * 60 for remaining in range(time_remaining, 0, -1): str_time = "Còn <strong>{:2d}s</strong> để chạy lần kế tiếp".format(remaining) dict_time_remaining = { 'mop_id': self.mop_id, 'text': str_time, 'time_count': "{:2d}".format(remaining).strip() } self.socketio_iron.emit('running_time_remain_mop', dict_time_remaining) time.sleep(1) if self.is_stop: break #STOP MOP break time.sleep(1) except Exception as _exError: stringhelpers.err("[ERROR] %s" % (_exError)) except Exception as error: stringhelpers.err("[ERROR] %s %s" % (self.name, error))
class Schedule(threading.Thread): ''' Schedule threading''' def __init__(self, name=None, mop_data=None, template_data=None, dict_schedule=None, is_stop=None, mechanism=None, schedule_id=0, queue=None, output_mapping=None): threading.Thread.__init__(self) self.name = name self.mop_data = mop_data self.template_data = template_data self.dict_schedule = dict_schedule self.is_stop = is_stop self.mechanism = mechanism self.requestURL = RequestURL() self._request = RequestHelpers() self.is_waiting = True self.schedule_id = schedule_id self.queue = queue self.output_mapping = output_mapping def run(self): try: #---------------------------- waiting time for time start ------------------------------------------------ while self.is_waiting: run_time = self.mop_data['run_datetime'].split("-")[1].strip() time_start = datetime.strptime(run_time, '%H:%M').time() time_current = datetime.strptime( "%d:%d" % (datetime.now().hour, datetime.now().minute), "%H:%M").time() if time_current >= time_start: self.is_waiting = False #--------------------------------------------------------------------------------------------------------- while not self.is_stop: # -------------------- run device from mop ------------------------------------------- array_device_mop = self.mop_data['devices'] run_devices = {} for item in array_device_mop: self._request.url = self.requestURL.URL_GET_DEVICE_DETAIL % ( int(item)) # get device detail device = self._request.get().json() run_devices[str(item)] = device['role'] self.template_data['run_devices'] = run_devices key_mop = 'main_schedule_%d' % (self.mop_data['mop_id']) try: table_id = int(self.mop_data['save_to_table']) tableImpl = TABLEImpl() table_name = tableImpl.get(table_id)['table_name'] irondiscovery = IronDiscovery( "IRONMAN-Thread-Template-%s" % (self.template_data['template_id']), self.template_data, {}, self.mop_data['mop_id'], table_name, self.output_mapping) # insert to queue discovery self.queue.put(irondiscovery) # irondiscovery.start() # irondiscovery.join() if self.mechanism.upper() == 'MANUAL': self.is_stop = True del self.dict_schedule[key_mop] else: weekday = datetime.now().strftime('%A') if weekday not in list(self.mop_data['day_in_week']): del self.dict_schedule[key_mop] self.is_stop = True else: while True: if irondiscovery.done == True: stringhelpers.info( '[IRON][DISCOVERY][WAITING][%d minutes][%s]' % (int(self.mop_data['return_after']), self.name)) time.sleep( int(self.mop_data['return_after']) * 60) break except Exception as _exError: stringhelpers.err("[ERROR] %s" % (_exError)) except Exception as error: stringhelpers.err("[ERROR] %s %s" % (self.name, error))
class Action(threading.Thread): """ Thread instance each process mega """ def __init__(self, name, data_action = None, action_id = None, params_action=None, param_rollback_action=None, vendor_os=None, session_fang=None, is_rolback=False, file_log=None, deviceid=None, table_name=None, data_fields=None): threading.Thread.__init__(self) self.name = name self.data_action = data_action self.requestURL = RequestURL() self.action_id = action_id self._request = RequestHelpers() self.data_command = None self.action_log = {'result':{'outputs':dict()}} self.fang = session_fang self.log_output_file_name = file_log self.vendor_os = vendor_os self.params_action = params_action self.param_rollback_action = param_rollback_action self.is_rollback = is_rolback self.final_result = False self.dict_state_result = dict() self.deviceid = deviceid self.dict_table_impl = { 'interfaces': InterfaceImpl, 'lldp': LLDPImpl } self.table_name = table_name self.data_fields = data_fields # sao the nay def run(self): try: key_list_command = self.vendor_os key_rollback = 'rollback' self.action_log['result']['outputs'][key_list_command] = {} self.action_log['result']['outputs'][key_list_command]['config'] = [] self.action_log['result']['outputs'][key_list_command]['rollback'] = [] action_type = self.data_action['category_1'] _list_action_commands = self.data_action['commands'][key_list_command]['config'] # list action_command config _list_action_rollback = self.data_action['commands'][key_list_command]['rollback'] # list action_command rollback # --------------- list dict action command ----------------------------------------------------------------- _dict_list_command = dict() _dict_list_params = self.params_action _array_step = [] if len(_list_action_commands) > 0: count = 0 for _command in _list_action_commands: count = count + 1 _dict_list_command[str(count)] = _command _array_step.append(str(count)) # save step command else: pass # ---------------------------------------------------------------------------------------------------------- # --------------- list dict action command rollback--------------------------------------------------------- _dict_list_command_rollback = dict() _dict_list_params_rollback = self.param_rollback_action _array_step_rollback = [] if len(_list_action_rollback) > 0: count = 0 for _command in _list_action_rollback: count = count + 1 _dict_list_command_rollback[str(count)] = _command _array_step_rollback.append(str(count)) # save step command rollback else: pass # ------------------------------------------------------------------------------------------------------ '''#############################process command by dependency############################################''' if len(_array_step) > 0 and self.is_rollback == False: compare_final_output = [] previous_final_output = [] for step in _array_step: _command_running = _dict_list_command[step] # if _command_running['dependency'] == '0': command_id = _command_running.get('command_id', 0) if command_id > 0: # command_id > 0 dependency = int(_command_running['dependency']) if dependency > 0: # run need compare dependStep = dependency if (int(_command_running['condition']) == int(previous_final_output[dependStep - 1])): command_type = _command_running.get('type', None) if command_type is not None: if command_type == '5': # process delay command output_info = self.process_each_command(command_id, _dict_list_params, step) previous_final_output.append(True) else: output_info = self.process_each_command(command_id, _dict_list_params, step) if output_info is not None: previous_final_output.append(output_info[str(command_id)]['final_output']) self.action_log['result']['outputs'][key_list_command]['config'].append( output_info) stringhelpers.info( "\nAction: [%s]-- config step [%s]: filnal-output: %s" % ( self.action_id, step, str(output_info[str(command_id)]['final_output']))) else: # previous_final_output.append(False) previous_final_output.append(True) else: stringhelpers.err( "MEGA ACTIONS STEP: %s NOT AVAIABLE WITH FINAL_OUTPUT OF STEP %d| THREAD %s" % (step, dependStep, self.name)) previous_final_output.append(False) continue else: # dependency == 0 command_type = _command_running.get('type', None) if command_type is not None: if command_type == '5': # process delay command output_info = self.process_each_command(command_id, _dict_list_params, step) previous_final_output.append(True) else: output_info = self.process_each_command(command_id, _dict_list_params, step) if output_info is not None: previous_final_output.append(output_info[str(command_id)]['final_output']) self.action_log['result']['outputs'][key_list_command]['config'].append( output_info) stringhelpers.info("\nAction: [%s]-- config step [%s]: filnal-output: %s" % ( self.action_id, step, str(output_info[str(command_id)]['final_output']))) if int(step) > 1: if int(output_info[str(command_id)]['final_output']) == int( _command_running.get('condition', 0)): compare_final_output.append(True) else: # self.action_log['final_output'] = False self.action_log['final_output'] = True compare_final_output = [] break else: # previous_final_output.append(False) previous_final_output.append(True) if int(step) > 1: # self.action_log['final_output'] = False self.action_log['final_output'] = True compare_final_output = [] break else: # last command in actions check point try: dependency = int(_command_running['dependency']) if dependency > 0 and int(step) > 0: continue if (int(_command_running['condition']) == int(previous_final_output[dependency - 1])): compare_final_output.append(True) else: #compare_final_output.append(False) compare_final_output.append(True) except: #compare_final_output.append(False) compare_final_output.append(True) # -------------- compare final_output for action ---------------------------------------------------- try: if len(compare_final_output) > 0: first_value = None count = 0 for x in compare_final_output: if count == 0: first_value = x else: first_value = func_compare('=', first_value, x) count = count + 1 self.action_log['final_output'] = first_value self.final_result = first_value #final result run action self.dict_state_result['final_result_action'] = self.final_result else: if len(previous_final_output) > 0: for x in previous_final_output: self.dict_state_result['final_result_action'] = x else: #self.dict_state_result['final_result_action'] = False self.dict_state_result['final_result_action'] = True except Exception as ex: stringhelpers.err("MEGA ACTIONS THREAD ERROR COMAPRE ACTION FINAL-OUTPUT: %s | THREAD %s" % (ex, self.name)) # --------------------------------------------------------------------------------------------------- '''######################################################################################################''' except Exception as e: stringhelpers.err("MEGA ACTIONS THREAD ERROR %s | THREAD %s" % (e, self.name)) except ConnectionError as errConn: stringhelpers.err("MEGA ACTIONS CONNECT API URL ERROR %s | THREAD %s" % (self._request.url, self.name)) def process_each_command(self, command_id = 0, _dict_list_params = [], step=''): '''process command contains params''' try: self._request.url = self.requestURL.MEGA_URL_COMMAND_DETAIL % (command_id) self.data_command = self._request.get().json() command = None #-------------------- process delay command --------------------------------------------------- try: if int(self.data_command['type']) == 5: sleep(int(self.data_command['delay'])) stringhelpers.info("[DELAY COMMAND %s %s]" % (command_id, self.data_command['delay'])) return None except Exception as ex_error: errror = ex_error #---------------------------------------------------------------------------------------------- ################### process args for command ############################################## #self.thread_lock.acquire() if len(_dict_list_params) > 0: for item in _dict_list_params: #array contain dict param argument for k, v in item.items(): if command is None: command = self.data_command['command'] command = command.replace('@{%s}' % (k), v) else: command = command.replace('@{%s}' % (k), v) else: command = self.data_command['command'] #self.thread_lock.release() ########################################################################################### if command is not None: commands = [command] #stringhelpers.info_green(command) self.fang.execute_template_action_command(commands, blanks=2, error_reporting=True, timeout=-1, terminal=False) #result_fang = self.fang.get_output() result_fang = self.fang.get_action_output(self.log_output_file_name) # processing parsing command follow output ########################################### action_command_log = self.parsing(command_id ,result_fang, commands[0], step) action_command_log = None return action_command_log ###################################################################################### else: return None except Exception as e: stringhelpers.err("[DISCOVERY] MEGA ACTION PROCESS EACH COMMAND ERROR %s | THREAD %s" % (e, self.name)) return None except ConnectionError as errConn: stringhelpers.err("[DISCOVERY] MEGA ACTION CONNECT API URL ERROR %s | THREAD %s" % (errConn, self.name)) return None def parsing(self, command_id = 0, result_fang = None, commandtext=None, step=''): final_result_output = [] output_result = dict(deviceid=self.deviceid) output_result['rows'] = [] key = str(command_id) output_result[key] = dict() output_result[key]['output'] = [] array_network_id = [] array_index_length = [] dict_index_header = dict() array_header_map = {} try: if command_id == 11019 or command_id == 11020 or command_id == 11025: longhk = "test" if self.data_fields is None: return None step = 0#int(step) - 1 start_by = self.data_command['output'][step].get('start_by', None) end_by = self.data_command['output'][step].get('end_by', None) if start_by is not None and end_by is not None: result_fang = stringhelpers.string_between(result_fang, start_by, end_by) else: return None array_row_data = stringhelpers.text_to_arrayrow(result_fang) array_row_data = stringhelpers.remove_duplicates(array_row_data) #string_contain_header = self.data_command['output'][step].get('header', None) # default item 0 in array string_contain_header = self.data_command['output'][step].get('template_header', None) # default item 0 in array #string_table_name = self.data_command['output'][step].get('db_table', None).lower() # table name string_table_name = self.table_name if string_contain_header is not None: array_header = [] arrayRow = [x for x in array_row_data if x is not None] list_header = [] list_header.append(string_contain_header) arrayRow[:0] = list_header is_next = False row_count = 0 for row in arrayRow: if row == '': continue if '#' not in row and '---------' not in row: if is_next: rows_dict = dict() array_value = row.split() data_build = dict(versions=[]) data_version = {} #-------- get value follow colums ------------------------------------------------------ is_insert = False index_column = 0 for config_output in self.data_command['output']: try: #index_column = int(config_output.get('column', None)) # value index key = config_output['header_start'] start, end = dict_index_header.get(key, None) value = row[start:end].strip() #field = array_header_map.get(key, None) field = self.data_fields[str(command_id)][config_output['name']] #array_check_whitespace = [v for v in value if v.isspace()] if value is not '': data_build[field] = value data_version[field] = value is_insert = True except: pass data_version['modifieddate'] = datetime.now() #--------------------------------------------------------------------------------------- if is_insert == True: index_column = index_column + 1 for index, name in enumerate(array_header): try: rows_dict[name] = array_value[index] except: rows_dict[name] = '' data_build['device_id'] = self.deviceid data_build['data'] = rows_dict data_build['table'] = string_table_name data_build['column'] = index_column data_build['row'] = row_count data_build['command_id'] = command_id #------------------- process insert/update/check database tablet interfaces -------- try: netwImpl = NetworkObjectImpl() intf = netwImpl.get(self.deviceid, string_table_name, index_column, row_count, command_id) if intf: #exist interfaces then update array_network_id.append(intf.networkobject_id) versions = intf['versions'] if versions is not None: versions.append(data_version) data_build['versions'] = versions netwImpl.update(**data_build) else: #not exist then insert data_build['versions'].append(data_version) intf = netwImpl.save(**data_build) array_network_id.append(intf.networkobject_id) except Exception as ex: _strError = "[DISCOVERY][INSERT][UPDATE][%s]: %s | THREAD %s" % (ex, string_table_name, self.name) stringhelpers.err(_strError) #------------------------------------------------------------------------------------------- output_result['rows'].append(rows_dict) if(string_contain_header in row): # build header and extract size header for value for config_output in self.data_command['output']: try: #header_start #header_end header_start = config_output['header_start'] header_end = config_output.get('header_end', None) index_start = row.index(header_start) name = config_output.get('name', None) #save field to db if header_end is None: index_end = len(row) * 15 else: index_end = row.index(header_end) header = row[index_start:index_end].strip() array_header_map[header] = name dict_index_header[header] = (index_start, index_end) array_header.append(header) except Exception as _outputHeaderError: _strError = "[DISCOVERY][PARSING][HEADER][%s]: %s | THREAD %s" % (self.name, _outputHeaderError) stringhelpers.err(_strError) is_next = True row_count = row_count + 1 #-------------------------------process delete interfaces & lldp if device not exist interface and lldp----- array_delete_networkobject = [] if len(array_network_id) > 0: netwImpl = NetworkObjectImpl() list = netwImpl.get_list(self.deviceid, string_table_name, command_id) if len(list) > 0: for x in list: if x.networkobject_id not in array_network_id: array_delete_networkobject.append(x.networkobject_id) if len(array_delete_networkobject) > 0: for d in array_delete_networkobject: netwImpl.delete(d) stringhelpers.err('[DELETE][NETWORK_OBJECT_ID] - %s' % (str(d)), '\n\n') # dang xu ly #----------------------------------------------------------------------------------------------------------- return output_result except Exception as _errorException: output_result[key]['parsing_status'] = 'ERROR' _strError = "[DISCOVERY] MEGA ACTION PARSING %s ERROR %s | THREAD %s" % (_errorException, self.name) stringhelpers.err(_strError) return output_result def join(self): threading.Thread.join(self) return self.dict_state_result
class MegaAction(threading.Thread): """ Thread instance each process mega """ def __init__(self, name, data_action=None, dict_action={}): threading.Thread.__init__(self) self.name = name self.data_action = data_action self.requestURL = RequestURL() self.dict_action = dict_action self._request = RequestHelpers() self.data_command = None self.action_log = {'result': {'outputs': dict()}} self.fang = None self.log_output_file_name = None # sao the nay def run(self): self._request.url = self.requestURL.URL_GET_DEVICE_DETAIL % ( self.data_action["test_device"]) try: device = self._request.get().json() try: if device['status_code'] == 500: #device not exist stringhelpers.err( "DEVICE ID %s NOT EXIST | THREAD %s" % (self.data_action["test_device"], self.name)) except: #process fang test device by command host = device['ip_mgmt'] port = int(device['port_mgmt']) username = device['username'] password = device['password'] device_type = device['os'] method = device['method'] #ssh, telnet parameters = { 'device_type': device_type, 'host': host, 'protocol': method.lower(), 'username': username, 'password': password, 'port': port, 'timeout': 300 } key_list_command = "%s|%s" % (device['vendor'], device['os']) key_rollback = 'rollback' self.log_output_file_name = "action_%s.log" % ( self.data_action['action_id']) self.action_log['result']['outputs'][key_list_command] = {} self.action_log['result']['outputs'][key_list_command][ 'config'] = [] self.action_log['result']['outputs'][key_list_command][ 'rollback'] = [] _list_action_commands = self.data_action['commands'][ key_list_command]['config'] #list action_command config _list_action_rollback = self.data_action['commands'][ key_list_command][ 'rollback'] #list action_command rollback #--------------- list dict action command -------------------------------------------------------------- _dict_list_command = dict() _dict_list_params = self.data_action['test_args'] _array_step = [] if len(_list_action_commands) > 0: count = 0 for _command in _list_action_commands: count = count + 1 _dict_list_command[str(count)] = _command _array_step.append(str(count)) # save step command else: pass #------------------------------------------------------------------------------------------------------- # --------------- list dict action command rollback--------------------------------------------------------------------- _dict_list_command_rollback = dict() _dict_list_params_rollback = self.data_action['rollback_args'] _array_step_rollback = [] if len(_list_action_rollback) > 0: count = 0 for _command in _list_action_rollback: count = count + 1 _dict_list_command_rollback[str(count)] = _command _array_step_rollback.append( str(count)) # save step command rollback else: pass # ------------------------------------------------------------------------------------------------------- fac = FactoryConnector(**parameters) '''#############################process command by dependency########################################''' if len(_array_step) > 0: print( "MEGA ACTION FANG DEVICE: host=%s, port=%s, devicetype=%s \n\n" % (parameters['host'], parameters['port'], parameters['device_type'])) self.fang = fac.execute_keep_alive( loginfo=self.log_output_file_name) compare_final_output = [] previous_final_output = [] for step in _array_step: _command_running = _dict_list_command[step] #if _command_running['dependency'] == '0': command_id = _command_running.get('command_id', 0) if command_id > 0: #command_id > 0 dependency = int(_command_running['dependency']) if dependency > 0: # run need compare dependStep = dependency if (int(_command_running['condition']) == int( previous_final_output[dependStep - 1])): output_info = self.process_each_command( command_id, _dict_list_params) if output_info is not None: previous_final_output.append( output_info[str( command_id)]['final_output']) self.action_log['result']['outputs'][ key_list_command]['config'].append( output_info) stringhelpers.info( "\nstep %s: %s" % (step, str(output_info))) else: previous_final_output.append(False) else: stringhelpers.err( "MEGA ACTIONS STEP: %s NOT AVAIABLE WITH FINAL_OUTPUT OF STEP %d| THREAD %s" % (step, dependStep, self.name)) previous_final_output.append(False) continue else: # dependency == 0 output_info = self.process_each_command( command_id, _dict_list_params) if output_info is not None: previous_final_output.append(output_info[ str(command_id)]['final_output']) self.action_log['result']['outputs'][ key_list_command]['config'].append( output_info) stringhelpers.info( "\nstep %s: %s" % (step, str(output_info))) if int(step) > 1: if int(output_info[str(command_id)] ['final_output']) == int( _command_running.get( 'condition', 0)): compare_final_output.append(True) else: self.action_log[ 'final_output'] = False compare_final_output = [] break else: previous_final_output.append(False) if int(step) > 1: self.action_log['final_output'] = False compare_final_output = [] break else: #last command in actions check point try: dependency = int( _command_running['dependency']) if dependency > 0 and int(step) > 0: continue if (int(_command_running['condition']) == int( previous_final_output[dependency - 1])): compare_final_output.append(True) else: compare_final_output.append(False) except: pass #-------------- compare final_output for action ---------------------------------------------------- try: if len(compare_final_output) > 0: first_value = None count = 0 for x in compare_final_output: if count == 0: first_value = x else: first_value = func_compare( '=', first_value, x) count = count + 1 self.action_log['final_output'] = first_value else: if len(previous_final_output) > 0: for x in previous_final_output: self.dict_state_result[ 'final_result_action'] = x else: self.dict_state_result[ 'final_result_action'] = False except Exception as ex: stringhelpers.err( "MEGA ACTIONS THREAD ERROR COMAPRE ACTION FINAL-OUTPUT: %s | THREAD %s" % (ex, self.name)) #--------------------------------------------------------------------------------------------------- '''##################################################################################################''' '''#############################process command by rollback dependency########################################''' if len(_array_step_rollback) > 0: print( "MEGA ACTION ROLLBACK FANG DEVICE: host=%s, port=%s, devicetype=%s \n\n" % (parameters['host'], parameters['port'], parameters['device_type'])) #self.fang = fac.execute_keep_alive(loginfo=self.log_output_file_name) compare_final_output = [] previous_final_output = [] for step in _array_step_rollback: _command_running = _dict_list_command_rollback[step] # if _command_running['dependency'] == '0': command_id = _command_running.get('command_id', 0) if command_id > 0: # command_id > 0 dependency = int(_command_running['dependency']) if dependency > 0: # run need compare dependStep = dependency if (int(_command_running['condition']) == int( previous_final_output[dependStep - 1])): output_info = self.process_each_command( command_id, _dict_list_params_rollback) if output_info is not None: previous_final_output.append( output_info[str( command_id)]['final_output']) self.action_log['result']['outputs'][ key_list_command][ 'rollback'].append(output_info) stringhelpers.info( "\nstep %s: %s" % (step, str(output_info))) else: previous_final_output.append(False) else: stringhelpers.err( "MEGA ACTIONS ROLLBACK STEP: %s NOT AVAIABLE WITH FINAL_OUTPUT OF STEP %d| THREAD %s" % (step, dependStep, self.name)) previous_final_output.append(False) continue else: # dependency == 0 output_info = self.process_each_command( command_id, _dict_list_params_rollback) if output_info is not None: previous_final_output.append(output_info[ str(command_id)]['final_output']) self.action_log['result']['outputs'][ key_list_command]['rollback'].append( output_info) stringhelpers.info( "\nstep %s: %s" % (step, str(output_info))) if int(step) > 1: if int(output_info[str(command_id)] ['final_output']) == int( _command_running.get( 'condition', 0)): compare_final_output.append(True) else: self.action_log[ 'final_output'] = False compare_final_output = [] break else: previous_final_output.append(False) if int(step) > 1: self.action_log['final_output'] = False compare_final_output = [] break else: # last command in actions check point try: #catch error not command rollback dependency = int( _command_running['dependency']) if dependency > 0 and int(step) > 0: continue if (int(_command_running['condition']) == int( previous_final_output[dependency - 1])): compare_final_output.append(True) else: compare_final_output.append(False) except: print("DO NOT COMMAND ROLLBACK\n") stringhelpers.err( "MEGA ACTIONS THREAD ROLLBACK FINISHED: | THREAD %s" % (self.name)) # -------------- compare final_output for action ---------------------------------------------------- try: if len(compare_final_output) > 0: first_value = None count = 0 for x in compare_final_output: if count == 0: first_value = x else: first_value = func_compare( '=', first_value, x) count = count + 1 self.action_log['final_output'] = first_value except Exception as ex: stringhelpers.err( "MEGA ACTIONS THREAD ERROR ROLLBACK COMAPRE ACTION FINAL-OUTPUT: %s | THREAD %s" % (ex, self.name)) # --------------------------------------------------------------------------------------------------- '''##################################################################################################''' # ------------------------------------ process save log action -------------------------------------- self._request.url = self.requestURL.MEGA_URL_ACTIONLOG_GETBY_ACTIONID % ( self.data_action['action_id']) _request_action_log = self._request.get().json() if len(_request_action_log) > 0: # update action log self.action_log['action_id'] = self.data_action[ 'action_id'] self.action_log['device_id'] = self.data_action[ "test_device"] try: self._request.url = self.requestURL.MEGA_URL_ACTIONLOG_UPDATE % ( _request_action_log[0]['log_id']) self._request.params = self.action_log self._request.put() stringhelpers.info( "MEGA ACTIONS THREAD INFO: %s | THREAD %s" % ("UPDATE ACTIONS LOG SUCCESS", self.name)) # ---------------update mega_status to action------------------------------------------------ self._request.url = self.requestURL.MEGA_URL_ACTION_UPDATE % ( self.data_action['action_id']) self._request.params = {'mega_status': 'tested'} self._request.put() key_action = 'action_%d' % ( self.data_action['action_id']) del self.dict_action[key_action] # -------------------------------------------------------------------------------------------- except ConnectionError as _conErr: stringhelpers.info( "MEGA ACTIONS THREAD ERROR: %s | THREAD %s" % (_conErr, self.name)) else: # insert action log self.action_log['action_id'] = self.data_action[ 'action_id'] self.action_log['device_id'] = self.data_action[ "test_device"] try: self._request.url = self.requestURL.MEGA_URL_ACTIONLOG_CREATE self._request.params = self.action_log self._request.post() stringhelpers.info( "MEGA ACTIONS THREAD INFO: %s | THREAD %s" % ("INSERT ACTIONS LOG SUCCESS", self.name)) # ---------------update mega_status to action------------------------------------------------ self._request.url = self.requestURL.MEGA_URL_ACTION_UPDATE % ( self.data_action['action_id']) self._request.params = {'mega_status': 'tested'} self._request.put() key_action = 'action_%d' % ( self.data_action['action_id']) del self.dict_action[key_action] # -------------------------------------------------------------------------------------------- except ConnectionError as _conErr: stringhelpers.err( "MEGA ACTIONS THREAD ERROR: %s | THREAD %s" % (_conErr, self.name)) # --------------------------------------------------------------------------------------------------- self.fang.remove_file_log(self.log_output_file_name) # stringhelpers.warn(str(self.action_log)) self.fang.terminal() # finished fang command except Exception as e: stringhelpers.err("MEGA ACTIONS THREAD ERROR %s | THREAD %s" % (e, self.name)) except ConnectionError as errConn: stringhelpers.err( "MEGA ACTIONS CONNECT API URL ERROR %s | THREAD %s" % (self._request.url, self.name)) def process_each_command(self, command_id=0, _dict_list_params={}): '''process command contains params''' try: self._request.url = self.requestURL.MEGA_URL_COMMAND_DETAIL % ( command_id) self.data_command = self._request.get().json() command = None ################### process args for command ############################################## if len(_dict_list_params.items()) > 0: for k, v in _dict_list_params.items(): if command is None: command = self.data_command['command'] command = command.replace('@{%s}' % (k), v) else: command = command.replace('@{%s}' % (k), v) else: command = self.data_command['command'] ########################################################################################### commands = [command] #stringhelpers.info_green(command) self.fang.execute_action_command(commands, blanks=2, error_reporting=True, timeout=30, terminal=False) #result_fang = self.fang.get_output() result_fang = self.fang.get_action_output( self.log_output_file_name) # processing parsing command follow output ########################################### command_type = self.data_command['type'] action_command_log = self.parsing(command_type, command_id, result_fang, commands[0]) return action_command_log ###################################################################################### except Exception as e: stringhelpers.err( "MEGA ACTION PROCESS EACH COMMAND ERROR %s | THREAD %s" % (e, self.name)) return None except ConnectionError as errConn: stringhelpers.err( "MEGA ACTION CONNECT API URL ERROR %s | THREAD %s" % (errConn, self.name)) return None def parsing(self, command_type=0, command_id=0, result_fang=None, command_text=None): final_result_output = [] output_result = dict() key = str(command_id) output_result[key] = dict() output_result[key]['output'] = [] try: if command_type == 3: # alway using for ironman if isinstance(self.data_command['output'], str): data_command_output = json.loads( self.data_command['output']) else: data_command_output = self.data_command['output'] for output_item in data_command_output: start_by = output_item['start_by'] end_by = output_item['end_by'] if start_by == '' and end_by == '': result = { 'value': '0', 'compare': True, 'command_type': str(command_type), 'command_id': str(command_id), 'command_text': command_text, 'console_log': result_fang } output_result[key]['output'].append(result) #output_result[key]['console_log'] = result_fang output_result[key]['final_output'] = True else: if end_by == 'end_row': end_by = '\r\n' _ret_value = stringhelpers.find_between( result_fang, start_by, end_by).strip() result = { 'value': _ret_value, 'compare': True, 'command_type': str(command_type), 'command_id': str(command_id), 'command_text': command_text, 'console_log': result_fang } output_result[key]['output'].append(result) #output_result[key]['console_log'] = result_fang output_result[key]['final_output'] = True return output_result elif command_type == 2 or command_type == 1: if isinstance(self.data_command['output'], str): data_command_output = json.loads( self.data_command['output']) else: data_command_output = self.data_command['output'] for output_item in data_command_output: #if output_item['start_by'] is not '' and output_item['end_by'] is not '': try: start_by = output_item['start_by'] end_by = output_item['end_by'] standard_value = output_item['standard_value'] compare = output_item['compare'] if end_by == 'end_row': end_by = '\r\n' compare_value = stringhelpers.find_between( result_fang, start_by, end_by) if compare_value is not None or compare_value is not "": compare_value = compare_value.strip() if compare_value is '' or compare_value is None: compare_value = result_fang #if compare_value is not None or compare_value is not '': if compare != "contains": compare_value = int(compare_value) standard_value = int(standard_value) retvalue_compare = func_compare( compare, standard_value, compare_value) if compare_value == '': result = { 'value': compare_value, 'compare': retvalue_compare, 'compare_operator': compare, 'command_type': str(command_type), 'command_id': str(command_id), 'command_text': command_text, 'console_log': result_fang } # if compare_value empty save raw data else: result = { 'value': compare_value, 'compare': retvalue_compare, 'compare_operator': compare, 'command_type': str(command_type), 'command_id': str(command_id), 'command_text': command_text, 'console_log': result_fang } output_result[key]['output'].append(result) # save final result of each output final_result_output.append(retvalue_compare) except Exception as _error: _strError = "MEGA ACTION PARSING COMMAND TYPE %d ERROR %s | THREAD %s" % ( command_type, _error, self.name) result = { 'value': compare_value, 'compare': retvalue_compare, 'error': _strError, 'command_type': command_type, 'command_id': str(command_id), 'command_text': command_text, 'console_log': result_fang } output_result[key]['output'].append(result) output_result[key]['parsing_status'] = 'ERROR' stringhelpers.err(_strError) final_result_output.append(False) # determine operator for final output try: final_operator = [] for x in self.data_command['final_output']: if x == '&' or x == '|': final_operator.append(x) else: pass # compare final output number_operator = 0 first_value = None for x in final_result_output: if len(final_operator) > 0: if number_operator == 0: first_value = x else: first_value = func_compare( final_operator[number_operator - 1], first_value, x) number_operator = number_operator + 1 if number_operator == len(final_result_output): output_result[key][ 'final_output'] = first_value else: output_result[key]['final_output'] = x except Exception as _errorFinal: if len(final_result_output) > 0: output_result[key][ 'final_output'] = final_result_output[0] _strError = "\nMEGA ACTION CALCULATOR FINAL_OUTPUT COMMAND_TYPE %d ERROR %s | THREAD %s" % ( command_type, _errorFinal, self.name) stringhelpers.err(_strError) return output_result except Exception as _errorException: output_result[key]['parsing_status'] = 'ERROR' _strError = "MEGA ACTION PARSING COMMAND TYPE %d ERROR %s | THREAD %s" % ( command_type, _errorException, self.name) stringhelpers.err(_strError) return output_result
def run(self): _request = RequestHelpers() dict_schedule = dict() dict_schedule_queue = dict() list_time = list() queue_discovery = queue.Queue() # run queue listining discovery ------------------------------------------------------------------------------ _ironQueue = IronQueue(queue_discovery, self.socketio, self.socketio_iron) _ironQueue.start() # ------------------------------------------------------------------------------------------------------------ while not self.is_stop: try: self.counter = self.counter + 1 arr_schedule_manage = [] # -------------- IRONMAN RUN SCHEDULE ---------------------------------------------------------------- #stringhelpers.print_bold("IRONMAN SCHEDULE RUN NUMBER: " + str(self.counter), "\n") # get current day name #weekday = datetime.now().strftime('%A') #_request.url = self.requestURL.IRONMAN_URL_GET_SCHEDULE % (weekday) _request.url = self.requestURL.IRONMAN_URL_GET_MOP_RUN_IRON _list_schedules = _request.get().json() if len(_list_schedules) > 0: for x in _list_schedules: key_mop = 'main_schedule_%d' % (int(x['mop_id'])) mop_id = int(x['mop_id']) mechanism = x['run_type'] run_time = x['run_datetime'].split("-")[1].strip() dict_schedule_queue[str(x['mop_id'])] = run_time if dict_schedule.get(key_mop, None) is not None: pass else: stringhelpers.info( "IRONMAN RUNNING MOP: " + str(x['mop_id']), "\n") list_time.append(run_time) _sub_mops = x.get('sub_mops', None) dict_schedule[key_mop] = key_mop schedule = Schedule( "SCHEDULE-%d" % (mop_id), x, _sub_mops, dict_schedule, False, mechanism, mop_id, queue_discovery, x['output_mapping'], self.socketio, self.socketio_iron) #arr_schedule_manage.append(schedule) #-----------------------process call api sockbot mop--------------------------------------- params_mops = dict() params_mops['name'] = x['name'] params_mops['app_secret_id'] = os.environ.get( 'SOCKBOT_APPCLIENT_SECRET') params_mops['mop_id'] = x['mop_id'] params_mops['status'] = 'RUNNING' params_mops['belong'] = 'IRONMAN' params_mops['command'] = 'STARTED' submops = [] for x_sub in _sub_mops: arr_devices = [] for k, v in x_sub['devices'].items(): arr_devices.append({ 'mop_id': x['mop_id'], 'device_id': v['device_id'], 'vendor': '%s|%s' % (v['vendor'], v['os']), 'port': v['port'] }) submops.append({ 'subNo': x_sub['subNo'], 'name': x_sub['name'], 'devices': arr_devices, 'num_devices': len(arr_devices), 'mop_id': x['mop_id'] }) params_mops['submops'] = submops self._sockbotAPIHelpers.url = self._sockbotAPIURL.SOCKBOT_API_CREATE_MOP self._sockbotAPIHelpers.params = params_mops self._sockbotAPIHelpers.post_json() schedule.start() #------------------------------------------------------------------------------------------ if len(arr_schedule_manage) > 0: #for schedule in arr_schedule_manage: # schedule.start() arr_schedule_manage.clear() except Exception as e: stringhelpers.print_bold("IRONMAN SCHEDULE [ERROR]: " + str(e), "\n") time.sleep(60) # stop iron queue ---------------------------------------------------------------------------------------------- _ironQueue.stop()