def _GetPluginInConfig(self): pluginCfgPath = util.ConvertToSDKFilePath(PLUGIN_CFG_FILE) if os.path.exists(pluginCfgPath): return self._LoadPlugInParams(pluginCfgPath) oldPluginCfgPath = util.ConvertToSDKFilePath(OLD_PLUGIN_CFG_FILE) if os.path.exists(oldPluginCfgPath): return self._LoadOldPlugInParams(oldPluginCfgPath) self.__logger.error( 'agentai cfg file {0} not exist.'.format(pluginCfgPath)) return False
def Init(self, agentEnv): self.__agentEnv = agentEnv try: ui_auto_cfg_path = util.ConvertToSDKFilePath(UI_AUTO_CFG) self.logger.info('read cfg path:%s' % ui_auto_cfg_path) uiAutoCfg = json.load(open(ui_auto_cfg_path, 'r')) except Exception as err: self.logger.error('cannot read {}, error: {}'.format( UI_AUTO_CFG, err)) return False # UI explore setting self.__maxClickNumber = uiAutoCfg['UiExplore'].get('MaxClickNumber') self.__waitTime = uiAutoCfg['UiExplore'].get('WaitTime') outputFolder = os.path.join(util.ConvertToSDKFilePath(''), uiAutoCfg['UiExplore'].get('OutputFolder')) if not os.path.exists(outputFolder): os.makedirs(outputFolder) self.__graphFolder = os.path.join(outputFolder, 'Graph') if not os.path.exists(self.__graphFolder): os.makedirs(self.__graphFolder) self.__actionFolder = os.path.join(outputFolder, 'Action') if not os.path.exists(self.__actionFolder): os.makedirs(self.__actionFolder) self.__explorePath = os.path.join(outputFolder, 'explore_path.txt') ret = self._InitDetector(uiAutoCfg) if ret is False: self.logger.error('initialize Detector failed') return False # UI coverage setting self.__computeCoverage = uiAutoCfg['UiCoverage'].get('ComputeCoverage') self.__sampleFolder = util.ConvertToSDKFilePath( uiAutoCfg['UiCoverage'].get('SampleFolder')) self.__orbThreshold = uiAutoCfg['UiCoverage'].get('ORBThreshold') # debug setting if uiAutoCfg.get('Debug', '') != '': self.__showButton = uiAutoCfg['Debug'].get('ShowButton') self.__showCostTime = uiAutoCfg['Debug'].get('ShowCostTime') self.__showImageRatio = uiAutoCfg['Debug'].get('ShowImageRatio') self.__testGetState = uiAutoCfg['Debug'].get('TestGetState') return True
def Init(self): """ Int function for env """ taskCfgFile = util.ConvertToSDKFilePath(TASK_CFG_FILE) taskReferCfgFile = util.ConvertToSDKFilePath(TASK_REFER_CFG_FILE) ret = self.__agentAPI.Initialize(taskCfgFile, referFile=taskReferCfgFile) if not ret: self.logger.error('Agent API Init Failed') return False ret = self.__agentAPI.SendCmd(AgentAPIMgr.MSG_SEND_GROUP_ID, REG_GROUP_ID) if not ret: self.logger.error('send message failed') return False return True
def Init(self): """ Load plugin config file, compatible with old style """ # 首先加载自研的算法,如果加载失败,则加载插件信息 if self.__load_ai_algorithm(): return True pluginCfgPath = util.ConvertToSDKFilePath(PLUGIN_CFG_FILE) if os.path.exists(pluginCfgPath): return self._LoadPlugInParams(pluginCfgPath) oldPluginCfgPath = util.ConvertToSDKFilePath(OLD_PLUGIN_CFG_FILE) if os.path.exists(oldPluginCfgPath): return self._LoadOldPlugInParams(oldPluginCfgPath) self.__logger.error('agentai cfg file {0} not exist.'.format(pluginCfgPath)) return False
def _LoadDQNPrams(self): learnArgs = {} learnCfgFile = util.ConvertToSDKFilePath(LEARN_CFG_FILE) if not os.path.exists(learnCfgFile): self.logger.error( 'DQN param file {} not exist.'.format(learnCfgFile)) return False, learnArgs try: config = configparser.ConfigParser() config.read(learnCfgFile) learnArgs['dueling_network'] = config.getboolean('DQN', 'DuelingNetwork', fallback=True) learnArgs['input_img_width'] = config.getint( 'DQN', 'InputImgWidth') learnArgs['input_img_height'] = config.getint( 'DQN', 'InputImgHeight') learnArgs['state_recent_frame'] = config.getint( 'DQN', 'StateRecentFrame') learnArgs['terminal_delay_frame'] = config.getint( 'DQN', 'TerminalDelayFrame') learnArgs['reward_discount'] = config.getfloat( 'DQN', 'RewardDiscount') learnArgs['learn_rate'] = config.getfloat('DQN', 'LearnRate') learnArgs['frame_per_action'] = config.getint( 'DQN', 'FramePerAction') learnArgs['observe_frame'] = config.getint('DQN', 'ObserveFrame') learnArgs['explore_frame'] = config.getint('DQN', 'ExploreFrame') learnArgs['initial_epsilon'] = config.getfloat( 'DQN', 'InitialEpsilon') learnArgs['final_epsilon'] = config.getfloat('DQN', 'FinalEpsilon') learnArgs['qnet_update_step'] = config.getint( 'DQN', 'QNetworkUpdateStep') learnArgs['memory_size'] = config.getint('DQN', 'MemorySize') learnArgs['show_img_state'] = config.getboolean( 'DQN', 'ShowImgState') learnArgs['mini_batch_size'] = config.getint( 'DQN', 'MiniBatchSize') learnArgs['train_with_double_q'] = config.getboolean( 'DQN', 'TrainWithDoubleQ') learnArgs['gpu_memory_fraction'] = config.getfloat( 'DQN', 'GPUMemoryFraction') learnArgs['gpu_memory_growth'] = config.getboolean( 'DQN', 'GPUMemoryGrowth') learnArgs['checkpoint_path'] = config.get('DQN', 'CheckPointPath') learnArgs['train_frame_rate'] = config.getint( 'DQN', 'TrainFrameRate') learnArgs['run_type'] = config.getint('DQN', 'RunType', fallback=1) except Exception as e: self.logger.error('Load file {} failed, error: {}.'.format( learnCfgFile, e)) return False, learnArgs return True, learnArgs
def Init(self): taskCfgFile = util.ConvertToSDKFilePath(TASK_CFG_FILE) referCfgFile = util.ConvertToSDKFilePath(REFER_CFG_FILE) ret = self.__agentAPI.Initialize(taskCfgFile, referCfgFile) if not ret: self.logger.error('agent API init failed') return False ret = self.__agentAPI.SendCmd(AgentAPIMgr.MSG_SEND_GROUP_ID, 1) if not ret: self.logger.error('send message failed') return False ret = self.__actionAPI.Initialize() if not ret: self.logger.error('action API init failed') return False return True
def __load_ai_algorithm(self): algorithm_config_path = util.ConvertToSDKFilePath(ALGORITHM_CONFIG_FILE) self.__logger.info("begin to load the ai algorithm configure, path:{}".format(algorithm_config_path)) # 获取算法配置 config = util.get_configure(algorithm_config_path) if config is None: self.__logger.warning("load the algorithm failed, please check other plugin file") return False alg_type = config['type'] model_parameter = self.__model_parameter_dict[alg_type] if model_parameter is None: self.__logger.warning("get the model parameter failed, alg_type:{}".format(alg_type)) return False if model_parameter.use_plugin_env == 0: self.__usePluginEnv = False else: self.__usePluginEnv = True if model_parameter.use_plugin_model == 0: self.__usePluginAIModel = False else: self.__usePluginAIModel = True if model_parameter.use_default_run_func == 0: self.__useDefaultRunFunc = False else: self.__useDefaultRunFunc = True # if self.__usePluginEnv is True: self.__envPackage = model_parameter.env_package self.__envModule = model_parameter.env_module self.__envClass = model_parameter.env_class # if self.__usePluginAIModel is True: self.__aiModelPackage = model_parameter.model_package self.__aiModelModule = model_parameter.model_module self.__aiModelClass = model_parameter.model_class self.__logger.info("get algorithm success, usePluginEnv:{}, usePluginAIModel:{}, useDefaultRunFunc:{}, " "env_package:{}, env_model:{}, env_class:{}, " "model_package:{}, model_module:{}, model_class:{}" .format(self.__usePluginEnv, self.__usePluginAIModel, self.__useDefaultRunFunc, self.__envPackage, self.__envModule, self.__envClass, self.__aiModelPackage, self.__aiModelModule, self.__aiModelClass)) # if self.__useDefaultRunFunc is not True: # self.__runFuncPackage = "" # self.__runFuncModule = "" # self.__runFuncName = "" return True
def _GetImageShape(self, imgFile): if not imgFile: return None image_path = util.ConvertToSDKFilePath(imgFile) if not os.path.exists(image_path): return None img = cv2.imread(image_path) if img is None: self.__logger.error('failed to read image(%s)' % image_path) return None return img.shape
def __init__(self): GameEnv.__init__(self) # convert action cfg path and create action controller object self.__actionCfgPath = util.ConvertToSDKFilePath(ACTION_CFG_FILE) self.__actionController = ActionController.ActionController() # convert env cfg path and load env parameters self.__envCfgPath = util.ConvertToSDKFilePath(ENV_CFG_FILE) self._load_env_params(self.__envCfgPath) # convert task cfg path and create agent API object self.__taskCfgPath = util.ConvertToSDKFilePath(TASK_CFG_FILE) self.__agentAPI = AgentAPIMgr.AgentAPIMgr() self.__gameState = GAME_STATE_INVALID self.__frameIndex = 0 self.__episodeCount = 0 self.__episodeReward = 0 self.Reset()
def _ProcArgs(self, learnArgs): learnArgs['action_space'] = self.actionSpace learnArgs['checkpoint_path'] = util.ConvertToSDKFilePath( learnArgs['checkpoint_path']) runType = learnArgs['run_type'] if runType == 0: self.testAgent = False elif runType == 1: self.testAgent = True learnArgs['memory_size'] = 200 learnArgs['initial_epsilon'] = 0.001 else: pass
def _LoadGameState(self): imEnvFile = util.ConvertToSDKFilePath(IM_ENV_CFG_FILE) try: with open(imEnvFile, 'r', encoding='utf-8') as file: jsonStr = file.read() gameStateCfg = json.loads(jsonStr) self.logger.info( "the config of env is {}".format(gameStateCfg)) self.__beginTaskID.extend(gameStateCfg['beginTaskID']) self.__overTaskID.extend(gameStateCfg['overTaskID']) except Exception as err: self.logger.error('Load game state file %s error! Error msg: %s', imEnvFile, str(err)) return False return True
def Init(self, agentEnv): self.logger.info("enter the ProxyAI init") self.__env = agentEnv self.__envCfgPath = util.ConvertToSDKFilePath(LEARNING_CFG_FILE) if not self._GetServerAddr(): self.logger.error('GetServerAddr failed!') return False # 连接server self.__proxySocket.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) ret = self.__proxySocket.connect_ex((self.__address, self.__port)) if ret != 0: self.logger.error('connect server failed, error = {}'.format(ret)) return False self.logger.info("leave the ProxyAI init") return True
def _LoadGraphPath(self, graphPathFile): pathCfgFile = util.ConvertToSDKFilePath(graphPathFile) if not os.path.exists(pathCfgFile): self.__logger.error( 'Graph path cfg file {} not exists!'.format(pathCfgFile)) return False try: with open(pathCfgFile, 'rb') as file: jsonstr = file.read() pathCfg = json.loads(str(jsonstr, encoding='utf-8')) self.__pointDict = self._LoadPints(pathCfg['points']) self.__pathDict = self._LoadPaths(pathCfg['paths']) except Exception as e: self.__logger.error( 'Load graph path file {} failed, error: {}.'.format( pathCfgFile, e)) return False return True
def _LoadPath(self, pathFile): cfgFile = util.ConvertToSDKFilePath(pathFile) if not os.path.exists(cfgFile): self.__logger.error( 'Path config file {} not exists!'.format(cfgFile)) return False try: with open(cfgFile, 'rb') as file: jsonstr = file.read() pathCfg = json.loads(str(jsonstr, encoding='utf-8')) self.__mapScene = pathCfg.get('mapScene') self.__pathPoints = pathCfg.get( self.__mapScene).get('Path').get('line0') except Exception as e: self.__logger.error('Load file {} failed, error: {}.'.format( cfgFile, e)) return False return True
def _LoadCfgFilePath(self): self.__actionCfgFile = util.ConvertToSDKFilePath(ACTION_CFG_FILE) self.__envCfgFile = util.ConvertToSDKFilePath(ENV_CFG_FILE) self.__recognizeCfgFile = util.ConvertToSDKFilePath(TASK_CFG_FILE)
def _LoadAIParams(self): """ Load AI parameters for training and test """ self.data['imageSize'] = 224 self.data['actionAheadNum'] = 1 self.data['classImageTimes'] = 2 imitationCfgFile = util.ConvertToSDKFilePath(IMITATION_CFG_FILE) if not os.path.exists(imitationCfgFile): self.__logger.error('Imitation cfg {} is not existed.'.format(imitationCfgFile)) return False try: with open(imitationCfgFile, 'r', encoding='utf-8') as file: jsonStr = file.read() aiCfg = json.loads(jsonStr) networkCfg = aiCfg.get('network') if networkCfg is None: self.__logger.error('No network params in imitation cfg!') return False # training data params self.data['trainDataDir'] = util.ConvertToSDKFilePath(networkCfg.get('dataDir')) self.data['testDataDir'] = util.ConvertToSDKFilePath(networkCfg.get('dataDir')) self.data['validDataRatio'] = networkCfg.get('validDataRatio') # Whether use small network self.data['isSmallNet'] = networkCfg.get('isSmallNet') # Whether choose action with max score during test period self.data['isMax'] = networkCfg.get('isMax') # Random ratio for choosing action if "isMax" is set as 0 self.data['randomRatio'] = networkCfg.get('randomRatio') # Frame number of ahead action considering action delay self.data['actionAheadNum'] = networkCfg.get('actionAheadNum') # Times for each class self.data['classImageTimes'] = networkCfg.get('classImageTimes') # Input height of image self.data['inputHeight'] = networkCfg.get('inputHeight') # Input width of image self.data['inputWidth'] = networkCfg.get('inputWidth') # whether use lstm self.data['useLstm'] = networkCfg.get('useLstm') # train Iter self.data['trainIter'] = networkCfg.get('trainIter') # timeStep is the frame length for LSTM (memory) self.data['timeStep'] = networkCfg.get('timeStep') # Whether use class balance self.data['useClassBalance'] = networkCfg.get('useClassBalance', True) # Whether use resNet self.data['useResNet'] = networkCfg.get('useResNet', True) actionCfg = aiCfg.get('action') if actionCfg is None: self.__logger.error('No action params in imitation cfg!') return False # action number for one second self.data['actionPerSecond'] = actionCfg.get('actionPerSecond') # action click time self.data['actionTimeMs'] = actionCfg.get('actionTimeMs') roiRegionCfg = aiCfg.get('roiRegion') if roiRegionCfg is None: self.__logger.error('No roiRegion params in imitation cfg!') return False # Input region for network imgShape = self._GetImageShape(roiRegionCfg['path']) if imgShape is None: self.__logger.error('Get image(%s) shape error!' % roiRegionCfg['path']) return False heightRatio = self.data['inputHeight']/imgShape[0] widthRatio = self.data['inputWidth']/imgShape[1] x = int(roiRegionCfg['region']['x'] * widthRatio) y = int(roiRegionCfg['region']['y'] * heightRatio) w = int(roiRegionCfg['region']['w'] * widthRatio) h = int(roiRegionCfg['region']['h'] * heightRatio) self.data['roiRegion'] = [x, y, w, h] if self.data['isSmallNet']: self.data['imageSize'] = 50 else: self.data['imageSize'] = 150 self.__logger.info('The imitation cfg is parsed.') except Exception as err: traceback.print_exc() self.__logger.error('Load imitation cfg {} error! Error msg: {}'.format(imitationCfgFile, err)) return False return True
def _LoadActionCfg(self): actionCfgFile = util.ConvertToSDKFilePath(ACTION_CFG_FILE) if not os.path.exists(actionCfgFile): self.__logger.error('Action cfg {} not existed.'.format(actionCfgFile)) return False try: # 加载action文件信息, with open(actionCfgFile, 'r', encoding='utf-8') as file: jsonStr = file.read() actionCfg = json.loads(jsonStr) # read action define # Set default task id for action define self.data['actionDefine'] = actionCfg['aiAction'] for n in range(len(self.data['actionDefine'])): if "task" not in self.data['actionDefine'][n].keys(): self.data['actionDefine'][n]["task"] = [0] # 获取gameAction for actionContext in actionCfg['gameAction']: actionType = actionContext.get('type') actionId = actionContext.get('id') if actionType != 'none': imageShape = self._GetImageShape(actionContext['actionRegion']['path']) imageHeight = imageShape[0] self.__ratio = self.data['inputHeight']/imageHeight if actionType == 'none': pass elif actionType == 'down' or actionType == 'up' or actionType == 'click': region = actionContext['actionRegion']['region'] buttonX = region['x'] + region['w']/2 buttonY = region['y'] + region['h']/2 actionContext['buttonX'] = int(buttonX * self.__ratio) actionContext['buttonY'] = int(buttonY * self.__ratio) actionContext['updateBtn'] = False actionContext['updateBtnX'] = -1 actionContext['updateBtnY'] = -1 elif actionType == 'key': region = actionContext['actionRegion']['region'] buttonX = region['x'] + region['w'] / 2 buttonY = region['y'] + region['h'] / 2 actionContext['buttonX'] = int(buttonX * self.__ratio) actionContext['buttonY'] = int(buttonY * self.__ratio) actionContext['updateBtn'] = False actionContext['updateBtnX'] = -1 actionContext['updateBtnY'] = -1 actionContext['alphabet'] = actionContext['actionRegion']['alphabet'] actionContext['action_type'] = actionContext['actionRegion']['actionType'] actionContext['action_text'] = actionContext['actionRegion']['text'] elif actionType == 'swipe': #regionX1, regionY1, regionX2, regionY2 = self._GetRegionSwipe(actionContext) startPoint = actionContext['actionRegion']['startPoint'] endPoint = actionContext['actionRegion']['endPoint'] actionContext['swipeStartX'] = int(startPoint['x'] * self.__ratio) actionContext['swipeStartY'] = int(startPoint['y'] * self.__ratio) actionContext['swipeEndX'] = int(endPoint['x'] * self.__ratio) actionContext['swipeEndY'] = int(endPoint['y'] * self.__ratio) elif actionType == 'joystick': center = actionContext['actionRegion']['center'] inner = actionContext['actionRegion']['inner'] outer = actionContext['actionRegion']['outer'] actionContext['centerx'] = int(center['x'] * self.__ratio) actionContext['centery'] = int(center['y'] * self.__ratio) actionContext['rangeInner'] = int((inner['w'] + inner['h']) * 0.25 * self.__ratio) actionContext['rangeOuter'] = int((outer['w'] + outer['h']) * 0.25 * self.__ratio) self._GetContextJoystick(actionContext, actionId) continue else: self.__logger.error('Error action type') self.actionsContextDict[actionId] = actionContext self.__logger.info("the actionsContextDict is {}".format(self.actionsContextDict)) self.data['actionsContextDict'] = self.actionsContextDict except Exception as err: self.__logger.error('Load action cfg {} error! Error msg: {}'.format(actionCfgFile, err)) return False return True
def _InitDetector(self, uiAutoCfg): self.__modelType = uiAutoCfg['ButtonDetection'].get('ModelType') if self.__modelType != 'Yolov3' and self.__modelType != 'RefineNet': self.logger.error('unknown model type: {}'.format( self.__modelType)) return False sys.path.append(pluginPath) if self.__modelType == 'Yolov3': if _is_windows_system: from .windows.Yolov3.OpencvYolov3 import OpencvYolov3 as Yolov3 else: from .ubuntu.Yolov3.OpencvYolov3 import OpencvYolov3 as Yolov3 param = dict() param['cfgPath'] = util.ConvertToSDKFilePath( uiAutoCfg['ButtonDetection'].get('CfgPath')) param['weightsPath'] = util.ConvertToSDKFilePath( uiAutoCfg['ButtonDetection'].get('WeightsPath')) param['namesPath'] = util.ConvertToSDKFilePath( uiAutoCfg['ButtonDetection'].get('NamesPath')) param['threshold'] = uiAutoCfg['ButtonDetection'].get('Threshold') self.__detector = Yolov3() ret = self.__detector.Init(param) if ret is False: self.logger.error('initialize Yolov3 failed') return False if self.__modelType == 'RefineNet': if _is_windows_system: from .windows.DetectRefineNet import DetectRefineNet else: from .ubuntu.DetectRefineNet import DetectRefineNet pthModelPath = util.ConvertToSDKFilePath( uiAutoCfg['ButtonDetection'].get('PthModelPath')) threshold = uiAutoCfg['ButtonDetection'].get('Threshold') labels = ('__background__', 'return', 'close', 'tag', 'other') self.__detector = DetectRefineNet.DetectRefineNet( labels, img_dim=320, num_classes=5, obj_thresh=threshold, nms_thresh=0.10, version='Refine_hc2net_version3', onnx_model='', trained_model=pthModelPath, nmsType='normal', platform='pytorch') if not self.__detector: self.logger.error('initialize RefineNet failed') return False if uiAutoCfg['ButtonDetection'].get('MaskPath', '') != '': maskPath = util.ConvertToSDKFilePath( uiAutoCfg['ButtonDetection'].get('MaskPath')) maskImage = cv2.imread(maskPath) if not maskImage: self.logger.error( 'cannot read mask image in {}'.format(maskPath)) return False grayImage = cv2.cvtColor(maskImage, cv2.COLOR_BGR2GRAY) _, self.__mask = cv2.threshold(grayImage, 127, 255, 0) else: self.__mask = None return True