def _GetPluginInConfig(self):
pluginCfgPath = util.ConvertToSDKFilePath(PLUGIN_CFG_FILE)
if os.path.exists(pluginCfgPath):
return self._LoadPlugInParams(pluginCfgPath)
oldPluginCfgPath = util.ConvertToSDKFilePath(OLD_PLUGIN_CFG_FILE)
if os.path.exists(oldPluginCfgPath):
return self._LoadOldPlugInParams(oldPluginCfgPath)
self.__logger.error(
'agentai cfg file {0} not exist.'.format(pluginCfgPath))
return False
def Init(self, agentEnv):
self.__agentEnv = agentEnv
try:
ui_auto_cfg_path = util.ConvertToSDKFilePath(UI_AUTO_CFG)
self.logger.info('read cfg path:%s' % ui_auto_cfg_path)
uiAutoCfg = json.load(open(ui_auto_cfg_path, 'r'))
except Exception as err:
self.logger.error('cannot read {}, error: {}'.format(
UI_AUTO_CFG, err))
return False
# UI explore setting
self.__maxClickNumber = uiAutoCfg['UiExplore'].get('MaxClickNumber')
self.__waitTime = uiAutoCfg['UiExplore'].get('WaitTime')
outputFolder = os.path.join(util.ConvertToSDKFilePath(''),
uiAutoCfg['UiExplore'].get('OutputFolder'))
if not os.path.exists(outputFolder):
os.makedirs(outputFolder)
self.__graphFolder = os.path.join(outputFolder, 'Graph')
if not os.path.exists(self.__graphFolder):
os.makedirs(self.__graphFolder)
self.__actionFolder = os.path.join(outputFolder, 'Action')
if not os.path.exists(self.__actionFolder):
os.makedirs(self.__actionFolder)
self.__explorePath = os.path.join(outputFolder, 'explore_path.txt')
ret = self._InitDetector(uiAutoCfg)
if ret is False:
self.logger.error('initialize Detector failed')
return False
# UI coverage setting
self.__computeCoverage = uiAutoCfg['UiCoverage'].get('ComputeCoverage')
self.__sampleFolder = util.ConvertToSDKFilePath(
uiAutoCfg['UiCoverage'].get('SampleFolder'))
self.__orbThreshold = uiAutoCfg['UiCoverage'].get('ORBThreshold')
# debug setting
if uiAutoCfg.get('Debug', '') != '':
self.__showButton = uiAutoCfg['Debug'].get('ShowButton')
self.__showCostTime = uiAutoCfg['Debug'].get('ShowCostTime')
self.__showImageRatio = uiAutoCfg['Debug'].get('ShowImageRatio')
self.__testGetState = uiAutoCfg['Debug'].get('TestGetState')
return True
def Init(self):
"""
Int function for env
"""
taskCfgFile = util.ConvertToSDKFilePath(TASK_CFG_FILE)
taskReferCfgFile = util.ConvertToSDKFilePath(TASK_REFER_CFG_FILE)
ret = self.__agentAPI.Initialize(taskCfgFile, referFile=taskReferCfgFile)
if not ret:
self.logger.error('Agent API Init Failed')
return False
ret = self.__agentAPI.SendCmd(AgentAPIMgr.MSG_SEND_GROUP_ID, REG_GROUP_ID)
if not ret:
self.logger.error('send message failed')
return False
return True
def Init(self):
"""
Load plugin config file, compatible with old style
"""
# 首先加载自研的算法，如果加载失败，则加载插件信息
if self.__load_ai_algorithm():
return True
pluginCfgPath = util.ConvertToSDKFilePath(PLUGIN_CFG_FILE)
if os.path.exists(pluginCfgPath):
return self._LoadPlugInParams(pluginCfgPath)
oldPluginCfgPath = util.ConvertToSDKFilePath(OLD_PLUGIN_CFG_FILE)
if os.path.exists(oldPluginCfgPath):
return self._LoadOldPlugInParams(oldPluginCfgPath)
self.__logger.error('agentai cfg file {0} not exist.'.format(pluginCfgPath))
return False
def _LoadDQNPrams(self):
learnArgs = {}
learnCfgFile = util.ConvertToSDKFilePath(LEARN_CFG_FILE)
if not os.path.exists(learnCfgFile):
self.logger.error(
'DQN param file {} not exist.'.format(learnCfgFile))
return False, learnArgs
try:
config = configparser.ConfigParser()
config.read(learnCfgFile)
learnArgs['dueling_network'] = config.getboolean('DQN',
'DuelingNetwork',
fallback=True)
learnArgs['input_img_width'] = config.getint(
'DQN', 'InputImgWidth')
learnArgs['input_img_height'] = config.getint(
'DQN', 'InputImgHeight')
learnArgs['state_recent_frame'] = config.getint(
'DQN', 'StateRecentFrame')
learnArgs['terminal_delay_frame'] = config.getint(
'DQN', 'TerminalDelayFrame')
learnArgs['reward_discount'] = config.getfloat(
'DQN', 'RewardDiscount')
learnArgs['learn_rate'] = config.getfloat('DQN', 'LearnRate')
learnArgs['frame_per_action'] = config.getint(
'DQN', 'FramePerAction')
learnArgs['observe_frame'] = config.getint('DQN', 'ObserveFrame')
learnArgs['explore_frame'] = config.getint('DQN', 'ExploreFrame')
learnArgs['initial_epsilon'] = config.getfloat(
'DQN', 'InitialEpsilon')
learnArgs['final_epsilon'] = config.getfloat('DQN', 'FinalEpsilon')
learnArgs['qnet_update_step'] = config.getint(
'DQN', 'QNetworkUpdateStep')
learnArgs['memory_size'] = config.getint('DQN', 'MemorySize')
learnArgs['show_img_state'] = config.getboolean(
'DQN', 'ShowImgState')
learnArgs['mini_batch_size'] = config.getint(
'DQN', 'MiniBatchSize')
learnArgs['train_with_double_q'] = config.getboolean(
'DQN', 'TrainWithDoubleQ')
learnArgs['gpu_memory_fraction'] = config.getfloat(
'DQN', 'GPUMemoryFraction')
learnArgs['gpu_memory_growth'] = config.getboolean(
'DQN', 'GPUMemoryGrowth')
learnArgs['checkpoint_path'] = config.get('DQN', 'CheckPointPath')
learnArgs['train_frame_rate'] = config.getint(
'DQN', 'TrainFrameRate')
learnArgs['run_type'] = config.getint('DQN', 'RunType', fallback=1)
except Exception as e:
self.logger.error('Load file {} failed, error: {}.'.format(
learnCfgFile, e))
return False, learnArgs
return True, learnArgs
def Init(self):
taskCfgFile = util.ConvertToSDKFilePath(TASK_CFG_FILE)
referCfgFile = util.ConvertToSDKFilePath(REFER_CFG_FILE)
ret = self.__agentAPI.Initialize(taskCfgFile, referCfgFile)
if not ret:
self.logger.error('agent API init failed')
return False
ret = self.__agentAPI.SendCmd(AgentAPIMgr.MSG_SEND_GROUP_ID, 1)
if not ret:
self.logger.error('send message failed')
return False
ret = self.__actionAPI.Initialize()
if not ret:
self.logger.error('action API init failed')
return False
return True
def __load_ai_algorithm(self):
algorithm_config_path = util.ConvertToSDKFilePath(ALGORITHM_CONFIG_FILE)
self.__logger.info("begin to load the ai algorithm configure, path:{}".format(algorithm_config_path))
# 获取算法配置
config = util.get_configure(algorithm_config_path)
if config is None:
self.__logger.warning("load the algorithm failed, please check other plugin file")
return False
alg_type = config['type']
model_parameter = self.__model_parameter_dict[alg_type]
if model_parameter is None:
self.__logger.warning("get the model parameter failed, alg_type:{}".format(alg_type))
return False
if model_parameter.use_plugin_env == 0:
self.__usePluginEnv = False
else:
self.__usePluginEnv = True
if model_parameter.use_plugin_model == 0:
self.__usePluginAIModel = False
else:
self.__usePluginAIModel = True
if model_parameter.use_default_run_func == 0:
self.__useDefaultRunFunc = False
else:
self.__useDefaultRunFunc = True
# if self.__usePluginEnv is True:
self.__envPackage = model_parameter.env_package
self.__envModule = model_parameter.env_module
self.__envClass = model_parameter.env_class
# if self.__usePluginAIModel is True:
self.__aiModelPackage = model_parameter.model_package
self.__aiModelModule = model_parameter.model_module
self.__aiModelClass = model_parameter.model_class
self.__logger.info("get algorithm success, usePluginEnv:{}, usePluginAIModel:{}, useDefaultRunFunc:{}, "
"env_package:{}, env_model:{}, env_class:{}, "
"model_package:{}, model_module:{}, model_class:{}"
.format(self.__usePluginEnv, self.__usePluginAIModel, self.__useDefaultRunFunc,
self.__envPackage, self.__envModule, self.__envClass,
self.__aiModelPackage, self.__aiModelModule, self.__aiModelClass))
# if self.__useDefaultRunFunc is not True:
# self.__runFuncPackage = ""
# self.__runFuncModule = ""
# self.__runFuncName = ""
return True
def _GetImageShape(self, imgFile):
if not imgFile:
return None
image_path = util.ConvertToSDKFilePath(imgFile)
if not os.path.exists(image_path):
return None
img = cv2.imread(image_path)
if img is None:
self.__logger.error('failed to read image(%s)' % image_path)
return None
return img.shape
def __init__(self):
GameEnv.__init__(self)
# convert action cfg path and create action controller object
self.__actionCfgPath = util.ConvertToSDKFilePath(ACTION_CFG_FILE)
self.__actionController = ActionController.ActionController()
# convert env cfg path and load env parameters
self.__envCfgPath = util.ConvertToSDKFilePath(ENV_CFG_FILE)
self._load_env_params(self.__envCfgPath)
# convert task cfg path and create agent API object
self.__taskCfgPath = util.ConvertToSDKFilePath(TASK_CFG_FILE)
self.__agentAPI = AgentAPIMgr.AgentAPIMgr()
self.__gameState = GAME_STATE_INVALID
self.__frameIndex = 0
self.__episodeCount = 0
self.__episodeReward = 0
self.Reset()
def _ProcArgs(self, learnArgs):
learnArgs['action_space'] = self.actionSpace
learnArgs['checkpoint_path'] = util.ConvertToSDKFilePath(
learnArgs['checkpoint_path'])
runType = learnArgs['run_type']
if runType == 0:
self.testAgent = False
elif runType == 1:
self.testAgent = True
learnArgs['memory_size'] = 200
learnArgs['initial_epsilon'] = 0.001
else:
pass
def _LoadGameState(self):
imEnvFile = util.ConvertToSDKFilePath(IM_ENV_CFG_FILE)
try:
with open(imEnvFile, 'r', encoding='utf-8') as file:
jsonStr = file.read()
gameStateCfg = json.loads(jsonStr)
self.logger.info(
"the config of env is {}".format(gameStateCfg))
self.__beginTaskID.extend(gameStateCfg['beginTaskID'])
self.__overTaskID.extend(gameStateCfg['overTaskID'])
except Exception as err:
self.logger.error('Load game state file %s error! Error msg: %s',
imEnvFile, str(err))
return False
return True
def Init(self, agentEnv):
self.logger.info("enter the ProxyAI init")
self.__env = agentEnv
self.__envCfgPath = util.ConvertToSDKFilePath(LEARNING_CFG_FILE)
if not self._GetServerAddr():
self.logger.error('GetServerAddr failed!')
return False
# 连接server
self.__proxySocket.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1)
ret = self.__proxySocket.connect_ex((self.__address, self.__port))
if ret != 0:
self.logger.error('connect server failed, error = {}'.format(ret))
return False
self.logger.info("leave the ProxyAI init")
return True
def _LoadGraphPath(self, graphPathFile):
pathCfgFile = util.ConvertToSDKFilePath(graphPathFile)
if not os.path.exists(pathCfgFile):
self.__logger.error(
'Graph path cfg file {} not exists!'.format(pathCfgFile))
return False
try:
with open(pathCfgFile, 'rb') as file:
jsonstr = file.read()
pathCfg = json.loads(str(jsonstr, encoding='utf-8'))
self.__pointDict = self._LoadPints(pathCfg['points'])
self.__pathDict = self._LoadPaths(pathCfg['paths'])
except Exception as e:
self.__logger.error(
'Load graph path file {} failed, error: {}.'.format(
pathCfgFile, e))
return False
return True
def _LoadPath(self, pathFile):
cfgFile = util.ConvertToSDKFilePath(pathFile)
if not os.path.exists(cfgFile):
self.__logger.error(
'Path config file {} not exists!'.format(cfgFile))
return False
try:
with open(cfgFile, 'rb') as file:
jsonstr = file.read()
pathCfg = json.loads(str(jsonstr, encoding='utf-8'))
self.__mapScene = pathCfg.get('mapScene')
self.__pathPoints = pathCfg.get(
self.__mapScene).get('Path').get('line0')
except Exception as e:
self.__logger.error('Load file {} failed, error: {}.'.format(
cfgFile, e))
return False
return True
def _LoadCfgFilePath(self):
self.__actionCfgFile = util.ConvertToSDKFilePath(ACTION_CFG_FILE)
self.__envCfgFile = util.ConvertToSDKFilePath(ENV_CFG_FILE)
self.__recognizeCfgFile = util.ConvertToSDKFilePath(TASK_CFG_FILE)
def _LoadAIParams(self):
"""
Load AI parameters for training and test
"""
self.data['imageSize'] = 224
self.data['actionAheadNum'] = 1
self.data['classImageTimes'] = 2
imitationCfgFile = util.ConvertToSDKFilePath(IMITATION_CFG_FILE)
if not os.path.exists(imitationCfgFile):
self.__logger.error('Imitation cfg {} is not existed.'.format(imitationCfgFile))
return False
try:
with open(imitationCfgFile, 'r', encoding='utf-8') as file:
jsonStr = file.read()
aiCfg = json.loads(jsonStr)
networkCfg = aiCfg.get('network')
if networkCfg is None:
self.__logger.error('No network params in imitation cfg!')
return False
# training data params
self.data['trainDataDir'] = util.ConvertToSDKFilePath(networkCfg.get('dataDir'))
self.data['testDataDir'] = util.ConvertToSDKFilePath(networkCfg.get('dataDir'))
self.data['validDataRatio'] = networkCfg.get('validDataRatio')
# Whether use small network
self.data['isSmallNet'] = networkCfg.get('isSmallNet')
# Whether choose action with max score during test period
self.data['isMax'] = networkCfg.get('isMax')
# Random ratio for choosing action if "isMax" is set as 0
self.data['randomRatio'] = networkCfg.get('randomRatio')
# Frame number of ahead action considering action delay
self.data['actionAheadNum'] = networkCfg.get('actionAheadNum')
# Times for each class
self.data['classImageTimes'] = networkCfg.get('classImageTimes')
# Input height of image
self.data['inputHeight'] = networkCfg.get('inputHeight')
# Input width of image
self.data['inputWidth'] = networkCfg.get('inputWidth')
# whether use lstm
self.data['useLstm'] = networkCfg.get('useLstm')
# train Iter
self.data['trainIter'] = networkCfg.get('trainIter')
# timeStep is the frame length for LSTM (memory)
self.data['timeStep'] = networkCfg.get('timeStep')
# Whether use class balance
self.data['useClassBalance'] = networkCfg.get('useClassBalance', True)
# Whether use resNet
self.data['useResNet'] = networkCfg.get('useResNet', True)
actionCfg = aiCfg.get('action')
if actionCfg is None:
self.__logger.error('No action params in imitation cfg!')
return False
# action number for one second
self.data['actionPerSecond'] = actionCfg.get('actionPerSecond')
# action click time
self.data['actionTimeMs'] = actionCfg.get('actionTimeMs')
roiRegionCfg = aiCfg.get('roiRegion')
if roiRegionCfg is None:
self.__logger.error('No roiRegion params in imitation cfg!')
return False
# Input region for network
imgShape = self._GetImageShape(roiRegionCfg['path'])
if imgShape is None:
self.__logger.error('Get image(%s) shape error!' % roiRegionCfg['path'])
return False
heightRatio = self.data['inputHeight']/imgShape[0]
widthRatio = self.data['inputWidth']/imgShape[1]
x = int(roiRegionCfg['region']['x'] * widthRatio)
y = int(roiRegionCfg['region']['y'] * heightRatio)
w = int(roiRegionCfg['region']['w'] * widthRatio)
h = int(roiRegionCfg['region']['h'] * heightRatio)
self.data['roiRegion'] = [x, y, w, h]
if self.data['isSmallNet']:
self.data['imageSize'] = 50
else:
self.data['imageSize'] = 150
self.__logger.info('The imitation cfg is parsed.')
except Exception as err:
traceback.print_exc()
self.__logger.error('Load imitation cfg {} error! Error msg: {}'.format(imitationCfgFile, err))
return False
return True
def _LoadActionCfg(self):
actionCfgFile = util.ConvertToSDKFilePath(ACTION_CFG_FILE)
if not os.path.exists(actionCfgFile):
self.__logger.error('Action cfg {} not existed.'.format(actionCfgFile))
return False
try:
# 加载action文件信息,
with open(actionCfgFile, 'r', encoding='utf-8') as file:
jsonStr = file.read()
actionCfg = json.loads(jsonStr)
# read action define
# Set default task id for action define
self.data['actionDefine'] = actionCfg['aiAction']
for n in range(len(self.data['actionDefine'])):
if "task" not in self.data['actionDefine'][n].keys():
self.data['actionDefine'][n]["task"] = [0]
# 获取gameAction
for actionContext in actionCfg['gameAction']:
actionType = actionContext.get('type')
actionId = actionContext.get('id')
if actionType != 'none':
imageShape = self._GetImageShape(actionContext['actionRegion']['path'])
imageHeight = imageShape[0]
self.__ratio = self.data['inputHeight']/imageHeight
if actionType == 'none':
pass
elif actionType == 'down' or actionType == 'up' or actionType == 'click':
region = actionContext['actionRegion']['region']
buttonX = region['x'] + region['w']/2
buttonY = region['y'] + region['h']/2
actionContext['buttonX'] = int(buttonX * self.__ratio)
actionContext['buttonY'] = int(buttonY * self.__ratio)
actionContext['updateBtn'] = False
actionContext['updateBtnX'] = -1
actionContext['updateBtnY'] = -1
elif actionType == 'key':
region = actionContext['actionRegion']['region']
buttonX = region['x'] + region['w'] / 2
buttonY = region['y'] + region['h'] / 2
actionContext['buttonX'] = int(buttonX * self.__ratio)
actionContext['buttonY'] = int(buttonY * self.__ratio)
actionContext['updateBtn'] = False
actionContext['updateBtnX'] = -1
actionContext['updateBtnY'] = -1
actionContext['alphabet'] = actionContext['actionRegion']['alphabet']
actionContext['action_type'] = actionContext['actionRegion']['actionType']
actionContext['action_text'] = actionContext['actionRegion']['text']
elif actionType == 'swipe':
#regionX1, regionY1, regionX2, regionY2 = self._GetRegionSwipe(actionContext)
startPoint = actionContext['actionRegion']['startPoint']
endPoint = actionContext['actionRegion']['endPoint']
actionContext['swipeStartX'] = int(startPoint['x'] * self.__ratio)
actionContext['swipeStartY'] = int(startPoint['y'] * self.__ratio)
actionContext['swipeEndX'] = int(endPoint['x'] * self.__ratio)
actionContext['swipeEndY'] = int(endPoint['y'] * self.__ratio)
elif actionType == 'joystick':
center = actionContext['actionRegion']['center']
inner = actionContext['actionRegion']['inner']
outer = actionContext['actionRegion']['outer']
actionContext['centerx'] = int(center['x'] * self.__ratio)
actionContext['centery'] = int(center['y'] * self.__ratio)
actionContext['rangeInner'] = int((inner['w'] + inner['h']) * 0.25 * self.__ratio)
actionContext['rangeOuter'] = int((outer['w'] + outer['h']) * 0.25 * self.__ratio)
self._GetContextJoystick(actionContext, actionId)
continue
else:
self.__logger.error('Error action type')
self.actionsContextDict[actionId] = actionContext
self.__logger.info("the actionsContextDict is {}".format(self.actionsContextDict))
self.data['actionsContextDict'] = self.actionsContextDict
except Exception as err:
self.__logger.error('Load action cfg {} error! Error msg: {}'.format(actionCfgFile, err))
return False
return True
def _InitDetector(self, uiAutoCfg):
self.__modelType = uiAutoCfg['ButtonDetection'].get('ModelType')
if self.__modelType != 'Yolov3' and self.__modelType != 'RefineNet':
self.logger.error('unknown model type: {}'.format(
self.__modelType))
return False
sys.path.append(pluginPath)
if self.__modelType == 'Yolov3':
if _is_windows_system:
from .windows.Yolov3.OpencvYolov3 import OpencvYolov3 as Yolov3
else:
from .ubuntu.Yolov3.OpencvYolov3 import OpencvYolov3 as Yolov3
param = dict()
param['cfgPath'] = util.ConvertToSDKFilePath(
uiAutoCfg['ButtonDetection'].get('CfgPath'))
param['weightsPath'] = util.ConvertToSDKFilePath(
uiAutoCfg['ButtonDetection'].get('WeightsPath'))
param['namesPath'] = util.ConvertToSDKFilePath(
uiAutoCfg['ButtonDetection'].get('NamesPath'))
param['threshold'] = uiAutoCfg['ButtonDetection'].get('Threshold')
self.__detector = Yolov3()
ret = self.__detector.Init(param)
if ret is False:
self.logger.error('initialize Yolov3 failed')
return False
if self.__modelType == 'RefineNet':
if _is_windows_system:
from .windows.DetectRefineNet import DetectRefineNet
else:
from .ubuntu.DetectRefineNet import DetectRefineNet
pthModelPath = util.ConvertToSDKFilePath(
uiAutoCfg['ButtonDetection'].get('PthModelPath'))
threshold = uiAutoCfg['ButtonDetection'].get('Threshold')
labels = ('__background__', 'return', 'close', 'tag', 'other')
self.__detector = DetectRefineNet.DetectRefineNet(
labels,
img_dim=320,
num_classes=5,
obj_thresh=threshold,
nms_thresh=0.10,
version='Refine_hc2net_version3',
onnx_model='',
trained_model=pthModelPath,
nmsType='normal',
platform='pytorch')
if not self.__detector:
self.logger.error('initialize RefineNet failed')
return False
if uiAutoCfg['ButtonDetection'].get('MaskPath', '') != '':
maskPath = util.ConvertToSDKFilePath(
uiAutoCfg['ButtonDetection'].get('MaskPath'))
maskImage = cv2.imread(maskPath)
if not maskImage:
self.logger.error(
'cannot read mask image in {}'.format(maskPath))
return False
grayImage = cv2.cvtColor(maskImage, cv2.COLOR_BGR2GRAY)
_, self.__mask = cv2.threshold(grayImage, 127, 255, 0)
else:
self.__mask = None
return True