def build_response(self, state_cache, state_index, hero_name): action_strs=[] restart = False # 对于模型,分析当前帧的行为 if self.real_hero != hero_name: state_info = state_cache[state_index] prev_hero = state_cache[state_index-1].get_hero(hero_name) if len(state_cache) >= 2 is not None else None # 如果有真实玩家,我们需要一些历史数据,所以分析3帧前的行为 elif len(state_cache) > 3: state_info = state_cache[state_index-3] next1_state_info = state_cache[state_index-2] next2_state_info = state_cache[state_index-1] next3_state_info = state_cache[state_index] else: return action_strs, False # 决定是否购买道具 buy_action = EquipUtil.buy_equip(state_info, hero_name) if buy_action is not None: buy_str = StateUtil.build_command(buy_action) action_strs.append(buy_str) # 如果有可以升级的技能,优先升级技能3 hero = state_info.get_hero(hero_name) skills = StateUtil.get_skills_can_upgrade(hero) if len(skills) > 0: skillid = 3 if 3 in skills else skills[0] update_cmd = CmdAction(hero.hero_name, CmdActionEnum.UPDATE, skillid, None, None, None, None, None, None) update_str = StateUtil.build_command(update_cmd) action_strs.append(update_str) # 回城相关逻辑 # 如果在回城中且没有被打断则继续回城,什么也不用返回 if prev_hero is not None: if hero.hero_name in self.hero_strategy and self.hero_strategy[hero.hero_name] == ActionEnum.town_ing \ and prev_hero.hp <= hero.hp \ and not StateUtil.if_hero_at_basement(hero): if not hero.skills[6].canuse: print(self.battle_id, hero.hero_name, '回城中,继续回城') return action_strs, False else: print(self.battle_id, hero.hero_name, '回城失败') town_action = CmdAction(hero.hero_name, CmdActionEnum.CAST, 6, hero.hero_name, None, None, None, None, None) action_str = StateUtil.build_command(town_action) action_strs.append(action_str) return action_strs, False if hero.hp <= 0: self.hero_strategy[hero.hero_name] = None return action_strs, False # # 补血逻辑 # if prev_hero is not None and hero.hero_name in self.hero_strategy and self.hero_strategy[ # hero.hero_name] == ActionEnum.hp_restore: # if StateUtil.cal_distance2(prev_hero.pos, hero.pos) < 100: # print(self.battle_id, hero_name, '到达补血点', '血量增长', hero.hp - prev_hero.hp) # del self.hero_strategy[hero_name] # if hero == self.model1_hero: # self.model1_hp_restore = time.time() # else: # self.model2_hp_restore = time.time() # 撤退逻辑 # TODO 甚至可以使用移动技能移动 if prev_hero is not None and hero.hero_name in self.hero_strategy and self.hero_strategy[hero.hero_name] == ActionEnum.retreat_to_town: if StateUtil.cal_distance2(prev_hero.pos, hero.pos) < 100: print(self.battle_id, hero_name, '开始回城') self.hero_strategy[hero.hero_name] = ActionEnum.town_ing town_action = CmdAction(hero.hero_name, CmdActionEnum.CAST, 6, hero.hero_name, None, None, None, None, None) action_str = StateUtil.build_command(town_action) action_strs.append(action_str) else: print(self.battle_id, hero_name, '还在撤退中', StateUtil.cal_distance2(prev_hero.pos, hero.pos)) return action_strs, False # 如果击杀了对方英雄,扫清附近小兵之后则启动撤退回城逻辑 if prev_hero is not None: if hero.hero_name in self.hero_strategy and self.hero_strategy[hero.hero_name] == ActionEnum.town_ing and prev_hero.hp <= hero.hp \ and not StateUtil.if_hero_at_basement(hero): if not hero.skills[6].canuse: return action_strs, False else: town_action = CmdAction(hero.hero_name, CmdActionEnum.CAST, 6, hero.hero_name, None, None, None, None, None) action_str = StateUtil.build_command(town_action) action_strs.append(action_str) if hero.hp <= 0: self.hero_strategy[hero.hero_name] = None return action_strs, False # 检查周围状况 near_enemy_heroes = StateUtil.get_nearby_enemy_heros(state_info, hero.hero_name, StateUtil.LINE_MODEL_RADIUS) near_enemy_units = StateUtil.get_nearby_enemy_units(state_info, hero.hero_name, StateUtil.LINE_MODEL_RADIUS) nearest_enemy_tower = StateUtil.get_nearest_enemy_tower(state_info, hero.hero_name, StateUtil.LINE_MODEL_RADIUS + 3) nearest_friend_units = StateUtil.get_nearby_friend_units(state_info, hero.hero_name, StateUtil.LINE_MODEL_RADIUS) line_index = 1 near_enemy_units_in_line = StateUtil.get_units_in_line(near_enemy_units, line_index) nearest_enemy_tower_in_line = StateUtil.get_units_in_line([nearest_enemy_tower], line_index) # 如果击杀对面英雄就回城补血。整体逻辑为,周围没有兵的情况下启动撤退逻辑,到达撤退地点之后启动回城。补满血之后再跟兵出来 # 处在泉水之中的时候设置策略层为吃线 if len(near_enemy_units_in_line) == 0 and len(near_enemy_heroes) == 0: if (hero_name == self.model1_hero and self.model2_just_dead == 1 and not StateUtil.if_hero_at_basement(hero)) \ or (hero_name == self.model2_hero and self.model1_just_dead == 1 and not StateUtil.if_hero_at_basement(hero)): if hero.hp / float(hero.maxhp) > 0.8: if hero_name == self.model1_hero: self.model2_just_dead = 0 else: self.model1_just_dead = 0 else: print(self.battle_id, hero_name, '选择撤退') self.hero_strategy[hero_name] = ActionEnum.retreat_to_town retreat_pos = StateUtil.get_retreat_pos(state_info, hero, line_index=1) action = CmdAction(hero_name, CmdActionEnum.MOVE, None, None, retreat_pos, None, None, -1, None) action_str = StateUtil.build_command(action) action_strs.append(action_str) if hero_name == self.model1_hero: self.model2_just_dead = 0 else: self.model1_just_dead = 0 return action_strs, False if StateUtil.if_hero_at_basement(hero): if hero_name == self.model1_hero: self.model2_just_dead = 0 else: self.model1_just_dead = 0 if hero.hp < hero.maxhp: if hero_name in self.hero_strategy: del self.hero_strategy[hero_name] return action_strs, False # # 残血并且周围没有敌人的情况下,可以去塔后吃加血 # if hero.hp / float(hero.maxhp) < 0.9 and hero not in self.hero_strategy: # print('补血条件', self.battle_id, hero_name, time.time(), self.model1_hp_restore, self.model2_hp_restore) # if hero == self.model1_hero and time.time() - self.model1_hp_restore > LineTrainerPPO.HP_RESTORE_GAP: # print(self.battle_id, hero_name, '选择加血') # self.hero_strategy[hero_name] = ActionEnum.hp_restore # elif hero == self.model2_hero and time.time() - self.model2_hp_restore > LineTrainerPPO.HP_RESTORE_GAP: # print(self.battle_id, hero_name, '选择加血') # self.hero_strategy[hero_name] = ActionEnum.hp_restore # # if self.hero_strategy[hero_name] == ActionEnum.hp_restore: # restore_pos = StateUtil.get_hp_restore_place(state_info, hero) # action = CmdAction(hero_name, CmdActionEnum.MOVE, None, None, restore_pos, None, None, -1, None) # action_str = StateUtil.build_command(action) # action_strs.append(action_str) # return action_strs, False # 开始根据策略决定当前的行动 # 对线情况下,首先拿到兵线,朝最前方的兵线移动 # 如果周围有危险(敌方单位)则启动对线模型 # 如果周围有小兵或者塔,需要他们都是在指定线上的小兵或者塔 if (len(near_enemy_units_in_line) == 0 and len(nearest_enemy_tower_in_line) == 0 and ( len(near_enemy_heroes) == 0 or StateUtil.if_in_line(hero, line_index, 4000) == -1) ) or\ (len(nearest_friend_units) == 0 and len(near_enemy_units_in_line) == 0 and len(near_enemy_heroes) == 0 and len(nearest_enemy_tower_in_line) == 1): # 跟兵线或者跟塔,优先跟塔 self.hero_strategy[hero.hero_name] = ActionEnum.line_1 # print("策略层:因为附近没有指定兵线的敌人所以开始吃线 " + hero.hero_name) front_soldier = StateUtil.get_frontest_soldier_in_line(state_info, line_index, hero.team) first_tower = StateUtil.get_first_tower(state_info, hero) if front_soldier is None or (hero.team == 0 and first_tower.pos.x > front_soldier.pos.x) or (hero.team == 1 and first_tower.pos.x < front_soldier.pos.x): # 跟塔,如果塔在前面的话 follow_tower_pos = StateUtil.get_tower_behind(first_tower, hero, line_index=1) move_action = CmdAction(hero.hero_name, CmdActionEnum.MOVE, None, None, follow_tower_pos, None, None, None, None) action_str = StateUtil.build_command(move_action) action_strs.append(action_str) else: # 得到最前方的兵线位置 move_action = CmdAction(hero.hero_name, CmdActionEnum.MOVE, None, None, front_soldier.pos, None, None, None, None) action_str = StateUtil.build_command(move_action) action_strs.append(action_str) else: if self.real_hero != hero_name: # 使用模型进行决策 # print("使用对线模型决定英雄%s的行动" % hero.hero_name) self.hero_strategy[hero.hero_name] = ActionEnum.line_model # 目前对线只涉及到两名英雄 rival_hero = '28' if hero.hero_name == '27' else '27' action, explorer_ratio, action_ratios = self.get_action(state_info, hero_name, rival_hero) # 考虑使用固定策略 # 如果决定使用策略,会连续n条行为全都采用策略(比如确保对方残血时候连续攻击的情况) # 如果策略返回为空则表示策略中断 if self.policy_ratio > 0 and ( 0 < self.cur_policy_act_idx_map[hero_name] < self.policy_continue_acts or random.uniform(0, 1) <= self.policy_ratio ): policy_action = LineTrainerPolicy.choose_action(state_info, action_ratios, hero_name, rival_hero, near_enemy_units, nearest_friend_units) if policy_action is not None: policy_action.vpred = action.vpred action = policy_action self.cur_policy_act_idx_map[hero_name] += 1 print("英雄 " + hero_name + " 使用策略,策略行为计数 idx " + str(self.cur_policy_act_idx_map[hero_name])) if self.cur_policy_act_idx_map[hero_name] >= self.policy_continue_acts: self.cur_policy_act_idx_map[hero_name] = 0 else: # 策略中断,清零 if self.cur_policy_act_idx_map[hero_name] > 0: print("英雄 " + hero_name + " 策略中断,清零") self.cur_policy_act_idx_map[hero_name] = 0 action_str = StateUtil.build_command(action) action_strs.append(action_str) # 如果是要求英雄施法回城,更新英雄状态,这里涉及到后续多帧是否等待回城结束 if action.action == CmdActionEnum.CAST and int(action.skillid) == 6: print("英雄%s释放了回城" % hero_name) self.hero_strategy[hero.hero_name] = ActionEnum.town_ing # 如果是选择了撤退,进行特殊标记,会影响到后续的行为 if action.action == CmdActionEnum.RETREAT: print("英雄%s释放了撤退,撤退点为%s" % (hero_name, action.tgtpos.to_string())) self.hero_strategy[hero.hero_name] = ActionEnum.retreat self.retreat_pos = action.tgtpos # 如果批量训练结束了,这时候需要清空未使用的训练集,然后重启游戏 if action.action == CmdActionEnum.RESTART: restart = True else: # 保存action信息到状态帧中 state_info.add_action(action) else: # 还是需要模型来计算出一个vpred rival_hero = '28' if hero.hero_name == '27' else '27' action, explorer_ratio, action_ratios = self.get_action(state_info, hero_name, rival_hero) # 推测玩家的行为 guess_action = Replayer.guess_player_action(state_info, next1_state_info, next2_state_info, next3_state_info, hero_name, rival_hero) guess_action.vpred = action.vpred action_str = StateUtil.build_command(guess_action) action_str['tick'] = state_info.tick print('猜测玩家行为为:' + JSON.dumps(action_str)) # 保存action信息到状态帧中 state_info.add_action(guess_action) return action_strs, restart
def build_response(self, state_info, prev_state_info, line_model, hero_names=None): battle_id = state_info.battleid tick = state_info.tick if tick >= 139062: db = 1 action_strs=[] if hero_names is None: hero_names = [hero.hero_name for hero in state_info.heros] for hero_name in hero_names: hero = state_info.get_hero(hero_name) prev_hero = prev_state_info.get_hero(hero.hero_name) if prev_state_info is not None else None # 检查是否重启游戏 # 线上第一个塔被摧毁 # 如果有可以升级的技能,优先升级技能3 skills = StateUtil.get_skills_can_upgrade(hero) if len(skills) > 0: skillid = 3 if 3 in skills else skills[0] update_cmd = CmdAction(hero.hero_name, CmdActionEnum.UPDATE, skillid, None, None, None, None, None, None) update_str = StateUtil.build_command(update_cmd) action_strs.append(update_str) # 检查周围状况 near_enemy_heroes = StateUtil.get_nearby_enemy_heros(state_info, hero.hero_name, StateUtil.LINE_MODEL_RADIUS) near_enemy_units = StateUtil.get_nearby_enemy_units(state_info, hero.hero_name, StateUtil.LINE_MODEL_RADIUS) nearest_enemy_tower = StateUtil.get_nearest_enemy_tower(state_info, hero.hero_name, StateUtil.LINE_MODEL_RADIUS + 3) # 回城相关逻辑 # 如果在回城中且没有被打断则继续回城,什么也不用返回 if prev_hero is not None: if self.hero_strategy[hero.hero_name] == ActionEnum.town_ing and prev_hero.hp <= hero.hp \ and not StateUtil.if_hero_at_basement(hero): if not hero.skills[6].canuse: print('回城中,继续回城') continue else: print('回城失败') if hero.hp <= 0: self.hero_strategy[hero.hero_name] = None continue # 处在少血状态是,且周围没有地方单位的情况下选择回城 # if len(near_enemy_heroes) == 0 and len(near_enemy_units) == 0 and nearest_enemy_tower is None: # if hero.hp/float(hero.maxhp) < LineTrainer.TOWN_HP_THRESHOLD: # print('策略层:回城') # # 检查英雄当前状态,如果在回城但是上一帧中受到了伤害,则将状态设置为正在回城,开始回城 # if self.hero_strategy[hero.hero_name] == ActionEnum.town_ing: # if prev_hero.hp > hero.hp: # town_action = CmdAction(hero.hero_name, CmdActionEnum.CAST, 6, hero.hero_name, None, None, None, None, None) # action_str = StateUtil.build_command(town_action) # action_strs.append(action_str) # # 检查英雄当前状态,如果不在回城,则将状态设置为正在回城,开始回城 # elif self.hero_strategy[hero.hero_name] != ActionEnum.town_ing: # self.hero_strategy[hero.hero_name] = ActionEnum.town_ing # town_action = CmdAction(hero.hero_name, CmdActionEnum.CAST, 6, hero.hero_name, None, None, None, None, None) # action_str = StateUtil.build_command(town_action) # action_strs.append(action_str) # # # 无论上面怎么操作,玩家下面的动作应该都是在回城中,所以跳过其它的操作 # continue # 处在泉水之中的时候设置策略层为吃线 if StateUtil.if_hero_at_basement(hero): if hero.hp < hero.maxhp: continue # 撤退逻辑 # TODO 甚至可以使用移动技能移动 if hero.hero_name in self.hero_strategy and self.hero_strategy[hero.hero_name] == ActionEnum.retreat: dist = StateUtil.cal_distance(hero.pos, self.retreat_pos) if dist <= 2: print('到达撤退点附近') self.hero_strategy[hero.hero_name] = None elif prev_hero is not None and prev_hero.pos.to_string() == hero.pos.to_string(): print('英雄卡住了,取消撤退') self.hero_strategy[hero.hero_name] = None else: print('仍然在撤退 ' + str(dist)) continue # 开始根据策略决定当前的行动 # 对线情况下,首先拿到兵线,朝最前方的兵线移动 # 如果周围有危险(敌方单位)则启动对线模型 # 如果周围有小兵或者塔,需要他们都是在指定线上的小兵或者塔 line_index = 1 near_enemy_units_in_line = StateUtil.get_units_in_line(near_enemy_units, line_index) nearest_enemy_tower_in_line = StateUtil.get_units_in_line([nearest_enemy_tower], line_index) if len(near_enemy_units_in_line) == 0 and len(nearest_enemy_tower_in_line) == 0 and (len(near_enemy_heroes) == 0 or StateUtil.if_in_line(hero, line_index, 4000) == -1): self.hero_strategy[hero.hero_name] = ActionEnum.line_1 # print("策略层:因为附近没有指定兵线的敌人所以开始吃线 " + hero.hero_name) # 跟兵线 front_soldier = StateUtil.get_frontest_soldier_in_line(state_info, line_index, hero.team) if front_soldier is None: action_str = StateUtil.build_action_command(hero.hero_name, 'HOLD', {}) action_strs.append(action_str) else: # 得到最前方的兵线位置 move_action = CmdAction(hero.hero_name, CmdActionEnum.MOVE, None, None, front_soldier.pos, None, None, None, None) action_str = StateUtil.build_command(move_action) action_strs.append(action_str) else: # 使用模型进行决策 # print("使用对线模型决定英雄%s的行动" % hero.hero_name) self.hero_strategy[hero.hero_name] = ActionEnum.line_model enemies = [] enemies.extend((hero.hero_name for hero in near_enemy_heroes)) enemies.extend((unit.unit_name for unit in near_enemy_units)) if nearest_enemy_tower is not None: enemies.append(nearest_enemy_tower.unit_name) # print('对线模型决策,因为周围有敌人 ' + ' ,'.join(enemies)) # 目前对线只涉及到两名英雄 rival_hero = '28' if hero.hero_name == '27' else '27' action = line_model.get_action(prev_state_info, state_info, hero.hero_name, rival_hero) action_str = StateUtil.build_command(action) action_strs.append(action_str) # 如果是要求英雄施法回城,更新英雄状态,这里涉及到后续多帧是否等待回城结束 if action.action == CmdActionEnum.CAST and int(action.skillid) == 6: print("英雄%s释放了回城" % hero_name) self.hero_strategy[hero.hero_name] = ActionEnum.town_ing # 如果是选择了撤退,进行特殊标记,会影响到后续的行为 if action.action == CmdActionEnum.RETREAT: print("英雄%s释放了撤退,撤退点为%s" % (hero_name, action.tgtpos.to_string())) self.hero_strategy[hero.hero_name] = ActionEnum.retreat self.retreat_pos = action.tgtpos # 保存action信息到状态帧中 state_info.add_action(action) return action_strs