def __init__(self, env, args, itr, seed):
# 随机设置种子
if seed is not None:
self.setup_seed(seed)
self.args = args
# 获取环境
self.env = env
# 进程编号
self.pid = itr
self.replay_buffer = ReplayBuffer(self.args)
self.win_rates = []
'''
这里，episode_reward 代表一个episode的累加奖赏，
episodes_reward代表多个episode的累加奖赏，
episodes_rewards代表多次评价的多个episode的累加奖赏
'''
self.episodes_rewards = []
self.evaluate_itr = []
self.max_win_rate = 0
self.time_steps = 0
# 保存结果和模型的位置，增加计数，帮助一次运行多个实例
alg_dir = self.args.alg + '_' + str(self.args.epsilon_anneal_steps // 10000) + 'w' + '_' + \
str(self.args.target_update_period)
self.alg_tag = '_' + self.args.optim
if self.args.her:
self.alg_tag += str(self.args.her)
alg_dir += '_her=' + str(self.args.her)
# self.save_path = self.args.result_dir + '/' + alg_dir + '/' + self.args.map + '/' + itr
self.save_path = self.args.result_dir + '/' + self.args.map + '/' + alg_dir + '/' + itr
if not os.path.exists(self.save_path):
os.makedirs(self.save_path)
self.args.model_dir = args.model_dir + '/' + args.map + '/' + alg_dir + '/' + itr
self.agents = Agents(args, itr=itr)
print('step runner 初始化')
if self.args.her:
print('使用HER')
def __init__(self, params):
seed = params['general_params']['seed']
self.__set_seed(seed=seed)
env_params = params['env_params']
env_params['seed'] = seed
self.env = UnityEnv(params=env_params)
agent_params = params['agent_params']
self.__num_of_agents = self.env.observation_space.shape[0]
state_size = self.env.observation_space.shape[1]
action_size = self.env.action_space_size
agent_params['num_of_agents'] = self.__num_of_agents
agent_params['state_size'] = state_size
agent_params['action_size'] = action_size
self.agents = Agents(params=agent_params)
trainer_params = params['trainer_params']
self.learning_rate_decay = trainer_params['learning_rate_decay']
self.results_path = trainer_params['results_path']
self.model_path = trainer_params['model_path']
self.t_max = trainer_params['t_max']
self.exploration_noise = UOProcess()
# data gathering variables
self.avg_rewards = []
self.scores = []
self.score = 0
self.sigma = 0.5
print("MADDPG agent.")
print("Configuration:")
pprint(params)
logging.info("Configuration: {}".format(params))
def __init__(self, env, args, itr):
# 获取参数
# self.args = get_common_args()
self.args = args
# 获取环境
self.env = env
# 进程编号
self.pid = itr
self.agents = Agents(args, itr=itr)
# 不复用网络，就会有多个agent，训练的时候共享参数，就是一个网络
# if not self.args.reuse_network:
# self.agents = []
# for i in range(self.args.n_agents):
# self.agents.append(Agents(self.args, i))
# self.rollout = RollOut(self.agents, self.args)
self.replay_buffer = ReplayBuffer(self.args)
self.win_rates = []
'''
这里，episode_reward 代表一个episode的累加奖赏，
episodes_reward代表多个episode的累加奖赏，
episodes_rewards代表多次评价的多个episode的累加奖赏
'''
self.episodes_rewards = []
self.evaluate_itr = []
self.max_win_rate = 0
# 保存结果和模型的位置，增加计数，帮助一次运行多个实例
self.save_path = self.args.result_dir + '/' + self.args.alg + '/' + self.args.map + '/' + str(
itr)
if not os.path.exists(self.save_path):
os.makedirs(self.save_path)
print('runner 初始化')
def import_agent_file(scenario_settings, con, cur, engine, model_settings,
agent_file_status, input_name):
"""
Generates new agents or uses pre-generated agents from provided .pkl file
Parameters
----------
**scenario_settings** : 'SQL schema'
Schema of the scenario settings
**con** : 'SQL connection'
SQL connection to connect to database
**cur** : 'SQL cursor'
Cursor
**engine** : 'SQL engine'
SQL engine to intepret SQL query
**model_settings** : 'object'
Model settings that apply to all scenarios
**agent_file_status** : 'attribute'
Attribute that describes whether to use pre-generated agent file or create new
**input_name** : 'string'
.Pkl file name substring of pre-generated agent table
Returns
-------
**solar_agents** : 'Class'
Instance of Agents class with either user pre-generated or new data
"""
schema = scenario_settings.schema
input_agent_dir = model_settings.input_agent_dir
state_to_model = scenario_settings.state_to_model
ISO_List = ['ERCOT', 'NEISO', 'NYISO', 'CAISO', 'PJM', 'MISO', 'SPP']
if agent_file_status == 'Use pre-generated Agents':
userdefined_table_name = "input_" + input_name + "_user_defined"
scenario_userdefined_name = get_userdefined_scenario_settings(
schema, userdefined_table_name, con)
scenario_userdefined_value = scenario_userdefined_name['val'].values[0]
solar_agents_df = pd.read_pickle(
os.path.join(input_agent_dir, scenario_userdefined_value + ".pkl"))
if scenario_settings.region in ISO_List:
solar_agents_df = pd.read_pickle(
os.path.join(input_agent_dir,
scenario_userdefined_value + ".pkl"))
else:
solar_agents_df = solar_agents_df[
solar_agents_df['state_abbr'].isin(state_to_model)]
if solar_agents_df.empty:
raise ValueError(
'Region not present within pre-generated agent file - Edit Inputsheet'
)
solar_agents = Agents(solar_agents_df)
solar_agents.on_frame(agent_mutation.elec.reassign_agent_tariffs, con)
else:
raise ValueError(
'Generating agents is not supported at this time. Please select "Use pre-generated Agents" in the input sheet'
)
return solar_agents
Windows (x86_64): "path/to/Reacher_Windows_x86_64/Reacher.exe"
Linux (x86): "path/to/Reacher_Linux/Reacher.x86"
Linux (x86_64): "path/to/Reacher_Linux/Reacher.x86_64"
'''
env = UnityEnvironment(file_name="./Reacher_Linux/Reacher.x86")
brain_name = env.brain_names[0] # get the default brain
brain = env.brains[brain_name]
env_info = env.reset(train_mode=True)[brain_name] # reset the environment
num_agents = len(env_info.agents) # get number of agents
states = env_info.vector_observations # get the current state
state_size = states.shape[1]
action_size = brain.vector_action_space_size
agents = Agents(
num_agents=num_agents, # create RL agents
state_size=state_size,
action_size=action_size,
random_seed=0)
checkpoint = torch.load('solution.pth', map_location={'cuda:0':
'cpu'}) # load weights
agents.actor_local.load_state_dict(checkpoint['actor_local_state_dict'])
agents.actor_target.load_state_dict(checkpoint['actor_target_state_dict'])
agents.actor_optimizer.load_state_dict(
checkpoint['actor_optimizer_state_dict'])
agents.critic_local.load_state_dict(checkpoint['critic_local_state_dict'])
agents.critic_target.load_state_dict(checkpoint['critic_target_state_dict'])
agents.critic_optimizer.load_state_dict(
checkpoint['critic_optimizer_state_dict'])
scores = np.zeros(num_agents) # initialize the score (for each agent)
def make_env(n_substeps=3,
horizon=250,
deterministic_mode=False,
n_agents=1,
env_no=1):
env = Base(n_agents=n_agents,
n_substeps=n_substeps,
horizon=horizon,
floor_size=10,
grid_size=50,
deterministic_mode=deterministic_mode,
env_no=env_no)
# Add Walls
#env.add_module(RandomWalls(grid_size=5, num_rooms=2, min_room_size=5, door_size=5, low_outside_walls=True, outside_wall_rgba="1 1 1 0.1"))
# Add Agents
first_agent_placement = uniform_placement
second_agent_placement = uniform_placement
agent_placement_fn = [first_agent_placement] + [second_agent_placement]
env.add_module(Agents(n_agents, placement_fn=agent_placement_fn))
# Add LidarSites
n_lidar_per_agent = 1
visualize_lidar = False
compress_lidar_scale = None
if visualize_lidar:
env.add_module(
LidarSites(n_agents=n_agents, n_lidar_per_agent=n_lidar_per_agent))
env.reset()
keys_self = ['agent_qpos_qvel']
keys_mask_self = [] #['mask_aa_obs']
keys_external = [] #['agent_qpos_qvel']
keys_mask_external = []
keys_copy = []
env = AddConstantObservationsWrapper(
env, new_obs={'agents_health': np.full((n_agents, 1), 100.0)})
keys_self += ['agents_health']
env = ProjectileWrapper(env)
#env = HealthWrapper(env)
env = SplitMultiAgentActions(env)
env = DiscretizeActionWrapper(env, 'action_movement')
env = AgentAgentObsMask2D(env)
if n_lidar_per_agent > 0:
env = Lidar(env,
n_lidar_per_agent=n_lidar_per_agent,
visualize_lidar=visualize_lidar,
compress_lidar_scale=compress_lidar_scale)
keys_copy += ['lidar']
keys_external += ['lidar']
env = SplitObservations(env,
keys_self + keys_mask_self,
keys_copy=keys_copy)
env = DiscardMujocoExceptionEpisodes(env)
env = SelectKeysWrapper(env,
keys_self=keys_self,
keys_external=keys_external,
keys_mask=keys_mask_self + keys_mask_external,
flatten=False)
return env
def __init__(self, agent_id, evaluation):
Agents.__init__(self, agent_id, evaluation)
self.wins = {} # a (agent_id, state): count dict
self.plays = {} # a (agent_id, state): count dict
self.depth = 60
self.simulation_time = 10
def __init__(self, agent_id, evaluation, depth=2):
Agents.__init__(self, agent_id, evaluation)
self.max_depth = depth
def main(mode=None, resume_year=None, endyear=None, ReEDS_inputs=None):
"""
Compute the economic adoption of distributed generation resources on an agent-level basis.
Model output is saved to a `/runs` file within the dGen directory.
"""
try:
# =====================================================================
# SET UP THE MODEL TO RUN
# =====================================================================
# initialize Model Settings object
# (this controls settings that apply to all scenarios to be executed)
model_settings = settings.init_model_settings()
prerun_test.check_dependencies()
# make output directory
# create the logger and stamp with git hash
logger = utilfunc.get_logger(
os.path.join(model_settings.out_dir, 'dg_model.log'))
logger.info("Model version is git commit {:}".format(
model_settings.git_hash))
# =====================================================================
# LOOP OVER SCENARIOS
# =====================================================================
out_subfolders = {'solar': []}
for i, scenario_file in enumerate(model_settings.input_scenarios):
logger.info('============================================')
logger.info('============================================')
logger.info("Running Scenario {i} of {n}".format(
i=i + 1, n=len(model_settings.input_scenarios)))
# initialize ScenarioSettings object
# (this controls settings that apply only to this specific scenario)
scenario_settings = settings.init_scenario_settings(
scenario_file, model_settings)
# log summary high level secenario settings
logger.info('Scenario Settings:')
logger.info('\tScenario Name: %s' %
scenario_settings.scenario_name)
logger.info('\tSectors: %s' %
list(scenario_settings.sector_data.keys()))
logger.info('\tTechnologies: %s' % scenario_settings.techs)
logger.info(
'\tYears: %s - %s' %
(scenario_settings.start_year, scenario_settings.end_year))
logger.info('Results Path: %s' % (scenario_settings.out_scen_path))
#==========================================================================================================
# CREATE AGENTS
#==========================================================================================================
logger.info("-------------- Agent Preparation ---------------")
if scenario_settings.generate_agents:
logger.info('\tCreating Agents')
solar_agents = Agents(
agent_mutation.init_solar_agents(scenario_settings))
logger.info('....{} agents in input csv'.format(
len(solar_agents)))
# Write base agents to disk
solar_agents.df.to_pickle(scenario_settings.out_scen_path +
'/agent_df_base.pkl')
else:
logger.info('Loading %s' % scenario_settings.agents_file_name)
with open(scenario_settings.agents_file_name, "r") as f:
solar_agents = Agents(pickle.load(f))
# Get set of columns that define agent's immutable attributes
cols_base = list(solar_agents.df.columns.values)
#==============================================================================
# TECHNOLOGY DEPLOYMENT
#==============================================================================
logger.info("-------------- Yearly Analysis ---------------")
complete_df = pd.DataFrame()
if scenario_settings.techs == ['solar']:
solar_agents.df['tech'] = 'solar'
for i, year in enumerate(scenario_settings.model_years):
is_first_year = year == model_settings.start_year
logger.info('\tWorking on %s' % year)
# determine any non-base columns and drop them
cols = list(solar_agents.df.columns.values)
cols_to_drop = [x for x in cols if x not in cols_base]
if len(cols_to_drop) != 0:
solar_agents.df.drop(cols_to_drop,
axis=1,
inplace=True)
# copy the core agent object and set their year
solar_agents.df['year'] = year
# get and apply load growth
load_growth_yearly = scenario_settings.get_load_growth(
year)
solar_agents.on_frame(
agent_mutation.elec.apply_load_growth,
(load_growth_yearly))
# Normalize the hourly load profile to updated total load which includes load growth multiplier
solar_agents.on_frame(agent_mutation.elec.
apply_scale_normalized_load_profiles)
# Get and apply net metering parameters
net_metering_yearly = scenario_settings.get_nem_settings(
year)
solar_agents.on_frame(
agent_mutation.elec.apply_export_tariff_params,
(net_metering_yearly))
# Apply each agent's electricity price change and assumption about increases
solar_agents.on_frame(
agent_mutation.elec.
apply_elec_price_multiplier_and_escalator,
[year, scenario_settings.get_rate_escalations()])
# Apply PV Specs
solar_agents.on_frame(agent_mutation.elec.apply_pv_specs,
scenario_settings.get_pv_specs())
solar_agents.on_frame(
agent_mutation.elec.apply_storage_specs, [
scenario_settings.get_batt_price_trajectories(),
year, scenario_settings
])
# Apply financial terms
solar_agents.on_frame(
agent_mutation.elec.apply_financial_params, [
scenario_settings.get_financing_terms(),
scenario_settings.
financial_options['annual_inflation_pct']
])
# Apply wholesale electricity prices
solar_agents.on_frame(
agent_mutation.elec.apply_wholesale_elec_prices,
scenario_settings.get_wholesale_elec_prices())
# Size S+S system and calculate electric bills
if 'ix' not in os.name:
cores = None
else:
cores = model_settings.local_cores
solar_agents.on_row(
fFuncs.calc_system_size_and_financial_performance,
cores=cores)
solar_agents.df['agent_id'] = solar_agents.df.index.values
# Calculate the financial performance of the S+S systems
solar_agents.on_frame(
financial_functions.calc_financial_performance)
# Calculate Maximum Market Share
solar_agents.on_frame(
financial_functions.calc_max_market_share,
scenario_settings.get_max_market_share())
# determine "developable" population
solar_agents.on_frame(
agent_mutation.elec.
calculate_developable_customers_and_load)
# Apply market_last_year
if is_first_year:
solar_agents.on_frame(
agent_mutation.elec.estimate_initial_market_shares)
market_last_year_df = None
else:
solar_agents.on_frame(
agent_mutation.elec.apply_market_last_year,
market_last_year_df)
# Calculate diffusion based on economics and bass diffusion
solar_agents.df, market_last_year_df = diffusion_functions.calc_diffusion_solar(
solar_agents.df, is_first_year,
scenario_settings.get_bass_params())
# Estimate total generation
solar_agents.on_frame(
agent_mutation.elec.estimate_total_generation)
# Aggregate results
scenario_settings.output_batt_dispatch_profiles = False
if is_first_year == True:
interyear_results_aggregations = agent_mutation.elec.aggregate_outputs_solar(
solar_agents.df, year, is_first_year,
scenario_settings)
else:
interyear_results_aggregations = agent_mutation.elec.aggregate_outputs_solar(
solar_agents.df, year, is_first_year,
scenario_settings, interyear_results_aggregations)
# --- Check to ensure that agent_df isn't growing (i.e. merges are failing silently) ---
df_print = solar_agents.df.copy()
df_print = df_print.loc[df_print['year'] == year]
df_print = df_print.groupby(['sector_abbr'
])['pv_kw_cum'].sum()
#==========================================================================================================
# WRITE AGENT DF AS PICKLES FOR POST-PROCESSING
#==========================================================================================================
# Write Outputs to the database
drop_fields = [
'consumption_hourly_initial', 'bill_savings',
'consumption_hourly', 'solar_cf_profile',
'tariff_dict', 'deprec_sch', 'batt_dispatch_profile'
] #dropping because are arrays or json
df_write = solar_agents.df.drop(drop_fields, axis=1)
write_annual = False
if write_annual:
df_write.to_pickle(scenario_settings.out_scen_path +
'/agent_df_%s.pkl' % year)
if i == 0:
complete_df = df_write
else:
complete_df = pd.concat([complete_df, df_write],
sort=False)
#==============================================================================
# Outputs & Visualization
#==============================================================================
logger.info("---------Saving Model Results---------")
complete_df.to_csv(scenario_settings.out_scen_path +
'/agent_outputs.csv')
logger.info("-------------Model Run Complete-------------")
logger.info('Completed in: %.1f seconds' %
(time.time() - model_settings.model_init))
except Exception as e:
if 'logger' in locals():
logger.error(e.__str__(), exc_info=True)
logger.info('Error on line {}'.format(
sys.exc_info()[-1].tb_lineno))
logger.info('Type of error {}'.format(type(e)))
logger.info('Error Text: {}'.format(e))
if 'logger' not in locals():
raise
finally:
if 'logger' in locals():
utilfunc.shutdown_log(logger)
utilfunc.code_profiler(model_settings.out_dir)
def decode_msg(self, lines):
'''
decode message for reinforcement learning
show:0
team_l:0
team_r:0
ball_x:0.000000, ball_y:0.000000, ball_vx:0.000000, ball_vy:0.000000
side: 1, num:1, x:-49.000000, y:0.000000, vel_x:0.000000, vel_y:0.000000, kick_count:0,
stamina, staminaCapacity
side: 1, num:2, x:-25.000000, y:-5.000000, vel_x:0.000000, vel_y:0.000000, kick_count:0
side: 1, num:3, x:-25.000000, y:5.000000, vel_x:0.000000, vel_y:0.000000, kick_count:0
side: 1, num:4, x:-25.000000, y:-10.000000, vel_x:0.000000, vel_y:0.000000, kick_count:0
side: 1, num:5, x:-25.000000, y:10.000000, vel_x:0.000000, vel_y:0.000000, kick_count:0
side: 1, num:6, x:-25.000000, y:0.000000, vel_x:0.000000, vel_y:0.000000, kick_count:0
side: 1, num:7, x:-15.000000, y:-5.000000, vel_x:0.000000, vel_y:0.000000, kick_count:0
side: 1, num:8, x:-15.000000, y:5.000000, vel_x:0.000000, vel_y:0.000000, kick_count:0
side: 1, num:9, x:-15.000000, y:-10.000000, vel_x:0.000000, vel_y:0.000000, kick_count:0
side: 1, num:10, x:-15.000000, y:10.000000, vel_x:0.000000, vel_y:0.000000, kick_count:0
side: 1, num:11, x:-15.000000, y:0.000000, vel_x:0.000000, vel_y:0.000000, kick_count:0
side: -1, num:2, x:25.000000, y:5.000000, vel_x:0.000000, vel_y:0.000000, kick_count:0
side: -1, num:3, x:25.000000, y:-5.000000, vel_x:0.000000, vel_y:0.000000, kick_count:0
side: -1, num:4, x:25.000000, y:10.000000, vel_x:0.000000, vel_y:0.000000, kick_count:0
side: -1, num:5, x:25.000000, y:-10.000000, vel_x:0.000000, vel_y:0.000000, kick_count:0
side: -1, num:6, x:25.000000, y:-0.000000, vel_x:0.000000, vel_y:0.000000, kick_count:0
side: -1, num:7, x:15.000000, y:5.000000, vel_x:0.000000, vel_y:0.000000, kick_count:0
side: -1, num:8, x:15.000000, y:-5.000000, vel_x:0.000000, vel_y:0.000000, kick_count:0
side: -1, num:9, x:15.000000, y:10.000000, vel_x:0.000000, vel_y:0.000000, kick_count:0
side: -1, num:10, x:15.000000, y:-10.000000, vel_x:0.000000, vel_y:0.000000, kick_count:0
side: -1, num:11, x:15.000000, y:-0.000000, vel_x:0.000000, vel_y:0.000000, kick_count:0
'''
try:
self.show = int(lines[0].split(':')[-1])
self.scores = np.array(
[int(lines[1].split(':')[-1]),
int(lines[2].split(':')[-1])])
except ValueError:
print('score and show error')
self.ball = np.array(
[np.float(val.split(':')[-1]) for val in lines[3].split(',')])
try:
players = []
for i in range(4, len(lines)):
player = [
np.float(val.split(':')[-1]) for val in lines[i].split(',')
]
players.append(player)
self.players = np.vstack(players)
except ValueError:
pass
# print('line error')
if len(self.agents) < 1:
self.agents = [
Agents(self.players, self.ball, i) for i in range(11)
]
self.move_message = self.format_move_message(self.players[:11,
2:4])
# print(self.move_message)
onball = np.argwhere(self.players[:, -3] - self.kickcount > 0)
# onball = np.argwhere(np.array([caldist(self.ball, self.players[i,2:4]) for i in range(22)]))
if len(onball) > 0:
# self.actioned = False
for i in range(11):
self.agents[i].onball = int(onball[0])
self.onball = int(onball[0])
self.kickcount = self.players[:, -3]
class Runner:
def __init__(self, env, args, itr, seed):
# 随机设置种子
if seed is not None:
self.setup_seed(seed)
self.args = args
# 获取环境
self.env = env
# 进程编号
self.pid = itr
self.replay_buffer = ReplayBuffer(self.args)
self.win_rates = []
'''
这里，episode_reward 代表一个episode的累加奖赏，
episodes_reward代表多个episode的累加奖赏，
episodes_rewards代表多次评价的多个episode的累加奖赏
'''
self.episodes_rewards = []
self.evaluate_itr = []
self.max_win_rate = 0
self.time_steps = 0
# 保存结果和模型的位置，增加计数，帮助一次运行多个实例
alg_dir = self.args.alg + '_' + str(self.args.epsilon_anneal_steps // 10000) + 'w' + '_' + \
str(self.args.target_update_period)
self.alg_tag = '_' + self.args.optim
if self.args.her:
self.alg_tag += str(self.args.her)
alg_dir += '_her=' + str(self.args.her)
# self.save_path = self.args.result_dir + '/' + alg_dir + '/' + self.args.map + '/' + itr
self.save_path = self.args.result_dir + '/' + self.args.map + '/' + alg_dir + '/' + itr
if not os.path.exists(self.save_path):
os.makedirs(self.save_path)
self.args.model_dir = args.model_dir + '/' + args.map + '/' + alg_dir + '/' + itr
self.agents = Agents(args, itr=itr)
print('step runner 初始化')
if self.args.her:
print('使用HER')
@staticmethod
def setup_seed(seed):
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
np.random.seed(seed)
random.seed(seed)
torch.backends.cudnn.deterministic = True
def generate_episode(self, episode_num, evaluate=False):
# 为保存评价的回放做准备
if self.args.replay_dir != '' and evaluate and episode_num == 0:
self.env.close()
# 变量初始化，使用her需要记录goal
self.env.reset()
done = False
info = None
win = False
last_action = np.zeros((self.args.n_agents, self.args.n_actions))
# epsilon 递减
epsilon = 0 if evaluate else self.args.epsilon
# epsilon 递减的方式
if self.args.epsilon_anneal_scale == 'episode' or \
(self.args.epsilon_anneal_scale == 'itr' and episode_num == 0):
epsilon = epsilon - self.args.epsilon_decay if epsilon > self.args.min_epsilon else epsilon
# 记录一个episode的信息
episode_buffer = None
if not evaluate:
episode_buffer = {
'o':
np.zeros([
self.args.episode_limit, self.args.n_agents,
self.args.obs_shape
]),
's':
np.zeros([self.args.episode_limit, self.args.state_shape]),
'a':
np.zeros([self.args.episode_limit, self.args.n_agents, 1]),
'onehot_a':
np.zeros([
self.args.episode_limit, self.args.n_agents,
self.args.n_actions
]),
'avail_a':
np.zeros([
self.args.episode_limit, self.args.n_agents,
self.args.n_actions
]),
'r':
np.zeros([self.args.episode_limit, 1]),
'next_o':
np.zeros([
self.args.episode_limit, self.args.n_agents,
self.args.obs_shape
]),
'next_s':
np.zeros([self.args.episode_limit, self.args.state_shape]),
'next_avail_a':
np.zeros([
self.args.episode_limit, self.args.n_agents,
self.args.n_actions
]),
'done':
np.ones([self.args.episode_limit, 1]),
'padded':
np.ones([self.args.episode_limit, 1])
}
# 开始进行一波 episode
states, former_states = [], []
obs = self.env.get_obs()
if self.args.her:
obs = np.concatenate((obs, self.env.goal), axis=1)
state = self.env.get_state()
if self.args.her:
states.append(self.env.state)
former_states.append(self.env.former_states)
avail_actions = []
self.agents.policy.init_hidden(1)
for agent_id in range(self.args.n_agents):
avail_action = self.env.get_avail_agent_actions(agent_id)
avail_actions.append(avail_action)
episode_reward = 0
for step in range(self.args.episode_limit):
if done:
break
else:
actions, onehot_actions = [], []
for agent_id in range(self.args.n_agents):
# avail_action = self.env.get_avail_agent_actions(agent_id)
action, _ = self.agents.choose_action(
obs[agent_id], last_action[agent_id], agent_id,
avail_actions[agent_id], epsilon, evaluate)
# 得到该动作的独热编码
onehot_action = np.zeros(self.args.n_actions)
onehot_action[action] = 1
onehot_actions.append(onehot_action)
# 加入联合动作
actions.append(action)
# avail_actions.append(avail_action)
# 记录该动作
last_action[agent_id] = onehot_action
# 对环境执行联合动作
reward, done, info = self.env.step(actions)
# 记录时间步
if not evaluate:
self.time_steps += 1
# 获取改变后的信息
if not done:
next_obs = self.env.get_obs()
if self.args.her:
next_obs = np.concatenate((next_obs, self.env.goal),
axis=1)
next_state = self.env.get_state()
if self.args.her:
states.append(self.env.state)
former_states.append(self.env.former_states)
else:
next_obs = obs
next_state = state
# 添加可得动作
next_avail_actions = []
for agent_id in range(self.args.n_agents):
avail_action = self.env.get_avail_agent_actions(agent_id)
next_avail_actions.append(avail_action)
# 添加经验
if not evaluate:
episode_buffer['o'][step] = obs
episode_buffer['s'][step] = state
episode_buffer['a'][step] = np.reshape(
actions, [self.args.n_agents, 1])
episode_buffer['onehot_a'][step] = onehot_actions
episode_buffer['avail_a'][step] = avail_actions
episode_buffer['r'][step] = [reward]
episode_buffer['next_o'][step] = next_obs
episode_buffer['next_s'][step] = next_state
episode_buffer['next_avail_a'][step] = next_avail_actions
episode_buffer['done'][step] = [done]
episode_buffer['padded'][step] = [0.]
# 更新变量
episode_reward += reward
obs = next_obs
state = next_state
avail_actions = next_avail_actions
if self.args.epsilon_anneal_scale == 'step':
epsilon = epsilon - self.args.epsilon_decay if epsilon > self.args.min_epsilon else epsilon
# 是训练则记录新的epsilon
if not evaluate:
self.args.epsilon = epsilon
# 获取对局信息
if info.__contains__('battle_won'):
win = True if done and info['battle_won'] else False
if evaluate and episode_num == self.args.evaluate_num - 1 and self.args.replay_dir != '':
self.env.save_replay()
self.env.close()
if not evaluate and self.args.her:
return episode_buffer, states, former_states
return episode_buffer, episode_reward, win
def run(self):
train_steps = 0
early_stop = 10
num_eval = 0
self.max_win_rate = 0
self.time_steps = 0
last_test_step = 0
begin_time = None
begin_step = None
# for itr in range(self.args.n_itr):
while self.time_steps < self.args.max_steps:
if begin_step is None:
begin_time = datetime.utcnow().astimezone(
timezone(timedelta(hours=8)))
begin_step = self.time_steps
# 收集 n_episodes 的数据
if self.args.her:
episode_batch, states, former_states = self.generate_episode(0)
self.her_k(episode_batch, states, former_states)
else:
episode_batch, _, _ = self.generate_episode(0)
for key in episode_batch.keys():
episode_batch[key] = np.array([episode_batch[key]])
for e in range(1, self.args.n_episodes):
if self.args.her:
episode_batch, states, former_states = self.generate_episode(
e)
self.her_k(episode_batch, states, former_states)
else:
episode, _, _ = self.generate_episode(e)
for key in episode_batch.keys():
episode[key] = np.array([episode[key]])
episode_batch[key] = np.concatenate(
(episode_batch[key], episode[key]), axis=0)
# 添加到 replay buffer
self.replay_buffer.store(episode_batch)
# 训练 TODO 12.5
if self.replay_buffer.size < self.args.batch_size * self.args.bs_rate:
print('replay buffer 还没 batch size * {} 大 ！'.format(
self.args.bs_rate))
begin_time = None
begin_step = None
continue
for _ in range(self.args.train_steps):
batch = self.replay_buffer.sample(self.args.batch_size)
self.agents.train(batch, train_steps)
train_steps += 1
# 周期性评价
# if itr % self.args.evaluation_period == 0:
if (self.time_steps -
last_test_step) / self.args.evaluation_steps_period >= 1.0:
num_eval += 1
last_test_step = self.time_steps
print(
f'进程 {self.pid}: {self.time_steps} step / {self.args.max_steps} steps'
)
# print('幂为：{}'.format(self.agents.policy.power))
win_rate, episodes_reward = self.evaluate()
# 保存测试结果
self.evaluate_itr.append(self.time_steps)
self.win_rates.append(win_rate)
self.episodes_rewards.append(episodes_reward)
# 表现好的模型要额外保存
if win_rate > self.max_win_rate:
self.max_win_rate = win_rate
self.agents.policy.save_model(str(win_rate))
# 不时刻保存，从而减少时间花费
if num_eval % 50 == 0:
self.save_results()
self.plot()
# 记录经历50次测试花费了多久
now = datetime.utcnow().astimezone(
timezone(timedelta(hours=8)))
elapsed_time = now - begin_time
expected_remain_time = (elapsed_time / (self.time_steps - begin_step)) * \
(self.args.max_steps - self.time_steps)
expected_end_time = now + expected_remain_time
print("预计还需: {}".format(str(expected_remain_time)))
print("预计结束时间为: {}".format(
expected_end_time.strftime("%Y-%m-%d_%H-%M-%S")))
# 最后把所有的都保存一下
self.save_results()
self.plot()
self.env.close()
def evaluate(self):
"""
得到平均胜率和每次测试的累加奖赏，方便画误差阴影图
:return:
"""
win_number = 0
episodes_reward = []
for itr in range(self.args.evaluate_num):
if self.args.didactic:
episode_reward, win = self.get_eval_qtot()
else:
_, episode_reward, win = self.generate_episode(itr,
evaluate=True)
episodes_reward.append(episode_reward)
if win:
win_number += 1
return win_number / self.args.evaluate_num, episodes_reward
def save_results(self):
"""
保存数据，方便后面多种算法结果画在一张图里比较
:return:
"""
# 如果已经有图片就删掉
for filename in os.listdir(self.save_path):
if filename.endswith('.npy'):
os.remove(self.save_path + '/' + filename)
np.save(self.save_path + '/evaluate_itr.npy', self.evaluate_itr)
if self.args.didactic and self.args.power is None and 'strapped' in self.args.alg:
np.save(self.save_path + '/train_steps.npy',
self.agents.policy.train_steps)
np.save(self.save_path + '/differences.npy',
self.agents.policy.differences)
else:
np.save(self.save_path + '/win_rates.npy', self.win_rates)
np.save(self.save_path + '/episodes_rewards.npy',
self.episodes_rewards)
def plot(self):
"""
定期绘图
:return:
"""
fig = plt.figure()
ax1 = fig.add_subplot(211)
if self.args.didactic and self.args.power is None and 'strapped' in self.args.alg:
win_x = np.array(self.agents.policy.train_steps)[:,
None] / 1000000.
win_y = np.array(self.agents.policy.differences)[:, None]
plot_win = pd.DataFrame(np.concatenate((win_x, win_y), axis=1),
columns=['T (mil)', self.args.which_diff])
sns.lineplot(x="T (mil)",
y=self.args.which_diff,
data=plot_win,
ax=ax1)
else:
win_x = np.array(self.evaluate_itr)[:, None] / 1000000.
win_y = np.array(self.win_rates)[:, None]
plot_win = pd.DataFrame(np.concatenate((win_x, win_y), axis=1),
columns=['T (mil)', 'Test Win'])
sns.lineplot(x="T (mil)", y="Test Win", data=plot_win, ax=ax1)
ax2 = fig.add_subplot(212)
reward_x = np.repeat(self.evaluate_itr,
self.args.evaluate_num)[:, None] / 1000000.
reward_y = np.array(self.episodes_rewards).flatten()[:, None]
plot_reward = pd.DataFrame(np.concatenate((reward_x, reward_y),
axis=1),
columns=['T (mil)', 'Median Test Returns'])
sns.lineplot(x="T (mil)",
y="Median Test Returns",
data=plot_reward,
ax=ax2,
ci='sd',
estimator=np.median)
plt.tight_layout()
# 格式化成2016-03-20-11_45_39形式
# tag = self.args.alg + '-' + time.strftime("%Y-%m-%d_%H-%M-%S", time.localtime())
tag = self.args.alg + '_' + str(self.args.target_update_period)
# if 'averaged' in self.args.alg:
tag += (self.alg_tag + '_' + datetime.utcnow().astimezone(
timezone(timedelta(hours=8))).strftime("%Y-%m-%d_%H-%M-%S"))
# 如果已经有图片就删掉
for filename in os.listdir(self.save_path):
if filename.endswith('.png'):
os.remove(self.save_path + '/' + filename)
fig.savefig(self.save_path + "/%s.png" % tag)
plt.close()
def get_eval_qtot(self):
"""
得到eval qtot
"""
self.env.reset()
all_last_action = np.zeros((self.args.n_agents, self.args.n_actions))
# 开始进行一波 episode
all_obs = self.env.get_obs()
state = self.env.get_state()
avail_actions = []
self.agents.policy.init_hidden(1)
eval_qs = []
actions = []
one_hot_actions = []
hidden_evals = None
for agent_idx in range(self.args.n_agents):
obs = all_obs[agent_idx]
last_action = all_last_action[agent_idx]
avail_action = self.env.get_avail_agent_actions(agent_idx)
avail_actions.append(avail_action)
onehot_agent_idx = np.zeros(self.args.n_agents)
onehot_agent_idx[agent_idx] = 1.
if self.args.last_action:
# 在水平方向上平铺
obs = np.hstack((obs, last_action))
if self.args.reuse_network:
obs = np.hstack((obs, onehot_agent_idx))
hidden_state = self.agents.policy.eval_hidden[:, agent_idx, :]
# 转置
obs = torch.Tensor(obs).unsqueeze(0)
# 是否使用 GPU
if self.args.cuda:
obs = obs.cuda()
hidden_state = hidden_state.cuda()
# 获取 Q(s, a)
qsa, hidden_eval = self.agents.policy.eval_rnn(obs, hidden_state)
qsa[avail_action == 0.0] = -float("inf")
eval_qs.append(torch.max(qsa))
action = torch.argmax(qsa)
actions.append(action)
onehot_action = np.zeros(self.args.n_actions)
onehot_action[action] = 1
one_hot_actions.append(onehot_action)
if hidden_evals is None:
hidden_evals = hidden_eval
else:
hidden_evals = torch.cat([hidden_evals, hidden_eval], dim=0)
s = torch.Tensor(state)
eval_qs = torch.Tensor(eval_qs).unsqueeze(0)
actions = torch.Tensor(actions).unsqueeze(0)
one_hot_actions = torch.Tensor(one_hot_actions).unsqueeze(0)
hidden_evals = hidden_evals.unsqueeze(0)
# 是否使用GPU
if self.args.cuda:
s = s.cuda()
eval_qs = eval_qs.cuda()
actions = actions.cuda()
one_hot_actions = one_hot_actions.cuda()
hidden_evals = hidden_evals.cuda()
# 计算Q_tot
eval_q_total = None
if self.args.alg == 'qatten':
eval_q_total, _, _ = self.agents.policy.eval_mix_net(
eval_qs, s, actions)
elif self.args.alg == 'qmix' \
or 'wqmix' in self.args.alg \
or 'strapped' in self.args.alg:
eval_q_total = self.agents.policy.eval_mix_net(eval_qs, s)
elif 'dmaq' in self.args.alg:
if self.args.alg == "dmaq_qatten":
ans_chosen, _, _ = self.agents.policy.mixer(eval_qs,
s,
is_v=True)
ans_adv, _, _ = self.agents.policy.mixer(
eval_qs,
s,
actions=one_hot_actions,
max_q_i=eval_qs,
is_v=False)
eval_q_total = ans_chosen + ans_adv
else:
ans_chosen = self.agents.policy.mixer(eval_qs, s, is_v=True)
ans_adv = self.agents.policy.mixer(eval_qs,
s,
actions=one_hot_actions,
max_q_i=eval_qs,
is_v=False)
eval_q_total = ans_chosen + ans_adv
elif self.args.alg == 'qtran_base':
one_hot_actions = one_hot_actions.unsqueeze(0)
hidden_evals = hidden_evals.unsqueeze(0)
eval_q_total = self.agents.policy.eval_joint_q(
s, hidden_evals, one_hot_actions)
eval_q_total = eval_q_total.squeeze().item()
return eval_q_total, 0
def her_k(self, episode, states, former_states):
import copy
for _ in range(self.args.her):
episode_buffer = {
'o':
np.zeros([
self.args.episode_limit, self.args.n_agents,
self.args.obs_shape
]),
's':
np.zeros([self.args.episode_limit, self.args.state_shape]),
'a':
np.zeros([self.args.episode_limit, self.args.n_agents, 1]),
'onehot_a':
np.zeros([
self.args.episode_limit, self.args.n_agents,
self.args.n_actions
]),
'avail_a':
np.zeros([
self.args.episode_limit, self.args.n_agents,
self.args.n_actions
]),
'r':
np.zeros([self.args.episode_limit, 1]),
'next_o':
np.zeros([
self.args.episode_limit, self.args.n_agents,
self.args.obs_shape
]),
'next_s':
np.zeros([self.args.episode_limit, self.args.state_shape]),
'next_avail_a':
np.zeros([
self.args.episode_limit, self.args.n_agents,
self.args.n_actions
]),
'done':
np.ones([self.args.episode_limit, 1]),
'padded':
np.ones([self.args.episode_limit, 1])
}
# 重新生成goal，order等信息
self.env.reset()
# 使用新生成的goals重构整个episode
for i in range(len(episode)):
reward = self.env.get_reward(states[i], former_states[i])
done = episode['done'][i]
if reward >= 0:
reward = 0
done = True
episode_buffer['o'][i] = episode['o'][i]
episode_buffer['o'][i, :, -2:] = np.array(self.env.goal)[:]
episode_buffer['s'][i] = episode['s'][i]
episode_buffer['a'][i] = episode['a'][i]
episode_buffer['onehot_a'][i] = episode['onehot_a'][i]
episode_buffer['avail_a'][i] = episode['avail_a'][i]
episode_buffer['r'][i] = [reward]
episode_buffer['next_o'][i] = episode['next_o'][i]
episode_buffer['next_o'][i, :,
-2:] = np.array(self.env.goal)[:]
episode_buffer['next_s'][i] = episode['next_s'][i]
episode_buffer['next_avail_a'][i] = episode['next_avail_a'][i]
episode_buffer['done'][i] = [done]
episode_buffer['padded'][i] = [0.]
if done:
break
for key in episode_buffer.keys():
episode_buffer[key] = np.array([episode_buffer[key]])
self.replay_buffer.store(episode_buffer)
class Trainer:
def __init__(self, params):
seed = params['general_params']['seed']
self.__set_seed(seed=seed)
env_params = params['env_params']
env_params['seed'] = seed
self.env = UnityEnv(params=env_params)
agent_params = params['agent_params']
self.__num_of_agents = self.env.observation_space.shape[0]
state_size = self.env.observation_space.shape[1]
action_size = self.env.action_space_size
agent_params['num_of_agents'] = self.__num_of_agents
agent_params['state_size'] = state_size
agent_params['action_size'] = action_size
self.agents = Agents(params=agent_params)
trainer_params = params['trainer_params']
self.learning_rate_decay = trainer_params['learning_rate_decay']
self.results_path = trainer_params['results_path']
self.model_path = trainer_params['model_path']
self.t_max = trainer_params['t_max']
self.exploration_noise = UOProcess()
# data gathering variables
self.avg_rewards = []
self.scores = []
self.score = 0
self.sigma = 0.5
print("MADDPG agent.")
print("Configuration:")
pprint(params)
logging.info("Configuration: {}".format(params))
def train(self, num_of_episodes):
logging.info("Training:")
reward_window = deque(maxlen=100)
for episode_i in range(1, num_of_episodes):
states = self.env.reset()
self.agents.reset(self.sigma)
scores = np.zeros(self.env.observation_space.shape[0])
total_loss = 0
self.sigma *= 0.99
counter = 0
for t in range(self.t_max):
actions = self.agents.choose_action(states)
next_states, rewards, dones, _ = self.env.step(actions)
self.agents.step(states, actions, rewards, next_states, dones)
states = next_states
# DEBUG
# logging.info("epsiode: {}, reward: {}, counter: {}, action: {}".
# format(episode_i, reward, counter, action))
total_loss += self.agents.agent_loss
scores += rewards
counter += 1
if any(dones):
break
reward_window.append(np.max(scores))
self.avg_rewards.append(np.mean(np.array(reward_window)))
print(
'\rEpisode {}\tCurrent Score: {:.4f}\tAverage Score: {:.4f} '
'\t\tTotal loss: {:.2f}\tLearning rate (actor): {:.4f}\tLearning rate (critic): {:.4f}'
.format(episode_i, np.max(scores), np.mean(reward_window),
total_loss, self.agents.learning_rate_actor,
self.agents.learning_rate_critic),
end="")
logging.info(
'Episode {}\tCurrent Score: {:.4f}\tAverage Score (over episodes): {:.4f} '
'\t\tTotal loss: {:.2f}\tLearning rate (actors): {:.4f}\tLearning rate (critic): {:.4f}'
.format(episode_i, np.max(scores), np.mean(reward_window),
total_loss, self.agents.learning_rate_actor,
self.agents.learning_rate_critic))
self.agents.learning_rate_actor *= self.learning_rate_decay
self.agents.learning_rate_critic *= self.learning_rate_decay
self.agents.set_learning_rate(self.agents.learning_rate_actor,
self.agents.learning_rate_critic)
if episode_i % 100 == 0:
avg_reward = np.mean(np.array(reward_window))
print("\rEpisode: {}\tAverage total reward: {:.2f}".format(
episode_i, avg_reward))
if avg_reward >= 1.0:
print(
'\nEnvironment solved in {:d} episodes!\tAverage Score: {:.2f}'
.format(episode_i - 100, avg_reward))
if not os.path.exists(self.model_path):
os.makedirs(self.model_path)
t = datetime.datetime.today().strftime('%Y-%m-%d_%H-%M')
torch.save(
self.agents.get_actor()[0].state_dict(),
self.model_path + 'checkpoint_actor1_{}.pth'.format(t))
torch.save(
self.agents.get_actor()[1].state_dict(),
self.model_path + 'checkpoint_actor2_{}.pth'.format(t))
torch.save(
self.agents.get_critic().state_dict(),
self.model_path + 'checkpoint_critic_{}.pth'.format(t))
np.array(self.avg_rewards).dump(
self.results_path + 'average_rewards_{}.dat'.format(t))
t = datetime.datetime.today().strftime('%Y-%m-%d_%H-%M')
# reward_matrix.dump(self.results_path + 'reward_matrix_new_{}.dat'.format(t))
np.array(self.avg_rewards).dump(self.results_path +
'average_rewards_{}.dat'.format(t))
def test(self,
checkpoint_actor1_filename,
checkpoint_actor2_filename,
checkpoint_critic_filename,
time_span=10):
checkpoint_actor1_path = self.model_path + checkpoint_actor1_filename
checkpoint_actor2_path = self.model_path + checkpoint_actor2_filename
checkpoint_critic_path = self.model_path + checkpoint_critic_filename
self.agents.get_actor()[0].load_state_dict(
torch.load(checkpoint_actor1_path))
self.agents.get_actor()[1].load_state_dict(
torch.load(checkpoint_actor2_path))
self.agents.get_critic().load_state_dict(
torch.load(checkpoint_critic_path))
for t in range(time_span):
state = self.env.reset(train_mode=False)
self.score = 0
#done = False
while True:
action = self.agents.choose_action(state, 'test')
state, reward, done, _ = self.env.step(action)
self.score += np.array(np.max(reward))
if any(done):
break
print('\nFinal score:', self.score)
self.env.close()
@staticmethod
def __set_seed(seed):
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
random.seed(seed)
np.random.seed(seed)
class Runner:
def __init__(self, env, args, itr):
# 获取参数
# self.args = get_common_args()
self.args = args
# 获取环境
self.env = env
# 进程编号
self.pid = itr
self.agents = Agents(args, itr=itr)
# 不复用网络，就会有多个agent，训练的时候共享参数，就是一个网络
# if not self.args.reuse_network:
# self.agents = []
# for i in range(self.args.n_agents):
# self.agents.append(Agents(self.args, i))
# self.rollout = RollOut(self.agents, self.args)
self.replay_buffer = ReplayBuffer(self.args)
self.win_rates = []
'''
这里，episode_reward 代表一个episode的累加奖赏，
episodes_reward代表多个episode的累加奖赏，
episodes_rewards代表多次评价的多个episode的累加奖赏
'''
self.episodes_rewards = []
self.evaluate_itr = []
self.max_win_rate = 0
# 保存结果和模型的位置，增加计数，帮助一次运行多个实例
self.save_path = self.args.result_dir + '/' + self.args.alg + '/' + self.args.map + '/' + str(
itr)
if not os.path.exists(self.save_path):
os.makedirs(self.save_path)
print('runner 初始化')
def generate_episode(self, episode_num, evaluate=False):
# 为保存评价的回放做准备
if self.args.replay_dir != '' and evaluate and episode_num == 0:
self.env.close()
# 变量初始化
self.env.reset()
done = False
info = None
win = False
last_action = np.zeros((self.args.n_agents, self.args.n_actions))
# epsilon 递减
epsilon = 0 if evaluate else self.args.epsilon
# epsilon 递减的方式
if self.args.epsilon_anneal_scale == 'episode' or \
(self.args.epsilon_anneal_scale == 'itr' and episode_num == 0):
epsilon = epsilon - self.args.epsilon_decay if epsilon > self.args.min_epsilon else epsilon
# 记录一个episode的信息
episode_buffer = None
if not evaluate:
episode_buffer = {
'o':
np.zeros([
self.args.episode_limit, self.args.n_agents,
self.args.obs_shape
]),
's':
np.zeros([self.args.episode_limit, self.args.state_shape]),
'a':
np.zeros([self.args.episode_limit, self.args.n_agents, 1]),
'onehot_a':
np.zeros([
self.args.episode_limit, self.args.n_agents,
self.args.n_actions
]),
'avail_a':
np.zeros([
self.args.episode_limit, self.args.n_agents,
self.args.n_actions
]),
'r':
np.zeros([self.args.episode_limit, 1]),
'next_o':
np.zeros([
self.args.episode_limit, self.args.n_agents,
self.args.obs_shape
]),
'next_s':
np.zeros([self.args.episode_limit, self.args.state_shape]),
'next_avail_a':
np.zeros([
self.args.episode_limit, self.args.n_agents,
self.args.n_actions
]),
'done':
np.ones([self.args.episode_limit, 1]),
'padded':
np.ones([self.args.episode_limit, 1])
}
# 开始进行一波 episode
obs = self.env.get_obs()
state = self.env.get_state()
avail_actions = []
self.agents.policy.init_hidden(1)
for agent_id in range(self.args.n_agents):
avail_action = self.env.get_avail_agent_actions(agent_id)
avail_actions.append(avail_action)
episode_reward = 0
for step in range(self.args.episode_limit):
if done:
break
else:
actions, onehot_actions = [], []
for agent_id in range(self.args.n_agents):
# avail_action = self.env.get_avail_agent_actions(agent_id)
action = self.agents.choose_action(obs[agent_id],
last_action[agent_id],
agent_id,
avail_actions[agent_id],
epsilon, evaluate)
# 得到该动作的独热编码
onehot_action = np.zeros(self.args.n_actions)
onehot_action[action] = 1
onehot_actions.append(onehot_action)
# 加入联合动作
actions.append(action)
# avail_actions.append(avail_action)
# 记录该动作
last_action[agent_id] = onehot_action
# 对环境执行联合动作
reward, done, info = self.env.step(actions)
# 获取改变后的信息
if not done:
next_obs = self.env.get_obs()
next_state = self.env.get_state()
else:
next_obs = obs
next_state = state
# 添加可得动作
next_avail_actions = []
for agent_id in range(self.args.n_agents):
avail_action = self.env.get_avail_agent_actions(agent_id)
next_avail_actions.append(avail_action)
# 添加经验
if not evaluate:
episode_buffer['o'][step] = obs
episode_buffer['s'][step] = state
episode_buffer['a'][step] = np.reshape(
actions, [self.args.n_agents, 1])
episode_buffer['onehot_a'][step] = onehot_actions
episode_buffer['avail_a'][step] = avail_actions
episode_buffer['r'][step] = [reward]
episode_buffer['next_o'][step] = next_obs
episode_buffer['next_s'][step] = next_state
episode_buffer['next_avail_a'][step] = next_avail_actions
episode_buffer['done'][step] = [done]
episode_buffer['padded'][step] = [0.]
# 更新变量
episode_reward += reward
obs = next_obs
state = next_state
avail_actions = next_avail_actions
if self.args.epsilon_anneal_scale == 'step':
epsilon = epsilon - self.args.epsilon_decay if epsilon > self.args.min_epsilon else epsilon
# 是训练则记录新的epsilon
if not evaluate:
self.args.epsilon = epsilon
# 获取对局信息
if info.__contains__('battle_won'):
win = True if done and info['battle_won'] else False
if evaluate and episode_num == self.args.evaluate_num - 1 and self.args.replay_dir != '':
self.env.save_replay()
self.env.close()
return episode_buffer, episode_reward, win
def run(self):
train_steps = 0
early_stop = 10
num_eval = 0
self.max_win_rate = 0
for itr in range(self.args.n_itr):
# 收集 n_episodes 的数据
episode_batch, _, _ = self.generate_episode(0)
for key in episode_batch.keys():
episode_batch[key] = np.array([episode_batch[key]])
for e in range(1, self.args.n_episodes):
episode, _, _ = self.generate_episode(e)
for key in episode_batch.keys():
episode[key] = np.array([episode[key]])
episode_batch[key] = np.concatenate(
(episode_batch[key], episode[key]), axis=0)
# 添加到 replay buffer
self.replay_buffer.store(episode_batch)
# 训练
if self.replay_buffer.size < self.args.batch_size * 12.5:
# print('replay buffer 还没 batch size 大')
continue
for _ in range(self.args.train_steps):
batch = self.replay_buffer.sample(self.args.batch_size)
self.agents.train(batch, train_steps)
# if self.args.reuse_network:
# self.agents.train(batch, train_steps)
# else:
# for i in range(self.args.n_agents):
# self.agents[i].train(batch, train_steps)
train_steps += 1
# 周期性评价
if itr % self.args.evaluation_period == 0:
num_eval += 1
print(f'进程 {self.pid}: {itr} / {self.args.n_itr}')
win_rate, episodes_reward = self.evaluate()
# 保存测试结果
self.evaluate_itr.append(itr)
self.win_rates.append(win_rate)
self.episodes_rewards.append(episodes_reward)
# 表现好的模型要额外保存
if win_rate > self.max_win_rate:
self.max_win_rate = win_rate
self.agents.policy.save_model(str(win_rate))
# 不时刻保存，从而减少时间花费
if num_eval % 50 == 0:
self.save_results()
self.plot()
# 最后把所有的都保存一下
self.save_results()
self.plot()
self.env.close()
def evaluate(self):
"""
得到平均胜率和每次测试的累加奖赏，方便画误差阴影图
:return:
"""
win_number = 0
episodes_reward = []
for itr in range(self.args.evaluate_num):
_, episode_reward, win = self.generate_episode(itr, evaluate=True)
episodes_reward.append(episode_reward)
if win:
win_number += 1
return win_number / self.args.evaluate_num, episodes_reward
def save_results(self):
"""
保存数据，方便后面多种算法结果画在一张图里比较
:return:
"""
# 如果已经有图片就删掉
for filename in os.listdir(self.save_path):
if filename.endswith('.npy'):
os.remove(self.save_path + '/' + filename)
np.save(self.save_path + '/evaluate_itr.npy', self.evaluate_itr)
np.save(self.save_path + '/win_rates.npy', self.win_rates)
np.save(self.save_path + '/episodes_rewards.npy',
self.episodes_rewards)
def plot(self):
"""
定期绘图
:return:
"""
fig = plt.figure()
ax1 = fig.add_subplot(211)
win_x = np.array(self.evaluate_itr)[:, None]
win_y = np.array(self.win_rates)[:, None]
plot_win = pd.DataFrame(np.concatenate((win_x, win_y), axis=1),
columns=['evaluate_itr', 'win_rates'])
sns.lineplot(x="evaluate_itr", y="win_rates", data=plot_win, ax=ax1)
ax2 = fig.add_subplot(212)
reward_x = np.repeat(self.evaluate_itr, self.args.evaluate_num)[:,
None]
reward_y = np.array(self.episodes_rewards).flatten()[:, None]
plot_reward = pd.DataFrame(
np.concatenate((reward_x, reward_y), axis=1),
columns=['evaluate_itr', 'episodes_rewards'])
sns.lineplot(x="evaluate_itr",
y="episodes_rewards",
data=plot_reward,
ax=ax2,
ci=68,
estimator=np.median)
# 格式化成2016-03-20-11_45_39形式
tag = self.args.alg + '-' + time.strftime("%Y-%m-%d_%H-%M-%S",
time.localtime())
# 如果已经有图片就删掉
for filename in os.listdir(self.save_path):
if filename.endswith('.png'):
os.remove(self.save_path + '/' + filename)
fig.savefig(self.save_path + "/%s.png" % tag)
plt.close()
num_agents = len(env_info.agents)
print('Number of agents:', num_agents)
# size of each action
action_size = brain.vector_action_space_size
print('Size of each action:', action_size)
# examine the state space
states = env_info.vector_observations
state_size = states.shape[1]
print('There are {} agents. Each observes a state with length: {}'.format(
states.shape[0], state_size))
print('The state for the first agent looks like:', states[0])
agents = Agents(num_agents=num_agents,
state_size=state_size,
action_size=action_size,
random_seed=0)
scores = ddpg(env,
brain_name,
agents,
n_episodes=n_episodes,
eps_start=eps_start,
eps_end=eps_end,
eps_decay=eps_decay,
resume=resume)
# plot the scores
plt.plot(np.arange(1, len(scores) + 1), np.mean(scores, axis=-1))
plt.ylabel('Score')
plt.xlabel('Episode #')
plt.show()
def import_agent_file(scenario_settings, con, cur, engine, model_settings,
agent_file_status, input_name):
"""
Generates new agents or uses pre-generated agents from provided .pkl file
Parameters
----------
**scenario_settings** : 'SQL schema'
Schema of the scenario settings
**con** : 'SQL connection'
SQL connection to connect to database
**cur** : 'SQL cursor'
Cursor
**engine** : 'SQL engine'
SQL engine to intepret SQL query
**model_settings** : 'object'
Model settings that apply to all scenarios
**agent_file_status** : 'attribute'
Attribute that describes whether to use pre-generated agent file or create new
**input_name** : 'string'
.Pkl file name substring of pre-generated agent table
Returns
-------
**solar_agents** : 'Class'
Instance of Agents class with either user pre-generated or new data
"""
schema = scenario_settings.schema
input_agent_dir = model_settings.input_agent_dir
state_to_model = scenario_settings.state_to_model
if agent_file_status == 'Use pre-generated Agents':
userdefined_table_name = 'input_' + input_name + '_user_defined'
scenario_userdefined_name = get_userdefined_scenario_settings(
schema, userdefined_table_name, con)
scenario_userdefined_value = scenario_userdefined_name['val'].values[0]
agents_df = pd.read_pickle(
os.path.join(input_agent_dir, scenario_userdefined_value + '.pkl'))
agents_df = agents_df[agents_df['state_abbr'].isin(state_to_model)]
if agents_df.empty:
raise ValueError(
'Region not present within pre-generated agent file - Edit Inputsheet'
)
# Convert dtypes of specific columns to floats
cols = [
'customers_in_bin_initial', 'load_kwh_per_customer_in_bin_initial',
'load_kwh_in_bin_initial', 'max_demand_kw', 'avg_monthly_kwh',
'cap_cost_multiplier', 'developable_roof_sqft',
'pct_of_bldgs_developable'
]
agents_df[cols] = agents_df[cols].astype(float)
agents = Agents(agents_df)
# Re-assign tariffs to agents with incompatible rate units
agents.on_frame(agent_mutation.elec.reassign_agent_tariffs, con)
# For wind: Ingest agent core attributes to database from pickle file
if scenario_settings.techs in [['wind']]:
ingest_agent_core_attributes(agents.df, con, cur, engine, schema,
model_settings.role,
scenario_settings.sectors,
scenario_settings.techs)
else:
raise ValueError(
'Generating agents is not supported at this time. Please select "Use pre-generated Agents" in the input sheet'
)
return agents