def HSRA(D, D_, pb1, pt1, ps1, pb2, ps2, iteration=int(1e6)): aloha = ALOHA_AGENT(D=D, arrival_rate=pb1, trans_prob=pt1) aloha.initialize() HSRA_agent = HSRA_AGENT(D=D_, arrival_rate=pb2, learning_rate=0.01, gamma=0.9, length=1) HSRA_agent.initailize() env = ENVIRONMENT(aloha_channel=ps1, agent_channel=ps2, aloha=aloha, agent=HSRA_agent) HSRA_reward = [] # begin = time.time() for _ in tqdm(range(iteration)): aloha_reward, agent_reward, observation = env.step() env.aloha.update(observation=observation) env.agent.update(observation=observation) HSRA_reward.append(aloha_reward + agent_reward) HSRA_timely_throughput = np.mean(HSRA_reward[-int(1e5):]) print('HSRA_timely_throughput:', HSRA_timely_throughput)
def main(N1, N2, D, parameter, iteration=int(1e5)): agent_list = [ALOHA_AGENT(D=D, arrival_rate=parameter[i], trans_prob=1/N2) \ for i in range(N2)] # parameterss pb2 n1_list = [ ALOHA_AGENT(D=D, arrival_rate=0.5, trans_prob=1 / (4 * N1)) for _ in range(N1) ] agent_list.extend(n1_list) channels = list(parameter[N2:]) # parameters ps2 n1_channels = [0.5 for _ in range(N1)] channels.extend(n1_channels) env = ENVIRONMENT(channels=channels, agent_list=agent_list) reward_list = [] energy_list = [] for t in tqdm(range(iteration)): reward, energy, observations = env.step(time=t) for i in range(N1 + N2): env.agent_list[i].update(observation=observations[i]) reward_list.append(reward) energy_list.append(energy) throughput, power = np.mean(reward_list[-int(1e4):]), np.mean( energy_list[-int(1e4):]) print('Throu = {}'.format(throughput)) print('Energy = {}'.format(power)) return throughput, power
def main(N, D, parameter, iteration=int(1e5)): agent_list = [HSRA_AGENT(D=D, arrival_rate=parameter[i], learning_rate=0.01, gamma=0.9, length=1) \ for i in range(N)] # parameterss pb2 channels = parameter[N:] # parameters ps2 env = ENVIRONMENT(channels=channels, agent_list=agent_list) reward_list = [] energy_list = [] for time in tqdm(range(iteration)): reward, energy, observations = env.step(time=time) for i in range(N): env.agent_list[i].update(observation=observations[i], time=time, N=N) reward_list.append(reward) energy_list.append(energy) throughput, power = np.mean(reward_list[-int(1e4):]), np.mean( energy_list[-int(1e4):]) print('Throu = {}'.format(throughput)) print('Energy = {}'.format(power)) return throughput, power
def upper_bound(D, D_, pb1, pt1, ps1, pb2, ps2, iteration=int(1e5)): aloha = ALOHA_AGENT(D=D, arrival_rate=pb1, trans_prob=pt1) aloha.initialize() # get LP agent policy LP_policy = multichainLP(D=D, D_=D_, pb1=pb1, pt1=pt1, ps1=ps1, pb2=pb2, ps2=ps2) sp_agent = SPECIFY_AGENT(D=D_, arrival_rate=pb2, policy=LP_policy) sp_agent.initialize() env = ENVIRONMENT(aloha_channel=ps1, agent_channel=ps2, aloha=aloha, agent=sp_agent) UP_reward = [] for _ in tqdm(range(iteration)): aloha_reward, agent_reward, observation = env.step() env.aloha.update(observation=observation) env.agent.update(observation=observation, aloha_queue=env.aloha.queue) UP_reward.append(aloha_reward + agent_reward) Upper_bound_timely_throughput = np.mean(UP_reward) print('Upper_bound_timely_throughput:', Upper_bound_timely_throughput)
def DLMA_FNN(D, D_, pb1, pt1, ps1, pb2, ps2, iteration=int(1e5)): aloha = ALOHA_AGENT(D=D, arrival_rate=pb1, trans_prob=pt1) aloha.initialize() DLMA = DQN_AGENT(D=D_, arrival_rate=pb2, state_size=160, n_actions=2, n_nodes=2, memory_size=1000, replace_target_iter=20, batch_size=64, learning_rate=0.01, gamma=0.9, epsilon=1, epsilon_min=0.005, epsilon_decay=0.995, alpha=0) DLMA.initailize() env = ENVIRONMENT(aloha_channel=ps1, agent_channel=ps2, aloha=aloha, agent=DLMA) state = [0] * DLMA.state_size DLMA_FNN_reward = [] begin = time.time() for i in tqdm(range(iteration)): aloha_reward, agent_reward, observation = env.step() env.aloha.update(observation) env.agent.update(observation, state) DLMA_FNN_reward.append(aloha_reward + agent_reward) next_state = state[8:] + return_action( env.agent.action) + return_observation(observation) + [ agent_reward, aloha_reward ] env.agent.store_transition(state, env.agent.action, agent_reward, aloha_reward, next_state) if i > 100 and (i % 5 == 0): env.agent.learn() # internally iterates default (prediction) model state = next_state DLMA_FNN_timely_throughput = np.mean(DLMA_FNN_reward) print('DLMA_FNN_timely_throughput:', DLMA_FNN_timely_throughput) end = time.time() print(u'当前进程的运行时间: ', (end - begin), 's') print(u'当前进程的内存使用:%.4f MB' % (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024))
def FSRA(D, D_, pb1, pt1, ps1, pb2, ps2, iteration=int(1e7)): aloha = ALOHA_AGENT(D=D, arrival_rate=pb1, trans_prob=pt1) aloha.initialize() FSRA_agent = FSRA_AGENT(D=D_, arrival_rate=pb2, learning_rate=0.01) FSRA_agent.initailize() env = ENVIRONMENT(aloha_channel=ps1, agent_channel=ps2, aloha=aloha, agent=FSRA_agent) FSRA_reward = [] for _ in tqdm(range(iteration)): aloha_reward, agent_reward ,observation = env.step() env.aloha.update(observation=observation) env.agent.update(observation=observation) FSRA_reward.append(aloha_reward + agent_reward) FSRA_timely_throughput = np.mean(FSRA_reward[-int(1e5):]) print('FSRA_timely_throughput:', FSRA_timely_throughput)
def main(n2, D, parameter, iteration=int(1e5)): agent_list = [DQN_AGENT(D=D, arrival_rate=parameter[i], state_size=int(8*M), n_actions=2, n_nodes=2, memory_size=E, replace_target_iter=F, batch_size=B, ) for i in range(n2)] env = ENVIRONMENT(channels=parameter[n2:], agent_list=agent_list) reward_list = [] energy_list = [] state = [[0] * int(8*M) for _ in range(n2)] next_state = [[0] * int(8*M) for _ in range(n2)] for t in tqdm(range(iteration)): for i in range(n2): env.agent_list[i].choose_action(np.array(state[i])) reward, energy, observations = env.step(time=t) reward_list.append(reward) energy_list.append(energy) for i in range(n2): env.agent_list[i].update_queue(observation=observations[i]) next_state[i], agent_reward, others_reward = return_next_state(i, state[i], env.agent_list, observations, reward) env.agent_list[i].store_transition(state[i], env.agent_list[i].action, agent_reward, others_reward, next_state[i]) if t > 100 and t % 5 == 0: for i in range(n2): env.agent_list[i].learn() state = copy.deepcopy(next_state) throughput, power = np.mean(reward_list[-int(1e4):]), np.mean(energy_list[-int(1e4):]) print('Throu = {}'.format(throughput)) print('Energy = {}'.format(power)) return throughput, power
def TSRA(D, D_, pb1, pt1, ps1, pb2, ps2, iteration=int(1e5)): aloha = ALOHA_AGENT(D=D, arrival_rate=pb1, trans_prob=pt1) TSRA_agent = TSRA_AGENT(D=D_, arrival_rate=pb2, learning_rate=0.01, gamma=0.9, length=1) env = ENVIRONMENT(aloha_channel=ps1, agent_channel=ps2, aloha=aloha, agent=TSRA_agent) TSRA_reward = [] begin = time.time() for _ in tqdm(range(iteration)): aloha_reward, agent_reward ,observation = env.step() env.aloha.update(observation=observation) env.agent.update(observation=observation) TSRA_reward.append(aloha_reward + agent_reward) TSRA_timely_throughput = np.mean(TSRA_reward) print('TSRA_timely_throughput:', TSRA_timely_throughput) end = time.time() print('time: ' , (end - begin), 's') print('memory: %.4f MB' % (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024) )
def DLMA_RNN(D, D_, pb1, pt1, ps1, pb2, ps2, iteration=int(1e5)): aloha = ALOHA_AGENT(D=D, arrival_rate=pb1, trans_prob=pt1) aloha.initialize() DLMA = DQN(D=D_, arrival_rate=pb2, features=8, n_actions=2, n_nodes=2, state_length=4, memory_size=1000, replace_target_iter=20, batch_size=64, learning_rate=0.01, gamma=0.9, epsilon=1, epsilon_min=0.005, epsilon_decay=0.995, alpha=0) DLMA.initailize() env = ENVIRONMENT(aloha_channel=ps1, agent_channel=ps2, aloha=aloha, agent=DLMA) channel_state = [0] * DLMA.features state = np.zeros((4, len(channel_state))) DLMA_RNN_reward = [] begin = time.time() for i in tqdm(range(iteration)): state = np.vstack([state[1:], channel_state]) aloha_reward, agent_reward, observation = env.step() env.aloha.update(observation) env.agent.update(observation, state) DLMA_RNN_reward.append(aloha_reward + agent_reward) next_channel_state = return_action( env.agent.action) + return_observation(observation) + [ agent_reward, agent_reward ] experience = np.concatenate([ channel_state, [env.agent.action, agent_reward, agent_reward], next_channel_state ]) env.agent.add_experience(experience) if i > 100 and (i % 5 == 0): env.agent.learn() # internally iterates default (prediction) model channel_state = next_channel_state DLMA_RNN_timely_throughput = np.mean(DLMA_RNN_reward) print('DLMA_RNN_timely_throughput:', DLMA_RNN_timely_throughput) end = time.time() print('time: ', (end - begin), 's') print('memory: %.4f MB' % (psutil.Process(os.getpid()).memory_info().rss / 1024 / 1024))