index += 1 print("seed: ", i) seed = i resource_coefficient = resource_coefficient_original * number_of_tasks / time_length print("resource coefficient: ", resource_coefficient) # generate synthetic data for the simulation df_tasks, df_nodes, n_time, n_tasks, n_nodes = \ generate_synthetic_data_edge_cloud(n_tasks=number_of_tasks, n_time=time_length, seed=seed, n_nodes=n_nodes, p_high_value_tasks=high_value_proportion, high_value_slackness_lower_limit=high_value_slackness, high_value_slackness_upper_limit=high_value_slackness + 2, low_value_slackness_lower_limit=low_value_slackness, low_value_slackness_upper_limit=low_value_slackness + 2, resource_demand_high=resource_ratio, vc_ratio=valuation_coefficient_ratio, k_resource=resource_coefficient) df_tasks = df_tasks.rename(columns={"storage": "DISK"}) df_nodes = df_nodes.rename(columns={ "storage": "DISK", "storage_cost": "DISK_cost" }) print(f"low value slackness = {low_value_slackness}") print(f"high value slackness = {high_value_slackness}") print("df_tasks:") print(df_tasks.head())
n_tasks = 300 n_time = int(n_tasks / 10) seed = 0 n_nodes = 6 resource_coefficient_original = 0.3 resource_coefficient = (resource_coefficient_original * n_tasks / n_time) high_value_slackness = 0 low_value_slackness = 6 valuation_ratio = 3 # generate synthetic data for the simulation df_tasks, df_nodes, n_time, n_tasks, n_nodes = \ generate_synthetic_data_edge_cloud(n_tasks=n_tasks, n_time=n_time, n_nodes=n_nodes, high_value_slackness_lower_limit=high_value_slackness, high_value_slackness_upper_limit=high_value_slackness + 2, low_value_slackness_lower_limit=low_value_slackness, low_value_slackness_upper_limit=low_value_slackness + 2, k_resource=resource_coefficient, vc_ratio=valuation_ratio, seed=seed) df_tasks = df_tasks.rename(columns={"storage": "DISK"}) df_nodes = df_nodes.rename( columns={"storage": "DISK", "storage_cost": "DISK_cost"}) print("df_tasks:") print(df_tasks.head()) print("df_nodes:") print(df_nodes.head()) # run Offline Optimal algo. social_welfare, social_welfare_solution, number_of_allocated_tasks, allocation_scheme = \
def run_sarsa(number_of_runs=50, total_number_of_steps=500, alpha=0.001, beta=0.1, epsilon=0.1, plot_bool=False): # # set some parameters # number_of_runs = 50 # total_number_of_steps = 500 # alpha = 0.003 # beta = 0.06 # epsilon = 0.1 # run the trials result_sarsa_list = [] for j in range(number_of_runs): try: print(f"run ID = {j}") # initialise some parameters n_steps = total_number_of_steps + 1 np.random.seed(j) # generate the two types of tasks number_of_tasks = 2 V = {} # value function: state -> value pi = {} # policy: (state+price) -> probability actions = [0, 1, 2, 3] # [reject, 0.3, 0.6, 1.0] of the value of the task # generate a seqence of analytics tasks k_resource = 7 df_tasks, df_nodes, n_time, n_tasks, n_nodes = \ generate_synthetic_data_edge_cloud(n_tasks=number_of_tasks, n_time=10, k_resource=k_resource, p_high_value_tasks=0.5, resource_demand_low=5, seed=1) # print("df_tasks:") # print(df_tasks) # normalize the state (df_tasks) df_tasks_normalised = df_tasks.copy() df_tasks_normalised['deadline'] = df_tasks_normalised['deadline'] - \ df_tasks_normalised['start_time'] + 1 df_tasks_normalised = df_tasks_normalised / (df_tasks_normalised.max()) # print("normalised df_tasks") # print(df_tasks_normalised) # generate one sample of tasks list_sample = [] list_sample_normalised = [] sample_sequence = np.random.randint(0, 2, n_steps) list_tasks_normalised = df_tasks_normalised.values.tolist() df1_normalised = list_tasks_normalised[0] # type one df2_normalised = list_tasks_normalised[1] # type two list_tasks = df_tasks.values.tolist() df1 = list_tasks[0] df2 = list_tasks[1] # print(df1_normalised) # print(df2_normalised) for i in sample_sequence: if i == 0: list_sample.append(df1) list_sample_normalised.append(df1_normalised) else: list_sample.append(df2) list_sample_normalised.append(df2_normalised) # convert to a dataframe # print("A sample of tasks") # print(list_sample) df_sample = pd.DataFrame(list_sample, columns=df_tasks.columns) df_sample_normalised = pd.DataFrame(list_sample_normalised, columns=df_tasks_normalised.columns) # print("the data frame of a sample of tasks") # print(df_sample) # print("the data frame of a sample of tasks (normalised)") # print(df_sample_normalised) current_time = 0 for i in range(len(df_sample.index)): df_sample.loc[i, "arrive_time"] += current_time df_sample.loc[i, "start_time"] += current_time + int() df_sample.loc[i, "deadline"] += current_time current_time = int(df_sample.loc[i, "arrive_time"] + 1) # print(df_sample.head()) average_reward_sarsa_list = [] # with tqdm(total=100) as pbar: mdp = ReverseAuctionMDP(df_tasks, df_nodes, k=1) # only one fog node agent = FogNodeAgent(mdp=mdp, n_steps=n_steps - 1, alpha=alpha, beta=beta, fog_index=0, df_tasks=df_sample, df_tasks_normalised=df_sample_normalised, df_nodes=df_nodes, epsilon=epsilon) agent.run_episode() total_reward = agent.total_reward # print(f"total rewards = {total_reward}") # print(f"list of rewards: {agent.reward_list}") # max_vc = 564.06 # average_reward = total_reward * max_vc / (n_steps-1) # average_reward_sarsa_list.append(average_reward) # pbar.update(1) max_vc = 564.06 reward_list = [i * max_vc for i in agent.reward_list] average_reward_sarsa_list = [] average_reward = 0 for i in range(n_steps - 1): average_reward = reward_list[i] / (i + 1) + (i) / (i + 1) * average_reward average_reward_sarsa_list.append(average_reward) result_sarsa_list.append(average_reward_sarsa_list.copy()) except Exception: # just continue if some runs fail pass if plot_bool == True: result_sarsa_y = [item for sublist in result_sarsa_list for item in sublist] result_sarsa_y = pd.Series(result_sarsa_y) print(result_sarsa_y) # plot the result sns.lineplot(data=result_sarsa_y) plt.show() else: # save the result for jupyter notebook with open(f'test{alpha}{beta}{epsilon}.txt', 'w') as f: f.write(json.dumps(result_sarsa_list))
# Restore def enable_print(): sys.stdout = stdout # set the parameters n_tasks = 100 n_time = 10 seed = 3 n_nodes = 6 # generate synthetic data for the simulation df_tasks, df_nodes, n_time, n_tasks, n_nodes = \ generate_synthetic_data_edge_cloud(n_tasks=n_tasks, n_time=n_time, n_nodes=n_nodes, k_resource=3, seed=seed) df_tasks = df_tasks.rename(columns={"storage": "DISK"}) df_nodes = df_nodes.rename(columns={ "storage": "DISK", "storage_cost": "DISK_cost" }) print("df_tasks:") print(df_tasks.head()) print("df_nodes:") print(df_nodes.head()) # run Optimal Pricing algo. # set price range
def execute_multi_agent_sarsa(avg_resource_capacity, avg_unit_cost, number_of_runs=50, total_number_of_steps=500, num_fog_nodes=6, resource_coefficient_original=3, plot_bool=False, num_actions=4, time_length=100, high_value_proportion=0.2, high_value_slackness=0, low_value_slackness=6, valuation_coefficient_ratio=10, resource_ratio=1.2, agents_list=None, bool_decay=True, training_seed=0, verbose=False, auction_type="second-price"): """execute multi-agent sarsa Args: agents_list: a list of trained agents :param number_of_runs: number of trials :param total_number_of_steps: steps of RL (allocate how many tasks) :param num_fog_nodes: number of fog nodes :param resource_coefficient_original: a coefficient for computing the resource coefficient :param alpha: step size of weights :param beta: step size of estimated average rewards :param epsilon_tuple: probability of exploration :param epsilon_steps_tuple: number of steps to run for each epsilon except for the last one :param plot_bool: whether plot the results :param num_actions: number of actions :param time_length: tasks arrive within this time length :param low_value_slackness: deadline slackness of low-value tasks :param resource_ratio: resource demand ratio between high-value and low-value tasks :param valuation_coefficient_ratio: valuation coefficient ratio between high-value and low-value tasks :param high_value_slackness: deadline slackness of high-value tasks :param high_value_proportion: the proportion of high-value tasks """ # run the trials result_sarsa_list = [ ] # a list of the lists of average rewards for each step of sarsa social_welfare_list = [] # a list of total social welfare of each trial # record the allocation scheme allocation_scheme = pd.DataFrame( columns=['node_id', 'start_time', 'end_time']) for j in tqdm(range(number_of_runs)): # run all the trials # for j in tqdm(range(number_of_runs - 1, number_of_runs)): # just run one trial if verbose: print(f"run ID = {j}") sw_list = [] # a list of social welfare after a new task arrives # initialise some parameters n_steps = total_number_of_steps + 1 np.random.seed(j) # generate the two types of tasks number_of_tasks = total_number_of_steps V = {} # value function: state -> value pi = {} # policy: (state+price) -> probability actions = list(range( num_actions)) # bid [reject, 1/3, 2/3, 1] the value of the task # generate a seqence of analytics tasks # compute the resource coefficient resource_coefficient = (resource_coefficient_original * number_of_tasks / time_length) # generate the synthetic data for simulations df_tasks, df_nodes, n_time, n_tasks, num_fog_nodes = \ generate_synthetic_data_edge_cloud(avg_resource_capacity, avg_unit_cost, n_tasks=total_number_of_steps, n_time=time_length, seed=j, n_nodes=num_fog_nodes, p_high_value_tasks=high_value_proportion, high_value_slackness_lower_limit=high_value_slackness, high_value_slackness_upper_limit=high_value_slackness + 2, low_value_slackness_lower_limit=low_value_slackness, low_value_slackness_upper_limit=low_value_slackness + 2, resource_demand_high=resource_ratio, vc_ratio=valuation_coefficient_ratio, k_resource=resource_coefficient) if verbose: print("resource coefficient: ", resource_coefficient) print(f"low value slackness = {low_value_slackness}") print(f"high value slackness = {high_value_slackness}") print("df_tasks:") print(df_tasks.head()) print("df_nodes:") print(df_nodes.head()) # print the total value of tasks total_value = 0 for i in range(total_number_of_steps): total_value += (df_tasks.loc[i, "valuation_coefficient"] * df_tasks.loc[i, "usage_time"]) if verbose: print(f"total_number_of_steps={total_number_of_steps}") print(f"total value of tasks = {total_value}") # with tqdm(total=100) as pbar: mdp = ReverseAuctionMDP(df_tasks, df_nodes, num_nodes=num_fog_nodes, num_actions=num_actions) # several fog nodes # reset the states of the fog node agents for i in range(mdp.num_fog_nodes): agents_list[i].reset_state(df_tasks) # actions taken by each node actions = {i: [] for i in range(mdp.num_fog_nodes)} # the reverse reverse_auction for k in tqdm(range(total_number_of_steps)): # fog nodes decide their bidding price, and allocation scheme for the current task if verbose: print() print(f"step: {k}") bids_list = [] # bidding price for one time step max_usage_time_list = [] # maximum usage time a fog node can offer start_time_list = [ ] # start time according to the planned allocation relative_start_time_list = [ ] # relative start time according to the current task for i in range(mdp.num_fog_nodes): (bidding_price, max_usage_time, relative_start_time, action) = \ agents_list[i].differential_sarsa_decide_action( verbose=verbose) # tranfer relative start_time to absolute start_time start_time = int(df_tasks.loc[k, 'arrive_time'] + relative_start_time + 1) bids_list.append(bidding_price) max_usage_time_list.append(max_usage_time) start_time_list.append(start_time) relative_start_time_list.append(relative_start_time) actions[i].append(action) # find the winner (winner_index, winner_num_time, winner_utility, max_utility) = \ mdp.step(bids_list, max_usage_time_list, start_time_list, verbose=verbose, auction_type=auction_type) if verbose: print() print(f"nodes' bids = {bids_list}") print(f"nodes' usage times = {max_usage_time_list}") print(f"nodes' start times = {start_time_list}") print(f"winner's index = {winner_index}") print(f"number of usage time = {winner_num_time}") print(f"winner's utility = {winner_utility}") print(f"user's utility = {max_utility}") # a list of social welfare after a new task arrives sw_list.append(mdp.social_welfare) # modify the overall allocation scheme if winner_num_time is not None and winner_num_time > 0: allocation_scheme.loc[k] = [ winner_index, start_time_list[winner_index], start_time_list[winner_index] + winner_num_time - 1 ] else: # the task is rejected allocation_scheme.loc[k] = [None, None, None] # Do not update weights during execution if k < total_number_of_steps - 1: # update sarsa weights for i in range(mdp.num_fog_nodes): if i == winner_index: # if fog node i wins this task agents_list[i].differential_sarsa_update_weights( 1, max_usage_time_list[i], relative_start_time_list[i], winner_revenue=winner_utility, bool_update_weights=False, verbose=verbose) else: # if fog node i lose the reverse_auction agents_list[i].differential_sarsa_update_weights( 0, max_usage_time_list[i], relative_start_time_list[i], winner_revenue=winner_utility, bool_update_weights=False, verbose=verbose) else: if verbose: print( "This is the last task.") # no need to update weights # enable_print() if verbose: print(f"social welfare = {sw_list[-1]}") social_welfare_list.append(sw_list[-1]) # generate a list of average rewards average_reward_sarsa_list = [] for i in range(total_number_of_steps): average_reward = sw_list[i] / (i + 1) average_reward_sarsa_list.append(average_reward) result_sarsa_list.append(average_reward_sarsa_list.copy()) # print(result_sarsa_list) if plot_bool: # plot the result fig, axes = plt.subplots(1 + num_fog_nodes, 1, figsize=(12, 30)) fig.suptitle('Figures') # plot the social welfare result_df = None for item in result_sarsa_list: result_sarsa_y = item.copy() x_list = range(len(result_sarsa_y)) auction_df = pd.DataFrame({ 'algorithm': 'reverse reverse_auction', 'steps': x_list, 'average_social_welfare': result_sarsa_y }) result_df = pd.concat([result_df, auction_df]) # print(result_df) sns.lineplot(ax=axes[0], data=result_df, x="steps", y="average_social_welfare") # plot the actions taken by each node for i in range(num_fog_nodes): actions_of_i = actions[i] x_list = range(len(actions_of_i)) actions_of_i_df = pd.DataFrame({ 'steps': x_list, 'action options': actions_of_i }) sns.lineplot(ax=axes[i + 1], data=actions_of_i_df, x='steps', y='action options') plt.show() else: # save the result for jupyter notebook if bool_decay: with open( f'../simulation_results/auction_v1_{j + 1}trials' f'_rc={resource_coefficient_original}_seed={training_seed}_decay.txt', 'w') as f: f.write(json.dumps(social_welfare_list)) else: with open( f'../simulation_results/auction_v1_{j + 1}trials' f'_rc={resource_coefficient_original}_seed={training_seed}.txt', 'w') as f: f.write(json.dumps(social_welfare_list)) return sw_list, total_value, df_tasks, df_nodes, agents_list, allocation_scheme
def train_multi_agent_sarsa(avg_resource_capacity, avg_unit_cost, seed=0, total_number_of_steps=500, num_fog_nodes=6, resource_coefficient_original=3, alpha=0.001, beta=0.1, epsilon_tuple=(0.2, 0.1, 0.05), epsilon_steps_tuple=(500, 500, 100), plot_bool=False, num_actions=4, time_length=100, high_value_proportion=0.2, high_value_slackness=0, low_value_slackness=6, valuation_coefficient_ratio=10, resource_ratio=1.2, trained_agents=None, verbose=False, auction_type="second-price"): """run multi-agent sarsa Args: :param auction_type: the type of the reverse reverse_auction :param verbose: whether print the details of the execution :param number_of_runs: number of trials :param total_number_of_steps: steps of RL (allocate how many tasks) :param num_fog_nodes: number of fog nodes :param resource_coefficient_original: a coefficient for computing the resource coefficient :param alpha: step size of weights :param beta: step size of estimated average rewards :param epsilon_tuple: probability of exploration :param epsilon_steps_tuple: number of steps to run for each epsilon :param plot_bool: whether plot the results :param num_actions: number of actions :param time_length: tasks arrive within this time length :param low_value_slackness: deadline slackness of low-value tasks :param resource_ratio: resource demand ratio between high-value and low-value tasks :param valuation_coefficient_ratio: valuation coefficient ratio between high-value and low-value tasks :param high_value_slackness: deadline slackness of high-value tasks :param high_value_proportion: the proportion of high-value tasks """ # record the allocation scheme allocation_scheme = pd.DataFrame( columns=['node_id', 'start_time', 'end_time']) # run the trials result_sarsa_list = [ ] # a list of the lists of average rewards for each step of sarsa social_welfare_list = [] # a list of total social welfare of each trial if verbose: print(f"seed={seed}") sw_list = [] # a list of social welfare after a new task arrives # initialise some parameters n_steps = total_number_of_steps + 1 np.random.seed(seed) # generate the two types of tasks number_of_tasks = total_number_of_steps V = {} # value function: state -> value pi = {} # policy: (state+price) -> probability actions = list( range(num_actions)) # bid [reject, 1/3, 2/3, 1] the value of the task # generate a seqence of analytics tasks # compute the resource coefficient resource_coefficient = (resource_coefficient_original * number_of_tasks / time_length) # generate the synthetic data for simulations df_tasks, df_nodes, n_time, n_tasks, num_fog_nodes = \ generate_synthetic_data_edge_cloud(avg_resource_capacity, avg_unit_cost, n_tasks=total_number_of_steps, n_time=time_length, seed=seed, n_nodes=num_fog_nodes, p_high_value_tasks=high_value_proportion, high_value_slackness_lower_limit=high_value_slackness, high_value_slackness_upper_limit=high_value_slackness + 2, low_value_slackness_lower_limit=low_value_slackness, low_value_slackness_upper_limit=low_value_slackness + 2, resource_demand_high=resource_ratio, vc_ratio=valuation_coefficient_ratio, k_resource=resource_coefficient) if verbose: print("resource coefficient: ", resource_coefficient) print(f"low value slackness = {low_value_slackness}") print(f"high value slackness = {high_value_slackness}") print('df_tasks:') print(df_tasks.head(10)) print('df_nodes:') print(df_nodes.head()) average_reward_sarsa_list = [] agents_list = [] # with tqdm(total=100) as pbar: mdp = ReverseAuctionMDP(df_tasks, df_nodes, num_nodes=num_fog_nodes, num_actions=num_actions) # several fog nodes # generate several agents representing several fog nodes if trained_agents is not None: for i in range(mdp.num_fog_nodes): agent = FogNodeAgent(n_steps=n_steps - 1, alpha=alpha, beta=beta, fog_index=i, df_tasks=df_tasks, df_nodes=df_nodes, num_actions=num_actions, epsilon=epsilon_tuple[0], mdp=mdp, trained_agent=trained_agents[i]) agents_list.append(agent) else: for i in range(mdp.num_fog_nodes): agent = FogNodeAgent(n_steps=n_steps - 1, alpha=alpha, beta=beta, fog_index=i, df_tasks=df_tasks, df_nodes=df_nodes, num_actions=num_actions, epsilon=epsilon_tuple[0], mdp=mdp) agents_list.append(agent) # actions taken by each node actions = {i: [] for i in range(mdp.num_fog_nodes)} # the reverse reverse_auction for k in tqdm(range(total_number_of_steps)): # fog nodes decide their bidding price, and allocation scheme for the current task if verbose: print() print(f"step: {k}") # epsilon decreases as the number of steps increases if k < epsilon_steps_tuple[0]: epsilon = epsilon_tuple[0] elif k < epsilon_steps_tuple[0] + epsilon_steps_tuple[1]: epsilon = epsilon_tuple[1] elif k < (epsilon_steps_tuple[0] + epsilon_steps_tuple[1] + epsilon_steps_tuple[2]): epsilon = epsilon_tuple[2] else: epsilon = 0 if verbose: print(f'epsilon = {epsilon}') # change the epsilon of all agents for i in range(mdp.num_fog_nodes): agents_list[i].epsilon = epsilon bids_list = [] # bidding price for one time step max_usage_time_list = [] # maximum usage time a fog node can offer start_time_list = [] # start time according to the planned allocation relative_start_time_list = [ ] # relative start time according to the current task for i in range(mdp.num_fog_nodes): (bidding_price, max_usage_time, relative_start_time, action) = \ agents_list[i].differential_sarsa_decide_action(verbose=verbose) # tranfer relative start_time to absolute start_time start_time = int(df_tasks.loc[k, 'arrive_time'] + relative_start_time + 1) bids_list.append(bidding_price) max_usage_time_list.append(max_usage_time) start_time_list.append(start_time) relative_start_time_list.append(relative_start_time) actions[i].append(action) # find the winner (winner_index, winner_num_time, winner_utility, max_utility) = \ mdp.step(bids_list, max_usage_time_list, start_time_list, verbose=verbose, auction_type=auction_type) sw_list.append(mdp.social_welfare ) # a list of social welfare after a new task arrives # modify the allocation scheme if winner_num_time is not None and winner_num_time > 0: allocation_scheme.loc[k] = [ winner_index, start_time_list[winner_index], start_time_list[winner_index] + max_usage_time_list[winner_index] - 1 ] else: # the task is rejected allocation_scheme.loc[k] = [None, None, None] if verbose: print() print(f"nodes' bids = {bids_list}") print(f"nodes' usage times = {max_usage_time_list}") print(f"nodes' start times = {start_time_list}") print(f"winner's index = {winner_index}") print(f"number of usage time = {winner_num_time}") print(f"winner's utility = {winner_utility}") print(f"user's utility = {max_utility}") # print(f"social_welfare_list={sw_list}") if k < total_number_of_steps - 1: # update sarsa weights for i in range(mdp.num_fog_nodes): if verbose: print(f"updating weights of node{i}:") if i == winner_index: # if fog node i wins this task agents_list[i].differential_sarsa_update_weights( 1, max_usage_time_list[i], relative_start_time_list[i], winner_revenue=winner_utility, verbose=verbose) else: # if fog node i lose the reverse_auction agents_list[i].differential_sarsa_update_weights( 0, max_usage_time_list[i], relative_start_time_list[i], winner_revenue=winner_utility, verbose=verbose) else: if verbose: print("This is the last task.") # no need to update weights if verbose: print(f"social welfare = {sw_list[-1]}") social_welfare_list.append(sw_list[-1]) # generate a list of average rewards of recent 100 tasks average_reward_sarsa_list = [] for i in range(total_number_of_steps): if i < 100: average_reward = sw_list[i] / (i + 1) average_reward_sarsa_list.append(average_reward) else: average_reward = (sw_list[i] - sw_list[i - 100]) / 100 average_reward_sarsa_list.append(average_reward) result_sarsa_list.append(average_reward_sarsa_list.copy()) # print(result_sarsa_list) # print the total value of tasks total_value = 0 for i in range(total_number_of_steps): total_value += (df_tasks.loc[i, "valuation_coefficient"] * df_tasks.loc[i, "usage_time"]) if verbose: print(f"total value of tasks = {total_value}") print("df_tasks:") print(df_tasks.head()) print("df_nodes:") print(df_nodes.head()) if plot_bool: # plot the result fig, axes = plt.subplots(1 + 2 * num_fog_nodes, 1, figsize=(12, 30)) fig.suptitle('Figures') # plot the social welfare result_df = None for item in result_sarsa_list: result_sarsa_y = item.copy() x_list = range(len(result_sarsa_y)) auction_df = pd.DataFrame({ 'algorithm': 'reverse reverse_auction', 'steps': x_list, 'average_social_welfare (recent 100 tasks)': result_sarsa_y }) result_df = pd.concat([result_df, auction_df]) # print(result_df) sns.lineplot(ax=axes[0], data=result_df, x="steps", y="average_social_welfare (recent 100 tasks)") # plt.show() # plot the learned average rewards of each node for i in range(num_fog_nodes): avg_reward = agents_list[i].list_avg_reward x_list = range(len(avg_reward)) avg_reward_df = pd.DataFrame({ 'steps': x_list, 'average rewards of node 1': avg_reward }) sns.lineplot(ax=axes[i + 1], data=avg_reward_df, x="steps", y="average rewards of node 1") # plot the actions taken by each node for i in range(num_fog_nodes): actions_of_i = actions[i] x_list = range(len(actions_of_i)) actions_of_i_df = pd.DataFrame({ 'steps': x_list, 'action options': actions_of_i }) sns.lineplot(ax=axes[i + num_fog_nodes + 1], data=actions_of_i_df, x='steps', y='action options') # TODO: plot the actions taken by each node plt.show() else: pass return sw_list, total_value, df_tasks, df_nodes, agents_list, allocation_scheme
def train_multi_agent_sarsa(seed=0, total_number_of_steps=500, num_fog_nodes=6, resource_coefficient_original=3, alpha=0.001, beta=0.1, epsilon_tuple=(0.2, 0.1, 0.05), epsilon_steps_tuple=(500, 500), plot_bool=False, num_actions=4, time_length=100, high_value_proportion=0.2, high_value_slackness=0, low_value_slackness=6, valuation_coefficient_ratio=10, resource_ratio=1.2, trained_agents=None): """run multi-agent sarsa Args: :param number_of_runs: number of trials :param total_number_of_steps: steps of RL (allocate how many tasks) :param num_fog_nodes: number of fog nodes :param resource_coefficient_original: a coefficient for computing the resource coefficient :param alpha: step size of weights :param beta: step size of estimated average rewards :param epsilon_tuple: probability of exploration :param epsilon_steps_tuple: number of steps to run for each epsilon except for the last one :param plot_bool: whether plot the results :param num_actions: number of actions :param time_length: tasks arrive within this time length :param low_value_slackness: deadline slackness of low-value tasks :param resource_ratio: resource demand ratio between high-value and low-value tasks :param valuation_coefficient_ratio: valuation coefficient ratio between high-value and low-value tasks :param high_value_slackness: deadline slackness of high-value tasks :param high_value_proportion: the proportion of high-value tasks """ # run the trials result_sarsa_list = [ ] # a list of the lists of average rewards for each step of sarsa social_welfare_list = [] # a list of total social welfare of each trial print(f"seed={seed}") block_print() sw_list = [] # a list of social welfare after a new task arrives # initialise some parameters n_steps = total_number_of_steps + 1 np.random.seed(seed) # generate the two types of tasks number_of_tasks = total_number_of_steps V = {} # value function: state -> value pi = {} # policy: (state+price) -> probability actions = list( range(num_actions)) # bid [reject, 1/3, 2/3, 1] the value of the task # generate a seqence of analytics tasks resource_coefficient = (resource_coefficient_original * number_of_tasks / time_length) # compute the resource coefficient # generate the synthetic data for simulations df_tasks, df_nodes, n_time, n_tasks, num_fog_nodes = \ generate_synthetic_data_edge_cloud(n_tasks=total_number_of_steps, n_time=time_length, seed=seed, n_nodes=num_fog_nodes, p_high_value_tasks=high_value_proportion, high_value_slackness_lower_limit=high_value_slackness, high_value_slackness_upper_limit=high_value_slackness + 2, low_value_slackness_lower_limit=low_value_slackness, low_value_slackness_upper_limit=low_value_slackness + 2, resource_demand_high=resource_ratio, vc_ratio=valuation_coefficient_ratio, k_resource=resource_coefficient) print("resource coefficient: ", resource_coefficient) print(f"low value slackness = {low_value_slackness}") print(f"high value slackness = {high_value_slackness}") average_reward_sarsa_list = [] agents_list = [] # with tqdm(total=100) as pbar: # several fog nodes mdp = ReverseAuctionMDP(df_tasks, df_nodes, num_nodes=num_fog_nodes, num_actions=num_actions) # generate several agents representing several fog nodes if trained_agents is not None: for i in range(mdp.num_fog_nodes): agent = MenuFogNodeAgent(n_steps=n_steps - 1, alpha=alpha, beta=beta, fog_index=i, df_tasks=df_tasks, df_nodes=df_nodes, num_actions=num_actions, epsilon=epsilon_tuple[0], mdp=mdp, trained_agent=trained_agents[i]) agents_list.append(agent) else: for i in range(mdp.num_fog_nodes): agent = MenuFogNodeAgent(n_steps=n_steps - 1, alpha=alpha, beta=beta, fog_index=i, df_tasks=df_tasks, df_nodes=df_nodes, num_actions=num_actions, epsilon=epsilon_tuple[0], mdp=mdp) agents_list.append(agent) # the reverse reverse_auction for k in range(total_number_of_steps): # fog nodes decide their bidding price, and allocation scheme for the current task print() print(f"step: {k}") # epsilon decreases as the number of steps increases if k < epsilon_steps_tuple[0]: epsilon = epsilon_tuple[0] elif k < epsilon_steps_tuple[0] + epsilon_steps_tuple[1]: epsilon = epsilon_tuple[1] else: epsilon = epsilon_tuple[2] print(f'epsilon = {epsilon}') # change the epsilon of all agents for i in range(mdp.num_fog_nodes): agents_list[i].epsilon = epsilon bids_list = [] # bidding price for one time step max_usage_time_list = [] # maximum usage time a fog node can offer start_time_list = [] # start time according to the planned allocation for i in range(mdp.num_fog_nodes): (prices_list, max_usage_time, start_time) = \ agents_list[i].differential_sarsa_decide_action() bids_list.append(prices_list) max_usage_time_list.append(max_usage_time) start_time_list.append(start_time) # find the winner (winner_index, winner_num_time, winner_utility, max_utility) = \ mdp.step(bids_list, max_usage_time_list) print() print(f"winner's index = {winner_index}") print(f"number of usage time = {winner_num_time}") print(f"winner's utility = {winner_utility}") print(f"user's utility = {max_utility}") sw_list.append(mdp.social_welfare ) # a list of social welfare after a new task arrives if k < total_number_of_steps - 1: # update sarsa weights for i in range(mdp.num_fog_nodes): agents_list[i].differential_sarsa_update_weights( winner_index, max_usage_time_list[i], winner_num_time, start_time_list[i]) else: print("This is the last task.") # no need to update weights enable_print() print(f"social welfare = {sw_list[-1]}") social_welfare_list.append(sw_list[-1]) # generate a list of average rewards average_reward_sarsa_list = [] for i in range(total_number_of_steps): average_reward = sw_list[i] / (i + 1) average_reward_sarsa_list.append(average_reward) result_sarsa_list.append(average_reward_sarsa_list.copy()) # print(result_sarsa_list) # print the total value of tasks total_value = 0 for i in range(total_number_of_steps): total_value += (df_tasks.loc[i, "valuation_coefficient"] * df_tasks.loc[i, "usage_time"]) print(f"total value of tasks = {total_value}") print("df_tasks:") print(df_tasks.head()) print("df_nodes:") print(df_nodes.head()) if plot_bool: # plot the result result_df = None for item in result_sarsa_list: result_sarsa_y = item.copy() x_list = range(len(result_sarsa_y)) auction_df = pd.DataFrame({ 'algorithm': 'reverse reverse_auction', 'steps': x_list, 'average_social_welfare': result_sarsa_y }) result_df = pd.concat([result_df, auction_df]) # print(result_df) sns.lineplot(data=result_df, x="steps", y="average_social_welfare") plt.show() return sw_list, total_value, df_tasks, df_nodes, agents_list else: # save the result for jupyter notebook return sw_list, total_value, df_tasks, df_nodes, agents_list