def scheduler(rid, sid):
one_route_crowd(rid)
served = []
dfirst = df.loc[rid]['departure_from_origin'].split(",")[sid]
afirst = df.loc[rid]['arrival_at_destination'].split(',')[sid]
dsec = get_sec(dfirst + ":00")
asec = get_sec(afirst + ":00")
print(dsec, asec)
dif = (asec - dsec) / len(json.loads(df.loc[rid]['map_json_content']))
timetable = []
timetable.append(dsec)
for i in range(len(json.loads(df.loc[rid]['map_json_content'])) - 2):
timetable.append(dsec + ((i + 1) * dif))
timetable.append(asec)
enviro = env(crowd_new)
agent = Agent(enviro.stops, timetable)
for i in range(20000):
ppl_served = 0
while True:
action = agent.decide_action(enviro.state)
# print(len(agent.passengers))
(done, nextState, (noppl,
pass1)) = enviro.step(action, agent.capacity,
agent.passengers)
ppl_served += noppl
agent.descending(nextState - 1)
# print(len(pass1))
agent.update_reward(action, nextState - 1, noppl)
if done:
if i % 1000 == 0:
print("done" + str(i))
break
served.append(ppl_served)
return {"time": agent.time, "optimise": served}
def PCRW(task_relation='<isAffiliatedTo>', MAXLEN=5):
rootpath = './data/yagosmall/' + task_relation + '/'
datapath = {'type2id': rootpath + 'type2id.json', 'relation2id': rootpath + 'relation2id.json', \
'graph': rootpath + 'graph.pkl', 'ent2type': rootpath + 'ent2type.json', \
'entity2id': rootpath + 'entity2id.json'}
Env_a = env(datapath)
Env_a.init_relation_query_state(task_relation)
maxlen = MAXLEN
Env_a.filter_query(maxlen)
pairs = Env_a.filter_query
random.shuffle(pairs)
training_pairs = pairs[int(len(pairs) * 0.9):]
test_pairs = pairs[:int(len(pairs) * 0.5)]
if len(test_pairs) > 5000:
test_pairs = test_pairs[:5000]
allpaths = []
for i in test_pairs:
e1, e2 = i[0], i[1]
paths = Env_a.path_traverse_BFS(e1, e2, max_len=MAXLEN)
allpaths.append(paths)
with open(rootpath + 'PCRW-%d' % MAXLEN + task_relation, 'w') as fin:
wrt_str = []
for paths in allpaths:
M = []
for path in paths:
M.append('\t'.join(path))
wrt_str.append('\n'.join(M))
wrt_str = '\n'.join(wrt_str)
fin.write(wrt_str)
with open(rootpath + 'PCRW-%d' % MAXLEN + task_relation, 'r') as fin:
Feature_SET = {}
for i in fin.readlines():
line = i.strip().split()
feature = []
for k, i in enumerate(line):
if k % 2 == 1:
feature.append(i)
feature = tuple(feature)
if feature in Feature_SET:
Feature_SET[feature] += 1
else:
Feature_SET[feature] = 1
print('Path feature:', Feature_SET)
def write_test_pairs(task_relation="<diedIn>", rootpath=None):
datapath = {'type2id': rootpath + 'type2id.json', 'relation2id': rootpath + 'relation2id.json', \
'graph': rootpath + 'graph.pkl', 'ent2type': rootpath + 'ent2type.json', \
'entity2id': rootpath + 'entity2id.json'}
Env_a = env(datapath)
Env_a.init_relation_query_state(task_relation)
maxlen = 8
Env_a.filter_query(maxlen, 5000)
pairs = Env_a.filter_query
random.shuffle(pairs)
test_pairs = pairs[:int(len(pairs) * 0.5)]
with open(rootpath + 'test_positive_pairs', 'w') as fin:
wstr = []
for i in test_pairs:
wstr.append(str(i[0] + '\t' + str(i[1])))
wstr = '\n'.join(wstr)
fin.write(wstr)
return
def test_empty_dict(self):
self.assertEqual(env('EMPTY_DICT'), {})
def train(task_relation="<diedIn>",rootpath=None,epoch=5):
datapath = {'type2id': rootpath + 'type2id.json', 'relation2id': rootpath + 'relation2id.json', \
'graph': rootpath + 'graph.pkl', 'ent2type': rootpath + 'ent2type.json' ,\
'entity2id': rootpath + 'entity2id.json'}
Env_a = env(datapath)
Env_a.init_relation_query_state(task_relation)
batchsize=20
maxlen=5
po = Policy_memory(Env_a,300, 100, Env_a.rel_num)
# Env_a.filter_query(maxlen,5000)
# pairs = Env_a.filter_query
# random.shuffle(pairs)
# training_pairs=pairs
# test_pairs=pairs[:int(len(pairs)*0.5)]
# valid_paris=pairs[int(len(pairs)*0.5):int(len(pairs)*0.6)]
train_path=rootpath+'/'+task_relation+'train_pairs'
valid_path = rootpath + '/' + task_relation + 'valid_pairs'
training_pairs=load_pair(train_path)
valid_paris=load_pair(valid_path)
print('Train pairs:',len(training_pairs))
print('valid pairs:',len(valid_paris))
#print('Test pairs:',len(test_pairs))
agent_a = agent(po, Env_a,policymethod='GRU')
if global_device=='cuda:0':
po=po.cuda()
try_count, batch_loss, ave_reward, ave_success = 0, 0, 0, 0
opt=torch.optim.Adam(agent_a.parameters()+Env_a.parameters(),lr=0.001)
for ep in range(epoch):
opt.zero_grad()
random.shuffle(training_pairs)
for query in training_pairs:
try:
e1, e2 = query[0], query[1]
e1, e2 = Env_a.entity2id[e1], Env_a.entity2id[e2]
with torch.no_grad():
traj, success = agent_a.trajectory(e1, e2,max_len=maxlen)
try_count += 1
except KeyError:
continue
logger.MARK(Env_a.traj_for_showing(traj))
traj_loss=0
po.zero_history()
traj_reward=0
for i in traj:
ave_reward+=i[4]
traj_reward+=i[4]
loss=agent_a.update_memory_policy(i)
loss.backward()
traj_loss+=loss.cpu()
if success:
ave_success+=1
batch_loss+=traj_loss/len(traj)
if try_count%batchsize==0 and try_count>0:
opt.step()
opt.zero_grad()
logger.info('|%d epoch|%d eposide|Batch_loss:%.4f|Ave_reward:%.3f|Ave_success:%%%.2f|'%(ep,try_count,batch_loss*100/batchsize,ave_reward/batchsize,ave_success*100/batchsize))
batch_loss,ave_reward,ave_success=0,0,0
if try_count%(20*batchsize)==0 and try_count>0:
valid(valid_paris,Env_a,agent_a,batchsize,maxlen)
generate_paths(Env_a,agent_a,test_pairs,rootpath+task_relation+'.paths',maxlen)
def test_env_driver_with_undefined_browser(self):
self.assertEqual(env_driver(env('DB')), False)
def test_dict_values(self):
self.assertEqual(env('DICT')['key1'], 'value1')
self.assertEqual(env('DICT')['key2'], 'value2')
def test_env_with_defined_key(self):
self.assertEqual(env('BROWSER'), 'chrome')
def test_env_driver_with_undefined_browser(self):
self.assertEqual(env_driver(env('DB')), False)
def test_dict_values(self):
self.assertEqual(env('DICT')['key1'], 'value1')
self.assertEqual(env('DICT')['key2'], 'value2')
# Pull in the command factory for the second example.
from Helpers.Commands import CommandFactory, Kwargs
from selenium.webdriver.support.wait import WebDriverWait
# This is where the WebDriver is instantiated. Instead
# of instantiating it directly, use the `env` and
# `env_driver` functions to grab it based on the
# `.env` configuration file.
# This could be written as:
#
# browser = env("BROWSER")
# web_driver = env_driver(browser)
# with quitting(web_driver()) as driver:
# pass
#
with quitting(env_driver(env("BROWSER"))()) as driver:
wait = WebDriverWait(driver, 30)
# Pass the web driver to the site automation along with anything
# else it might need to do its job. This could include an
# instance of WebDriverWait, and even the collection of
# Models.
google_search = GoogleExample.GoogleSearch(driver, wait, Models)
bing_search = BingExample.BingSearch(driver, wait, Models)
# Do stuff with your controllers.
google_search.do_search('google wiki')
sleep(5)
bing_search.do_search('bing wiki')
sleep(5)
def test_get_dict(self):
self.assertIsInstance(env('DICT'), dict)
def test_list_values(self):
self.assertEqual(env('LIST')[0], 'Item1')
self.assertEqual(env('LIST')[1], 'Item2')
def test_get_list(self):
self.assertIsInstance(env('LIST'), list)
def test_get_database_with_undefined_database_type(self):
self.assertEqual(get_database(env('BROWSER')), False)
def test_get_database_with_defined_database_type(self):
self.assertIsInstance(get_database(env('DB_TYPE')), SqliteDatabase)
def init_env(self, task_relation):
self.Env_a = env(self.datapath)
self.Env_a.init_relation_query_state(task_relation)
def test_empty_list(self):
self.assertEqual(env('EMPTY_LIST'), [])
res = self.policy.forward(cur_e, cur_r, target_e)
res = -torch.log(res) * y
loss = torch.sum(res)
return loss
def get_state_from_env(self, ent):
pass
if __name__ == '__main__':
torch.device('cpu')
rootpath = './data/yago10w_RL/'
datapath = {'type2id': rootpath + 'type2id.json', 'relation2id': rootpath + 'relation2id.json' \
, 'graph': rootpath + 'graph.pkl', 'ent2type': rootpath + 'ent2type.json' \
, 'entity2id': rootpath + 'entity2id.json' }
hu = env(datapath)
hu.init_relation_query_state('<isCitizenOf>')
po = Policy(300, 32, 200)
po.init()
ag = agent(po, hu)
#print(hu.relation2id)
count = 0
for k, fact in enumerate(hu.query):
e1, e2 = fact[0], fact[1]
e1, e2 = hu.entity2id[e1], hu.entity2id[e2]
res = ag.trajectory(e1, e2, 5)
print(res)
def test_empty_dict(self):
self.assertEqual(env('EMPTY_DICT'), {})
def test_env_with_defined_key(self):
self.assertEqual(env('BROWSER'), 'chrome')
def test_get_database_with_undefined_database_type(self):
self.assertEqual(get_database(env('BROWSER')), False)
def test_env_driver_with_defined_browser(self):
self.assertEqual(env_driver(env('BROWSER')), webdriver.Chrome)
def test_get_dict(self):
self.assertIsInstance(env('DICT'), dict)
def test_get_database_with_defined_database_type(self):
self.assertIsInstance(get_database(env('DB_TYPE')), SqliteDatabase)
def test_env_driver_with_defined_browser(self):
self.assertEqual(env_driver(env('BROWSER')), webdriver.Chrome)
def test_get_list(self):
self.assertIsInstance(env('LIST'), list)
def train(task_relation="<diedIn>", rootpath=None, epoch=5):
datapath = {'type2id': rootpath + 'type2id.json', 'relation2id': rootpath + 'relation2id.json', \
'graph': rootpath + 'graph.pkl', 'ent2type': rootpath + 'ent2type.json' ,\
'entity2id': rootpath + 'entity2id.json'}
Env_a = env(datapath)
Env_a.init_relation_query_state(task_relation)
batchsize = 20
maxlen = 5
po = Policy_memory(Env_a, 300, 100, Env_a.rel_num)
Env_a.filter_query(maxlen, 5000)
pairs = Env_a.filter_query
random.shuffle(pairs)
training_pairs = pairs
test_pairs = pairs[:int(len(pairs) * 0.5)]
reward_record = []
success_record = []
path_length = 0
valid_paris = pairs[int(len(pairs) * 0.5):int(len(pairs) * 0.6)]
print('Train pairs:', len(training_pairs))
print('valid pairs:', len(valid_paris))
print('Test pairs:', len(test_pairs))
agent_a = agent(po, Env_a, policymethod='GRU')
if global_device == 'cuda:0':
po = po.cuda()
try_count, batch_loss, ave_reward, ave_success = 0, 0, 0, 0
opt = torch.optim.Adam(agent_a.parameters() + Env_a.parameters(), lr=0.001)
for ep in range(epoch):
opt.zero_grad()
random.shuffle(training_pairs)
for query in training_pairs:
try:
e1, e2 = query[0], query[1]
e1, e2 = Env_a.entity2id[e1], Env_a.entity2id[e2]
with torch.no_grad():
traj, success = agent_a.trajectory(e1, e2, max_len=maxlen)
try_count += 1
except KeyError:
continue
logger.MARK(Env_a.traj_for_showing(traj))
traj_loss = 0
po.zero_history()
traj_reward = 0
for i in traj:
ave_reward += i[4]
traj_reward += i[4]
loss = agent_a.update_memory_policy(i)
loss.backward()
traj_loss += loss.cpu()
if success:
ave_success += 1
path_length += len(traj) - 1
success_record.append(1)
else:
success_record.append(0)
reward_record.append(traj_reward)
batch_loss += traj_loss / len(traj)
if try_count % batchsize == 0 and try_count > 0:
opt.step()
opt.zero_grad()
logger.info(
'|%d epoch|%d eposide|Batch_loss:%.4f|Ave_reward:%.3f|Ave_success:%%%.2f|ave path lenghth:%.2f|'
% (ep, try_count, batch_loss * 100 / batchsize,
ave_reward / batchsize, ave_success * 100 / batchsize,
path_length / ave_success))
batch_loss, ave_reward, ave_success, path_length = 0, 0, 0, 0
if try_count % (20 * batchsize) == 0 and try_count > 0:
valid(valid_paris, Env_a, agent_a, batchsize, maxlen)
generate_paths(Env_a, agent_a, test_pairs,
rootpath + task_relation + '.paths', maxlen)
success = ave_smooth(success_record, 20)
reward = ave_smooth(reward_record, 20)
with open(rootpath + task_relation + 'sucess_record_without.txt',
'w') as fin:
wstr = '\n'.join([str(i) for i in success])
fin.write(wstr)
with open(rootpath + task_relation + 'reward_record_without.txt',
'w') as fin:
wstr = '\n'.join([str(i) for i in reward])
fin.write(wstr)
with open(rootpath + task_relation + 'test_positive_pairs', 'w') as fin:
wstr = []
for i in test_pairs:
wstr.append(str(i[0] + '\t' + str(i[1])))
wstr = '\n'.join(wstr)
fin.write(wstr)
def test_list_values(self):
self.assertEqual(env('LIST')[0], 'Item1')
self.assertEqual(env('LIST')[1], 'Item2')
options = get_options()
if options.Test:
vis_update_params['test_mode_on'] = True
sumoBinary = checkBinary('sumo-gui')
if options.Train:
vis_update_params['test_mode_on'] = False
sumoBinary = checkBinary('sumo')
else:
print("Initizlizing Test mood as default")
vis_update_params['test_mode_on'] = True
sumoBinary = checkBinary('sumo-gui')
## ----- ##
Proudhon = env(sumoBinary=sumoBinary) # env.__init__ and template loading
# environment reset inside episode()
## ----- ##
episode_num = 0
Algorithm_for_RL, environment_for_next_episode, episode_reward, episode_reward_list = SimTools.episode(sumoBinary=sumoBinary, Proudhon=Proudhon)
# total_reward_per_episode.append(episode_reward) #RLcomment
# reward_history_per_episode.append(episode_reward_list) #RLcomment
## --- ##
environment_for_next_episode.reset(sumoBinary=sumoBinary)
## --- ##
while(episode_num < max_num_episodes):
def test_empty_list(self):
self.assertEqual(env('EMPTY_LIST'), [])
def __init__(self, bunshin_name, bunshin_type, God_brain):
self.environment = env(bunshin_name=bunshin_name,
bunshin_type=bunshin_type,
God_brain=God_brain)
self.bunshin_type = bunshin_type
def episode(sumoBinary, RB_RLAlgorithm=None, Proudhon=None, episode_num=0):
########################
# 1 inits
done = False # are we done with the episode or not
step = 0 # step number
if (Proudhon is None):
Proudhon = env(sumoBinary=sumoBinary)
# Proudhon.reset() #TempComment
# if(RB_RLAlgorithm is None): #RLcomment
# algo_params = q_learning_params # from Config.py #RLcomment
# RB_RLAlgorithm = RLAlgorithm(Proudhon, algo_params= algo_params,
# load_q_table = load_q_table, test_mode_on = vis_update_params['test_mode_on']) # Algorithm for RB Agent #RLcomment
# ## ######################
########################
# 2 init measurements
traci.simulationStep() # After stepping
Proudhon.get_emer_start_lane()
# (communication from vehicle):
getters(Proudhon.list_of_vehicles) # measure all values from environment (local variable.. since some vehicles exited)
# (communication to environment):
Proudhon.measure_full_state() # measure all values into our agents
# (communication to algorithm/agent):
# new_observed_state_for_this_agent = Proudhon.observed_state[0] #RLcomment
# Chose Action from Feasible Actions:
# feasible_actions_for_current_state = Proudhon.get_feasible_actions(Proudhon.list_of_vehicles[1]) #RLcomment
# chosen_action = RB_RLAlgorithm.pickAction(feasible_actions_for_current_state, new_observed_state_for_this_agent) #RLcomment
# RB_RLAlgorithm.applyAction(chosen_action, Proudhon.list_of_vehicles[1]) # Request Action on Agent #RLcomment
# episode_reward = 0 #RLcomment
# episode_reward_list = [] #RLcomment
########################
# 3: MAIN LOOP
if (episode_num % vis_update_params['every_n_episodes'] == 0): #TempComment
print(f'E:{episode_num: <{6}}|S:{0: <{4}} | '
f'MaxPossible: {Proudhon.max_possible_cars: <{4}} | '
f'ActualPerLane: { [ vehicles_data[i] for i in range(num_lanes) ] } |'
f'NumVehicles: {fill_str(str(len(Proudhon.list_of_vehicles)), 5)}')
while traci.simulation.getMinExpectedNumber() > 0:
# 3.1: Store last states
amb_last_velocity = Proudhon.emer.spd
# last_observed_state = Proudhon.observed_state #RLComment
# last_observed_state_for_this_agent = last_observed_state #RLcomment
# ----------------------------------------------------------------- #
# 3.2: M O V E O N E S I M U L A T I O N S T E P
# ----------------------------------------------------------------- #
traci.simulationStep() # actual action applying
step += 1
# TODO: Turn this into are_we_ok function
if (Proudhon.list_of_vehicles[0].getL() != Proudhon.emer_start_lane and enable_checks):
raise ValueError(
f"Ambulance Changed lane from {Proudhon.emer_start_lane} to {Proudhon.list_of_vehicles[0].getL()} on step {step}. "
f"\nAmbulance Should not change lane. Quitting.")
if (step % vis_update_params['every_n_iters'] == 0 and episode_num % vis_update_params['every_n_episodes'] == 0): # print step info #TempComment
print(f'E:{episode_num: <{6}}|S:{step: <{4}} |' #TempComment
f'EmerVel: {fill_str(str(Proudhon.emer.spd), 5)} |'
f'EmerGoalDist: {fill_str(str(Proudhon.amb_goal_dist-Proudhon.emer.lane_pose), 5)} |'
f'NumVehicles: {fill_str(str(len(Proudhon.list_of_vehicles)), 5)}')
# ----------------------------------------------------------------- #
# 3.3: measurements and if we are done check
getters(Proudhon.list_of_vehicles)
Proudhon.measure_full_state()
# new_observed_state_for_this_agent = Proudhon.observed_state[0] #RLcomment
done = Proudhon.are_we_done(full_state=Proudhon.full_state, step_number=step)
# 3.4: reward last step's chosen action
# reward = Proudhon.calc_reward(amb_last_velocity, done, step) #RLcomment
# episode_reward += reward # for history #RLcomment
# episode_reward_list.append(reward) # for history #RLcomment
# 3.6: Feasibility check for current_state (for next step)
# feasible_actions_for_current_state = Proudhon.get_feasible_actions(Proudhon.list_of_vehicles[1]) #RLcomment
# 3.5: update q table using backward reward logic
# RB_RLAlgorithm.update_q_table(chosen_action, reward, new_observed_state_for_this_agent, #RLcomment
# last_observed_state_for_this_agent, feasible_actions_for_current_state) #RLcomment
if (done): # DO NOT REMOVE THIS (IT BREAKS IF WE ARE DONE)
if (episode_num % vis_update_params['every_n_episodes'] == 0):
if (
done == 1): # TODO: Remove episode_end_reason outsisde the print check -- we might need it elsewehere
episode_end_reason = "max steps"
elif (done == 2):
episode_end_reason = "ambulance goal"
else:
raise ValueError(f"Episode: {episode_num} done is True ={done} but reason not known !")
print(f'E:{episode_num: <{6}}|EndStep:{step: <{4}}')
break
# 3.7: Actually Choose Action from feasible ones (for next step)
# chosen_action = RB_RLAlgorithm.pickAction(feasible_actions_for_current_state, new_observed_state_for_this_agent) #RLcomment
# 3.8: Request environment to apply new action (Request action on Agent for next step)
# Action is still not applied here, but on #3.2
# RB_RLAlgorithm.applyAction(chosen_action, Proudhon.list_of_vehicles[1]) #RLcomment
# Episode end
sys.stdout.flush()
# 4: Update Epsilon after episode is done
# old_epsilon = RB_RLAlgorithm.epsilon #RLcomment
# RB_RLAlgorithm.epsilon = RB_RLAlgorithm.min_epsilon + (RB_RLAlgorithm.max_epsilon - RB_RLAlgorithm.min_epsilon) * \ #RLcomment
# np.exp(-RB_RLAlgorithm.decay_rate * episode_num) # DONE: Change epsilon to update every episode not every iteration #RLcomment
if (episode_num % vis_update_params['every_n_episodes'] == 0):
print(f'\n\nE:{episode_num: <{6}}| END:{step: <{4}} |'
f'reason: {episode_end_reason: <{15}}')
print('-'*157)
print('=' * 157)
print('\n')
if (vis_update_params['print_reward_every_episode'] and episode_num % vis_update_params['every_n_episodes'] != 0):
print(f'E:{episode_num: <{6}}| END:{step: <{4}} | ')
# f'finalCumReward: ' + str(episode_reward)[:6] + ' ' * max(0, 6 - len(str(episode_reward))) +" | ") #RLcomment
# return RB_RLAlgorithm, Proudhon, episode_reward, episode_reward_list #RLcomment
return None, Proudhon, None, None
from selenium.webdriver.support.wait import WebDriverWait
# This is where the WebDriver is instantiated. Instead
# of instantiating it directly, use the `env` and
# `env_driver` functions to grab it based on the
# `.env` configuration file.
# This could be written as:
#
# browser = env("BROWSER")
# web_driver = env_driver(browser)
# with quitting(web_driver()) as driver:
# pass
#
with quitting(env_driver(env("BROWSER"))()) as driver:
wait = WebDriverWait(driver, 30)
# Pass the web driver to the site automation along with anything
# else it might need to do its job. This could include an
# instance of WebDriverWait, and even the collection of
# Models.
google_search = GoogleExample.GoogleSearch(driver, wait, Models)
bing_search = BingExample.BingSearch(driver, wait, Models)
# Do stuff with your controllers.
google_search.do_search('google wiki')
sleep(5)
bing_search.do_search('bing wiki')
sleep(5)
