def get_avg_speed_at_edge(self,
start,
end,
attr,
sensor_data=None,
time_window=None):
tmc_id = None
length = None
# See JOURNAL_localdata_generator notebook
time_traversed = 10.791
if 0 not in attr:
key = random.choice(list(attr))
tmc_id = attr[key]['tmc_id']
length = attr[key]['length']
else:
if 0 in attr:
if 'tmc_id' in attr[0]:
tmc_id = attr[0]['tmc_id']
if 'length' in attr[0]:
length = attr[0]['length']
if tmc_id and tmc_id in sensor_data:
# TODO: Check if tmc_id is within the local data
# Give some delay if not the local, depends on distance from optimal rsu
# For now just give a delay of 1 hour
parent_grid = GRID_ID
for k, v in LOCAL_RSU_VARS.SUB_GRIDS.items():
if GRID_ID in v:
parent_grid = k
if tmc_id in LOCAL_RSU_VARS.TMC_DICT[parent_grid]:
actual_time_window = time_window
else:
# Get the grid_id where this tmc_id is from
for k, v in LOCAL_RSU_VARS.TMC_DICT.items():
if tmc_id in v:
other_rsu = k
break
# Use manhattan distance as the delay
r1 = geo_utils.get_rsu_by_grid_id(self.rsu_arr, parent_grid)
r2 = geo_utils.get_rsu_by_grid_id(self.rsu_arr, other_rsu)
delay = r1.get_manhattan_distance(r2)
# TODO: Fix this hardcoded
if GLOBAL_VARS.NEIGHBOR_LEVEL == 0:
delay = 0
actual_time_window = (time_window - delay) % 24
average_speed_at_time_window = sensor_data[tmc_id]['speeds'][
actual_time_window]
time_traversed = length / average_speed_at_time_window
return time_traversed
def get_dataframe_historical_data(grid, rsu_arr, with_neighbors=True):
# utils.print_log("get_speeds_hash_for_grid({})".format(grid))
# print("RSU_ID:", RSU_ID, "GRID_ID:", GRID_ID)
if not os.path.exists(os.path.join(os.getcwd(), 'data')):
raise OSError("Must first download data, see README.md")
data_dir = os.path.join(os.getcwd(), 'data')
if not os.path.exists(os.path.join(data_dir, 'historical_speeds')):
os.mkdir(os.path.join(data_dir, 'historical_speeds'))
historical_speeds_dir = os.path.join(data_dir, 'historical_speeds')
if with_neighbors and rsu_arr:
r = geo_utils.get_rsu_by_grid_id(rsu_arr, grid)
d = r.get_neighbors(rsu_arr)
hist_dfs = []
for _, n in d.items():
if n:
if n.grid_id != GRID_ID:
# print("Must get old data for {}".format(n.grid_id))
name = "{}_historical_speeds.pkl".format(n.grid_id)
hist_df = get_pickled_df(historical_speeds_dir, name)
# print(hist_df.head())
# print(hist_df.shape)
hist_dfs.append(hist_df)
all_hist_dfs = pd.concat(hist_dfs, ignore_index=True)
# print(all_hist_dfs)
# print(all_hist_dfs.shape)
return all_hist_dfs
def generate_RSU_arr(x, y):
target_area = GLOBAL_VARS.EXTENDED_DOWNTOWN_NASH_POLY
# Polys are just the larger boundaries to be used to "decrease" the number since we still use geohashing and it is still limited.
polys = gb.divide_grid(target_area, (x, y))
rsu_arr = []
for i in range(x):
for j in range(y):
idx = j + (i * y)
p = polys[idx]
gid = ghh.encode(p.centroid.x, p.centroid.y, precision=6)
r = ag.adjustable_RSU(gid, p, (i, j))
r.set_max_size(x, y)
rsu_arr.append(r)
if not os.path.exists(os.path.join(os.getcwd(), 'data')):
raise OSError("Must first download data, see README.md")
data_dir = os.path.join(os.getcwd(), 'data')
file_path = os.path.join(data_dir, '{}-{}-grids_df.pkl'.format(x, y))
with open(file_path, 'rb') as handle:
df = pd.read_pickle(handle)
sub_grid_rsus = []
TMC_THRESH = df['tmc_count'].quantile(0.75)
DIV_X, DIV_Y = 2, 2
for i, row in df.iterrows():
gid = row['grid_id']
tmc_count = row['tmc_count']
if tmc_count > TMC_THRESH:
# BUG: If DIV_X and DIV_Y are odd, the central grid has the same gid as the parent grid
# if tmc_count / (DIV_X * DIV_Y) > TMC_THRESH:
# DIV_X, DIV_Y = 3, 3
r = geo_utils.get_rsu_by_grid_id(rsu_arr, gid)
r_poly = r.poly
sub_polys = divide_grid_temp(r_poly, (DIV_X, DIV_Y))
for p in sub_polys:
new_gid = ghh.encode(p.centroid.x, p.centroid.y, precision=6)
new_r = ag.adjustable_RSU(new_gid, p, (1000, 1000))
new_r.queue_limit = GLOBAL_VARS.QUEUE_THRESHOLD
new_r.set_max_size(x, y)
sub_grid_rsus.append(new_r)
# print("Adding: {}".format(new_gid))
r.add_sub_grid(new_r)
# main_no_sub_polys = [r.poly for r in rsu_arr if not r.get_sub_grids()]
# sub_grid_polys = [r.poly for r in sub_grid_rsus]
# SUB_GRIDS_DICT = {}
# for r in rsu_arr:
# if r.get_sub_grids():
# SUB_GRIDS_DICT[r.grid_id] = [sg.grid_id for sg in r.get_sub_grids()]
file_path = os.path.join(data_dir, "{}-{}-rsu_arr.pkl".format(x, y))
with open(file_path, 'wb') as handle:
pickle.dump(rsu_arr, handle)
utils.print_log("Generated rsu_arr for broker.")
def get_dataframe_historical_data(self, grid, with_neighbors=True):
print("get_dataframe_historical_data({})".format(grid))
if with_neighbors and self.rsu_arr:
r = geo_utils.get_rsu_by_grid_id(self.rsu_arr, grid)
d = r.get_neighbors(self.rsu_arr)
hist_dfs = []
for _, n in d.items():
if n:
if n.grid_id != GRID_ID:
# print("Must get old data for {}".format(n.grid_id))
name = "{}_historical_speeds.pkl".format(n.grid_id)
hist_df = self.get_pickled_df(
self.historical_speeds_dir, name)
hist_dfs.append(hist_df)
all_hist_dfs = pd.concat(hist_dfs, ignore_index=True)
return all_hist_dfs
def get_speeds_hash_for_grid(self, grid_id, with_neighbors=False):
G = {}
file_path = os.path.join(
self.avg_speeds_dir,
f'one_minute/{grid_id}_dict_avg_speeds_minute.pkl')
# file_path = os.path.join(self.avg_speeds_dir, '5x5/{}-avg_speeds.pkl'.format(grid_id))
with open(file_path, 'rb') as handle:
g = pickle.load(handle)
G = {**G, **g}
if with_neighbors and self.rsu_arr:
r = geo_utils.get_rsu_by_grid_id(self.rsu_arr, grid_id)
d = r.get_neighbors(self.rsu_arr)
for _, n in d.items():
if n:
file_path = os.path.join(
self.avg_speeds_dir,
f'one_minute/{n.grid_id}_dict_avg_speeds_minute.pkl')
# file_path = os.path.join(self.avg_speeds_dir, '5x5/{}-avg_speeds.pkl'.format(n.grid_id))
with open(file_path, 'rb') as handle:
g = pickle.load(handle)
G = {**G, **g}
return G
def subtask_allocation(self, nlevel, subtask):
data = subtask.get_json()
gridA = data['gridA']
gridB = data['gridB']
if isinstance(gridA, int):
optimal_rsu = gridB
else:
optimal_rsu = gridA
if DEBUG == 1:
print("Checking for: {}".format(optimal_rsu))
if nlevel == 0:
self._mongodb_c._db.tasks.update_one({"_id": data['_id']}, {
'$set': {
'rsu_assigned_to': optimal_rsu,
'allocation_time': utils.time_print(int)
}
})
return
if nlevel > 0:
r = geo_utils.get_rsu_by_grid_id(self._rsu_arr, optimal_rsu)
nn = geo_utils.get_neighbors_level(self._rsu_arr, r.get_idx(),
nlevel)
nn.insert(0, r.get_idx())
found = False
candidate_rsus = []
# Arranges the neighbors by distance from the most optimal grid in question
# DEBUG: This will cause some additional delays
nn = geo_utils.sort_idx_by_distance(GLOBAL_VARS.X_AXIS,
GLOBAL_VARS.Y_AXIS,
r.get_idx(), nn)
nn.sort(key=lambda x: x[1])
nn = [n[0] for n in nn]
for n in nn:
rsu = self._rsu_arr[n]
candidate_rsus.append(rsu)
res, gid = rsu.add_task(
self._nxg,
self._rsu_arr,
subtask,
use_sub_grids=GLOBAL_VARS.USE_SUB_GRIDS)
if res:
found = True
self._mongodb_c._db.tasks.update_one(
{"_id": data['_id']}, {
'$set': {
'rsu_assigned_to': gid,
'allocation_time': utils.time_print(int)
}
})
# utils.print_log("Assigned to: {}".format(gid))
break
'''
If not ok, move to the next one (keep which has lowest number)
'''
# Get RSU with least number of queue
if not found:
if DEBUG == 1:
print("Not found by looking, must force...")
# NOTE: Just shuffling so there is a chance that subtasks will be assigned to different RSUs and not just in the order they come in (if they are both tied for minimum queue lengths)
# IDEA: Check if this has an effect
# Right now as the last resort, it appends to the RSU with least queue length
# If i don't randomize this, then it will assign to the least distance, which is just the most optimal one (may cause delay)
random.shuffle(candidate_rsus)
candidate_rsus = sorted(candidate_rsus,
key=lambda rsu: len(rsu.queue),
reverse=False)
if DEBUG == 1:
[print(r) for r in candidate_rsus]
if candidate_rsus[0]._sub_grids:
[print(r) for r in candidate_rsus[0]._sub_grids]
print("\n")
_, gid = candidate_rsus[0].add_task_forced(
subtask, use_sub_grids=GLOBAL_VARS.USE_SUB_GRIDS)
# utils.print_log("Forced to: {}".format(gid))
self._mongodb_c._db.tasks.update_one({"_id": data['_id']}, {
'$set': {
'rsu_assigned_to': gid,
'allocation_time': utils.time_print(int)
}
})
return
def add_task(self,
G,
rsu_arr,
task,
use_sub_grids=True,
constraints='queue'):
if DEBUG == 1:
print("Trying to add task {} to {} with q[{}/{}]".format(
task._id, self.grid_id, len(self.queue), self.queue_limit))
ideal_grid = task.gridA
if isinstance(task.gridA, int):
ideal_grid = task.gridB
elif isinstance(task.gridA, str):
ideal_grid = task.gridA
if isinstance(task.gridB, int):
next_grid = geo_utils.get_grid_id_from_node(G, task.gridB)
elif isinstance(task.gridB, str):
next_grid = task.gridB
elif task.gridB is None:
next_grid = ideal_grid
ideal_rsu = geo_utils.get_rsu_by_grid_id(rsu_arr, ideal_grid)
next_rsu = geo_utils.get_rsu_by_grid_id(rsu_arr, next_grid)
# Not affected by change in actual grid
delay_ideal = self.get_manhattan_distance(ideal_rsu)
delay_next = self.get_manhattan_distance(next_rsu)
total_delay = delay_ideal + delay_next
# constraints
met_delay_constraint = True
met_queue_constraint = True
use_delay_constraint = False
use_queue_constraint = False
constraint_arr = constraints.split("|")
if 'delay' in constraint_arr:
use_delay_constraint = True
if 'queue' in constraint_arr:
use_queue_constraint = True
if use_delay_constraint:
if total_delay > self.delay_threshold:
met_delay_constraint = False
if use_queue_constraint:
if (len(self.queue) + 1) > self.queue_limit:
met_queue_constraint = False
if met_delay_constraint and met_queue_constraint:
if DEBUG == 1:
print("{} Adding task: {}".format(self.grid_id, task._id))
# This will fill up the parent rsu first before moving on to subgrids
if (len(self.queue) + 1) < self.queue_limit:
self.queue.append(task)
res = True
grid_id = self.grid_id
return res, grid_id
elif use_sub_grids and len(self._sub_grids) > 0:
res, grid_id = self.add_task_to_sub_grids(G,
rsu_arr,
task,
constraints='queue')
return res, grid_id
else:
self.queue.append(task)
res = True
grid_id = self.grid_id
return res, grid_id
else:
return False, ""