def run(server_class=HTTPServer, handler_class=S, port=8000):
server_address = ('', port)
httpd = server_class(server_address, handler_class)
mylogger.info('Starting httpd...\n')
check_tasks_to_schedule()
try:
httpd.serve_forever()
except KeyboardInterrupt:
pass
httpd.server_close()
mylogger.info('Stopping httpd...\n')
def do_POST(self):
content_length = int(self.headers['Content-Length']) # <--- Gets the size of data
post_data = self.rfile.read(content_length) # <--- Gets the data itself
mylogger.info("POST request,\nPath: %s\nHeaders:\n%s\n\nBody:\n%s\n", str(self.path), str(self.headers), post_data.decode('utf-8'))
mylogger.info("TYPE: {}".format(type(post_data.decode('utf-8'))))
data_json = json.loads(post_data.decode('utf-8'))
mylogger.info("JSON data {}".format(data_json))
if data_json["content"] == "newTaskInserted":
mylogger.info("calling check_tasks_to_schedule")
check_tasks_to_schedule()
def find_nearby_devices(self, need_immediate_number):
mylogger.info('Immediate need ' + str(need_immediate_number) +
" devices for task " + str(self.id))
cur_time = current_milli_time()
nearby_idss = locations.find(
{"timeStamp": {
"$gte": cur_time - 15 * MINUTE,
"$lte": cur_time
}})
nearby_idsss = [
x['uid'] for x in nearby_idss
if vincenty(x['location'], (self.latitude,
self.longitude)).meters < self.radius
]
mylogger.info("found " + str(nearby_idsss))
nearby_ids = list(
set(nearby_idsss) - set(self.assignment_list) -
set(self.waiting_pool))
mylogger.info("after elimination" + str(nearby_ids))
device_list = []
for device_id in nearby_ids:
loc = list(
locations.find({
"uid": device_id,
"location": {
"$ne": [0, 0]
}
}).sort("timeStamp", -1).limit(1))[0]['location']
bat = db.Batteries.find_one({"_id": device_id})
user = db.Users.find_one({'_id': device_id})
token = user.get('token', '')
selected_time = user.get('selected_times', 0)
if token != '':
device_list.append(
Device(id=device_id,
latitude=loc[0],
longitude=loc[1],
bat_level=float(bat["level"]),
bat_status=bat["status"],
selected_times=selected_time,
token=token))
if len(device_list) > need_immediate_number:
return list(
sorted(update_device_scores(device_list),
key=lambda d: d.score)[:need_immediate_number])
else:
return device_list
def path_model(device_list, location, radius, cur_time, target_time,
tolerance):
# device_list: list of Device objects
# location: [latitude: float, longitude: float]
# radius: location tolerance (meters)
# cur_time: current time (milliseconds)
# target_time: epoch time (milliseconds) at which the task should occur
# tolerance: time tolerance in milliseconds
table = locations
# Digits of significance for latitude/longitude values
degree_digits = 3
# The predicted trajectory starts at the current time and ends at the target time
trajectory_length = target_time - cur_time
# The current trajectory ends where the predicted trajectory starts and is the same length
cur_trajectory_start = cur_time - trajectory_length
# When comparing the current trajectory to the predicted trajectory, these are the steps
# (in milliseconds) that they are normalized to
steps = trajectory_length / (5 * 60 * 1000)
tolerable_radius = 100
for device in device_list:
mylogger.info('\n\n\n\n\npredicting for device ' + str(device.id))
# Get the current trajectory so it can be compared against the predicted trajectories
cur_trajectory = list(
table.find({
"uid": device.id,
"timeStamp": {
"$gte": cur_trajectory_start,
"$lt": cur_time
},
"location": {
"$ne": [0, 0]
}
}).sort("timeStamp", 1))
mylogger.info('cur_trajectory length=' + str(len(cur_trajectory)))
if len(cur_trajectory) == 0:
device.probability = None
continue
# Find points near to the first point of the current trajectory
cur_tra_first_point = cur_trajectory[0]["location"]
#near_points = table.find({
# "uid": device.id,
# "timeStamp": {
# "$lt": cur_time - trajectory_length
# }
#}).sort("timeStamp", 1)
#near_points = [x for x in near_points if vincenty(x['location'], cur_tra_first_point).meters <= tolerable_radius]
near_points = range_query(table, cur_tra_first_point[::-1],
tolerable_radius, 0,
cur_time - trajectory_length, {
'location': 1,
'timeStamp': 1
}).sort('timeStamp', 1)
# Starting times of the predicted trajectories
start_times = [
last(g)["timeStamp"] for k, g in groupby(
near_points, lambda x: roundv(x["location"], degree_digits))
]
# TODO: If there are many start_times then many adjust probability to reflect uncertainty
mylogger.info('start_times length=' + str(len(start_times)))
if len(start_times) == 0:
device.probability = None
continue
# All the predicted trajectories will be contained in this list
historical_trajectory = list(
table.find({
"uid": device.id,
"timeStamp": {
"$gte": start_times[0],
"$lt": start_times[-1] + trajectory_length
}
}))
mylogger.info('start_times first/last= ' + str(start_times[0]) + " " +
str((start_times[-1])))
mylogger.info('historical_trajectory length=' +
str(len(historical_trajectory)))
def compare_trajectory_starting_at(x):
i = binary_search(historical_trajectory,
x,
key=lambda x: x["timeStamp"])
tra = historical_trajectory[i:]
c = compare_trajectory(tra, cur_trajectory, 5 * 60 * 1000)
# mylogger.info('compare_trajectory', c)
return tra, c
best_tra, best_error = min(
(compare_trajectory_starting_at(x) for x in start_times),
key=lambda x: x[1])
predicted_location = best_tra[-1]["location"]
mylogger.info('best_error' + str(best_error))
mylogger.info('predicted location' + str(predicted_location))
mylogger.info('distance from target location' +
str(vincenty(predicted_location, location).meters))
device.probability = int(vincenty(best_tra[-1]["location"], location).meters < radius) \
if best_error < steps * tolerable_radius else None
return list(
filter(lambda d: d.probability is not None and d.probability > 0,
device_list))
def send_to_devices(self, devices):
task_dic = tasks.find_one({"_id": self.id}, {
"waiting_pool": 1,
"assignment_list": 1
})
self.assignment_list = task_dic["assignment_list"]
self.waiting_pool = task_dic.get("waiting_pool", [])
cur_time = current_milli_time()
need_immediate = 0
tokens = []
for device in devices:
loc_list = list(
locations.find({
"uid": device.id,
"location": {
"$ne": [0, 0]
},
"timeStamp": {
"$gte": cur_time - 15 * MINUTE,
"$lte": cur_time
}
}).sort("timeStamp", -1).limit(1))
if loc_list:
current_device_location = loc_list[0]['location']
else:
current_device_location = []
if current_device_location:
mylogger.info('try assigning task ' + str(self.id) + ' to ' +
str(device.id))
user = users.find_one({'_id': device.id})
if vincenty(
current_device_location,
(self.latitude, self.longitude)).meters < self.radius:
users.update({'_id': device.id},
{'$inc': {
'predictability': 1
}},
upsert=True)
if 'token' in user and user[
'_id'] not in self.assignment_list and user[
'_id'] in self.waiting_pool:
#send_message(user["token"], self.to_json(device.id))
mylogger.info('assigned task' + str(self.id) + ' to' +
str(device.id))
self.assignment_list.append(device.id)
tokens.append((user['token'], device.id))
self.waiting_pool.remove(device.id)
users.update({'_id': device.id},
{'$inc': {
'selected_times': 1
}},
upsert=True)
else:
mylogger.info("something wrong when assigning task " +
str(self.id) + " to device " +
str(device.id))
mylogger.info("Device info: " + str(device))
mylogger.info("Task info " + str(self))
if device.id in self.assignment_list and device.id in self.waiting_pool:
mylogger.info(
"removed duplicate entries in waiting pool and assignment list device : "
+ str(device.id))
self.waiting_pool.remove(device.id)
else:
mylogger.info(
"device " + str(device.id) +
" is no longer in task region, removed from waiting pool"
)
mylogger.info("the distance is " + str(
vincenty(current_device_location,
(self.latitude, self.longitude)).meters) +
" radius is " + str(self.radius))
users.update({'_id': device.id},
{'$inc': {
'unpredictability': 1
}},
upsert=True)
if device.id in self.assignment_list:
self.assignment_list.remove(device.id)
if device.id in self.waiting_pool:
self.waiting_pool.remove(device.id)
need_immediate += 1
else:
need_immediate += 1
self.waiting_pool.remove(device.id)
mylogger.info("Can't find the current location for " +
str(device.id) + " removed from waiting pool")
users.update({'_id': device.id},
{'$inc': {
'unpredictability': 1
}},
upsert=True)
if need_immediate > 0:
nearby_devices = self.find_nearby_devices(need_immediate)
if len(nearby_devices) == 0:
mylogger.info('Cannot find any nearby devices for task ' +
str(self.id))
for nearby_device in nearby_devices:
tokens.append((nearby_device.token, device.id))
mylogger.info('immediate assigned task ' + str(self.id) +
' to ' + str(nearby_device.id))
self.assignment_list.append(nearby_device.id)
if nearby_device.id in self.waiting_pool:
self.waiting_pool.remove(nearby_device.id)
users.update({'_id': nearby_device.id},
{'$inc': {
'selected_times': 1
}},
upsert=True)
if len(self.assignment_list) >= self.num_devices:
self.assigned = True
tasks.update({'_id': self.id}, {'$set': {'assigned': True}})
if self.time + self.tolerance / 2 < cur_time:
tasks.update_one({"_id": self.id}, {'$set': {"compromised": True}})
mylogger.info('current assignment list: ' + str(self.assignment_list))
mylogger.info('current waiting pool: ' + str(self.waiting_pool))
tasks.update({'_id': self.id},
{'$set': {
'assignment_list': self.assignment_list
}})
tasks.update({'_id': self.id},
{'$set': {
'waiting_pool': self.waiting_pool
}})
for token, device_id in tokens:
send_message(token, self.to_json(device_id))
def check_tasks_to_schedule(start_daemon=start_daemon):
cur_time = current_milli_time()
window_left = cur_time
window_right = window_left + 20 * MINUTE
mylogger.info('start checking tasks at ' + str(milliseconds_to_string(window_left)))
mylogger.info('tasks interval to be included: ' + str(milliseconds_to_string(window_left)) + ' and ' + str(milliseconds_to_string(window_right)))
task_list = list(map(Task, tasks.aggregate([
{
'$match': {'assigned': False}
},
{
'$addFields': {
'time2': {
'$add': [
'$time',
{'$multiply': ['$tolerance', 0.5]}
]
},
'time3': {
'$subtract': [
'$time',
{'$multiply': ['$tolerance', 0.5]}
]
}
}
},
{
'$match': {
'time2': {'$gte': window_left},
'time3': {'$lt': window_right}
}
},
{
'$sort': {'time': 1}
}
])))
mylogger.info("task_list:" + str([str(t) for t in task_list]))
if len(task_list) > 0:
un_finished_tasks = schedule_tasks(task_list, cur_time)
resumed = False
for task in sorted(un_finished_tasks, key=lambda t: t.time):
soonest = task.time - window_left - task.tolerance / 2
latest = task.time - window_left + task.tolerance / 2 - 10 * SECOND
# Note: latest cannot be the exact task.time + tolerance/2, because of the processing delay, when reschedule, the right bound will not be included.
interval = max( MINUTE, soonest)
if latest > interval:
mylogger.info("recall check_task_to_schedule due to" + str(task.id) + ' after ' + str(interval/SECOND) + ' seconds. Now is ' + str(milliseconds_to_string(window_left)))
start_daemon(interval / SECOND, check_tasks_to_schedule, [])
resumed = True
break
else:
mylogger.info("task " + str(task.id) +" is compromised")
tasks.update_one({"_id": task.id}, {'$set': {"compromised": True, "num_devices": len(task.assignment_list)}})
if not resumed:
next_time = (window_right - window_left) / SECOND
mylogger.info("tasks are scheduled, next calling is" + str(next_time) + ' seconds ' + ' Now is '+ str(milliseconds_to_string(window_left)))
start_daemon(next_time, check_tasks_to_schedule, [])
else:
next_tasks = list(tasks.find({
"finished": False,
"assigned": False,
"time": {"$gte": window_right}
}).sort('time', 1).limit(1))
mylogger.info('next_tasks: ' + str([str(t) for t in next_tasks]))
if len(next_tasks) > 0:
next_task = Task(next_tasks[0])
gap = (next_task.time - next_task.tolerance / 2 - window_left)
mylogger.info('start next check_tasks_to_schedule after' + str(gap/SECOND))
start_daemon(gap / SECOND, check_tasks_to_schedule, [])
else:
mylogger.info('start next check_tasks_to_schedule after 20 minutes')
start_daemon(20 * 60, check_tasks_to_schedule, [])
def schedule_tasks(task_list, cur_time):
# take in a list of Task objects
un_finished_tasks = []
for task in task_list:
mylogger.info("\nscheduling " + str(task))
potential_devices = get_potential_devices(task, radius=10 * task.radius, advance_min=15, cur_time=cur_time)
mylogger.info("potentialDevices:")
mylogger.info(str([str(d.id) for d in potential_devices]))
in_range_devices = path_model(potential_devices, (task.latitude, task.longitude), task.radius, cur_time, task.time, task.tolerance)
mylogger.info("in_range_devices:")
mylogger.info(str([str(d.id) for d in in_range_devices]))
if len(in_range_devices) < task.num_devices and task.time - cur_time < 5 * 60 * 1000:
mylogger.info("time is closing. going to find nearby devices")
cur_devices = get_nearby_devices(task)
in_range_devices.extend(cur_devices)
if len(in_range_devices) < task.num_devices:
mylogger.info('not enough in_range_devices ' + str(task.num_devices - len(in_range_devices)) + " more needed")
candidates = filter_devices_by_battery_level(in_range_devices)
mylogger.info("candidates for, " + str(task.id) + " are "+ str([str(d) for d in candidates]))
def increment(device_obj):
device_obj.tasksSatisfied += 1
task.potential_devices = candidates
list(map(increment, task.potential_devices))
for task in task_list:
scored_candidates = update_device_scores(task.potential_devices)
mylogger.info("scored candidates for task" + str(task.id) + " are " + str([str(d) for d in scored_candidates]))
if len(scored_candidates) > 0:
if not assign_task(task, scored_candidates, cur_time):
un_finished_tasks.append(task)
else:
un_finished_tasks.append(task)
return un_finished_tasks
def do_GET(self):
mylogger.info("GET request,\nPath: %s\nHeaders:\n%s\n", str(self.path), str(self.headers))
self._set_response()
self.wfile.write("GET request for {}".format(self.path).encode('utf-8'))
def assign_task(task, scored_candidates, cur_time, daemon=start_daemon):
try:
need_number = task.num_devices - len(task.assignment_list) - len(task.waiting_pool)
if need_number > 0:
if len(scored_candidates) >= need_number:
devices = sorted(scored_candidates, key=lambda d: d.score)[:need_number]
status = True
else:
devices = scored_candidates
mylogger.info("WARNING: assigning task " + str(task.id) + ' is not finished, found ' + str(len(scored_candidates)) + ' devices '+ str(task.num_devices - len(scored_candidates)) + ' more are needed')
status = False
timer = (task.time - cur_time - task.tolerance / 2) / SECOND
if timer > 0:
mylogger.info("assigning " + str(task.id) + " after " + str(timer) + " seconds to " + str(len(devices)) + ' devices')
daemon(timer, task.send_to_devices, [devices])
for d in devices:
if d.id not in task.assignment_list and d.id not in task.waiting_pool:
task.waiting_pool.append(d.id)
tasks.update({'_id': task.id}, {'$set': {'waiting_pool': task.waiting_pool}})
else:
if cur_time < task.time + task.tolerance / 2 :
#within window send right now
mylogger.info("schedule task " + str(task.id) + " within tolerance window")
daemon(5, task.send_to_devices, [devices])
for d in devices:
if d.id not in task.assignment_list and d.id not in task.waiting_pool:
task.waiting_pool.append(d.id)
mylogger.info("devices: " + str([d for d in task.waiting_pool]) + ' are waiting to be assigned')
tasks.update({'_id': task.id}, {'$set': {'waiting_pool': task.waiting_pool}})
else:
mylogger.info("Too late to schedule " + str(task.id) )
status = False
tasks.update({'_id': task.id}, {'$set': {'waiting_pool': []}})
else:
mylogger.info("There are enough devices for task " + str(task.id) + ". No need to assign to new devices.")
mylogger.info("Assignment List is: " + str([d for d in task.assignment_list]))
mylogger.info("waiting pool is: " + str([d for d in task.waiting_pool]))
mylogger.info("candidates were going to assign but not necessary: " + str(scored_candidates))
status = True
return status
except Exception:
traceback.print_exc()
return False