def find_the_gaps_user_info_to_mp_fetched(dbcc, user_id, start_date, start_time, end_date, end_time, user_info_list):
start_unix_minutes = support.date_time_to_unix_minutes_timestamp(datetime.datetime.combine(start_date, start_time))
end_unix_minutes = support.date_time_to_unix_minutes_timestamp(datetime.datetime.combine(end_date, end_time))
missing_ranges_unix_minutes = support.reduce_to_missing_ranges(
user_info_list,
lambda x: support.date_time_to_unix_minutes_timestamp(datetime.datetime.combine(x.date, x.time)),
1,
start_unix_minutes,
end_unix_minutes,
)
mp_fetched = []
for missing_range_unix_minutes in missing_ranges_unix_minutes:
progression = missing_range_unix_minutes.get_progression()
for missing_unix_minute in progression:
date_time = support.unix_minutes_timestamp_to_date_time(missing_unix_minute)
time = date_time.time()
date = date_time.date()
minute_of_day = support.calc_minute_of_day(time)
day = support.calc_day(date)
cell_id = 0
new_mp_fetched = db_layout.MPFetched.new(dbcc, user_id, minute_of_day, date, day, cell_id)
mp_fetched.append(new_mp_fetched)
return mp_fetched
def find_the_gaps_user_info_to_mp_fetched(dbcc, user_id, start_date,
start_time, end_date, end_time,
user_info_list):
start_unix_minutes = support.date_time_to_unix_minutes_timestamp(
datetime.datetime.combine(start_date, start_time))
end_unix_minutes = support.date_time_to_unix_minutes_timestamp(
datetime.datetime.combine(end_date, end_time))
missing_ranges_unix_minutes = support.reduce_to_missing_ranges(
user_info_list, lambda x: support.date_time_to_unix_minutes_timestamp(
datetime.datetime.combine(x.date, x.time)), 1, start_unix_minutes,
end_unix_minutes)
mp_fetched = []
for missing_range_unix_minutes in missing_ranges_unix_minutes:
progression = missing_range_unix_minutes.get_progression()
for missing_unix_minute in progression:
date_time = support.unix_minutes_timestamp_to_date_time(
missing_unix_minute)
time = date_time.time()
date = date_time.date()
minute_of_day = support.calc_minute_of_day(time)
day = support.calc_day(date)
cell_id = 0
new_mp_fetched = db_layout.MPFetched.new(dbcc, user_id,
minute_of_day, date, day,
cell_id)
mp_fetched.append(new_mp_fetched)
return mp_fetched
def retrieve_user_mobility_data_single_user(dbcc, user_id, current_date, predicted_date):
history_date = calculate_mp_history_date(current_date)
current_week_date_range = support.unix_weeks_timestamp_to_date_range(
support.date_to_unix_weeks_timestamp(current_date)
)
history_week_date_range = support.unix_weeks_timestamp_to_date_range(
support.date_to_unix_weeks_timestamp(history_date)
)
begin_of_history_week = history_week_date_range.min_range
end_of_current_week = current_week_date_range.max_range
current_day = support.calc_day(current_date)
predicted_day = support.calc_day(predicted_date)
applicable_days = support.difference_of_days(current_day, predicted_day)
con = dbcc.retrieve_connection(db_layout.MPFetched.database_name)
select = "*"
table = db_layout.MPFetched.table_name
layout_dict = dict(db_layout.MPFetched.layout)
user_id_sql = layout_dict["user_id"].format_to_db_value(user_id)
end_date_sql = layout_dict["date"].format_to_db_value(end_of_current_week)
start_date_sql = layout_dict["date"].format_to_db_value(begin_of_history_week)
applicable_days_sql = map(layout_dict["day"].format_to_db_value, applicable_days)
applicable_days_query = " AND (day = " + " OR day = ".join(applicable_days_sql) + ")"
where_query = (
"WHERE user_id = "
+ user_id_sql
+ " AND date <= "
+ end_date_sql
+ " AND date >= "
+ start_date_sql
+ applicable_days_query
)
err.log_error(err.INFO, "Retrieving data for MP between " + str(history_date) + " and " + str(current_date))
mysql_rows = con.retrieve(select, table, where_query)
result = []
for mysql_row in mysql_rows:
result.append(db_layout.MPFetched.object_from_mysql_row(dbcc, mysql_row))
return result
def retrieve_user_mobility_data_single_user(dbcc, user_id, current_date,
predicted_date):
history_date = calculate_mp_history_date(current_date)
current_week_date_range = support.unix_weeks_timestamp_to_date_range(
support.date_to_unix_weeks_timestamp(current_date))
history_week_date_range = support.unix_weeks_timestamp_to_date_range(
support.date_to_unix_weeks_timestamp(history_date))
begin_of_history_week = history_week_date_range.min_range
end_of_current_week = current_week_date_range.max_range
current_day = support.calc_day(current_date)
predicted_day = support.calc_day(predicted_date)
applicable_days = support.difference_of_days(current_day, predicted_day)
con = dbcc.retrieve_connection(db_layout.MPFetched.database_name)
select = "*"
table = db_layout.MPFetched.table_name
layout_dict = dict(db_layout.MPFetched.layout)
user_id_sql = layout_dict["user_id"].format_to_db_value(user_id)
end_date_sql = layout_dict["date"].format_to_db_value(end_of_current_week)
start_date_sql = layout_dict["date"].format_to_db_value(
begin_of_history_week)
applicable_days_sql = map(layout_dict["day"].format_to_db_value,
applicable_days)
applicable_days_query = " AND (day = " + " OR day = ".join(
applicable_days_sql) + ")"
where_query = ("WHERE user_id = " + user_id_sql + " AND date <= " +
end_date_sql + " AND date >= " + start_date_sql +
applicable_days_query)
err.log_error(
err.INFO, "Retrieving data for MP between " + str(history_date) +
" and " + str(current_date))
mysql_rows = con.retrieve(select, table, where_query)
result = []
for mysql_row in mysql_rows:
result.append(
db_layout.MPFetched.object_from_mysql_row(dbcc, mysql_row))
return result
def clear_user_information_maas_message(dbcc, msg):
user_id = msg.user_id
db_user_mobility_information_list = []
#Save it in mean database and convert to db layout object
for movement in msg.user_information:
db_user_mobility_information = db_layout.UserMobilityInformation.new(
dbcc, user_id, movement.time, movement.date, movement.cell_id)
db_user_mobility_information_list.append(db_user_mobility_information)
db_layout.UserMobilityInformation.save_many(
dbcc, db_user_mobility_information_list)
mp_fetched_list = []
#Clear the data and save it in the fetched database
for db_user_mobility_information in db_user_mobility_information_list:
minute_of_day = support.calc_minute_of_day(
db_user_mobility_information.time)
day = support.calc_day(db_user_mobility_information.date)
mp_fetched = db_layout.MPFetched.new(
dbcc, user_id, minute_of_day, db_user_mobility_information.date,
day, db_user_mobility_information.cell_id)
mp_fetched_list.append(mp_fetched)
#Fill in gaps
end_date_time = datetime.datetime.combine(
msg.start_date,
msg.start_time) + datetime.timedelta(seconds=msg.time_span)
gaps_mp_fetched = find_the_gaps_user_info_to_mp_fetched(
dbcc, msg.user_id,
msg.start_date, msg.start_time, end_date_time.date(),
end_date_time.time(), db_user_mobility_information_list)
mp_fetched_list.extend(gaps_mp_fetched)
db_layout.MPFetched.save_many(dbcc, mp_fetched_list)
err.log_error(
err.INFO, "Cleared " + str(len(msg.user_information)) +
" rows user information data")
err.log_error(
err.INFO, "Filled in " + str(len(gaps_mp_fetched)) +
" holes with mp fetched data")
return sr.SuppliedUserInfo.from_maas_user_info_message_to_supplied_info(
msg)
def mobility_prediction(pykov_chain, current_cell_id, current_date, current_time, predicted_date, predicted_time):
initial_minute_of_day = support.calc_minute_of_day(current_time)
initial_day = support.calc_day(current_date)
initial_key = UserEntry(initial_day, initial_minute_of_day, current_cell_id).to_pykov_key()
initial_pykov_vector = pykov.Vector({initial_key: 1})
difference_in_seconds = (
datetime.datetime.combine(predicted_date, predicted_time)
- datetime.datetime.combine(current_date, current_time)
).total_seconds()
difference_in_minutes = int(difference_in_seconds / 60)
distribution_dict = pykov_chain.pow(initial_pykov_vector, difference_in_minutes)
print "eeeeeee", distribution_dict
return distribution_dict
def clear_user_information_maas_message(dbcc, msg):
user_id = msg.user_id
db_user_mobility_information_list = []
# Save it in mean database and convert to db layout object
for movement in msg.user_information:
db_user_mobility_information = db_layout.UserMobilityInformation.new(
dbcc, user_id, movement.time, movement.date, movement.cell_id
)
db_user_mobility_information_list.append(db_user_mobility_information)
db_layout.UserMobilityInformation.save_many(dbcc, db_user_mobility_information_list)
mp_fetched_list = []
# Clear the data and save it in the fetched database
for db_user_mobility_information in db_user_mobility_information_list:
minute_of_day = support.calc_minute_of_day(db_user_mobility_information.time)
day = support.calc_day(db_user_mobility_information.date)
mp_fetched = db_layout.MPFetched.new(
dbcc, user_id, minute_of_day, db_user_mobility_information.date, day, db_user_mobility_information.cell_id
)
mp_fetched_list.append(mp_fetched)
# Fill in gaps
end_date_time = datetime.datetime.combine(msg.start_date, msg.start_time) + datetime.timedelta(
seconds=msg.time_span
)
gaps_mp_fetched = find_the_gaps_user_info_to_mp_fetched(
dbcc,
msg.user_id,
msg.start_date,
msg.start_time,
end_date_time.date(),
end_date_time.time(),
db_user_mobility_information_list,
)
mp_fetched_list.extend(gaps_mp_fetched)
db_layout.MPFetched.save_many(dbcc, mp_fetched_list)
err.log_error(err.INFO, "Cleared " + str(len(msg.user_information)) + " rows user information data")
err.log_error(err.INFO, "Filled in " + str(len(gaps_mp_fetched)) + " holes with mp fetched data")
return sr.SuppliedUserInfo.from_maas_user_info_message_to_supplied_info(msg)
def mobility_prediction(pykov_chain, current_cell_id, current_date,
current_time, predicted_date, predicted_time):
initial_minute_of_day = support.calc_minute_of_day(current_time)
initial_day = support.calc_day(current_date)
initial_key = UserEntry(initial_day, initial_minute_of_day,
current_cell_id).to_pykov_key()
initial_pykov_vector = pykov.Vector({initial_key: 1})
difference_in_seconds = (
datetime.datetime.combine(predicted_date, predicted_time) -
datetime.datetime.combine(current_date, current_time)).total_seconds()
difference_in_minutes = int(difference_in_seconds / 60)
distribution_dict = pykov_chain.pow(initial_pykov_vector,
difference_in_minutes)
print 'eeeeeee', distribution_dict
return distribution_dict
def prepare_pykov_chain_with_single_user_mobility_states(
grouped_by_date_grid_by_minute_of_day_mp_fetched, current_day_str, predicted_day_str
):
list_of_day_str_applicable = support.difference_of_days(current_day_str, predicted_day_str)
from_to_user_entry_dict = {}
for date, grid_by_minute_of_day_mp_fetched in grouped_by_date_grid_by_minute_of_day_mp_fetched.items():
if support.calc_day(date) in list_of_day_str_applicable:
# It seems this day is a contender in the prediction.
# For each minute of the day look at the cell id(source) and look at the next because that is the destination
# Each pair of source and destination should be added to the from_to user_entry_dict (date doesn't matter anymore as link is formed)
# Using the source as the key and adding the destination to the list
i = 0
for source_mp_fetched in grid_by_minute_of_day_mp_fetched:
if i == common_config.MINUTES_IN_A_DAY - 1:
# We have reached the end of the day, must look at the next if possible in the next day
next_date = date + support.Range.RESOLUTION_DATETIME_DAY
if support.calc_day(next_date) in list_of_day_str_applicable:
# It seems the adjoining day is also a contender in the prediction. Configure the destination correctly
destination_mp_fetched = grouped_by_date_grid_by_minute_of_day_mp_fetched[next_date][0]
else:
# There is no connection to the next day
destination_mp_fetched = None
else:
# In the middle of the day, destination is next in the grid
destination_mp_fetched = grid_by_minute_of_day_mp_fetched[i + 1]
if destination_mp_fetched:
source_user_entry = UserEntry(
source_mp_fetched.day, source_mp_fetched.minute_of_day, source_mp_fetched.cell_id
)
destination_user_entry = UserEntry(
destination_mp_fetched.day, destination_mp_fetched.minute_of_day, destination_mp_fetched.cell_id
)
# Add it to the from to user entry dict but with the source_user_entry as key
# Add it as a key if it doesn't exist yet
if source_user_entry in from_to_user_entry_dict:
from_to_user_entry_dict[source_user_entry].append(destination_user_entry)
else:
from_to_user_entry_dict[source_user_entry] = [destination_user_entry]
i += 1
# Calculate all percentages the same from and to user entry coincide
pykov_chain_entries = {}
for starting_user_entry, destinations in from_to_user_entry_dict.items():
starting_key = starting_user_entry.to_pykov_key()
total_amount = float(len(destinations))
grouped_by_cell_id = support.group_by(destinations, lambda user_entry: user_entry.cell_id)
for destination_cell_id, destinations_with_same_cell_id in grouped_by_cell_id.items():
destination_key = destinations_with_same_cell_id[0].to_pykov_key()
amount_of_destinations_with_same_cell_id = len(destinations_with_same_cell_id)
percentage = float(amount_of_destinations_with_same_cell_id) / total_amount
pykov_chain_entries[starting_key, destination_key] = percentage
return pykov.Chain(pykov_chain_entries)
def prepare_pykov_chain_with_single_user_mobility_states(
grouped_by_date_grid_by_minute_of_day_mp_fetched, current_day_str,
predicted_day_str):
list_of_day_str_applicable = support.difference_of_days(
current_day_str, predicted_day_str)
from_to_user_entry_dict = {}
for date, grid_by_minute_of_day_mp_fetched in grouped_by_date_grid_by_minute_of_day_mp_fetched.items(
):
if support.calc_day(date) in list_of_day_str_applicable:
#It seems this day is a contender in the prediction.
#For each minute of the day look at the cell id(source) and look at the next because that is the destination
#Each pair of source and destination should be added to the from_to user_entry_dict (date doesn't matter anymore as link is formed)
#Using the source as the key and adding the destination to the list
i = 0
for source_mp_fetched in grid_by_minute_of_day_mp_fetched:
if i == common_config.MINUTES_IN_A_DAY - 1:
#We have reached the end of the day, must look at the next if possible in the next day
next_date = date + support.Range.RESOLUTION_DATETIME_DAY
if support.calc_day(
next_date) in list_of_day_str_applicable:
#It seems the adjoining day is also a contender in the prediction. Configure the destination correctly
destination_mp_fetched = grouped_by_date_grid_by_minute_of_day_mp_fetched[
next_date][0]
else:
#There is no connection to the next day
destination_mp_fetched = None
else:
#In the middle of the day, destination is next in the grid
destination_mp_fetched = grid_by_minute_of_day_mp_fetched[
i + 1]
if destination_mp_fetched:
source_user_entry = UserEntry(
source_mp_fetched.day, source_mp_fetched.minute_of_day,
source_mp_fetched.cell_id)
destination_user_entry = UserEntry(
destination_mp_fetched.day,
destination_mp_fetched.minute_of_day,
destination_mp_fetched.cell_id)
#Add it to the from to user entry dict but with the source_user_entry as key
#Add it as a key if it doesn't exist yet
if source_user_entry in from_to_user_entry_dict:
from_to_user_entry_dict[source_user_entry].append(
destination_user_entry)
else:
from_to_user_entry_dict[source_user_entry] = [
destination_user_entry
]
i += 1
#Calculate all percentages the same from and to user entry coincide
pykov_chain_entries = {}
for starting_user_entry, destinations in from_to_user_entry_dict.items():
starting_key = starting_user_entry.to_pykov_key()
total_amount = float(len(destinations))
grouped_by_cell_id = support.group_by(
destinations, lambda user_entry: user_entry.cell_id)
for destination_cell_id, destinations_with_same_cell_id in grouped_by_cell_id.items(
):
destination_key = destinations_with_same_cell_id[0].to_pykov_key()
amount_of_destinations_with_same_cell_id = len(
destinations_with_same_cell_id)
percentage = float(
amount_of_destinations_with_same_cell_id) / total_amount
pykov_chain_entries[starting_key, destination_key] = percentage
return pykov.Chain(pykov_chain_entries)
def handle_incoming_mp_single_user_request(frontend, connection, msg):
err.log_error(err.INFO, "Using MP algorithm")
predicted_date_time = datetime.datetime.combine(msg.current_date, msg.current_time) + datetime.timedelta(seconds=msg.future_time_delta)
predicted_date = predicted_date_time.date()
predicted_time = predicted_date_time.time()
'''
if user_id is 0000 (i.e. 0, because it is an integer), we need to use multi-user prediction, not single user prediction
'''
if msg.user_id != 0:
mp_fetched = mp.retrieve_user_mobility_data_single_user(frontend.dbcc, msg.user_id, msg.current_date, predicted_date)
grouped_by_date_mp_fetched = support.group_by(mp_fetched, lambda x: x.date)
grouped_by_date_grid_by_minute_of_day_mp_fetched = {}
for date, mp_fetched_list in grouped_by_date_mp_fetched.items():
minute_of_day_grid = support.to_grid( mp_fetched_list
, 1
, lambda x: x.minute_of_day
, 0
, common_config.MINUTES_IN_A_DAY - 1
)
grouped_by_date_grid_by_minute_of_day_mp_fetched[date] = minute_of_day_grid
missing_ranges = mp.check_user_mobility_data_single_user_integrity( grouped_by_date_grid_by_minute_of_day_mp_fetched
, msg.current_date
)
if len(missing_ranges) > 0:
#Missing user information data, queue request and ask data from supplier
missing_resource = mr.MissingUserInfo(missing_ranges)
frontend.queue_as_pending_request(connection, msg, [missing_resource])
err.log_error(err.INFO, "Did not have enough user info data. Asking supplier for: " + str(missing_ranges))
error_message = mo.Error(102, "MP Request will take longer than expected because user data is missing at MOBaaS")
connection.add_message(error_message)
for missing_range in missing_ranges:
start_time = missing_range.min_range.time()
start_date = missing_range.min_range.date()
time_span = int((missing_range.max_range - missing_range.min_range).total_seconds())
supplier_message = mo.MaaSSingleUserDataRequest(msg.user_id, start_time, start_date, time_span, 60)
frontend.ask_at_supplier(supplier_message)
else:
#There is all the info we need, calculate the spots
current_day_str = support.calc_day(msg.current_date)
predicted_day_str = support.calc_day(predicted_date)
pykov_chain = mp.prepare_pykov_chain_with_single_user_mobility_states(grouped_by_date_grid_by_minute_of_day_mp_fetched
, current_day_str
, predicted_day_str
)
chance_distribution_dict = mp.mobility_prediction(pykov_chain, msg.current_cell_id, msg.current_date, msg.current_time, predicted_date, predicted_time)
#Create mobaas_protocol.prediction objects from the answers
mobaas_protocol_predictions = mp.from_pykov_distribution_dict_to_predictions(chance_distribution_dict)
answer_message = mo.MPSingleUserAnswer(msg.user_id, predicted_time, predicted_date, mobaas_protocol_predictions)
connection.add_message(answer_message)
err.log_error(err.INFO, "Found an answer to the request! Answer:" + str(answer_message))
else:
print 'error: should not get here!'
def test_calc_day():
monday = support.calc_day(datetime.date(year=2014, month=06, day=30))
time4 = datetime.time(hour=23, minute=59, second=59)
value1 = support.calc_minute_of_day(time1)
value2 = support.calc_minute_of_day(time2)
value3 = support.calc_minute_of_day(time3)
value4 = support.calc_minute_of_day(time4)
run_tests.compare_answer(value1, 1, "minute of day = 1")
run_tests.compare_answer(value2, 60, "minute of day = 2")
run_tests.compare_answer(value3, 0, "minute of day = 3")
run_tests.compare_answer(value4, 1439, "minute of day = 4")
def test_calc_day():
monday = support.calc_day(datetime.date(year=2014, month=06, day=30))
tuesday = support.calc_day(datetime.date(year=2014, month=07, day=01))
wednesday = support.calc_day(datetime.date(year=2014, month=07, day=02))
thursday = support.calc_day(datetime.date(year=2015, month=01, day=15))
friday = support.calc_day(datetime.date(year=2015, month=01, day=16))
saturday = support.calc_day(datetime.date(year=2015, month=01, day=17))
sunday = support.calc_day(datetime.date(year=2015, month=01, day=18))
run_tests.compare_answer(monday, config.MONDAY, "monday")
run_tests.compare_answer(tuesday, config.TUESDAY, "tuesday")
run_tests.compare_answer(wednesday, config.WEDNESDAY, "wednesday")
run_tests.compare_answer(thursday, config.THURSDAY, "thursday")
run_tests.compare_answer(friday, config.FRIDAY, "friday")
run_tests.compare_answer(saturday, config.SATURDAY, "saturday")
run_tests.compare_answer(sunday, config.SUNDAY, "sunday")
