def find_the_gaps_user_info_to_mp_fetched(dbcc, user_id, start_date, start_time, end_date, end_time, user_info_list): start_unix_minutes = support.date_time_to_unix_minutes_timestamp(datetime.datetime.combine(start_date, start_time)) end_unix_minutes = support.date_time_to_unix_minutes_timestamp(datetime.datetime.combine(end_date, end_time)) missing_ranges_unix_minutes = support.reduce_to_missing_ranges( user_info_list, lambda x: support.date_time_to_unix_minutes_timestamp(datetime.datetime.combine(x.date, x.time)), 1, start_unix_minutes, end_unix_minutes, ) mp_fetched = [] for missing_range_unix_minutes in missing_ranges_unix_minutes: progression = missing_range_unix_minutes.get_progression() for missing_unix_minute in progression: date_time = support.unix_minutes_timestamp_to_date_time(missing_unix_minute) time = date_time.time() date = date_time.date() minute_of_day = support.calc_minute_of_day(time) day = support.calc_day(date) cell_id = 0 new_mp_fetched = db_layout.MPFetched.new(dbcc, user_id, minute_of_day, date, day, cell_id) mp_fetched.append(new_mp_fetched) return mp_fetched
def find_the_gaps_user_info_to_mp_fetched(dbcc, user_id, start_date, start_time, end_date, end_time, user_info_list): start_unix_minutes = support.date_time_to_unix_minutes_timestamp( datetime.datetime.combine(start_date, start_time)) end_unix_minutes = support.date_time_to_unix_minutes_timestamp( datetime.datetime.combine(end_date, end_time)) missing_ranges_unix_minutes = support.reduce_to_missing_ranges( user_info_list, lambda x: support.date_time_to_unix_minutes_timestamp( datetime.datetime.combine(x.date, x.time)), 1, start_unix_minutes, end_unix_minutes) mp_fetched = [] for missing_range_unix_minutes in missing_ranges_unix_minutes: progression = missing_range_unix_minutes.get_progression() for missing_unix_minute in progression: date_time = support.unix_minutes_timestamp_to_date_time( missing_unix_minute) time = date_time.time() date = date_time.date() minute_of_day = support.calc_minute_of_day(time) day = support.calc_day(date) cell_id = 0 new_mp_fetched = db_layout.MPFetched.new(dbcc, user_id, minute_of_day, date, day, cell_id) mp_fetched.append(new_mp_fetched) return mp_fetched
def retrieve_user_mobility_data_single_user(dbcc, user_id, current_date, predicted_date): history_date = calculate_mp_history_date(current_date) current_week_date_range = support.unix_weeks_timestamp_to_date_range( support.date_to_unix_weeks_timestamp(current_date) ) history_week_date_range = support.unix_weeks_timestamp_to_date_range( support.date_to_unix_weeks_timestamp(history_date) ) begin_of_history_week = history_week_date_range.min_range end_of_current_week = current_week_date_range.max_range current_day = support.calc_day(current_date) predicted_day = support.calc_day(predicted_date) applicable_days = support.difference_of_days(current_day, predicted_day) con = dbcc.retrieve_connection(db_layout.MPFetched.database_name) select = "*" table = db_layout.MPFetched.table_name layout_dict = dict(db_layout.MPFetched.layout) user_id_sql = layout_dict["user_id"].format_to_db_value(user_id) end_date_sql = layout_dict["date"].format_to_db_value(end_of_current_week) start_date_sql = layout_dict["date"].format_to_db_value(begin_of_history_week) applicable_days_sql = map(layout_dict["day"].format_to_db_value, applicable_days) applicable_days_query = " AND (day = " + " OR day = ".join(applicable_days_sql) + ")" where_query = ( "WHERE user_id = " + user_id_sql + " AND date <= " + end_date_sql + " AND date >= " + start_date_sql + applicable_days_query ) err.log_error(err.INFO, "Retrieving data for MP between " + str(history_date) + " and " + str(current_date)) mysql_rows = con.retrieve(select, table, where_query) result = [] for mysql_row in mysql_rows: result.append(db_layout.MPFetched.object_from_mysql_row(dbcc, mysql_row)) return result
def retrieve_user_mobility_data_single_user(dbcc, user_id, current_date, predicted_date): history_date = calculate_mp_history_date(current_date) current_week_date_range = support.unix_weeks_timestamp_to_date_range( support.date_to_unix_weeks_timestamp(current_date)) history_week_date_range = support.unix_weeks_timestamp_to_date_range( support.date_to_unix_weeks_timestamp(history_date)) begin_of_history_week = history_week_date_range.min_range end_of_current_week = current_week_date_range.max_range current_day = support.calc_day(current_date) predicted_day = support.calc_day(predicted_date) applicable_days = support.difference_of_days(current_day, predicted_day) con = dbcc.retrieve_connection(db_layout.MPFetched.database_name) select = "*" table = db_layout.MPFetched.table_name layout_dict = dict(db_layout.MPFetched.layout) user_id_sql = layout_dict["user_id"].format_to_db_value(user_id) end_date_sql = layout_dict["date"].format_to_db_value(end_of_current_week) start_date_sql = layout_dict["date"].format_to_db_value( begin_of_history_week) applicable_days_sql = map(layout_dict["day"].format_to_db_value, applicable_days) applicable_days_query = " AND (day = " + " OR day = ".join( applicable_days_sql) + ")" where_query = ("WHERE user_id = " + user_id_sql + " AND date <= " + end_date_sql + " AND date >= " + start_date_sql + applicable_days_query) err.log_error( err.INFO, "Retrieving data for MP between " + str(history_date) + " and " + str(current_date)) mysql_rows = con.retrieve(select, table, where_query) result = [] for mysql_row in mysql_rows: result.append( db_layout.MPFetched.object_from_mysql_row(dbcc, mysql_row)) return result
def clear_user_information_maas_message(dbcc, msg): user_id = msg.user_id db_user_mobility_information_list = [] #Save it in mean database and convert to db layout object for movement in msg.user_information: db_user_mobility_information = db_layout.UserMobilityInformation.new( dbcc, user_id, movement.time, movement.date, movement.cell_id) db_user_mobility_information_list.append(db_user_mobility_information) db_layout.UserMobilityInformation.save_many( dbcc, db_user_mobility_information_list) mp_fetched_list = [] #Clear the data and save it in the fetched database for db_user_mobility_information in db_user_mobility_information_list: minute_of_day = support.calc_minute_of_day( db_user_mobility_information.time) day = support.calc_day(db_user_mobility_information.date) mp_fetched = db_layout.MPFetched.new( dbcc, user_id, minute_of_day, db_user_mobility_information.date, day, db_user_mobility_information.cell_id) mp_fetched_list.append(mp_fetched) #Fill in gaps end_date_time = datetime.datetime.combine( msg.start_date, msg.start_time) + datetime.timedelta(seconds=msg.time_span) gaps_mp_fetched = find_the_gaps_user_info_to_mp_fetched( dbcc, msg.user_id, msg.start_date, msg.start_time, end_date_time.date(), end_date_time.time(), db_user_mobility_information_list) mp_fetched_list.extend(gaps_mp_fetched) db_layout.MPFetched.save_many(dbcc, mp_fetched_list) err.log_error( err.INFO, "Cleared " + str(len(msg.user_information)) + " rows user information data") err.log_error( err.INFO, "Filled in " + str(len(gaps_mp_fetched)) + " holes with mp fetched data") return sr.SuppliedUserInfo.from_maas_user_info_message_to_supplied_info( msg)
def mobility_prediction(pykov_chain, current_cell_id, current_date, current_time, predicted_date, predicted_time): initial_minute_of_day = support.calc_minute_of_day(current_time) initial_day = support.calc_day(current_date) initial_key = UserEntry(initial_day, initial_minute_of_day, current_cell_id).to_pykov_key() initial_pykov_vector = pykov.Vector({initial_key: 1}) difference_in_seconds = ( datetime.datetime.combine(predicted_date, predicted_time) - datetime.datetime.combine(current_date, current_time) ).total_seconds() difference_in_minutes = int(difference_in_seconds / 60) distribution_dict = pykov_chain.pow(initial_pykov_vector, difference_in_minutes) print "eeeeeee", distribution_dict return distribution_dict
def clear_user_information_maas_message(dbcc, msg): user_id = msg.user_id db_user_mobility_information_list = [] # Save it in mean database and convert to db layout object for movement in msg.user_information: db_user_mobility_information = db_layout.UserMobilityInformation.new( dbcc, user_id, movement.time, movement.date, movement.cell_id ) db_user_mobility_information_list.append(db_user_mobility_information) db_layout.UserMobilityInformation.save_many(dbcc, db_user_mobility_information_list) mp_fetched_list = [] # Clear the data and save it in the fetched database for db_user_mobility_information in db_user_mobility_information_list: minute_of_day = support.calc_minute_of_day(db_user_mobility_information.time) day = support.calc_day(db_user_mobility_information.date) mp_fetched = db_layout.MPFetched.new( dbcc, user_id, minute_of_day, db_user_mobility_information.date, day, db_user_mobility_information.cell_id ) mp_fetched_list.append(mp_fetched) # Fill in gaps end_date_time = datetime.datetime.combine(msg.start_date, msg.start_time) + datetime.timedelta( seconds=msg.time_span ) gaps_mp_fetched = find_the_gaps_user_info_to_mp_fetched( dbcc, msg.user_id, msg.start_date, msg.start_time, end_date_time.date(), end_date_time.time(), db_user_mobility_information_list, ) mp_fetched_list.extend(gaps_mp_fetched) db_layout.MPFetched.save_many(dbcc, mp_fetched_list) err.log_error(err.INFO, "Cleared " + str(len(msg.user_information)) + " rows user information data") err.log_error(err.INFO, "Filled in " + str(len(gaps_mp_fetched)) + " holes with mp fetched data") return sr.SuppliedUserInfo.from_maas_user_info_message_to_supplied_info(msg)
def mobility_prediction(pykov_chain, current_cell_id, current_date, current_time, predicted_date, predicted_time): initial_minute_of_day = support.calc_minute_of_day(current_time) initial_day = support.calc_day(current_date) initial_key = UserEntry(initial_day, initial_minute_of_day, current_cell_id).to_pykov_key() initial_pykov_vector = pykov.Vector({initial_key: 1}) difference_in_seconds = ( datetime.datetime.combine(predicted_date, predicted_time) - datetime.datetime.combine(current_date, current_time)).total_seconds() difference_in_minutes = int(difference_in_seconds / 60) distribution_dict = pykov_chain.pow(initial_pykov_vector, difference_in_minutes) print 'eeeeeee', distribution_dict return distribution_dict
def prepare_pykov_chain_with_single_user_mobility_states( grouped_by_date_grid_by_minute_of_day_mp_fetched, current_day_str, predicted_day_str ): list_of_day_str_applicable = support.difference_of_days(current_day_str, predicted_day_str) from_to_user_entry_dict = {} for date, grid_by_minute_of_day_mp_fetched in grouped_by_date_grid_by_minute_of_day_mp_fetched.items(): if support.calc_day(date) in list_of_day_str_applicable: # It seems this day is a contender in the prediction. # For each minute of the day look at the cell id(source) and look at the next because that is the destination # Each pair of source and destination should be added to the from_to user_entry_dict (date doesn't matter anymore as link is formed) # Using the source as the key and adding the destination to the list i = 0 for source_mp_fetched in grid_by_minute_of_day_mp_fetched: if i == common_config.MINUTES_IN_A_DAY - 1: # We have reached the end of the day, must look at the next if possible in the next day next_date = date + support.Range.RESOLUTION_DATETIME_DAY if support.calc_day(next_date) in list_of_day_str_applicable: # It seems the adjoining day is also a contender in the prediction. Configure the destination correctly destination_mp_fetched = grouped_by_date_grid_by_minute_of_day_mp_fetched[next_date][0] else: # There is no connection to the next day destination_mp_fetched = None else: # In the middle of the day, destination is next in the grid destination_mp_fetched = grid_by_minute_of_day_mp_fetched[i + 1] if destination_mp_fetched: source_user_entry = UserEntry( source_mp_fetched.day, source_mp_fetched.minute_of_day, source_mp_fetched.cell_id ) destination_user_entry = UserEntry( destination_mp_fetched.day, destination_mp_fetched.minute_of_day, destination_mp_fetched.cell_id ) # Add it to the from to user entry dict but with the source_user_entry as key # Add it as a key if it doesn't exist yet if source_user_entry in from_to_user_entry_dict: from_to_user_entry_dict[source_user_entry].append(destination_user_entry) else: from_to_user_entry_dict[source_user_entry] = [destination_user_entry] i += 1 # Calculate all percentages the same from and to user entry coincide pykov_chain_entries = {} for starting_user_entry, destinations in from_to_user_entry_dict.items(): starting_key = starting_user_entry.to_pykov_key() total_amount = float(len(destinations)) grouped_by_cell_id = support.group_by(destinations, lambda user_entry: user_entry.cell_id) for destination_cell_id, destinations_with_same_cell_id in grouped_by_cell_id.items(): destination_key = destinations_with_same_cell_id[0].to_pykov_key() amount_of_destinations_with_same_cell_id = len(destinations_with_same_cell_id) percentage = float(amount_of_destinations_with_same_cell_id) / total_amount pykov_chain_entries[starting_key, destination_key] = percentage return pykov.Chain(pykov_chain_entries)
def prepare_pykov_chain_with_single_user_mobility_states( grouped_by_date_grid_by_minute_of_day_mp_fetched, current_day_str, predicted_day_str): list_of_day_str_applicable = support.difference_of_days( current_day_str, predicted_day_str) from_to_user_entry_dict = {} for date, grid_by_minute_of_day_mp_fetched in grouped_by_date_grid_by_minute_of_day_mp_fetched.items( ): if support.calc_day(date) in list_of_day_str_applicable: #It seems this day is a contender in the prediction. #For each minute of the day look at the cell id(source) and look at the next because that is the destination #Each pair of source and destination should be added to the from_to user_entry_dict (date doesn't matter anymore as link is formed) #Using the source as the key and adding the destination to the list i = 0 for source_mp_fetched in grid_by_minute_of_day_mp_fetched: if i == common_config.MINUTES_IN_A_DAY - 1: #We have reached the end of the day, must look at the next if possible in the next day next_date = date + support.Range.RESOLUTION_DATETIME_DAY if support.calc_day( next_date) in list_of_day_str_applicable: #It seems the adjoining day is also a contender in the prediction. Configure the destination correctly destination_mp_fetched = grouped_by_date_grid_by_minute_of_day_mp_fetched[ next_date][0] else: #There is no connection to the next day destination_mp_fetched = None else: #In the middle of the day, destination is next in the grid destination_mp_fetched = grid_by_minute_of_day_mp_fetched[ i + 1] if destination_mp_fetched: source_user_entry = UserEntry( source_mp_fetched.day, source_mp_fetched.minute_of_day, source_mp_fetched.cell_id) destination_user_entry = UserEntry( destination_mp_fetched.day, destination_mp_fetched.minute_of_day, destination_mp_fetched.cell_id) #Add it to the from to user entry dict but with the source_user_entry as key #Add it as a key if it doesn't exist yet if source_user_entry in from_to_user_entry_dict: from_to_user_entry_dict[source_user_entry].append( destination_user_entry) else: from_to_user_entry_dict[source_user_entry] = [ destination_user_entry ] i += 1 #Calculate all percentages the same from and to user entry coincide pykov_chain_entries = {} for starting_user_entry, destinations in from_to_user_entry_dict.items(): starting_key = starting_user_entry.to_pykov_key() total_amount = float(len(destinations)) grouped_by_cell_id = support.group_by( destinations, lambda user_entry: user_entry.cell_id) for destination_cell_id, destinations_with_same_cell_id in grouped_by_cell_id.items( ): destination_key = destinations_with_same_cell_id[0].to_pykov_key() amount_of_destinations_with_same_cell_id = len( destinations_with_same_cell_id) percentage = float( amount_of_destinations_with_same_cell_id) / total_amount pykov_chain_entries[starting_key, destination_key] = percentage return pykov.Chain(pykov_chain_entries)
def handle_incoming_mp_single_user_request(frontend, connection, msg): err.log_error(err.INFO, "Using MP algorithm") predicted_date_time = datetime.datetime.combine(msg.current_date, msg.current_time) + datetime.timedelta(seconds=msg.future_time_delta) predicted_date = predicted_date_time.date() predicted_time = predicted_date_time.time() ''' if user_id is 0000 (i.e. 0, because it is an integer), we need to use multi-user prediction, not single user prediction ''' if msg.user_id != 0: mp_fetched = mp.retrieve_user_mobility_data_single_user(frontend.dbcc, msg.user_id, msg.current_date, predicted_date) grouped_by_date_mp_fetched = support.group_by(mp_fetched, lambda x: x.date) grouped_by_date_grid_by_minute_of_day_mp_fetched = {} for date, mp_fetched_list in grouped_by_date_mp_fetched.items(): minute_of_day_grid = support.to_grid( mp_fetched_list , 1 , lambda x: x.minute_of_day , 0 , common_config.MINUTES_IN_A_DAY - 1 ) grouped_by_date_grid_by_minute_of_day_mp_fetched[date] = minute_of_day_grid missing_ranges = mp.check_user_mobility_data_single_user_integrity( grouped_by_date_grid_by_minute_of_day_mp_fetched , msg.current_date ) if len(missing_ranges) > 0: #Missing user information data, queue request and ask data from supplier missing_resource = mr.MissingUserInfo(missing_ranges) frontend.queue_as_pending_request(connection, msg, [missing_resource]) err.log_error(err.INFO, "Did not have enough user info data. Asking supplier for: " + str(missing_ranges)) error_message = mo.Error(102, "MP Request will take longer than expected because user data is missing at MOBaaS") connection.add_message(error_message) for missing_range in missing_ranges: start_time = missing_range.min_range.time() start_date = missing_range.min_range.date() time_span = int((missing_range.max_range - missing_range.min_range).total_seconds()) supplier_message = mo.MaaSSingleUserDataRequest(msg.user_id, start_time, start_date, time_span, 60) frontend.ask_at_supplier(supplier_message) else: #There is all the info we need, calculate the spots current_day_str = support.calc_day(msg.current_date) predicted_day_str = support.calc_day(predicted_date) pykov_chain = mp.prepare_pykov_chain_with_single_user_mobility_states(grouped_by_date_grid_by_minute_of_day_mp_fetched , current_day_str , predicted_day_str ) chance_distribution_dict = mp.mobility_prediction(pykov_chain, msg.current_cell_id, msg.current_date, msg.current_time, predicted_date, predicted_time) #Create mobaas_protocol.prediction objects from the answers mobaas_protocol_predictions = mp.from_pykov_distribution_dict_to_predictions(chance_distribution_dict) answer_message = mo.MPSingleUserAnswer(msg.user_id, predicted_time, predicted_date, mobaas_protocol_predictions) connection.add_message(answer_message) err.log_error(err.INFO, "Found an answer to the request! Answer:" + str(answer_message)) else: print 'error: should not get here!'
def test_calc_day(): monday = support.calc_day(datetime.date(year=2014, month=06, day=30))
time4 = datetime.time(hour=23, minute=59, second=59) value1 = support.calc_minute_of_day(time1) value2 = support.calc_minute_of_day(time2) value3 = support.calc_minute_of_day(time3) value4 = support.calc_minute_of_day(time4) run_tests.compare_answer(value1, 1, "minute of day = 1") run_tests.compare_answer(value2, 60, "minute of day = 2") run_tests.compare_answer(value3, 0, "minute of day = 3") run_tests.compare_answer(value4, 1439, "minute of day = 4") def test_calc_day(): monday = support.calc_day(datetime.date(year=2014, month=06, day=30)) tuesday = support.calc_day(datetime.date(year=2014, month=07, day=01)) wednesday = support.calc_day(datetime.date(year=2014, month=07, day=02)) thursday = support.calc_day(datetime.date(year=2015, month=01, day=15)) friday = support.calc_day(datetime.date(year=2015, month=01, day=16)) saturday = support.calc_day(datetime.date(year=2015, month=01, day=17)) sunday = support.calc_day(datetime.date(year=2015, month=01, day=18)) run_tests.compare_answer(monday, config.MONDAY, "monday") run_tests.compare_answer(tuesday, config.TUESDAY, "tuesday") run_tests.compare_answer(wednesday, config.WEDNESDAY, "wednesday") run_tests.compare_answer(thursday, config.THURSDAY, "thursday") run_tests.compare_answer(friday, config.FRIDAY, "friday") run_tests.compare_answer(saturday, config.SATURDAY, "saturday") run_tests.compare_answer(sunday, config.SUNDAY, "sunday")