def clear_single_link_capacity_message(dbcc, msg):
link_capacity_data = db_layout.LinkCapacityData.new(dbcc, msg.link_id, msg.capacity)
link_capacity_data.save()
err.log_error(err.INFO, "Cleared link capacity information for link " + str(link_capacity_data.link_id))
return sr.SuppliedLinkCapacity.from_link_capacity_data_to_supplied_info(link_capacity_data)
def handle_user_data_request(dbcc, msg):
err.log_error(err.INFO, "Found a request for user data")
start_date_time = datetime.datetime.combine(msg.start_date, msg.start_time)
end_date_time = start_date_time + datetime.timedelta(seconds=msg.time_span)
end_time = end_date_time.time()
end_date = end_date_time.date()
conditions = [ ("user_id", "=", msg.user_id)
, ("date", ">=", msg.start_date)
, ("date", "<=", end_date)
]
user_information_objects = test_db_layout.MaaSUserMobilityInformation.retrieve_objects(dbcc, conditions)
valid_user_information_objects = []
#Limit between requested start and end time
for user_info_obj in user_information_objects:
user_info_date_time = datetime.datetime.combine(user_info_obj.date, user_info_obj.time)
if start_date_time <= user_info_date_time <= end_date_time:
valid_user_information_objects.append(user_info_obj)
#Limit to the correct resolution
reduced_user_information_objects = test_db_layout.MaaSUserMobilityInformation.reduce_to_time_resolution(valid_user_information_objects, msg.time_resolution)
err.log_error(err.INFO, "Sending " + str(len(reduced_user_information_objects)) + " rows of user data")
return test_db_layout.MaaSUserMobilityInformation.construct_maas_single_user_data_answer(msg.user_id, msg.start_date, msg.start_time, msg.time_span, reduced_user_information_objects)
def close_all_connections(self):
result = True
err.log_error(err.INFO, "All socket connections are being closed...")
for connection in self.connections:
result = result and connection.close()
return result
def read_buffer(self):
while self.has_data_to_read():
try:
self.read_message_buffer += self.sock.recv(config.RECEIVE_CHUNK_SIZE)
except socket.error as e:
err.log_error(err.ERROR, "Tried to read from self " + str(self) + " but an error occured. Error " + str(e.errno) + ": " + str(e.strerror))
break
def fill_table_from_file(cls, dbcc, delimiter, filepath, func_str_to_obj_args):
result = True
err.log_error(err.INFO, "Filling table " + cls.table_name + " with data from file " + filepath)
try:
f = open(filepath, 'r')
except IOError as e:
err.log_error(err.CRITICAL, "Tried to read a file but it failed: [" + str(e.errno) + "] " + str(e.message))
result = False
else:
i = 0
path, extension = os.path.splitext(filepath)
_, filename = os.path.split(path)
objects = []
for line in f:
i += 1
arguments = func_str_to_obj_args(filename, extension, line.split(delimiter))
obj = cls.new(dbcc, *arguments)
objects.append(obj)
cls.save_many(dbcc, objects)
f.close()
return result
def accept_connections(self):
while self.has_pending_connections():
sock, address = self.sock.accept()
connection_id = self.next_connection_id()
connection_obj = socket_connections.SocketConnection(connection_id, sock, address, self)
err.log_error(err.INFO, "A connection has been accepted " + str(connection_obj))
def clear_single_link_maas_message(dbcc, msg):
link_id = msg.link_id
db_flow_data_list = []
# Save it in mean database and convert to db layout object
for flow_data in msg.flow_data:
db_flow_data = db_layout.FlowData.new(
dbcc,
link_id,
flow_data.start_time,
flow_data.stop_time,
flow_data.source_ip,
flow_data.destination_ip,
flow_data.num_of_packets,
flow_data.num_of_bytes,
)
db_flow_data_list.append(db_flow_data)
db_layout.FlowData.save_many(dbcc, db_flow_data_list)
# Process flowdata
flows_ts_data = map(db_layout.FlowData.to_tuple_without_link_id, db_flow_data_list)
flows_ts_output_list = flows_ts.process_flow_data(flows_ts_data)
# Save it in fetched database
bp_fetched_objects = []
for unix_timestamp_time, throughput_bps in flows_ts_output_list:
bp_fetched_objects.append(db_layout.BPFetched.new(dbcc, link_id, unix_timestamp_time, throughput_bps))
db_layout.BPFetched.save_many(dbcc, bp_fetched_objects)
err.log_error(err.INFO, "Cleared " + str(len(msg.flow_data)) + " rows of flow data")
return sr.SuppliedFlowData.from_bp_fetched_list_to_supplied_info(bp_fetched_objects)
def close_all_connections(self):
result = True
err.log_error(err.INFO, "All database connections are being closed...")
for db_connection in self.connections:
result = result and db_connection.close()
return result
def _handle_pending_requests(self):
pending_requests = self.pending_request_queue.get_ready_to_be_handled_requests()
for pending_request in pending_requests:
connection = pending_request.connection
request = pending_request.request_message
err.log_error(err.INFO, "Pending request now possible. Rehandling request: " + str(request))
handle_messages.handle_incoming_message_consumer(self, connection, request)
def broadcast(self, message):
connections = self.can_send_connections()
for connection in connections:
connection.add_message(message)
if len(connections) == 0:
err.log_error(err.ERROR, "Requested to broadcast a message to all connections but there are not connections in " + str(self))
err.log_error(err.ERROR, "Message was: " + str(message))
def _start_x_with_y(x, y):
x_started = y(x)
if x_started:
err.log_error(err.INFO, str(x) + " has been started")
else:
err.log_error(err.CRITICAL, str(x) + " could not be started!")
return x_started
def start(self):
#Start connection to MOBaaS server frontend
connection_started = self.client.create_connection(config.SERVER_HOSTNAME, config.SERVER_PORT)
if connection_started:
err.log_error(err.INFO, "Started connection to MOBaaS frontend")
else:
err.log_error(err.CRITICAL, "Could not create a connection to MOBaaS frontend!")
return connection_started
def _handle_send_phase(self):
if self.time_to_send():
self.set_next_time()
next_message = self.pop_next_message()
err.log_error(err.INFO, "Sending message: " + str(next_message))
for connection in self.client.can_send_connections():
connection.add_message(next_message)
self.client.send_to_connections()
def handle_flow_data_request(dbcc, msg):
err.log_error(err.INFO, "Found a request for flow data")
end_date_time = datetime.datetime.combine(msg.start_date, msg.start_time) + datetime.timedelta(seconds=msg.time_span)
start_time_unix_ms = support.to_unix_timestamp_ms_from_date_and_time(msg.start_date, msg.start_time)
stop_time_unix_ms = support.to_unix_timestamp_ms_from_date_and_time(end_date_time.date(), end_date_time.time())
db_flow_data_objects = test_db_layout.MaaSFlowData.retrieve_flow_data(dbcc, msg.link_id, start_time_unix_ms, stop_time_unix_ms)
err.log_error(err.INFO, "Sending " + str(len(db_flow_data_objects)) + " rows of flow data")
return test_db_layout.MaaSFlowData.construct_maas_single_link_answer(msg.link_id, db_flow_data_objects)
def main():
global FRONTEND
FRONTEND = Frontend()
signal.signal(signal.SIGINT, shutdown)
if FRONTEND.start():
err.log_error(err.INFO, "Server started succesfully. Going to work!")
FRONTEND.go_to_work()
else:
err.log_error(err.CRITICAL, "Could not start server succesfully. Aborting...")
shutdown(1, None)
def close(self):
result = False
try:
self.db_connection.close()
result = True
except mdb.Error as e:
err.log_error(err.ERROR, "Tried to close the db connection but could not. Removed it anyways")
err.log_error(err.ERROR, "Error: " + e.message)
self.connection_collection.remove_connection(self.cid)
return result
def main():
global MAAS
MAAS = TestSupplier("Test MaaS", config.TEST_MAAS_DB)
signal.signal(signal.SIGINT, shutdown)
if MAAS.start():
err.log_error(err.INFO, "Server started succesfully. Going to work!")
MAAS.go_to_work()
else:
err.log_error(err.CRITICAL, "Could not start server succesfully. Aborting...")
shutdown(1, None)
def main():
global test_consumer_obj
test_consumer_obj = test_consumer.TestConsumer("MP Test Consumer", config.TEST_MP_MESSAGES, config.WAIT_BETWEEN_SENDS_MP)
signal.signal(signal.SIGINT, shutdown)
if test_consumer_obj.start():
err.log_error(err.INFO, "Test Consumer started!")
test_consumer_obj.go_to_work()
else:
err.log_error(err.CRITICAL, "Could not start the Test Consumer! Quitting...")
def start(self):
result = False
try:
con = mdb.connect(central_config.DB_SERVER_HOSTNAME, central_config.DB_USER, central_config.DB_PWD, self.db_name)
result = True
except mdb.Error as e:
err.log_error(err.CRITICAL, "Could not connect to database " + self.db_name + " because " + str(e.message))
else:
self.db_connection = con
return result
def close(self):
result = False
try:
self.sock.close()
result = True
except socket.error:
err.log_error(err.ERROR, "Tried to close a connection, but it seems something was wrong with the connection. Closed the connection anyway. Do not need to do anything about this.")
self.connection_collection.remove_connection(self.cid)
return result
def handle_incoming_message_consumer(frontend, connection, msg):
err.log_error(err.INFO, "Received request {" + str(msg) + "}")
if isinstance(msg, mo.BPSingleLinkRequest):
handle_incoming_bp_single_link_request(frontend, connection, msg)
elif isinstance(msg, mo.MPSingleUserRequest):
handle_incoming_mp_single_user_request(frontend, connection, msg)
elif isinstance(msg, mo.MPMultiUserRequest):
handle_incoming_mp_multi_user_request(frontend, connection, msg)
elif isinstance(msg, mo.EndConversation):
err.log_error(err.INFO, "Consumer Connection " + str(connection) + " is requested to end because: " + msg.reason)
connection.close()
def send_buffer(self):
while self.has_data_to_send():
try:
to_send = self.send_message_buffer[:config.SEND_CHUNK_SIZE]
has_send = self.sock.send(to_send)
except socket.error as e:
err.log_error(err.ERROR, "Could not send data over the connection " + str(self) + ". Error " + str(e.errno) + ": " + str(e.strerror))
break
self.send_message_buffer = self.send_message_buffer[has_send:]
if not self.can_send_data():
self.needs_closing = True
def _get_all_values(self):
values = []
result = True
for column_name, _ in self.layout:
if hasattr(self, column_name):
values.append(getattr(self, column_name))
else:
err.log_error(err.ERROR, "Tried to use an object from table " + self.table_name + " but the object \
didn't have the attribute " + column_name)
result = False
return values if result else None
def handle_link_capacity_request(dbcc, msg):
answer = None
conditions = [ ("link_id", "=", msg.link_id)
]
link_capacity_objects = test_db_layout.MaaSLinkCapacityData.retrieve_objects(dbcc, conditions)
if link_capacity_objects:
link_capacity_object = link_capacity_objects[0]
answer = test_db_layout.MaaSLinkCapacityData.construct_maas_single_link_capacity_answer(link_capacity_object.link_id, link_capacity_object.capacity)
err.log_error(err.INFO, "Sending 1 row of link capacity data")
return answer
def databases_setup():
dbcc = db_connections.DBConnectionCollection("Setting up all tables")
dbcc.create_multiple_connections(db_layout.DATABASE_NAMES)
all_started = dbcc.start_all_connections()
if all_started:
err.log_error(err.INFO, "Succesfully started all db connections. Doing maintenance...")
all_classes = inspect.getmembers(db_layout, inspect.isclass)
for _, class_p in all_classes:
class_p.create_table(dbcc)
else:
err.log_error(err.CRITICAL, "Could not connect to all databases. Not doing any maintence...")
dbcc.close_all_connections()
def close(self):
result = True
err.log_error(err.INFO, "The " + self.name + " server is being stopped")
if self.sock:
result = self.close_all_connections()
err.log_error(err.INFO, "Closing server socket")
try:
self.sock.close()
result = result and True
except socket.error as e:
err.log_error(err.ERROR, "Could not cleanly close socket. Error " + str(e.errno) + ": " + str(e.strerror))
err.log_error(err.INFO, "Server socket is closed")
else:
err.log_error(err.ERROR, "Tried to close the server, but the server was never correctly started!")
return result
def clear_user_information_maas_message(dbcc, msg):
user_id = msg.user_id
db_user_mobility_information_list = []
# Save it in mean database and convert to db layout object
for movement in msg.user_information:
db_user_mobility_information = db_layout.UserMobilityInformation.new(
dbcc, user_id, movement.time, movement.date, movement.cell_id
)
db_user_mobility_information_list.append(db_user_mobility_information)
db_layout.UserMobilityInformation.save_many(dbcc, db_user_mobility_information_list)
mp_fetched_list = []
# Clear the data and save it in the fetched database
for db_user_mobility_information in db_user_mobility_information_list:
minute_of_day = support.calc_minute_of_day(db_user_mobility_information.time)
day = support.calc_day(db_user_mobility_information.date)
mp_fetched = db_layout.MPFetched.new(
dbcc, user_id, minute_of_day, db_user_mobility_information.date, day, db_user_mobility_information.cell_id
)
mp_fetched_list.append(mp_fetched)
# Fill in gaps
end_date_time = datetime.datetime.combine(msg.start_date, msg.start_time) + datetime.timedelta(
seconds=msg.time_span
)
gaps_mp_fetched = find_the_gaps_user_info_to_mp_fetched(
dbcc,
msg.user_id,
msg.start_date,
msg.start_time,
end_date_time.date(),
end_date_time.time(),
db_user_mobility_information_list,
)
mp_fetched_list.extend(gaps_mp_fetched)
db_layout.MPFetched.save_many(dbcc, mp_fetched_list)
err.log_error(err.INFO, "Cleared " + str(len(msg.user_information)) + " rows user information data")
err.log_error(err.INFO, "Filled in " + str(len(gaps_mp_fetched)) + " holes with mp fetched data")
return sr.SuppliedUserInfo.from_maas_user_info_message_to_supplied_info(msg)
def create_connection(self, hostname, port):
result = True
cid = self.next_connection_id()
address = (hostname, port)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect(address)
except socket.error as e:
err.log_error(err.ERROR, "Could not connect to " + hostname + ":" + str(port) + " because: " + str(e.errno) + " " + str(e.strerror))
result = False
else:
connection = socket_connections.SocketConnection(cid, sock, address, self)
err.log_error(err.INFO, "A connection has been made: " + str(connection))
return result
def handle_incoming_message_supplier(frontend, connection, msg):
dbcc = frontend.dbcc
err.log_error(err.INFO, "Received Supplier message of type: " + str(msg.__class__))
supplied_resource = None
if isinstance(msg, mo.MaaSSingleUserDataAnswer):
supplied_resource = clear_data.clear_user_information_maas_message(dbcc, msg)
elif isinstance(msg, mo.MaaSSingleLinkDataAnswer):
supplied_resource = clear_data.clear_single_link_maas_message(dbcc, msg)
elif isinstance(msg, mo.MaaSSingleLinkCapacityAnswer):
supplied_resource = clear_data.clear_single_link_capacity_message(dbcc, msg)
elif isinstance(msg, mo.EndConversation):
err.log_error(err.INFO, "Supplier Connection " + str(connection) + " is requested to end because: " + msg.reason)
connection.close()
if supplied_resource:
frontend.process_supplied_resource(supplied_resource)
def retrieve_user_mobility_data_single_user(dbcc, user_id, current_date, predicted_date):
history_date = calculate_mp_history_date(current_date)
current_week_date_range = support.unix_weeks_timestamp_to_date_range(
support.date_to_unix_weeks_timestamp(current_date)
)
history_week_date_range = support.unix_weeks_timestamp_to_date_range(
support.date_to_unix_weeks_timestamp(history_date)
)
begin_of_history_week = history_week_date_range.min_range
end_of_current_week = current_week_date_range.max_range
current_day = support.calc_day(current_date)
predicted_day = support.calc_day(predicted_date)
applicable_days = support.difference_of_days(current_day, predicted_day)
con = dbcc.retrieve_connection(db_layout.MPFetched.database_name)
select = "*"
table = db_layout.MPFetched.table_name
layout_dict = dict(db_layout.MPFetched.layout)
user_id_sql = layout_dict["user_id"].format_to_db_value(user_id)
end_date_sql = layout_dict["date"].format_to_db_value(end_of_current_week)
start_date_sql = layout_dict["date"].format_to_db_value(begin_of_history_week)
applicable_days_sql = map(layout_dict["day"].format_to_db_value, applicable_days)
applicable_days_query = " AND (day = " + " OR day = ".join(applicable_days_sql) + ")"
where_query = (
"WHERE user_id = "
+ user_id_sql
+ " AND date <= "
+ end_date_sql
+ " AND date >= "
+ start_date_sql
+ applicable_days_query
)
err.log_error(err.INFO, "Retrieving data for MP between " + str(history_date) + " and " + str(current_date))
mysql_rows = con.retrieve(select, table, where_query)
result = []
for mysql_row in mysql_rows:
result.append(db_layout.MPFetched.object_from_mysql_row(dbcc, mysql_row))
return result
def handle_incoming_mp_multi_user_request(frontend, connection, msg, iteration=0):
err.log_error(err.INFO, "Using MP multi-user algorithm, iteration %d" % iteration)
start_time = timeit.default_timer()
# do this every msg.future_time_delta minutes
#result = mup.mp(msg.user_id, msg.current_time, msg.current_date, msg.future_time_delta)
current_time_in_min = sum(int(x) * 60 ** i for i,x in enumerate(reversed(msg.current_time.split(':'))))
next_time_in_min = current_time_in_min + (iteration * msg.future_time_delta)
next_time_formatted = '{:02d}:{:02d}'.format(*divmod(next_time_in_min, 60))
if next_time_in_min > 1440 - msg.future_time_delta:
return;
result = mup.mp(msg.user_id, next_time_formatted, msg.current_date, msg.future_time_delta)
mu_predictions = []
for k in result.keys():
(user_id, current_cell_id, next_time) = k
#err.log_error(err.INFO, "Processing result entry for %i %i %s " % (user_id, current_cell_id, next_time))
probabilities = result[k]
#user_entry = UserEntry.from_pykov_key(pykov_key)
#prediction = data_objects.Prediction(user_entry.cell_id, probability)
#prediction_obj = mp.from_pykov_distribution_dict_to_predictions(probabilities)
predictions = []
for p in probabilities:
(cell_id, probability) = p
predictions.append(data_objects.Prediction(cell_id, probability))
mu_prediction = data_objects.MultiUserPrediction(user_id, current_cell_id, next_time, predictions)
mu_predictions.append(mu_prediction)
answer_message = mo.MPMultiUserAnswer(mu_predictions)
err.log_error(err.INFO, "answer_message before sending it:")
pprint.pprint(answer_message)
connection.add_message(answer_message)
#self.mp_multi_user_loop_helper(frontend, connection, msg, iteration+1)
err.log_error(err.INFO, "Execution time for this run (iteration %d) is : %d , Second" % (iteration, timeit.default_timer() - start_time))
Timer(msg.future_time_delta, handle_incoming_mp_multi_user_request, (frontend, connection, msg, iteration+1)).start()
def clear_table(cls, dbcc):
db_connection = cls.retrieve_connection(dbcc)
if db_connection.table_exists(cls.table_name):
result = cls.delete_table(db_connection)
if result:
result = result and cls.create_table(db_connection)
if result:
err.log_error(
err.INFO,
"Table " + cls.table_name + " has been cleared")
if not result:
err.log_error(
err.CRITICAL,
"Table " + cls.table_name + " could not be cleared!")
else:
result = False
err.log_error(err.ERROR,
"Table " + cls.table_name + " does not exist!")
return result
def broadcast_close(self, close_message):
err.log_error(err.INFO,
"Broadcasting close message: " + str(close_message))
for connection in self.can_send_connections():
connection.send_close(close_message)
def handle_incoming_mp_single_user_request(frontend, connection, msg):
err.log_error(err.INFO, "Using MP algorithm")
predicted_date_time = datetime.datetime.combine(msg.current_date, msg.current_time) + datetime.timedelta(seconds=msg.future_time_delta)
predicted_date = predicted_date_time.date()
predicted_time = predicted_date_time.time()
'''
if user_id is 0000 (i.e. 0, because it is an integer), we need to use multi-user prediction, not single user prediction
'''
if msg.user_id != 0:
mp_fetched = mp.retrieve_user_mobility_data_single_user(frontend.dbcc, msg.user_id, msg.current_date, predicted_date)
grouped_by_date_mp_fetched = support.group_by(mp_fetched, lambda x: x.date)
grouped_by_date_grid_by_minute_of_day_mp_fetched = {}
for date, mp_fetched_list in grouped_by_date_mp_fetched.items():
minute_of_day_grid = support.to_grid( mp_fetched_list
, 1
, lambda x: x.minute_of_day
, 0
, common_config.MINUTES_IN_A_DAY - 1
)
grouped_by_date_grid_by_minute_of_day_mp_fetched[date] = minute_of_day_grid
missing_ranges = mp.check_user_mobility_data_single_user_integrity( grouped_by_date_grid_by_minute_of_day_mp_fetched
, msg.current_date
)
if len(missing_ranges) > 0:
#Missing user information data, queue request and ask data from supplier
missing_resource = mr.MissingUserInfo(missing_ranges)
frontend.queue_as_pending_request(connection, msg, [missing_resource])
err.log_error(err.INFO, "Did not have enough user info data. Asking supplier for: " + str(missing_ranges))
error_message = mo.Error(102, "MP Request will take longer than expected because user data is missing at MOBaaS")
connection.add_message(error_message)
for missing_range in missing_ranges:
start_time = missing_range.min_range.time()
start_date = missing_range.min_range.date()
time_span = int((missing_range.max_range - missing_range.min_range).total_seconds())
supplier_message = mo.MaaSSingleUserDataRequest(msg.user_id, start_time, start_date, time_span, 60)
frontend.ask_at_supplier(supplier_message)
else:
#There is all the info we need, calculate the spots
current_day_str = support.calc_day(msg.current_date)
predicted_day_str = support.calc_day(predicted_date)
pykov_chain = mp.prepare_pykov_chain_with_single_user_mobility_states(grouped_by_date_grid_by_minute_of_day_mp_fetched
, current_day_str
, predicted_day_str
)
chance_distribution_dict = mp.mobility_prediction(pykov_chain, msg.current_cell_id, msg.current_date, msg.current_time, predicted_date, predicted_time)
#Create mobaas_protocol.prediction objects from the answers
mobaas_protocol_predictions = mp.from_pykov_distribution_dict_to_predictions(chance_distribution_dict)
answer_message = mo.MPSingleUserAnswer(msg.user_id, predicted_time, predicted_date, mobaas_protocol_predictions)
connection.add_message(answer_message)
err.log_error(err.INFO, "Found an answer to the request! Answer:" + str(answer_message))
else:
print 'error: should not get here!'
def set_next_time(self):
td = datetime.timedelta(seconds=self.wait_between_sends)
self.next_send_time = self.next_send_time + td
err.log_error(
err.INFO, "Next send will be at: " +
self.next_send_time.strftime("[%H:%M:%S %d-%m-%Y]"))
def close(self):
err.log_error(err.INFO, "Closing Test Consumer.")
self.client.broadcast_close(
message_objects.EndConversation("Client is being closed"))
self.client.close_all_connections()
err.log_error(err.INFO, "Closed the Test Consumer. Now Quitting.")
def execute(self, query):
result = False
cur = self.db_connection.cursor(mdb.cursors.DictCursor)
try:
cur.execute(query)
self.db_connection.commit()
cur.close()
result = True
except mdb.Error as e:
err.log_error(
err.CRITICAL,
"Tried to perform a query on the database, but it wasn't able to!"
)
err.log_error(err.CRITICAL, "Query: " + query)
err.log_error(err.CRITICAL, "Database: " + self.db_name)
err.log_error(err.CRITICAL, "Error: " + e.message)
if self.db_connection:
try:
self.db_connection.rollback()
except mdb.Error as e_rollback:
err.log_error(
err.CRITICAL,
"Tried to rollback the failed query but this failed as well. Your database might have become corrupted!"
)
err.log_error(err.CRITICAL, "Error: " + e_rollback.message)
return result
def add_connection(self, connection):
err.log_error(
err.INFO, "Connection " + str(connection.cid) +
" has been added to Connection_Collection " + self.name)
self.connections.append(connection)
def create_multiple_connections(self, db_names):
err.log_error(err.INFO, "Create multiple database connections")
for db_name in db_names:
cid = self.next_connection_id()
DBConnection(cid, self, db_name)