def clear_single_link_capacity_message(dbcc, msg): link_capacity_data = db_layout.LinkCapacityData.new(dbcc, msg.link_id, msg.capacity) link_capacity_data.save() err.log_error(err.INFO, "Cleared link capacity information for link " + str(link_capacity_data.link_id)) return sr.SuppliedLinkCapacity.from_link_capacity_data_to_supplied_info(link_capacity_data)
def handle_user_data_request(dbcc, msg): err.log_error(err.INFO, "Found a request for user data") start_date_time = datetime.datetime.combine(msg.start_date, msg.start_time) end_date_time = start_date_time + datetime.timedelta(seconds=msg.time_span) end_time = end_date_time.time() end_date = end_date_time.date() conditions = [ ("user_id", "=", msg.user_id) , ("date", ">=", msg.start_date) , ("date", "<=", end_date) ] user_information_objects = test_db_layout.MaaSUserMobilityInformation.retrieve_objects(dbcc, conditions) valid_user_information_objects = [] #Limit between requested start and end time for user_info_obj in user_information_objects: user_info_date_time = datetime.datetime.combine(user_info_obj.date, user_info_obj.time) if start_date_time <= user_info_date_time <= end_date_time: valid_user_information_objects.append(user_info_obj) #Limit to the correct resolution reduced_user_information_objects = test_db_layout.MaaSUserMobilityInformation.reduce_to_time_resolution(valid_user_information_objects, msg.time_resolution) err.log_error(err.INFO, "Sending " + str(len(reduced_user_information_objects)) + " rows of user data") return test_db_layout.MaaSUserMobilityInformation.construct_maas_single_user_data_answer(msg.user_id, msg.start_date, msg.start_time, msg.time_span, reduced_user_information_objects)
def close_all_connections(self): result = True err.log_error(err.INFO, "All socket connections are being closed...") for connection in self.connections: result = result and connection.close() return result
def read_buffer(self): while self.has_data_to_read(): try: self.read_message_buffer += self.sock.recv(config.RECEIVE_CHUNK_SIZE) except socket.error as e: err.log_error(err.ERROR, "Tried to read from self " + str(self) + " but an error occured. Error " + str(e.errno) + ": " + str(e.strerror)) break
def fill_table_from_file(cls, dbcc, delimiter, filepath, func_str_to_obj_args): result = True err.log_error(err.INFO, "Filling table " + cls.table_name + " with data from file " + filepath) try: f = open(filepath, 'r') except IOError as e: err.log_error(err.CRITICAL, "Tried to read a file but it failed: [" + str(e.errno) + "] " + str(e.message)) result = False else: i = 0 path, extension = os.path.splitext(filepath) _, filename = os.path.split(path) objects = [] for line in f: i += 1 arguments = func_str_to_obj_args(filename, extension, line.split(delimiter)) obj = cls.new(dbcc, *arguments) objects.append(obj) cls.save_many(dbcc, objects) f.close() return result
def accept_connections(self): while self.has_pending_connections(): sock, address = self.sock.accept() connection_id = self.next_connection_id() connection_obj = socket_connections.SocketConnection(connection_id, sock, address, self) err.log_error(err.INFO, "A connection has been accepted " + str(connection_obj))
def clear_single_link_maas_message(dbcc, msg): link_id = msg.link_id db_flow_data_list = [] # Save it in mean database and convert to db layout object for flow_data in msg.flow_data: db_flow_data = db_layout.FlowData.new( dbcc, link_id, flow_data.start_time, flow_data.stop_time, flow_data.source_ip, flow_data.destination_ip, flow_data.num_of_packets, flow_data.num_of_bytes, ) db_flow_data_list.append(db_flow_data) db_layout.FlowData.save_many(dbcc, db_flow_data_list) # Process flowdata flows_ts_data = map(db_layout.FlowData.to_tuple_without_link_id, db_flow_data_list) flows_ts_output_list = flows_ts.process_flow_data(flows_ts_data) # Save it in fetched database bp_fetched_objects = [] for unix_timestamp_time, throughput_bps in flows_ts_output_list: bp_fetched_objects.append(db_layout.BPFetched.new(dbcc, link_id, unix_timestamp_time, throughput_bps)) db_layout.BPFetched.save_many(dbcc, bp_fetched_objects) err.log_error(err.INFO, "Cleared " + str(len(msg.flow_data)) + " rows of flow data") return sr.SuppliedFlowData.from_bp_fetched_list_to_supplied_info(bp_fetched_objects)
def close_all_connections(self): result = True err.log_error(err.INFO, "All database connections are being closed...") for db_connection in self.connections: result = result and db_connection.close() return result
def _handle_pending_requests(self): pending_requests = self.pending_request_queue.get_ready_to_be_handled_requests() for pending_request in pending_requests: connection = pending_request.connection request = pending_request.request_message err.log_error(err.INFO, "Pending request now possible. Rehandling request: " + str(request)) handle_messages.handle_incoming_message_consumer(self, connection, request)
def broadcast(self, message): connections = self.can_send_connections() for connection in connections: connection.add_message(message) if len(connections) == 0: err.log_error(err.ERROR, "Requested to broadcast a message to all connections but there are not connections in " + str(self)) err.log_error(err.ERROR, "Message was: " + str(message))
def _start_x_with_y(x, y): x_started = y(x) if x_started: err.log_error(err.INFO, str(x) + " has been started") else: err.log_error(err.CRITICAL, str(x) + " could not be started!") return x_started
def start(self): #Start connection to MOBaaS server frontend connection_started = self.client.create_connection(config.SERVER_HOSTNAME, config.SERVER_PORT) if connection_started: err.log_error(err.INFO, "Started connection to MOBaaS frontend") else: err.log_error(err.CRITICAL, "Could not create a connection to MOBaaS frontend!") return connection_started
def _handle_send_phase(self): if self.time_to_send(): self.set_next_time() next_message = self.pop_next_message() err.log_error(err.INFO, "Sending message: " + str(next_message)) for connection in self.client.can_send_connections(): connection.add_message(next_message) self.client.send_to_connections()
def handle_flow_data_request(dbcc, msg): err.log_error(err.INFO, "Found a request for flow data") end_date_time = datetime.datetime.combine(msg.start_date, msg.start_time) + datetime.timedelta(seconds=msg.time_span) start_time_unix_ms = support.to_unix_timestamp_ms_from_date_and_time(msg.start_date, msg.start_time) stop_time_unix_ms = support.to_unix_timestamp_ms_from_date_and_time(end_date_time.date(), end_date_time.time()) db_flow_data_objects = test_db_layout.MaaSFlowData.retrieve_flow_data(dbcc, msg.link_id, start_time_unix_ms, stop_time_unix_ms) err.log_error(err.INFO, "Sending " + str(len(db_flow_data_objects)) + " rows of flow data") return test_db_layout.MaaSFlowData.construct_maas_single_link_answer(msg.link_id, db_flow_data_objects)
def main(): global FRONTEND FRONTEND = Frontend() signal.signal(signal.SIGINT, shutdown) if FRONTEND.start(): err.log_error(err.INFO, "Server started succesfully. Going to work!") FRONTEND.go_to_work() else: err.log_error(err.CRITICAL, "Could not start server succesfully. Aborting...") shutdown(1, None)
def close(self): result = False try: self.db_connection.close() result = True except mdb.Error as e: err.log_error(err.ERROR, "Tried to close the db connection but could not. Removed it anyways") err.log_error(err.ERROR, "Error: " + e.message) self.connection_collection.remove_connection(self.cid) return result
def main(): global MAAS MAAS = TestSupplier("Test MaaS", config.TEST_MAAS_DB) signal.signal(signal.SIGINT, shutdown) if MAAS.start(): err.log_error(err.INFO, "Server started succesfully. Going to work!") MAAS.go_to_work() else: err.log_error(err.CRITICAL, "Could not start server succesfully. Aborting...") shutdown(1, None)
def main(): global test_consumer_obj test_consumer_obj = test_consumer.TestConsumer("MP Test Consumer", config.TEST_MP_MESSAGES, config.WAIT_BETWEEN_SENDS_MP) signal.signal(signal.SIGINT, shutdown) if test_consumer_obj.start(): err.log_error(err.INFO, "Test Consumer started!") test_consumer_obj.go_to_work() else: err.log_error(err.CRITICAL, "Could not start the Test Consumer! Quitting...")
def start(self): result = False try: con = mdb.connect(central_config.DB_SERVER_HOSTNAME, central_config.DB_USER, central_config.DB_PWD, self.db_name) result = True except mdb.Error as e: err.log_error(err.CRITICAL, "Could not connect to database " + self.db_name + " because " + str(e.message)) else: self.db_connection = con return result
def close(self): result = False try: self.sock.close() result = True except socket.error: err.log_error(err.ERROR, "Tried to close a connection, but it seems something was wrong with the connection. Closed the connection anyway. Do not need to do anything about this.") self.connection_collection.remove_connection(self.cid) return result
def handle_incoming_message_consumer(frontend, connection, msg): err.log_error(err.INFO, "Received request {" + str(msg) + "}") if isinstance(msg, mo.BPSingleLinkRequest): handle_incoming_bp_single_link_request(frontend, connection, msg) elif isinstance(msg, mo.MPSingleUserRequest): handle_incoming_mp_single_user_request(frontend, connection, msg) elif isinstance(msg, mo.MPMultiUserRequest): handle_incoming_mp_multi_user_request(frontend, connection, msg) elif isinstance(msg, mo.EndConversation): err.log_error(err.INFO, "Consumer Connection " + str(connection) + " is requested to end because: " + msg.reason) connection.close()
def send_buffer(self): while self.has_data_to_send(): try: to_send = self.send_message_buffer[:config.SEND_CHUNK_SIZE] has_send = self.sock.send(to_send) except socket.error as e: err.log_error(err.ERROR, "Could not send data over the connection " + str(self) + ". Error " + str(e.errno) + ": " + str(e.strerror)) break self.send_message_buffer = self.send_message_buffer[has_send:] if not self.can_send_data(): self.needs_closing = True
def _get_all_values(self): values = [] result = True for column_name, _ in self.layout: if hasattr(self, column_name): values.append(getattr(self, column_name)) else: err.log_error(err.ERROR, "Tried to use an object from table " + self.table_name + " but the object \ didn't have the attribute " + column_name) result = False return values if result else None
def handle_link_capacity_request(dbcc, msg): answer = None conditions = [ ("link_id", "=", msg.link_id) ] link_capacity_objects = test_db_layout.MaaSLinkCapacityData.retrieve_objects(dbcc, conditions) if link_capacity_objects: link_capacity_object = link_capacity_objects[0] answer = test_db_layout.MaaSLinkCapacityData.construct_maas_single_link_capacity_answer(link_capacity_object.link_id, link_capacity_object.capacity) err.log_error(err.INFO, "Sending 1 row of link capacity data") return answer
def databases_setup(): dbcc = db_connections.DBConnectionCollection("Setting up all tables") dbcc.create_multiple_connections(db_layout.DATABASE_NAMES) all_started = dbcc.start_all_connections() if all_started: err.log_error(err.INFO, "Succesfully started all db connections. Doing maintenance...") all_classes = inspect.getmembers(db_layout, inspect.isclass) for _, class_p in all_classes: class_p.create_table(dbcc) else: err.log_error(err.CRITICAL, "Could not connect to all databases. Not doing any maintence...") dbcc.close_all_connections()
def close(self): result = True err.log_error(err.INFO, "The " + self.name + " server is being stopped") if self.sock: result = self.close_all_connections() err.log_error(err.INFO, "Closing server socket") try: self.sock.close() result = result and True except socket.error as e: err.log_error(err.ERROR, "Could not cleanly close socket. Error " + str(e.errno) + ": " + str(e.strerror)) err.log_error(err.INFO, "Server socket is closed") else: err.log_error(err.ERROR, "Tried to close the server, but the server was never correctly started!") return result
def clear_user_information_maas_message(dbcc, msg): user_id = msg.user_id db_user_mobility_information_list = [] # Save it in mean database and convert to db layout object for movement in msg.user_information: db_user_mobility_information = db_layout.UserMobilityInformation.new( dbcc, user_id, movement.time, movement.date, movement.cell_id ) db_user_mobility_information_list.append(db_user_mobility_information) db_layout.UserMobilityInformation.save_many(dbcc, db_user_mobility_information_list) mp_fetched_list = [] # Clear the data and save it in the fetched database for db_user_mobility_information in db_user_mobility_information_list: minute_of_day = support.calc_minute_of_day(db_user_mobility_information.time) day = support.calc_day(db_user_mobility_information.date) mp_fetched = db_layout.MPFetched.new( dbcc, user_id, minute_of_day, db_user_mobility_information.date, day, db_user_mobility_information.cell_id ) mp_fetched_list.append(mp_fetched) # Fill in gaps end_date_time = datetime.datetime.combine(msg.start_date, msg.start_time) + datetime.timedelta( seconds=msg.time_span ) gaps_mp_fetched = find_the_gaps_user_info_to_mp_fetched( dbcc, msg.user_id, msg.start_date, msg.start_time, end_date_time.date(), end_date_time.time(), db_user_mobility_information_list, ) mp_fetched_list.extend(gaps_mp_fetched) db_layout.MPFetched.save_many(dbcc, mp_fetched_list) err.log_error(err.INFO, "Cleared " + str(len(msg.user_information)) + " rows user information data") err.log_error(err.INFO, "Filled in " + str(len(gaps_mp_fetched)) + " holes with mp fetched data") return sr.SuppliedUserInfo.from_maas_user_info_message_to_supplied_info(msg)
def create_connection(self, hostname, port): result = True cid = self.next_connection_id() address = (hostname, port) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: sock.connect(address) except socket.error as e: err.log_error(err.ERROR, "Could not connect to " + hostname + ":" + str(port) + " because: " + str(e.errno) + " " + str(e.strerror)) result = False else: connection = socket_connections.SocketConnection(cid, sock, address, self) err.log_error(err.INFO, "A connection has been made: " + str(connection)) return result
def handle_incoming_message_supplier(frontend, connection, msg): dbcc = frontend.dbcc err.log_error(err.INFO, "Received Supplier message of type: " + str(msg.__class__)) supplied_resource = None if isinstance(msg, mo.MaaSSingleUserDataAnswer): supplied_resource = clear_data.clear_user_information_maas_message(dbcc, msg) elif isinstance(msg, mo.MaaSSingleLinkDataAnswer): supplied_resource = clear_data.clear_single_link_maas_message(dbcc, msg) elif isinstance(msg, mo.MaaSSingleLinkCapacityAnswer): supplied_resource = clear_data.clear_single_link_capacity_message(dbcc, msg) elif isinstance(msg, mo.EndConversation): err.log_error(err.INFO, "Supplier Connection " + str(connection) + " is requested to end because: " + msg.reason) connection.close() if supplied_resource: frontend.process_supplied_resource(supplied_resource)
def retrieve_user_mobility_data_single_user(dbcc, user_id, current_date, predicted_date): history_date = calculate_mp_history_date(current_date) current_week_date_range = support.unix_weeks_timestamp_to_date_range( support.date_to_unix_weeks_timestamp(current_date) ) history_week_date_range = support.unix_weeks_timestamp_to_date_range( support.date_to_unix_weeks_timestamp(history_date) ) begin_of_history_week = history_week_date_range.min_range end_of_current_week = current_week_date_range.max_range current_day = support.calc_day(current_date) predicted_day = support.calc_day(predicted_date) applicable_days = support.difference_of_days(current_day, predicted_day) con = dbcc.retrieve_connection(db_layout.MPFetched.database_name) select = "*" table = db_layout.MPFetched.table_name layout_dict = dict(db_layout.MPFetched.layout) user_id_sql = layout_dict["user_id"].format_to_db_value(user_id) end_date_sql = layout_dict["date"].format_to_db_value(end_of_current_week) start_date_sql = layout_dict["date"].format_to_db_value(begin_of_history_week) applicable_days_sql = map(layout_dict["day"].format_to_db_value, applicable_days) applicable_days_query = " AND (day = " + " OR day = ".join(applicable_days_sql) + ")" where_query = ( "WHERE user_id = " + user_id_sql + " AND date <= " + end_date_sql + " AND date >= " + start_date_sql + applicable_days_query ) err.log_error(err.INFO, "Retrieving data for MP between " + str(history_date) + " and " + str(current_date)) mysql_rows = con.retrieve(select, table, where_query) result = [] for mysql_row in mysql_rows: result.append(db_layout.MPFetched.object_from_mysql_row(dbcc, mysql_row)) return result
def handle_incoming_mp_multi_user_request(frontend, connection, msg, iteration=0): err.log_error(err.INFO, "Using MP multi-user algorithm, iteration %d" % iteration) start_time = timeit.default_timer() # do this every msg.future_time_delta minutes #result = mup.mp(msg.user_id, msg.current_time, msg.current_date, msg.future_time_delta) current_time_in_min = sum(int(x) * 60 ** i for i,x in enumerate(reversed(msg.current_time.split(':')))) next_time_in_min = current_time_in_min + (iteration * msg.future_time_delta) next_time_formatted = '{:02d}:{:02d}'.format(*divmod(next_time_in_min, 60)) if next_time_in_min > 1440 - msg.future_time_delta: return; result = mup.mp(msg.user_id, next_time_formatted, msg.current_date, msg.future_time_delta) mu_predictions = [] for k in result.keys(): (user_id, current_cell_id, next_time) = k #err.log_error(err.INFO, "Processing result entry for %i %i %s " % (user_id, current_cell_id, next_time)) probabilities = result[k] #user_entry = UserEntry.from_pykov_key(pykov_key) #prediction = data_objects.Prediction(user_entry.cell_id, probability) #prediction_obj = mp.from_pykov_distribution_dict_to_predictions(probabilities) predictions = [] for p in probabilities: (cell_id, probability) = p predictions.append(data_objects.Prediction(cell_id, probability)) mu_prediction = data_objects.MultiUserPrediction(user_id, current_cell_id, next_time, predictions) mu_predictions.append(mu_prediction) answer_message = mo.MPMultiUserAnswer(mu_predictions) err.log_error(err.INFO, "answer_message before sending it:") pprint.pprint(answer_message) connection.add_message(answer_message) #self.mp_multi_user_loop_helper(frontend, connection, msg, iteration+1) err.log_error(err.INFO, "Execution time for this run (iteration %d) is : %d , Second" % (iteration, timeit.default_timer() - start_time)) Timer(msg.future_time_delta, handle_incoming_mp_multi_user_request, (frontend, connection, msg, iteration+1)).start()
def clear_table(cls, dbcc): db_connection = cls.retrieve_connection(dbcc) if db_connection.table_exists(cls.table_name): result = cls.delete_table(db_connection) if result: result = result and cls.create_table(db_connection) if result: err.log_error( err.INFO, "Table " + cls.table_name + " has been cleared") if not result: err.log_error( err.CRITICAL, "Table " + cls.table_name + " could not be cleared!") else: result = False err.log_error(err.ERROR, "Table " + cls.table_name + " does not exist!") return result
def broadcast_close(self, close_message): err.log_error(err.INFO, "Broadcasting close message: " + str(close_message)) for connection in self.can_send_connections(): connection.send_close(close_message)
def handle_incoming_mp_single_user_request(frontend, connection, msg): err.log_error(err.INFO, "Using MP algorithm") predicted_date_time = datetime.datetime.combine(msg.current_date, msg.current_time) + datetime.timedelta(seconds=msg.future_time_delta) predicted_date = predicted_date_time.date() predicted_time = predicted_date_time.time() ''' if user_id is 0000 (i.e. 0, because it is an integer), we need to use multi-user prediction, not single user prediction ''' if msg.user_id != 0: mp_fetched = mp.retrieve_user_mobility_data_single_user(frontend.dbcc, msg.user_id, msg.current_date, predicted_date) grouped_by_date_mp_fetched = support.group_by(mp_fetched, lambda x: x.date) grouped_by_date_grid_by_minute_of_day_mp_fetched = {} for date, mp_fetched_list in grouped_by_date_mp_fetched.items(): minute_of_day_grid = support.to_grid( mp_fetched_list , 1 , lambda x: x.minute_of_day , 0 , common_config.MINUTES_IN_A_DAY - 1 ) grouped_by_date_grid_by_minute_of_day_mp_fetched[date] = minute_of_day_grid missing_ranges = mp.check_user_mobility_data_single_user_integrity( grouped_by_date_grid_by_minute_of_day_mp_fetched , msg.current_date ) if len(missing_ranges) > 0: #Missing user information data, queue request and ask data from supplier missing_resource = mr.MissingUserInfo(missing_ranges) frontend.queue_as_pending_request(connection, msg, [missing_resource]) err.log_error(err.INFO, "Did not have enough user info data. Asking supplier for: " + str(missing_ranges)) error_message = mo.Error(102, "MP Request will take longer than expected because user data is missing at MOBaaS") connection.add_message(error_message) for missing_range in missing_ranges: start_time = missing_range.min_range.time() start_date = missing_range.min_range.date() time_span = int((missing_range.max_range - missing_range.min_range).total_seconds()) supplier_message = mo.MaaSSingleUserDataRequest(msg.user_id, start_time, start_date, time_span, 60) frontend.ask_at_supplier(supplier_message) else: #There is all the info we need, calculate the spots current_day_str = support.calc_day(msg.current_date) predicted_day_str = support.calc_day(predicted_date) pykov_chain = mp.prepare_pykov_chain_with_single_user_mobility_states(grouped_by_date_grid_by_minute_of_day_mp_fetched , current_day_str , predicted_day_str ) chance_distribution_dict = mp.mobility_prediction(pykov_chain, msg.current_cell_id, msg.current_date, msg.current_time, predicted_date, predicted_time) #Create mobaas_protocol.prediction objects from the answers mobaas_protocol_predictions = mp.from_pykov_distribution_dict_to_predictions(chance_distribution_dict) answer_message = mo.MPSingleUserAnswer(msg.user_id, predicted_time, predicted_date, mobaas_protocol_predictions) connection.add_message(answer_message) err.log_error(err.INFO, "Found an answer to the request! Answer:" + str(answer_message)) else: print 'error: should not get here!'
def set_next_time(self): td = datetime.timedelta(seconds=self.wait_between_sends) self.next_send_time = self.next_send_time + td err.log_error( err.INFO, "Next send will be at: " + self.next_send_time.strftime("[%H:%M:%S %d-%m-%Y]"))
def close(self): err.log_error(err.INFO, "Closing Test Consumer.") self.client.broadcast_close( message_objects.EndConversation("Client is being closed")) self.client.close_all_connections() err.log_error(err.INFO, "Closed the Test Consumer. Now Quitting.")
def execute(self, query): result = False cur = self.db_connection.cursor(mdb.cursors.DictCursor) try: cur.execute(query) self.db_connection.commit() cur.close() result = True except mdb.Error as e: err.log_error( err.CRITICAL, "Tried to perform a query on the database, but it wasn't able to!" ) err.log_error(err.CRITICAL, "Query: " + query) err.log_error(err.CRITICAL, "Database: " + self.db_name) err.log_error(err.CRITICAL, "Error: " + e.message) if self.db_connection: try: self.db_connection.rollback() except mdb.Error as e_rollback: err.log_error( err.CRITICAL, "Tried to rollback the failed query but this failed as well. Your database might have become corrupted!" ) err.log_error(err.CRITICAL, "Error: " + e_rollback.message) return result
def add_connection(self, connection): err.log_error( err.INFO, "Connection " + str(connection.cid) + " has been added to Connection_Collection " + self.name) self.connections.append(connection)
def create_multiple_connections(self, db_names): err.log_error(err.INFO, "Create multiple database connections") for db_name in db_names: cid = self.next_connection_id() DBConnection(cid, self, db_name)