def get_variables(self, data_source): '''Returns a list of all stored variables corresponding to the input data source Keyword arguments: @param data_source -- a string denoting the name of a data source; collection names corresponding to data from this data source are expected to start with the data source name ''' (host, port) = DBUtils.get_db_host_and_port() if port != self.db_port: port = self.db_port client = pm.MongoClient(host=host, port=port) database = client[self.db_name] collection_names = database.list_collection_names() variable_list = [] for collection_name in collection_names: if collection_name == 'system.indexes' or \ not collection_name.startswith(data_source): continue collection = database[collection_name] variable_names = DataUtils.get_variable_list( collection_name, collection) variable_list.extend(variable_names) return variable_list
def write_metadata(self, config_params): '''If a "black_box_metadata" collection doesn't exist in the black box database, creates the collection and stores the metadata for all logged variables there. @param config_params -- a black_box.config.config_params.ConfigParams instance ''' if config_params: (host, port) = DBUtils.get_db_host_and_port() if port != self.db_port: port = self.db_port client = pm.MongoClient(host=host, port=port) db = client[self.db_name] collection_names = db.list_collection_names() if 'black_box_metadata' in collection_names: print('[write_metadata] {0} already has a "black_box_metadata" collection'.format(self.db_name)) else: print('[write_metadata] Saving metadata') collection = db['black_box_metadata'] if config_params.ros: for topic_params in config_params.ros.topic: collection_name = ConfigUtils.get_full_variable_name('ros', topic_params.name) if topic_params.metadata: metadata = {} metadata['collection_name'] = collection_name metadata['ros'] = {} metadata['ros']['topic_name'] = topic_params.metadata.topic_name metadata['ros']['msg_type'] = topic_params.metadata.msg_type metadata['ros']['direct_msg_mapping'] = topic_params.metadata.direct_msg_mapping collection.insert_one(metadata) if config_params.zmq: for topic_params in config_params.zmq.topics: collection_name = ConfigUtils.get_full_variable_name('zmq', topic_params.name) if topic_params.metadata: metadata = {} metadata['collection_name'] = collection_name metadata['ros'] = {} metadata['ros']['topic_name'] = topic_params.metadata.topic_name metadata['ros']['msg_type'] = topic_params.metadata.msg_type metadata['ros']['direct_msg_mapping'] = topic_params.metadata.direct_msg_mapping collection.insert_one(metadata) else: raise AssertionError('[write_metadata] config_params needs to be of type ' + \ 'black_box.config.config_params.ConfigParams')
def log_data(self, variable, timestamp, data): '''Logs the data dictionary for the given variable; adds the timestamp to the data dictionary before logging. @param variable -- name of a variable to be logged @param timestamp -- a timestamp (epoch in seconds) @param data -- a dictionary to be logged ''' (host, port) = DBUtils.get_db_host_and_port() if port != self.db_port: port = self.db_port client = pm.MongoClient(host=host, port=port) db = client[self.db_name] collection = db[variable] data['timestamp'] = timestamp collection.insert_one(data)
def get_latest_data(self, collection_name, variable_names): '''Returns a dictionary in which each key is a full variable name (namely a variable name of the format "collection_name/variable_name", where "variable_name" is a flattened version of a variable stored in the collection) and the value is a list string "[timestamp, value]", namely the latest value of the variable together with its timestamp (the list is a string to allow "value" to be of different types). If the given collection does not exist or there are no documents in it, returns an empty dictionary. Keyword arguments: @param collection_name -- name corresponding to a collection from the log database @param variable_names -- list of variable names that should be retrieved from the collection ''' (host, port) = DBUtils.get_db_host_and_port() if port != self.db_port: port = self.db_port client = pm.MongoClient(host=host, port=port) database = client[self.db_name] collection = database[collection_name] doc = collection.find_one(sort=[('timestamp', pm.DESCENDING)]) var_data = {} var_full_names = {} for var_name in variable_names: full_var_name = '{0}/{1}'.format(collection_name, var_name) var_data[full_var_name] = None var_full_names[var_name] = full_var_name if doc: for var_name in variable_names: var_value = DataUtils.get_var_value(doc, var_name) var_data[var_full_names[var_name]] = '[{0}, {1}]'.format( doc['timestamp'], var_value) return var_data