def init_save_database(self):
from topic_store.database import MongoStorage
self.db_client = MongoStorage(config=self.scenario.storage["config"],
collection=self.scenario.context,
use_grid_fs=self.use_grid_fs)
self.info_logger("Initialised saving to database {} @ '{}/{}'".format(
self.db_client.uri, self.db_client.name, self.scenario.context))
def mongodb_to_ros_bag(scenario_file, output_file):
scenario = ScenarioFileParser(scenario_file)
if scenario.storage["method"] != "database":
raise ValueError(
"Scenario file '{}' storage.method is not set to 'database'".
format(scenario_file))
print("Converting MongoDB '{}' to ROS bag '{}'".format(
scenario.storage["uri"], output_file.name))
client = MongoStorage(uri=scenario.storage["uri"],
collection=scenario.context)
storage = get_mongo_storage_by_session(client)
count = storage.cursor.count()
ros_bag = rosbag.Bag(str(output_file), 'w')
try:
with tqdm(total=count) as progress_bar:
for item in storage:
msgs = item.to_ros_msg_list()
time = rospy.Time.from_sec(item["_ts_meta"]["ros_time"])
for msg in msgs:
if hasattr(msg, "_connection_header"):
source = getattr(msg, "_connection_header")["topic"]
if source:
try:
ros_bag.write(source, msg, time)
except Exception as e:
print("Could not write", source, 'because',
e.message)
progress_bar.update()
finally:
print("Closing the ROS bag '{}'".format(output_file))
ros_bag.close()
def test_image_encoding(self):
if not rospy.get_node_uri():
rospy.init_node("topic_store_tests")
client = MongoStorage(collection="python_tests")
# Test ROS_Message Serialisation with common Image Types
from topic_store.utils import _numpy_types as numpy_types
for encoding, np_type in numpy_types.items():
size = (32, 32, np_type[1]) if np_type[1] != 1 else (32, 32)
random_array = np.random.random(size)
typed_array = (random_array * 255.0).astype(np_type[0])
message = ros_numpy.msgify(Image, typed_array, encoding=encoding)
# Insert image message
im_document = TopicStore({"image": message})
im_result = client.insert_one(im_document)
# Get image message and check data is the same
returned_im_document = client.find_by_id(im_result.inserted_id)
assert returned_im_document.id == im_result.inserted_id
retrieved_array = ros_numpy.numpify(
returned_im_document.msgs["image"])
np.testing.assert_equal(typed_array, retrieved_array)
# Delete image message
client.delete_by_id(im_result.inserted_id)
def mongodb_to_topic_store(scenario_file, topic_store_file):
client = MongoStorage.load(scenario_file)
print("Converting MongoDB '{}' to '{}'".format(client.uri, topic_store_file.name))
storage = get_mongo_storage_by_session(client)
count = storage.cursor.count()
topic_storage = TopicStorage(topic_store_file)
with tqdm(total=count) as progress_bar:
for item in storage:
topic_storage.insert_one(item)
progress_bar.update()
def topic_store_to_mongodb(topic_store_file, scenario_file):
client = MongoStorage.load(scenario_file)
print("Converting '{}' to MongoDB '{}'".format(topic_store_file.name, client.uri))
storage = TopicStorage.load(topic_store_file)
count = len(storage) # TODO: very slow operation
with tqdm(total=count) as progress_bar:
for item in storage:
try:
client.insert_one(item)
except pymongo.errors.DuplicateKeyError:
print("Storage Item '_id: {}' already exists in the '{}/{}' collection".format(item.id, client.name,
client.collection_name))
progress_bar.update()
def client_from_uri(uri, collection):
if is_uri(uri):
# DB name will usually be specified as authSource in the URI, if not present use default=topic_store
db_name = None
if "authSource" in uri:
options = [
s.split('=') for s in urlparse(uri).query.split("&") if s
]
options = {k: v for k, v in options}
if "authSource" in options:
db_name = options["authSource"]
client = MongoStorage(collection=collection, uri=uri, db_name=db_name)
return client
else:
raise ValueError("Not a valid URI: {}".format(uri))
def test_documents(self):
if not rospy.get_node_uri():
rospy.init_node("topic_store_tests")
client = MongoStorage(collection="python_tests")
# Insert a test document
insert_result = client.insert_one(
TopicStore({
"name": "test_name",
"number": 1
}))
# Retrieve the inserted document
inserted_document = client.find_by_id(insert_result.inserted_id)
assert inserted_document.id == insert_result.inserted_id
# Update the document name and number fields
new_name = ''.join(random.sample('raymond', 7))
new_number = random.randint(0, 100)
update_result = client.update_one_by_id(inserted_document.id,
name=new_name,
number=new_number)
inserted_document_after_update = client.find_by_id(
insert_result.inserted_id)
assert inserted_document_after_update.id == insert_result.inserted_id
assert inserted_document_after_update.dict["number"] == new_number
assert inserted_document_after_update.dict["name"] == new_name
# Print all documents in the collection
cursor = client.find()
for x in cursor:
print(
"Doc:\n\t-As Structure: {}\n\t-As Dict: {}\n\t-As ROS Msgs: {}"
.format(str(x), x.dict, x.msgs))
# Or print using the same API as TopicStorage
for x in client:
print(
"Doc:\n\t-As Structure: {}\n\t-As Dict: {}\n\t-As ROS Msgs: {}"
.format(str(x), x.dict, x.msgs))
# Cleanup test by deleting document
delete_result = client.delete_by_id(insert_result.inserted_id)
def test_large_document(self):
if not rospy.get_node_uri():
rospy.init_node("topic_store_tests")
client = MongoStorage(collection="python_tests", use_grid_fs=True)
# Insert >16MB document
random_array = np.random.random((3000, 3000, 3)).astype(np.float32)
message = ros_numpy.msgify(Image, random_array, encoding="32FC3")
# Insert image message
im_document = TopicStore({"image": message})
im_result = client.insert_one(im_document)
# Get image message and check data is the same
returned_im_document = client.find_by_id(im_result.inserted_id)
assert returned_im_document.id == im_result.inserted_id
retrieved_array = ros_numpy.numpify(returned_im_document.msgs["image"])
np.testing.assert_equal(random_array, retrieved_array)
# Delete image message
client.delete_by_id(im_result.inserted_id)
def test_image_encoding(self):
if not rospy.get_node_uri():
rospy.init_node("topic_store_tests")
client = MongoStorage(collection="python_tests")
# Test ROS_Message Serialisation with common Image Types
numpy_types = {
"rgb8": (np.uint8, 3),
"rgba8": (np.uint8, 4),
"rgb16": (np.uint16, 3),
"rgba16": (np.uint16, 4),
"bgr8": (np.uint8, 3),
"bgra8": (np.uint8, 4),
"bgr16": (np.uint16, 3),
"bgra16": (np.uint16, 4),
"mono8": (np.uint8, 1),
"mono16": (np.uint16, 1),
"bayer_rggb8": (np.uint8, 1),
"bayer_bggr8": (np.uint8, 1),
"bayer_gbrg8": (np.uint8, 1),
"bayer_grbg8": (np.uint8, 1),
"bayer_rggb16": (np.uint16, 1),
"bayer_bggr16": (np.uint16, 1),
"bayer_gbrg16": (np.uint16, 1),
"bayer_grbg16": (np.uint16, 1),
"8UC1": (np.uint8, 1),
"8UC2": (np.uint8, 2),
"8UC3": (np.uint8, 3),
"8UC4": (np.uint8, 4),
"8SC1": (np.int8, 1),
"8SC2": (np.int8, 2),
"8SC3": (np.int8, 3),
"8SC4": (np.int8, 4),
"16UC1": (np.uint16, 1),
"16UC2": (np.uint16, 2),
"16UC3": (np.uint16, 3),
"16UC4": (np.uint16, 4),
"16SC1": (np.int16, 1),
"16SC2": (np.int16, 2),
"16SC3": (np.int16, 3),
"16SC4": (np.int16, 4),
"32SC1": (np.int32, 1),
"32SC2": (np.int32, 2),
"32SC3": (np.int32, 3),
"32SC4": (np.int32, 4),
"32FC1": (np.float32, 1),
"32FC2": (np.float32, 2),
"32FC3": (np.float32, 3),
"32FC4": (np.float32, 4),
"64FC1": (np.float64, 1),
"64FC2": (np.float64, 2),
"64FC3": (np.float64, 3),
"64FC4": (np.float64, 4)
}
for encoding, np_type in numpy_types.items():
size = (32, 32, np_type[1]) if np_type[1] != 1 else (32, 32)
random_array = np.random.random(size)
typed_array = (random_array * 255.0).astype(np_type[0])
message = ros_numpy.msgify(Image, typed_array, encoding=encoding)
# Insert image message
im_document = TopicStore({"image": message})
im_result = client.insert_one(im_document)
# Get image message and check data is the same
returned_im_document = client.find_by_id(im_result.inserted_id)
assert returned_im_document.id == im_result.inserted_id
retrieved_array = ros_numpy.numpify(
returned_im_document.msgs["image"])
np.testing.assert_equal(typed_array, retrieved_array)
# Delete image message
client.delete_by_id(im_result.inserted_id)
class ScenarioRunner:
def __init__(self,
scenario_file,
stabilise_time,
verbose=True,
queue_size=100,
n_io_threads=1,
threads_auto=False,
use_grid_fs=False):
self.saved_n = 0
self.scenario_file = scenario_file
self.stabilise_time = stabilise_time
self.verbose = verbose
self.thread_queue_size = queue_size
self.n_threads = n_io_threads
self.auto_threads = threads_auto
self.use_grid_fs = use_grid_fs
self.events = {}
# Load Scenario
rospy.loginfo("Loading scenario: '{}'".format(scenario_file))
self.scenario = ScenarioFileParser(scenario_file)
# Create publisher for topic_store scenario logs
self.logger = best_logger(verbose=verbose)
self.info_logger = DefaultLogger(verbose=verbose)
# Create subscriber tree for getting all the data
self.subscriber_tree = SubscriberTree(self.scenario.data)
if self.stabilise_time:
self.info_logger(
"Waiting for {}s before starting (stabilise_time)".format(
self.stabilise_time))
rospy.sleep(self.stabilise_time)
# Choose appropriate methods
# Setting up the scenario runner this way means it's easily extendable by inheriting from Scenario runner
# and declaring custom behaviours
save_init_function_name = "init_save_" + self.scenario.storage["method"]
self.save_method_init_function = getattr(self, save_init_function_name,
None)
if not callable(self.save_method_init_function):
raise Exception(
"Invalid storage value ('{}()' does not exist)".format(
save_init_function_name))
self.save_method_init_function()
save_function_name = "save_" + self.scenario.storage["method"]
self.save_method_function = getattr(self, save_function_name, None)
if not callable(self.save_method_function):
raise Exception(
"Invalid storage value ('{}()' does not exist)".format(
save_function_name))
collection_method_init_name = "init_way_point_" + self.scenario.collection[
"method"]
self.collection_method_init_function = getattr(
self, collection_method_init_name, None)
if not callable(self.collection_method_init_function):
raise Exception(
"Invalid way point value ('{}()' does not exist)".format(
collection_method_init_name))
self.save_callback_rate = FPSCounter()
self.event_callback_rate = FPSCounter()
self.extra_logs = {}
if self.n_threads > 0:
self.jobs_worker = LoadBalancer(maxsize=self.thread_queue_size,
threads=self.n_threads,
auto=self.auto_threads)
self.save_callback_rate = FPSCounter(self.jobs_worker.maxsize)
self.collection_method_init_function()
def init_way_point_action_server(self):
def __request(goal_msg):
success, save_msg = True, "Success!"
# TODO: self.save() no longer returns values of self.__save() because jobs are handled in a load_balancer
# TODO: for now action_server will return success no matter what
self.save()
result = CollectDataResult(success)
feedback = CollectDataFeedback(save_msg)
self.service_server.publish_feedback(feedback)
(self.service_server.set_succeeded
if success else self.service_server.set_aborted)(result)
action_lib_server_name = self.scenario.collection["action_server_name"]
self.info_logger(
"Starting '{}' actionlib server".format(action_lib_server_name))
self.service_server = actionlib.SimpleActionServer(
action_lib_server_name, CollectDataAction, __request, False)
self.service_server.start()
def init_way_point_action_server_video(self):
event_name = "action_lib_start_stop"
topic_to_watch = self.scenario.collection["watch_topic"]
self.events[event_name] = {"event": Event(), "data": False}
self.events[topic_to_watch] = {"event": Event(), "data": ""}
def __request(goal_msg):
should_start = str(
goal_msg.message).lower() in ["true", "t", "start"]
starting_string = "Starting" if should_start else "Stopping"
self.info_logger("{} data collection from '{}'".format(
starting_string, action_lib_server_name))
self.set_event_msg_callback(should_start, event_name)
result = CollectDataResult(should_start)
feedback = CollectDataFeedback(
"{} data capture".format(starting_string))
self.service_server.publish_feedback(feedback)
self.service_server.set_succeeded(result)
action_lib_server_name = self.scenario.collection["action_server_name"]
self.info_logger(
"Starting '{}' actionlib server".format(action_lib_server_name))
self.service_server = actionlib.SimpleActionServer(
action_lib_server_name, CollectDataAction, __request, False)
self.service_server.start()
AutoSubscriber(topic_to_watch,
callback=self.set_event_msg_callback,
callback_args=topic_to_watch)
while not rospy.is_shutdown():
self.events[topic_to_watch]["event"].wait()
self.events[topic_to_watch]["event"].clear()
if self.events[event_name]["data"]:
self.save()
# self.log("Waiting for event on '{}' topic before next data cycle".format(topic_to_watch))
def set_event_msg_callback(self, data, event_id):
if event_id in self.events:
self.event_callback_rate.toc()
self.event_callback_rate.tic()
self.extra_logs["event_callback_rate"] = "{:.2f}".format(
self.save_callback_rate.get_fps())
self.events[event_id]["data"] = data
self.events[event_id]["event"].set()
def init_way_point_timer(self):
delay = self.scenario.collection["timer_delay"]
while not rospy.is_shutdown():
self.save()
# self.log("Waiting for {}s before next data cycle".format(delay))
rospy.sleep(delay)
def init_way_point_event(self):
event_name = "topic_update"
topic_to_watch = self.scenario.collection["watch_topic"]
self.events[event_name] = {"event": Event(), "data": ""}
AutoSubscriber(topic_to_watch,
callback=self.set_event_msg_callback,
callback_args=event_name)
while not rospy.is_shutdown():
self.events[event_name]["event"].wait()
self.events[event_name]["event"].clear()
self.save()
# self.log("Waiting for event on '{}' topic before next data cycle".format(topic_to_watch))
def init_save_database(self):
from topic_store.database import MongoStorage
self.db_client = MongoStorage(config=self.scenario.storage["config"],
collection=self.scenario.context,
use_grid_fs=self.use_grid_fs)
self.info_logger("Initialised saving to database {} @ '{}/{}'".format(
self.db_client.uri, self.db_client.name, self.scenario.context))
def init_save_filesystem(self):
from topic_store.filesystem import TopicStorage
formatted_datetime = datetime.datetime.now().strftime(
'%Y_%m_%d_%H_%M_%S')
save_location = self.scenario.storage["location"]
if not save_location or save_location in [
"default", "auto", "topic_store"
]:
save_location = pathlib.Path(
os.path.expanduser("~/.ros/topic_store/filesystem"))
elif save_location.startswith("pkg="):
package_name = save_location.split('=')[-1]
save_location = pathlib.Path(
os.path.expanduser(
"~/.ros/topic_store/filesystem/{}/".format(package_name)))
else:
save_location = pathlib.Path(os.path.expanduser(save_location))
save_folder = save_location / self.scenario.context
self.info_logger(
"Configured save_location as '{}'".format(save_folder))
try:
save_folder.mkdir(parents=True)
except OSError as e:
if e.errno != 17: # File exists is okay
raise
save_file = save_folder / "{}{}".format(formatted_datetime,
TopicStorage.suffix)
self.filesystem_storage = TopicStorage(save_file)
self.info_logger("Initialised saving to the filesystem at '{}'".format(
self.filesystem_storage.path))
def save_database(self, message_tree):
insert_result = self.db_client.insert_one(message_tree)
self.saved_n += 1
# self.log("Inserted document to database result='acknowledged={}, inserted_id={}'".format(
# insert_result.acknowledged, insert_result.inserted_id))
return insert_result.inserted_id
def save_filesystem(self, message_tree):
# self.log("Saving documents to file system n={}".format(self.saved_n))
self.saved_n += 1
self.filesystem_storage.insert_one(message_tree)
return self.saved_n
def __save(self, data, job_meta=None, **kwargs):
try:
saved_data_id = self.save_method_function(data)
except Exception as e:
self.info_logger("Exception raised when saving! '{}'".format(
e.message))
return False, e.message
if job_meta is not None:
worker_id = job_meta.pop("worker_id", None)
# Pretty print some variables
info_str = {
k.replace("worker_", ""): "{:.2f}".format(v)
for k, v in job_meta.items()
}
self.extra_logs["save_callback_rate"] = "{:.2f}".format(
self.save_callback_rate.get_fps())
for k, v in self.extra_logs.items():
info_str[k] = v
self.logger(
"Worker {} successfully saved data id='{}'".format(
worker_id, saved_data_id), **info_str)
else:
self.logger(
"Successfully saved data id='{}'".format(saved_data_id),
**self.extra_logs)
return True, "Success!"
def save(self):
"""Collates data from the scenario topic structure and saves. Returns SaveSuccess, SaveMessage"""
self.save_callback_rate.toc()
self.save_callback_rate.tic()
data_hierarchy = self.subscriber_tree.get_message_tree()
if self.n_threads == 0: # don't use a threading model
self.__save(data_hierarchy)
else:
added_task = self.jobs_worker.add_task(self.__save,
[data_hierarchy], {},
wait=False)
if not added_task:
self.info_logger(
"IO Queue Full, cannot add jobs to the queue, please wait.",
verbose=True)
self.jobs_worker.add_task(self.__save, [data_hierarchy], {},
wait=True)
class ScenarioRunner:
def __init__(self, scenario_file, stabilise_time, verbose=True):
self.saved_n = 0
self.scenario_file = scenario_file
self.stabilise_time = stabilise_time
self.verbose = verbose
self.logger = print
self.events = {}
# Load Scenario
rospy.loginfo("Loading scenario: '{}'".format(scenario_file))
self.scenario = ScenarioFileParser(scenario_file)
# Create subscriber tree for getting all the data
self.subscriber_tree = SubscriberTree(self.scenario.data)
if self.stabilise_time:
self.log("Waiting for {}s before starting (stabilise_time)".format(self.stabilise_time))
rospy.sleep(self.stabilise_time)
# Choose appropriate methods
# Setting up the scenario runner this way means it's easily extendable by inheriting from Scenario runner
# and declaring custom behaviours
save_init_function_name = "init_save_" + self.scenario.storage["method"]
self.save_method_init_function = getattr(self, save_init_function_name, None)
if not callable(self.save_method_init_function):
raise Exception("Invalid storage value ('{}()' does not exist)".format(save_init_function_name))
self.save_method_init_function()
save_function_name = "save_" + self.scenario.storage["method"]
self.save_method_function = getattr(self, save_function_name, None)
if not callable(self.save_method_function):
raise Exception("Invalid storage value ('{}()' does not exist)".format(save_function_name))
collection_method_init_name = "init_way_point_" + self.scenario.collection["method"]
self.collection_method_init_function = getattr(self, collection_method_init_name, None)
if not callable(self.collection_method_init_function):
raise Exception("Invalid way point value ('{}()' does not exist)".format(collection_method_init_name))
self.collection_method_init_function()
def log(self, message, **kwargs):
if self.verbose:
self.logger("\033[93mScenarioRunner\033[0m: {}".format(message), **kwargs)
def init_way_point_action_server(self):
def __request(goal_msg):
success, save_msg = self.save()
result = CollectDataResult(success)
feedback = CollectDataFeedback(save_msg)
self.service_server.publish_feedback(feedback)
(self.service_server.set_succeeded if success else self.service_server.set_aborted)(result)
action_lib_server_name = self.scenario.collection["action_server_name"]
self.log("Starting '{}' actionlib server".format(action_lib_server_name))
self.service_server = actionlib.SimpleActionServer(action_lib_server_name, CollectDataAction, __request, False)
self.service_server.start()
def init_way_point_action_server_video(self):
event_name = "action_lib_start_stop"
topic_to_watch = self.scenario.collection["watch_topic"]
self.events[event_name] = {"event": Event(), "data": False}
self.events[topic_to_watch] = {"event": Event(), "data": ""}
def __request(goal_msg):
should_start = str(goal_msg.message).lower() in ["true", "t", "start"]
starting_string = "Starting" if should_start else "Stopping"
self.log("{} data collection from '{}'".format(starting_string, action_lib_server_name))
self.set_event_msg_callback(should_start, event_name)
result = CollectDataResult(should_start)
feedback = CollectDataFeedback("{} data capture".format(starting_string))
self.service_server.publish_feedback(feedback)
self.service_server.set_succeeded(result)
action_lib_server_name = self.scenario.collection["action_server_name"]
self.log("Starting '{}' actionlib server".format(action_lib_server_name))
self.service_server = actionlib.SimpleActionServer(action_lib_server_name, CollectDataAction, __request, False)
self.service_server.start()
AutoSubscriber(topic_to_watch, callback=self.set_event_msg_callback, callback_args=topic_to_watch)
while not rospy.is_shutdown():
self.events[topic_to_watch]["event"].wait()
self.events[topic_to_watch]["event"].clear()
if self.events[event_name]["data"]:
self.save()
self.log("Waiting for event on '{}' topic before next data cycle".format(topic_to_watch))
def set_event_msg_callback(self, data, event_id):
if event_id in self.events:
self.events[event_id]["data"] = data
self.events[event_id]["event"].set()
def init_way_point_timer(self):
delay = self.scenario.collection["timer_delay"]
while not rospy.is_shutdown():
self.save()
self.log("Waiting for {}s before next data cycle".format(delay))
rospy.sleep(delay)
def init_way_point_event(self):
event_name = "topic_update"
topic_to_watch = self.scenario.collection["watch_topic"]
self.events[event_name] = {"event": Event(), "data": ""}
AutoSubscriber(topic_to_watch, callback=self.set_event_msg_callback, callback_args=event_name)
while not rospy.is_shutdown():
self.events[event_name]["event"].wait()
self.events[event_name]["event"].clear()
self.save()
self.log("Waiting for event on '{}' topic before next data cycle".format(topic_to_watch))
def init_save_database(self):
from topic_store.database import MongoStorage
self.db_client = MongoStorage(config=self.scenario.storage["config"], collection=self.scenario.context)
self.log("Initialised saving to database {} @ '{}/{}'".format(self.db_client.uri,
self.db_client.name, self.scenario.context))
def init_save_filesystem(self):
from topic_store.filesystem import TopicStorage
formatted_datetime = datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S')
save_location = self.scenario.storage["location"]
if not save_location or save_location in ["default", "auto", "topic_store"]:
save_location = pathlib.Path(os.path.expanduser("~/.ros/topic_store/filesystem"))
elif save_location.startswith("pkg="):
package_name = save_location.split('=')[-1]
save_location = pathlib.Path(os.path.expanduser("~/.ros/topic_store/filesystem/{}/".format(package_name)))
else:
save_location = pathlib.Path(os.path.expanduser(save_location))
save_folder = save_location / self.scenario.context
self.log("Configured save_location as '{}'".format(save_folder))
try:
save_folder.mkdir(parents=True)
except OSError as e:
if e.errno != 17: # File exists is okay
raise
save_file = save_folder / "{}{}".format(formatted_datetime, TopicStorage.suffix)
self.filesystem_storage = TopicStorage(save_file)
self.log("Initialised saving to the filesystem at '{}'".format(self.filesystem_storage.path))
def save_database(self, message_tree):
insert_result = self.db_client.insert_one(message_tree)
self.log("Inserted document to database result='acknowledged={}, inserted_id={}'".format(
insert_result.acknowledged, insert_result.inserted_id))
def save_filesystem(self, message_tree):
self.log("Saving documents to file system n={}".format(self.saved_n))
self.saved_n += 1
self.filesystem_storage.insert_one(message_tree)
def save(self):
"""Collates data from the scenario topic structure and saves. Returns SaveSuccess, SaveMessage"""
data = self.subscriber_tree.get_message_tree()
try:
self.save_method_function(data)
except Exception as e:
self.log("Exception raised when saving! '{}'".format(e.message))
return False, e.message
return True, "Success!"
def _convert_cli():
parser = argparse.ArgumentParser()
parser.add_argument("-i",
"--input",
help="Input File",
type=str,
required=True)
parser.add_argument("-o",
"--output",
help="Output File",
type=str,
required=True)
parser.add_argument(
"-c",
"--collection",
help="MongoDB collection to use if URI passed as --input",
type=str,
required=False)
parser.add_argument("-q",
"--query",
help='MongoDB input query as dict (example: -q '
'\'{"_id": "ObjectId(5f718a354e5e8239dcd1eca1)"}\'',
type=str,
required=False,
default=None)
parser.add_argument(
"-p",
"--projection",
help='MongoDB input projection as dict (example: -p \'{"name": 1}\'',
type=str,
required=False,
default=None)
args = parser.parse_args()
rospy.init_node("topic_store_convert", anonymous=True)
input_path = pathlib.Path(args.input)
output_path = pathlib.Path(args.output)
# if not input_path.exists():
# raise IOError("Input file '{}' does not exist".format(input_path))
if input_path.suffix == ".bag":
raise NotImplementedError(
"Converting from ROS bags is not currently supported. "
"The conversion to ROS bags is lossy and requires adding meta data to reconstruct"
"the original .topic_store or database documents")
elif input_path.suffix == TopicStorage.suffix and output_path.suffix == ".bag":
topic_store_to_ros_bag(input_path, output_path)
elif input_path.suffix == ".yaml" and output_path.suffix == TopicStorage.suffix:
input_path = resolve_scenario_yaml(input_path)
mongodb_to_topic_store(MongoStorage.load(input_path), output_path)
elif input_path.suffix == ".yaml" and output_path.suffix == ".bag":
input_path = resolve_scenario_yaml(input_path)
mongodb_to_ros_bag(MongoStorage.load(input_path), output_path)
elif input_path.suffix == TopicStorage.suffix and output_path.suffix == ".yaml":
output_path = resolve_scenario_yaml(output_path)
topic_store_to_mongodb(input_path, output_path)
elif is_uri(args.input):
srv = args.input
collection = args.collection
query = args.query
projection = args.projection
# if not hasattr(args, "query") or not args.query:
# raise ValueError("If input is a MongoDB URI you must specify a DB query -q/--query to export data")
if not hasattr(args, "collection") or not args.collection:
raise ValueError(
"If input is a MongoDB URI you must specify a DB collection -c/--collection to query data"
)
# Try to parse a query/projection string to a dict and perform some basic cleaning
# The query string will filter the db documents by client.find(query)
if query is not None:
try:
query, projection = [
x if x is None else json.loads(x)
for x in [args.query, args.projection]
]
except ValueError:
print(
"Query/Projection parameter cannot be parsed as a python dict \nQ: '{}'\nP: '{}'"
.format(args.query, args.projection))
raise
# Some simple rules to support searching by ID from console
if query:
for k, v in query.items():
try:
unicode
except NameError:
unicode = str
if isinstance(v, (str, unicode)) and (v.startswith('ObjectId(')
and v.endswith(')')):
print("Converting query field '{}' to ObjectId".format(k))
query[k] = ObjectId(str(v[9:-1]))
client = client_from_uri(srv, collection=collection)
if is_uri(args.output):
db2 = client_from_uri(args.output, collection=collection)
mongodb_to_mongodb(client, db2, query=query, projection=projection)
elif output_path.suffix == ".bag":
mongodb_to_ros_bag(client,
output_path,
query=query,
projection=projection)
elif output_path.suffix == TopicStorage.suffix:
mongodb_to_topic_store(client,
output_path,
query=query,
projection=projection)
else:
raise ValueError(
"No valid conversion from Mongo URI '{}' to '{}' file".format(
client.uri, output_path))
elif input_path.suffix == output_path.suffix:
print("No conversion or migration for '{}' to '{}'".format(
input_path, output_path))
print(
"If you would like to copy the file please use 'cp {} {}'".format(
input_path, output_path))
else:
print("No conversion or migration for '{}' to '{}'".format(
input_path, output_path))