def mongodb_to_topic_store(scenario_file, topic_store_file):
client = MongoStorage.load(scenario_file)
print("Converting MongoDB '{}' to '{}'".format(client.uri, topic_store_file.name))
storage = get_mongo_storage_by_session(client)
count = storage.cursor.count()
topic_storage = TopicStorage(topic_store_file)
with tqdm(total=count) as progress_bar:
for item in storage:
topic_storage.insert_one(item)
progress_bar.update()
def topic_store_to_mongodb(topic_store_file, scenario_file):
client = MongoStorage.load(scenario_file)
print("Converting '{}' to MongoDB '{}'".format(topic_store_file.name, client.uri))
storage = TopicStorage.load(topic_store_file)
count = len(storage) # TODO: very slow operation
with tqdm(total=count) as progress_bar:
for item in storage:
try:
client.insert_one(item)
except pymongo.errors.DuplicateKeyError:
print("Storage Item '_id: {}' already exists in the '{}/{}' collection".format(item.id, client.name,
client.collection_name))
progress_bar.update()
def _convert_cli():
parser = argparse.ArgumentParser()
parser.add_argument("-i",
"--input",
help="Input File",
type=str,
required=True)
parser.add_argument("-o",
"--output",
help="Output File",
type=str,
required=True)
parser.add_argument(
"-c",
"--collection",
help="MongoDB collection to use if URI passed as --input",
type=str,
required=False)
parser.add_argument("-q",
"--query",
help='MongoDB input query as dict (example: -q '
'\'{"_id": "ObjectId(5f718a354e5e8239dcd1eca1)"}\'',
type=str,
required=False,
default=None)
parser.add_argument(
"-p",
"--projection",
help='MongoDB input projection as dict (example: -p \'{"name": 1}\'',
type=str,
required=False,
default=None)
args = parser.parse_args()
rospy.init_node("topic_store_convert", anonymous=True)
input_path = pathlib.Path(args.input)
output_path = pathlib.Path(args.output)
# if not input_path.exists():
# raise IOError("Input file '{}' does not exist".format(input_path))
if input_path.suffix == ".bag":
raise NotImplementedError(
"Converting from ROS bags is not currently supported. "
"The conversion to ROS bags is lossy and requires adding meta data to reconstruct"
"the original .topic_store or database documents")
elif input_path.suffix == TopicStorage.suffix and output_path.suffix == ".bag":
topic_store_to_ros_bag(input_path, output_path)
elif input_path.suffix == ".yaml" and output_path.suffix == TopicStorage.suffix:
input_path = resolve_scenario_yaml(input_path)
mongodb_to_topic_store(MongoStorage.load(input_path), output_path)
elif input_path.suffix == ".yaml" and output_path.suffix == ".bag":
input_path = resolve_scenario_yaml(input_path)
mongodb_to_ros_bag(MongoStorage.load(input_path), output_path)
elif input_path.suffix == TopicStorage.suffix and output_path.suffix == ".yaml":
output_path = resolve_scenario_yaml(output_path)
topic_store_to_mongodb(input_path, output_path)
elif is_uri(args.input):
srv = args.input
collection = args.collection
query = args.query
projection = args.projection
# if not hasattr(args, "query") or not args.query:
# raise ValueError("If input is a MongoDB URI you must specify a DB query -q/--query to export data")
if not hasattr(args, "collection") or not args.collection:
raise ValueError(
"If input is a MongoDB URI you must specify a DB collection -c/--collection to query data"
)
# Try to parse a query/projection string to a dict and perform some basic cleaning
# The query string will filter the db documents by client.find(query)
if query is not None:
try:
query, projection = [
x if x is None else json.loads(x)
for x in [args.query, args.projection]
]
except ValueError:
print(
"Query/Projection parameter cannot be parsed as a python dict \nQ: '{}'\nP: '{}'"
.format(args.query, args.projection))
raise
# Some simple rules to support searching by ID from console
if query:
for k, v in query.items():
try:
unicode
except NameError:
unicode = str
if isinstance(v, (str, unicode)) and (v.startswith('ObjectId(')
and v.endswith(')')):
print("Converting query field '{}' to ObjectId".format(k))
query[k] = ObjectId(str(v[9:-1]))
client = client_from_uri(srv, collection=collection)
if is_uri(args.output):
db2 = client_from_uri(args.output, collection=collection)
mongodb_to_mongodb(client, db2, query=query, projection=projection)
elif output_path.suffix == ".bag":
mongodb_to_ros_bag(client,
output_path,
query=query,
projection=projection)
elif output_path.suffix == TopicStorage.suffix:
mongodb_to_topic_store(client,
output_path,
query=query,
projection=projection)
else:
raise ValueError(
"No valid conversion from Mongo URI '{}' to '{}' file".format(
client.uri, output_path))
elif input_path.suffix == output_path.suffix:
print("No conversion or migration for '{}' to '{}'".format(
input_path, output_path))
print(
"If you would like to copy the file please use 'cp {} {}'".format(
input_path, output_path))
else:
print("No conversion or migration for '{}' to '{}'".format(
input_path, output_path))