def test_image_encoding(self):
if not rospy.get_node_uri():
rospy.init_node("topic_store_tests")
client = MongoStorage(collection="python_tests")
# Test ROS_Message Serialisation with common Image Types
from topic_store.utils import _numpy_types as numpy_types
for encoding, np_type in numpy_types.items():
size = (32, 32, np_type[1]) if np_type[1] != 1 else (32, 32)
random_array = np.random.random(size)
typed_array = (random_array * 255.0).astype(np_type[0])
message = ros_numpy.msgify(Image, typed_array, encoding=encoding)
# Insert image message
im_document = TopicStore({"image": message})
im_result = client.insert_one(im_document)
# Get image message and check data is the same
returned_im_document = client.find_by_id(im_result.inserted_id)
assert returned_im_document.id == im_result.inserted_id
retrieved_array = ros_numpy.numpify(
returned_im_document.msgs["image"])
np.testing.assert_equal(typed_array, retrieved_array)
# Delete image message
client.delete_by_id(im_result.inserted_id)
def __iter__(self):
if not self.path.exists():
raise StopIteration()
with self.path.open("rb") as fh:
while True:
try:
yield TopicStore(data_tree=pickle.load(fh))
except EOFError:
break
def test_topic_store(self):
if not rospy.get_node_uri():
rospy.init_node("topic_store_tests")
sample_tree = {"ros_msg": "/rosout", "int": 1, "float": 1.0, "str": "1", "dict": {0: 0}, "list": [0]}
tree = SubscriberTree(sample_tree)
rospy.sleep(1)
messages = tree.get_message_tree()
test_file = pathlib.Path(__file__).parent / "topic_store_tests.topic_store"
try:
test_file.unlink()
except OSError:
pass
storage = TopicStorage(test_file)
# Try getting an element that doesn't exist
try:
s = list(storage)[0]
raise AssertionError()
except IndexError:
pass
# Try looping over elements that don't exist
for _ in storage:
raise AssertionError()
# Add a message and check
write_n = 5
for _ in range(write_n):
storage.insert_one(messages)
# Check iterator indexing
stored_items = write_n
read_items = 0
for _ in storage:
read_items += 1
assert read_items == stored_items
# Check loading
storage = TopicStorage(test_file)
storage.insert_one(TopicStore({0: 0}))
stored_items += 1
for s in storage:
print(s)
# Test API
from topic_store import load
loaded_messages = load(test_file)
read_items = 0
for _ in storage:
read_items += 1
assert read_items == stored_items
print("All TopicStorage tests passed!")
def test_sanitation(self):
if not rospy.get_node_uri():
rospy.init_node("topic_store_tests")
test_dict = {
# Data heavy types (binary)
"1_Image": ("sensor_msgs/Image", sensor_msgs.msg.Image()),
"1_RegionOfInterest": ("sensor_msgs/RegionOfInterest", sensor_msgs.msg.RegionOfInterest()),
"1_CameraInfo": ("sensor_msgs/CameraInfo", sensor_msgs.msg.CameraInfo()),
# Time types
"1_Time_genpy": ("genpy.Time", rospy.rostime.Time(0)),
"1_Duration_genpy": ("genpy.Duration", rospy.rostime.Duration(0)),
"1_Time_rospy": ("genpy.Time", rospy.rostime.Time(0)),
"1_Duration_rospy": ("genpy.Duration", rospy.rostime.Duration(0)),
# Standard types
"1_Header": ("std_msgs/Header", std_msgs.msg.Header()),
# Duplicate the above to avoid memory copy not value copy errors
# Data heavy types (binary)
"2_Image": ("sensor_msgs/Image", sensor_msgs.msg.Image()),
"2_RegionOfInterest": ("sensor_msgs/RegionOfInterest", sensor_msgs.msg.RegionOfInterest()),
"2_CameraInfo": ("sensor_msgs/CameraInfo", sensor_msgs.msg.CameraInfo()),
# Time types
"2_Time_genpy": ("genpy.Time", rospy.rostime.Time(0)),
"2_Duration_genpy": ("genpy.Duration", rospy.rostime.Duration(0)),
"2_Time_rospy": ("genpy.Time", rospy.rostime.Time(0)),
"2_Duration_rospy": ("genpy.Duration", rospy.rostime.Duration(0)),
# Standard types
"2_Header": ("std_msgs/Header", std_msgs.msg.Header()),
}
test_dict.update({
# Test nested objects
"1_ImageHeader": ("std_msgs/Header", test_dict["1_Image"][1].header),
"1_CameraInfo": ("std_msgs/Header", test_dict["1_CameraInfo"][1].header),
"1_CameraInfoRegionOfInterest": ("sensor_msgs/RegionOfInterest", test_dict["1_CameraInfo"][1].roi),
"2_ImageHeader": ("std_msgs/Header", test_dict["2_Image"][1].header),
"2_CameraInfo": ("std_msgs/Header", test_dict["2_CameraInfo"][1].header),
"2_CameraInfoRegionOfInterest": ("sensor_msgs/RegionOfInterest", test_dict["2_CameraInfo"][1].roi),
})
# Check topic store functions
store = TopicStore(test_dict)
for key, (string_type, type_class) in test_dict.items():
assert store[key][1]["_ros_meta"]["type"] == string_type # check to python sanitation keeps type
assert type(store(key)[1]) == type(type_class) # check to rosify sanitation returns correct type
# Check main sanitation functions
sanitised = sanitise_dict(test_dict)
rosified = rosify_dict(test_dict)
for key, (string_type, type_class) in test_dict.items():
assert sanitised[key][1]["_ros_meta"]["type"] == string_type # check to python sanitation keeps type
assert type(rosified[key][1]) == type(type_class) # check to rosify sanitation returns correct type
print("All sanitation tests passed!")
def insert_one(self, topic_store):
"""Inserts a topic store object into the database
Returns:
pymongo.results.InsertOneResult: Contains the ID for the inserted document
"""
if isinstance(topic_store, dict):
topic_store = TopicStore(topic_store)
if not isinstance(topic_store, TopicStore):
raise ValueError("Can only insert TopicStore items into the database not '{}'".format(type(topic_store)))
parsed_store = self.__gridfs_ify(topic_store)
return self.collection.insert_one(parsed_store)
def __get(self, cursor_ret):
document = None
try:
document = self.parser(cursor_ret)
if self.apply_fn:
document = self.apply_fn(document)
return TopicStore(document)
except Exception as e:
if not self.skip_on_error:
raise
print("Skipping document '{}' because '{}'".format(
(document.get('id') + " ") if document else None, e.message))
return document
def insert_one(self, topic_store):
if isinstance(topic_store, dict):
topic_store = TopicStore(topic_store)
if not isinstance(topic_store, TopicStore):
raise ValueError("TopicStorage only supports TopicStore types")
# Create if doesn't exist on the system
if not self.path.exists():
try:
self.path.parent.mkdir(parents=True)
except OSError as e:
if e.errno != 17: # File exists is okay
raise
with self.path.open("ab" if self.path.exists() else "wb") as fh:
pickle.dump(topic_store.dict, fh, protocol=TopicStorage.PROTOCOL)
def test_documents(self):
if not rospy.get_node_uri():
rospy.init_node("topic_store_tests")
client = MongoStorage(collection="python_tests")
# Insert a test document
insert_result = client.insert_one(
TopicStore({
"name": "test_name",
"number": 1
}))
# Retrieve the inserted document
inserted_document = client.find_by_id(insert_result.inserted_id)
assert inserted_document.id == insert_result.inserted_id
# Update the document name and number fields
new_name = ''.join(random.sample('raymond', 7))
new_number = random.randint(0, 100)
update_result = client.update_one_by_id(inserted_document.id,
name=new_name,
number=new_number)
inserted_document_after_update = client.find_by_id(
insert_result.inserted_id)
assert inserted_document_after_update.id == insert_result.inserted_id
assert inserted_document_after_update.dict["number"] == new_number
assert inserted_document_after_update.dict["name"] == new_name
# Print all documents in the collection
cursor = client.find()
for x in cursor:
print(
"Doc:\n\t-As Structure: {}\n\t-As Dict: {}\n\t-As ROS Msgs: {}"
.format(str(x), x.dict, x.msgs))
# Or print using the same API as TopicStorage
for x in client:
print(
"Doc:\n\t-As Structure: {}\n\t-As Dict: {}\n\t-As ROS Msgs: {}"
.format(str(x), x.dict, x.msgs))
# Cleanup test by deleting document
delete_result = client.delete_by_id(insert_result.inserted_id)
def __iter__(self):
if not self.path.exists():
raise StopIteration()
with self.path.open("rb") as fh:
position = -1
while True:
try:
load_kwargs = {}
if sys.version_info[0] >= 3: # python3 loading
load_kwargs = dict(encoding="latin1")
position += 1
tree = pickle.load(fh, **load_kwargs)
yield TopicStore(data_tree=tree)
except EOFError:
break
except Exception as e:
print(
"File at position {} is corrupt. Skipping, error: {}".
format(position, e))
def test_large_document(self):
if not rospy.get_node_uri():
rospy.init_node("topic_store_tests")
client = MongoStorage(collection="python_tests", use_grid_fs=True)
# Insert >16MB document
random_array = np.random.random((3000, 3000, 3)).astype(np.float32)
message = ros_numpy.msgify(Image, random_array, encoding="32FC3")
# Insert image message
im_document = TopicStore({"image": message})
im_result = client.insert_one(im_document)
# Get image message and check data is the same
returned_im_document = client.find_by_id(im_result.inserted_id)
assert returned_im_document.id == im_result.inserted_id
retrieved_array = ros_numpy.numpify(returned_im_document.msgs["image"])
np.testing.assert_equal(random_array, retrieved_array)
# Delete image message
client.delete_by_id(im_result.inserted_id)
def find_one(self, query, *args, **kwargs):
"""Returns a matched TopicStore document"""
# TODO: remove FIND_ONE function in place of generic find function
skip_fetch_binary, skip_on_error, args, kwargs = self.__parse_find_args_kwargs(args, kwargs)
doc = None
try:
doc = self.collection.find_one(query, *args, **kwargs)
if not doc: # Return if doc not found
return doc
parsed_document = self.reverse_parser(doc)
if not skip_fetch_binary:
parsed_document = self.__ungridfs_ify(parsed_document)
return TopicStore(parsed_document)
except Exception as e:
if not skip_on_error:
raise
print("Skipping document '{}' because '{}'".format(
(doc.get('id') + " ") if doc else None, e.message)
)
return doc
def test_image_encoding(self):
if not rospy.get_node_uri():
rospy.init_node("topic_store_tests")
client = MongoStorage(collection="python_tests")
# Test ROS_Message Serialisation with common Image Types
numpy_types = {
"rgb8": (np.uint8, 3),
"rgba8": (np.uint8, 4),
"rgb16": (np.uint16, 3),
"rgba16": (np.uint16, 4),
"bgr8": (np.uint8, 3),
"bgra8": (np.uint8, 4),
"bgr16": (np.uint16, 3),
"bgra16": (np.uint16, 4),
"mono8": (np.uint8, 1),
"mono16": (np.uint16, 1),
"bayer_rggb8": (np.uint8, 1),
"bayer_bggr8": (np.uint8, 1),
"bayer_gbrg8": (np.uint8, 1),
"bayer_grbg8": (np.uint8, 1),
"bayer_rggb16": (np.uint16, 1),
"bayer_bggr16": (np.uint16, 1),
"bayer_gbrg16": (np.uint16, 1),
"bayer_grbg16": (np.uint16, 1),
"8UC1": (np.uint8, 1),
"8UC2": (np.uint8, 2),
"8UC3": (np.uint8, 3),
"8UC4": (np.uint8, 4),
"8SC1": (np.int8, 1),
"8SC2": (np.int8, 2),
"8SC3": (np.int8, 3),
"8SC4": (np.int8, 4),
"16UC1": (np.uint16, 1),
"16UC2": (np.uint16, 2),
"16UC3": (np.uint16, 3),
"16UC4": (np.uint16, 4),
"16SC1": (np.int16, 1),
"16SC2": (np.int16, 2),
"16SC3": (np.int16, 3),
"16SC4": (np.int16, 4),
"32SC1": (np.int32, 1),
"32SC2": (np.int32, 2),
"32SC3": (np.int32, 3),
"32SC4": (np.int32, 4),
"32FC1": (np.float32, 1),
"32FC2": (np.float32, 2),
"32FC3": (np.float32, 3),
"32FC4": (np.float32, 4),
"64FC1": (np.float64, 1),
"64FC2": (np.float64, 2),
"64FC3": (np.float64, 3),
"64FC4": (np.float64, 4)
}
for encoding, np_type in numpy_types.items():
size = (32, 32, np_type[1]) if np_type[1] != 1 else (32, 32)
random_array = np.random.random(size)
typed_array = (random_array * 255.0).astype(np_type[0])
message = ros_numpy.msgify(Image, typed_array, encoding=encoding)
# Insert image message
im_document = TopicStore({"image": message})
im_result = client.insert_one(im_document)
# Get image message and check data is the same
returned_im_document = client.find_by_id(im_result.inserted_id)
assert returned_im_document.id == im_result.inserted_id
retrieved_array = ros_numpy.numpify(
returned_im_document.msgs["image"])
np.testing.assert_equal(typed_array, retrieved_array)
# Delete image message
client.delete_by_id(im_result.inserted_id)
def next(self):
return TopicStore(self.parser(self.cursor.next()))
def get_message_tree(self):
"""TopicStore: Representation of the SubscriberTree topics snapshot"""
return TopicStore(self.__get_msg_tree(data_tree=self.tree))
def next(self):
document = self.parser(self.cursor.next())
if self.apply_fn:
document = self.apply_fn(document)
return TopicStore(document)
def __getitem__(self, item):
document = self.parser(self.cursor.__getitem__(item))
if self.apply_fn:
document = self.apply_fn(document)
return TopicStore(document)
def find_one(self, query, *args, **kwargs):
"""Returns a matched TopicStore document"""
doc = self.collection.find_one(query, *args, **kwargs)
parsed_document = self.reverse_parser(doc)
parsed_document = self.__ungridfs_ify(parsed_document)
return TopicStore(parsed_document)
def __getitem__(self, item):
return TopicStore(self.parser(self.cursor.__getitem__(item)))
def find_one(self, query, *args, **kwargs):
"""Returns a matched TopicStore document"""
return TopicStore(self.reverse_parser(self.collection.find_one(query, *args, **kwargs)))