def perform_test(self, capsule: BaseCapsule,
worker_pool: CapsuleThreadPool, num_samples: int) \
-> datetime.timedelta:
# Warm things up, such as getting model on GPU if capsule uses it
warmup_results = worker_pool.map(
lambda kwargs: capsule.process_frame(**kwargs),
[self.generate_input_kwargs(capsule) for _ in range(50)]
)
for _ in warmup_results:
pass
# Generate test args before starting the test, so that the
# benchmark is purely just for the capsule
test_inputs = [self.generate_input_kwargs(capsule)
for _ in range(num_samples)]
# Begin the benchmark
start_time = datetime.datetime.now()
results = worker_pool.map(
lambda kwargs: capsule.process_frame(**kwargs),
test_inputs)
for _ in results:
pass
end_time = datetime.datetime.now()
duration = end_time - start_time
return duration
def _run_inference_on_images(images: List[np.ndarray], capsule: BaseCapsule):
"""Run inference on a list of images"""
# Make multiple parallel requests with different images and different
# input DetectionNodes to the capsule.
request_input = []
with ThreadPoolExecutor(
thread_name_prefix="InputOutputValidation ") \
as executor:
for stream_id, image in enumerate(images):
if capsule.input_type.size == NodeDescription.Size.NONE:
input_node = None
elif capsule.input_type.size == NodeDescription.Size.SINGLE:
input_node = make_detection_node(image.shape,
capsule.input_type)
elif capsule.input_type.size == NodeDescription.Size.ALL:
input_node = [make_detection_node(image.shape,
capsule.input_type)
for _ in range(random.randint(0, 5))]
else:
raise NotImplementedError(
"The capsule did not have a NodeDescription.Size that was "
"known!")
options = make_capsule_options(capsule)
future = executor.submit(
capsule.process_frame,
frame=image,
detection_node=input_node,
options=options,
state=capsule.stream_state())
request_input.append((future, input_node))
for future, input_node in request_input:
# Postprocess the results
prediction = future.result(timeout=90)
# Verify that the capsule performed correctly
if isinstance(prediction, DetectionNode):
# Validate that what was returned by the capsule was valid
assert (capsule.output_type.size
is NodeDescription.Size.SINGLE)
output_nodes = [prediction]
elif prediction is None and isinstance(input_node, DetectionNode):
# If the capsule didn't output something, then it must have
# modified the input node in-place. Validate the changes.
assert (capsule.output_type.size
is NodeDescription.Size.SINGLE)
output_nodes = [input_node]
elif prediction is None and isinstance(input_node, list):
# If this capsule accepts size ALL as input, then it must
# have modified the detections within the input list
assert capsule.input_type.size is NodeDescription.Size.ALL
assert capsule.output_type.size is NodeDescription.Size.ALL
output_nodes = input_node
elif isinstance(prediction, list):
# Validate that every detection node in the list is correct
assert capsule.output_type.size is NodeDescription.Size.ALL
output_nodes = prediction
else:
raise RuntimeError(f"Unknown prediction type: {prediction}")
# Validate every output node against the capsules output_type
for output_node in output_nodes:
assert capsule.output_type.describes(output_node), \
("Capsule failed to output a prediction that matches "
"the NodeDescription it had for it's output type. "
f"Prediction: {prediction}")
# Assert the nodes "extra_data" attribute can be JSON encoded
# without errors. Typically this can happen if there's a numpy
# array carelessly left in the extra_data
json.loads(json.dumps(output_node.extra_data))
# If this capsule can encode things, verify that the backend
# correctly implemented the "distance" function
if (capsule.capability.encoded
and prediction is not None
and len(prediction) > 0):
# Get one of the predictions
pred = prediction[0]
# Measure the distance from an encoding to itself
# (should be 0)
distances = capsule.backends[0].distances(
pred.encoding, np.array([pred.encoding]))
assert len(distances) == 1
assert distances[0] == 0, \
("This assertion can be removed in the case that "
"there is some new distance function where two "
"encodings that are equal no longer have a distance "
"of 0. Until that case exists, keep this assertion.")
def _validate_capsule(capsule: BaseCapsule):
"""This will try calling different attributes on a capsule to make sure
they are there. This is based on the API compatibility version.
:raises: InvalidCapsuleError
"""
def check_arg_names(func: Callable,
correct: List[str],
ignore: Optional[List[str]] = None) \
-> bool:
"""Return False if a function has the wrong argument names. Return
true if they are correct.
Usage:
>>> def my_func(self, frame, detection_node):
... pass
>>> check_arg_names(my_func, ['self'], ['frame', 'detection_node'])
True
"""
# noinspection PyUnresolvedReferences
code = func.__code__
ignore = [] if ignore is None else ignore
all_var_names = code.co_varnames
arg_names = all_var_names[:code.co_argcount]
filtered = [n for n in arg_names if n not in ignore]
return filtered == correct
try:
# If the ASCII flag is used, only [a-zA-Z0-9_] is matched for \w.
# Otherwise, some other Unicode characters can be matched, depending on
# the locale. We don't want that.
if re.fullmatch(r"\w+", capsule.name, flags=re.ASCII) is None:
raise InvalidCapsuleError(
"Capsule names must only contain alphanumeric characters and "
"underscores")
# Validate the capsule class attributes
capsule_assertions = [
isinstance(capsule.name, str),
callable(capsule.backend_loader),
isinstance(capsule.version, int),
isinstance(capsule.input_type, NodeDescription),
isinstance(capsule.output_type, NodeDescription),
isinstance(capsule.options, dict)
]
if not all(capsule_assertions):
raise InvalidCapsuleError(
f"The capsule has an invalid internal configuration!\n" +
f"Capsule Assertions: {capsule_assertions}")
# Make sure that certain things are NOT attributes (we don't want
# accidental leftover code from previous capsule versions)
unwanted_attributes = ["backend_config"]
for unwanted_attr in unwanted_attributes:
try:
# This should throw an attribute error
capsule.__getattribute__(unwanted_attr)
raise InvalidCapsuleError(
f"The capsule has leftover attributes from a previous "
f"OpenVisionCapsules API version. Attribute name: "
f"{unwanted_attr}")
except AttributeError:
pass
# Validate the capsule's backend_loader takes the right args
loader = capsule.backend_loader
loader_assertions = [
callable(loader),
check_arg_names(func=loader, correct=["capsule_files", "device"])
]
if not all(loader_assertions):
raise InvalidCapsuleError(
f"The capsule's backend_loader has an invalid configuration!\n"
f"Loader Assertions: {loader_assertions}")
# Validate the backend class attributes
if capsule.backends is not None:
backend = capsule.backends[0]
backend_assertions = [
callable(backend.batch_predict),
callable(backend.process_frame),
callable(backend.close),
isinstance(capsule.backends[0], BaseBackend)
]
if not all(backend_assertions):
raise InvalidCapsuleError(
f"The capsule's backend has an invalid configuration!\n"
f"Backend Assertions: {backend_assertions}")
# Validate the stream state
stream_state = capsule.stream_state
stream_state_assertions = [
(stream_state is BaseStreamState
or BaseStreamState in stream_state.__bases__)
]
if not all(stream_state_assertions):
raise InvalidCapsuleError(
"The capsule's stream_state has an invalid configuration!\n"
f"Stream State Assertions: {stream_state_assertions}")
# Validate that if the capsule is an encoder, it has a threshold option
if capsule.capability.encoded:
if "recognition_threshold" not in capsule.options.keys():
raise InvalidCapsuleError(
"This capsule can encode, but doesn't have a option "
"called \"recognition_threshold\"!")
except InvalidCapsuleError:
# Don't catch this exception type in the "except Exception" below
raise
except AttributeError as e:
message = f"The capsule did not describe the necessary attributes. " \
f"Error: {e}"
raise InvalidCapsuleError(message)
except Exception as e:
message = f"This capsule is invalid for unknown reasons. Error: {e}"
raise InvalidCapsuleError(message)
def _run_inference_on_images(images: List[np.ndarray], capsule: BaseCapsule):
"""Run inference on a list of images"""
# Make multiple parallel requests with different images and different
# input DetectionNodes to the capsule.
request_input = []
with ThreadPoolExecutor(
thread_name_prefix="InputOutputValidation ") \
as executor:
for stream_id, image in enumerate(images):
if capsule.input_type.size == NodeDescription.Size.NONE:
node = None
elif capsule.input_type.size == NodeDescription.Size.SINGLE:
node = make_detection_node(image.shape,
capsule.input_type)
elif capsule.input_type.size == NodeDescription.Size.ALL:
node = [make_detection_node(image.shape,
capsule.input_type)
for _ in range(random.randint(0, 5))]
else:
raise NotImplementedError(
"The capsule did not have a NodeDescription.Size that was "
"known!")
options = make_capsule_options(capsule)
future = executor.submit(
capsule.process_frame,
frame=image,
detection_node=node,
options=options,
state=capsule.stream_state())
request_input.append((future, node))
for future, node in request_input:
# Postprocess the results
prediction = future.result(timeout=90)
# Verify that the capsule performed correctly
if node is not None and not isinstance(node, list):
assert capsule.output_type.describes(node)
elif prediction is not None:
assert all(capsule.output_type.describes(detection_node)
for detection_node in prediction), \
("Capsule failed to output a prediction that matches "
"the NodeDescription it had for it's output type. "
f"Prediction: {prediction}")
# If this capsule can encode things, verify that the backend
# correctly implemented the "distance" function
if (capsule.capability.encoded
and prediction is not None
and len(prediction) > 0):
# Get one of the predictions
pred = prediction[0]
# Measure the distance from an encoding to itself
# (should be 0)
distances = capsule.backends[0].distances(
pred.encoding, np.array([pred.encoding]))
assert len(distances) == 1
assert distances[0] == 0, \
("This assertion can be removed in the case that "
"there is some new distance function where two "
"encodings that are equal no longer have a distance "
"of 0. Until that case exists, keep this assertion.")