def update_frame_error_result_metric_properties(): # Start by finding the image source ids start_time = time.time() logging.getLogger(__name__).info( "Updating 'metric_properties' for all FrameErrorResult objects ...") logging.getLogger(__name__).info("Loading set of referenced metric IDs...") metric_ids = FrameErrorResult._mongometa.collection.distinct('metric') # Autoload the image source type logging.getLogger(__name__).info( f"Autoloading metric types ({time.time() - start_time}s) ...") autoload_modules(Metric, ids=metric_ids) logging.getLogger(__name__).info( f"Found {len(metric_ids)} metrics, updating information " f"({time.time() - start_time}s) ...") n_updated = 0 for metric_id in metric_ids: metric = Metric.objects.get({'_id': metric_id}) metric_properties = metric.get_properties() n_updated += FrameErrorResult.objects.raw({ 'metric': metric_id }).update( { '$set': { 'metric_properties': { str(k): json_value(v) for k, v in metric_properties.items() } } }, upsert=False) logging.getLogger(__name__).info( f"Updated {n_updated} FrameErrorResult objects in {time.time() - start_time}s " )
def load_referenced_models(self) -> None: """ Load the result type so we can save the task :return: """ with no_auto_dereference(ImportDatasetTask): if isinstance(self.result, bson.ObjectId): # result is an id and not a model, autoload the model autoload_modules(ImageSource, [self.result])
def invalidate_systems_by_name(system_names: typing.Iterable[str]) -> int: system_names = list(system_names) autoload_modules(VisionSystem, classes=system_names) removed = VisionSystem.objects.raw({ '_cls': { '$in': system_names } }).delete() logging.getLogger(__name__).info("removed {0} systems".format(removed)) return removed
def print_available_experiments() -> None: """ Print out the list of valid experiment ids. These will be human-readable names :return: Nothing """ autoload_modules( Experiment ) # To actually get all the experiments, pymodm needs to know their types. print("Valid experiments are:") for obj in Experiment.objects.all().only('_id').values(): print(" {0}".format(obj['_id']))
def invalidate_trial_results( trial_result_ids: typing.Iterable[bson.ObjectId]) -> int: trial_result_ids = list(trial_result_ids) autoload_modules(TrialResult, trial_result_ids) removed = TrialResult.objects.raw({ '_id': { '$in': trial_result_ids } }).delete() logging.getLogger(__name__).info( "removed {0} trial results".format(removed)) return removed
def invalidate_metric_results( metric_result_ids: typing.Iterable[bson.ObjectId]) -> int: metric_result_ids = list(metric_result_ids) autoload_modules(MetricResult, metric_result_ids) removed = MetricResult.objects.raw({ '_id': { '$in': metric_result_ids } }).delete() logging.getLogger(__name__).info( "removed {0} metric results".format(removed)) return removed
def get_columns_for_plots( plot_ids: typing.Iterable[bson.ObjectId]) -> typing.Set[str]: """ For a set of plot ids, get the required columns for those plots. Handles auto-loading the plot types, and the plot objects themselves will go out of scope once this helper completes :param plot_ids: The list of plot ids to search for :return: A combined set of the required columns for all the given plots """ plot_ids = list(plot_ids) autoload_modules(Plot, ids=plot_ids) plots = Plot.objects.raw({'_id': {'$in': plot_ids}}) columns = set(column for plot in plots for column in plot.get_required_columns()) return columns
def get_result(self) -> typing.Union[ImageSource, None]: """ Actually get the result object. This will auto-load the result model, and then attempt to construct it. :return: """ with no_auto_dereference(ImportDatasetTask): if self.result is None: return None if isinstance(self.result, bson.ObjectId): # result is an id and not a model, autoload the model autoload_modules(ImageSource, [self.result]) # This will now dereference correctly return self.result
def update_frame_errors_system_properties(): # Start with the frame error result, which will tell us which trial results have been measured start_time = time.time() logging.getLogger(__name__).info( "Updating 'system_properties' for all FrameError objects ...") logging.getLogger(__name__).info("Getting set of trial result ids...") trial_result_ids = FrameErrorResult._mongometa.collection.distinct( 'trial_results') # Get the unique system ids logging.getLogger(__name__).info( f"Getting set of system ids ({time.time() - start_time})...") system_ids = TrialResult._mongometa.collection.distinct('system') # Autoload the types for the trial and systems logging.getLogger(__name__).info( f"Autoloading types ({time.time() - start_time})...") autoload_modules(VisionSystem, ids=system_ids) autoload_modules(TrialResult, ids=trial_result_ids) # Get the set of system ids used in those trials logging.getLogger(__name__).info( f"Updating information from {len(trial_result_ids)} trials " f"over {len(system_ids)} systems...") n_updated = 0 for system_id in system_ids: system = VisionSystem.objects.get({'_id': system_id}) for trial_result in TrialResult.objects.raw({ '_id': { '$in': trial_result_ids }, 'system': system_id }).only('_id', 'settings'): system_properties = system.get_properties(None, trial_result.settings) n_updated += FrameError.objects.raw({ 'trial_result': trial_result.pk }).update( { '$set': { 'system_properties': { str(k): json_value(v) for k, v in system_properties.items() } } }, upsert=False) # We definitely don't want to upsert logging.getLogger(__name__).info( f"Updated {n_updated} FrameError objects in {time.time() - start_time}s " )
def main(task_id: str, config_file: str = 'config.yml', mongodb_host: str = None, mongodb_port: int = None): """ Run a particular task. :args: Only argument is the id of the task to run :return: """ task_id = ObjectId(task_id) # Load the configuration config = load_global_config(config_file) if __name__ == '__main__': # Only configure the logging if this is the main function, don't reconfigure logging.config.dictConfig(config['logging']) # Configure the database and the image manager dbconn.configure(config['database'], override_host=mongodb_host, override_port=mongodb_port) im_manager.configure(config['image_manager']) # Set up the path manager path_manager = PathManager(paths=config['paths'], temp_folder=config['temp_folder'], output_dir=config.get('output_dir', None)) # Try and get the task object autoload_modules(Task, [task_id]) # Try and autoload the task subclass try: task = Task.objects.get({'_id': task_id}) except Exception as ex: logging.getLogger(__name__).critical( "Exception occurred while loading Task({0}):\n{1}".format( str(task_id), traceback.format_exc())) raise ex # Since we got the task, try and run it try: task.run_task(path_manager) except Exception as ex: logging.getLogger(__name__).critical( "Exception occurred while running {0}({1}):\n{2}".format( type(task).__name__, str(task_id), traceback.format_exc())) task.mark_job_failed() raise ex finally: task.save()
def load_referenced_modules(self): logging.getLogger(__name__).info("Loading referenced models...") # Load the metric model metric_id = None with no_auto_dereference(MeasureTrialTask): if isinstance(self.metric, bson.ObjectId): metric_id = self.metric if metric_id is not None: autoload_modules(Metric, [metric_id]) # Load the trial results models with no_auto_dereference(MeasureTrialTask): model_ids = list( set(tr_id for tr_id in self.trial_results if isinstance(tr_id, bson.ObjectId))) if len(model_ids) > 0: autoload_modules(TrialResult, model_ids)
def load_referenced_models(self): """ Go through the models referenced by this experiment and ensure their model types have been loaded. This is necessary or accessing any of the reference fields will cause an exception. Clean references will access the metric results and plots, so we want to ensure that those models are loaded. :return: """ # Load metric result models with no_auto_dereference(type(self)): model_ids = set(result_id for result_id in self.metric_results if isinstance(result_id, bson.ObjectId)) if len(model_ids) > 0: autoload_modules(MetricResult, ids=list(model_ids)) # Load plot models with no_auto_dereference(type(self)): model_ids = set(plot_id for plot_id in self.plots if isinstance(plot_id, bson.ObjectId)) if len(model_ids) > 0: autoload_modules(Plot, ids=list(model_ids))
def invalidate_image_collections( image_source_ids: typing.Iterable[bson.ObjectId]) -> int: """ Invalidate the data associated with a particular image source. This cascades to derived trial results, and from there to benchmark results. Also cleans out tasks. :param image_source_ids: The ids of the image sources to remove :return: """ # Just delete the image collections, reference fields should handle the rest image_source_ids = list(image_source_ids) autoload_modules(ImageSource, image_source_ids) removed = ImageSource.objects.raw({ '_id': { '$in': image_source_ids } }).delete() logging.getLogger(__name__).info( "removed {0} image sources".format(removed)) return removed
def load_referenced_models(self): """ Go through the models referenced by this experiment and ensure their model types have been loaded. This is necessary or accessing any of the reference fields will cause an exception :return: """ # Load system models with no_auto_dereference(type(self)): model_ids = set(sys_id for sys_id in self.systems if isinstance(sys_id, bson.ObjectId)) if len(model_ids) > 0: autoload_modules(VisionSystem, ids=list(model_ids)) # Load image source models with no_auto_dereference(type(self)): model_ids = set(source_id for source_id in self.image_sources if isinstance(source_id, bson.ObjectId)) if len(model_ids) > 0: autoload_modules(ImageSource, ids=list(model_ids)) # Load metric models with no_auto_dereference(type(self)): model_ids = set(metric_id for metric_id in self.metrics if isinstance(metric_id, bson.ObjectId)) if len(model_ids) > 0: autoload_modules(Metric, ids=list(model_ids)) # Load superclass models - which may reference the types we just loaded, so we do that first super(SimpleExperiment, self).load_referenced_models()
def update_frame_error_result_image_source_properties(): # Start by finding the image source ids start_time = time.time() logging.getLogger(__name__).info( "Updating 'image_source_properties' for all FrameErrorResult objects ..." ) logging.getLogger(__name__).info( "Loading set of referenced image source IDs...") image_source_ids = FrameErrorResult._mongometa.collection.distinct( 'image_source') # Autoload the image source type logging.getLogger(__name__).info( f"Autoloading image source types ({time.time() - start_time}s) ...") autoload_modules(ImageSource, ids=image_source_ids) logging.getLogger(__name__).info( f"Found {len(image_source_ids)} image sources, updating information " f"({time.time() - start_time}s) ...") n_updated = 0 for image_source_id in image_source_ids: image_source = ImageSource.objects.get({'_id': image_source_id}) image_source_properties = image_source.get_properties() n_updated += FrameErrorResult.objects.raw({ 'image_source': image_source_id }).update( { '$set': { 'image_source_properties': { str(k): json_value(v) for k, v in image_source_properties.items() } } }, upsert=False) logging.getLogger(__name__).info( f"Updated {n_updated} FrameErrorResult objects in {time.time() - start_time}s " )
def load_referenced_models(self) -> None: """ Load the metric, trial, and result types so we can save the task :return: """ with no_auto_dereference(MeasureTrialTask): if isinstance(self.metric, bson.ObjectId): # The metric is just an ID, we will need the model to autoload_modules(Metric, [self.metric]) trials_to_load = [ trial_id for trial_id in self.trial_results if isinstance(trial_id, bson.ObjectId) ] if len(trials_to_load) > 0: autoload_modules(TrialResult, trials_to_load) if isinstance(self.result, bson.ObjectId): # result is an id and not a model, autoload the model autoload_modules(MetricResult, [self.result])
def export_plot_data(experiment_name: str, cache_folder: Path) -> None: """ Export the plot data for a particular experiment to the given folder :param experiment_name: The unique name of the experiment :param cache_folder: The folder to store the data in :return: """ # Autoload the experiment type autoload_modules(Experiment, ids=[experiment_name]) # Get the result ids from the experiment, bypassing the experiment object object_partials = list( Experiment.objects.raw({ '_id': experiment_name }).only('metric_results', 'plots').values()) metric_result_ids = [ result_id for experiment_doc in object_partials for result_id in experiment_doc.get('metric_results', []) ] plot_ids = [ plot_id for experiment_doc in object_partials for plot_id in experiment_doc.get('plots', []) ] if len(metric_result_ids) <= 0: logging.getLogger(__name__).info( f"No results for {experiment_name}, maybe re-run update_experiments?" ) return logging.getLogger(__name__).info( f"Caching plot data for {experiment_name} in {cache_folder} ...") # Get the columns columns = get_columns_for_plots(plot_ids) if len(columns) <= 0: # No columns means no data and no plots logging.getLogger(__name__).info( f"No data to cache for {experiment_name}, are there any plots?" ) return # Load the metric result types autoload_modules(MetricResult, ids=metric_result_ids) # Read all the results logging.getLogger(__name__).info( f"Accumulating columns [{columns}] over {len(metric_result_ids)} results..." ) data = [] for result_id in metric_result_ids: try: metric_result = MetricResult.objects.get({'_id': result_id}) except MetricResult.DoesNotExist: continue if metric_result is not None: data.extend(metric_result.get_results(columns)) del metric_result data = DataFrame(data) logging.getLogger(__name__).info(f"Collected {len(data)} rows.") # Save the data to file cache_file = Experiment.get_cache_file(cache_folder, experiment_name) if not cache_file.parent.exists(): cache_file.parent.mkdir(parents=True) logging.getLogger(__name__).info(f"Saving data to {cache_file}") data.to_pickle(str(cache_file))
def invalidate_systems(system_ids: typing.Iterable[bson.ObjectId]) -> int: system_ids = list(system_ids) autoload_modules(VisionSystem, system_ids) removed = VisionSystem.objects.raw({'_id': {'$in': system_ids}}).delete() logging.getLogger(__name__).info("removed {0} systems".format(removed)) return removed
def measure_results( self, trial_results: typing.Iterable[TrialResult]) -> FrameErrorResult: """ Collect the errors TODO: Track the error introduced by a loop closure, somehow. Might need to track loop closures in the FrameResult :param trial_results: The results of several trials to aggregate :return: :rtype BenchmarkResult: """ trial_results = list(trial_results) # preload model types for the models linked to the trial results. with no_auto_dereference(SLAMTrialResult): model_ids = set(tr.system for tr in trial_results if isinstance(tr.system, bson.ObjectId)) autoload_modules(VisionSystem, list(model_ids)) model_ids = set(tr.image_source for tr in trial_results if isinstance(tr.image_source, bson.ObjectId)) autoload_modules(ImageSource, list(model_ids)) # Check if the set of trial results is valid. Loads the models. invalid_reason = check_trial_collection(trial_results) if invalid_reason is not None: return MetricResult(metric=self, trial_results=trial_results, success=False, message=invalid_reason) # Make sure we have a non-zero number of trials to measure if len(trial_results) <= 0: return MetricResult(metric=self, trial_results=trial_results, success=False, message="Cannot measure zero trials.") # Ensure the trials all have the same number of results for repeat, trial_result in enumerate(trial_results[1:]): if len(trial_result.results) != len(trial_results[0].results): return MetricResult( metric=self, trial_results=trial_results, success=False, message= f"Repeat {repeat + 1} has a different number of frames " f"({len(trial_result.results)} != {len(trial_results[0].results)})" ) # Load the system, it must be the same for all trials (see check_trial_collection) system = trial_results[0].system # Pre-load the image objects in a batch, to avoid loading them piecemeal later images = [image for _, image in trial_results[0].image_source] # Build mappings between frame result timestamps and poses for each trial timestamps_to_pose = [{ frame_result.timestamp: frame_result.pose for frame_result in trial_result.results } for trial_result in trial_results] # Choose transforms between each trajectory and the ground truth estimate_origins_and_scales = [ robust_align_trajectory_to_ground_truth( [ frame_result.estimated_pose for frame_result in trial_result.results if frame_result.estimated_pose is not None ], [ frame_result.pose for frame_result in trial_result.results if frame_result.estimated_pose is not None ], compute_scale=not bool(trial_result.has_scale), use_symmetric_scale=True) for trial_result in trial_results ] motion_scales = [1.0] * len(trial_results) for idx in range(len(trial_results)): if not trial_results[idx].has_scale: motion_scales[idx] = robust_compute_motions_scale( [ frame_result.estimated_motion for frame_result in trial_results[idx].results if frame_result.estimated_motion is not None ], [ frame_result.motion for frame_result in trial_results[idx].results if frame_result.estimated_motion is not None ], ) # Then, tally all the errors for all the computed trajectories estimate_errors = [[] for _ in range(len(trial_results))] image_columns = set() distances_lost = [[] for _ in range(len(trial_results))] times_lost = [[] for _ in range(len(trial_results))] frames_lost = [[] for _ in range(len(trial_results))] distances_found = [[] for _ in range(len(trial_results))] times_found = [[] for _ in range(len(trial_results))] frames_found = [[] for _ in range(len(trial_results))] is_tracking = [False for _ in range(len(trial_results))] tracking_frames = [0 for _ in range(len(trial_results))] tracking_distances = [0 for _ in range(len(trial_results))] prev_tracking_time = [0 for _ in range(len(trial_results))] current_tracking_time = [0 for _ in range(len(trial_results))] for frame_idx, frame_results in enumerate( zip(*(trial_result.results for trial_result in trial_results))): # Get the estimated motions and absolute poses for each trial, # And convert them to the ground truth coordinate frame using # the scale, translation and rotation we chose scaled_motions = [ tf.Transform( location=frame_results[idx].estimated_motion.location * motion_scales[idx], rotation=frame_results[idx].estimated_motion.rotation_quat( True), w_first=True) if frame_results[idx].estimated_motion is not None else None for idx in range(len(frame_results)) ] scaled_poses = [ align_point(pose=frame_results[idx].estimated_pose, shift=estimate_origins_and_scales[idx][0], rotation=estimate_origins_and_scales[idx][1], scale=estimate_origins_and_scales[idx][2]) if frame_results[idx].estimated_pose is not None else None for idx in range(len(frame_results)) ] # Find the average estimated motion for this frame across all the different trials # The average is not available for frames with only a single estimate non_null_motions = [ motion for motion in scaled_motions if motion is not None ] if len(non_null_motions) > 1: average_motion = tf.compute_average_pose(non_null_motions) else: average_motion = None # Union the image columns for all the images for all the frame results image_columns |= set( column for frame_result in frame_results for column in frame_result.image.get_columns()) for repeat_idx, frame_result in enumerate(frame_results): # Record how long the current tracking state has persisted if frame_idx <= 0: # Cannot change to or from tracking on the first frame is_tracking[repeat_idx] = (frame_result.tracking_state is TrackingState.OK) prev_tracking_time[repeat_idx] = frame_result.timestamp elif is_tracking[ repeat_idx] and frame_result.tracking_state is not TrackingState.OK: # This trial has become lost, add to the list and reset the counters frames_found[repeat_idx].append( tracking_frames[repeat_idx]) distances_found[repeat_idx].append( tracking_distances[repeat_idx]) times_found[repeat_idx].append( current_tracking_time[repeat_idx] - prev_tracking_time[repeat_idx]) tracking_frames[repeat_idx] = 0 tracking_distances[repeat_idx] = 0 prev_tracking_time[repeat_idx] = current_tracking_time[ repeat_idx] is_tracking[repeat_idx] = False elif not is_tracking[ repeat_idx] and frame_result.tracking_state is TrackingState.OK: # This trial has started to track, record how long it was lost for frames_lost[repeat_idx].append(tracking_frames[repeat_idx]) distances_lost[repeat_idx].append( tracking_distances[repeat_idx]) times_lost[repeat_idx].append( current_tracking_time[repeat_idx] - prev_tracking_time[repeat_idx]) tracking_frames[repeat_idx] = 0 tracking_distances[repeat_idx] = 0 prev_tracking_time[repeat_idx] = current_tracking_time[ repeat_idx] is_tracking[repeat_idx] = True # Update the current tracking information tracking_frames[repeat_idx] += 1 tracking_distances[repeat_idx] += np.linalg.norm( frame_result.motion.location) current_tracking_time[repeat_idx] = frame_result.timestamp # Turn loop closures into distances. We don't need to worry about origins because everything is GT frame if len(frame_result.loop_edges) > 0: loop_distances, loop_angles = compute_loop_distances_and_angles( frame_result.pose, ( timestamps_to_pose[repeat_idx][timestamp] for timestamp in frame_result.loop_edges if timestamp in timestamps_to_pose[ repeat_idx] # they should all be in there, but for safety, check )) else: loop_distances, loop_angles = [], [] # Build the frame error frame_error = make_frame_error( trial_result=trial_results[repeat_idx], frame_result=frame_result, image=images[frame_idx], system=system, repeat_index=repeat_idx, loop_distances=loop_distances, loop_angles=loop_angles, # Compute the error in the absolute estimated pose (if available) absolute_error=make_pose_error( scaled_poses[repeat_idx], # The frame_result.pose) if scaled_poses[repeat_idx] is not None else None, # Compute the error of the motion relative to the true motion relative_error=make_pose_error(scaled_motions[repeat_idx], frame_result.motion) if scaled_motions[repeat_idx] is not None else None, # Compute the error between the motion and the average estimated motion noise=make_pose_error(scaled_motions[repeat_idx], average_motion) if scaled_motions[repeat_idx] is not None and average_motion is not None else None, systemic_error=make_pose_error(average_motion, frame_result.motion) if average_motion is not None else None) estimate_errors[repeat_idx].append(frame_error) # Add any accumulated tracking information left over at the end if len(trial_results[0].results) > 0: for repeat_idx, tracking in enumerate(is_tracking): if tracking: frames_found[repeat_idx].append( tracking_frames[repeat_idx]) distances_found[repeat_idx].append( tracking_distances[repeat_idx]) times_found[repeat_idx].append( current_tracking_time[repeat_idx] - prev_tracking_time[repeat_idx]) else: frames_lost[repeat_idx].append(tracking_frames[repeat_idx]) distances_lost[repeat_idx].append( tracking_distances[repeat_idx]) times_lost[repeat_idx].append( current_tracking_time[repeat_idx] - prev_tracking_time[repeat_idx]) # Once we've tallied all the results, either succeed or fail based on the number of results. if len(estimate_errors) <= 0 or any( len(trial_errors) <= 0 for trial_errors in estimate_errors): return FrameErrorResult( metric=self, trial_results=trial_results, success=False, message="No measurable errors for these trajectories") return make_frame_error_result( metric=self, trial_results=trial_results, errors=[ TrialErrors(frame_errors=estimate_errors[repeat], frames_lost=frames_lost[repeat], frames_found=frames_found[repeat], times_lost=times_lost[repeat], times_found=times_found[repeat], distances_lost=distances_lost[repeat], distances_found=distances_found[repeat]) for repeat, trial_result in enumerate(trial_results) ])
def invalidate_failed_metric_results() -> int: autoload_modules(MetricResult) removed = MetricResult.objects.raw({'success': False}).delete() logging.getLogger(__name__).info( "removed {0} metric results".format(removed)) return removed