def anti_instagram_annotations_test(): base = "${DUCKIETOWN_DATA}/phase3-misc-files/so1/" base = expand_environment(base) dirs = locate_files(base, "*.iids1", alsodirs=True) directory_results = {} overall_results = [] if not dirs: raise ValueError("No IIDS1 directories") for d in dirs: import getpass uname = getpass.getuser() out = os.path.join(os.path.dirname(d), uname, os.path.basename(d) + ".v") if not os.path.exists(out): os.makedirs(out) results = examine_dataset(d, out) overall_results = merge_comparison_results(results, overall_results) directory_results[d] = results db = shelve.open("tests_results", flag="n") db["directory_results"] = directory_results db["overall_results"] = overall_results db.close() print ("overall average error: %f" % (overall_results["total_error"] / overall_results["total_pixels"])) print ("overall regions checked: %f" % (overall_results["total_regions"])) for t in overall_results["v_vals"].keys(): print ( "region %f: RGB %f,%f,%f, HSV %f,%f,%f" % ( t, np.mean(overall_results["r_vals"][t]), np.mean(overall_results["g_vals"][t]), np.mean(overall_results["b_vals"][t]), np.mean(overall_results["h_vals"][t]), np.mean(overall_results["s_vals"][t]), np.mean(overall_results["v_vals"][t]), ) ) ipython_if_guy()
def search_all_configuration_files(): sources = [] # We look in $DUCKIETOWN_ROOT/catkin_ws/src sources.append(get_catkin_ws_src()) # then we look in $DUCKIETOWN_FLEET sources.append(get_duckiefleet_root()) configs = [] pattern = '*' + SUFFIX logger.info('Looking for %s files.' % pattern) for s in sources: fs = locate_files(s, pattern) logger.info('%4d files in %s' % (len(fs), s)) configs.extend(fs) # logger.debug('I found:\n' + "\n".join(configs)) return configs
def check(self): # find a try: python_files = locate_files(self.dirname, '*.py') for filename in python_files: try: check_no_half_merges(filename) if DuckietownConstants.enforce_no_tabs: check_no_tabs(filename) if DuckietownConstants.enforce_naming_conventions: check_good_name(filename) except CheckFailed as e: msg = 'Check failed for file %s:' % filename raise_wrapped(CheckFailed, e, msg, compact=True) except CheckFailed as e: msg = 'Checks failed for package %s.' % self.package_name l = str(e) raise CheckFailed(msg, l)
def anti_instagram_annotations_test(dirname, out_base): base = expand_all(dirname) if not os.path.exists(base): msg = 'Could not find directory %s' % base raise Exception(msg) dirs = locate_files(base, '*.iids1', alsodirs=True) directory_results = {} overall_results = [] if not dirs: raise ValueError('No IIDS1 directories found') for d in dirs: out = os.path.join(out_base, os.path.basename(d) + '.v') if not os.path.exists(out): os.makedirs(out) results = examine_dataset(d, out) overall_results = merge_comparison_results(results, overall_results) directory_results[d] = results db = shelve.open('tests_results', flag='n') db['directory_results'] = directory_results db['overall_results'] = overall_results db.close() logger.info( "overall average error: %f" % (overall_results['total_error'] / overall_results['total_pixels'])) logger.info("overall regions checked: %f" % (overall_results['total_regions'])) for t in overall_results['v_vals'].keys(): logger.info("region %f: RGB %f,%f,%f, HSV %f,%f,%f" % (t, np.mean(overall_results['r_vals'][t]), np.mean(overall_results['g_vals'][t]), np.mean(overall_results['b_vals'][t]), np.mean(overall_results['h_vals'][t]), np.mean(overall_results['s_vals'][t]), np.mean(overall_results['v_vals'][t])))
def anti_instagram_annotations_test(): base = "${DUCKIETOWN_DATA}/phase3-misc-files/so1/" base = expand_environment(base) dirs = locate_files(base, '*.iids1', alsodirs=True) directory_results = {} overall_results = [] if not dirs: raise ValueError('No IIDS1 directories') for d in dirs: import getpass uname = getpass.getuser() out = os.path.join(os.path.dirname(d), uname, os.path.basename(d) + '.v') if not os.path.exists(out): os.makedirs(out) results = examine_dataset(d, out) overall_results = merge_comparison_results(results, overall_results) directory_results[d] = results db = shelve.open('tests_results', flag='n') db['directory_results'] = directory_results db['overall_results'] = overall_results db.close() print("overall average error: %f" % (overall_results['total_error'] / overall_results['total_pixels'])) print("overall regions checked: %f" % (overall_results['total_regions'])) for t in overall_results['v_vals'].keys(): print("region %f: RGB %f,%f,%f, HSV %f,%f,%f" % (t, np.mean(overall_results['r_vals'][t]), np.mean(overall_results['g_vals'][t]), np.mean(overall_results['b_vals'][t]), np.mean(overall_results['h_vals'][t]), np.mean(overall_results['s_vals'][t]), np.mean(overall_results['v_vals'][t]))) ipython_if_guy()
def examine_dataset(dirname, out): logger.info(dirname) dirname = expand_environment(dirname) jpgs = locate_files(dirname, "*.jpg") mats = locate_files(dirname, "*.mat") logger.debug("I found %d jpgs and %d mats" % (len(jpgs), len(mats))) if len(jpgs) == 0: msg = "Not enough jpgs." raise ValueError(msg) # if len(mats) == 0: # msg = 'Not enough mats.' # raise ValueError(msg) first_jpg = sorted(jpgs)[0] logger.debug("Using jpg %r to learn transformation." % first_jpg) first_jpg_image = image_cv_from_jpg_fn(first_jpg) success, health, parameters = calculate_transform(first_jpg_image) s = "" s += "success: %s\n" % str(success) s += "health: %s\n" % str(health) s += "parameters: %s\n" % str(parameters) w = os.path.join(out, "learned_transform.txt") with open(w, "w") as f: f.write(s) logger.info(s) transform = ScaleAndShift(**parameters) config_dir = "${DUCKIETOWN_ROOT}/catkin_ws/src/duckietown/config/baseline/line_detector/line_detector_node/" config_dir = expand_environment(config_dir) configurations = locate_files(config_dir, "*.yaml") # logger.info('configurations: %r' % configurations) for j in jpgs: summaries = [] shape = (200, 160) interpolation = cv2.INTER_NEAREST bn = os.path.splitext(os.path.basename(j))[0] fn = os.path.join(out, "%s.all.png" % (bn)) if os.path.exists(fn): logger.debug("Skipping because file exists: %r" % fn) else: for c in configurations: logger.info("Trying %r" % c) name = os.path.splitext(os.path.basename(c))[0] if name in ["oreo", "myrtle", "bad_lighting", "226-night"]: continue with open(c) as f: stuff = yaml.load(f) if not "detector" in stuff: msg = 'Cannot find "detector" section in %r' % c raise ValueError(msg) detector = stuff["detector"] logger.info(detector) if not isinstance(detector, list) and len(detector) == 2: raise ValueError(detector) from duckietown_utils.instantiate_utils import instantiate def LineDetectorClass(): return instantiate(detector[0], detector[1]) s = run_detection( transform, j, out, shape=shape, interpolation=interpolation, name=name, LineDetectorClass=LineDetectorClass, ) summaries.append(s) together = make_images_grid(summaries, cols=1, pad=10, bgcolor=[0.5, 0.5, 0.5]) cv2.imwrite(fn, zoom_image(together, 4)) # ipython_if_guy() overall_results = [] comparison_results = {} for m in mats: logger.debug(m) jpg = os.path.splitext(m)[0] + ".jpg" if not os.path.exists(jpg): msg = "JPG %r for mat %r does not exist" % (jpg, m) logger.error(msg) else: frame_results = test_pair(transform, jpg, m, out) comparison_results[m] = frame_results overall_results = merge_comparison_results(comparison_results[m], overall_results) print "comparison_results[m]=frame_results" # ipython_if_guy() print "finished mats: " + dirname return overall_results
def examine_dataset(dirname, out): logger.info(dirname) dirname = expand_all(dirname) jpgs = locate_files(dirname, '*.jpg') mats = locate_files(dirname, '*.mat') logger.debug('I found %d JPGs and %d .mat.' % (len(jpgs), len(mats))) if len(jpgs) == 0: msg = 'Not JPGs found in %r.' % dirname raise ValueError(msg) # if len(mats) == 0: # msg = 'Not enough mats.' # raise ValueError(msg) first_jpg = sorted(jpgs)[0] logger.debug('Using jpg %r to learn transformation.' % first_jpg) first_jpg_image = image_cv_from_jpg_fn(first_jpg) success, health, parameters = calculate_transform(first_jpg_image) s = "" s += 'success: %s\n' % str(success) s += 'health: %s\n' % str(health) s += 'parameters: %s\n' % str(parameters) w = os.path.join(out, 'learned_transform.txt') with open(w, 'w') as f: f.write(s) logger.info(s) transform = ScaleAndShift(**parameters) duckietown_package_dir = get_ros_package_path('duckietown') config_dir = os.path.join( duckietown_package_dir, 'config/baseline/line_detector/line_detector_node') if not os.path.exists(config_dir): msg = 'Could not find configuration dir %s' % config_dir raise Exception(msg) config_dir = expand_all(config_dir) configurations = locate_files(config_dir, '*.yaml') if not configurations: msg = 'Could not find any configuration file in %s.' % config_dir raise Exception(msg) #logger.info('configurations: %r' % configurations) for j in jpgs: summaries = [] shape = (200, 160) interpolation = cv2.INTER_NEAREST bn = os.path.splitext(os.path.basename(j))[0] fn = os.path.join(out, '%s.all.png' % (bn)) if os.path.exists(fn): logger.debug('Skipping because file exists: %r' % fn) else: for c in configurations: logger.info('Trying %r' % c) name = os.path.splitext(os.path.basename(c))[0] if name in ['oreo', 'myrtle', 'bad_lighting', '226-night']: continue with open(c) as f: stuff = yaml.load(f) if not 'detector' in stuff: msg = 'Cannot find "detector" section in %r' % c raise ValueError(msg) detector = stuff['detector'] logger.info(detector) if not isinstance(detector, list) and len(detector) == 2: raise ValueError(detector) def LineDetectorClass(): return instantiate(detector[0], detector[1]) s = run_detection(transform, j, out, shape=shape, interpolation=interpolation, name=name, LineDetectorClass=LineDetectorClass) summaries.append(s) together = make_images_grid(summaries, cols=1, pad=10, bgcolor=[.5, .5, .5]) cv2.imwrite(fn, zoom_image(together, 4)) overall_results = [] comparison_results = {} for m in mats: logger.debug(m) jpg = os.path.splitext(m)[0] + '.jpg' if not os.path.exists(jpg): msg = 'JPG %r for mat %r does not exist' % (jpg, m) logger.error(msg) else: frame_results = test_pair(transform, jpg, m, out) comparison_results[m] = frame_results overall_results = merge_comparison_results(comparison_results[m], overall_results) print "comparison_results[m]=frame_results" print "finished mats: " + dirname return overall_results