def test_image(runs_dirpath, dataset_raw_dirpath, image_info, batch_size, ds_fac_list, run_name_list, model_disp_max_abs_value, output_dir, output_shapefiles): # --- Load data --- # # CHANGE the arguments of the load_gt_data() function if using your own and it does not take the same arguments: ori_image, ori_metadata, gt_polygons = read_bradbury_buildings.load_gt_data(dataset_raw_dirpath, image_info["city"], image_info["number"]) if INPUT_POLYGONS_FILENAME_EXTENSION is not None: gt_polygons = read_bradbury_buildings.load_polygons(dataset_raw_dirpath, image_info["city"], image_info["number"], INPUT_POLYGONS_FILENAME_EXTENSION) else: gt_polygons = gt_polygons if gt_polygons is not None: # CHANGE the arguments of the IMAGE_NAME_FORMAT format string if using your own and it does not take the same arguments: image_name = read_bradbury_buildings.IMAGE_NAME_FORMAT.format(city=image_info["city"], number=image_info["number"]) print_utils.print_info("Processing image {}".format(image_name)) aligned_gt_polygons = test.test_align_gt(runs_dirpath, ori_image, ori_metadata, gt_polygons, batch_size, ds_fac_list, run_name_list, model_disp_max_abs_value, output_dir, image_name, output_shapefiles=output_shapefiles) # Save aligned_gt_polygons in dataset dir: aligned_gt_polygons_filepath = read_bradbury_buildings.get_polygons_filepath(dataset_raw_dirpath, image_info["city"], image_info["number"], ALIGNED_GT_POLYGONS_FILENAME_EXTENSION) os.makedirs(os.path.dirname(aligned_gt_polygons_filepath), exist_ok=True) np.save(aligned_gt_polygons_filepath, aligned_gt_polygons)
def generate_test(config, params, split_name="train", seed=None, sobol_generator=None): # Find data_dir data_dirpath = python_utils.choose_first_existing_path( config["data_dir_candidates"]) if data_dirpath is None: print_utils.print_error("ERROR: Data directory not found!") exit() data_dirpath = os.path.expanduser(data_dirpath) print_utils.print_info("Using data from {}".format(data_dirpath)) root_dir = os.path.join(data_dirpath, config["data_root_partial_dirpath"]) alpha, x, density, gt, noise, curvature = generate_data( root_dir, params, split_name=split_name, seed=seed, sobol_generator=sobol_generator) noisy_gt = gt + noise import matplotlib.pyplot as plt f = plt.figure() f.set_tight_layout({"pad": .0}) ax = f.gca() # plt.scatter(alpha, noisy_gt, s=10) ax.plot(alpha, noisy_gt) ax.set_xlabel("alpha") ax.set_ylabel("y") # plt.title("Sinusoid, freq = {}".format(params["f"])) plt.show()
def main(): # --- Process args --- # args = get_args() config = run_utils.load_config(args.config) if config is None: print_utils.print_error( "ERROR: cannot continue without a config file. Exiting now...") exit() if args.batch_size is not None: config["batch_size"] = args.batch_size distribution = "uniform" dataset_params = { "n": args.sample_count, "f": args.frequency, "s": args.noise_std, "d": distribution, } # Find data_dir data_dirpath = python_utils.choose_first_existing_path( config["data_dir_candidates"]) if data_dirpath is None: print_utils.print_error("ERROR: Data directory not found!") exit() data_dirpath = os.path.expanduser(data_dirpath) print_utils.print_info("Using data from {}".format(data_dirpath)) root_dir = os.path.join(data_dirpath, config["data_root_partial_dirpath"]) sobol_generator = rand_utils.SobolGenerator() train_ds = Synthetic1DDataset(root_dir=root_dir, params=dataset_params, split_name="train", sobol_generator=sobol_generator, transform=torchvision.transforms.Compose([ transforms.ToTensor(), transforms.ToDevice(device="cuda") ])) train_dl = DataLoader(train_ds, batch_size=config["batch_size"], shuffle=True, num_workers=4) for i_batch, sample_batched in enumerate(train_dl): print( i_batch, sample_batched['density'].max(), # sample_batched['gt'], # sample_batched['noise'], )
def get_termlangs(repositories, all_languages): print_info("Searching for manifests ..") termlangs = {} for m in find_files(repositories, ".manifest"): print_info("Found Manifest:" +str(m)) try: term = parse_manifest(m) if term.language not in termlangs: termlangs[term.language] = [] termlangs[term.language].append(term) except Exception as e: import traceback traceback.print_exc() print_error("Failed" + e) return termlangs
def get_osm_annotations(filepath): filename_no_extension = os.path.splitext(filepath)[0] npy_filepath = filename_no_extension + ".npy" if os.path.exists(npy_filepath): print_utils.print_info("Loading OSM building data from disc...") gt_polygons = np.load(npy_filepath, allow_pickle=True) else: print_utils.print_info( "Fetching OSM building data from the internet...") gt_polygons = geo_utils.get_polygons_from_osm(filepath, tag="building") # Save npy to avoid re-fetching: np.save(npy_filepath, gt_polygons) # Save shapefile for visualisation: shp_filepath = filename_no_extension + ".shp" geo_utils.save_shapefile_from_polygons(gt_polygons, filepath, shp_filepath) return gt_polygons
def validate(self, sess, dataset_tensors, merged_summaries, summaries_writer, summary_index, plot=False): val_image, \ val_gt_polygons, \ val_disp_polygons, \ val_gt_polygon_map, \ val_gt_disp_field_map, \ val_disp_polygon_map = dataset_tensors val_image_batch, val_gt_polygons_batch, val_disp_polygons_batch, val_gt_polygon_map_batch, val_gt_disp_field_map_batch, val_disp_polygon_map_batch = sess.run( [val_image, val_gt_polygons, val_disp_polygons, val_gt_polygon_map, val_gt_disp_field_map, val_disp_polygon_map]) feed_dict = { self.input_image: val_image_batch, self.input_disp_polygon_map: val_disp_polygon_map_batch, self.gt_disp_field_map: val_gt_disp_field_map_batch, self.gt_seg: val_gt_polygon_map_batch, self.gt_polygons: val_gt_polygons_batch, self.disp_polygons: val_disp_polygons_batch, self.keep_prob: 1.0 } input_list = [merged_summaries, self.total_loss] if self.add_disp_output: input_list.append(self.level_0_disp_pred) if self.add_seg_output: input_list.append(self.level_0_seg_pred) output_list = sess.run(input_list, feed_dict=feed_dict) extra_output_count = self.add_disp_output + self.add_seg_output val_summary, val_loss, = output_list[:-extra_output_count] val_pred_disp_field_map_batch = val_pred_seg_batch = None if self.add_disp_output: index = -extra_output_count val_pred_disp_field_map_batch = output_list[index] if self.add_seg_output: index = -extra_output_count + self.add_disp_output val_pred_seg_batch = output_list[index] summaries_writer.add_summary(val_summary, summary_index) print_utils.print_info("step {}, validation loss = {}".format(summary_index, val_loss)) # print("\t validation threshold accuracies = {}".format(val_threshold_accuracies)) return val_image_batch, val_gt_polygons_batch, val_disp_polygons_batch, val_gt_polygon_map_batch, val_gt_disp_field_map_batch, val_disp_polygon_map_batch, val_pred_disp_field_map_batch, val_pred_seg_batch
def build_from_resource(resource, term, term_dir, language, theme): try: print_info("Project building..\t" + str(resource.filename)) except: print_error("Project failed due to filename encoding: " + str(resource.filename)) return None project = Project.parse_project_meta(resource) project_dir = os.path.join(term_dir,"%.02d"%(project.number)) makedirs(project_dir) try: built_project = Project.build_project(term, project, language, theme, project_dir) except: print_error("Project failed while building: " + str(resource.filename)) return None print_info("Project done!\t\t" + str(resource.filename)) return built_project
def restore_checkpoint(self, sess, saver, checkpoints_dir): """ :param sess: :param saver: :param checkpoints_dir: :return: True if a checkpoint was found and restored, False if no checkpoint was found """ checkpoint = tf.train.get_checkpoint_state(checkpoints_dir) if checkpoint and checkpoint.model_checkpoint_path: # Check if the model has a checkpoint print_utils.print_info("Restoring {} checkpoint {}".format( self.model_name, checkpoint.model_checkpoint_path)) try: saver.restore(sess, checkpoint.model_checkpoint_path) except tf.errors.InvalidArgumentError: print_utils.print_error( "ERROR: could not load checkpoint.\n" "\tThis is likely due to: .\n" "\t\t - the model graph definition has changed from the checkpoint thus weights do not match\n" .format(checkpoints_dir)) exit() return True else: return False
def train(self, sess, dataset_tensors, dropout_keep_prob, with_summaries=False, merged_summaries=None, summaries_writer=None, summary_index=None, plot=False): """ :param sess: :param with_summaries: (Default: False) :param merged_summaries: Must be not None if with_summaries is True :param summaries_writer: Must be not None if with_summaries is True :return: """ if with_summaries: assert merged_summaries is not None and summaries_writer is not None, \ "merged_summaries and writer should be specified if with_summaries is True" train_image, \ _, \ _, \ train_gt_polygon_map, \ train_gt_disp_field_map, \ train_disp_polygon_map = dataset_tensors train_image_batch, train_gt_polygon_map_batch, train_gt_disp_field_map_batch, train_disp_polygon_map_batch = sess.run( [ train_image, train_gt_polygon_map, train_gt_disp_field_map, train_disp_polygon_map ]) feed_dict = { self.input_image: train_image_batch, self.input_disp_polygon_map: train_disp_polygon_map_batch, self.gt_disp_field_map: train_gt_disp_field_map_batch, self.gt_seg: train_gt_polygon_map_batch, self.gt_polygons: tf_utils.create_array_to_feed_placeholder(self.gt_polygons), self.disp_polygons: tf_utils.create_array_to_feed_placeholder(self.disp_polygons), self.keep_prob: dropout_keep_prob, } if with_summaries: if summary_index == 0: run_options = tf.RunOptions( trace_level=tf.RunOptions.FULL_TRACE) run_metadata = tf.RunMetadata() else: run_options = run_metadata = None input_list = [merged_summaries, self.train_step, self.total_loss] if self.add_disp_output: input_list.append(self.level_0_disp_pred) if self.add_seg_output: input_list.append(self.level_0_seg_pred) output_list = sess.run(input_list, feed_dict=feed_dict, options=run_options, run_metadata=run_metadata) extra_output_count = self.add_disp_output + self.add_seg_output train_summary, _, train_loss = output_list[:-extra_output_count] train_pred_disp_field_map_batch = train_pred_seg_batch = None if self.add_disp_output: index = -extra_output_count train_pred_disp_field_map_batch = output_list[index] if self.add_seg_output: index = -extra_output_count + self.add_disp_output train_pred_seg_batch = output_list[index] # TODO: If uncommenting below code, also add relevant code to the "else" block below # train_summary, _, train_loss, train_pred_disp_field_map_batch = sess.run( # [merged_summaries, train_step, total_loss, pred_disp_field_map], # feed_dict={input_image: train_gt_polygon_map_batch, input_disp_polygon_map: train_disp_polygon_map_batch, # gt_disp_field_map: train_gt_disp_field_map_batch, # keep_prob: DROPOUT_KEEP_PROB, # mode_training: True}, options=run_options, run_metadata=run_metadata) summaries_writer.add_summary(train_summary, summary_index) if summary_index == 0: summaries_writer.add_run_metadata(run_metadata, 'step%03d' % summary_index) print_utils.print_info("step {}, training loss = {}".format( summary_index, train_loss)) if plot: train_image_batch = ( train_image_batch - self.image_dynamic_range[0] ) / (self.image_dynamic_range[1] - self.image_dynamic_range[0]) # train_gt_disp_field_map_batch = train_gt_disp_field_map_batch * 2 # Within [-1, 1] # train_gt_disp_field_map_batch = train_gt_disp_field_map_batch * self.disp_max_abs_value # Within [-disp_max_abs_value, disp_max_abs_value] # train_pred_disp_field_map_batch = train_pred_disp_field_map_batch * 2 # Within [-1, 1] # train_pred_disp_field_map_batch = train_pred_disp_field_map_batch * self.disp_max_abs_value # Within [-disp_max_abs_value, disp_max_abs_value] # visualization.plot_batch(["Training gt disp", "Training pred disp"], train_image_batch, # train_gt_polygon_map_batch, # [train_gt_disp_field_map_batch, train_pred_disp_field_map_batch], # train_disp_polygon_map_batch) if self.add_seg_output: visualization.plot_batch_seg("Training pred seg", train_image_batch, train_pred_seg_batch) return train_image_batch, train_gt_polygon_map_batch, train_gt_disp_field_map_batch, train_disp_polygon_map_batch, train_pred_disp_field_map_batch, train_pred_seg_batch else: _ = sess.run([self.train_step], feed_dict=feed_dict) return train_image_batch, train_gt_polygon_map_batch, train_gt_disp_field_map_batch, train_disp_polygon_map_batch, None, None
def main(): # --- Process args --- # args = get_args() config = run_utils.load_config(args.config) if config is None: print_utils.print_error( "ERROR: cannot continue without a config file. Exiting now...") exit() print_utils.print_info("Using downscaling factors: {}".format(args.ds_fac)) run_name_list = [RUN_NAME_FORMAT.format(ds_fac) for ds_fac in args.ds_fac] # --- Read image --- # print_utils.print_info("Reading image...") image_filepath = get_abs_path(args.image) image, image_metadata = read_image(image_filepath, args.pixelsize) image = clip_image(image, 0, 255) # hist = np.histogram(image) # print_hist(hist) im_min, im_max = get_min_max(image, std_factor=3) # print("min: {}, max: {}".format(im_min, im_max)) image = stretch_image(image, im_min, im_max, 0, 255) image = clip_image(image, 0, 255) # hist = np.histogram(image) # print_hist(hist) print("Image stats:") print("\tShape: {}".format(image.shape)) print("\tMin: {}".format(image.min())) print("\tMax: {}".format(image.max())) # --- Read shapefile if it exists --- # if args.shapefile is not None: shapefile_filepath = get_abs_path(args.shapefile) gt_polygons = get_shapefile_annotations(image_filepath, shapefile_filepath) else: # --- Load or fetch OSM building data --- # gt_polygons = get_osm_annotations(image_filepath) # --- Print polygon info --- # print("Polygons stats:") print("\tCount: {}".format(len(gt_polygons))) print("\tMin: {}".format(min([polygon.min() for polygon in gt_polygons]))) print("\tMax: {}".format(max([polygon.max() for polygon in gt_polygons]))) if not check_polygons_in_image(image, gt_polygons): print_utils.print_error( "ERROR: polygons are not inside the image. This is most likely due to using the wrong projection when reading the input shapefile. Aborting..." ) exit() print_utils.print_info("Aligning building annotations...") aligned_polygons = test.test_align_gt(args.runs_dirpath, image, image_metadata, gt_polygons, args.batch_size, args.ds_fac, run_name_list, config["disp_max_abs_value"], output_shapefiles=False) print_utils.print_info("Saving aligned building annotations...") save_annotations(args.image, aligned_polygons)
def build(repositories, theme, all_languages, output_dir): termlangs = get_termlangs(repositories, all_languages) print_info("Copying assets") copydir(html_assets, output_dir) css_dir = os.path.join(output_dir, "css") makedirs(css_dir) make_css(css_assets, theme, css_dir) languages = {} project_count = {} for language_code, terms in termlangs.items(): if language_code not in all_languages: all_languages[language_code] = Language( code=language_code, name=language_code, legal={}, translations={} ) language = all_languages[language_code] print_info("Language "+language.name) out_terms = [] count = 0 lang_dir = os.path.join(output_dir, language.code) for term in terms: term_dir = os.path.join(lang_dir, "%s.%d"%(term.id, term.number)) makedirs(term_dir) print_info("Building Term:\t\t" + str(term.title)) projects = [] for p in term.projects: built_project = Project.build_from_resource(p, \ term, \ term_dir, \ language, \ theme) if built_project: count += 1 projects.append(built_project) else: continue extras = [] for r in term.extras: extras.append(build_extra(term, r, language, theme, term_dir)) term = Term( id=term.id, manifest=term.manifest, number=term.number, language=term.language, title=term.title, description=term.description, projects=projects, extras=extras, ) out_terms.append(make_term_index(term, language, theme, term_dir)) print_info("Term built!") print_info("Building " + language.name +" index") languages[language_code] = \ make_lang_index(language, out_terms, theme, lang_dir) project_count[language_code] = count print_info("Building " + theme.name + " index: " + output_dir) sorted_languages = [] for lang in sorted(project_count.keys(), \ key=lambda x: project_count[x], reverse=True): sorted_languages.append((all_languages[lang], languages[lang])) make_index(sorted_languages, \ all_languages[theme.language], \ theme, \ output_dir) print_info("Complete")