def process_files(save_dir, file_list, code_list): for f, c in zip(file_list, code_list): print('processing: {} with code {}'.format(f,c)) sub_dir = os.path.join(save_dir, '/'.join(f.split('/')[5:-1])) ersa_utils.make_dir_if_not_exist(sub_dir) save_name = os.path.join(sub_dir, os.path.basename(f)) rgb = ersa_utils.load_file(f) rgb_new = makeup_aemo_img(rgb, c) ersa_utils.save_file(save_name, rgb_new)
def match_files(save_dir, root_dir, source_dist): for i in range(1, 6): if i != 2: for p, s, f in os.walk(root_dir.format(i)): save_sub_dir = os.path.join(save_dir, '/'.join(p.split('/')[8:])) ersa_utils.make_dir_if_not_exist(save_sub_dir) target_dist = get_aemo_stats(p) align_files(p, save_sub_dir, source_dist, target_dist) else: for p, s, f in os.walk(os.path.join(root_dir.format(i), '1')): save_sub_dir = os.path.join(save_dir, '/'.join(p.split('/')[8:])) ersa_utils.make_dir_if_not_exist(save_sub_dir) target_dist = get_aemo_stats(p) align_files(p, save_sub_dir, source_dist, target_dist) for p, s, f in os.walk(os.path.join(root_dir.format(i), '2')): save_sub_dir = os.path.join(save_dir, '/'.join(p.split('/')[8:])) ersa_utils.make_dir_if_not_exist(save_sub_dir) target_dist = get_aemo_stats(p) align_files(p, save_sub_dir, source_dist, target_dist)
model_dir = r'/hdd/Results/towers' weight_range = [30] iou_list = np.zeros(len(weight_range)) IMAGE_SIZE = (500, 500) PATH_TO_TEST_IMAGES_DIR = r'/media/ei-edl01/data/uab_datasets/towers/data/Original_Tiles' test_gt = get_test_images(PATH_TO_TEST_IMAGES_DIR) img_dir, task_dir = sis_utils.get_task_img_folder() IOU_TH = 0.5 for cnt, weight in enumerate(tqdm(weight_range)): model_name = 'confmap_uab_UnetCrop_towers_pw{}_0_PS(572, 572)_BS5_EP100_LR0.0001_DS60_DR0.1_SFN32'.format( weight) predicted_dir = os.path.join(task_dir, 'predicted{}'.format(model_name)) ground_truth_dir = os.path.join(task_dir, 'ground-truth{}'.format(model_name)) ersa_utils.make_dir_if_not_exist(predicted_dir) ersa_utils.make_dir_if_not_exist(ground_truth_dir) pred_file_names = natsorted( glob(os.path.join(task_dir, model_name, '*.npy'))) for pred_file_name, gt_file_name in zip(pred_file_names, test_gt): pred = ersa_utils.load_file(pred_file_name) pred = pred gt = ersa_utils.load_file(gt_file_name) h_steps, w_steps = extract_grids(pred, IMAGE_SIZE[0], IMAGE_SIZE[1]) patch_cnt = 0 for h_cnt, h in enumerate(h_steps): for w_cnt, w in enumerate(w_steps): pred_patch = pred[h:h + IMAGE_SIZE[0], w:w + IMAGE_SIZE[1]] gt_patch = gt[h:h + IMAGE_SIZE[0], w:w + IMAGE_SIZE[1]]
tile_size = (5000, 5000) bs = 5 suffix = 'test' sfn = 32 gpu = 0 nn_utils.set_gpu(gpu) # get test data # gammas = [2.5, 1, 2.5] sample_id = 3 data_dir = r'/media/ei-edl01/data/aemo/samples/0584007740{}0_01'.format( sample_id) files = sorted(glob(os.path.join(data_dir, 'hist_match_ct', '*.tif'))) # adjust gamma '''gamma_save_dir = os.path.join(data_dir, 'gamma_adjust') ersa_utils.make_dir_if_not_exist(gamma_save_dir) ga = gammaAdjust.GammaAdjust(gamma=gammas[sample_id - 1], path=gamma_save_dir) ga.run(force_run=False, file_list=files) files = sorted(glob(os.path.join(gamma_save_dir, '*.tif')))''' # get image mean img_mean = cm.get_channel_mean('', [[f] for f in files]) model_dir = r'/hdd6/Models/UNET_city/UnetCrop_spca_aug_xcity_0_PS(572, 572)_BS5_EP100_LR0.0001_DS60_DR0.1_SFN32' # test model '''nn_utils.tf_warn_level(3) unet = unet.UNet(class_num, patch_size, suffix=suffix, batch_size=bs) model_dir = r'/hdd6/Models/UNET_city/UnetCrop_spca_aug_xcity_0_PS(572, 572)_BS5_EP100_LR0.0001_DS60_DR0.1_SFN32' # model_dir = r'/hdd6/Models/Inria_decay/UnetCrop_inria_decay_0_PS(572, 572)_BS5_EP100_LR0.0001_DS60.0_DR0.1_SFN32'
def main(flags): np.random.seed(int(flags.run_id)) tf.set_random_seed(int(flags.run_id)) # make network # define place holder X = tf.placeholder( tf.float32, shape=[None, flags.input_size[0], flags.input_size[1], 3], name='X') y = tf.placeholder( tf.int32, shape=[None, flags.input_size[0], flags.input_size[1], 1], name='y') mode = tf.placeholder(tf.bool, name='mode') model = myUnetModelCrop({ 'X': X, 'Y': y }, trainable=mode, model_name=flags.model_name, input_size=flags.input_size, batch_size=flags.batch_size, learn_rate=flags.learning_rate, decay_step=flags.decay_step, decay_rate=flags.decay_rate, epochs=flags.epochs, start_filter_num=flags.sfn) model.create_graph('X', class_num=flags.num_classes) # create collection # the original file is in /ei-edl01/data/uab_datasets/inria blCol = uab_collectionFunctions.uabCollection(flags.ds_name) blCol.readMetadata() img_mean = blCol.getChannelMeans([0, 1, 2]) # get mean of rgb info print(img_mean) img_dir, task_dir = sis_utils.get_task_img_folder() save_dir = os.path.join(task_dir, 'bihar_patches') ersa_utils.make_dir_if_not_exist(save_dir) files, par_dir = blCol.getAllTileByDirAndExt([0, 1, 2, 3]) resize_patches(files, par_dir, flags.input_size, save_dir) patchDir = save_dir # make data reader # use first 5 tiles for validation idx, file_list = uabCrossValMaker.uabUtilGetFolds(patchDir, 'fileList.txt', 'tile') # use first city for validation #assert len(file_list) == flags.n_train + flags.n_valid file_list_train = [a for a in file_list[:45]] file_list_valid = [a for a in file_list[-5:]] with tf.name_scope('image_loader'): # GT has no mean to subtract, append a 0 for block mean dataReader_train = uabDataReader.ImageLabelReader( [3], [0, 1, 2], patchDir, file_list_train, flags.input_size, None, flags.batch_size, dataAug='flip,rotate', block_mean=np.append([0], img_mean)) # no augmentation needed for validation dataReader_valid = uabDataReader.ImageLabelReader([3], [0, 1, 2], patchDir, file_list_valid, flags.input_size, None, flags.batch_size, dataAug=' ', block_mean=np.append( [0], img_mean)) # train start_time = time.time() model.train_config('X', 'Y', flags.n_train, flags.n_valid, flags.input_size, uabRepoPaths.modelPath, loss_type='xent', par_dir='{}'.format(flags.ds_name)) model.run( train_reader=dataReader_train, valid_reader=dataReader_valid, pretrained_model_dir= None, # train from scratch, no need to load pre-trained model isTrain=True, img_mean=img_mean, verb_step=100, # print a message every 100 step(sample) save_epoch=200, # save the model every 5 epochs gpu=GPU, patch_size=flags.input_size) duration = time.time() - start_time print('duration {:.2f} hours'.format(duration / 60 / 60))
def process(self): """ Evaluate the network :return: """ nn_utils.set_gpu(self.gpu) if self.score_results: with open(os.path.join(self.score_save_dir, 'result.txt'), 'w'): pass iou_record = [] # prepare the reader if self.score_results: init_op, reader_op = dataReaderSegmentation.DataReaderSegmentationTesting( self.input_size, self.tile_size, self.file_list, overlap=self.model.get_overlap(), pad=self.model.get_overlap() // 2, batch_size=self.batch_size, chan_mean=self.img_mean, is_train=False, has_gt=True, random=False, gt_dim=1, include_gt=True).read_op() feature, label = reader_op self.model.create_graph(feature, **self.kwargs) else: init_op, reader_op = dataReaderSegmentation.DataReaderSegmentationTesting( self.input_size, self.tile_size, self.file_list, overlap=self.model.get_overlap(), pad=self.model.get_overlap() // 2, batch_size=self.batch_size, chan_mean=self.img_mean, is_train=False, has_gt=False, random=False, gt_dim=0, include_gt=False).read_op() feature = reader_op self.model.create_graph(feature[0], **self.kwargs) pad = self.model.get_overlap() for file_cnt, (file_name_list) in enumerate(self.file_list): file_name_truth = None if self.score_results: file_name, file_name_truth = file_name_list tile_name = os.path.basename(file_name_truth).split( self.split_char)[0] else: file_name = file_name_list[0] tile_name = os.path.basename(file_name).split( self.split_char)[0] if self.verb: print('Evaluating {} ... '.format(tile_name)) # read tile size if no tile size is given if self.tile_size is None or self.compute_shape_flag: self.compute_shape_flag = True tile = ersa_utils.load_file(file_name) self.tile_size = tile.shape[:2] start_time = time.time() # run the model if self.model.config is None: self.model.config = tf.ConfigProto(allow_soft_placement=True) with tf.Session(config=self.model.config) as sess: init = tf.global_variables_initializer() sess.run(init) self.model.load(self.model_dir, sess, epoch=self.load_epoch_num, best_model=self.best_model) result = self.model.test_sample(sess, init_op[file_cnt]) image_pred = patchExtractor.unpatch_block( result, tile_dim=[self.tile_size[0] + pad, self.tile_size[1] + pad], patch_size=self.input_size, tile_dim_output=self.tile_size, patch_size_output=[ self.input_size[0] - pad, self.input_size[1] - pad ], overlap=pad) if self.compute_shape_flag: self.tile_size = None pred = nn_utils.get_pred_labels(image_pred) * self.truth_val if self.score_results: truth_label_img = ersa_utils.load_file(file_name_truth) iou = nn_utils.iou_metric(truth_label_img, pred, divide_flag=True) iou_record.append(iou) duration = time.time() - start_time if self.verb: print('{} mean IoU={:.3f}, duration: {:.3f}'.format( tile_name, iou[0] / iou[1], duration)) # save results pred_save_dir = os.path.join(self.score_save_dir, 'pred') ersa_utils.make_dir_if_not_exist(pred_save_dir) ersa_utils.save_file( os.path.join(pred_save_dir, '{}.png'.format(tile_name)), pred.astype(np.uint8)) if self.score_results: with open(os.path.join(self.score_save_dir, 'result.txt'), 'a+') as file: file.write('{} {}\n'.format(tile_name, iou)) if self.score_results: iou_record = np.array(iou_record) mean_iou = np.sum(iou_record[:, 0]) / np.sum(iou_record[:, 1]) print('Overall mean IoU={:.3f}'.format(mean_iou)) with open(os.path.join(self.score_save_dir, 'result.txt'), 'a+') as file: file.write('{}'.format(mean_iou))
def process(self): """ Evaluate the network :return: """ nn_utils.set_gpu(self.gpu) if self.score_results: with open(os.path.join(self.score_save_dir, 'result.txt'), 'w'): pass iou_record = [] # prepare the reader if self.score_results: feature, label = self.reader_op self.model.create_graph(feature, **self.kwargs) else: feature = self.reader_op self.model.create_graph(feature[0], **self.kwargs) # run the model with tf.Session() as sess: init = tf.global_variables_initializer() sess.run([init, self.init_op]) self.model.load(self.model_dir, sess, epoch=self.load_epoch_num, best_model=self.best_model) for file_cnt, (file_name_list) in enumerate(self.file_list): file_name_truth = None if self.score_results: file_name, file_name_truth = file_name_list tile_name = os.path.basename(file_name_truth).split( self.split_char)[0] else: file_name = file_name_list[0] tile_name = os.path.basename(file_name).split( self.split_char)[0] if self.verb: print('Evaluating {} ... '.format(tile_name)) start_time = time.time() result = sess.run(self.model.output) pred = np.argmax(np.squeeze(result, axis=0), axis=-1) if self.post_func is not None: pred = self.post_func(pred) if self.save_func is not None: save_img = self.save_func(pred) else: save_img = pred if self.score_results: truth_label_img = ersa_utils.load_file(file_name_truth) iou = nn_evals.mean_IU(pred, truth_label_img, self.ignore_label) iou_record.append(iou) duration = time.time() - start_time if self.verb: print('{} mean IoU={:.3f}, duration: {:.3f}'.format( tile_name, iou, duration)) # save results pred_save_dir = os.path.join(self.score_save_dir, 'pred') ersa_utils.make_dir_if_not_exist(pred_save_dir) ersa_utils.save_file( os.path.join(pred_save_dir, '{}.png'.format(tile_name)), save_img.astype(np.uint8)) if self.score_results: with open( os.path.join(self.score_save_dir, 'result.txt'), 'a+') as file: file.write('{} {}\n'.format(tile_name, iou)) if self.score_results: iou_record = np.array(iou_record) mean_iou = np.mean(iou_record) print('Overall mean IoU={:.3f}'.format(mean_iou)) with open(os.path.join(self.score_save_dir, 'result.txt'), 'a+') as file: file.write('{}'.format(mean_iou))
batch_size = 1 input_size = [572, 572] city_name = 'DC' nn_utils.tf_warn_level(3) if city_name == 'Atlanta': tile_size = [2000, 3000] elif city_name == 'Austin': tile_size = [2000, 3000] elif city_name == 'DC': tile_size = [2500, 2500] img_dir, task_dir = sis_utils.get_task_img_folder() path_to_save = os.path.join(task_dir, 'dtda_new', city_name, 'valid') ersa_utils.make_dir_if_not_exist(path_to_save) model_dir = r'/hdd6/Models/UNET_rand_gird/UnetCrop_inria_aug_grid_0_PS(572, 572)_BS5_' \ r'EP100_LR0.0001_DS60_DR0.1_SFN32' tf.reset_default_graph() blCol = uab_collectionFunctions.uabCollection(city_name) blCol.readMetadata() file_list, parent_dir = blCol.getAllTileByDirAndExt([0, 1, 2]) file_list_truth, parent_dir_truth = blCol.getAllTileByDirAndExt(3) img_mean = blCol.getChannelMeans([0, 1, 2]) # make the model # define place holder X = tf.placeholder(tf.float32,
def make_dataset(is_dcc, city_name, patch_size, encoder): # settings if not is_dcc: data_dir = r'/home/lab/Documents/bohao/data/transmission_line' else: data_dir = r'/work/bh163/misc/object_detection' base_dir = os.path.join(data_dir, 'yolo') save_dir = os.path.join(base_dir, 'temp') img_dir = os.path.join(base_dir, 'images', 'all') csv_dir = os.path.join(base_dir, 'csv') ersa_utils.make_dir_if_not_exist(img_dir) ersa_utils.make_dir_if_not_exist(csv_dir) ersa_utils.make_dir_if_not_exist(save_dir) ersa_utils.make_dir_if_not_exist(os.path.join(base_dir, 'images', 'train')) ersa_utils.make_dir_if_not_exist(os.path.join(base_dir, 'images', 'test')) ersa_utils.make_dir_if_not_exist(os.path.join(base_dir, 'labels', 'train')) ersa_utils.make_dir_if_not_exist(os.path.join(base_dir, 'labels', 'test')) # get files rgb_files = natsorted([ a for a in glob(os.path.join(data_dir, 'raw', '*.tif')) if 'multiclass' not in a and city_name in a ]) csv_files = natsorted([ a for a in glob(os.path.join(data_dir, 'raw', '*.csv')) if city_name in a ]) write_data_info(rgb_files, csv_files, save_dir, patch_size, encoder) write_dataset(rgb_files, save_dir, img_dir, csv_dir, patch_size, city_name)
od_graph_def.ParseFromString(serialized_graph) tf.import_graph_def(od_graph_def, name='') for tile_id in [1, 2, 3]: # get prediction raw_rgb = ersa_utils.load_file( os.path.join(raw_dir, 'USA_{}_{}.tif'.format(city_list[city_id], tile_id))) pred_files = natsorted( glob(os.path.join(task_fold, pred_dir, '*_{}.txt'.format(tile_id)))) npy_file_name = os.path.join( info_dir, 'USA_{}_{}.npy'.format(city_list[city_id], tile_id)) coords = ersa_utils.load_file(npy_file_name) text_save_dir = os.path.join(task_dir, 'faster_rcnn_all') ersa_utils.make_dir_if_not_exist(text_save_dir) text_file_name = os.path.join( text_save_dir, 'USA_{}_{}.txt'.format(city_list[city_id], tile_id)) h_steps, w_steps = extract_grids(raw_rgb, patch_size[0], patch_size[1]) with open(text_file_name, 'w+') as f: patch_cnt = 0 for h_cnt, h in enumerate(tqdm(h_steps)): for w_cnt, w in enumerate(tqdm(w_steps)): image_patch = raw_rgb[h:h + patch_size[0], w:w + patch_size[1], :] image_np_expanded = np.expand_dims(image_patch, axis=0) output_dict = run_inference_for_single_image( image_patch, detection_graph) for left, top, right, bottom, confidence, class_name in get_predict_info( output_dict, patch_size, (h, w), ('T', ), th):
data_dir = r'/home/lab/Documents/bohao/data/transmission_line/raw' city_list = ['Tucson', 'Colwich', 'Clyde', 'Wilmington'] img_dir, task_dir = sis_utils.get_task_img_folder() tower_vals = [3, 1] tower_names = ['Transmission_Tower', 'Distribution_Tower'] total_counts = np.zeros((len(tower_names), len(city_list))) tower_dict = {} for cn in city_list: tower_type = {} for tt in tower_names: tower_type[tt] = [0] tower_dict[cn] = tower_type for tn in tower_names: ersa_utils.make_dir_if_not_exist(os.path.join(img_dir, tn)) for city_id in range(len(city_list)): gt_files = natsorted(glob(os.path.join(data_dir, '*{}*_multiclass.tif'.format(city_list[city_id])))) rgb_files = ['_'.join(a.split('_')[:-1])+'.tif' for a in gt_files] for rgb_file_name, gt_file_name in zip(rgb_files, gt_files): rgb = ersa_utils.load_file(rgb_file_name) gt = ersa_utils.load_file(gt_file_name) prefix = os.path.basename(rgb_file_name)[7:-4] print('{:<20}: '.format(prefix), end='') for t_cnt, (tower_val, tower_name) in enumerate(zip(tower_vals, tower_names)): binary_map = (gt == tower_val).astype(np.int) tower_cnts = count_objects(binary_map) print('{:<15} {:<4}\t'.format(tower_name, tower_cnts), end='') total_counts[t_cnt][city_id] += tower_cnts
dirs['conf'].format(conf_dict[city_id]), '{}{}.png'.format(city_list[city_id].split('_')[1], tile_id))) line_gt = ersa_utils.load_file( os.path.join( dirs['line'], '{}{}_GT.png'.format(city_list[city_id].split('_')[1], tile_id))) tower_gt = get_tower_truth_pred(dirs, city_id, tile_id) tower_pred, tower_conf, _ = local_maxima_suppression(preds, th=merge_range) conf_img = scipy.misc.imresize(conf_img, line_gt.shape) return preds, raw_rgb, conf_img, line_gt, tower_gt, tower_pred, tower_conf if __name__ == '__main__': # directories img_dir, task_dir = sis_utils.get_task_img_folder() ersa_utils.make_dir_if_not_exist(os.path.join(img_dir, 'new_annotation')) dirs = { 'task': task_dir, 'image': os.path.join(img_dir, 'new_annotation'), 'raw': r'/home/lab/Documents/bohao/data/transmission_line/raw2', 'conf': r'/media/ei-edl01/user/bh163/tasks/2018.11.16.transmission_line/confmap_uab_UnetCrop_linesv3_city0_' r'pw50_0_PS(572, 572)_BS5_EP100_LR0.0001_DS80_DR0.1_SFN32', 'line': r'/media/ei-edl01/data/uab_datasets/lines_v3/data/Original_Tiles' } # settings
def eval_tiles(): blCol = uab_collectionFunctions.uabCollection(ds_name) blCol.readMetadata() file_list, parent_dir = blCol.getAllTileByDirAndExt([1, 2, 3]) file_list_truth, parent_dir_truth = blCol.getAllTileByDirAndExt(0) idx, file_list = uabCrossValMaker.uabUtilGetFolds(None, file_list, 'tile') idx_truth, file_list_truth = uabCrossValMaker.uabUtilGetFolds( None, file_list_truth, 'tile') # use first 5 tiles for validation file_list_valid = uabCrossValMaker.make_file_list_by_key( idx, file_list, [0, 1, 2, 3, 4, 5]) file_list_valid_truth = uabCrossValMaker.make_file_list_by_key( idx_truth, file_list_truth, [0, 1, 2, 3, 4, 5]) img_mean = blCol.getChannelMeans([1, 2, 3]) # make the model # define place holder X = tf.placeholder(tf.float32, shape=[None, input_size[0], input_size[1], 3], name='X') y = tf.placeholder(tf.int32, shape=[None, input_size[0], input_size[1], 1], name='y') mode = tf.placeholder(tf.bool, name='mode') model = UnetModelCrop({ 'X': X, 'Y': y }, trainable=mode, input_size=input_size, batch_size=batch_size, start_filter_num=32) # create graph model.create_graph('X', class_num=2) # evaluate on each sub folder root_dir = r'/home/lab/Documents/bohao/data/aemo_all/align/0584270470{}0_01' for fl, p_dir in get_file_list(root_dir): for f in fl: print('Evaluating {} in {}'.format(f, p_dir)) pred_save_dir = os.path.join(task_dir, 'aemo_all', '/'.join(p_dir.split('/')[7:])) ersa_utils.make_dir_if_not_exist(pred_save_dir) # prepare the reader reader = uabDataReader.ImageLabelReader( gtInds=[0], dataInds=[0], nChannels=3, parentDir=p_dir, chipFiles=[[f]], chip_size=input_size, tile_size=tile_size, batchSize=batch_size, block_mean=img_mean, overlap=model.get_overlap(), padding=np.array( (model.get_overlap() / 2, model.get_overlap() / 2)), isTrain=False) rManager = reader.readManager # run the model pred, conf_map = model.run(pretrained_model_dir=model_dir, test_reader=rManager, tile_size=tile_size, patch_size=input_size, gpu=gpu, load_epoch_num=75, best_model=False) pred_name = 'pred_{}.png'.format(f.split('.')[0]) conf_name = 'conf_{}.npy'.format(f.split('.')[0]) ersa_utils.save_file(os.path.join(pred_save_dir, pred_name), pred.astype(np.uint8)) ersa_utils.save_file(os.path.join(pred_save_dir, conf_name), conf_map)
fig = make_cmp_plot(rgb[loc], gt[loc], '{}_{}'.format(title_str, cnt), building_size) yield fig, building_size spca_dir = r'/media/ei-edl01/data/uab_datasets/spca/data/Original_Tiles' aemo_dir = r'/home/lab/Documents/bohao/data/aemo/aemo_pad' aemohist_dir = r'/hdd/ersa/preprocess/aemo_pad/hist_matching' img_dir, task_dir = sis_utils.get_task_img_folder() spca_files = glob(os.path.join(spca_dir, '*_RGB.jpg')) idx = np.random.permutation(20) spca_files = [spca_files[i] for i in idx] aemo_files = glob(os.path.join(aemo_dir, '*rgb.tif')) aemo_hist_files = glob(os.path.join(aemohist_dir, '*histRGB.tif')) ersa_utils.make_dir_if_not_exist(os.path.join(img_dir, 'spca')) ersa_utils.make_dir_if_not_exist(os.path.join(img_dir, 'aemo')) ersa_utils.make_dir_if_not_exist(os.path.join(img_dir, 'aemohist')) for spca_file in spca_files: title_str = os.path.basename(spca_file).split('_')[0] rgb = ersa_utils.load_file(spca_file) gt_file = spca_file[:-7] + 'GT.png' gt = ersa_utils.load_file(gt_file) for cnt, (fig, _) in enumerate(visualize_fn_object(rgb, gt, title_str)): plt.savefig( os.path.join(img_dir, 'spca', '{}_{}.png'.format(title_str, cnt))) plt.close(fig)
file_list_valid = uabCrossValMaker.make_file_list_by_key( idx, file_list, [1, 2, 3]) file_list_valid_truth = uabCrossValMaker.make_file_list_by_key( idx_truth, file_list_truth, [1, 2, 3]) img_mean = blCol.getChannelMeans([2, 3, 4]) img_mean = np.concatenate([np.array([0]), img_mean]) # make the model # define place holder for weight in [50]: for city_id in range(4): model_dir = r'/hdd6/Models/lines_tw1/UnetCrop_lines_city{}_pw{}_0_PS(572, 572)_BS5_' \ r'EP50_LR0.0001_DS30_DR0.1_SFN32'.format(city_id, weight) SAVE_DIR = os.path.join( task_dir, 'confmap_uab_{}'.format(os.path.basename(model_dir))) ersa_utils.make_dir_if_not_exist(SAVE_DIR) tf.reset_default_graph() X = tf.placeholder(tf.float32, shape=[None, input_size[0], input_size[1], 4], name='X') y = tf.placeholder(tf.int32, shape=[None, input_size[0], input_size[1], 1], name='y') mode = tf.placeholder(tf.bool, name='mode') model = UnetModelCrop({ 'X': X, 'Y': y }, trainable=mode, input_size=input_size, batch_size=batch_size,
return fig, iou, size if __name__ == '__main__': window_size = 200 stride = 200 img_dir, task_dir = sis_utils.get_task_img_folder() good_th = 80 bad_th = 5 truth_dir = r'/media/ei-edl01/data/uab_datasets/aemo_comb/data/Original_Tiles' ft_dir = r'/hdd/Results/aemo/uab/UnetCrop_aemo_comb_xfold2_1_PS(572, 572)_BS5_EP80_LR0.001_DS30_DR0.1_SFN32/aemo_comb/pred' img_save_dir = os.path.join(img_dir, 'uab_pred_patches_comb', ft_dir.split('/')[-3]) ersa_utils.make_dir_if_not_exist(img_save_dir) eval_files = [ '_'.join(os.path.basename(f)[:-4].split('_')[:2]) for f in glob(os.path.join(ft_dir, '*.png')) ] for f in eval_files: rgb = imageio.imread(os.path.join(truth_dir, '{}_RGB.tif'.format(f))) truth_img_name = os.path.join(truth_dir, '{}_GT.tif'.format(f)) ft_img_name = os.path.join(ft_dir, '{}.png'.format(f)) truth, ft = imageio.imread(truth_img_name), imageio.imread(ft_img_name) assert np.all(np.unique(truth) == np.array([0, 1])) and np.all( np.unique(ft) == np.array([0, 1]))
'image/object/bbox/ymin': dataset_util.float_list_feature(ymins), 'image/object/bbox/ymax': dataset_util.float_list_feature(ymaxs), 'image/object/class/text': dataset_util.bytes_list_feature(classes_text), 'image/object/class/label': dataset_util.int64_list_feature(classes), })) return tf_example if __name__ == '__main__': # settings if not IS_DCC: data_dir = r'/home/lab/Documents/bohao/data/transmission_line' save_dir = os.path.join(data_dir, 'info') store_dir = os.path.join(data_dir, 'patches') tf_dir = os.path.join(data_dir, 'data') ersa_utils.make_dir_if_not_exist(save_dir) ersa_utils.make_dir_if_not_exist(store_dir) ersa_utils.make_dir_if_not_exist(tf_dir) else: data_dir = r'/work/bh163/misc/object_detection' save_dir = os.path.join(data_dir, 'info') store_dir = os.path.join(data_dir, 'patches') tf_dir = os.path.join(data_dir, 'data') ersa_utils.make_dir_if_not_exist(save_dir) ersa_utils.make_dir_if_not_exist(store_dir) ersa_utils.make_dir_if_not_exist(tf_dir) # get files rgb_files = natsorted([a for a in glob(os.path.join(data_dir, 'raw', '*.tif')) if 'multiclass' not in a and CITY_NAME in a]) csv_files = natsorted([a for a in glob(os.path.join(data_dir, 'raw', '*.csv')) if CITY_NAME in a])
writer = csv.writer(f, lineterminator='\n') writer.writerow(['{}'.format(x) for x in fc1000]) f2.write('{}\n'.format(patch_name)) if __name__ == '__main__': # settings patch_size = (572, 572) tile_size = (5000, 5000) np.random.seed(1004) gpu = 0 img_dir, task_dir = sis_utils.get_task_img_folder() use_hist = False cm = collectionMaker.read_collection('spca') cm.print_meta_data() chan_mean = cm.meta_data['chan_mean'][:3] file_list = cm.load_files(field_id=','.join(str(i) for i in range(0, 663)), field_ext='RGB,GT') patch_list_train = patchExtractor.PatchExtractor(patch_size, tile_size, 'spca_all', 184, 184 // 2). \ run(file_list=file_list, file_exts=['jpg', 'png'], force_run=False).get_filelist() feature_dir = os.path.join(task_dir, 'spca_patches') dr = data_reader(patch_list_train, chan_mean) ersa_utils.make_dir_if_not_exist(feature_dir) processBlock.BasicProcess( 'make_feature', feature_dir, func=lambda: make_res_features(dr, feature_dir, gpu)).run()