def resize_patches(files, par_dir, patch_size, save_dir): def load_and_resize(par, f, patch_size, preserve, pad=92): patch_size = np.array(patch_size) - np.array([pad * 2, pad * 2]) data = ersa_utils.load_file(os.path.join(par, f)) data = ersa_utils.pad_image(data, pad) return reader_utils.resize_image(data, patch_size, preserve) f_name_list = [] for file in files: r_patch = load_and_resize(par_dir[0], file[0], patch_size, True).astype(np.uint8) g_patch = load_and_resize(par_dir[1], file[1], patch_size, True).astype(np.uint8) b_patch = load_and_resize(par_dir[2], file[2], patch_size, True).astype(np.uint8) gt_patch = load_and_resize(par_dir[3], file[3], patch_size, True).astype(np.uint8) ersa_utils.save_file( os.path.join(save_dir, '{}jpg'.format(file[0][:-3])), r_patch) ersa_utils.save_file( os.path.join(save_dir, '{}jpg'.format(file[1][:-3])), g_patch) ersa_utils.save_file( os.path.join(save_dir, '{}jpg'.format(file[2][:-3])), b_patch) ersa_utils.save_file( os.path.join(save_dir, '{}png'.format(file[3][:-3])), gt_patch) f_line = '{}jpg {}jpg {}jpg {}png\n'.format(file[0][:-3], file[1][:-3], file[2][:-3], file[3][:-3]) f_name_list.append(f_line) file_name = os.path.join(save_dir, 'fileList.txt') ersa_utils.save_file(file_name, f_name_list)
def process(self, **kwargs): """ process to make the new field :param kwargs: file_list: the list of the files, if not given, use all the files with selected field extension file_ext: the new file extension, if not given, use the same as the old one d_type: the new data type, if not given, use the same as the old one :return: """ if 'file_list' not in kwargs: file_list = self.clc.load_files(','.join(self.clc.field_name), ','.join(self.clc.field_id), self.field_ext_pair[0]) else: file_list = kwargs['file_list'] assert len(file_list[0]) == 1 pbar = tqdm(file_list) for img_file in pbar: save_name = img_file[0].replace(''.join([a for a in self.field_ext_pair[0] if a != '.' and a != '*']), self.field_ext_pair[1]) if 'file_ext' in kwargs: # user specified a new file extension save_name = save_name.replace(save_name.split('.')[-1], kwargs['file_ext']) save_name = os.path.join(self.path, os.path.basename(save_name)) pbar.set_description('Making {}'.format(os.path.basename(save_name))) img = ersa_utils.load_file(img_file[0]) for old_val, new_val in self.switch_dict.items(): img[np.where(img == old_val)] = new_val if 'd_type' in kwargs: img = img.astype(kwargs['d_type']) ersa_utils.save_file(save_name, img) self.files.append(save_name)
def get_pretrained_weights(flags): save_name = os.path.join(flags.weight_dir, 'weight.pkl') if not os.path.exists(save_name): X = tf.placeholder(tf.float32, shape=[None, flags.input_size[0], flags.input_size[1], 3], name='X') y = tf.placeholder(tf.int32, shape=[None, flags.input_size[0], flags.input_size[1], 1], name='y') mode = tf.placeholder(tf.bool, name='mode') model = uabMakeNetwork_UNet.UnetModelCrop({'X': X, 'Y': y}, trainable=mode, model_name=flags.model_name, input_size=flags.input_size, batch_size=flags.batch_size, learn_rate=flags.learning_rate, decay_step=flags.decay_step, decay_rate=flags.decay_rate, epochs=flags.epochs, start_filter_num=flags.sfn) model.create_graph('X', class_num=flags.num_classes) train_vars = [v for v in tf.global_variables() if 'global_step' not in v.name] weight_dict = dict() with tf.Session() as sess: model.load(flags.model_dir, sess, epoch=95) for v in train_vars: theta = sess.run(v) weight_dict[v.name] = theta ersa_utils.save_file(save_name, weight_dict) else: weight_dict = ersa_utils.load_file(save_name) tf.reset_default_graph() return weight_dict
def extract_patches(img_pair, patch_size, pad, overlap, patch_dir, prefix, file_exts=('jpg', 'png'), file_suffix=('rgb', 'gt')): grid_list = patchExtractor.make_grid(tile_size + 2 * pad, patch_size, overlap) record_file = open(os.path.join(patch_dir, 'file_list.txt'), 'a+') patch_list = [] for suffix_cnt, (img, ext) in enumerate(zip(img_pair, file_exts)): patch_list_ext = [] # extract images for patch, y, x in patchExtractor.patch_block(img, pad, grid_list, patch_size, return_coord=True): patch_name = '{}_y{}x{}.{}'.format( prefix + '_{}'.format(file_suffix[suffix_cnt]), int(y), int(x), ext) patch_name = os.path.join(patch_dir, patch_name) ersa_utils.save_file(patch_name, patch) patch_list_ext.append(patch_name) patch_list.append(patch_list_ext) patch_list = ersa_utils.rotate_list(patch_list) for items in patch_list: record_file.write('{}\n'.format(' '.join(items))) record_file.close()
def process(self, **kwargs): """ Extract the patches :param kwargs: file_list: list of lists of the files, can be generated by using collectionMaker.load_files() file_exts: extensions of the new files :return: """ assert len(kwargs['file_exts']) == len(kwargs['file_list'][0]) grid_list = None if self.tile_size is not None: grid_list = make_grid(self.tile_size + 2*self.pad, self.patch_size, self.overlap) pbar = tqdm(kwargs['file_list']) record_file = open(os.path.join(self.path, 'file_list.txt'), 'w') for files in pbar: pbar.set_description('Extracting {}'.format(os.path.basename(files[0]))) patch_list = [] for f, ext in zip(files, kwargs['file_exts']): patch_list_ext = [] img = ersa_utils.load_file(f) if self.tile_size is None: grid_list = make_grid(np.array(img.shape[:2])+2*self.pad, self.patch_size, self.overlap) # extract images for patch, y, x in patch_block(img, self.pad, grid_list, self.patch_size, return_coord=True): patch_name = '{}_y{}x{}.{}'.format(os.path.basename(f).split('.')[0], int(y), int(x), ext) patch_name = os.path.join(self.path, patch_name) ersa_utils.save_file(patch_name, patch.astype(np.uint8)) patch_list_ext.append(patch_name) patch_list.append(patch_list_ext) patch_list = ersa_utils.rotate_list(patch_list) for items in patch_list: record_file.write('{}\n'.format(' '.join(items))) record_file.close()
def get_pretrained_weights(weight_dir, model_dir): save_name = os.path.join(weight_dir, 'weight.pkl') if not os.path.exists(save_name): X = tf.placeholder(tf.float32, shape=[None, input_size[0], input_size[1], 3], name='X') y = tf.placeholder(tf.int32, shape=[None, input_size[0], input_size[1], 1], name='y') mode = tf.placeholder(tf.bool, name='mode') model = uabMakeNetwork_UNet.UnetModelCrop({'X': X, 'Y': y}, trainable=mode, input_size=input_size, start_filter_num=32) model.create_graph('X', class_num=2) train_vars = [v for v in tf.trainable_variables()] weight_dict = dict() with tf.Session() as sess: model.load(model_dir, sess, epoch=95) for v in train_vars: theta = sess.run(v) weight_dict[v.name] = theta ersa_utils.save_file(save_name, weight_dict) else: weight_dict = ersa_utils.load_file(save_name) tf.reset_default_graph() return weight_dict
def align_files(data_dir, save_dir, source_dist, target_dist): rgb_files = glob(os.path.join(data_dir, '*.tif')) for file in rgb_files: print('aligning {}'.format(file)) im_s = ersa_utils.load_file(file) im_res = cust_hist_match(target_dist, source_dist, im_s) ersa_utils.save_file(os.path.join(save_dir, os.path.basename(file)), im_res)
def save_activations(self, rgb_list, gt_list, rgb_dir, img_mean, gpu, pretrained_model_dir, path_to_save, input_size, batch_size, load_epoch_num): os.environ['CUDA_DEVICE_ORDER'] = 'PCI_BUS_ID' os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu) activation_dict = dict() for file_name, file_name_truth in zip(rgb_list, gt_list): tile_name = file_name_truth.split('_')[0] print('Evaluating {} ... '.format(tile_name)) # get tile size sample_img = imageio.imread(os.path.join(rgb_dir[0], file_name[0])) tile_size = sample_img.shape[:2] # prepare the reader reader = uabDataReader.ImageLabelReader( gtInds=[0], dataInds=[0], nChannels=3, parentDir=rgb_dir, chipFiles=[file_name], chip_size=input_size, tile_size=tile_size, batchSize=batch_size, block_mean=img_mean, overlap=self.get_overlap(), padding=np.array( (self.get_overlap() / 2, self.get_overlap() / 2)), isTrain=False) rManager = reader.readManager total_len = np.ceil((tile_size[0] + self.get_overlap()) / (input_size[0] - self.get_overlap())) * \ np.ceil((tile_size[1] + self.get_overlap()) / (input_size[1] - self.get_overlap())) if self.config is None: self.config = tf.ConfigProto(allow_soft_placement=True) with tf.Session(config=self.config) as sess: init = tf.global_variables_initializer() sess.run(init) self.load(pretrained_model_dir, sess, epoch=load_epoch_num, best_model=False) for X_batch in tqdm(rManager, total=total_len): for layer_id in range(len(self.activations)): layer_val = sess.run(self.activations[layer_id], feed_dict={ self.inputs['X']: X_batch, self.trainable: False }) for chan_id in range(layer_val.shape[-1]): f_i_t = layer_val[:, :, :, chan_id].flatten() act_name = 'f_{}_{}'.format(layer_id, chan_id) if act_name not in activation_dict: activation_dict[act_name] = bayes_update() activation_dict[act_name].update(f_i_t) save_name = os.path.join(path_to_save, 'activation_list.pkl') ersa_utils.save_file(save_name, activation_dict)
def make_dataset(rgb_files, info_dir, store_dir, tf_dir, city_name=''): writer_train = tf.python_io.TFRecordWriter( os.path.join(tf_dir, 'train_v2_{}.record'.format(city_name))) writer_valid = tf.python_io.TFRecordWriter( os.path.join(tf_dir, 'valid_v2_{}.record'.format(city_name))) for rgb_file_name in rgb_files: file_name = os.path.basename(rgb_file_name[:-4]) if 'NZ' not in file_name: city_id = int(file_name.split('_')[-1]) else: city_id = int(file_name.split('_')[-2]) if city_id <= 3: print('Processing file {} in validation set'.format(file_name)) is_val = True else: print('Processing file {} in training set'.format(file_name)) is_val = False rgb = ersa_utils.load_file(rgb_file_name) npy_file_name = os.path.join( info_dir, os.path.basename(rgb_file_name[:-4] + '.npy')) coords = ersa_utils.load_file(npy_file_name) patch_cnt = 0 for line in coords: for cell in line: patch_cnt += 1 save_name = os.path.join( store_dir, os.path.basename(rgb_file_name[:-4] + '_{}.jpg'.format(patch_cnt))) img = rgb[cell['h']:cell['h'] + PATCH_SIZE[0], cell['w']:cell['w'] + PATCH_SIZE[1], :3] label = cell['label'] # assert np.unique(label) == ['DT'] or label == [] box = cell['box'] '''import matplotlib.pyplot as plt import matplotlib.patches as patches if len(label) > 0: fig, ax = plt.subplots(1) ax.imshow(img) for l, b in zip(label, box): rect = patches.Rectangle((b[1], b[0]), b[3]-b[1], b[2]-b[0], linewidth=1, edgecolor='r', facecolor='none') ax.add_patch(rect) plt.show()''' ersa_utils.save_file(save_name, img) tf_example = create_tf_example(save_name, label, box) if is_val: writer_valid.write(tf_example.SerializeToString()) else: writer_train.write(tf_example.SerializeToString()) writer_train.close() writer_valid.close()
def process_files(save_dir, file_list, code_list): for f, c in zip(file_list, code_list): print('processing: {} with code {}'.format(f,c)) sub_dir = os.path.join(save_dir, '/'.join(f.split('/')[5:-1])) ersa_utils.make_dir_if_not_exist(sub_dir) save_name = os.path.join(sub_dir, os.path.basename(f)) rgb = ersa_utils.load_file(f) rgb_new = makeup_aemo_img(rgb, c) ersa_utils.save_file(save_name, rgb_new)
def run(self, train_reader=None, valid_reader=None, test_reader=None, pretrained_model_dir=None, layers2load=None, isTrain=False, img_mean=np.array((0, 0, 0), dtype=np.float32), verb_step=100, save_epoch=5, gpu=None, tile_size=(5000, 5000), patch_size=(572, 572), truth_val=1, continue_dir=None, load_epoch_num=None, valid_iou=False, best_model=True): if gpu is not None: os.environ['CUDA_DEVICE_ORDER'] = 'PCI_BUS_ID' os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu) if isTrain: coord = tf.train.Coordinator() with tf.Session(config=self.config) as sess: # init model init = [tf.global_variables_initializer(), tf.local_variables_initializer()] sess.run(init) saver = tf.train.Saver(var_list=tf.global_variables(), max_to_keep=1) # load model if pretrained_model_dir is not None: if layers2load is not None: self.load_weights(pretrained_model_dir, layers2load) else: self.load(pretrained_model_dir, sess, saver, epoch=load_epoch_num) threads = tf.train.start_queue_runners(coord=coord, sess=sess) try: train_summary_writer = tf.summary.FileWriter(self.ckdir, sess.graph) self.train('X', 'Y', self.n_train, sess, train_summary_writer, n_valid=self.n_valid, train_reader=train_reader, valid_reader=valid_reader, image_summary=util_functions.image_summary, img_mean=img_mean, verb_step=verb_step, save_epoch=save_epoch, continue_dir=continue_dir, valid_iou=valid_iou) finally: coord.request_stop() coord.join(threads) saver.save(sess, '{}/model.ckpt'.format(self.ckdir), global_step=self.global_step) else: if self.config is None: self.config = tf.ConfigProto(allow_soft_placement=True) pad = self.get_overlap() with tf.Session(config=self.config) as sess: init = tf.global_variables_initializer() sess.run(init) self.load(pretrained_model_dir, sess, epoch=load_epoch_num, best_model=best_model) self.model_name = pretrained_model_dir.split('/')[-1] result = self.test('X', sess, test_reader) image_pred = uabUtilreader.un_patchify_shrink(result, [tile_size[0] + pad, tile_size[1] + pad], tile_size, patch_size, [patch_size[0] - pad, patch_size[1] - pad], overlap=pad) global TILE_CNT ersa_utils.save_file(os.path.join(SAVE_DIR, '{}.npy'.format(TILE_NAME[TILE_CNT])), image_pred[:, :, 1]) TILE_CNT += 1 return util_functions.get_pred_labels(image_pred) * truth_val
def make_dataset(rgb_files, info_dir, store_dir, city_name=''): writer_train = open( os.path.join(store_dir, 'data', 'train_{}_T.txt'.format(city_name)), 'w+') writer_valid = open( os.path.join(store_dir, 'data', 'test_{}_T.txt'.format(city_name)), 'w+') for rgb_file_name in rgb_files: file_name = os.path.basename(rgb_file_name[:-4]) city_id = int(file_name.split('_')[-1]) if city_id <= 3: print('Processing file {} in validation set'.format(file_name)) is_val = True else: print('Processing file {} in training set'.format(file_name)) is_val = False rgb = ersa_utils.load_file(rgb_file_name) npy_file_name = os.path.join( info_dir, os.path.basename(rgb_file_name[:-4] + '.npy')) coords = ersa_utils.load_file(npy_file_name) patch_cnt = 0 for line in coords: for cell in line: patch_cnt += 1 patch_file_name = os.path.basename(rgb_file_name[:-4] + '_{}.jpg'.format(patch_cnt)) img_name = os.path.join(store_dir, 'build/darknet/x64/data/obj', patch_file_name) lbl_name = os.path.join( store_dir, 'build/darknet/x64/data/obj', os.path.basename(rgb_file_name[:-4] + '_{}.txt'.format(patch_cnt))) img = rgb[cell['h']:cell['h'] + PATCH_SIZE[0], cell['w']:cell['w'] + PATCH_SIZE[1], :] label = cell['label'] # assert np.unique(label) == ['DT'] or label == [] box = cell['box'] ersa_utils.save_file(img_name, img) write_lbl(lbl_name, label, box, PATCH_SIZE) if is_val: writer_valid.write('{}\n'.format(img_name)) else: writer_train.write('{}\n'.format(img_name)) writer_train.close() writer_valid.close()
def make_patches(files, patch_size, save_dir, overlap=0): for f in tqdm(files): tile_name = '_'.join(os.path.basename(f).split('_')[:2]) rgb = ersa_utils.load_file(f) h, w, _ = rgb.shape grid = patchExtractor.make_grid((h, w), patch_size, overlap) file_list = os.path.join(save_dir, 'file_list.txt') with open(file_list, 'w+') as f: for cnt, patch in enumerate( patchExtractor.patch_block(rgb, overlap // 2, grid, patch_size)): file_name = '{}_{:04d}.jpg'.format(tile_name, cnt) ersa_utils.save_file(os.path.join(save_dir, file_name), patch) f.write('{}\n'.format(os.path.join(save_dir, file_name)))
def make_dataset_all(rgb_files, info_dir, store_dir, tf_dir): writer_train = tf.python_io.TFRecordWriter( os.path.join(tf_dir, 'train_v2_xcity.record')) writer_valid = tf.python_io.TFRecordWriter( os.path.join(tf_dir, 'valid_v2_xcity.record')) for rgb_file_name in rgb_files: file_name = os.path.basename(rgb_file_name[:-4]) if 'NZ' not in file_name: city_id = int(file_name.split('_')[-1]) else: city_id = int(file_name.split('_')[-2]) if city_id <= 3: print('Processing file {} in validation set'.format(file_name)) is_val = True else: print('Processing file {} in training set'.format(file_name)) is_val = False rgb = ersa_utils.load_file(rgb_file_name) npy_file_name = os.path.join( info_dir, os.path.basename(rgb_file_name[:-4] + '.npy')) coords = ersa_utils.load_file(npy_file_name) patch_cnt = 0 for line in coords: for cell in line: patch_cnt += 1 save_name = os.path.join( store_dir, os.path.basename(rgb_file_name[:-4] + '_{}.jpg'.format(patch_cnt))) img = rgb[cell['h']:cell['h'] + PATCH_SIZE[0], cell['w']:cell['w'] + PATCH_SIZE[1], :3] label = cell['label'] # assert np.unique(label) == ['DT'] or label == [] box = cell['box'] ersa_utils.save_file(save_name, img) tf_example = create_tf_example(save_name, label, box) if is_val: writer_valid.write(tf_example.SerializeToString()) else: writer_train.write(tf_example.SerializeToString()) writer_train.close() writer_valid.close()
def write_data_info(rgb_files, csv_files, save_dir, patch_size, encoder): for rgb_file, csv_file in zip(rgb_files, csv_files): print('Processing data {}'.format(os.path.basename(rgb_file)[:-3])) save_name = os.path.basename(rgb_file)[:-4] + '.npy' rgb = ersa_utils.load_file(rgb_file) coords, h_steps, w_steps = extract_grids(rgb, patch_size[0], patch_size[1]) for label, y, x in read_polygon_csv_data(csv_file, encoder): h_id_0, w_id_0 = get_cell_id(y, x, h_steps, w_steps, patch_size) h_start = coords[h_id_0][w_id_0]['h'] w_start = coords[h_id_0][w_id_0]['w'] box = get_bounding_box(y - h_start, x - w_start, patch_size) coords[h_id_0][w_id_0]['label'].append(label) coords[h_id_0][w_id_0]['box'].append(box) ersa_utils.save_file(os.path.join(save_dir, save_name), coords)
def create_pred_weight_map(gt_files, save_dir, thresh=200): for gt_file in gt_files: tile_name = os.path.basename(gt_file)[:-9] gt = ersa_utils.load_file(gt_file) lbl = measure.label(gt) building_idx = np.unique(lbl) gt_wm = np.zeros_like(gt, dtype=np.uint8) for idx in building_idx[1:]: building_cord = np.where((lbl == idx)) building_size = np.sum(gt[building_cord]) if building_size > thresh: gt_wm[building_cord] = 2 else: gt_wm[building_cord] = 1 ersa_utils.save_file( os.path.join(save_dir, '{}_wm.tif'.format(tile_name)), gt_wm)
def write_dataset(rgb_files, info_dir, img_dir, csv_dir, patch_size, city_name): df = pd.DataFrame(columns=[ 'filenames', 'width', 'height', 'class', 'xmin', 'ymin', 'xmax', 'ymax', 'train_test' ]) for rgb_file_name in rgb_files: file_name = os.path.basename(rgb_file_name[:-4]) city_id = int(file_name.split('_')[-1]) if city_id <= 3: print('Processing file {} in validation set'.format(file_name)) train_test = 'test' else: print('Processing file {} in training set'.format(file_name)) train_test = 'train' rgb = ersa_utils.load_file(rgb_file_name) npy_file_name = os.path.join( info_dir, os.path.basename(rgb_file_name[:-4] + '.npy')) coords = ersa_utils.load_file(npy_file_name) patch_cnt = 0 record_cnt = 0 for line in coords: for cell in line: patch_cnt += 1 img_name = os.path.basename(rgb_file_name[:-4] + '_{}.png'.format(patch_cnt)) save_name = os.path.join(img_dir, img_name) img = rgb[cell['h']:cell['h'] + patch_size[0], cell['w']:cell['w'] + patch_size[1], :] label = cell['label'] box = cell['box'] ersa_utils.save_file(save_name, img) if len(box) > 0: for lbl, bbox in zip(label, box): df.loc[patch_cnt] = [ img_name, patch_size[0], patch_size[1], lbl, bbox[1], bbox[0], bbox[3], bbox[2], train_test ] record_cnt += 1 df.to_csv(os.path.join(csv_dir, 'labels_{}.csv'.format(city_name)), index=False)
def process(self, **kwargs): """ Extract the patches :param kwargs: file_list: list of files :return: """ pbar = tqdm(kwargs['file_list']) for files in pbar: pbar.set_description('Adjusting {} with gamma={}'.format( os.path.basename(files), self.gamma)) img = ersa_utils.load_file(files) img = self.adjust_gamma(img, self.gamma) old_name = os.path.basename(files) tile_name = '{}_gamma{}.{}'.format( old_name.split('.')[0], ersa_utils.float2str(self.gamma), old_name.split('.')[1]) tile_name = os.path.join(self.path, tile_name) ersa_utils.save_file(tile_name, img.astype(np.uint8))
def write_data_info(rgb_files, csv_files, save_dir): for rgb_file, csv_file in zip(rgb_files, csv_files): print('Processing data {} ...'.format(os.path.basename(rgb_file)[:-3]), end='') city_name = os.path.basename(rgb_file[:-4]).split('_')[2] city_id = os.path.basename(rgb_file[:-4]).split('_')[-1] rgb_save_name = '{}{}_RGB.tif'.format(city_name, city_id) gt_save_name = '{}{}_GT.png'.format(city_name, city_id) rgb = ersa_utils.load_file(rgb_file) h, w = rgb.shape[:2] gt = np.zeros((h, w), dtype=np.uint8) for label, y, x in read_polygon_csv_data(csv_file): gt[y, x] = ENCODER[label] ersa_utils.save_file(os.path.join(save_dir, rgb_save_name), rgb) ersa_utils.save_file(os.path.join(save_dir, gt_save_name), gt) print('Done!')
def make_groundtruth(): data_dir = r'/home/lab/Documents/bohao/data/transmission_line/raw2' csv_files_temp = natsorted(glob(os.path.join(data_dir, '*.csv'))) csv_files = [] for c in csv_files_temp: if 'NZ' in c: if 'resize' in c: csv_files.append(c) else: csv_files.append(c) rgb_files = natsorted(glob(os.path.join(data_dir, '*.tif'))) assert len(csv_files) == len(rgb_files) for cn, rn in zip(csv_files, rgb_files): rgb = ersa_utils.load_file(rn) h, w = rgb.shape[:2] gt = np.zeros((h, w), dtype=np.uint8) for label, y, x in read_polygon_csv_data(cn): gt[y, x] = ENCODER[label] gt_file_name = os.path.join(data_dir, '{}_multiclass.tif'.format(os.path.splitext(os.path.basename(rn))[0])) ersa_utils.save_file(gt_file_name, gt)
def process(self, **kwargs): """ Extract the patches :param kwargs: file_list: list of files :return: """ pbar = tqdm(kwargs['file_list']) for files in pbar: pbar.set_description('Upsampling {} to {}*{}'.format(os.path.basename(files), self.up_size[0], self.up_size[1])) old_name = os.path.basename(files) tile_name = '{}_upsample{}.{}'.format(old_name.split('.')[0], self.up_size[0], old_name.split('.')[1]) tile_name = os.path.join(self.path, tile_name) old_file = ersa_utils.load_file(files) if len(old_file.shape) == 3: new_file = scipy.misc.imresize(old_file, self.up_size, 'bilinear') else: new_file = scipy.misc.imresize(old_file, self.up_size, 'nearest') ersa_utils.save_file(tile_name, new_file)
def get_height_width_list(rgb_files, csv_files, save_dir): for rgb_file, csv_file in zip(rgb_files, csv_files): city_name = os.path.basename(rgb_file[:-4]).split('_')[2] city_id = os.path.basename(rgb_file[:-4]).split('_')[-1] print('Processing {}_{} ...'.format(city_name, city_id), end='') height_list = [] width_list = [] for label, y, x in read_polygon_csv_data(csv_file, encoder): height, width = get_bounding_box(y, x) height_list.append(height) width_list.append(width) height_width_stats = np.stack([height_list, width_list]) ersa_utils.save_file( os.path.join( save_dir, 'boxes_height_width_stats_{}{}.npy'.format(city_name, city_id)), height_width_stats) print('Done!')
def create_polygon_fig(csv_files, patch_size=(300, 300)): """ Create and save the polygon ground truth image :param csv_files: list of csv files :param patch_size: used if the ground truth file is empty :return: """ for csv_file in tqdm(csv_files): df = pd.read_csv(csv_file) if not df.empty: gt = np.zeros((df['height'][0], df['width'][0])) for name, group in df.groupby('Object', sort=False): y, x = polygon(group['Y'].values, group['X'].values) y, x = check_bounds(y, x, df['height'][0] - 1, df['width'][0] - 1) gt[y, x] = 1 else: gt = np.zeros((patch_size)) save_name = csv_file[:-7] + 'GT.png' ersa_utils.save_file(save_name, gt.astype(np.uint8))
def write_data_info(rgb_files, csv_files, save_dir): for rgb_file, csv_file in zip(rgb_files, csv_files): print('Processing data {}'.format(os.path.basename(rgb_file)[:-3])) save_name = os.path.basename(rgb_file)[:-4] + '.npy' rgb = ersa_utils.load_file(rgb_file) coords, h_steps, w_steps = extract_grids(rgb, PATCH_SIZE[0], PATCH_SIZE[1]) for label, y, x in read_polygon_csv_data(csv_file): if len(x) and len(y): h_id_0, w_id_0 = get_cell_id(y, x, h_steps, w_steps) h_start = coords[h_id_0][w_id_0]['h'] w_start = coords[h_id_0][w_id_0]['w'] box = get_bounding_box(y - h_start, x - w_start) # FIXME only label them as T #coords[h_id_0][w_id_0]['label'].append(label) coords[h_id_0][w_id_0]['label'].append('T') coords[h_id_0][w_id_0]['box'].append(box) else: print(csv_file) ersa_utils.save_file(os.path.join(save_dir, save_name), coords)
def check_stats(check_fig=False): root_dir = r'/home/lab/Documents/bohao/data/aemo_all/align/0584270470{}0_01' data_dir = r'/media/ei-edl01/data/aemo/TILES/' for fl, p_dir in get_file_list(root_dir): rgb_file_dir = os.path.join(data_dir, '/'.join(p_dir.split('/')[8:])) pred_save_dir = os.path.join(task_dir, 'aemo_all', '/'.join(p_dir.split('/')[7:])) pred_files = sorted(glob(os.path.join(pred_save_dir, '*.png'))) conf_files = sorted(glob(os.path.join(pred_save_dir, '*.npy'))) for file_pred, file_conf in zip(pred_files, conf_files): print('Processing file {}'.format(file_pred)) pred = ersa_utils.load_file(file_pred) conf = ersa_utils.load_file(file_conf) rgb = ersa_utils.load_file( os.path.join(rgb_file_dir, os.path.basename(file_pred)[5:-3]) + 'tif') bm = 1 - get_blank_regions(rgb) pred = bm * pred conf = bm * conf if check_fig: visualize_utils.compare_three_figure(rgb, pred, conf) ersa_utils.save_file(file_pred, pred) ersa_utils.save_file(file_conf, conf)
def make_collection(self): """ Make meta data of the collection, including tile dimension, ground truth and rgb files list means of all channels in rgb files :return: """ # collect files selection gt_regexp = self.make_regexp(self.field_name, self.field_id, self.gt_ext, self.file_ext[-1]) gt_files = self.get_files(gt_regexp, full_path=True) # rgb data can have multiple channels and be stored in multiple files rgb_files = [] for cnt, ext in enumerate(self.rgb_ext): rgb_regexp = self.make_regexp(self.field_name, self.field_id, ext, self.file_ext[cnt]) rgb_files.append(self.get_files(rgb_regexp, full_path=True)) rgb_files = ersa_utils.rotate_list(rgb_files) # make meta_data tile_dim = ersa_utils.load_file(rgb_files[0][0]).shape[:2] channel_mean = get_channel_mean(self.raw_data_path, rgb_files) meta_data = { 'raw_data_path': self.raw_data_path, 'field_name': self.field_name, 'field_id': self.field_id, 'rgb_ext': self.rgb_ext, 'gt_ext': self.gt_ext, 'file_ext': self.file_ext, 'clc_name': self.clc_name, 'tile_dim': tile_dim, 'gt_files': gt_files, 'rgb_files': rgb_files, 'chan_mean': channel_mean, 'files': self.files } ersa_utils.save_file(os.path.join(self.clc_dir, 'meta.pkl'), meta_data)
def run(self, force_run=False, **kwargs): """ Run the process :param force_run: if True, then the process will run no matter it has completed before :param kwargs: :return: """ # check if state file exists state_exist = os.path.exists(self.state_file) # run the function if force run or haven't run before if force_run or state_exist == 0: print(('Start running {}'.format(self.name))) # write state log as incomplete with open(self.state_file, 'w') as f: f.write('Incomplete\n') # run the process self.val = self.func(**kwargs) # write state log as complete with open(self.state_file, 'w') as f: f.write('Finished\n') ersa_utils.save_file(self.save_path, self.val) else: # if haven't run before, run the process if not self.check_finish(): self.val = self.func(**kwargs) ersa_utils.save_file(self.save_path, self.val) # if already exists, load the file self.val = ersa_utils.load_file(self.save_path) # write state log as complete with open(self.state_file, 'w') as f: f.write('Finished\n') return self
def write_data_info(rgb_files, csv_files, save_dir): for rgb_file, csv_file in zip(rgb_files, csv_files): print('Processing data {} ...'.format(os.path.basename(rgb_file)[:-3]), end='') if 'NZ' in rgb_file and 'resize' in rgb_file: city_name = os.path.basename(rgb_file[:-4]).split('_')[1] if 'Palmerston North' in city_name: city_name = 'PalmerstonNorth' city_id = os.path.basename(rgb_file[:-4]).split('_')[-2] else: city_name = os.path.basename(rgb_file[:-4]).split('_')[2] city_id = os.path.basename(rgb_file[:-4]).split('_')[-1] rgb_save_name = '{}{}_RGB.tif'.format(city_name, city_id) gt_save_name = '{}{}_GT.png'.format(city_name, city_id) tw_save_name = '{}{}_TW.png'.format(city_name, city_id) rgb = ersa_utils.load_file(rgb_file) h, w = rgb.shape[:2] gt = np.zeros((h, w), dtype=np.uint8) gt_towers = np.zeros((h, w), dtype=np.uint8) for label, y, x in read_polygon_csv_data(csv_file): try: gt[x, y] = ENCODER[label] except IndexError: pass for label, y, x in read_polygon_csv_data_towers(csv_file): try: gt_towers[y, x] = ENCODER[label] except IndexError: pass # dilation kernel = np.ones((15, 15), np.uint8) gt = cv2.dilate(gt, kernel, iterations=1) '''from visualize import visualize_utils visualize_utils.compare_figures([rgb, gt, gt_towers], (1, 3), fig_size=(15, 5))''' ersa_utils.save_file(os.path.join(save_dir, rgb_save_name), rgb[:, :, :3]) ersa_utils.save_file(os.path.join(save_dir, gt_save_name), gt) ersa_utils.save_file(os.path.join(save_dir, tw_save_name), gt_towers) print('Done!')
conf, true, area, ious = cmos.scoring_func(gt_obj, cm_obj) conf_all.append(conf) true_all.append(true) area_all.append(area) ious_all.append(ious) conf_agg.append(conf) true_agg.append(true) true_fold = np.concatenate(true_all) conf_fold = np.concatenate(conf_all) area_fold = np.concatenate(area_all) ious_fold = np.concatenate(ious_all) ersa_utils.save_file(os.path.join(task_dir, '{}_true.npy'.format(mn)), true_fold) ersa_utils.save_file(os.path.join(task_dir, '{}_conf.npy'.format(mn)), conf_fold) ersa_utils.save_file(os.path.join(task_dir, '{}_area.npy'.format(mn)), area_fold) ersa_utils.save_file(os.path.join(task_dir, '{}_ious.npy'.format(mn)), ious_fold) p, r, _ = precision_recall_curve(true_fold, conf_fold) plt.plot(r[1:], p[1:], linewidth=3, label=mn + ' largest recall={:.3f}'.format(r[1])) p, r, _ = precision_recall_curve(np.concatenate(true_agg), np.concatenate(conf_agg))
for c in range(3): rgb_cnt, _ = np.histogram(rgb[:, :, c], bins=np.arange(256)) aemo[c, :] += rgb_cnt aemo = aemo / len(rgb_files) aemo[:, :2] = 0 aemo[:, -1] = aemo[:, -2] for i in range(3): aemo[i, :] = ersa_utils.savitzky_golay(aemo[i, :], 11, 2) spca_stat_file = os.path.join(task_dir, 'spca_rgb_stats.npy') spca = ersa_utils.load_file(spca_stat_file) aemo_stat_file = os.path.join(task_dir, 'aemo_panel_stats.npy') aemo = ersa_utils.load_file(aemo_stat_file) spca[:, 0] = 0 aemo[:, 0] = 0 aemo[:, -1] = aemo[:, -2] # get test data data_dir = r'/home/lab/Documents/bohao/data/aemo/aemo_pad' files = sorted(glob(os.path.join(data_dir, '*rgb.tif'))) save_dir = r'/home/lab/Documents/bohao/data/aemo/aemo_hist2' for file in tqdm(files): im_s = ersa_utils.load_file(file) im_res = cust_hist_match(aemo, spca, im_s) ersa_utils.save_file(os.path.join(save_dir, os.path.basename(file)), im_res)