def process_files(save_dir, file_list, code_list):
for f, c in zip(file_list, code_list):
print('processing: {} with code {}'.format(f,c))
sub_dir = os.path.join(save_dir, '/'.join(f.split('/')[5:-1]))
ersa_utils.make_dir_if_not_exist(sub_dir)
save_name = os.path.join(sub_dir, os.path.basename(f))
rgb = ersa_utils.load_file(f)
rgb_new = makeup_aemo_img(rgb, c)
ersa_utils.save_file(save_name, rgb_new)
def match_files(save_dir, root_dir, source_dist):
for i in range(1, 6):
if i != 2:
for p, s, f in os.walk(root_dir.format(i)):
save_sub_dir = os.path.join(save_dir, '/'.join(p.split('/')[8:]))
ersa_utils.make_dir_if_not_exist(save_sub_dir)
target_dist = get_aemo_stats(p)
align_files(p, save_sub_dir, source_dist, target_dist)
else:
for p, s, f in os.walk(os.path.join(root_dir.format(i), '1')):
save_sub_dir = os.path.join(save_dir, '/'.join(p.split('/')[8:]))
ersa_utils.make_dir_if_not_exist(save_sub_dir)
target_dist = get_aemo_stats(p)
align_files(p, save_sub_dir, source_dist, target_dist)
for p, s, f in os.walk(os.path.join(root_dir.format(i), '2')):
save_sub_dir = os.path.join(save_dir, '/'.join(p.split('/')[8:]))
ersa_utils.make_dir_if_not_exist(save_sub_dir)
target_dist = get_aemo_stats(p)
align_files(p, save_sub_dir, source_dist, target_dist)
model_dir = r'/hdd/Results/towers'
weight_range = [30]
iou_list = np.zeros(len(weight_range))
IMAGE_SIZE = (500, 500)
PATH_TO_TEST_IMAGES_DIR = r'/media/ei-edl01/data/uab_datasets/towers/data/Original_Tiles'
test_gt = get_test_images(PATH_TO_TEST_IMAGES_DIR)
img_dir, task_dir = sis_utils.get_task_img_folder()
IOU_TH = 0.5
for cnt, weight in enumerate(tqdm(weight_range)):
model_name = 'confmap_uab_UnetCrop_towers_pw{}_0_PS(572, 572)_BS5_EP100_LR0.0001_DS60_DR0.1_SFN32'.format(
weight)
predicted_dir = os.path.join(task_dir, 'predicted{}'.format(model_name))
ground_truth_dir = os.path.join(task_dir,
'ground-truth{}'.format(model_name))
ersa_utils.make_dir_if_not_exist(predicted_dir)
ersa_utils.make_dir_if_not_exist(ground_truth_dir)
pred_file_names = natsorted(
glob(os.path.join(task_dir, model_name, '*.npy')))
for pred_file_name, gt_file_name in zip(pred_file_names, test_gt):
pred = ersa_utils.load_file(pred_file_name)
pred = pred
gt = ersa_utils.load_file(gt_file_name)
h_steps, w_steps = extract_grids(pred, IMAGE_SIZE[0], IMAGE_SIZE[1])
patch_cnt = 0
for h_cnt, h in enumerate(h_steps):
for w_cnt, w in enumerate(w_steps):
pred_patch = pred[h:h + IMAGE_SIZE[0], w:w + IMAGE_SIZE[1]]
gt_patch = gt[h:h + IMAGE_SIZE[0], w:w + IMAGE_SIZE[1]]
tile_size = (5000, 5000)
bs = 5
suffix = 'test'
sfn = 32
gpu = 0
nn_utils.set_gpu(gpu)
# get test data
# gammas = [2.5, 1, 2.5]
sample_id = 3
data_dir = r'/media/ei-edl01/data/aemo/samples/0584007740{}0_01'.format(
sample_id)
files = sorted(glob(os.path.join(data_dir, 'hist_match_ct', '*.tif')))
# adjust gamma
'''gamma_save_dir = os.path.join(data_dir, 'gamma_adjust')
ersa_utils.make_dir_if_not_exist(gamma_save_dir)
ga = gammaAdjust.GammaAdjust(gamma=gammas[sample_id - 1], path=gamma_save_dir)
ga.run(force_run=False, file_list=files)
files = sorted(glob(os.path.join(gamma_save_dir, '*.tif')))'''
# get image mean
img_mean = cm.get_channel_mean('', [[f] for f in files])
model_dir = r'/hdd6/Models/UNET_city/UnetCrop_spca_aug_xcity_0_PS(572, 572)_BS5_EP100_LR0.0001_DS60_DR0.1_SFN32'
# test model
'''nn_utils.tf_warn_level(3)
unet = unet.UNet(class_num, patch_size, suffix=suffix, batch_size=bs)
model_dir = r'/hdd6/Models/UNET_city/UnetCrop_spca_aug_xcity_0_PS(572, 572)_BS5_EP100_LR0.0001_DS60_DR0.1_SFN32'
# model_dir = r'/hdd6/Models/Inria_decay/UnetCrop_inria_decay_0_PS(572, 572)_BS5_EP100_LR0.0001_DS60.0_DR0.1_SFN32'
def main(flags):
np.random.seed(int(flags.run_id))
tf.set_random_seed(int(flags.run_id))
# make network
# define place holder
X = tf.placeholder(
tf.float32,
shape=[None, flags.input_size[0], flags.input_size[1], 3],
name='X')
y = tf.placeholder(
tf.int32,
shape=[None, flags.input_size[0], flags.input_size[1], 1],
name='y')
mode = tf.placeholder(tf.bool, name='mode')
model = myUnetModelCrop({
'X': X,
'Y': y
},
trainable=mode,
model_name=flags.model_name,
input_size=flags.input_size,
batch_size=flags.batch_size,
learn_rate=flags.learning_rate,
decay_step=flags.decay_step,
decay_rate=flags.decay_rate,
epochs=flags.epochs,
start_filter_num=flags.sfn)
model.create_graph('X', class_num=flags.num_classes)
# create collection
# the original file is in /ei-edl01/data/uab_datasets/inria
blCol = uab_collectionFunctions.uabCollection(flags.ds_name)
blCol.readMetadata()
img_mean = blCol.getChannelMeans([0, 1, 2]) # get mean of rgb info
print(img_mean)
img_dir, task_dir = sis_utils.get_task_img_folder()
save_dir = os.path.join(task_dir, 'bihar_patches')
ersa_utils.make_dir_if_not_exist(save_dir)
files, par_dir = blCol.getAllTileByDirAndExt([0, 1, 2, 3])
resize_patches(files, par_dir, flags.input_size, save_dir)
patchDir = save_dir
# make data reader
# use first 5 tiles for validation
idx, file_list = uabCrossValMaker.uabUtilGetFolds(patchDir, 'fileList.txt',
'tile')
# use first city for validation
#assert len(file_list) == flags.n_train + flags.n_valid
file_list_train = [a for a in file_list[:45]]
file_list_valid = [a for a in file_list[-5:]]
with tf.name_scope('image_loader'):
# GT has no mean to subtract, append a 0 for block mean
dataReader_train = uabDataReader.ImageLabelReader(
[3], [0, 1, 2],
patchDir,
file_list_train,
flags.input_size,
None,
flags.batch_size,
dataAug='flip,rotate',
block_mean=np.append([0], img_mean))
# no augmentation needed for validation
dataReader_valid = uabDataReader.ImageLabelReader([3], [0, 1, 2],
patchDir,
file_list_valid,
flags.input_size,
None,
flags.batch_size,
dataAug=' ',
block_mean=np.append(
[0], img_mean))
# train
start_time = time.time()
model.train_config('X',
'Y',
flags.n_train,
flags.n_valid,
flags.input_size,
uabRepoPaths.modelPath,
loss_type='xent',
par_dir='{}'.format(flags.ds_name))
model.run(
train_reader=dataReader_train,
valid_reader=dataReader_valid,
pretrained_model_dir=
None, # train from scratch, no need to load pre-trained model
isTrain=True,
img_mean=img_mean,
verb_step=100, # print a message every 100 step(sample)
save_epoch=200, # save the model every 5 epochs
gpu=GPU,
patch_size=flags.input_size)
duration = time.time() - start_time
print('duration {:.2f} hours'.format(duration / 60 / 60))
def process(self):
"""
Evaluate the network
:return:
"""
nn_utils.set_gpu(self.gpu)
if self.score_results:
with open(os.path.join(self.score_save_dir, 'result.txt'), 'w'):
pass
iou_record = []
# prepare the reader
if self.score_results:
init_op, reader_op = dataReaderSegmentation.DataReaderSegmentationTesting(
self.input_size,
self.tile_size,
self.file_list,
overlap=self.model.get_overlap(),
pad=self.model.get_overlap() // 2,
batch_size=self.batch_size,
chan_mean=self.img_mean,
is_train=False,
has_gt=True,
random=False,
gt_dim=1,
include_gt=True).read_op()
feature, label = reader_op
self.model.create_graph(feature, **self.kwargs)
else:
init_op, reader_op = dataReaderSegmentation.DataReaderSegmentationTesting(
self.input_size,
self.tile_size,
self.file_list,
overlap=self.model.get_overlap(),
pad=self.model.get_overlap() // 2,
batch_size=self.batch_size,
chan_mean=self.img_mean,
is_train=False,
has_gt=False,
random=False,
gt_dim=0,
include_gt=False).read_op()
feature = reader_op
self.model.create_graph(feature[0], **self.kwargs)
pad = self.model.get_overlap()
for file_cnt, (file_name_list) in enumerate(self.file_list):
file_name_truth = None
if self.score_results:
file_name, file_name_truth = file_name_list
tile_name = os.path.basename(file_name_truth).split(
self.split_char)[0]
else:
file_name = file_name_list[0]
tile_name = os.path.basename(file_name).split(
self.split_char)[0]
if self.verb:
print('Evaluating {} ... '.format(tile_name))
# read tile size if no tile size is given
if self.tile_size is None or self.compute_shape_flag:
self.compute_shape_flag = True
tile = ersa_utils.load_file(file_name)
self.tile_size = tile.shape[:2]
start_time = time.time()
# run the model
if self.model.config is None:
self.model.config = tf.ConfigProto(allow_soft_placement=True)
with tf.Session(config=self.model.config) as sess:
init = tf.global_variables_initializer()
sess.run(init)
self.model.load(self.model_dir,
sess,
epoch=self.load_epoch_num,
best_model=self.best_model)
result = self.model.test_sample(sess, init_op[file_cnt])
image_pred = patchExtractor.unpatch_block(
result,
tile_dim=[self.tile_size[0] + pad, self.tile_size[1] + pad],
patch_size=self.input_size,
tile_dim_output=self.tile_size,
patch_size_output=[
self.input_size[0] - pad, self.input_size[1] - pad
],
overlap=pad)
if self.compute_shape_flag:
self.tile_size = None
pred = nn_utils.get_pred_labels(image_pred) * self.truth_val
if self.score_results:
truth_label_img = ersa_utils.load_file(file_name_truth)
iou = nn_utils.iou_metric(truth_label_img,
pred,
divide_flag=True)
iou_record.append(iou)
duration = time.time() - start_time
if self.verb:
print('{} mean IoU={:.3f}, duration: {:.3f}'.format(
tile_name, iou[0] / iou[1], duration))
# save results
pred_save_dir = os.path.join(self.score_save_dir, 'pred')
ersa_utils.make_dir_if_not_exist(pred_save_dir)
ersa_utils.save_file(
os.path.join(pred_save_dir, '{}.png'.format(tile_name)),
pred.astype(np.uint8))
if self.score_results:
with open(os.path.join(self.score_save_dir, 'result.txt'),
'a+') as file:
file.write('{} {}\n'.format(tile_name, iou))
if self.score_results:
iou_record = np.array(iou_record)
mean_iou = np.sum(iou_record[:, 0]) / np.sum(iou_record[:, 1])
print('Overall mean IoU={:.3f}'.format(mean_iou))
with open(os.path.join(self.score_save_dir, 'result.txt'),
'a+') as file:
file.write('{}'.format(mean_iou))
def process(self):
"""
Evaluate the network
:return:
"""
nn_utils.set_gpu(self.gpu)
if self.score_results:
with open(os.path.join(self.score_save_dir, 'result.txt'), 'w'):
pass
iou_record = []
# prepare the reader
if self.score_results:
feature, label = self.reader_op
self.model.create_graph(feature, **self.kwargs)
else:
feature = self.reader_op
self.model.create_graph(feature[0], **self.kwargs)
# run the model
with tf.Session() as sess:
init = tf.global_variables_initializer()
sess.run([init, self.init_op])
self.model.load(self.model_dir,
sess,
epoch=self.load_epoch_num,
best_model=self.best_model)
for file_cnt, (file_name_list) in enumerate(self.file_list):
file_name_truth = None
if self.score_results:
file_name, file_name_truth = file_name_list
tile_name = os.path.basename(file_name_truth).split(
self.split_char)[0]
else:
file_name = file_name_list[0]
tile_name = os.path.basename(file_name).split(
self.split_char)[0]
if self.verb:
print('Evaluating {} ... '.format(tile_name))
start_time = time.time()
result = sess.run(self.model.output)
pred = np.argmax(np.squeeze(result, axis=0), axis=-1)
if self.post_func is not None:
pred = self.post_func(pred)
if self.save_func is not None:
save_img = self.save_func(pred)
else:
save_img = pred
if self.score_results:
truth_label_img = ersa_utils.load_file(file_name_truth)
iou = nn_evals.mean_IU(pred, truth_label_img,
self.ignore_label)
iou_record.append(iou)
duration = time.time() - start_time
if self.verb:
print('{} mean IoU={:.3f}, duration: {:.3f}'.format(
tile_name, iou, duration))
# save results
pred_save_dir = os.path.join(self.score_save_dir, 'pred')
ersa_utils.make_dir_if_not_exist(pred_save_dir)
ersa_utils.save_file(
os.path.join(pred_save_dir,
'{}.png'.format(tile_name)),
save_img.astype(np.uint8))
if self.score_results:
with open(
os.path.join(self.score_save_dir,
'result.txt'), 'a+') as file:
file.write('{} {}\n'.format(tile_name, iou))
if self.score_results:
iou_record = np.array(iou_record)
mean_iou = np.mean(iou_record)
print('Overall mean IoU={:.3f}'.format(mean_iou))
with open(os.path.join(self.score_save_dir, 'result.txt'),
'a+') as file:
file.write('{}'.format(mean_iou))
batch_size = 1
input_size = [572, 572]
city_name = 'DC'
nn_utils.tf_warn_level(3)
if city_name == 'Atlanta':
tile_size = [2000, 3000]
elif city_name == 'Austin':
tile_size = [2000, 3000]
elif city_name == 'DC':
tile_size = [2500, 2500]
img_dir, task_dir = sis_utils.get_task_img_folder()
path_to_save = os.path.join(task_dir, 'dtda_new', city_name, 'valid')
ersa_utils.make_dir_if_not_exist(path_to_save)
model_dir = r'/hdd6/Models/UNET_rand_gird/UnetCrop_inria_aug_grid_0_PS(572, 572)_BS5_' \
r'EP100_LR0.0001_DS60_DR0.1_SFN32'
tf.reset_default_graph()
blCol = uab_collectionFunctions.uabCollection(city_name)
blCol.readMetadata()
file_list, parent_dir = blCol.getAllTileByDirAndExt([0, 1, 2])
file_list_truth, parent_dir_truth = blCol.getAllTileByDirAndExt(3)
img_mean = blCol.getChannelMeans([0, 1, 2])
# make the model
# define place holder
X = tf.placeholder(tf.float32,
def make_dataset(is_dcc, city_name, patch_size, encoder):
# settings
if not is_dcc:
data_dir = r'/home/lab/Documents/bohao/data/transmission_line'
else:
data_dir = r'/work/bh163/misc/object_detection'
base_dir = os.path.join(data_dir, 'yolo')
save_dir = os.path.join(base_dir, 'temp')
img_dir = os.path.join(base_dir, 'images', 'all')
csv_dir = os.path.join(base_dir, 'csv')
ersa_utils.make_dir_if_not_exist(img_dir)
ersa_utils.make_dir_if_not_exist(csv_dir)
ersa_utils.make_dir_if_not_exist(save_dir)
ersa_utils.make_dir_if_not_exist(os.path.join(base_dir, 'images', 'train'))
ersa_utils.make_dir_if_not_exist(os.path.join(base_dir, 'images', 'test'))
ersa_utils.make_dir_if_not_exist(os.path.join(base_dir, 'labels', 'train'))
ersa_utils.make_dir_if_not_exist(os.path.join(base_dir, 'labels', 'test'))
# get files
rgb_files = natsorted([
a for a in glob(os.path.join(data_dir, 'raw', '*.tif'))
if 'multiclass' not in a and city_name in a
])
csv_files = natsorted([
a for a in glob(os.path.join(data_dir, 'raw', '*.csv'))
if city_name in a
])
write_data_info(rgb_files, csv_files, save_dir, patch_size, encoder)
write_dataset(rgb_files, save_dir, img_dir, csv_dir, patch_size, city_name)
od_graph_def.ParseFromString(serialized_graph)
tf.import_graph_def(od_graph_def, name='')
for tile_id in [1, 2, 3]:
# get prediction
raw_rgb = ersa_utils.load_file(
os.path.join(raw_dir,
'USA_{}_{}.tif'.format(city_list[city_id], tile_id)))
pred_files = natsorted(
glob(os.path.join(task_fold, pred_dir,
'*_{}.txt'.format(tile_id))))
npy_file_name = os.path.join(
info_dir, 'USA_{}_{}.npy'.format(city_list[city_id], tile_id))
coords = ersa_utils.load_file(npy_file_name)
text_save_dir = os.path.join(task_dir, 'faster_rcnn_all')
ersa_utils.make_dir_if_not_exist(text_save_dir)
text_file_name = os.path.join(
text_save_dir, 'USA_{}_{}.txt'.format(city_list[city_id], tile_id))
h_steps, w_steps = extract_grids(raw_rgb, patch_size[0], patch_size[1])
with open(text_file_name, 'w+') as f:
patch_cnt = 0
for h_cnt, h in enumerate(tqdm(h_steps)):
for w_cnt, w in enumerate(tqdm(w_steps)):
image_patch = raw_rgb[h:h + patch_size[0],
w:w + patch_size[1], :]
image_np_expanded = np.expand_dims(image_patch, axis=0)
output_dict = run_inference_for_single_image(
image_patch, detection_graph)
for left, top, right, bottom, confidence, class_name in get_predict_info(
output_dict, patch_size, (h, w), ('T', ), th):
data_dir = r'/home/lab/Documents/bohao/data/transmission_line/raw'
city_list = ['Tucson', 'Colwich', 'Clyde', 'Wilmington']
img_dir, task_dir = sis_utils.get_task_img_folder()
tower_vals = [3, 1]
tower_names = ['Transmission_Tower', 'Distribution_Tower']
total_counts = np.zeros((len(tower_names), len(city_list)))
tower_dict = {}
for cn in city_list:
tower_type = {}
for tt in tower_names:
tower_type[tt] = [0]
tower_dict[cn] = tower_type
for tn in tower_names:
ersa_utils.make_dir_if_not_exist(os.path.join(img_dir, tn))
for city_id in range(len(city_list)):
gt_files = natsorted(glob(os.path.join(data_dir, '*{}*_multiclass.tif'.format(city_list[city_id]))))
rgb_files = ['_'.join(a.split('_')[:-1])+'.tif' for a in gt_files]
for rgb_file_name, gt_file_name in zip(rgb_files, gt_files):
rgb = ersa_utils.load_file(rgb_file_name)
gt = ersa_utils.load_file(gt_file_name)
prefix = os.path.basename(rgb_file_name)[7:-4]
print('{:<20}: '.format(prefix), end='')
for t_cnt, (tower_val, tower_name) in enumerate(zip(tower_vals, tower_names)):
binary_map = (gt == tower_val).astype(np.int)
tower_cnts = count_objects(binary_map)
print('{:<15} {:<4}\t'.format(tower_name, tower_cnts), end='')
total_counts[t_cnt][city_id] += tower_cnts
dirs['conf'].format(conf_dict[city_id]),
'{}{}.png'.format(city_list[city_id].split('_')[1], tile_id)))
line_gt = ersa_utils.load_file(
os.path.join(
dirs['line'],
'{}{}_GT.png'.format(city_list[city_id].split('_')[1], tile_id)))
tower_gt = get_tower_truth_pred(dirs, city_id, tile_id)
tower_pred, tower_conf, _ = local_maxima_suppression(preds, th=merge_range)
conf_img = scipy.misc.imresize(conf_img, line_gt.shape)
return preds, raw_rgb, conf_img, line_gt, tower_gt, tower_pred, tower_conf
if __name__ == '__main__':
# directories
img_dir, task_dir = sis_utils.get_task_img_folder()
ersa_utils.make_dir_if_not_exist(os.path.join(img_dir, 'new_annotation'))
dirs = {
'task':
task_dir,
'image':
os.path.join(img_dir, 'new_annotation'),
'raw':
r'/home/lab/Documents/bohao/data/transmission_line/raw2',
'conf':
r'/media/ei-edl01/user/bh163/tasks/2018.11.16.transmission_line/confmap_uab_UnetCrop_linesv3_city0_'
r'pw50_0_PS(572, 572)_BS5_EP100_LR0.0001_DS80_DR0.1_SFN32',
'line':
r'/media/ei-edl01/data/uab_datasets/lines_v3/data/Original_Tiles'
}
# settings
def eval_tiles():
blCol = uab_collectionFunctions.uabCollection(ds_name)
blCol.readMetadata()
file_list, parent_dir = blCol.getAllTileByDirAndExt([1, 2, 3])
file_list_truth, parent_dir_truth = blCol.getAllTileByDirAndExt(0)
idx, file_list = uabCrossValMaker.uabUtilGetFolds(None, file_list, 'tile')
idx_truth, file_list_truth = uabCrossValMaker.uabUtilGetFolds(
None, file_list_truth, 'tile')
# use first 5 tiles for validation
file_list_valid = uabCrossValMaker.make_file_list_by_key(
idx, file_list, [0, 1, 2, 3, 4, 5])
file_list_valid_truth = uabCrossValMaker.make_file_list_by_key(
idx_truth, file_list_truth, [0, 1, 2, 3, 4, 5])
img_mean = blCol.getChannelMeans([1, 2, 3])
# make the model
# define place holder
X = tf.placeholder(tf.float32,
shape=[None, input_size[0], input_size[1], 3],
name='X')
y = tf.placeholder(tf.int32,
shape=[None, input_size[0], input_size[1], 1],
name='y')
mode = tf.placeholder(tf.bool, name='mode')
model = UnetModelCrop({
'X': X,
'Y': y
},
trainable=mode,
input_size=input_size,
batch_size=batch_size,
start_filter_num=32)
# create graph
model.create_graph('X', class_num=2)
# evaluate on each sub folder
root_dir = r'/home/lab/Documents/bohao/data/aemo_all/align/0584270470{}0_01'
for fl, p_dir in get_file_list(root_dir):
for f in fl:
print('Evaluating {} in {}'.format(f, p_dir))
pred_save_dir = os.path.join(task_dir, 'aemo_all',
'/'.join(p_dir.split('/')[7:]))
ersa_utils.make_dir_if_not_exist(pred_save_dir)
# prepare the reader
reader = uabDataReader.ImageLabelReader(
gtInds=[0],
dataInds=[0],
nChannels=3,
parentDir=p_dir,
chipFiles=[[f]],
chip_size=input_size,
tile_size=tile_size,
batchSize=batch_size,
block_mean=img_mean,
overlap=model.get_overlap(),
padding=np.array(
(model.get_overlap() / 2, model.get_overlap() / 2)),
isTrain=False)
rManager = reader.readManager
# run the model
pred, conf_map = model.run(pretrained_model_dir=model_dir,
test_reader=rManager,
tile_size=tile_size,
patch_size=input_size,
gpu=gpu,
load_epoch_num=75,
best_model=False)
pred_name = 'pred_{}.png'.format(f.split('.')[0])
conf_name = 'conf_{}.npy'.format(f.split('.')[0])
ersa_utils.save_file(os.path.join(pred_save_dir, pred_name),
pred.astype(np.uint8))
ersa_utils.save_file(os.path.join(pred_save_dir, conf_name),
conf_map)
fig = make_cmp_plot(rgb[loc], gt[loc], '{}_{}'.format(title_str, cnt),
building_size)
yield fig, building_size
spca_dir = r'/media/ei-edl01/data/uab_datasets/spca/data/Original_Tiles'
aemo_dir = r'/home/lab/Documents/bohao/data/aemo/aemo_pad'
aemohist_dir = r'/hdd/ersa/preprocess/aemo_pad/hist_matching'
img_dir, task_dir = sis_utils.get_task_img_folder()
spca_files = glob(os.path.join(spca_dir, '*_RGB.jpg'))
idx = np.random.permutation(20)
spca_files = [spca_files[i] for i in idx]
aemo_files = glob(os.path.join(aemo_dir, '*rgb.tif'))
aemo_hist_files = glob(os.path.join(aemohist_dir, '*histRGB.tif'))
ersa_utils.make_dir_if_not_exist(os.path.join(img_dir, 'spca'))
ersa_utils.make_dir_if_not_exist(os.path.join(img_dir, 'aemo'))
ersa_utils.make_dir_if_not_exist(os.path.join(img_dir, 'aemohist'))
for spca_file in spca_files:
title_str = os.path.basename(spca_file).split('_')[0]
rgb = ersa_utils.load_file(spca_file)
gt_file = spca_file[:-7] + 'GT.png'
gt = ersa_utils.load_file(gt_file)
for cnt, (fig, _) in enumerate(visualize_fn_object(rgb, gt, title_str)):
plt.savefig(
os.path.join(img_dir, 'spca', '{}_{}.png'.format(title_str, cnt)))
plt.close(fig)
file_list_valid = uabCrossValMaker.make_file_list_by_key(
idx, file_list, [1, 2, 3])
file_list_valid_truth = uabCrossValMaker.make_file_list_by_key(
idx_truth, file_list_truth, [1, 2, 3])
img_mean = blCol.getChannelMeans([2, 3, 4])
img_mean = np.concatenate([np.array([0]), img_mean])
# make the model
# define place holder
for weight in [50]:
for city_id in range(4):
model_dir = r'/hdd6/Models/lines_tw1/UnetCrop_lines_city{}_pw{}_0_PS(572, 572)_BS5_' \
r'EP50_LR0.0001_DS30_DR0.1_SFN32'.format(city_id, weight)
SAVE_DIR = os.path.join(
task_dir, 'confmap_uab_{}'.format(os.path.basename(model_dir)))
ersa_utils.make_dir_if_not_exist(SAVE_DIR)
tf.reset_default_graph()
X = tf.placeholder(tf.float32,
shape=[None, input_size[0], input_size[1], 4],
name='X')
y = tf.placeholder(tf.int32,
shape=[None, input_size[0], input_size[1], 1],
name='y')
mode = tf.placeholder(tf.bool, name='mode')
model = UnetModelCrop({
'X': X,
'Y': y
},
trainable=mode,
input_size=input_size,
batch_size=batch_size,
return fig, iou, size
if __name__ == '__main__':
window_size = 200
stride = 200
img_dir, task_dir = sis_utils.get_task_img_folder()
good_th = 80
bad_th = 5
truth_dir = r'/media/ei-edl01/data/uab_datasets/aemo_comb/data/Original_Tiles'
ft_dir = r'/hdd/Results/aemo/uab/UnetCrop_aemo_comb_xfold2_1_PS(572, 572)_BS5_EP80_LR0.001_DS30_DR0.1_SFN32/aemo_comb/pred'
img_save_dir = os.path.join(img_dir, 'uab_pred_patches_comb',
ft_dir.split('/')[-3])
ersa_utils.make_dir_if_not_exist(img_save_dir)
eval_files = [
'_'.join(os.path.basename(f)[:-4].split('_')[:2])
for f in glob(os.path.join(ft_dir, '*.png'))
]
for f in eval_files:
rgb = imageio.imread(os.path.join(truth_dir, '{}_RGB.tif'.format(f)))
truth_img_name = os.path.join(truth_dir, '{}_GT.tif'.format(f))
ft_img_name = os.path.join(ft_dir, '{}.png'.format(f))
truth, ft = imageio.imread(truth_img_name), imageio.imread(ft_img_name)
assert np.all(np.unique(truth) == np.array([0, 1])) and np.all(
np.unique(ft) == np.array([0, 1]))
'image/object/bbox/ymin': dataset_util.float_list_feature(ymins),
'image/object/bbox/ymax': dataset_util.float_list_feature(ymaxs),
'image/object/class/text': dataset_util.bytes_list_feature(classes_text),
'image/object/class/label': dataset_util.int64_list_feature(classes),
}))
return tf_example
if __name__ == '__main__':
# settings
if not IS_DCC:
data_dir = r'/home/lab/Documents/bohao/data/transmission_line'
save_dir = os.path.join(data_dir, 'info')
store_dir = os.path.join(data_dir, 'patches')
tf_dir = os.path.join(data_dir, 'data')
ersa_utils.make_dir_if_not_exist(save_dir)
ersa_utils.make_dir_if_not_exist(store_dir)
ersa_utils.make_dir_if_not_exist(tf_dir)
else:
data_dir = r'/work/bh163/misc/object_detection'
save_dir = os.path.join(data_dir, 'info')
store_dir = os.path.join(data_dir, 'patches')
tf_dir = os.path.join(data_dir, 'data')
ersa_utils.make_dir_if_not_exist(save_dir)
ersa_utils.make_dir_if_not_exist(store_dir)
ersa_utils.make_dir_if_not_exist(tf_dir)
# get files
rgb_files = natsorted([a for a in glob(os.path.join(data_dir, 'raw', '*.tif'))
if 'multiclass' not in a and CITY_NAME in a])
csv_files = natsorted([a for a in glob(os.path.join(data_dir, 'raw', '*.csv')) if CITY_NAME in a])
writer = csv.writer(f, lineterminator='\n')
writer.writerow(['{}'.format(x) for x in fc1000])
f2.write('{}\n'.format(patch_name))
if __name__ == '__main__':
# settings
patch_size = (572, 572)
tile_size = (5000, 5000)
np.random.seed(1004)
gpu = 0
img_dir, task_dir = sis_utils.get_task_img_folder()
use_hist = False
cm = collectionMaker.read_collection('spca')
cm.print_meta_data()
chan_mean = cm.meta_data['chan_mean'][:3]
file_list = cm.load_files(field_id=','.join(str(i) for i in range(0, 663)),
field_ext='RGB,GT')
patch_list_train = patchExtractor.PatchExtractor(patch_size, tile_size, 'spca_all',
184, 184 // 2). \
run(file_list=file_list, file_exts=['jpg', 'png'], force_run=False).get_filelist()
feature_dir = os.path.join(task_dir, 'spca_patches')
dr = data_reader(patch_list_train, chan_mean)
ersa_utils.make_dir_if_not_exist(feature_dir)
processBlock.BasicProcess(
'make_feature',
feature_dir,
func=lambda: make_res_features(dr, feature_dir, gpu)).run()