#aemo_hist_files = glob(os.path.join(aemohist_dir, '*histRGB.tif'))
aemo_hist_files = glob(os.path.join(r'/home/lab/Documents/bohao/data/aemo/aemo_hist2', '*rgb.tif'))
save_file = os.path.join(task_dir, 'spca_tile_stats.npy')
def get_spcastats():
print('Extacting panel pixels in spca...')
spca_stats = np.zeros((3, 255))
for rgb_file in tqdm(spca_files):
rgb = ersa_utils.load_file(rgb_file)
for c in range(3):
cnt, _ = np.histogram(rgb[:, :, c], bins=np.arange(256))
spca_stats[c, :] += cnt
spca_stats = spca_stats / len(spca_files)
return spca_stats
spca = processBlock.ValueComputeProcess('spca_tile_stats', task_dir, save_file, get_spcastats).run(force_run).val
save_file = os.path.join(task_dir, 'aemo_tile_stats.npy')
def get_spcastats():
print('Extracting panel pixels in aemo...')
aemo_stats = np.zeros((3, 255))
for rgb_file in tqdm(aemo_files):
rgb = ersa_utils.load_file(rgb_file)
for c in range(3):
cnt, _ = np.histogram(rgb[:, :, c], bins=np.arange(256))
aemo_stats[c, :] += cnt
aemo_stats = aemo_stats / len(aemo_files)
return aemo_stats
aemo = processBlock.ValueComputeProcess('aemo_tile_stats', task_dir, save_file, get_spcastats).run(force_run).val
def get_spcastats():
spca_dir = r'/media/ei-edl01/data/uab_datasets/spca/data/Original_Tiles'
spca_files = glob(os.path.join(spca_dir, '*_RGB.jpg'))
idx = np.random.permutation(len(spca_files))
spca = np.zeros((3, 255))
for i in tqdm(idx[:100]):
rgb = ersa_utils.load_file(spca_files[i])
for c in range(3):
cnt, _ = np.histogram(rgb[:, :, c], bins=np.arange(256))
spca[c, :] += cnt
spca = spca / 100
return spca
spca = processBlock.ValueComputeProcess('spca_rgb_stats', task_dir, save_file,
get_spcastats).run().val
n_col = 1 + len(rgb_files)
fig = plt.figure(figsize=(18, 8))
c_list = ['r', 'g', 'b']
for c in range(3):
plt.subplot(3, n_col, 1 + n_col * c)
plt.bar(np.arange(255), spca[c, :], color=c_list[c])
plt.ylim([0, 40e4])
if c == 0:
plt.title('California')
for plt_cnt, rgb_file in enumerate(rgb_hist_files):
rgb = ersa_utils.load_file(rgb_file[0])
for c in range(3):
rgb_cnt, _ = np.histogram(rgb[:, :, c], bins=np.arange(256))
uniq_vals = np.sort(np.unique(np.concatenate(uniq_vals)))
ious_a = np.zeros(len(uniq_vals))
ious_b = np.zeros(len(uniq_vals))'''
uniq_vals = np.linspace(0, 1, 1000)
ious_a = np.zeros(len(uniq_vals))
ious_b = np.zeros(len(uniq_vals))
for conf, truth in zip(conf_files, truth_files):
c = ersa_utils.load_file(conf)
t = ersa_utils.load_file(truth)
for cnt, th in enumerate(tqdm(uniq_vals)):
c_th = (c > th).astype(np.int)
a, b = nn_utils.iou_metric(c_th, t, truth_val=1, divide_flag=True)
ious_a[cnt] = a
ious_b[cnt] = b
return np.stack([uniq_vals, ious_a, ious_b], axis=0)
save_file = os.path.join(task_dir, 'iou_vary_th_{}_2.npy'.format(model_name))
iou = processBlock.ValueComputeProcess('iou_vary_th_{}_2'.format(model_name),
task_dir, save_file, get_ious).run().val
ious = iou[1, :] / iou[2, :]
vals = iou[0, :]
plt.plot(vals, ious)
plt.show()
# load spca features
spca_patch_name_file = os.path.join(spca_ftr_dir, 'res50_patches.txt')
spca_patch_names = ersa_utils.load_file(spca_patch_name_file)
spca_feature_name_file = os.path.join(spca_ftr_dir, 'res50_feature.csv')
spca_feature = np.genfromtxt(spca_feature_name_file, delimiter=',')
spca_feature = spca_feature[select_patch, :]
# do tsne
features = np.concatenate([aemo_feature, spca_feature], axis=0)
save_file = os.path.join(
task_dir, 'hist_tsne_pp{}_lr{}_top{}.npy'.format(perplex, learn_rate,
top_num))
feature_encode = processBlock.ValueComputeProcess(
'hist_tsnp_pp{}_lr{}_top{}'.format(perplex, learn_rate,
top_num), task_dir, save_file,
lambda: get_tsne_features(features, perplex, learn_rate)).run().val
aemo_num = len(aemo_patch_names)
# show figure
plt.figure(figsize=(8, 6))
plt.scatter(feature_encode[:aemo_num, 0],
feature_encode[:aemo_num, 1],
label='AEMO')
plt.scatter(feature_encode[aemo_num:, 0],
feature_encode[aemo_num:, 1],
label='California')
plt.xlabel('Feature 0')
plt.ylabel('Feature 1')
plt.legend()
save_file = os.path.join(task_dir, 'spca_panel_stats.npy')
def get_spcastats():
print('Extacting panel pixels in spca...')
spca_stats = np.zeros((3, 255))
for rgb_file in tqdm(spca_files):
gt_file = rgb_file[:-7] + 'GT.png'
rgb = ersa_utils.load_file(rgb_file)
gt = ersa_utils.load_file(gt_file)
for c in range(3):
cnt, _ = np.histogram(rgb[:, :, c] * gt, bins=np.arange(256))
if np.sum(gt) > 0:
spca_stats[c, :] += cnt / np.sum(gt)
spca_stats = spca_stats / len(spca_files)
return spca_stats
spca = processBlock.ValueComputeProcess('spca_panel_stats', task_dir, save_file, get_spcastats).run().val
save_file = os.path.join(task_dir, 'aemo_panel_stats.npy')
def get_spcastats():
print('Extracting panel pixels in aemo...')
aemo_stats = np.zeros((3, 255))
for rgb_file in tqdm(aemo_files):
gt_file = rgb_file[:-7] + 'gt_d255.tif'
rgb = ersa_utils.load_file(rgb_file)
gt = ersa_utils.load_file(gt_file)
for c in range(3):
cnt, _ = np.histogram(rgb[:, :, c] * gt, bins=np.arange(256))
aemo_stats[c, :] += cnt / np.sum(gt)
aemo_stats = aemo_stats / len(aemo_files)
return aemo_stats