def get_data_mean_std(path, data_type='c3'):
''' deprecated
get mean and std value of a whole dataset of PolSAR data,
assume that the data distribution is even among all the PolSAR files, which is actually not
'''
# first, get mean value
print('calculating mean value')
mean = np.zeros(6, dtype=np.complex64)
num_caled = 0
for root, dirs, files in os.walk(path):
if 'C3' == root[-2:]:
c3 = psr.read_c3(root, out='complex_vector_6', is_print=True)
num_now = c3.shape[1] * c3.shape[2]
mean_now = c3.mean(axis=(-1, -2))
mean = num_caled / (num_caled + num_now) * mean + num_now / (
num_now + num_caled) * mean_now
num_caled += num_now
print(f'mean of whole: {mean}\nmean of now: {mean_now}')
# second get std value
print('calculating stc value')
var = np.zeros(6) #should be a real number
num_caled = 0
for root, _, _ in os.walk(path):
if 'C3' in root:
c3 = psr.read_c3(root, out='complex_vector_6', is_print=True)
num_now = c3.shape[1] * c3.shape[2]
var_now = np.sum(np.abs(c3 - mean)**2) / num_now
var = num_caled / (num_caled + num_now) * var + num_now / (
num_caled + num_now) * var_now
num_caled += num_now
std = np.sqrt(var)
return mean, std
def extract_H_A_alpha_span(path):
''' Extract H/A/alpha/Span data
Args:
path (str): to the the H/A/alpha data
'''
if osp.isdir(path):
path = osp.join(path, 'unnormed.npy')
if not osp.isfile(path):
raise IOError(f'{path} is not a valid path')
HAalpha = np.load(path)
assert HAalpha.shape[0] == 3, 'Wrong shape of H/A/ahpha data'
c3 = psr.read_c3(osp.dirname(path.replace('HAalpha', 'C3')),
out='save_space')
assert np.array_equal(
c3.shape[1:],
HAalpha.shape[1:]), 'Unmatched C3 and H/A/alpha data pair'
span = c3[0, ...] + c3[5, ...] + c3[8, ...]
span[span < mathlib.eps] = mathlib.eps
span = np.expand_dims(np.log(span), 0)
HAalphaSpan = np.concatenate((HAalpha, span), axis=0)
dst_file = path.replace('HAalpha', 'HAalphaSpan')
print(f'extract H/A/alpha/Span data from {path} to {dst_file}')
fu.mkdir_if_not_exist(osp.dirname(dst_file))
np.save(dst_file, HAalphaSpan)
def hoekman(src_path: str, norm=False) -> None:
''' Hoekman decomposition and save to file, the hoekman_and_norm() is an older version of this func, and is a subset of this func
@in -src_path -source path, where should contains 'C3' folder
@in -norm -normalize or not, default: false
'''
if 'C3' in os.listdir(src_path):
# read C3 file
print(f'hoekman on dir (norm={norm}): {src_path}', end='')
c3 = psr.read_c3(osp.join(src_path, 'C3'))
h = psr.Hokeman_decomposition(c3)
dst_path = osp.join(src_path, 'Hoekman')
fu.mkdir_if_not_exist(dst_path)
# np.save(osp.join(dst_path, 'ori'), h) # save the unnormalized file
# normalize
if norm:
for ii in range(9):
h[ii, :, :] = psr.min_max_contrast_median_map(
10 * np.log10(h[ii, :, :]))
# cv2.imwrite(osp.join(dst_path, f'{ii}.jpg'), (h[0, :, :]*255).astype(np.uint8))
# plt.hist() can take very long time to process a 2D array, but little time to process a 1D array, so flatten the array if possible
# plt.hist(h[ii, :, :].flatten())
# plt.savefig(osp.join(dst_path, f'log-hist-{ii}.jpg'))
# save to file
if norm:
np.save(osp.join(dst_path, 'normed'), h)
else:
np.save(osp.join(dst_path, 'unnormed'), h)
print('\tdone')
else:
raise ValueError('wrong src path')
def check_data_value_scale(path):
''' check the scale of PolSAR data value '''
for root, _, _ in os.walk(path):
if 'C3' == root[-2:]:
c3 = psr.read_c3(root, out='save_space')
if c3.mean() > 0:
print(f'{root} : mean value is {c3.mean()}')
def __getitem__(self, index):
''' 先不管高分三号的数据集，先弄RADATSAT-2的数据 '''
# generate label and its mask
label_path = self.labels_path[index]
label = lbm.read_change_label_png(label_path) - 1
mask = label < 2 # 1 表示存在有效标记，0表示没有标记
label[~mask] = 0 # 1 表示存在变化，0表示没有变化或没有标记数据
# get the file path
label_dir = osp.split(label_path)[0]
# two date time display format
if osp.isfile(osp.join(label_dir, 'pin.txt')): # mode 1
re_exp = r'20\d{6}'
elif osp.isfile(osp.join(label_dir, 'pin-2.txt')): # mode 2
re_exp = r'\d{2}[A-Z][a-z]{2}\d{4}'
else:
raise ValueError('unknown mode of label format')
[a_time, b_time] = re.findall(re_exp, label_path)
file_path = []
file_path.append(
osp.join(label_dir.replace('label', 'data'), a_time, 'C3'))
file_path.append(
osp.join(label_dir.replace('label', 'data'), b_time, 'C3'))
# get the file data
slice_idx = re.search(r'-\d{4}-', label_path)
if slice_idx is None:
raise ValueError('can not find the wave code')
slice_idx = int(slice_idx.group()[1:5])
file = []
for ii in range(2):
# meta_info = psr.read_hdr(file_path[ii])
# het = meta_info['lines']
# wes = meta_info['samples']
# patch_y, patch_x = lbm.get_corrds_from_slice_idx([het, wes], (512,512), slice_idx)
file.append(
psr.read_c3(osp.join(file_path[ii], str(slice_idx)),
out=self.data_format)[[0, 5, 8]])
# patch_y, patch_x = int(patch_y), int(patch_x)
# file_a = file_a[:, patch_y: patch_y+512, patch_x:patch_x+512]
# file_b = psr.read_c3(file_b_path, out=self.data_format)[:, patch_y: patch_y+512, patch_x:patch_x+512]
# clip data
for ii in range(2):
file[ii][file[ii] > 1] = 1
file[ii][file[ii] < -1] = -1
return torch.from_numpy(file[0]), torch.from_numpy(
file[1]), torch.from_numpy(label).long(), torch.from_numpy(mask)
def get_files_data(self, index):
''' read file a and file b data, in torch.tensor format
if the data is images, then it is normed into [0,1 ],
'''
label_path = self.labels_path[index]
# get the file path
label_dir = osp.split(label_path)[0]
# two date time display format
if osp.isfile(osp.join(label_dir, 'pin.txt')): # mode 1
re_exp = r'20\d{6}'
elif osp.isfile(osp.join(label_dir, 'pin-2.txt')): # mode 2
re_exp = r'\d{2}[A-Z][a-z]{2}\d{4}'
else:
raise ValueError('unknown mode of label format')
[a_time, b_time] = re.findall(re_exp, label_path)
files_path = []
files_path.append(
osp.join(label_dir.replace('label', 'data'), a_time, 'C3'))
files_path.append(
osp.join(label_dir.replace('label', 'data'), b_time, 'C3'))
# get the file data
slice_idx = re.search(r'-\d{4}-', label_path)
if slice_idx is None:
raise ValueError('can not find the wave code')
slice_idx = int(slice_idx.group()[1:5])
files = []
if self.data_type == 'original':
for ii in range(2):
files.append(
torch.from_numpy(
psr.read_c3(osp.join(files_path[ii], str(slice_idx)),
out=self.data_format)))
elif self.data_type == 'pauli':
for ii in range(2):
files.append(
psr.read_bmp(osp.join(files_path[ii], str(slice_idx))))
if self.to_tensor:
for ii in range(2):
files[ii] = self.tf(files[ii])
else:
for ii in range(2):
files[ii] = files[ii].permute(2, 0, 1)
else:
raise NotImplementedError
return files
def __getitem__(self, index: int):
label, mask = self.get_label_and_mask(index)
# get the file path
label_path = self.labels_path[index]
label_dir = osp.split(label_path)[0]
# two date time display format
if osp.isfile(osp.join(label_dir, 'pin.txt')): # mode 1
re_exp = r'20\d{6}'
elif osp.isfile(osp.join(label_dir, 'pin-2.txt')): # mode 2
re_exp = r'\d{2}[A-Z][a-z]{2}\d{4}'
else:
raise ValueError('unknown mode of label format')
[a_time, b_time] = re.findall(re_exp, label_path)
file_path = []
file_path.append(
osp.join(label_dir.replace('label', 'data'), a_time, 'C3'))
file_path.append(
osp.join(label_dir.replace('label', 'data'), b_time, 'C3'))
# get the file data
slice_idx = re.search(r'-\d{4}-', label_path)
if slice_idx is None:
raise ValueError('can not find the wave code')
slice_idx = int(slice_idx.group()[1:5])
file = []
for ii in range(2):
tmp = psr.read_c3(osp.join(file_path[ii], str(slice_idx)))
tmp = psr.Hokeman_decomposition(tmp)
# for jj in range(9):
# tmp[jj, :, :] = psr.min_max_contrast_median_map(10*np.log10(tmp[jj, :, :]))
tmp = np.log10(tmp)
file.append(tmp)
if self.time_shuffle and np.random.binomial(1, 0.5):
return torch.from_numpy(file[0]), torch.from_numpy(
file[1]), label, mask
else:
return torch.from_numpy(file[1]), torch.from_numpy(
file[0]), label, mask
def hoekman_and_norm(src_path: str) -> None:
''' hoekman decomposition, normalization as pauliRGB, and save to file
@in -src_path -source path, where should contains 'C3' folder
'''
if 'C3' in os.listdir(src_path):
print(f'hoekman and norm on dir: {src_path}', end='')
c3 = psr.read_c3(osp.join(src_path, 'C3'))
h = psr.Hokeman_decomposition(c3)
dst_path = osp.join(src_path, 'Hoekman')
fu.mkdir_if_not_exist(dst_path)
# np.save(osp.join(dst_path, 'ori'), h) # save the unnormalized file
for ii in range(9):
h[ii, :, :] = psr.min_max_contrast_median_map(
10 * np.log10(h[ii, :, :]))
# cv2.imwrite(osp.join(dst_path, f'{ii}.jpg'), (h[0, :, :]*255).astype(np.uint8))
# plt.hist() can take very long time to process a 2D array, but little time to process a 1D array, so flatten the array if possible
# plt.hist(h[ii, :, :].flatten())
# plt.savefig(osp.join(dst_path, f'log-hist-{ii}.jpg'))
np.save(osp.join(dst_path, 'normed'), h)
print('\tdone')
else:
raise ValueError('wrong src path')
def flatten_directory(path: str,
folder='uni_rot',
ori_data_type='mat',
select_features=None):
''' Flatten hierarchical directory according to label files
Args:
path (str): path to label files, not been divided into train, val and
test sets
folder (str): folder in which file data stored, also indicates the
data type
ori_data_type (str): original data type, 'mat': Matlab file, 'npy':
numpy file, 'C3' diagonal elements of C3 matrix with logarithm
select_features (tuple): features to be selected, None indicates to
select all. Default: None
'''
src_path = path.replace('label', 'data')
pngs = glob.glob(osp.join(path, '*.png'))
for png in pngs:
png = osp.basename(png)
loc = re.findall(r'[a-z]+', png)[0]
date = re.findall(r'\d{8}', png)[0]
idx = re.findall(r'_(\d{3}).', png)[0]
idx = idx.lstrip('0') if idx.lstrip('0') else '0'
if folder == 'C3':
src_file_path = osp.join(src_path, loc, date, folder, idx)
else:
src_file_path = osp.join(src_path, loc, date, folder, idx,
'unnormed.' + ori_data_type)
dst_file_path = osp.join(path.replace('label', folder + '_unnormed'),
png.replace('png', 'npy'))
fu.mkdir_if_not_exist(osp.dirname(dst_file_path))
print(
f'copy {src_file_path.replace(src_path, "")} to {dst_file_path.replace(src_path, "")}'
)
# different original file type
if ori_data_type == 'mat':
file = load_uni_rot_mat_file(src_file_path,
select_features=select_features)
elif ori_data_type == 'npy':
file = np.load(src_file_path)
# check in nan or inf
num_nan = np.isnan(file).sum()
num_inf = np.isinf(file).sum()
if num_nan > 0:
raise ValueError(f'{src_file_path}: nan value exist')
if num_inf > 0:
raise ValueError(f'{src_file_path}: inf value exist')
elif ori_data_type == 'C3':
file = psr.read_c3(src_file_path, out='save_space')
file = file[(0, 5, 8), :, :]
file[file < mathlib.eps] = mathlib.eps
file = np.log(file)
mathlib.check_inf_nan(file)
np.save(dst_file_path, file)
print('flatten directory finished\n')
def get_files_data(self, index):
''' read file a and file b data, in torch.tensor format
if the data is images, then it is normed into [0, 1],
'''
label_path = self.labels_path[index]
# get the file path
label_dir = osp.split(label_path)[0]
# two date time display format
re_exp = r'20\d{6}'
[a_time, b_time] = re.findall(re_exp, label_path)
files_path = []
if 's2' in self.data_format:
files_path.append(
osp.join(label_dir.replace('label', 'data'), a_time, 's2'))
files_path.append(
osp.join(label_dir.replace('label', 'data'), b_time, 's2'))
else:
files_path.append(
osp.join(label_dir.replace('label', 'data'), a_time, 'C3'))
files_path.append(
osp.join(label_dir.replace('label', 'data'), b_time, 'C3'))
# get the file data
if self.sensor == 'GF3':
slice_idx = re.search(r'-\d{3}-', label_path)
elif self.sensor == 'RS2':
slice_idx = re.search(r'-\d{4}-', label_path)
if slice_idx is None:
raise ValueError('can not find the wave code')
slice_idx = int(slice_idx.group()[1:-1])
files = []
if self.data_format in ('save_space', 'complex_vector_9',
'complex_vector_6'):
for ii in range(2):
psr_data = psr.read_c3(osp.join(files_path[ii],
str(slice_idx)),
out=self.data_format)
if self.norm:
mean = np.load(
osp.join(files_path[ii],
self.data_format + '_mean.npy'))
std = np.load(
osp.join(files_path[ii],
self.data_format + '_std.npy'))
_, _, psr_data = psr.norm_3_sigma(psr_data, mean, std)
files.append(torch.from_numpy(psr_data).type(torch.complex64))
# polar coordinate form, i.e. magnitude and angle
elif 'polar' in self.data_format:
if 'c3' in self.data_format:
# complex vector 6
for ii in range(2):
c6 = psr.read_c3(osp.join(files_path[ii], str(slice_idx)),
out='complex_vector_6').astype(
np.complex64)
if self.norm:
mean = np.load(
osp.join(files_path[ii],
'complex_vector_6' + '_mean.npy'))
std = np.load(
osp.join(files_path[ii],
'complex_vector_6' + '_std.npy'))
_, _, c6 = psr.norm_3_sigma(c6, mean, std)
abs = np.expand_dims(np.abs(c6), axis=0)
agl = np.expand_dims(np.angle(c6), axis=0)
polar = torch.cat(
(torch.from_numpy(agl), torch.from_numpy(abs)), dim=0)
if '4D' in self.data_format:
x = torch.from_numpy(c6.real.astype(np.float32))
y = torch.from_numpy(c6.imag.astype(np.float32))
polar = torch.cat((polar, x, y), dim=0)
files.append(polar)
elif 's2' in self.data_format:
# s2 matrix
for ii in range(2):
s2 = psr.read_s2(osp.join(
files_path[ii], str(slice_idx))).astype(np.complex64)
if self.norm:
mean = np.load(
osp.join(files_path[ii], 's2_abs_mean.npy'))
std = np.load(
osp.join(files_path[ii], 's2_abs_std.npy'))
_, _, s2 = psr.norm_3_sigma(s2, mean, std, type='abs')
abs = np.expand_dims(np.abs(s2), axis=0)
agl = np.expand_dims(np.angle(s2), axis=0)
polar = torch.cat(
(torch.from_numpy(agl), torch.from_numpy(abs)), dim=0)
if '4D' in self.data_format:
x = torch.from_numpy(s2.real.astype(
np.float32)).unsqueeze(dim=0)
y = torch.from_numpy(s2.imag.astype(
np.float32)).unsqueeze(dim=0)
polar = torch.cat((polar, x, y), dim=0)
files.append(polar)
elif self.data_format == 's2':
for ii in range(2):
psr_data = psr.read_s2(osp.join(files_path[ii],
str(slice_idx)))
if self.norm:
mean = np.load(osp.join(files_path[ii], 's2_abs_mean.npy'))
std = np.load(osp.join(files_path[ii], 's2_abs_std.npy'))
_, _, psr_data = psr.norm_3_sigma(psr_data,
mean,
std,
type='abs')
files.append(torch.from_numpy(psr_data).type(torch.complex64))
elif self.data_format == 'pauli':
for ii in range(2):
files.append(
psr.read_bmp(osp.join(files_path[ii], str(slice_idx))))
if self.to_tensor:
for ii in range(2):
files[ii] = self.tf(files[ii])
else:
for ii in range(2):
files[ii] = files[ii].permute(2, 0, 1)
elif self.data_format == 'hoekman':
for ii in range(2):
files.append(
torch.from_numpy(
np.load(
osp.join(files_path[ii], str(slice_idx),
'normed.npy').replace('C3', 'Hoekman'))))
else:
raise NotImplementedError
return files
file_b = psr.as_format(file_b, 'complex_vector_9')
file_a = psr.wishart_noise(file_a.reshape(3, 3, -1),
ENL=self.ENL).reshape(9, h, w)
file_b = psr.wishart_noise(file_b.reshape(3, 3, -1),
ENL=self.ENL).reshape(9, h, w)
file_a = psr.as_format(file_a, ori_format)
file_b = psr.as_format(file_b, ori_format)
return torch.from_numpy(file_a), torch.from_numpy(file_b), label, mask
if __name__ == '__main__':
''' test WishartNoise() '''
path_a = r'data/GF3/data/E115_N39_中国河北/降轨/1/20161209'
path_b = r'data/GF3/data/E115_N39_中国河北/降轨/1/20170306'
file_a = psr.read_c3(path_a)
file_b = psr.read_c3(path_b)
# boxcar smoothing
b = BoxcarSmooth(3)
file_a, file_b, _, _ = b(file_a, file_b, None, None)
ba = psr.rgb_by_c3(file_a)
bb = psr.rgb_by_c3(file_b)
cv2.imwrite('/home/csl/code/PolSAR_CD/tmp/ba.png',
cv2.cvtColor((255 * ba).astype(np.uint8), cv2.COLOR_RGB2BGR))
cv2.imwrite('/home/csl/code/PolSAR_CD/tmp/bb.png',
cv2.cvtColor((255 * bb).astype(np.uint8), cv2.COLOR_RGB2BGR))
# generate Wishart noise
w = WishartNoise(3)
file_a, file_b, _, _ = w(file_a, file_b, None, None)
