def test_add_subvrts_only_to_one_nansat(self):
d = Domain(4326, "-te 25 70 35 72 -ts 500 500")
arr = np.random.randn(500, 500)
n1 = Nansat.from_domain(d, log_level=40)
n2 = Nansat.from_domain(d, log_level=40)
n1.add_band(arr, {'name': 'band1'})
self.assertEqual(type(n1.vrt.band_vrts), dict)
self.assertTrue(len(n1.vrt.band_vrts) > 0)
self.assertEqual(n2.vrt.band_vrts, {})
def update_icemap_mosaic(inp_filename, inp_data, out_filename, out_domain,
out_metadata):
if os.path.exists(out_filename):
mos_array = Nansat(out_filename)[1]
else:
mos_array = np.zeros(out_domain.shape(), np.uint8) + 255
# read classification data and reproject onto mosaic domain
n = Nansat(inp_filename)
if inp_data is None:
n.reproject_gcps()
n.reproject(out_domain)
inp_data = dict(arr=n[1], mask=n[2])
# put data into mosaic array
gpi = (inp_data['mask'] == 1) * (inp_data['arr'] < 255)
mos_array[gpi] = inp_data['arr'][gpi]
# export
n_out = Nansat.from_domain(out_domain)
n_out.add_band(array=mos_array, parameters={'name': 'classification'})
n_out.set_metadata(n.get_metadata())
n_out.set_metadata(out_metadata)
n_out = add_colortable(n_out)
n_out.export(out_filename, driver='GTiff', options=['COMPRESS=LZW'])
return inp_data
def test_from_domain_nansat(self):
n1 = Nansat(self.test_file_gcps, log_level=40, mapper=self.default_mapper)
n2 = Nansat.from_domain(n1, n1[1])
self.assertEqual(type(n2), Nansat)
self.assertEqual(len(n2.bands()), 1)
self.assertEqual(type(n2[1]), np.ndarray)
def test_export_gcps_complex_to_netcdf(self):
""" Should export file with GCPs and write correct complex bands"""
n0 = Nansat(self.test_file_gcps,
log_level=40,
mapper=self.default_mapper)
b0 = n0['L_469']
n1 = Nansat.from_domain(n0)
n1.add_band(b0.astype('complex64'), parameters={'name': 'L_469'})
tmpfilename = os.path.join(self.tmp_data_path,
'nansat_export_gcps_complex.nc')
n1.export(tmpfilename)
ncf = Dataset(tmpfilename)
self.assertTrue(os.path.exists(tmpfilename))
self.assertTrue('GCPX' in ncf.variables)
self.assertTrue('GCPY' in ncf.variables)
self.assertTrue('GCPPixel' in ncf.variables)
self.assertTrue('GCPLine' in ncf.variables)
n2 = Nansat(tmpfilename, mapper=self.default_mapper)
b2 = n2['L_469']
lon0, lat0 = n0.get_geolocation_grids()
lon2, lat2 = n1.get_geolocation_grids()
np.testing.assert_allclose(lon0, lon2)
np.testing.assert_allclose(lat0, lat2)
def test_export_gcps_complex_to_netcdf(self):
""" Should export file with GCPs and write correct complex bands"""
n0 = Nansat(self.test_file_gcps, log_level=40, mapper=self.default_mapper)
b0 = n0['L_469']
n1 = Nansat.from_domain(n0)
n1.add_band(b0.astype('complex64'), parameters={'name': 'L_469'})
tmpfilename = os.path.join(self.tmp_data_path, 'nansat_export_gcps_complex.nc')
n1.export(tmpfilename)
ncf = Dataset(tmpfilename)
self.assertTrue(os.path.exists(tmpfilename))
self.assertTrue('GCPX' in ncf.variables)
self.assertTrue('GCPY' in ncf.variables)
self.assertTrue('GCPPixel' in ncf.variables)
self.assertTrue('GCPLine' in ncf.variables)
n2 = Nansat(tmpfilename, mapper=self.default_mapper)
b2 = n2['L_469']
lon0, lat0 = n0.get_geolocation_grids()
lon2, lat2 = n1.get_geolocation_grids()
np.testing.assert_allclose(lon0, lon2)
np.testing.assert_allclose(lat0, lat2)
def test_reproject_pure_geolocation(self):
n0 = Nansat(self.test_file_gcps)
b0 = n0[1]
lon0, lat0 = n0.get_geolocation_grids()
d1 = Domain.from_lonlat(lon=lon0, lat=lat0)
d2 = Domain.from_lonlat(lon=lon0, lat=lat0, add_gcps=False)
d3 = Domain(NSR().wkt, '-te 27 70 31 72 -ts 500 500')
n1 = Nansat.from_domain(d1, b0)
n2 = Nansat.from_domain(d2, b0)
n1.reproject(d3)
n2.reproject(d3)
b1 = n1[1]
b2 = n2[1]
self.assertTrue(np.allclose(b1, b2))
def test_dont_export2thredds_gcps(self):
n = Nansat(self.test_file_gcps, log_level=40, mapper=self.default_mapper)
n2 = Nansat.from_domain(n)
n.add_band(np.ones(n2.shape(), np.float32))
tmpfilename = os.path.join(self.tmp_data_path,
'nansat_export2thredds.nc')
self.assertRaises(ValueError, n2.export2thredds, tmpfilename,
['L_645'])
def test_get_item_inf_expressions(self):
""" inf should be replaced with nan """
self.mock_pti['get_wkv_variable'].return_value=dict(short_name='newband')
d = Domain(4326, "-te 25 70 35 72 -ts 500 500")
n = Nansat.from_domain(d, log_level=40)
arr = np.empty((500, 500))
n.add_band(arr, {'expression': 'np.array([0,1,2,3,np.inf,5,6,7])'})
self.assertIsInstance(n[1], np.ndarray)
self.assertTrue(np.isnan(n[1][4]))
def test_get_item_basic_expressions(self):
""" Testing get_item with some basic expressions """
self.mock_pti['get_wkv_variable'].return_value=dict(short_name='newband')
d = Domain(4326, "-te 25 70 35 72 -ts 500 500")
n = Nansat.from_domain(d, np.zeros((500, 500)), {'expression': 'np.ones((500, 500))'})
self.assertIsInstance(n[1], np.ndarray)
self.assertEqual(n[1].shape, (500, 500))
band1 = n[1]
self.assertTrue(np.allclose(band1, np.ones((500, 500))))
def test_add_band(self):
d = Domain(4326, "-te 25 70 35 72 -ts 500 500")
arr = np.random.randn(500, 500)
n = Nansat.from_domain(d, log_level=40)
n.add_band(arr, {'name': 'band1'})
self.assertEqual(type(n), Nansat)
self.assertEqual(type(n[1]), np.ndarray)
self.assertEqual(n.get_metadata('name', 1), 'band1')
self.assertEqual(n[1].shape, (500, 500))
def test_dont_export2thredds_gcps(self):
n = Nansat(self.test_file_gcps,
log_level=40,
mapper=self.default_mapper)
n2 = Nansat.from_domain(n)
n.add_band(np.ones(n2.shape(), np.float32))
tmpfilename = os.path.join(self.tmp_data_path,
'nansat_export2thredds.nc')
self.assertRaises(ValueError, n2.export2thredds, tmpfilename,
['L_645'])
def test_add_bands_no_parameter(self):
d = Domain(4326, "-te 25 70 35 72 -ts 500 500")
arr = np.random.randn(500, 500)
n = Nansat.from_domain(d, log_level=40)
n.add_bands([arr, arr])
self.assertEqual(type(n), Nansat)
self.assertEqual(type(n[1]), np.ndarray)
self.assertEqual(type(n[2]), np.ndarray)
def test_dont_export2thredds_gcps(self):
n = Nansat(self.test_file_gcps,
log_level=40,
mapper=self.default_mapper)
n2 = Nansat.from_domain(n)
n.add_band(np.ones(n2.shape(), np.float32))
tmpfilename = os.path.join(self.tmp_data_path,
'nansat_export2thredds.nc')
with self.assertRaises(ValueError) as e:
n2.export2thredds(tmpfilename)
self.assertIn('Cannot export dataset with GCPS', e.exception.args[0])
def test_from_domain_array(self):
d = Domain(4326, "-te 25 70 35 72 -ts 500 500")
n = Nansat.from_domain(d, np.random.randn(500, 500), {'name': 'band1'})
self.assertEqual(type(n), Nansat)
self.assertEqual(type(n[1]), np.ndarray)
self.assertEqual(n.get_metadata('name', 1), 'band1')
self.assertEqual(n[1].shape, (500, 500))
self.assertEqual(n.filename, '')
self.assertIsInstance(n.logger, logging.Logger)
self.assertEqual(n.name, '')
self.assertEqual(n.path, '')
def test_add_bands(self):
d = Domain(4326, "-te 25 70 35 72 -ts 500 500")
arr = np.random.randn(500, 500)
n = Nansat.from_domain(d, log_level=40)
n.add_bands([arr, arr], [{'name': 'band1'}, {'name': 'band2'}])
self.assertIsInstance(n, Nansat)
self.assertEqual(n.vrt.vrt.vrt, None)
self.assertIsInstance(n[1], np.ndarray)
self.assertIsInstance(n[2], np.ndarray)
self.assertEqual(n.get_metadata('name', 1), 'band1')
self.assertEqual(n.get_metadata('name', 2), 'band2')
def save_ice_map(inp_filename, raw_filename, classifier_filename, threads,
source, quicklook, force):
""" Load texture features, apply classifier and save ice map """
# get filenames
out_filename = inp_filename.replace('_texture_features.npz',
'_classified_%s.tif' % source)
if os.path.exists(out_filename) and not force:
print('Processed file %s already exists.' % out_filename)
return out_filename
# import classifier
clf = pickle.load(open(classifier_filename, "rb"))
clf.n_jobs = threads
# get texture features
npz = np.load(inp_filename)
features = np.vstack([
npz['textureFeatures'].item()['HH'],
npz['textureFeatures'].item()['HV'],
])
imgSize = features.shape[1:]
features = features.reshape((26, np.prod(imgSize))).T
gpi = np.isfinite(features.sum(axis=1))
result = clf.predict(features[gpi, :])
classImage = np.ones(np.prod(imgSize)) * 255
classImage[gpi] = result
classImage = classImage.reshape(imgSize)
img_shape = classImage.shape
# open original file to get geometry
raw_nansat = Nansat(raw_filename)
# crop and resize original Nansat to match the ice map
raw_shape = raw_nansat.shape()
crop = [rshape % ishape for (rshape, ishape) in zip(raw_shape, img_shape)]
raw_nansat.crop(0, 0, raw_shape[1] - crop[1], raw_shape[0] - crop[0])
raw_nansat.resize(height=img_shape[0])
raw_nansat.reproject_gcps()
# create new Nansat object and add ice map
ice_map = Nansat.from_domain(domain=raw_nansat,
array=classImage.astype(np.uint8))
ice_map.set_metadata(raw_nansat.get_metadata())
ice_map.set_metadata('entry_title', 'S1_SAR_ICE_MAP')
ice_map = add_colortable(ice_map, source)
ice_map.export(out_filename, bands=[1], driver='GTiff')
if quicklook:
rgb = colorcode_array(classImage, source)
plt.imsave(out_filename.replace('.tif', '.png'), rgb)
return out_filename
def test_export2thredds_longlat_dict(self):
d = Domain("+proj=latlong +datum=WGS84 +ellps=WGS84 +no_defs",
"-te 27 70 31 72 -ts 200 200")
n = Nansat.from_domain(d)
n.add_band(np.ones(d.shape(), np.float32),
parameters={'name': 'L_469'})
n.set_metadata('time_coverage_start', '2016-01-19')
tmpfilename = os.path.join(self.tmp_data_path,
'nansat_export2thredds_longlat.nc')
n.export2thredds(tmpfilename, {'L_469': {'type': '>i1'}})
ncI = Dataset(tmpfilename, 'r')
ncIVar = ncI.variables['L_469']
self.assertTrue(ncIVar.grid_mapping in ncI.variables.keys())
self.assertEqual(ncIVar[:].dtype, np.int8)
def test_export2thredds_longlat_dict(self):
d = Domain("+proj=latlong +datum=WGS84 +ellps=WGS84 +no_defs",
"-te 27 70 31 72 -ts 200 200")
n = Nansat.from_domain(d)
n.add_band(np.ones(d.shape(), np.float32),
parameters={'name': 'L_469'})
n.set_metadata('time_coverage_start', '2016-01-19')
tmpfilename = os.path.join(self.tmp_data_path,
'nansat_export2thredds_longlat.nc')
n.export2thredds(tmpfilename, {'L_469': {'type': '>i1'}})
ncI = Dataset(tmpfilename, 'r')
ncIVar = ncI.variables['L_469']
self.assertTrue(ncIVar.grid_mapping in ncI.variables.keys())
self.assertEqual(ncIVar[:].dtype, np.int8)
def run_correction(ifile,
angular_scale_copol=-0.2,
angular_scale_crpol=-0.025,
angular_offset=34.5,
output_dtype=np.float32):
""" Run thermal, textural and angular correction of input Sentinel-1 file
Parameters
----------
ifile : str
input file
angular_scale_hh : float
Scale for angular correction of sigma0 in HH or VV
angular_scale_hv : float
Scale for angular correction of sigma0 in HV or VH
angular_offset : float
Central angle for sigma0 normalization
output_dtype : dtype
Type of output array
Returns
--------
s1 : Nansat
object with corrected bands and metadata
"""
scale = {
'HH': angular_scale_copol,
'HV': angular_scale_crpol,
'VH': angular_scale_crpol,
'VV': angular_scale_copol,
}
s1 = Sentinel1Image(ifile)
n = Nansat.from_domain(s1)
inc = s1['incidence_angle']
for pol in s1.pols:
print('Correct %s band' % pol)
parameters = s1.get_metadata(band_id='sigma0_%s' % pol)
for i in [
'dataType', 'PixelFunctionType', 'SourceBand', 'SourceFilename'
]:
parameters.pop(i)
array = s1.remove_texture_noise(pol)
array = 10 * np.log10(array) - scale[pol] * (inc - angular_offset)
n.add_band(array=array.astype(output_dtype), parameters=parameters)
n.set_metadata(s1.get_metadata())
return n
def test_repr_basic(self):
""" repr should include some basic elements """
d = Domain(4326, "-te 25 70 35 72 -ts 500 500")
n = Nansat.from_domain(d, log_level=40)
arr = np.empty((500, 500))
exp = 'np.array([0,1,2,3,np.inf,5,6,7])'
n.add_band(arr, {'expression': exp})
n_repr = repr(n)
self.assertIn(exp, n_repr, 'The expressions should be in repr')
self.assertIn('SourceFilename', n_repr)
self.assertIn('/vsimem/', n_repr)
self.assertIn('500 x 500', n_repr)
self.assertIn('Projection:', n_repr)
self.assertIn('25', n_repr)
self.assertIn('72', n_repr)
self.assertIn('35', n_repr)
self.assertIn('70', n_repr)
def get_n(filename,
bandName='sigma0_HV',
factor=0.5,
vmin=-30,
vmax=-5,
denoise=False,
dB=True,
**kwargs):
''' Get Nansat object with image data scaled to UInt8
Parameters
----------
filename : str - input file name
bandName : str - name of band in the file
factor : float - subsampling factor
vmin : float - minimum allowed value in the band
vmax : float - maximum allowed value in the band
denoise : bool - apply denoising of sigma0 ?
dB : bool - apply conversion to dB ?
**kwargs : parameters for get_denoised_object()
Returns
-------
n : Nansat object with one band scaled to UInt8
'''
if denoise:
# run denoising
n = get_denoised_object(filename, bandName, factor, **kwargs)
else:
# open data with Nansat and downsample
n = Nansat(filename)
if factor != 1:
n.resize(factor, resample_alg=-1)
# get matrix with data
img = n[bandName]
# convert to dB
if not denoise and dB:
img = 10 * np.log10(img)
# convert to 0 - 255
img = get_uint8_image(img, vmin, vmax)
nout = Nansat.from_domain(n, img, parameters={'name': bandName})
nout.set_metadata(n.get_metadata())
# improve geolocation accuracy
if len(nout.vrt.dataset.GetGCPs()) > 0:
nout.reproject_gcps()
nout.vrt.tps = True
return nout
def run_denoising(ifile,
pols=['HV'],
db=False,
filter_negative=False,
**kwargs):
""" Run denoising of input file
Parameters
----------
ifile : str
input file
pols : str
polarisoation options, ['HH'], ['HV'] or ['HH','HV']
db : bool
convert to decibel?
data_format : str
format of data in output file
filter_negative : bool
replace negative values with smallest nearest positive?
Returns
--------
s1 : Nansat
object with denoised bands and metadata
"""
s1 = Sentinel1Image(ifile)
n = Nansat.from_domain(s1)
for pol in pols:
print('Denoise %s band' % pol)
s1.add_denoised_band(pol)
array = s1['sigma0_%s_denoised' % pol]
parameters = s1.get_metadata(band_id='sigma0_%s' % pol)
if filter_negative:
print('Remove negaive pixels')
array = remove_negative(array)
if db:
print('Convert to dB')
array = 10 * np.log10(array)
parameters['units'] = 'dB'
n.add_band(array=array, parameters=parameters)
n.set_metadata(s1.get_metadata())
return n
kmeans = KMeans(n_clusters=9, n_jobs=cfg.numberOfThreads).fit(
pca.transform(scaler.transform(features_all))[:, :n_components])
pickle.dump([scaler, pca, n_components, kmeans], open(cfg.kmeansFilename,
"wb"))
# apply clustering
print('*** Exporting files to:')
for li, ifile in enumerate(ifiles):
ofile = ifile.replace('_texture_features.npz', '_kmeans_clustered.tif')
print('[%d/%d] %s' %
(li + 1, len(ifiles),
ifile.replace('_texture_features.npz', '_kmeans.tif')))
npz = np.load(ifile)
tfsHH = npz['textureFeatures'].item()['HH']
tfsHV = npz['textureFeatures'].item()['HV']
imgSize = tfsHH.shape[1:]
features = np.vstack([tfsHH, tfsHV]).reshape(26, np.prod(imgSize)).T
gpi = np.isfinite(features.sum(axis=1))
kmeansZones = np.zeros(np.prod(imgSize)) # 0 is reserved for void cells
kmeansZones[gpi] = 1 + kmeans.predict(
pca.transform(scaler.transform(features[gpi]))[:, :n_components])
kmeansZones = kmeansZones.reshape(imgSize)
nansatObjSigma0 = Nansat(
ifile.replace('_texture_features.npz', '_sigma0_HH_denoised.tif'))
if nansatObjSigma0.shape() != imgSize:
nansatObjSigma0.crop(0, 0, imgSize[1], imgSize[0])
nansatObjCluster = Nansat.from_domain(array=kmeansZones.astype(np.uint8),
domain=nansatObjSigma0)
nansatObjCluster.export(ofile, bands=[1], driver='GTiff')
if cfg.quicklook:
plt.imsave(ofile.replace('.tif', '.png'), kmeansZones, cmap='tab10')
def get_n(filename,
bandName='sigma0_HV',
factor=0.5,
denoise=False,
dB=True,
mask_invalid=True,
landmask_border=20,
correct_hh=False,
correct_hh_factor=-0.27,
remove_spatial_mean=False,
vmin=None,
vmax=None,
pmin=10,
pmax=99,
**kwargs):
""" Get Nansat object with image data scaled to UInt8
Parameters
----------
filename : str
input file name
bandName : str
name of band in the file
factor : float
subsampling factor
denoise : bool
apply denoising of sigma0 ?
dB : bool
apply conversion to dB ?
mask_invalid : bool
mask invalid pixels (land, inf, etc) with 0 ?
landmask_border : int
border around landmask
correct_hh : bool
perform angular correction of sigma0_HH ?
correct_hh_factor : float
coefficient in the correction factor sigma0_HH_cor = sigma0_HH + correct_hh_factor * incidence_angle
remove_spatial_mean : bool
remove spatial mean from image ?
vmin : float or None
minimum value to convert to 1
vmax : float or None
maximum value to convert to 255
pmin : float
lower percentile for data scaling if vmin is None
pmax : float
upper percentile for data scaling if vmax is None
**kwargs : dummy parameters for
get_denoised_object()
Returns
-------
n : Nansat object with one band scaled to UInt8
"""
if denoise:
# run denoising
n = get_denoised_object(filename, bandName, factor, **kwargs)
else:
# open data with Nansat and downsample
n = Nansat(filename)
if factor != 1:
n.resize(factor, resample_alg=-1)
# get matrix with data
img = n[bandName]
# convert to dB
if not denoise and dB:
img[img <= 0] = np.nan
img = 10 * np.log10(img)
if correct_hh:
img = hh_angular_correction(n, img, bandName, correct_hh_factor)
if mask_invalid:
mask = get_invalid_mask(img, n, landmask_border)
img[mask] = np.nan
if remove_spatial_mean:
img -= get_spatial_mean(img)
# convert to 0 - 255
img = get_uint8_image(img, vmin, vmax, pmin, pmax)
# create Nansat with one band only
nout = Nansat.from_domain(n, img, parameters={'name': bandName})
nout.set_metadata(n.get_metadata())
# improve geolocation accuracy
if len(nout.vrt.dataset.GetGCPs()) > 0:
nout.reproject_gcps()
nout.vrt.tps = True
return nout
if S == 98 and F == 10:
CC = 98 + 10 # color code for iceberg
CT = ft.GetFieldAsInteger('CT')
else:
CC = 2 # color code for bergy water
if CT == 1:
CC = 1 # color code for open water
if CT == 98:
CC = 0 # color code for ice free
ft.SetField('classID', int(CC))
oLayer.SetFeature(ft)
sigma0 = gdal.Open(ifile).ReadAsArray()
fp_classID = gdal.Open(ifile)
gdal.RasterizeLayer(fp_classID, [1], oLayer, options=["ATTRIBUTE=classID"])
classID = fp_classID.ReadAsArray()
classID[classID == sigma0] = 255
classID[np.isnan(classID)] = 99
nansatObjSigma0 = Nansat(ifile)
nansatObjIceChart = Nansat.from_domain(array=classID.astype(np.uint8),
domain=nansatObjSigma0)
nansatObjIceChart.set_metadata(nansatObjSigma0.get_metadata())
nansatObjIceChart.set_metadata('entry_title',
'REPROJECTED_%s_ICE_CHART' % cfg.sourceType)
nansatObjIceChart = add_colortable(nansatObjIceChart, cfg.sourceType)
nansatObjIceChart.export(ofile, bands=[1], driver='GTiff')
if cfg.quicklook:
rgb = np.zeros((classID.shape[0], classID.shape[1], 3), 'uint8')
for k in colorDict[cfg.sourceType].keys():
rgb[classID == k, :] = colorDict[cfg.sourceType][k]
plt.imsave(ofile.replace('.tif', '.png'), rgb)
