def test_add_subvrts_only_to_one_nansat(self): d = Domain(4326, "-te 25 70 35 72 -ts 500 500") arr = np.random.randn(500, 500) n1 = Nansat.from_domain(d, log_level=40) n2 = Nansat.from_domain(d, log_level=40) n1.add_band(arr, {'name': 'band1'}) self.assertEqual(type(n1.vrt.band_vrts), dict) self.assertTrue(len(n1.vrt.band_vrts) > 0) self.assertEqual(n2.vrt.band_vrts, {})
def update_icemap_mosaic(inp_filename, inp_data, out_filename, out_domain, out_metadata): if os.path.exists(out_filename): mos_array = Nansat(out_filename)[1] else: mos_array = np.zeros(out_domain.shape(), np.uint8) + 255 # read classification data and reproject onto mosaic domain n = Nansat(inp_filename) if inp_data is None: n.reproject_gcps() n.reproject(out_domain) inp_data = dict(arr=n[1], mask=n[2]) # put data into mosaic array gpi = (inp_data['mask'] == 1) * (inp_data['arr'] < 255) mos_array[gpi] = inp_data['arr'][gpi] # export n_out = Nansat.from_domain(out_domain) n_out.add_band(array=mos_array, parameters={'name': 'classification'}) n_out.set_metadata(n.get_metadata()) n_out.set_metadata(out_metadata) n_out = add_colortable(n_out) n_out.export(out_filename, driver='GTiff', options=['COMPRESS=LZW']) return inp_data
def test_from_domain_nansat(self): n1 = Nansat(self.test_file_gcps, log_level=40, mapper=self.default_mapper) n2 = Nansat.from_domain(n1, n1[1]) self.assertEqual(type(n2), Nansat) self.assertEqual(len(n2.bands()), 1) self.assertEqual(type(n2[1]), np.ndarray)
def test_export_gcps_complex_to_netcdf(self): """ Should export file with GCPs and write correct complex bands""" n0 = Nansat(self.test_file_gcps, log_level=40, mapper=self.default_mapper) b0 = n0['L_469'] n1 = Nansat.from_domain(n0) n1.add_band(b0.astype('complex64'), parameters={'name': 'L_469'}) tmpfilename = os.path.join(self.tmp_data_path, 'nansat_export_gcps_complex.nc') n1.export(tmpfilename) ncf = Dataset(tmpfilename) self.assertTrue(os.path.exists(tmpfilename)) self.assertTrue('GCPX' in ncf.variables) self.assertTrue('GCPY' in ncf.variables) self.assertTrue('GCPPixel' in ncf.variables) self.assertTrue('GCPLine' in ncf.variables) n2 = Nansat(tmpfilename, mapper=self.default_mapper) b2 = n2['L_469'] lon0, lat0 = n0.get_geolocation_grids() lon2, lat2 = n1.get_geolocation_grids() np.testing.assert_allclose(lon0, lon2) np.testing.assert_allclose(lat0, lat2)
def test_reproject_pure_geolocation(self): n0 = Nansat(self.test_file_gcps) b0 = n0[1] lon0, lat0 = n0.get_geolocation_grids() d1 = Domain.from_lonlat(lon=lon0, lat=lat0) d2 = Domain.from_lonlat(lon=lon0, lat=lat0, add_gcps=False) d3 = Domain(NSR().wkt, '-te 27 70 31 72 -ts 500 500') n1 = Nansat.from_domain(d1, b0) n2 = Nansat.from_domain(d2, b0) n1.reproject(d3) n2.reproject(d3) b1 = n1[1] b2 = n2[1] self.assertTrue(np.allclose(b1, b2))
def test_dont_export2thredds_gcps(self): n = Nansat(self.test_file_gcps, log_level=40, mapper=self.default_mapper) n2 = Nansat.from_domain(n) n.add_band(np.ones(n2.shape(), np.float32)) tmpfilename = os.path.join(self.tmp_data_path, 'nansat_export2thredds.nc') self.assertRaises(ValueError, n2.export2thredds, tmpfilename, ['L_645'])
def test_get_item_inf_expressions(self): """ inf should be replaced with nan """ self.mock_pti['get_wkv_variable'].return_value=dict(short_name='newband') d = Domain(4326, "-te 25 70 35 72 -ts 500 500") n = Nansat.from_domain(d, log_level=40) arr = np.empty((500, 500)) n.add_band(arr, {'expression': 'np.array([0,1,2,3,np.inf,5,6,7])'}) self.assertIsInstance(n[1], np.ndarray) self.assertTrue(np.isnan(n[1][4]))
def test_get_item_basic_expressions(self): """ Testing get_item with some basic expressions """ self.mock_pti['get_wkv_variable'].return_value=dict(short_name='newband') d = Domain(4326, "-te 25 70 35 72 -ts 500 500") n = Nansat.from_domain(d, np.zeros((500, 500)), {'expression': 'np.ones((500, 500))'}) self.assertIsInstance(n[1], np.ndarray) self.assertEqual(n[1].shape, (500, 500)) band1 = n[1] self.assertTrue(np.allclose(band1, np.ones((500, 500))))
def test_add_band(self): d = Domain(4326, "-te 25 70 35 72 -ts 500 500") arr = np.random.randn(500, 500) n = Nansat.from_domain(d, log_level=40) n.add_band(arr, {'name': 'band1'}) self.assertEqual(type(n), Nansat) self.assertEqual(type(n[1]), np.ndarray) self.assertEqual(n.get_metadata('name', 1), 'band1') self.assertEqual(n[1].shape, (500, 500))
def test_add_bands_no_parameter(self): d = Domain(4326, "-te 25 70 35 72 -ts 500 500") arr = np.random.randn(500, 500) n = Nansat.from_domain(d, log_level=40) n.add_bands([arr, arr]) self.assertEqual(type(n), Nansat) self.assertEqual(type(n[1]), np.ndarray) self.assertEqual(type(n[2]), np.ndarray)
def test_dont_export2thredds_gcps(self): n = Nansat(self.test_file_gcps, log_level=40, mapper=self.default_mapper) n2 = Nansat.from_domain(n) n.add_band(np.ones(n2.shape(), np.float32)) tmpfilename = os.path.join(self.tmp_data_path, 'nansat_export2thredds.nc') with self.assertRaises(ValueError) as e: n2.export2thredds(tmpfilename) self.assertIn('Cannot export dataset with GCPS', e.exception.args[0])
def test_from_domain_array(self): d = Domain(4326, "-te 25 70 35 72 -ts 500 500") n = Nansat.from_domain(d, np.random.randn(500, 500), {'name': 'band1'}) self.assertEqual(type(n), Nansat) self.assertEqual(type(n[1]), np.ndarray) self.assertEqual(n.get_metadata('name', 1), 'band1') self.assertEqual(n[1].shape, (500, 500)) self.assertEqual(n.filename, '') self.assertIsInstance(n.logger, logging.Logger) self.assertEqual(n.name, '') self.assertEqual(n.path, '')
def test_add_bands(self): d = Domain(4326, "-te 25 70 35 72 -ts 500 500") arr = np.random.randn(500, 500) n = Nansat.from_domain(d, log_level=40) n.add_bands([arr, arr], [{'name': 'band1'}, {'name': 'band2'}]) self.assertIsInstance(n, Nansat) self.assertEqual(n.vrt.vrt.vrt, None) self.assertIsInstance(n[1], np.ndarray) self.assertIsInstance(n[2], np.ndarray) self.assertEqual(n.get_metadata('name', 1), 'band1') self.assertEqual(n.get_metadata('name', 2), 'band2')
def save_ice_map(inp_filename, raw_filename, classifier_filename, threads, source, quicklook, force): """ Load texture features, apply classifier and save ice map """ # get filenames out_filename = inp_filename.replace('_texture_features.npz', '_classified_%s.tif' % source) if os.path.exists(out_filename) and not force: print('Processed file %s already exists.' % out_filename) return out_filename # import classifier clf = pickle.load(open(classifier_filename, "rb")) clf.n_jobs = threads # get texture features npz = np.load(inp_filename) features = np.vstack([ npz['textureFeatures'].item()['HH'], npz['textureFeatures'].item()['HV'], ]) imgSize = features.shape[1:] features = features.reshape((26, np.prod(imgSize))).T gpi = np.isfinite(features.sum(axis=1)) result = clf.predict(features[gpi, :]) classImage = np.ones(np.prod(imgSize)) * 255 classImage[gpi] = result classImage = classImage.reshape(imgSize) img_shape = classImage.shape # open original file to get geometry raw_nansat = Nansat(raw_filename) # crop and resize original Nansat to match the ice map raw_shape = raw_nansat.shape() crop = [rshape % ishape for (rshape, ishape) in zip(raw_shape, img_shape)] raw_nansat.crop(0, 0, raw_shape[1] - crop[1], raw_shape[0] - crop[0]) raw_nansat.resize(height=img_shape[0]) raw_nansat.reproject_gcps() # create new Nansat object and add ice map ice_map = Nansat.from_domain(domain=raw_nansat, array=classImage.astype(np.uint8)) ice_map.set_metadata(raw_nansat.get_metadata()) ice_map.set_metadata('entry_title', 'S1_SAR_ICE_MAP') ice_map = add_colortable(ice_map, source) ice_map.export(out_filename, bands=[1], driver='GTiff') if quicklook: rgb = colorcode_array(classImage, source) plt.imsave(out_filename.replace('.tif', '.png'), rgb) return out_filename
def test_export2thredds_longlat_dict(self): d = Domain("+proj=latlong +datum=WGS84 +ellps=WGS84 +no_defs", "-te 27 70 31 72 -ts 200 200") n = Nansat.from_domain(d) n.add_band(np.ones(d.shape(), np.float32), parameters={'name': 'L_469'}) n.set_metadata('time_coverage_start', '2016-01-19') tmpfilename = os.path.join(self.tmp_data_path, 'nansat_export2thredds_longlat.nc') n.export2thredds(tmpfilename, {'L_469': {'type': '>i1'}}) ncI = Dataset(tmpfilename, 'r') ncIVar = ncI.variables['L_469'] self.assertTrue(ncIVar.grid_mapping in ncI.variables.keys()) self.assertEqual(ncIVar[:].dtype, np.int8)
def run_correction(ifile, angular_scale_copol=-0.2, angular_scale_crpol=-0.025, angular_offset=34.5, output_dtype=np.float32): """ Run thermal, textural and angular correction of input Sentinel-1 file Parameters ---------- ifile : str input file angular_scale_hh : float Scale for angular correction of sigma0 in HH or VV angular_scale_hv : float Scale for angular correction of sigma0 in HV or VH angular_offset : float Central angle for sigma0 normalization output_dtype : dtype Type of output array Returns -------- s1 : Nansat object with corrected bands and metadata """ scale = { 'HH': angular_scale_copol, 'HV': angular_scale_crpol, 'VH': angular_scale_crpol, 'VV': angular_scale_copol, } s1 = Sentinel1Image(ifile) n = Nansat.from_domain(s1) inc = s1['incidence_angle'] for pol in s1.pols: print('Correct %s band' % pol) parameters = s1.get_metadata(band_id='sigma0_%s' % pol) for i in [ 'dataType', 'PixelFunctionType', 'SourceBand', 'SourceFilename' ]: parameters.pop(i) array = s1.remove_texture_noise(pol) array = 10 * np.log10(array) - scale[pol] * (inc - angular_offset) n.add_band(array=array.astype(output_dtype), parameters=parameters) n.set_metadata(s1.get_metadata()) return n
def test_repr_basic(self): """ repr should include some basic elements """ d = Domain(4326, "-te 25 70 35 72 -ts 500 500") n = Nansat.from_domain(d, log_level=40) arr = np.empty((500, 500)) exp = 'np.array([0,1,2,3,np.inf,5,6,7])' n.add_band(arr, {'expression': exp}) n_repr = repr(n) self.assertIn(exp, n_repr, 'The expressions should be in repr') self.assertIn('SourceFilename', n_repr) self.assertIn('/vsimem/', n_repr) self.assertIn('500 x 500', n_repr) self.assertIn('Projection:', n_repr) self.assertIn('25', n_repr) self.assertIn('72', n_repr) self.assertIn('35', n_repr) self.assertIn('70', n_repr)
def get_n(filename, bandName='sigma0_HV', factor=0.5, vmin=-30, vmax=-5, denoise=False, dB=True, **kwargs): ''' Get Nansat object with image data scaled to UInt8 Parameters ---------- filename : str - input file name bandName : str - name of band in the file factor : float - subsampling factor vmin : float - minimum allowed value in the band vmax : float - maximum allowed value in the band denoise : bool - apply denoising of sigma0 ? dB : bool - apply conversion to dB ? **kwargs : parameters for get_denoised_object() Returns ------- n : Nansat object with one band scaled to UInt8 ''' if denoise: # run denoising n = get_denoised_object(filename, bandName, factor, **kwargs) else: # open data with Nansat and downsample n = Nansat(filename) if factor != 1: n.resize(factor, resample_alg=-1) # get matrix with data img = n[bandName] # convert to dB if not denoise and dB: img = 10 * np.log10(img) # convert to 0 - 255 img = get_uint8_image(img, vmin, vmax) nout = Nansat.from_domain(n, img, parameters={'name': bandName}) nout.set_metadata(n.get_metadata()) # improve geolocation accuracy if len(nout.vrt.dataset.GetGCPs()) > 0: nout.reproject_gcps() nout.vrt.tps = True return nout
def run_denoising(ifile, pols=['HV'], db=False, filter_negative=False, **kwargs): """ Run denoising of input file Parameters ---------- ifile : str input file pols : str polarisoation options, ['HH'], ['HV'] or ['HH','HV'] db : bool convert to decibel? data_format : str format of data in output file filter_negative : bool replace negative values with smallest nearest positive? Returns -------- s1 : Nansat object with denoised bands and metadata """ s1 = Sentinel1Image(ifile) n = Nansat.from_domain(s1) for pol in pols: print('Denoise %s band' % pol) s1.add_denoised_band(pol) array = s1['sigma0_%s_denoised' % pol] parameters = s1.get_metadata(band_id='sigma0_%s' % pol) if filter_negative: print('Remove negaive pixels') array = remove_negative(array) if db: print('Convert to dB') array = 10 * np.log10(array) parameters['units'] = 'dB' n.add_band(array=array, parameters=parameters) n.set_metadata(s1.get_metadata()) return n
kmeans = KMeans(n_clusters=9, n_jobs=cfg.numberOfThreads).fit( pca.transform(scaler.transform(features_all))[:, :n_components]) pickle.dump([scaler, pca, n_components, kmeans], open(cfg.kmeansFilename, "wb")) # apply clustering print('*** Exporting files to:') for li, ifile in enumerate(ifiles): ofile = ifile.replace('_texture_features.npz', '_kmeans_clustered.tif') print('[%d/%d] %s' % (li + 1, len(ifiles), ifile.replace('_texture_features.npz', '_kmeans.tif'))) npz = np.load(ifile) tfsHH = npz['textureFeatures'].item()['HH'] tfsHV = npz['textureFeatures'].item()['HV'] imgSize = tfsHH.shape[1:] features = np.vstack([tfsHH, tfsHV]).reshape(26, np.prod(imgSize)).T gpi = np.isfinite(features.sum(axis=1)) kmeansZones = np.zeros(np.prod(imgSize)) # 0 is reserved for void cells kmeansZones[gpi] = 1 + kmeans.predict( pca.transform(scaler.transform(features[gpi]))[:, :n_components]) kmeansZones = kmeansZones.reshape(imgSize) nansatObjSigma0 = Nansat( ifile.replace('_texture_features.npz', '_sigma0_HH_denoised.tif')) if nansatObjSigma0.shape() != imgSize: nansatObjSigma0.crop(0, 0, imgSize[1], imgSize[0]) nansatObjCluster = Nansat.from_domain(array=kmeansZones.astype(np.uint8), domain=nansatObjSigma0) nansatObjCluster.export(ofile, bands=[1], driver='GTiff') if cfg.quicklook: plt.imsave(ofile.replace('.tif', '.png'), kmeansZones, cmap='tab10')
def get_n(filename, bandName='sigma0_HV', factor=0.5, denoise=False, dB=True, mask_invalid=True, landmask_border=20, correct_hh=False, correct_hh_factor=-0.27, remove_spatial_mean=False, vmin=None, vmax=None, pmin=10, pmax=99, **kwargs): """ Get Nansat object with image data scaled to UInt8 Parameters ---------- filename : str input file name bandName : str name of band in the file factor : float subsampling factor denoise : bool apply denoising of sigma0 ? dB : bool apply conversion to dB ? mask_invalid : bool mask invalid pixels (land, inf, etc) with 0 ? landmask_border : int border around landmask correct_hh : bool perform angular correction of sigma0_HH ? correct_hh_factor : float coefficient in the correction factor sigma0_HH_cor = sigma0_HH + correct_hh_factor * incidence_angle remove_spatial_mean : bool remove spatial mean from image ? vmin : float or None minimum value to convert to 1 vmax : float or None maximum value to convert to 255 pmin : float lower percentile for data scaling if vmin is None pmax : float upper percentile for data scaling if vmax is None **kwargs : dummy parameters for get_denoised_object() Returns ------- n : Nansat object with one band scaled to UInt8 """ if denoise: # run denoising n = get_denoised_object(filename, bandName, factor, **kwargs) else: # open data with Nansat and downsample n = Nansat(filename) if factor != 1: n.resize(factor, resample_alg=-1) # get matrix with data img = n[bandName] # convert to dB if not denoise and dB: img[img <= 0] = np.nan img = 10 * np.log10(img) if correct_hh: img = hh_angular_correction(n, img, bandName, correct_hh_factor) if mask_invalid: mask = get_invalid_mask(img, n, landmask_border) img[mask] = np.nan if remove_spatial_mean: img -= get_spatial_mean(img) # convert to 0 - 255 img = get_uint8_image(img, vmin, vmax, pmin, pmax) # create Nansat with one band only nout = Nansat.from_domain(n, img, parameters={'name': bandName}) nout.set_metadata(n.get_metadata()) # improve geolocation accuracy if len(nout.vrt.dataset.GetGCPs()) > 0: nout.reproject_gcps() nout.vrt.tps = True return nout
if S == 98 and F == 10: CC = 98 + 10 # color code for iceberg CT = ft.GetFieldAsInteger('CT') else: CC = 2 # color code for bergy water if CT == 1: CC = 1 # color code for open water if CT == 98: CC = 0 # color code for ice free ft.SetField('classID', int(CC)) oLayer.SetFeature(ft) sigma0 = gdal.Open(ifile).ReadAsArray() fp_classID = gdal.Open(ifile) gdal.RasterizeLayer(fp_classID, [1], oLayer, options=["ATTRIBUTE=classID"]) classID = fp_classID.ReadAsArray() classID[classID == sigma0] = 255 classID[np.isnan(classID)] = 99 nansatObjSigma0 = Nansat(ifile) nansatObjIceChart = Nansat.from_domain(array=classID.astype(np.uint8), domain=nansatObjSigma0) nansatObjIceChart.set_metadata(nansatObjSigma0.get_metadata()) nansatObjIceChart.set_metadata('entry_title', 'REPROJECTED_%s_ICE_CHART' % cfg.sourceType) nansatObjIceChart = add_colortable(nansatObjIceChart, cfg.sourceType) nansatObjIceChart.export(ofile, bands=[1], driver='GTiff') if cfg.quicklook: rgb = np.zeros((classID.shape[0], classID.shape[1], 3), 'uint8') for k in colorDict[cfg.sourceType].keys(): rgb[classID == k, :] = colorDict[cfg.sourceType][k] plt.imsave(ofile.replace('.tif', '.png'), rgb)