def test_export_gcps_complex_to_netcdf(self):
''' Should export file with GCPs and write correct complex bands'''
n0 = Nansat(self.test_file_gcps, logLevel=40)
b0 = n0['L_469']
n1 = Nansat(domain=n0)
n1.add_band(b0.astype('complex64'), parameters={'name': 'L_469'})
tmpfilename = os.path.join(ntd.tmp_data_path,
'nansat_export_gcps_complex.nc')
n1.export(tmpfilename)
ncf = netcdf_file(tmpfilename)
self.assertTrue(os.path.exists(tmpfilename))
self.assertTrue('GCPX' in ncf.variables)
self.assertTrue('GCPY' in ncf.variables)
self.assertTrue('GCPPixel' in ncf.variables)
self.assertTrue('GCPLine' in ncf.variables)
n2 = Nansat(tmpfilename)
b2 = n2['L_469']
np.testing.assert_allclose(b0, b2)
lon0, lat0 = n0.get_geolocation_grids()
lon2, lat2 = n1.get_geolocation_grids()
np.testing.assert_allclose(lon0, lon2)
np.testing.assert_allclose(lat0, lat2)
def test_export_gcps_complex_to_netcdf(self):
''' Should export file with GCPs and write correct complex bands'''
n0 = Nansat(self.test_file_gcps, logLevel=40)
b0 = n0['L_469']
n1 = Nansat(domain=n0)
n1.add_band(b0.astype('complex64'),
parameters={'name': 'L_469'})
tmpfilename = os.path.join(ntd.tmp_data_path, 'nansat_export_gcps_complex.nc')
n1.export(tmpfilename)
ncf = netcdf_file(tmpfilename)
self.assertTrue(os.path.exists(tmpfilename))
self.assertTrue('GCPX' in ncf.variables)
self.assertTrue('GCPY' in ncf.variables)
self.assertTrue('GCPPixel' in ncf.variables)
self.assertTrue('GCPLine' in ncf.variables)
n2 = Nansat(tmpfilename)
b2 = n2['L_469']
np.testing.assert_allclose(b0, b2)
lon0, lat0 = n0.get_geolocation_grids()
lon2, lat2 = n1.get_geolocation_grids()
np.testing.assert_allclose(lon0, lon2)
np.testing.assert_allclose(lat0, lat2)
def test_get_item_inf_expressions(self):
''' inf should be replaced with nan '''
d = Domain(4326, "-te 25 70 35 72 -ts 500 500")
n = Nansat(domain=d, logLevel=40)
arr = np.empty((500, 500))
n.add_band(arr, {'expression': 'np.array([0,1,2,3,np.inf,5,6,7])'})
self.assertIsInstance(n[1], np.ndarray)
self.assertTrue(np.isnan(n[1][4]))
def test_get_item_inf_expressions(self):
''' inf should be replaced with nan '''
d = Domain(4326, "-te 25 70 35 72 -ts 500 500")
n = Nansat(domain=d, logLevel=40)
arr = np.empty((500, 500))
n.add_band(arr, {'expression': 'np.array([0,1,2,3,np.inf,5,6,7])'})
self.assertIsInstance(n[1], np.ndarray)
self.assertTrue(np.isnan(n[1][4]))
def test_dont_export2thredds_gcps(self):
n = Nansat(self.test_file_gcps, logLevel=40)
n2 = Nansat(domain=n)
n.add_band(np.ones(n2.shape(), np.float32))
tmpfilename = os.path.join(ntd.tmp_data_path,
'nansat_export2thredds.nc')
self.assertRaises(OptionError, n2.export2thredds, tmpfilename,
['L_645'])
def test_dont_export2thredds_gcps(self):
n = Nansat(self.test_file_gcps, log_level=40, mapper=self.default_mapper)
n2 = Nansat.from_domain(n)
n.add_band(np.ones(n2.shape(), np.float32))
tmpfilename = os.path.join(self.tmp_data_path,
'nansat_export2thredds.nc')
self.assertRaises(ValueError, n2.export2thredds, tmpfilename,
['L_645'])
def test_dont_export2thredds_gcps(self):
n = Nansat(self.test_file_gcps, logLevel=40)
n2 = Nansat(domain=n)
n.add_band(np.ones(n2.shape(), np.float32))
tmpfilename = os.path.join(ntd.tmp_data_path,
'nansat_export2thredds.nc')
self.assertRaises(OptionError, n2.export2thredds, tmpfilename,
['L_645'])
def test_add_band(self):
d = Domain(4326, "-te 25 70 35 72 -ts 500 500")
arr = np.random.randn(500, 500)
n = Nansat(domain=d, logLevel=40)
n.add_band(arr, {'name': 'band1'})
self.assertEqual(type(n), Nansat)
self.assertEqual(type(n[1]), np.ndarray)
self.assertEqual(n.get_metadata('name', 1), 'band1')
self.assertEqual(n[1].shape, (500, 500))
def test_dont_export2thredds_gcps(self):
n = Nansat(self.test_file_gcps,
log_level=40,
mapper=self.default_mapper)
n2 = Nansat.from_domain(n)
n.add_band(np.ones(n2.shape(), np.float32))
tmpfilename = os.path.join(self.tmp_data_path,
'nansat_export2thredds.nc')
self.assertRaises(ValueError, n2.export2thredds, tmpfilename,
['L_645'])
def test_add_band(self):
d = Domain(4326, "-te 25 70 35 72 -ts 500 500")
arr = np.random.randn(500, 500)
n = Nansat(domain=d, logLevel=40)
n.add_band(arr, {'name': 'band1'})
self.assertEqual(type(n), Nansat)
self.assertEqual(type(n[1]), np.ndarray)
self.assertEqual(n.get_metadata('name', 1), 'band1')
self.assertEqual(n[1].shape, (500, 500))
def test_add_subvrts_only_to_one_nansat(self):
d = Domain(4326, "-te 25 70 35 72 -ts 500 500")
arr = np.random.randn(500, 500)
n1 = Nansat(domain=d, logLevel=40)
n2 = Nansat(domain=d, logLevel=40)
n1.add_band(arr, {'name': 'band1'})
self.assertEqual(type(n1.vrt.bandVRTs), dict)
self.assertTrue(len(n1.vrt.bandVRTs) > 0)
self.assertEqual(n2.vrt.bandVRTs, {})
def test_add_subvrts_only_to_one_nansat(self):
d = Domain(4326, "-te 25 70 35 72 -ts 500 500")
arr = np.random.randn(500, 500)
n1 = Nansat(domain=d, logLevel=40)
n2 = Nansat(domain=d, logLevel=40)
n1.add_band(arr, {'name': 'band1'})
self.assertEqual(type(n1.vrt.bandVRTs), dict)
self.assertTrue(len(n1.vrt.bandVRTs) > 0)
self.assertEqual(n2.vrt.bandVRTs, {})
def test_dont_export2thredds_gcps(self):
n = Nansat(self.test_file_gcps,
log_level=40,
mapper=self.default_mapper)
n2 = Nansat.from_domain(n)
n.add_band(np.ones(n2.shape(), np.float32))
tmpfilename = os.path.join(self.tmp_data_path,
'nansat_export2thredds.nc')
with self.assertRaises(ValueError) as e:
n2.export2thredds(tmpfilename)
self.assertIn('Cannot export dataset with GCPS', e.exception.args[0])
def test_get_item_basic_expressions(self):
''' Testing get_item with some basic expressions '''
d = Domain(4326, "-te 25 70 35 72 -ts 500 500")
n = Nansat(domain=d, logLevel=40)
arr = np.empty((500, 500))
n.add_band(arr, {'expression': '1+1'})
n.add_band(arr, {'expression': 'np.random.randn(500, 500)'})
self.assertIsInstance(n[1], int)
self.assertIsInstance(n[2], np.ndarray)
self.assertEqual(n[1], 2)
self.assertEqual(len(n[2]), 500)
self.assertEqual(len(n[2][0]), 500)
def test_export_selected_bands(self):
n = Nansat(self.test_file_gcps)
resfile = 'tmp.nc'
new_band = np.random.randn(n.shape()[0], n.shape()[1])
n.add_band(new_band, {'name': 'newBand'})
# Test with band numbers
n.export(resfile, bands=[4, 2])
self.assertTrue(os.path.exists(resfile))
nn = Nansat(resfile)
self.assertTrue(nn.has_band('newBand'))
self.assertTrue(nn.has_band('L_555'))
os.unlink(resfile)
def get_denoised_object(filename, bandName, factor, **kwargs):
from sentinel1denoised.S1_EW_GRD_NoiseCorrection import Sentinel1Image
s = Sentinel1Image(filename)
s.add_denoised_band('sigma0_HV', **kwargs)
s.resize(factor, eResampleAlg=-1)
img = s[bandName + '_denoised']
n = Nansat(domain=s)
n.add_band(img, parameters=s.get_metadata(bandID=bandName))
n.set_metadata(s.get_metadata())
return n
def test_export_selected_bands(self):
n = Nansat(self.test_file_gcps)
resfile = 'tmp.nc'
new_band = np.random.randn(n.shape()[0], n.shape()[1])
n.add_band(new_band, {'name': 'newBand'})
# Test with band numbers
n.export(resfile, bands=[4, 2])
self.assertTrue(os.path.exists(resfile))
nn = Nansat(resfile)
self.assertTrue(nn.has_band('newBand'))
self.assertTrue(nn.has_band('L_555'))
os.unlink(resfile)
def test_get_item_basic_expressions(self):
''' Testing get_item with some basic expressions '''
d = Domain(4326, "-te 25 70 35 72 -ts 500 500")
n = Nansat(domain=d, logLevel=40)
arr = np.empty((500, 500))
n.add_band(arr, {'expression': '1+1'})
n.add_band(arr, {'expression': 'np.random.randn(500, 500)'})
self.assertIsInstance(n[1], int)
self.assertIsInstance(n[2], np.ndarray)
self.assertEqual(n[1], 2)
self.assertEqual(len(n[2]), 500)
self.assertEqual(len(n[2][0]), 500)
def test_export2thredds_longlat_list(self):
d = Domain("+proj=latlong +datum=WGS84 +ellps=WGS84 +no_defs",
"-te 27 70 31 72 -ts 200 200")
n = Nansat(domain=d)
n.add_band(np.ones(d.shape(), np.float32),
parameters={'name': 'L_469'})
n.set_metadata('time_coverage_start', '2016-01-19')
tmpfilename = os.path.join(ntd.tmp_data_path,
'nansat_export2thredds_longlat.nc')
n.export2thredds(tmpfilename, ['L_469'])
ncI = netcdf_file(tmpfilename, 'r')
ncIVar = ncI.variables['L_469']
self.assertTrue(ncIVar.grid_mapping in ncI.variables.keys())
def test_export2thredds_longlat_list(self):
d = Domain("+proj=latlong +datum=WGS84 +ellps=WGS84 +no_defs",
"-te 27 70 31 72 -ts 200 200")
n = Nansat(domain=d)
n.add_band(np.ones(d.shape(), np.float32),
parameters={'name': 'L_469'})
n.set_metadata('time_coverage_start', '2016-01-19')
tmpfilename = os.path.join(ntd.tmp_data_path,
'nansat_export2thredds_longlat.nc')
n.export2thredds(tmpfilename, ['L_469'])
ncI = netcdf_file(tmpfilename, 'r')
ncIVar = ncI.variables['L_469']
self.assertTrue(ncIVar.grid_mapping in ncI.variables.keys())
def test_add_band_and_reproject(self):
""" Should add band and swath mask and return np.nan in areas out of swath """
n = Nansat(self.test_file_gcps, log_level=40, mapper=self.default_mapper)
d = Domain(4326, "-te 27 70 30 72 -ts 500 500")
n.add_band(np.ones(n.shape(), np.uint8))
n.reproject(d)
b4 = n[4] # added, reprojected band
b5 = n[5] # swathmask
self.assertTrue(n.has_band('swathmask')) # the added band
self.assertTrue(n.has_band('swathmask_0000')) # the actual swathmask
self.assertTrue(b4[0, 0]==0)
self.assertTrue(b4[300, 300] == 1)
self.assertTrue(b5[0, 0]==0)
self.assertTrue(b5[300, 300] == 1)
def test_add_band_and_reproject(self):
''' Should add band and swath mask
and return 0 in areas out of swath '''
n = Nansat(self.test_file_gcps, logLevel=40)
d = Domain(4326, "-te 27 70 30 72 -ts 500 500")
n.add_band(np.ones(n.shape()))
n.reproject(d)
b1 = n[1]
b4 = n[4]
self.assertTrue(n.has_band('swathmask'))
self.assertTrue(b1[0, 0] == 0)
self.assertTrue(b1[300, 300] > 0)
self.assertTrue(np.isnan(b4[0, 0]))
self.assertTrue(b4[300, 300] == 1.)
def get_denoised_object(filename, bandName, factor, **kwargs):
''' Use sentinel1denoised and preform thermal noise removal
Import is done within the function to make the dependency not so strict
'''
from sentinel1denoised.S1_EW_GRD_NoiseCorrection import Sentinel1Image
s = Sentinel1Image(filename)
s.add_denoised_band('sigma0_HV', **kwargs)
s.resize(factor, eResampleAlg=-1)
img = s[bandName + '_denoised']
n = Nansat(domain=s)
n.add_band(img, parameters=s.get_metadata(bandID=bandName))
n.set_metadata(s.get_metadata())
return n
def test_add_band_and_reproject(self):
''' Should add band and swath mask
and return 0 in areas out of swath '''
n = Nansat(self.test_file_gcps, logLevel=40)
d = Domain(4326, "-te 27 70 30 72 -ts 500 500")
n.add_band(np.ones(n.shape()))
n.reproject(d)
b1 = n[1]
b4 = n[4]
self.assertTrue(n.has_band('swathmask'))
self.assertTrue(b1[0, 0] == 0)
self.assertTrue(b1[300, 300] > 0)
self.assertTrue(np.isnan(b4[0, 0]))
self.assertTrue(b4[300, 300] == 1.)
def test_add_band_and_reproject(self):
""" Should add band and swath mask and return 0 in areas out of swath """
n = Nansat(self.test_file_gcps, log_level=40, mapper=self.default_mapper)
d = Domain(4326, "-te 27 70 30 72 -ts 500 500")
n.add_band(np.ones(n.shape()))
n.reproject(d)
b1 = n[1]
b4 = n[4]
self.assertTrue(n.has_band('swathmask')) # the added band
self.assertTrue(n.has_band('swathmask_0000')) # the actual swathmask
self.assertTrue(b1[0, 0] == 0)
self.assertTrue(b1[300, 300] > 0)
self.assertTrue(np.isnan(b4[0, 0]))
self.assertTrue(b4[300, 300] == 1.)
def test_export_netcdf(self):
""" Test export and following import of data with bands containing np.nan values """
n = Nansat(self.test_file_gcps, mapper=self.default_mapper)
arrNoNaN = np.random.randn(n.shape()[0], n.shape()[1])
n.add_band(arrNoNaN, {'name': 'testBandNoNaN'})
arrWithNaN = arrNoNaN.copy()
arrWithNaN[int(n.shape()[0] / 2.) - 10:int(n.shape()[0] / 2 + 10),
int(n.shape()[1] / 2.) - 10:int(n.shape()[1] / 2 + 10)] = np.nan
n.add_band(arrWithNaN, {'name': 'testBandWithNaN'})
n.export(self.tmp_filename)
exported = Nansat(self.tmp_filename, mapper=self.default_mapper)
earrNoNaN = exported['testBandNoNaN']
# Use allclose to allow some roundoff errors
self.assertTrue(np.allclose(arrNoNaN, earrNoNaN))
earrWithNaN = exported['testBandWithNaN']
np.testing.assert_allclose(arrWithNaN, earrWithNaN)
def test_repr_basic(self):
''' repr should include some basic elements '''
d = Domain(4326, "-te 25 70 35 72 -ts 500 500")
n = Nansat(domain=d, logLevel=40)
arr = np.empty((500, 500))
exp = 'np.array([0,1,2,3,np.inf,5,6,7])'
n.add_band(arr, {'expression': exp})
n_repr = repr(n)
self.assertIn(exp, n_repr, 'The expressions should be in repr')
self.assertIn('SourceFilename', n_repr)
self.assertIn('/vsimem/', n_repr)
self.assertIn('500 x 500', n_repr)
self.assertIn('Projection:', n_repr)
self.assertIn('25', n_repr)
self.assertIn('72', n_repr)
self.assertIn('35', n_repr)
self.assertIn('70', n_repr)
def test_export_netcdf(self):
""" Test export and following import of data with bands containing np.nan values """
n = Nansat(self.test_file_gcps, mapper=self.default_mapper)
arrNoNaN = np.random.randn(n.shape()[0], n.shape()[1])
n.add_band(arrNoNaN, {'name': 'testBandNoNaN'})
arrWithNaN = arrNoNaN.copy()
arrWithNaN[int(n.shape()[0] / 2.) - 10:int(n.shape()[0] / 2 + 10),
int(n.shape()[1] / 2.) - 10:int(n.shape()[1] / 2 +
10)] = np.nan
n.add_band(arrWithNaN, {'name': 'testBandWithNaN'})
n.export(self.tmp_filename)
exported = Nansat(self.tmp_filename, mapper=self.default_mapper)
earrNoNaN = exported['testBandNoNaN']
# Use allclose to allow some roundoff errors
self.assertTrue(np.allclose(arrNoNaN, earrNoNaN))
earrWithNaN = exported['testBandWithNaN']
np.testing.assert_allclose(arrWithNaN, earrWithNaN)
def test_repr_basic(self):
''' repr should include some basic elements '''
d = Domain(4326, "-te 25 70 35 72 -ts 500 500")
n = Nansat(domain=d, logLevel=40)
arr = np.empty((500, 500))
exp = 'np.array([0,1,2,3,np.inf,5,6,7])'
n.add_band(arr, {'expression': exp})
n_repr = repr(n)
self.assertIn(exp, n_repr, 'The expressions should be in repr')
self.assertIn('SourceFilename', n_repr)
self.assertIn('/vsimem/', n_repr)
self.assertIn('500 x 500', n_repr)
self.assertIn('Projection:', n_repr)
self.assertIn('25', n_repr)
self.assertIn('72', n_repr)
self.assertIn('35', n_repr)
self.assertIn('70', n_repr)
def test_export_netcdf(self):
''' Test export and following import of data with bands containing
np.nan values
'''
n = Nansat(self.test_file_gcps)
arrNoNaN = np.random.randn(n.shape()[0], n.shape()[1])
n.add_band(arrNoNaN, {'name': 'testBandNoNaN'})
arrWithNaN = arrNoNaN.copy()
arrWithNaN[n.shape()[0] / 2 - 10:n.shape()[0] / 2 + 10,
n.shape()[1] / 2 - 10:n.shape()[1] / 2 + 10] = np.nan
n.add_band(arrWithNaN, {'name': 'testBandWithNaN'})
n.export(self.tmpfilename)
exported = Nansat(self.tmpfilename)
earrNoNaN = exported['testBandNoNaN']
# Use allclose to allow some roundoff errors
self.assertTrue(np.allclose(arrNoNaN, earrNoNaN))
earrWithNaN = exported['testBandWithNaN']
np.testing.assert_allclose(arrWithNaN, earrWithNaN)
os.unlink(self.tmpfilename)
def test_export_netcdf(self):
''' Test export and following import of data with bands containing
np.nan values
'''
n = Nansat(self.test_file_gcps)
arrNoNaN = np.random.randn(n.shape()[0], n.shape()[1])
n.add_band(arrNoNaN, {'name': 'testBandNoNaN'})
arrWithNaN = arrNoNaN.copy()
arrWithNaN[n.shape()[0] / 2 - 10:n.shape()[0] / 2 + 10,
n.shape()[1] / 2 - 10:n.shape()[1] / 2 + 10] = np.nan
n.add_band(arrWithNaN, {'name': 'testBandWithNaN'})
n.export(self.tmpfilename)
exported = Nansat(self.tmpfilename)
earrNoNaN = exported['testBandNoNaN']
# Use allclose to allow some roundoff errors
self.assertTrue(np.allclose(arrNoNaN, earrNoNaN))
earrWithNaN = exported['testBandWithNaN']
np.testing.assert_allclose(arrWithNaN, earrWithNaN)
os.unlink(self.tmpfilename)
def main(argv=None):
year = '2012'
useMask = False
if argv is None:
argv = sys.argv
if argv is None:
print("Please specify the path/year to the asar folder! \n")
return
# Parse arguments
try:
opts, args = getopt.getopt(argv, "hi:o:",
["year=", "oPath=", "iPath=", "useMask="])
except getopt.GetoptError:
print 'readASAR.py -year <year> ...'
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print 'readASAR.py -year <year> ...'
sys.exit()
elif opt in ("-year", "--year"):
year = arg
elif opt in ("-oPath", "--oPath"):
oPath = arg
elif opt in ("-iPath", "--iPath"):
iPath = arg
elif opt in ("-useMask", "--useMask"):
useMask = arg
oPath = '/media/SOLabNFS2/tmp/roughness/' + year + '/'
iPath = '/media/SOLabNFS2/store/satellite/asar/' + year + '/'
if not os.path.exists(oPath):
os.makedirs(oPath)
dirNames = os.listdir(iPath)
for dirName in dirNames:
fileNames = os.listdir(iPath + dirName)
for fileName in fileNames:
figureName = oPath + fileName[0:27] + '/' + fileName + '_proj.png'
kmlName = oPath + fileName[0:27] + '/' + fileName + '.kml'
if not os.path.exists(oPath + fileName[0:27] + '/'):
os.makedirs(oPath + fileName[0:27] + '/')
if os.path.isfile(kmlName):
print "%s already processed" % (fileName)
continue
else:
print "%s" % (fileName)
# try to create Nansat object
try:
n = Nansat(iPath + dirName + '/' + fileName,
mapperName='asar',
logLevel=27)
except Exception as e:
print "Failed to create Nansat object:"
print str(e)
os.rmdir(oPath + fileName[0:27] + '/')
continue
#~ Get the bands
raw_counts = n[1]
inc_angle = n[2]
#~ NICE image (roughness)
pol = n.bands()[3]['polarization']
if pol == 'HH':
ph = (2.20495, -14.3561e-2, 11.28e-4)
sigma0_hh_ref = exp(
(ph[0] + inc_angle * ph[1] + inc_angle**2 * ph[2]) *
log(10))
roughness = n[3] / sigma0_hh_ref
elif pol == 'VV':
pv = (2.29373, -15.393e-2, 15.1762e-4)
sigma0_vv_ref = exp(
(pv[0] + inc_angle * pv[1] + inc_angle**2 * pv[2]) *
log(10))
roughness = n[3] / sigma0_vv_ref
#~ Create new band
n.add_band(bandID=4, array=roughness, \
parameters={'name':'roughness', \
'wkv': 'surface_backwards_scattering_coefficient_of_radar_wave', \
'dataType': 6})
# Reproject image into Lat/Lon WGS84 (Simple Cylindrical) projection
# 1. Cancel previous reprojection
# 2. Get corners of the image and the pixel resolution
# 3. Create Domain with stereographic projection, corner coordinates 1000m
# 4. Reproject
# 5. Write image
n.reproject() # 1.
lons, lats = n.get_corners() # 2.
# Pixel resolution
#~ pxlRes = distancelib.getPixelResolution(array(lats), array(lons), n.shape())
#~ pxlRes = array(pxlRes)*360/40000 # great circle distance
pxlRes = array(
distancelib.getPixelResolution(array(lats), array(lons),
n.shape(), 'deg'))
ipdb.set_trace()
if min(lats) >= 65 and max(
lats) >= 75 and max(lats) - min(lats) >= 13:
pxlRes = array([0.00065, 0.00065
]) * 2 # make the resolution 150x150m
#~ pxlRes = pxlRes*7 # make the resolution worser
srsString = "+proj=latlong +datum=WGS84 +ellps=WGS84 +no_defs"
#~ extentString = '-lle %f %f %f %f -ts 3000 3000' % (min(lons), min(lats), max(lons), max(lats))
extentString = '-lle %f %f %f %f -tr %f %f' % (min(lons), min(lats), \
max(lons), max(lats), pxlRes[1], pxlRes[0])
d = Domain(srs=srsString, ext=extentString) # 3.
n.reproject(d) # 4.
if useMask:
# get array with watermask (landmask) b
# it must be done after reprojection!
# 1. Get Nansat object with watermask
# 2. Get array from Nansat object. 0 - land, 1 - water
#wm = n.watermask(mod44path='/media/magDesk/media/SOLabNFS/store/auxdata/coastline/mod44w/')
wm = n.watermask(
mod44path='/media/data/data/auxdata/coastline/mod44w/')
wmArray = wm[1]
#~ ОШИБКА numOfColor=255 не маскирует, потому что в figure.apply_mask: availIndeces = range(self.d['numOfColor'], 255 - 1)
#~ n.write_figure(fileName=figureName, bands=[3], \
#~ numOfColor=255, mask_array=wmArray, mask_lut={0: 0},
#~ clim=[0,0.15], cmapName='gray', transparency=0) # 5.
n.write_figure(fileName=figureName, bands=[4], \
mask_array=wmArray, mask_lut={0: [0,0,0]},
clim=[0,2], cmapName='gray', transparency=[0,0,0]) # 5.
else:
n.write_figure(fileName=figureName, bands=[1], \
clim=[0,2], cmapName='gray', transparency=[0,0,0]) # 5.
# open the input image and convert to RGBA for further tiling with slbtiles
input_img = Image.open(figureName)
output_img = input_img.convert("RGBA")
output_img.save(figureName)
# make KML image
n.write_kml_image(kmlFileName=kmlName, kmlFigureName=figureName)
#~ Change the file permissions
os.chmod(oPath, 0777)
os.chmod(oPath + fileName[0:27] + '/', 0777)
os.chmod(kmlName, 0777)
os.chmod(figureName, 0777)
#~ Change the owner and group
#~ os.chown(oPath, 1111, 1111)
#~ os.chown(oPath + fileName[0:27] + '/', 1111, 1111)
#~ os.chown(kmlName, 1111, 1111)
#~ os.chown(figureName, 1111, 1111)
#~ garbage collection
gc.collect()
def boreali_processing(obj, final_path):
wavelen = [412, 443, 469, 488, 531, 547, 555, 645, 667, 678]
cpa_limits = [0.01, 2,
0.01, 1,
0.01, 1, 10]
b = Boreali('michigan', wavelen)
n = Nansat(obj)
dom = Domain('+proj=latlong +datum=WGS84 +ellps=WGS84 +no_defs', '-lle -86.3 44.6 -85.2 45.3 -ts 300 200')
n.reproject(dom)
theta = numpy.zeros_like(n[2])
custom_n = Nansat(domain=n)
band_rrs_numbers = list(map(lambda x: n._get_band_number('Rrs_' + str(x)),
wavelen))
for index in range(0, len(wavelen)):
# Преобразуем в Rrsw
rrsw = n[band_rrs_numbers[index]] / (0.52 + 1.7 * n[band_rrs_numbers[index]])
custom_n.add_band(rrsw, parameters={'name': 'Rrsw_' + str(wavelen[index]),
'units': 'sr-1',
'wavelength': wavelen[index]})
custom_n = create_mask(custom_n)
cpa = b.process(custom_n, cpa_limits, mask=custom_n['mask'], theta=theta, threads=4)
custom_n.add_band(array=cpa[0], parameters={'name': 'chl',
'long_name': 'Chlorophyl-a',
'units': 'mg m-3'})
custom_n.add_band(array=cpa[1], parameters={'name': 'tsm',
'long_name': 'Total suspended matter',
'units': 'g m-3'})
custom_n.add_band(array=cpa[2], parameters={'name': 'doc',
'long_name': 'Dissolved organic carbon',
'units': 'gC m-3'})
custom_n.add_band(array=cpa[3], parameters={'name': 'mse',
'long_name': 'Root Mean Square Error',
'units': 'sr-1'})
custom_n.add_band(array=cpa[4], parameters={'name': 'mask',
'long_name': 'L2 Boreali mask',
'units': '1'})
custom_n.export(final_path + obj.split('/')[-1] + 'cpa_deep.nc')
fig_params = {'legend': True,
'LEGEND_HEIGHT': 0.5,
'NAME_LOCATION_Y': 0,
'mask_array': cpa[4],
'mask_lut': {1: [255, 255, 255], 2: [128, 128, 128], 4: [200, 200, 255]}}
custom_n.write_figure(final_path + obj.split('/')[-1] + 'chl_deep.png', 'chl', clim=[0, 1.], **fig_params)
custom_n.write_figure(final_path + obj.split('/')[-1] + 'tsm_deep.png', 'tsm', clim=[0, 1.], **fig_params)
custom_n.write_figure(final_path + obj.split('/')[-1] + 'doc_deep.png', 'doc', clim=[0, .2], **fig_params)
custom_n.write_figure(final_path + obj.split('/')[-1] + 'mse_deep.png', 'mse', clim=[1e-5, 1e-2], logarithm=True, **fig_params)
n.write_figure(final_path + obj.split('/')[-1] + 'rgb_deep.png',
[16, 14, 6],
clim=[[0, 0, 0], [0.006, 0.04, 0.024]],
mask_array=cpa[4],
mask_lut={2: [128, 128, 128]})
def boreali_osw_processing(obj, final_path):
"""
Мой код в данной функции основан на tutorial.py который я нашел в репозитории boreali.
:param obj: путь до изображения
:param final_path: Путь для сохранения файлов
:return:
"""
wavelen = [412, 443, 469, 488, 531, 547, 555, 645, 667, 678]
cpa_limits = [0.01, 2,
0.01, 1,
0.01, 1, 10]
h = get_deph() # Глубина исследуемого района по батиметрии
b = Boreali('michigan', wavelen)
n = Nansat(obj)
dom = Domain('+proj=latlong +datum=WGS84 +ellps=WGS84 +no_defs', '-lle -86.3 44.6 -85.2 45.3 -ts 300 200')
n.reproject(dom)
theta = numpy.zeros_like(n[2])
custom_n = Nansat(domain=n)
band_rrs_numbers = list(map(lambda x: n._get_band_number('Rrs_' + str(x)),
wavelen)) # Получаем список номеров бандов в которых лежат значения Rrs
# для корректной работы складываем в custom_n значения и Rrs и Rrsw
for index in range(0, len(wavelen)):
rrsw = n[band_rrs_numbers[index]] / (0.52 + 1.7 * n[band_rrs_numbers[index]]) # Пересчитываем Rrs в Rrsw
custom_n.add_band(rrsw, parameters={'name': 'Rrsw_' + str(wavelen[index]), # Складываем в новый объект Rrsw
'units': 'sr-1',
'wavelength': wavelen[index]})
# Складываем в новый объект значения Rrs
custom_n.add_band(n[band_rrs_numbers[index]], parameters={'name': 'Rrs_' + str(wavelen[index]),
'units': 'sr-1',
'wavelength': wavelen[index]})
custom_n = create_mask(custom_n)
cpa = b.process(custom_n, cpa_limits, mask=custom_n['mask'], depth=h, theta=theta, threads=4)
custom_n.add_band(array=cpa[0], parameters={'name': 'chl',
'long_name': 'Chlorophyl-a',
'units': 'mg m-3'})
custom_n.add_band(array=cpa[1], parameters={'name': 'tsm',
'long_name': 'Total suspended matter',
'units': 'g m-3'})
custom_n.add_band(array=cpa[2], parameters={'name': 'doc',
'long_name': 'Dissolved organic carbon',
'units': 'gC m-3'})
custom_n.add_band(array=cpa[3], parameters={'name': 'mse',
'long_name': 'Root Mean Square Error',
'units': 'sr-1'})
custom_n.add_band(array=cpa[4], parameters={'name': 'mask',
'long_name': 'L2 Boreali mask',
'units': '1'})
custom_n.export(final_path + obj.split('/')[-1] + 'cpa_OSW.nc')
fig_params = {'legend': True,
'LEGEND_HEIGHT': 0.5,
'NAME_LOCATION_Y': 0,
'mask_array': cpa[4],
'mask_lut': {1: [255, 255, 255],
2: [128, 128, 128],
4: [200, 200, 255]}}
custom_n.write_figure(final_path + obj.split('/')[-1] + 'chl_OSW.png', 'chl', clim=[0, 1.], **fig_params)
custom_n.write_figure(final_path + obj.split('/')[-1] + 'tsm_OSW.png', 'tsm', clim=[0, 1.], **fig_params)
custom_n.write_figure(final_path + obj.split('/')[-1] + 'doc_OSW.png', 'doc', clim=[0, .2], **fig_params)
custom_n.write_figure(final_path + obj.split('/')[-1] + 'mse_OSW.png', 'mse', clim=[1e-5, 1e-2],
logarithm=True, **fig_params)
n.write_figure(final_path + obj.split('/')[-1] + 'rgb_OSW.png',
[16, 14, 6],
clim=[[0, 0, 0], [0.006, 0.04, 0.024]],
mask_array=cpa[4],
mask_lut={2: [128, 128, 128]})
print n.shape()
# get list with coordinates of the object corners
print n.get_corners()
# get lists with coordinates of the object borders
print n.get_border()
raw_counts = n[1]
inc_angle = n[2]
#~ sigma0 = n[3]
sigma0 = raw_counts**2.0 * sin(deg2rad(inc_angle))
sigma0 = 10*log10(sigma0)
n.add_band(bandID=4, array=sigma0)
# 1. Remove speckle noise (using Lee-Wiener filter)
speckle_filter('wiener', 7)
# Reprojected image into Lat/Lon WGS84 (Simple Cylindrical) projection
# 1. Cancel previous reprojection
# 2. Get corners of the image and the pixel resolution
# 3. Create Domain with stereographic projection, corner coordinates and resolution 1000m
# 4. Reproject
# 5. Write image
n.reproject() # 1.
lons, lats = n.get_corners() # 2.
pxlRes = distancelib.getPixelResolution(array(lats), array(lons), n.shape(), units="deg")
srsString = "+proj=latlong +datum=WGS84 +ellps=WGS84 +no_defs"
#~ extentString = '-lle %f %f %f %f -ts 3000 3000' % (min(lons), min(lats), max(lons), max(lats))
# get time of the image aquisition
print 'Time:', n.get_time()[0], '\n'
# set GlobalMetadata
n.set_metadata(key='GlobalKey', value='GlobalVal')
# get Global Metadata
print 'Global Metadata:', n.get_metadata(), '\n'
# set BandMetadata to the 1st band
n.set_metadata(key='BandKey', value='BandVal', bandID=1)
# get 1st Band Metadata
print '1st Band Metadata:', n.get_metadata(bandID=1), '\n'
# add a band from file (copy the 2nd band to the end (4th band)
n.add_band(fileName=n.fileName, bandID=2)
# add a band from numpy array (copy the 1st band to the end (5th band))
n.add_band(array=n[1], parameters={'name': 'Name1',
'info': 'copy from the 1st band array'})
# print band list
n.list_bands()
# get GDAL raster band (2nd band)
band = n.get_GDALRasterBand(bandID=2)
# Get band data and do some operations
# -- Get data from 1st band as numpy array
a = n[1]
# -- Plot the array (pyplot image is save to a PNG file)
plt.imshow(a);plt.colorbar();plt.savefig(oFileName + '01_imshow.png');plt.close()
# -- Save as Matlab file
savemat(oFileName + '01.mat', {'band_1': a})
def main( argv=None ):
year = '2012'
useMask = False
if argv is None:
argv = sys.argv
if argv is None:
print ( "Please specify the path/year to the asar folder! \n")
return
# Parse arguments
try:
opts, args = getopt.getopt(argv,"hi:o:",["year=","oPath=","iPath=","useMask="])
except getopt.GetoptError:
print 'readASAR.py -year <year> ...'
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print 'readASAR.py -year <year> ...'
sys.exit()
elif opt in ("-year", "--year"):
year = arg
elif opt in ("-oPath", "--oPath"):
oPath = arg
elif opt in ("-iPath", "--iPath"):
iPath = arg
elif opt in ("-useMask", "--useMask"):
useMask = arg
oPath = '/media/SOLabNFS2/tmp/roughness/' + year + '/'
iPath = '/media/SOLabNFS2/store/satellite/asar/' + year + '/'
if not os.path.exists(oPath):
os.makedirs(oPath)
dirNames=os.listdir(iPath)
for dirName in dirNames:
fileNames=os.listdir(iPath+dirName)
for fileName in fileNames:
figureName = oPath + fileName[0:27] + '/' + fileName + '_proj.png'
kmlName = oPath + fileName[0:27] + '/' + fileName + '.kml'
if not os.path.exists(oPath + fileName[0:27] + '/'):
os.makedirs(oPath + fileName[0:27] + '/')
if os.path.isfile(kmlName):
print "%s already processed" % (fileName)
continue
else:
print "%s" % (fileName)
# try to create Nansat object
try:
n = Nansat(iPath + dirName + '/' + fileName, mapperName='asar', logLevel=27)
except Exception as e:
print "Failed to create Nansat object:"
print str(e)
os.rmdir(oPath + fileName[0:27] + '/' )
continue
#~ Get the bands
raw_counts = n[1]
inc_angle = n[2]
#~ NICE image (roughness)
pol = n.bands()[3]['polarization']
if pol == 'HH':
ph = (2.20495, -14.3561e-2, 11.28e-4)
sigma0_hh_ref = exp( ( ph[0]+inc_angle*ph[1]+inc_angle**2*ph[2])*log(10) )
roughness = n[3]/sigma0_hh_ref
elif pol == 'VV':
pv = (2.29373, -15.393e-2, 15.1762e-4)
sigma0_vv_ref = exp( ( pv[0]+inc_angle*pv[1]+inc_angle**2*pv[2])*log(10) )
roughness = n[3]/sigma0_vv_ref
#~ Create new band
n.add_band(bandID=4, array=roughness, \
parameters={'name':'roughness', \
'wkv': 'surface_backwards_scattering_coefficient_of_radar_wave', \
'dataType': 6})
# Reproject image into Lat/Lon WGS84 (Simple Cylindrical) projection
# 1. Cancel previous reprojection
# 2. Get corners of the image and the pixel resolution
# 3. Create Domain with stereographic projection, corner coordinates 1000m
# 4. Reproject
# 5. Write image
n.reproject() # 1.
lons, lats = n.get_corners() # 2.
# Pixel resolution
#~ pxlRes = distancelib.getPixelResolution(array(lats), array(lons), n.shape())
#~ pxlRes = array(pxlRes)*360/40000 # great circle distance
pxlRes = array(distancelib.getPixelResolution(array(lats), array(lons), n.shape(), 'deg'))
ipdb.set_trace()
if min(lats) >= 65 and max(lats) >= 75 and max(lats)-min(lats) >= 13:
pxlRes = array([0.00065, 0.00065])*2 # make the resolution 150x150m
#~ pxlRes = pxlRes*7 # make the resolution worser
srsString = "+proj=latlong +datum=WGS84 +ellps=WGS84 +no_defs"
#~ extentString = '-lle %f %f %f %f -ts 3000 3000' % (min(lons), min(lats), max(lons), max(lats))
extentString = '-lle %f %f %f %f -tr %f %f' % (min(lons), min(lats), \
max(lons), max(lats), pxlRes[1], pxlRes[0])
d = Domain(srs=srsString, ext=extentString) # 3.
n.reproject(d) # 4.
if useMask:
# get array with watermask (landmask) b
# it must be done after reprojection!
# 1. Get Nansat object with watermask
# 2. Get array from Nansat object. 0 - land, 1 - water
#wm = n.watermask(mod44path='/media/magDesk/media/SOLabNFS/store/auxdata/coastline/mod44w/')
wm = n.watermask(mod44path='/media/data/data/auxdata/coastline/mod44w/')
wmArray = wm[1]
#~ ОШИБКА numOfColor=255 не маскирует, потому что в figure.apply_mask: availIndeces = range(self.d['numOfColor'], 255 - 1)
#~ n.write_figure(fileName=figureName, bands=[3], \
#~ numOfColor=255, mask_array=wmArray, mask_lut={0: 0},
#~ clim=[0,0.15], cmapName='gray', transparency=0) # 5.
n.write_figure(fileName=figureName, bands=[4], \
mask_array=wmArray, mask_lut={0: [0,0,0]},
clim=[0,2], cmapName='gray', transparency=[0,0,0]) # 5.
else:
n.write_figure(fileName=figureName, bands=[1], \
clim=[0,2], cmapName='gray', transparency=[0,0,0]) # 5.
# open the input image and convert to RGBA for further tiling with slbtiles
input_img = Image.open(figureName)
output_img = input_img.convert("RGBA")
output_img.save(figureName)
# make KML image
n.write_kml_image(kmlFileName=kmlName, kmlFigureName=figureName)
#~ Change the file permissions
os.chmod(oPath, 0777)
os.chmod(oPath + fileName[0:27] + '/', 0777)
os.chmod(kmlName, 0777)
os.chmod(figureName, 0777)
#~ Change the owner and group
#~ os.chown(oPath, 1111, 1111)
#~ os.chown(oPath + fileName[0:27] + '/', 1111, 1111)
#~ os.chown(kmlName, 1111, 1111)
#~ os.chown(figureName, 1111, 1111)
#~ garbage collection
gc.collect()
# 1. Reproject the current object
# 2. Get array with data
# 2. Create new Nansat object from the given array and for given domain
n.reproject(dStereo)
array = n[1]
nStereo = Nansat(domain=dStereo, array=array, parameters={'name': 'band1'})
print 'Stereo Nansat:', nStereo
# add band from array to existing object
# 0. Cancel reprojection. Adding bands works only on non-reprojected data
# 1. Get the data from the object and modify
# 2. Add band with modified data to the object
# 3. Check that the band is in the object
n.reproject()
array = n[1] * 10
n.add_band(array=array, parameters={'name': 'new_band', 'about': 'test'})
print 'Nansat with new band:', n
# add band from another file to existing object
n.add_band(fileName='stere.tif', bandID='L_645', parameters={'name': 'L_645_stere'})
print 'Nansat with new band from another file:', n
# Prepare the image for Google Earth exporting
# reproject image into Lat/Lon WGS84 (Simple Cylindrical) projection
# 1. Cancel previous reprojection
# 2. Get corners of the image and the pixel resolution
# 3. Create Domain with stereographic projection, corner coordinates 1000m
# 4. Reproject
n.reproject() # 1.
lons, lats = n.get_corners() # 2.
srsString = "+proj=latlong +datum=WGS84 +ellps=WGS84 +no_defs"
def process_boreali(self, opts):
'''Advanced processing of MODIS images:
retrieve chl, tsm, doc with boreali
generate images
'''
pnDefaults = {
'lmchl': [0, 5, False],
'lmtsm': [0, 3, False],
'lmdoc': [0, 2, False],
'lmmse': [1e-8, 1e-5, True]}
borMinMax = [[pnDefaults['lmchl'][0], pnDefaults['lmchl'][1]],
[pnDefaults['lmtsm'][0], pnDefaults['lmtsm'][1]],
[pnDefaults['lmdoc'][0], pnDefaults['lmdoc'][1]]]
dtsDomain = Domain(opts['srs'], opts['ext'])
fileName = self.get_metadata('name')
oBaseFileName = self.get_metadata('name').strip('"').strip("'")
ncName = opts['oDir'] + oBaseFileName + '.nc'
print ncName
prodFileNames = {}
for pn in opts['prods']:
prodFileNames[pn] = '%s/%s.%s.png' % (opts['oDir'], oBaseFileName, pn)
if os.path.exists(ncName):
print '%s already exist!' % ncName
else:
# good bits for NRT
#self.add_mask(cloudBits=[1, 4, 5, 6, 9, 10, 13, 15, 20, 21, 23, 28, 29, 30])
try:
self.reproject(dtsDomain)
except:
print 'Cannot reproject %s. Skipping' % fileName
return 1
else:
Rrsw_412 = self['Rrsw_412']
if Rrsw_412 is None:
return 1
# process input with BOREALI
b = Boreali(model='northsea', zone='northsea')
cImg = b.process_lm(self, wavelen=[412, 443, 488, 531, 555, 667],
start=opts['start'],
minmax=borMinMax)
# generate Nansat with results
img2 = Nansat(domain=self)
for i, pn in enumerate(opts['prods']):
img2.add_band(array=cImg[i, :, :], parameters={'name': pn})
img2.add_band(array=self['mask'], parameters={'name': 'mask'})
# export results into NC-file
img2.export(ncName)
# write images with concentrations
for pn in opts['prods']:
pnd = pnDefaults[pn]
img2.write_figure(prodFileNames[pn], pn, clim=[pnd[0], pnd[1]], legend=True, logarithm=pnd[2])
return 0
