def __init__(self, url=None, sltype=None, mode=READ_ONLY, **kwargs): if mode != READ_ONLY: raise Exception("This mapper can only be used in read_only mode.") if sltype not in ['i', 'c', 'a', 'b']: raise Exception("Unknown SLSTR product type") super(SAFESLFile, self).__init__(url=url, mode=mode, **kwargs) self.__sltype = sltype self.__data_handlers = [] self.__oblique_fields = [] # coordinate files geodetic = "geodetic_%sn.nc" % sltype times = "time_%sn.nc" % sltype coordinate_files = [geodetic, times] # detect the data files and instanciate mappers for each one datafiles = [] for fname in glob.glob(os.path.join(url, "*_%s[n,o].nc" % sltype)): if os.path.basename(fname) not in coordinate_files: datafiles.append(os.path.basename(fname)) for f in datafiles: fname = os.path.join(url, f) self.__data_handlers.append(NCFile(url=fname, mode=mode, **kwargs)) # instantiate mappers for each coordinate file self.__geod_handler = NCFile(os.path.join(url, geodetic), mode=mode, **kwargs) self.__time_handler = NCFile(os.path.join(url, times), mode=mode, **kwargs) self.__fieldlocator = {} self.__geofieldlocator = {} self.__fieldtranslate = {} # offset between nadir and oblique swath edge self.nadir_to_oblique_offset = None
def __init__(self, url=None, mode=READ_ONLY, **kwargs): if mode != READ_ONLY: raise Exception("This mapper can only be used in read_only mode.") super(SAFEOLFile, self).__init__(url=url, mode=mode, **kwargs) self.__data_handlers = [] self.__fields = {} # ancillary files cartesian = "geo_coordinates.nc" instrument = "instrument_data.nc" times = "time_coordinates.nc" # handlers for ancillary fields self.__time_handler = NCFile(os.path.join(url, times), mode=mode, **kwargs) self.__fields[times] = [] self.__coord_handler = NCFile(os.path.join(url, cartesian), mode=mode, **kwargs) self.__fields[cartesian] = [] self.__instr_handler = NCFile(os.path.join(url, instrument), mode=mode, **kwargs) self.__fields[instrument] = [] # get product type safefolder = os.path.basename(os.path.normpath(url)) print safefolder if "_OL_1_ERR" in safefolder or "_OL_1_EFR" in safefolder: datafiles = DATAFILES["L1B"] elif "OL_2_LRR" in safefolder or "OL_2_LFR" in safefolder: datafiles = DATAFILES["L2LAND"] elif "OL_2_WRR" in safefolder or "OL_2_WFR" in safefolder: datafiles = DATAFILES["L2WATER"] else: raise Exception("Unknown product type") # detect the data files and instanciate mappers for each one for f in datafiles: if '*' in f: fnames = glob.glob(os.path.join(url, f)) for fname in fnames: self.__data_handlers.append(NCFile(url=fname, mode=mode, **kwargs)) self.__fields[f] = [] else: fname = os.path.join(url, f) self.__data_handlers.append(NCFile(url=fname, mode=mode, **kwargs)) self.__fields[f] = [] self.__fieldlocator = {} self.__geofieldlocator = {}
def read_field(self, fieldname): """ Return the field, without its values. Actual values can be retrieved with read_values() method. """ if fieldname in ['wind_speed', 'wind_direction']: # create a virtual field variable = Variable( shortname=fieldname, description=VIRTUALFIELD_DESCR[fieldname], authority=self.get_naming_authority(), standardname=VIRTUALFIELD_STDNAME[fieldname] ) field = Field( variable, OrderedDict([('time', 1), #('z', 1), ('y', self.get_dimsize('y')), ('x', self.get_dimsize('x')) ]), datatype=numpy.dtype(numpy.float32), units=VIRTUALFIELD_UNITS[fieldname] ) field.attach_storage(self.get_field_handler(fieldname)) else: field = NCFile.read_field(self, fieldname) vava=field.variable uni=field.units if 'z' in field.get_dimnames(): field = Field( vava, OrderedDict([('time', 1), #('z', self.get_dimsize('z')), ('y', self.get_dimsize('y')), ('x', self.get_dimsize('x')) ]), datatype=numpy.dtype(numpy.float32), units=uni ) field.attach_storage(self.get_field_handler(fieldname)) return field
def eodyn_current(infile, outdir, vmin=0., vmax=5.08, vmin_pal=0., vmax_pal=2., write_netcdf=False): """ """ # Read/Process data print 'Read/Process data' ncfile = NCFile(infile) if 'id' in ncfile.read_global_attributes(): l4id = ncfile.read_global_attribute('id') # l4id = 'e-Odyn' #ncfile.read_global_attribute('id') elif re.match(r'^e-Odyn_.*\.nc', os.path.basename(infile)) is not None: l4id = 'e-Odyn' else: raise Exception('Unknown GlobCurrent L4 file.') # /TMP ucur = ncfile.read_values(L4_MAPS[l4id]['uname'])[::, ::-1, 0] ucur = np.transpose(ucur) vcur = ncfile.read_values(L4_MAPS[l4id]['vname'])[::, ::-1, 0] vcur = np.transpose(vcur) masku = [ucur == -9999] maskv = [vcur == -9999] if l4id not in ['CourantGeostr']: lon = ncfile.read_values('lon')[0:2].astype('float64') lat = ncfile.read_values('lat')[-1:-3:-1].astype('float64') for i in range(2): # avoid rounding errors lon[i] = np.round(lon[i] * 10000) / 10000 lat[i] = np.round(lat[i] * 10000) / 10000 else: lon = ncfile.read_values('lon')[:] shift = -np.where(lon < 0)[0][0] ucur = np.roll(ucur, shift, axis=1) vcur = np.roll(vcur, shift, axis=1) lon = lon[shift:shift + 2] lat = ncfile.read_values('lat')[-1:-3:-1] lon0, dlon, lat0, dlat = lon[0], lon[1] - lon[0], lat[0], lat[1] - lat[0] #dtime_units = ncfile.read_field('time').units #dtime = num2date(ncfile.read_values('time')[0], dtime_units) timefmt = '%Y-%m-%dT%H:%M:%S.%fZ' start_time = datetime.strptime( ncfile.read_global_attribute('time_coverage_start'), timefmt) stop_time = datetime.strptime( ncfile.read_global_attribute('time_coverage_end'), timefmt) (dtime, time_range) = stfmt.format_time_and_range(start_time, stop_time, units='ms') # rundtime = ncfile.read_global_attribute('date_modified') # rundtime = datetime.strptime(rundtime, '%Y%m%dT%H%M%SZ') # Construct metadata/geolocation/band(s) print 'Construct metadata/geolocation/band(s)' metadata = {} metadata['product_name'] = L4_MAPS[l4id]['productname'] metadata['name'] = os.path.splitext(os.path.basename(infile))[0] metadata['datetime'] = dtime metadata['time_range'] = time_range #metadata['time_range'] = L4_MAPS[l4id]['timerange'] metadata['source_URI'] = infile metadata['source_provider'] = 'e-Odyn' metadata['processing_center'] = '' metadata['conversion_software'] = 'Syntool' metadata['conversion_version'] = '0.0.0' metadata['conversion_datetime'] = stfmt.format_time(datetime.utcnow()) metadata['parameter'] = ['current velocity', 'current direction'] # metadata['type'] = 'model' # metadata['model_longitude_resolution'] = abs(dlon) # metadata['model_latitude_resolution'] = abs(dlat) # metadata['model_analysis_datetime'] = stfmt.format_time(rundtime) geolocation = {} geolocation['projection'] = stfmt.format_gdalprojection() geolocation['geotransform'] = [ lon0 - dlon / 2., dlon, 0, lat0 - dlat / 2., 0, dlat ] band = [] mask = ucur.mask | vcur.mask print(mask) curvel = np.sqrt(ucur.data**2 + vcur.data**2) curdir = np.mod( np.arctan2(vcur.data, ucur.data) * 180. / np.pi + 360., 360.) offset, scale = vmin, (vmax - vmin) / 254. np.clip(curvel, vmin, vmax, out=curvel) array = np.round((curvel - offset) / scale).astype('uint8') array[mask] = 255 array[masku] = 255 array[maskv] = 255 print(array) colortable = stfmt.format_colortable('matplotlib_jet', vmin=vmin, vmax=vmax, vmin_pal=vmin_pal, vmax_pal=vmax_pal) band.append({ 'array': array, 'scale': scale, 'offset': offset, 'description': 'current velocity', 'unittype': 'm/s', 'nodatavalue': 255, 'parameter_range': [vmin, vmax], 'colortable': colortable }) array = np.round(curdir / 360. * 254.).astype('uint8') array[mask] = 255 array[masku] = 255 array[maskv] = 255 band.append({ 'array': array, 'scale': 360. / 254., 'offset': 0., 'description': 'current direction', 'unittype': 'deg', 'nodatavalue': 255, 'parameter_range': [0, 360.] }) # Write geotiff if write_netcdf == False: print 'Write geotiff' tifffile = stfmt.format_tifffilename(outdir, metadata, create_dir=True) stfmt.write_geotiff(tifffile, metadata, geolocation, band) elif write_netcdf == True: print 'Write netcdf' # u/v -> bands band = [] mask = ucur.mask | vcur.mask vmin = -vmax offset, scale = vmin, (vmax - vmin) / 254. u = np.clip(ucur.data, vmin, vmax) array = np.round((u - offset) / scale).astype('uint8') array[mask] = 255 band.append({ 'array': array, 'scale': scale, 'offset': offset, 'description': 'current u', 'unittype': 'm/s', 'nodatavalue': 255, 'parameter_range': [vmin, vmax] }) v = np.clip(vcur.data, vmin, vmax) array = np.round((v - offset) / scale).astype('uint8') array[mask] = 255 band.append({ 'array': array, 'scale': scale, 'offset': offset, 'description': 'current v', 'unittype': 'm/s', 'nodatavalue': 255, 'parameter_range': [vmin, vmax] }) # Write ncfile = stfmt.format_ncfilename(outdir, metadata, create_dir=True) stfmt.write_netcdf(ncfile, metadata, geolocation, band, dgcpy=1., dgcpx=1.)
def current(infile, outdir, vmin=0., vmax=1.50, vmin_pal=0., vmax_pal=1.5, write_netcdf=False): """ """ # Read/Process data print('Read/Process data') ncfile = NCFile(infile) if 'id' in ncfile.read_global_attributes(): l4id = ncfile.read_global_attribute('id') elif (re.match(r'^dt_global_allsat_madt_uv.*\.nc', os.path.basename(infile)) is not None): l4id = 'Surface_height' # vmin = 0.; vmax = 2.; vmin_pal = 0.; vmax_pal = 2. elif (re.match(r'^dt_global_allsat_msla_uv.*\.nc', os.path.basename(infile)) is not None): l4id = 'Sea_Level_Anomaly' # vmin = 0; vmax = 1; vmin_pal = 0.; vmax_pal = 1 elif re.match(r'^mdt.*\.nc', os.path.basename(infile)) is not None: l4id = 'Mean_Dynamic_Topo' # vmin = 0; vmax = 1.5; vmin_pal = 0; vmax_pal = 1.5 elif re.match(r'^Tide_.*\.nc', os.path.basename(infile)) is not None: l4id = 'Tide' # vmin = 0; vmax = 1.5; vmin_pal = 0; vmax_pal = 1.5 else: raise Exception('Unknown file.') # /TMP ucur = ncfile.read_values(L4_MAPS[l4id]['uname'])[0, ::-1, :] vcur = ncfile.read_values(L4_MAPS[l4id]['vname'])[0, ::-1, :] lon = ncfile.read_values('lon')[:].astype('float64') lat = ncfile.read_values('lat')[-1:-3:-1].astype('float64') lon[lon > 180.] = lon[lon > 180.] - 360. indsorted = np.argsort(lon) lon = lon[indsorted] ucur = ucur[:, indsorted] vcur = vcur[:, indsorted] lon0, dlon, lat0, dlat = lon[0], lon[1] - lon[0], lat[0], lat[1] - lat[0] if l4id in [ 'Mean_Dynamic_Topo', ]: dtime = datetime(2014, 12, 1) else: dtime_units = ncfile.read_field('time').units dtime = num2date(ncfile.read_values('time')[0], dtime_units) # rundtime = ncfile.read_global_attribute('date_modified') # rundtime = datetime.strptime(rundtime, '%Y%m%dT%H%M%SZ') # Construct metadata/geolocation/band(s) print('Construct metadata/geolocation/band(s)') metadata = {} metadata['product_name'] = L4_MAPS[l4id]['productname'] metadata['name'] = os.path.splitext(os.path.basename(infile))[0] metadata['datetime'] = stfmt.format_time(dtime) metadata['time_range'] = L4_MAPS[l4id]['timerange'] metadata['source_URI'] = infile metadata['source_provider'] = '*****@*****.**' metadata['processing_center'] = '' metadata['conversion_software'] = 'Syntool' metadata['conversion_version'] = '0.0.0' metadata['conversion_datetime'] = stfmt.format_time(datetime.utcnow()) metadata['parameter'] = [L4_MAPS[l4id]['parameter']] # metadata['type'] = 'model' # metadata['model_longitude_resolution'] = abs(dlon) # metadata['model_latitude_resolution'] = abs(dlat) # metadata['model_analysis_datetime'] = stfmt.format_time(rundtime) geolocation = {} geolocation['projection'] = stfmt.format_gdalprojection() if l4id in [ 'Tide', ]: ucur = ucur[:-1, ::] vcur = vcur[:-1, ::] geolocation['geotransform'] = [lon0, dlon, 0, lat0, 0, dlat] else: geolocation['geotransform'] = [ lon0 - dlon / 2., dlon, 0, lat0 - dlat / 2., 0, dlat ] band = [] offset, scale = vmin, (vmax - vmin) / 254. mask = ucur.mask | vcur.mask curvel = np.sqrt(ucur.data**2 + vcur.data**2) curdir = np.mod( np.arctan2(vcur.data, ucur.data) * 180. / np.pi + 360., 360.) np.clip(curvel, vmin, vmax, out=curvel) array = np.round((curvel - offset) / scale).astype('uint8') array[mask] = 255 colortable = stfmt.format_colortable('matplotlib_jet', vmin=vmin, vmax=vmax, vmin_pal=vmin_pal, vmax_pal=vmax_pal) band.append({ 'array': array, 'scale': scale, 'offset': offset, 'description': L4_MAPS[l4id]['productname'], 'unittype': 'm/s', 'nodatavalue': 255, 'parameter_range': [vmin, vmax], 'colortable': colortable }) array = np.round(curdir / 360. * 254.).astype('uint8') array[mask] = 255 band.append({ 'array': array, 'scale': 360. / 254., 'offset': 0., 'description': 'current direction', 'unittype': 'deg', 'nodatavalue': 255, 'parameter_range': [0, 360.] }) # Write geotiff print('Write geotiff') tifffile = stfmt.format_tifffilename(outdir, metadata, create_dir=True) stfmt.write_geotiff(tifffile, metadata, geolocation, band)
import os import sys import numpy import netCDF4 from cerbere.mapper.ncfile import NCFile from matplotlib import pyplot as plt if __name__ == '__main__': filou = "/home/cerdata/provider/neodc/l4/esacci_sst/2010/03/02/20100302120000-ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_LT-v02.0-fv01.0.nc" print filou nc = NCFile(filou) filout = '/tmp/ostia_sst.png' filout2 = '/tmp/ostia_sst2.png' sst = nc.read_values('analysed_sst') sst = numpy.squeeze(sst) print sst.shape nc.close() nc = netCDF4.Dataset(filou) sst2 = nc.variables['analysed_sst'][:] sst2 = numpy.squeeze(sst2) print sst2.shape nc.close() #plot plt.figure() plt.pcolor(sst[200:800, 100:600]) plt.colorbar() plt.savefig(filout) #plot plt.figure() plt.pcolor(sst2[200:800, 100:600])
def fsle_gridded(infile, outdir, vmin=-1., vmax=0., vmin_pal=-1., vmax_pal=0., write_netcdf=False): """ """ # Read/Process data print('Read/Process data') ncfile = NCFile(infile) if 'id' in ncfile.read_global_attributes(): l4id = ncfile.read_global_attribute('id') elif (re.match(r'^dt_global_allsat_madt_fsle.*\.nc', os.path.basename(infile)) is not None): l4id = 'FSLE' else: raise Exception('Unknown file.') h = ncfile.read_values(L4_MAPS[l4id]['hname'])[0, ::-1, :] lon = ncfile.read_values('lon')[:].astype('float64') lat = ncfile.read_values('lat')[-1:-3:-1].astype('float64') lon[lon > 180.] = lon[lon > 180.] - 360. indsorted = np.argsort(lon) lon = lon[indsorted] h = h[:, indsorted] for i in range(2): # avoid rounding errors lon[i] = np.round(lon[i] * 10000) / 10000 lat[i] = np.round(lat[i] * 10000) / 10000 lon0, dlon, lat0, dlat = lon[0], lon[1] - lon[0], lat[0], lat[1] - lat[0] dlon = 0.04 dlat = -0.04 dtime_units = ncfile.read_field('time').units dtime = num2date(ncfile.read_values('time')[0], dtime_units) # Construct metadata/geolocation/band(s) print('Construct metadata/geolocation/band(s)') metadata = {} metadata['product_name'] = L4_MAPS[l4id]['productname'] metadata['name'] = os.path.splitext(os.path.basename(infile))[0] metadata['datetime'] = stfmt.format_time(dtime) metadata['time_range'] = L4_MAPS[l4id]['timerange'] metadata['source_URI'] = infile metadata['source_provider'] = '*****@*****.**' metadata['processing_center'] = '' metadata['conversion_software'] = 'Syntool' metadata['conversion_version'] = '0.0.0' metadata['conversion_datetime'] = stfmt.format_time(datetime.utcnow()) metadata['parameter'] = [L4_MAPS[l4id]['parameter']] # metadata['type'] = 'model' # metadata['model_longitude_resolution'] = abs(dlon) # metadata['model_latitude_resolution'] = abs(dlat) # metadata['model_analysis_datetime'] = stfmt.format_time(rundtime) geolocation = {} geolocation['projection'] = stfmt.format_gdalprojection() geolocation['geotransform'] = [ lon0 - dlon / 2., dlon, 0, lat0 - dlat / 2., 0, dlat ] band = [] offset, scale = vmin, (vmax - vmin) / 254. np.clip(h, vmin, vmax, out=h) array = np.round((h - offset) / scale).astype('uint8') array[h.mask] = 255 colortable = stfmt.format_colortable('matplotlib_jet_r', vmin=vmin, vmax=vmax, vmin_pal=vmin_pal, vmax_pal=vmax_pal) band.append({ 'array': array, 'scale': scale, 'offset': offset, 'description': 'FSLE', 'unittype': 'day', 'nodatavalue': 255, 'parameter_range': [vmin, vmax], 'colortable': colortable }) # Write geotiff print('Write geotiff') tifffile = stfmt.format_tifffilename(outdir, metadata, create_dir=True) stfmt.write_geotiff(tifffile, metadata, geolocation, band)
def sea_level_gridded(infile, outdir, vmin=-1., vmax=1.0, vmin_pal=-1., vmax_pal=1., write_netcdf=False): """ """ # Read/Process data print('Read/Process data') ncfile = NCFile(infile) if 'id' in ncfile.read_global_attributes(): l4id = ncfile.read_global_attribute('id') elif (re.match(r'^MSL_Map_MERGED_Global_IB_RWT_NoGIA.*\.nc', os.path.basename(infile)) is not None): l4id = 'Mean_Sea_Level' # vmin = -10; vmax = 10; vmin_pal = -10.; vmax_pal = 10 elif (re.match(r'^dt_global_allsat_madt_h.*\.nc', os.path.basename(infile)) is not None): l4id = 'Surface_height' # vmin = -2; vmax = 2; vmin_pal = -2; vmax_pal = 2 elif (re.match(r'^dt_global_allsat_msla_h.*\.nc', os.path.basename(infile)) is not None): l4id = 'Sea_Level_Anomaly' # vmin = -0.2; vmax = 0.2; vmin_pal = -0.2; vmax_pal = 0.2 elif re.match(r'^mdt.*\.nc', os.path.basename(infile)) is not None: l4id = 'Mean_Dynamic_Topo' # vmin = -1.5; vmax = 1.5; vmin_pal = -1.5; vmax_pal = 1.5 elif re.match(r'^mss.*\.nc', os.path.basename(infile)) is not None: l4id = 'Mean_Sea_Surface' # vmin = -80.; vmax = 80.; vmin_pal = -80.; vmax_pal = 80. elif (re.match(r'^nrt_merged_mswh.*\.nc', os.path.basename(infile)) is not None): l4id = 'Sea_Wave_Height' # vmin = 0.; vmax = 6.0; vmin_pal = 0.; vmax_pal = 6.0 elif (re.match(r'^nrt_merged_mwind.*\.nc', os.path.basename(infile)) is not None): l4id = 'Wind' # vmin = 0.; vmax = 20.0; vmin_pal = 0.; vmax_pal = 20.0 elif re.match(r'^Tide.*\.nc', os.path.basename(infile)) is not None: l4id = 'Tide' # vmin = -1.5; vmax = 1.5; vmin_pal = -1.5; vmax_pal = 1.50 else: raise Exception('Unknown file.') # /TMP if l4id in [ 'Mean_Sea_Level', ]: h = ncfile.read_values(L4_MAPS[l4id]['hname'])[::-1, :] lon = ncfile.read_values('longitude')[:].astype('float64') lat = ncfile.read_values('latitude')[-1:-3:-1].astype('float64') elif l4id in ['Mean_Sea_Surface', 'Sea_Wave_Height', 'Wind']: h = ncfile.read_values(L4_MAPS[l4id]['hname'])[:, ::-1] h = np.transpose(h) lon = ncfile.read_values('NbLongitudes')[:].astype('float64') lat = ncfile.read_values('NbLatitudes')[-1:-3:-1].astype('float64') else: h = ncfile.read_values(L4_MAPS[l4id]['hname'])[0, ::-1, :] lon = ncfile.read_values('lon')[:].astype('float64') lat = ncfile.read_values('lat')[-1:-3:-1].astype('float64') lon[lon > 180.] = lon[lon > 180.] - 360. indsorted = np.argsort(lon) lon = lon[indsorted] h = h[:, indsorted] lon0, dlon, lat0, dlat = lon[0], lon[1] - lon[0], lat[0], lat[1] - lat[0] if l4id in ['Mean_Sea_Level', 'Mean_Dynamic_Topo', 'Mean_Sea_Surface']: dtime = datetime(2014, 12, 1) elif l4id in ['Sea_Wave_Height', 'Wind']: dtime = datetime(int(infile[-20:-16]), int(infile[-16:-14]), int(infile[-14:-12])) else: dtime_units = ncfile.read_field('time').units dtime = num2date(ncfile.read_values('time')[0], dtime_units) # rundtime = ncfile.read_global_attribute('date_modified') # rundtime = datetime.strptime(rundtime, '%Y%m%dT%H%M%SZ') # Construct metadata/geolocation/band(s) print('Construct metadata/geolocation/band(s)') metadata = {} metadata['product_name'] = L4_MAPS[l4id]['productname'] metadata['name'] = os.path.splitext(os.path.basename(infile))[0] metadata['datetime'] = stfmt.format_time(dtime) metadata['time_range'] = L4_MAPS[l4id]['timerange'] metadata['source_URI'] = infile metadata['source_provider'] = '*****@*****.**' metadata['processing_center'] = '' metadata['conversion_software'] = 'Syntool' metadata['conversion_version'] = '0.0.0' metadata['conversion_datetime'] = stfmt.format_time(datetime.utcnow()) metadata['parameter'] = [L4_MAPS[l4id]['parameter']] # metadata['type'] = 'model' # metadata['model_longitude_resolution'] = abs(dlon) # metadata['model_latitude_resolution'] = abs(dlat) # metadata['model_analysis_datetime'] = stfmt.format_time(rundtime) geolocation = {} geolocation['projection'] = stfmt.format_gdalprojection() if l4id in ['Mean_Sea_Surface', 'Sea_Wave_Height', 'Wind']: geolocation['geotransform'] = [ lon0 - dlon, dlon, 0, lat0 - dlat, 0, dlat ] elif l4id in [ 'Tide', ]: h = h[:-1, ::] geolocation['geotransform'] = [lon0, dlon, 0, lat0, 0, dlat] else: geolocation['geotransform'] = [ lon0 - dlon / 2., dlon, 0, lat0 - dlat / 2., 0, dlat ] band = [] offset, scale = vmin, (vmax - vmin) / 254. np.clip(h, vmin, vmax, out=h) array = np.round((h - offset) / scale).astype('uint8') array[h.mask] = 255 colortable = stfmt.format_colortable('matplotlib_jet', vmin=vmin, vmax=vmax, vmin_pal=vmin_pal, vmax_pal=vmax_pal) band.append({ 'array': array, 'scale': scale, 'offset': offset, 'description': L4_MAPS[l4id]['hname'], 'unittype': 'm', 'nodatavalue': 255, 'parameter_range': [vmin, vmax], 'colortable': colortable }) # Write geotiff print('Write geotiff') tifffile = stfmt.format_tifffilename(outdir, metadata, create_dir=True) stfmt.write_geotiff(tifffile, metadata, geolocation, band)
def sar_doppler(infile, outdir): """ """ # tmp #infile = '/local/home/fab/data/sar/ASA/agulhas/ASA_WSM_1PNPDE20110518_210602_000002143102_00330_48189_1274/SAR_doppler.nc' # infile = '/local/home/fab/data/sar/ASA/agulhas/ASA_WSM_1PNPDE20110824_211403_000002143106_00014_49597_2093/SAR_doppler.nc' # outdir = '/local/home/data/syntool_inputs' # /tmp # Read/Process data print 'Read/Process data' sardop = NCFile(infile) product_ref = sardop.read_global_attribute('SOURCE_PRODUCT_REF') start_time = sardop.read_global_attribute('SOURCE_START_DATE') start_time = datetime.strptime(start_time, '%Y%m%d%H%M%S.%f') duration = sardop.read_global_attribute('SOURCE_ACQ_DURATION') stop_time = start_time + timedelta(seconds=duration) polarisation = sardop.read_global_attribute('SOURCE_POLARIZATION') lon = sardop.read_values('longitude')[::-1, :] lat = sardop.read_values('latitude')[::-1, :] #dopano = sardop.read_values('dopanomaly')[::-1, :] radvel = sardop.read_values('radial_vel')[::-1, :] validity = sardop.read_values('validity')[::-1, :] track_angle = sardop.read_global_attribute('SOURCE_TRACK_ANGLE') if track_angle < 0: radvel *= -1 shp = lon.shape nlines = np.ceil(shp[0] / 4.) + 1 lines = np.round(np.linspace(0, shp[0] - 1, num=nlines)).astype('int32') npixels = np.ceil(shp[1] / 4) + 1 pixels = np.round(np.linspace(0, shp[1] - 1, num=npixels)).astype('int32') gcplin = np.tile(lines.reshape(nlines, 1), (1, npixels)) gcppix = np.tile(pixels.reshape(1, npixels), (nlines, 1)) gcplon = lon[gcplin, gcppix] gcplat = lat[gcplin, gcppix] gcphei = np.zeros((nlines, npixels)) gcppix = gcppix + 0.5 gcplin = gcplin + 0.5 # Construct metadata/geolocation/band(s) print 'Construct metadata/geolocation/band(s)' metadata = {} (dtime, time_range) = stfmt.format_time_and_range(start_time, stop_time, units='ms') metadata['product_name'] = 'SAR_doppler' metadata['name'] = product_ref metadata['datetime'] = dtime metadata['time_range'] = time_range metadata['source_URI'] = infile metadata['source_provider'] = 'ESA' metadata['processing_center'] = 'CLS' metadata['conversion_software'] = 'Syntool' metadata['conversion_version'] = '0.0.0' metadata['conversion_datetime'] = stfmt.format_time(datetime.utcnow()) metadata['parameter'] = 'radial horizontal velocities' metadata['type'] = 'remote sensing' metadata['sensor_type'] = 'SAR' metadata['sensor_name'] = 'ASAR' metadata['sensor_platform'] = 'ENVISAT' metadata['sensor_mode'] = 'WSM' #metadata['sensor_swath'] = sensor_swath metadata['sensor_polarisation'] = polarisation #metadata['sensor_pass'] = sensor_pass geolocation = {} geolocation['projection'] = stfmt.format_gdalprojection(geogcs='WGS84') geolocation['gcps'] = stfmt.format_gdalgcps(gcplon, gcplat, gcphei, gcppix, gcplin) # band = [] # scale = (vmax-vmin)/254. # offset = vmin # indzero = np.where(validity == 0) # array = np.clip(np.round((radvel-offset)/scale), 0, 254).astype('uint8') # array[indzero] = 255 # band.append({'array':array, 'scale':scale, 'offset':offset, # 'description':'radial horizontal velocities', 'unittype':'m/s', # 'nodatavalue':255, 'parameter_range':[vmin, vmax]}) band = [] cmap = doppler_colormap() norm = Normalize(vmin=-2.5, vmax=2.5) rgb = cmap(norm(radvel)) indnodata = np.where(validity == 0) for ich in range(3): channel = np.round(rgb[:, :, ich] * 255).astype('uint8') channel[indnodata] = 0 band.append({'array': channel, 'nodatavalue': 0}) # Write geotiff print 'Write geotiff' tifffile = stfmt.format_tifffilename(outdir, metadata, create_dir=True) stfmt.write_geotiff(tifffile, metadata, geolocation, band)
elif model == 'Image': rows, cells = geodims r0, r1 = rows / 2 - width, rows / 2 + width c0, c1 = cells / 2 - width, cells / 2 + width print "Subset " print "row : ", r0, r1 print "cell: ", c0, c1 subset = modelobj.extract_subset(slices={'row': slice(r0, r1, 1), 'cell': slice(c0, c1, 1)}) elif model == 'Grid': nj, ni = geodims j0, j1 = nj / 2 - width, nj / 2 + width i0, i1 = ni / 2 - width, ni / 2 + width subset = modelobj.extract_subset(slices={'y': slice(j0, j1, 1), 'x': slice(i0, i1, 1)}) # save subset logging.info("Save subset") subsetfname = 'hrdds21.nc' if os.path.exists(subsetfname): os.remove(subsetfname) oncf = NCFile(url=subsetfname, mode=WRITE_NEW, ncformat='NETCDF4') subset.save(oncf) oncf.close() # read subset logging.info("Read subset") f = NCFile(url=subsetfname) modelobj = modelreader() modelobj.load(f)
class SAFEOLFile(AbstractMapper): """Abstract class for SAFE OLCI files. This mapper concatenates together the files within a SAFE folder that share the same dimensions. url: the path to the product SAFE folder L1B geo_coordinates.nc, instrument_data.nc, *_radiance.nc, time_coordinates.nc L2 LAND geo_coordinates.nc, instrument_data.nc, iwv.nc, lqsf.nc, ogvi.nc, otci.nc, rc_ogvi.nc, time_coordinates.nc L2 WATER geo_coordinates.nc, instrument_data.nc, chl_nn, chl_oc4me.nc, iop_nn.nc, *_reflectance.nc, par.nc, trsp.nc, tsm_nn, w_aer.nc, wqsf.nc, time_coordinates.nc """ def __init__(self, url=None, mode=READ_ONLY, **kwargs): if mode != READ_ONLY: raise Exception("This mapper can only be used in read_only mode.") super(SAFEOLFile, self).__init__(url=url, mode=mode, **kwargs) self.__data_handlers = [] self.__fields = {} # ancillary files cartesian = "geo_coordinates.nc" instrument = "instrument_data.nc" times = "time_coordinates.nc" # handlers for ancillary fields self.__time_handler = NCFile(os.path.join(url, times), mode=mode, **kwargs) self.__fields[times] = [] self.__coord_handler = NCFile(os.path.join(url, cartesian), mode=mode, **kwargs) self.__fields[cartesian] = [] self.__instr_handler = NCFile(os.path.join(url, instrument), mode=mode, **kwargs) self.__fields[instrument] = [] # get product type safefolder = os.path.basename(os.path.normpath(url)) print safefolder if "_OL_1_ERR" in safefolder or "_OL_1_EFR" in safefolder: datafiles = DATAFILES["L1B"] elif "OL_2_LRR" in safefolder or "OL_2_LFR" in safefolder: datafiles = DATAFILES["L2LAND"] elif "OL_2_WRR" in safefolder or "OL_2_WFR" in safefolder: datafiles = DATAFILES["L2WATER"] else: raise Exception("Unknown product type") # detect the data files and instanciate mappers for each one for f in datafiles: if '*' in f: fnames = glob.glob(os.path.join(url, f)) for fname in fnames: self.__data_handlers.append(NCFile(url=fname, mode=mode, **kwargs)) self.__fields[f] = [] else: fname = os.path.join(url, f) self.__data_handlers.append(NCFile(url=fname, mode=mode, **kwargs)) self.__fields[f] = [] self.__fieldlocator = {} self.__geofieldlocator = {} def open(self, view=None, datamodel=None, datamodel_geolocation_dims=None): """ Args: view (dict, optional): a dictionary where keys are dimension names and values are slices. A view can be set on a file, meaning that only the subset defined by this view will be accessible. This view is expressed as any subset (see :func:`get_values`). For example:: view = {'row':slice(200,250), 'cell':slice(200,300)} datamodel (str): type of feature read or written. Internal argument only used by the classes from :mod:`~cerbere.datamodel` package. Can be 'Grid', 'Swath', etc... datamodel_geolocation_dims (list, optional): list of the name of the geolocation dimensions defining the data model to be read in the file. Optional argument, only used by the datamodel classes, in case the mapper class can store different types of data models. Returns: an handler on the opened file """ # open each related file in the SAFE repo if view is None: rowview = None else: rowview = {'row': view['row']} for hdlr in self.__data_handlers: hdlr.open(view, datamodel, datamodel_geolocation_dims) self.__coord_handler.open(view, datamodel, datamodel_geolocation_dims) self.__time_handler.open(rowview, datamodel, datamodel_geolocation_dims) self.__instr_handler.open(view, datamodel, datamodel_geolocation_dims) # build the two-way dictionaries of fields # ...for data for hdlr in self.__data_handlers: self.__fields[os.path.basename(hdlr.get_url())]\ = hdlr.get_fieldnames() for fieldname in hdlr.get_fieldnames(): self.__fieldlocator[fieldname] = hdlr def close(self): """Close handler on storage""" for hdlr in self.__data_handlers: hdlr.close() self.__data_handlers = None self.__coord_handler.close() self.__time_handler.close() self.__instr_handler.close() self.__coord_handler = None self.__time_handler = None self.__instr_handler = None def get_dimsize(self, dimname): """Return the size of a dimension. Args: dimname (str): name of the dimension. Returns: int: size of the dimension. """ dim = self.get_matching_dimname(dimname) return self.__coord_handler.get_dimsize(dim) def get_dimensions(self, fieldname=None): """Return the dimension names of a file or a field in the file. For temporal and spatial dimensions, the cerbere standard names are returned. Args: fieldname (str): the name of the field from which to get the dimensions. For a geolocation field, use the cerbere standard name (time, lat, lon), though native field name will work too. Returns: tuple<str>: the standard dimensions of the field or file. """ if fieldname is None: return self.__coord_handler.get_dimensions() if fieldname in ['time', 'lat', 'lon']: # Should all have the same dimension as lat native_fieldname = self.get_geolocation_field('lat') dims = self.__coord_handler.get_dimensions(native_fieldname) else: handler = self.__fieldlocator[fieldname] dims = handler.get_dimensions(fieldname) # convert geolocation dims to standard names newdims = [] for dim in list(dims): newdims.append(self.get_standard_dimname(dim)) return tuple(newdims) def get_matching_dimname(self, dimname): """Return the equivalent name in the native format for a standard dimension. This is a translation of the standard names to native ones. It is used for internal purpose only and should not be called directly. The standard dimension names are: * x, y, time for :class:`~cerbere.datamodel.grid.Grid` * row, cell, time for :class:`~cerbere.datamodel.swath.Swath` or :class:`~cerbere.datamodel.image.Image` To be derived when creating an inherited data mapper class. This is mandatory for geolocation dimensions which must be standard. Args: dimname (str): standard dimension name. Returns: str: return the native name for the dimension. Return `dimname` if the input dimension has no standard name. See Also: see :func:`get_standard_dimname` for the reverse operation """ matching = {'time': 'time', 'row': 'rows', 'cell': 'columns'} if dimname in matching: return matching[dimname] return dimname def get_standard_dimname(self, dimname): """ Returns the equivalent standard dimension name for a dimension in the native format. This is a translation of the native names to standard ones. It is used for internal purpose and should not be called directly. To be derived when creating an inherited data mapper class. This is mandatory for geolocation dimensions which must be standard. Args: dimname (string): native dimension name Return: str: the (translated) standard name for the dimension. Return `dimname` if the input dimension has no standard name. See Also: see :func:`get_matching_dimname` for the reverse operation """ matching = {'time': 'time', 'rows': 'row', 'columns': 'cell'} if dimname in matching: return matching[dimname] return dimname def get_fieldnames(self): """Returns the list of geophysical fields stored for the feature. The geolocation field names are excluded from this list. Returns: list<string>: list of field names """ return self.__fieldlocator.keys() def __get_native_fieldname(self, fieldname): """Returns the native name of a field. Args: fieldname (str): name of the field. Returns: str: the native name of the field. The same as input if the field is not a geolocation field. """ if fieldname in ['lat', 'lon', 'time', 'z']: return self.get_geolocation_field(fieldname) return fieldname def get_geolocation_field(self, fieldname): """Return the equivalent field name in the file format for a standard geolocation field (lat, lon, time, z). Used for internal purpose and should not be called directly. Args: fieldname (str): name of the standard geolocation field (lat, lon or time) Return: str: name of the corresponding field in the native file format. Returns None if no matching is found """ MATCHES = {'lat': 'latitude', 'lon': 'longitude', 'time': 'time'} if fieldname in MATCHES: return MATCHES[fieldname] return None def read_field(self, fieldname): """ Return the :class:`cerbere.field.Field` object corresponding to the requested fieldname. The :class:`cerbere.field.Field` class contains all the metadata describing a field (equivalent to a variable in netCDF). Args: fieldname (str): name of the field Returns: :class:`cerbere.field.Field`: the corresponding field object """ if fieldname == 'time': rows = self.get_dimsize('row') cols = self.get_dimsize('cell') variable = Variable( shortname='time', description='time of measurement', authority=None, standardname=None ) field = Field( variable, OrderedDict([('row', rows), ('cell', cols)]), datatype=dtype(int64) ) field.attach_storage(self.get_field_handler(fieldname)) field.units = self.__time_handler.get_handler().\ variables['time_stamp'].units return field elif fieldname in ['lat', 'lon']: native_name = self.get_geolocation_field(fieldname) geofield = self.__coord_handler.read_field( native_name ) geofield.name = fieldname geofield.attach_storage(self.get_field_handler(fieldname)) return geofield else: native_name = self.__get_native_fieldname(fieldname) return self.__fieldlocator[native_name].read_field(native_name) def read_values(self, fieldname, slices=None): """Read the data of a field. Args: fieldname (str): name of the field which to read the data from slices (list of slice, optional): list of slices for the field if subsetting is requested. A slice must then be provided for each field dimension. The slices are relative to the opened view (see :func:open) if a view was set when opening the file. Return: MaskedArray: array of data read. Array type is the same as the storage type. """ native_name = self.__get_native_fieldname(fieldname) if fieldname == 'time': if slices is not None: tslices = [slices[0]] else: tslices = slices time = self.__time_handler.read_values('time_stamp', slices=tslices) # reshape as a 2D field rows = self.get_dimsize('row') cols = self.get_dimsize('cell') if slices is None: shape = (cols, rows) else: newslices = self._fill_slices(slices, (rows, cols)) shape = (newslices[1].stop - newslices[1].start, newslices[0].stop - newslices[0].start) time = ma.resize(time, shape).transpose() return time elif fieldname in ['lat', 'lon']: return self.__coord_handler.read_values(native_name, slices) else: return self.__fieldlocator[native_name].read_values(native_name, slices) def read_fillvalue(self, fieldname): """Read the fill value of a field. Args: fieldname (str): name of the field. Returns: number or char or str: fill value of the field. The type is the as the type of the data in the field. """ return self.__fieldlocator[fieldname].read_fillvalue(fieldname) def read_global_attributes(self): """Returns the names of the global attributes. Returns: list<str>: the list of the attribute names. """ # all files seem to have the same list og global attributes. return self.__coord_handler.read_global_attributes() def read_global_attribute(self, name): """Returns the value of a global attribute. Args: name (str): name of the global attribute. Returns: str, number or datetime: value of the corresponding attribute. """ # all files seem to have the same list or global attributes. return self.__coord_handler.read_global_attribute(name) def read_field_attributes(self, fieldname): """Return the specific attributes of a field. Args: fieldname (str): name of the field. Returns: dict<string, string or number or datetime>: a dictionary where keys are the attribute names. """ return self.__fieldlocator[fieldname].read_field_attributes(fieldname) def get_start_time(self): """Returns the minimum date of the file temporal coverage. Returns: datetime: start time of the data in file. """ varname = 'time_stamp' vardate = self.__time_handler.get_handler().variables[varname] return num2date(vardate[0], vardate.units) def get_end_time(self): """Returns the maximum date of the file temporal coverage. Returns: datetime: end time of the data in file. """ # WRONG!!! varname = 'time_stamp' vardate = self.__time_handler.get_handler().variables[varname] return num2date(vardate[-1], vardate.units) def get_bbox(self): """Returns the bounding box of the feature, as a tuple. Returns: tuple: bbox expressed as (lonmin, latmin, lonmax, latmax) """ return None def write_global_attributes(self, attrs): """Write the global attributes of the file. Args: attrs (dict<string, string or number or datetime>): a dictionary containing the attributes names and values to be written. """ raise NotImplementedError def create_field(self, field, dim_translation=None): """Creates a new field in the mapper. Creates the field structure but don't write yet its values array. Args: field (Field): the field to be created. See also: :func:`write_field` for writing the values array. """ raise NotImplementedError def create_dim(self, dimname, size=None): """Add a new dimension. Args: dimname (str): name of the dimension. size (int): size of the dimension (unlimited if None) """ raise NotImplementedError def write_field(self, fieldname): """Writes the field data on disk. Args: fieldname (str): name of the field to write. """ raise NotImplementedError
class SAFESLFile(AbstractMapper): """Abstract class for SAFE SLSTR files (except L2P). """ def __init__(self, url=None, sltype=None, mode=READ_ONLY, **kwargs): if mode != READ_ONLY: raise Exception("This mapper can only be used in read_only mode.") if sltype not in ['i', 'c', 'a', 'b']: raise Exception("Unknown SLSTR product type") super(SAFESLFile, self).__init__(url=url, mode=mode, **kwargs) self.__sltype = sltype self.__data_handlers = [] self.__oblique_fields = [] # coordinate files geodetic = "geodetic_%sn.nc" % sltype times = "time_%sn.nc" % sltype coordinate_files = [geodetic, times] # detect the data files and instanciate mappers for each one datafiles = [] for fname in glob.glob(os.path.join(url, "*_%s[n,o].nc" % sltype)): if os.path.basename(fname) not in coordinate_files: datafiles.append(os.path.basename(fname)) for f in datafiles: fname = os.path.join(url, f) self.__data_handlers.append(NCFile(url=fname, mode=mode, **kwargs)) # instantiate mappers for each coordinate file self.__geod_handler = NCFile(os.path.join(url, geodetic), mode=mode, **kwargs) self.__time_handler = NCFile(os.path.join(url, times), mode=mode, **kwargs) self.__fieldlocator = {} self.__geofieldlocator = {} self.__fieldtranslate = {} # offset between nadir and oblique swath edge self.nadir_to_oblique_offset = None def __is_oblique(self, fieldname): """Test if a field corresponds to an oblique view subproduct. Returns: bool: True f a field corresponds to an oblique view """ return (fieldname in self.__oblique_fields) def open(self, view=None, datamodel=None, datamodel_geolocation_dims=None): """ Args: view (dict, optional): a dictionary where keys are dimension names and values are slices. A view can be set on a file, meaning that only the subset defined by this view will be accessible. This view is expressed as any subset (see :func:`get_values`). For example:: view = {'row':slice(200,250), 'cell':slice(200,300)} datamodel (str): type of feature read or written. Internal argument only used by the classes from :mod:`~cerbere.datamodel` package. Can be 'Grid', 'Swath', etc... datamodel_geolocation_dims (list, optional): list of the name of the geolocation dimensions defining the data model to be read in the file. Optional argument, only used by the datamodel classes, in case the mapper class can store different types of data models. Returns: an handler on the opened file """ # open each related file in the SAFE repo if view is None: rowview = None else: rowview = {'row': view['row']} # modify view for oblique fields which are narrower if 'cell' in view: obliqueview = view obliqueview['cell'] = (view['cell'] - self.nadir_to_oblique_offset) for hdlr in self.__data_handlers: f = os.path.basename(hdlr.get_url()) is_oblique = (f[-4] == 'o') newview = view if is_oblique and view is not None: newview = obliqueview hdlr.open(newview, datamodel, datamodel_geolocation_dims) self.__geod_handler.open(view, datamodel, datamodel_geolocation_dims) self.__time_handler.open(rowview, datamodel, datamodel_geolocation_dims) # build the two-way dictionaries of fields # ...for data for hdlr in self.__data_handlers: f = os.path.basename(hdlr.get_url()) is_oblique = (f[-4] == 'o') for fieldname in hdlr.get_fieldnames(): ncvar = hdlr.get_handler().variables[fieldname] if 'long_name' in (ncvar.ncattrs()): longname = ncvar.long_name newfieldname = self.__get_fieldname(fieldname, longname) else: newfieldname = fieldname self.__fieldtranslate[newfieldname] = fieldname self.__fieldlocator[newfieldname] = hdlr if is_oblique: self.__oblique_fields.append(newfieldname) # ...for geodetic coordinates for fieldname in self.__geod_handler.get_fieldnames(): self.__geofieldlocator[fieldname] = self.__geod_handler # define nadir/oblique offset nadir_track_offset = (self.__geofieldlocator[ 'latitude_orphan_%sn' % self.__sltype].read_global_attribute('track_offset')) oblique_track_offset = (self.__fieldlocator[ 'latitude_orphan_%so' % self.__sltype].read_global_attribute('track_offset')) self.nadir_to_oblique_offset = int( round(nadir_track_offset - oblique_track_offset)) def close(self): """Close handler on storage""" for hdlr in self.__data_handlers: hdlr.close() self.__data_handlers = None self.__geod_handler.close() self.__time_handler.close() self.__geod_handler = None self.__time_handler = None def get_dimsize(self, dimname): """Return the size of a dimension. Args: dimname (str): name of the dimension. Returns: int: size of the dimension. """ dim = self.get_matching_dimname(dimname) return self.__geod_handler.get_dimsize(dim) def get_dimensions(self, fieldname=None): """Return the dimension names of a file or a field in the file. For temporal and spatial dimensions, the cerbere standard names are returned. Args: fieldname (str): the name of the field from which to get the dimensions. For a geolocation field, use the cerbere standard name (time, lat, lon), though native field name will work too. Returns: tuple<str>: the standard dimensions of the field or file. """ if fieldname is None: dims = self.__geod_handler.get_dimensions() if 'orphan_pixels' in dims: # same dimension name (but not size) in oblique/nadir views so # we have to create two dimensions since we merge the two views newdims = [] for dim in dims: if dim != 'orphan_pixels': newdims.append(dim) else: newdims.extend([dim + '_n', dim + '_o']) return newdims return dims if fieldname in ['time', 'lat', 'lon', 'z']: # Should all have the same dimension as lat native_fieldname = self.get_geolocation_field('lat') dims = self.__geod_handler.get_dimensions(native_fieldname) else: handler = self.__fieldlocator[fieldname] dims = handler.get_dimensions( self.__get_native_fieldname(fieldname)) # convert geolocation dims to standard names newdims = [] for dim in list(dims): if self.__is_oblique(fieldname): view = 'o' else: view = 'n' newdims.append(self.get_standard_dimname(dim, view)) return tuple(newdims) def get_matching_dimname(self, dimname): """Return the equivalent name in the native format for a standard dimension. This is a translation of the standard names to native ones. It is used for internal purpose only and should not be called directly. The standard dimension names are: * x, y, time for :class:`~cerbere.datamodel.grid.Grid` * row, cell, time for :class:`~cerbere.datamodel.swath.Swath` or :class:`~cerbere.datamodel.image.Image` To be derived when creating an inherited data mapper class. This is mandatory for geolocation dimensions which must be standard. Args: dimname (str): standard dimension name. Returns: str: return the native name for the dimension. Return `dimname` if the input dimension has no standard name. See Also: see :func:`get_standard_dimname` for the reverse operation """ matching = { 'time': 'time', 'row': 'rows', 'cell': 'columns', 'z': 'elevation' } # remove the oblique/nadir suffix if dimname[-2:] in ['_n', '_o']: dimname = dimname.strip('_n').strip('_o') if dimname in matching: return matching[dimname] return dimname def get_standard_dimname(self, dimname, view=None): """ Returns the equivalent standard dimension name for a dimension in the native format. This is a translation of the native names to standard ones. It is used for internal purpose and should not be called directly. To be derived when creating an inherited data mapper class. This is mandatory for geolocation dimensions which must be standard. Args: dimname (string): native dimension name Return: str: the (translated) standard name for the dimension. Return `dimname` if the input dimension has no standard name. See Also: see :func:`get_matching_dimname` for the reverse operation """ matching = { 'time': 'time', 'rows': 'row', 'columns': 'cell', 'elevation': 'z' } if dimname in matching: return matching[dimname] # for other dimensions, add the oblique/nadir suffix if dimname in ['row', 'cell', 'time', 'z']: return dimname if view is 'n': dimname += '_n' elif view is 'o': dimname += '_o' return dimname def get_fieldnames(self): """Returns the list of geophysical fields stored for the feature. The geolocation field names are excluded from this list. Returns: list<string>: list of field names """ return self.__fieldlocator.keys() def __get_native_fieldname(self, fieldname): """Returns the native name of a field. Args: fieldname (str): name of the field. Returns: str: the native name of the field. The same as input if the field is not a geolocation field. """ if fieldname in ['lat', 'lon', 'time', 'z']: return self.get_geolocation_field(fieldname) if fieldname in self.__fieldtranslate: return self.__fieldtranslate[fieldname] return fieldname def __get_fieldname(self, fieldname, longname): """Returns a unique field name built from the long name for ambiguous field names. Used because some sub files use the same variable names. Args: fieldname (str): field name to replace with a new name, if not unique. longname (str): longname from which to build a new unique field name Returns: str: a unique field name among all the files in a SAFE container. """ if fieldname in ["SST", "SST_uncertainty", "exception"]: return longname.replace(' ', '_') return fieldname def get_geolocation_field(self, fieldname): """Return the equivalent field name in the file format for a standard geolocation field (lat, lon, time, z). Used for internal purpose and should not be called directly. Args: fieldname (str): name of the standard geolocation field (lat, lon or time) Return: str: name of the corresponding field in the native file format. Returns None if no matching is found """ if fieldname == 'time': return 'time' matching = { 'lat': 'latitude', 'lon': 'longitude', 'time': 'time', 'z': 'elevation' }[fieldname] native_fieldname = matching + '_%sn' % self.__sltype return native_fieldname def read_field(self, fieldname): """ Return the :class:`cerbere.field.Field` object corresponding to the requested fieldname. The :class:`cerbere.field.Field` class contains all the metadata describing a field (equivalent to a variable in netCDF). Args: fieldname (str): name of the field Returns: :class:`cerbere.field.Field`: the corresponding field object """ if fieldname == 'time': rows = self.get_dimsize('row') cols = self.get_dimsize('cell') variable = Variable(shortname='time', description='time of measurement', authority=None, standardname=None) field = Field(variable, OrderedDict([('row', rows), ('cell', cols)]), datatype=dtype(int64)) field.attach_storage(self.get_field_handler(fieldname)) field.units = self.__time_handler.get_handler().\ variables['time_stamp_%s' % self.__sltype[0]].units return field elif fieldname in ['lat', 'lon', 'z']: native_name = self.get_geolocation_field(fieldname) geofield = self.__geofieldlocator[native_name].read_field( native_name) geofield.name = fieldname return geofield else: native_name = self.__get_native_fieldname(fieldname) field = self.__fieldlocator[fieldname].read_field(native_name) field.name = fieldname field.attach_storage(self.get_field_handler(fieldname)) dims = field.dimensions renamed_dims = OrderedDict() for dim in dims: newdim = dim if dim not in ['row', 'cell', 'time', 'z']: if self.__is_oblique(fieldname): newdim += '_o' else: newdim += '_n' renamed_dims[newdim] = dims[dim] field.dimensions = renamed_dims # for oblique view, the swath is narrower. It is padded with # dummy values to stack it over nadir view fields if fieldname in self.__oblique_fields: if 'cell' in field.get_dimnames(): field.dimensions['cell'] = self.get_dimsize('cell') return field def read_values(self, fieldname, slices=None): """Read the data of a field. Args: fieldname (str): name of the field which to read the data from slices (list of slice, optional): list of slices for the field if subsetting is requested. A slice must then be provided for each field dimension. The slices are relative to the opened view (see :func:open) if a view was set when opening the file. Return: MaskedArray: array of data read. Array type is the same as the storage type. """ native_name = self.__get_native_fieldname(fieldname) rowslice = None if slices is not None: rowslice = [slices[0]] if fieldname == 'time': suffix = self.__sltype SCANSYNC = self.__time_handler.read_values('SCANSYNC')[0] PIXSYNC_i = self.__time_handler.read_values('PIXSYNC_%s' % suffix)[0] prefix = PREFIX['n'] first_scan_i\ = self.__time_handler.read_values( '%s_First_scan_%s' % (prefix, suffix), slices=rowslice) first_min_ts\ = self.__time_handler.read_values( '%s_Minimal_ts_%s' % (prefix, suffix), slices=rowslice) scanfield = 'scan_%sn' % suffix pixelfield = 'pixel_%sn' % suffix indices_handler = self.__fieldlocator[scanfield] scan = indices_handler.read_values(scanfield, slices=slices) pixel = indices_handler.read_values(pixelfield, slices=slices) time = first_min_ts.reshape((-1, 1))\ + (scan - first_scan_i.reshape((-1, 1)))\ * SCANSYNC + pixel * PIXSYNC_i # mask wrong times (which occur in test data) maxdate = date2num(self.get_end_time(), "microseconds since 2000-01-01T00:00:00Z") mindate = date2num(self.get_start_time(), "microseconds since 2000-01-01T00:00:00Z") time = ma.masked_where(((time < mindate) | (time > maxdate)), time, copy=False) return time elif fieldname in ['lat', 'lon', 'z']: return self.__geofieldlocator[native_name].read_values( native_name, slices) elif self.__is_oblique(fieldname): # oblique views has not the same cell dimension than nadir # we want to stack fields from both views by padding fillvalues celldim = None try: dims = self.__fieldlocator[fieldname].get_dimensions( native_name) empty = False if 'cell' in dims: dimsizes = [self.get_dimsize(dim) for dim in dims] celldim = list(dims).index('cell') nadir_slices = cerbere.mapper.slices.get_nice_slices( slices, dimsizes) sli = nadir_slices[celldim] nad_start, nad_end, step = sli.start, sli.stop, sli.step obliquewidth = ( self.__fieldlocator[fieldname].get_dimsize('cell')) obl_start, obl_end = nad_start, nad_end obl_start = max(0, obl_start - self.nadir_to_oblique_offset) if obl_start > obliquewidth: obl_start = obliquewidth obl_end = max( 0, min(obliquewidth, obl_end - self.nadir_to_oblique_offset)) newslices = list(nadir_slices) newslices[celldim] = slice(obl_start, obl_end, step) if obl_start >= obl_end: empty = True else: # case of some fields such as orphan pixels which don't # have a cell dimension newslices = list(slices) if slices is not None else None except ValueError: raise # read values in oblique view grid if celldim is not None and empty: values = ma.masked_all((), dtype=self.__fieldlocator[fieldname]. _handler.variables[native_name].dtype) else: values = self.__fieldlocator[fieldname].read_values( self.__fieldtranslate[fieldname], newslices) # padding for missing oblique values to match nadir view grid if celldim is not None: shape = cerbere.mapper.slices.get_shape_from_slice( cerbere.mapper.slices.get_nice_slices(slices, dimsizes)) padded_values = ma.masked_all( tuple(shape), dtype=values.dtype, ) if values.shape == () or min(list(values.shape)) <= 0: # empty result => return padded values only return padded_values padded_slice = [] for dim in dims: if dim != 'cell': padded_slice.append(slice(None, None, None)) else: offset = (self.nadir_to_oblique_offset - nad_start + obl_start) padded_slice.append( slice(offset, offset + values.shape[celldim])) padded_values[padded_slice] = values return padded_values else: return values else: return self.__fieldlocator[fieldname].read_values( native_name, slices) def read_fillvalue(self, fieldname): """Read the fill value of a field. Args: fieldname (str): name of the field. Returns: number or char or str: fill value of the field. The type is the as the type of the data in the field. """ return self.__fieldlocator[fieldname].read_fillvalue(fieldname) def read_global_attributes(self): """Returns the names of the global attributes. Returns: list<str>: the list of the attribute names. """ # all files seem to have the same list og global attributes. return self.__geod_handler.read_global_attributes() def read_global_attribute(self, name): """Returns the value of a global attribute. Args: name (str): name of the global attribute. Returns: str, number or datetime: value of the corresponding attribute. """ # all files seem to have the same list or global attributes. return self.__geod_handler.read_global_attribute(name) def read_field_attributes(self, fieldname): """Return the specific attributes of a field. Args: fieldname (str): name of the field. Returns: dict<string, string or number or datetime>: a dictionary where keys are the attribute names. """ return self.__fieldlocator[fieldname].read_field_attributes(fieldname) def get_start_time(self): """Returns the minimum date of the file temporal coverage. Returns: datetime: start time of the data in file. """ varname = 'time_stamp_%s' % self.__sltype[0] vardate = self.__time_handler.get_handler().variables[varname] return num2date(vardate[0], vardate.units) def get_end_time(self): """Returns the maximum date of the file temporal coverage. Returns: datetime: end time of the data in file. """ # WRONG!!! varname = 'time_stamp_%s' % self.__sltype[0] vardate = self.__time_handler.get_handler().variables[varname] return num2date(vardate[-1], vardate.units) def get_bbox(self): """Returns the bounding box of the feature, as a tuple. Returns: tuple: bbox expressed as (lonmin, latmin, lonmax, latmax) """ return None def write_global_attributes(self, attrs): """Write the global attributes of the file. Args: attrs (dict<string, string or number or datetime>): a dictionary containing the attributes names and values to be written. """ raise NotImplementedError def create_field(self, field, dim_translation=None): """Creates a new field in the mapper. Creates the field structure but don't write yet its values array. Args: field (Field): the field to be created. See also: :func:`write_field` for writing the values array. """ raise NotImplementedError def create_dim(self, dimname, size=None): """Add a new dimension. Args: dimname (str): name of the dimension. size (int): size of the dimension (unlimited if None) """ raise NotImplementedError def write_field(self, fieldname): """Writes the field data on disk. Args: fieldname (str): name of the field to write. """ raise NotImplementedError
def ww3_model_wave(infile, outdir, date=None, max_forecast_hours=None, vmin=0., vmax=25.4, vmin_pal=0., vmax_pal=10., v2=False): """ """ # Read/Process data print 'Read/Process data' ncfile = NCFile(infile) ww3 = {} ww3['time'] = ncfile.read_values('time') ww3['time_units'] = ncfile.read_field('time').units ww3uniqtime = ww3['time'].size == 1 if ww3uniqtime: # assume 3h ww3['deltatime'] = 180 else: t01 = num2date(np.array(ww3['time'][0:2]), ww3['time_units']) ww3['deltatime'] = np.round( (t01[1] - t01[0]).total_seconds() / 60.).astype('int') if date != None: tind = np.where(ww3['time'] == date2num(date, ww3['time_units']))[0] if tind.size == 1: tsl = slice(tind[0], tind[0] + 1) ww3['time'] = ww3['time'][tsl] else: raise Exception('Date not found in WW3 file.') else: tsl = slice(0, ww3['time'].size) if max_forecast_hours is not None: if ww3['time'].size != 1: raise Exception( 'max_forecast_hours option works with only 1 time.') ww3time = num2date(ww3['time'][0], ww3['time_units']) if 'date_cycle' not in ncfile.read_global_attributes(): raise Exception( 'max_forecast_hours option works with date_cycle attribute.') cycletime_str = ncfile.read_global_attribute('date_cycle') if 10 == len(cycletime_str): cycletime_format = '%Y%m%d%H' elif 20 == len(cycletime_str) and cycletime_str.endswith('Z'): cycletime_format = '%Y-%m-%dT%H:%M:%SZ' else: raise Exception( 'Cycletime format is not supported: {}'.format(cycletime_str)) cycletime = datetime.strptime(cycletime_str, cycletime_format) forecast_hours = (ww3time - cycletime).total_seconds() / 3600. if forecast_hours > max_forecast_hours: raise Exception('Exceeds max_forecast_hours.') slices = [ tsl, slice(0, ncfile.get_dimsize('latitude')), slice(0, ncfile.get_dimsize('longitude')) ] # slices = [tsl, # slice(ncfile.get_dimsize('latitude')-1, None, -1), # slice(0, 2*ncfile.get_dimsize('longitude'))] ncfieldnames = ncfile.get_fieldnames() fieldnames = [ 'hs', 'phs0', 'phs1', 'phs2', 'phs3', 'ptp0', 'ptp1', 'ptp2', 'ptp3', 'pdir0', 'pdir1', 'pdir2', 'pdir3' ] ww3['source'] = [infile] for fieldname in fieldnames: if fieldname in ncfieldnames: ww3[fieldname] = ncfile.read_values(fieldname, slices=slices)[:, ::-1, :] else: infile2 = split_ww3_fname(infile, fieldname) if not os.path.exists(infile2): infile2 = other_split_ww3_fname(infile, fieldname) if os.path.exists(infile2): ncfile2 = NCFile(infile2) ww3[fieldname] = ncfile2.read_values(fieldname, slices=slices)[:, ::-1, :] ncfile2.close() ww3['source'].append(infile2) ww3['npart'] = 0 for indp in range(4): pin = [name + str(indp) in ww3 for name in ['phs', 'ptp', 'pdir']] if all(pin): ww3['npart'] += 1 else: break if ww3['npart'] == 0: raise Exception('Could not find all partition variables.') ww3['area'] = ncfile.read_global_attribute('area') if 'global' in ww3['area'].lower(): ww3['lon'] = ncfile.read_values('lon') ww3['lat'] = ncfile.read_values('lat')[::-1] ww3['lon_res'] = float( ncfile.read_global_attribute('longitude_resolution')) ww3['lat_res'] = float( ncfile.read_global_attribute('latitude_resolution')) elif ww3['area'] == 'ARCTIC-12km': ww3['lon'] = ncfile.read_values('lon')[::-1, :] ww3['lat'] = ncfile.read_values('lat')[::-1, :] else: raise Exception('Not implemented : area = "{}"'.format(ww3['area'])) if 'run_time' in ncfile.read_global_attributes(): run_time = ncfile.read_global_attribute('run_time') ww3['rundtime'] = datetime.strptime(run_time, '%Y-%m-%dT%H:%M:%SZ') ncfile.close() # Construct metadata/geolocation/band(s) print 'Construct metadata/geolocation/band(s)' metadata = {} #metadata['product_name'] = #metadata['name'] = #metadata['datetime'] = metadata['time_range'] = [ '-{:d}m'.format(ww3['deltatime'] / 2), '+{:d}m'.format(ww3['deltatime'] / 2) ] metadata['source_URI'] = ww3['source'] metadata['source_provider'] = ['SHOM', 'Ifremer'] metadata['processing_center'] = '' metadata['conversion_software'] = 'Syntool' metadata['conversion_version'] = '0.0.0' metadata['conversion_datetime'] = stfmt.format_time(datetime.utcnow()) #metadata['parameter'] = metadata['type'] = 'model' #metadata['model_longitude_resolution'] = #metadata['model_latitude_resolution'] = if 'rundtime' in ww3: metadata['model_analysis_datetime'] = stfmt.format_time( ww3['rundtime']) geolocation = {} if 'global' in ww3['area'].lower(): metadata['model_longitude_resolution'] = ww3['lon_res'] metadata['model_latitude_resolution'] = ww3['lat_res'] geolocation['projection'] = stfmt.format_gdalprojection() lon0, dlon = ww3['lon'][0], ww3['lon'][1] - ww3['lon'][0] lat0, dlat = ww3['lat'][0], ww3['lat'][1] - ww3['lat'][0] geolocation['geotransform'] = [ lon0 - dlon / 2., dlon, 0, lat0 - dlat / 2., 0, dlat ] ww3['grid'] = 'GLOBAL' elif ww3['area'] == 'ARCTIC-12km': import pyproj srs = osr.SpatialReference() srs.ImportFromEPSG(3411) proj = pyproj.Proj(srs.ExportToProj4()) x01, dummy = proj(ww3['lon'][:, [0, -1]], ww3['lat'][:, [0, -1]]) x0, x1 = x01.mean(axis=0) dx = (x1 - x0) / (ww3['lon'].shape[1] - 1) dummy, y01 = proj(ww3['lon'][[0, -1], :], ww3['lat'][[0, -1], :]) y0, y1 = y01.mean(axis=1) dy = (y1 - y0) / (ww3['lon'].shape[0] - 1) geolocation['projection'] = srs.ExportToWkt() geolocation['geotransform'] = [ x0 - dx / 2., dx, 0, y0 - dy / 2., 0, dy ] # geolocation['geotransform'] = [-2600051.73564, 12500.2285676, 0, # 2787547.79214, 0, -12500.2262608] ww3['grid'] = 'ARCTIC' else: raise Exception('Not implemented : area = "{}"'.format(ww3['area'])) # Loop on time for itime in range(ww3['time'].size): dtime = num2date(ww3['time'][itime], ww3['time_units']) metadata['datetime'] = stfmt.format_time(dtime) if len(ww3['source']) == 1: basename = os.path.splitext(os.path.basename(infile))[0] if not ww3uniqtime: _date = dtime.strftime('%Y%m%d') _datetime = dtime.strftime('%Y%m%dT%H') if _date in basename and _datetime not in basename: basename = basename.replace(_date, dtime.strftime('%Y%m%dT%HZ')) else: raise Exception else: basename = 'WW3-' + ww3['grid'] + '-' + dtime.strftime( '%Y%m%dT%HZ') if 'hindcast' in infile.lower(): basename = 'HINDCAST_' + basename ### Total HS ### # Update metadata metadata['product_name'] = 'WW3_model_wave_hs' if v2 == True: metadata['product_name'] += '_v2' metadata['name'] = basename + '-hs' metadata['parameter'] = 'wave significant height' # Make band band = [] hs = ww3['hs'][itime, :, :] offset, scale = vmin, (vmax - vmin) / 254.0 np.clip(hs, vmin, vmax, out=hs) array = np.round((hs - offset) / scale).astype('uint8') array[hs.mask] = 255 colortable = stfmt.format_colortable('matplotlib_jet', vmin=vmin, vmax=vmax, vmin_pal=vmin_pal, vmax_pal=vmax_pal) band.append({ 'array': array, 'scale': scale, 'offset': offset, 'description': metadata['parameter'], 'unittype': 'm', 'nodatavalue': 255, 'parameter_range': [vmin, vmax], 'colortable': colortable }) # Write geotiff print 'Write geotiff' tifffile = stfmt.format_tifffilename(outdir, metadata, create_dir=True) stfmt.write_geotiff(tifffile, metadata, geolocation, band) ### Partitions ### phslst = [ ww3['phs' + str(i)][itime, :, :] for i in range(ww3['npart']) ] phs = np.ma.dstack(phslst) ptplst = [ ww3['ptp' + str(i)][itime, :, :] for i in range(ww3['npart']) ] ptp = np.ma.dstack(ptplst) pdirlst = [ ww3['pdir' + str(i)][itime, :, :] for i in range(ww3['npart']) ] pdir = np.ma.dstack(pdirlst) # Reorder partitions by HS -> Keep WW3 order # phs.data[phs.mask] = -1000 # make sure masked values don't interfere with sorting # index = np.ogrid[:phs.shape[0], :phs.shape[1], :phs.shape[2]] # index[2] = (-phs).argsort(axis=2, kind='mergesort') # phs = phs[index] # ptp = ptp[index] # pdir = pdir[index] # pdir from_direction -> to_direction pdir = np.mod(pdir + 180.0, 360.0) # pdir clockwise from north -> counter clockwise from east pdir = np.mod(90.0 - pdir, 360.0) # Write each partition in a geotiff for i in range(ww3['npart']): # Update metadata lpartnum = 'partition ' + str(i) spartnum = 'part' + str(i) metadata['product_name'] = 'WW3_model_wave_' + spartnum if v2 == True: metadata['product_name'] += '_v2' metadata['name'] = basename + '-' + spartnum metadata['parameter'] = [ 'wave significant height ' + lpartnum, 'wave peak period ' + lpartnum, 'wave mean direction ' + lpartnum ] # Make bands band = [] iphs, iptp, ipdir = phs[:, :, i], ptp[:, :, i], pdir[:, :, i] # HS #_vmin, _vmax = 0.0, 25.4 offset, scale = vmin, (vmax - vmin) / 254.0 np.clip(iphs, vmin, vmax, out=iphs) array = np.round((iphs - offset) / scale).astype('uint8') array[iphs.mask] = 255 band.append({ 'array': array, 'scale': scale, 'offset': offset, 'description': metadata['parameter'][0], 'unittype': 'm', 'nodatavalue': 255, 'parameter_range': [vmin, vmax] }) # Period _vmin, _vmax = 0.0, 25.4 offset, scale = _vmin, (_vmax - _vmin) / 254.0 np.clip(iptp, _vmin, _vmax, out=iptp) array = np.round((iptp - offset) / scale).astype('uint8') array[iptp.mask] = 255 band.append({ 'array': array, 'scale': scale, 'offset': offset, 'description': metadata['parameter'][1], 'unittype': 's', 'nodatavalue': 255, 'parameter_range': [_vmin, _vmax] }) # Direction _vmin, _vmax = 0.0, 360.0 offset, scale = _vmin, (_vmax - _vmin) / 254.0 np.clip(ipdir, _vmin, _vmax, out=ipdir) array = np.round((ipdir - offset) / scale).astype('uint8') array[ipdir.mask] = 255 band.append({ 'array': array, 'scale': scale, 'offset': offset, 'description': metadata['parameter'][2], 'unittype': 'degree', 'nodatavalue': 255, 'parameter_range': [vmin, vmax] }) # Write geotiff print 'Write geotiff' tifffile = stfmt.format_tifffilename(outdir, metadata, create_dir=True) stfmt.write_geotiff(tifffile, metadata, geolocation, band, drv_opts=['PHOTOMETRIC=MINISBLACK'])
def bathymetry_gebco(infile, outdir, vmin=-6000, vmax=0, vmin_pal=-6000., vmax_pal=0.): """ """ # Read/Process data print 'Read/Process data' ncfile = NCFile(infile) bat = ncfile.read_values('elevation')[:, :] bat = bat.astype('float32') bat[(bat < 0) & (bat >= -25)] = -25 bat[(bat < -25) & (bat >= -50)] = -50 bat[(bat < -50) & (bat >= -100)] = -100 bat[(bat < -100) & (bat >= -500)] = -500 bat[(bat < -500) & (bat >= -1000)] = -1000 bat[(bat < -1000) & (bat >= -2000)] = -2000 bat[(bat < -2000) & (bat >= -3000)] = -3000 bat[(bat < -3000) & (bat >= -4000)] = -4000 bat[(bat < -4000) & (bat >= -5000)] = -5000 bat[(bat < -5000) & (bat >= -6000)] = -6000 bat[(bat < -6000) & (bat >= -10000)] = -10000 mask = [bat >= 0] offset, scale = vmin, (vmax - vmin) / 254. np.clip(bat, vmin, vmax, out=bat) bat -= offset bat /= scale bat = np.round(bat).astype('uint8') lon = ncfile.read_values('lon')[:2:1] lat = ncfile.read_values('lat')[:2:1] lon0 = lon[0] dlon = lon[-1] - lon[0] lat0 = lat[0] dlat = lat[-1] - lat[0] # Construct metadata/geolocation/band(s) print 'Construct metadata/geolocation/band(s)' metadata = {} metadata['product_name'] = 'GEBCO bathymetry' metadata['name'] = os.path.splitext(os.path.basename(infile))[0] metadata['datetime'] = stfmt.format_time(datetime(2012, 1, 1)) metadata['time_range'] = ['-3660d', '+3660d'] metadata['source_URI'] = infile metadata['source_provider'] = 'GEBCO' metadata['processing_center'] = '' metadata['conversion_software'] = 'Syntool' metadata['conversion_version'] = '0.0.0' metadata['conversion_datetime'] = stfmt.format_time(datetime.utcnow()) metadata['parameter'] = 'bathymetry ' metadata['type'] = 'remote sensing' metadata['longitude_resolution'] = abs(dlon) metadata['latitude_resolution'] = abs(dlat) geolocation = {} geolocation['projection'] = stfmt.format_gdalprojection() geolocation['geotransform'] = [ lon0 - dlon / 2., dlon, 0, lat0 - dlat / 2., 0, dlat ] band = [] bat[mask] = 255 colortable = stfmt.format_colortable('ibcso', vmax=vmax, vmax_pal=vmax_pal, vmin=vmin, vmin_pal=vmin_pal) band.append({ 'array': bat, 'scale': scale, 'offset': offset, 'description': 'bathymetry', 'unittype': 'm', 'nodatavalue': 255, 'parameter_range': [vmin, vmax], 'colortable': colortable }) # Write geotiff print 'Write geotiff' tifffile = stfmt.format_tifffilename(outdir, metadata, create_dir=True) stfmt.write_geotiff(tifffile, metadata, geolocation, band)
import os import sys import numpy import netCDF4 from cerbere.mapper.ncfile import NCFile from matplotlib import pyplot as plt if __name__ =='__main__': filou = "/home/cerdata/provider/neodc/l4/esacci_sst/2010/03/02/20100302120000-ESACCI-L4_GHRSST-SSTdepth-OSTIA-GLOB_LT-v02.0-fv01.0.nc" print filou nc = NCFile(filou) filout = '/tmp/ostia_sst.png' filout2 = '/tmp/ostia_sst2.png' sst = nc.read_values('analysed_sst') sst = numpy.squeeze(sst) print sst.shape nc.close() nc = netCDF4.Dataset(filou) sst2 = nc.variables['analysed_sst'][:] sst2 = numpy.squeeze(sst2) print sst2.shape nc.close() #plot plt.figure() plt.pcolor(sst[200:800,100:600]) plt.colorbar() plt.savefig(filout) #plot plt.figure() plt.pcolor(sst2[200:800,100:600])
class SAFESLFile(AbstractMapper): """Abstract class for SAFE SLSTR files (except L2P). """ def __init__(self, url=None, sltype=None, mode=READ_ONLY, **kwargs): if mode != READ_ONLY: raise Exception("This mapper can only be used in read_only mode.") if sltype not in ['i', 'c', 'a', 'b']: raise Exception("Unknown SLSTR product type") super(SAFESLFile, self).__init__(url=url, mode=mode, **kwargs) self.__sltype = sltype self.__data_handlers = [] self.__oblique_fields = [] # coordinate files geodetic = "geodetic_%sn.nc" % sltype times = "time_%sn.nc" % sltype coordinate_files = [geodetic, times] # detect the data files and instanciate mappers for each one datafiles = [] for fname in glob.glob(os.path.join(url, "*_%s[n,o].nc" % sltype)): if os.path.basename(fname) not in coordinate_files: datafiles.append(os.path.basename(fname)) for f in datafiles: fname = os.path.join(url, f) self.__data_handlers.append(NCFile(url=fname, mode=mode, **kwargs)) # instantiate mappers for each coordinate file self.__geod_handler = NCFile(os.path.join(url, geodetic), mode=mode, **kwargs) self.__time_handler = NCFile(os.path.join(url, times), mode=mode, **kwargs) self.__fieldlocator = {} self.__geofieldlocator = {} self.__fieldtranslate = {} # offset between nadir and oblique swath edge self.nadir_to_oblique_offset = None def __is_oblique(self, fieldname): """Test if a field corresponds to an oblique view subproduct. Returns: bool: True f a field corresponds to an oblique view """ return (fieldname in self.__oblique_fields) def open(self, view=None, datamodel=None, datamodel_geolocation_dims=None): """ Args: view (dict, optional): a dictionary where keys are dimension names and values are slices. A view can be set on a file, meaning that only the subset defined by this view will be accessible. This view is expressed as any subset (see :func:`get_values`). For example:: view = {'row':slice(200,250), 'cell':slice(200,300)} datamodel (str): type of feature read or written. Internal argument only used by the classes from :mod:`~cerbere.datamodel` package. Can be 'Grid', 'Swath', etc... datamodel_geolocation_dims (list, optional): list of the name of the geolocation dimensions defining the data model to be read in the file. Optional argument, only used by the datamodel classes, in case the mapper class can store different types of data models. Returns: an handler on the opened file """ # open each related file in the SAFE repo if view is None: rowview = None else: rowview = {'row': view['row']} # modify view for oblique fields which are narrower if 'cell' in view: obliqueview = view obliqueview['cell'] = ( view['cell'] - self.nadir_to_oblique_offset) for hdlr in self.__data_handlers: f = os.path.basename(hdlr.get_url()) is_oblique = (f[-4] == 'o') newview = view if is_oblique and view is not None: newview = obliqueview hdlr.open(newview, datamodel, datamodel_geolocation_dims) self.__geod_handler.open(view, datamodel, datamodel_geolocation_dims) self.__time_handler.open(rowview, datamodel, datamodel_geolocation_dims) # build the two-way dictionaries of fields # ...for data for hdlr in self.__data_handlers: f = os.path.basename(hdlr.get_url()) is_oblique = (f[-4] == 'o') for fieldname in hdlr.get_fieldnames(): ncvar = hdlr.get_handler().variables[fieldname] if 'long_name' in (ncvar.ncattrs()): longname = ncvar.long_name newfieldname = self.__get_fieldname(fieldname, longname) else: newfieldname = fieldname self.__fieldtranslate[newfieldname] = fieldname self.__fieldlocator[newfieldname] = hdlr if is_oblique: self.__oblique_fields.append(newfieldname) # ...for geodetic coordinates for fieldname in self.__geod_handler.get_fieldnames(): self.__geofieldlocator[fieldname] = self.__geod_handler # define nadir/oblique offset nadir_track_offset = ( self.__geofieldlocator['latitude_orphan_%sn' % self.__sltype] .read_global_attribute('track_offset')) oblique_track_offset = ( self.__fieldlocator['latitude_orphan_%so' % self.__sltype] .read_global_attribute('track_offset')) self.nadir_to_oblique_offset = int(round( nadir_track_offset - oblique_track_offset)) def close(self): """Close handler on storage""" for hdlr in self.__data_handlers: hdlr.close() self.__data_handlers = None self.__geod_handler.close() self.__time_handler.close() self.__geod_handler = None self.__time_handler = None def get_dimsize(self, dimname): """Return the size of a dimension. Args: dimname (str): name of the dimension. Returns: int: size of the dimension. """ dim = self.get_matching_dimname(dimname) return self.__geod_handler.get_dimsize(dim) def get_dimensions(self, fieldname=None): """Return the dimension names of a file or a field in the file. For temporal and spatial dimensions, the cerbere standard names are returned. Args: fieldname (str): the name of the field from which to get the dimensions. For a geolocation field, use the cerbere standard name (time, lat, lon), though native field name will work too. Returns: tuple<str>: the standard dimensions of the field or file. """ if fieldname is None: dims = self.__geod_handler.get_dimensions() if 'orphan_pixels' in dims: # same dimension name (but not size) in oblique/nadir views so # we have to create two dimensions since we merge the two views newdims = [] for dim in dims: if dim != 'orphan_pixels': newdims.append(dim) else: newdims.extend([dim + '_n', dim + '_o']) return newdims return dims if fieldname in ['time', 'lat', 'lon', 'z']: # Should all have the same dimension as lat native_fieldname = self.get_geolocation_field('lat') dims = self.__geod_handler.get_dimensions(native_fieldname) else: handler = self.__fieldlocator[fieldname] dims = handler.get_dimensions( self.__get_native_fieldname(fieldname)) # convert geolocation dims to standard names newdims = [] for dim in list(dims): if self.__is_oblique(fieldname): view = 'o' else: view = 'n' newdims.append(self.get_standard_dimname(dim, view)) return tuple(newdims) def get_matching_dimname(self, dimname): """Return the equivalent name in the native format for a standard dimension. This is a translation of the standard names to native ones. It is used for internal purpose only and should not be called directly. The standard dimension names are: * x, y, time for :class:`~cerbere.datamodel.grid.Grid` * row, cell, time for :class:`~cerbere.datamodel.swath.Swath` or :class:`~cerbere.datamodel.image.Image` To be derived when creating an inherited data mapper class. This is mandatory for geolocation dimensions which must be standard. Args: dimname (str): standard dimension name. Returns: str: return the native name for the dimension. Return `dimname` if the input dimension has no standard name. See Also: see :func:`get_standard_dimname` for the reverse operation """ matching = {'time': 'time', 'row': 'rows', 'cell': 'columns', 'z': 'elevation'} # remove the oblique/nadir suffix if dimname[-2:] in ['_n', '_o']: dimname = dimname.strip('_n').strip('_o') if dimname in matching: return matching[dimname] return dimname def get_standard_dimname(self, dimname, view=None): """ Returns the equivalent standard dimension name for a dimension in the native format. This is a translation of the native names to standard ones. It is used for internal purpose and should not be called directly. To be derived when creating an inherited data mapper class. This is mandatory for geolocation dimensions which must be standard. Args: dimname (string): native dimension name Return: str: the (translated) standard name for the dimension. Return `dimname` if the input dimension has no standard name. See Also: see :func:`get_matching_dimname` for the reverse operation """ matching = {'time': 'time', 'rows': 'row', 'columns': 'cell', 'elevation': 'z'} if dimname in matching: return matching[dimname] # for other dimensions, add the oblique/nadir suffix if dimname in ['row', 'cell', 'time', 'z']: return dimname if view is 'n': dimname += '_n' elif view is 'o': dimname += '_o' return dimname def get_fieldnames(self): """Returns the list of geophysical fields stored for the feature. The geolocation field names are excluded from this list. Returns: list<string>: list of field names """ return self.__fieldlocator.keys() def __get_native_fieldname(self, fieldname): """Returns the native name of a field. Args: fieldname (str): name of the field. Returns: str: the native name of the field. The same as input if the field is not a geolocation field. """ if fieldname in ['lat', 'lon', 'time', 'z']: return self.get_geolocation_field(fieldname) if fieldname in self.__fieldtranslate: return self.__fieldtranslate[fieldname] return fieldname def __get_fieldname(self, fieldname, longname): """Returns a unique field name built from the long name for ambiguous field names. Used because some sub files use the same variable names. Args: fieldname (str): field name to replace with a new name, if not unique. longname (str): longname from which to build a new unique field name Returns: str: a unique field name among all the files in a SAFE container. """ if fieldname in ["SST", "SST_uncertainty", "exception"]: return longname.replace(' ', '_') return fieldname def get_geolocation_field(self, fieldname): """Return the equivalent field name in the file format for a standard geolocation field (lat, lon, time, z). Used for internal purpose and should not be called directly. Args: fieldname (str): name of the standard geolocation field (lat, lon or time) Return: str: name of the corresponding field in the native file format. Returns None if no matching is found """ if fieldname == 'time': return 'time' matching = {'lat': 'latitude', 'lon': 'longitude', 'time': 'time', 'z': 'elevation'}[fieldname] native_fieldname = matching + '_%sn' % self.__sltype return native_fieldname def read_field(self, fieldname): """ Return the :class:`cerbere.field.Field` object corresponding to the requested fieldname. The :class:`cerbere.field.Field` class contains all the metadata describing a field (equivalent to a variable in netCDF). Args: fieldname (str): name of the field Returns: :class:`cerbere.field.Field`: the corresponding field object """ if fieldname == 'time': rows = self.get_dimsize('row') cols = self.get_dimsize('cell') variable = Variable( shortname='time', description='time of measurement', authority=None, standardname=None ) field = Field( variable, OrderedDict([('row', rows), ('cell', cols)]), datatype=dtype(int64) ) field.attach_storage(self.get_field_handler(fieldname)) field.units = self.__time_handler.get_handler().\ variables['time_stamp_%s' % self.__sltype[0]].units return field elif fieldname in ['lat', 'lon', 'z']: native_name = self.get_geolocation_field(fieldname) geofield = self.__geofieldlocator[native_name].read_field( native_name ) geofield.name = fieldname return geofield else: native_name = self.__get_native_fieldname(fieldname) field = self.__fieldlocator[fieldname].read_field(native_name) field.name = fieldname field.attach_storage(self.get_field_handler(fieldname)) dims = field.dimensions renamed_dims = OrderedDict() for dim in dims: newdim = dim if dim not in ['row', 'cell', 'time', 'z']: if self.__is_oblique(fieldname): newdim += '_o' else: newdim += '_n' renamed_dims[newdim] = dims[dim] field.dimensions = renamed_dims # for oblique view, the swath is narrower. It is padded with # dummy values to stack it over nadir view fields if fieldname in self.__oblique_fields: if 'cell' in field.get_dimnames(): field.dimensions['cell'] = self.get_dimsize('cell') return field def read_values(self, fieldname, slices=None): """Read the data of a field. Args: fieldname (str): name of the field which to read the data from slices (list of slice, optional): list of slices for the field if subsetting is requested. A slice must then be provided for each field dimension. The slices are relative to the opened view (see :func:open) if a view was set when opening the file. Return: MaskedArray: array of data read. Array type is the same as the storage type. """ native_name = self.__get_native_fieldname(fieldname) rowslice = None if slices is not None: rowslice = [slices[0]] if fieldname == 'time': suffix = self.__sltype SCANSYNC = self.__time_handler.read_values('SCANSYNC')[0] PIXSYNC_i = self.__time_handler.read_values( 'PIXSYNC_%s' % suffix)[0] prefix = PREFIX['n'] first_scan_i\ = self.__time_handler.read_values( '%s_First_scan_%s' % (prefix, suffix), slices=rowslice) first_min_ts\ = self.__time_handler.read_values( '%s_Minimal_ts_%s' % (prefix, suffix), slices=rowslice) scanfield = 'scan_%sn' % suffix pixelfield = 'pixel_%sn' % suffix indices_handler = self.__fieldlocator[scanfield] scan = indices_handler.read_values(scanfield, slices=slices) pixel = indices_handler.read_values(pixelfield, slices=slices) time = first_min_ts.reshape((-1, 1))\ + (scan - first_scan_i.reshape((-1, 1)))\ * SCANSYNC + pixel * PIXSYNC_i # mask wrong times (which occur in test data) maxdate = date2num(self.get_end_time(), "microseconds since 2000-01-01T00:00:00Z") mindate = date2num(self.get_start_time(), "microseconds since 2000-01-01T00:00:00Z") time = ma.masked_where( ((time < mindate) | (time > maxdate)), time, copy=False ) return time elif fieldname in ['lat', 'lon', 'z']: return self.__geofieldlocator[native_name].read_values(native_name, slices) elif self.__is_oblique(fieldname): # oblique views has not the same cell dimension than nadir # we want to stack fields from both views by padding fillvalues celldim = None try: dims = self.__fieldlocator[fieldname].get_dimensions( native_name ) empty = False if 'cell' in dims: dimsizes = [self.get_dimsize(dim) for dim in dims] celldim = list(dims).index('cell') nadir_slices = cerbere.mapper.slices.get_nice_slices( slices, dimsizes) sli = nadir_slices[celldim] nad_start, nad_end, step = sli.start, sli.stop, sli.step obliquewidth = (self.__fieldlocator[fieldname] .get_dimsize('cell')) obl_start, obl_end = nad_start, nad_end obl_start = max(0, obl_start - self.nadir_to_oblique_offset) if obl_start > obliquewidth: obl_start = obliquewidth obl_end = max(0, min(obliquewidth, obl_end - self.nadir_to_oblique_offset) ) newslices = list(nadir_slices) newslices[celldim] = slice(obl_start, obl_end, step) if obl_start >= obl_end: empty = True else: # case of some fields such as orphan pixels which don't # have a cell dimension newslices = list(slices) if slices is not None else None except ValueError: raise # read values in oblique view grid if celldim is not None and empty: values = ma.masked_all( (), dtype=self.__fieldlocator[fieldname]._handler.variables[native_name].dtype) else: values = self.__fieldlocator[fieldname].read_values( self.__fieldtranslate[fieldname], newslices) # padding for missing oblique values to match nadir view grid if celldim is not None: shape = cerbere.mapper.slices.get_shape_from_slice( cerbere.mapper.slices.get_nice_slices(slices, dimsizes)) padded_values = ma.masked_all( tuple(shape), dtype=values.dtype, ) if values.shape == () or min(list(values.shape)) <= 0: # empty result => return padded values only return padded_values padded_slice = [] for dim in dims: if dim != 'cell': padded_slice.append(slice(None, None, None)) else: offset = (self.nadir_to_oblique_offset - nad_start + obl_start) padded_slice.append(slice( offset, offset + values.shape[celldim] )) padded_values[padded_slice] = values return padded_values else: return values else: return self.__fieldlocator[fieldname].read_values(native_name, slices) def read_fillvalue(self, fieldname): """Read the fill value of a field. Args: fieldname (str): name of the field. Returns: number or char or str: fill value of the field. The type is the as the type of the data in the field. """ return self.__fieldlocator[fieldname].read_fillvalue(fieldname) def read_global_attributes(self): """Returns the names of the global attributes. Returns: list<str>: the list of the attribute names. """ # all files seem to have the same list og global attributes. return self.__geod_handler.read_global_attributes() def read_global_attribute(self, name): """Returns the value of a global attribute. Args: name (str): name of the global attribute. Returns: str, number or datetime: value of the corresponding attribute. """ # all files seem to have the same list or global attributes. return self.__geod_handler.read_global_attribute(name) def read_field_attributes(self, fieldname): """Return the specific attributes of a field. Args: fieldname (str): name of the field. Returns: dict<string, string or number or datetime>: a dictionary where keys are the attribute names. """ return self.__fieldlocator[fieldname].read_field_attributes(fieldname) def get_start_time(self): """Returns the minimum date of the file temporal coverage. Returns: datetime: start time of the data in file. """ varname = 'time_stamp_%s' % self.__sltype[0] vardate = self.__time_handler.get_handler().variables[varname] return num2date(vardate[0], vardate.units) def get_end_time(self): """Returns the maximum date of the file temporal coverage. Returns: datetime: end time of the data in file. """ # WRONG!!! varname = 'time_stamp_%s' % self.__sltype[0] vardate = self.__time_handler.get_handler().variables[varname] return num2date(vardate[-1], vardate.units) def get_bbox(self): """Returns the bounding box of the feature, as a tuple. Returns: tuple: bbox expressed as (lonmin, latmin, lonmax, latmax) """ return None def write_global_attributes(self, attrs): """Write the global attributes of the file. Args: attrs (dict<string, string or number or datetime>): a dictionary containing the attributes names and values to be written. """ raise NotImplementedError def create_field(self, field, dim_translation=None): """Creates a new field in the mapper. Creates the field structure but don't write yet its values array. Args: field (Field): the field to be created. See also: :func:`write_field` for writing the values array. """ raise NotImplementedError def create_dim(self, dimname, size=None): """Add a new dimension. Args: dimname (str): name of the dimension. size (int): size of the dimension (unlimited if None) """ raise NotImplementedError def write_field(self, fieldname): """Writes the field data on disk. Args: fieldname (str): name of the field to write. """ raise NotImplementedError
''' Created on 18 juin 2012 Test reading/displaying time series file @author: jfpiolle ''' import logging import matplotlib.pyplot as plt import matplotlib.colors from cerbere.datamodel.pointtimeseries import PointTimeSeries from cerbere.mapper.ncfile import NCFile logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)-5s %(message)s', datefmt='%d/%m/%Y %I:%M:%S') ff = '/home/cercache/project/globwave/data/globwave/oceansites/ocean_temperature_sensor/2015/05/WMO61280_20150501T0000_20150515T0100_Lat_40.69N_Lon_1.48E.nc' ncf = NCFile( url = ff) traj = PointTimeSeries() traj.load( ncf ) #val = traj.getData('greyscale_threshold') traj.display_timeseries('sea_surface_temperature', palette=plt.cm.gray, pretty=True, range=[273,305,0.25], output='toto.png')
def add_swh_era(feature_file, outpath, linear=False, suffix=None): """ :param feature_file: the file in which you want to append a new field :param outpath: (str) :param linear: bool :param suffix: str :return: """ # ERA5 model definition model = RegularGriddedModel('/home/ref-ecmwf/ERA5/', '%Y/%m/era_5-copernicus__%Y%m%d.nc', 1, 0.50, 'ERA5025NCDataset', -90, 0, 720, 361, modelfeature='CylindricalGridTimeSeries') new = None # open feature file if new is not None: mode = 'r' else: mode = 'r+' # feature = cerbere.open_as_feature( # options.feature, feature_file, options.reader, mode=mode # ) hh = NCFile(feature_file) feature = Trajectory.load(hh) # remap model fields intmode = 'closest' if linear: intmode = 'linear' fieldnames = NAMING_CONVENTION.keys() fields = model.get_model_fields(feature, fieldnames, mode=intmode) # add metadata for fieldname, field in fields.items(): newname = NAMING_CONVENTION[fieldname] field.name = newname field.standardname = STANDARD_NAME[fieldname] if field.units in UNITS: field.units = UNITS[field.units] field.attrs['source'] = "Copernicus ERA5 Reanalysis by ECMWF" # add to current feature if suffix is None: for fieldname, field in fields.items(): if field in feature_file.fieldnames: raise Exception('%s field already in file' % fieldname) feature.add_field(field) # or create a new file with same structure else: basef, ext = os.path.splitext(os.path.basename(feature_file)) if outpath is None: outpath = os.path.dirname(feature_file) auxf = os.path.join(outpath, basef + suffix + '.nc') if os.path.exists(auxf): os.remove(auxf) print("Save to: {}".format(auxf)) # create empty feature auxfeature = feature.extract(fields=[]) for fieldname, field in fields.items(): auxfeature.add_field(field) # coordinates attribute exfield = feature.get_field(feature.fieldnames[0]) if 'coordinates' in exfield.attrs: field.attrs['coordinates'] = \ exfield.attrs['coordinates'] # save auxfeature.save(auxf) feature.close()