def test_open_ecmwf_y_wind_and_x_wind_at_given_time(self):
n = Nansat(self.test_file_ecmwf, bands=['y_wind', 'x_wind'],
netcdf_dim={'time': '1488409200'}, mapperName='netcdf_cf')
self.assertIsInstance(n, Nansat)
self.assertEqual(2, len(n.bands()))
self.assertTrue(n['x_wind_10m'].any())
self.assertTrue(n['y_wind_10m'].any())
def test_open_ecmwf_y_wind_and_x_wind_at_given_time(self):
n = Nansat(self.test_file_ecmwf,
bands=['y_wind', 'x_wind'],
netcdf_dim={'time': '1488409200'},
mapperName='netcdf_cf')
self.assertIsInstance(n, Nansat)
self.assertEqual(2, len(n.bands()))
self.assertTrue(n['x_wind_10m'].any())
self.assertTrue(n['y_wind_10m'].any())
def test_open_arome_arctic_y_wind_and_x_wind_at_given_datetime(self):
n = Nansat(self.test_file_arome_arctic, bands=['y_wind', 'x_wind'],
netcdf_dim={'time':
np.datetime64(datetime.datetime(2017,2,28,15,30,0))},
mapperName='netcdf_cf')
self.assertIsInstance(n, Nansat)
self.assertEqual(2, len(n.bands()))
self.assertTrue(n['x_wind_10m'].any())
self.assertTrue(n['y_wind_10m'].any())
def test_open_arome_arctic_y_wind_and_x_wind_at_given_datetime(self):
n = Nansat(self.test_file_arome_arctic,
bands=['y_wind', 'x_wind'],
netcdf_dim={
'time':
np.datetime64(datetime.datetime(2017, 2, 28, 15, 30, 0))
},
mapperName='netcdf_cf')
self.assertIsInstance(n, Nansat)
self.assertEqual(2, len(n.bands()))
self.assertTrue(n['x_wind_10m'].any())
self.assertTrue(n['y_wind_10m'].any())
def get_or_create(self,
uri,
n_points=10,
uri_filter_args=None,
uri_service_name=FILE_SERVICE_NAME,
uri_service_type=LOCAL_FILE_SERVICE,
*args,
**kwargs):
""" Create dataset and corresponding metadata
Parameters:
----------
uri : str
URI to file or stream openable by Nansat
n_points : int
Number of border points (default is 10)
uri_filter_args : dict
Extra DatasetURI filter arguments if several datasets can refer to the same URI
uri_service_name : str
name of the service which is used ('dapService', 'fileService', 'http' or 'wms')
uri_service_type : str
type of the service which is used ('OPENDAP', 'local', 'HTTPServer' or 'WMS')
Returns:
-------
dataset and flag
"""
if not uri_filter_args:
uri_filter_args = {}
# Validate uri - this should raise an exception if the uri doesn't point to a valid
# file or stream
validate_uri(uri)
# Several datasets can refer to the same uri (e.g., scatterometers and svp drifters), so we
# need to pass uri_filter_args
uris = DatasetURI.objects.filter(uri=uri, **uri_filter_args)
if len(uris) > 0:
return uris[0].dataset, False
# Open file with Nansat
n = Nansat(nansat_filename(uri), **kwargs)
# get metadata from Nansat and get objects from vocabularies
n_metadata = n.get_metadata()
entry_id = n_metadata.get('entry_id', None)
# set compulsory metadata (source)
platform, _ = Platform.objects.get_or_create(
json.loads(n_metadata['platform']))
instrument, _ = Instrument.objects.get_or_create(
json.loads(n_metadata['instrument']))
specs = n_metadata.get('specs', '')
source, _ = Source.objects.get_or_create(platform=platform,
instrument=instrument,
specs=specs)
default_char_fields = {
# Adding NERSC_ in front of the id violates the string representation of the uuid
#'entry_id': lambda: 'NERSC_' + str(uuid.uuid4()),
'entry_id': lambda: str(uuid.uuid4()),
'entry_title': lambda: 'NONE',
'summary': lambda: 'NONE',
}
# set optional CharField metadata from Nansat or from default_char_fields
options = {}
try:
existing_ds = Dataset.objects.get(entry_id=entry_id)
except Dataset.DoesNotExist:
existing_ds = None
for name in default_char_fields:
if name not in n_metadata:
warnings.warn('%s is not provided in Nansat metadata!' % name)
# prevent overwriting of existing values by defaults
if existing_ds:
options[name] = existing_ds.__getattribute__(name)
else:
options[name] = default_char_fields[name]()
else:
options[name] = n_metadata[name]
default_foreign_keys = {
'gcmd_location': {
'model': Location,
'value': pti.get_gcmd_location('SEA SURFACE')
},
'data_center': {
'model': DataCenter,
'value': pti.get_gcmd_provider('NERSC')
},
'ISO_topic_category': {
'model': ISOTopicCategory,
'value': pti.get_iso19115_topic_category('Oceans')
},
}
# set optional ForeignKey metadata from Nansat or from default_foreign_keys
for name in default_foreign_keys:
value = default_foreign_keys[name]['value']
model = default_foreign_keys[name]['model']
if name not in n_metadata:
warnings.warn('%s is not provided in Nansat metadata!' % name)
else:
try:
value = json.loads(n_metadata[name])
except:
warnings.warn(
'%s value of %s metadata provided in Nansat is wrong!'
% (n_metadata[name], name))
if existing_ds:
options[name] = existing_ds.__getattribute__(name)
else:
options[name], _ = model.objects.get_or_create(value)
# Find coverage to set number of points in the geolocation
if len(n.vrt.dataset.GetGCPs()) > 0:
n.reproject_gcps()
geolocation = GeographicLocation.objects.get_or_create(
geometry=WKTReader().read(n.get_border_wkt(nPoints=n_points)))[0]
# create dataset
# - the get_or_create method should use get_or_create here as well,
# or its name should be changed - see issue #127
ds, created = Dataset.objects.update_or_create(
entry_id=options['entry_id'],
defaults={
'time_coverage_start': n.get_metadata('time_coverage_start'),
'time_coverage_end': n.get_metadata('time_coverage_end'),
'source': source,
'geographic_location': geolocation,
'gcmd_location': options["gcmd_location"],
'ISO_topic_category': options["ISO_topic_category"],
"data_center": options["data_center"],
'entry_title': options["entry_title"],
'summary': options["summary"]
})
# create parameter
all_band_meta = n.bands()
for band_id in range(1, len(all_band_meta) + 1):
band_meta = all_band_meta[band_id]
standard_name = band_meta.get('standard_name', None)
short_name = band_meta.get('short_name', None)
units = band_meta.get('units', None)
if standard_name in ['latitude', 'longitude', None]:
continue
params = Parameter.objects.filter(standard_name=standard_name)
if params.count() > 1 and short_name is not None:
params = params.filter(short_name=short_name)
if params.count() > 1 and units is not None:
params = params.filter(units=units)
if params.count() >= 1:
ds.parameters.add(params[0])
# create dataset URI
DatasetURI.objects.get_or_create(name=uri_service_name,
service=uri_service_type,
uri=uri,
dataset=ds)
return ds, created
def get_or_create(self, uri, reprocess=False, *args, **kwargs):
# ingest file to db
ds, created = super(DatasetManager,
self).get_or_create(uri, *args, **kwargs)
# set Dataset entry_title
ds.entry_title = 'SAR NRCS'
ds.save()
# Unless reprocess==True, we may not need to do the following... (see
# managers.py in sar doppler processor)
#visExists = ... # check if visualization(s) already created
#if visExists and not reprocess:
# warnings.warn('NO VISUALISATIONS CREATED - update managers.py')
# return ds, created
n = Nansat(nansat_filename(uri))
n.reproject_GCPs()
n.resize(pixelsize=500)
lon, lat = n.get_corners()
lat_max = min(lat.max(), 85)
d = Domain(
NSR(3857), '-lle %f %f %f %f -ts %d %d' %
(lon.min(), lat.min(), lon.max(), lat_max, n.shape()[1],
n.shape()[0]))
# Get all NRCS bands
s0bands = []
pp = []
for key, value in n.bands().iteritems():
try:
if value['standard_name'] == standard_name:
s0bands.append(key)
pp.append(value['polarization'])
except KeyError:
continue
''' Create data products
'''
mm = self.__module__.split('.')
module = '%s.%s' % (mm[0], mm[1])
mp = media_path(module, n.fileName)
# ppath = product_path(module, n.fileName)
# Create png's for each band
num_products = 0
for band in s0bands:
print 'Visualize', band
s0_tmp = n[band]
n_tmp = Nansat(domain=n, array=s0_tmp)
n_tmp.reproject_GCPs()
n_tmp.reproject(d)
s0 = n_tmp[1]
n_tmp = None
mask = np.ones(s0.shape, np.uint8)
mask[np.isnan(s0) + (s0 <= 0)] = 0
s0 = np.log10(s0) * 10.
meta = n.bands()[band]
product_filename = '%s_%s.png' % (meta['short_name'],
meta['polarization'])
nansatFigure(s0, mask, polarization_clims[meta['polarization']][0],
polarization_clims[meta['polarization']][1], mp,
product_filename)
# Get DatasetParameter
param = Parameter.objects.get(short_name=meta['short_name'])
dsp, created = DatasetParameter.objects.get_or_create(
dataset=ds, parameter=param)
# Create Visualization
geom, created = GeographicLocation.objects.get_or_create(
geometry=WKTReader().read(n.get_border_wkt()))
vv, created = Visualization.objects.get_or_create(
uri='file://localhost%s/%s' % (mp, product_filename),
title='%s %s polarization' %
(param.standard_name, meta['polarization']),
geographic_location=geom)
# Create VisualizationParameter
vp, created = VisualizationParameter.objects.get_or_create(
visualization=vv, ds_parameter=dsp)
return ds, True
