def verify_output(self, files, output, config):
tested = map(lambda f:
self.translate_file('tests/products/estimated', f),
files)
with nc.loader(tested, self.tile_cut) as old:
valid = nc.getvar(old, 'globalradiation')
max_vaild = valid[:].max()
# It allow a 1% of the maximum value as the maximum error
# threshold.
threshold = max_vaild * 0.01
calculated = output.globalradiation
if config['product']:
products = map(lambda f:
self.translate_file('products/estimated', f),
files)
with nc.loader(products, self.tile_cut) as new:
calculated = nc.getvar(new, 'globalradiation')
gtz = lambda m: m[calculated[:] >= 0]
diff = gtz(calculated[:] - valid[:])
print('thr: {:}'.format(threshold))
print('min: {:} ({:})'.format(gtz(calculated[:]).min(),
gtz(valid[:]).min()))
print('max: {:} ({:})'.format(gtz(calculated[:]).max(),
gtz(valid[:]).max()))
self.assertTrue((diff < threshold).all())
shape = valid.shape
return shape
def construct(cls, static_file, ref_filename):
# At first it should have: lat, lon, dem, linke
logging.info("This is the first execution from the deployment... ")
with nc.loader(ref_filename) as root_ref:
with nc.loader(static_file) as root:
lat = nc.getvar(root_ref, 'lat')
lon = nc.getvar(root_ref, 'lon')
nc.getvar(root, 'lat', source=lat)
nc.getvar(root, 'lon', source=lon)
cls.project_dem(root, lat, lon)
cls.project_linke(root, lat, lon)
def verify_output(self):
with nc.loader('tests/products/estimated/*.nc', DIMS) as old_root:
with nc.loader('products/estimated/*.nc', DIMS) as new_root:
valid = nc.getvar(old_root, 'globalradiation')
max_vaild = valid[:].max()
# It allow a 1% of the maximum value as the maximum error
# threshold.
threshold = max_vaild * 0.01
calculated = nc.getvar(new_root, 'globalradiation')
gtz = lambda m: m[calculated[:] >= 0]
diff = gtz(calculated[:] - valid[:])
print 'min: ', gtz(calculated[:]).min(), '(', gtz(valid[:]).min(), ')'
print 'max: ', gtz(calculated[:]).max(), '(', gtz(valid[:]).max(), ')'
self.assertTrue((diff < threshold).all())
def initialize(radiance_filename, radiation_filename, callback=lambda r: r):
ref, _ = nc.open(radiation_filename)
ref_radiation = nc.getvar(ref, 'globalradiation')
with nc.loader(radiance_filename) as radiance_root:
radiance = nc.getvar(radiance_root, 'radiance', source=ref_radiation)
radiance[0, :] = callback(radiance[0, :])
nc.close(ref)
def __exit__(self, type, value, traceback):
self.initial_size = len(self.initial_files)
final_size = len(glob.glob('%s/*.nc' % self.directory))
files = self.files
if files:
# Test if it downloaded something.
self.assertNotEquals(final_size, self.initial_size)
the_size = final_size - self.initial_size
self.assertEquals(len(files), the_size)
# Test if it calibrated the downloaded files.
with nc.loader(files) as root:
counts = nc.getvar(root, 'counts_shift')
self.assertEquals(counts.shape, (the_size, 1, 1))
self.assertEquals(counts[0], 32.)
space = nc.getvar(root, 'space_measurement')
self.assertEquals(space.shape, (the_size, 1, 1))
self.assertEquals(space[0], 29.)
prelaunch_0 = nc.getvar(root, 'prelaunch_0')
self.assertEquals(prelaunch_0.shape, (the_size, 1, 1))
self.assertAlmostEqual(prelaunch_0[0], 0.61180001, 6)
prelaunch_1 = nc.getvar(root, 'prelaunch_1')
self.assertEquals(prelaunch_1.shape, (the_size, 1, 1))
self.assertAlmostEqual(prelaunch_1[0], 0.00116, 6)
postlaunch = nc.getvar(root, 'postlaunch')
self.assertEquals(postlaunch.shape, (the_size, 1, 1))
def initialize(radiance_filename, radiation_filename, callback=lambda r: r):
ref, _ = nc.open(radiation_filename)
ref_radiation = nc.getvar(ref, 'globalradiation')
with nc.loader(radiance_filename) as radiance_root:
radiance = nc.getvar(radiance_root, 'radiance', source=ref_radiation)
radiance[0, :] = callback(radiance[0, :])
nc.close(ref)
def test_calibrate(self):
instrument.calibrate('goes13.2014.124.123422.BAND_01.nc')
with nc.loader('goes13.2014.124.123422.BAND_01.nc') as root:
var = lambda v: float(nc.getvar(root, v)[0])
self.assertAlmostEquals(var('space_measurement'), 29.0)
self.assertAlmostEquals(var('postlaunch'), 1.293)
self.assertAlmostEquals(var('counts_shift'), 32.0)
# 0: radiance, 1: albedo
self.assertAlmostEquals(var('prelaunch_0'), 0.6118)
self.assertAlmostEquals(var('prelaunch_1'), 0.00116)
def obtain_data(self, y, x):
now = localize(datetime.utcnow())
now = now.replace(year=2015, month=2, day=17)
today_pattern = path + 'goes13.%Y.%j.*.BAND_01.nc'
files = glob(now.strftime(today_pattern))
if not files:
return []
with nc.loader(files) as root:
time = nc.getvar(root, 'time')[:,0]
data = nc.getvar(root, 'globalradiation')[:,y,x]
return zip(time.tolist(), data.tolist())
def extend_cache(self, filenames):
cached_files = glob.glob('%s/*.nc' % self.temporal_path)
not_cached = filter(lambda f: self.get_cached_file(f)
not in cached_files,
filenames)
if not_cached:
loader = Loader(not_cached)
new_files = pmap(self.get_cached_file, not_cached)
with nc.loader(new_files, dimensions=DIMS) as cache:
self.algorithm.update_temporalcache(loader, cache)
loader.dump()
def __init__(self, algorithm):
super(OutputCache, self).__init__(algorithm)
self.output = Loader(pmap(self.get_output_file, self.filenames))
self.root = self.output.root
with nc.loader(self.filenames, dimensions=DIMS) as images:
map(algorithm.create_1px_dimensions, self.root.roots)
self.root.getvar('time', source=images.getvar('time'))
self.root.getvar('cloudindex',
'f4', source=images.getvar('data'))
self.root.getvar('globalradiation',
'f4', source=images.getvar('data'))
def test_open_close_using_with(self):
# check if open the pattern selection using using a package instance.
with nc.loader('unittest0*.nc') as root:
self.assertEquals(root.files,
['unittest0%i.nc' % i for i in range(5)])
self.assertEquals(root.pattern, 'unittest0*.nc')
self.assertEquals(len(root.roots), 5)
self.assertFalse(root.is_new)
self.assertFalse(root.read_only)
# check if close the package with all the files.
with self.assertRaisesRegexp(RuntimeError, u'NetCDF: Not a valid ID'):
nc.close(root)
def test_open_close_using_with(self):
# check if open the pattern selection using using a package instance.
with nc.loader('unittest0*.nc') as root:
self.assertEquals(root.files,
['unittest0%i.nc' % i for i in range(5)])
self.assertEquals(root.pattern, 'unittest0*.nc')
self.assertEquals(len(root.roots), 5)
self.assertFalse(root.is_new)
self.assertFalse(root.read_only)
# check if close the package with all the files.
with self.assertRaisesRegexp(RuntimeError, u'NetCDF: Not a valid ID'):
nc.close(root)
def initialize_variables(self, filenames):
self.path = '/'.join(filenames[0].split('/')[0:-1])
self.output_path = self.product
with nc.loader(self.filenames, dimensions=self.tile_cut) as images:
if self.output_path:
self.initialize_path(filenames, images)
else:
data_shape = images.getvar('data').shape
self.time = np.zeros(images.getvar('time').shape)
self.ref_cloudindex = np.zeros(data_shape)
self.cloudindex = self.ref_cloudindex
self.ref_globalradiation = np.zeros(data_shape)
self.globalradiation = self.ref_globalradiation
def only_incompleted(url, destfolder):
dest = calculate_destiny(url, destfolder)
completed = False
if os.path.exists(dest):
try:
with nc.loader(dest) as root:
nc.getvar(root, 'data')
nc.getvar(root, 'lat')
nc.getvar(root, 'lon')
completed = True
except (OSError, IOError, Exception):
logger.error("The file %s was broken." % dest)
return not completed
def only_incompleted(url, destfolder):
dest = calculate_destiny(url, destfolder)
completed = False
if os.path.exists(dest):
try:
with nc.loader(dest) as root:
nc.getvar(root, 'data')
nc.getvar(root, 'lat')
nc.getvar(root, 'lon')
completed = True
except (OSError, IOError, Exception):
logger.error("The file %s was broken." % dest)
return not completed
def __init__(self, filenames):
# At first it should have: lat, lon, dem, linke
self.root, is_new = nc.open('static.nc')
if is_new:
logging.info("This is the first execution from the deployment... ")
with nc.loader(filenames[0]) as root_ref:
self.lat = nc.getvar(root_ref, 'lat')
self.lon = nc.getvar(root_ref, 'lon')
nc.getvar(self.root, 'lat', source=self.lat)
nc.getvar(self.root, 'lon', source=self.lon)
self.project_dem()
self.project_linke()
nc.sync(self.root)
self.root = nc.tailor(self.root, dimensions=DIMS)
def __init__(self, filenames, tile_cut={}):
# At first it should have: lat, lon, dem, linke
self.root, is_new = nc.open("static.nc")
if is_new:
logging.info("This is the first execution from the deployment... ")
with nc.loader(filenames[0]) as root_ref:
self.lat = nc.getvar(root_ref, "lat")
self.lon = nc.getvar(root_ref, "lon")
nc.getvar(self.root, "lat", source=self.lat)
nc.getvar(self.root, "lon", source=self.lon)
self.project_dem()
self.project_linke()
nc.sync(self.root)
self.root = nc.tailor(self.root, dimensions=tile_cut)
def calibrate(filename):
"""
Append the calibration parameters as variables of the netcdf file.
Keyword arguments:
filename -- the name of a netcdf file.
"""
params = calibration_to(filename)
with nc.loader(filename) as root:
for key, value in params.items():
nc.getdim(root, 'xc_1', 1)
nc.getdim(root, 'yc_1', 1)
if isinstance(value, list):
for i in range(len(value)):
nc.getvar(root, '%s_%i' % (key, i), 'f4', ('time', 'yc_1', 'xc_1' ))[:] = value[i]
else:
nc.getvar(root, key, 'f4', ('time', 'yc_1', 'xc_1'))[:] = value
def test_load_calibration_using_netcdf_package(self):
instrument.calibrate('goes13.2014.124.123422.BAND_01.nc')
array_g = lambda x: [x for i in range(5)]
with nc.loader(array_g('goes13.2014.124.123422.BAND_01.nc')) as root:
var = lambda v: map(float, nc.getvar(root, v)[:])
for i in range(5):
self.assertAlmostEquals(var('space_measurement')[i],
array_g(29.0)[i])
self.assertAlmostEquals(var('postlaunch')[i],
array_g(1.293)[i])
self.assertAlmostEquals(var('counts_shift')[i],
array_g(32.0)[i])
# 0: radiance, 1: albedo
self.assertAlmostEquals(var('prelaunch_0')[i],
array_g(0.6118)[i])
self.assertAlmostEquals(var('prelaunch_1')[i],
array_g(0.00116)[i])
def radiance(filename):
prefix = short(filename, 0, 3)
slot = int(round(decimalhour(to_datetime(filename))*2))
suffix = short(filename, 4, 6)
output_filename = create_output_path(prefix, slot, suffix)
root, is_new = nc.open(filename)
radiation = nc.getvar(root, 'globalradiation')
with nc.loader(output_filename) as radiance_root:
dims_names = list(reversed(radiation.dimensions.keys()))
dims_values = list(reversed(radiation.dimensions.values()))
create_dims = (lambda name, dimension:
radiance_root.create_dimension(name, len(dimension)))
(map(lambda name, dimension: create_dims(name, dimension),
dims_names, dims_values))
radiance = (nc.getvar(radiance_root, 'radiance', vtype='f4',
dimensions=tuple(dims_names)))
radiance[:] = radiation[:]*30.*60.*10**-6
def radiance(filename):
prefix = short(filename, 0, 3)
slot = int(round(decimalhour(to_datetime(filename)) * 2))
suffix = short(filename, 4, 6)
output_filename = create_output_path(prefix, slot, suffix)
root, is_new = nc.open(filename)
radiation = nc.getvar(root, 'globalradiation')
with nc.loader(output_filename) as radiance_root:
dims_names = list(reversed(radiation.dimensions.keys()))
dims_values = list(reversed(radiation.dimensions.values()))
create_dims = (lambda name, dimension: radiance_root.create_dimension(
name, len(dimension)))
(map(lambda name, dimension: create_dims(name, dimension), dims_names,
dims_values))
radiance = (nc.getvar(radiance_root,
'radiance',
vtype='f4',
dimensions=tuple(dims_names)))
radiance[:] = radiation[:] * 30. * 60. * 10**-6
def setUp(self):
with nc.loader('goes13.2014.124.123422.BAND_01.nc') as root:
nc.getdim(root, 'xc', 200)
nc.getdim(root, 'yc', 100)
nc.getdim(root, 'time')
nc.getvar(root, 'data', 'i4', ('time', 'yc', 'xc'), fill_value=1)
from flask_restful import Resource, Api
from datetime import datetime
import pytz
from glob import glob
from netcdf import netcdf as nc
import numpy as np
app = Flask(__name__)
api = Api(app)
log = open('queries.txt', 'a')
path = './products/estimated/'
gmt = pytz.timezone('GMT')
local = pytz.timezone('America/Argentina/Buenos_Aires')
localize = lambda dt: (gmt.localize(dt)).astimezone(local)
with nc.loader('%sstatic.nc' % path) as root:
lat_m, lon_m = root.getvar('lat')[:], root.getvar('lon')[:]
class Irradiance(Resource):
def translate(self, georef):
try:
lat, lon = map(float, georef.split(','))[:2]
except Exception:
lat, lon = None, None
return lat, lon
def register(self, lat, lon):
now = localize(datetime.utcnow())
log.write('%s: (%.5f, %.5f)' % (str(now), lat, lon))
def time(self):
with nc.loader(self.filenames_from_a_month_ago) as loader:
self.times = nc.getvar(loader, 'time')[:]
return self.times
