def verify_output(self, files, output, config): tested = map(lambda f: self.translate_file('tests/products/estimated', f), files) with nc.loader(tested, self.tile_cut) as old: valid = nc.getvar(old, 'globalradiation') max_vaild = valid[:].max() # It allow a 1% of the maximum value as the maximum error # threshold. threshold = max_vaild * 0.01 calculated = output.globalradiation if config['product']: products = map(lambda f: self.translate_file('products/estimated', f), files) with nc.loader(products, self.tile_cut) as new: calculated = nc.getvar(new, 'globalradiation') gtz = lambda m: m[calculated[:] >= 0] diff = gtz(calculated[:] - valid[:]) print('thr: {:}'.format(threshold)) print('min: {:} ({:})'.format(gtz(calculated[:]).min(), gtz(valid[:]).min())) print('max: {:} ({:})'.format(gtz(calculated[:]).max(), gtz(valid[:]).max())) self.assertTrue((diff < threshold).all()) shape = valid.shape return shape
def construct(cls, static_file, ref_filename): # At first it should have: lat, lon, dem, linke logging.info("This is the first execution from the deployment... ") with nc.loader(ref_filename) as root_ref: with nc.loader(static_file) as root: lat = nc.getvar(root_ref, 'lat') lon = nc.getvar(root_ref, 'lon') nc.getvar(root, 'lat', source=lat) nc.getvar(root, 'lon', source=lon) cls.project_dem(root, lat, lon) cls.project_linke(root, lat, lon)
def verify_output(self): with nc.loader('tests/products/estimated/*.nc', DIMS) as old_root: with nc.loader('products/estimated/*.nc', DIMS) as new_root: valid = nc.getvar(old_root, 'globalradiation') max_vaild = valid[:].max() # It allow a 1% of the maximum value as the maximum error # threshold. threshold = max_vaild * 0.01 calculated = nc.getvar(new_root, 'globalradiation') gtz = lambda m: m[calculated[:] >= 0] diff = gtz(calculated[:] - valid[:]) print 'min: ', gtz(calculated[:]).min(), '(', gtz(valid[:]).min(), ')' print 'max: ', gtz(calculated[:]).max(), '(', gtz(valid[:]).max(), ')' self.assertTrue((diff < threshold).all())
def initialize(radiance_filename, radiation_filename, callback=lambda r: r): ref, _ = nc.open(radiation_filename) ref_radiation = nc.getvar(ref, 'globalradiation') with nc.loader(radiance_filename) as radiance_root: radiance = nc.getvar(radiance_root, 'radiance', source=ref_radiation) radiance[0, :] = callback(radiance[0, :]) nc.close(ref)
def __exit__(self, type, value, traceback): self.initial_size = len(self.initial_files) final_size = len(glob.glob('%s/*.nc' % self.directory)) files = self.files if files: # Test if it downloaded something. self.assertNotEquals(final_size, self.initial_size) the_size = final_size - self.initial_size self.assertEquals(len(files), the_size) # Test if it calibrated the downloaded files. with nc.loader(files) as root: counts = nc.getvar(root, 'counts_shift') self.assertEquals(counts.shape, (the_size, 1, 1)) self.assertEquals(counts[0], 32.) space = nc.getvar(root, 'space_measurement') self.assertEquals(space.shape, (the_size, 1, 1)) self.assertEquals(space[0], 29.) prelaunch_0 = nc.getvar(root, 'prelaunch_0') self.assertEquals(prelaunch_0.shape, (the_size, 1, 1)) self.assertAlmostEqual(prelaunch_0[0], 0.61180001, 6) prelaunch_1 = nc.getvar(root, 'prelaunch_1') self.assertEquals(prelaunch_1.shape, (the_size, 1, 1)) self.assertAlmostEqual(prelaunch_1[0], 0.00116, 6) postlaunch = nc.getvar(root, 'postlaunch') self.assertEquals(postlaunch.shape, (the_size, 1, 1))
def test_calibrate(self): instrument.calibrate('goes13.2014.124.123422.BAND_01.nc') with nc.loader('goes13.2014.124.123422.BAND_01.nc') as root: var = lambda v: float(nc.getvar(root, v)[0]) self.assertAlmostEquals(var('space_measurement'), 29.0) self.assertAlmostEquals(var('postlaunch'), 1.293) self.assertAlmostEquals(var('counts_shift'), 32.0) # 0: radiance, 1: albedo self.assertAlmostEquals(var('prelaunch_0'), 0.6118) self.assertAlmostEquals(var('prelaunch_1'), 0.00116)
def obtain_data(self, y, x): now = localize(datetime.utcnow()) now = now.replace(year=2015, month=2, day=17) today_pattern = path + 'goes13.%Y.%j.*.BAND_01.nc' files = glob(now.strftime(today_pattern)) if not files: return [] with nc.loader(files) as root: time = nc.getvar(root, 'time')[:,0] data = nc.getvar(root, 'globalradiation')[:,y,x] return zip(time.tolist(), data.tolist())
def extend_cache(self, filenames): cached_files = glob.glob('%s/*.nc' % self.temporal_path) not_cached = filter(lambda f: self.get_cached_file(f) not in cached_files, filenames) if not_cached: loader = Loader(not_cached) new_files = pmap(self.get_cached_file, not_cached) with nc.loader(new_files, dimensions=DIMS) as cache: self.algorithm.update_temporalcache(loader, cache) loader.dump()
def __init__(self, algorithm): super(OutputCache, self).__init__(algorithm) self.output = Loader(pmap(self.get_output_file, self.filenames)) self.root = self.output.root with nc.loader(self.filenames, dimensions=DIMS) as images: map(algorithm.create_1px_dimensions, self.root.roots) self.root.getvar('time', source=images.getvar('time')) self.root.getvar('cloudindex', 'f4', source=images.getvar('data')) self.root.getvar('globalradiation', 'f4', source=images.getvar('data'))
def test_open_close_using_with(self): # check if open the pattern selection using using a package instance. with nc.loader('unittest0*.nc') as root: self.assertEquals(root.files, ['unittest0%i.nc' % i for i in range(5)]) self.assertEquals(root.pattern, 'unittest0*.nc') self.assertEquals(len(root.roots), 5) self.assertFalse(root.is_new) self.assertFalse(root.read_only) # check if close the package with all the files. with self.assertRaisesRegexp(RuntimeError, u'NetCDF: Not a valid ID'): nc.close(root)
def initialize_variables(self, filenames): self.path = '/'.join(filenames[0].split('/')[0:-1]) self.output_path = self.product with nc.loader(self.filenames, dimensions=self.tile_cut) as images: if self.output_path: self.initialize_path(filenames, images) else: data_shape = images.getvar('data').shape self.time = np.zeros(images.getvar('time').shape) self.ref_cloudindex = np.zeros(data_shape) self.cloudindex = self.ref_cloudindex self.ref_globalradiation = np.zeros(data_shape) self.globalradiation = self.ref_globalradiation
def only_incompleted(url, destfolder): dest = calculate_destiny(url, destfolder) completed = False if os.path.exists(dest): try: with nc.loader(dest) as root: nc.getvar(root, 'data') nc.getvar(root, 'lat') nc.getvar(root, 'lon') completed = True except (OSError, IOError, Exception): logger.error("The file %s was broken." % dest) return not completed
def __init__(self, filenames): # At first it should have: lat, lon, dem, linke self.root, is_new = nc.open('static.nc') if is_new: logging.info("This is the first execution from the deployment... ") with nc.loader(filenames[0]) as root_ref: self.lat = nc.getvar(root_ref, 'lat') self.lon = nc.getvar(root_ref, 'lon') nc.getvar(self.root, 'lat', source=self.lat) nc.getvar(self.root, 'lon', source=self.lon) self.project_dem() self.project_linke() nc.sync(self.root) self.root = nc.tailor(self.root, dimensions=DIMS)
def __init__(self, filenames, tile_cut={}): # At first it should have: lat, lon, dem, linke self.root, is_new = nc.open("static.nc") if is_new: logging.info("This is the first execution from the deployment... ") with nc.loader(filenames[0]) as root_ref: self.lat = nc.getvar(root_ref, "lat") self.lon = nc.getvar(root_ref, "lon") nc.getvar(self.root, "lat", source=self.lat) nc.getvar(self.root, "lon", source=self.lon) self.project_dem() self.project_linke() nc.sync(self.root) self.root = nc.tailor(self.root, dimensions=tile_cut)
def calibrate(filename): """ Append the calibration parameters as variables of the netcdf file. Keyword arguments: filename -- the name of a netcdf file. """ params = calibration_to(filename) with nc.loader(filename) as root: for key, value in params.items(): nc.getdim(root, 'xc_1', 1) nc.getdim(root, 'yc_1', 1) if isinstance(value, list): for i in range(len(value)): nc.getvar(root, '%s_%i' % (key, i), 'f4', ('time', 'yc_1', 'xc_1' ))[:] = value[i] else: nc.getvar(root, key, 'f4', ('time', 'yc_1', 'xc_1'))[:] = value
def test_load_calibration_using_netcdf_package(self): instrument.calibrate('goes13.2014.124.123422.BAND_01.nc') array_g = lambda x: [x for i in range(5)] with nc.loader(array_g('goes13.2014.124.123422.BAND_01.nc')) as root: var = lambda v: map(float, nc.getvar(root, v)[:]) for i in range(5): self.assertAlmostEquals(var('space_measurement')[i], array_g(29.0)[i]) self.assertAlmostEquals(var('postlaunch')[i], array_g(1.293)[i]) self.assertAlmostEquals(var('counts_shift')[i], array_g(32.0)[i]) # 0: radiance, 1: albedo self.assertAlmostEquals(var('prelaunch_0')[i], array_g(0.6118)[i]) self.assertAlmostEquals(var('prelaunch_1')[i], array_g(0.00116)[i])
def radiance(filename): prefix = short(filename, 0, 3) slot = int(round(decimalhour(to_datetime(filename))*2)) suffix = short(filename, 4, 6) output_filename = create_output_path(prefix, slot, suffix) root, is_new = nc.open(filename) radiation = nc.getvar(root, 'globalradiation') with nc.loader(output_filename) as radiance_root: dims_names = list(reversed(radiation.dimensions.keys())) dims_values = list(reversed(radiation.dimensions.values())) create_dims = (lambda name, dimension: radiance_root.create_dimension(name, len(dimension))) (map(lambda name, dimension: create_dims(name, dimension), dims_names, dims_values)) radiance = (nc.getvar(radiance_root, 'radiance', vtype='f4', dimensions=tuple(dims_names))) radiance[:] = radiation[:]*30.*60.*10**-6
def radiance(filename): prefix = short(filename, 0, 3) slot = int(round(decimalhour(to_datetime(filename)) * 2)) suffix = short(filename, 4, 6) output_filename = create_output_path(prefix, slot, suffix) root, is_new = nc.open(filename) radiation = nc.getvar(root, 'globalradiation') with nc.loader(output_filename) as radiance_root: dims_names = list(reversed(radiation.dimensions.keys())) dims_values = list(reversed(radiation.dimensions.values())) create_dims = (lambda name, dimension: radiance_root.create_dimension( name, len(dimension))) (map(lambda name, dimension: create_dims(name, dimension), dims_names, dims_values)) radiance = (nc.getvar(radiance_root, 'radiance', vtype='f4', dimensions=tuple(dims_names))) radiance[:] = radiation[:] * 30. * 60. * 10**-6
def setUp(self): with nc.loader('goes13.2014.124.123422.BAND_01.nc') as root: nc.getdim(root, 'xc', 200) nc.getdim(root, 'yc', 100) nc.getdim(root, 'time') nc.getvar(root, 'data', 'i4', ('time', 'yc', 'xc'), fill_value=1)
from flask_restful import Resource, Api from datetime import datetime import pytz from glob import glob from netcdf import netcdf as nc import numpy as np app = Flask(__name__) api = Api(app) log = open('queries.txt', 'a') path = './products/estimated/' gmt = pytz.timezone('GMT') local = pytz.timezone('America/Argentina/Buenos_Aires') localize = lambda dt: (gmt.localize(dt)).astimezone(local) with nc.loader('%sstatic.nc' % path) as root: lat_m, lon_m = root.getvar('lat')[:], root.getvar('lon')[:] class Irradiance(Resource): def translate(self, georef): try: lat, lon = map(float, georef.split(','))[:2] except Exception: lat, lon = None, None return lat, lon def register(self, lat, lon): now = localize(datetime.utcnow()) log.write('%s: (%.5f, %.5f)' % (str(now), lat, lon))
def time(self): with nc.loader(self.filenames_from_a_month_ago) as loader: self.times = nc.getvar(loader, 'time')[:] return self.times