def test_write_overwrite_true(self):
testFile = os.path.join(self.testDir, 'delete_me.nc')
CDRWriter.write(self.dataset, testFile, overwrite=True)
self.assertTrue(os.path.isfile(testFile))
CDRWriter.write(self.dataset, testFile, overwrite=True)
self.assertTrue(os.path.isfile(testFile))
def test_write_overwrite_false(self):
testFile = os.path.join(self.testDir, 'delete_me.nc')
CDRWriter.write(self.dataset, testFile)
self.assertTrue(os.path.isfile(testFile))
try:
CDRWriter.write(self.dataset, testFile, overwrite=False)
self.fail("IOError expected")
except IOError:
pass
def test_create_file_name_CDR(self):
start = datetime.datetime(2015, 8, 23, 14, 24, 52)
end = datetime.datetime(2015, 8, 23, 15, 25, 53)
self.assertEqual(
"FIDUCEO_CDR_AOT_MVIRI_MET7-0.00_20150823142452_20150823152553_L2_v02.3_fv2.0.0.nc",
CDRWriter.create_file_name_CDR("AOT", "MVIRI", "MET7-0.00", start,
end, "L2", "02.3"))
start = datetime.datetime(2018, 1, 9, 23, 24, 52)
end = datetime.datetime(2018, 1, 10, 1, 25, 53)
self.assertEqual(
"FIDUCEO_CDR_SST_AVHRR_NOAA18_20180109232452_20180110012553_ENSEMBLE_v03.3_fv2.0.0.nc",
CDRWriter.create_file_name_CDR("SST", "AVHRR", "NOAA18", start,
end, "ENSEMBLE", "03.3"))
def create_albedo_dataset(self):
dataset = CDRWriter.createTemplate("ALBEDO", PRODUCT_WIDTH,
PRODUCT_HEIGHT)
dataset.attrs["institution"] = "Brockmann Consult GmbH"
dataset.attrs["title"] = "integration test dataset"
dataset.attrs["source"] = "made-up data"
dataset.attrs["history"] = "initial version"
dataset.attrs["references"] = "no"
dataset.attrs["comment"] = "just testing"
dataset.attrs["auxiliary_data"] = "nope"
dataset.attrs["configuration"] = "test"
dataset.attrs["time_coverage_start"] = "now"
dataset.attrs["time_coverage_end"] = "later"
dataset.attrs["time_coverage_duration"] = "P8S"
dataset.attrs["time_coverage_resolution"] = "P1S"
for x in range(0, PRODUCT_WIDTH):
dataset["quality_pixel_bitmask"].data[:, x] = np.ones(
(PRODUCT_HEIGHT), np.int8) * x + 7
dataset["surface_albedo"].data[:, x] = np.ones(
(PRODUCT_HEIGHT), np.int8) * x * 0.9
dataset["time"].data[:] = np.ones((PRODUCT_HEIGHT), np.int32) * 8
return dataset
def testTemplateEasy_UTH(self):
ds = CDRWriter.createTemplate('UTH', 360, 120)
self.assertIsNotNone(ds)
Assertions.assert_global_attributes(self, ds.attrs)
Assertions.assert_cdr_global_attributes(self, ds.attrs)
Assertions.assert_gridded_global_attributes(self, ds.attrs)
def test_write_albedo(self):
template = self.create_albedo_dataset()
start = datetime.datetime(2016, 11, 22, 13, 24, 52)
end = datetime.datetime(2016, 11, 22, 14, 25, 53)
file_name = CDRWriter.create_file_name_CDR("ALBEDO", "MVIRI",
"METEOSAT-7", start, end,
"L2", "1.1")
self.target_path = os.path.join(self.temp_dir, file_name)
start_time = datetime.datetime.now()
CDRWriter.write(template, self.target_path)
end_time = datetime.datetime.now()
elapsed_time = end_time - start_time
print("ALBEDO write time: " + str(elapsed_time.seconds) + "." +
str(round(elapsed_time.microseconds / 1000)))
self.assertTrue(os.path.isfile(self.target_path))
target_data = xr.open_dataset(self.target_path)
try:
variable = target_data["quality_pixel_bitmask"]
self.assertEqual(8, variable.data[1, 1])
self.assertEqual(EXPECTED_CHUNKING,
variable.encoding["chunksizes"])
variable = target_data["time"]
self.assertEqual(8, variable.data[2])
variable = target_data["surface_albedo"]
self.assertAlmostEqual(2.7, variable.data[3, 3], 7)
self.assertEqual(EXPECTED_CHUNKING,
variable.encoding["chunksizes"])
finally:
target_data.close()
def testTemplateEasy_ALBEDO(self):
ds = CDRWriter.createTemplate('ALBEDO', 5000, 5000)
self.assertIsNotNone(ds)
Assertions.assert_global_attributes(self, ds.attrs)
Assertions.assert_cdr_global_attributes(self, ds.attrs)
def testTemplateEasy_SST_ENSEMBLE(self):
ds = CDRWriter.createTemplate('SST_ENSEMBLE', 409, 11732, 20)
self.assertIsNotNone(ds)
Assertions.assert_global_attributes(self, ds.attrs)
Assertions.assert_cdr_global_attributes(self, ds.attrs)
def testTemplateEasy_AOT(self):
ds = CDRWriter.createTemplate('AOT', 409, 12941)
self.assertIsNotNone(ds)
Assertions.assert_global_attributes(self, ds.attrs)
Assertions.assert_cdr_global_attributes(self, ds.attrs)
def toNetCDF(self, CDR_path, version, comment_on_version, overwrite=False):
""" Saves the attributes of an averaged CDR created by
AveragedCDRFromCDRs to a NetCDF file using the CDRWriter by Tom Block.
Parameters:
CDR_path (str): path to directory where CDRs are saved
version (str): Version of this CDR (for CDR filename)
comment_on_version (str): Comment on this version of the CDR.
"""
u_types = ['structured', 'common', 'independent']
branches = ['ascend', 'descend']
u_quantities = ['uth', 'BT', 'BT_full']
DATE_PATTERN = "%Y%m%d%H%M%S"
numlons = len(self.lon)
numlats = len(self.lat)
start_time = self.time_coverage_start
end_time = self.time_coverage_end
start_time_for_filename = datetime(start_time.year, start_time.month,
1, 0, 0, 0)
end_time_for_filename = datetime(
start_time.year, start_time.month,
monthrange(start_time.year, start_time.month)[1], 23, 59, 59)
CDR_filename = CDRWriter.create_file_name_CDR(
'UTH',
sensor=self.instrument,
platform=self.satellite,
start=start_time_for_filename,
end=end_time_for_filename,
type='L3',
version=version)
# create directory for CDRs if it does not exist
if not exists(CDR_path):
makedirs(CDR_path)
ds = CDRWriter.createTemplate('UTH', numlons, numlats)
# Variables:
simple_vars = ['lat', 'lon', 'lat_bnds', 'lon_bnds']
branch_vars = [
'observation_count', 'observation_count_all', 'overpass_count',
'uth', 'BT', 'BT_full', 'uth_inhomogeneity', 'BT_inhomogeneity',
'BT_full_inhomogeneity', 'time_ranges'
]
simple_attrs = [
'source', 'time_coverage_duration', 'geospatial_lon_resolution',
'geospatial_lat_resolution'
]
for var in simple_vars:
ds.variables[var].data = getattr(self, var)
for attr in simple_attrs:
ds.attrs[attr] = getattr(self, attr)
for var in branch_vars:
for b in branches:
ds.variables['{}_{}'.format(var, b)].data =\
getattr(self, '{}'.format(var))['{}ing'.format(b)]
for q in u_quantities:
for t in u_types:
for b in branches:
ds.variables['u_{}_{}_{}'.format(t, q, b)].data =\
getattr(self, 'u_{}'.format(q))[t]['{}ing'.format(b)]
# Attributes:
ds.attrs['institution'] = 'University of Hamburg'
ds.attrs['title'] = 'Upper Tropospheric Humidity (UTH)'
ds.attrs['time_coverage_start'] = start_time.strftime(DATE_PATTERN)
ds.attrs['time_coverage_end'] = end_time.strftime(DATE_PATTERN)
ds.attrs['time_coverage_resolution'] = pd.Timedelta(
1, unit='M').isoformat()
ds.attrs['geospatial_lat_units'] = 'deg'
ds.attrs['geospatial_lon_units'] = 'deg'
ds.attrs['history'] = 'Created on {}.'.format(
datetime.now().strftime('%Y-%m-%d %H:%M'))
ds.attrs['references'] = ''
ds.attrs['comment'] = comment_on_version
ds.attrs['auxiliary_data'] = ''
ds.attrs['configuration'] = ''
# Write to NetCDF (with CDRWriter)
CDRWriter.write(ds, join(CDR_path, CDR_filename), overwrite=overwrite)
print('CDR {} for {} on {} saved as {}'.format(
start_time.strftime('%Y/%m'), self.instrument, self.satellite,
CDR_filename))
def generate_products(ctx, period_index, logger):
# numpy.seterr(divide='ignore', invalid='ignore')
if not os.path.isdir(ctx.dir_products): os.makedirs(ctx.dir_products)
start_date_acc_period = ctx.v_acc_periods[period_index].start_date
end_date_acc_period = start_date_acc_period + timedelta(minutes=23 * 60 +
59)
st_date = start_date_acc_period.strftime('%Y-%m-%d')
st_time = start_date_acc_period.strftime('%H-%M')
fiduceo_product_albedo = CDRWriter.createTemplate('ALBEDO', 5000, 5000)
fiduceo_product_aot = CDRWriter.createTemplate('AOT', 5000, 5000)
for tile in ctx.v_tiles:
start_line = tile.first_line
end_line = tile.last_line
start_column = tile.first_column
end_column = tile.last_column
root_dir = ctx.rootdir_tile_maker[tile.machine]
dir_input_tiles = os.path.join(root_dir, ctx.subdir_input_tiles)
dir_solution_tiles = os.path.join(root_dir, ctx.subdir_solution_tiles)
file_static_data_tile = os.path.join(
dir_input_tiles,
get_tile_filename("STATIC", ctx.satellite, ctx.radiometer,
ctx.projection,
str(ctx.ssp).zfill(4), ctx.platform, "", "",
tile.id))
file_solution_tile = os.path.join(
dir_solution_tiles,
get_tile_filename("SOLUTION", ctx.satellite, ctx.radiometer,
ctx.projection,
str(ctx.ssp).zfill(4), ctx.platform, st_date,
st_time, tile.id))
if os.path.isfile(file_static_data_tile) and os.path.isfile(
file_solution_tile):
# ============= Extract data from static tile =============
static_data_tile = StaticDataTile(file_static_data_tile)
fiduceo_product_aot.latitude[start_line:end_line+1, start_column:end_column+1] = \
static_data_tile.latitude[:, :]
fiduceo_product_aot.longitude[start_line:end_line + 1, start_column:end_column + 1] = \
static_data_tile.longitude[:, :]
# ============= AOT CDR =============
solution_tile = SolutionTile(file_solution_tile)
fiduceo_product_aot.attrs['source'] = solution_tile.source
fiduceo_product_aot.attrs[
'auxiliary_data'] = solution_tile.auxiliary_data
if solution_tile.acquisition_time.shape[0] == 1:
# time has to be expressed in seconds from 1st January 1970
t0 = datetime(1970, 1, 1)
st_date_noon = start_date_acc_period + timedelta(hours=12)
delta = st_date_noon - t0
fiduceo_product_aot.time[start_line:end_line +
1, ] = t0 + timedelta(
seconds=delta.total_seconds())
fiduceo_product_albedo.time[start_line:end_line +
1, ] = t0 + timedelta(
seconds=delta.total_seconds())
fiduceo_product_aot.aot[start_line:end_line + 1,
start_column:end_column +
1] = numpy.where(
solution_tile.total_tau[0, 0, :, :]
< 0.01, numpy.nan,
solution_tile.total_tau[0,
0, :, :])
aot = numpy.where(solution_tile.total_tau[0, 0, :, :] < 0.01,
numpy.nan, solution_tile.total_tau[0,
0, :, :])
sigma_aot = numpy.where(
solution_tile.total_tau[0, 0, :, :] < 0.01, numpy.nan,
solution_tile.sigma_total_tau[0, 0, :, :])
fiduceo_product_aot.u_independent_aot[start_line:end_line + 1,
start_column:end_column +
1] = sigma_aot / aot
# fiduceo_product_aot.u_independent_aot[start_line:end_line + 1, start_column:end_column + 1] = numpy.where(solution_tile.total_tau[0, 0, :, :] < 0.01, numpy.nan, numpy.divide(solution_tile.sigma_total_tau[0, 0, :, :], solution_tile.total_tau[0, 0, :, :]))
not_enough_obs = solution_tile.error_flag[:, :] == 202
quality_cut_aot = solution_tile.total_tau[0, 0, :, :] < 0.01
mask_aot = numpy.logical_or(not_enough_obs, quality_cut_aot)
fiduceo_product_aot.quality_pixel_bitmask[
start_line:end_line + 1,
start_column:end_column + 1] = numpy.where(mask_aot, 1, 0)
# ============= ALBEDO CDR =============
fiduceo_product_albedo.attrs['source'] = solution_tile.source
fiduceo_product_albedo.attrs[
'auxiliary_data'] = solution_tile.auxiliary_data
quality_cut_albedo = solution_tile.BHR[0, :, :] < 0.001
land_cover = static_data_tile.land_cover[:, :] == 2.0
mask_albedo = numpy.logical_or(quality_cut_albedo, land_cover)
fiduceo_product_albedo.surface_albedo[
start_line:end_line + 1,
start_column:end_column + 1] = numpy.where(
mask_albedo, numpy.nan, solution_tile.BHR[0, :, :])
sigma_BHR = numpy.where(mask_albedo, numpy.nan,
solution_tile.sigma_BHR[0, :, :])
BHR = numpy.where(mask_albedo, numpy.nan,
solution_tile.BHR[0, :, :])
fiduceo_product_albedo.u_independent_surface_albedo[
start_line:end_line + 1,
start_column:end_column + 1] = sigma_BHR / BHR
fiduceo_product_albedo.quality_pixel_bitmask[
start_line:end_line + 1,
start_column:end_column + 1] = numpy.where(
mask_albedo, 1, 0)
else:
raise SystemExit(
'No Solution Tile and Static Tile associated to {}'.format(
tile.id))
fiduceo_product_aot.attrs['institution'] = 'Rayference SPRL'
fiduceo_product_aot.attrs['title'] = 'MVIRI Aerosol FIDUCEO CDR'
fiduceo_product_aot.attrs['history'] = "Created: " + datetime.now(
).strftime("%a %b %H:%M:%S %Y")
fiduceo_product_aot.attrs['references'] = ''
fiduceo_product_aot.attrs['comment'] = ''
fiduceo_product_aot.attrs['configuration'] = ''
fiduceo_product_aot.attrs[
'time_coverage_start'] = start_date_acc_period.strftime(
"%Y%m%dT%H%M%SZ")
fiduceo_product_aot.attrs[
'time_coverage_end'] = end_date_acc_period.strftime("%Y%m%dT%H%M%SZ")
fiduceo_product_aot.attrs['time_coverage_duration'] = 'P1D'
fiduceo_product_aot.attrs['time_coverage_resolution'] = 'P1D'
platform = ctx.platform + '-' + ctx.ssp_cdr
filename_aot_product = CDRWriter.create_file_name_CDR(
'AOT', ctx.radiometer, platform, start_date_acc_period,
end_date_acc_period, 'L2', str(ctx.gedap_version))
CDRWriter.write(fiduceo_product_aot,
os.path.join(ctx.dir_products, filename_aot_product))
fiduceo_product_albedo.attrs['institution'] = 'Rayference SPRL'
fiduceo_product_albedo.attrs['title'] = 'MVIRI Albedo FIDUCEO CDR'
fiduceo_product_albedo.attrs['history'] = 'Created: ' + datetime.now(
).strftime("%a %b %H:%M:%S %Y")
fiduceo_product_albedo.attrs['references'] = ''
fiduceo_product_albedo.attrs['comment'] = ''
fiduceo_product_albedo.attrs['configuration'] = ''
fiduceo_product_albedo.attrs[
'time_coverage_start'] = start_date_acc_period.strftime(
"%Y%m%dT%H%M%SZ")
fiduceo_product_albedo.attrs[
'time_coverage_end'] = end_date_acc_period.strftime("%Y%m%dT%H%M%SZ")
fiduceo_product_albedo.attrs['time_coverage_duration'] = 'P1D'
fiduceo_product_albedo.attrs['time_coverage_resolution'] = 'P1D'
filename_albedo_product = CDRWriter.create_file_name_CDR(
'ALBEDO', ctx.radiometer, platform, start_date_acc_period,
end_date_acc_period, 'L2', str(ctx.gedap_version))
CDRWriter.write(fiduceo_product_albedo,
os.path.join(ctx.dir_products, filename_albedo_product))
