def Test_get_mapset(self):
mapset = querydb.get_mapset(mapsetcode='SPOTV-Africa-1km')
logger.info("Mapset: %s", mapset.pixel_shift_lat)
mapset_dict = functions.row2dict(mapset)
self.assertEqual(1, 1)
def test_product_data_management(self):
def row2dict(row):
d = {}
for column_name in row.c.keys(): # all_cols:
d[column_name] = str(getattr(row, column_name))
return d
# Select prod/vers
productcode = 'vgt-ndvi'
version = 'proba-v2.1'
product = Product(product_code=productcode, version=version)
# does the product have mapsets AND subproducts?
all_prod_mapsets = product.mapsets
all_prod_subproducts = product.subproducts
if all_prod_mapsets.__len__() > 0 and all_prod_subproducts.__len__() > 0:
for mapset in all_prod_mapsets:
mapset_info = querydb.get_mapset(mapsetcode=mapset, allrecs=False, echo=False)
mapset_dict = row2dict(mapset_info)
mapset_dict['mapsetdatasets'] = []
all_mapset_datasets = product.get_subproducts(mapset=mapset)
for subproductcode in all_mapset_datasets:
dataset_info = querydb.get_subproduct(productcode=productcode,
version=version,
subproductcode=subproductcode,
echo=False)
dataset_dict = row2dict(dataset_info)
dataset = product.get_dataset(mapset=mapset, sub_product_code=subproductcode)
completeness = dataset.get_dataset_normalized_info()
dataset_dict['datasetcompleteness'] = completeness
mapset_dict['mapsetdatasets'].append(dataset_dict)
def test_product_data_management(self):
def row2dict(row):
d = {}
for column_name in row.c.keys(): # all_cols:
d[column_name] = str(getattr(row, column_name))
return d
# Select prod/vers
productcode = 'vgt-ndvi'
version = 'proba-v2.1'
product = Product(product_code=productcode, version=version)
# does the product have mapsets AND subproducts?
all_prod_mapsets = product.mapsets
all_prod_subproducts = product.subproducts
if all_prod_mapsets.__len__() > 0 and all_prod_subproducts.__len__(
) > 0:
for mapset in all_prod_mapsets:
mapset_info = querydb.get_mapset(mapsetcode=mapset,
allrecs=False)
mapset_dict = row2dict(mapset_info)
mapset_dict['mapsetdatasets'] = []
all_mapset_datasets = product.get_subproducts(mapset=mapset)
for subproductcode in all_mapset_datasets:
dataset_info = querydb.get_subproduct(
productcode=productcode,
version=version,
subproductcode=subproductcode)
dataset_dict = row2dict(dataset_info)
dataset = product.get_dataset(
mapset=mapset, sub_product_code=subproductcode)
completeness = dataset.get_dataset_normalized_info()
dataset_dict['datasetcompleteness'] = completeness
mapset_dict['mapsetdatasets'].append(dataset_dict)
def test_all_products_to_json(self):
def row2dict(row):
d = {}
for column_name in list(row.c.keys()): # all_cols:
d[column_name] = str(getattr(row, column_name))
return d
# get full distinct list of products (native only)
db_products = querydb.get_products()
try:
db_products.__len__()
except AttributeError:
db_products = querydb.get_product_native(allrecs=True)
self.assertTrue(db_products.__len__() > 0)
products_dict_all = []
# loop the products list
for product in db_products:
if python_version == 2:
prod_dict = row2dict(product)
if python_version == 3:
prod_dict = product
productcode = prod_dict['productcode']
version = prod_dict['version']
p = Product(product_code=productcode, version=version)
# does the product have mapsets AND subproducts?
all_prod_mapsets = p.mapsets
all_prod_subproducts = p.subproducts
if all_prod_mapsets.__len__() > 0 and all_prod_subproducts.__len__() > 0:
prod_dict['productmapsets'] = []
for mapset in all_prod_mapsets:
mapset_info = querydb.get_mapset(mapsetcode=mapset, allrecs=False)
mapset_dict = row2dict(mapset_info)
mapset_dict['mapsetdatasets'] = []
all_mapset_datasets = p.get_subproducts(mapset=mapset)
for subproductcode in all_mapset_datasets:
dataset_info = querydb.get_subproduct(productcode=productcode,
version=version,
subproductcode=subproductcode)
dataset_dict = row2dict(dataset_info)
dataset = p.get_dataset(mapset=mapset, sub_product_code=subproductcode)
completeness = dataset.get_dataset_normalized_info()
dataset_dict['datasetcompleteness'] = completeness
mapset_dict['mapsetdatasets'].append(dataset_dict)
prod_dict['productmapsets'].append(mapset_dict)
products_dict_all.append(prod_dict)
# See ES2-596
self.assertEqual(len(db_products), len(products_dict_all))
def assigndb(self, mapsetcode):
mapset = querydb.get_mapset(mapsetcode, echo=False)
spatial_ref_wkt = mapset.srs_wkt
geo_transform = [
mapset.upper_left_long, mapset.pixel_shift_long,
mapset.rotation_factor_long, mapset.upper_left_lat,
mapset.rotation_factor_lat, mapset.pixel_shift_lat
]
self.spatial_ref.ImportFromWkt(spatial_ref_wkt)
self.geo_transform = geo_transform
self.size_x = int(mapset.pixel_size_x)
self.size_y = int(mapset.pixel_size_y)
self.short_name = mapset.mapsetcode
def assigndb(self, mapsetcode):
mapset = querydb.get_mapset(mapsetcode, echo=False)
spatial_ref_wkt = mapset.srs_wkt
geo_transform = [mapset.upper_left_long,
mapset.pixel_shift_long,
mapset.rotation_factor_long,
mapset.upper_left_lat,
mapset.rotation_factor_lat,
mapset.pixel_shift_lat]
self.spatial_ref.ImportFromWkt(spatial_ref_wkt)
self.geo_transform = geo_transform
self.size_x = int(mapset.pixel_size_x)
self.size_y = int(mapset.pixel_size_y)
self.short_name = mapset.mapsetcode
def Test_get_processingchains_input_products(self):
import json
processingchain_products = querydb.get_processingchains_input_products(
)
if processingchain_products.__len__() > 0:
products_dict_all = []
# loop the products list
for input_product in processingchain_products:
process_id = input_product.process_id
output_mapsetcode = input_product.output_mapsetcode
prod_dict = functions.row2dict(input_product)
# prod_dict = input_product
# del prod_dict['_labels']
prod_dict['productmapsets'] = []
mapset_info = querydb.get_mapset(mapsetcode=output_mapsetcode)
mapset_dict = functions.row2dict(mapset_info)
mapset_dict['mapsetoutputproducts'] = []
output_products = querydb.get_processingchain_output_products(
process_id)
for outputproduct in output_products:
outputproduct_dict = functions.row2dict(outputproduct)
# outputproduct_dict = outputproduct
# del outputproduct_dict['_labels']
mapset_dict['mapsetoutputproducts'].append(
outputproduct_dict)
prod_dict['productmapsets'].append(mapset_dict)
products_dict_all.append(prod_dict)
prod_json = json.dumps(products_dict_all,
ensure_ascii=False,
sort_keys=True,
indent=4,
separators=(', ', ': '))
# logger.info("Processing chains: %s", processingchain_products)
# for row in processingchain_products:
# logger.info("row.dict: %s", row.__dict__)
# logger.info("row.process_id: %s", row.process_id)
# logger.info("row.output_mapsetcode: %s", row.output_mapsetcode)
# logger.info("row.mapsetcode: %s", row.mapsetcode)
# print row.process_id
# print row.output_mapsetcode
# print row.mapsetcode
self.assertEqual(1, 1)
def test_all_products_to_json(self):
def row2dict(row):
d = {}
for column_name in row.c.keys(): # all_cols:
d[column_name] = str(getattr(row, column_name))
return d
# get full distinct list of products (native only)
db_products = querydb.get_products(echo=False)
try:
db_products.__len__()
except AttributeError:
db_products = querydb.get_product_native(allrecs=True, echo=False)
self.assertTrue(db_products.__len__() > 0)
products_dict_all = []
# loop the products list
for product in db_products:
prod_dict = row2dict(product)
productcode = prod_dict['productcode']
version = prod_dict['version']
p = Product(product_code=productcode, version=version)
# does the product have mapsets AND subproducts?
all_prod_mapsets = p.mapsets
all_prod_subproducts = p.subproducts
if all_prod_mapsets.__len__() > 0 and all_prod_subproducts.__len__() > 0:
prod_dict['productmapsets'] = []
for mapset in all_prod_mapsets:
mapset_info = querydb.get_mapset(mapsetcode=mapset, allrecs=False, echo=False)
mapset_dict = row2dict(mapset_info)
mapset_dict['mapsetdatasets'] = []
all_mapset_datasets = p.get_subproducts(mapset=mapset)
for subproductcode in all_mapset_datasets:
dataset_info = querydb.get_subproduct(productcode=productcode,
version=version,
subproductcode=subproductcode,
echo=False)
dataset_dict = row2dict(dataset_info)
dataset = p.get_dataset(mapset=mapset, sub_product_code=subproductcode)
completeness = dataset.get_dataset_normalized_info()
dataset_dict['datasetcompleteness'] = completeness
mapset_dict['mapsetdatasets'].append(dataset_dict)
prod_dict['productmapsets'].append(mapset_dict)
products_dict_all.append(prod_dict)
self.assertEquals(len(db_products), 31)
def Test_get_processingchains_input_products(self):
import json
processingchain_products = querydb.get_processingchains_input_products()
if processingchain_products.__len__() > 0:
products_dict_all = []
# loop the products list
for input_product in processingchain_products:
process_id = input_product.process_id
output_mapsetcode = input_product.output_mapsetcode
prod_dict = functions.row2dict(input_product)
# prod_dict = input_product
# del prod_dict['_labels']
prod_dict['productmapsets'] = []
mapset_info = querydb.get_mapset(mapsetcode=output_mapsetcode)
mapset_dict = functions.row2dict(mapset_info)
mapset_dict['mapsetoutputproducts'] = []
output_products = querydb.get_processingchain_output_products(process_id)
for outputproduct in output_products:
outputproduct_dict = functions.row2dict(outputproduct)
# outputproduct_dict = outputproduct
# del outputproduct_dict['_labels']
mapset_dict['mapsetoutputproducts'].append(outputproduct_dict)
prod_dict['productmapsets'].append(mapset_dict)
products_dict_all.append(prod_dict)
prod_json = json.dumps(products_dict_all,
ensure_ascii=False,
sort_keys=True,
indent=4,
separators=(', ', ': '))
# logger.info("Processing chains: %s", processingchain_products)
# for row in processingchain_products:
# logger.info("row.dict: %s", row.__dict__)
# logger.info("row.process_id: %s", row.process_id)
# logger.info("row.output_mapsetcode: %s", row.output_mapsetcode)
# logger.info("row.mapsetcode: %s", row.mapsetcode)
# print row.process_id
# print row.output_mapsetcode
# print row.mapsetcode
self.assertEqual(1, 1)
def assigndb(self, mapsetcode):
mapset = querydb.get_mapset(mapsetcode)
self.spatial_ref_wkt = mapset.srs_wkt
geo_transform = [
float(mapset.upper_left_long),
float(mapset.pixel_shift_long),
float(mapset.rotation_factor_long),
float(mapset.upper_left_lat),
float(mapset.rotation_factor_lat),
float(mapset.pixel_shift_lat)
]
self.spatial_ref.ImportFromWkt(self.spatial_ref_wkt)
self.geo_transform = geo_transform
self.size_x = int(mapset.pixel_size_x)
self.size_y = int(mapset.pixel_size_y)
self.short_name = mapset.mapsetcode
self.bbox = self.get_bbox(mapset)
def Test_get_mapset(self):
mapset = querydb.get_mapset(mapsetcode='WGS84_Africa_1km')
logger.info("Mapset: %s", mapset)
self.assertEqual(1, 1)
def getTimeseries(productcode, subproductcode, version, mapsetcode, geom,
start_date, end_date, aggregate):
# Extract timeseries from a list of files and return as JSON object
# It applies to a single dataset (prod/sprod/version/mapset) and between 2 dates
# Several types of aggregation foreseen:
#
# mean : Sum(Xi)/N(Xi) -> min/max not considered e.g. Rain
# cumulate: Sum(Xi) -> min/max not considered e.g. Fire
#
# count: N(Xi where min < Xi < max) e.g. Vegetation anomalies
# surface: count * PixelArea e.g. Water Bodies
# percent: count/Ntot e.g. Vegetation anomalies
# precip: compute the precipitation volume in m3*1E6 Rain (only)
#
# History: 1.0 : Initial release - since 2.0.1 -> now renamed '_green' from greenwich package
# 1.1 : Since Feb. 2017, it is based on a different approach (gdal.RasterizeLayer instead of greenwich)
# in order to solve the issue with MULTIPOLYGON
#
ogr.UseExceptions()
# Get Mapset Info
mapset_info = querydb.get_mapset(mapsetcode=mapsetcode)
# Prepare for computing conversion to area: the pixel size at Lat=0 is computed
# The correction to the actual latitude (on AVERAGE value - will be computed below)
const_d2km = 12364.35
area_km_equator = abs(float(mapset_info.pixel_shift_lat)) * abs(
float(mapset_info.pixel_shift_long)) * const_d2km
# Get Product Info
product_info = querydb.get_product_out_info(productcode=productcode,
subproductcode=subproductcode,
version=version)
if product_info.__len__() > 0:
# Get info from product_info
scale_factor = 0
scale_offset = 0
nodata = 0
date_format = ''
for row in product_info:
scale_factor = row.scale_factor
scale_offset = row.scale_offset
nodata = row.nodata
date_format = row.date_format
date_type = row.data_type_id
# Create an output/temp shapefile, for managing the output layer (really mandatory ?? Can be simplified ???)
try:
tmpdir = tempfile.mkdtemp(prefix=__name__,
suffix='_getTimeseries',
dir=es_constants.base_tmp_dir)
except:
logger.error('Cannot create temporary dir ' +
es_constants.base_tmp_dir + '. Exit')
raise NameError('Error in creating tmpdir')
out_shape = tmpdir + os.path.sep + "output_shape.shp"
outDriver = ogr.GetDriverByName('ESRI Shapefile')
# Create the output shapefile
outDataSource = outDriver.CreateDataSource(out_shape)
dest_srs = ogr.osr.SpatialReference()
dest_srs.ImportFromEPSG(4326)
outLayer = outDataSource.CreateLayer("Layer", dest_srs)
# outLayer = outDataSource.CreateLayer("Layer")
idField = ogr.FieldDefn("id", ogr.OFTInteger)
outLayer.CreateField(idField)
featureDefn = outLayer.GetLayerDefn()
feature = ogr.Feature(featureDefn)
feature.SetGeometry(geom)
# area = geom.GetArea()
feature.SetField("id", 1)
outLayer.CreateFeature(feature)
feature = None
[list_files,
dates_list] = getFilesList(productcode, subproductcode, version,
mapsetcode, date_format, start_date,
end_date)
# Built a dictionary with filenames/dates
dates_to_files_dict = dict(list(zip(dates_list, list_files)))
# Generate unique list of files
unique_list = set(list_files)
uniqueFilesValues = []
geo_mask_created = False
for infile in unique_list:
single_result = {
'filename': '',
'meanvalue_noscaling': nodata,
'meanvalue': None
}
if infile.strip() != '' and os.path.isfile(infile):
# try:
# Open input file
orig_ds = gdal.Open(infile, gdal.GA_ReadOnly)
orig_cs = osr.SpatialReference()
orig_cs.ImportFromWkt(orig_ds.GetProjectionRef())
orig_geoT = orig_ds.GetGeoTransform()
x_origin = orig_geoT[0]
y_origin = orig_geoT[3]
pixel_size_x = orig_geoT[1]
pixel_size_y = -orig_geoT[5]
in_data_type_gdal = conv_data_type_to_gdal(date_type)
# Create a mask from the geometry, with the same georef as the input file[s]
if not geo_mask_created:
# Read polygon extent and round to raster resolution
x_min, x_max, y_min, y_max = outLayer.GetExtent()
x_min_round = int(old_div(
(x_min - x_origin),
pixel_size_x)) * pixel_size_x + x_origin
x_max_round = (
int(old_div(
(x_max - x_origin),
(pixel_size_x))) + 1) * pixel_size_x + x_origin
y_min_round = (
int(old_div(
(y_min - y_origin),
(pixel_size_y))) - 1) * pixel_size_y + y_origin
y_max_round = int(
old_div((y_max - y_origin),
(pixel_size_y))) * pixel_size_y + y_origin
#
# # Create the destination data source
x_res = int(
round(
old_div((x_max_round - x_min_round),
pixel_size_x)))
y_res = int(
round(
old_div((y_max_round - y_min_round),
pixel_size_y)))
#
# # Create mask in memory
mem_driver = gdal.GetDriverByName('MEM')
mem_ds = mem_driver.Create('', x_res, y_res, 1,
in_data_type_gdal)
mask_geoT = [
x_min_round, pixel_size_x, 0, y_max_round, 0,
-pixel_size_y
]
mem_ds.SetGeoTransform(mask_geoT)
mem_ds.SetProjection(orig_cs.ExportToWkt())
#
# # Create a Layer with '1' for the pixels to be selected
gdal.RasterizeLayer(mem_ds, [1], outLayer, burn_values=[1])
# gdal.RasterizeLayer(mem_ds, [1], outLayer, None, None, [1])
# Read the polygon-mask
band = mem_ds.GetRasterBand(1)
geo_values = mem_ds.ReadAsArray()
# Create a mask from geo_values (mask-out the '0's)
geo_mask = ma.make_mask(geo_values == 0)
geo_mask_created = True
#
# # Clean/Close objects
mem_ds = None
mem_driver = None
outDriver = None
outLayer = None
# Read data from input file
x_offset = int(old_div((x_min - x_origin), pixel_size_x))
y_offset = int(old_div((y_origin - y_max), pixel_size_y))
band_in = orig_ds.GetRasterBand(1)
data = band_in.ReadAsArray(x_offset, y_offset, x_res, y_res)
# Catch the Error ES2-105 (polygon not included in Mapset)
if data is None:
logger.error(
'ERROR: polygon extends out of file mapset for file: %s'
% infile)
return []
# Create a masked array from the data (considering Nodata)
masked_data = ma.masked_equal(data, nodata)
# Apply on top of it the geo mask
mxnodata = ma.masked_where(geo_mask, masked_data)
# Test ONLY
# write_ds_to_geotiff(mem_ds, '/data/processing/exchange/Tests/mem_ds.tif')
if aggregate['aggregation_type'] == 'count' or aggregate[
'aggregation_type'] == 'percent' or aggregate[
'aggregation_type'] == 'surface' or aggregate[
'aggregation_type'] == 'precip':
if mxnodata.count() == 0:
meanResult = None
else:
mxrange = mxnodata
min_val = aggregate['aggregation_min']
max_val = aggregate['aggregation_max']
if min_val is not None:
min_val_scaled = old_div((min_val - scale_offset),
scale_factor)
mxrange = ma.masked_less(mxnodata, min_val_scaled)
# See ES2-271
if max_val is not None:
# Scale threshold from physical to digital value
max_val_scaled = old_div(
(max_val - scale_offset), scale_factor)
mxrange = ma.masked_greater(
mxrange, max_val_scaled)
elif max_val is not None:
# Scale threshold from physical to digital value
max_val_scaled = old_div((max_val - scale_offset),
scale_factor)
mxrange = ma.masked_greater(
mxnodata, max_val_scaled)
if aggregate['aggregation_type'] == 'percent':
# 'percent'
meanResult = float(mxrange.count()) / float(
mxnodata.count()) * 100
elif aggregate['aggregation_type'] == 'surface':
# 'surface'
# Estimate 'average' Latitude
y_avg = (y_min + y_max) / 2.0
pixelAvgArea = area_km_equator * math.cos(
old_div(y_avg, 180) * math.pi)
meanResult = float(mxrange.count()) * pixelAvgArea
elif aggregate['aggregation_type'] == 'precip':
# 'surface'
# Estimate 'average' Latitude
y_avg = (y_min + y_max) / 2.0
pixelAvgArea = area_km_equator * math.cos(
old_div(y_avg, 180) * math.pi)
n_pixels = mxnodata.count()
avg_precip = mxnodata.mean()
# Result is in km * km * mmm i.e. 1E3 m*m*m -> we divide by 1E3 to get 1E6 m*m*m
meanResult = float(
n_pixels) * pixelAvgArea * avg_precip * 0.001
else:
# 'count'
meanResult = float(mxrange.count())
# Both results are equal
finalvalue = meanResult
else: # if aggregate['type'] == 'mean' or if aggregate['type'] == 'cumulate':
if mxnodata.count() == 0:
finalvalue = None
meanResult = None
else:
if aggregate['aggregation_type'] == 'mean':
# 'mean'
meanResult = mxnodata.mean()
else:
# 'cumulate'
meanResult = mxnodata.sum()
finalvalue = (meanResult * scale_factor + scale_offset)
# Assign results
single_result['filename'] = infile
single_result['meanvalue_noscaling'] = meanResult
single_result['meanvalue'] = finalvalue
else:
logger.debug('ERROR: raster file does not exist - %s' % infile)
uniqueFilesValues.append(single_result)
# Define a dictionary to associate filenames/values
files_to_values_dict = dict(
(x['filename'], x['meanvalue']) for x in uniqueFilesValues)
# Prepare array for result
resultDatesValues = []
# Returns a list of 'filenames', 'dates', 'values'
for mydate in dates_list:
my_result = {'date': datetime.date.today(), 'meanvalue': nodata}
# Assign the date
my_result['date'] = mydate
# Assign the filename
my_filename = dates_to_files_dict[mydate]
# Map from array of Values
my_result['meanvalue'] = files_to_values_dict[my_filename]
# Map from array of dates
resultDatesValues.append(my_result)
try:
shutil.rmtree(tmpdir)
except:
logger.debug('ERROR: Error in deleting tmpdir. Exit')
# Return result
return resultDatesValues
else:
logger.debug(
'ERROR: product not registered in the products table! - %s %s %s' %
(productcode, subproductcode, version))
return []
def getTimeseries_green(productcode, subproductcode, version, mapsetcode, wkt,
start_date, end_date, aggregate):
# Extract timeseries from a list of files and return as JSON object
# It applies to a single dataset (prod/sprod/version/mapset) and between 2 dates
# Several types of aggregation foreseen:
#
# mean : Sum(Xi)/N(Xi) -> min/max not considered e.g. Rain
# cumulate: Sum(Xi) -> min/max not considered e.g. Fire
#
# count: N(Xi where min < Xi < max) e.g. Vegetation anomalies
# surface: count * PixelArea e.g. Water Bodies
# percent: count/Ntot e.g. Vegetation anomalies
ogr.UseExceptions()
theGeomWkt = ' '.join(wkt.strip().split())
geom = Geometry(wkt=str(theGeomWkt), srs=4326)
# Get Mapset Info
mapset_info = querydb.get_mapset(mapsetcode=mapsetcode)
# Compute pixel area by converting degree to km
pixelArea = abs(mapset_info.pixel_shift_lat) * abs(
mapset_info.pixel_shift_lat) * 12544.0
# Get Product Info
product_info = querydb.get_product_out_info(productcode=productcode,
subproductcode=subproductcode,
version=version)
if product_info.__len__() > 0:
scale_factor = 0
scale_offset = 0
nodata = 0
date_format = ''
for row in product_info:
scale_factor = row.scale_factor
scale_offset = row.scale_offset
nodata = row.nodata
unit = row.unit
date_format = row.date_format
[list_files,
dates_list] = getFilesList(productcode, subproductcode, version,
mapsetcode, date_format, start_date,
end_date)
# Built a dictionary with filenames/dates
dates_to_files_dict = dict(zip(dates_list, list_files))
# Generate unique list of files
unique_list = set(list_files)
uniqueFilesValues = []
for infile in unique_list:
single_result = {
'filename': '',
'meanvalue_noscaling': nodata,
'meanvalue': None
}
if os.path.isfile(infile):
try:
mx = []
with Raster(infile) as img:
# Assign nodata from prod_info
img._nodata = nodata
with img.clip(geom) as clipped:
# Save clipped image (for debug only)
# clipped.save('/data/processing/exchange/clipped_test.tif')
mx = clipped.array()
nodata_array_masked = ma.masked_equal(mx, nodata)
merged_mask = ma.mask_or(ma.getmask(mx),
ma.getmask(nodata_array_masked))
mxnodata = ma.masked_array(ma.getdata(mx), merged_mask)
if aggregate['aggregation_type'] == 'count' or aggregate[
'aggregation_type'] == 'percent' or aggregate[
'aggregation_type'] == 'surface':
min_val = aggregate['aggregation_min']
max_val = aggregate['aggregation_max']
# Scale threshold from physical to digital value
min_val_scaled = (min_val -
scale_offset) / scale_factor
max_val_scaled = (max_val -
scale_offset) / scale_factor
mxrange = ma.masked_outside(mxnodata, min_val_scaled,
max_val_scaled)
if aggregate['aggregation_type'] == 'percent':
# 'percent'
meanResult = float(mxrange.count()) / float(
mxnodata.count()) * 100
elif aggregate['aggregation_type'] == 'surface':
# 'surface'
meanResult = float(mxrange.count()) * pixelArea
else:
# 'count'
meanResult = float(mxrange.count())
# Both results are equal
finalvalue = meanResult
else: #if aggregate['type'] == 'mean' or if aggregate['type'] == 'cumulate':
if mxnodata.count() == 0:
meanResult = 0.0
else:
if aggregate['aggregation_type'] == 'mean':
# 'mean'
meanResult = mxnodata.mean()
else:
# 'cumulate'
meanResult = mxnodata.sum()
# Scale to physical value
finalvalue = (meanResult * scale_factor + scale_offset)
# Assign results
single_result['filename'] = infile
single_result['meanvalue_noscaling'] = meanResult
single_result['meanvalue'] = finalvalue
except Exception, e:
logger.debug('ERROR: clipping - %s' % (e))
# sys.exit (1)
else:
logger.debug('ERROR: raster file does not exist - %s' % infile)
# sys.exit (1)
uniqueFilesValues.append(single_result)
# Define a dictionary to associate filenames/values
files_to_values_dict = dict(
(x['filename'], x['meanvalue']) for x in uniqueFilesValues)
# Prepare array for result
resultDatesValues = []
# Returns a list of 'filenames', 'dates', 'values'
for mydate in dates_list:
# my_result = {'date': datetime.date.today(), 'filename':'', 'meanvalue':nodata}
my_result = {'date': datetime.date.today(), 'meanvalue': nodata}
# Assign the date
my_result['date'] = mydate
# Assign the filename
my_filename = dates_to_files_dict[mydate]
# my_result['filename'] = my_filename
# Map from array of Values
my_result['meanvalue'] = files_to_values_dict[my_filename]
# Map from array of dates
resultDatesValues.append(my_result)
return resultDatesValues
def __init__(self, mapset_code):
self.mapset_code = mapset_code
kwargs = {'mapsetcode': self.mapset_code}
self._mapset = querydb.get_mapset(**kwargs)
if not self._mapset:
raise NoMapsetFound(kwargs)
def Test_get_mapset(self):
mapset = querydb.get_mapset(mapsetcode='WGS84_Africa_1km')
logger.info("Mapset: %s", mapset)
self.assertEqual(1, 1)
def __init__(self, mapset_code):
self.mapset_code = mapset_code
kwargs = {'mapsetcode': self.mapset_code}
self._mapset = querydb.get_mapset(**kwargs)
if not self._mapset:
raise NoMapsetFound(kwargs)
def test_data_management(self):
import json
db_products = querydb.get_products(activated=True)
if db_products.__len__() > 0:
products_dict_all = []
# loop the products list
for row in db_products:
prod_dict = functions.row2dict(row)
productcode = prod_dict['productcode']
version = prod_dict['version']
p = Product(product_code=productcode, version=version)
# print productcode
# does the product have mapsets AND subproducts?
all_prod_mapsets = p.mapsets
all_prod_subproducts = p.subproducts
if all_prod_mapsets.__len__(
) > 0 and all_prod_subproducts.__len__() > 0:
prod_dict['productmapsets'] = []
for mapset in all_prod_mapsets:
mapset_dict = []
# print mapset
mapset_info = querydb.get_mapset(mapsetcode=mapset,
allrecs=False)
# if mapset_info.__len__() > 0:
mapset_dict = functions.row2dict(mapset_info)
# else:
# mapset_dict['mapsetcode'] = mapset
mapset_dict['mapsetdatasets'] = []
all_mapset_datasets = p.get_subproducts(mapset=mapset)
for subproductcode in all_mapset_datasets:
# print 'productcode: ' + productcode
# print 'version: ' + version
# print 'subproductcode: ' + subproductcode
dataset_info = querydb.get_subproduct(
productcode=productcode,
version=version,
subproductcode=subproductcode)
# print dataset_info
# dataset_info = querydb.db.product.get(productcode, version, subproductcode)
# dataset_dict = {}
if dataset_info is not None:
dataset_dict = functions.row2dict(dataset_info)
# dataset_dict = dataset_info.__dict__
# del dataset_dict['_labels']
if hasattr(dataset_info, 'frequency_id'):
if dataset_info.frequency_id == 'e15minute' or dataset_info.frequency_id == 'e30minute':
dataset_dict[
'nodisplay'] = 'no_minutes_display'
# To be implemented in dataset.py
elif dataset_info.frequency_id == 'e1year':
dataset_dict[
'nodisplay'] = 'no_minutes_display'
else:
dataset = p.get_dataset(
mapset=mapset,
sub_product_code=subproductcode)
completeness = dataset.get_dataset_normalized_info(
)
dataset_dict[
'datasetcompleteness'] = completeness
dataset_dict['nodisplay'] = 'false'
dataset_dict['mapsetcode'] = mapset_dict[
'mapsetcode']
dataset_dict[
'mapset_descriptive_name'] = mapset_dict[
'descriptive_name']
mapset_dict['mapsetdatasets'].append(
dataset_dict)
else:
pass
prod_dict['productmapsets'].append(mapset_dict)
products_dict_all.append(prod_dict)
prod_json = json.dumps(products_dict_all,
ensure_ascii=False,
sort_keys=True,
indent=4,
separators=(', ', ': '))
datamanagement_json = '{"success":"true", "total":'\
+ str(db_products.__len__())\
+ ',"products":'+prod_json+'}'
else:
datamanagement_json = '{"success":false, "error":"No data sets defined!"}'
def push_data_ftp(dry_run=False,
user=None,
psw=None,
url=None,
trg_dir=None,
masked=True):
# Synchronized data towards an ftp server (only for JRC)
# It replaces, since the new srv-ies-ftp.jrc.it ftp is set, the bash script: mirror_to_ftp.sh
# Configuration: it looks at all 'non-masked' products and pushes them
# For the mapsets, find what is in the filesystem, and pushes only the 'largest'
# It uses a command like:
# lftp -e "mirror -RLe /data/processing/vgt-ndvi/sv2-pv2.1/SPOTV-Africa-1km/derived/10dmax-linearx2/
# /narma/eStation_2.0/processing/vgt-ndvi/sv2-pv2.1/SPOTV-Africa-1km/derived/10dmax-linearx2/;exit"
# -u narma:JRCVRw2960 sftp://srv-ies-ftp.jrc.it"" >> /eStation2/log/push_data_ftp.log
#
spec_logger = log.my_logger('apps.es2system.push_data_ftp')
try:
from config import server_ftp
except:
logger.warning('Configuration file for ftp sync not found. Exit')
return 1
if user is None:
user = server_ftp.server['user']
if psw is None:
psw = server_ftp.server['psw']
if url is None:
url = server_ftp.server['url']
if trg_dir is None:
trg_dir = server_ftp.server['data_dir']
# Create an ad-hoc file for the lftp command output (beside the standard logger)
logfile = es_constants.es2globals['log_dir'] + 'push_data_ftp.log'
message = time.strftime(
"%Y-%m-%d %H:%M") + ' INFO: Running the ftp sync now ... \n'
logger.debug("Entering routine %s" % 'push_data_ftp')
# Loop over 'not-masked' products
products = querydb.get_products(masked=False)
# produts=products[21:23] # test a subset
for row in products:
prod_dict = functions.row2dict(row)
productcode = prod_dict['productcode']
version = prod_dict['version']
spec_logger.info('Working on product {}/{}'.format(
productcode, version))
# TEMP - For testing only
# if productcode!='vgt-ndvi' or version !='sv2-pv2.2':
# continue
# Check it if is in the list of 'exclusions' defined in ./config/server_ftp.py
key = '{}/{}'.format(productcode, version)
skip = False
if key in server_ftp.exclusions:
skip = True
logger.debug('Do not sync for {}/{}'.format(productcode, version))
p = Product(product_code=productcode, version=version)
all_prod_mapsets = p.mapsets
all_prod_subproducts = p.subproducts
# Check there is at least one mapset and one subproduct
if all_prod_mapsets.__len__() > 0 and all_prod_subproducts.__len__(
) > 0 and not skip:
# In case of several mapsets, check if there is a 'larger' one
if len(all_prod_mapsets) > 1:
mapset_to_use = []
for my_mapset in all_prod_mapsets:
mapset_info = querydb.get_mapset(mapsetcode=my_mapset,
allrecs=False)
if hasattr(mapset_info, "mapsetcode"):
my_mapobj = MapSet()
my_mapobj.assigndb(my_mapset)
larger_mapset = my_mapobj.get_larger_mapset()
if larger_mapset is not None:
if larger_mapset not in mapset_to_use:
mapset_to_use.append(larger_mapset)
else:
if my_mapset not in mapset_to_use:
mapset_to_use.append(my_mapset)
else:
mapset_to_use = all_prod_mapsets
# Loop over existing mapset
for mapset in mapset_to_use:
all_mapset_datasets = p.get_subproducts(mapset=mapset)
# Loop over existing subproducts
for subproductcode in all_mapset_datasets:
# Get info - and ONLY for NOT masked products
dataset_info = querydb.get_subproduct(
productcode=productcode,
version=version,
subproductcode=subproductcode,
masked=masked) # -> TRUE means only NOT masked sprods
if dataset_info is not None:
dataset_dict = functions.row2dict(dataset_info)
dataset_dict['mapsetcode'] = mapset
logger.debug('Working on {}/{}/{}/{}'.format(
productcode, version, mapset, subproductcode))
subdir = functions.set_path_sub_directory(
productcode, subproductcode,
dataset_dict['product_type'], version, mapset)
source = data_dir + subdir
target = trg_dir + subdir
# command = 'lftp -e "mirror -RLe {} {};exit" -u {}:{} {}"" >> {}'.format(source,target,user,psw,url,logfile)
command = 'lftp -e "mirror -RLe {} {};exit" -u {}:{} {}"" >> /dev/null'.format(
source, target, user, psw, url)
logger.debug("Executing %s" % command)
spec_logger.info(
'Working on mapset/subproduct {}/{} \n'.format(
mapset, subproductcode))
# return
try:
status = os.system(command)
if status:
logger.error("Error in executing %s" % command)
spec_logger.error("Error in executing %s" %
command)
except:
logger.error(
'Error in executing command: {}'.format(
command))
spec_logger.error(
'Error in executing command: {}'.format(
command))