def test_ingest_vgt_ndvi(self):
productcode = 'vgt_ndvi'
productversion = 'undefined'
subproductcode = 'ndvi_native'
mapsetcode = 'WGS84_Africa_1km'
product = {"productcode": productcode,
"version": productversion}
args = {"productcode": productcode,
"subproductcode": subproductcode,
"datasource_descr_id": '',
"version": productversion}
product_in_info = querydb.get_product_in_info(echo=echo_query, **args)
re_process = product_in_info.re_process
re_extract = product_in_info.re_extract
sprod = {'subproduct': ingest.subproductcode,
'mapsetcode': ingest.mapsetcode,
're_extract': re_extract,
're_process': re_process}
subproducts.append(sprod)
ingestion(date_fileslist, in_date, product, subproducts, datasource_descr, echo_query=echo_query)
self.assertEqual(1, 1)
def Test_get_product_in_info(self):
product_in = querydb.get_product_in_info(productcode='fewsnet-rfe',
subproductcode='10d',
version='undefined',
datasource_descr_id='USGS:EARLWRN:FEWSNET')
logger.info("Product IN info: %s", product_in)
self.assertEqual(1, 1)
def Test_get_product_in_info2(self):
product_in = querydb.get_product_in_info(productcode='modis-firms',
subproductcode='1day',
version='v5.0',
datasource_descr_id='FIRMS:NASA')
logger.info("Product IN info: %s", product_in)
self.assertEqual(1, 1)
def Test_get_product_in_info3(self):
product_in = querydb.get_product_in_info(productcode='modis-chla',
subproductcode='chla-day',
version='v2013.1',
datasource_descr_id='GSFC:CGI:MODIS:CHLA:1D')
logger.info("Product IN info: %s", product_in)
self.assertEqual(1, 1)
def Test_get_product_in_info(self):
product_in = querydb.get_product_in_info(productcode='fewsnet-rfe',
subproductcode='10d',
version='undefined',
datasource_descr_id='USGS:EARLWRN:FEWSNET')
logger.info("Product IN info: %s", product_in)
self.assertEqual(1, 1)
def Test_get_product_in_info1(self):
product_in = querydb.get_product_in_info(productcode='vgt-ndvi',
subproductcode='ndv',
version='spot-v1',
datasource_descr_id='EO:EUM:DAT:SPOT1:S10NDVI')
logger.info("Product IN info: %s", product_in)
self.assertEqual(1, 1)
def Test_get_product_in_info1(self):
product_in = querydb.get_product_in_info(productcode='vgt-ndvi',
subproductcode='ndv',
version='spot-v1',
datasource_descr_id='EO:EUM:DAT:SPOT1:S10NDVI')
logger.info("Product IN info: %s", product_in)
self.assertEqual(1, 1)
def Test_get_product_in_info2(self):
product_in = querydb.get_product_in_info(productcode='modis-firms',
subproductcode='1day',
version='v5.0',
datasource_descr_id='FIRMS:NASA')
logger.info("Product IN info: %s", product_in)
self.assertEqual(1, 1)
def Test_get_product_in_info4(self):
product_in = querydb.get_product_in_info(
productcode='lsasaf-et',
subproductcode='et',
version='undefined',
datasource_descr_id='EO:EUM:DAT:MSG:ET-SEVIRI')
logger.info("Product IN info: %s", product_in)
self.assertEqual(1, 1)
def Test_get_product_in_info3(self):
product_in = querydb.get_product_in_info(
productcode='modis-chla',
subproductcode='chla-day',
version='v2013.1',
datasource_descr_id='GSFC:CGI:MODIS:CHLA:1D')
logger.info("Product IN info: %s", product_in)
self.assertEqual(1, 1)
def ingest_jrc_wbd(input_dir, in_date=None, avg=None):
if avg:
date_fileslist = glob.glob(input_dir + '/JRC-WBD_AVG2000-' + in_date +
'*')
subproductcode = 'avg'
mapsetcode = 'WD-GEE-ECOWAS-AVG'
datasource_descrID = 'JRC:WBD:GEE:AVG'
else:
date_fileslist = glob.glob(input_dir + '/JRC-WBD_' + in_date + '*')
subproductcode = 'occurr'
mapsetcode = 'WD-GEE-ECOWAS-1'
datasource_descrID = 'JRC:WBD:GEE'
productcode = 'wd-gee'
productversion = '1.0'
product = {"productcode": productcode, "version": productversion}
args = {
"productcode": productcode,
"subproductcode": subproductcode,
"datasource_descr_id": datasource_descrID,
"version": productversion
}
product_in_info = querydb.get_product_in_info(**args)
re_process = product_in_info.re_process
re_extract = product_in_info.re_extract
sprod = {
'subproduct': subproductcode,
'mapsetcode': mapsetcode,
're_extract': re_extract,
're_process': re_process
}
subproducts = []
subproducts.append(sprod)
output_file = es_constants.es2globals['processing_dir']+\
functions.set_path_sub_directory(productcode, subproductcode, 'Ingest', productversion, mapsetcode) +\
functions.set_path_filename(in_date,productcode,subproductcode,mapsetcode,productversion,'.tif')
for internet_filter, datasource_descr in querydb.get_datasource_descr(
source_type='INTERNET', source_id=datasource_descrID):
ingestion.ingestion(date_fileslist,
in_date,
product,
subproducts,
datasource_descr,
logger,
echo_query=1)
return output_file
def test_ingest_g_cls_ndvi_200_1Km(self):
# Test Copernicus Products version 2.2 (starting with NDVI 2.2.1)
productcode = 'vgt-ndvi'
productversion = 'proba-v2.2'
subproductcode = 'ndv'
mapsetcode = 'SPOTV-Africa-1km'
datasource_descrID = 'PDF:GLS:PROBA-V1:NDVI300'
# input_dir = self.test_ingest_dir + os.path.sep + productcode + os.path.sep + self.native_dir
#date_fileslist = [os.path.join(input_dir, 'c_gls_NDVI_202003010000_AFRI_PROBAV_V2.2.1.zip')]
date_fileslist = glob.glob(
'/eos/jeodpp/home/users/venkavi/data/processing/vgt-ndvi/sv2-pv2.2/archive/c_gls_NDVI300_202007110000_GLOBE_PROBAV_V1.0.1.nc*'
)
in_date = '202007110000'
out_date = '20200711'
product = {"productcode": productcode, "version": productversion}
args = {
"productcode": productcode,
"subproductcode": subproductcode,
"datasource_descr_id": datasource_descrID,
"version": productversion
}
product_in_info = querydb.get_product_in_info(**args)
re_process = product_in_info.re_process
re_extract = product_in_info.re_extract
sprod = {
'subproduct': subproductcode,
'mapsetcode': mapsetcode,
're_extract': re_extract,
're_process': re_process
}
subproducts = [sprod]
# Remove existing output
# self.remove_output_file(productcode, subproductcode, productversion, mapsetcode, out_date)
datasource_descr = querydb.get_datasource_descr(
source_type='INTERNET', source_id=datasource_descrID)
ingestion.ingestion(date_fileslist,
in_date,
product,
subproducts,
datasource_descr[0],
logger,
echo_query=1,
test_mode=False)
# status = self.checkIngestedFile(productcode=productcode, subproductcode=subproductcode,
# version=productversion, mapsetcode=mapsetcode, date=out_date)
self.assertEqual(1, 1)
def Test_get_product_in_info1(self):
product_in = querydb.get_product_in_info(
productcode='vgt-ndvi',
subproductcode='ndv',
version='spot-v1',
datasource_descr_id='EO:EUM:DAT:SPOT1:S10NDVI')
logger.info("Product IN info: %s", product_in)
if hasattr(product_in, "c"): # There is a result so test succeeds!
result = 1
else:
result = 0
self.assertEqual(1, result)
def Test_get_product_in_info(self):
product_in = querydb.get_product_in_info(
productcode='fewsnet-rfe',
subproductcode='10d',
version='undefined',
datasource_descr_id='USGS:EARLWRN:FEWSNET')
# Test fails because version has to be 2.0 for fewsnet-rfe
logger.info("Product IN info: %s", product_in)
if product_in.__len__() == 0: # No result so test succeeds!
result = 1
else:
result = 0
self.assertEqual(1, result)
def get_product_in_info(product, subproduct, datasource_descr_id, my_logger):
try:
# Get information about the dataset
args = {
"productcode": product['productcode'],
"subproductcode": subproduct,
"datasource_descr_id": datasource_descr_id,
"version": product['version']
}
# Get information from sub_dataset_source table
product_in_info = querydb.get_product_in_info(**args)
return product_in_info
except:
my_logger.error('Error defining Input product info ')
def get_subrproducts_from_ingestion(product, datasource_descr_id):
# Get list of ingestions triggers [prod/subprod/mapset]
ingestions = querydb.get_ingestion_subproduct(allrecs=False, **product)
# Loop over ingestion triggers
subproducts = list()
for ingest in ingestions:
dates_not_in_filename = False
if ingest.input_to_process_re == 'dates_not_in_filename':
dates_not_in_filename = True
logger.debug(" --> processing subproduct: %s" % ingest.subproductcode)
args = {
"productcode": product['productcode'],
"subproductcode": ingest.subproductcode,
"datasource_descr_id": datasource_descr_id,
"version": product['version']
}
product_in_info = querydb.get_product_in_info(**args)
try:
re_process = product_in_info.re_process
re_extract = product_in_info.re_extract
nodata_value = product_in_info.no_data
in_scale_factor = product_in_info.scale_factor
in_offset = product_in_info.scale_offset
sprod = {
'subproduct': ingest.subproductcode,
'mapsetcode': ingest.mapsetcode,
're_extract': re_extract,
're_process': re_process,
'nodata': nodata_value,
'in_scale_factor': in_scale_factor,
'in_offset': in_offset
}
subproducts.append(sprod)
except:
logger.warning("Subproduct %s not defined for source %s" %
(ingest.subproductcode, datasource_descr_id))
return subproducts
def loop_ingestion_drive(dry_run=False, test_one_product=None):
echo_query = False
# Get all active product ingestion records with a subproduct count.
active_product_ingestions = querydb.get_ingestion_product(allrecs=True)
for active_product_ingest in active_product_ingestions:
productcode = active_product_ingest[0]
productversion = active_product_ingest[1]
# Verify the test-one-product case
do_ingest_product = is_test_one_product(test_one_product, productcode)
if do_ingest_product:
logger.info("Ingestion active for product: [%s] subproduct N. %s" %
(active_product_ingest[0], active_product_ingest[2]))
# For the current active product ingestion: get all
product = {"productcode": productcode, "version": productversion}
logger.debug("Processing product: %s - version %s" %
(productcode, productversion))
# Get the list of acquisition sources that are defined for this ingestion 'trigger'
# (i.e. prod/version)
# NOTE: the following implies there is 1 and only 1 '_native' subproduct associated to a 'product';
native_product = {
"productcode": productcode,
"subproductcode": productcode + "_native",
"version": productversion
}
sources_list = querydb.get_product_sources(**native_product)
logger.debug("For product [%s] N. %s source is/are found" %
(productcode, len(sources_list)))
systemsettings = functions.getSystemSettings()
for source in sources_list:
logger_spec = log.my_logger('apps.ingestion.' + productcode +
'.' + productversion)
logger.debug("Processing Source type [%s] with id [%s]" %
(source.type, source.data_source_id))
# Re-initialize the datasource_descr
# datasource_descr = None
# Get datasource desctiption
datasource_descr = querydb.get_datasource_descr(
source_type=source.type, source_id=source.data_source_id)
datasource_descr = datasource_descr[0]
# TODO optimize this in order to get direct file filter expression
my_filter_expr = get_filenaming_info(source, datasource_descr)
files = get_files_matching_with_file_expression(my_filter_expr)
# See ES2-204
logger_spec.debug(
"Number of files found for product [%s] is: %s" %
(active_product_ingest[0], len(files)))
if len(files) > 0:
# Get list of ingestions triggers [prod/subprod/mapset]
ingestions = querydb.get_ingestion_subproduct(
allrecs=False, **product)
# Loop over ingestion triggers
subproducts = list()
for ingest in ingestions:
# TODO if one ingest gives true and another false?
dates_not_in_filename = is_date_not_in_filename(
ingest.input_to_process_re)
logger.debug(" --> processing subproduct: %s" %
ingest.subproductcode)
args = {
"productcode": product['productcode'],
"subproductcode": ingest.subproductcode,
"datasource_descr_id":
datasource_descr.datasource_descr_id,
"version": product['version']
}
product_in_info = querydb.get_product_in_info(**args)
# TODO verify the approach Should we get subproduct from single query
subproduct = get_subproduct(
ingest, product_in_info,
datasource_descr.datasource_descr_id)
if subproduct is not None:
subproducts.append(subproduct)
if subproducts is None:
#TODO what to do?
logger.error(
"For current active ingestion No subproducts for Product [%s] "
% (productcode))
# Get the list of unique dates by extracting the date from all files.
dates_list = get_list_unique_dates(datasource_descr, files,
dates_not_in_filename,
product_in_info,
ingest.mapsetcode)
# Loop over dates and get list of files
for in_date in dates_list:
date_fileslist = get_dates_file_list(
dates_not_in_filename, files, my_filter_expr,
in_date, logger_spec)
# Pass list of files to ingestion routine
if (not dry_run):
try:
result = ingestion(date_fileslist,
in_date,
product,
subproducts,
datasource_descr,
logger_spec,
echo_query=echo_query)
except:
logger.error(
"Error in ingestion of file [%s] " %
(functions.conv_list_2_string(
date_fileslist)))
else:
# Result is None means we are still waiting for some files to be received. Keep files in /data/ingest
# dates_not_in_filename means the input files contains many dates (e.g. GSOD precip)
if result is not None and not dates_not_in_filename:
if source.store_original_data or systemsettings[
'type_installation'] == 'Server':
store_native_files(
product, date_fileslist,
logger_spec)
else:
delete_files(date_fileslist,
logger_spec)
else:
time.sleep(10)