def ingest_netcdf_cds(internet_source, downloaded_file, processed_item):
ingestion_status = False
try:
product = {
"productcode": internet_source.productcode,
"version": internet_source.version
}
# Datasource description
datasource_descr = querydb.get_datasource_descr(
source_type='INTERNET', source_id=internet_source.internet_id)
datasource_descr = datasource_descr[0]
# Get list of subproducts
sub_datasource = ingestion.get_subrproducts_from_ingestion(
product, datasource_descr.datasource_descr_id)
ingestion_status = ingestion_netcdf.ingestion_netcdf(
downloaded_file,
processed_item.split(':')[0], product, sub_datasource,
datasource_descr, logger)
except Exception as inst:
logger.debug("Error in CDS Ingestion: %s" % internet_source)
raise
return ingestion_status
def debug_CDS_MSLP_hour_netcdf(self):
internet_id = "CDS:ERA5:REANALYSIS:MSLP:HOUR"
product = {"productcode": "era5-hourly-mslp", "version": "1.0"}
downloaded_file = '/data/processing/era5-hourly-mslp/1.0/archive/202101010100_reanalysis-era5-single-levels_reanalysis_mean_sea_level_pressure.nc'
in_date = '202101010000'
# Datasource description
datasource_descr = querydb.get_datasource_descr(source_type='INTERNET',
source_id=internet_id)
datasource_descr = datasource_descr[0]
# Get list of subproducts
sub_datasource = ingestion.get_subrproducts_from_ingestion(
product, datasource_descr.datasource_descr_id)
ingestion_status = ingestion_netcdf.ingestion_netcdf(downloaded_file,
in_date,
product,
sub_datasource,
datasource_descr,
logger,
test_mode=True)
def debug_CDS_SST_MONTH_netcdf(self):
internet_id = "CDS:ERA5:REANALYSIS:SST:MONTH"
product = {"productcode": "era5-monthly-sst", "version": "1.0"}
downloaded_file = '/data/ingest/20210101_sst_monthly_average.nc'
in_date = '202101010000'
# Datasource description
datasource_descr = querydb.get_datasource_descr(source_type='INTERNET',
source_id=internet_id)
datasource_descr = datasource_descr[0]
# Get list of subproducts
sub_datasource = ingestion.get_subrproducts_from_ingestion(
product, datasource_descr.datasource_descr_id)
ingestion_status = ingestion_netcdf.ingestion_netcdf(downloaded_file,
in_date,
product,
sub_datasource,
datasource_descr,
logger,
test_mode=True)
def debug_IRI_PRCP_1Month_ingest_netcdf(self):
internet_id = "IRI:NOAA:PRCP:MONTH" #'CDS:ERA5:REANALYSIS:SST:MONTH'
product = {"productcode": "iri_prcp", "version": "1.0"}
downloaded_file = '/data/processing/iri_prcp/1.0/archive/20210101-iri_prcp.nc'
in_date = '20210101'
# Datasource description
datasource_descr = querydb.get_datasource_descr(source_type='INTERNET',
source_id=internet_id)
datasource_descr = datasource_descr[0]
# Get list of subproducts
sub_datasource = ingestion.get_subrproducts_from_ingestion(
product, datasource_descr.datasource_descr_id)
ingestion_status = ingestion_netcdf.ingestion_netcdf(downloaded_file,
in_date,
product,
sub_datasource,
datasource_descr,
logger,
test_mode=True)
def debug_CDS_RFE_DAY_netcdf(self):
internet_id = "CDS:ERA5:REANALYSIS:RFE:DAY"
product = {"productcode": "era5-rfe", "version": "1.0"}
downloaded_file = '/data/ingest/202103200000_reanalysis-era5-single-levels_reanalysis_total_precipitation.nc'
in_date = '202103200000'
# Datasource description
datasource_descr = querydb.get_datasource_descr(source_type='INTERNET',
source_id=internet_id)
datasource_descr = datasource_descr[0]
# Get list of subproducts
sub_datasource = ingestion.get_subrproducts_from_ingestion(
product, datasource_descr.datasource_descr_id)
ingestion_status = ingestion_netcdf.ingestion_netcdf(downloaded_file,
in_date,
product,
sub_datasource,
datasource_descr,
logger,
test_mode=True)
def debug_IRI_surfacetemp_1Month_ingest_netcdf(self):
internet_id = "IRI:NOAA:SURFACETEMP:MONTH" #'CDS:ERA5:REANALYSIS:SST:MONTH'
product = {"productcode": "iri-surface-temp", "version": "1.0"}
downloaded_file = '/tmp/climatestation/surface_temp_Jan_2020.nc'
in_date = '20200101'
# Datasource description
datasource_descr = querydb.get_datasource_descr(source_type='INTERNET',
source_id=internet_id)
datasource_descr = datasource_descr[0]
# Get list of subproducts
sub_datasource = ingestion.get_subrproducts_from_ingestion(
product, datasource_descr.datasource_descr_id)
ingestion_status = ingestion_netcdf.ingestion_netcdf(downloaded_file,
in_date,
product,
sub_datasource,
datasource_descr,
logger,
test_mode=True)
def iri_api_loop_internet(internet_source):
logger_spec = log.my_logger('apps.get_datastore.' +
internet_source.internet_id)
if internet_source.user_name is None:
user_name = "anonymous"
else:
user_name = internet_source.user_name
if internet_source.password is None:
password = "******"
else:
password = internet_source.password
usr_pwd = str(user_name) + ':' + str(password)
# Create the full filename from a 'template' which contains
internet_url = str(internet_source.url)
# processed_list = []
# processed_list_filename = es_constants.get_datastore_processed_list_prefix + internet_source.internet_id.replace(":", "_") + '.list'
# processed_list = functions.restore_obj_from_json(processed_list,
# processed_list_filename)
try:
# Check if template is dict or string them create resources_parameters
# if type(template_paramater) is dict:
# resources_parameters = template_paramater
# else:
# resources_parameters = json.loads(template_paramater)
if internet_source.productcode is None or internet_source.version is None:
logger.error("Product is not passed")
return
product = {
"productcode": internet_source.productcode,
"version": internet_source.version
}
# Datasource description
datasource_descr = querydb.get_datasource_descr(
source_type='INTERNET', source_id=internet_source.internet_id)
datasource_descr = datasource_descr[0]
# Get list of subproducts
subproducts = ingestion.get_subrproducts_from_ingestion(
product, datasource_descr.datasource_descr_id)
dates = build_list_dates_generic(from_date=internet_source.start_date,
to_date=internet_source.end_date,
frequency_id=str(
internet_source.frequency_id))
# Dates defined are dynamic not based on the configuration file
iri_api.process_list_matching_url(datasource_descr, product,
subproducts, dates)
# functions.dump_obj_to_json(processed_list, processed_list_filename)
except:
logger.error("Error in IRI datastore service. Continue")
b_error = True
finally:
logger.info("IRI datastore service Ending")
current_list = []
return current_list
