def set_var_att_value(dataset, var_name, att_name, cds_type, att_value):
var = dsproc.get_var(dataset, var_name)
if var is None:
error_string = "Required variable {} not found in dataset".format(var_name)
dsproc.error(dsproc.EREQVAR, error_string)
return 0
status = dsproc.set_att_value(var, att_name, cds_type, att_value)
return status
def get_var_data(dataset, var_name, data_type):
var = dsproc.get_var(dataset, var_name)
if var is None:
error_string = "Required variable {} not found in dataset".format(var_name)
dsproc.error(dsproc.EREQVAR, error_string)
return None
var_data, missing = dsproc.get_var_data(var, data_type, 0)
if var_data is None:
return None
return var_data
def init_var_data(dataset, var_name, start, nsamples, use_missing=False):
var = dsproc.get_var(dataset, var_name)
if var is None:
error_string = "Required variable {} not found in dataset".format(var_name)
dsproc.error(dsproc.EREQVAR, error_string)
return None
var_data = dsproc.init_var_data(var, start, nsamples, use_missing)
if var_data is None:
return None
return var_data
def create_quicklook(proc_data, begin_date, end_date):
status_string = 'create_quicklook error'
out_dsid = proc_data.out_dsid
# **************************
# Create plots using run_dq_inpsector function
# **************************
dq_inputs = ['-t','-v','all','-E','time_offset','time',
'ncomparisons','time_bounds', None]
status = dsproc.run_dq_inspector(out_dsid, begin_date, end_date, dq_inputs, 0)
if status < 0:
error_string = 'Could not execute call to dq_inspector'
dsproc.error(status_string, error_string)
return -1
return 1
def pre_transform_hook(user_data, begin_date, end_date, ret_data):
status = 1
begin_time = strftime('%Y%m%d.%H%M%S', gmtime(begin_date))
end_time = strftime('%Y%m%d.%H%M%S', gmtime(end_date))
FAIL_MIN = 0
WARN_MAX = 1
# Get QC, set every intermediata/bad value to bad
CDS_INT = 4
CDS_FLOAT = 5
ret_groups = ret_data.get_groups()
group_names = [ g.get_name() for g in ret_groups ]
wavelengths = [500, 870]
# For each one, set values above threshold (10) to missing value
for top_group in ret_groups:
name = top_group.get_name()
dsproc.debug_lv1("Retrieving: {}".format( name ) )
if 'csphot' in name or 'aeronet' in name:
var_name_prefix = 'aod_cimel'
elif 'sgpmfrsraod1michC1' in name:
var_name_prefix = 'aod_mfrsr_C1'
elif 'sgpmfrsraod1michE13' in name:
var_name_prefix = 'aod_mfrsr_E13'
else:
var_name_prefix = 'aod_nimfr'
sub_groups = top_group.get_groups()
sub_group_with_data = sub_groups[0]
for wavelength in wavelengths:
var_name = "{}_{}".format(var_name_prefix, wavelength)
var = dsproc.get_var(sub_group_with_data, var_name)
if var is None:
error_string = "Required variable {} not found in dataset".format(var_name)
dsproc.error(dsproc.EREQVAR, error_string)
return -1
var_data, missing = dsproc.get_var_data(var, CDS_FLOAT, 0)
# Below 0, Above 1
below_fail_min = var_data < FAIL_MIN
above_warn_max = var_data > WARN_MAX
var_data[below_fail_min] = MISSING_VALUE
var_data[above_warn_max] = MISSING_VALUE
nsamples = len(var_data)
status = dsproc.set_var_data(var, 0, nsamples, None, var_data)
if status is None:
return -1
qc_var = dsproc.get_qc_var(var)
if qc_var is not None:
qc_var_data, missing = dsproc.get_var_data(qc_var, CDS_INT, 0)
# Testing setting anything not good, as bad
not_good_qc = qc_var_data != 0
qc_var_data[not_good_qc] = 1
# Below 0, Above 1
qc_var_data[below_fail_min] = 1
qc_var_data[above_warn_max] = 1
status = dsproc.set_var_data(qc_var, 0, nsamples, None, qc_var_data)
if status is None:
return -1
# If only one value for cimel, artificially add a sample to make
# transformation work
# Note, this is a hack and not currently supported by ADI
cimel_name = ""
for name in group_names:
if 'csphot' in name:
cimel_name = name
break
var_name_prefix = 'aod_cimel'
if cimel_name:
dsproc.debug_lv1("Retrieving: {}".format(cimel_name) )
top_group = ret_data.get_group(cimel_name)
if not top_group:
dsproc.debug_lv1("No retrieved group for {}, skipping".format(cimel_name))
else:
sub_groups = top_group.get_groups()
sub_group_with_data = sub_groups[0]
# 500nm cimel data
var_name_500 = "{}_500".format(var_name_prefix, wavelength)
var_500 = dsproc.get_var(sub_group_with_data, var_name_500)
if var_500 is None:
error_string = "Required variable {} not found in dataset".format(var_name_500)
dsproc.error(dsproc.EREQVAR, error_string)
return -1
var_500_data, missing = dsproc.get_var_data(var_500, CDS_FLOAT, 0)
# Check if have only one value
nsamples_500 = len(var_500_data)
# 870nm cimel var
var_name_870 = "{}_870".format(var_name_prefix, wavelength)
var_870 = dsproc.get_var(sub_group_with_data, var_name_870)
if var_870 is None:
error_string = "Required variable {} not found in dataset".format(var_name_870)
dsproc.error(dsproc.EREQVAR, error_string)
return -1
if nsamples_500 == 1:
dsproc.debug_lv1("Only 1 cimel sample, artificially adding another")
dsproc.log("Only 1 cimel sample, artificially adding another")
times = dsproc.get_sample_times(sub_group_with_data, 0)
times[0] = times[0] + 1
dsproc.set_sample_times(sub_group_with_data, 1, times)
var_type = var_500_data.dtype
fake_value = [MISSING_VALUE]
fake_value = np.array(fake_value)
fake_value = fake_value.astype(var_type)
status = dsproc.set_var_data(var_500, 1, 1, None, fake_value)
if status is None:
return -1
status = dsproc.set_var_data(var_870, 1, 1, None, fake_value)
if status is None:
return -1
if dsproc.get_debug_level() > 1:
dsproc.dump_retrieved_datasets("./debug_dumps", "retrieved_data.debug", 0)
return 1