def masking_operation( reference_time_string,
operation_index,
analysis_directory,
iteration,
flag_directory,
flag_directory2,
output_directory ):
# get dates in the month to be processed in this operation
reference_time = parser.parse(reference_time_string)
processing_dates = operation_dates(reference_time, operation_index)
print processing_dates
# derive flags for each day
for date in processing_dates:
year = date.year
month = date.month
day = date.day
datestring = '%04i%02i%02i' % (year, month, day)
flags_to_mask = copy.deepcopy( FLAGS_TO_MASK )
# mask marine regions post 2012
if year > 2012:
flags_to_mask.append( [MISSING_MARINE_FLAG, NULL_FLAG] )
apply_mask(analysis_directory, iteration, flag_directory, flag_directory2, flags_to_mask, date, output_directory)
def flagging_operation(reference_time_string, operation_index, input_directory,
iteration, output_directory, window_range,
count_threshold, target_flag):
# get dates in the month to be processed in this operation
reference_time = parser.parse(reference_time_string)
processing_dates = operation_dates(reference_time, operation_index)
# derive flags for each day
for processdate in processing_dates:
# check that directory exists and if not then make it
if not os.path.exists(
os.path.join(output_directory, str(processdate.year))):
os.makedirs(os.path.join(output_directory, str(processdate.year)))
flag_values = local_window_constraint_checks(
input_directory, output_directory, iteration, processdate,
window_range, count_threshold, target_flag)
# set flag values for output
#flag_values = numpy.zeros( definitions.GLOBAL_FIELD_SHAPE[1:], FLAG_TYPE )
# save to NetCDF
outputfile = os.path.join(
output_directory, '{:04d}'.format(processdate.year),
'eustace_analysis_{:d}_qc_flags_{:04d}{:02d}{:02d}.nc'.format(
iteration, processdate.year, processdate.month,
processdate.day))
save_flag_file(flag_values, processdate, outputfile)
def flagging_operation(
reference_time_string,
operation_index,
analysis_directory,
iteration,
output_directory,
percentile_directory,
):
# get dates in the month to be processed in this operation
reference_time = parser.parse(reference_time_string)
processing_dates = operation_dates(reference_time, operation_index)
# derive flags for each day
for processdate in processing_dates:
# check that directory exists and if not then make it
if not os.path.exists(
os.path.join(output_directory, str(processdate.year))):
os.makedirs(os.path.join(output_directory, str(processdate.year)))
# compute flags
flag_values = location_threshold_checks(analysis_directory,
percentile_directory,
output_directory, iteration,
processdate)
#save
outputfile = os.path.join(
output_directory, '{:04d}'.format(processdate.year),
'eustace_analysis_{:d}_qc_flags_{:04d}{:02d}{:02d}.nc'.format(
iteration, processdate.year, processdate.month,
processdate.day))
save_flag_file(flag_values, processdate, outputfile)
def flagging_operation(reference_time_string, operation_index, input_directory,
iteration, output_directory, start_year, end_year):
# get dates in the month to be processed in this operation
reference_time = parser.parse(reference_time_string)
processing_dates = operation_dates(reference_time, operation_index)
# derive flags for each day
for processdate in processing_dates:
# check that directory exists and if not then make it
if not os.path.exists(
os.path.join(output_directory, str(processdate.year))):
os.makedirs(os.path.join(output_directory, str(processdate.year)))
flag_values = calendar_constraint_checks(input_directory,
output_directory, iteration,
processdate, start_year,
end_year, CALENDAR_DAY_FLAG)
# save to NetCDF
outputfile = os.path.join(
output_directory, '{:04d}'.format(processdate.year),
'eustace_analysis_{:d}_qc_flags_{:04d}{:02d}{:02d}.nc'.format(
iteration, processdate.year, processdate.month,
processdate.day))
save_flag_file(flag_values, processdate, outputfile)
def flagging_operation( reference_time_string,
operation_index,
analysis_directory,
iteration,
output_directory ):
# get dates in the month to be processed in this operation
reference_time = parser.parse(reference_time_string)
processing_dates = operation_dates(reference_time, operation_index)
# check that directory exists and if not then make it
if not os.path.exists(os.path.join(output_directory, str(processing_dates[0].year))):
os.makedirs(os.path.join(output_directory, str(processing_dates[0].year)))
latent_variable_flag(analysis_directory, output_directory, iteration, processing_dates)
def flagging_operation(
reference_time_string,
operation_index,
analysis_directory,
iteration,
output_directory,
):
# get dates in the month to be processed in this operation
reference_time = parser.parse(reference_time_string)
processing_dates = operation_dates(reference_time, operation_index)
# derive flags for each day
for processdate in processing_dates:
# check that directory exists and if not then make it
if not os.path.exists(
os.path.join(output_directory, str(processdate.year))):
os.makedirs(os.path.join(output_directory, str(processdate.year)))
# compute flags
areal_flag_values = areal_checks(analysis_directory, output_directory,
iteration, processdate,
QUARTILE_LIMIT)
extreme_flag_values = extremes_checks(analysis_directory,
output_directory, iteration,
processdate, EXTREME_LOWER_LIMIT,
EXTREME_UPPER_LIMIT)
# join flags
flag_values = areal_flag_values | extreme_flag_values
#save
outputfile = os.path.join(
output_directory, '{:04d}'.format(processdate.year),
'eustace_analysis_{:d}_qc_flags_{:04d}{:02d}{:02d}.nc'.format(
iteration, processdate.year, processdate.month,
processdate.day))
save_flag_file(flag_values, processdate, outputfile)
def flagging_operation(reference_time_string, operation_index,
analysis_directory, iteration, output_directory,
climatology_limit, largescale_limit,
constraint_threshold):
# get dates in the month to be processed in this operation
reference_time = parser.parse(reference_time_string)
processing_dates = operation_dates(reference_time, operation_index)
# derive flags for each day
for processdate in processing_dates:
# check that directory exists and if not then make it
if not os.path.exists(
os.path.join(output_directory, str(processdate.year))):
os.makedirs(os.path.join(output_directory, str(processdate.year)))
# compute flags
climatology_flag_values = climatology_uncertainty_checks(
analysis_directory, output_directory, iteration, processdate,
climatology_limit)
large_scale_flag_values = large_scale_uncertainty_checks(
analysis_directory, output_directory, iteration, processdate,
largescale_limit)
local_influence_flag_values = local_influence_checks(
analysis_directory, output_directory, iteration, processdate,
constraint_threshold)
marine_flag = get_marine_flag()
# join flags
flag_values = climatology_flag_values | large_scale_flag_values | local_influence_flag_values | marine_flag
#save
outputfile = os.path.join(
output_directory, '{:04d}'.format(processdate.year),
'eustace_analysis_{:d}_qc_flags_{:04d}{:02d}{:02d}.nc'.format(
iteration, processdate.year, processdate.month,
processdate.day))
save_flag_file(flag_values, processdate, outputfile)