def modify_faimms_netcdf(netcdf_file_path, channel_id_info):
""" Modify the downloaded netCDF file so it passes both CF and IMOS checker
input:
netcdf_file_path(str) : path of netcdf file to modify
channel_id_index(tupple) : information from xml for the channel
"""
modify_aims_netcdf(netcdf_file_path, channel_id_info)
netcdf_file_obj = Dataset(netcdf_file_path, 'a', format='NETCDF4')
netcdf_file_obj.aims_channel_id = int(channel_id_info['channel_id'])
if not (channel_id_info['metadata_uuid'] == 'Not Available'):
netcdf_file_obj.metadata_uuid = channel_id_info['metadata_uuid']
# some weather stations channels don't have a depth variable if sensor above water
if 'depth' in netcdf_file_obj.variables.keys():
var = netcdf_file_obj.variables['depth']
var.long_name = 'nominal depth'
var.positive = 'down'
var.axis = 'Z'
var.reference_datum = 'sea surface'
var.valid_min = -10.0
var.valid_max = 30.0
netcdf_file_obj.renameVariable('depth', 'NOMINAL_DEPTH')
if 'DEPTH' in netcdf_file_obj.variables.keys():
var = netcdf_file_obj.variables['DEPTH']
var.coordinates = "TIME LATITUDE LONGITUDE NOMINAL_DEPTH"
var.long_name = 'actual depth'
var.reference_datum = 'sea surface'
var.positive = 'down'
var.valid_min = -10.0
var.valid_max = 30.0
netcdf_file_obj.close()
netcdf_file_obj = Dataset(
netcdf_file_path, 'a', format='NETCDF4'
) # need to close to save to file. as we call get_main_netcdf_var just after
main_var = get_main_netcdf_var(netcdf_file_path)
# DEPTH, LATITUDE and LONGITUDE are not dimensions, so we make them into auxiliary cooordinate variables by adding this attribute
if 'NOMINAL_DEPTH' in netcdf_file_obj.variables.keys():
netcdf_file_obj.variables[
main_var].coordinates = "TIME LATITUDE LONGITUDE NOMINAL_DEPTH"
else:
netcdf_file_obj.variables[
main_var].coordinates = "TIME LATITUDE LONGITUDE"
netcdf_file_obj.close()
if not convert_time_cf_to_imos(netcdf_file_path):
return False
remove_dimension_from_netcdf(
netcdf_file_path) # last modification to do in this order!
return True
def get_main_var_folder_name(netcdf_file_path):
main_var = get_main_netcdf_var(netcdf_file_path)
netcdf_file_obj = Dataset(netcdf_file_path, mode='r')
var_folder_name = netcdf_file_obj.variables[main_var].long_name.replace(
' ', '_')
aims_channel_id = netcdf_file_obj.aims_channel_id
if hasattr(netcdf_file_obj.variables[main_var], 'sensor_depth'):
sensor_depth = netcdf_file_obj.variables[main_var].sensor_depth
retval = '%s@%sm_channel_%s' % (var_folder_name, str(sensor_depth),
str(aims_channel_id))
else:
retval = '%s_channel_%s' % (var_folder_name, str(aims_channel_id))
netcdf_file_obj.close()
return retval
def process_monthly_channel(channel_id, aims_xml_info, level_qc):
""" Downloads all the data available for one channel_id and moves the file to a wip_path dir
channel_id(str)
aims_xml_info(tuple)
level_qc(int)
aims_service : 1 -> FAIMMS data
100 -> SOOP TRV data
300 -> NRS DATA
for monthly data download, only 1 and 300 should be use
"""
logger.info('>> QC%s - Processing channel %s' % (str(level_qc), str(channel_id)))
channel_id_info = aims_xml_info[channel_id]
from_date = channel_id_info['from_date']
thru_date = channel_id_info['thru_date']
[start_dates, end_dates] = create_list_of_dates_to_download(channel_id, level_qc, from_date, thru_date)
if len(start_dates) != 0:
# download monthly file
for start_date, end_date in zip(start_dates, end_dates):
start_date = start_date.strftime("%Y-%m-%dT%H:%M:%SZ")
end_date = end_date.strftime("%Y-%m-%dT%H:%M:%SZ")
netcdf_tmp_file_path = download_channel(channel_id, start_date, end_date, level_qc)
contact_aims_msg = "Process of channel aborted - CONTACT AIMS"
if netcdf_tmp_file_path is None:
logger.error(' Channel %s - not valid zip file - %s' % (str(channel_id), contact_aims_msg))
break
# NO_DATA_FOUND file only means there is no data for the selected time period. Could be some data afterwards
if is_no_data_found(netcdf_tmp_file_path):
logger.warning(' Channel %s - No data for the time period:%s - %s' % (str(channel_id), start_date, end_date))
shutil.rmtree(os.path.dirname(netcdf_tmp_file_path))
else:
if is_time_var_empty(netcdf_tmp_file_path):
logger.error(' Channel %s - No values in TIME variable - %s' % (str(channel_id), contact_aims_msg))
shutil.rmtree(os.path.dirname(netcdf_tmp_file_path))
break
if not modify_anmn_nrs_netcdf(netcdf_tmp_file_path, channel_id_info):
logger.error(' Channel %s - Could not modify the NetCDF file - Process of channel aborted' % str(channel_id))
shutil.rmtree(os.path.dirname(netcdf_tmp_file_path))
break
main_var = get_main_netcdf_var(netcdf_tmp_file_path)
if has_var_only_fill_value(netcdf_tmp_file_path, main_var):
logger.error(' Channel %s - _Fillvalues only in main variable - %s' % (str(channel_id), contact_aims_msg))
shutil.rmtree(os.path.dirname(netcdf_tmp_file_path))
break
if get_anmn_nrs_site_name(netcdf_tmp_file_path) == []:
logger.error(' Channel %s - Unknown site_code gatt value - %s' % (str(channel_id), contact_aims_msg))
shutil.rmtree(os.path.dirname(netcdf_tmp_file_path))
break
if not is_time_monotonic(netcdf_tmp_file_path):
logger.error(' Channel %s - TIME value is not strickly monotonic - %s' % (str(channel_id), contact_aims_msg))
shutil.rmtree(os.path.dirname(netcdf_tmp_file_path))
break
# check every single file of the list. We don't assume that if one passes, all pass ... past proved this
wip_path = os.environ.get('data_wip_path')
checker_retval = pass_netcdf_checker(netcdf_tmp_file_path, tests=['cf:latest', 'imos:1.3'])
if not checker_retval:
logger.error(' Channel %s - File does not pass CF/IMOS compliance checker - Process of channel aborted' % str(channel_id))
shutil.copy(netcdf_tmp_file_path, os.path.join(wip_path, 'errors'))
logger.error(' File copied to %s for debugging' % (os.path.join(wip_path, 'errors', os.path.basename(netcdf_tmp_file_path))))
shutil.rmtree(os.path.dirname(netcdf_tmp_file_path))
break
netcdf_tmp_file_path = fix_data_code_from_filename(netcdf_tmp_file_path)
netcdf_tmp_file_path = fix_provider_code_from_filename(netcdf_tmp_file_path, 'IMOS_ANMN')
if re.search('IMOS_ANMN_[A-Z]{1}_', netcdf_tmp_file_path) is None:
logger.error(' Channel %s - File name Data code does not pass REGEX - Process of channel aborted' % str(channel_id))
shutil.copy(netcdf_tmp_file_path, os.path.join(wip_path, 'errors'))
logger.error(' File copied to %s for debugging' % (os.path.join(wip_path, 'errors', os.path.basename(netcdf_tmp_file_path))))
shutil.rmtree(os.path.dirname(netcdf_tmp_file_path))
break
move_to_tmp_incoming(netcdf_tmp_file_path)
if TESTING:
# The 2 next lines download the first month only for every single channel. This is only used for testing
save_channel_info(channel_id, aims_xml_info, level_qc, end_date)
break
save_channel_info(channel_id, aims_xml_info, level_qc, end_date)
else:
logger.info('QC%s - Channel %s already up to date' % (str(level_qc), str(channel_id)))
close_logger(logger)