def process_qc_level(level_qc):
""" Downloads all channels for a QC level
level_qc(int) : 0 or 1"""
logger.info(
'Process SOOP-TRV download from AIMS web service - QC level {level_qc}'
.format(level_qc=str(level_qc)))
xml_url = 'http://data.aims.gov.au/gbroosdata/services/rss/netcdf/level%s/100' % str(
level_qc)
try:
aims_xml_info = parse_aims_xml(xml_url)
except Exception as err:
logger.error('RSS feed not available')
exit(1)
for channel_id in aims_xml_info.keys():
try:
is_channel_processed = process_channel(channel_id, aims_xml_info,
level_qc)
if is_channel_processed:
save_channel_info(channel_id, aims_xml_info, level_qc)
except Exception as err:
logger.error('QC%s - Channel %s:ailed, unknown reason - manual \
debug required' % (str(level_qc), str(channel_id)))
logger.error(str(err))
logger.error(traceback.print_exc())
def process_qc_level(level_qc):
""" Downloads all channels for a QC level
level_qc(int) : 0 or 1"""
logger.info('Process SOOP-TRV download from AIMS web service - QC level %s' % str(level_qc))
xml_url = 'http://data.aims.gov.au/gbroosdata/services/rss/netcdf/level%s/100' % str(level_qc)
try:
aims_xml_info = parse_aims_xml(xml_url)
except:
logger.error('RSS feed not available')
exit(1)
for channel_id in aims_xml_info.keys():
try:
is_channel_processed = process_channel(channel_id, aims_xml_info,
level_qc)
if is_channel_processed:
save_channel_info(channel_id, aims_xml_info, level_qc)
except Exception, e:
logger.error(' Channel %s QC%s - Failed, unknown reason - manual \
debug required' % (str(channel_id), str(level_qc)))
logger.error(str(e))
logger.error(traceback.print_exc())
def process_monthly_channel(channel_id, aims_xml_info, level_qc):
""" Downloads all the data available for one channel_id and moves the file to a wip_path dir
channel_id(str)
aims_xml_info(tuple)
level_qc(int)
aims_service : 1 -> FAIMMS data
100 -> SOOP TRV data
300 -> NRS DATA
for monthly data download, only 1 and 300 should be use
"""
logger.info('>> QC%s - Processing channel %s' % (str(level_qc), str(channel_id)))
channel_id_info = aims_xml_info[channel_id]
from_date = channel_id_info['from_date']
thru_date = channel_id_info['thru_date']
[start_dates, end_dates] = create_list_of_dates_to_download(channel_id, level_qc, from_date, thru_date)
if len(start_dates) != 0:
# download monthly file
for start_date, end_date in zip(start_dates, end_dates):
start_date = start_date.strftime("%Y-%m-%dT%H:%M:%SZ")
end_date = end_date.strftime("%Y-%m-%dT%H:%M:%SZ")
netcdf_tmp_file_path = download_channel(channel_id, start_date, end_date, level_qc)
contact_aims_msg = "Process of channel aborted - CONTACT AIMS"
if netcdf_tmp_file_path is None:
logger.error(' Channel %s - not valid zip file - %s' % (str(channel_id), contact_aims_msg))
break
# NO_DATA_FOUND file only means there is no data for the selected time period. Could be some data afterwards
if is_no_data_found(netcdf_tmp_file_path):
logger.warning(' Channel %s - No data for the time period:%s - %s' % (str(channel_id), start_date, end_date))
shutil.rmtree(os.path.dirname(netcdf_tmp_file_path))
else:
if is_time_var_empty(netcdf_tmp_file_path):
logger.error(' Channel %s - No values in TIME variable - %s' % (str(channel_id), contact_aims_msg))
shutil.rmtree(os.path.dirname(netcdf_tmp_file_path))
break
if not modify_anmn_nrs_netcdf(netcdf_tmp_file_path, channel_id_info):
logger.error(' Channel %s - Could not modify the NetCDF file - Process of channel aborted' % str(channel_id))
shutil.rmtree(os.path.dirname(netcdf_tmp_file_path))
break
main_var = get_main_netcdf_var(netcdf_tmp_file_path)
if has_var_only_fill_value(netcdf_tmp_file_path, main_var):
logger.error(' Channel %s - _Fillvalues only in main variable - %s' % (str(channel_id), contact_aims_msg))
shutil.rmtree(os.path.dirname(netcdf_tmp_file_path))
break
if get_anmn_nrs_site_name(netcdf_tmp_file_path) == []:
logger.error(' Channel %s - Unknown site_code gatt value - %s' % (str(channel_id), contact_aims_msg))
shutil.rmtree(os.path.dirname(netcdf_tmp_file_path))
break
if not is_time_monotonic(netcdf_tmp_file_path):
logger.error(' Channel %s - TIME value is not strickly monotonic - %s' % (str(channel_id), contact_aims_msg))
shutil.rmtree(os.path.dirname(netcdf_tmp_file_path))
break
# check every single file of the list. We don't assume that if one passes, all pass ... past proved this
wip_path = os.environ.get('data_wip_path')
checker_retval = pass_netcdf_checker(netcdf_tmp_file_path, tests=['cf:latest', 'imos:1.3'])
if not checker_retval:
logger.error(' Channel %s - File does not pass CF/IMOS compliance checker - Process of channel aborted' % str(channel_id))
shutil.copy(netcdf_tmp_file_path, os.path.join(wip_path, 'errors'))
logger.error(' File copied to %s for debugging' % (os.path.join(wip_path, 'errors', os.path.basename(netcdf_tmp_file_path))))
shutil.rmtree(os.path.dirname(netcdf_tmp_file_path))
break
netcdf_tmp_file_path = fix_data_code_from_filename(netcdf_tmp_file_path)
netcdf_tmp_file_path = fix_provider_code_from_filename(netcdf_tmp_file_path, 'IMOS_ANMN')
if re.search('IMOS_ANMN_[A-Z]{1}_', netcdf_tmp_file_path) is None:
logger.error(' Channel %s - File name Data code does not pass REGEX - Process of channel aborted' % str(channel_id))
shutil.copy(netcdf_tmp_file_path, os.path.join(wip_path, 'errors'))
logger.error(' File copied to %s for debugging' % (os.path.join(wip_path, 'errors', os.path.basename(netcdf_tmp_file_path))))
shutil.rmtree(os.path.dirname(netcdf_tmp_file_path))
break
move_to_tmp_incoming(netcdf_tmp_file_path)
if TESTING:
# The 2 next lines download the first month only for every single channel. This is only used for testing
save_channel_info(channel_id, aims_xml_info, level_qc, end_date)
break
save_channel_info(channel_id, aims_xml_info, level_qc, end_date)
else:
logger.info('QC%s - Channel %s already up to date' % (str(level_qc), str(channel_id)))
close_logger(logger)
def process_monthly_channel(channel_id, aims_xml_info, level_qc):
""" Downloads all the data available for one channel_id and moves the file to a wip_path dir
channel_id(str)
aims_xml_info(tuple)
level_qc(int)
aims_service : 1 -> FAIMMS data
100 -> SOOP TRV data
300 -> NRS DATA
for monthly data download, only 1 and 300 should be use
"""
logger.info('>> QC%s - Processing channel %s' % (str(level_qc), str(channel_id)))
channel_id_info = aims_xml_info[channel_id]
from_date = channel_id_info['from_date']
thru_date = channel_id_info['thru_date']
[start_dates, end_dates] = create_list_of_dates_to_download(channel_id, level_qc, from_date, thru_date)
if len(start_dates) != 0:
# download monthly file
for start_date, end_date in zip(start_dates, end_dates):
start_date = start_date.strftime("%Y-%m-%dT%H:%M:%SZ")
end_date = end_date.strftime("%Y-%m-%dT%H:%M:%SZ")
netcdf_tmp_file_path = download_channel(channel_id, start_date, end_date, level_qc)
contact_aims_msg = "Process of channel aborted - CONTACT AIMS"
if netcdf_tmp_file_path is None:
logger.error(' Channel %s - not valid zip file - %s' % (str(channel_id), contact_aims_msg))
break
# NO_DATA_FOUND file only means there is no data for the selected time period. Could be some data afterwards
if is_no_data_found(netcdf_tmp_file_path):
logger.warning(' Channel %s - No data for the time period:%s - %s' % (str(channel_id), start_date, end_date))
shutil.rmtree(os.path.dirname(netcdf_tmp_file_path))
else:
if is_time_var_empty(netcdf_tmp_file_path):
logger.error(' Channel %s - No values in TIME variable - %s' % (str(channel_id), contact_aims_msg))
shutil.rmtree(os.path.dirname(netcdf_tmp_file_path))
break
if not modify_faimms_netcdf(netcdf_tmp_file_path, channel_id_info):
logger.error(' Channel %s - Could not modify the NetCDF file - Process of channel aborted' % str(channel_id))
shutil.rmtree(os.path.dirname(netcdf_tmp_file_path))
break
main_var = get_main_faimms_var(netcdf_tmp_file_path)
if has_var_only_fill_value(netcdf_tmp_file_path, main_var):
logger.error(' Channel %s - _Fillvalues only in main variable - %s' % (str(channel_id), contact_aims_msg))
shutil.rmtree(os.path.dirname(netcdf_tmp_file_path))
break
if get_faimms_site_name(netcdf_tmp_file_path) == [] or get_faimms_platform_type(netcdf_tmp_file_path) == []:
logger.error(' Channel %s - Unknown site_code gatt value - %s' % (str(channel_id), contact_aims_msg))
shutil.rmtree(os.path.dirname(netcdf_tmp_file_path))
break
if not is_time_monotonic(netcdf_tmp_file_path):
logger.error(' Channel %s - TIME value is not strickly monotonic - %s' % (str(channel_id), contact_aims_msg))
shutil.rmtree(os.path.dirname(netcdf_tmp_file_path))
break
# check every single file of the list. We don't assume that if one passes, all pass ... past proved this
wip_path = DATA_WIP_PATH
checker_retval = pass_netcdf_checker(netcdf_tmp_file_path, tests=['cf:latest', 'imos:1.3'])
if not checker_retval:
logger.error(' Channel %s - File does not pass CF/IMOS compliance checker - Process of channel aborted' % str(channel_id))
shutil.copy(netcdf_tmp_file_path, os.path.join(wip_path, 'errors'))
logger.error(' File copied to %s for debugging' % (os.path.join(wip_path, 'errors', os.path.basename(netcdf_tmp_file_path))))
shutil.rmtree(os.path.dirname(netcdf_tmp_file_path))
break
netcdf_tmp_file_path = fix_data_code_from_filename(netcdf_tmp_file_path)
netcdf_tmp_file_path = fix_provider_code_from_filename(netcdf_tmp_file_path, 'IMOS_FAIMMS')
if re.search('IMOS_FAIMMS_[A-Z]{1}_', netcdf_tmp_file_path) is None:
logger.error(' Channel %s - File name Data code does not pass REGEX - Process of channel aborted' % str(channel_id))
shutil.copy(netcdf_tmp_file_path, os.path.join(wip_path, 'errors'))
logger.error(' File copied to %s for debugging' % (os.path.join(wip_path, 'errors', os.path.basename(netcdf_tmp_file_path))))
shutil.rmtree(os.path.dirname(netcdf_tmp_file_path))
break
move_to_tmp_incoming(netcdf_tmp_file_path)
if TESTING:
# The 2 next lines download the first month only for every single channel. This is only used for testing
save_channel_info(channel_id, aims_xml_info, level_qc, end_date)
break
save_channel_info(channel_id, aims_xml_info, level_qc, end_date)
else:
logger.info('QC%s - Channel %s already up to date' % (str(level_qc), str(channel_id)))
close_logger(logger)