def setUp(self):
""" Check that a the AIMS system or this script hasn't been modified.
This function checks that a downloaded file still has the same md5.
"""
logging_aims()
channel_id = '8365'
from_date = '2008-09-30T00:27:27Z'
thru_date = '2008-09-30T00:30:00Z'
level_qc = 1
aims_rss_val = 100
xml_url = 'http://data.aims.gov.au/gbroosdata/services/rss/netcdf/level%s/%s' % (str(level_qc), str(aims_rss_val))
aims_xml_info = parse_aims_xml(xml_url)
channel_id_info = aims_xml_info[channel_id]
self.netcdf_tmp_file_path = download_channel(channel_id, from_date, thru_date, level_qc)
modify_soop_trv_netcdf(self.netcdf_tmp_file_path, channel_id_info)
# force values of attributes which change all the time
netcdf_file_obj = Dataset(self.netcdf_tmp_file_path, 'a', format='NETCDF4')
netcdf_file_obj.date_created = "1970-01-01T00:00:00Z"
netcdf_file_obj.history = 'data validation test only'
netcdf_file_obj.close()
shutil.move(self.netcdf_tmp_file_path, remove_creation_date_from_filename(self.netcdf_tmp_file_path))
self.netcdf_tmp_file_path = remove_creation_date_from_filename(self.netcdf_tmp_file_path)
def setUp(self):
""" Check that a the AIMS system or this script hasn't been modified.
This function checks that a downloaded file still has the same md5.
"""
logging_aims()
channel_id = '9272'
from_date = '2016-01-01T00:00:00Z'
thru_date = '2016-01-02T00:00:00Z'
level_qc = 1
faimms_rss_val = 1
xml_url = 'http://data.aims.gov.au/gbroosdata/services/rss/netcdf/level%s/%s' % (
str(level_qc), str(faimms_rss_val))
aims_xml_info = parse_aims_xml(xml_url)
channel_id_info = aims_xml_info[channel_id]
self.netcdf_tmp_file_path = download_channel(channel_id, from_date,
thru_date, level_qc)
modify_faimms_netcdf(self.netcdf_tmp_file_path, channel_id_info)
# force values of attributes which change all the time
netcdf_file_obj = Dataset(self.netcdf_tmp_file_path,
'a',
format='NETCDF4')
netcdf_file_obj.date_created = "1970-01-01T00:00:00Z" # epoch
netcdf_file_obj.history = 'data validation test only'
netcdf_file_obj.close()
def setUp(self):
""" Check that a the AIMS system or this script hasn't been modified.
This function checks that a downloaded file still has the same md5.
"""
channel_id = '8365'
from_date = '2008-09-30T00:27:27Z'
thru_date = '2008-09-30T00:30:00Z'
level_qc = 1
aims_rss_val = 100
xml_url = 'http://data.aims.gov.au/gbroosdata/services/rss/netcdf/level%s/%s' % (
str(level_qc), str(aims_rss_val))
logger.info('Data validation unittests...')
aims_xml_info = parse_aims_xml(xml_url)
channel_id_info = aims_xml_info[channel_id]
self.netcdf_tmp_file_path = download_channel(channel_id, from_date,
thru_date, level_qc)
modify_soop_trv_netcdf(self.netcdf_tmp_file_path, channel_id_info)
# force values of attributes which change all the time
netcdf_file_obj = Dataset(self.netcdf_tmp_file_path,
'a',
format='NETCDF4')
netcdf_file_obj.date_created = "1970-01-01T00:00:00Z"
netcdf_file_obj.history = 'data validation test only'
netcdf_file_obj.NCO = 'NCO_VERSION'
netcdf_file_obj.close()
shutil.move(
self.netcdf_tmp_file_path,
remove_creation_date_from_filename(self.netcdf_tmp_file_path))
self.netcdf_tmp_file_path = remove_creation_date_from_filename(
self.netcdf_tmp_file_path)
def process_qc_level(level_qc):
""" Downloads all channels for a QC level
level_qc(int) : 0 or 1"""
logger.info(
'Process SOOP-TRV download from AIMS web service - QC level {level_qc}'
.format(level_qc=str(level_qc)))
xml_url = 'http://data.aims.gov.au/gbroosdata/services/rss/netcdf/level%s/100' % str(
level_qc)
try:
aims_xml_info = parse_aims_xml(xml_url)
except Exception as err:
logger.error('RSS feed not available')
exit(1)
for channel_id in aims_xml_info.keys():
try:
is_channel_processed = process_channel(channel_id, aims_xml_info,
level_qc)
if is_channel_processed:
save_channel_info(channel_id, aims_xml_info, level_qc)
except Exception as err:
logger.error('QC%s - Channel %s:ailed, unknown reason - manual \
debug required' % (str(level_qc), str(channel_id)))
logger.error(str(err))
logger.error(traceback.print_exc())
def process_qc_level(level_qc):
""" Downloads all channels for a QC level
level_qc(int) : 0 or 1
"""
logger.info('Process ANMN NRS download from AIMS web service - QC level {level_qc}'.format(level_qc=level_qc))
xml_url = 'http://data.aims.gov.au/gbroosdata/services/rss/netcdf/level{level_qc}/300'.format(level_qc=level_qc)
try:
aims_xml_info = parse_aims_xml(xml_url)
except Exception as err:
logger.error('RSS feed not available')
exit(1)
for channel_id in aims_xml_info.keys():
try:
process_monthly_channel(channel_id, aims_xml_info, level_qc)
except Exception as err:
logger.error(' Channel %s QC%s - Failed, unknown reason - manual debug required' % (str(channel_id), str(level_qc)))
logger.error(traceback.print_exc())
def process_qc_level(level_qc):
""" Downloads all channels for a QC level
level_qc(int) : 0 or 1
"""
logger.info('Process FAIMMS download from AIMS web service - QC level %s' % str(level_qc))
xml_url = 'http://data.aims.gov.au/gbroosdata/services/rss/netcdf/level%s/1' % str(level_qc)
try:
aims_xml_info = parse_aims_xml(xml_url)
except:
logger.error('RSS feed not available')
exit(1)
for channel_id in aims_xml_info.keys():
try:
process_monthly_channel(channel_id, aims_xml_info, level_qc)
except Exception, e:
logger.error(' Channel %s QC%s - Failed, unknown reason - manual debug required' % (str(channel_id), str(level_qc)))
logger.error(traceback.print_exc())