def test_bgc_timeseries(self):
filename = 'IMOS_ANMN-NRS_KOSTUZ_20150330T080039Z_NRSROT_FV01_NRSROT-1503-WQM-55_END-20150727T063234Z_C-20150731T040136Z.nc'
testfile = os.path.join(self.tempdir, filename)
make_test_file(testfile, {
'site_code': 'NRSROT',
'featureType': 'timeSeries'
},
TEMP={},
PRES_REL={},
DEPTH={},
PSAL={},
DOX2={},
CPHL={},
TURB={})
dest_dir, dest_filename = os.path.split(
MooringsFileClassifier.dest_path(testfile))
self.assertEqual(dest_dir,
'IMOS/ANMN/NRS/NRSROT/Biogeochem_timeseries')
self.assertEqual(dest_filename, filename)
filename = 'IMOS_ANMN-QLD_KUZ_20150328T203001Z_GBROTE_FV01_GBROTE-1503-ECO-FLNTUSB-18_END-20151013T030538Z_C-20160225T042413Z.nc'
testfile = os.path.join(self.tempdir, filename)
make_test_file(testfile, {
'site_code': 'GBROTE',
'featureType': 'timeseries'
},
DEPTH={},
CPHL={},
TURB={})
dest_dir, dest_filename = os.path.split(
MooringsFileClassifier.dest_path(testfile))
self.assertEqual(dest_dir,
'IMOS/ANMN/QLD/GBROTE/Biogeochem_timeseries')
self.assertEqual(dest_filename, filename)
def test_bgc_profiles(self):
filename = 'IMOS_ANMN-NRS_CDEKOSTUZ_20121113T001841Z_NRSMAI_FV00_Profile-SBE-19plus_C-20151030T034432Z.nc'
testfile = os.path.join(self.tempdir, filename)
make_test_file(testfile, {
'site_code': 'NRSMAI',
'featureType': 'profile'
},
TEMP={},
PRES={},
CNDC={})
dest_dir, dest_filename = os.path.split(
MooringsFileClassifier.dest_path(testfile))
self.assertEqual(dest_dir,
'IMOS/ANMN/NRS/NRSMAI/Biogeochem_profiles/non-QC')
self.assertEqual(dest_filename, filename)
filename = 'IMOS_ANMN-WA_CDEKOSTUZ_20121113T013800Z_WACA20_FV01_3052.0-1-SBE19plus-70_C-20140211T090215Z.nc'
testfile = os.path.join(self.tempdir, filename)
make_test_file(testfile, {
'site_code': 'WACA20',
'featureType': 'profile'
},
TEMP={},
PRES_REL={},
PSAL={},
DOX2={})
dest_dir, dest_filename = os.path.split(
MooringsFileClassifier.dest_path(testfile))
self.assertEqual(dest_dir, 'IMOS/ANMN/WA/WACA20/Biogeochem_profiles')
self.assertEqual(dest_filename, filename)
def test_velocity(self):
filename = 'IMOS_ANMN-NRS_AETVZ_20150703T053000Z_NRSROT-ADCP_FV01_NRSROT-ADCP-1507-Workhorse-ADCP-43_END-20151023T034500Z_C-20151117T074309Z.nc'
testfile = os.path.join(self.tempdir, filename)
make_test_file(testfile, {'site_code': 'NRSROT'},
TEMP={},
PRES_REL={},
DEPTH={},
UCUR={},
VCUR={},
WCUR={})
dest_dir, dest_filename = os.path.split(
MooringsFileClassifier.dest_path(testfile))
self.assertEqual(dest_dir, 'IMOS/ANMN/NRS/NRSROT/Velocity')
self.assertEqual(dest_filename, filename)
filename = 'IMOS_ANMN-NRS_AETVZ_20150703T053000Z_NRSROT-ADCP_FV00_NRSROT-ADCP-1507-Workhorse-ADCP-43_END-20151023T034500Z_C-20151117T074309Z.nc'
testfile = os.path.join(self.tempdir, filename)
make_test_file(testfile, {'site_code': 'NRSROT'},
UCUR_MAG={},
VCUR_MAG={},
CSPD={})
dest_dir, dest_filename = os.path.split(
MooringsFileClassifier.dest_path(testfile))
self.assertEqual(dest_dir, 'IMOS/ANMN/NRS/NRSROT/Velocity/non-QC')
self.assertEqual(dest_filename, filename)
def test_ctd_timeseries(self):
filename = 'IMOS_ANMN-WA_CSTZ_20141117T080001Z_WATR10_FV01_WATR10-1411-SBE37SM-RS232-52.nc'
testfile = os.path.join(self.tempdir, filename)
make_test_file(testfile, {
'site_code': 'WATR10',
'featureType': 'timeSeries'
},
TEMP={},
PRES={},
CNDC={})
dest_dir, dest_filename = os.path.split(
MooringsFileClassifier.dest_path(testfile))
self.assertEqual(dest_dir, 'IMOS/ANMN/WA/WATR10/CTD_timeseries')
self.assertEqual(dest_filename, filename)
filename = 'IMOS_ANMN-SA_ACESTZ_20141201T030411Z_SAM8SG-1412_FV01_SAM8SG-1412-NXIC-CTD-44.71_END-20150411T020421Z_C-20150730T044018Z.nc'
testfile = os.path.join(self.tempdir, filename)
make_test_file(testfile, {
'site_code': 'SAM8SG',
'featureType': 'timeSeries'
},
TEMP={},
PRES_REL={},
DEPTH={},
PSAL={},
CNDC={},
SSPD={})
dest_dir, dest_filename = os.path.split(
MooringsFileClassifier.dest_path(testfile))
self.assertEqual(dest_dir, 'IMOS/ANMN/SA/SAM8SG/CTD_timeseries')
self.assertEqual(dest_filename, filename)
def test_missing_site_code(self):
testfile = os.path.join(
self.tempdir,
'IMOS_ANMN-NRS_CDEKOSTUZ_20121113T001841Z_BADBAD_FV01_Profile.nc')
make_test_file(testfile)
with self.assertRaisesRegex(InvalidFileContentError,
r"has no attribute 'site_code'"):
MooringsFileClassifier.dest_path(testfile)
def test_missing_featuretype(self):
testfile = os.path.join(
self.tempdir,
'IMOS_ANMN-NRS_CDEKOSTUZ_20121113T001841Z_NRSMAI_FV01_Profile-SBE-19plus_C-20151030T034432Z.nc'
)
make_test_file(testfile, {'site_code': 'NRSMAI'})
with self.assertRaisesRegex(InvalidFileContentError,
r"has no attribute 'featureType'"):
MooringsFileClassifier.dest_path(testfile)
def test_sofs_waves_rt(self):
filename = 'IMOS_DWM-SOTS_W_20190318_SOFS_FV00_SOFS-8-2019-MRU-Surface-wave-height-realtime.nc'
testfile = os.path.join(self.tempdir, filename)
make_test_file(
testfile, {
'time_coverage_start': "2019-03-18T00:00:00Z",
'time_coverage_end': "2019-10-31T00:00:00Z"
})
dest_dir, dest_filename = os.path.split(
DwmFileClassifier.dest_path(testfile))
self.assertEqual(dest_dir, 'IMOS/DWM/SOTS/2019/real-time')
self.assertEqual(dest_filename, filename)
def test_unknown_profiles(self):
filename = 'IMOS_ANMN-NRS_CDEKOSTUZ_20121113T001841Z_NRSMAI_FV00_mystery-profile.nc'
testfile = os.path.join(self.tempdir, filename)
make_test_file(testfile, {
'site_code': 'NRSMAI',
'featureType': 'profile'
},
TIME={},
DEPTH={})
with self.assertRaisesRegex(InvalidFileContentError,
r"Could not determine data category"):
MooringsFileClassifier.dest_path(testfile)
def test_unknown_product(self):
filename = 'IMOS_ANMN-NRS_STZ_20170101T000000Z_NRSMAI_FV02.nc'
testfile = os.path.join(self.tempdir, filename)
make_test_file(testfile, {
'site_code': 'NRSMAI',
'featureType': 'timeSeriesProfile'
},
TEMP={},
PSAL={})
with self.assertRaisesRegex(
InvalidFileNameError,
r"Can't determine product type from file name"):
MooringsFileClassifier.dest_path(testfile)
def test_long_timeseries(self):
filename = 'IMOS_ANMN-NRS_STZ_19441015T000000Z_NRSMAI_FV02_NRSMAI-long-timeseries_END-20140703T000000Z_C-20160525T064856Z.nc'
testfile = os.path.join(self.tempdir, filename)
make_test_file(testfile, {
'site_code': 'NRSMAI',
'featureType': 'timeSeriesProfile'
},
TEMP={},
PSAL={})
dest_dir, dest_filename = os.path.split(
MooringsFileClassifier.dest_path(testfile))
self.assertEqual(dest_dir, 'IMOS/ANMN/NRS/NRSMAI/aggregated_products')
self.assertEqual(dest_filename, filename)
def test_wave(self):
filename = 'IMOS_ANMN-NRS_WZ_20140914T075900Z_NRSDAR_FV01_NRSDAR-1409-SUB-Workhorse-ADCP-24.3_END-20150205T225900Z_C-20150326T055936Z.nc'
testfile = os.path.join(self.tempdir, filename)
make_test_file(testfile, {
'site_code': 'NRSDAR',
'featureType': 'doesntmatter'
},
DEPTH={},
VAVH={})
dest_dir, dest_filename = os.path.split(
MooringsFileClassifier.dest_path(testfile))
self.assertEqual(dest_dir, 'IMOS/ANMN/NRS/NRSDAR/Wave')
self.assertEqual(dest_filename, filename)
def test_temperature_gridded(self):
filename = 'IMOS_ANMN-NSW_Temperature_20100702T003500Z_CH070_FV02_CH070-1007-regridded_END-20100907T000500Z_C-20141211T025746Z.nc'
testfile = os.path.join(self.tempdir, filename)
make_test_file(testfile, {
'site_code': 'CH070',
'featureType': 'timeSeriesProfile'
},
TEMP={},
DEPTH={})
dest_dir, dest_filename = os.path.split(
MooringsFileClassifier.dest_path(testfile))
self.assertEqual(dest_dir, 'IMOS/ANMN/NSW/CH070/Temperature/gridded')
self.assertEqual(dest_filename, filename)
def test_sofs_one_day_delayed(self):
filename = 'IMOS_DWM-ASFS_CMST_20150101T000000Z_SOFS_FV02_C-20171002T000000Z.nc'
testfile = os.path.join(self.tempdir, filename)
make_test_file(
testfile, {
'data_mode': 'D',
'time_coverage_start': '2015-01-01T00:00:00Z',
'time_coverage_end': '2015-01-01T23:30:00Z'
})
dest_dir, dest_filename = os.path.split(
DwmFileClassifier.dest_path(testfile))
self.assertEqual(dest_dir, 'IMOS/DWM/SOTS/2015')
self.assertEqual(dest_filename, filename)
def test_pulse_realtime(self):
filename = 'IMOS_DWM-SOTS_W_20150325T110000Z_Pulse_FV00_Pulse-11-2015-MRU-Surface-wave-height-realtime.nc'
testfile = os.path.join(self.tempdir, filename)
make_test_file(
testfile, {
'site_code': 'SOTS',
'platform_code': 'Pulse',
'time_coverage_start': '2015-03-25T11:00:00Z'
})
dest_dir, dest_filename = os.path.split(
DwmFileClassifier.dest_path(testfile))
self.assertEqual(dest_dir, 'IMOS/DWM/SOTS/2015/real-time')
self.assertEqual(dest_filename, filename)
def test_da_temperature(self):
filename = 'IMOS_DWM-DA_TZ_20120426T092000Z_EAC5_FV01_EAC5-2012-STARMON-MINI-300_END-20130826T222000Z_C' \
'-20140722T061401Z.nc '
testfile = os.path.join(self.tempdir, filename)
make_test_file(testfile, {
'platform_code': 'EAC5',
'featureType': 'timeSeries'
},
TEMP={},
DEPTH={})
dest_dir, dest_filename = os.path.split(
DwmFileClassifier.dest_path(testfile))
self.assertEqual(dest_dir, 'IMOS/DWM/DA/EAC5/Temperature')
self.assertEqual(dest_filename, filename)
def test_dwm_sots_aggregated_timeseries(self):
expected_prefix = 'IMOS/DWM/SOTS/aggregated_timeseries'
filename = 'IMOS_DWM-SOTS_OZ_20170319_SOTS_FV01_DOX2-aggregated-timeseries_END-20171101_C-20190819.nc'
testfile = os.path.join(self.temp_dir, filename)
make_test_file(
testfile, {
'site_code': 'EAC4700',
'source': 'mooring',
'featureType': 'timeSeries'
})
dest_dir, dest_filename = os.path.split(
MooringsProductsHandler.dest_path(testfile))
self.assertEqual(dest_dir, expected_prefix)
self.assertEqual(dest_filename, filename)
def test_dwm_hourly_timeseries_nonqc(self):
expected_prefix = 'IMOS/DWM/DA/hourly_timeseries'
filename = 'IMOS_DWM-DA_STZ_20110613_ITFTSL_FV02_hourly-timeseries-including-non-QC_END-20151024_C-20191010.nc'
testfile = os.path.join(self.temp_dir, filename)
make_test_file(
testfile, {
'site_code': 'ITFTSL',
'source': 'mooring',
'featureType': 'timeSeries'
})
dest_dir, dest_filename = os.path.split(
MooringsProductsHandler.dest_path(testfile))
self.assertEqual(dest_dir, expected_prefix)
self.assertEqual(dest_filename, filename)
def test_dwm_gridded_timeseries(self):
expected_prefix = 'IMOS/DWM/DA/gridded_timeseries'
filename = 'IMOS_DWM-DA_TZ_20150519_EAC4700_FV02_TEMP-gridded-timeseries_END-20180422_C-20191216.nc'
testfile = os.path.join(self.temp_dir, filename)
make_test_file(
testfile, {
'site_code': 'EAC4700',
'source': 'mooring',
'featureType': 'timeSeries'
})
dest_dir, dest_filename = os.path.split(
MooringsProductsHandler.dest_path(testfile))
self.assertEqual(dest_dir, expected_prefix)
self.assertEqual(dest_filename, filename)
def test_anmn_hourly_timeseries_nonqc(self):
expected_prefix = 'IMOS/ANMN/QLD/GBRLSL/hourly_timeseries'
filename = 'IMOS_ANMN-QLD_BSTUZ_20071103_GBRLSL_FV02_hourly-timeseries-including-non-QC_END-20140523_C-20191010.nc'
testfile = os.path.join(self.temp_dir, filename)
make_test_file(
testfile, {
'site_code': 'GBRLSL',
'source': 'mooring',
'featureType': 'timeSeries'
})
dest_dir, dest_filename = os.path.split(
MooringsProductsHandler.dest_path(testfile))
self.assertEqual(dest_dir, expected_prefix)
self.assertEqual(dest_filename, filename)
def test_anmn_gridded_timeseries(self):
expected_prefix = 'IMOS/ANMN/NRS/NRSROT/gridded_timeseries'
filename = 'IMOS_ANMN-NRS_SZ_20081120_NRSROT_FV02_PSAL-gridded-timeseries_END-20190523_C-20191121.nc'
testfile = os.path.join(self.temp_dir, filename)
make_test_file(
testfile, {
'site_code': 'NRSROT',
'source': 'mooring',
'featureType': 'timeSeriesProfile'
})
dest_dir, dest_filename = os.path.split(
MooringsProductsHandler.dest_path(testfile))
self.assertEqual(dest_dir, expected_prefix)
self.assertEqual(dest_filename, filename)
def test_pressure_only(self):
# Files will only pressure are also classified as "Temperature".
filename = 'IMOS_ANMN-WA_Z_20120914T032100Z_WATR50_FV01_WATR50-1209-DR-1050-517_END-20130319T053000Z_C-20130325T032512Z.nc'
testfile = os.path.join(self.tempdir, filename)
make_test_file(testfile, {
'site_code': 'WATR50',
'featureType': 'timeSeries'
},
PRES={},
DEPTH={})
dest_dir, dest_filename = os.path.split(
MooringsFileClassifier.dest_path(testfile))
self.assertEqual(dest_dir, 'IMOS/ANMN/WA/WATR50/Temperature')
self.assertEqual(dest_filename, filename)
def test_anmn_aggregated_timeseries(self):
expected_prefix = 'IMOS/ANMN/QLD/GBRLSL/aggregated_timeseries'
filename = 'IMOS_ANMN-QLD_BZ_20121103_GBRLSL_FV01_CPHL-aggregated-timeseries_END-20140522_C-20191120.nc'
testfile = os.path.join(self.temp_dir, filename)
make_test_file(
testfile, {
'site_code': 'GBRLSL',
'source': 'mooring',
'featureType': 'timeSeries'
})
dest_dir, dest_filename = os.path.split(
MooringsProductClassifier.dest_path(testfile))
self.assertEqual(dest_dir, expected_prefix)
self.assertEqual(dest_filename, filename)
def test_temperature(self):
filename = 'IMOS_ANMN-NSW_TZ_20150310T130000Z_PH100_FV01_PH100-1503-Aqualogger-520T-16_END-20150606T025000Z_C-20150804T234610Z.nc'
testfile = os.path.join(self.tempdir, filename)
make_test_file(testfile, {
'site_code': 'PH100',
'featureType': 'timeSeries'
},
TEMP={},
PRES={},
DEPTH={})
dest_dir, dest_filename = os.path.split(
MooringsFileClassifier.dest_path(testfile))
self.assertEqual(dest_dir, 'IMOS/ANMN/NSW/PH100/Temperature')
self.assertEqual(dest_filename, filename)
def test_pulse_delayed(self):
filename = 'IMOS_DWM-SOTS_20130507T080000Z_Pulse_FV01_Pulse-10-2013_END-20131013T210000Z_C-20160315T000000Z.nc'
testfile = os.path.join(self.tempdir, filename)
make_test_file(
testfile, {
'site_code': 'SOTS',
'platform_code': 'Pulse',
'time_deployment_start': '2013-05-07T08:00:00Z',
'time_coverage_start': '2013-05-07T08:00:00Z',
'time_coverage_end': '2013-10-13T21:00:00Z'
})
dest_dir, dest_filename = os.path.split(
DwmFileClassifier.dest_path(testfile))
self.assertEqual(dest_dir, 'IMOS/DWM/SOTS/2013')
self.assertEqual(dest_filename, filename)
def test_fluxpulse_realtime(self):
filename = 'IMOS_DWM-SOTS_W_20160316T140000Z_FluxPulse_FV00_FluxPulse-1-2016-MRU-Surface-wave-height.nc '
testfile = os.path.join(self.tempdir, filename)
make_test_file(
testfile, {
'site_code': 'SOTS',
'platform_code': 'FluxPulse',
'data_mode': 'R',
'time_coverage_start': '2016-03-16T14:00:00Z',
'time_coverage_end': '2017-10-02T00:00:00Z'
})
dest_dir, dest_filename = os.path.split(
DwmFileClassifier.dest_path(testfile))
self.assertEqual(dest_dir, 'IMOS/DWM/SOTS/2016/real-time')
self.assertEqual(dest_filename, filename)
def test_eco_parsb(self):
filename = (
'IMOS_ANMN-QLD_FZ_20180514T083000Z_GBRPPS_FV01_GBRPPS-1805-ECO-PARSB-26.5_END-20181110T074743Z'
'_C-20190807T070104Z.nc')
testfile = os.path.join(self.tempdir, filename)
make_test_file(testfile, {
'site_code': 'GBRPPS',
'featureType': 'timeseries'
},
PAR={})
dest_dir, dest_filename = os.path.split(
MooringsFileClassifier.dest_path(testfile))
self.assertEqual(dest_dir,
'IMOS/ANMN/QLD/GBRPPS/Biogeochem_timeseries')
self.assertEqual(dest_filename, filename)
def test_da_ctd_timeseries(self):
filename = 'IMOS_DWM-DA_STZ_20120426T092000Z_EAC5_FV01_EAC5-2012-SBE37SMP-202_END-20130826T222000Z_C' \
'-20140722T061531Z.nc '
testfile = os.path.join(self.tempdir, filename)
make_test_file(testfile, {
'platform_code': 'EAC5',
'featureType': 'timeSeries'
},
TEMP={},
PSAL={},
PRES_REL={},
DEPTH={})
dest_dir, dest_filename = os.path.split(
DwmFileClassifier.dest_path(testfile))
self.assertEqual(dest_dir, 'IMOS/DWM/DA/EAC5/CTD_timeseries')
self.assertEqual(dest_filename, filename)
def test_acidification_mooring_realtime(self):
filename = 'IMOS_ANMN-AM_GST_20180926T000000Z_NRSMAI_FV01_NRSMAI-CO2-1809-realtime.nc'
testfile = os.path.join(self.tempdir, filename)
make_test_file(testfile, {'site_code': 'NRSMAI'},
TEMP={},
PSAL={},
Press_ATM={},
xCO2EQ_PPM={},
xCO2ATM_PPM={},
fCO2SW_UATM={},
DOX1={},
TPH={})
dest_dir, dest_filename = os.path.split(
MooringsFileClassifier.dest_path(testfile))
self.assertEqual(dest_dir, 'IMOS/ANMN/AM/NRSMAI/CO2/real-time')
self.assertEqual(dest_filename, filename)
def test_acidification_mooring_delayed(self):
filename = 'IMOS_ANMN-AM_GST_20170912T060000Z_NRSMAI_FV01_NRSMAI-CO2-1709-delayed_END_20180419T230000Z_C-20180716T102404Z.nc'
testfile = os.path.join(self.tempdir, filename)
make_test_file(testfile, {'site_code': 'NRSMAI'},
TEMP={},
PSAL={},
Press_ATM={},
xCO2EQ_PPM={},
xCO2ATM_PPM={},
fCO2SW_UATM={},
DOX1={},
TPH={})
dest_dir, dest_filename = os.path.split(
MooringsFileClassifier.dest_path(testfile))
self.assertEqual(dest_dir, 'IMOS/ANMN/AM/NRSMAI/CO2/delayed')
self.assertEqual(dest_filename, filename)
def test_eco_bb2flwb(self):
filename = (
'IMOS_ANMN-NRS_EKZ_20130125T140759Z_NRSROT_FV01_NRSROT-1301-ECO-BB2FLWB-21_END-20130520T135245Z'
'_C-20191115T024327Z.nc')
testfile = os.path.join(self.tempdir, filename)
make_test_file(testfile, {
'site_code': 'NRSROT',
'featureType': 'timeseries'
},
CDOM={},
VSF470={})
dest_dir, dest_filename = os.path.split(
MooringsFileClassifier.dest_path(testfile))
self.assertEqual(dest_dir,
'IMOS/ANMN/NRS/NRSROT/Biogeochem_timeseries')
self.assertEqual(dest_filename, filename)
