def test_site_data_update_site_list_with_multiple_updates(test_file_path):
first_timestamp = '2013-01-01T01:01:01'
second_timestamp = '2013-02-02T02:02:02'
site_code = '01117800'
site_data_file = test_util.get_test_file_path(
'usgs/nwis/site_%s_daily.xml' % site_code)
with test_util.mocked_urls(site_data_file):
with freezegun.freeze_time(first_timestamp):
nwis.hdf5.update_site_data(site_code, path=test_file_path,
autorepack=False)
site_data = nwis.hdf5.get_site_data(site_code, path=test_file_path)
last_value = site_data['00060:00003']['values'][-1]
assert first_timestamp == last_value['last_checked'] == last_value['last_modified']
update_data_file = test_util.get_test_file_path(os.path.join(
'usgs', 'nwis', 'site_%s_daily_update.xml' % site_code))
with test_util.mocked_urls(update_data_file):
with freezegun.freeze_time(second_timestamp):
nwis.hdf5.update_site_data(site_code, path=test_file_path,
autorepack=False)
updated_site_data = nwis.hdf5.get_site_data(site_code, path=test_file_path)
updated_values = updated_site_data['00060:00003']['values']
last_value = updated_values[-1]
assert last_value['last_checked'] != first_timestamp
assert second_timestamp == last_value['last_checked'] == last_value['last_modified']
original_timestamp = first_timestamp
modified_timestamp = second_timestamp
test_values = [
dict(datetime="1963-01-23T00:00:00", last_checked=modified_timestamp, last_modified=modified_timestamp, qualifiers="A", value='7'),
dict(datetime="1964-01-23T00:00:00", last_checked=modified_timestamp, last_modified=modified_timestamp, qualifiers="A", value='1017'),
dict(datetime="1964-01-24T00:00:00", last_checked=original_timestamp, last_modified=original_timestamp, qualifiers="A", value='191'),
dict(datetime="1964-08-22T00:00:00", last_checked=original_timestamp, last_modified=original_timestamp, qualifiers="A", value='7.9'),
dict(datetime="1969-05-26T00:00:00", last_checked=modified_timestamp, last_modified=modified_timestamp, qualifiers="A", value='1080'),
dict(datetime="2011-12-06T00:00:00", last_checked=modified_timestamp, last_modified=modified_timestamp, qualifiers="P", value='222'),
dict(datetime="2011-12-15T00:00:00", last_checked=original_timestamp, last_modified=original_timestamp, qualifiers="P Eqp", value='-999999'),
dict(datetime="2012-01-15T00:00:00", last_checked=original_timestamp, last_modified=original_timestamp, qualifiers="P e", value='97'),
dict(datetime="2012-05-25T00:00:00", last_checked=modified_timestamp, last_modified=original_timestamp, qualifiers="P", value='56'),
dict(datetime="2012-05-26T00:00:00", last_checked=modified_timestamp, last_modified=original_timestamp, qualifiers="P", value='55'),
dict(datetime="2012-05-27T00:00:00", last_checked=modified_timestamp, last_modified=modified_timestamp, qualifiers="A", value='52'),
dict(datetime="2012-05-28T00:00:00", last_checked=modified_timestamp, last_modified=original_timestamp, qualifiers="P", value='48'),
dict(datetime="2012-05-29T00:00:00", last_checked=modified_timestamp, last_modified=modified_timestamp, qualifiers="P", value='1099'),
dict(datetime="2012-05-30T00:00:00", last_checked=modified_timestamp, last_modified=modified_timestamp, qualifiers="P", value='1098'),
dict(datetime="2012-05-31T00:00:00", last_checked=modified_timestamp, last_modified=original_timestamp, qualifiers="P", value='41'),
dict(datetime="2012-06-01T00:00:00", last_checked=modified_timestamp, last_modified=original_timestamp, qualifiers="P", value='37'),
dict(datetime="2012-06-02T00:00:00", last_checked=modified_timestamp, last_modified=modified_timestamp, qualifiers="P", value='1097'),
dict(datetime="2012-06-03T00:00:00", last_checked=modified_timestamp, last_modified=original_timestamp, qualifiers="P", value='69'),
dict(datetime="2012-06-04T00:00:00", last_checked=modified_timestamp, last_modified=original_timestamp, qualifiers="P", value='81'),
dict(datetime="2012-06-05T00:00:00", last_checked=modified_timestamp, last_modified=modified_timestamp, qualifiers="P", value='1071'),
dict(datetime="2012-06-06T00:00:00", last_checked=modified_timestamp, last_modified=modified_timestamp, qualifiers="P", value='2071'),
]
for test_value in test_values:
assert updated_values.index(test_value) >= 0
def test_site_data_update_site_list_with_multiple_updates(test_file_path):
first_timestamp = '2013-01-01T01:01:01'
second_timestamp = '2013-02-02T02:02:02'
site_code = '01117800'
site_data_file = test_util.get_test_file_path(
'usgs/nwis/site_%s_daily.xml' % site_code)
with test_util.mocked_urls(site_data_file):
with freezegun.freeze_time(first_timestamp):
nwis.hdf5.update_site_data(site_code, path=test_file_path,
autorepack=False)
site_data = nwis.hdf5.get_site_data(site_code, path=test_file_path)
last_value = site_data['00060:00003']['values'][-1]
assert first_timestamp == last_value['last_checked'] == last_value['last_modified']
update_data_file = test_util.get_test_file_path(
'usgs/nwis/site_%s_daily_update.xml' % site_code)
with test_util.mocked_urls(update_data_file):
with freezegun.freeze_time(second_timestamp):
nwis.hdf5.update_site_data(site_code, path=test_file_path,
autorepack=False)
updated_site_data = nwis.hdf5.get_site_data(site_code, path=test_file_path)
updated_values = updated_site_data['00060:00003']['values']
last_value = updated_values[-1]
assert last_value['last_checked'] != first_timestamp
assert second_timestamp == last_value['last_checked'] == last_value['last_modified']
original_timestamp = first_timestamp
modified_timestamp = second_timestamp
test_values = [
dict(datetime="1963-01-23T00:00:00", last_checked=modified_timestamp, last_modified=modified_timestamp, qualifiers="A", value='7'),
dict(datetime="1964-01-23T00:00:00", last_checked=modified_timestamp, last_modified=modified_timestamp, qualifiers="A", value='1017'),
dict(datetime="1964-01-24T00:00:00", last_checked=original_timestamp, last_modified=original_timestamp, qualifiers="A", value='191'),
dict(datetime="1964-08-22T00:00:00", last_checked=original_timestamp, last_modified=original_timestamp, qualifiers="A", value='7.9'),
dict(datetime="1969-05-26T00:00:00", last_checked=modified_timestamp, last_modified=modified_timestamp, qualifiers="A", value='1080'),
dict(datetime="2011-12-06T00:00:00", last_checked=modified_timestamp, last_modified=modified_timestamp, qualifiers="P", value='222'),
dict(datetime="2011-12-15T00:00:00", last_checked=original_timestamp, last_modified=original_timestamp, qualifiers="P Eqp", value='-999999'),
dict(datetime="2012-01-15T00:00:00", last_checked=original_timestamp, last_modified=original_timestamp, qualifiers="P e", value='97'),
dict(datetime="2012-05-25T00:00:00", last_checked=modified_timestamp, last_modified=original_timestamp, qualifiers="P", value='56'),
dict(datetime="2012-05-26T00:00:00", last_checked=modified_timestamp, last_modified=original_timestamp, qualifiers="P", value='55'),
dict(datetime="2012-05-27T00:00:00", last_checked=modified_timestamp, last_modified=modified_timestamp, qualifiers="A", value='52'),
dict(datetime="2012-05-28T00:00:00", last_checked=modified_timestamp, last_modified=original_timestamp, qualifiers="P", value='48'),
dict(datetime="2012-05-29T00:00:00", last_checked=modified_timestamp, last_modified=modified_timestamp, qualifiers="P", value='1099'),
dict(datetime="2012-05-30T00:00:00", last_checked=modified_timestamp, last_modified=modified_timestamp, qualifiers="P", value='1098'),
dict(datetime="2012-05-31T00:00:00", last_checked=modified_timestamp, last_modified=original_timestamp, qualifiers="P", value='41'),
dict(datetime="2012-06-01T00:00:00", last_checked=modified_timestamp, last_modified=original_timestamp, qualifiers="P", value='37'),
dict(datetime="2012-06-02T00:00:00", last_checked=modified_timestamp, last_modified=modified_timestamp, qualifiers="P", value='1097'),
dict(datetime="2012-06-03T00:00:00", last_checked=modified_timestamp, last_modified=original_timestamp, qualifiers="P", value='69'),
dict(datetime="2012-06-04T00:00:00", last_checked=modified_timestamp, last_modified=original_timestamp, qualifiers="P", value='81'),
dict(datetime="2012-06-05T00:00:00", last_checked=modified_timestamp, last_modified=modified_timestamp, qualifiers="P", value='1071'),
dict(datetime="2012-06-06T00:00:00", last_checked=modified_timestamp, last_modified=modified_timestamp, qualifiers="P", value='2071'),
]
for test_value in test_values:
assert updated_values.index(test_value) >= 0
def test_get_site_data_multiple_methods():
site_code = '08054500'
site_data_file = 'usgs/nwis/site_08054500_multiple_methods.xml'
with test_util.mocked_urls(site_data_file):
site_data = ulmo.usgs.nwis.get_site_data(site_code, methods={'62614': 'all', '45592': 'all'})
assert len(site_data['00054:00003']['values']) == 1
assert len(site_data.keys()) == 17
def test_get_site_data_bad_service_raises_error():
site_code = '08068500'
site_data_file = 'usgs/nwis/site_%s_daily.xml' % site_code
with test_util.mocked_urls(site_data_file):
with pytest.raises(ValueError):
ulmo.usgs.nwis.get_site_data(site_code,
service="bad_service")
def test_multi_message_download():
for test_set in multi_message_test_sets:
with test_util.mocked_urls(test_set['data_files']):
data = ulmo.usgs.eddn.get_data(test_set['dcp_address'], start=test_set['start'])
assert data['message_timestamp_utc'][-1] == test_set['first_row_message_timestamp_utc']
assert data['message_timestamp_utc'][0] == test_set['last_row_message_timestamp_utc']
assert len(data) == test_set['number_of_lines']
def test_get_stations_with_elements():
test_elements = [
{
'elements': 'PRCP',
'includes': [
'ASN00008230',
'WA006567710',
'VQC00672823',
],
'excludes': ['SWE00136141', 'USR0000OHOR', 'USC00450935', 'USR0000THEN', 'USR0000MDRY'],
}, {
'elements': ['SNOW', 'TMAX'],
'includes': [
'ACW00011604',
'USW00094895',
'VQW00011640',
'ZI000067991',
],
'excludes': ['BR037642870', 'BR00C8-0100', 'BR048519530', 'BR002548070', 'IN009081600']
},
]
url_files = {
'ghcnd-stations': 'ncdc/ghcnd/ghcnd-stations.txt',
'ghcnd-inventory': 'ncdc/ghcnd/ghcnd-inventory.txt'
}
with test_util.mocked_urls(url_files):
for test_element in test_elements:
elements = test_element.get('elements')
stations = ghcn_daily.get_stations(elements=elements,
as_dataframe=True)
_check_stations_dataframe(stations,
test_element.get('includes'),
test_element.get('excludes'))
def test_get_sites_by_bounding_box():
bounding_box_values = '-83.0,36.5,-81.0,38.5'
sites_data_file = 'usgs/nwis/sites_%s_daily.xml' % bounding_box_values
with test_util.mocked_urls(sites_data_file):
sites = ulmo.usgs.nwis.get_sites(bounding_box=bounding_box_values,
service='dv')
assert len(sites) > 244
def test_last_refresh_gets_updated(test_file_path):
test_file_path = os.path.join(test_file_path, "test.h5")
first_timestamp = '2013-01-01T01:01:01'
second_timestamp = '2013-02-02T02:02:02'
forth_timestamp = '2013-03-03T03:03:03'
site_code = '01117800'
site_data_file = test_util.get_test_file_path(
'usgs/nwis/site_%s_daily.xml' % site_code)
with test_util.mocked_urls(site_data_file):
with freezegun.freeze_time(first_timestamp):
nwis.hdf5.update_site_data(site_code, path=test_file_path,
autorepack=False)
first_refresh = nwis.hdf5._get_last_refresh(site_code, test_file_path)
assert first_refresh == first_timestamp
with freezegun.freeze_time(second_timestamp):
nwis.hdf5.update_site_data(site_code, path=test_file_path,
autorepack=False)
second_refresh = nwis.hdf5._get_last_refresh(site_code, test_file_path)
assert second_refresh == second_timestamp
nwis.hdf5.update_site_data(site_code, path=test_file_path,
input_file=site_data_file, autorepack=False)
third_refresh = nwis.hdf5._get_last_refresh(site_code, test_file_path)
assert third_refresh == None
with freezegun.freeze_time(forth_timestamp):
nwis.hdf5.update_site_data(site_code, path=test_file_path,
autorepack=False)
forth_refresh = nwis.hdf5._get_last_refresh(site_code, test_file_path)
assert forth_refresh is not None
assert forth_refresh == forth_timestamp
def test_get_data_as_dataframe():
with test_util.mocked_urls('cpc/drought/palmer10'):
data = ulmo.cpc.drought.get_data(start='2010-5-20',
end='2010-6-13',
as_dataframe=True)
assert isinstance(data, pandas.DataFrame)
def test_multi_message_download():
for test_set in multi_message_test_sets:
with test_util.mocked_urls(test_set["data_files"]):
data = ulmo.usgs.eddn.get_data(test_set["dcp_address"], start=test_set["start"])
assert data["message_timestamp_utc"][-1] == test_set["first_row_message_timestamp_utc"]
assert data["message_timestamp_utc"][0] == test_set["last_row_message_timestamp_utc"]
assert len(data) == test_set["number_of_lines"]
def test_get_sensors():
sensors_file = 'cdec/historical/sensors.htm'
with test_util.mocked_urls(sensors_file):
sensors = ulmo.cdec.historical.get_sensors()
assert 200 < len(sensors)
sensors.index = sensors['Sensor No'].astype(int)
assert u'FLOW, RIVER DISCHARGE' == sensors.xs(20)['Description']
def test_get_sites_with_extra_kwarg():
sites_data_file = 'usgs/nwis/sites_kwarg_agencyCD.xml'
with test_util.mocked_urls(sites_data_file):
sites = ulmo.usgs.nwis.get_sites(state_code='TX',
agencyCD='USCE',
service='dv')
assert len(sites) == 1
def test_last_refresh_gets_updated(test_file_path):
test_file_path = test_file_path + "test.h5"
first_timestamp = '2013-01-01T01:01:01'
second_timestamp = '2013-02-02T02:02:02'
forth_timestamp = '2013-03-03T03:03:03'
site_code = '01117800'
site_data_file = test_util.get_test_file_path(
'usgs/nwis/site_%s_daily.xml' % site_code)
with test_util.mocked_urls(site_data_file):
with freezegun.freeze_time(first_timestamp):
nwis.hdf5.update_site_data(site_code, path=test_file_path,
autorepack=False)
first_refresh = nwis.hdf5._get_last_refresh(site_code, test_file_path)
assert first_refresh == first_timestamp
with freezegun.freeze_time(second_timestamp):
nwis.hdf5.update_site_data(site_code, path=test_file_path,
autorepack=False)
second_refresh = nwis.hdf5._get_last_refresh(site_code, test_file_path)
assert second_refresh == second_timestamp
nwis.hdf5.update_site_data(site_code, path=test_file_path,
input_file=site_data_file, autorepack=False)
third_refresh = nwis.hdf5._get_last_refresh(site_code, test_file_path)
assert third_refresh == None
with freezegun.freeze_time(forth_timestamp):
nwis.hdf5.update_site_data(site_code, path=test_file_path,
autorepack=False)
forth_refresh = nwis.hdf5._get_last_refresh(site_code, test_file_path)
assert forth_refresh is not None
assert forth_refresh == forth_timestamp
def test_get_station_data():
test_station_data = [
('MYST2', '2018-02-03', {
'code': 'MYST2',
'description': 'Pat Mayse Lake',
'station_type': 'RESERVOIR',
'timezone': 'US/Central',
'values': {
'2018-02-03 01:00:00': {'PRECIP PRE': 0.0, 'CIP(A) ELE': 0.0, 'VATION S': 451.61, 'TORAGE': 121347.0, 'INFLOW R': 0.0, 'ELEASE AI': 59.0, 'R-TEMP WI': 35.4, 'ND-DIRWIND': 75.0, '-SPEED REL': 5.36, '-HUMID SOL': 41.52, 'AR-RAD V': -2.0, 'OLTAGE BA': 12.37}
},
}),
]
for code, date, test_data in test_station_data:
url_date = date.replace('-', '')
filename = '%s.%s.html' % (code, url_date)
data_file = 'usace/swtwc/' + filename
with test_util.mocked_urls(data_file):
station_data = ulmo.usace.swtwc.get_station_data(code, date)
for key, value in test_data.items():
if key == 'values':
_compare_values(test_data['values'], station_data['values'])
else:
assert station_data[key] == test_data[key]
def test_get_sites_by_multiple_serving_parameter_code():
site_code = '08068500'
parameter_code_values = '00060,00065'
sites_data_file = 'usgs/nwis/sites_%s_%s_daily.xml' % (site_code, parameter_code_values)
with test_util.mocked_urls(sites_data_file):
sites = ulmo.usgs.nwis.get_sites(sites=site_code, parameter_code=parameter_code_values, service='dv')
assert len(sites) == 1
def test_parse_dcp_message_timestamp():
for test_set in message_test_sets:
dcp_data_file = 'usgs/eddn/' + test_set['dcp_address'] + '.txt'
with test_util.mocked_urls(dcp_data_file):
data = ulmo.usgs.eddn.get_data(test_set['dcp_address'])
assert data['message_timestamp_utc'][-1] == test_set[
'first_row_message_timestamp_utc']
def test_get_sensors():
sensors_file = 'cdec/historical/sensors.htm'
with test_util.mocked_urls(sensors_file):
sensors = ulmo.cdec.historical.get_sensors()
assert 200 < len(sensors)
sensors.index = sensors['Sensor No'].astype(int)
assert u'FLOW, RIVER DISCHARGE' == sensors.xs(20)['Description']
def test_get_stations_with_elements():
test_elements = [
{
'elements': 'PRCP',
'includes': [
'ASN00008230',
'WA006567710',
'VQC00672823',
],
'excludes': [
'AR000870470',
'AR000875850',
'BC000068234',
'GME00111464',
'UY000864400',
],
}, {
'elements': 'PRCP',
'includes': [
'ASN00008230',
'WA006567710',
'VQC00672823',
],
'excludes': [
'AR000870470',
'AR000875850',
'BC000068234',
'GME00111464',
'UY000864400',
],
}, {
'elements': ['SNOW', 'TMAX'],
'includes': [
'ACW00011604',
'USW00094895',
'VQW00011640',
'ZI000067991',
],
'excludes': [
'BR00B4-0010',
'IN003070101',
'KZ000038223',
'ZA000067753',
],
},
]
url_files = {
'ghcnd-stations': 'ncdc/ghcnd/ghcnd-stations.txt',
'ghcnd-inventory': 'ncdc/ghcnd/ghcnd-inventory.txt'
}
with test_util.mocked_urls(url_files):
for test_element in test_elements:
elements = test_element.get('elements')
stations = ghcn_daily.get_stations(elements=elements,
as_dataframe=True)
_check_stations_dataframe(stations,
test_element.get('includes'),
test_element.get('excludes'))
def test_get_stations_as_dicts():
with test_util.mocked_urls('ncdc/ghcnd/ghcnd-stations.txt'):
stations = ghcn_daily.get_stations()
assert len(stations) > 80000
for test_station in test_stations:
station_id = test_station.get('id')
assert stations.get(station_id) == test_station
def test_get_station_data_current():
# can't easily test current since it is a moving target changes, but mostly
# just make sure it parses correctl: current will have '---' values where
# previous days do not
data_file = "usace/swtwc/MYST2.current.html"
with test_util.mocked_urls(data_file):
station_data = ulmo.usace.swtwc.get_station_data("MYST2")
assert len(station_data.get("values")) > 0
def test_get_site_data_single_site_with_start_and_end():
site_code = '08068500'
site_data_file = 'usgs/nwis/site_08068500_instantaneous_2011-11-05_2011-11-18.xml'
with test_util.mocked_urls(site_data_file):
site_data = ulmo.usgs.nwis.get_site_data(site_code, start='2011-11-05',
end='2011-11-18', service='instantaneous')
assert len(site_data) == 7
assert len(site_data['63680:00011']['values']) == 1250
def test_get_stations_as_dicts():
with test_util.mocked_urls('ncdc/ghcnd/ghcnd-stations.txt'):
stations = ghcn_daily.get_stations()
assert len(stations) > 80000
for test_station in test_stations:
station_id = test_station.get('id')
assert stations.get(station_id) == test_station
def test_get_station_data_out_of_range():
# can't easily test current since it is a moving target changes, but mostly
# just make sure it parses correctl: current will have '---' values where
# previous days do not
data_file = 'usace/swtwc/empty.html'
with test_util.mocked_urls(data_file):
with pytest.raises(ValueError):
station_data = ulmo.usace.swtwc.get_station_data('MYST2', '1945-01-01')
def test_get_station_data_current():
# can't easily test current since it is a moving target changes, but mostly
# just make sure it parses correctl: current will have '---' values where
# previous days do not
data_file = 'usace/swtwc/MYST2.current.html'
with test_util.mocked_urls(data_file):
station_data = ulmo.usace.swtwc.get_station_data('MYST2')
assert len(station_data.get('values')) > 0
def test_get_data_by_state():
for test_set in test_sets:
with test_util.mocked_urls(test_set['filename']):
data = ulmo.cpc.drought.get_data(
state=test_set['state'], start=test_set['start'],
end=test_set['end'])
assert len(data) == 1
assert test_set['state'] in data
def test_get_station_data_out_of_range():
# can't easily test current since it is a moving target changes, but mostly
# just make sure it parses correctl: current will have '---' values where
# previous days do not
data_file = "usace/swtwc/empty.html"
with test_util.mocked_urls(data_file):
with pytest.raises(ValueError):
station_data = ulmo.usace.swtwc.get_station_data("MYST2", "1945-01-01")
def test_get_site_data_single_site_with_period():
site_data_file = 'usgs/nwis/site_01117800_instantaneous_P45D.xml'
site_code = '01117800'
with test_util.mocked_urls(site_data_file):
site_data = ulmo.usgs.nwis.get_site_data(site_code, period='P45D',
service='daily')
assert len(site_data) >= 1
assert len(site_data['00060:00003']['values']) == 45
def test_get_data_by_state():
for test_set in test_sets:
with test_util.mocked_urls(test_set['filename']):
data = ulmo.cpc.drought.get_data(state=test_set['state'],
start=test_set['start'],
end=test_set['end'])
assert len(data) == 1
assert test_set['state'] in data
def test_get_site_data_multiple_methods():
site_code = '08054500'
site_data_file = 'usgs/nwis/site_08054500_multiple_methods.xml'
with test_util.mocked_urls(site_data_file):
site_data = ulmo.usgs.nwis.get_site_data(site_code, methods={'00062': 'all'})
assert len(site_data['00062:00011:1']['values']) == 288
assert len(site_data['00062:00011:20']['values']) == 288
assert len(site_data.keys()) == 2
def test_get_site_data_single_site_with_period():
site_data_file = 'usgs/nwis/site_01117800_instantaneous_P45D.xml'
site_code = '01117800'
with test_util.mocked_urls(site_data_file):
site_data = ulmo.usgs.nwis.get_site_data(site_code,
period='P45D',
service='daily')
assert len(site_data) >= 1
assert len(site_data['00060:00003']['values']) == 45
def test_parse_dcp_message_timestamp():
for test_set in message_test_sets:
dcp_data_file = 'noaa/goes/' + test_set['dcp_address'] + '.txt'
with test_util.mocked_urls(dcp_data_file, force=True):
data = goes.get_data(test_set['dcp_address'], hours=12)
assert data['message_timestamp_utc'][-1] == datetime.fromtimestamp(
int(test_set['message_timestamp'].strip('/Date()')) / 1000)
assert data['message_timestamp_utc'][-1] == datetime.fromtimestamp(
int(test_set['message_timestamp'].strip('/Date()')) / 1000)
def test_get_site_data_multiple_methods():
site_code = '08054500'
site_data_file = 'usgs/nwis/site_08054500_multiple_methods.xml'
with test_util.mocked_urls(site_data_file):
site_data = ulmo.usgs.nwis.get_site_data(site_code,
methods={'00062': 'all'})
assert len(site_data['00062:00011:1']['values']) == 288
assert len(site_data['00062:00011:20']['values']) == 288
assert len(site_data.keys()) == 2
def test_get_historical_data():
parameters_file = 'lcra/waterquality/12147_params.html'
data_file = 'lcra/waterquality/12147_results.html'
with test_util.mocked_urls(data_file):
results = get_historical_data(12147)
assert len(results) == 12
for data in results:
assert data['Site'] == u'12147'
def test_get_site_data_single_site_with_start_and_end():
site_code = '08068500'
site_data_file = 'usgs/nwis/site_08068500_instantaneous_2011-11-05_2011-11-18.xml'
with test_util.mocked_urls(site_data_file):
site_data = ulmo.usgs.nwis.get_site_data(site_code,
start='2011-11-05',
end='2011-11-18',
service='instantaneous')
assert len(site_data) == 7
assert len(site_data['63680:00011']['values']) == 1250
def test_get_stations_as_dataframe():
with test_util.mocked_urls('ncdc/ghcnd/ghcnd-stations.txt'):
stations = ghcn_daily.get_stations(as_dataframe=True)
assert len(stations) > 80000
for test_station in test_stations:
station_id = test_station.get('id')
station = stations.xs(station_id)
station_dict = util.misc._nans_to_nones(station.to_dict())
assert station_dict == test_station
def test_get_historical_data():
parameters_file = 'lcra/waterquality/12147_params.html'
data_file = 'lcra/waterquality/12147_results.html'
with test_util.mocked_urls(data_file):
results = get_historical_data(12147)
assert len(results) == 12
for data in results:
assert data['Site'] == u'12147'
def test_get_stations_as_dataframe():
with test_util.mocked_urls('ncdc/ghcnd/ghcnd-stations.txt'):
stations = ghcn_daily.get_stations(as_dataframe=True)
assert len(stations) > 80000
for test_station in test_stations:
station_id = test_station.get('id')
station = stations.xs(station_id)
station_dict = util.misc._nans_to_nones(station.to_dict())
assert station_dict == test_station
def test_get_data():
for test_set in test_sets:
with test_util.mocked_urls(test_set['filename']):
data = ulmo.cpc.drought.get_data(start=test_set['start'], end=test_set['end'])
values = data.get(test_set['state'], {}).get(test_set['climate_division'])
test_values = test_set['values']
for test_value in test_values:
assert test_value in values
def test_get_sites_by_multiple_serving_parameter_code():
site_code = '08068500'
parameter_code_values = '00060,00065'
sites_data_file = 'usgs/nwis/sites_%s_%s_daily.xml' % (
site_code, parameter_code_values)
with test_util.mocked_urls(sites_data_file):
sites = ulmo.usgs.nwis.get_sites(sites=site_code,
parameter_code=parameter_code_values,
service='dv')
assert len(sites) == 1
def test_get_sites_by_state_code():
mocked_urls = {
'http://waterservices.usgs.gov/nwis/dv/': 'usgs/nwis/RI_daily.xml',
'http://waterservices.usgs.gov/nwis/iv/':
'usgs/nwis/RI_instantaneous.xml',
}
with test_util.mocked_urls(mocked_urls):
sites = ulmo.usgs.nwis.get_sites(state_code='RI')
assert len(sites) > 64
def test_get_data_as_dataframe():
with test_util.temp_dir() as data_dir:
with test_util.mocked_urls(MOCKED_URLS):
data = ulmo.twc.kbdi.get_data(
start='2013-04-09',
end='2013-04-09',
as_dataframe=True,
data_dir=data_dir,
)
assert isinstance(data, pandas.DataFrame)
def test_get_stations_with_fips():
with test_util.mocked_urls("ncdc/gsod/ish-history.csv"):
stations = ulmo.ncdc.gsod.get_stations(fips="SW")
assert 400 <= len(stations) <= 500
assert "026850-99999" in stations
stations = ulmo.ncdc.gsod.get_stations(fips=["US", "MX", "CA"])
assert 9500 <= len(stations) <= 10000
assert "768420-99999" in stations
assert "720025-99999" in stations
assert "729675-99999" in stations
def test_data_dir_used():
with test_util.temp_dir() as data_dir:
with test_util.mocked_urls(MOCKED_URLS):
ulmo.twc.kbdi.get_data(
start='2013-04-09',
end='2013-04-11',
as_dataframe=True,
data_dir=data_dir,
)
files_glob = glob.glob(os.path.join(data_dir, '*'))
assert len(files_glob) == 3
def test_get_stations_with_state():
with test_util.mocked_urls('ncdc/gsod/isd-history.csv'):
stations = ulmo.ncdc.gsod.get_stations(state='TX')
assert 450 <= len(stations) <= 500
assert '999999-93987' in stations
stations = ulmo.ncdc.gsod.get_stations(state=['TX', 'AR', 'LA'])
assert 750 <= len(stations) <= 800
assert '999999-93987' in stations
assert '999999-13963' in stations
assert '994780-99999' in stations
def test_get_stations_with_country():
with test_util.mocked_urls('ncdc/gsod/isd-history.csv'):
stations = ulmo.ncdc.gsod.get_stations(country='AF')
assert 100 <= len(stations) <= 150
assert '409900-99999' in stations
stations = ulmo.ncdc.gsod.get_stations(country=['US', 'MX', 'CA'])
assert 9500 <= len(stations) <= 10000
assert '768420-99999' in stations
assert '911660-99999' in stations
assert '729675-99999' in stations
def test_get_stations_with_state():
with test_util.mocked_urls("ncdc/gsod/ish-history.csv"):
stations = ulmo.ncdc.gsod.get_stations(state="TX")
assert 500 <= len(stations) <= 550
assert "999999-93987" in stations
stations = ulmo.ncdc.gsod.get_stations(state=["TX", "AR", "LA"])
assert 800 <= len(stations) <= 850
assert "999999-93987" in stations
assert "999999-13963" in stations
assert "994780-99999" in stations
def test_get_stations_with_country():
with test_util.mocked_urls('ncdc/gsod/ish-history.csv'):
stations = ulmo.ncdc.gsod.get_stations(country='AH')
assert 250 <= len(stations) <= 300
assert '999032-99999' in stations
stations = ulmo.ncdc.gsod.get_stations(country=['US', 'MX', 'CN'])
assert 9500 <= len(stations) <= 10000
assert '768420-99999' in stations
assert '720025-99999' in stations
assert '729675-99999' in stations
def test_get_stations_with_fips():
with test_util.mocked_urls('ncdc/gsod/ish-history.csv'):
stations = ulmo.ncdc.gsod.get_stations(fips='SW')
assert 400 <= len(stations) <= 500
assert '026850-99999' in stations
stations = ulmo.ncdc.gsod.get_stations(fips=['US', 'MX', 'CA'])
assert 9500 <= len(stations) <= 10000
assert '768420-99999' in stations
assert '720025-99999' in stations
assert '729675-99999' in stations
def test_get_stations_with_fips():
with test_util.mocked_urls('ncdc/gsod/ish-history.csv'):
stations = ulmo.ncdc.gsod.get_stations(fips='SW')
assert 400 <= len(stations) <= 500
assert '026850-99999' in stations
stations = ulmo.ncdc.gsod.get_stations(fips=['US', 'MX', 'CA'])
assert 9500 <= len(stations) <= 10000
assert '768420-99999' in stations
assert '720025-99999' in stations
assert '729675-99999' in stations
def test_get_site_data_multiple_methods():
site_code = '08054500'
site_data_file = 'usgs/nwis/site_08054500_multiple_methods.xml'
with test_util.mocked_urls(site_data_file):
site_data = ulmo.usgs.nwis.get_site_data(site_code,
methods={
'62614': 'all',
'45592': 'all'
})
assert len(site_data['00054:00003']['values']) == 1
assert len(site_data.keys()) == 17
def test_get_stations_with_state():
with test_util.mocked_urls('ncdc/gsod/isd-history.csv'):
stations = ulmo.ncdc.gsod.get_stations(state='TX')
assert 450 <= len(stations) <= 500
assert '999999-93987' in stations
stations = ulmo.ncdc.gsod.get_stations(state=['TX', 'AR', 'LA'])
assert 750 <= len(stations) <= 800
assert '999999-93987' in stations
assert '999999-13963' in stations
assert '994780-99999' in stations
def test_get_sites_by_state_code():
mocked_urls = {
'http://waterservices.usgs.gov/nwis/dv/':
'usgs/nwis/RI_daily.xml',
'http://waterservices.usgs.gov/nwis/iv/':
'usgs/nwis/RI_instantaneous.xml',
}
with test_util.mocked_urls(mocked_urls):
sites = ulmo.usgs.nwis.get_sites(state_code='RI')
assert len(sites) == 64
def test_get_stations_with_country():
with test_util.mocked_urls('ncdc/gsod/isd-history.csv'):
stations = ulmo.ncdc.gsod.get_stations(country='AF')
assert 100 <= len(stations) <= 150
assert '409900-99999' in stations
stations = ulmo.ncdc.gsod.get_stations(country=['US', 'MX', 'CA'])
assert 9500 <= len(stations) <= 10000
assert '768420-99999' in stations
assert '911660-99999' in stations
assert '729675-99999' in stations
def test_get_stations_with_country():
with test_util.mocked_urls('ncdc/gsod/ish-history.csv'):
stations = ulmo.ncdc.gsod.get_stations(country='AH')
assert 250 <= len(stations) <= 300
assert '999032-99999' in stations
stations = ulmo.ncdc.gsod.get_stations(country=['US', 'MX', 'CN'])
assert 9500 <= len(stations) <= 10000
assert '768420-99999' in stations
assert '720025-99999' in stations
assert '729675-99999' in stations
def test_get_data_by_county():
with test_util.temp_dir() as data_dir:
for test_set in test_sets:
with test_util.mocked_urls(MOCKED_URLS):
data = ulmo.twc.kbdi.get_data(
county=test_set['fips'],
start=test_set['start'],
end=test_set['end'],
data_dir=data_dir,
)
assert len(data) == 1
assert test_set['fips'] in data
def test_get_station_data():
test_sets = [('CE7F42E6', [(datetime.date(2013, 1, 1), 168.04),
(datetime.date(2013, 1, 15), 168.69)])]
for station_code, test_values in test_sets:
stations_file = 'usace/rivergages/data_%s.cfm' % station_code
with test_util.mocked_urls(stations_file):
station_data = ulmo.usace.rivergages.get_station_data(
'CE7F42E6', 'HP', start='2013-1-1', end='2013-1-15')
for test_value in test_values:
assert test_value in station_data.iteritems()
def test_get_station_sensors():
test_sets = [('PRA', [6, 6, 15, 15, 22, 23, 76])]
for station_id, test_value in test_sets:
stations_file = 'cdec/historical/%s.htm' % (station_id)
with test_util.mocked_urls(stations_file):
available_sensors = ulmo.cdec.historical.get_station_sensors(
[station_id])
assert list(available_sensors[station_id].sensor_id.to_dict().values()
) == test_value
