def test_fetch_data(self):
""" Test download agrimet data within time slice.
Test refomatting of data, test unit converstion to std units.
Checked data is in a Pandas.DataFrame object.
:return:
"""
agrimet = Agrimet(station=self.fetch_site,
start_date='2015-01-01',
end_date='2015-12-31',
interval='daily')
raw = agrimet.fetch_met_data(return_raw=True)
formed = agrimet.fetch_met_data()
a = raw.iloc[1, :].tolist()
b = formed.iloc[1, :].tolist()
# dates equality
self.assertEqual(a[0], b[0])
self.assertEqual(a[0], '2015-01-02')
# in to mm
self.assertEqual(a[2], b[2] / 25.4)
# deg F to deg C
self.assertAlmostEqual(a[4], b[4] * 1.8 + 32, delta=0.01)
# in to mm
self.assertEqual(a[7], b[7] / 25.4)
# Langleys to J m-2
self.assertEqual(a[9], b[9] / 41868.)
# mph to m sec-1
self.assertEqual(a[12], b[12] / 0.44704)
def test_agrimet_mesonet_gridmet_etr(self):
agrimet = Agrimet(station=self.fetch_site,
start_date=self.start,
end_date=self.end,
interval='daily')
formed = agrimet.fetch_met_data()
agri_etr = formed['ETRS'].values
s, e = datetime.strptime(self.start, '%Y-%m-%d'), \
datetime.strptime(self.end, '%Y-%m-%d')
gridmet = GridMet('etr', start=s, end=e, lat=self.lat, lon=self.lon)
gridmet_etr = gridmet.get_point_timeseries()
gridmet_etr = gridmet_etr.values
mco = Mesonet(self.covm_mco, start=self.start, end=self.end)
mesonet_daily = mco.mesonet_etr(lat=46.3, elevation=1000.0)
mesonet_etr = mesonet_daily['ETR'].values
plt.plot(gridmet_etr[121:273], label='gridmet')
plt.plot(mesonet_etr[121:273], label='mesonet')
plt.plot(agri_etr[121:273], label='agrimet')
plt.xlabel('GROWING SEASON DAY (MAY 01 - SEP 30)')
plt.ylabel('Tall Crop Reference ET (mm) daily')
plt.legend()
# plt.show()
saved = os.path.join(os.path.dirname(__file__),
'grid_agri_meso_fig.png')
print('saved to {}'.format(saved))
plt.savefig(saved)
ga_ratio = gridmet_etr[121:273].sum() / agri_etr[121:273].sum()
print('gridmet - agrimet ratio: {}'.format(ga_ratio))
ma_ratio = mesonet_etr[121:273].sum() / agri_etr[121:273].sum()
print('mesonet - agrimet ratio: {}'.format(ma_ratio))
def test_fetch_data_many_stations(self):
""" Test download nultiple agrimet station data download.
This runs through a list of stations, reformats data, checks unit conversion,
and Pandas.DataFrame
:return:
"""
for site in self.outside_PnGp_sites:
agrimet = Agrimet(station=site,
start_date='2015-05-15',
end_date='2015-05-15',
interval='daily')
raw = agrimet.fetch_met_data(return_raw=True)
formed = agrimet.fetch_met_data()
params = ['et', 'mm', 'pc', 'sr', 'wr']
for param in params:
key = '{}_{}'.format(site, param)
converted = formed[param.upper()].values.flatten()[0]
unconverted = raw[key].values.flatten()[0]
if param in ['et', 'pc']:
unconverted *= 25.4
if param == 'mm':
unconverted = (unconverted - 32) * 5 / 9
if param == 'sr':
unconverted *= 41868.
if param == 'wr':
unconverted *= 1609.34
if isnan(converted):
pass
elif agrimet.empty_df:
pass
else:
self.assertAlmostEqual(converted, unconverted, delta=0.01)
def get_agrimet_crop(self, yr):
agrimet = Agrimet(station=self.station, start_date=START.format(yr),
end_date=END.format(yr), interval='daily')
data = agrimet.fetch_crop_data()
alfalfa = data['ALFM']
m_alfalfa = alfalfa.groupby(lambda x: x.month).sum().values
m_alfalfa = m_alfalfa.reshape((m_alfalfa.shape[0], 1))
return m_alfalfa
def test_great_plains_crop(self):
a = Agrimet(station=self.gp_site,
start_date=self.start,
end_date=self.end,
interval='daily')
df = a.fetch_crop_data()
self.assertIsInstance(a, Agrimet)
def test_pacific_nw_crop(self):
a = Agrimet(station=self.pn_site,
start_date=self.start,
end_date=self.end,
interval='daily')
df = a.fetch_crop_data()
self.assertIsInstance(a, Agrimet)
def test_web_retrieval_all_stations_crop(self):
for s in self.all_stations:
a = Agrimet(station=s,
start_date=self.start_season,
end_date=self.end_season,
interval='daily')
# try:
a.fetch_crop_data()
print('{} appears valid'.format(a.station))
def __init__(self, project, csv, table, station=None, lat=None, lon=None, monthly=False):
Agrimet.__init__(self, station=station, lat=lat, lon=lon)
self.monthly = monthly
self.make_empty_df()
self.first = True
self.project = project
self.csv = csv
self.table = table
self.lat = lat
self.lon = lon
self.elev = None
self.station_rank = 0
def test_web_retrieval_all_stations_crop(self):
fails = []
for s in self.all_stations:
a = Agrimet(station=s,
start_date=self.start_season,
end_date=self.end_season,
interval='daily')
try:
a.fetch_crop_data()
print('{} appears valid'.format(a.station))
except Exception as e:
print('{} appears invalid: {}'.format(a.station, e))
fails.append(a.station)
print(fails)
def test_write_agrimet_shapefile(self):
agrimet = Agrimet(write_stations=True)
station_data = agrimet.load_stations()
epsg = '4326'
outfile = self.out_shape
agrimet.write_agrimet_sation_shp(station_data, epsg, outfile)
with fopen(outfile, 'r') as shp:
count = 0
for _ in shp:
count += 1
self.assertEqual(186, count)
file_list = os.listdir(self.points_dir)
for f in file_list:
if 'write_test' in f:
os.remove(os.path.join(self.points_dir, f))
def test_find_image_station(self):
""" Test find closest agrimet station to Landsat image centroid.
:return:
"""
l8 = Landsat8(self.dirname_image)
agrimet = Agrimet(sat_image=l8)
self.assertEqual(agrimet.station, self.fetch_site)
def test_agrimet_gridmet_precip(self):
agrimet = Agrimet(station=self.fetch_site,
start_date=self.start,
end_date=self.end,
interval='daily')
formed = agrimet.fetch_met_data()
agri_ppt = formed['PP'].values
s, e = datetime.strptime(self.start, '%Y-%m-%d'), \
datetime.strptime(self.end, '%Y-%m-%d')
gridmet = GridMet('pr', start=s, end=e, lat=self.lat, lon=self.lon)
gridmet_ppt = gridmet.get_point_timeseries()
gridmet_ppt = gridmet_ppt.values
difference = abs(gridmet_ppt - agri_ppt)
self.assertLess(nanmean(difference), -100000)
def test_instantiate_Agrimet(self):
""" Test object instantiation.
:return:
"""
ag = Agrimet(start_date='2000-01-01',
end_date='2000-12-31',
station=self.fetch_site,
sat_image=Landsat8(self.dirname_image))
self.assertIsInstance(ag, Agrimet)
def test_find_closest_station(self):
""" Test find closest agrimet station to any point.
:return:
"""
coords = []
with fopen(self.point_file, 'r') as src:
for feature in src:
coords.append(feature['geometry']['coordinates'])
for coord in coords:
agrimet = Agrimet(lon=coord[0], lat=coord[1])
self.assertTrue(agrimet.station in self.site_ids)
def get_agrimet_etr(self, yr, first=False):
agrimet = Agrimet(station=self.station, start_date=START.format(yr),
end_date=END.format(yr), interval='daily')
formed = agrimet.fetch_met_data()
if isnull(formed['ETRS']).values.sum() == formed['ETRS'].shape[0]:
agri_etr = formed['ETRS'].groupby(lambda x: x.month).sum().values
else:
agri_etr = formed['ETRS'].groupby(lambda x: x.month).sum().values
if first:
self.elev = get_elevation(agrimet.station_coords[0], agrimet.station_coords[1])
print('Station {}'.format(self.station))
if formed['YM'].values.mean() != nan:
t_dew = formed['YM'].values
ea = calcs._sat_vapor_pressure(t_dew)[0]
tmin, tmax = formed['MN'].values, formed['MX'].values
try:
doy = formed.index.strftime('%j').astype(int).values
except AttributeError:
doy = formed.index.strftime('%j').astype(int)
doy = doy.reshape((len(doy), 1))[0]
lat = agrimet.station_coords[0]
rs = formed['SR'].values
uz = formed['UA'].values
zw = 2.0
formed['calc_etr'] = Daily(tmin=tmin, tmax=tmax, ea=ea, rs=rs, uz=uz, zw=zw, elev=self.elev,
lat=lat, doy=doy).etr()
calc_etr = formed['calc_etr'].groupby(lambda x: x.month).sum().values
else:
calc_etr = empty(agri_etr.shape)
calc_etr[:] = 0.0
return agri_etr, calc_etr
def test_agrimet_gridmet_etr(self):
agrimet = Agrimet(station=self.fetch_site,
start_date=self.start,
end_date=self.end,
interval='daily')
formed = agrimet.fetch_met_data()
agri_etr = formed['ETRS'].values
s, e = datetime.strptime(self.start, '%Y-%m-%d'), \
datetime.strptime(self.end, '%Y-%m-%d')
gridmet = GridMet('etr', start=s, end=e, lat=self.lat, lon=self.lon)
gridmet_etr = gridmet.get_point_timeseries()
gridmet_etr = gridmet_etr.values
# plt.plot(gridmet_etr, label='gridmet')
# plt.plot(agri_etr, label='agrimet')
# plt.legend()
# plt.show()
# plt.close()
ratio = agri_etr.sum() / gridmet_etr.sum()
print('ratio: {}'.format(ratio))
def test_find_sites(self):
with fopen(self.test_locations, 'r') as src:
for feat in src:
lat, lon = feat['geometry']['coordinates'][1], feat[
'geometry']['coordinates'][0]
expected_site = feat['properties']['siteid']
# don't test on neighboring sites
if expected_site not in self.excepted_sites:
agrimet = Agrimet(lat=lat,
lon=lon,
start_date=self.start,
end_date=self.end)
found_site = agrimet.station
self.assertEqual(expected_site, found_site)