def test_igra2():
"""Test that we are properly parsing data from the IGRA2 archive."""
df, header = IGRAUpperAir.request_data(datetime(2010, 6, 1, 12),
'USM00070026')
assert_almost_equal(df['lvltyp1'][5], 1, 1)
assert_almost_equal(df['lvltyp2'][5], 0, 1)
assert_almost_equal(df['etime'][5], 126, 2)
assert_almost_equal(df['pressure'][5], 925.0, 2)
assert_almost_equal(df['pflag'][5], 0, 1)
assert_almost_equal(df['height'][5], 696., 2)
assert_almost_equal(df['zflag'][5], 2, 1)
assert_almost_equal(df['temperature'][5], -3.2, 2)
assert_almost_equal(df['tflag'][5], 2, 1)
assert_almost_equal(df['relative_humidity'][5], 96.3, 2)
assert_almost_equal(df['direction'][5], 33.0, 2)
assert_almost_equal(df['speed'][5], 8.2, 2)
assert_almost_equal(df['u_wind'][5], -4.5, 2)
assert_almost_equal(df['v_wind'][5], -6.9, 2)
assert_almost_equal(df['dewpoint'][5], -3.7, 2)
assert (df.units['pressure'] == 'hPa')
assert (df.units['height'] == 'meter')
assert (df.units['temperature'] == 'degC')
assert (df.units['dewpoint'] == 'degC')
assert (df.units['u_wind'] == 'meter / second')
assert (df.units['v_wind'] == 'meter / second')
assert (df.units['speed'] == 'meter / second')
assert (df.units['direction'] == 'degrees')
assert (df.units['etime'] == 'second')
def test_igra2(monkeypatch):
"""Test that we are properly parsing data from the IGRA2 archive."""
with monkeypatch.context() as m:
m.setattr(
request, 'urlopen', lambda _: open(
os.path.join(os.path.dirname(__file__), 'fixtures',
'USM00070026.zip'), 'rb'))
df, header = IGRAUpperAir.request_data(datetime(2010, 6, 1, 12),
'USM00070026')
assert_almost_equal(df['lvltyp1'][5], 1, 1)
assert_almost_equal(df['lvltyp2'][5], 0, 1)
assert_almost_equal(df['etime'][5], 126, 2)
assert_almost_equal(df['pressure'][5], 925.0, 2)
assert_almost_equal(df['pflag'][5], 0, 1)
assert_almost_equal(df['height'][5], 696., 2)
assert_almost_equal(df['zflag'][5], 2, 1)
assert_almost_equal(df['temperature'][5], -3.2, 2)
assert_almost_equal(df['tflag'][5], 2, 1)
assert_almost_equal(df['relative_humidity'][5], 96.3, 2)
assert_almost_equal(df['direction'][5], 33.0, 2)
assert_almost_equal(df['speed'][5], 8.2, 2)
assert_almost_equal(df['u_wind'][5], -4.5, 2)
assert_almost_equal(df['v_wind'][5], -6.9, 2)
assert_almost_equal(df['dewpoint'][5], -3.7, 2)
assert (df.units['pressure'] == 'hPa')
assert (df.units['height'] == 'meter')
assert (df.units['temperature'] == 'degC')
assert (df.units['dewpoint'] == 'degC')
assert (df.units['u_wind'] == 'meter / second')
assert (df.units['v_wind'] == 'meter / second')
assert (df.units['speed'] == 'meter / second')
assert (df.units['direction'] == 'degrees')
assert (df.units['etime'] == 'second')
def test_igra2(monkeypatch):
"""Test that we are properly parsing data from the IGRA2 archive."""
with monkeypatch.context() as m:
m.setattr(request, 'urlopen',
lambda _: open(os.path.join(os.path.dirname(__file__),
'fixtures', 'USM00070026.zip'), 'rb'))
df, header = IGRAUpperAir.request_data(datetime(2010, 6, 1, 12), 'USM00070026')
assert_almost_equal(df['lvltyp1'][5], 1, 1)
assert_almost_equal(df['lvltyp2'][5], 0, 1)
assert_almost_equal(df['etime'][5], 126, 2)
assert_almost_equal(df['pressure'][5], 925.0, 2)
assert_almost_equal(df['pflag'][5], 0, 1)
assert_almost_equal(df['height'][5], 696., 2)
assert_almost_equal(df['zflag'][5], 2, 1)
assert_almost_equal(df['temperature'][5], -3.2, 2)
assert_almost_equal(df['tflag'][5], 2, 1)
assert_almost_equal(df['relative_humidity'][5], 96.3, 2)
assert_almost_equal(df['direction'][5], 33.0, 2)
assert_almost_equal(df['speed'][5], 8.2, 2)
assert_almost_equal(df['u_wind'][5], -4.5, 2)
assert_almost_equal(df['v_wind'][5], -6.9, 2)
assert_almost_equal(df['dewpoint'][5], -3.7, 2)
assert(df.units['pressure'] == 'hPa')
assert(df.units['height'] == 'meter')
assert(df.units['temperature'] == 'degC')
assert(df.units['dewpoint'] == 'degC')
assert(df.units['u_wind'] == 'meter / second')
assert(df.units['v_wind'] == 'meter / second')
assert(df.units['speed'] == 'meter / second')
assert(df.units['direction'] == 'degrees')
assert(df.units['etime'] == 'second')
def data_uploader():
z = 0
y = 0
stations = ['USM00072250']
while z < len(stations):
beginning = [datetime(2000, 1, 1), datetime(2001, 12, 31)]
station_name = str(stations[z])
df, header = IGRAUpperAir.request_data(beginning, station_name)
x = 0
station = StationModel(station_name, header['latitude'][0],
header['longitude'][0])
station.save_to_db()
test = df.notnull()
date_test = 0
while x < len(df['height']):
year = df['date'][x].strftime("%Y")
month = df['date'][x].strftime("%m")
oni = OniData.find_by_date(int(year), int(month))
if test['temperature'][x] and test['height'][x]:
if date_test != df['date'][x]:
launch_data = Launch(df['date'][x], oni, z + 1)
launch_data.save_to_db()
date_test = df['date'][x]
y += 1
reading = Reading(int(df['height'][x]), df['temperature'][x],
y)
reading.save_to_db()
x += 1
z += 1
def test_igra2_drvd(monkeypatch):
"""Test that we are properly parsing data from the IGRA2 archive."""
with monkeypatch.context() as m:
m.setattr(
request, 'urlopen', lambda _: open(
os.path.join(os.path.dirname(__file__), 'fixtures',
'USM00070026-derived.zip'), 'rb'))
df, header = IGRAUpperAir.request_data(datetime(2014, 9, 10, 0),
'USM00070026',
derived=True)
assert_almost_equal(df['pressure'][5], 947.43, 2)
assert_almost_equal(df['reported_height'][5], 610., 2)
assert_almost_equal(df['calculated_height'][5], 610., 2)
assert_almost_equal(df['temperature'][5], 269.1, 2)
assert_almost_equal(df['temperature_gradient'][5], 0.0, 2)
assert_almost_equal(df['potential_temperature'][5], 273.2, 2)
assert_almost_equal(df['potential_temperature_gradient'][5], 11.0, 2)
assert_almost_equal(df['virtual_temperature'][5], 269.5, 2)
assert_almost_equal(df['virtual_potential_temperature'][5], 273.7, 2)
assert_almost_equal(df['vapor_pressure'][5], 4.268, 2)
assert_almost_equal(df['saturation_vapor_pressure'][5], 4.533, 2)
assert_almost_equal(df['reported_relative_humidity'][5], 93.9, 2)
assert_almost_equal(df['calculated_relative_humidity'][5], 94.1, 2)
assert_almost_equal(df['u_wind'][5], -75.3, 2)
assert_almost_equal(df['u_wind_gradient'][5], -7.8, 2)
assert_almost_equal(df['v_wind'][5], 9.6, 2)
assert_almost_equal(df['v_wind_gradient'][5], -1.2, 2)
assert_almost_equal(df['refractive_index'][5], 27.0, 2)
assert (df.units['pressure'] == 'hPa')
assert (df.units['reported_height'] == 'meter')
assert (df.units['calculated_height'] == 'meter')
assert (df.units['temperature'] == 'Kelvin')
assert (df.units['temperature_gradient'] == 'Kelvin / kilometer')
assert (df.units['potential_temperature'] == 'Kelvin')
assert (df.units['potential_temperature_gradient'] == 'Kelvin / kilometer')
assert (df.units['virtual_temperature'] == 'Kelvin')
assert (df.units['virtual_potential_temperature'] == 'Kelvin')
assert (df.units['vapor_pressure'] == 'Pascal')
assert (df.units['saturation_vapor_pressure'] == 'Pascal')
assert (df.units['reported_relative_humidity'] == 'percent')
assert (df.units['calculated_relative_humidity'] == 'percent')
assert (df.units['u_wind'] == 'meter / second')
assert (df.units['u_wind_gradient'] == '(meter / second) / kilometer)')
assert (df.units['v_wind'] == 'meter / second')
assert (df.units['v_wind_gradient'] == '(meter / second) / kilometer)')
assert (df.units['refractive_index'] == 'unitless')
def test_igra2_drvd(monkeypatch):
"""Test that we are properly parsing data from the IGRA2 archive."""
with monkeypatch.context() as m:
m.setattr(request, 'urlopen',
lambda _: open(os.path.join(os.path.dirname(__file__),
'fixtures', 'USM00070026-derived.zip'), 'rb'))
df, header = IGRAUpperAir.request_data(datetime(2014, 9, 10, 0),
'USM00070026', derived=True)
assert_almost_equal(df['pressure'][5], 947.43, 2)
assert_almost_equal(df['reported_height'][5], 610., 2)
assert_almost_equal(df['calculated_height'][5], 610., 2)
assert_almost_equal(df['temperature'][5], 269.1, 2)
assert_almost_equal(df['temperature_gradient'][5], 0.0, 2)
assert_almost_equal(df['potential_temperature'][5], 273.2, 2)
assert_almost_equal(df['potential_temperature_gradient'][5], 11.0, 2)
assert_almost_equal(df['virtual_temperature'][5], 269.5, 2)
assert_almost_equal(df['virtual_potential_temperature'][5], 273.7, 2)
assert_almost_equal(df['vapor_pressure'][5], 4.268, 2)
assert_almost_equal(df['saturation_vapor_pressure'][5], 4.533, 2)
assert_almost_equal(df['reported_relative_humidity'][5], 93.9, 2)
assert_almost_equal(df['calculated_relative_humidity'][5], 94.1, 2)
assert_almost_equal(df['u_wind'][5], -75.3, 2)
assert_almost_equal(df['u_wind_gradient'][5], -7.8, 2)
assert_almost_equal(df['v_wind'][5], 9.6, 2)
assert_almost_equal(df['v_wind_gradient'][5], -1.2, 2)
assert_almost_equal(df['refractive_index'][5], 27.0, 2)
assert(df.units['pressure'] == 'hPa')
assert(df.units['reported_height'] == 'meter')
assert(df.units['calculated_height'] == 'meter')
assert(df.units['temperature'] == 'Kelvin')
assert(df.units['temperature_gradient'] == 'Kelvin / kilometer')
assert(df.units['potential_temperature'] == 'Kelvin')
assert(df.units['potential_temperature_gradient'] == 'Kelvin / kilometer')
assert(df.units['virtual_temperature'] == 'Kelvin')
assert(df.units['virtual_potential_temperature'] == 'Kelvin')
assert(df.units['vapor_pressure'] == 'Pascal')
assert(df.units['saturation_vapor_pressure'] == 'Pascal')
assert(df.units['reported_relative_humidity'] == 'percent')
assert(df.units['calculated_relative_humidity'] == 'percent')
assert(df.units['u_wind'] == 'meter / second')
assert(df.units['u_wind_gradient'] == '(meter / second) / kilometer)')
assert(df.units['v_wind'] == 'meter / second')
assert(df.units['v_wind_gradient'] == '(meter / second) / kilometer)')
assert(df.units['refractive_index'] == 'unitless')
def test_igra2_drvd():
"""Test that we are properly parsing data from the IGRA2 archive."""
df, header = IGRAUpperAir.request_data(datetime(2014, 9, 10, 0),
'USM00070026',
derived=True)
assert_almost_equal(df['pressure'][5], 947.43, 2)
assert_almost_equal(df['reported_height'][5], 610., 2)
assert_almost_equal(df['calculated_height'][5], 610., 2)
assert_almost_equal(df['temperature'][5], 269.1, 2)
assert_almost_equal(df['temperature_gradient'][5], 0.0, 2)
assert_almost_equal(df['potential_temperature'][5], 273.2, 2)
assert_almost_equal(df['potential_temperature_gradient'][5], 11.0, 2)
assert_almost_equal(df['virtual_temperature'][5], 269.5, 2)
assert_almost_equal(df['virtual_potential_temperature'][5], 273.7, 2)
assert_almost_equal(df['vapor_pressure'][5], 4.268, 2)
assert_almost_equal(df['saturation_vapor_pressure'][5], 4.533, 2)
assert_almost_equal(df['reported_relative_humidity'][5], 93.9, 2)
assert_almost_equal(df['calculated_relative_humidity'][5], 94.1, 2)
assert_almost_equal(df['relative_humidity_gradient'][5], -75.3, 2)
assert_almost_equal(df['u_wind'][5], -7.8, 2)
assert_almost_equal(df['u_wind_gradient'][5], 9.6, 2)
assert_almost_equal(df['v_wind'][5], -1.2, 2)
assert_almost_equal(df['v_wind_gradient'][5], 2.7, 2)
assert_almost_equal(df['refractive_index'][5], 295., 2)
assert (df.units['pressure'] == 'hPa')
assert (df.units['reported_height'] == 'meter')
assert (df.units['calculated_height'] == 'meter')
assert (df.units['temperature'] == 'Kelvin')
assert (df.units['temperature_gradient'] == 'Kelvin / kilometer')
assert (df.units['potential_temperature'] == 'Kelvin')
assert (df.units['potential_temperature_gradient'] == 'Kelvin / kilometer')
assert (df.units['virtual_temperature'] == 'Kelvin')
assert (df.units['virtual_potential_temperature'] == 'Kelvin')
assert (df.units['vapor_pressure'] == 'Pascal')
assert (df.units['saturation_vapor_pressure'] == 'Pascal')
assert (df.units['reported_relative_humidity'] == 'percent')
assert (df.units['calculated_relative_humidity'] == 'percent')
assert (df.units['u_wind'] == 'meter / second')
assert (df.units['u_wind_gradient'] == '(meter / second) / kilometer)')
assert (df.units['v_wind'] == 'meter / second')
assert (df.units['v_wind_gradient'] == '(meter / second) / kilometer)')
assert (df.units['refractive_index'] == 'unitless')
"""Downloads all Arctic (Lat > 65) radiosonde data that extend at least from 1990-2020"""
import pandas as pd
import numpy as np
from siphon.simplewebservice.igra2 import IGRAUpperAir
from datetime import datetime
save_loc_soundings = '/Users/watkdani/Documents/Research/IGRA2 Paper/Data/IGRA_derived_soundings/'
save_loc_headers = '/Users/watkdani/Documents/Research/IGRA2 Paper/Data/IGRA_derived_headers/'
station_list_loc = '/Users/watkdani/Documents/Research/IGRA2 Paper/igra2-station-list.txt'
station_list = pd.read_fwf(station_list_loc,
header=None, names=['station_id', 'lat', 'lon', 'elevation', 'name', 'start_year', 'end_year', 'number'])
idx = (station_list.lat >= 65) & (station_list.end_year == 2020)
idx = idx & (station_list.begin_year <= 1990)
arctic_stations = station_list.loc[idx]
for station in arctic_stations.station_id:
dates = [datetime(1990, 1, 1, 0), datetime(2020, 7, 1, 23)]
df, header = IGRAUpperAir.request_data(dates, station, derived=True)
df.to_csv(save_loc_soundings + station + '-igra2-derived.csv')
header.to_csv(save_loc_headers + station + '-igra2-derived.csv')
print(station + ' finished')
def siphon_igra2_to_xarray(station, path=sound_path,
fields=['temperature', 'pressure'],
times=['1984-01-01', '2019-12-31'], derived=False):
from siphon.simplewebservice.igra2 import IGRAUpperAir
import pandas as pd
import numpy as np
import xarray as xr
from urllib.error import URLError
import logging
from aux_functions_strat import path_glob
logger = logging.getLogger('strato_sounding')
# logging.basicConfig(filename=path / 'siphon.log', level=logging.INFO,
# format='%(asctime)s %(levelname)-10s %(processName)s %(name)s %(message)s')
# check for already d/l files:
files = path_glob(path, '*_derived.nc')
names = [x.as_posix().split('/')[-1].split('.')[0] for x in files]
if station in names:
logging.warning('station {} already downloaded, skipping'.format(station))
return '1'
logger.info('fields chosen are: {}'.format(fields))
logger.info('dates chosen are: {}'.format(times))
dates = pd.to_datetime(times)
dates = [x.to_pydatetime() for x in dates]
logger.info('getting {} from IGRA2...'.format(station))
try:
df, header = IGRAUpperAir.request_data(dates, station, derived=derived)
except URLError:
logger.warning('file not found using siphon.skipping...')
return '2'
header = header[header['number_levels'] > 25] # enough resolution
dates = header['date'].values
logger.info('splicing dataframe and converting to xarray dataset...')
ds_list = []
for date in dates:
dff = df[fields].loc[df['date'] == date]
# release = dff.iloc[0, 1]
dss = dff.to_xarray()
# dss.attrs['release'] = release
ds_list.append(dss)
max_ind = np.max([ds.index.size for ds in ds_list])
vars_ = np.nan * np.ones((len(dates), len(fields), max_ind))
for i, ds in enumerate(ds_list):
size = ds[[x for x in ds.data_vars][0]].size
vars_[i, :, 0:size] = ds.to_array().values
Vars = xr.DataArray(vars_, dims=['time', 'var', 'point'])
Vars['time'] = dates
Vars['var'] = fields
ds = Vars.to_dataset(dim='var')
for field in fields:
ds[field].attrs['units'] = df.units[field]
ds.attrs['site_id'] = header.loc[:, 'site_id'].values[0]
ds.attrs['lat'] = header.loc[:, 'latitude'].values[0]
ds.attrs['lon'] = header.loc[:, 'longitude'].values[0]
logger.info('Done!')
if derived:
filename = station + '_derived' + '.nc'
else:
filename = station + '_not_derived' + '.nc'
comp = dict(zlib=True, complevel=9) # best compression
encoding = {var: comp for var in ds.data_vars}
ds.to_netcdf(path / filename, 'w', encoding=encoding)
logger.info('saved {} to {}.'.format(filename, path))
return ds
def updated_data_uploader(station, begin_date, end_date):
global date_for_reading
z = 0
y = 0
stations = [station]
while z < len(stations):
beginning = [begin_date, end_date]
station_name = str(stations[z])
# api request for igra2 data, returns pandas data frame stored in df, header contains header info
df, header = IGRAUpperAir.request_data(beginning, station_name)
x = 0
# creates a matching dataframe with boolean values representing if the index is null
test_for_null = df.notnull()
# drops nan values
df.dropna(subset=['height'])
# initializes at first date before loop, is adjusted through out loop for every station
date = df['date'][1]
# initialize empty list that will hold temperature values used later to determine
temperatures = []
while x < len(df['height']):
# tests to make sure the values in the dataframe are not null before adding anything to list
if test_for_null['temperature'][x] or test_for_null['height'][x]:
# adds current temperature to list
temp = df['temperature'][x]
temperatures.append(temp)
# print("{} temp for day: {}".format(temp, df['date'][x].strftime("%Y, %m, %d")))
# while the date is the same as it was previously initialized temperature records are stored in the
# temps list for further use in determining the altitude at which the temperature is the lowest
# the if statement below checks for when all the temperature records for a particular date have been
# recorded.
if date != df['date'][x]:
# set the new date
date = df['date'][x]
# the below statements are used to convert the date from the dataframe which is a timestamp object
# to a datetime object. DateTime is required to use relativeDelta which is needed for iteration by
# date at a later time
for_reading_year = df['date'][x - 1].strftime("%Y")
for_reading_month = df['date'][x - 1].strftime("%m")
for_reading_day = df['date'][x - 1].strftime("%d")
for_reading_hour = df['date'][x - 1].strftime("%H")
# the below method call returns the oceanic nino index for the current month/year (either -1, 0, 1)
oni = OniData.find_by_date(int(for_reading_year),
int(for_reading_month))
# initialize datetime object
date_for_reading = datetime(int(for_reading_year),
int(for_reading_month),
int(for_reading_day),
int(for_reading_hour))
# determines the index of the lowest temperature in the temperatures list
# the length of temperatures - the index of the min temperature minus - 1 because the current
# position is 1 over
x -= 1
min_temp = len(temperatures) - temperatures.index(
min(temperatures))
# determines the elevation as the lowest temp
elevation_value = df['height'][x - min_temp]
# instantiates reading object
reading = JustReadings(station_name, date_for_reading,
elevation_value, oni)
# saves reading object to database
reading.save_to_db()
# clears temp list
temperatures = []
x += 1
z += 1
def test_igra2_nonexistent():
"""Test behavior when requesting non-existent data."""
with pytest.raises(ValueError) as err:
IGRAUpperAir.request_data(datetime.utcnow(), 'NOSUCHSTATION')
assert 'No data' in str(err.value)
query to the Integrated Global Radiosonde Archive version 2. """ from datetime import datetime from siphon.simplewebservice.igra2 import IGRAUpperAir #################################################### # Create a datetime object for the sounding and string of the station identifier. date = datetime(2014, 9, 10, 0) station = 'USM00070026' #################################################### # Make the request. IGRAUpperAir returns a dataframe containing the sounding data and # a dataframe with station metadata from the sounding header. df, header = IGRAUpperAir.request_data(date, station) #################################################### # Inspect data columns in the dataframe. print(df.columns) # Inspect metadata from the data headers print(header.columns) #################################################### # Pull out a specific column of data. print(df['pressure']) print(header['latitude']) #################################################### # Units are stored in a dictionary with the variable name as the key in the `units`