def test_site_nc(self):
'''Pull met data using the nc files
'''
attributes = ["power", "wind_direction", "wind_speed", "temperature",
"pressure","density"]
leap_day = True
utc = True
start = pandas.Timestamp('2007-08-01', tz='utc')
end = pandas.Timestamp('2007-08-15', tz='utc')
wind_data = get_nc_data("102445", start, end, attributes=attributes, leap_day=leap_day, utc=utc, nc_dir=WIND_MET_NC_DIR)
expected = [9.53277647e-01, 2.16432190e+02, 4.99592876e+00, 2.92580750e+02,
1.01280258e+05, 1.17889750e+00]
numpy.testing.assert_allclose(expected, wind_data.ix[0].values)
self.assertEqual(14*24*12+1, len(wind_data)) # End is inclusive of midnight
utc = False
start = pandas.Timestamp('2007-08-01', tz='America/New_York')
end = pandas.Timestamp('2007-08-15', tz='America/New_York')
# The files return Standard Time rather than Daylight Savings time
#2007,7,31,23,0,1.176,2.7840000000000003,101219.832,290.957,251.209,6.839
#2007,8,1,0,0,1.177,2.458,101206.096,290.874,248.353,6.595
# Testing as though local time should use DST where appropriate
expected = [2.78404403e+00, 2.51209885e+02, 6.83912182e+00, 2.90957336e+02,
1.01219828e+05, 1.17690361e+00]
wind_data = get_nc_data("102445", start, end, attributes=attributes, leap_day=leap_day, utc=utc, nc_dir=WIND_MET_NC_DIR)
numpy.testing.assert_allclose(expected, wind_data.ix[0].values)
self.assertEqual(14*24*12+1, len(wind_data)) # End is inclusive of midnight
def test_partial_year(self):
'''Match site data from forecast nc files
'''
# UTC
start = pandas.Timestamp('2007-01-01', tz='utc')
end = pandas.Timestamp('2007-01-02', tz='utc')
fcst_data = get_nc_data("53252",
start,
end,
utc=True,
nc_dir=WIND_FCST_DIR)
self.assertEqual(start, fcst_data.index[0])
# From ncdump, all values are float32 which do not compare easily to
# python floats which are float64
expected = numpy.array([
6.2671943, 8.6079865, 6.7353525, 6.384234, 0.26309761, 3.6874273,
1.4196928, 0.53551841, 10.572015, 13.249797, 10.526829, 10.306773
],
dtype='float32')
self.assertEqual(25, len(fcst_data))
self.assertTrue(numpy.array_equal(expected, list(fcst_data.ix[0])))
ex_dict = dict(zip(FORECAST_ATTRS, expected))
# Verify column names are correct
for k, v in ex_dict.items():
self.assertEqual(v, fcst_data.ix[0][k])
# Local
#start = pandas.Timestamp('2007-01-01', tz='America/Denver')
#end = pandas.Timestamp('2007-01-02', tz='America/Denver')
fcst_data = get_nc_data("53252",
start,
end,
utc=False,
nc_dir=WIND_FCST_DIR)
self.assertEqual(start, fcst_data.index[0])
# From ncdump, all values are float32 which do not compare easily to
# python floats which are float64
expected = numpy.array([
6.2671943, 8.6079865, 6.7353525, 6.384234, 0.26309761, 3.6874273,
1.4196928, 0.53551841, 10.572015, 13.249797, 10.526829, 10.306773
],
dtype='float32')
self.assertEqual(25, len(fcst_data))
self.assertTrue(numpy.array_equal(expected, list(fcst_data.ix[0])))
ex_dict = dict(zip(FORECAST_ATTRS, expected))
# Verify column names are correct
for k, v in ex_dict.items():
self.assertEqual(v, fcst_data.ix[0][k])
def test_multiple_years(self):
'''Match site data from forecast nc files for multiple years
'''
# UTC
start = pandas.Timestamp('2007-01-01', tz='utc')
end = pandas.Timestamp('2008-08-02', tz='utc')
fcst_data = get_nc_data("53252",
start,
end,
utc=True,
nc_dir=WIND_FCST_DIR)
self.assertEqual(start, fcst_data.index[0])
self.assertEqual(end, fcst_data.index[-1])
def test_aws_wind_data(self):
'''Pull nc data from AWS
'''
#filename = os.path.join(os.path.dirname(__file__), "53252.nc")
nc_dir = "s3://nrel-pds-wtk/wtk-techno-economic/pywtk-data/met_data"
#wkt = "POINT(-103.128662109375 40.24179856487036)"
attributes = [
"power", "wind_direction", "wind_speed", "temperature", "pressure",
"density"
]
#attributes = ["power"]
#names = ["2011"]
leap_day = False
utc = False
start = pandas.Timestamp('2011-01-01', tz='America/Denver')
end = pandas.Timestamp('2011-12-31 23:59:59', tz='America/Denver')
#wind_data = get_nc_data_from_file(filename, start, end, attributes=attributes, leap_day=leap_day, utc=utc)
wind_data = get_nc_data("53252",
start,
end,
attributes=attributes,
leap_day=leap_day,
utc=utc,
nc_dir=nc_dir)
#wind_data = wind_dict["53252"]
#Year,Month,Day,Hour,Minute,density at hub height (kg/m^3),power (MW),surface air pressure (Pa),air temperature at 2m (K),wind direction at 100m (deg),wind speed at 100m (m/s)
#first_line = [2011,1,1,0,0,1.1320000000000001,15.359,85467.688,259.591,318.124,11.844]
# Match to the higher accuracy of the nc dataset
expected = [
15.35958480834961, 318.1243896484375, 11.844223022460938,
259.5919494628906, 85467.6875, 1.132704496383667
]
#expected_dict = dict(zip(["density", "power", "pressure", "temperature", "wind_direction", "wind_speed"], first_line[5:]))
expected_dict = dict(
zip([
'power', 'wind_direction', 'wind_speed', 'temperature',
'pressure', 'density'
], expected))
self.assertEqual(expected_dict, wind_data.iloc[0].to_dict())
#print map(float, wind_data.ix[0].values)
#print wind_data.columns.values
self.assertEqual(365 * 24 * 12, len(wind_data))
def met_data():
'''Return met data from the nc files as to_json representation of pandas
dataframe.
Required parameters:
sites | wkt - string of comma-separated site_ids or Well known text geometry
start - unix timestamp of start time
end - unix timestamp of end time
Optional parameters:
orient - Pandas dataframe to_json orientation, defaults to records:
split, records, index, columns or values
See Pandas documentation for more info
max_point_return - Maximum number of closest sites to a POINT wkt, defaults
to 1. Will be ignored for all other cases.
attributes - string attributes separated by commas to return, will fail
if attribute is not valid for the data set
leap_day - Bool to include leap day. Defaults to True
utc - Bool to use UTC rather than site local time. Defaults to True
Returns:
dict of site id to json representation of dataframes
'''
if len(request.args) == 0:
return send_from_directory('static', 'met.html')
try:
sites = get_sites_from_request(request)
start = pandas.Timestamp(request.args.get('start', type=int),
unit="s",
tz='utc')
end = pandas.Timestamp(request.args.get('end', type=int),
unit="s",
tz='utc')
orient = request.args.get("orient", "records")
if orient not in ["split", "records", "index", "columns", "values"]:
return jsonify({
"success":
False,
"message":
"Orient must be one of split, records, index, columns or values"
}), 400
if "attributes" in request.args:
attributes = request.args.get("attributes", "").split(",")
if not set(attributes) <= set(MET_ATTRS):
return jsonify({
"success":
False,
"message":
"Attributes must be a subset of %s" % MET_ATTRS
}), 400
else:
attributes = MET_ATTRS
ret_dict = {}
for site_id in sites['site_id']:
#ret_dict[site_id] = get_wind_data(site_id, start, end).to_json()
ret_dict[str(site_id)] = json.loads(
get_nc_data(
site_id,
start,
end,
attributes=attributes,
leap_day=True,
utc=False,
nc_dir=MET_LOC).reset_index().to_json(orient=orient))
return jsonify(ret_dict)
except Exception as e:
return jsonify({"success": False, "message": str(e)}), 400