class historicalWeather():
"""Class to get historical weather data and write to postgres database
"""
def __init__(self, pg, ds_key):
self.pg_username = pg['username']
self.pg_password = pg['password']
self.pg_host = pg['host']
self.pg_db = pg['database']
self.pg_port = pg['port']
self.ds_key = ds_key
self.atx_lat = 30.267151
self.atx_lon = -97.743057
self.ds_key = ds_key
self.init_ds_obj()
def write_to_sql(self, times, temps, precips, rain_prob, humidities, wind,
clouds, uv):
with psycopg2.connect(dbname=self.pg_db,
user=self.pg_username,
password=self.pg_password,
host=self.pg_host,
port=self.pg_port) as conn:
with conn.cursor() as curs:
pg_query = """INSERT INTO weather
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT DO NOTHING"""
curs.executemany(
pg_query,
zip(times, temps, precips, rain_prob, humidities, wind,
clouds, uv))
conn.commit()
def init_ds_obj(self):
self.ds = DarkSky(self.ds_key)
def fetch_day_history(self, day):
try:
hist = self.ds.get_time_machine_forecast(
self.atx_lat,
self.atx_lon,
extend=False,
lang=languages.ENGLISH,
units=units.AUTO,
exclude=[weather.MINUTELY, weather.ALERTS],
timezone='UTC',
time=day)
times = [x.time for x in hist.hourly.data]
temps = [x.temperature for x in hist.hourly.data]
precips = [x.precip_intensity for x in hist.hourly.data]
rain_prob = [x.precip_probability for x in hist.hourly.data]
humidities = [x.humidity for x in hist.hourly.data]
wind = [x.wind_speed for x in hist.hourly.data]
clouds = [x.cloud_cover for x in hist.hourly.data]
uv = [x.uv_index for x in hist.hourly.data]
self.write_to_sql(times, temps, precips, rain_prob, humidities,
wind, clouds, uv)
except:
pass
def get_weather(steps):
API_KEY = 'X'
darksky = DarkSky(API_KEY)
for step in steps:
a = darksky.get_time_machine_forecast(step.latitude,
step.longitude,
time=step.time)
step.weather = a.currently
def get_weather_data():
# establish connection to dark sky
darksky = DarkSky(API_KEY)
stadium_locations = pd.read_csv(
os.path.join(data_dir, 'stadium_locations.csv'))
# list of timestamps to collect data for
time_stamps = []
current_date = start_date
while current_date < end_date:
time_stamps.append(current_date)
current_date += relativedelta(days=+1)
# TODO: REMOVE THIS
# time_stamps = time_stamps[:5]
# get weather data
weather_forecast = {}
i = 0
for index, row in stadium_locations.iterrows():
try:
print(
f"Getting data for stadium number {i} / {stadium_locations.shape[0]}: {row['stadium_name']}"
)
forecast_list = []
for time_stamp in time_stamps:
forecast = darksky.get_time_machine_forecast(
row['latitude'],
row['longitude'],
extend=False, # default `False`
lang=languages.ENGLISH, # default `ENGLISH`
values_units=units.AUTO, # default `auto`
exclude=[weather.MINUTELY,
weather.ALERTS], # default `[]`,
timezone=
'UTC', # default None - will be set by DarkSky API automatically
time=time_stamp)
forecast_list.append(forecast)
weather_forecast[row['stadium_name']] = forecast_list
except Exception:
print(f"We failed to get weather data for {row['stadium_name']}")
traceback.print_exc()
finally:
i += 1
# save weather data
pickle.dump(weather_forecast,
open(os.path.join(data_dir, 'weather_forecast.p'), "wb"))
print('Weather data has been obtained')
def execute(self, context):
# Authenticate Socrata client
self.log.info('Authenticate DarkSky client')
darksky = DarkSky(self.dark_sky_api_token)
# render macros to for date
rendered_forecast_date_str = self.forecast_date_str.format(**context)
forecast_date_obj = datetime.datetime.strptime(
rendered_forecast_date_str, '%Y-%m-%d')
# Get Daily Weather results from API endpoint
self.log.info('Query API for hourly weather results')
forecast = darksky.get_time_machine_forecast(
self.latitude,
self.longitude,
forecast_date_obj,
extend=False, # default `False`
lang=languages.ENGLISH, # default `ENGLISH`
units=units.AUTO, # default `auto`
exclude=[
weather.CURRENTLY, weather.MINUTELY, weather.DAILY,
weather.ALERTS
] # default `[]`
)
forecast_hourly_data_object_list = forecast.hourly.data
forecast_hourly_data_dict_list = []
for forecast_hourly_data_object in forecast_hourly_data_object_list:
forecast_hourly_data_dict_list.append(
vars(forecast_hourly_data_object))
self.log.info('Got {} results'.format(
len(forecast_hourly_data_dict_list)))
# Write response to file
self.log.info('Write response to CSV')
rendered_csv_output_filepath = self.csv_output_filepath.format(
**context)
keys = forecast_hourly_data_dict_list[0].keys()
with open(rendered_csv_output_filepath, 'w') as outfile:
dict_writer = csv.DictWriter(outfile, keys)
dict_writer.writeheader()
dict_writer.writerows(forecast_hourly_data_dict_list)
self.log.info(
'Wrote response to {}'.format(rendered_csv_output_filepath))
def get_old_weather(latlong, time):
#latlong = (float,float)
#time = a datetime object
from darksky.api import DarkSky, DarkSkyAsync
from darksky.types import languages, units, weather
API_KEY = get_API_from_file()
# Synchronous way
darksky = DarkSky(API_KEY)
#print (latlong)
latitude, longitude = latlong #[:]
forecast = darksky.get_time_machine_forecast(
latitude,
longitude,
time,
extend=False, # default `False`
lang=languages.ENGLISH, # default `ENGLISH`
#values_units= units.AUTO, # default `auto`
exclude=[weather.MINUTELY] #, weather.ALERTS] # default `[]`
)
return forecast
def get_forecast_sync(self, coordinate):
# Synchronous way Time Machine
from datetime import datetime as dt
API_KEY = self.config.get_config('darksky', 'api', '9bd0b8315895a1c550505b281fc082ff')
darksky = DarkSky(API_KEY)
t = dt(2018, 5, 6, 12)
#latitude = 42.3601
#longitude = -71.0589
latitude, longitude = coordinate
forecast = darksky.get_time_machine_forecast(
latitude, longitude,
extend=False, # default `False`
lang=languages.ENGLISH, # default `ENGLISH`
values_units=units.AUTO, # default `auto`
exclude=[weather.MINUTELY, weather.ALERTS], # default `[]`,
timezone='UTC', # default None - will be set by DarkSky API automatically
time=t
)
print(forecast)
return forecast
passwd="faelb",
database="smaroo_db")
myCursor = mydb.cursor()
format = '%Y-%m-%d %H:%M:%S'
for i in range(2, 30):
t = dt(
2020, 2, i, 3, 0, 0
) # for some reason the API gives day -1 thats why we start at 2 to 30
latitude = 48.210033
longitude = 16.363449
forecast = darksky.get_time_machine_forecast(
latitude,
longitude,
extend=False, # default `False`
lang=languages.ENGLISH, # default `ENGLISH`
values_units=units.SI, # default `auto`
exclude=[weather.MINUTELY, weather.ALERTS], # default `[]`,
timezone=
'GMT', # default None - will be set by DarkSky API automatically
time=t)
for item in forecast.daily:
a = item.time
b = item.apparent_temperature_high
c = item.humidity
sql = "INSERT INTO temperatur (Zeitpunkt, temperatur) VALUES (%s, %s)"
values = (a.strftime(format), int(b))
myCursor.execute(sql, values)
sql = "INSERT INTO humidity (Zeitpunkt, humidity) VALUES (%s, %s)"
values = (a.strftime(format), c)
locations_by_day = {}
weather_by_day = {}
with open(location_file, 'r') as f:
locations_by_day = json.load(f)
for day in sorted(locations_by_day.keys()):
locations_list = locations_by_day[day]
for location in locations_list:
timestamp = datetime.datetime.fromisoformat(location['timestamp'])
latitiude = location['latitude']
longitude = location['longitude']
forecast = darksky.get_time_machine_forecast(
latitiude,
longitude,
extend=False,
lang=languages.ENGLISH,
values_units=units.US,
exclude=[weather.MINUTELY],
timezone='UTC',
time=timestamp)
if not day in weather_by_day:
weather_by_day[day] = []
weather_by_day[day].append(convert_forecast_to_json(forecast))
if not os.path.exists(DATA_DIR):
os.mkdir(DATA_DIR)
with open(os.path.join(DATA_DIR, 'weather_{}.json'.format(year)), 'w') as f:
f.write(json.dumps(weather_by_day, indent=2))
class historicalWeather():
"""Class to get historical weather data and write to postgres database
"""
def __init__(self, pg, ds_key, max_date=None):
self.max_date = max_date
self.pg_username = pg['username']
self.pg_password = pg['password']
self.pg_host = pg['host']
self.pg_db = pg['database']
self.pg_port = pg['port']
self.ds_key = ds_key
self.atx_lat = 30.267151
self.atx_lon = -97.743057
self.ds_key = ds_key
self.init_ds_obj()
def write_to_sql(self, times, temps, precips, rain_prob, humidities, wind,
clouds, uv):
with psycopg2.connect(dbname=self.pg_db,
user=self.pg_username,
password=self.pg_password,
host=self.pg_host,
port=self.pg_port) as conn:
with conn.cursor() as curs:
pg_query = """INSERT INTO weather
VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT DO NOTHING"""
curs.executemany(
pg_query,
zip(times, temps, precips, rain_prob, humidities, wind,
clouds, uv))
conn.commit()
def init_ds_obj(self):
self.ds = DarkSky(self.ds_key)
def fetch_day_history(self, day):
try:
hist = self.ds.get_time_machine_forecast(
self.atx_lat,
self.atx_lon,
extend=False,
lang=languages.ENGLISH,
units=units.AUTO,
exclude=[weather.MINUTELY, weather.ALERTS],
timezone='UTC',
time=day)
times = [x.time for x in hist.hourly.data]
time_check = [
x > pd.to_datetime(self.max_date, utc=True) for x in times
]
times = [
val for (val, boolean) in zip(times, time_check) if boolean
]
temps = [x.temperature for x in hist.hourly.data]
temps = [
val for (val, boolean) in zip(temps, time_check) if boolean
]
precips = [x.precip_intensity for x in hist.hourly.data]
precips = [
val for (val, boolean) in zip(precips, time_check) if boolean
]
rain_prob = [x.precip_probability for x in hist.hourly.data]
rain_prob = [
val for (val, boolean) in zip(rain_prob, time_check) if boolean
]
humidities = [x.humidity for x in hist.hourly.data]
humidities = [
val for (val, boolean) in zip(humidities, time_check)
if boolean
]
wind = [x.wind_speed for x in hist.hourly.data]
wind = [val for (val, boolean) in zip(wind, time_check) if boolean]
clouds = [x.cloud_cover for x in hist.hourly.data]
clouds = [
val for (val, boolean) in zip(clouds, time_check) if boolean
]
uv = [x.uv_index for x in hist.hourly.data]
uv = [val for (val, boolean) in zip(uv, time_check) if boolean]
# self.new_weather = pd.DataFrame({'time': times,
# 'temp': temps,
# 'current_rain': precips,
# 'rain_prob': rain_prob,
# 'humidity': humidities,
# 'wind': wind,
# 'cloud_cover': clouds,
# 'uv': uv})
self.write_to_sql(times, temps, precips, rain_prob, humidities,
wind, clouds, uv)
except:
pass
date_time_str = '2017-07-01 00:00:00'
date_time_obj = datetime.strptime(date_time_str, '%Y-%m-%d %H:%M:%S')
latitude = 43.581381
longitude = -79.647606
# This will do from 2017-07-01 to 2017-07-05 both inclusive
amount_of_days_from_initial_date = 5
for day_to_add in range(amount_of_days_from_initial_date):
date_to_use = date_time_obj + timedelta(days=day_to_add)
forecast = darksky.get_time_machine_forecast(
latitude,
longitude,
date_to_use,
extend=True,
lang=languages.ENGLISH,
units=units.CA,
exclude=[weather.CURRENTLY, weather.DAILY, weather.FLAGS])
print(date_to_use)
# This goes from 9 am to 5 pm, if it is until 4pm replace 18 by 17
for hour_item in forecast.hourly.data[9:18]:
temperature = hour_item.temperature # float
uv_index = hour_item.uv_index # int
wind_speed = hour_item.wind_speed # float
cloud_cover = hour_item.cloud_cover # float
precip_intensity = hour_item.precip_intensity # float
print(str(hour_item.temperature) + '***' + str(hour_item.time))