str(url_base) + "?monyr=" + str(date_last.month) + "/1/" +
str(date_last.year) + "&view=table")
url_current = str(
str(url_base) + "?monyr=" + str(date_current.month) + "/1/" +
str(date_current.year) + "&view=table")
url_next = str(
str(url_base) + "?monyr=" + str(date_next.month) + "/1/" +
str(date_next.year) + "&view=table")
generated_urls = [url_current, url_last, url_next, url_last1
] #will give priority to first in this list
for url in generated_urls:
url_list.append(url)
bulk_scraper(url_list)
for location in locations:
print(location)
#print('updating weather forecasts')
#get_weather_forecast(location,"update")
#print('updating weather temp historic')
#historic_weather_data(location, "T", 'update')
#print('updating weather precip historic')
#historic_weather_data(location, "P", 'update')
if switch == 'a':
get_weather(location, 'update')
elif switch == 'h':
get_weather(location, 'history')
else:
get_weather(location, 'forecast')
def main(filename):
with open(filename) as f:
corn_dict = json.load(f)
print(corn_dict)
corn_weather_dict = {}
total_pct = 0
weather_dict_temp = {}
weather_dict_precip = {}
for key, items in corn_dict.items():
try:
weather_dict_temp = corn_weather_dict["weather_dict_temp"]
weather_dict_precip = corn_weather_dict["weather_dict_precip"]
except Exception:
None
state = str(key)
print(state)
pct = float(items.strip('%'))/100
weather = get_weather(state)
precip_dict = weather[0]
temp_dict = weather[1]
total_pct = float(total_pct) + float(pct)
for key1, data in precip_dict.items():
if key1 in weather_dict_precip.keys():
weather_dict_precip[key1] = pct * precip_dict[key1] + weather_dict_precip[key1]
if key1 not in weather_dict_precip.keys():
weather_dict_precip[key1] = pct * precip_dict[key1]
for key1, data in temp_dict.items():
if key1 in weather_dict_temp.keys():
weather_dict_temp[key1] = pct * temp_dict[key1] + weather_dict_temp[key1]
if key1 not in weather_dict_temp.keys():
weather_dict_temp[key1] = pct * temp_dict[key1]
corn_weather_dict["weather_dict_temp"] = weather_dict_temp
corn_weather_dict["weather_dict_precip"] = weather_dict_precip
weather_dict_temp = corn_weather_dict["weather_dict_temp"]
weather_dict_precip = corn_weather_dict["weather_dict_precip"]
precip_keys = []
temp_keys = []
count = 0
for key, items in corn_dict.items():
url = items[1]
state = str(key)
weather = get_weather(state)
precip_dict = weather[0]
temp_dict = weather[1]
for subkey in precip_dict.keys():
if count < 1:
precip_keys.append(subkey)
for subkey in temp_dict.keys():
if count < 1:
temp_keys.append(subkey)
for subkey in precip_keys:
formattedas_date = datetime.strptime(subkey, "%Y-%m-%d")
if subkey not in precip_dict.keys() and int(formattedas_date.year) != int(date.today().year):
precip_keys.remove(subkey)
for subkey in temp_keys:
formattedas_date = datetime.strptime(subkey, "%Y-%m-%d")
if subkey not in temp_dict.keys() and int(formattedas_date.year) != int(date.today().year):
temp_keys.remove(subkey)
count += 1
date_adj_weather_dict_temp = {}
date_adj_weather_dict_precip = {}
for key, items in weather_dict_temp.items():
if key in temp_keys:
date_adj_weather_dict_temp[key] = float("{0:.2f}".format(items / total_pct))
for key, items in weather_dict_precip.items():
if key in temp_keys:
date_adj_weather_dict_precip[key] = float("{0:.2f}".format(items / total_pct))
corn_weather_dict["weather_dict_temp"] = date_adj_weather_dict_temp
corn_weather_dict["weather_dict_precip"] = date_adj_weather_dict_precip
actual_name = str(filename).replace(path,'')
filename_mix = (destination_path + str(actual_name).replace('.txt', '_weather_profile.txt'))
with open(filename_mix, 'w') as f:
json.dump(corn_weather_dict, f)
def main():
filename = (
r"C:\Users\thoma\Desktop\Python\TH Projects\Port\alpha\corn\corn_dict.txt"
)
with open(filename) as f:
corn_dict = json.load(f)
print(corn_dict)
corn_weather_dict = {}
total_pct = 0
weather_dict_temp = {}
weather_dict_precip = {}
for key, items in corn_dict.items():
try:
weather_dict_temp = corn_weather_dict["weather_dict_temp"]
weather_dict_precip = corn_weather_dict["weather_dict_precip"]
except Exception:
None
url = items[1]
if url != '':
state = key.strip()
print(state)
pct = float(items[0].strip('%'))
weather = get_weather(state)
precip_dict = weather[0]
temp_dict = weather[1]
total_pct = float(total_pct) + float(pct)
for key1, data in precip_dict.items():
if key1 in weather_dict_precip.keys():
weather_dict_precip[key1] = pct * precip_dict[
key1] + weather_dict_precip[key1]
if key1 not in weather_dict_precip.keys():
weather_dict_precip[key1] = pct * precip_dict[key1]
for key1, data in temp_dict.items():
if key1 in weather_dict_temp.keys():
weather_dict_temp[
key1] = pct * temp_dict[key1] + weather_dict_temp[key1]
if key1 not in weather_dict_temp.keys():
weather_dict_temp[key1] = pct * temp_dict[key1]
corn_weather_dict["weather_dict_temp"] = weather_dict_temp
corn_weather_dict["weather_dict_precip"] = weather_dict_precip
weather_dict_temp = corn_weather_dict["weather_dict_temp"]
weather_dict_precip = corn_weather_dict["weather_dict_precip"]
precip_keys = []
temp_keys = []
count = 0
for key, items in corn_dict.items():
url = items[1]
if url != '':
state = key.strip()
print(state)
weather = get_weather(state)
precip_dict = weather[0]
temp_dict = weather[1]
for subkey in precip_dict.keys():
if count < 1:
precip_keys.append(subkey)
for subkey in temp_dict.keys():
if count < 1:
temp_keys.append(subkey)
for subkey in precip_keys:
formattedas_date = datetime.strptime(subkey, "%Y-%m-%d")
if subkey not in precip_dict.keys() and int(
formattedas_date.year) != int(date.today().year):
precip_keys.remove(subkey)
for subkey in temp_keys:
formattedas_date = datetime.strptime(subkey, "%Y-%m-%d")
if subkey not in temp_dict.keys() and int(
formattedas_date.year) != int(date.today().year):
temp_keys.remove(subkey)
count += 1
date_adj_weather_dict_temp = {}
date_adj_weather_dict_precip = {}
for key, items in weather_dict_temp.items():
if key in temp_keys:
date_adj_weather_dict_temp[key] = items / total_pct
for key, items in weather_dict_precip.items():
if key in temp_keys:
date_adj_weather_dict_precip[key] = items / total_pct
corn_weather_dict["weather_dict_temp"] = date_adj_weather_dict_temp
corn_weather_dict["weather_dict_precip"] = date_adj_weather_dict_precip
filename_c = r"C:\Users\thoma\Desktop\Python\TH Projects\Port\tools\weather\weather_profiles\corn_weather_profile.txt"
with open(filename_c, 'w') as f:
json.dump(corn_weather_dict, f)
forecast_dest = (
r"C:\Users\thoma\Desktop\Python\TH Projects\Port\alpha\corn\corn_crop_forecast_time_series_dict.txt"
)
with open(forecast_dest) as f:
forecast_dict = json.load(f)
forecast_dict = diff_from_seasonal(forecast_dict,
5) #seasonal(forecast_dict, 5)
#print(forecast_dict)
inverted_progress_dict = {}
for key, item in forecast_dict.items(): #progress_dict.items():
inverted_progress_dict[key] = -1 * item
weather = get_weather("corn")
precip_dict = weather[0]
temp_dict = weather[1]
#seasonal_patterns_w_price(precip_dict, temp_dict, price_dict, '2015')
#below shows that reported corn progress drives the corn price
seasonal_patterns_w_price(price_dict, price_dict, inverted_progress_dict,
'2019')
seasonal_patterns_w_price(price_dict, price_dict, inverted_progress_dict,
'2018')
seasonal_patterns_w_price(price_dict, price_dict, inverted_progress_dict,
'2017')
seasonal_patterns_w_price(price_dict, price_dict, inverted_progress_dict,
'2016')
seasonal_patterns_w_price(price_dict, price_dict, inverted_progress_dict,
'2015')
seasonal_patterns_w_price(price_dict, price_dict, inverted_progress_dict,
def put_it_together():
progress_dict = get_from_xls()
category_dict = detail_analyse_progress(progress_dict)
#gets a list of dicts with each stage's data e.g. harvest [week, absolute progress, change]
#get max weather date
weather = get_weather("corn")
precip_dict = dict_unpacker(weather[0])
precip_list = []
for key in precip_dict.keys():
precip_list.append(key)
precip_list = sorted(precip_list)
max_date = precip_list[len(precip_list) - 1]
print(max_date)
progress_list = []
for item in category_dict:
progress_forecast = corn_weather_correls(2016, item)
progress_list.append(progress_forecast)
overall_progress_dict = {}
for key in progress_list[0].keys():
try:
add_list = []
for j in range(0, len(progress_list)): #this was 1
if key.year > 1980:
add_list.append(progress_list[j][key])
season_weighter = [2, 1, 1, 1, 1, 1, 2]
weighted_avg = 0
for i in range(0, len(add_list)):
plus = season_weighter[i] * add_list[i]
weighted_avg += plus
weighted_avg = weighted_avg / len(add_list)
if key <= max_date:
overall_progress_dict[str(key).replace(
" 00:00:00", ''
)] = weighted_avg #statistics.mean(add_list)#[statistics.mean(add_list), add_list] #keep for debugging purposes to see all components
except Exception:
None
#print(overall_progress_dict)
key_list = []
for key in overall_progress_dict.keys():
key_list.append(key)
adj_key_list = sorted(key_list)
for key in adj_key_list:
try:
None
print(key, overall_progress_dict[key])
except Exception:
None
progress_dest = (
r"C:\Users\thoma\Desktop\Python\TH Projects\Port\alpha\corn\corn_crop_forecast_time_series_dict.txt"
)
with open(progress_dest, 'w') as f:
json.dump(overall_progress_dict, f)
def corn_weather_correls(train_date_year, category_dict):
season_list = []
season_type = weather_correlation(train_date_year, category_dict,
'n') #'n'no graphs
season_list.append(season_type)
#saving as a list meerley son json.dump will work
with open('season_correlation.txt', 'w') as f:
json.dump(season_list, f)
correl_dict = season_list[0]
weather = get_weather("corn")
precip_dict = dict_unpacker(weather[0])
temp_dict = dict_unpacker(weather[1])
rolling_precip = rolling_day(7, precip_dict)
rolling_temp = rolling_day(7, temp_dict)
#category_dict = detail_analyse_progress(progress_dict)
remaining_dict = {}
exception_list = []
#category dict is date : [week, progress, progress chg]
#this gets the remaining_dict i.e. for the week before how much planting was left
for key, items in category_dict.items():
try:
key = date_to_datetime(key)
remaining_dict[key] = (1 - float(items[1]) + float(items[2]))
except Exception:
exception_list.append(key)
#-------------- START get forecast definition START----------------
#need input item = [week, date, precip, temp, remaining, forecast progress_chg]
def get_item(year, week):
item_date = date_from_isoweek(year, week, 7)
item_date_next = date_from_isoweek(year, week + 1, 7)
try:
item_precip = rolling_precip[item_date]
except Exception:
item_precip = 'precip_error'
try:
item_temp = rolling_temp[item_date]
except Exception:
item_temp = 'temp_error'
#need to fix item remaining to tie in with forecast
try:
item_remaining = remaining_dict[item_date]
except Exception:
item_remaining = 'remaining_error'
input_list = [item_precip, item_temp, item_remaining]
key_list = []
for key in correl_dict.keys():
key_list.append(key)
max_correl_no = str(max(key_list))
used_correl_week = min([str(week), max_correl_no])
correl_list = correl_dict[int(used_correl_week)] #need the int
#for reference this is the correl list, linear_correl = [coeff, ax, b]
#correl_list = [linear_correl(p_values, y_values),
# linear_correl(t_values, y_values),
# linear_correl(r_values, y_values)]
coeff_sum = 0
total_weighted_forecast = 0
for i in range(len(input_list)):
#print(input_list, correl_list)
forecast = correl_list[i][1] * input_list[i] + correl_list[i][2]
coeff = correl_list[i][0]
try:
coeff_sum = coeff_sum + coeff
weighted_forecast = forecast * coeff
total_weighted_forecast += weighted_forecast
except Exception:
coeff_sum = 3
weighted_forecast = forecast
total_weighted_forecast += weighted_forecast
forecast_progress_chg = total_weighted_forecast / coeff_sum
#for when the forecast has no remaining already
try:
next_remaining_check = remaining_dict[
item_date_next] #testing for error in next remaining amount
except Exception:
remaining_dict[
item_date_next] = item_remaining + forecast_progress_chg
return [
week, item_date, item_precip, item_temp, item_remaining,
forecast_progress_chg
]
#--------------END get forecast definition END ----------------
year_list = []
week_list = []
#find the weeks and years in the category dict and the max/mins
for key in category_dict:
#formattedas_date = datetime.strptime(str(key), "%Y-%m-%d")
formattedas_date = key
week = formattedas_date.strftime("%V")
if formattedas_date.year not in year_list:
year_list.append(int(formattedas_date.year))
if week not in week_list:
week_list.append(int(week))
#to ensure this year is in the list
this_year = datetime.today().year
if this_year not in year_list:
year_list.append(this_year)
max_year = max(year_list)
min_year = min(year_list)
max_week = max(week_list)
min_week = min(week_list)
#find the forecast chg for those weeks and years in the category dict and add cum forecast
forecast_dict = {}
for i in range(min_year, max_year + 1):
cum_forecast = 0
for j in range(min_week, max_week + 1):
try:
item = get_item(i, j)
add = abs(
float(item[5])
) #abs because later readings provide big -ves as it uses the correlation all the way
if str(add) == 'nan':
add = 0
cum_forecast += add
item.append(min(
[cum_forecast,
1])) #this is needed to cap forecast progress at 100%
forecast_dict[item[1]] = item
except Exception:
None
output_dict = {}
for key, items in forecast_dict.items():
output_dict[key] = items[6]
forecast_key_list = []
for key in forecast_dict.keys():
forecast_key_list.append(key)
for i in range(min_year, max_year + 1):
for j in range(1, 53):
output_key = date_from_isoweek(i, j, 7) #- timedelta(days=1)
if output_key not in forecast_key_list and j < min_week:
output_dict[output_key] = 0
elif output_key not in forecast_key_list and j > max_week:
output_dict[output_key] = 1
elif output_key not in forecast_key_list:
output_dict[
output_key] = 0 # this fills any errors with zeros, consider removing
#sort output dict for convenience
sorted_output_dict = {}
sorted_list = []
for key in output_dict.keys():
sorted_list.append(key)
sorted_list = sorted(sorted_list)
for key in sorted_list:
sorted_output_dict[key] = output_dict[key]
#print(output_dict)
return sorted_output_dict
def weather_correlation(train_date_year, sample_dict, graph_y_or_n):
"""for progress dict returns weekly correlation list of progress vs precip, temp and last week progress [correl coeff, ax, b]"""
category_dict = sample_dict
weather = get_weather("corn")
precip_dict = dict_unpacker(weather[0])
temp_dict = dict_unpacker(weather[1])
rolling_precip = rolling_day(7, precip_dict)
rolling_temp = rolling_day(7, temp_dict)
unique_week_list = []
week_list = []
change_list = []
progress_list = []
date_list = []
precip_list = []
temp_list = []
exception_list = []
for key, items in category_dict.items():
try:
precip_list.append(rolling_precip[key])
temp_list.append(rolling_temp[key])
week_list.append(items[0])
change_list.append(items[2])
date_list.append(key)
progress_list.append(items[1])
except Exception:
exception_list.append(key)
if items[0] not in unique_week_list:
unique_week_list.append(items[0])
print("These dates not found in the weather dictionary: ", exception_list)
unique_week_list = sorted(unique_week_list)
correl_dict = {}
for i in range(len(unique_week_list)):
p_values, t_values, r_values, y_values, n_values = [], [], [], [], []
for j in range(len(week_list)):
#TRAIN DATE LIMIT HAPPENS HERE
if unique_week_list[i] == week_list[
j] and date_list[j].year <= train_date_year:
y_values.append(change_list[j])
p_values.append(precip_list[j])
t_values.append(temp_list[j])
n_values.append(str(date_list[j]).replace(" 00:00:00", ""))
r_values.append(1 - float(progress_list[j]) +
float(change_list[j]))
if n_values != []:
#if you want graphs == y
if graph_y_or_n == 'y':
identifier1 = "Week " + str(unique_week_list[i]) + ", precip"
identifier2 = "Week " + str(unique_week_list[i]) + ", temp"
identifier3 = "Week " + str(
unique_week_list[i]) + ", amount left"
scatter_graph_correl(p_values, y_values, n_values, identifier1)
scatter_graph_correl(t_values, y_values, n_values, identifier2)
scatter_graph_correl(r_values, y_values, n_values, identifier3)
#otherwise make the correl dict
else:
correl_list = [
linear_correl(p_values, y_values),
linear_correl(t_values, y_values),
linear_correl(r_values, y_values)
]
correl_dict[int(unique_week_list[i])] = correl_list
return correl_dict
# -*- coding: utf-8 -*-
"""
Created on Mon Jul 15 17:45:32 2019
@author: thoma
"""
import sys
import_list = [
r"C:\Users\thoma\Desktop\Python\TH Projects\Port\tools",
r"C:\Users\thoma\Desktop\Python\TH Projects\Port\tools\general\scraper",
r"C:\Users\thoma\Desktop\Python\TH Projects\Port\tools\weather\weather_programs"
]
for import_location in import_list:
if import_location not in sys.path:
sys.path.append(import_location)
from weather_downloader import get_weather
from weather_tools import seasonal_patterns
#import sys
#for p in sys.path:
# print(p)
#weather = get_weather("corn")
weather = get_weather(["Corsham", "", "", "UK"])
precip_dict = weather[0]
temp_dict = weather[1]
for i in range(2009, 2020):
seasonal_patterns(precip_dict, temp_dict, i)