Example #1
0
                str(url_base) + "?monyr=" + str(date_last.month) + "/1/" +
                str(date_last.year) + "&view=table")
            url_current = str(
                str(url_base) + "?monyr=" + str(date_current.month) + "/1/" +
                str(date_current.year) + "&view=table")
            url_next = str(
                str(url_base) + "?monyr=" + str(date_next.month) + "/1/" +
                str(date_next.year) + "&view=table")

            generated_urls = [url_current, url_last, url_next, url_last1
                              ]  #will give priority to first in this list
            for url in generated_urls:
                url_list.append(url)

        bulk_scraper(url_list)

    for location in locations:
        print(location)
        #print('updating weather forecasts')
        #get_weather_forecast(location,"update")
        #print('updating weather temp historic')
        #historic_weather_data(location, "T", 'update')
        #print('updating weather precip historic')
        #historic_weather_data(location, "P", 'update')
        if switch == 'a':
            get_weather(location, 'update')
        elif switch == 'h':
            get_weather(location, 'history')
        else:
            get_weather(location, 'forecast')
def main(filename):
    with open(filename) as f:
        corn_dict = json.load(f)

    print(corn_dict)
    
    corn_weather_dict = {}
    
    total_pct = 0
    weather_dict_temp = {}
    weather_dict_precip = {}
    
    for key, items in corn_dict.items():
        try:
            weather_dict_temp = corn_weather_dict["weather_dict_temp"]
            weather_dict_precip = corn_weather_dict["weather_dict_precip"]
        except Exception:
            None
        
        state = str(key)
        print(state)
        pct = float(items.strip('%'))/100
        weather = get_weather(state)
        precip_dict = weather[0]
        temp_dict = weather[1]
        total_pct = float(total_pct) + float(pct)
        for key1, data in precip_dict.items():
            if key1 in weather_dict_precip.keys():
                weather_dict_precip[key1] = pct * precip_dict[key1] + weather_dict_precip[key1]
            if key1 not in weather_dict_precip.keys():
                weather_dict_precip[key1] = pct * precip_dict[key1]
            
                
        for key1, data in temp_dict.items():
            if key1 in weather_dict_temp.keys():
                weather_dict_temp[key1] = pct * temp_dict[key1] + weather_dict_temp[key1]
            if key1 not in weather_dict_temp.keys():
                weather_dict_temp[key1] = pct * temp_dict[key1]
                
                        
        corn_weather_dict["weather_dict_temp"] = weather_dict_temp
        corn_weather_dict["weather_dict_precip"] = weather_dict_precip
    
    
    weather_dict_temp = corn_weather_dict["weather_dict_temp"]
    weather_dict_precip = corn_weather_dict["weather_dict_precip"]    
    
    precip_keys = []
    temp_keys = []
    count = 0
    for key, items in corn_dict.items():
        url = items[1]
        state = str(key)
        weather = get_weather(state)
        precip_dict = weather[0]
        temp_dict = weather[1]
        
        for subkey in precip_dict.keys():
            if count < 1:
                precip_keys.append(subkey)
        
        for subkey in temp_dict.keys():
            if count < 1:
                temp_keys.append(subkey)        
    
        for subkey in precip_keys:
            formattedas_date = datetime.strptime(subkey, "%Y-%m-%d")
            if subkey not in precip_dict.keys() and int(formattedas_date.year) != int(date.today().year):
                precip_keys.remove(subkey)
            
        for subkey in temp_keys:
            formattedas_date = datetime.strptime(subkey, "%Y-%m-%d")
            if subkey not in temp_dict.keys() and int(formattedas_date.year) != int(date.today().year):
                temp_keys.remove(subkey)
        
        count += 1
    
    
    date_adj_weather_dict_temp = {}
    date_adj_weather_dict_precip = {}
    for key, items in weather_dict_temp.items():
        if key in temp_keys:
            date_adj_weather_dict_temp[key] = float("{0:.2f}".format(items / total_pct))
    
            
    for key, items in weather_dict_precip.items():
        if key in temp_keys:
            date_adj_weather_dict_precip[key] = float("{0:.2f}".format(items / total_pct))
    
        
    corn_weather_dict["weather_dict_temp"] = date_adj_weather_dict_temp
    corn_weather_dict["weather_dict_precip"] = date_adj_weather_dict_precip

    actual_name = str(filename).replace(path,'')    
    filename_mix = (destination_path + str(actual_name).replace('.txt', '_weather_profile.txt'))
    with open(filename_mix, 'w') as f:
        json.dump(corn_weather_dict, f)
Example #3
0
def main():
    filename = (
        r"C:\Users\thoma\Desktop\Python\TH Projects\Port\alpha\corn\corn_dict.txt"
    )
    with open(filename) as f:
        corn_dict = json.load(f)

    print(corn_dict)

    corn_weather_dict = {}

    total_pct = 0
    weather_dict_temp = {}
    weather_dict_precip = {}

    for key, items in corn_dict.items():
        try:
            weather_dict_temp = corn_weather_dict["weather_dict_temp"]
            weather_dict_precip = corn_weather_dict["weather_dict_precip"]
        except Exception:
            None

        url = items[1]
        if url != '':
            state = key.strip()
            print(state)
            pct = float(items[0].strip('%'))
            weather = get_weather(state)
            precip_dict = weather[0]
            temp_dict = weather[1]
            total_pct = float(total_pct) + float(pct)
            for key1, data in precip_dict.items():
                if key1 in weather_dict_precip.keys():
                    weather_dict_precip[key1] = pct * precip_dict[
                        key1] + weather_dict_precip[key1]
                if key1 not in weather_dict_precip.keys():
                    weather_dict_precip[key1] = pct * precip_dict[key1]

            for key1, data in temp_dict.items():
                if key1 in weather_dict_temp.keys():
                    weather_dict_temp[
                        key1] = pct * temp_dict[key1] + weather_dict_temp[key1]
                if key1 not in weather_dict_temp.keys():
                    weather_dict_temp[key1] = pct * temp_dict[key1]

            corn_weather_dict["weather_dict_temp"] = weather_dict_temp
            corn_weather_dict["weather_dict_precip"] = weather_dict_precip

    weather_dict_temp = corn_weather_dict["weather_dict_temp"]
    weather_dict_precip = corn_weather_dict["weather_dict_precip"]

    precip_keys = []
    temp_keys = []
    count = 0
    for key, items in corn_dict.items():
        url = items[1]
        if url != '':
            state = key.strip()
            print(state)
            weather = get_weather(state)
            precip_dict = weather[0]
            temp_dict = weather[1]

            for subkey in precip_dict.keys():
                if count < 1:
                    precip_keys.append(subkey)

            for subkey in temp_dict.keys():
                if count < 1:
                    temp_keys.append(subkey)

            for subkey in precip_keys:
                formattedas_date = datetime.strptime(subkey, "%Y-%m-%d")
                if subkey not in precip_dict.keys() and int(
                        formattedas_date.year) != int(date.today().year):
                    precip_keys.remove(subkey)

            for subkey in temp_keys:
                formattedas_date = datetime.strptime(subkey, "%Y-%m-%d")
                if subkey not in temp_dict.keys() and int(
                        formattedas_date.year) != int(date.today().year):
                    temp_keys.remove(subkey)

            count += 1

    date_adj_weather_dict_temp = {}
    date_adj_weather_dict_precip = {}
    for key, items in weather_dict_temp.items():
        if key in temp_keys:
            date_adj_weather_dict_temp[key] = items / total_pct

    for key, items in weather_dict_precip.items():
        if key in temp_keys:
            date_adj_weather_dict_precip[key] = items / total_pct

    corn_weather_dict["weather_dict_temp"] = date_adj_weather_dict_temp
    corn_weather_dict["weather_dict_precip"] = date_adj_weather_dict_precip

    filename_c = r"C:\Users\thoma\Desktop\Python\TH Projects\Port\tools\weather\weather_profiles\corn_weather_profile.txt"
    with open(filename_c, 'w') as f:
        json.dump(corn_weather_dict, f)
Example #4
0
forecast_dest = (
    r"C:\Users\thoma\Desktop\Python\TH Projects\Port\alpha\corn\corn_crop_forecast_time_series_dict.txt"
)
with open(forecast_dest) as f:
    forecast_dict = json.load(f)

forecast_dict = diff_from_seasonal(forecast_dict,
                                   5)  #seasonal(forecast_dict, 5)
#print(forecast_dict)

inverted_progress_dict = {}
for key, item in forecast_dict.items():  #progress_dict.items():
    inverted_progress_dict[key] = -1 * item

weather = get_weather("corn")
precip_dict = weather[0]
temp_dict = weather[1]
#seasonal_patterns_w_price(precip_dict, temp_dict, price_dict, '2015')
#below shows that reported corn progress drives the corn price
seasonal_patterns_w_price(price_dict, price_dict, inverted_progress_dict,
                          '2019')
seasonal_patterns_w_price(price_dict, price_dict, inverted_progress_dict,
                          '2018')
seasonal_patterns_w_price(price_dict, price_dict, inverted_progress_dict,
                          '2017')
seasonal_patterns_w_price(price_dict, price_dict, inverted_progress_dict,
                          '2016')
seasonal_patterns_w_price(price_dict, price_dict, inverted_progress_dict,
                          '2015')
seasonal_patterns_w_price(price_dict, price_dict, inverted_progress_dict,
Example #5
0
    def put_it_together():
        progress_dict = get_from_xls()
        category_dict = detail_analyse_progress(progress_dict)
        #gets a list of dicts with each stage's data e.g. harvest [week, absolute progress, change]

        #get max weather date
        weather = get_weather("corn")
        precip_dict = dict_unpacker(weather[0])
        precip_list = []
        for key in precip_dict.keys():
            precip_list.append(key)
        precip_list = sorted(precip_list)
        max_date = precip_list[len(precip_list) - 1]
        print(max_date)

        progress_list = []

        for item in category_dict:
            progress_forecast = corn_weather_correls(2016, item)
            progress_list.append(progress_forecast)

        overall_progress_dict = {}

        for key in progress_list[0].keys():
            try:
                add_list = []
                for j in range(0, len(progress_list)):  #this was 1
                    if key.year > 1980:
                        add_list.append(progress_list[j][key])

                season_weighter = [2, 1, 1, 1, 1, 1, 2]
                weighted_avg = 0
                for i in range(0, len(add_list)):
                    plus = season_weighter[i] * add_list[i]
                    weighted_avg += plus
                weighted_avg = weighted_avg / len(add_list)

                if key <= max_date:

                    overall_progress_dict[str(key).replace(
                        " 00:00:00", ''
                    )] = weighted_avg  #statistics.mean(add_list)#[statistics.mean(add_list), add_list] #keep for debugging purposes to see all components
            except Exception:
                None

        #print(overall_progress_dict)

        key_list = []
        for key in overall_progress_dict.keys():
            key_list.append(key)

        adj_key_list = sorted(key_list)
        for key in adj_key_list:
            try:
                None
                print(key, overall_progress_dict[key])
            except Exception:
                None

        progress_dest = (
            r"C:\Users\thoma\Desktop\Python\TH Projects\Port\alpha\corn\corn_crop_forecast_time_series_dict.txt"
        )
        with open(progress_dest, 'w') as f:
            json.dump(overall_progress_dict, f)
Example #6
0
def corn_weather_correls(train_date_year, category_dict):
    season_list = []
    season_type = weather_correlation(train_date_year, category_dict,
                                      'n')  #'n'no graphs
    season_list.append(season_type)

    #saving as a list meerley son json.dump will work
    with open('season_correlation.txt', 'w') as f:
        json.dump(season_list, f)

    correl_dict = season_list[0]

    weather = get_weather("corn")
    precip_dict = dict_unpacker(weather[0])
    temp_dict = dict_unpacker(weather[1])

    rolling_precip = rolling_day(7, precip_dict)
    rolling_temp = rolling_day(7, temp_dict)

    #category_dict = detail_analyse_progress(progress_dict)
    remaining_dict = {}
    exception_list = []

    #category dict is date : [week, progress, progress chg]
    #this gets the remaining_dict i.e. for the week before how much planting was left
    for key, items in category_dict.items():
        try:
            key = date_to_datetime(key)
            remaining_dict[key] = (1 - float(items[1]) + float(items[2]))

        except Exception:
            exception_list.append(key)

    #-------------- START get forecast definition START----------------
    #need input item = [week, date, precip, temp, remaining, forecast progress_chg]
    def get_item(year, week):
        item_date = date_from_isoweek(year, week, 7)
        item_date_next = date_from_isoweek(year, week + 1, 7)

        try:
            item_precip = rolling_precip[item_date]
        except Exception:
            item_precip = 'precip_error'

        try:
            item_temp = rolling_temp[item_date]
        except Exception:
            item_temp = 'temp_error'

        #need to fix item remaining to tie in with forecast
        try:
            item_remaining = remaining_dict[item_date]
        except Exception:
            item_remaining = 'remaining_error'

        input_list = [item_precip, item_temp, item_remaining]

        key_list = []
        for key in correl_dict.keys():
            key_list.append(key)
        max_correl_no = str(max(key_list))
        used_correl_week = min([str(week), max_correl_no])

        correl_list = correl_dict[int(used_correl_week)]  #need the int
        #for reference this is the correl list, linear_correl = [coeff, ax, b]
        #correl_list = [linear_correl(p_values, y_values),
        #       linear_correl(t_values, y_values),
        #       linear_correl(r_values, y_values)]

        coeff_sum = 0
        total_weighted_forecast = 0
        for i in range(len(input_list)):
            #print(input_list, correl_list)
            forecast = correl_list[i][1] * input_list[i] + correl_list[i][2]
            coeff = correl_list[i][0]
            try:
                coeff_sum = coeff_sum + coeff
                weighted_forecast = forecast * coeff
                total_weighted_forecast += weighted_forecast
            except Exception:
                coeff_sum = 3
                weighted_forecast = forecast
                total_weighted_forecast += weighted_forecast

        forecast_progress_chg = total_weighted_forecast / coeff_sum

        #for when the forecast has no remaining already
        try:
            next_remaining_check = remaining_dict[
                item_date_next]  #testing for error in next remaining amount
        except Exception:
            remaining_dict[
                item_date_next] = item_remaining + forecast_progress_chg

        return [
            week, item_date, item_precip, item_temp, item_remaining,
            forecast_progress_chg
        ]
        #--------------END get forecast definition END ----------------

    year_list = []
    week_list = []

    #find the weeks and years in the category dict and the max/mins
    for key in category_dict:
        #formattedas_date = datetime.strptime(str(key), "%Y-%m-%d")
        formattedas_date = key
        week = formattedas_date.strftime("%V")
        if formattedas_date.year not in year_list:
            year_list.append(int(formattedas_date.year))
        if week not in week_list:
            week_list.append(int(week))

    #to ensure this year is in the list
    this_year = datetime.today().year
    if this_year not in year_list:
        year_list.append(this_year)

    max_year = max(year_list)
    min_year = min(year_list)
    max_week = max(week_list)
    min_week = min(week_list)

    #find the forecast chg for those weeks and years in the category dict and add cum forecast
    forecast_dict = {}
    for i in range(min_year, max_year + 1):
        cum_forecast = 0
        for j in range(min_week, max_week + 1):
            try:
                item = get_item(i, j)
                add = abs(
                    float(item[5])
                )  #abs because later readings provide big -ves as it uses the correlation all the way
                if str(add) == 'nan':
                    add = 0
                cum_forecast += add
                item.append(min(
                    [cum_forecast,
                     1]))  #this is needed to cap forecast progress at 100%
                forecast_dict[item[1]] = item
            except Exception:
                None

    output_dict = {}
    for key, items in forecast_dict.items():
        output_dict[key] = items[6]

    forecast_key_list = []
    for key in forecast_dict.keys():
        forecast_key_list.append(key)

    for i in range(min_year, max_year + 1):
        for j in range(1, 53):
            output_key = date_from_isoweek(i, j, 7)  #- timedelta(days=1)
            if output_key not in forecast_key_list and j < min_week:
                output_dict[output_key] = 0
            elif output_key not in forecast_key_list and j > max_week:
                output_dict[output_key] = 1
            elif output_key not in forecast_key_list:
                output_dict[
                    output_key] = 0  # this fills any errors with zeros, consider removing

    #sort output dict for convenience
    sorted_output_dict = {}
    sorted_list = []
    for key in output_dict.keys():
        sorted_list.append(key)

    sorted_list = sorted(sorted_list)
    for key in sorted_list:
        sorted_output_dict[key] = output_dict[key]
    #print(output_dict)
    return sorted_output_dict
Example #7
0
def weather_correlation(train_date_year, sample_dict, graph_y_or_n):
    """for progress dict returns weekly correlation list of progress vs precip, temp and last week progress [correl coeff, ax, b]"""

    category_dict = sample_dict

    weather = get_weather("corn")
    precip_dict = dict_unpacker(weather[0])
    temp_dict = dict_unpacker(weather[1])

    rolling_precip = rolling_day(7, precip_dict)
    rolling_temp = rolling_day(7, temp_dict)

    unique_week_list = []
    week_list = []
    change_list = []
    progress_list = []
    date_list = []
    precip_list = []
    temp_list = []

    exception_list = []
    for key, items in category_dict.items():
        try:
            precip_list.append(rolling_precip[key])
            temp_list.append(rolling_temp[key])
            week_list.append(items[0])
            change_list.append(items[2])
            date_list.append(key)
            progress_list.append(items[1])
        except Exception:
            exception_list.append(key)

        if items[0] not in unique_week_list:
            unique_week_list.append(items[0])

    print("These dates not found in the weather dictionary: ", exception_list)

    unique_week_list = sorted(unique_week_list)
    correl_dict = {}
    for i in range(len(unique_week_list)):
        p_values, t_values, r_values, y_values, n_values = [], [], [], [], []
        for j in range(len(week_list)):
            #TRAIN DATE LIMIT HAPPENS HERE
            if unique_week_list[i] == week_list[
                    j] and date_list[j].year <= train_date_year:
                y_values.append(change_list[j])
                p_values.append(precip_list[j])
                t_values.append(temp_list[j])
                n_values.append(str(date_list[j]).replace(" 00:00:00", ""))
                r_values.append(1 - float(progress_list[j]) +
                                float(change_list[j]))

        if n_values != []:
            #if you want graphs == y
            if graph_y_or_n == 'y':
                identifier1 = "Week " + str(unique_week_list[i]) + ", precip"
                identifier2 = "Week " + str(unique_week_list[i]) + ", temp"
                identifier3 = "Week " + str(
                    unique_week_list[i]) + ", amount left"

                scatter_graph_correl(p_values, y_values, n_values, identifier1)
                scatter_graph_correl(t_values, y_values, n_values, identifier2)
                scatter_graph_correl(r_values, y_values, n_values, identifier3)
            #otherwise make the correl dict
            else:
                correl_list = [
                    linear_correl(p_values, y_values),
                    linear_correl(t_values, y_values),
                    linear_correl(r_values, y_values)
                ]
                correl_dict[int(unique_week_list[i])] = correl_list

    return correl_dict
Example #8
0
# -*- coding: utf-8 -*-
"""
Created on Mon Jul 15 17:45:32 2019

@author: thoma
"""
import sys
import_list = [
    r"C:\Users\thoma\Desktop\Python\TH Projects\Port\tools",
    r"C:\Users\thoma\Desktop\Python\TH Projects\Port\tools\general\scraper",
    r"C:\Users\thoma\Desktop\Python\TH Projects\Port\tools\weather\weather_programs"
]
for import_location in import_list:
    if import_location not in sys.path:
        sys.path.append(import_location)

from weather_downloader import get_weather
from weather_tools import seasonal_patterns

#import sys
#for p in sys.path:
#    print(p)
#weather = get_weather("corn")
weather = get_weather(["Corsham", "", "", "UK"])
precip_dict = weather[0]
temp_dict = weather[1]
for i in range(2009, 2020):
    seasonal_patterns(precip_dict, temp_dict, i)