def plot_water_level_with_fit(station, dates, levels, p): x = matplotlib.dates.date2num(dates) #changing the dates to floats y = levels #plot original data points plt.plot(dates, y, label='water level') typical_lower, typical_higher = station.typical_range[ 0], station.typical_range[1] plt.axhline(y=typical_lower, color='b', label='typical low') plt.axhline(y=typical_higher, color='r', label='typical high') #call ployfit to generate the best fit line and the shift of the x values poly, d0 = polyfit(dates, y, p) #Plot the best fit line plt.plot(dates, poly(x - d0), label='best fit') #labels plt.xlabel('date') plt.ylabel('water level (m)') plt.xticks(rotation=45) plt.title(str(station.name)) plt.legend() # Display plot plt.tight_layout() plt.show()
def prediction_func(): """Function that wraps the prediction code.""" predict_plots = [] for i, station in enumerate(highrisk_stations): try: date, level = predict(station.name, dataset_size=1000, lookback=200, iteration=100, display=300, use_pretrained=True, batch_size=256, epoch=20) except Exception: logger.error('NN prediction failed') date, level = ([], []), ([], [], []) predict_plot = plot_prediction(date, level) try: poly, d0 = polyfit(date[0], level[0], 4) predict_plot.line(date[0] + date[1], [poly(date - d0) for date in date2num(date[0] + date[1])], line_width=2, line_color='gray', legend_label='Polynomial Fit', line_dash='dashed') except TypeError: logger.error('No data for polyfit') predict_plot.plot_width = 400 predict_plot.plot_height = 400 predict_plot.sizing_mode = 'scale_width' predict_plots.append(Panel(child=predict_plot, title=station.name)) predicting_text = Div(text='<p><i>Prediction is running... {:.0%}</i></p>'.format(i / 6)) doc.add_next_tick_callback(partial(update_layout, old=predict_column, new=column(predict_text, predicting_text, width=500, height=650))) predict_tabs = Tabs(tabs=predict_plots) doc.add_next_tick_callback(partial(update_layout, old=predict_column, new=column(predict_text, predict_tabs, width=500, height=650)))
def test_polyfit(): dates = [datetime(2020, 1, 1), datetime(2020, 1, 2), datetime(2020, 1, 3)] levels = [4, 5, 6] poly, d0 = polyfit(dates, levels, 1) assert round(poly(0.1), 1) == 4.1 assert d0 == 737425
def test_polyfit_generates_a_polynomial_with_a_date_offset_of_the_first_date(): initial_date = dt.datetime(2020, 1, 2, tzinfo=dt.timezone.utc) dates = [initial_date, initial_date + dt.timedelta(days=1), initial_date + dt.timedelta(days=2), initial_date + dt.timedelta(days=3), initial_date + dt.timedelta(days=4), initial_date + dt.timedelta(days=5)] expected_polynomial = np.poly1d([1, 2, 3, 4, 5]) levels = [expected_polynomial(0), expected_polynomial(1), expected_polynomial(2), expected_polynomial(3), expected_polynomial(4), expected_polynomial(5)] poly, d0 = polyfit(dates, levels, 5) assert initial_date == d0 assert expected_polynomial[0] == round(poly[0]) assert expected_polynomial[1] == round(poly[1]) assert expected_polynomial[2] == round(poly[2]) assert expected_polynomial[3] == round(poly[3]) assert expected_polynomial[4] == round(poly[4]) assert expected_polynomial[5] == round(poly[5])
def run(): #Building list of stations to fetch data for stations = build_station_list() update_water_levels(stations) N_stations_at_risk = stations_highest_rel_level(stations, 5) #Variables dt = 2 # Fetch data over past 2 days p = 4 #Polynomial degree i = 0 #Counter for s in N_stations_at_risk: i += 1 dates, levels = fetch_measure_levels( s[0].measure_id, dt=datetime.timedelta(days=dt)) #Compiling dates and water levels if not dates: print('Insufficient Data') else: d0, poly = polyfit(dates, levels, p) #Creating polynomial #Plotting data, polynomial and typical high/low plt.subplot(3, 3, i + 1) plt.plot(dates, levels) plt.plot(dates, poly((mpl.dates.date2num(dates) - d0))) plt.axhline(y=s[0].typical_range[0], color='y') plt.axhline(y=s[0].typical_range[1], color='r') plt.xlabel('date') plt.ylabel('water level (m)') plt.xticks(rotation=45) plt.title(s[0].name) plt.show()
def plot_water_level_with_fit(station, dates, levels, p): ''' This function plot the best-fit polynomial of the water level about dates of a certain station with the original data plotted as well Input: station : a MonitoringStation object containing data about a station dates : a list of sample time in a certain period levels : levels of water at the sample times p : the degree of polynomial ''' #get best-fit polynomial from dates and levels poly, d0 = polyfit(dates, levels, p) #shift date so that the first value of the date we use to plot is 0 date_f = matplotlib.dates.date2num(dates) - d0 colors = choice(['b', 'g', 'r', 'c', 'y', 'k']) #plot original data points and best-fit polynomial x = np.linspace(date_f[0], date_f[-1], len(dates)) y = poly(x) plt.plot(dates, y, color=colors) plt.plot(dates, levels, '.', color=colors) plt.xticks(rotation=45) # plot horizontal lines showing the typical range of water level date_length = len(dates) low_r = station.typical_range[0] y0 = [low_r] * date_length high_r = station.typical_range[1] y1 = [high_r] * date_length plt.plot(dates, y0, '--', color=colors) plt.plot(dates, y1, '--', color=colors)
def plot_water_levels_with_fit(station, dates, levels, p): """Inputs: station as MonitoringStation class, dates of data recording and the river levels at that point, and the degree of polynomial to be plotted. Outputs: Plot of fitted polynomial, original data, and typical ranges""" try: #fetch data for each station poly, d0 = polyfit(dates, levels, 10) x = matplotlib.dates.date2num(dates) #plot polynomial plt.plot(dates, poly(x - d0), label="poly fit") #plot original data plot_water_levels(station.name, dates, levels) #plot low and high levels high_low_ranges = station.typical_range low = [high_low_ranges[0]] * len(dates) high = [high_low_ranges[1]] * len(dates) plt.plot(dates, low, label="low level") plt.plot(dates, high, label="high level") plt.legend(loc='upper left') #plt.legend("Bestfit", "Original data", "low level", "high level") plt.show() except: print( "{} does not have full data available to plot for the given period" .format(station.name))
def run(): # Build list of stations stations = build_station_list() update_water_levels(stations) station_and_relative_water_levels = [] for station in stations: station_and_relative_water_levels.append( (station.relative_water_level(), station.name, station)) stations_with_values_for_water_levels = [] for x in station_and_relative_water_levels: if x[0] is None: pass else: stations_with_values_for_water_levels.append(x) stations_with_values_for_water_levels.sort(reverse=True) greatest_5 = stations_with_values_for_water_levels[1:6] p = 4 for n in greatest_5: #Fetch data over past 2 days dt = 2 dates, levels = fetch_measure_levels(n[2].measure_id, dt=datetime.timedelta(days=dt)) #fit data to a polynomial k = polyfit(dates, levels, p) #plot graph with data and also polynomial plot_water_level_with_fit(n[2], dates, levels, k) #plot lines of typical high and low plt.axhline(n[2].typical_range[0], linewidth=2, color='r') plt.axhline(n[2].typical_range[1], linewidth=2, color='r') plt.show()
def plot_water_level_with_fit(station, dates, levels, p): #Collect outputs from polyfit function poly, numdates, d0 = polyfit(dates, levels, p) #Plot the dates against the output function plt.plot(dates, poly(numdates - d0)) #plot the actual value of plot_water_levels(station, dates, levels) plt.show()
def test_polyfit(): stations = build_station_list() station = stations[0] dt = 2 dates, levels = fetch_measure_levels(station.measure_id, dt=datetime.timedelta(days=dt)) N = randint(1, 5) assert type(polyfit(dates, levels, N)) == tuple
def plot_water_level_with_polyfit(station, dates, levels): """Plots water level using real time data and polyfit""" if len(dates) == 0 or len(levels) == 0: print("not enough data available for {}".format(station.name)) else: polynomial, shift = polyfit(dates, levels, 12) x = np.linspace(matplotlib.dates.date2num(dates[-1]), matplotlib.dates.date2num(dates[0]), 200) #creates an array of 30 points between starting and end date plt.plot(x, polynomial(x - shift), label="polyfit") plot_water_levels(station, dates, levels)
def test_polyfit(): #Create dates by converting a series of number into dates x = np.linspace(10001, 10005, 20) dates = mpld.num2date(x) #Ensure that the levels follow a known function x^2 levels = x**2 #Retrieve function outputs from polyfit poly, numdates, d0 = a.polyfit(dates, levels, 2) for i in range(20): #Check if the polyfit function is approximately equal to x**2, up to a certain tolerance assert math.isclose(poly(numdates - d0)[i], x[i]**2, abs_tol=1e-3) == True
def test_polyfit(): dt = 10 #create time and level data to be testerd times, levels = fetch_measure_levels(stations[0].measure_id, dt=timedelta(days=dt)) x = dates.date2num(times) #run polyfit poly, d0 = polyfit(times, levels, 2) assert d0 == x[0] assert isinstance(poly, np.poly1d)
def run(): # Build list of stations stations = build_station_list() N=5 #number of stations we consider of having highest risk of flood update_water_levels(stations) list_of_5_stations_greatest_level=stations_highest_rel_level(stations , N) dt=2 p=4 #degree 4 against time for station in list_of_5_stations_greatest_level: dates, levels = fetch_measure_levels(station.measure_id, dt=datetime.timedelta(days=dt)) poly, d0 = polyfit(dates, levels, p) plot_water_level_with_fit(station, dates, levels, p)
def test_analyse(): stations = build_station_list() dt = 10 stations=sorted(stations, key=lambda x: x.name) for station in stations[:10]: dates, levels = fetch_measure_levels( station.measure_id, dt=datetime.timedelta(days=dt)) p=4 poly, d0 = polyfit(dates,levels,p) dydx=np.polyder(poly) assert dydx==np.poly1d.deriv(poly)
def test_polyfit(): #Create sets of data base_d = datetime.datetime.today() dates = [base_d - datetime.timedelta(seconds=x) for x in range(0, 100000)] levels = np.linspace(0, 10, 100000) poly, x = polyfit(dates, levels, 3) #Round both shifts to same number of decimal places r1 = round(matplotlib.dates.date2num(base_d), 6) r2 = round(x, 6) assert r1 == r2
def test_polyfit(): """For an example set of dates, assert that the level at the date is equal to the level predicted by the polyfit function at that point""" dates = np.array([date(2020, 1, 1), date(2020, 1, 4), date(2020, 1, 8)]) levels = np.array([1.1, 7.6, 2.7]) # a polynomial of order 1 less than the total number of datapoints will pass through them all N = len(dates) - 1 poly, d0 = polyfit(dates, levels, N) # iterate through the dates, and compare the actual level to the estimate value from poly for i, d in enumerate(dates): d_num = date2num(d) assert round(levels[i] - poly(d_num - d0), 6) == 0
def plot_2D(row, col, stations, dt, Loc, Fmt, fit=False, p=0): """ Function that plots data in two dimensional grid""" #Initialise variables i = 0 j = 0 fig, axarr = plt.subplots(row, col) for station in stations: dates, levels = fetch_measure_levels(station.measure_id, dt=datetime.timedelta(days=dt)) #Pass inconsistent data if levels == []: pass else: #Set details for each subplot axarr[i, j].plot(dates, levels) axarr[i, j].xaxis.set_major_locator(Loc) axarr[i, j].xaxis.set_major_formatter(Fmt) axarr[i, j].set_title(station.name) line1 = axarr[i, j].axhline(y=station.typical_range[0], c='g', label='Typical low', linestyle='--') line2 = axarr[i, j].axhline(y=station.typical_range[1], c='r', label='Typical high', linestyle='--') if fit == True: #Convert datetime object into integers x = matplotlib.dates.date2num(dates) y = levels #Get line of best fit object and x-axis shift poly, d0 = polyfit(dates, levels, p) # Plot polynomial fit at 30 points along interval x1 = np.linspace(d0, d0 - dt, 30) line3, = axarr[i, j].plot(x1 , poly(x1 - d0), label='Line of best fit') plt.legend(handles = [line3, line2, line1]) else: plt.legend(handles=[line2, line1]) pass #Algorithm to move from top to bottom, left to right if i < row-1: i += 1 elif i == row-1: j += 1 i = 0 plt.subplots_adjust(bottom=0.02, right=0.98, left=0.02, top=0.98)
def plot_water_level_with_fit(station, dates, levels, p): """Plots a graph of dates against time on the current figure Note that this DOES NOT show the figure - must call plt.plot() after this function This allows for data to be added to the plot after this function is called""" today = date2num(dates[0]) poly, d0 = polyfit(dates, levels, p) xs = np.linspace(today - 2, today, 1000) ys = poly(xs - d0) plt.plot(xs - d0, ys) plt.title(station.name + " Water Level") plt.xlim(-2, 0) plt.xlabel("Time until last reading (days)") plt.ylabel("Level measurement")
def test_polyfit(): stations = build_station_list() list_of_5_stations_random = [] p = 0 for station in stations: list_of_5_stations_random.append(station) p = p + 1 if p == 5: break dt = 2 p = 4 #degree 4 against time for station in list_of_5_stations_random: dates, levels = fetch_measure_levels(station.measure_id, dt=datetime.timedelta(days=dt)) poly, d0 = polyfit(dates, levels, p) assert isinstance(poly, tuple) == True
def plot_water_level_with_fit(station, dates, levels, p): function, offset = polyfit( dates, levels, p) #Find the expression and offset for the funciton of best fit days = matplotlib.dates.date2num(dates) #Change dates to list of days net_days = (offset - days) #start counted from day one plt.plot(net_days, function(net_days), label="Height from best fit") #Plot the graph new_plot(station, net_days, levels) #plot the height coming from data typical = station.typical_range plt.axhline(y=typical[0], label="min typical level", color='g') #plots the minimum typical level as line on graph plt.axhline(y=typical[-1], label="max typical level", color='r') #plots the maximum typical level as line on graph plt.xlabel('$days$') plt.ylabel('$heights$') plt.legend() plt.show()
def plot_water_level_with_fit(station, dates, levels, p): x = matplotlib.dates.date2num(dates) poli = polyfit(dates, levels, p) plt.plot(dates, levels) x1 = np.linspace(x[0], x[-1], len(dates)) plt.plot(dates, poli(x1)) low_range = [station.typical_range[0]] * (len(levels)) high_range = [station.typical_range[1]] * (len(levels)) plt.plot(dates, low_range) plt.plot(dates, high_range) # Add axis labels, rotate date labels and add plot title plt.xlabel('date') plt.ylabel('water level (m)') plt.xticks(rotation=45) plt.title(station.name) plt.tight_layout() # This makes sure plot does not cut off date labels plt.show()
def test_polyfit_returns_false_if_the_dates_and_levels_arrays_are_not_the_same_length(): initial_date = dt.datetime(2020, 1, 2, tzinfo=dt.timezone.utc) dates = [initial_date, initial_date + dt.timedelta(days=1), initial_date + dt.timedelta(days=2), initial_date + dt.timedelta(days=3), initial_date + dt.timedelta(days=4), initial_date + dt.timedelta(days=5)] polynomial = np.poly1d([1, 2, 3, 4, 5]) levels = [polynomial(0), polynomial(1), polynomial(2), polynomial(3), polynomial(4)] poly, d0 = polyfit(dates, levels, 5) assert poly is False assert d0 is False
def plot_water_levels_with_fit(listinput, p): """Add best-fit line to water level graphs, and display them. Parameters ---------- listinput : list list of station (MonitoringStation), dates (list), and levels (list), in this order. List must be of length multiple of 3. p : int order of polynomial fit Returns ------- None. """ fig = create_water_levels_plot(listinput) for i in range(len(listinput) // 3): # initialize values to plot dates = listinput[3 * i + 1] levels = listinput[3 * i + 2] poly, d0 = polyfit(dates, levels, p) # convert dates to minutes (integers), normalized to start at zero x = [date.timestamp() / 60 for date in dates] offset = d0.timestamp() / 60 x = [a - offset for a in x] # calculate levels from fit x_levels = poly(x) # plot curve fig.add_trace(go.Scatter(x=dates, y=x_levels, mode='lines', name='Fitted water level', showlegend=(i == 0), legendgroup="fittedlevel", line_color='gray'), row=i + 1, col=1) plot(fig, auto_open=True)
def plot_0D(stations, dt, Loc, loc, Fmt, fmt, fit=False, p=0): """ Function that plots data in one graph""" #Initialise variables fig, axarr = plt.subplots(1, 1) for station in stations: dates, levels = fetch_measure_levels(station.measure_id, dt=datetime.timedelta(days=dt)) #Pass inconsistent data if levels == []: pass else: plt.plot(dates,levels, label=station.name) axarr.xaxis.set_major_locator(Loc) axarr.xaxis.set_major_formatter(Fmt) if loc and fmt != 0: axarr.xaxis.set_minor_locator(loc) axarr.xaxis.set_minor_formatter(fmt) axarr.tick_params(which = 'major', labelsize = '14') #line1 = axarr.axhline(y=station.typical_range[0], c='g', #label='Typical low', linestyle='--') #line2 = axarr.axhline(y=station.typical_range[1], c='r', #label='Typical high', linestyle='--') if fit == True: #Convert datetime object into integers x = matplotlib.dates.date2num(dates) y = levels #Get line of best fit object and x-axis shift poly, d0 = polyfit(dates, levels, p) # Plot polynomial fit at 30 points along interval x1 = np.linspace(d0, d0 - dt, 30) line3, = axarr.plot(x1 , poly(x1 - d0), label='Best fit ' + station.name) plt.legend(handles = [line3])#, line2, line1]) #else: #plt.legend(handles=[line2, line1]) #pass plt.subplots_adjust(bottom=0.02, right=0.9, left=0.02, top=0.98) plt.legend(bbox_to_anchor=(1.005, 1), loc=2, borderaxespad=0.)
def plot_water_level_with_fit(station, dates, levels, p): """plot_water_level_with_fit(station, dates, levels, p) -- Plots the water level data and the best-fit polynomial.""" poly, d0 = polyfit(dates, levels, p) x = matplotlib.dates.date2num(dates) plt.plot(dates, levels, '.') x1 = np.linspace(x[0], x[-1], 30) plt.plot(x1, poly(x1 - d0)) plt.plot([min(dates), max(dates)], [station.typical_range[0], station.typical_range[0]]) plt.plot([min(dates), max(dates)], [station.typical_range[1], station.typical_range[1]]) plt.xlabel('Date/time since %s' % dates[0]) plt.ylabel('water level (m)') plt.xticks(rotation=45) plt.title("{}".format(station.name)) # Display plot plt.tight_layout() # This makes sure plot does not cut off date labels plt.show()
def plot_water_level_with_fit(station, dates, levels, p): pol, d0 = polyfit(dates, levels, p) predicted_levels = [] for date in dates: cd = matplotlib.dates.date2num(date) pl = pol(cd - d0) predicted_levels.append(pl) highest = [station.typical_range[1]] * len(dates) lowest = [station.typical_range[0]] * len(dates) plt.plot(dates, predicted_levels) plt.plot(dates, levels) plt.plot(dates, lowest) plt.plot(dates, highest) plt.xlabel("date") plt.ylabel("water level (m)") plt.xticks(rotation=45) plt.title("Station: {}\n".format(station.name)) plt.tight_layout plt.show() return "No Error"
def evaluate_risk(station, gradient_weight, time_weight, height_weight): dt = 2 p = 4 #degree 4 against time risk_level = 0 dates, levels = fetch_measure_levels(station.measure_id, dt=datetime.timedelta(days=dt)) try: #Tries to see if there is data and can fit a polynomial to it data_consistent = True poly, d0 = polyfit(dates, levels, p) gradient = np.gradient(poly) risk_level += gradient_weight * gradient[-1] if gradient[:1] > 0: gradient2 = list(np.gradient(gradient)) gradient2.reverse() try: #tries to find lenght of time that the water has been rising index_min = gradient.index(0) risk_level += (len(gradient) - index_min) * time_weight except: pass risk_level += station.relative_water_level() * height_weight except: data_consistent = False return risk_level, data_consistent
def run(): # Build list of stations stations = build_station_list() # Update latest level data for all stations update_water_levels(stations) #create list of names stations to be plotted with their measure_id highest_levels = stations_highest_rel_level(stations, 60) stations_to_check = [] #retrieve the rest of station data for the high for j in highest_levels: for i in stations: if j[0] == i.name: stations_to_check.append(i) #time period for plot dt = 2 polydegree = 10 severe = [] high = [] low = [] moderate = [] rating_store = [low, moderate, high, severe] ratings = ['severe', 'high', 'moderate', 'low'] for i in stations_to_check: #reset score to determin rating category = 0 #if there is no value for relative water level, we need to check the rest of the data anyway so set it to greater than 1 if i.relative_water_level() is None: rel_level = 2 else: rel_level = i.relative_water_level() #check to see if level is above average if rel_level > 1: category = category + 2 #fetch data for each station times, levels = fetch_measure_levels(i.measure_id, dt=timedelta(days=dt)) x = dates.date2num(times) #create polyfit for stations, if there is no data available in this period it cannot be done. if len(times) != 0: poly, d0 = polyfit(times, levels, polydegree) trend = poly.deriv() #gradient of the polyfit being positive shows a general worsening of the situation if trend(x[0] - d0) > 0: category = category + 1 #add the station's town to the appropriate rating list rating_store[category].append(i.town) #print each rating list for i in range(len(ratings)): print("\n", "The following Towns have a {} warning:".format(ratings[i])) for j in rating_store[len(ratings) - i - 1]: print(j) return
def run(): """Task2G: Using your implementation, list the towns where you assess the risk of flooding to be greatest. Explain the criteria that you have used in making your assessment, and rate the risk at ‘severe’, ‘high’, ‘moderate’ or ‘low’.""" # Build list of stations stations = build_station_list() update_water_levels(stations) # Define N N = 20 # Setting the time interval to 2 days dt = 2 # Run curve fitting with degree of 4 p = 4 shortlist = flood.stations_highest_rel_level(stations, N) # First find the stations at risk target_stations = [] for station in shortlist: dates, levels = fetch_measure_levels(station.measure_id, dt=datetime.timedelta(days=dt)) # print('station.measure_id =', station.measure_id) # print('len(dates) =', len(dates)) # print('len(levels) =', len(levels)) if len(dates) < 1 or len(levels) < 1: # if there is no data, fitting will throw an error continue # calculate the fitting polynomial poly, d0 = polyfit(dates, levels, p) x = matplotlib.dates.date2num(dates) # get the latest data point now = max(x - d0) # predict the water level one day later # using the fitted polynomial prediction = poly(now + 1) r_level = station.relative_water_level() predicted_r_level = predicted_relative_water_level(station, prediction) # calculate the predicted rise in relative water level rise = predicted_r_level - r_level if predicted_r_level > r_level: target_stations.append([station.name, rise]) print("{}:\n\tRelative water level: {}\n\tPredicted relative water level: {}\n\tRise: {}".format( station.name, r_level, predicted_r_level, rise)) # now the stations at risk are marked # get towns at risk # elements: [town, number_of_nearest_stations_at_risk] target_towns = [] for i, stations_risk in enumerate(target_stations): if stations_risk[0] in [towns for towns, count in target_towns]: target_towns[i][1] += stations_risk[1] else: target_towns.append(stations_risk[:]) # sort target_towns by risks in descending order target_towns.sort(key=lambda x: x[1], reverse=True) print('-' * 70) print('List of target towns and the estimated flood risk:') print(target_towns) print('-' * 70) print('The towns where the risk of flooding is assessed to be the greatest:') ratings = ['low', 'moderate', 'high', 'severe'] for town, risk in target_towns: rating_factor = 0 # low if risk > 0.5: rating_factor = 1 # moderate if risk > 5: rating_factor = 2 # high if risk > 10: rating_factor = 3 # severe print('{}:\n\t{}'.format(town, ratings[rating_factor]))