def object_minimize(x,t,cumdata_cases,cum_deaths): SUCQ = odeint(sucq, [x[9],x[10],x[11],x[12],x[13],x[14],x[15],0,x[16]],t, args=(x[0],x[1],x[2],x[3],x[4],x[5],x[6],x[7],x[8])) w1=1 w2=1 r1 = -1*hy.nse(SUCQ[:,3],cumdata_cases) r2 = -1*hy.nse(SUCQ[:,7],cum_deaths) return (w1*r1+w2*r2)/(w1+w2)
def metrics_computation(data_list): """Calculates the metrics values based on the model output. (RMSE, MSE, MAE and R²) Parameters ---------- data_list : list with the separated data corresponding to each station. Returns ------- m_list : list with the separated data corresponding to each station with metrics included. """ m_list = [] for el in data_list: el["rmse"], el["mse"], el["mae"], el["r2"] = 0, 0, 0, 0 el["vaf"], el["nse"] = 0, 0 lines = el.shape[0] for i in range(lines): yt = np.asarray(el.iloc[i, 6], dtype=np.float32) yp = np.asarray(el.iloc[i, 7], dtype=np.float32) el.iloc[i, 10] = metrics.mean_squared_error(yt, yp, squared=False) el.iloc[i, 11] = metrics.mean_squared_error(yt, yp, squared=True) el.iloc[i, 12] = metrics.mean_absolute_error(yt, yp) el.iloc[i, 13] = metrics.r2_score(yt, yp) el.iloc[i, 14] = vaf_metric(yt, yp) el.iloc[i, 15] = he.nse(yt, yp) m_list.append(el) return m_list
def object_minimize(x,t,cumdata_cases): SUCQ = odeint(sucq, [x[3],x[4],x[5],x[6]],t, args=(x[0],x[1],x[2])) # # if cumdata_cases[-1:] < 1000: # return sum( ( np.log10(cumdata_cases)-np.log10(SUCQ[:,3].ravel()) )**2) # if cumdata_cases[-1:] >= 1000: # return sum((cumdata_cases-SUCQ[:,3].ravel())**2) return -1*hy.nse(SUCQ[:,3],cumdata_cases)
def Ajust_SUCQ(FILE,pop,extrapolação,day_0,variavel,pasta): data_covid = pd.read_csv(pasta+"/"+FILE, header = 0, sep = ";") data_covid = data_covid[['DateRep',variavel]] nome_data = 'DateRep' df = data_covid day_0_str=df[nome_data][0][:4]+'-'+df[nome_data][0][5:7]+'-'+df[nome_data][0][-2:] #day_0_str = data_covid['DateRep'][0][-4:]+'-'+data_covid['DateRep'][0][3:5]+'-'+data_covid['DateRep'][0][:2] date = np.array(day_0_str, dtype=np.datetime64)+ np.arange(len(data_covid)) if date[0]>=np.array(day_0, dtype=np.datetime64): t0 = 0 else: dif_dias =np.array('2020-03-18', dtype=np.datetime64)-date[0] t0 = dif_dias.astype(int) date= date[t0:] cumdata_covid = data_covid[['Cases']].cumsum() cumdata_cases = cumdata_covid['Cases'].values[t0:] days_mens = np.linspace(1,len(cumdata_cases),len(cumdata_cases)) p0,bsup,binf = bounds(FILE) p0.append(cumdata_cases[0]) bsup.append(cumdata_cases[0]+10**-9) binf.append(cumdata_cases[0]-10**-9) popt = min_minimize(cumdata_cases,C,p0,days_mens,bsup,binf) r,p,K,Co = popt solution = C(days_mens, r, p, K,Co) NSE = hy.nse(solution,cumdata_cases) RMSE = hy.rmse(solution,cumdata_cases) MARE = hy.mare(solution,cumdata_cases) print(FILE[9:-4]) print("r = %f " % (r)) print("p = %f " % (p)) print("K = %f " % (K)) print("NSE = %.5f " % (NSE)) print("#######################") days_mens = np.linspace(1,len(cumdata_cases)+365,len(cumdata_cases)+365) solution = C(days_mens, r, p, K,Co) date_future = np.array(date[0], dtype=np.datetime64)+ np.arange(len(date)+extrapolação) saída = pd.DataFrame(solution, columns=["Cases"]) saída["date"] = date_future path_out = "C:/Users/ravel/OneDrive/Área de Trabalho/DataScientist/sklearn/COVID-19/CasosPorEstado/GLM_covid19/data/data_simulated" saída.to_csv(path_out+"/"+FILE,sep=";") return [r,p,K,NSE,RMSE,MARE]
def Ajust_SUCQ(data_covid, pop, passo, j): data_covid = data_covid[['DateRep', 'Cases']] t0 = j tf = j + passo cumdata_covid = data_covid[['Cases']].cumsum() cumdata_cases = cumdata_covid['Cases'].values[t0:tf] # data=data_covid[['DateRep']].values[t0:tf] # day_last = data[-1:][0][0] data = data_covid[['DateRep']].values[t0:tf][-1:][0][0] day_last_str = data[:4] + '-' + data[5:7] + '-' + data[-2:] day_last = day_last_str #np.array(day_last_str, dtype=np.datetime64) N = pop * 10**6 t = np.linspace(1, len(cumdata_cases), len(cumdata_cases)) So, Uo, Qo, Co = [ .9 * N, 6 * cumdata_cases[0], cumdata_cases[0], cumdata_cases[0] ] # padrão [.8*N,6*cumdata_cases[0],cumdata_cases[0],cumdata_cases[0]] alfa_0, beta_0, gama1_0 = [.2 / So, .3, .1] # padrão [.5/N,.1,.19] p0 = [alfa_0, beta_0, gama1_0, So, Uo, Qo, Co] bsup = [0.9 / So, .50, .20, N, Uo * 2., Qo * 2.0, Co + 10**-9] binf = [0.1 / So, .05, .01, .7 * N, Uo * .5, Qo * 0.5, Co - 10**-9] #p0 = ajust_curvefit(days_mens,cumdata_cases,p0,bsup,binf) popt = min_minimize(cumdata_cases, sucq_solve, p0, t, bsup, binf) alfa_0, beta_0, gama1_0, So, Uo, Qo, Co = popt solution = SUCQ(t, alfa_0, beta_0, gama1_0, So, Uo, Qo, Co) NSE = hy.nse(np.log10(cumdata_cases), np.log10(solution[:, 3])) if NSE >= 0.25: return [alfa_0 * So / gama1_0, day_last] else: return [np.nan, day_last]
def Ajust_SUCQ(FILE,pop,extrapolação): path = 'C:/Users/ravellys/Documents/GitHub/COVID-19-Brasil/COVID-19-Brasil/data/DADOS/' data_covid = pd.read_csv(path+FILE, header = 0, sep = ";") data_covid=data_covid[['DateRep','Cases']] nome_data = 'DateRep' df = data_covid[['DateRep','Cases']] day_0_str=df[nome_data][0][:4]+'-'+df[nome_data][0][5:7]+'-'+df[nome_data][0][-2:] date = np.array(day_0_str, dtype=np.datetime64)+ np.arange(len(data_covid)) if date[0]>=np.array('2020-05-09', dtype=np.datetime64): t0 = 0 else: dif_dias =np.array('2020-05-09', dtype=np.datetime64)-date[0] t0 = dif_dias.astype(int) date= date[t0:] cumdata_covid = df[['Cases']].cumsum() cumdata_cases = cumdata_covid['Cases'].values[t0:] days_mens = np.linspace(1,len(cumdata_cases),len(cumdata_cases)) N = pop*10**6 t = days_mens So,Uo,Qo,Co = [.9*N,6*cumdata_cases[0],cumdata_cases[0],cumdata_cases[0]] # padrão [.8*N,6*cumdata_cases[0],cumdata_cases[0],cumdata_cases[0]] alfa_0,beta_0,gama1_0= [.2/So,.3,.1] # padrão [.5/N,.1,.19] p0 = [alfa_0,beta_0,gama1_0,So,Uo,Qo,Co] bsup = [0.4/So,.50,.20, N,Uo*2.,Qo*2.0,Co+10**-9] binf = [0.09/So,.05,.01,.7*N,Uo*.5,Qo*0.5,Co-10**-9] #p0 = ajust_curvefit(days_mens,cumdata_cases,p0,bsup,binf) popt = min_minimize(cumdata_cases,sucq_solve,p0,t,bsup,binf) alfa_0,beta_0,gama1_0,So,Uo,Qo,Co = popt solution = SUCQ(t,alfa_0,beta_0,gama1_0,So,Uo,Qo,Co) NSE = hy.nse(solution[:,3],cumdata_cases) RMSE = hy.rmse(solution[:,3],cumdata_cases) MARE = hy.mare(solution[:,3],cumdata_cases) print(FILE[9:-4]) print("alfa = %f " % (alfa_0*So)) print("beta = %f " % (beta_0)) print("gamma = %f " % (gama1_0)) print("R = %f" %(alfa_0*So/gama1_0)) print("NSE = %.5f " % (NSE)) print("#######################") date_future = np.array(date[0], dtype=np.datetime64)+ np.arange(len(date)+extrapolação) days_future = np.linspace(1,len(cumdata_cases)+extrapolação,len(cumdata_cases)+extrapolação) Cum_cases_estimated = SUCQ(days_future, *popt) estimativafutura_saída=pd.DataFrame(Cum_cases_estimated[:,3], columns = ["Cases"]) estimativafutura_saída["S"]=Cum_cases_estimated[:,0] estimativafutura_saída["U"]=Cum_cases_estimated[:,1] estimativafutura_saída["Q"]=Cum_cases_estimated[:,2] estimativafutura_saída["I"] = Cum_cases_estimated[:,1]+Cum_cases_estimated[:,2]+Cum_cases_estimated[:,3] estimativafutura_saída["date"] = date_future fileout = 'C:/Users/ravellys/Documents/GitHub/COVID-19-Brasil/COVID-19-Brasil/data/DADOS_estimados/' estimativafutura_saída.to_csv(fileout+FILE,sep=";") #par_SUQC_head = ["estado","população","data inicial","data final", "R","alfa","beta","gamma","So","Uo","Qo","Co","NSE"] #par_SUQC = {"estado":FILE[9:-4],"população":pop,"data inicial":date[0],"data final": date[-1:][0],"R": alfa_0*N/gama1_0,"alfa":popt[0]*N,"beta":popt[1],"gamma":popt[2],"So":popt[3],"Uo":popt[4],"Qo":popt[5],"Co":popt[6],"NSE":NSE} #append_dict_as_row("C:/Users/ravel/OneDrive/Área de Trabalho/DataScientist/sklearn/COVID-19/par_SUCQ.csv",par_SUQC,par_SUQC_head) return [alfa_0*So/gama1_0,alfa_0*So, beta_0, gama1_0, So,Uo,Qo,Co, NSE, RMSE, MARE]
def Sensitivity(mypath, FILE, pop): print(FILE) t0 = 0 data_mensured = pd.read_csv(mypath + "/" + FILE, header=0, sep=";") data_mensured = data_mensured[['DateRep', 'Cases']] cumdata_covid = data_mensured[['Cases']].cumsum() cumdata_cases = cumdata_covid['Cases'].values[t0:] t = np.linspace(1, len(cumdata_cases), len(cumdata_cases)) for i in população: if i[0] == FILE[9:-4]: pop = float(i[1]) problem = { 'num_vars': 4, 'names': ['r', 'p', 'K', 'alfa'], 'bounds': [[0.001, 50], [0., 1.0], [np.log10(0.01 * pop * 10**6), np.log10(.5 * pop * 10**6)], [-10, 1]] } nsamples = 5000 param_values = saltelli.sample(problem, nsamples) Y = [] W = [] for i in param_values: r, p, K, alfa = i alfa = 10**alfa K = 10**K Co = cumdata_cases[0] cases_simulated = C(t, r, p, K, 10**alfa, Co) NSE = hy.nse(cases_simulated, cumdata_cases) if NSE <= 1: NSE = NSE else: NSE = -1000000000 if NSE >= 0.75: W.append([r, p, K, alfa, NSE]) Y.append([r, p, K, alfa, NSE]) Y = np.array(Y) W = np.array(W) print(W) df_Y = pd.DataFrame(Y, columns=['r', 'p', 'K', 'alfa', 'NSE']) df_W = pd.DataFrame(W, columns=['r', 'p', 'K', 'alfa', 'NSE']) Si = sobol.analyze(problem, df_Y["NSE"].values) df_S = pd.DataFrame(Si["S1"], columns=["S1"]) df_S["ST"] = Si["ST"] df_S.to_csv( "C:/Users/ravel/OneDrive/Área de Trabalho/DataScientist/sklearn/COVID-19/CasosPorEstado/Richards_covid19/data/sensibilidade/" + FILE, sep=";") binf = np.array([min(df_W.r), min(df_W.p), min(df_W.K), min(df_W.alfa)]) bsup = np.array([max(df_W.r), max(df_W.p), max(df_W.K), max(df_W.alfa)]) NSE_max = max(df_W.NSE) for i in range(len(df_W)): if df_W.NSE[i] == NSE_max: pos_max = i p0 = np.array([ df_W.r[pos_max], df_W.p[pos_max], df_W.K[pos_max], df_W.alfa[pos_max] ]) par = np.array([binf, bsup, p0]) df_par = pd.DataFrame(par, ["binf", "bsup", "p0"]).transpose() df_par.to_csv( "C:/Users/ravel/OneDrive/Área de Trabalho/DataScientist/sklearn/COVID-19/CasosPorEstado/Richards_covid19/data/range/" + FILE, sep=";")
def Ajust_(FILE,pop,extrapolação,day_0,variavel,pasta, path_out): data_covid = pd.read_csv(pasta+"/"+FILE, header = 0, sep = ";") data_covid = data_covid[['DateRep',variavel,"cum-Deaths"]] nome_data = 'DateRep' df = data_covid day_0_str=df[nome_data][0][:4]+'-'+df[nome_data][0][5:7]+'-'+df[nome_data][0][-2:] #day_0_str = data_covid['DateRep'][0][-4:]+'-'+data_covid['DateRep'][0][3:5]+'-'+data_covid['DateRep'][0][:2] date = np.array(day_0_str, dtype=np.datetime64)+ np.arange(len(data_covid)) if date[0]>=np.array(day_0, dtype=np.datetime64): t0 = 0 else: dif_dias =np.array(day_0, dtype=np.datetime64)-date[0] t0 = dif_dias.astype(int) date= date[t0:] cumdata_covid = data_covid[['Cases']].cumsum() cum_deaths = data_covid[['cum-Deaths']] cumdata_cases = cumdata_covid['Cases'].values[t0:] cum_deaths = cum_deaths['cum-Deaths'].values[t0:] t = np.linspace(1,len(cumdata_cases),len(cumdata_cases)) N = pop*10**6 #n1,r1,beta,gama1,N,So,Uo,Qo,Co,Ro n1_0,r1_0,n2_0,r2_0,beta_0,gama1_0,gama2_0,eta_0=[1/100,10,1/50,-4,.15,.05,.15,5/100] So,Uo,Qo,Co,R1o,R2o,nCo = [.99*N,6*cumdata_cases[0],cumdata_cases[0],cumdata_cases[0],-4,14,3*cumdata_cases[0]] p0 = [n1_0,r1_0,n2_0,r2_0,beta_0,gama1_0,gama2_0,eta_0,N,So,Uo,Qo,Co,R1o,R2o,3,nCo] bsup = [n1_0*1.01,r1_0*1.1,n2_0*1.1, 0,0.50,.300,0.50, 20/100,N + 1, N,Uo*2.,Qo*2.0,Co+10**-9, 0,18,6,nCo*2.0] binf = [n1_0*0.99,r1_0*0.9,n2_0*0.9,-5,0.01,.001,0.05,.01/100,N - 1,.9*N,Uo*.5,Qo*0.5,Co-10**-9,-6, 8,4,nCo*0.5] # popt,pcov = ajust_curvefit(t,cum_deaths,p0,bsup,binf) # perr = np.sqrt(np.diag(pcov)) # Nstd = nstf(.99) # bsup = popt + Nstd * perr # binf = popt - Nstd * perr popt = min_minimize(cum_deaths,cumdata_cases,sucq_solve,p0,t,bsup,binf) n1_0,r1_0,n2_0,r2_0,beta_0,gama1_0,gama2_0,eta_0,N,So,Uo,Qo,Co,R1o,R2o,Ro,nCo = popt solution = SUCQ(t,*popt) NSE = hy.nse(solution[:,3],cumdata_cases) RMSE = hy.rmse(solution[:,3],cumdata_cases) MARE = hy.mare(solution[:,3],cumdata_cases) NSE_deaths = hy.nse(solution[:,7],cum_deaths) RMSE_deaths = hy.rmse(solution[:,7],cum_deaths) MARE_deaths = hy.mare(solution[:,7],cum_deaths) print(FILE[9:-4]) print("n1 = %f " % (n1_0)) print("n2 = %f " % (n2_0)) print("r1m = %f" %(r1_0)) print("r2m = %f" %(r2_0)) print("R1o = %f" %(R1o)) print("R2o = %f" %(R2o)) print("beta = %f " % (beta_0)) print("gamma1 = %f " % (gama1_0)) print("gamma2 = %f " % (gama2_0)) print("eta = %f " % (eta_0)) print("NSE = %.5f " % (NSE)) print("NSE Deaths = %.5f " % (NSE_deaths)) print("#######################") date_future = np.array(date[0], dtype=np.datetime64)+ np.arange(len(date)+extrapolação) days_future = np.linspace(1,len(cumdata_cases)+extrapolação,len(cumdata_cases)+extrapolação) Cum_cases_estimated = SUCQ(days_future, *popt) estimativafutura_saída=pd.DataFrame(Cum_cases_estimated[:,3], columns = ["Cases"]) estimativafutura_saída["S"]=Cum_cases_estimated[:,0] estimativafutura_saída["U"]=Cum_cases_estimated[:,1] estimativafutura_saída["Q"]=Cum_cases_estimated[:,2] estimativafutura_saída["I"] = Cum_cases_estimated[:,1]+Cum_cases_estimated[:,2]+Cum_cases_estimated[:,3]+Cum_cases_estimated[:,8] estimativafutura_saída["uC"] =Cum_cases_estimated[:,8] estimativafutura_saída["Rt"]=Cum_cases_estimated[:,6] # R = [] # for i in days_future: # R.append(R_s(i,n1_0,r1_0,n2_0,r2_0,R1o,R2o)) # #estimativafutura_saída["Rt"]= np.array(R) estimativafutura_saída["M"]=Cum_cases_estimated[:,7] estimativafutura_saída["date"] = date_future estimativafutura_saída.to_csv(path_out+'/'+FILE,sep=";") return [n1_0,r1_0,n2_0,r2_0,beta_0,gama1_0,gama2_0,eta_0,N,So,Uo,Qo,Co,R1o,R2o,nCo, NSE, RMSE, MARE,NSE_deaths, RMSE_deaths, MARE_deaths]
def Ajust_SUCQ(FILE, pop, extrapolação): path = 'C:/Users/ravellys/Documents/GitHub/COVID-19-Brasil/COVID-19-Brasil/data/DADOS/' data_covid = pd.read_csv(path + FILE, header=0, sep=";") data_covid = data_covid[['DateRep', 'Cases']] nome_data = 'DateRep' df = data_covid[['DateRep', 'Cases']] day_0_str = df[nome_data][0][:4] + '-' + df[nome_data][0][5:7] + '-' + df[ nome_data][0][-2:] day_0 = np.array(day_0_str, dtype=np.datetime64) date = np.array(day_0_str, dtype=np.datetime64) + np.arange( len(data_covid)) if date[0] >= np.array('2020-02-28', dtype=np.datetime64): t0 = 0 else: dif_dias = np.array('2020-02-28', dtype=np.datetime64) - date[0] t0 = dif_dias.astype(int) day_isol = '2020-03-20' dif_dias = date[len(date) - 1] - np.array(day_isol, dtype=np.datetime64) tf = dif_dias.astype(int) if (day_0 <= np.array(day_isol, dtype=np.datetime64)): date = date[t0:-tf] cumdata_covid = data_covid[['Cases']].cumsum() cumdata_cases = cumdata_covid['Cases'].values[t0:-tf] days_mens = np.linspace(1, len(cumdata_cases), len(cumdata_cases)) N = pop * 10**6 t = days_mens alfa_0, beta_0, gama1_0 = [.8 / N, .1, .19] So, Uo, Qo, Co = [ .8 * N, 6 * cumdata_cases[0], cumdata_cases[0], cumdata_cases[0] ] p0 = [alfa_0, beta_0, gama1_0, So, Uo, Qo, Co] bsup = [1 / N, 1, 1, N, Uo * 1.5, Qo * 1.2, Co + 10**-9] binf = [0, 0, 0, .5 * N, Uo * .5, Qo * 0.8, Co - 10**-9] popt = ajust_curvefit(days_mens, cumdata_cases, p0, bsup, binf) p0 = popt popt = min_minimize(cumdata_cases, sucq_solve, p0, t, bsup, binf) alfa_0, beta_0, gama1_0, So, Uo, Qo, Co = popt solution = SUCQ(t, alfa_0, beta_0, gama1_0, So, Uo, Qo, Co) NSE = hy.nse(cumdata_cases, solution[:, 3]) # print(FILE[9:-4]) # print("alfa = %f " % (alfa_0*N)) # print("beta = %f " % (beta_0)) # print("gamma = %f " % (gama1_0)) # print("R = %f" %(alfa_0*N/gama1_0)) # print("NSE = %.5f " % (NSE)) # print("#######################") date_future = np.array( date[0], dtype=np.datetime64) + np.arange(len(date) + extrapolação) days_future = np.linspace(1, len(cumdata_cases) + extrapolação, len(cumdata_cases) + extrapolação) Cum_cases_estimated = SUCQ(days_future, *popt) estimativafutura_saída = pd.DataFrame(Cum_cases_estimated[:, 3], columns=["Cases"]) estimativafutura_saída["S"] = Cum_cases_estimated[:, 0] estimativafutura_saída["U"] = Cum_cases_estimated[:, 1] estimativafutura_saída["Q"] = Cum_cases_estimated[:, 2] estimativafutura_saída[ "I"] = Cum_cases_estimated[:, 1] + Cum_cases_estimated[:, 2] + Cum_cases_estimated[:, 3] estimativafutura_saída["date"] = date_future fileout = 'C:/Users/ravellys/Documents/GitHub/COVID-19-Brasil/COVID-19-Brasil/data/DADOS_estimados_sem_isolamento/' estimativafutura_saída.to_csv(fileout + FILE, sep=";") if NSE >= 0.9: return [alfa_0 * N / gama1_0, FILE[9:-4]]
c='green', markersize=2, markerfacecolor="None", label='train') plt.plot(x_test[:, len(x_test[1, :]) - 1], y_test[:, i], 'o', c='red', markersize=2, markerfacecolor="None", label='test') plt.plot(dias, TH[:, i], 'gray', label=r'$Simulado \theta\ %d$' % (i + 1)) NSEtest = he.nse( TH[int(x_test[:, len(x_test[1, :]) - 1][0] - init - 1):, i], y_test[:, i]) plt.text(260, .21, r"$NSE_{test}: %.3f $" % NSEtest, fontsize=12) NSEtrain = he.nse( TH[1:int(x_test[:, len(x_test[1, :]) - 1][0] - init - 1), i], y_train[:, i]) plt.text(260, .23, r"$NSE_{train}: %.3f $" % NSEtrain, fontsize=12) plt.xlabel('$dias$') plt.ylabel(r'$ \theta\ (cm³.cm^{-3})$') plt.ylim(.01, .25) plt.legend(loc='upper left') plt.show() """ Actual Evapotranspiration (ETa)