def writer(self): weather_data = CABOWeatherDataProvider(self.FName, fpath=self.Path) keys = weather_data.export()[0].keys() with open(f'{self.FName}.csv', 'w') as f: file = csv.DictWriter(f, keys) file.writeheader() file.writerows(weather_data.export())
def load_data(): weather_data = CABOWeatherDataProvider('NL2', fpath="./Data/METEO/CABOWE") #file = cabo_weather.CABOWeatherDataProvider("./Data/METEO/CABOWE") keys = weather_data.export()[0].keys() print(keys) with open('amin.csv', 'w') as f: file = csv.DictWriter(f, keys) file.writeheader() file.writerows(weather_data.export())
def get_weather_rain(name, path='./Input/Data/CABOWE'): # load meteo file with pcse name = str(name).split('.')[0] weather_data = CABOWeatherDataProvider(fname=name, fpath=path) # export data = weather_data.export() # extar rain = [item["RAIN"] for item in data] return rain
def set_up_wofost(crop_start_date, crop_end_date, meteo, crop, variety, soil, wav=100, co2=400, rdmsol=100.): cropdata = YAMLCropDataProvider(fpath="./WOFOST_crop_parameters") cropdata.set_active_crop(crop, variety) soildata = CABOFileReader(soil) soildata['RDMSOL'] = rdmsol sitedata = WOFOST71SiteDataProvider(WAV=wav, CO2=co2) parameters = ParameterProvider(cropdata=cropdata, soildata=soildata, sitedata=sitedata) with open("temporal.amgt", 'w') as fp: fp.write( agromanagement_contents.format(year=crop_start_date.year, crop=crop, variety=variety, crop_start_date=crop_start_date, crop_end_date=crop_end_date)) agromanagement = YAMLAgroManagementReader("temporal.amgt") wdp = CABOWeatherDataProvider(meteo, fpath=f"./data/meteo/{meteo}/") return parameters, agromanagement, wdp
def set_wofost_up(crop="maize", variety="Maize_VanHeemst_1988", meteo="Upper_East", soil="ec4.new", wav=60, co2=400, rdmsol=100): cropdata = YAMLCropDataProvider(fpath="./WOFOST_crop_parameters") cropdata.set_active_crop(crop, variety) soildata = CABOFileReader(soil) soildata["RDMSOL"] = rdmsol sitedata = WOFOST71SiteDataProvider(WAV=wav, CO2=co2) parameters = ParameterProvider(cropdata=cropdata, soildata=soildata, sitedata=sitedata) agromanagement = YAMLAgroManagementReader("ghana_maize.amgt") wdp = CABOWeatherDataProvider(meteo, fpath=f"./data/meteo/{meteo}/") return parameters, agromanagement, wdp
def run(crop: str, soil: str, agro: str, day: int, weather_filename: str, saved_name="output"): # load argo from directory agromanagement = YAMLAgroManagementReader(f"{base_dir}/{agro}") sitedata = WOFOST71SiteDataProvider(WAV=100, CO2=360) # load soil from directory soildata = CABOFileReader(f"{base_dir}/{soil}") # load crop from directory cropdata = CABOFileReader(f"{base_dir}/{crop}") # load weather data from directory wdp = CABOWeatherDataProvider(fname=weather_filename, fpath=base_dir) # packaing parameters parameters = ParameterProvider(cropdata=cropdata, soildata=soildata, sitedata=sitedata) # create model wofost = Wofost71_WLP_FD(parameters, wdp, agromanagement) # run till [day] wofost.run(day) # save output az a csv in OUT directory model_out_put = wofost.get_output() df = pd.DataFrame(model_out_put) df.to_csv(f"{out_dir}/{saved_name}.csv")
def wofost_parameter_sweep_func(crop_start_date=dt.date(2011, 7, 1), crop_end_date=dt.datetime(2011, 11, 1), span=40.0, tdwi=20., tsum1=750., tsum2=859., tsumem=70, rgrlai=0.05, cvo=0.05, cvl=0.05, meteo="Upper_East", crop="maize", variety="Maize_VanHeemst_1988", soil="ec4.new", wav=100, co2=400, rdmsol=100., potential=False): cropdata = YAMLCropDataProvider(fpath="./WOFOST_crop_parameters") cropdata.set_active_crop(crop, variety) soildata = CABOFileReader(soil) soildata["RDMSOL"] = rdmsol sitedata = WOFOST71SiteDataProvider(WAV=wav, CO2=co2) parameters = ParameterProvider(cropdata=cropdata, soildata=soildata, sitedata=sitedata) for p, v in zip( ["SPAN", "TSUM1", "TSUM2", "TSUMEM", "TDWI", "RGRLAI", "CVO", "CVL"], [span, tsum1, tsum2, tsumem, tdwi, rgrlai, cvo, cvl]): parameters.set_override(p, v, check=True) with open("temporal.amgt", 'w') as fp: fp.write( agromanagement_contents.format(year=crop_start_date.year, crop=crop, variety=variety, crop_start_date=crop_start_date, crop_end_date=crop_end_date)) agromanagement = YAMLAgroManagementReader("temporal.amgt") wdp = CABOWeatherDataProvider(meteo, fpath=f"./data/meteo/{meteo}/") df_results, simulator = run_wofost(parameters, agromanagement, wdp, potential=potential) fig, axs = plt.subplots(nrows=5, ncols=2, sharex=True, squeeze=True, figsize=(16, 16)) axs = axs.flatten() for j, p in enumerate(WOFOST_PARAMETERS): axs[j].plot_date(df_results.index, df_results[p], '-') axs[j].set_ylabel(WOFOST_LABELS[p], fontsize=8) plt.gcf().autofmt_xdate() plt.gca().fmt_xdata = matplotlib.dates.DateFormatter('%Y-%m-%d') axs[8].set_xlabel("Time [d]") axs[9].set_xlabel("Time [d]") key = f"span_{span}-tdwi_{tdwi}-tsum1_{tsum1}-tsum2_{tsum2}-tsumem_{tsumem}" key += f"-rgrlai_{rgrlai}-wav_{wav}-cvo_{cvo}-cvl_{cvl}" if potential: key += "-POT.csv" else: key += "-LIM.csv" df_results.to_csv(key, encoding="utf-8", index=False)
def ensemble_wofost(lon=115.55, lat=38.05, start=dt.date(2008, 10, 12), end=None, en_size=3, prior_file=None, weather_type="NASA", weather_path=None, out_en_file=None, data_dir=None): """ This is a function to generate a emsemble of WOFOST paramters and corresponding output. you need to specify Longitude (lon), Latitude (lat), start time of crop (start), end time of crop (end, 270 days duration by default), emsemble size (en_size), configuration file for prior distributions of pramaters (prior_file), weather driver dataset type (weather_type), it's set to NASA Power dataset "NASA" by default, you could use ERA5 "ERA5" or ECMWF TIGGE "ITGEE" instead or use your own CABO file (%your_cabo_files_name%).) """ if data_dir is None: #home = os.path.dirname(os.path.realpath("__file__")) home = os.path.split(os.path.realpath(__file__))[ 0] #os.path.dirname(os.path.abspath(__file__)) data_dir = home + "/data/" #print(data_dir) if prior_file is None: prior_file = data_dir + "par_prior.csv" if out_en_file is None: out_en_file = "WOFOST_par_ensemble.npy" if lat < -90 or lat > 90: msg = "Latitude should be between -90 and 90 degrees." raise ValueError(msg) if lon < -180 or lon > 180: msg = "Longitude should be between -180 and 180 degrees." raise ValueError(msg) if end == None: end = start + dt.timedelta(days=270) if start >= end: msg = "Start time should be earlier than end time." raise ValueError(msg) if weather_type == "NASA": print("Downloading weather driver from NASA Power...") weather = NASAPowerWeatherDataProvider(latitude=lat, longitude=lon) elif weather_type[:3] == "ERA" or weather_type[:3] == "era": print("ERA5 reanalysis dataset used.") if weather_path is None or not os.path.isdir(weather_path): msg = "Please provide a valid path for weahter driver data." raise ValueError(msg) gen_era_cabo(lat, lon, start.year, end.year, inputfile=weather_path, data_dir=data_dir) size = 0.25 weather_name = "ERA_%5.2f_%5.2f" % (int( (lon + size / 2.) / size) * size, int( (lat + size / 2.) / size) * size) weather = CABOWeatherDataProvider(weather_name, fpath=weather_path) elif weather_type[:5].upper() == "TIGGE": print("TIGGE forecast from ECMWF used.") if weather_path is None or not os.path.isdir(weather_path): msg = "Please provide a valid path for weahter driver data." raise ValueError(msg) gen_tigge_cabo(lat, lon, start.year, end.year, inputfile=weather_path, data_dir=data_dir) size = 0.25 weather_name = "TIGGE_%5.2f_%5.2f" % (int( (lon + size / 2.) / size) * size, int( (lat + size / 2.) / size) * size) weather = CABOWeatherDataProvider(weather_name, fpath=weather_path) else: if weather_path == None: raise ValueError("Please provide your weather driver path!") weather = CABOWeatherDataProvider(weather_type, fpath=weather_path) sdoy = retrieve_pixel_value([lon, lat], data_dir + "mean_wheat_sdoy_china_kriging_int.tif") tsum1 = retrieve_pixel_value([lon, lat], data_dir + "mean_wheat_TSUM1_china_kriging.tif") tsum2 = retrieve_pixel_value([lon, lat], data_dir + "TSUM2_aver_0.1deg.tif") varnames = ["day", "TAGP", "LAI", "TWSO", "DVS"] tmp = {} cropfile = os.path.join(data_dir, 'WWH108.CAB') crop = CABOFileReader(cropfile) soilfile = os.path.join(data_dir, 'Hengshui.soil') soil = CABOFileReader(soilfile) site = WOFOST71SiteDataProvider(WAV=100, CO2=360) parameters = ParameterProvider(soildata=soil, cropdata=crop, sitedata=site) agromanagement_file = os.path.join(data_dir, 'shenzhou_wheat.amgt') agromanagement = YAMLAgroManagementReader(agromanagement_file) (key, value), = agromanagement[0].items() agromanagement[0][dt.datetime.strptime( "%d%03d" % (start.year, sdoy), "%Y%j").date()] = agromanagement[0].pop(key) value['CropCalendar']['crop_start_date'] = dt.datetime.strptime( "%d%03d" % (start.year, sdoy), "%Y%j").date() print("Crop is sowed at %s" % dt.datetime.strftime( value['CropCalendar']['crop_start_date'], "%Y-%m-%d")) value['CropCalendar']['crop_end_date'] = end prior_dist, prior_list, param_xvalue, param_type = define_prior_distributions( chunk=prior_file, tsum1=tsum1, tsum2=tsum2) z_start = np.empty((len(prior_list), en_size)) for i, param in enumerate(prior_list): z_start[i, :] = prior_dist[param].rvs(en_size) outdata = [] for i in range(en_size): theta_dict = dict(zip(prior_list, z_start[:, i])) cropdata = copy.deepcopy(crop) tb_x = {} tb_y = {} tb_t = {} tmp_dict = {} for par in theta_dict.keys(): try: if param_type[par] != 'S': tb_index = par.find("TB") if tb_index < 0: print(param_xvparam_typealue[par]) raise Exception("Are you sure %s is a table value?" % par) tb_name = par[:tb_index + 2] tmp_list = [param_xvalue[par], theta_dict[par]] if not tb_name in tb_x: tb_x[tb_name] = np.array([param_xvalue[par]]) tb_y[tb_name] = np.array([theta_dict[par]]) tb_t[tb_name] = param_type[par] else: tb_x[tb_name] = np.append(tb_x[tb_name], param_xvalue[par]) tb_y[tb_name] = np.append(tb_y[tb_name], theta_dict[par]) except KeyError: raise Exception( "There's something wrong with %s, please check it." % par) tmp_dict = {} for par in tb_x.keys(): # Table parameters s_i = np.argsort(tb_x[par]) s_x = tb_x[par][s_i] s_v = tb_y[par][s_i] par_tb = [] # print(par,tb_t[par],cropdata[par],s_x,s_v) if tb_t[par][1] == 'P': for i in range(len(tb_x[par])): if tb_t[par][0] == 'Y': # Partly change table Y values if s_x[i] in cropdata[par][::2]: # change old value c_i = cropdata[par][::2].index(s_x[i]) cropdata[par][c_i * 2] = s_v[i] else: # insert new value array_X = cropdata[par][::2] array_Y = cropdata[par][1:][::2] ins_i = bisect(array_X, s_x[i]) cropdata[par].insert(ins_i * 2, s_x[i]) cropdata[par].insert(ins_i * 2 + 1, s_v[i]) #print(cropdata[par]) else: # Partly change table X values if s_x[i] in cropdata[par][ 1:][::2]: # change old value c_i = cropdata[par][1:][::2].index(s_x[i]) cropdata[par][c_i * 2] = s_v[i] else: # insert new value array_X = cropdata[par][::2] array_Y = cropdata[par][1:][::2] ins_i = bisect(array_X, s_x[i]) cropdata[par].insert(ins_i * 2, s_x[i]) cropdata[par].insert(ins_i * 2 + 1, s_v[i]) #print(cropdata[par]) elif tb_t[par][1] == 'A': if tb_t[par][0] == 'Y': # Totally change table Y values for i in range(len(tb_x[par])): par_tb.append(s_x[i]) par_tb.append(s_v[i]) else: # Totally change table X values for i in range(len(tb_x[par])): par_tb.append(s_v[i]) par_tb.append(s_x[i]) tmp_dict[par] = par_tb #print(tmp_dict[par]) theta_dict.update(tmp_dict) else: raise Exception( "There's something wrong with %s, please check it." % par) ########################################################################## cropdata.update(theta_dict) parameters = ParameterProvider(cropdata=cropdata, soildata=soil, sitedata=site) wofwof = Wofost71_PP(parameters, weather, agromanagement) wofwof.run_till_terminate() output = wofwof.get_output() summary_output = wofwof.get_summary_output() msg = "Reached maturity at {DOM} with max LAI of {LAIMAX} "\ "and a yield of {TWSO} kg/ha." print(msg.format(**summary_output[0])) for var in varnames: tmp[var] = [t[var] for t in output] theta_dict["LAI"] = tmp["LAI"][-181:] theta_dict["day"] = tmp["day"][-181:] theta_dict["Yield"] = tmp["TWSO"][-1] outdata.append(theta_dict) np.save(out_en_file, outdata)
def Generate_With_Dists_From_Scratch(self, distribution_file, crop_file, soil_file, weather_point, timer_file, central_value='absolute'): """ Generate ensembles using strings pointing to the wofost files from a parameter distribution file. - distribution_file - string of the location where the parameter distribution is. - crop_file - the parameter file location string. - soil_file - the soil parameter file location string. - weather_point - the unix wildcard search which identifies the weather data. - timer_file - the timer file location string. """ if central_value not in ['absolute', 'relative']: raise ValueError('central_value must be absolute or relative.\ \nAbsolute using the exact distributions from the distribution file.\ \nRelative creates distribuitions around the input crop file.' ) manager = multiprocessing.Manager() self.repo = manager.list() self.param_files = [] self.generated_agromanagers = [] self.distribution_file = distribution_file self.central_value = central_value try: self.params = pa.read_csv(distribution_file) except: raise NameError('Cant open the distribution file %s' % distribution_file) self.new_param_vals = {} for n, i in enumerate(self.params['Param'].iloc[:]): if np.isnan(self.params['Function_Value'].iloc[n]) == True: self.new_param_vals[i] = [] else: if i not in list(self.new_param_vals.keys()): self.new_param_vals[i] = {} self.new_param_vals[i][ self.params['Function_Value'].iloc[n]] = [] else: self.new_param_vals[i][ self.params['Function_Value'].iloc[n]] = [] # Read in the parameter files: crop = CABOFileReader(crop_file) soil = CABOFileReader(soil_file) # # the site parameters cover extra stuff not covered by the parameter files # # wav is the initial soil moisture content. site = WOFOST71SiteDataProvider(WAV=100, CO2=360) # # Read in the weather file weather = CABOWeatherDataProvider(weather_point) # get the agromanager agromanagement_object = YAMLAgroManagementReader(timer_file) # define a function that is multiprocessable def multiproc_wofost(input_wofost_object): input_wofost_object.run_till_terminate() self.repo.append(input_wofost_object.get_output()) # setup somewhere to put the processes active_processes = [] run_on = multiprocessing.cpu_count() - 1 process_counter = 0 while process_counter < self.en_number: if len(active_processes) < run_on: # get a clean version of the parameters new = copy.deepcopy(crop) # loop through the parameters in the file for j in range(len(self.params)): name, mu, min_val, max_val, sigma, func = self.params.iloc[ j] if name == 'PDATE': continue if self.central_value is 'relative': if type(crop_object[name]) in [int, float]: mu = crop_object[name] # min and max are 3 sigma away from the mean min_val = mu - (self.rel_rng * sigma) max_val = mu + (self.rel_rng * sigma) else: if func in new[name]: loc = np.where( np.array(new[name]) == func)[0][0] mu = new[name][loc + 1] min_val = mu - (self.rel_rng * sigma) max_val = mu + (self.rel_rng * sigma) else: # WARNING: # if we have gone down this route, it means there # is no current function value for this parameter. # this could lead to potentially weird results. # blind_obedience means to put it in anyway. blind_obedience = True if blind_obedience == True: pass else: continue # get the distributions dist = scipy.stats.truncnorm((min_val - mu) / sigma, (max_val - mu) / sigma, loc=mu, scale=sigma) # get a new value new_val = dist.rvs(1)[0] # first, reasign the simple single parameters if np.isnan(func) == True: new[name] = new_val self.new_param_vals[name].append(new_val) else: # first check if there already is a function value in place already prs_keys = np.array(new[name])[::2] prs_vals = np.array(new[name])[1::2] # quickly add the val to the new _param_values self.new_param_vals[name][func].append(new_val) # reasign the values if the function value is there if func in prs_keys: prs_vals[np.where( prs_keys == func)[0][0]] = new_val new[name] = np.hstack(zip(prs_keys, prs_vals)) # or put a new one in if it is not there already else: new_keys = np.concatenate( [prs_keys, np.array([func])]) new_vals = np.concatenate( [prs_vals, np.array([new_val])]) sort_index = np.argsort(new_keys) new_keys = new_keys[sort_index] new_vals = new_vals[sort_index] new[name] = np.hstack(zip(new_keys, new_vals)) # reassign the planting date based off the normal distribution: # grab the row in the param file that is the planting date if 'PDATE' in self.params['Param'].values: pdate_row = np.where( self.params['Param'].values == 'PDATE')[0][0] # get the aspects to make the normal distribution pdate_min = self.params['Min'].values[pdate_row] pdate_max = self.params['Max'].values[pdate_row] pdate_mu = self.params['Mean'].values[pdate_row] pdate_sigma = self.params['StdDev'].values[pdate_row] # generate the distributions pdate_dist = scipy.stats.truncnorm( (pdate_min - pdate_mu) / pdate_sigma, (pdate_max - pdate_mu) / pdate_sigma, loc=pdate_mu, scale=pdate_sigma) # pull out the key for the agromanager campaign_start = list(agromanagement_object[0].keys())[0] # create a new planting date new_pdate = agromanagement_object[0][campaign_start]['CropCalendar']['crop_start_date'] + \ dt.timedelta(days=pdate_dist.rvs(1)[0]) # make all ensembles have the same campaign length so everything fits new_campdate = campaign_start - dt.timedelta( days=abs(pdate_min) - 1) # create the new agromanager with the new planting date new_agromanager = copy.deepcopy( agromanagement_object)[0][campaign_start] new_agromanager['CropCalendar'][ 'crop_start_date'] = new_pdate new_agro_obj = [{new_campdate: new_agromanager}] # add it to a repo so we have a record of it self.generated_agromanagers.append(new_agro_obj) else: new_agro_obj = agromanagement_object self.param_files.append(new) new_parameter_object = ParameterProvider(new, soil, site) # instantiate the new version of wofost iter_wof = self.runner(new_parameter_object, weather, new_agro_obj) # and process it using multiprocessing p = multiprocessing.Process(target=multiproc_wofost, args=(iter_wof, )) p.daemon = True p.name = str(process_counter) p.start() active_processes.append(p) process_counter += 1 else: for pr in active_processes: if pr.is_alive() == False: active_processes.remove(pr)
def get_weather_et0(self): weather_data = CABOWeatherDataProvider(self.FName, fpath=self.Path) data = weather_data.export() et0s = [item["ET0"] for item in data] return et0s
def Process(self): # 1 - first we need to intialize the wofost components crop = CABOFileReader(self.crop_file) soil = CABOFileReader(self.soil_file) # # the site parameters cover extra stuff not covered by the parameter files # # wav is the initial soil moisture content. site = WOFOST71SiteDataProvider(WAV=100, CO2=360) # # and ciompile them into a single object. parameters = ParameterProvider(crop, soil, site) # # Read in the weather file weather = CABOWeatherDataProvider(self.weather_point) vanilla_timer = """Version: 1.0 AgroManagement: - 2006-01-01: CropCalendar: crop_name: 'winter-wheat' variety_name: 'Shenzhou_wheat' crop_start_date: 2006-10-12 crop_start_type: sowing crop_end_date: 2007-06-30 crop_end_type: harvest max_duration: 300 TimedEvents: null StateEvents: null""" agromanagement = yaml.load(vanilla_timer)['AgroManagement'] # 2 - now we need to enter all the years as campaigns into the agromanager available_days = list(weather.store.keys()) available_days = [i[0] for i in available_days] available_years = np.unique([i.year for i in available_days]) # define the month and day to start and finish for each year start_month, start_day = self.planting_date.month, self.planting_date.day end_month, end_day = self.upto_date.month, self.upto_date.day # get the key into the original agromanager entry key = list(agromanagement[0].keys())[0] # loop through the years in the data for i in available_years: # define the years starting date crop_start = dt.date(i, start_month, start_day) # define the years end date if start_month > end_month: # add a year to the datetime if the upto crosses into the next year crop_end = dt.date(i + 1, end_month, end_day) else: crop_end = dt.date(i, end_month, end_day) # copy the bulk of the campain so it can be edited new_campain = copy.deepcopy(agromanagement[0][key]) # change the relivant bits new_campain['CropCalendar']['crop_start_date'] = crop_start new_campain['CropCalendar']['crop_end_date'] = crop_end # format it to_insert = {crop_start: new_campain} # delete the originals if i == available_years[-1]: del agromanagement[0] # dont add a campaign that is over the unto_datetime if crop_start > self.upto_date: continue # finally add that to the agromanager agromanagement.append(to_insert) # 3 - now we run WOFOST # instantiate a wofost instance wofost = Wofost71_PP(parameters, weather, agromanagement) # # run wofost.run_till_terminate() # # get the output self.output = wofost.get_output() # and save the agromanager self.agromanager = agromanagement # define all the starting and ending times for the campains is quick lists self.starts = [list(i.keys())[0] for i in self.agromanager] self.ends = [ i[self.starts[n]]['CropCalendar']['crop_end_date'] for n, i in enumerate(self.agromanager) ]
# Set up input paramter files describing soil and crop. # You will always use the same files here soil = CABOFileReader('Hengshui.soil') site = WOFOST71SiteDataProvider(WAV=100, CO2=360) crop = YAMLCropDataProvider('.') # directory containing crop file crop.set_active_crop('maize', 'Grain_maize_204') parameters = ParameterProvider(crop, soil, site) # Set up the parameters that describe sowing, harvest and crop management. agromanagement = YAMLAgroManagementReader('timer_china_maize.amgt') # Update agromanagement to the year we are interested in. This needs to # match the year of the weather data you are using new_agromanagement = change_year(agromanagement, year_of_interest) # Set up the weather file with the weather data we wrote above weather = CABOWeatherDataProvider(cabo_weather_file) # Initialise the model wofost = Wofost71_PP(parameters, weather, new_agromanagement) # Run the crop model wofost.run_till_terminate() # Get the out put output = wofost.get_output() # Final yield, This is what we are interested in. # We get the output on the last day (output[-1]) and # we are interested in the Total Weight of Storage Organs (TWSO), # which is strongly related to yield. TWSO_final = output[-1]['TWSO'] print(f"The final yield is {TWSO_final} kg ha-1")