def run_producer(server={"server": None, "port": None}, shared_id=None): context = zmq.Context() socket = context.socket(zmq.PUSH) config = { "port": server["port"] if server["port"] else "6666", "server": server["server"] if server["server"] else "localhost", "sim.json": os.path.join(os.path.dirname(__file__), '../sim-min.json'), "crop.json": os.path.join(os.path.dirname(__file__), '../crop-min.json'), "site.json": os.path.join(os.path.dirname(__file__), '../site-min.json'), "climate.csv": os.path.join(os.path.dirname(__file__), '../climate-min.csv'), "shared_id": shared_id } # read commandline args only if script is invoked directly from commandline if len(sys.argv) > 1 and __name__ == "__main__": for arg in sys.argv[1:]: k, v = arg.split("=") if k in config: config[k] = v print "config:", config socket.connect("tcp://" + config["server"] + ":" + config["port"]) with open(config["sim.json"]) as _: sim_json = json.load(_) with open(config["site.json"]) as _: site_json = json.load(_) with open(config["crop.json"]) as _: crop_json = json.load(_) with open(config["climate.csv"]) as _: climate_csv = _.read() env = monica_io.create_env_json_from_json_config({ "crop": crop_json, "site": site_json, "sim": sim_json, "climate": climate_csv }) #print env # add shared ID if env to be sent to routable monicas if config["shared_id"]: env["sharedId"] = config["shared_id"] socket.send_json(env) print "done"
def run_producer(): # calibration options calibration_mode = False # simulation options activate_debug = False output_dir = "runs/2019-02-15/" # calibration ------------------------------------- # technical initialisation config = { "user": "******", "port": "66666", "server": "localhost" } paths = PATHS[config["user"]] sent_id = 0 start_send = time.clock() socket = initialise_sockets(config) print(socket) # output file output_filename = "agmip_calibration_phase2_step2_evaluation.csv" if calibration_mode: output_filename = "agmip_calibration_phase2_step2_calibration.csv" # read in management information management_file = paths["INCLUDE_FILE_BASE_PATH"] + "info/CSIRO_data_set/simulation_plan.xlsx" management_df = pd.read_excel(management_file, sheader=0, index_col=None, keep_default_na=False, encoding="latin1") print "Shape", management_df.shape # iterate over every row/environment where row contains the setup of one simulation for sim_row, environment in management_df.iterrows(): print "Run simulation " + str(sim_row) id = int(environment["ID"]) calibration_mode = bool(environment["Calibration"]) year = int(environment["Year"]) site_name = environment["Site"] sowing_day = environment["sowDay"] sim_parameters = None site_parameters = None crop_parameters = None simulation_dir = "monica_simulation_setup/evaluation" if calibration_mode: simulation_dir = "monica_simulation_setup/calibration" sim_file = "sim-%d.json" % id with open(simulation_dir + "/" + sim_file) as fp: sim_parameters = json.load(fp) site_file = sim_parameters["site.json"] with open(site_file) as fp: site_parameters = json.load(fp) #site_parameters["include-file-base-path"] = paths["INCLUDE_FILE_BASE_PATH"] sim_parameters["include-file-base-path"] = paths["INCLUDE_FILE_BASE_PATH"] # spring wheat parameters for stage temperature sum stage1_sum = 142.74 stage2_sum = 218.43 stage3_sum = 302.21 stage4_sum = 67.51 stage5_sum = 273.15 tsum_zadok30 = stage2_sum + (stage3_sum * 0.25) tsum_zadok65 = stage2_sum + stage3_sum + (stage4_sum * 0.25) output_map = { "events": [ "anthesis", [ "Date|Ant" ], "maturity", [ "Date|Mat", ], ["while", "Stage", "=", 2], [ ["Date|EmergeDate", "FIRST"] ], ["while", "Stage", "=", 3], [ ["Date|BeginStage3", "FIRST"] ], ["while", "Stage", "=", 4], [ ["Date|BeginStage4", "FIRST"] ], ["while", "Stage", "=", 5], [ ["Date|BeginStage5", "FIRST"] ], ["while", "Stage", "=", 6], [ ["Date|BeginStage6", "FIRST"] ], ["while", "TempSum", ">=", tsum_zadok30], [ ["Date|ZADOK30", "FIRST"] ], ["while", "TempSum", ">=", tsum_zadok65], [ ["Date|ZADOK65", "FIRST"] ], ["while", "Stage", "=", 6], [ ["Date|ZADOK90", "FIRST"] ] ] } sim_parameters["output"] = output_map crop_file = sim_parameters["crop.json"] with open(crop_file) as fp: crop_parameters = json.load(fp) env_map = { "crop": crop_parameters, "site": site_parameters, "sim": sim_parameters, "climate": "" } env = monica_io.create_env_json_from_json_config(env_map) env["csvViaHeaderOptions"] = sim_parameters["climate.csv-options"] env["csvViaHeaderOptions"]["start-date"] = sim_parameters["climate.csv-options"]["start-date"] env["csvViaHeaderOptions"]["end-date"] = sim_parameters["climate.csv-options"]["end-date"] env["pathToClimateCSV"] = sim_parameters["climate.csv"] # overwrite stage temperature sum values with tommaso's calibrated ones # stage 1 env["cropRotation"][0]["worksteps"][0]["crop"]["cropParams"]["cultivar"]\ ["StageTemperatureSum"][0][0] = stage1_sum env["cropRotation"][0]["worksteps"][0]["crop"]["cropParams"]["cultivar"]\ ["StageTemperatureSum"][0][1] = stage2_sum env["cropRotation"][0]["worksteps"][0]["crop"]["cropParams"]["cultivar"] \ ["StageTemperatureSum"][0][2] = stage3_sum env["cropRotation"][0]["worksteps"][0]["crop"]["cropParams"]["cultivar"] \ ["StageTemperatureSum"][0][3] = stage4_sum env["cropRotation"][0]["worksteps"][0]["crop"]["cropParams"]["cultivar"] \ ["StageTemperatureSum"][0][4] = stage5_sum # final env object with all necessary information env["customId"] = { "id": id, "calibration": calibration_mode, "output_dir": output_dir, "output_filename": output_filename, "sowing_day": sowing_day, "site_name": site_name, "year": year } with open("monica_simulation_setup/env.json", "w") as fp: json.dump(env, fp=fp, indent=4) # sending env object to MONICA ZMQ print("sent env ", sent_id, " customId: ", env["customId"]) socket.send_json(env) sent_id += 1 # break stop_send = time.clock() # just tell the sending time if objects have really been sent print("sending ", sent_id, " envs took ", (stop_send - start_send), " seconds")
def producer(setup=None): "main function" # reload monica_io module so that cached references will be cleared # this is a problem because the "crops" key in crop.json is changed on every call to this function # and thus for every setup, but calls to ["ref", "crops", "XXX"] will cache the result of # resolving references to XXX and thus the updates to XXX for the secondary yield further up won't happen reload(monica_io) paths = PATHS[USER] #Configure producer if setup == None: #playground run_id = "custom" PRODUCTION_LEVEL = 'WL.NL.rain' #options: "Pot", "WL.NL.rain" TF = "continuous" FERT_STRATEGY = "BASE" #options: "NDEM", "NMIN", "BASE" COVER_CROP_FREQ = { #always use int for insert-cc-every and out-of #keep out-of as small as possible (to ensure uniform temporal distribution) "insert-cc-every": 1, #CM "out-of": 4, #CM "suffix": "25" #TODO set it (for output file name) } EXPORT_RATE = "base" RESIDUES_HUMUS_BALANCE = False #mgt complying with humus balance approach of NRW HUMBAL_CORRECTION = { "heavy": 200, "medium": 300, "light": 400 } else: run_id = setup["id"] PRODUCTION_LEVEL = setup["PRODUCTION_LEVEL"] TF = setup["TF"] FERT_STRATEGY = setup["FERT_STRATEGY"] COVER_CROP_FREQ = setup["COVER_CROP_FREQ"] if setup["res_mgt"] == "RESIDUES_HUMUS_BALANCE": RESIDUES_HUMUS_BALANCE = True else: RESIDUES_HUMUS_BALANCE = False EXPORT_RATE = setup["res_mgt"] HUMBAL_CORRECTION = setup["HUMBAL_CORRECTION"] #end of user configuration #assemble file name suffix for out suffix = "_id" + run_id + "_" suffix += TF + "_" suffix += "fert-" + FERT_STRATEGY.lower() + "_" if RESIDUES_HUMUS_BALANCE: suffix += "res-humbal_" else: suffix += "res-" + EXPORT_RATE + "_" suffix += "cc-" + COVER_CROP_FREQ["suffix"] + "_" suffix += "pl-" + PRODUCTION_LEVEL.replace(".", "") + "_" if FERT_STRATEGY == "NMIN": rotations_file = "json_templates/rotations_dynamic_harv.json" elif FERT_STRATEGY == "NDEM": rotations_file = "json_templates/rotations_dynamic_harv_Ndem.json" elif FERT_STRATEGY == "BASE": rotations_file = "json_templates/rotations_dynamic_harv_Nbaseline.json" rotations_spinup_file = "json_templates/rotations_dynamic_harv_Nbaseline_SU.json" if LOCAL_RUN: PATH_TO_CLIMATE_DATA_DIR = timeframes[TF]["local-path-to-climate"] else: PATH_TO_CLIMATE_DATA_DIR = timeframes[TF]["cluster-path-to-climate"] context = zmq.Context() socket = context.socket(zmq.PUSH) port = 6666 if len(sys.argv) == 1 else sys.argv[1] if LOCAL_RUN: socket.connect(PATHS[USER]["local-monica-server"]) else: socket.connect("tcp://cluster1:" + str(port)) soil_db_con = sqlite3.connect("soil.sqlite") with open("json_templates/sim.json") as _: sim = json.load(_) sim["start-date"] = timeframes[TF]["start-date"] sim["end-date"] = timeframes[TF]["end-date"] with open("json_templates/site.json") as _: site = json.load(_) with open("json_templates/crop.json") as _: crop = json.load(_) with open("json_templates/cover-crop.json") as _: cover_crop = json.load(_)["CM"] if RESIDUES_HUMUS_BALANCE: #inject additional harvest params for ws in range(len(cover_crop["worksteps"])): if cover_crop["worksteps"][ws]["type"] == "AutomaticHarvest": cover_crop["worksteps"][ws]["opt-carbon-conservation"] = True cover_crop["worksteps"][ws]["crop-impact-on-humus-balance"] = [humus_equivalent["crop"]["CC"], "Humus equivalent [Heq]"] cover_crop["worksteps"][ws]["residue-heq"] = [humus_equivalent["material"]["green-manure"], "Heq ton-1 DM"] cover_crop["worksteps"][ws]["crop-usage"] = CROP_USAGE_HUMBAL["CC"] cover_crop["worksteps"][ws]["exported"] = True #Needed to fire the humus balance approach; if true and crop-usage="green-manure" --> the crop is anyway returned to the soil with open("json_templates/cover-crop_SU.json") as _: cover_crop_spinup = json.load(_)["CM_SU"] with open("json_templates/sims.json") as _: sims = json.load(_) if SOC_STUDY: sims = sims["soc_study"] else: sims = sims["potentials_study"] if FERT_STRATEGY != "NMIN": #turn off Nmin automatic fertilization for setting in sims.iteritems(): setting[1]["UseNMinMineralFertilisingMethod"] = False #load initialization SOC values SOC_ini_vals = defaultdict(lambda: defaultdict()) with open("settings_SOC_study/SOCini_unique_combinations.csv") as _: reader = csv.reader(_) reader.next() for row in reader: param_id = int(row[0]) sim_id = row[1] corgs = [] for i in range(3, 8): if len(row) > i and row[i] != "": corgs.append(float(row[i])) SOC_ini_vals[param_id][sim_id] = corgs param_vals = defaultdict(lambda: defaultdict(float)) best_p_id = -999 best_like = -999 #load parameters sets (used only for SOC study) with open("settings_SOC_study/" + PARAMETERS_FILE) as _: reader = csv.reader(_) reader.next() for row in reader: param_id = int(row[0]) param_vals[param_id]["PartSOM_Fast_to_SOM_Slow"] = float(row[1]) param_vals[param_id]["SOM_FastDecCoeffStandard"] = float(row[2]) param_vals[param_id]["PartSMB_Fast_to_SOM_Fast"] = float(row[3]) param_vals[param_id]["PartSMB_Slow_to_SOM_Fast"] = float(row[4]) param_vals[param_id]["AOM_SlowDecCoeffStandard"] = float(row[5]) param_vals[param_id]["SOM_SlowDecCoeffStandard"] = float(row[6]) param_vals[param_id]["AOM_SlowUtilizationEfficiency"] = float(row[7]) #following two params were calibrated using exps 25 and 35 from V140 Muencheberg param_vals[param_id]["PartAOM_to_AOM_Fast"] = 0.81 param_vals[param_id]["PartAOM_to_AOM_Slow"] = 0.19 like = float(row[8]) if like > best_like: best_like = like best_p_id = param_id def load_rotations(rotations_file, spinup): with open(rotations_file) as _: rotations = json.load(_) #identify rotations with codes rots_info = {} for bkr, rots in rotations.iteritems(): for rot in rots.iteritems(): rot_code = int(rot[0]) my_rot = [] for cm in rot[1]:#["worksteps"]: for ws in range(len(cm["worksteps"])): if cm["worksteps"][ws]["type"] == "Sowing": my_rot.append(cm["worksteps"][ws]["crop"][2]) #for each crop, identify previous main one (needed to determine expected N availability) rot_info = [] for i in range(len(my_rot)): cm_info = {} current_cp = my_rot[i] if i != 0: previous_cp = my_rot[i-1] else: previous_cp = my_rot[-1] cm_info["current"] = current_cp cm_info["previous"] = previous_cp rot_info.append(cm_info) rots_info[rot_code] = rot_info if RESIDUES_HUMUS_BALANCE and spinup==False: #inject additional harvest params for bkr, rots in rotations.iteritems(): for rot in rots.iteritems(): rot_code = int(rot[0]) rot_info = rots_info[rot_code] cp_index = 0 for cm in rot[1]:#["worksteps"]: for ws in range(len(cm["worksteps"])): cp = rot_info[cp_index]["current"] if cm["worksteps"][ws]["type"] == "AutomaticHarvest": cm["worksteps"][ws]["opt-carbon-conservation"] = True cm["worksteps"][ws]["crop-impact-on-humus-balance"] = [humus_equivalent["crop"][cp], "Humus equivalent [Heq]"] if cp == "SBee": cm["worksteps"][ws]["residue-heq"] = [humus_equivalent["material"]["green-manure"], "Heq ton-1 DM"] else: cm["worksteps"][ws]["residue-heq"] = [humus_equivalent["material"]["straw"], "Heq ton-1 DM"] cm["worksteps"][ws]["organic-fertilizer-heq"] = [humus_equivalent["material"]["pig-slurry"], "Heq ton-1 DM"] cm["worksteps"][ws]["crop-usage"] = CROP_USAGE_HUMBAL[cp] cm["worksteps"][ws]["max-residue-recover-fraction"] = CROP_USAGE_HUMBAL["max-residue-recover-fraction"] cp_index += 1 return rotations, rots_info rotations, rots_info = load_rotations(rotations_file, spinup=False) rotations_spinup, rots_info_spinup = load_rotations(rotations_spinup_file, spinup=True) #find the latest crop harvested to determine when to finish the spinup last_date_spinup = defaultdict(lambda: defaultdict(date)) for bkr_id in rotations_spinup.keys(): for rot_id in rotations_spinup[bkr_id].keys(): rotation_spinup = rotations_spinup[bkr_id][rot_id] my_year = 1971 last_date = None i = 0 while True: first_ws = rotation_spinup[i]["worksteps"][0]["date"].split("-") last_ws = rotation_spinup[i]["worksteps"][-1]["latest-date"].split("-") first_date = date(my_year, int(first_ws[1]), int(first_ws[2])) if last_date != None and first_date < last_date: my_year += 1 my_year += int(last_ws[0]) last_date = date(my_year, int(last_ws[1]), int(last_ws[2])) if last_date.year == 2004: last_date_spinup[bkr_id][rot_id] = last_date + timedelta(days=1) break i += 1 if i == len(rotation_spinup): i = 0 #for spinup and "BASE" fertilization: #read additional info #1. expected mineral N availability expected_N_availability = defaultdict(lambda: defaultdict()) with open ("expected_N_availability.csv") as _: reader = csv.reader(_) reader.next() for row in reader: cp_sequence = (row[0], row[1]) soil_type = row[2] expected_N_value = float(row[3]) expected_N_availability[cp_sequence][soil_type] = expected_N_value expected_N_availability[cp_sequence]["target_depth"] = float(row[4]) #2. rules to split mineral fertilization mineralN_split = defaultdict(lambda: defaultdict()) with open ("MineralN_topdressing.csv") as _: reader = csv.reader(_) reader.next() for row in reader: cp = row[0] mineralN_split[cp]["target"] = float(row[2]) for i in range(4, 7): if row[i] != "": mineralN_split[cp][i-4] = float(row[i]) def modify_2ry_yield_params(cp, export_r): for k in EXPORT_PRESETS[export_r].keys(): if cp[0] in k: my_rate = EXPORT_PRESETS[export_r][k] for organ in cp[1]["cropParams"]["cultivar"]["OrganIdsForSecondaryYield"]: organ["yieldPercentage"] *= my_rate for cp in crop["crops"].iteritems(): #correct version if "_SU" in cp[0]: modify_2ry_yield_params(cp, "base") elif not RESIDUES_HUMUS_BALANCE: #"_SU" not in cp[0] modify_2ry_yield_params(cp, EXPORT_RATE) #old version (bugged) #if "_SU" in cp[0]: # modify_2ry_yield_params(cp, "base") #if RESIDUES_HUMUS_BALANCE: # continue #else: # modify_2ry_yield_params(cp, EXPORT_RATE) sim["UseSecondaryYields"] = True sim["include-file-base-path"] = paths["INCLUDE_FILE_BASE_PATH"] def read_general_metadata(path_to_file): "read metadata file" with open(path_to_file) as file_: data = {} reader = csv.reader(file_, delimiter="\t") reader.next() for row in reader: if int(row[1]) != 1: continue data[(int(row[2]), int(row[3]))] = { "subpath-climate.csv": row[9], "latitude": float(row[13]), "elevation": float(row[14]) } return data general_metadata = read_general_metadata("NRW_General_Metadata.csv") def load_mapping(row_offset=0, col_offset=0): to_climate_index = {} with(open("working_resolution_to_climate_lat_lon_indices.json")) as _: l = json.load(_) for i in xrange(0, len(l), 2): cell = (row_offset + l[i][0], col_offset + l[i][1]) to_climate_index[cell] = tuple(l[i+1]) return to_climate_index def read_orgN_kreise(path_to_file, col_index): "read organic N info for kreise" with open(path_to_file) as file_: data = {} reader = csv.reader(file_, delimiter=",") reader.next() reader.next() for row in reader: for kreis_code in row[1].split("|"): if kreis_code != "": data[int(kreis_code)] = float(row[col_index]) return data orgN_kreise = read_orgN_kreise("NRW_orgN_balance_T27.csv", 8) #orgN_kreise = read_orgN_kreise("NRW_orgN_balance_T28.csv", 6) def soil_OID(con, profile_id): "return soil OID from the database connection for given profile id" query = """ select id, soil_OID from soil_profile where id = ? order by id """ con.row_factory = sqlite3.Row for row in con.cursor().execute(query, (profile_id,)): OID = row["soil_OID"] break return OID def update_soil(row, col): "in place update the env" site["SiteParameters"]["Latitude"] = general_metadata[(row, col)]["latitude"] site["SiteParameters"]["HeightNN"] = [general_metadata[(row, col)]["elevation"], "m"] site["SiteParameters"]["SoilProfileParameters"] = soil_io.soil_parameters(soil_db_con, soil_ids[(row, col)]) KA5_txt = soil_io.sand_and_clay_to_ka5_texture(site["SiteParameters"]["SoilProfileParameters"][0]["Sand"][0], site["SiteParameters"]["SoilProfileParameters"][0]["Clay"][0]) for layer in site["SiteParameters"]["SoilProfileParameters"]: layer["SoilBulkDensity"][0] = max(layer["SoilBulkDensity"][0], 600) layer["SoilOrganicCarbon"][0] = layer["SoilOrganicCarbon"][0] * 0.6 #correction factor suggested by TGaiser return KA5_txt def set_initial_SOC(SOC_vals): for layer, val in enumerate(SOC_vals): if len(site["SiteParameters"]["SoilProfileParameters"]) > layer: site["SiteParameters"]["SoilProfileParameters"][layer]["SoilOrganicCarbon"][0] = val def read_header(path_to_ascii_grid_file): "read metadata from esri ascii grid file" metadata = {} header_str = "" with open(path_to_ascii_grid_file) as _: for i in range(0, 6): line = _.readline() header_str += line sline = [x for x in line.split() if len(x) > 0] if len(sline) > 1: metadata[sline[0].strip().lower()] = float(sline[1].strip()) return metadata, header_str def read_ascii_grid(path_to_file, include_no_data=False, row_offset=0, col_offset=0): "read an ascii grid into a map, without the no-data values" def int_or_float(s): try: return int(s) except ValueError: return float(s) with open(path_to_file) as file_: data = {} #skip the header (first 6 lines) for _ in range(0, 6): file_.next() row = -1 for line in file_: row += 1 col = -1 for col_str in line.strip().split(" "): col += 1 if not include_no_data and int_or_float(col_str) == -9999: continue data[(row_offset+row, col_offset+col)] = int_or_float(col_str) return data #offset is used to match info in general metadata and soil database soil_ids = read_ascii_grid("asc_grids/soil-profile-id_nrw_gk3.asc", row_offset=282) bkr_ids = read_ascii_grid("asc_grids/bkr_nrw_gk3.asc", row_offset=282) lu_ids = read_ascii_grid("asc_grids/lu_resampled.asc", row_offset=282) kreise_ids = read_ascii_grid("asc_grids/kreise_matrix.asc", row_offset=282) meteo_ids = load_mapping(row_offset=282) soil_metadata, _ = read_header("asc_grids/soil-profile-id_nrw_gk3.asc") wgs84 = Proj(init="epsg:4326") gk3 = Proj(init="epsg:3396") gk5 = Proj(init="epsg:31469") def create_ascii_grid_interpolator(arr, meta, ignore_nodata=True): "read an ascii grid into a map, without the no-data values" rows, cols = arr.shape cellsize = int(meta["cellsize"]) xll = int(meta["xllcorner"]) yll = int(meta["yllcorner"]) nodata_value = meta["nodata_value"] xll_center = xll + cellsize // 2 yll_center = yll + cellsize // 2 yul_center = yll_center + (rows - 1)*cellsize points = [] values = [] for row in range(rows): for col in range(cols): value = arr[row, col] if ignore_nodata and value == nodata_value: continue r = xll_center + col * cellsize h = yul_center - row * cellsize points.append([r, h]) values.append(value) return NearestNDInterpolator(np.array(points), np.array(values)) path_to_slope_grid = paths["path-to-data-dir"] + "/germany/slope_1000_gk5.asc" slope_metadata, _ = read_header(path_to_slope_grid) slope_grid = np.loadtxt(path_to_slope_grid, dtype=float, skiprows=6) slope_gk5_interpolate = create_ascii_grid_interpolator(slope_grid, slope_metadata) #counter = 0 #for k, v in bkr_ids.iteritems(): # if v == 134: # if k in lu_ids: # counter += 1 #print counter def insert_cc(crop_rotation, cc_data): "insert cover crops in the rotation" insert_cover_here = [] for cultivation_method in range(len(crop_rotation)): for workstep in crop_rotation[cultivation_method]["worksteps"]: if workstep["type"] == "Sowing": if workstep["crop"][2] in COVER_BEFORE: insert_cover_here.append((cultivation_method, workstep["date"])) break for position, mydate in reversed(insert_cover_here): mydate = mydate.split("-") main_crop_sowing = date(2017, int(mydate[1]), int(mydate[2])) latest_harvest_cc = main_crop_sowing - timedelta(days = 10) latest_harvest_cc = unicode("0001-" + str(latest_harvest_cc.month).zfill(2) + "-" + str(latest_harvest_cc.day).zfill(2)) crop_rotation.insert(position, copy.deepcopy(cc_data)) crop_rotation[position]["worksteps"][1]["latest-date"] = latest_harvest_cc def calculate_orgfert_amount(N_applied, fert_type, soilCN=10): "convert N applied in amount of fresh org fert" AOM_DryMatterContent = fert_type["AOM_DryMatterContent"][0] AOM_NH4Content = fert_type["AOM_NH4Content"][0] AOM_NO3Content = fert_type["AOM_NO3Content"][0] CN_Ratio_AOM_Fast = fert_type["CN_Ratio_AOM_Fast"][0] CN_Ratio_AOM_Slow = fert_type["CN_Ratio_AOM_Slow"][0] PartAOM_to_AOM_Fast = fert_type["PartAOM_to_AOM_Fast"][0] PartAOM_to_AOM_Slow = fert_type["PartAOM_to_AOM_Slow"][0] AOM_to_C = 0.45 AOM_fast_factor = (AOM_to_C * PartAOM_to_AOM_Fast)/CN_Ratio_AOM_Fast AOM_slow_factor = (AOM_to_C * PartAOM_to_AOM_Slow)/CN_Ratio_AOM_Slow AOM_SOM_factor = (1- (PartAOM_to_AOM_Fast + PartAOM_to_AOM_Slow)) * AOM_to_C / soilCN conversion_coeff = AOM_NH4Content + AOM_NO3Content + AOM_fast_factor + AOM_slow_factor + AOM_SOM_factor AOM_dry = N_applied / conversion_coeff AOM_fresh = AOM_dry / AOM_DryMatterContent return AOM_fresh def update_fert_values(rotation, rot_info, cc_in_cm, expected_N_availability, mineralN_split, soil_type, orgN_applied, spinup): "function to mimic baseline fertilization" cow_unit = 100 GVs = orgN_applied/cow_unit orgN_effect = GVs * 10 #insert cc in rotation info for cm in reversed(range(len(rot_info))): rot_info[cm]["has_cover_before"] = False if rot_info[cm]["current"] in COVER_BEFORE and cc_in_cm: rot_info[cm]["has_cover_before"] = True cc_info = {"current": "CC"} rot_info.insert(cm, cc_info) for cm in range(len(rotation)): if rot_info[cm]["current"] == "CC": #cover crops do not receive any fertilization continue current_cp = rot_info[cm]["current"] previous_cp = rot_info[cm]["previous"] has_cover = rot_info[cm]["has_cover_before"] if spinup: #spinup crops contain "_SU" by convention current_cp = current_cp.replace("_SU", "") previous_cp = previous_cp.replace("_SU", "") N_target = mineralN_split[current_cp]["target"] expected_Nmin = expected_N_availability[(current_cp, previous_cp)][soil_type] #modify expected Nmin depending on livestock pressure and presence of cover crop if has_cover: expected_Nmin += 20 expected_Nmin += orgN_effect #calculate N to be applied with mineral fertilization sum_Nfert = max(N_target - expected_Nmin, 0) #map the fertilization worksteps ref_fert = 0 for ws in range(len(rotation[cm]["worksteps"])): workstep = rotation[cm]["worksteps"][ws] if workstep["type"] == "MineralFertilization" and workstep["amount"][0] == 0: workstep["amount"][0] = sum_Nfert * mineralN_split[current_cp][ref_fert] ref_fert += 1 def extend_rotation(rot, ccfreq_denominator): #ccfreq = COVER_CROP_FREQ["out-of"] ext_rot = [] for i in range(ccfreq_denominator): ext_rot.append(copy.deepcopy(rot)) return ext_rot def insert_CC(ext_rot, cover_crop, ccfreq_numerator): #COVER_CROP_FREQ["insert-cc-every"] cc_in_cm = {} for cm in range(len(ext_rot)): cc_in_cm[cm] = False if (cm+1) <= ccfreq_numerator: insert_cc(ext_rot[cm], cover_crop) cc_in_cm[cm] = True return ext_rot, cc_in_cm def update_mineral_fert(ext_rot, cc_in_cm, rot_id, rots_info, spinup=False): 'for baseline scenario' for rot in range(len(ext_rot)): update_fert_values(ext_rot[rot], copy.deepcopy(rots_info[int(rot_id)]), cc_in_cm[rot], expected_N_availability, mineralN_split, soil_type, orgN_kreise[kreis_id], spinup) def compose_rotation(ext_rot): #crop["cropRotation"] = composed_rot composed_rot = [] for rot in ext_rot: for cm in rot: composed_rot.append(cm) return composed_rot def rotate(crop_rotation): "rotate the crops in the rotation" crop_rotation.insert(0, crop_rotation.pop()) def set_params(env, params): 'set values of sensitive params' soilorg_params = env['params']['userSoilOrganicParameters']["DEFAULT"] crop_rotations = env["cropRotations"] soilorg_params["PartSOM_Fast_to_SOM_Slow"][0] = params["PartSOM_Fast_to_SOM_Slow"] soilorg_params["SOM_FastDecCoeffStandard"][0] = params["SOM_FastDecCoeffStandard"] soilorg_params["PartSMB_Fast_to_SOM_Fast"][0] = params["PartSMB_Fast_to_SOM_Fast"] soilorg_params["PartSMB_Slow_to_SOM_Fast"][0] = params["PartSMB_Slow_to_SOM_Fast"] soilorg_params["SOM_SlowDecCoeffStandard"][0] = params["SOM_SlowDecCoeffStandard"] soilorg_params["AOM_SlowUtilizationEfficiency"][0] = params["AOM_SlowUtilizationEfficiency"] for rot in crop_rotations: for cm in rot["cropRotation"]: for ws in cm["worksteps"]: if ws["type"] == "OrganicFertilization": ws["parameters"]["AOM_SlowDecCoeffStandard"][0] = params["AOM_SlowDecCoeffStandard"] ws["parameters"]["PartAOM_to_AOM_Fast"][0] = params["PartAOM_to_AOM_Fast"] ws["parameters"]["PartAOM_to_AOM_Slow"][0] = params["PartAOM_to_AOM_Slow"] #elif ws["type"] == "Sowing": # ws["crop"]["residueParams"]["PartAOM_to_AOM_Fast"][0] = params["PartAOM_to_AOM_Fast"] sent_id = 0 start_send = time.clock() simulated_cells = 0 no_kreis = 0 counter_debug = 0 #bkr2lk = defaultdict(set) #for additional info #soilty2iniSOC = defaultdict(list) #for additional info export_lat_lon_coords = False export_lat_lon_file = None srows = int(soil_metadata["nrows"]) scellsize = int(soil_metadata["cellsize"]) sxll = int(soil_metadata["xllcorner"]) syll = int(soil_metadata["yllcorner"]) sxll_center = sxll + scellsize // 2 syll_center = syll + scellsize // 2 syul_center = syll_center + (srows - 1)*scellsize unique_combos = {} #for calibration/SA envs run_params_id = [] if SOC_STUDY: #investigate parameter uncertainty for p_id in param_vals.keys(): run_params_id.append(int(p_id)) run_params_id = sorted(run_params_id) else: #use only best parameter set run_params_id.append(best_p_id) ###Read missing data to be run file_missing_data = "report_missing_data/report_missing_sims_id" + run_id + ".csv" rerun_cells = set() with open(file_missing_data) as _: reader = csv.reader(_) reader.next() for missing_info in reader: rerun_cells.add(missing_info[2]) for (row, col), gmd in general_metadata.iteritems(): #run only what's needed: row_col = "{}{:03d}".format(row, col) if row_col not in rerun_cells: continue #if row_col != "487040": # continue #test #if int(row) != 504 or int(col) != 62: # continue if (row, col) in soil_ids and (row, col) in bkr_ids and (row, col) in lu_ids: # get gk3 coordinates for soil row/col sr_gk3 = sxll_center + col * scellsize sh_gk3 = syul_center - row * scellsize if export_lat_lon_coords: if not export_lat_lon_file: export_lat_lon_file = open("soil_row_col_to_lat_lon_coords.csv", "w") export_lat_lon_file.write("row,col,lat,lon\n") slon, slat = transform(gk3, wgs84, sr_gk3, sh_gk3) export_lat_lon_file.write(",".join(map(str, [row, col, slat, slon])) + "\n") sr_gk5, sh_gk5 = transform(gk3, gk5, sr_gk3, sh_gk3) site["SiteParameters"]["Slope"] = slope_gk5_interpolate(sr_gk5, sh_gk5) / 100.0 #continue bkr_id = bkr_ids[(row, col)] ########for testing #if bkr_id != 129: # continue soil_id = soil_ids[(row, col)] meteo_id = meteo_ids[(row, col)] if (row, col) in kreise_ids: kreis_id = kreise_ids[(row, col)] #bkr2lk[bkr_id].add(kreis_id) else: no_kreis += 1 print "-----------------------------------------------------------" print "kreis not found for calculation of organic N, skipping cell" print "-----------------------------------------------------------" continue simulated_cells += 1 KA5_txt = update_soil(row, col) s_OID = soil_OID(soil_db_con, soil_ids[(row, col)]) light_soils = ["Ss", "Su2", "Su3", "Su4", "St2", "Sl3", "Sl2"] heavy_soils = ["Tu3", "Tu4", "Lt3", "Ts2", "Tl", "Tu2", "Tt"] soil_type = "medium" if KA5_txt in light_soils: soil_type = "light" elif KA5_txt in heavy_soils: soil_type = "heavy" site["SiteParameters"]["SoilSpecificHumusBalanceCorrection"] = HUMBAL_CORRECTION[soil_type] #soilty2iniSOC[soil_type].append(site["SiteParameters"]["SoilProfileParameters"][0]["SoilOrganicCarbon"][0]) #continue #row_col = "{}{:03d}".format(row, col) #topsoil_carbon[row_col] = site["SiteParameters"]["SoilProfileParameters"][0]["SoilOrganicCarbon"][0] #continue for rot_id, rotation in rotations[str(bkr_id)].iteritems(): #retrieve info for calibration/SA ''' dump_env = False if (s_OID, meteo_id, rot_id, orgN_kreise[kreis_id]) not in unique_combos.keys(): profs = [] dump_env = True for prof in range(len(site["SiteParameters"]["SoilProfileParameters"])): prof_info = { "SOC": site["SiteParameters"]["SoilProfileParameters"][prof]["SoilOrganicCarbon"][0], "thickness": site["SiteParameters"]["SoilProfileParameters"][prof]["Thickness"][0], "N_layers": int(round(site["SiteParameters"]["SoilProfileParameters"][prof]["Thickness"][0] * 10)) } profs.append(prof_info) unique_combos[(s_OID, meteo_id, rot_id, orgN_kreise[kreis_id])] = profs print "added combo OID: {0}, meteo: {1}, rot: {2}, org N: {3}".format(str(s_OID), str(meteo_id), str(rot_id), str(orgN_kreise[kreis_id])) else: continue ''' m_id = str(meteo_id).replace("(", "").replace(")", "").replace(", ", "_") unique_id = str(s_OID) + "_" + m_id + "_" + str(rot_id) + "_" + str(int(orgN_kreise[kreis_id])) if SOC_STUDY: #avoid repeating simulations which will yield the same results if unique_id not in unique_combos.keys(): unique_combos[unique_id] = True else: continue for p_id in run_params_id: if p_id in SOC_ini_vals.keys() and unique_id in SOC_ini_vals[p_id].keys(): set_initial_SOC(SOC_ini_vals[p_id][unique_id])# vals at the beginning of spinup #print("p_id {0}, unique_id {1} - OK!".format(str(p_id), str(unique_id))) #counter_debug += 1 #print counter_debug bbb=1 else: print ("initial SOC not found for p_id: {0} and unique_id: {1}".format(str(p_id), str(unique_id))) print (row_col) exit() #spinup period (1971-2004): rotation_spinup = rotations_spinup[str(bkr_id)][rot_id] ext_rot_spinup = extend_rotation(rotation_spinup, 4) ext_rot_spinup, cc_in_cm_spinup = insert_CC(ext_rot_spinup, cover_crop_spinup, 1) update_mineral_fert(ext_rot_spinup, cc_in_cm_spinup, rot_id, rots_info_spinup, spinup=True) #following period (2005-2050): ext_rot = extend_rotation(rotation, COVER_CROP_FREQ["out-of"]) ext_rot, cc_in_cm = insert_CC(ext_rot, cover_crop, COVER_CROP_FREQ["insert-cc-every"]) if FERT_STRATEGY == "BASE": update_mineral_fert(ext_rot, cc_in_cm, rot_id, rots_info) crop_rot1 = compose_rotation(ext_rot_spinup) crop_rot2 = compose_rotation(ext_rot) crop["cropRotations"] = [ { "start": "1971-01-01", "end": last_date_spinup[str(bkr_id)][rot_id].isoformat(), "cropRotation": crop_rot1 }, { "start": (last_date_spinup[str(bkr_id)][rot_id] + timedelta(days=1)).isoformat(), "end": "2060-12-31", "cropRotation": crop_rot2 } ] env = monica_io.create_env_json_from_json_config({ "crop": crop, "site": site, "sim": sim, "climate": "" }) set_params(env, param_vals[p_id]) #assign amount of organic fertilizer for sim_period in range(2): for cultivation_method in env["cropRotations"][sim_period]["cropRotation"]: for workstep in cultivation_method["worksteps"]: if workstep["type"] == "OrganicFertilization": workstep["amount"][0] = calculate_orgfert_amount(orgN_kreise[kreis_id], workstep["parameters"]) #with open("test_crop.json", "w") as _: # _.write(json.dumps(crop, indent=4)) #climate is read by the server env["csvViaHeaderOptions"] = sim["climate.csv-options"] env["csvViaHeaderOptions"]["start-date"] = sim["start-date"] env["csvViaHeaderOptions"]["end-date"] = sim["end-date"] env["pathToClimateCSV"] = [] for PATH in PATH_TO_CLIMATE_DATA_DIR: env["pathToClimateCSV"].append(PATH + "row-" + str(meteo_id[0]) + "/col-" + str(meteo_id[1]) + ".csv") #read dumped event for test purpose: #comment this after test #basepath = os.path.dirname(os.path.abspath(__file__)) #d_env = basepath + "/dumped_envs/" + unique_id + ".json" #with open(d_env) as _: # env = json.load(_) ##### for sim_id, sim_ in sims.iteritems(): if sim_id != PRODUCTION_LEVEL: continue env["events"] = sim_["output"] env["params"]["simulationParameters"]["NitrogenResponseOn"] = sim_["NitrogenResponseOn"] env["params"]["simulationParameters"]["WaterDeficitResponseOn"] = sim_["WaterDeficitResponseOn"] env["params"]["simulationParameters"]["UseAutomaticIrrigation"] = sim_["UseAutomaticIrrigation"] env["params"]["simulationParameters"]["UseNMinMineralFertilisingMethod"] = sim_["UseNMinMineralFertilisingMethod"] env["params"]["simulationParameters"]["FrostKillOn"] = sim_["FrostKillOn"] #if dump_env: # #save the env for calibration/SA # basepath = os.path.dirname(os.path.abspath(__file__)) # filename = basepath + "/dumped_envs/" + unique_id + ".json" # with open(filename, "w") as _: # _.write(json.dumps(env, indent=4)) # print("dumped env: " + unique_id) for main_cp_iteration in range(0, len(rots_info[int(rot_id)])): #do not allow crop rotation of sim_period2 to start with a CC if "is-cover-crop" in env["cropRotations"][1]["cropRotation"][0].keys() and env["cropRotations"][1]["cropRotation"][0]["is-cover-crop"] == True: rotate(env["cropRotations"][1]["cropRotation"]) env["customId"] = rot_id \ + "|" + sim_id \ + "|" + str(soil_id) \ + "|(" + str(row) + "/" + str(col) + ")" \ + "|" + str(bkr_id) \ + "|" + str(main_cp_iteration) \ + "|" + str(sim["UseSecondaryYields"]) \ + "|" + str(timeframes[TF]["start-recording-out"]) \ + "|" + str(RESIDUES_HUMUS_BALANCE) \ + "|" + suffix \ + "|" + KA5_txt \ + "|" + soil_type \ + "|" + str(p_id) \ + "|" + str(orgN_kreise[kreis_id]) \ + "|" + unique_id socket.send_json(env) print "sent env ", sent_id, " customId: ", env["customId"] #exit() sent_id += 1 rotate(env["cropRotations"][1]["cropRotation"]) #only simperiod2 is rotated #exit() #print(len(unique_combos.keys())) if export_lat_lon_file: export_lat_lon_file.close() stop_send = time.clock() print "sending ", sent_id, " envs took ", (stop_send - start_send), " seconds" print "simulated cells: ", simulated_cells, "; not found kreise for org N: ", no_kreis '''
def main(): "main" context = zmq.Context() socket = context.socket(zmq.PUSH) #port = 6666 if len(sys.argv) == 1 else sys.argv[1] config = {"port": 6666, "start": 1, "end": 8157} if len(sys.argv) > 1: for arg in sys.argv[1:]: k, v = arg.split("=") if k in config: config[k] = int(v) if LOCAL_RUN: socket.connect("tcp://localhost:" + str(config["port"])) else: socket.connect("tcp://cluster2:" + str(config["port"])) with open("sim.json") as _: sim = json.load(_) with open("site.json") as _: site = json.load(_) with open("crop.json") as _: crop = json.load(_) with open("sims.json") as _: sims = json.load(_) sim["include-file-base-path"] = PATHS[USER]["INCLUDE_FILE_BASE_PATH"] period_gcms = [ { "grcp": "0", "period": "0", "gcm-rcp": "0_0" }, { "grcp": "1", "period": "2", "gcm-rcp": "GFDL-CM3_45" }, { "grcp": "2", "period": "2", "gcm-rcp": "GFDL-CM3_85" }, { "grcp": "3", "period": "2", "gcm-rcp": "GISS-E2-R_45" }, { "grcp": "4", "period": "2", "gcm-rcp": "GISS-E2-R_85" }, { "grcp": "5", "period": "2", "gcm-rcp": "HadGEM2-ES_26" }, { "grcp": "6", "period": "2", "gcm-rcp": "HadGEM2-ES_45" }, { "grcp": "7", "period": "2", "gcm-rcp": "HadGEM2-ES_85" }, { "grcp": "8", "period": "2", "gcm-rcp": "MIROC5_45" }, { "grcp": "9", "period": "2", "gcm-rcp": "MIROC5_85" }, { "grcp": "10", "period": "2", "gcm-rcp": "MPI-ESM-MR_26" }, { "grcp": "11", "period": "2", "gcm-rcp": "MPI-ESM-MR_45" }, { "grcp": "12", "period": "2", "gcm-rcp": "MPI-ESM-MR_85" }, { "grcp": "1", "period": "3", "gcm-rcp": "GFDL-CM3_45" }, { "grcp": "2", "period": "3", "gcm-rcp": "GFDL-CM3_85" }, { "grcp": "3", "period": "3", "gcm-rcp": "GISS-E2-R_45" }, { "grcp": "4", "period": "3", "gcm-rcp": "GISS-E2-R_85" }, { "grcp": "5", "period": "3", "gcm-rcp": "HadGEM2-ES_26" }, { "grcp": "6", "period": "3", "gcm-rcp": "HadGEM2-ES_45" }, { "grcp": "7", "period": "3", "gcm-rcp": "HadGEM2-ES_85" }, { "grcp": "8", "period": "3", "gcm-rcp": "MIROC5_45" }, { "grcp": "9", "period": "3", "gcm-rcp": "MIROC5_85" }, { "grcp": "10", "period": "3", "gcm-rcp": "MPI-ESM-MR_26" }, { "grcp": "11", "period": "3", "gcm-rcp": "MPI-ESM-MR_45" }, { "grcp": "12", "period": "3", "gcm-rcp": "MPI-ESM-MR_85" }, ] start_year = {"0": "1980", "2": "2040", "3": "2070"} def read_latitude(filename): "" defs = defaultdict(list) lats = {} with open(filename) as _: reader = csv.reader(_) reader.next() for row in reader: res = int(row[1]) if res < 25: continue row_col = (int(row[2]), int(row[3])) lat = float(row[13]) lats[row_col] = lat defs[res].append(lat) for res, vals in defs.iteritems(): lats[(res, -1)] = sum(vals) / len(vals) return lats latitudes = read_latitude(PATHS[USER]["LOCAL_ARCHIVE_PATH_TO_PROJECT"] + "Soil_data/add_x_y_lon_lat_ele_centred.csv") def read_lookup_file(filename): "" lll = [] with open(filename) as _: reader = csv.reader(_) reader.next() for row in reader: lll.append({ 25: (int(row[0]), int(row[1])), 50: (int(row[2]), int(row[3])), 100: (int(row[4]), int(row[5])) }) return lll lookup = read_lookup_file(PATHS[USER]["LOCAL_ARCHIVE_PATH_TO_PROJECT"] + "/unit_description/MCSUR_NRW_CC_cell_lookup.csv") def read_soil_properties(filename): "" soil = defaultdict(list) with open(filename) as _: reader = csv.reader(_) reader.next() for row in reader: row_col = (int(row[3]), int(row[2])) #skip horizons with no properties defined if float(row[45]) <= 0.00001: continue #avoid BD to be too low (soilT crashes) bulk_d = max(600.0, float(row[32]) * 1000) soil[row_col].append({ "depth": float(row[15]), "thickness": float(row[16]), "pwp": float(row[44]), "fc": float(row[45]), "sat": float(row[46]), "corg": float(row[34]), "bulk-density": bulk_d, "sand": float(row[29]) / 100.0, "clay": float(row[27]) / 100.0, "ph": float(row[36]), "sceleton": float(row[31]) / 100.0 }) return soil soil = { 25: read_soil_properties(PATHS[USER]["LOCAL_ARCHIVE_PATH_TO_PROJECT"] + "Soil_data/climate_change_NRW_soil_r25.csv"), 50: read_soil_properties(PATHS[USER]["LOCAL_ARCHIVE_PATH_TO_PROJECT"] + "Soil_data/climate_change_NRW_soil_r50.csv"), 100: read_soil_properties(PATHS[USER]["LOCAL_ARCHIVE_PATH_TO_PROJECT"] + "Soil_data/climate_change_NRW_soil_r100.csv") } def update_soil(soil_res, row, col, crop_id): "update function" crop["cropRotation"][2] = crop_id soil_layers = soil[soil_res][(row, col)] site["Latitude"] = latitudes[(row, col)] if ( row, col) in latitudes else latitudes[(soil_res, -1)] layers = [] profile_depth = 0 for iii, layer in enumerate(soil_layers): lll = { "SoilOrganicCarbon": [layer["corg"], "%"], "SoilBulkDensity": [layer["bulk-density"], "kg m-3"], "FieldCapacity": [layer["fc"], "m3 m-3"], "PermanentWiltingPoint": [layer["pwp"], "m3 m-3"], "PoreVolume": [layer["sat"], "m3 m-3"], "SoilMoisturePercentFC": [80.0 if crop_id == "M" else 50.0, "% [0-100]"], "Sand": layer["sand"], "Clay": layer["clay"], "Sceleton": layer["sceleton"], "pH": layer["ph"], "Thickness": layer["thickness"] } profile_depth = layer["depth"] #!!!! danger, this is changing in memory sims events structure and events can only then be assigned to env sims["output"][crop_id][1][6][1][1] = int( min(math.ceil(profile_depth * 10), 20)) layers.append(lll) site["SiteParameters"]["SoilProfileParameters"] = layers return profile_depth #print site["SiteParameters"]["SoilProfileParameters"] #assert len(row_cols) == len(pheno["GM"].keys()) == len(pheno["WW"].keys()) #print "# of rowsCols = ", len(row_cols) climate_to_soil = [{ "step": 1, "climate": 25, "soil": 25 }, { "step": 1, "climate": 25, "soil": 50 }, { "step": 1, "climate": 25, "soil": 100 }, { "step": 2, "climate": 50, "soil": 25 }, { "step": 2, "climate": 100, "soil": 25 }, { "step": 3, "climate": 50, "soil": 50 }, { "step": 3, "climate": 100, "soil": 100 }] production_situations = { "PLP": { "water-response": False, "nitrogen-response": False }, "PLW": { "water-response": True, "nitrogen-response": False }, "PLN": { "water-response": True, "nitrogen-response": True } } i = 1 start_store = time.clock() #start = config["start"] - 1 #end = config["end"] - 1 #row_cols_ = row_cols[start:end+1] #print "running from ", start, "/", row_cols[start], " to ", end, "/", row_cols[end] for c2s in climate_to_soil: step = c2s["step"] climate_resolution = c2s["climate"] soil_resolution = c2s["soil"] climate_to_soils = defaultdict(set) for mmm in lookup: climate_to_soils[mmm[climate_resolution]].add(mmm[soil_resolution]) print "step: ", step, " ", sum( [len(s[1]) for s in climate_to_soils.iteritems()] ), " runs in climate_res: ", climate_resolution, " and soil_res: ", soil_resolution for (climate_row, climate_col), soil_coords in climate_to_soils.iteritems(): #if (climate_row, climate_col) != (6,6): # continue for soil_row, soil_col in soil_coords: #if (soil_row, soil_col) != (11,11): # continue for crop_id in ["W", "M"]: rootdepth_soillimited = update_soil( soil_resolution, soil_row, soil_col, crop_id) env = monica_io.create_env_json_from_json_config({ "crop": crop, "site": site, "sim": sim, "climate": "" }) env["csvViaHeaderOptions"] = sim["climate.csv-options"] env["events"] = sims["output"][crop_id] for workstep in env["cropRotation"][0]["worksteps"]: if workstep["type"] == "Seed": current_rootdepth = float( workstep["crop"]["cropParams"]["cultivar"] ["CropSpecificMaxRootingDepth"]) workstep["crop"]["cropParams"]["cultivar"][ "CropSpecificMaxRootingDepth"] = min( current_rootdepth, rootdepth_soillimited) break for production_id, switches in production_situations.iteritems( ): env["params"]["simulationParameters"][ "WaterDeficitResponseOn"] = switches[ "water-response"] env["params"]["simulationParameters"][ "NitrogenResponseOn"] = switches[ "nitrogen-response"] for period_gcm in period_gcms: grcp = period_gcm["grcp"] period = period_gcm["period"] gcm = period_gcm["gcm-rcp"] #if period != "0": # continue #if climate_resolution != 50: # continue #if soil_resolution != 25: # continue #if crop_id != "W": # continue if period != "0": climate_filename = "daily_mean_P{}_GRCP_{}_RES{}_C0{}R{}.csv".format( period, grcp, climate_resolution, climate_col, climate_row) else: climate_filename = "daily_mean_P{}_RES{}_C0{}R{}.csv".format( period, climate_resolution, climate_col, climate_row) if LOCAL_RUN: env["pathToClimateCSV"] = PATHS[USER][ "LOCAL_ARCHIVE_PATH_TO_PROJECT"] + "Climate_data/NRW_weather_climate_change_v3/res_" + str( climate_resolution ) + "/period_" + period + "/GRCP_" + grcp + "/" + climate_filename else: env["pathToClimateCSV"] = PATHS[USER][ "ARCHIVE_PATH_TO_PROJECT"] + "Climate_data/NRW_weather_climate_change_v3/res_" + str( climate_resolution ) + "/period_" + period + "/GRCP_" + grcp + "/" + climate_filename #initialize nitrate/ammonium in soil layers at start of simulation #for i in range(3): # env["cropRotation"][0]["worksteps"][i]["date"] = start_year[period] + "-01-01" env["customId"] = crop_id \ + "|" + str(climate_resolution) \ + "|(" + str(climate_row) + "/" + str(climate_col) + ")" \ + "|" + str(soil_resolution) \ + "|(" + str(soil_row) + "/" + str(soil_col) + ")" \ + "|" + period \ + "|" + grcp \ + "|" + gcm \ + "|" + production_id #with open("envs/env-"+str(i)+".json", "w") as _: # _.write(json.dumps(env)) socket.send_json(env) print "sent env ", i, " customId: ", env[ "customId"] #exit() i += 1 stop_store = time.clock() print "sending ", (i - 1), " envs took ", (stop_store - start_store), " seconds" #print "ran from ", start, "/", row_cols[start], " to ", end, "/", row_cols[end] return
def run_producer(): "main function" #site_name = "FI" site_name = "FI" simulations = ["F"] simulation_dir = "simulation/" + site_name + "/" context = zmq.Context() socket = context.socket(zmq.PUSH) config = {"user": "******", "port": "66666", "server": "localhost"} if len(sys.argv) > 1: for arg in sys.argv[1:]: k, v = arg.split("=") if k in config: config[k] = v paths = PATHS[config["user"]] socket.connect("tcp://" + config["server"] + ":" + str(config["port"])) for sim_type in simulations: c_files = climate_files[sim_type][site_name] co2_values = climate_files[sim_type]["CO2"] print c_files for climate_file, co2 in zip(c_files, co2_values): output_file = "O" + os.path.splitext( os.path.basename(climate_file))[0] + ".txt" if sim_type == "C": output_file = "O" + os.path.splitext( os.path.basename(climate_file))[0] + str(co2) + ".txt" if sim_type == "F": output_file = "MO_FU2050_YR_" + os.path.splitext( os.path.basename( climate_file))[0] + "_" + site_name + ".txt" with open(simulation_dir + "sim-" + site_name + ".json") as _: sim = json.load(_) with open(simulation_dir + "site-" + site_name + ".json") as _: site = json.load(_) with open(simulation_dir + "crop-" + site_name + ".json") as _: crop = json.load(_) sim["include-file-base-path"] = paths["INCLUDE_FILE_BASE_PATH"] if sim_type == "F": sim["start-date"] = timeframes[site_name][climate_file][ "start-date"] sim["end-date"] = timeframes[site_name][climate_file][ "end-date"] sent_id = 0 start_send = time.clock() env = monica_io.create_env_json_from_json_config({ "crop": crop, "site": site, "sim": sim, "climate": "" }) #monica_io.add_climate_data_to_env(env, sim) env["csvViaHeaderOptions"] = sim["climate.csv-options"] env["csvViaHeaderOptions"]["start-date"] = sim["start-date"] env["csvViaHeaderOptions"]["end-date"] = sim["end-date"] env["pathToClimateCSV"] = [] climate_path = paths[ "INCLUDE_FILE_BASE_PATH"] + simulation_dir + "climate/" + climate_file + ".csv" print(climate_path) env["pathToClimateCSV"].append(climate_path) env["params"]["userEnvironmentParameters"]["AtmosphericCO2"] = co2 env["customId"] = { "id": "BCD3", "site": site_name, "output_filename": output_file } #{ #"id": "BCD3", #"site_name": site_name #} socket.send_json(env) print("sent env ", sent_id, " customId: ", env["customId"]) sent_id += 1 stop_send = time.clock() print("sending ", sent_id, " envs took ", (stop_send - start_send), " seconds")
def __init__(self, exp_maps, host): self.host = host #for multi-experiment: create a M-2 relationship between exp_IDs and param files self.IDs_paramspaths = {} for exp_map in exp_maps: self.IDs_paramspaths[exp_map["exp_ID"]] = {} self.IDs_paramspaths[exp_map["exp_ID"]]["species"] = exp_map["species_file"] self.IDs_paramspaths[exp_map["exp_ID"]]["cultivar"] = exp_map["cultivar_file"] #load rules (for MONICA output) with open("calibration_rules.json") as _: self.calibration_rules = json.load(_) #data structures for spotpy self.expected_outcome = [] #former "observations"; stores success scores self.simulation_outcome = [] #former "simulations"; stores outcome after checking rules self.exp_ids = [] #mirrors spotpy data structure to provide the user with info on the exp id (success/failure) self.checked_rules = [] #mirrors spotpy data structure to provide the user with info on the checked rule (success/failure) self.species_params={} #map to store different species params sets avoiding repetition self.cultivar_params={} #map to store different cultivar params sets avoiding repetition #create envs self.envs = [] for exp_map in exp_maps: with open(exp_map["sim_file"]) as simfile: sim = json.load(simfile) sim["crop.json"] = exp_map["crop_file"] sim["site.json"] = exp_map["site_file"] #sim["climate.csv"] = exp_map["climate_file"] with open(exp_map["site_file"]) as sitefile: site = json.load(sitefile) with open(exp_map["crop_file"]) as cropfile: crop = json.load(cropfile) mycrop = exp_map["crop_ID"] crop["crops"][mycrop]["cropParams"]["species"][1] = exp_map["species_file"] crop["crops"][mycrop]["cropParams"]["cultivar"][1] = exp_map["cultivar_file"] env = monica_io.create_env_json_from_json_config({ "crop": crop, "site": site, "sim": sim, "climate": "" }) #customize monica output env["events"] = [] for rule_id, rule_specs in self.calibration_rules.iteritems(): env["events"].append(rule_specs["custom event"][0]) #customize output env["events"].append(rule_specs["custom event"][1]) #climate is read by the server env["csvViaHeaderOptions"] = sim["climate.csv-options"] env["csvViaHeaderOptions"]["start-date"] = sim["climate.csv-options"]["start-date"] env["csvViaHeaderOptions"]["end-date"] = sim["climate.csv-options"]["end-date"] env["pathToClimateCSV"] = [] env["pathToClimateCSV"].append(exp_map["climate_file"]) #monica_io.add_climate_data_to_env(env, sim) this might not be supported anymore for ws in env["cropRotation"][0]["worksteps"]: if ws["type"] == "Seed" or ws["type"] == "Sowing": self.species_params[exp_map["species_file"]] = ws["crop"]["cropParams"]["species"] self.cultivar_params[exp_map["cultivar_file"]] = ws["crop"]["cropParams"]["cultivar"] break env["customId"] = exp_map["exp_ID"] self.envs.append(env) self.context = zmq.Context() self.socket_producer = self.context.socket(zmq.PUSH) self.socket_producer.connect("tcp://" + self.host["server"] + ":" + self.host["push-port"])
def __init__(self, exp_maps, obslist): #for multi-experiment: create a M-2 relationship between exp_IDs and param files self.IDs_paramspaths = {} for exp_map in exp_maps: self.IDs_paramspaths[exp_map["exp_ID"]] = {} self.IDs_paramspaths[ exp_map["exp_ID"]]["species"] = exp_map["species_file"] self.IDs_paramspaths[ exp_map["exp_ID"]]["cultivar"] = exp_map["cultivar_file"] #custom events (for MONICA output) custom_events = defaultdict(list) with open("template_events.json") as _: template_events = json.load(_) #observations data structures self.observations = [] #for spotpy for record in obslist: my_event = deepcopy( template_events[record["stage"]]) #deepcopy just to be sure :) if record["stage"] == "Zadok12": #special case, "Stage" (=2) instead of "DOY" is used self.observations.append(2) my_event[0] = unicode(record["date"].isoformat()) else: self.observations.append(record["DOY"]) custom_events[record["exp_ID"]].append(my_event[0]) custom_events[record["exp_ID"]].append(my_event[1]) self.species_params = { } #map to store different species params sets avoiding repetition self.cultivar_params = { } #map to store different cultivar params sets avoiding repetition #create envs self.envs = [] for exp_map in exp_maps: with open(exp_map["sim_file"]) as simfile: sim = json.load(simfile) #sim["events"] = custom_events[exp_map["exp_ID"]] #customize output, if here monica.io will ignore it (why?) #with open("test_sim_events.json", "w") as out_f: # json.dump(sim, out_f, indent=4) sim["crop.json"] = exp_map["crop_file"] sim["site.json"] = exp_map["site_file"] sim["EmergenceMoistureControlOn"] = True sim["EmergenceFloodingControlOn"] = True #sim["climate.csv"] = exp_map["climate_file"] with open(exp_map["site_file"]) as sitefile: site = json.load(sitefile) with open(exp_map["crop_file"]) as cropfile: crop = json.load(cropfile) mycrop = exp_map["crop_ID"] crop["crops"][mycrop]["cropParams"]["species"][1] = exp_map[ "species_file"] crop["crops"][mycrop]["cropParams"]["cultivar"][1] = exp_map[ "cultivar_file"] env = monica_io.create_env_json_from_json_config({ "crop": crop, "site": site, "sim": sim, "climate": "" }) env["events"] = custom_events[exp_map["exp_ID"]] #customize output #climate is read by the server env["csvViaHeaderOptions"] = sim["climate.csv-options"] env["csvViaHeaderOptions"]["start-date"] = sim[ "climate.csv-options"]["start-date"] env["csvViaHeaderOptions"]["end-date"] = sim[ "climate.csv-options"]["end-date"] env["pathToClimateCSV"] = [] env["pathToClimateCSV"].append(exp_map["climate_file"]) position = int(exp_map["where_in_rotation"][0]) for position in exp_map["where_in_rotation"]: for ws in env["cropRotation"][position]["worksteps"]: if ws["type"] == "Seed" or ws["type"] == "Sowing": self.species_params[exp_map["species_file"]] = ws[ "crop"]["cropParams"]["species"] self.cultivar_params[exp_map["cultivar_file"]] = ws[ "crop"]["cropParams"]["cultivar"] break #monica_io.add_climate_data_to_env(env, sim) this does not work anymore properly env["customId"] = exp_map["exp_ID"] env["where_in_rotation"] = exp_map["where_in_rotation"] self.envs.append(env) self.context = zmq.Context() self.socket_producer = self.context.socket(zmq.PUSH) #self.socket_producer.connect("tcp://cluster2:6666") self.socket_producer.connect("tcp://localhost:6666")
def main(): "main function" config = { "port": 6666, } if len(sys.argv) > 1: for arg in sys.argv[1:]: k,v = arg.split("=") if k in config: config[k] = int(v) context = zmq.Context() socket = context.socket(zmq.PUSH) if LOCAL_RUN: socket.connect("tcp://localhost:" + str(config["port"])) else: socket.connect("tcp://cluster" + str(RUN_ON_CLUSTER) + ":" + str(config["port"])) soil_db_con = sqlite3.connect("soil.sqlite") with open("sim-voce.json") as _: sim = json.load(_) with open("site-voce.json") as _: site = json.load(_) with open("crop-voce.json") as _: crop = json.load(_) with open("rotations-voce.json") as _: rotations = json.load(_) sim["include-file-base-path"] = PATHS[USER]["INCLUDE_FILE_BASE_PATH"] def read_general_metadata(path_to_file): "read metadata file" with open(path_to_file) as file_: data = {} reader = csv.reader(file_, delimiter="\t") reader.next() for row in reader: if int(row[1]) != 1: continue data[(int(row[2]), int(row[3]))] = { "subpath-climate.csv": row[9], "latitude": float(row[13]), "elevation": float(row[14]) } return data general_metadata = read_general_metadata("NRW_General_Metadata.csv") def load_mapping(row_offset=0, col_offset=0): to_climate_index = {} with(open("working_resolution_to_climate_lat_lon_indices.json")) as _: l = json.load(_) for i in xrange(0, len(l), 2): cell = (row_offset + l[i][0], col_offset + l[i][1]) to_climate_index[cell] = tuple(l[i+1]) return to_climate_index def read_orgN_kreise(path_to_file): "read organic N info for kreise" with open(path_to_file) as file_: data = {} reader = csv.reader(file_, delimiter=",") reader.next() reader.next() for row in reader: for kreis_code in row[1].split("|"): if kreis_code != "": data[int(kreis_code)] = float(row[8]) return data orgN_kreise = read_orgN_kreise("NRW_orgN_balance.csv") #print orgN_kreise.keys() def update_soil_crop_dates(row, col): "in place update the env" #startDate = date(1980, 1, 1)# + timedelta(days = p["sowing-doy"]) #sim["start-date"] = startDate.isoformat() #sim["end-date"] = date(2010, 12, 31).isoformat() #sim["debug?"] = True site["Latitude"] = general_metadata[(row, col)]["latitude"] site["HeightNN"] = [general_metadata[(row, col)]["elevation"], "m"] site["SiteParameters"]["SoilProfileParameters"] = soil_io.soil_parameters(soil_db_con, soil_ids[(row, col)]) for layer in site["SiteParameters"]["SoilProfileParameters"]: layer["SoilBulkDensity"][0] = max(layer["SoilBulkDensity"][0], 600) def read_ascii_grid(path_to_file, include_no_data=False, row_offset=0, col_offset=0): "read an ascii grid into a map, without the no-data values" def int_or_float(s): try: return int(s) except ValueError: return float(s) with open(path_to_file) as file_: data = {} #skip the header (first 6 lines) for _ in range(0, 6): file_.next() row = -1 for line in file_: row += 1 col = -1 for col_str in line.strip().split(" "): col += 1 if not include_no_data and int_or_float(col_str) == -9999: continue data[(row_offset+row, col_offset+col)] = int_or_float(col_str) return data #offset is used to match info in general metadata and soil database soil_ids = read_ascii_grid("soil-profile-id_nrw_gk3.asc", row_offset=282) bkr_ids = read_ascii_grid("bkr_nrw_gk3.asc", row_offset=282) lu_ids = read_ascii_grid("lu_resampled.asc", row_offset=282) kreise_ids = read_ascii_grid("kreise_matrix.asc", row_offset=282) meteo_ids = load_mapping(row_offset=282) #counter = 0 #for k, v in bkr_ids.iteritems(): # if v == 134: # if k in lu_ids: # counter += 1 #print counter i = 0 start_send = time.clock() def calculate_orgfert_amount(N_applied, fert_type): "convert N applied in amount of fresh org fert" AOM_DryMatterContent = fert_type["AOM_DryMatterContent"][0] AOM_NH4Content = fert_type["AOM_NH4Content"][0] AOM_NO3Content = fert_type["AOM_NO3Content"][0] CN_Ratio_AOM_Fast = fert_type["CN_Ratio_AOM_Fast"][0] CN_Ratio_AOM_Slow = fert_type["CN_Ratio_AOM_Slow"][0] PartAOM_to_AOM_Fast = fert_type["PartAOM_to_AOM_Fast"][0] PartAOM_to_AOM_Slow = fert_type["PartAOM_to_AOM_Slow"][0] AOM_to_C = 0.45 AOM_fast_factor = 1/(CN_Ratio_AOM_Fast/(AOM_to_C * PartAOM_to_AOM_Fast)) AOM_slow_factor = 1/(CN_Ratio_AOM_Slow/(AOM_to_C * PartAOM_to_AOM_Slow)) conversion_coeff = AOM_NH4Content + AOM_NO3Content + AOM_fast_factor + AOM_slow_factor AOM_dry = N_applied / conversion_coeff AOM_fresh = AOM_dry / AOM_DryMatterContent return AOM_fresh simulated_cells = 0 no_kreis = 0 for (row, col), gmd in general_metadata.iteritems(): if (row, col) in soil_ids and (row, col) in bkr_ids and (row, col) in lu_ids: bkr_id = bkr_ids[(row, col)] #if bkr_id != 143: # continue soil_id = soil_ids[(row, col)] meteo_id = meteo_ids[(row, col)] if (row, col) in kreise_ids: kreis_id = kreise_ids[(row, col)] else: no_kreis += 1 print "-----------------------------------------------------" print "kreis not found for calculation of organic N" #TODO find out a solution print "-----------------------------------------------------" simulated_cells += 1 update_soil_crop_dates(row, col) for rot_id, rotation in rotations.iteritems(): crop["cropRotation"] = rotation env = monica_io.create_env_json_from_json_config({ "crop": crop, "site": site, "sim": sim, "climate": "" }) #assign amount of organic fertilizer #for cultivation_method in env["cropRotation"]: # for workstep in cultivation_method["worksteps"]: # if workstep["type"] == "OrganicFertilization": # workstep["amount"] = calculate_orgfert_amount(orgN_kreise[kreis_id], workstep["parameters"]) #climate is read by the server env["csvViaHeaderOptions"] = sim["climate.csv-options"] if LOCAL_RUN: env["pathToClimateCSV"] = LOCAL_PATH_TO_CLIMATE_DATA_DIR + "row-" + str(meteo_id[0]) + "/col-" + str(meteo_id[1]) + ".csv" else: env["pathToClimateCSV"] = PATH_TO_CLIMATE_DATA_DIR + "row-" + str(meteo_id[0]) + "/col-" + str(meteo_id[1]) + ".csv" #env["pathToClimateCSV"] = PATH_TO_CLIMATE_DATA_DIR + gmd["subpath-climate.csv"] env["customId"] = rot_id \ + "|" + str(soil_id) \ + "|(" + str(row) + "/" + str(col) + ")" \ + "|" + str(bkr_id) socket.send_json(env) print "sent env ", i, " customId: ", env["customId"] #if i > 10: # exit() i += 1 stop_send = time.clock() print "sending ", i, " envs took ", (stop_send - start_send), " seconds" print "simulated cells: ", simulated_cells, "; not found kreise for org N: ", no_kreis
def __init__(self, exp_maps, observations, config, finalrun): #read params - they will be modified in the run method with open(config["soilorg_params_path"]) as _: self.soil_organic_params = json.load(_) with open(config["fertorg_params_path"]) as _: self.fert_organic_params = json.load(_) #following params were calibrated using exps 25 and 35 from V140 Muencheberg self.fert_organic_params["PartAOM_to_AOM_Fast"][0] = 0.81 self.fert_organic_params["PartAOM_to_AOM_Slow"][0] = 0.19 #observations data structures self.observations = observations self.obslist = helper.create_spotpylist(self.observations) #create envs self.envs = [] for exp_map in exp_maps: with open(exp_map["sim_file"]) as simfile: sim = json.load(simfile) sim["crop.json"] = exp_map["crop_file"] sim["site.json"] = exp_map["site_file"] sim["climate.csv"] = exp_map["climate_file"] with open(exp_map["site_file"]) as sitefile: site = json.load(sitefile) with open(exp_map["crop_file"]) as cropfile: crop = json.load(cropfile) crop["cropRotations"] = [] env = monica_io.create_env_json_from_json_config({ "crop": crop, "site": site, "sim": sim }) #create references to self.soil_organic_params and self.fert_organic_params env["params"][ "userSoilOrganicParameters"] = self.soil_organic_params for cm in env["cropRotation"]: for ws in cm["worksteps"]: if ws["type"] == "OrganicFertilization": ws["parameters"] = self.fert_organic_params #customize events section env["events"] = [] with open(config["events-file"]) as _: json_out = json.load(_) env["events"] = json_out["events"] #climate is read by the server env["csvViaHeaderOptions"] = sim["climate.csv-options"] env["pathToClimateCSV"] = [] if config["server"] == "localhost": env["pathToClimateCSV"].append(sim["climate.csv"]) else: local_path_to_climate = os.path.dirname( os.path.abspath(__file__)) + "\\climate_files" cluster_path_to_climate = "/archiv-daten/md/projects/sustag/climate_files_uncertainty_analysis" env["pathToClimateCSV"].append(sim["climate.csv"].replace( local_path_to_climate, cluster_path_to_climate).replace("\\", "/")) #print env["pathToClimateCSV"][0] env["customId"] = str(exp_map["treatment"]) + "|" + str( exp_map["parcel"]) + "|" + config["events-file"] self.envs.append(env) #open sockets self.context = zmq.Context() self.socket_push = self.context.socket(zmq.PUSH) s_push = "tcp://" + config["server"] + ":" + config["push-port"] self.socket_push.connect(s_push) self.socket_pull = self.context.socket(zmq.PULL) s_pull = "tcp://" + config["server"] + ":" + config["pull-port"] self.socket_pull.connect(s_pull)
def __init__(self, exp_maps, obslist): #for multi-experiment: create a M-2 relationship between exp_IDs and param files self.IDs_paramspaths = {} for exp_map in exp_maps: self.IDs_paramspaths[exp_map["exp_ID"]] = {} self.IDs_paramspaths[exp_map["exp_ID"]]["species"] = exp_map["species_file"] self.IDs_paramspaths[exp_map["exp_ID"]]["cultivar"] = exp_map["cultivar_file"] #observations data structures self.observations = [] #for spotpy for record in obslist: self.observations.append(record["value"]) self.species_params={} #map to store different species params sets avoiding repetition self.cultivar_params={} #map to store different cultivar params sets avoiding repetition #create envs self.envs = [] for exp_map in exp_maps: with open(exp_map["sim_file"]) as simfile: sim = json.load(simfile) sim["crop.json"] = exp_map["crop_file"] sim["site.json"] = exp_map["site_file"] #sim["climate.csv"] = exp_map["climate_file"] with open(exp_map["site_file"]) as sitefile: site = json.load(sitefile) with open(exp_map["crop_file"]) as cropfile: crop = json.load(cropfile) mycrop = exp_map["crop_ID"] crop["crops"][mycrop]["cropParams"]["species"][1] = exp_map["species_file"] crop["crops"][mycrop]["cropParams"]["cultivar"][1] = exp_map["cultivar_file"] env = monica_io.create_env_json_from_json_config({ "crop": crop, "site": site, "sim": sim, "climate": "" }) #climate is read by the server env["csvViaHeaderOptions"] = sim["climate.csv-options"] env["csvViaHeaderOptions"]["start-date"] = sim["climate.csv-options"]["start-date"] env["csvViaHeaderOptions"]["end-date"] = sim["climate.csv-options"]["end-date"] env["pathToClimateCSV"] = [] env["pathToClimateCSV"].append(exp_map["climate_file"]) position = int(exp_map["where_in_rotation"][0]) for position in exp_map["where_in_rotation"]: for ws in env["cropRotation"][position]["worksteps"]: if ws["type"] == "Seed" or ws["type"] == "Sowing": self.species_params[exp_map["species_file"]] = ws["crop"]["cropParams"]["species"] self.cultivar_params[exp_map["cultivar_file"]] = ws["crop"]["cropParams"]["cultivar"] break #monica_io.add_climate_data_to_env(env, sim) this does not work anymore properly env["customId"] = exp_map["exp_ID"] env["where_in_rotation"] = exp_map["where_in_rotation"] self.envs.append(env) self.context = zmq.Context() self.socket_producer = self.context.socket(zmq.PUSH) #self.socket_producer.connect("tcp://cluster2:6666") self.socket_producer.connect("tcp://localhost:6666")
def __init__(self, exp_maps, obslist, finalrun): #for multi-experiment: create a M-2 relationship between exp_IDs and param files self.IDs_paramspaths = {} for exp_map in exp_maps: self.IDs_paramspaths[exp_map["exp_ID"]] = {} self.IDs_paramspaths[exp_map["exp_ID"]]["species"] = exp_map["species_file"] self.IDs_paramspaths[exp_map["exp_ID"]]["cultivar"] = exp_map["cultivar_file"] #observations data structures self.observations = [] #for spotpy self.evaluationdates = {} #for plotting outputs self.obsdict = {} #for plotting outputs i = 0 #i is a reference to match the element in result array (from spotpy) for record in obslist: var_name = record["variable"] if "aggregation" in record.keys(): #for plotting purposes the variable name must be unique (and match variable in collect results method) fromL = str(record["aggregation"][1][0]) toL = str(record["aggregation"][1][1]) var_name += " " + fromL + " to " + toL self.observations.append(record["value"]) if var_name not in self.evaluationdates: #add the variable as a key self.evaluationdates[var_name] = {} self.obsdict[var_name] = {} if record["exp_ID"] not in self.evaluationdates[var_name]: #add the experiment as a key self.evaluationdates[var_name][record["exp_ID"]] = [] self.obsdict[var_name][record["exp_ID"]] = [] thisdate = record["date"].split("-")#self.evaluationdates needs a date type (not isoformat) self.evaluationdates[var_name][record["exp_ID"]].append([i, date(int(thisdate[0]), int(thisdate[1]), int(thisdate[2]))]) self.obsdict[var_name][record["exp_ID"]].append([i, record["value"]]) i += 1 #normalization data structure self.normalize = defaultdict(lambda: defaultdict(dict)) for var in self.obsdict: var_indexes = [] self.normalize[var]["max_obs_value"] = 0.01 for exp_id in self.obsdict[var]: inds_values = self.obsdict[var][exp_id] for index_value in inds_values: var_indexes.append(index_value[0]) #index in evallist self.normalize[var]["max_obs_value"] = max(self.normalize[var]["max_obs_value"], index_value[1]) #max observed value self.normalize[var]["where"] = var_indexes #apply normalization to obs for anorm in self.normalize: n_factor = self.normalize[anorm]["max_obs_value"] / 100 for jjj in self.normalize[anorm]["where"]: self.observations[jjj] /= n_factor self.species_params={} #map to store different species params sets avoiding repetition self.cultivar_params={} #map to store different cultivar params sets avoiding repetition #create envs self.envs = [] for exp_map in exp_maps: with open(exp_map["sim_file"]) as simfile: sim = json.load(simfile) sim["crop.json"] = exp_map["crop_file"] sim["site.json"] = exp_map["site_file"] sim["climate.csv"] = exp_map["climate_file"] with open(exp_map["site_file"]) as sitefile: site = json.load(sitefile) with open(exp_map["crop_file"]) as cropfile: crop = json.load(cropfile) mycrop = exp_map["crop_ID"] crop["crops"][mycrop]["cropParams"]["species"][1] = exp_map["species_file"] crop["crops"][mycrop]["cropParams"]["cultivar"][1] = exp_map["cultivar_file"] env = monica_io.create_env_json_from_json_config({ "crop": crop, "site": site, "sim": sim }) #add required outputs for record in obslist: if record["exp_ID"] == exp_map["exp_ID"]: if not finalrun: #output only info required by spotpy env["events"].append(unicode(record["date"])) var = [unicode(record["variable"])] if "aggregation" in record.keys(): var = [] var.append(record["aggregation"]) env["events"].append(var) #TODO:Add weight here? elif finalrun: #daily output for plots if "daily" not in env["events"]: env["events"].append(unicode("daily")) env["events"].append([]) #empty list of daily variables env["events"][1].append(unicode("Date")) var = [unicode(record["variable"])] if "aggregation" in record.keys(): var = record["aggregation"] if var not in env["events"][1]: #avoid to ask twice the same var as out env["events"][1].append(var) position = int(exp_map["where_in_rotation"][0]) for ws in env["cropRotation"][position]["worksteps"]: if ws["type"] == "Seed" or ws["type"] == "Sowing": self.species_params[exp_map["species_file"]] = ws["crop"]["cropParams"]["species"] self.cultivar_params[exp_map["cultivar_file"]] = ws["crop"]["cropParams"]["cultivar"] break monica_io.add_climate_data_to_env(env, sim) env["customId"] = exp_map["exp_ID"] env["where_in_rotation"] = exp_map["where_in_rotation"] self.envs.append(env) self.context = zmq.Context() self.socket_producer = self.context.socket(zmq.PUSH) #self.socket_producer.connect("tcp://cluster2:6666") self.socket_producer.connect("tcp://localhost:6666")
def main(): "main" context = zmq.Context() socket = context.socket(zmq.PUSH) #port = 6666 if len(sys.argv) == 1 else sys.argv[1] config = {"port": 6666, "start": 1, "end": 8157} if len(sys.argv) > 1: for arg in sys.argv[1:]: kkk, vvv = arg.split("=") if kkk in config: config[kkk] = int(vvv) #socket.bind("tcp://*:" + str(config["port"])) #socket.connect("tcp://localhost:" + str(config["port"])) socket.connect("tcp://cluster2:" + str(config["port"])) with open("sim.json") as _: sim = json.load(_) with open("site.json") as _: site = json.load(_) with open("crop.json") as _: crop = json.load(_) with open("sims.json") as _: sims = json.load(_) sim["include-file-base-path"] = "C:/Users/berg.ZALF-AD/MONICA" def read_pheno(path_to_file): "read phenology data" with open(path_to_file) as _: ppp = {} reader = csv.reader(_) reader.next() for row in reader: ppp[(int(row[0]), int(row[1]))] = { "sowing-doy": int(row[6]), "flowering-doy": int(row[7]), "harvest-doy": int(row[8]) } return ppp pheno = { "GM": read_pheno("Maize_pheno_v3.csv"), "WW": read_pheno("WW_pheno_v3.csv") } soil = {} row_cols = [] with open("JRC_soil_macsur_v3.csv") as _: reader = csv.reader(_) reader.next() for row in reader: row_col = (int(row[1]), int(row[0])) row_cols.append(row_col) soil[row_col] = { "elevation": float(row[4]), "latitude": float(row[5]), "depth": float(row[6]), "pwp": float(row[7]), "fc": float(row[8]), "sat": float(row[9]), "sw-init": float(row[10]), "oc-topsoil": float(row[11]), "oc-subsoil": float(row[12]), "bd-topsoil": float(row[13]), "bd-subsoil": float(row[14]), "sand-topsoil": float(row[15]), "sand-subsoil": float(row[18]), "clay-topsoil": float(row[16]), "clay-subsoil": float(row[19]), } def read_calibrated_tsums(path_to_file, crop_id): "read calibrated tsums into dict" with open(path_to_file) as _: rrr = {} reader = csv.reader(_) reader.next() for line in reader: ddd = {} row_, col_ = line[0].split("_") row, col = (int(row_), int(col_)) ddd["tsums"] = [ int(line[1]), int(line[2]), int(line[3]), int(line[4]), int(line[5]), int(line[6]) ] delta = 0 if crop_id == "GM": delta = 1 ddd["tsums"].append(int(line[7])) ddd["CriticalTemperatureHeatStress"] = float(line[10]) ddd["BeginSensitivePhaseHeatStress"] = 0 ddd["EndSensitivePhaseHeatStress"] = 0 ddd["HeatSumIrrigationStart"] = float(line[delta + delta + 9]) ddd["HeatSumIrrigationEnd"] = float(line[delta + delta + 10]) rrr[(row, col)] = ddd return rrr calib = { "GM": read_calibrated_tsums("Calibrated_TSUM_Maize.csv", "GM"), "WW": read_calibrated_tsums("Calibrated_TSUM_WW.csv", "WW") } def update_soil_crop_dates(row, col, crop_id): "update function" sss = soil[(row, col)] ppp = pheno[crop_id][(row, col)] extended_harvest_doy = ppp["harvest-doy"] + 10 start_date = date(START_YEAR, 1, 1) sim["climate.csv-options"]["start-date"] = start_date.isoformat() end_date = date(2010, 12, 31) sim["climate.csv-options"]["end-date"] = end_date.isoformat() #sim["debug?"] = True pwp = sss["pwp"] fc_ = sss["fc"] sm_percent_fc = sss["sw-init"] / fc_ * 100.0 is_wintercrop = ppp["sowing-doy"] > ppp["harvest-doy"] seeding_date = date(START_YEAR, 1, 1) + timedelta(days=ppp["sowing-doy"]) crop["cropRotation"][0]["worksteps"][0][ "date"] = seeding_date.strftime("0000-%m-%d") crop["cropRotation"][0]["worksteps"][0][ "soilMoisturePercentFC"] = sm_percent_fc crop["cropRotation"][0]["worksteps"][1][ "date"] = seeding_date.strftime("0000-%m-%d") crop["cropRotation"][0]["worksteps"][1]["crop"][2] = crop_id harvest_date = date(START_YEAR + (1 if is_wintercrop else 0), 1, 1) + timedelta(days=extended_harvest_doy) #harvest_date = date(1980, 12, 31) if crop_id == "GM" else date(1980 + (1 if is_wintercrop else 0), 1, 1) + timedelta(days=ppp["harvest-doy"]) crop["cropRotation"][0]["worksteps"][2][ "date"] = harvest_date.strftime("000" + ("1" if is_wintercrop else "0") + "-%m-%d") site["Latitude"] = sss["latitude"] top = { "Thickness": [0.3, "m"], "SoilOrganicCarbon": [sss["oc-topsoil"], "%"], "SoilBulkDensity": [sss["bd-topsoil"] * 1000, "kg m-3"], "FieldCapacity": [fc_, "m3 m-3"], "PermanentWiltingPoint": [pwp, "m3 m-3"], "PoreVolume": [sss["sat"], "m3 m-3"], "SoilMoisturePercentFC": [sm_percent_fc, "% [0-100]"], "Sand": sss["sand-topsoil"] / 100.0, "Clay": sss["clay-topsoil"] / 100.0 } sub = { "Thickness": [1.7, "m"], "SoilOrganicCarbon": [sss["oc-subsoil"], "%"], "SoilBulkDensity": [sss["bd-subsoil"] * 1000, "kg m-3"], "FieldCapacity": [fc_, "m3 m-3"], "PermanentWiltingPoint": [pwp, "m3 m-3"], "PoreVolume": [sss["sat"], "m3 m-3"], "SoilMoisturePercentFC": [sm_percent_fc, "% [0-100]"], "Sand": sss["sand-subsoil"] / 100.0, "Clay": sss["clay-subsoil"] / 100.0 } site["SiteParameters"]["SoilProfileParameters"] = [top, sub] #print site["SiteParameters"]["SoilProfileParameters"] print "# of rowsCols = ", len(row_cols) i = 0 start_store = time.clock() start = config["start"] - 1 end = config["end"] - 1 row_cols_ = row_cols[start:end + 1] print "running from ", start, "/", row_cols[ start], " to ", end, "/", row_cols[end] for row, col in row_cols_: if soil[(row, col)]["bd-topsoil"] < 0.6: #avoid to simulate peat soils continue for crop_id in ["WW", "GM"]: update_soil_crop_dates(row, col, crop_id) env = monica_io.create_env_json_from_json_config({ "crop": crop, "site": site, "sim": sim, "climate": "" }) env["csvViaHeaderOptions"] = sim["climate.csv-options"] env["params"]["userEnvironmentParameters"]["AtmosphericCO2"] = 360 climate_filename = "{}_{:03d}_v1.csv".format(row, col) #read climate data on the server and send just the path to the climate data csv file env["pathToClimateCSV"] = PATH_TO_CLIMATE_DATA_SERVER + climate_filename cal = calib[crop_id][(row, col)] cultivar = env["cropRotation"][0]["worksteps"][1]["crop"][ "cropParams"]["cultivar"] cultivar["CropSpecificMaxRootingDepth"] = 1.5 cultivar["StageTemperatureSum"] = cal["tsums"] cultivar["BeginSensitivePhaseHeatStress"] = 0 cultivar["EndSensitivePhaseHeatStress"] = 0 cultivar["HeatSumIrrigationStart"] = cal["HeatSumIrrigationStart"] cultivar["HeatSumIrrigationEnd"] = cal["HeatSumIrrigationEnd"] env["customId"] = crop_id + "|(" + str(row) + "/" + str(col) + ")" socket.send_json(env) print "sent env ", i, " customId: ", env["customId"] i += 1 #break #break stop_store = time.clock() print "sending ", i, " envs took ", (stop_store - start_store), " seconds" print "ran from ", start, "/", row_cols[start], " to ", end, "/", row_cols[ end] return
def run_producer(setup, custom_crop, server={ "server": None, "port": None, "nd-port": None }, preprocessed_data=None): "main" config = { "user": USER, "port": server["port"] if server["port"] else "66663", "no-data-port": server["nd-port"] if server["nd-port"] else "5555", "server": server["server"] if server["server"] else "localhost" } if len(sys.argv) > 1: for arg in sys.argv[1:]: k, v = arg.split("=") if k in config: config[k] = v context = zmq.Context() socket = context.socket(zmq.PUSH) prod_cons_socket = context.socket(zmq.PUSH) soil_db_con = sqlite3.connect(paths["path-to-data-dir"] + "germany/buek1000.sqlite") #connect producer and consumer directly prod_cons_socket.bind("tcp://*:" + str(config["no-data-port"])) if LOCAL_RUN: socket.connect("tcp://localhost:" + str(config["port"])) else: socket.connect("tcp://" + config["server"] + ":" + str(config["port"])) with open("sim.json") as _: sim_json = json.load(_) sim_json["include-file-base-path"] = paths["include-file-base-path"] with open("site.json") as _: site_json = json.load(_) with open("crop.json") as _: crop_json = json.load(_) def get_value(list_or_value): return list_or_value[0] if isinstance(list_or_value, list) else list_or_value sent_env_count = 1 start_time = time.clock() if preprocessed_data == None: preprocessed_data = preprocess_data() dem_gk5_interpolate = preprocessed_data["dem_gk5_interpolate"] slope_gk5_interpolate = preprocessed_data["slope_gk5_interpolate"] soil_gk5_interpolate = preprocessed_data["soil_gk5_interpolate"] cdict = preprocessed_data["cdict"] climate_gk5_interpolate = preprocessed_data["climate_gk5_interpolate"] stat_infos = preprocessed_data["stat_infos"] #put custom crop in place for stat_info in stat_infos: for cm in stat_info["rotation"]: for ws in cm["worksteps"]: if "crop" in ws.keys(): ws["crop"] = custom_crop #send stations list to the consumer sim_stats = [] for stat_info in stat_infos: sim_stats.append(int(stat_info["station_id"])) prod_cons_socket.send_json(sim_stats) for stat_info in stat_infos: station_id = stat_info["station_id"] r_gk5, h_gk5 = stat_info["gk5_rh"] env_template = monica_io.create_env_json_from_json_config({ "crop": crop_json, "site": site_json, "sim": sim_json, "climate": "" }) env_template["cropRotation"] = stat_info["rotation"] env_template["events"] = stat_info["events"] soil_id = soil_gk5_interpolate(r_gk5, h_gk5) if soil_id == -9999: continue if soil_id < 1 or soil_id > 71: #print "row/col:", row, "/", col, "has unknown soil_id:", soil_id #unknown_soil_ids.add(soil_id) continue crow, ccol = climate_gk5_interpolate(r_gk5, h_gk5) height_nn = dem_gk5_interpolate(r_gk5, h_gk5) slope = slope_gk5_interpolate(r_gk5, h_gk5) clat, clon = cdict[(crow, ccol)] #slon, slat = transform(gk5, wgs84, r, h) #print "srow:", srow, "scol:", scol, "h:", h, "r:", r, " inter:", inter, "crow:", crow, "ccol:", ccol, "slat:", slat, "slon:", slon, "clat:", clat, "clon:", clon env_template["params"]["userCropParameters"][ "__enable_T_response_leaf_expansion__"] = setup[ "LeafExtensionModifier"] # set soil-profile sp_json = soil_io.soil_parameters(soil_db_con, soil_id) soil_profile = monica_io.find_and_replace_references(sp_json, sp_json)["result"] #print "soil:", soil_profile env_template["params"]["siteParameters"][ "SoilProfileParameters"] = soil_profile # setting groundwater level if setup["groundwater-level"]: groundwaterlevel = 20 layer_depth = 0 for layer in soil_profile: if layer.get("is_in_groundwater", False): groundwaterlevel = layer_depth #print "setting groundwaterlevel of soil_id:", str(soil_id), "to", groundwaterlevel, "m" break layer_depth += get_value(layer["Thickness"]) env_template["params"]["userEnvironmentParameters"][ "MinGroundwaterDepthMonth"] = 3 env_template["params"]["userEnvironmentParameters"][ "MinGroundwaterDepth"] = [max(0, groundwaterlevel - 0.2), "m"] env_template["params"]["userEnvironmentParameters"][ "MaxGroundwaterDepth"] = [groundwaterlevel + 0.2, "m"] # setting impenetrable layer if setup["impenetrable-layer"]: impenetrable_layer_depth = get_value( env_template["params"]["userEnvironmentParameters"] ["LeachingDepth"]) layer_depth = 0 for layer in soil_profile: if layer.get("is_impenetrable", False): impenetrable_layer_depth = layer_depth #print "setting leaching depth of soil_id:", str(soil_id), "to", impenetrable_layer_depth, "m" break layer_depth += get_value(layer["Thickness"]) env_template["params"]["userEnvironmentParameters"][ "LeachingDepth"] = [impenetrable_layer_depth, "m"] env_template["params"]["siteParameters"][ "ImpenetrableLayerDepth"] = [impenetrable_layer_depth, "m"] if setup["elevation"]: env_template["params"]["siteParameters"]["heightNN"] = height_nn if setup["slope"]: env_template["params"]["siteParameters"]["slope"] = slope / 100.0 if setup["latitude"]: env_template["params"]["siteParameters"]["Latitude"] = clat env_template["params"]["simulationParameters"][ "UseNMinMineralFertilisingMethod"] = setup["fertilization"] env_template["params"]["simulationParameters"][ "UseAutomaticIrrigation"] = setup["irrigation"] env_template["params"]["simulationParameters"][ "NitrogenResponseOn"] = setup["NitrogenResponseOn"] env_template["params"]["simulationParameters"][ "WaterDeficitResponseOn"] = setup["WaterDeficitResponseOn"] env_template["params"]["simulationParameters"][ "EmergenceMoistureControlOn"] = setup["EmergenceMoistureControlOn"] env_template["params"]["simulationParameters"][ "EmergenceFloodingControlOn"] = setup["EmergenceFloodingControlOn"] env_template["csvViaHeaderOptions"] = sim_json["climate.csv-options"] if LOCAL_RUN: env_template["pathToClimateCSV"] = paths[ "path-to-climate-csvs-dir"] + "row-" + str( crow) + "/col-" + str(ccol) + ".csv" else: env_template["pathToClimateCSV"] = paths[ "archive-path-to-climate-csvs-dir"] + "row-" + str( crow) + "/col-" + str(ccol) + ".csv" env_template["customId"] = {"station_id": int(station_id)} #with open("envs/env-"+str(sent_env_count)+".json", "w") as _: # _.write(json.dumps(env)) socket.send_json(env_template) print "sent env ", sent_env_count, " customId: ", env_template[ "customId"] #exit() sent_env_count += 1 stop_time = time.clock() print "sending ", (sent_env_count - 1), " envs took ", (stop_time - start_time), " seconds" #print "ran from ", start, "/", row_cols[start], " to ", end, "/", row_cols[end] print "exiting run_producer()"
def run_producer(): # some options and configurations create_simulation_files = False # calibration options training_mode = False calibrate_apache = False # simulation options activate_debug = False run_via_simulation_files = False output_dir = "runs/2018-07-16/" # calibration ------------------------------------- # original "StageTemperatureSum": [[148, 284, 380, 180, 420, 25 ], "\u00b0C d"] # overwrite some settings if creation of simulation files is active if create_simulation_files: run_via_simulation_files = False training_mode = False # technical initialisation config = {"user": "******", "port": "66666", "server": "localhost"} sent_id = 0 start_send = time.clock() socket = initialise_sockets(config) print(socket) # output file output_filename = "agmip_calibration_phase2_step1_evaluation.csv" if training_mode: output_filename = "agmip_calibration_phase2_step1_calibration.csv" paths = PATHS[config["user"]] # read in management information management_file = paths[ "INCLUDE_FILE_BASE_PATH"] + "monica_simulation_setup/input_data/cal2_phenology_mgt_soil_data.txt" management_df = pd.read_csv(management_file, sep='\t', header=0, index_col=None, keep_default_na=False, encoding="latin1") # iterate over every row/environment where row contains the setup of one simulation for environment in management_df.iterrows(): simulation_row = environment[1] sim_id = int(simulation_row["n"]) if training_mode: # calibration mode if simulation_row["Date_observee_Epi_1cm"] == "NA": print( "Skip evaluation simulation because testing mode is active" ) continue if calibrate_apache: if simulation_row["Variete"] != "Apache": # calibration mode active but sim_id refers to wrong cultivar continue else: # Bermude if simulation_row["Variete"] != "Bermude": continue # apache parameters stage1_sum = 148 stage2_sum = 101 # 340 stage3_sum = 429 # 407 if simulation_row["Variete"] == "Bermude": stage1_sum = 148 stage2_sum = 83 # 340 stage3_sum = 477 # 407 # ------------------------------------------------ # based on Christian Kersebaums assumptions tsum_bbch55 = (stage2_sum + stage3_sum) tsum_bbch30 = (stage3_sum) * 0.25 + stage2_sum print("Run simulation " + str(sim_id)) sim_parameters = None site_parameters = None crop_parameters = None if run_via_simulation_files: # run from existing simulation files print("Run from MONICA simulation files ...") simulation_dir = "monica_simulation_setup/calibration/" if simulation_row["Date_observee_Epi_1cm"] == "NA": simulation_dir = "monica_simulation_setup/evaluation/" with open(simulation_dir + "sim-" + str(sim_id) + ".json") as fp: sim_parameters = json.load(fp) with open(simulation_dir + "site-" + str(sim_id) + ".json") as fp: site_parameters = json.load(fp) with open(simulation_dir + "crop-" + str(sim_id) + ".json") as fp: crop_parameters = json.load(fp) else: # dynamically create simulation objects print( "Dynamic run by creating simulation objects directly from AgMIP management file." ) site_parameters = create_site_parameters(simulation_row) crop_parameters = create_crop_parameters(simulation_row) sim_parameters = create_sim_parameters( simulation_row, paths["INCLUDE_FILE_BASE_PATH"], sim_id, output_dir, activate_debug, create_simulation_files, tsum_bbch30, tsum_bbch55) if site_parameters is None: print("ERROR: site_parameters == None") if crop_parameters is None: print("ERROR: crop_parameters == None") if sim_parameters is None: print("ERROR: sim_parameters == None") # check if just MONICA simulation files should be created from the provided input files # do'nt send any information if input files should be created if create_simulation_files: dir = "monica_simulation_setup/calibration/" if simulation_row["Date_observee_Epi_1cm"] == "NA": dir = "monica_simulation_setup/evaluation/" with open(dir + "site-" + str(sim_id) + ".json", "w") as fp: json.dump(site_parameters, fp=fp, indent=4) with open(dir + "crop-" + str(sim_id) + ".json", "w") as fp: json.dump(crop_parameters, fp=fp, indent=4) with open(dir + "sim-" + str(sim_id) + ".json", "w") as fp: json.dump(sim_parameters, fp=fp, indent=4) continue env_map = { "crop": crop_parameters, "site": site_parameters, "sim": sim_parameters } env = monica_io.create_env_json_from_json_config(env_map) # calibration # for calibration overwrite stage temperature sum values env["cropRotation"][0]["worksteps"][0]["crop"]["cropParams"][ "cultivar"]["StageTemperatureSum"][0][1] = stage2_sum env["cropRotation"][0]["worksteps"][0]["crop"]["cropParams"][ "cultivar"]["StageTemperatureSum"][0][2] = stage3_sum env["cropRotation"][0]["worksteps"][0]["crop"]["cropParams"][ "cultivar"]["StageTemperatureSum"][0][0] = stage1_sum temp_sum = env["cropRotation"][0]["worksteps"][0]["crop"][ "cropParams"]["cultivar"]["StageTemperatureSum"][0] bbch30_tempsum = temp_sum[0] + temp_sum[1] + 0.25 * (temp_sum[3]) # final env object with all necessary information env["customId"] = { "id": sim_id, "calibration": training_mode, "sim_files": output_dir, "output_filename": output_filename, "cultivar": simulation_row["Variete"], "sowing_date": simulation_row["Date_Semis"], "bbch30": simulation_row["Date_observee_Epi_1cm"], # observation bbch 30 "bbch55": simulation_row["Date_observee_Epiaison"], # observation bbch 55 "site": simulation_row["Libelle"] } with open("monica_simulation_setup/env.json", "w") as fp: json.dump(env, fp=fp, indent=4) # sending env object to MONICA ZMQ print("sent env ", sent_id, " customId: ", env["customId"]) socket.send_json(env) sent_id += 1 stop_send = time.clock() # just tell the sending time if objects have really been sent if not create_simulation_files: print("sending ", sent_id, " envs took ", (stop_send - start_send), " seconds")
def producer(setup=None): "main function" paths = PATHS[USER] #Configure producer if setup == None: #playground run_id = "custom" PRODUCTION_LEVEL = 'WL.NL.rain' #options: "Pot", "WL.NL.rain" TF = "historical" FERT_STRATEGY = "BASE" #options: "NDEM", "NMIN", "BASE" COVER_CROP_FREQ = { #always use int for insert-cc-every and out-of #keep out-of as small as possible (to ensure uniform spatial distribution) "insert-cc-every": 1, #CM "out-of": 4, #CM "suffix": "25" #REMEMBER to set it (for output file name) } RESIDUES_EXPORTED = True EXPORT_RATE = "base" RESIDUES_HUMUS_BALANCE = False #mgt complying with humus balance approach of NRW: if true, RESIDUE_EXPORTED has no effect! else: run_id = setup["id"] PRODUCTION_LEVEL = setup["PRODUCTION_LEVEL"] TF = setup["TF"] FERT_STRATEGY = setup["FERT_STRATEGY"] COVER_CROP_FREQ = setup["COVER_CROP_FREQ"] if setup["res_mgt"] == "RESIDUES_HUMUS_BALANCE": RESIDUES_HUMUS_BALANCE = True RESIDUES_EXPORTED = False else: RESIDUES_HUMUS_BALANCE = False RESIDUES_EXPORTED = True EXPORT_RATE = setup["res_mgt"] HUMBAL_CORRECTION = setup["HUMBAL_CORRECTION"] #end of user configuration #assemble file name suffix for out suffix = "_id" + run_id + "_" suffix += TF + "_" suffix += "fert-" + FERT_STRATEGY.lower() + "_" if RESIDUES_HUMUS_BALANCE: suffix += "res-humbal_" else: suffix += "res-" + EXPORT_RATE + "_" suffix += "cc-" + COVER_CROP_FREQ["suffix"] + "_" suffix += "pl-" + PRODUCTION_LEVEL.replace(".", "") + "_" if FERT_STRATEGY == "NMIN": rotations_file = "rotations_dynamic_harv.json" elif FERT_STRATEGY == "NDEM": rotations_file = "rotations_dynamic_harv_Ndem.json" elif FERT_STRATEGY == "BASE": rotations_file = "rotations_dynamic_harv_Nbaseline.json" if LOCAL_RUN: PATH_TO_CLIMATE_DATA_DIR = timeframes[TF]["local-path-to-climate"] else: PATH_TO_CLIMATE_DATA_DIR = timeframes[TF]["cluster-path-to-climate"] context = zmq.Context() socket = context.socket(zmq.PUSH) port = 6666 if len(sys.argv) == 1 else sys.argv[1] if LOCAL_RUN: socket.connect("tcp://localhost:66663") else: socket.connect("tcp://cluster1:" + str(port)) soil_db_con = sqlite3.connect("soil.sqlite") with open("sim.json") as _: sim = json.load(_) sim["start-date"] = timeframes[TF]["start-date"] sim["end-date"] = timeframes[TF]["end-date"] with open("site.json") as _: site = json.load(_) with open("crop.json") as _: crop = json.load(_) with open("cover-crop.json") as _: cover_crop = json.load(_)["CM"] if RESIDUES_HUMUS_BALANCE: #inject additional harvest params for ws in range(len(cover_crop["worksteps"])): if cover_crop["worksteps"][ws]["type"] == "AutomaticHarvest": cover_crop["worksteps"][ws][ "opt-carbon-conservation"] = True cover_crop["worksteps"][ws][ "crop-impact-on-humus-balance"] = [ humus_equivalent["crop"]["CC"], "Humus equivalent [Heq]" ] cover_crop["worksteps"][ws]["residue-heq"] = [ humus_equivalent["material"]["green-manure"], "Heq ton-1 DM" ] cover_crop["worksteps"][ws][ "crop-usage"] = CROP_USAGE_HUMBAL["CC"] cover_crop["worksteps"][ws][ "exported"] = True #Needed to fire the humus balance approach; if true and crop-usage="green-manure" --> the crop is anyway returned to the soil with open("sims.json") as _: sims = json.load(_) if FERT_STRATEGY != "NMIN": #turn off Nmin automatic fertilization for setting in sims.iteritems(): setting[1]["UseNMinMineralFertilisingMethod"] = False with open(rotations_file) as _: rotations = json.load(_) #identify rotations with codes rots_info = {} for bkr, rots in rotations.iteritems(): for rot in rots.iteritems(): rot_code = int(rot[0]) my_rot = [] for cm in rot[1]: #["worksteps"]: for ws in range(len(cm["worksteps"])): if cm["worksteps"][ws]["type"] == "Sowing": my_rot.append(cm["worksteps"][ws]["crop"][2]) #for each crop, identify previous main one (needed to determine expected N availability) rot_info = [] for i in range(len(my_rot)): cm_info = {} current_cp = my_rot[i] if i != 0: previous_cp = my_rot[i - 1] else: previous_cp = my_rot[-1] cm_info["current"] = current_cp cm_info["previous"] = previous_cp rot_info.append(cm_info) rots_info[rot_code] = rot_info if RESIDUES_HUMUS_BALANCE: #inject additional harvest params for bkr, rots in rotations.iteritems(): for rot in rots.iteritems(): rot_code = int(rot[0]) rot_info = rots_info[rot_code] cp_index = 0 for cm in rot[1]: #["worksteps"]: for ws in range(len(cm["worksteps"])): cp = rot_info[cp_index]["current"] if cm["worksteps"][ws][ "type"] == "AutomaticHarvest": cm["worksteps"][ws][ "opt-carbon-conservation"] = True cm["worksteps"][ws][ "crop-impact-on-humus-balance"] = [ humus_equivalent["crop"][cp], "Humus equivalent [Heq]" ] if cp == "SBee": cm["worksteps"][ws]["residue-heq"] = [ humus_equivalent["material"] ["green-manure"], "Heq ton-1 DM" ] else: cm["worksteps"][ws]["residue-heq"] = [ humus_equivalent["material"]["straw"], "Heq ton-1 DM" ] cm["worksteps"][ws][ "organic-fertilizer-heq"] = [ humus_equivalent["material"] ["pig-slurry"], "Heq ton-1 DM" ] cm["worksteps"][ws][ "crop-usage"] = CROP_USAGE_HUMBAL[cp] cm["worksteps"][ws][ "max-residue-recover-fraction"] = CROP_USAGE_HUMBAL[ "max-residue-recover-fraction"] cp_index += 1 if FERT_STRATEGY == "BASE": #read additional info required for baseline fert strategy: #1. expected mineral N availability expected_N_availability = defaultdict(lambda: defaultdict()) with open("expected_N_availability.csv") as _: reader = csv.reader(_) reader.next() for row in reader: cp_sequence = (row[0], row[1]) soil_type = row[2] expected_N_value = float(row[3]) expected_N_availability[cp_sequence][ soil_type] = expected_N_value expected_N_availability[cp_sequence]["target_depth"] = float( row[4]) #2. rules to split mineral fertilization mineralN_split = defaultdict(lambda: defaultdict()) with open("MineralN_topdressing.csv") as _: reader = csv.reader(_) reader.next() for row in reader: cp = row[0] mineralN_split[cp]["target"] = float(row[2]) for i in range(4, 7): if row[i] != "": mineralN_split[cp][i - 4] = float(row[i]) sim["UseSecondaryYields"] = RESIDUES_EXPORTED if RESIDUES_EXPORTED: for cp in crop["crops"].iteritems(): for k in EXPORT_PRESETS[EXPORT_RATE].keys(): if cp[0] in k: my_rate = EXPORT_PRESETS[EXPORT_RATE][k] for organ in cp[1]["cropParams"]["cultivar"][ "OrganIdsForSecondaryYield"]: organ["yieldPercentage"] *= my_rate sim["include-file-base-path"] = paths["INCLUDE_FILE_BASE_PATH"] def read_general_metadata(path_to_file): "read metadata file" with open(path_to_file) as file_: data = {} reader = csv.reader(file_, delimiter="\t") reader.next() for row in reader: if int(row[1]) != 1: continue data[(int(row[2]), int(row[3]))] = { "subpath-climate.csv": row[9], "latitude": float(row[13]), "elevation": float(row[14]) } return data general_metadata = read_general_metadata("NRW_General_Metadata.csv") def load_mapping(row_offset=0, col_offset=0): to_climate_index = {} with (open("working_resolution_to_climate_lat_lon_indices.json")) as _: l = json.load(_) for i in xrange(0, len(l), 2): cell = (row_offset + l[i][0], col_offset + l[i][1]) to_climate_index[cell] = tuple(l[i + 1]) return to_climate_index def read_orgN_kreise(path_to_file): "read organic N info for kreise" with open(path_to_file) as file_: data = {} reader = csv.reader(file_, delimiter=",") reader.next() reader.next() for row in reader: for kreis_code in row[1].split("|"): if kreis_code != "": data[int(kreis_code)] = float(row[8]) return data orgN_kreise = read_orgN_kreise("NRW_orgN_balance.csv") def update_soil(row, col): "in place update the env" site["SiteParameters"]["Latitude"] = general_metadata[( row, col)]["latitude"] site["SiteParameters"]["HeightNN"] = [ general_metadata[(row, col)]["elevation"], "m" ] site["SiteParameters"][ "SoilProfileParameters"] = soil_io.soil_parameters( soil_db_con, soil_ids[(row, col)]) KA5_txt = soil_io.sand_and_clay_to_ka5_texture( site["SiteParameters"]["SoilProfileParameters"][0]["Sand"][0], site["SiteParameters"]["SoilProfileParameters"][0]["Clay"][0]) for layer in site["SiteParameters"]["SoilProfileParameters"]: layer["SoilBulkDensity"][0] = max(layer["SoilBulkDensity"][0], 600) layer["SoilOrganicCarbon"][0] = layer["SoilOrganicCarbon"][ 0] * 0.6 #correction factor suggested by TGaiser return KA5_txt def read_header(path_to_ascii_grid_file): "read metadata from esri ascii grid file" metadata = {} header_str = "" with open(path_to_ascii_grid_file) as _: for i in range(0, 6): line = _.readline() header_str += line sline = [x for x in line.split() if len(x) > 0] if len(sline) > 1: metadata[sline[0].strip().lower()] = float( sline[1].strip()) return metadata, header_str def read_ascii_grid(path_to_file, include_no_data=False, row_offset=0, col_offset=0): "read an ascii grid into a map, without the no-data values" def int_or_float(s): try: return int(s) except ValueError: return float(s) with open(path_to_file) as file_: data = {} #skip the header (first 6 lines) for _ in range(0, 6): file_.next() row = -1 for line in file_: row += 1 col = -1 for col_str in line.strip().split(" "): col += 1 if not include_no_data and int_or_float(col_str) == -9999: continue data[(row_offset + row, col_offset + col)] = int_or_float(col_str) return data #offset is used to match info in general metadata and soil database soil_ids = read_ascii_grid("soil-profile-id_nrw_gk3.asc", row_offset=282) bkr_ids = read_ascii_grid("bkr_nrw_gk3.asc", row_offset=282) lu_ids = read_ascii_grid("lu_resampled.asc", row_offset=282) kreise_ids = read_ascii_grid("kreise_matrix.asc", row_offset=282) meteo_ids = load_mapping(row_offset=282) soil_metadata, _ = read_header("soil-profile-id_nrw_gk3.asc") wgs84 = Proj(init="epsg:4326") gk3 = Proj(init="epsg:3396") gk5 = Proj(init="epsg:31469") def create_ascii_grid_interpolator(arr, meta, ignore_nodata=True): "read an ascii grid into a map, without the no-data values" rows, cols = arr.shape cellsize = int(meta["cellsize"]) xll = int(meta["xllcorner"]) yll = int(meta["yllcorner"]) nodata_value = meta["nodata_value"] xll_center = xll + cellsize // 2 yll_center = yll + cellsize // 2 yul_center = yll_center + (rows - 1) * cellsize points = [] values = [] for row in range(rows): for col in range(cols): value = arr[row, col] if ignore_nodata and value == nodata_value: continue r = xll_center + col * cellsize h = yul_center - row * cellsize points.append([r, h]) values.append(value) return NearestNDInterpolator(np.array(points), np.array(values)) path_to_slope_grid = paths[ "path-to-data-dir"] + "/germany/slope_1000_gk5.asc" slope_metadata, _ = read_header(path_to_slope_grid) slope_grid = np.loadtxt(path_to_slope_grid, dtype=float, skiprows=6) slope_gk5_interpolate = create_ascii_grid_interpolator( slope_grid, slope_metadata) #counter = 0 #for k, v in bkr_ids.iteritems(): # if v == 134: # if k in lu_ids: # counter += 1 #print counter def rotate(crop_rotation): "rotate the crops in the rotation" crop_rotation.insert(0, crop_rotation.pop()) def insert_cc(crop_rotation, cc_data): "insert cover crops in the rotation" insert_cover_here = [] for cultivation_method in range(len(crop_rotation)): for workstep in crop_rotation[cultivation_method]["worksteps"]: if workstep["type"] == "Sowing": if workstep["crop"][2] in COVER_BEFORE: insert_cover_here.append( (cultivation_method, workstep["date"])) break for position, mydate in reversed(insert_cover_here): mydate = mydate.split("-") main_crop_sowing = date(2017, int(mydate[1]), int(mydate[2])) latest_harvest_cc = main_crop_sowing - timedelta(days=10) latest_harvest_cc = unicode("0001-" + str(latest_harvest_cc.month).zfill(2) + "-" + str(latest_harvest_cc.day).zfill(2)) crop_rotation.insert(position, copy.deepcopy(cc_data)) crop_rotation[position]["worksteps"][1][ "latest-date"] = latest_harvest_cc def calculate_orgfert_amount(N_applied, fert_type, soilCN=10): "convert N applied in amount of fresh org fert" AOM_DryMatterContent = fert_type["AOM_DryMatterContent"][0] AOM_NH4Content = fert_type["AOM_NH4Content"][0] AOM_NO3Content = fert_type["AOM_NO3Content"][0] CN_Ratio_AOM_Fast = fert_type["CN_Ratio_AOM_Fast"][0] CN_Ratio_AOM_Slow = fert_type["CN_Ratio_AOM_Slow"][0] PartAOM_to_AOM_Fast = fert_type["PartAOM_to_AOM_Fast"][0] PartAOM_to_AOM_Slow = fert_type["PartAOM_to_AOM_Slow"][0] AOM_to_C = 0.45 AOM_fast_factor = (AOM_to_C * PartAOM_to_AOM_Fast) / CN_Ratio_AOM_Fast AOM_slow_factor = (AOM_to_C * PartAOM_to_AOM_Slow) / CN_Ratio_AOM_Slow AOM_SOM_factor = ( 1 - (PartAOM_to_AOM_Fast + PartAOM_to_AOM_Slow)) * AOM_to_C / soilCN conversion_coeff = AOM_NH4Content + AOM_NO3Content + AOM_fast_factor + AOM_slow_factor + AOM_SOM_factor AOM_dry = N_applied / conversion_coeff AOM_fresh = AOM_dry / AOM_DryMatterContent return AOM_fresh def update_fert_values(rotation, rot_info, cc_in_cm, expected_N_availability, mineralN_split, soil_type, orgN_applied): "function to mimic baseline fertilization" cow_unit = 100 GVs = orgN_applied / cow_unit orgN_effect = GVs * 10 #insert cc in rotation info for cm in reversed(range(len(rot_info))): rot_info[cm]["has_cover_before"] = False if rot_info[cm]["current"] in COVER_BEFORE and cc_in_cm: rot_info[cm]["has_cover_before"] = True cc_info = {"current": "CC"} rot_info.insert(cm, cc_info) for cm in range(len(rotation)): if rot_info[cm]["current"] == "CC": #cover crops do not receive any fertilization continue current_cp = rot_info[cm]["current"] previous_cp = rot_info[cm]["previous"] has_cover = rot_info[cm]["has_cover_before"] N_target = mineralN_split[current_cp]["target"] expected_Nmin = expected_N_availability[(current_cp, previous_cp)][soil_type] #modify expected Nmin depending on livestock pressure and presence of cover crop if has_cover: expected_Nmin += 20 expected_Nmin += orgN_effect #calculate N to be applied with mineral fertilization sum_Nfert = max(N_target - expected_Nmin, 0) #map the fertilization worksteps ref_fert = 0 for ws in range(len(rotation[cm]["worksteps"])): workstep = rotation[cm]["worksteps"][ws] if workstep["type"] == "MineralFertilization" and workstep[ "amount"][0] == 0: workstep["amount"][ 0] = sum_Nfert * mineralN_split[current_cp][ref_fert] ref_fert += 1 sent_id = 0 start_send = time.clock() simulated_cells = 0 no_kreis = 0 #bkr2lk = defaultdict(set) #for additional info #soilty2iniSOC = defaultdict(list) #for additional info export_lat_lon_coords = False export_lat_lon_file = None srows = int(soil_metadata["nrows"]) scellsize = int(soil_metadata["cellsize"]) sxll = int(soil_metadata["xllcorner"]) syll = int(soil_metadata["yllcorner"]) sxll_center = sxll + scellsize // 2 syll_center = syll + scellsize // 2 syul_center = syll_center + (srows - 1) * scellsize for (row, col), gmd in general_metadata.iteritems(): #test #if int(row) != 505 or int(col) != 58: # continue if (row, col) in soil_ids and (row, col) in bkr_ids and (row, col) in lu_ids: # get gk3 coordinates for soil row/col sr_gk3 = sxll_center + col * scellsize sh_gk3 = syul_center - row * scellsize if export_lat_lon_coords: if not export_lat_lon_file: export_lat_lon_file = open( "soil_row_col_to_lat_lon_coords.csv", "w") export_lat_lon_file.write("row,col,lat,lon\n") slon, slat = transform(gk3, wgs84, sr_gk3, sh_gk3) export_lat_lon_file.write( ",".join(map(str, [row, col, slat, slon])) + "\n") sr_gk5, sh_gk5 = transform(gk3, gk5, sr_gk3, sh_gk3) site["SiteParameters"]["Slope"] = slope_gk5_interpolate( sr_gk5, sh_gk5) / 100.0 #continue bkr_id = bkr_ids[(row, col)] ########for testing #if bkr_id != 129: # continue soil_id = soil_ids[(row, col)] meteo_id = meteo_ids[(row, col)] if (row, col) in kreise_ids: kreis_id = kreise_ids[(row, col)] #bkr2lk[bkr_id].add(kreis_id) else: no_kreis += 1 print "-----------------------------------------------------" print "kreis not found for calculation of organic N" print "-----------------------------------------------------" simulated_cells += 1 KA5_txt = update_soil(row, col) light_soils = ["Ss", "Su2", "Su3", "Su4", "St2", "Sl3", "Sl2"] heavy_soils = ["Tu3", "Tu4", "Lt3", "Ts2", "Tl", "Tu2", "Tt"] soil_type = "medium" if KA5_txt in light_soils: soil_type = "light" elif KA5_txt in heavy_soils: soil_type = "heavy" site["SiteParameters"][ "SoilSpecificHumusBalanceCorrection"] = HUMBAL_CORRECTION[ soil_type] #soilty2iniSOC[soil_type].append(site["SiteParameters"]["SoilProfileParameters"][0]["SoilOrganicCarbon"][0]) #continue #row_col = "{}{:03d}".format(row, col) #topsoil_carbon[row_col] = site["SiteParameters"]["SoilProfileParameters"][0]["SoilOrganicCarbon"][0] #continue for rot_id, rotation in rotations[str(bkr_id)].iteritems(): ########for testing #if rot_id not in ["9120", "7120", "7130", "8120", "6110", "6120", "5120", "1110", "1130", "3110", "3130", "2110", "2120", "2130", "4110", "4120"]: # continue #extend rotation ext_rot = [] for i in range(COVER_CROP_FREQ["out-of"]): ext_rot.append(copy.deepcopy(rotation)) #insert CC in a subset of CM cc_in_cm = {} for cm in range(len(ext_rot)): cc_in_cm[cm] = False if (cm + 1) <= COVER_CROP_FREQ["insert-cc-every"]: insert_cc(ext_rot[cm], cover_crop) cc_in_cm[cm] = True #update mineral fert (baseline N scenario) if FERT_STRATEGY == "BASE": for rot in range(len(ext_rot)): update_fert_values( ext_rot[rot], copy.deepcopy(rots_info[int(rot_id)]), cc_in_cm[rot], expected_N_availability, mineralN_split, soil_type, orgN_kreise[kreis_id]) #compose the rotation composed_rot = [] for rot in ext_rot: for cm in rot: composed_rot.append(cm) crop["cropRotation"] = composed_rot env = monica_io.create_env_json_from_json_config({ "crop": crop, "site": site, "sim": sim, "climate": "" }) env["sharedId"] = "ts_sustag_nrw" #assign amount of organic fertilizer for cultivation_method in env["cropRotation"]: for workstep in cultivation_method["worksteps"]: if workstep["type"] == "OrganicFertilization": workstep["amount"][0] = calculate_orgfert_amount( orgN_kreise[kreis_id], workstep["parameters"] ) #TODO: assign soilCN param dynamically #with open("test_crop.json", "w") as _: # _.write(json.dumps(crop, indent=4)) #climate is read by the server env["csvViaHeaderOptions"] = sim["climate.csv-options"] env["csvViaHeaderOptions"]["start-date"] = sim["start-date"] env["csvViaHeaderOptions"]["end-date"] = sim["end-date"] env["pathToClimateCSV"] = [] for PATH in PATH_TO_CLIMATE_DATA_DIR: env["pathToClimateCSV"].append(PATH + "row-" + str(meteo_id[0]) + "/col-" + str(meteo_id[1]) + ".csv") for sim_id, sim_ in sims.iteritems(): if sim_id != PRODUCTION_LEVEL: continue env["events"] = sim_["output"] env["params"]["simulationParameters"][ "NitrogenResponseOn"] = sim_["NitrogenResponseOn"] env["params"]["simulationParameters"][ "WaterDeficitResponseOn"] = sim_[ "WaterDeficitResponseOn"] env["params"]["simulationParameters"][ "UseAutomaticIrrigation"] = sim_[ "UseAutomaticIrrigation"] env["params"]["simulationParameters"][ "UseNMinMineralFertilisingMethod"] = sim_[ "UseNMinMineralFertilisingMethod"] env["params"]["simulationParameters"][ "FrostKillOn"] = sim_["FrostKillOn"] for main_cp_iteration in range( 0, len(rots_info[int(rot_id)])): #do not allow crop rotation to start with a CC if "is-cover-crop" in env["cropRotation"][0].keys( ) and env["cropRotation"][0]["is-cover-crop"] == True: rotate(env["cropRotation"]) env["customId"] = rot_id \ + "|" + sim_id \ + "|" + str(soil_id) \ + "|(" + str(row) + "/" + str(col) + ")" \ + "|" + str(bkr_id) \ + "|" + str(main_cp_iteration) \ + "|" + str(sim["UseSecondaryYields"]) \ + "|" + str(timeframes[TF]["start-recording-out"]) \ + "|" + str(RESIDUES_HUMUS_BALANCE) \ + "|" + suffix \ + "|" + KA5_txt \ + "|" + soil_type socket.send_json(env) print "sent env ", sent_id, "customId:", env[ "customId"], "shared_id:", env["sharedId"] sent_id += 1 rotate(env["cropRotation"]) if export_lat_lon_file: export_lat_lon_file.close() stop_send = time.clock() print "sending ", sent_id, " envs took ", (stop_send - start_send), " seconds" print "simulated cells: ", simulated_cells, "; not found kreise for org N: ", no_kreis
def main(): "main" context = zmq.Context() socket = context.socket(zmq.PUSH) #port = 6666 if len(sys.argv) == 1 else sys.argv[1] config = {"port": 16666, "start": 1, "end": 8157} if len(sys.argv) > 1: for arg in sys.argv[1:]: k, v = arg.split("=") if k in config: config[k] = int(v) soil_db_con = sqlite3.connect(PATHS[USER]["PATH_TO_SOIL_DIR"] + "soil.sqlite") if LOCAL_RUN: socket.connect("tcp://localhost:6666") # + str(config["port"])) else: socket.connect("tcp://cluster2:" + str(config["port"])) with open("sim.json") as _: sim = json.load(_) with open("site.json") as _: site = json.load(_) with open("crop.json") as _: crop = json.load(_) #sim["include-file-base-path"] = PATHS[USER]["INCLUDE_FILE_BASE_PATH"] crows = 938 ccols = 720 xll = 5137800 yll = 5562800 cellsize = 100 scols = 3653 srows = 5001 wgs84 = Proj(init="epsg:4326") #gk3 = Proj(init="epsg:31467") gk5 = Proj(init="epsg:31469") #cdict = {} def create_interpolator(path_to_file, wgs84, gk5): "read an ascii grid into a map, without the no-data values" with open(path_to_file) as file_: # skip headerlines file_.next() file_.next() nrows = 938 ncols = 720 points = np.zeros((ccols * crows, 2), np.int32) values = np.zeros((ccols * crows), np.int32) i = -1 row = -1 for line in file_: row += 1 col = -1 for col_str in line.strip().split(" "): col += 1 i += 1 clat, clon = col_str.split("|") #cdict[(row, col)] = (clat, clon) r, h = transform(wgs84, gk5, clon, clat) points[i, 0] = h points[i, 1] = r values[i] = 1000 * row + col #print "row:", row, "col:", col, "clat:", clat, "clon:", clon, "h:", h, "r:", r, "val:", values[i] return NearestNDInterpolator(points, values) interpol = create_interpolator( PATHS[USER]["PATH_TO_CLIMATE_CSVS_DIR"] + "germany-lat-lon-coordinates.grid", wgs84, gk5) soil_profiles = {} envs = {} start = time.clock() i = 1 srow = -1 with open(PATHS[USER]["PATH_TO_SOIL_DIR"] + "buek1000_100_gk5.asc") as _: for i in range(0, 6): _.next() for line in _: srow += 1 scol = -1 for col_str in line.strip().split(" "): scol += 1 soil_id = int(col_str) if soil_id == -9999: continue if soil_id not in envs: site["SiteParameters"][ "SoilProfileParameters"] = soil_io.soil_parameters( soil_db_con, soil_id) envs[soil_id] = monica_io.create_env_json_from_json_config( { "crop": crop, "site": site, "sim": sim, "climate": "" }) envs[soil_id]["csvViaHeaderOptions"] = sim[ "climate.csv-options"] env = envs[soil_id] sh = yll + (cellsize / 2) + (srows - srow) * cellsize sr = xll + (cellsize / 2) + scol * cellsize inter = interpol(sh, sr) crow = int(inter / 1000) ccol = inter - (crow * 1000) #clat, clon = cdict[(crow, ccol)] #slon, slat = transform(gk5, wgs84, r, h) #print "srow:", srow, "scol:", scol, "h:", sh, "r:", sr, " inter:", inter, "crow:", crow, "ccol:", ccol, "slat:", slat, "slon:", slon, "clat:", clat, "clon:", clon if LOCAL_RUN: env["pathToClimateCSV"] = PATHS[USER][ "PATH_TO_CLIMATE_CSVS_DIR"] + "germany/row-" + str( crow) + "/col-" + str(ccol) + ".csv" env["pathToClimateCSV"] = PATHS[USER][ "PATH_TO_CLIMATE_CSVS_DIR"] + "germany/row-851/col-543.csv" else: env["pathToClimateCSV"] = PATHS[USER][ "ARCHIVE_PATH_TO_CLIMATE_CSVS_DIR"] + "germany/row-" + str( crow) + "/col-" + str(ccol) + ".csv" env["pathToClimateCSV"] = PATHS[USER][ "ARCHIVE_PATH_TO_CLIMATE_CSVS_DIR"] + "germany/row-851/col-543.csv" #print env["pathToClimateCSV"] env["customId"] = "(" + str(crow) + "/" + str(ccol) + ")" \ + "|(" + str(srow) + "/" + str(scol) + ")" #with open("envs/env-"+str(i)+".json", "w") as _: # _.write(json.dumps(env)) socket.send_json(env) print "sent env ", i, " customId: ", env["customId"] #if i > 100: # exit() i += 1 stop = time.clock() print "sending ", (i - 1), " envs took ", (stop - start), " seconds" #print "ran from ", start, "/", row_cols[start], " to ", end, "/", row_cols[end] return