def generate_and_send_env(station, soil_type, rot_id, climate, rcp, realization, counter): def limit_rootdepth(): for cultivation_method in env["cropRotation"]: for workstep in cultivation_method["worksteps"]: if workstep["type"] == "Seed": current_rootdepth = float( workstep["crop"]["cropParams"]["cultivar"] ["CropSpecificMaxRootingDepth"]) workstep["crop"]["cropParams"]["cultivar"][ "CropSpecificMaxRootingDepth"] = min( current_rootdepth, 0.8) break crop = copy.deepcopy(crop_template) crop["cropRotation"] += rotations[rot_id] site = sites[soil_type] site["SiteParameters"]["Latitude"] = site_parameters[station][ "Latitude"] site["SiteParameters"]["NDeposition"] = site_parameters[station][ "NDeposition"] env = monica_io3.create_env_json_from_json_config({ "crop": crop, "site": site, "sim": sim }) env["csvViaHeaderOptions"] = sim["climate.csv-options"] if soil_type == "poor_soil": limit_rootdepth() weather_file_name = station + "~" + climate + "_" + rcp + "_30" + "~lq~trans" weather_file_name += "-" + str(realization).zfill(2) + ".csv" #weather_file_name = station + "-" + climate #if climate != "now" and climate != "naw": # weather_file_name += "-RCP85" #weather_file_name += "_" + str(realization).zfill(2) + ".csv" print(weather_file_name) if LOCAL_RUN: env["pathToClimateCSV"] = PATHS[USER][ "LOCAL_ARCHIVE_PATH_TO_PROJECT"] + "converted/no_snow_cover_assumed/" + weather_file_name else: env["pathToClimateCSV"] = PATHS[USER][ "ARCHIVE_PATH_TO_PROJECT"] + "converted/no_snow_cover_assumed/" + weather_file_name ## currently its the same folder... env["customId"] = station + "|" + soil + "|" + rotation_id + "|" + climate + "|" + rcp + "|" + str( realization).zfill(2) socket.send_json(env) print(("sent env ", counter, " customId: ", env["customId"])) return counter + 1
def run_producer(server = {"server": None, "port": None}, shared_id = None): context = zmq.Context() socket = context.socket(zmq.PUSH) config = { "port": server["port"] if server["port"] else "6666", "server": server["server"] if server["server"] else "localhost", "sim.json": os.path.join(os.path.dirname(__file__), '../sim-min.json'), "crop.json": os.path.join(os.path.dirname(__file__), '../crop-min.json'), "site.json": os.path.join(os.path.dirname(__file__), '../site-min.json'), "climate.csv": os.path.join(os.path.dirname(__file__), '../climate-min.csv'), "shared_id": shared_id } # read commandline args only if script is invoked directly from commandline if len(sys.argv) > 1 and __name__ == "__main__": for arg in sys.argv[1:]: k, v = arg.split("=") if k in config: config[k] = v print("config:", config) socket.connect("tcp://" + config["server"] + ":" + config["port"]) with open(config["sim.json"]) as _: sim_json = json.load(_) with open(config["site.json"]) as _: site_json = json.load(_) with open(config["crop.json"]) as _: crop_json = json.load(_) #with open(config["climate.csv"]) as _: # climate_csv = _.read() env = monica_io3.create_env_json_from_json_config({ "crop": crop_json, "site": site_json, "sim": sim_json, "climate": "" #climate_csv }) env["csvViaHeaderOptions"] = sim_json["climate.csv-options"] env["pathToClimateCSV"] = config["climate.csv"] #print(env) # add shared ID if env to be sent to routable monicas if config["shared_id"]: env["sharedId"] = config["shared_id"] socket.send_json(env) print("done")
def monica_env(): with open(PATH_TO_REPO / "data/monica/sim-min.json") as _: sim_json = json.load(_) with open(PATH_TO_REPO / "data/monica/site-min.json") as _: site_json = json.load(_) with open(PATH_TO_REPO / "data/monica/crop-min.json") as _: crop_json = json.load(_) env = monica_io3.create_env_json_from_json_config({ "crop": crop_json, "site": site_json, "sim": sim_json, "climate": "" #climate_csv }) return env
def run_producer(config): "main" if not config["server"]: config["server"] = server[config["mode"]] def rotate(crop_rotation): "rotate the crops in the rotation" crop_rotation.insert(0, crop_rotation.pop()) print("config:", config) context = zmq.Context() socket = context.socket(zmq.PUSH) # pylint: disable=no-member # select paths paths = PATHS[config["mode"]] # connect to monica proxy (if local, it will try to connect to a locally started monica) socket.connect("tcp://" + config["server"] + ":" + str(config["server-port"])) template_folder = script_path + "/json_templates/" with open(template_folder + "sim.json") as _: sim = json.load(_) sim["include-file-base-path"] = paths["monica-parameters-path"] if USER_MODE == "localProducer-localMonica": sim["climate.csv-options"]["no-of-climate-file-header-lines"] = 1 elif USER_MODE == "localProducer-remoteMonica": sim["climate.csv-options"]["no-of-climate-file-header-lines"] = 2 elif USER_MODE == "remoteProducer-remoteMonica": sim["climate.csv-options"]["no-of-climate-file-header-lines"] = 2 with open(template_folder + "site.json") as _: site = json.load(_) with open(template_folder + "crop.json") as _: crop = json.load(_) with open(template_folder + "sims.json") as _: sims = json.load(_) with open(template_folder + "irrigations.json") as _: irrigation = json.load(_) period_gcm_co2s = [ { "id": "C1", "period": "0", "gcm": "0_0", "co2_value": 360 }, { "id": "C26", "period": "2", "gcm": "GFDL-CM3_45", "co2_value": 499 }, #{"id": "C28", "period": "2", "gcm": "GFDL-CM3_85", "co2_value": 571}, { "id": "C30", "period": "2", "gcm": "GISS-E2-R_45", "co2_value": 499 }, #{"id": "C32", "period": "2", "gcm": "GISS-E2-R_85", "co2_value": 571}, { "id": "C36", "period": "2", "gcm": "HadGEM2-ES_45", "co2_value": 499 }, #{"id": "C38", "period": "2", "gcm": "HadGEM2-ES_85", "co2_value": 571}, { "id": "C40", "period": "2", "gcm": "MIROC5_45", "co2_value": 499 }, #{"id": "C42", "period": "2", "gcm": "MIROC5_85", "co2_value": 571}, { "id": "C46", "period": "2", "gcm": "MPI-ESM-MR_45", "co2_value": 499 }, #{"id": "C48", "period": "2", "gcm": "MPI-ESM-MR_85", "co2_value": 571} ] soil = {} row_cols = [] with open( os.path.join(paths["monica-project-data"], "stu_eu_layer_ref.csv")) as _: reader = csv.reader(_) next(reader) for row in reader: #Column_,Row,Grid_Code,Location,CLocation,elevation,latitude,longitude,depth,OC_topsoil,OC_subsoil,BD_topsoil,BD_subsoil,Sand_topsoil,Clay_topsoil,Silt_topsoil,Sand_subsoil,Clay_subsoil,Silt_subsoil #soil_ref,CLocation,latitude,depth,OC_topsoil,OC_subsoil,BD_topsoil,BD_subsoil,Sand_topsoil,Clay_topsoil,Silt_topsoil,Sand_subsoil,Clay_subsoil,Silt_subsoil soil_ref = row[0] #row_col = (int(row[1]), int(row[0])) row_cols.append(soil_ref) soil[soil_ref] = { "climate_location": row[1], "latitude": float(row[2]), "depth": float(row[3]), "oc-topsoil": float(row[4]), "oc-subsoil": float(row[5]), "bd-topsoil": float(row[6]), "bd-subsoil": float(row[7]), "sand-topsoil": float(row[8]), "clay-topsoil": float(row[9]), "silt-topsoil": float(row[10]), "sand-subsoil": float(row[11]), "clay-subsoil": float(row[12]), "silt-subsoil": float(row[13]), } def get_custom_site(soil_ref): "update function" cell_soil = soil[soil_ref] # pwp = cell_soil["pwp"] # fc_ = cell_soil["fc"] # sm_percent_fc = cell_soil["sw-init"] / fc_ * 100.0 top = { "Thickness": [0.3, "m"], "SoilOrganicCarbon": [cell_soil["oc-topsoil"], "%"], "SoilBulkDensity": [cell_soil["bd-topsoil"], "kg m-3"], #"FieldCapacity": [fc_, "m3 m-3"], #"PermanentWiltingPoint": [pwp, "m3 m-3"], #"PoreVolume": [cell_soil["sat"], "m3 m-3"], #"SoilMoisturePercentFC": [sm_percent_fc, "% [0-100]"], "Sand": cell_soil["sand-topsoil"] / 100.0, "Clay": cell_soil["clay-topsoil"] / 100.0, "Silt": cell_soil["silt-topsoil"] / 100.0 } sub = { "Thickness": [1.7, "m"], "SoilOrganicCarbon": [cell_soil["oc-subsoil"], "%"], "SoilBulkDensity": [cell_soil["bd-subsoil"], "kg m-3"], #"FieldCapacity": [fc_, "m3 m-3"], #"PermanentWiltingPoint": [pwp, "m3 m-3"], #"PoreVolume": [cell_soil["sat"], "m3 m-3"], #"SoilMoisturePercentFC": [sm_percent_fc, "% [0-100]"], "Sand": cell_soil["sand-subsoil"] / 100.0, "Clay": cell_soil["clay-subsoil"] / 100.0, "Silt": cell_soil["silt-subsoil"] / 100.0 } custom_site = { "soil-profile": [top, sub], "latitude": cell_soil["latitude"], #"sw-init": cell_soil["sw-init"], } return custom_site #assert len(row_cols) == len(pheno["GM"].keys()) == len(pheno["WW"].keys()) print("# of rowsCols = ", len(row_cols)) i = 0 start_store = time.process_time() start = config["start-row"] - 1 end = config["end-row"] - 1 if end < 0: row_cols_ = row_cols[start:] else: row_cols_ = row_cols[start:end + 1] #row_cols_ = [(108,106), (89,82), (71,89), (58,57), (77,109), (66,117), (46,151), (101,139), (116,78), (144,123)] #row_cols_ = ["64500"] #row_cols_ = [(3015,2836)] # 2836,3015 print("running from ", start, "/", row_cols[start], " to ", end, "/", row_cols[end]) run_periods = list(map(str, json.loads(config["run-periods"]))) for soil_ref in row_cols_: custom_site = get_custom_site(soil_ref) site["SiteParameters"]["Latitude"] = custom_site["latitude"] site["SiteParameters"]["SoilProfileParameters"] = custom_site[ "soil-profile"] for crop_id in crop["soybean"].keys(): #if crop_id not in ["0000", "II"]: # continue for ws in crop["cropRotation"][0]["worksteps"]: if ws["type"] == "AutomaticSowing": #set crop ref ws["crop"][2] = crop_id #if ws["type"] == "SetValue": # #set mois # ws["value"] = custom_site["sw-init"] #force max rooting depth #site["SiteParameters"]["ImpenetrableLayerDepth"] = crop["crops"][crop_id]["cropParams"]["cultivar"]["CropSpecificMaxRootingDepth"] env = monica_io3.create_env_json_from_json_config({ "crop": crop, "site": site, "sim": sim, "climate": "" }) env["csvViaHeaderOptions"] = sim["climate.csv-options"] for pgc in period_gcm_co2s: co2_id = pgc["id"] co2_value = pgc["co2_value"] period = pgc["period"] gcm = pgc["gcm"] if period not in run_periods: continue env["params"]["userEnvironmentParameters"][ "AtmosphericCO2"] = co2_value if USER_MODE == "localProducer-localMonica": climatefile_version = "v1" elif USER_MODE == "localProducer-remoteMonica": climatefile_version = "v3test" elif USER_MODE == "remoteProducer-remoteMonica": climatefile_version = "v3test" climateLocation = soil[soil_ref]["climate_location"] climate_filename = "{}_{}.csv".format(climateLocation, climatefile_version) #if not os.path.exists(path_to_climate_file): # continue #read climate data on the server and send just the path to the climate data csv file env["pathToClimateCSV"] = paths[ "monica-path-to-climate-dir"] + period + "/" + gcm + "/" + climate_filename env["events"] = sims["output"] for sim_ in sims["treatments"]: env["params"]["simulationParameters"][ "UseAutomaticIrrigation"] = False env["params"]["simulationParameters"][ "WaterDeficitResponseOn"] = sim_[ "WaterDeficitResponseOn"] env["params"]["simulationParameters"][ "FrostKillOn"] = sim_["FrostKillOn"] n_steps = len(env["cropRotation"][0]["worksteps"]) if sim_["Irrigate"]: if n_steps == 2: #add irrigation for irri in irrigation["irristeps"]: env["cropRotation"][0]["worksteps"].append( irri) env["cropRotation"][1]["worksteps"].append( irri) if not sim_["Irrigate"]: if n_steps == 35: #remove irrigation env["cropRotation"][0]["worksteps"] = [ env["cropRotation"][0]["worksteps"][0], env["cropRotation"][0]["worksteps"][1] ] env["cropRotation"][1]["worksteps"] = [ env["cropRotation"][1]["worksteps"][0], env["cropRotation"][1]["worksteps"][1] ] for _ in range(len(env["cropRotation"])): rotate(env["cropRotation"]) try: first_cp = env["cropRotation"][0]["worksteps"][0][ "crop"]["cropParams"]["species"]["="][ "SpeciesName"] except: first_cp = env["cropRotation"][0]["worksteps"][0][ "crop"]["cropParams"]["species"]["SpeciesName"] env["customId"] = { #"row": row, "col": col, "soil_ref": soil_ref, "period": period, "gcm": gcm, "co2_id": co2_id, "co2_value": co2_value, "trt_no": sim_["TrtNo"], "prod_case": sim_["ProdCase"], "crop_id": crop_id, "first_cp": first_cp } print("sent env ", i, " customId: ", list(env["customId"].values())) #filename = "./V" + str(i) + "_" + soil_ref +"_"+ env["customId"]["trt_no"] +"_"+ env["customId"]["gcm"] +"_"+ env["customId"]["crop_id"] +".json" #WriteEnv(filename, env) socket.send_json(env) i += 1 #exit() stop_store = time.process_time() print("sending ", i, " envs took ", (stop_store - start_store), " seconds") print("ran from ", start, "/", row_cols[start], " to ", end, "/", row_cols[end]) return
def main(): config = { "port": "6666", "server": "localhost", "sim.json": os.path.join(os.path.dirname(__file__), '../sim-min.json'), "crop.json": os.path.join(os.path.dirname(__file__), '../crop-min.json'), "site.json": os.path.join(os.path.dirname(__file__), '../site-min.json'), "climate.csv": os.path.join(os.path.dirname(__file__), '../climate-min.csv'), "shared_id": None, "climate_service_address": "10.10.24.186:11000", "admin_master_address": "10.10.24.186:8000" } # read commandline args only if script is invoked directly from commandline if len(sys.argv) > 1 and __name__ == "__main__": for arg in sys.argv[1:]: k, v = arg.split("=") if k in config: config[k] = v print("config:", config) with open(config["sim.json"]) as _: sim_json = json.load(_) with open(config["site.json"]) as _: site_json = json.load(_) with open(config["crop.json"]) as _: crop_json = json.load(_) env = monica_io3.create_env_json_from_json_config({ "crop": crop_json, "site": site_json, "sim": sim_json, "climate": "" # climate_csv }) # env["csvViaHeaderOptions"] = sim_json["climate.csv-options"] env["pathToClimateCSV"] = "blbla" # config["climate.csv"] with open("env.json", "r") as _: env = json.load(_) # with open("env.json", "w") as _: # _.write(json.dumps(env)) # rust_client = capnp.TwoPartyClient("localhost:4000") # client = capnp.TwoPartyClient("localhost:8000") csv_time_series = capnp.TwoPartyClient(config["climate_service_address"]).bootstrap().cast_as( climate_data_capnp.Climate.TimeSeries) # monica = capnp.TwoPartyClient("localhost:9999").bootstrap().cast_as(model_capnp.Model.EnvInstance) master_available = False while not master_available: try: admin_master = capnp.TwoPartyClient(config["admin_master_address"]).bootstrap().cast_as( cluster_admin_service_capnp.Cluster.AdminMaster) master_available = True except: # time.sleep(1) pass model_factories = admin_master.availableModels().wait().factories if len(model_factories) > 0: model_factory = model_factories[0] # get instance id print(model_factory.modelId().wait().id) if True: # get a single instance print("requesting single instance ...") cap_holder = model_factory.newInstance().instance # cap_holder = model_factory.restoreSturdyRef( # "b37e2e43-8a72-4dc0-8cb1-2bdc416d8148").cap monica = cap_holder.cap().wait().cap.as_interface(model_capnp.Model.EnvInstance) sturdy_ref = cap_holder.save().wait().sturdyRef print("single instance sturdy ref:", sturdy_ref) proms = [] print("running jobs on single instance ...") for i in range(5): env["customId"] = str(i) proms.append(monica.run({"rest": {"value": json.dumps(env), "structure": { "json": None}}, "timeSeries": csv_time_series})) print("results from single instance ...") for res in capnp.join_promises(proms).wait(): if len(res.result.value) > 0: # .result["customId"]) print(json.loads(res.result.value)["customId"]) # cap_holder.release().wait() if True: # get multiple instances print("requesting multiple instances ...") cap_holders = model_factory.newInstances(5).instances restore = True # cap_holders = model_factory.restoreSturdyRef( # "96495c8f-e241-4289-bee4-f314fef5b16d").cap sturdy_ref_p = cap_holders.save() entries_p = cap_holders.cap() sturdy_ref_p, entries_p = capnp.join_promises( [sturdy_ref_p, entries_p]).wait() entries = entries_p.cap sturdy_ref = sturdy_ref_p.sturdyRef print("multi instance sturdy ref:", sturdy_ref) monica_proms = [] for ent in entries: monica_proms.append(ent.entry.cap().then( lambda res: res.cap.as_interface(model_capnp.Model.EnvInstance))) monicas = capnp.join_promises(monica_proms).wait() proms = [] print("running jobs on multiple instances ...") for i in range(len(monicas)): env["customId"] = str(i) proms.append(monicas[i].run({"rest": {"value": json.dumps( env), "structure": {"json": None}}, "timeSeries": csv_time_series})) print("results from multiple instances ...") for res in capnp.join_promises(proms).wait(): if len(res.result.value) > 0: # .result["customId"]) print(json.loads(res.result.value)["customId"]) return """ climate_service = capnp.TwoPartyClient("localhost:8000").bootstrap().cast_as(climate_data_capnp.Climate.DataService) sims_prom = climate_service.simulations_request().send() sims = sims_prom.wait() if len(sims.simulations) > 0: info_prom = sims.simulations[1].info() info = info_prom.wait() print("id:", info.info.id) sim = sims.simulations[1] scens_p = sim.scenarios() scens = scens_p.wait() if len(scens.scenarios) > 0: scen = scens.scenarios[0] reals_p = scen.realizations() reals = reals_p.wait() if len(reals.realizations) > 0: real = reals.realizations[0] ts_p = real.closestTimeSeriesAt({"latlon": {"lat": 52.5, "lon": 14.1}}) ts = ts_p.wait().timeSeries ts.range().then(lambda r: print(r)).wait() ts.realizationInfo().then(lambda r: print(r.realInfo)).wait() ts.scenarioInfo().then(lambda r: print(r.scenInfo)).wait() ts.simulationInfo().then(lambda r: print(r.simInfo)).wait() """ csv_time_series = capnp.TwoPartyClient("localhost:8000").bootstrap().cast_as( climate_data_capnp.Climate.TimeSeries) # header = csv_time_series.header().wait().header monica_instance = capnp.TwoPartyClient( "localhost:9999").bootstrap().cast_as(model_capnp.Model.EnvInstance) # monica_instance = capnp.TwoPartyClient("login01.cluster.zalf.de:9999").bootstrap().cast_as(model_capnp.Model.EnvInstance) proms = [] for i in range(10): env["customId"] = str(i) proms.append(monica_instance.run({"rest": {"value": json.dumps( env), "structure": {"json": None}}, "timeSeries": csv_time_series})) for i in range(10): ps = proms[i * 50:i * 50 + 50] for res in capnp.join_promises(ps).wait(): if len(res.result.value) > 0: # .result["customId"]) print(json.loads(res.result.value)["customId"]) return reslist = capnp.join_promises(proms).wait() # reslist.wait() for res in reslist: print(json.loads(res.result.value)["customId"]) # .result["customId"]) # .then(lambda res: setattr(_context.results, "result", \ # self.calc_yearly_tavg(res[2].startDate, res[2].endDate, res[0].header, res[1].data))) # result_j = monica_instance.runEnv({"jsonEnv": json.dumps(env), "timeSeries": csv_time_series}).wait().result # result = json.loads(result_j) # print("result:", result) """ # req = model.run_request() # req.data = ts # result = req.send().wait().result tavg_ts = ts.subheader(["tavg"]).wait().timeSeries start_time = time.perf_counter() result = model.run(tavg_ts).wait().result end_time = time.perf_counter() print("rust:", result, "time:", (end_time - start_time), "s") """ """
def run_producer(config): "main" if not config["server"]: config["server"] = server[config["mode"]] def rotate(crop_rotation): "rotate the crops in the rotation" crop_rotation.insert(0, crop_rotation.pop()) print("config:", config) context = zmq.Context() socket = context.socket(zmq.PUSH) # pylint: disable=no-member # select paths paths = PATHS[config["mode"]] # connect to monica proxy (if local, it will try to connect to a locally started monica) socket.connect("tcp://" + config["server"] + ":" + str(config["server-port"])) template_folder = script_path + "/json_templates_wra/" with open(template_folder + "sim.json") as _: sim = json.load(_) sim["include-file-base-path"] = paths["monica-parameters-path"] if USER_MODE == "localProducer-localMonica": sim["climate.csv-options"]["no-of-climate-file-header-lines"] = 1 elif USER_MODE == "localProducer-remoteMonica": sim["climate.csv-options"]["no-of-climate-file-header-lines"] = 2 elif USER_MODE == "remoteProducer-remoteMonica": sim["climate.csv-options"]["no-of-climate-file-header-lines"] = 2 with open(template_folder + "site.json") as _: site = json.load(_) with open(template_folder + "crop.json") as _: crop = json.load(_) with open(template_folder + "sims.json") as _: sims = json.load(_) # load extra files for crop rotations with open(template_folder + "crop_wb_wra.json") as _: crop_rot_wb_wra = json.load(_) with open(template_folder + "crop_wra_wb.json") as _: crop_rot_wra_wb = json.load(_) period_gcm_co2s = [ { "id": "C1", "period": "0", "gcm": "0_0", "co2_value": 360 }, # #{"id": "C26", "period": "2", "gcm": "GFDL-CM3_45", "co2_value": 499}, #{"id": "C28", "period": "2", "gcm": "GFDL-CM3_85", "co2_value": 571}, #{"id": "C30", "period": "2", "gcm": "GISS-E2-R_45", "co2_value": 499}, #{"id": "C32", "period": "2", "gcm": "GISS-E2-R_85", "co2_value": 571}, { "id": "C34", "period": "2", "gcm": "HadGEM2-ES_26", "co2_value": 443 }, #{"id": "C36", "period": "2", "gcm": "HadGEM2-ES_45", "co2_value": 499}, #{"id": "C38", "period": "2", "gcm": "HadGEM2-ES_85", "co2_value": 571}, #{"id": "C40", "period": "2", "gcm": "MIROC5_45", "co2_value": 499}, #{"id": "C42", "period": "2", "gcm": "MIROC5_85", "co2_value": 571}, { "id": "C44", "period": "2", "gcm": "MPI-ESM-MR_26", "co2_value": 443 }, #{"id": "C46", "period": "2", "gcm": "MPI-ESM-MR_45", "co2_value": 499}, #{"id": "C48", "period": "2", "gcm": "MPI-ESM-MR_85", "co2_value": 571} ] soil = {} row_cols = [] dates = dict() with open( os.path.join(paths["monica-project-data"], "stu_eu_layer_grid_management_rapeseed.csv")) as _: reader = csv.reader(_) next(reader) for row in reader: soil_ref = row[0] #soil_ref,latitude,longitude,sowStart,sowEnd,harvest dates[soil_ref] = { "sowStart": row[3], "sowEnd": row[4], "harvest": row[5], } print(os.path.join(paths["monica-project-data"], "stu_eu_layer_ref.csv")) with open( os.path.join(paths["monica-project-data"], "stu_eu_layer_ref.csv")) as _: reader = csv.reader(_) next(reader) for row in reader: #Column_,Row,Grid_Code,Location,CLocation,elevation,latitude,longitude,depth,OC_topsoil,OC_subsoil,BD_topsoil,BD_subsoil,Sand_topsoil,Clay_topsoil,Silt_topsoil,Sand_subsoil,Clay_subsoil,Silt_subsoil #soil_ref,CLocation,latitude,depth,OC_topsoil,OC_subsoil,BD_topsoil,BD_subsoil,Sand_topsoil,Clay_topsoil,Silt_topsoil,Sand_subsoil,Clay_subsoil,Silt_subsoil soil_ref = row[0] row_cols.append(soil_ref) soil[soil_ref] = { "climate_location": row[1], "latitude": float(row[2]), "depth": float(row[3]), "oc-topsoil": float(row[4]), "oc-subsoil": float(row[5]), "bd-topsoil": float(row[6]), "bd-subsoil": float(row[7]), "sand-topsoil": float(row[8]), "clay-topsoil": float(row[9]), "silt-topsoil": float(row[10]), "sand-subsoil": float(row[11]), "clay-subsoil": float(row[12]), "silt-subsoil": float(row[13]), } def get_custom_site(soil_ref): "update function" cell_soil = soil[soil_ref] top = { "Thickness": [0.3, "m"], "SoilOrganicCarbon": [cell_soil["oc-topsoil"], "%"], "SoilBulkDensity": [cell_soil["bd-topsoil"], "kg m-3"], "Sand": cell_soil["sand-topsoil"] / 100.0, "Clay": cell_soil["clay-topsoil"] / 100.0, "Silt": cell_soil["silt-topsoil"] / 100.0 } sub = { "Thickness": [1.7, "m"], "SoilOrganicCarbon": [cell_soil["oc-subsoil"], "%"], "SoilBulkDensity": [cell_soil["bd-subsoil"], "kg m-3"], "Sand": cell_soil["sand-subsoil"] / 100.0, "Clay": cell_soil["clay-subsoil"] / 100.0, "Silt": cell_soil["silt-subsoil"] / 100.0 } custom_site = { "soil-profile": [top, sub], "latitude": cell_soil["latitude"], } return custom_site print("# of rowsCols = ", len(row_cols)) i = 0 start_store = time.process_time() start = config["start-row"] - 1 end = config["end-row"] - 1 if end < 0: row_cols_ = row_cols[start:] else: row_cols_ = row_cols[start:end + 1] #row_cols_ = ["64500"] #row_cols_ = [(3015,2836)] # 2836,3015 print("running from ", start, "/", row_cols[start], " to ", end, "/", row_cols[end]) run_periods = list(map(str, json.loads(config["run-periods"]))) for soil_ref in row_cols_: custom_site = get_custom_site(soil_ref) site["SiteParameters"]["Latitude"] = custom_site["latitude"] site["SiteParameters"]["SoilProfileParameters"] = custom_site[ "soil-profile"] def setDates(sowFs, sowFe, harF, isfirst): if isfirst: #sowing first crop str = crop["cropRotation"][0]["worksteps"][0]["earliest-date"] crop["cropRotation"][0]["worksteps"][0][ "earliest-date"] = doyToDate(1981, sowFs, str) str = crop["cropRotation"][0]["worksteps"][0]["latest-date"] crop["cropRotation"][0]["worksteps"][0][ "latest-date"] = doyToDate(1981, sowFe, str) # harvest str = crop["cropRotation"][0]["worksteps"][4]["latest-date"] crop["cropRotation"][0]["worksteps"][4][ "latest-date"] = doyToDate(1981, harF, str) else: #sowing first crop str = crop["cropRotation"][0]["worksteps"][0]["earliest-date"] crop["cropRotation"][0]["worksteps"][0][ "earliest-date"] = doyToDate(1981, sowFs, str) str = crop["cropRotation"][0]["worksteps"][0]["latest-date"] crop["cropRotation"][0]["worksteps"][0][ "latest-date"] = doyToDate(1981, sowFe, str) # harvest str = crop["cropRotation"][0]["worksteps"][4]["latest-date"] crop["cropRotation"][0]["worksteps"][4][ "latest-date"] = doyToDate(1981, harF, str) for idxcr in range(0, 2): mDat = dates[soil_ref] setDates(mDat["sowStart"], mDat["sowEnd"], mDat["harvest"]) if idxcr == 0: crop["cropRotation"] = crop_rot_wb_wra["cropRotation"] setDates(mDat["sowStart"], mDat["sowEnd"], mDat["harvest"], False) else: crop["cropRotation"] = crop_rot_wra_wb["cropRotation"] setDates(mDat["sowStart"], mDat["sowEnd"], mDat["harvest"], True) env = monica_io3.create_env_json_from_json_config({ "crop": crop, "site": site, "sim": sim, "climate": "" }) env["csvViaHeaderOptions"] = sim["climate.csv-options"] if idxcr == 0: env["csvViaHeaderOptions"]["start-date"] = "" enddate = doyToDate(2010, dates[soil_ref]["harvest"], "2010-01-01", datestr="%m-%d") env["csvViaHeaderOptions"]["end-date"] = enddate else: env["csvViaHeaderOptions"]["start-date"] = "1981-01-01" env["csvViaHeaderOptions"]["end-date"] = "" for pgc in period_gcm_co2s: co2_id = pgc["id"] co2_value = pgc["co2_value"] period = pgc["period"] gcm = pgc["gcm"] if period not in run_periods: continue env["params"]["userEnvironmentParameters"][ "AtmosphericCO2"] = co2_value if USER_MODE == "localProducer-localMonica": climatefile_version = "v1" elif USER_MODE == "localProducer-remoteMonica": climatefile_version = "v3test" elif USER_MODE == "remoteProducer-remoteMonica": climatefile_version = "v3test" climateLocation = soil[soil_ref]["climate_location"] climate_filename = "{}_{}.csv".format(climateLocation, climatefile_version) #read climate data on the server and send just the path to the climate data csv file env["pathToClimateCSV"] = paths[ "monica-path-to-climate-dir"] + period + "/" + gcm + "/" + climate_filename env["events"] = sims["output"] for sim_ in sims["treatments"]: env["params"]["simulationParameters"][ "UseAutomaticIrrigation"] = sim_["Irrigate"] env["params"]["simulationParameters"][ "WaterDeficitResponseOn"] = sim_[ "WaterDeficitResponseOn"] env["params"]["simulationParameters"][ "FrostKillOn"] = sim_["FrostKillOn"] try: first_cp = env["cropRotation"][0]["worksteps"][0][ "crop"]["cropParams"]["species"]["="][ "SpeciesName"] except: first_cp = env["cropRotation"][0]["worksteps"][0][ "crop"]["cropParams"]["species"]["SpeciesName"] env["customId"] = { #"row": row, "col": col, "soil_ref": soil_ref, "period": period, "gcm": gcm, "co2_id": co2_id, "co2_value": co2_value, "trt_no": sim_["TrtNo"], "prod_case": sim_["ProdCase"], "crop_id": first_cp, "first_cp": first_cp } print("sent env ", i, " customId: ", list(env["customId"].values())) #filename = "./envtest/V" + str(i) + "_" + soil_ref +"_"+ env["customId"]["trt_no"] +"_"+ env["customId"]["gcm"] +"_"+ env["customId"]["crop_id"] +".json" #WriteEnv(filename, env) socket.send_json(env) i += 1 #exit() stop_store = time.process_time() print("sending ", i, " envs took ", (stop_store - start_store), " seconds") print("ran from ", start, "/", row_cols[start], " to ", end, "/", row_cols[end]) return
def run_producer(server={"server": None, "port": None}, shared_id=None): "main" context = zmq.Context() socket = context.socket(zmq.PUSH) #config_and_no_data_socket = context.socket(zmq.PUSH) config = { "mode": "mbm-local-remote", "server-port": server["port"] if server["port"] else DEFAULT_PORT, "server": server["server"] if server["server"] else DEFAULT_HOST, "start-row": "0", "end-row": "-1", "sim.json": TEMPLATE_SIM_JSON, "crop.json": TEMPLATE_CROP_JSON, "site.json": TEMPLATE_SITE_JSON, "setups-file": SETUP_FILE, "run-setups": RUN_SETUP, "shared_id": shared_id } # read commandline args only if script is invoked directly from commandline if len(sys.argv) > 1 and __name__ == "__main__": for arg in sys.argv[1:]: k, v = arg.split("=") if k in config: config[k] = v print("config:", config) # select paths paths = PATHS[config["mode"]] # open soil db connection soil_db_con = sqlite3.connect(paths["path-to-data-dir"] + DATA_SOIL_DB) #soil_db_con = cas_sq3.connect(paths["path-to-data-dir"] + DATA_SOIL_DB) #CAS. # connect to monica proxy (if local, it will try to connect to a locally started monica) socket.connect("tcp://" + config["server"] + ":" + str(config["server-port"])) # read setup from csv file setups = Mrunlib.read_sim_setups(config["setups-file"]) run_setups = json.loads(config["run-setups"]) print("read sim setups: ", config["setups-file"]) #transforms geospatial coordinates from one coordinate reference system to another # transform wgs84 into gk5 wgs84 = Proj(init="epsg:4326" ) #proj4 -> (World Geodetic System 1984 https://epsg.io/4326) gk5 = Proj(init=GEO_TARGET_GRID) # Load grids ## note numpy is able to load from a compressed file, ending with .gz or .bz2 # height data for germany path_to_dem_grid = paths["path-to-data-dir"] + DATA_GRID_HEIGHT dem_metadata, _ = Mrunlib.read_header(path_to_dem_grid) dem_grid = np.loadtxt(path_to_dem_grid, dtype=int, skiprows=6) dem_gk5_interpolate = Mrunlib.create_ascii_grid_interpolator( dem_grid, dem_metadata) print("read: ", path_to_dem_grid) # slope data path_to_slope_grid = paths["path-to-data-dir"] + DATA_GRID_SLOPE slope_metadata, _ = Mrunlib.read_header(path_to_slope_grid) slope_grid = np.loadtxt(path_to_slope_grid, dtype=float, skiprows=6) slope_gk5_interpolate = Mrunlib.create_ascii_grid_interpolator( slope_grid, slope_metadata) print("read: ", path_to_slope_grid) # land use data path_to_corine_grid = paths["path-to-data-dir"] + DATA_GRID_LAND_USE corine_meta, _ = Mrunlib.read_header(path_to_corine_grid) corine_grid = np.loadtxt(path_to_corine_grid, dtype=int, skiprows=6) corine_gk5_interpolate = Mrunlib.create_ascii_grid_interpolator( corine_grid, corine_meta) print("read: ", path_to_corine_grid) # soil data path_to_soil_grid = paths["path-to-data-dir"] + DATA_GRID_SOIL soil_metadata, _ = Mrunlib.read_header(path_to_soil_grid) soil_grid = np.loadtxt(path_to_soil_grid, dtype=int, skiprows=6) print("read: ", path_to_soil_grid) cdict = {} climate_data_to_gk5_interpolator = {} for run_id in run_setups: setup = setups[run_id] climate_data = setup["climate_data"] if not climate_data in climate_data_to_gk5_interpolator: # path to latlon-to-rowcol.json path = TEMPLATE_PATH_LATLON.format( path_to_climate_dir=paths["path-to-climate-dir"], climate_data=climate_data) climate_data_to_gk5_interpolator[ climate_data] = Mrunlib.create_climate_geoGrid_interpolator_from_json_file( path, wgs84, gk5, cdict) print("created climate_data to gk5 interpolator: ", path) sent_env_count = 1 start_time = time.clock() listOfClimateFiles = set() # run calculations for each setup for _, setup_id in enumerate(run_setups): if setup_id not in setups: continue start_setup_time = time.clock() setup = setups[setup_id] climate_data = setup["climate_data"] climate_model = setup["climate_model"] climate_scenario = setup["climate_scenario"] climate_region = setup["climate_region"] # read template sim.json with open(setup.get("sim.json", config["sim.json"])) as _: sim_json = json.load(_) # change start and end date acording to setup if setup["start_year"]: sim_json["climate.csv-options"]["start-date"] = str( setup["start_year"]) + "-01-01" if setup["end_year"]: sim_json["climate.csv-options"]["end-date"] = str( setup["end_year"]) + "-12-31" sim_json["include-file-base-path"] = paths["include-file-base-path"] # read template site.json with open(setup.get("site.json", config["site.json"])) as _: site_json = json.load(_) # read template crop.json with open(setup.get("crop.json", config["crop.json"])) as _: crop_json = json.load(_) crop_json["cropRotation"][0]["worksteps"][0]["date"] = crop_json[ "cropRotation"][0]["worksteps"][0]["date"].replace( "XXXX", str(setup["start_year"])) # create environment template from json templates env_template = monica_io3.create_env_json_from_json_config({ "crop": crop_json, "site": site_json, "sim": sim_json, "climate": "" }) # set shared id in template if config["shared_id"]: env_template["sharedId"] = config["shared_id"] scols = int(soil_metadata["ncols"]) srows = int(soil_metadata["nrows"]) scellsize = int(soil_metadata["cellsize"]) xllcorner = int(soil_metadata["xllcorner"]) yllcorner = int(soil_metadata["yllcorner"]) #print("All Rows x Cols: " + str(srows) + "x" + str(scols), flush=True) for srow in range(0, srows): #print(srow,) if srow < int(config["start-row"]): continue elif int(config["end-row"]) > 0 and srow > int(config["end-row"]): break for scol in range(0, scols): soil_id = soil_grid[srow, scol] if soil_id == -9999: continue if soil_id < 1 or soil_id > 71: #print("row/col:", srow, "/", scol, "has unknown soil_id:", soil_id) #unknown_soil_ids.add(soil_id) continue #get coordinate of clostest climate element of real soil-cell sh_gk5 = yllcorner + (scellsize / 2) + (srows - srow - 1) * scellsize sr_gk5 = xllcorner + (scellsize / 2) + scol * scellsize #inter = crow/ccol encoded into integer crow, ccol = climate_data_to_gk5_interpolator[climate_data]( sr_gk5, sh_gk5) # check if current grid cell is used for agriculture if setup["landcover"]: corine_id = corine_gk5_interpolate(sr_gk5, sh_gk5) if corine_id not in [ 200, 210, 211, 212, 240, 241, 242, 243, 244 ]: continue height_nn = dem_gk5_interpolate(sr_gk5, sh_gk5) slope = slope_gk5_interpolate(sr_gk5, sh_gk5) #print("scol:", scol, "crow/col:", (crow, ccol), "soil_id:", soil_id, "height_nn:", height_nn, "slope:", slope, "seed_harvest_cs:", seed_harvest_cs) #with open("dump-" + str(c) + ".json", "w") as jdf: # json.dump({"id": (str(resolution) \ # + "|" + str(vrow) + "|" + str(vcol) \ # + "|" + str(crow) + "|" + str(ccol) \ # + "|" + str(soil_id) \ # + "|" + str(uj_id)), "sowing": worksteps[0], "harvest": worksteps[1]}, jdf, indent=2) # c += 1 env_template["params"]["userCropParameters"][ "__enable_T_response_leaf_expansion__"] = setup[ "LeafExtensionModifier"] # set soil-profile sp_json = soil_io3.soil_parameters(soil_db_con, int(soil_id)) soil_profile = monica_io3.find_and_replace_references( sp_json, sp_json)["result"] #print("soil:", soil_profile) env_template["params"]["siteParameters"][ "SoilProfileParameters"] = soil_profile # setting groundwater level if setup["groundwater-level"]: groundwaterlevel = 20 layer_depth = 0 for layer in soil_profile: if layer.get("is_in_groundwater", False): groundwaterlevel = layer_depth #print("setting groundwaterlevel of soil_id:", str(soil_id), "to", groundwaterlevel, "m") break layer_depth += Mrunlib.get_value(layer["Thickness"]) env_template["params"]["userEnvironmentParameters"][ "MinGroundwaterDepthMonth"] = 3 env_template["params"]["userEnvironmentParameters"][ "MinGroundwaterDepth"] = [ max(0, groundwaterlevel - 0.2), "m" ] env_template["params"]["userEnvironmentParameters"][ "MaxGroundwaterDepth"] = [groundwaterlevel + 0.2, "m"] # setting impenetrable layer if setup["impenetrable-layer"]: impenetrable_layer_depth = Mrunlib.get_value( env_template["params"]["userEnvironmentParameters"] ["LeachingDepth"]) layer_depth = 0 for layer in soil_profile: if layer.get("is_impenetrable", False): impenetrable_layer_depth = layer_depth #print("setting leaching depth of soil_id:", str(soil_id), "to", impenetrable_layer_depth, "m") break layer_depth += Mrunlib.get_value(layer["Thickness"]) env_template["params"]["userEnvironmentParameters"][ "LeachingDepth"] = [impenetrable_layer_depth, "m"] env_template["params"]["siteParameters"][ "ImpenetrableLayerDepth"] = [ impenetrable_layer_depth, "m" ] if setup["elevation"]: env_template["params"]["siteParameters"][ "heightNN"] = float(height_nn) if setup["slope"]: env_template["params"]["siteParameters"][ "slope"] = slope / 100.0 if setup["latitude"]: clat, _ = cdict[(crow, ccol)] env_template["params"]["siteParameters"]["Latitude"] = clat if setup["CO2"]: env_template["params"]["userEnvironmentParameters"][ "AtmosphericCO2"] = float(setup["CO2"]) if setup["O3"]: env_template["params"]["userEnvironmentParameters"][ "AtmosphericO3"] = float(setup["O3"]) env_template["params"]["simulationParameters"][ "UseNMinMineralFertilisingMethod"] = setup["fertilization"] env_template["params"]["simulationParameters"][ "UseAutomaticIrrigation"] = setup["irrigation"] env_template["params"]["simulationParameters"][ "NitrogenResponseOn"] = setup["NitrogenResponseOn"] env_template["params"]["simulationParameters"][ "WaterDeficitResponseOn"] = setup["WaterDeficitResponseOn"] env_template["params"]["simulationParameters"][ "EmergenceMoistureControlOn"] = setup[ "EmergenceMoistureControlOn"] env_template["params"]["simulationParameters"][ "EmergenceFloodingControlOn"] = setup[ "EmergenceFloodingControlOn"] env_template["csvViaHeaderOptions"] = sim_json[ "climate.csv-options"] subpath_to_csv = TEMPLATE_PATH_CLIMATE_CSV.format( climate_data=climate_data, climate_model_folder=(climate_model + "/" if climate_model else ""), climate_scenario_folder=(climate_scenario + "/" if climate_scenario else ""), climate_region=climate_region, crow=str(crow), ccol=str(ccol)) # subpath_to_csv = climate_data + "/csvs/" \ # + (climate_model + "/" if climate_model else "") \ # + (climate_scenario + "/" if climate_scenario else "") \ # + climate_region + "/row-" + str(crow) + "/col-" + str(ccol) + ".csv" env_template["pathToClimateCSV"] = paths[ "monica-path-to-climate-dir"] + subpath_to_csv #print(env_template["pathToClimateCSV"]) if DEBUG_WRITE_CLIMATE: listOfClimateFiles.add(subpath_to_csv) env_template["customId"] = { "setup_id": setup_id, "srow": srow, "scol": scol, "crow": int(crow), "ccol": int(ccol), "soil_id": int(soil_id) } if not DEBUG_DONOT_SEND: socket.send_json(env_template) print("sent env ", sent_env_count, " customId: ", env_template["customId"], flush=True) sent_env_count += 1 # write debug output, as json file if DEBUG_WRITE: if not os.path.exists(DEBUG_WRITE_FOLDER): os.makedirs(DEBUG_WRITE_FOLDER) if sent_env_count < DEBUG_ROWS: path_to_debug_file = DEBUG_WRITE_FOLDER + "/row_" + str( sent_env_count - 1) + "_" + str(setup_id) + ".json" if not os.path.isfile(path_to_debug_file): with open(path_to_debug_file, "w") as _: _.write(json.dumps(env_template)) else: print("WARNING: Row ", (sent_env_count - 1), " already exists") #print("unknown_soil_ids:", unknown_soil_ids) #print("crows/cols:", crows_cols) stop_setup_time = time.clock() print("Setup ", (sent_env_count - 1), " envs took ", (stop_setup_time - start_setup_time), " seconds", flush=True) stop_time = time.clock() # write summary of used json files if DEBUG_WRITE_CLIMATE: if not os.path.exists(DEBUG_WRITE_FOLDER): os.makedirs(DEBUG_WRITE_FOLDER) path_to_climate_summary = DEBUG_WRITE_FOLDER + "/climate_file_list" + ".csv" with open(path_to_climate_summary, "w") as _: _.write('\n'.join(listOfClimateFiles)) try: print("sending ", (sent_env_count - 1), " envs took ", (stop_time - start_time), " seconds", flush=True) #print("ran from ", start, "/", row_cols[start], " to ", end, "/", row_cols[end] print("exiting run_producer()", flush=True) except Exception: raise
def main(): config = { "port": "6666", "server": "localhost", "sim.json": os.path.join(os.path.dirname(__file__), '../sim-min.json'), "crop.json": os.path.join(os.path.dirname(__file__), '../crop-min.json'), "site.json": os.path.join(os.path.dirname(__file__), '../site-min.json'), "climate.csv": os.path.join(os.path.dirname(__file__), '../climate-min.csv'), "shared_id": None } # read commandline args only if script is invoked directly from commandline if len(sys.argv) > 1 and __name__ == "__main__": for arg in sys.argv[1:]: k, v = arg.split("=") if k in config: config[k] = v print("config:", config) with open(config["sim.json"]) as _: sim_json = json.load(_) with open(config["site.json"]) as _: site_json = json.load(_) with open(config["crop.json"]) as _: crop_json = json.load(_) env = monica_io3.create_env_json_from_json_config({ "crop": crop_json, "site": site_json, "sim": sim_json, "climate": "" #climate_csv }) #env["csvViaHeaderOptions"] = sim_json["climate.csv-options"] #env["pathToClimateCSV"] = config["climate.csv"] #rust_client = capnp.TwoPartyClient("localhost:4000") #client = capnp.TwoPartyClient("localhost:8000") """ climate_service = capnp.TwoPartyClient("localhost:8000").bootstrap().cast_as(climate_data_capnp.Climate.DataService) sims_prom = climate_service.simulations_request().send() sims = sims_prom.wait() if len(sims.simulations) > 0: info_prom = sims.simulations[1].info() info = info_prom.wait() print("id:", info.info.id) sim = sims.simulations[1] scens_p = sim.scenarios() scens = scens_p.wait() if len(scens.scenarios) > 0: scen = scens.scenarios[0] reals_p = scen.realizations() reals = reals_p.wait() if len(reals.realizations) > 0: real = reals.realizations[0] ts_p = real.closestTimeSeriesAt({"latlon": {"lat": 52.5, "lon": 14.1}}) ts = ts_p.wait().timeSeries ts.range().then(lambda r: print(r)).wait() ts.realizationInfo().then(lambda r: print(r.realInfo)).wait() ts.scenarioInfo().then(lambda r: print(r.scenInfo)).wait() ts.simulationInfo().then(lambda r: print(r.simInfo)).wait() """ csv_time_series = capnp.TwoPartyClient( "localhost:8000").bootstrap().cast_as( climate_data_capnp.Climate.TimeSeries) #header = csv_time_series.header().wait().header monica_instance = capnp.TwoPartyClient( "localhost:9999").bootstrap().cast_as(model_capnp.Model.EnvInstance) #monica_instance = capnp.TwoPartyClient("login01.cluster.zalf.de:9999").bootstrap().cast_as(model_capnp.Model.EnvInstance) proms = [] for i in range(10): env["customId"] = str(i) proms.append( monica_instance.run({ "rest": { "value": json.dumps(env), "structure": { "json": None } }, "timeSeries": csv_time_series })) for i in range(10): ps = proms[i * 50:i * 50 + 50] for res in capnp.join_promises(ps).wait(): if len(res.result.value) > 0: print(json.loads( res.result.value)["customId"]) #.result["customId"]) return reslist = capnp.join_promises(proms).wait() #reslist.wait() for res in reslist: print(json.loads(res.result.value)["customId"]) #.result["customId"]) # .then(lambda res: setattr(_context.results, "result", \ # self.calc_yearly_tavg(res[2].startDate, res[2].endDate, res[0].header, res[1].data))) #result_j = monica_instance.runEnv({"jsonEnv": json.dumps(env), "timeSeries": csv_time_series}).wait().result #result = json.loads(result_j) #print("result:", result) """ #req = model.run_request() #req.data = ts #result = req.send().wait().result tavg_ts = ts.subheader(["tavg"]).wait().timeSeries start_time = time.perf_counter() result = model.run(tavg_ts).wait().result end_time = time.perf_counter() print("rust:", result, "time:", (end_time - start_time), "s") """ """
def run_producer(server={"server": None, "port": None}, shared_id=None): "main" context = zmq.Context() socket = context.socket(zmq.PUSH) # pylint: disable=no-member #config_and_no_data_socket = context.socket(zmq.PUSH) config = { "mode": "remoteProducer-remoteMonica", "server-port": server["port"] if server["port"] else DEFAULT_PORT, "server": server["server"] if server["server"] else DEFAULT_HOST, "start-row": "0", "end-row": "-1", "sim.json": TEMPLATE_SIM_JSON, "crop.json": TEMPLATE_CROP_JSON, "site.json": TEMPLATE_SITE_JSON, "setups-file": SETUP_FILE, "run-setups": RUN_SETUP, "shared_id": shared_id } # read commandline args only if script is invoked directly from commandline if len(sys.argv) > 1 and __name__ == "__main__": for arg in sys.argv[1:]: k, v = arg.split("=") if k in config: config[k] = v print("config:", config) # select paths paths = PATHS[config["mode"]] # open soil db connection soil_db_con = sqlite3.connect(paths["path-to-data-dir"] + DATA_SOIL_DB) #soil_db_con = cas_sq3.connect(paths["path-to-data-dir"] + DATA_SOIL_DB) #CAS. # connect to monica proxy (if local, it will try to connect to a locally started monica) socket.connect("tcp://" + config["server"] + ":" + str(config["server-port"])) # read setup from csv file setups = Mrunlib.read_sim_setups(paths["path-to-projects-dir"] + PROJECT_FOLDER + config["setups-file"]) run_setups = json.loads(config["run-setups"]) print( "read sim setups: ", paths["path-to-projects-dir"] + PROJECT_FOLDER + config["setups-file"]) #transforms geospatial coordinates from one coordinate reference system to another # transform wgs84 into gk5 wgs84 = Proj(init="epsg:4326" ) #proj4 -> (World Geodetic System 1984 https://epsg.io/4326) gk5 = Proj(init=GEO_TARGET_GRID) # dictionary key = cropId value = [interpolate, data dictionary, is-winter-crop] ilr_seed_harvest_data = defaultdict(lambda: { "interpolate": None, "data": defaultdict(dict), "is-winter-crop": None }) # add crop id from setup file crops_in_setups = set() for setup_id, setup in setups.items(): for crop_id in setup["crop-ids"].split("_"): crops_in_setups.add(crop_id) for crop_id in crops_in_setups: try: #read seed/harvest dates for each crop_id path_harvest = TEMPLATE_PATH_HARVEST.format( path_to_projects_dir=paths["path-to-projects-dir"], project_folder=PROJECT_FOLDER, crop_id=crop_id) print("created seed harvest gk5 interpolator and read data: ", path_harvest) Mrunlib.create_seed_harvest_geoGrid_interpolator_and_read_data( path_harvest, wgs84, gk5, ilr_seed_harvest_data) except IOError: print( "Couldn't read file:", paths["path-to-projects-dir"] + PROJECT_FOLDER + "ILR_SEED_HARVEST_doys_" + crop_id + ".csv") continue # Load grids ## note numpy is able to load from a compressed file, ending with .gz or .bz2 # height data for germany path_to_dem_grid = paths["path-to-data-dir"] + DATA_GRID_HEIGHT dem_metadata, _ = Mrunlib.read_header(path_to_dem_grid) dem_grid = np.loadtxt(path_to_dem_grid, dtype=int, skiprows=6) dem_gk5_interpolate = Mrunlib.create_ascii_grid_interpolator( dem_grid, dem_metadata) print("read: ", path_to_dem_grid) # slope data path_to_slope_grid = paths["path-to-data-dir"] + DATA_GRID_SLOPE slope_metadata, _ = Mrunlib.read_header(path_to_slope_grid) slope_grid = np.loadtxt(path_to_slope_grid, dtype=float, skiprows=6) slope_gk5_interpolate = Mrunlib.create_ascii_grid_interpolator( slope_grid, slope_metadata) print("read: ", path_to_slope_grid) # land use data path_to_corine_grid = paths["path-to-data-dir"] + DATA_GRID_LAND_USE corine_meta, _ = Mrunlib.read_header(path_to_corine_grid) corine_grid = np.loadtxt(path_to_corine_grid, dtype=int, skiprows=6) corine_gk5_interpolate = Mrunlib.create_ascii_grid_interpolator( corine_grid, corine_meta) print("read: ", path_to_corine_grid) # soil data path_to_soil_grid = paths["path-to-data-dir"] + DATA_GRID_SOIL soil_metadata, _ = Mrunlib.read_header(path_to_soil_grid) soil_grid = np.loadtxt(path_to_soil_grid, dtype=int, skiprows=6) print("read: ", path_to_soil_grid) # rNfactor data path_to_rnf_grid = paths["path-to-data-dir"] + DATA_GRID_RNFACTOR rnf_meta, _ = Mrunlib.read_header(path_to_rnf_grid) rnf_grid = np.loadtxt(path_to_rnf_grid, dtype=float, skiprows=6) rnf_gk5_interpolate = Mrunlib.create_ascii_grid_interpolator( rnf_grid, rnf_meta) print("read: ", path_to_rnf_grid) cdict = {} climate_data_to_gk5_interpolator = {} for run_id in run_setups: setup = setups[run_id] climate_data = setup["climate_data"] if not climate_data in climate_data_to_gk5_interpolator: # path to latlon-to-rowcol.json path = TEMPLATE_PATH_LATLON.format( path_to_climate_dir=paths["path-to-climate-dir"], climate_data=climate_data) climate_data_to_gk5_interpolator[ climate_data] = Mrunlib.create_climate_geoGrid_interpolator_from_json_file( path, wgs84, gk5, cdict) print("created climate_data to gk5 interpolator: ", path) sent_env_count = 1 start_time = time.clock() listOfClimateFiles = set() # run calculations for each setup for _, setup_id in enumerate(run_setups): if setup_id not in setups: continue start_setup_time = time.clock() setup = setups[setup_id] climate_data = setup["climate_data"] climate_model = setup["climate_model"] climate_scenario = setup["climate_scenario"] climate_region = setup["climate_region"] crop_ids = setup["crop-ids"].split("_") # read template sim.json with open(setup.get("sim.json", config["sim.json"])) as _: sim_json = json.load(_) # change start and end date acording to setup if setup["start_year"]: sim_json["climate.csv-options"]["start-date"] = str( setup["start_year"]) + "-01-01" if setup["end_year"]: sim_json["climate.csv-options"]["end-date"] = str( setup["end_year"]) + "-12-31" sim_json["include-file-base-path"] = paths["include-file-base-path"] # read template site.json with open(setup.get("site.json", config["site.json"])) as _: site_json = json.load(_) # read template crop.json with open(setup.get("crop.json", config["crop.json"])) as _: crop_json = json.load(_) # create environment template from json templates env_template = monica_io3.create_env_json_from_json_config({ "crop": crop_json, "site": site_json, "sim": sim_json, "climate": "" }) # set shared id in template if config["shared_id"]: env_template["sharedId"] = config["shared_id"] crop_rotation_copy = copy.deepcopy(env_template["cropRotation"]) # create crop rotation according to setup # clear crop rotation and get its template # crop_rotation_templates = env_template.pop("cropRotation") # env_template["cropRotation"] = [] # get correct template # env_template["cropRotation"] = crop_rotation_templates[crop_id] # we just got one cultivation method in our rotation # worksteps_templates_dict = env_template["cropRotation"][0].pop("worksteps") # clear the worksteps array and rebuild it out of the setup # worksteps = env_template["cropRotation"][0]["worksteps"] = [] # worksteps.append(worksteps_templates_dict["sowing"][setup["sowing-date"]]) # worksteps.append(worksteps_templates_dict["harvest"][setup["harvest-date"]]) scols = int(soil_metadata["ncols"]) srows = int(soil_metadata["nrows"]) scellsize = int(soil_metadata["cellsize"]) xllcorner = int(soil_metadata["xllcorner"]) yllcorner = int(soil_metadata["yllcorner"]) #unknown_soil_ids = set() soil_id_cache = {} print("All Rows x Cols: " + str(srows) + "x" + str(scols)) for srow in range(0, srows): print(srow, ) if srow != 238: continue if srow < int(config["start-row"]): continue elif int(config["end-row"]) > 0 and srow > int(config["end-row"]): break for scol in range(0, scols): if scol != 107: continue soil_id = int(soil_grid[srow, scol]) if soil_id == -9999: continue if soil_id in soil_id_cache: sp_json = soil_id_cache[soil_id] else: sp_json = soil_io3.soil_parameters(soil_db_con, soil_id) soil_id_cache[soil_id] = sp_json if len(sp_json) == 0: print("row/col:", srow, "/", scol, "has unknown soil_id:", soil_id) #unknown_soil_ids.add(soil_id) continue #get coordinate of clostest climate element of real soil-cell sh_gk5 = yllcorner + (scellsize / 2) + (srows - srow - 1) * scellsize sr_gk5 = xllcorner + (scellsize / 2) + scol * scellsize #inter = crow/ccol encoded into integer crow, ccol = climate_data_to_gk5_interpolator[climate_data]( sr_gk5, sh_gk5) # check if current grid cell is used for agriculture if setup["landcover"]: corine_id = corine_gk5_interpolate(sr_gk5, sh_gk5) if corine_id not in [ 200, 210, 211, 212, 240, 241, 242, 243, 244 ]: continue rNfactor = rnf_gk5_interpolate(sr_gk5, sh_gk5) height_nn = dem_gk5_interpolate(sr_gk5, sh_gk5) slope = slope_gk5_interpolate(sr_gk5, sh_gk5) for i, crop_id in enumerate(crop_ids): worksteps = env_template["cropRotation"][i]["worksteps"] worksteps_copy = crop_rotation_copy[i]["worksteps"] ilr_interpolate = ilr_seed_harvest_data[crop_id][ "interpolate"] seed_harvest_cs = ilr_interpolate( sr_gk5, sh_gk5) if ilr_interpolate else None print("scol:", scol, "crow/col:", (crow, ccol), "crop_id:", crop_id, "soil_id:", soil_id, "height_nn:", height_nn, "slope:", slope, "seed_harvest_cs:", seed_harvest_cs) # multiply rNFactor onto mineral nitrogen fertilizer amounts for k, workstep in enumerate(worksteps): workstep_copy = worksteps_copy[k] if workstep["type"] == "MineralFertilization": if type(workstep["amount"]) == list: workstep["amount"][ 0] = workstep_copy["amount"][0] * rNfactor elif type(workstep["amount"]) == float: workstep["amount"] = workstep_copy[ "amount"] * rNfactor # set external seed/harvest dates if seed_harvest_cs: seed_harvest_data = ilr_seed_harvest_data[crop_id][ "data"][seed_harvest_cs] if seed_harvest_data: is_winter_crop = ilr_seed_harvest_data[crop_id][ "is-winter-crop"] if setup["sowing-date"] == "fixed": sowing_date = seed_harvest_data["sowing-date"] elif setup["sowing-date"] == "auto": sowing_date = seed_harvest_data[ "latest-sowing-date"] sds = [int(x) for x in sowing_date.split("-")] sd = date(2001, sds[1], sds[2]) sdoy = sd.timetuple().tm_yday if setup["harvest-date"] == "fixed": harvest_date = seed_harvest_data[ "harvest-date"] elif setup["harvest-date"] == "auto": harvest_date = seed_harvest_data[ "latest-harvest-date"] #print("sowing_date:", sowing_date, "harvest_date:", harvest_date) hds = [int(x) for x in harvest_date.split("-")] hd = date(2001, hds[1], hds[2]) hdoy = hd.timetuple().tm_yday esds = [ int(x) for x in seed_harvest_data["earliest-sowing-date"]. split("-") ] esd = date(2001, esds[1], esds[2]) # sowing after harvest should probably never occur in both fixed setup! if setup["sowing-date"] == "fixed" and setup[ "harvest-date"] == "fixed": if is_winter_crop: calc_harvest_date = date( 2000, 12, 31) + timedelta( days=min(hdoy, sdoy - 1)) else: calc_harvest_date = date( 2000, 12, 31) + timedelta(days=hdoy) worksteps[0]["date"] = seed_harvest_data[ "sowing-date"] worksteps[-1][ "date"] = "{:04d}-{:02d}-{:02d}".format( hds[0], calc_harvest_date.month, calc_harvest_date.day) elif setup["sowing-date"] == "fixed" and setup[ "harvest-date"] == "auto": if is_winter_crop: calc_harvest_date = date( 2000, 12, 31) + timedelta( days=min(hdoy, sdoy - 1)) else: calc_harvest_date = date( 2000, 12, 31) + timedelta(days=hdoy) worksteps[0]["date"] = seed_harvest_data[ "sowing-date"] worksteps[1][ "latest-date"] = "{:04d}-{:02d}-{:02d}".format( hds[0], calc_harvest_date.month, calc_harvest_date.day) elif setup["sowing-date"] == "auto" and setup[ "harvest-date"] == "fixed": worksteps[0][ "earliest-date"] = seed_harvest_data[ "earliest-sowing-date"] if esd > date( esd.year, 6, 20 ) else "{:04d}-{:02d}-{:02d}".format( sds[0], 6, 20) calc_sowing_date = date( 2000, 12, 31) + timedelta(days=max(hdoy + 1, sdoy)) worksteps[0][ "latest-date"] = "{:04d}-{:02d}-{:02d}".format( sds[0], calc_sowing_date.month, calc_sowing_date.day) worksteps[1]["date"] = seed_harvest_data[ "harvest-date"] elif setup["sowing-date"] == "auto" and setup[ "harvest-date"] == "auto": worksteps[0][ "earliest-date"] = seed_harvest_data[ "earliest-sowing-date"] if esd > date( esd.year, 6, 20 ) else "{:04d}-{:02d}-{:02d}".format( sds[0], 6, 20) if is_winter_crop: calc_harvest_date = date( 2000, 12, 31) + timedelta( days=min(hdoy, sdoy - 1)) else: calc_harvest_date = date( 2000, 12, 31) + timedelta(days=hdoy) worksteps[0][ "latest-date"] = seed_harvest_data[ "latest-sowing-date"] worksteps[1][ "latest-date"] = "{:04d}-{:02d}-{:02d}".format( hds[0], calc_harvest_date.month, calc_harvest_date.day) #print("dates: ", int(seed_harvest_cs), ":", worksteps[0]["earliest-date"], "<", worksteps[0]["latest-date"] ) #print("dates: ", int(seed_harvest_cs), ":", worksteps[1]["latest-date"], "<", worksteps[0]["earliest-date"], "<", worksteps[0]["latest-date"] ) #print("dates: ", int(seed_harvest_cs), ":", worksteps[0]["date"]) #print("dates: ", int(seed_harvest_cs), ":", worksteps[-1]["date"]) #print("sowing:", worksteps[0], "harvest:", worksteps[1]) #with open("dump-" + str(c) + ".json", "w") as jdf: # json.dump({"id": (str(resolution) \ # + "|" + str(vrow) + "|" + str(vcol) \ # + "|" + str(crow) + "|" + str(ccol) \ # + "|" + str(soil_id) \ # + "|" + crop_id \ # + "|" + str(uj_id)), "sowing": worksteps[0], "harvest": worksteps[1]}, jdf, indent=2) #c += 1 env_template["params"]["userCropParameters"][ "__enable_T_response_leaf_expansion__"] = setup[ "LeafExtensionModifier"] # set soil-profile #sp_json = soil_io3.soil_parameters(soil_db_con, int(soil_id)) soil_profile = monica_io3.find_and_replace_references( sp_json, sp_json)["result"] #print("soil:", soil_profile) env_template["params"]["siteParameters"][ "SoilProfileParameters"] = soil_profile # setting groundwater level if setup["groundwater-level"]: groundwaterlevel = 20 layer_depth = 0 for layer in soil_profile: if layer.get("is_in_groundwater", False): groundwaterlevel = layer_depth #print("setting groundwaterlevel of soil_id:", str(soil_id), "to", groundwaterlevel, "m") break layer_depth += Mrunlib.get_value(layer["Thickness"]) env_template["params"]["userEnvironmentParameters"][ "MinGroundwaterDepthMonth"] = 3 env_template["params"]["userEnvironmentParameters"][ "MinGroundwaterDepth"] = [ max(0, groundwaterlevel - 0.2), "m" ] env_template["params"]["userEnvironmentParameters"][ "MaxGroundwaterDepth"] = [groundwaterlevel + 0.2, "m"] # setting impenetrable layer if setup["impenetrable-layer"]: impenetrable_layer_depth = Mrunlib.get_value( env_template["params"]["userEnvironmentParameters"] ["LeachingDepth"]) layer_depth = 0 for layer in soil_profile: if layer.get("is_impenetrable", False): impenetrable_layer_depth = layer_depth #print("setting leaching depth of soil_id:", str(soil_id), "to", impenetrable_layer_depth, "m") break layer_depth += Mrunlib.get_value(layer["Thickness"]) env_template["params"]["userEnvironmentParameters"][ "LeachingDepth"] = [impenetrable_layer_depth, "m"] env_template["params"]["siteParameters"][ "ImpenetrableLayerDepth"] = [ impenetrable_layer_depth, "m" ] if setup["elevation"]: env_template["params"]["siteParameters"][ "heightNN"] = float(height_nn) if setup["slope"]: env_template["params"]["siteParameters"][ "slope"] = slope / 100.0 clat, _ = cdict[(crow, ccol)] if setup["latitude"]: clat, _ = cdict[(crow, ccol)] env_template["params"]["siteParameters"]["Latitude"] = clat if setup["CO2"]: env_template["params"]["userEnvironmentParameters"][ "AtmosphericCO2"] = float(setup["CO2"]) if setup["O3"]: env_template["params"]["userEnvironmentParameters"][ "AtmosphericO3"] = float(setup["O3"]) env_template["params"]["simulationParameters"][ "UseNMinMineralFertilisingMethod"] = setup["fertilization"] env_template["params"]["simulationParameters"][ "UseAutomaticIrrigation"] = setup["irrigation"] env_template["params"]["simulationParameters"][ "NitrogenResponseOn"] = setup["NitrogenResponseOn"] env_template["params"]["simulationParameters"][ "WaterDeficitResponseOn"] = setup["WaterDeficitResponseOn"] env_template["params"]["simulationParameters"][ "EmergenceMoistureControlOn"] = setup[ "EmergenceMoistureControlOn"] env_template["params"]["simulationParameters"][ "EmergenceFloodingControlOn"] = setup[ "EmergenceFloodingControlOn"] env_template["csvViaHeaderOptions"] = sim_json[ "climate.csv-options"] subpath_to_csv = TEMPLATE_PATH_CLIMATE_CSV.format( climate_data=climate_data, climate_model_folder=(climate_model + "/" if climate_model else ""), climate_scenario_folder=(climate_scenario + "/" if climate_scenario else ""), climate_region=climate_region, crow=str(crow), ccol=str(ccol)) # subpath_to_csv = climate_data + "/csvs/" \ # + (climate_model + "/" if climate_model else "") \ # + (climate_scenario + "/" if climate_scenario else "") \ # + climate_region + "/row-" + str(crow) + "/col-" + str(ccol) + ".csv" env_template["pathToClimateCSV"] = paths[ "monica-path-to-climate-dir"] + subpath_to_csv print(env_template["pathToClimateCSV"]) if DEBUG_WRITE_CLIMATE: listOfClimateFiles.add(subpath_to_csv) env_template["customId"] = { "setup_id": setup_id, "srow": srow, "scol": scol, "crow": int(crow), "ccol": int(ccol), "soil_id": soil_id } if not DEBUG_DONOT_SEND: socket.send_json(env_template) print("sent env ", sent_env_count, " customId: ", env_template["customId"]) sent_env_count += 1 # write debug output, as json file if DEBUG_WRITE: debug_write_folder = paths["path-debug-write-folder"] if not os.path.exists(debug_write_folder): os.makedirs(debug_write_folder) if sent_env_count < DEBUG_ROWS: path_to_debug_file = debug_write_folder + "/row_" + str( sent_env_count - 1) + "_" + str(setup_id) + ".json" if not os.path.isfile(path_to_debug_file): with open(path_to_debug_file, "w") as _: _.write(json.dumps(env_template)) else: print("WARNING: Row ", (sent_env_count - 1), " already exists") #print("unknown_soil_ids:", unknown_soil_ids) #print("crows/cols:", crows_cols) stop_setup_time = time.clock() print("Setup ", (sent_env_count - 1), " envs took ", (stop_setup_time - start_setup_time), " seconds") stop_time = time.clock() # write summary of used json files if DEBUG_WRITE_CLIMATE: debug_write_folder = paths["path-debug-write-folder"] if not os.path.exists(debug_write_folder): os.makedirs(debug_write_folder) path_to_climate_summary = debug_write_folder + "/climate_file_list" + ".csv" with open(path_to_climate_summary, "w") as _: _.write('\n'.join(listOfClimateFiles)) try: print("sending ", (sent_env_count - 1), " envs took ", (stop_time - start_time), " seconds") #print("ran from ", start, "/", row_cols[start], " to ", end, "/", row_cols[end] print("exiting run_producer()") except Exception: raise
def run_producer(server={"server": None, "port": None}, shared_id=None): context = zmq.Context() socket = context.socket(zmq.PUSH) config = { "port": server["port"] if server["port"] else "6666", "server": server["server"] if server["server"] else "localhost", "sim.json": os.path.join(os.path.dirname(__file__), '../sim-min.json'), "crop.json": os.path.join(os.path.dirname(__file__), '../crop-min.json'), "site.json": os.path.join(os.path.dirname(__file__), '../site-min.json'), "climate.csv": os.path.join(os.path.dirname(__file__), '../climate-min.csv'), "shared_id": shared_id } # read commandline args only if script is invoked directly from commandline if len(sys.argv) > 1 and __name__ == "__main__": for arg in sys.argv[1:]: k, v = arg.split("=") if k in config: config[k] = v print("config:", config) socket.connect("tcp://" + config["server"] + ":" + config["port"]) with open(config["sim.json"]) as _: sim_json = json.load(_) with open(config["site.json"]) as _: site_json = json.load(_) with open(config["crop.json"]) as _: crop_json = json.load(_) #with open(config["climate.csv"]) as _: # climate_csv = _.read() env = monica_io3.create_env_json_from_json_config({ "crop": crop_json, "site": site_json, "sim": sim_json, "climate": "" #climate_csv }) env["csvViaHeaderOptions"] = sim_json["climate.csv-options"] env["pathToClimateCSV"] = config["climate.csv"] #print(env) # add shared ID if env to be sent to routable monicas if config["shared_id"]: env["sharedId"] = config["shared_id"] socket.send_json(env) print("done")