def run_producer(server={"server": None, "port": None}, shared_id=None):
context = zmq.Context()
socket = context.socket(zmq.PUSH)
config = {
"port":
server["port"] if server["port"] else "6666",
"server":
server["server"] if server["server"] else "localhost",
"sim.json":
os.path.join(os.path.dirname(__file__), '../sim-min.json'),
"crop.json":
os.path.join(os.path.dirname(__file__), '../crop-min.json'),
"site.json":
os.path.join(os.path.dirname(__file__), '../site-min.json'),
"climate.csv":
os.path.join(os.path.dirname(__file__), '../climate-min.csv'),
"shared_id":
shared_id
}
# read commandline args only if script is invoked directly from commandline
if len(sys.argv) > 1 and __name__ == "__main__":
for arg in sys.argv[1:]:
k, v = arg.split("=")
if k in config:
config[k] = v
print "config:", config
socket.connect("tcp://" + config["server"] + ":" + config["port"])
with open(config["sim.json"]) as _:
sim_json = json.load(_)
with open(config["site.json"]) as _:
site_json = json.load(_)
with open(config["crop.json"]) as _:
crop_json = json.load(_)
with open(config["climate.csv"]) as _:
climate_csv = _.read()
env = monica_io.create_env_json_from_json_config({
"crop": crop_json,
"site": site_json,
"sim": sim_json,
"climate": climate_csv
})
#print env
# add shared ID if env to be sent to routable monicas
if config["shared_id"]:
env["sharedId"] = config["shared_id"]
socket.send_json(env)
print "done"
def run_producer():
# calibration options
calibration_mode = False
# simulation options
activate_debug = False
output_dir = "runs/2019-02-15/"
# calibration -------------------------------------
# technical initialisation
config = {
"user": "******",
"port": "66666",
"server": "localhost"
}
paths = PATHS[config["user"]]
sent_id = 0
start_send = time.clock()
socket = initialise_sockets(config)
print(socket)
# output file
output_filename = "agmip_calibration_phase2_step2_evaluation.csv"
if calibration_mode:
output_filename = "agmip_calibration_phase2_step2_calibration.csv"
# read in management information
management_file = paths["INCLUDE_FILE_BASE_PATH"] + "info/CSIRO_data_set/simulation_plan.xlsx"
management_df = pd.read_excel(management_file, sheader=0, index_col=None, keep_default_na=False,
encoding="latin1")
print "Shape", management_df.shape
# iterate over every row/environment where row contains the setup of one simulation
for sim_row, environment in management_df.iterrows():
print "Run simulation " + str(sim_row)
id = int(environment["ID"])
calibration_mode = bool(environment["Calibration"])
year = int(environment["Year"])
site_name = environment["Site"]
sowing_day = environment["sowDay"]
sim_parameters = None
site_parameters = None
crop_parameters = None
simulation_dir = "monica_simulation_setup/evaluation"
if calibration_mode:
simulation_dir = "monica_simulation_setup/calibration"
sim_file = "sim-%d.json" % id
with open(simulation_dir + "/" + sim_file) as fp:
sim_parameters = json.load(fp)
site_file = sim_parameters["site.json"]
with open(site_file) as fp:
site_parameters = json.load(fp)
#site_parameters["include-file-base-path"] = paths["INCLUDE_FILE_BASE_PATH"]
sim_parameters["include-file-base-path"] = paths["INCLUDE_FILE_BASE_PATH"]
# spring wheat parameters for stage temperature sum
stage1_sum = 142.74
stage2_sum = 218.43
stage3_sum = 302.21
stage4_sum = 67.51
stage5_sum = 273.15
tsum_zadok30 = stage2_sum + (stage3_sum * 0.25)
tsum_zadok65 = stage2_sum + stage3_sum + (stage4_sum * 0.25)
output_map = {
"events": [
"anthesis", [
"Date|Ant"
],
"maturity", [
"Date|Mat",
],
["while", "Stage", "=", 2], [
["Date|EmergeDate", "FIRST"]
],
["while", "Stage", "=", 3], [
["Date|BeginStage3", "FIRST"]
],
["while", "Stage", "=", 4], [
["Date|BeginStage4", "FIRST"]
],
["while", "Stage", "=", 5], [
["Date|BeginStage5", "FIRST"]
],
["while", "Stage", "=", 6], [
["Date|BeginStage6", "FIRST"]
],
["while", "TempSum", ">=", tsum_zadok30], [
["Date|ZADOK30", "FIRST"]
],
["while", "TempSum", ">=", tsum_zadok65], [
["Date|ZADOK65", "FIRST"]
],
["while", "Stage", "=", 6], [
["Date|ZADOK90", "FIRST"]
]
]
}
sim_parameters["output"] = output_map
crop_file = sim_parameters["crop.json"]
with open(crop_file) as fp:
crop_parameters = json.load(fp)
env_map = {
"crop": crop_parameters,
"site": site_parameters,
"sim": sim_parameters,
"climate": ""
}
env = monica_io.create_env_json_from_json_config(env_map)
env["csvViaHeaderOptions"] = sim_parameters["climate.csv-options"]
env["csvViaHeaderOptions"]["start-date"] = sim_parameters["climate.csv-options"]["start-date"]
env["csvViaHeaderOptions"]["end-date"] = sim_parameters["climate.csv-options"]["end-date"]
env["pathToClimateCSV"] = sim_parameters["climate.csv"]
# overwrite stage temperature sum values with tommaso's calibrated ones
# stage 1
env["cropRotation"][0]["worksteps"][0]["crop"]["cropParams"]["cultivar"]\
["StageTemperatureSum"][0][0] = stage1_sum
env["cropRotation"][0]["worksteps"][0]["crop"]["cropParams"]["cultivar"]\
["StageTemperatureSum"][0][1] = stage2_sum
env["cropRotation"][0]["worksteps"][0]["crop"]["cropParams"]["cultivar"] \
["StageTemperatureSum"][0][2] = stage3_sum
env["cropRotation"][0]["worksteps"][0]["crop"]["cropParams"]["cultivar"] \
["StageTemperatureSum"][0][3] = stage4_sum
env["cropRotation"][0]["worksteps"][0]["crop"]["cropParams"]["cultivar"] \
["StageTemperatureSum"][0][4] = stage5_sum
# final env object with all necessary information
env["customId"] = {
"id": id,
"calibration": calibration_mode,
"output_dir": output_dir,
"output_filename": output_filename,
"sowing_day": sowing_day,
"site_name": site_name,
"year": year
}
with open("monica_simulation_setup/env.json", "w") as fp:
json.dump(env, fp=fp, indent=4)
# sending env object to MONICA ZMQ
print("sent env ", sent_id, " customId: ", env["customId"])
socket.send_json(env)
sent_id += 1
# break
stop_send = time.clock()
# just tell the sending time if objects have really been sent
print("sending ", sent_id, " envs took ", (stop_send - start_send), " seconds")
def producer(setup=None):
"main function"
# reload monica_io module so that cached references will be cleared
# this is a problem because the "crops" key in crop.json is changed on every call to this function
# and thus for every setup, but calls to ["ref", "crops", "XXX"] will cache the result of
# resolving references to XXX and thus the updates to XXX for the secondary yield further up won't happen
reload(monica_io)
paths = PATHS[USER]
#Configure producer
if setup == None:
#playground
run_id = "custom"
PRODUCTION_LEVEL = 'WL.NL.rain' #options: "Pot", "WL.NL.rain"
TF = "continuous"
FERT_STRATEGY = "BASE" #options: "NDEM", "NMIN", "BASE"
COVER_CROP_FREQ = {
#always use int for insert-cc-every and out-of
#keep out-of as small as possible (to ensure uniform temporal distribution)
"insert-cc-every": 1, #CM
"out-of": 4, #CM
"suffix": "25" #TODO set it (for output file name)
}
EXPORT_RATE = "base"
RESIDUES_HUMUS_BALANCE = False #mgt complying with humus balance approach of NRW
HUMBAL_CORRECTION = {
"heavy": 200,
"medium": 300,
"light": 400
}
else:
run_id = setup["id"]
PRODUCTION_LEVEL = setup["PRODUCTION_LEVEL"]
TF = setup["TF"]
FERT_STRATEGY = setup["FERT_STRATEGY"]
COVER_CROP_FREQ = setup["COVER_CROP_FREQ"]
if setup["res_mgt"] == "RESIDUES_HUMUS_BALANCE":
RESIDUES_HUMUS_BALANCE = True
else:
RESIDUES_HUMUS_BALANCE = False
EXPORT_RATE = setup["res_mgt"]
HUMBAL_CORRECTION = setup["HUMBAL_CORRECTION"]
#end of user configuration
#assemble file name suffix for out
suffix = "_id" + run_id + "_"
suffix += TF + "_"
suffix += "fert-" + FERT_STRATEGY.lower() + "_"
if RESIDUES_HUMUS_BALANCE:
suffix += "res-humbal_"
else:
suffix += "res-" + EXPORT_RATE + "_"
suffix += "cc-" + COVER_CROP_FREQ["suffix"] + "_"
suffix += "pl-" + PRODUCTION_LEVEL.replace(".", "") + "_"
if FERT_STRATEGY == "NMIN":
rotations_file = "json_templates/rotations_dynamic_harv.json"
elif FERT_STRATEGY == "NDEM":
rotations_file = "json_templates/rotations_dynamic_harv_Ndem.json"
elif FERT_STRATEGY == "BASE":
rotations_file = "json_templates/rotations_dynamic_harv_Nbaseline.json"
rotations_spinup_file = "json_templates/rotations_dynamic_harv_Nbaseline_SU.json"
if LOCAL_RUN:
PATH_TO_CLIMATE_DATA_DIR = timeframes[TF]["local-path-to-climate"]
else:
PATH_TO_CLIMATE_DATA_DIR = timeframes[TF]["cluster-path-to-climate"]
context = zmq.Context()
socket = context.socket(zmq.PUSH)
port = 6666 if len(sys.argv) == 1 else sys.argv[1]
if LOCAL_RUN:
socket.connect(PATHS[USER]["local-monica-server"])
else:
socket.connect("tcp://cluster1:" + str(port))
soil_db_con = sqlite3.connect("soil.sqlite")
with open("json_templates/sim.json") as _:
sim = json.load(_)
sim["start-date"] = timeframes[TF]["start-date"]
sim["end-date"] = timeframes[TF]["end-date"]
with open("json_templates/site.json") as _:
site = json.load(_)
with open("json_templates/crop.json") as _:
crop = json.load(_)
with open("json_templates/cover-crop.json") as _:
cover_crop = json.load(_)["CM"]
if RESIDUES_HUMUS_BALANCE:
#inject additional harvest params
for ws in range(len(cover_crop["worksteps"])):
if cover_crop["worksteps"][ws]["type"] == "AutomaticHarvest":
cover_crop["worksteps"][ws]["opt-carbon-conservation"] = True
cover_crop["worksteps"][ws]["crop-impact-on-humus-balance"] = [humus_equivalent["crop"]["CC"], "Humus equivalent [Heq]"]
cover_crop["worksteps"][ws]["residue-heq"] = [humus_equivalent["material"]["green-manure"], "Heq ton-1 DM"]
cover_crop["worksteps"][ws]["crop-usage"] = CROP_USAGE_HUMBAL["CC"]
cover_crop["worksteps"][ws]["exported"] = True #Needed to fire the humus balance approach; if true and crop-usage="green-manure" --> the crop is anyway returned to the soil
with open("json_templates/cover-crop_SU.json") as _:
cover_crop_spinup = json.load(_)["CM_SU"]
with open("json_templates/sims.json") as _:
sims = json.load(_)
if SOC_STUDY:
sims = sims["soc_study"]
else:
sims = sims["potentials_study"]
if FERT_STRATEGY != "NMIN":
#turn off Nmin automatic fertilization
for setting in sims.iteritems():
setting[1]["UseNMinMineralFertilisingMethod"] = False
#load initialization SOC values
SOC_ini_vals = defaultdict(lambda: defaultdict())
with open("settings_SOC_study/SOCini_unique_combinations.csv") as _:
reader = csv.reader(_)
reader.next()
for row in reader:
param_id = int(row[0])
sim_id = row[1]
corgs = []
for i in range(3, 8):
if len(row) > i and row[i] != "":
corgs.append(float(row[i]))
SOC_ini_vals[param_id][sim_id] = corgs
param_vals = defaultdict(lambda: defaultdict(float))
best_p_id = -999
best_like = -999
#load parameters sets (used only for SOC study)
with open("settings_SOC_study/" + PARAMETERS_FILE) as _:
reader = csv.reader(_)
reader.next()
for row in reader:
param_id = int(row[0])
param_vals[param_id]["PartSOM_Fast_to_SOM_Slow"] = float(row[1])
param_vals[param_id]["SOM_FastDecCoeffStandard"] = float(row[2])
param_vals[param_id]["PartSMB_Fast_to_SOM_Fast"] = float(row[3])
param_vals[param_id]["PartSMB_Slow_to_SOM_Fast"] = float(row[4])
param_vals[param_id]["AOM_SlowDecCoeffStandard"] = float(row[5])
param_vals[param_id]["SOM_SlowDecCoeffStandard"] = float(row[6])
param_vals[param_id]["AOM_SlowUtilizationEfficiency"] = float(row[7])
#following two params were calibrated using exps 25 and 35 from V140 Muencheberg
param_vals[param_id]["PartAOM_to_AOM_Fast"] = 0.81
param_vals[param_id]["PartAOM_to_AOM_Slow"] = 0.19
like = float(row[8])
if like > best_like:
best_like = like
best_p_id = param_id
def load_rotations(rotations_file, spinup):
with open(rotations_file) as _:
rotations = json.load(_)
#identify rotations with codes
rots_info = {}
for bkr, rots in rotations.iteritems():
for rot in rots.iteritems():
rot_code = int(rot[0])
my_rot = []
for cm in rot[1]:#["worksteps"]:
for ws in range(len(cm["worksteps"])):
if cm["worksteps"][ws]["type"] == "Sowing":
my_rot.append(cm["worksteps"][ws]["crop"][2])
#for each crop, identify previous main one (needed to determine expected N availability)
rot_info = []
for i in range(len(my_rot)):
cm_info = {}
current_cp = my_rot[i]
if i != 0:
previous_cp = my_rot[i-1]
else:
previous_cp = my_rot[-1]
cm_info["current"] = current_cp
cm_info["previous"] = previous_cp
rot_info.append(cm_info)
rots_info[rot_code] = rot_info
if RESIDUES_HUMUS_BALANCE and spinup==False:
#inject additional harvest params
for bkr, rots in rotations.iteritems():
for rot in rots.iteritems():
rot_code = int(rot[0])
rot_info = rots_info[rot_code]
cp_index = 0
for cm in rot[1]:#["worksteps"]:
for ws in range(len(cm["worksteps"])):
cp = rot_info[cp_index]["current"]
if cm["worksteps"][ws]["type"] == "AutomaticHarvest":
cm["worksteps"][ws]["opt-carbon-conservation"] = True
cm["worksteps"][ws]["crop-impact-on-humus-balance"] = [humus_equivalent["crop"][cp], "Humus equivalent [Heq]"]
if cp == "SBee":
cm["worksteps"][ws]["residue-heq"] = [humus_equivalent["material"]["green-manure"], "Heq ton-1 DM"]
else:
cm["worksteps"][ws]["residue-heq"] = [humus_equivalent["material"]["straw"], "Heq ton-1 DM"]
cm["worksteps"][ws]["organic-fertilizer-heq"] = [humus_equivalent["material"]["pig-slurry"], "Heq ton-1 DM"]
cm["worksteps"][ws]["crop-usage"] = CROP_USAGE_HUMBAL[cp]
cm["worksteps"][ws]["max-residue-recover-fraction"] = CROP_USAGE_HUMBAL["max-residue-recover-fraction"]
cp_index += 1
return rotations, rots_info
rotations, rots_info = load_rotations(rotations_file, spinup=False)
rotations_spinup, rots_info_spinup = load_rotations(rotations_spinup_file, spinup=True)
#find the latest crop harvested to determine when to finish the spinup
last_date_spinup = defaultdict(lambda: defaultdict(date))
for bkr_id in rotations_spinup.keys():
for rot_id in rotations_spinup[bkr_id].keys():
rotation_spinup = rotations_spinup[bkr_id][rot_id]
my_year = 1971
last_date = None
i = 0
while True:
first_ws = rotation_spinup[i]["worksteps"][0]["date"].split("-")
last_ws = rotation_spinup[i]["worksteps"][-1]["latest-date"].split("-")
first_date = date(my_year, int(first_ws[1]), int(first_ws[2]))
if last_date != None and first_date < last_date:
my_year += 1
my_year += int(last_ws[0])
last_date = date(my_year, int(last_ws[1]), int(last_ws[2]))
if last_date.year == 2004:
last_date_spinup[bkr_id][rot_id] = last_date + timedelta(days=1)
break
i += 1
if i == len(rotation_spinup):
i = 0
#for spinup and "BASE" fertilization:
#read additional info
#1. expected mineral N availability
expected_N_availability = defaultdict(lambda: defaultdict())
with open ("expected_N_availability.csv") as _:
reader = csv.reader(_)
reader.next()
for row in reader:
cp_sequence = (row[0], row[1])
soil_type = row[2]
expected_N_value = float(row[3])
expected_N_availability[cp_sequence][soil_type] = expected_N_value
expected_N_availability[cp_sequence]["target_depth"] = float(row[4])
#2. rules to split mineral fertilization
mineralN_split = defaultdict(lambda: defaultdict())
with open ("MineralN_topdressing.csv") as _:
reader = csv.reader(_)
reader.next()
for row in reader:
cp = row[0]
mineralN_split[cp]["target"] = float(row[2])
for i in range(4, 7):
if row[i] != "":
mineralN_split[cp][i-4] = float(row[i])
def modify_2ry_yield_params(cp, export_r):
for k in EXPORT_PRESETS[export_r].keys():
if cp[0] in k:
my_rate = EXPORT_PRESETS[export_r][k]
for organ in cp[1]["cropParams"]["cultivar"]["OrganIdsForSecondaryYield"]:
organ["yieldPercentage"] *= my_rate
for cp in crop["crops"].iteritems():
#correct version
if "_SU" in cp[0]:
modify_2ry_yield_params(cp, "base")
elif not RESIDUES_HUMUS_BALANCE: #"_SU" not in cp[0]
modify_2ry_yield_params(cp, EXPORT_RATE)
#old version (bugged)
#if "_SU" in cp[0]:
# modify_2ry_yield_params(cp, "base")
#if RESIDUES_HUMUS_BALANCE:
# continue
#else:
# modify_2ry_yield_params(cp, EXPORT_RATE)
sim["UseSecondaryYields"] = True
sim["include-file-base-path"] = paths["INCLUDE_FILE_BASE_PATH"]
def read_general_metadata(path_to_file):
"read metadata file"
with open(path_to_file) as file_:
data = {}
reader = csv.reader(file_, delimiter="\t")
reader.next()
for row in reader:
if int(row[1]) != 1:
continue
data[(int(row[2]), int(row[3]))] = {
"subpath-climate.csv": row[9],
"latitude": float(row[13]),
"elevation": float(row[14])
}
return data
general_metadata = read_general_metadata("NRW_General_Metadata.csv")
def load_mapping(row_offset=0, col_offset=0):
to_climate_index = {}
with(open("working_resolution_to_climate_lat_lon_indices.json")) as _:
l = json.load(_)
for i in xrange(0, len(l), 2):
cell = (row_offset + l[i][0], col_offset + l[i][1])
to_climate_index[cell] = tuple(l[i+1])
return to_climate_index
def read_orgN_kreise(path_to_file, col_index):
"read organic N info for kreise"
with open(path_to_file) as file_:
data = {}
reader = csv.reader(file_, delimiter=",")
reader.next()
reader.next()
for row in reader:
for kreis_code in row[1].split("|"):
if kreis_code != "":
data[int(kreis_code)] = float(row[col_index])
return data
orgN_kreise = read_orgN_kreise("NRW_orgN_balance_T27.csv", 8)
#orgN_kreise = read_orgN_kreise("NRW_orgN_balance_T28.csv", 6)
def soil_OID(con, profile_id):
"return soil OID from the database connection for given profile id"
query = """
select
id,
soil_OID
from soil_profile
where id = ?
order by id
"""
con.row_factory = sqlite3.Row
for row in con.cursor().execute(query, (profile_id,)):
OID = row["soil_OID"]
break
return OID
def update_soil(row, col):
"in place update the env"
site["SiteParameters"]["Latitude"] = general_metadata[(row, col)]["latitude"]
site["SiteParameters"]["HeightNN"] = [general_metadata[(row, col)]["elevation"], "m"]
site["SiteParameters"]["SoilProfileParameters"] = soil_io.soil_parameters(soil_db_con, soil_ids[(row, col)])
KA5_txt = soil_io.sand_and_clay_to_ka5_texture(site["SiteParameters"]["SoilProfileParameters"][0]["Sand"][0], site["SiteParameters"]["SoilProfileParameters"][0]["Clay"][0])
for layer in site["SiteParameters"]["SoilProfileParameters"]:
layer["SoilBulkDensity"][0] = max(layer["SoilBulkDensity"][0], 600)
layer["SoilOrganicCarbon"][0] = layer["SoilOrganicCarbon"][0] * 0.6 #correction factor suggested by TGaiser
return KA5_txt
def set_initial_SOC(SOC_vals):
for layer, val in enumerate(SOC_vals):
if len(site["SiteParameters"]["SoilProfileParameters"]) > layer:
site["SiteParameters"]["SoilProfileParameters"][layer]["SoilOrganicCarbon"][0] = val
def read_header(path_to_ascii_grid_file):
"read metadata from esri ascii grid file"
metadata = {}
header_str = ""
with open(path_to_ascii_grid_file) as _:
for i in range(0, 6):
line = _.readline()
header_str += line
sline = [x for x in line.split() if len(x) > 0]
if len(sline) > 1:
metadata[sline[0].strip().lower()] = float(sline[1].strip())
return metadata, header_str
def read_ascii_grid(path_to_file, include_no_data=False, row_offset=0, col_offset=0):
"read an ascii grid into a map, without the no-data values"
def int_or_float(s):
try:
return int(s)
except ValueError:
return float(s)
with open(path_to_file) as file_:
data = {}
#skip the header (first 6 lines)
for _ in range(0, 6):
file_.next()
row = -1
for line in file_:
row += 1
col = -1
for col_str in line.strip().split(" "):
col += 1
if not include_no_data and int_or_float(col_str) == -9999:
continue
data[(row_offset+row, col_offset+col)] = int_or_float(col_str)
return data
#offset is used to match info in general metadata and soil database
soil_ids = read_ascii_grid("asc_grids/soil-profile-id_nrw_gk3.asc", row_offset=282)
bkr_ids = read_ascii_grid("asc_grids/bkr_nrw_gk3.asc", row_offset=282)
lu_ids = read_ascii_grid("asc_grids/lu_resampled.asc", row_offset=282)
kreise_ids = read_ascii_grid("asc_grids/kreise_matrix.asc", row_offset=282)
meteo_ids = load_mapping(row_offset=282)
soil_metadata, _ = read_header("asc_grids/soil-profile-id_nrw_gk3.asc")
wgs84 = Proj(init="epsg:4326")
gk3 = Proj(init="epsg:3396")
gk5 = Proj(init="epsg:31469")
def create_ascii_grid_interpolator(arr, meta, ignore_nodata=True):
"read an ascii grid into a map, without the no-data values"
rows, cols = arr.shape
cellsize = int(meta["cellsize"])
xll = int(meta["xllcorner"])
yll = int(meta["yllcorner"])
nodata_value = meta["nodata_value"]
xll_center = xll + cellsize // 2
yll_center = yll + cellsize // 2
yul_center = yll_center + (rows - 1)*cellsize
points = []
values = []
for row in range(rows):
for col in range(cols):
value = arr[row, col]
if ignore_nodata and value == nodata_value:
continue
r = xll_center + col * cellsize
h = yul_center - row * cellsize
points.append([r, h])
values.append(value)
return NearestNDInterpolator(np.array(points), np.array(values))
path_to_slope_grid = paths["path-to-data-dir"] + "/germany/slope_1000_gk5.asc"
slope_metadata, _ = read_header(path_to_slope_grid)
slope_grid = np.loadtxt(path_to_slope_grid, dtype=float, skiprows=6)
slope_gk5_interpolate = create_ascii_grid_interpolator(slope_grid, slope_metadata)
#counter = 0
#for k, v in bkr_ids.iteritems():
# if v == 134:
# if k in lu_ids:
# counter += 1
#print counter
def insert_cc(crop_rotation, cc_data):
"insert cover crops in the rotation"
insert_cover_here = []
for cultivation_method in range(len(crop_rotation)):
for workstep in crop_rotation[cultivation_method]["worksteps"]:
if workstep["type"] == "Sowing":
if workstep["crop"][2] in COVER_BEFORE:
insert_cover_here.append((cultivation_method, workstep["date"]))
break
for position, mydate in reversed(insert_cover_here):
mydate = mydate.split("-")
main_crop_sowing = date(2017, int(mydate[1]), int(mydate[2]))
latest_harvest_cc = main_crop_sowing - timedelta(days = 10)
latest_harvest_cc = unicode("0001-" + str(latest_harvest_cc.month).zfill(2) + "-" + str(latest_harvest_cc.day).zfill(2))
crop_rotation.insert(position, copy.deepcopy(cc_data))
crop_rotation[position]["worksteps"][1]["latest-date"] = latest_harvest_cc
def calculate_orgfert_amount(N_applied, fert_type, soilCN=10):
"convert N applied in amount of fresh org fert"
AOM_DryMatterContent = fert_type["AOM_DryMatterContent"][0]
AOM_NH4Content = fert_type["AOM_NH4Content"][0]
AOM_NO3Content = fert_type["AOM_NO3Content"][0]
CN_Ratio_AOM_Fast = fert_type["CN_Ratio_AOM_Fast"][0]
CN_Ratio_AOM_Slow = fert_type["CN_Ratio_AOM_Slow"][0]
PartAOM_to_AOM_Fast = fert_type["PartAOM_to_AOM_Fast"][0]
PartAOM_to_AOM_Slow = fert_type["PartAOM_to_AOM_Slow"][0]
AOM_to_C = 0.45
AOM_fast_factor = (AOM_to_C * PartAOM_to_AOM_Fast)/CN_Ratio_AOM_Fast
AOM_slow_factor = (AOM_to_C * PartAOM_to_AOM_Slow)/CN_Ratio_AOM_Slow
AOM_SOM_factor = (1- (PartAOM_to_AOM_Fast + PartAOM_to_AOM_Slow)) * AOM_to_C / soilCN
conversion_coeff = AOM_NH4Content + AOM_NO3Content + AOM_fast_factor + AOM_slow_factor + AOM_SOM_factor
AOM_dry = N_applied / conversion_coeff
AOM_fresh = AOM_dry / AOM_DryMatterContent
return AOM_fresh
def update_fert_values(rotation, rot_info, cc_in_cm, expected_N_availability, mineralN_split, soil_type, orgN_applied, spinup):
"function to mimic baseline fertilization"
cow_unit = 100
GVs = orgN_applied/cow_unit
orgN_effect = GVs * 10
#insert cc in rotation info
for cm in reversed(range(len(rot_info))):
rot_info[cm]["has_cover_before"] = False
if rot_info[cm]["current"] in COVER_BEFORE and cc_in_cm:
rot_info[cm]["has_cover_before"] = True
cc_info = {"current": "CC"}
rot_info.insert(cm, cc_info)
for cm in range(len(rotation)):
if rot_info[cm]["current"] == "CC":
#cover crops do not receive any fertilization
continue
current_cp = rot_info[cm]["current"]
previous_cp = rot_info[cm]["previous"]
has_cover = rot_info[cm]["has_cover_before"]
if spinup: #spinup crops contain "_SU" by convention
current_cp = current_cp.replace("_SU", "")
previous_cp = previous_cp.replace("_SU", "")
N_target = mineralN_split[current_cp]["target"]
expected_Nmin = expected_N_availability[(current_cp, previous_cp)][soil_type]
#modify expected Nmin depending on livestock pressure and presence of cover crop
if has_cover:
expected_Nmin += 20
expected_Nmin += orgN_effect
#calculate N to be applied with mineral fertilization
sum_Nfert = max(N_target - expected_Nmin, 0)
#map the fertilization worksteps
ref_fert = 0
for ws in range(len(rotation[cm]["worksteps"])):
workstep = rotation[cm]["worksteps"][ws]
if workstep["type"] == "MineralFertilization" and workstep["amount"][0] == 0:
workstep["amount"][0] = sum_Nfert * mineralN_split[current_cp][ref_fert]
ref_fert += 1
def extend_rotation(rot, ccfreq_denominator): #ccfreq = COVER_CROP_FREQ["out-of"]
ext_rot = []
for i in range(ccfreq_denominator):
ext_rot.append(copy.deepcopy(rot))
return ext_rot
def insert_CC(ext_rot, cover_crop, ccfreq_numerator): #COVER_CROP_FREQ["insert-cc-every"]
cc_in_cm = {}
for cm in range(len(ext_rot)):
cc_in_cm[cm] = False
if (cm+1) <= ccfreq_numerator:
insert_cc(ext_rot[cm], cover_crop)
cc_in_cm[cm] = True
return ext_rot, cc_in_cm
def update_mineral_fert(ext_rot, cc_in_cm, rot_id, rots_info, spinup=False):
'for baseline scenario'
for rot in range(len(ext_rot)):
update_fert_values(ext_rot[rot],
copy.deepcopy(rots_info[int(rot_id)]),
cc_in_cm[rot],
expected_N_availability,
mineralN_split,
soil_type,
orgN_kreise[kreis_id],
spinup)
def compose_rotation(ext_rot): #crop["cropRotation"] = composed_rot
composed_rot = []
for rot in ext_rot:
for cm in rot:
composed_rot.append(cm)
return composed_rot
def rotate(crop_rotation):
"rotate the crops in the rotation"
crop_rotation.insert(0, crop_rotation.pop())
def set_params(env, params):
'set values of sensitive params'
soilorg_params = env['params']['userSoilOrganicParameters']["DEFAULT"]
crop_rotations = env["cropRotations"]
soilorg_params["PartSOM_Fast_to_SOM_Slow"][0] = params["PartSOM_Fast_to_SOM_Slow"]
soilorg_params["SOM_FastDecCoeffStandard"][0] = params["SOM_FastDecCoeffStandard"]
soilorg_params["PartSMB_Fast_to_SOM_Fast"][0] = params["PartSMB_Fast_to_SOM_Fast"]
soilorg_params["PartSMB_Slow_to_SOM_Fast"][0] = params["PartSMB_Slow_to_SOM_Fast"]
soilorg_params["SOM_SlowDecCoeffStandard"][0] = params["SOM_SlowDecCoeffStandard"]
soilorg_params["AOM_SlowUtilizationEfficiency"][0] = params["AOM_SlowUtilizationEfficiency"]
for rot in crop_rotations:
for cm in rot["cropRotation"]:
for ws in cm["worksteps"]:
if ws["type"] == "OrganicFertilization":
ws["parameters"]["AOM_SlowDecCoeffStandard"][0] = params["AOM_SlowDecCoeffStandard"]
ws["parameters"]["PartAOM_to_AOM_Fast"][0] = params["PartAOM_to_AOM_Fast"]
ws["parameters"]["PartAOM_to_AOM_Slow"][0] = params["PartAOM_to_AOM_Slow"]
#elif ws["type"] == "Sowing":
# ws["crop"]["residueParams"]["PartAOM_to_AOM_Fast"][0] = params["PartAOM_to_AOM_Fast"]
sent_id = 0
start_send = time.clock()
simulated_cells = 0
no_kreis = 0
counter_debug = 0
#bkr2lk = defaultdict(set) #for additional info
#soilty2iniSOC = defaultdict(list) #for additional info
export_lat_lon_coords = False
export_lat_lon_file = None
srows = int(soil_metadata["nrows"])
scellsize = int(soil_metadata["cellsize"])
sxll = int(soil_metadata["xllcorner"])
syll = int(soil_metadata["yllcorner"])
sxll_center = sxll + scellsize // 2
syll_center = syll + scellsize // 2
syul_center = syll_center + (srows - 1)*scellsize
unique_combos = {} #for calibration/SA envs
run_params_id = []
if SOC_STUDY:
#investigate parameter uncertainty
for p_id in param_vals.keys():
run_params_id.append(int(p_id))
run_params_id = sorted(run_params_id)
else:
#use only best parameter set
run_params_id.append(best_p_id)
###Read missing data to be run
file_missing_data = "report_missing_data/report_missing_sims_id" + run_id + ".csv"
rerun_cells = set()
with open(file_missing_data) as _:
reader = csv.reader(_)
reader.next()
for missing_info in reader:
rerun_cells.add(missing_info[2])
for (row, col), gmd in general_metadata.iteritems():
#run only what's needed:
row_col = "{}{:03d}".format(row, col)
if row_col not in rerun_cells:
continue
#if row_col != "487040":
# continue
#test
#if int(row) != 504 or int(col) != 62:
# continue
if (row, col) in soil_ids and (row, col) in bkr_ids and (row, col) in lu_ids:
# get gk3 coordinates for soil row/col
sr_gk3 = sxll_center + col * scellsize
sh_gk3 = syul_center - row * scellsize
if export_lat_lon_coords:
if not export_lat_lon_file:
export_lat_lon_file = open("soil_row_col_to_lat_lon_coords.csv", "w")
export_lat_lon_file.write("row,col,lat,lon\n")
slon, slat = transform(gk3, wgs84, sr_gk3, sh_gk3)
export_lat_lon_file.write(",".join(map(str, [row, col, slat, slon])) + "\n")
sr_gk5, sh_gk5 = transform(gk3, gk5, sr_gk3, sh_gk3)
site["SiteParameters"]["Slope"] = slope_gk5_interpolate(sr_gk5, sh_gk5) / 100.0
#continue
bkr_id = bkr_ids[(row, col)]
########for testing
#if bkr_id != 129:
# continue
soil_id = soil_ids[(row, col)]
meteo_id = meteo_ids[(row, col)]
if (row, col) in kreise_ids:
kreis_id = kreise_ids[(row, col)]
#bkr2lk[bkr_id].add(kreis_id)
else:
no_kreis += 1
print "-----------------------------------------------------------"
print "kreis not found for calculation of organic N, skipping cell"
print "-----------------------------------------------------------"
continue
simulated_cells += 1
KA5_txt = update_soil(row, col)
s_OID = soil_OID(soil_db_con, soil_ids[(row, col)])
light_soils = ["Ss", "Su2", "Su3", "Su4", "St2", "Sl3", "Sl2"]
heavy_soils = ["Tu3", "Tu4", "Lt3", "Ts2", "Tl", "Tu2", "Tt"]
soil_type = "medium"
if KA5_txt in light_soils:
soil_type = "light"
elif KA5_txt in heavy_soils:
soil_type = "heavy"
site["SiteParameters"]["SoilSpecificHumusBalanceCorrection"] = HUMBAL_CORRECTION[soil_type]
#soilty2iniSOC[soil_type].append(site["SiteParameters"]["SoilProfileParameters"][0]["SoilOrganicCarbon"][0])
#continue
#row_col = "{}{:03d}".format(row, col)
#topsoil_carbon[row_col] = site["SiteParameters"]["SoilProfileParameters"][0]["SoilOrganicCarbon"][0]
#continue
for rot_id, rotation in rotations[str(bkr_id)].iteritems():
#retrieve info for calibration/SA
'''
dump_env = False
if (s_OID, meteo_id, rot_id, orgN_kreise[kreis_id]) not in unique_combos.keys():
profs = []
dump_env = True
for prof in range(len(site["SiteParameters"]["SoilProfileParameters"])):
prof_info = {
"SOC": site["SiteParameters"]["SoilProfileParameters"][prof]["SoilOrganicCarbon"][0],
"thickness": site["SiteParameters"]["SoilProfileParameters"][prof]["Thickness"][0],
"N_layers": int(round(site["SiteParameters"]["SoilProfileParameters"][prof]["Thickness"][0] * 10))
}
profs.append(prof_info)
unique_combos[(s_OID, meteo_id, rot_id, orgN_kreise[kreis_id])] = profs
print "added combo OID: {0}, meteo: {1}, rot: {2}, org N: {3}".format(str(s_OID), str(meteo_id), str(rot_id), str(orgN_kreise[kreis_id]))
else:
continue
'''
m_id = str(meteo_id).replace("(", "").replace(")", "").replace(", ", "_")
unique_id = str(s_OID) + "_" + m_id + "_" + str(rot_id) + "_" + str(int(orgN_kreise[kreis_id]))
if SOC_STUDY:
#avoid repeating simulations which will yield the same results
if unique_id not in unique_combos.keys():
unique_combos[unique_id] = True
else:
continue
for p_id in run_params_id:
if p_id in SOC_ini_vals.keys() and unique_id in SOC_ini_vals[p_id].keys():
set_initial_SOC(SOC_ini_vals[p_id][unique_id])# vals at the beginning of spinup
#print("p_id {0}, unique_id {1} - OK!".format(str(p_id), str(unique_id)))
#counter_debug += 1
#print counter_debug
bbb=1
else:
print ("initial SOC not found for p_id: {0} and unique_id: {1}".format(str(p_id), str(unique_id)))
print (row_col)
exit()
#spinup period (1971-2004):
rotation_spinup = rotations_spinup[str(bkr_id)][rot_id]
ext_rot_spinup = extend_rotation(rotation_spinup, 4)
ext_rot_spinup, cc_in_cm_spinup = insert_CC(ext_rot_spinup, cover_crop_spinup, 1)
update_mineral_fert(ext_rot_spinup, cc_in_cm_spinup, rot_id, rots_info_spinup, spinup=True)
#following period (2005-2050):
ext_rot = extend_rotation(rotation, COVER_CROP_FREQ["out-of"])
ext_rot, cc_in_cm = insert_CC(ext_rot, cover_crop, COVER_CROP_FREQ["insert-cc-every"])
if FERT_STRATEGY == "BASE":
update_mineral_fert(ext_rot, cc_in_cm, rot_id, rots_info)
crop_rot1 = compose_rotation(ext_rot_spinup)
crop_rot2 = compose_rotation(ext_rot)
crop["cropRotations"] = [
{
"start": "1971-01-01",
"end": last_date_spinup[str(bkr_id)][rot_id].isoformat(),
"cropRotation": crop_rot1
},
{
"start": (last_date_spinup[str(bkr_id)][rot_id] + timedelta(days=1)).isoformat(),
"end": "2060-12-31",
"cropRotation": crop_rot2
}
]
env = monica_io.create_env_json_from_json_config({
"crop": crop,
"site": site,
"sim": sim,
"climate": ""
})
set_params(env, param_vals[p_id])
#assign amount of organic fertilizer
for sim_period in range(2):
for cultivation_method in env["cropRotations"][sim_period]["cropRotation"]:
for workstep in cultivation_method["worksteps"]:
if workstep["type"] == "OrganicFertilization":
workstep["amount"][0] = calculate_orgfert_amount(orgN_kreise[kreis_id], workstep["parameters"])
#with open("test_crop.json", "w") as _:
# _.write(json.dumps(crop, indent=4))
#climate is read by the server
env["csvViaHeaderOptions"] = sim["climate.csv-options"]
env["csvViaHeaderOptions"]["start-date"] = sim["start-date"]
env["csvViaHeaderOptions"]["end-date"] = sim["end-date"]
env["pathToClimateCSV"] = []
for PATH in PATH_TO_CLIMATE_DATA_DIR:
env["pathToClimateCSV"].append(PATH + "row-" + str(meteo_id[0]) + "/col-" + str(meteo_id[1]) + ".csv")
#read dumped event for test purpose:
#comment this after test
#basepath = os.path.dirname(os.path.abspath(__file__))
#d_env = basepath + "/dumped_envs/" + unique_id + ".json"
#with open(d_env) as _:
# env = json.load(_)
#####
for sim_id, sim_ in sims.iteritems():
if sim_id != PRODUCTION_LEVEL:
continue
env["events"] = sim_["output"]
env["params"]["simulationParameters"]["NitrogenResponseOn"] = sim_["NitrogenResponseOn"]
env["params"]["simulationParameters"]["WaterDeficitResponseOn"] = sim_["WaterDeficitResponseOn"]
env["params"]["simulationParameters"]["UseAutomaticIrrigation"] = sim_["UseAutomaticIrrigation"]
env["params"]["simulationParameters"]["UseNMinMineralFertilisingMethod"] = sim_["UseNMinMineralFertilisingMethod"]
env["params"]["simulationParameters"]["FrostKillOn"] = sim_["FrostKillOn"]
#if dump_env:
# #save the env for calibration/SA
# basepath = os.path.dirname(os.path.abspath(__file__))
# filename = basepath + "/dumped_envs/" + unique_id + ".json"
# with open(filename, "w") as _:
# _.write(json.dumps(env, indent=4))
# print("dumped env: " + unique_id)
for main_cp_iteration in range(0, len(rots_info[int(rot_id)])):
#do not allow crop rotation of sim_period2 to start with a CC
if "is-cover-crop" in env["cropRotations"][1]["cropRotation"][0].keys() and env["cropRotations"][1]["cropRotation"][0]["is-cover-crop"] == True:
rotate(env["cropRotations"][1]["cropRotation"])
env["customId"] = rot_id \
+ "|" + sim_id \
+ "|" + str(soil_id) \
+ "|(" + str(row) + "/" + str(col) + ")" \
+ "|" + str(bkr_id) \
+ "|" + str(main_cp_iteration) \
+ "|" + str(sim["UseSecondaryYields"]) \
+ "|" + str(timeframes[TF]["start-recording-out"]) \
+ "|" + str(RESIDUES_HUMUS_BALANCE) \
+ "|" + suffix \
+ "|" + KA5_txt \
+ "|" + soil_type \
+ "|" + str(p_id) \
+ "|" + str(orgN_kreise[kreis_id]) \
+ "|" + unique_id
socket.send_json(env)
print "sent env ", sent_id, " customId: ", env["customId"]
#exit()
sent_id += 1
rotate(env["cropRotations"][1]["cropRotation"]) #only simperiod2 is rotated
#exit()
#print(len(unique_combos.keys()))
if export_lat_lon_file:
export_lat_lon_file.close()
stop_send = time.clock()
print "sending ", sent_id, " envs took ", (stop_send - start_send), " seconds"
print "simulated cells: ", simulated_cells, "; not found kreise for org N: ", no_kreis
'''
def main():
"main"
context = zmq.Context()
socket = context.socket(zmq.PUSH)
#port = 6666 if len(sys.argv) == 1 else sys.argv[1]
config = {"port": 6666, "start": 1, "end": 8157}
if len(sys.argv) > 1:
for arg in sys.argv[1:]:
k, v = arg.split("=")
if k in config:
config[k] = int(v)
if LOCAL_RUN:
socket.connect("tcp://localhost:" + str(config["port"]))
else:
socket.connect("tcp://cluster2:" + str(config["port"]))
with open("sim.json") as _:
sim = json.load(_)
with open("site.json") as _:
site = json.load(_)
with open("crop.json") as _:
crop = json.load(_)
with open("sims.json") as _:
sims = json.load(_)
sim["include-file-base-path"] = PATHS[USER]["INCLUDE_FILE_BASE_PATH"]
period_gcms = [
{
"grcp": "0",
"period": "0",
"gcm-rcp": "0_0"
},
{
"grcp": "1",
"period": "2",
"gcm-rcp": "GFDL-CM3_45"
},
{
"grcp": "2",
"period": "2",
"gcm-rcp": "GFDL-CM3_85"
},
{
"grcp": "3",
"period": "2",
"gcm-rcp": "GISS-E2-R_45"
},
{
"grcp": "4",
"period": "2",
"gcm-rcp": "GISS-E2-R_85"
},
{
"grcp": "5",
"period": "2",
"gcm-rcp": "HadGEM2-ES_26"
},
{
"grcp": "6",
"period": "2",
"gcm-rcp": "HadGEM2-ES_45"
},
{
"grcp": "7",
"period": "2",
"gcm-rcp": "HadGEM2-ES_85"
},
{
"grcp": "8",
"period": "2",
"gcm-rcp": "MIROC5_45"
},
{
"grcp": "9",
"period": "2",
"gcm-rcp": "MIROC5_85"
},
{
"grcp": "10",
"period": "2",
"gcm-rcp": "MPI-ESM-MR_26"
},
{
"grcp": "11",
"period": "2",
"gcm-rcp": "MPI-ESM-MR_45"
},
{
"grcp": "12",
"period": "2",
"gcm-rcp": "MPI-ESM-MR_85"
},
{
"grcp": "1",
"period": "3",
"gcm-rcp": "GFDL-CM3_45"
},
{
"grcp": "2",
"period": "3",
"gcm-rcp": "GFDL-CM3_85"
},
{
"grcp": "3",
"period": "3",
"gcm-rcp": "GISS-E2-R_45"
},
{
"grcp": "4",
"period": "3",
"gcm-rcp": "GISS-E2-R_85"
},
{
"grcp": "5",
"period": "3",
"gcm-rcp": "HadGEM2-ES_26"
},
{
"grcp": "6",
"period": "3",
"gcm-rcp": "HadGEM2-ES_45"
},
{
"grcp": "7",
"period": "3",
"gcm-rcp": "HadGEM2-ES_85"
},
{
"grcp": "8",
"period": "3",
"gcm-rcp": "MIROC5_45"
},
{
"grcp": "9",
"period": "3",
"gcm-rcp": "MIROC5_85"
},
{
"grcp": "10",
"period": "3",
"gcm-rcp": "MPI-ESM-MR_26"
},
{
"grcp": "11",
"period": "3",
"gcm-rcp": "MPI-ESM-MR_45"
},
{
"grcp": "12",
"period": "3",
"gcm-rcp": "MPI-ESM-MR_85"
},
]
start_year = {"0": "1980", "2": "2040", "3": "2070"}
def read_latitude(filename):
""
defs = defaultdict(list)
lats = {}
with open(filename) as _:
reader = csv.reader(_)
reader.next()
for row in reader:
res = int(row[1])
if res < 25:
continue
row_col = (int(row[2]), int(row[3]))
lat = float(row[13])
lats[row_col] = lat
defs[res].append(lat)
for res, vals in defs.iteritems():
lats[(res, -1)] = sum(vals) / len(vals)
return lats
latitudes = read_latitude(PATHS[USER]["LOCAL_ARCHIVE_PATH_TO_PROJECT"] +
"Soil_data/add_x_y_lon_lat_ele_centred.csv")
def read_lookup_file(filename):
""
lll = []
with open(filename) as _:
reader = csv.reader(_)
reader.next()
for row in reader:
lll.append({
25: (int(row[0]), int(row[1])),
50: (int(row[2]), int(row[3])),
100: (int(row[4]), int(row[5]))
})
return lll
lookup = read_lookup_file(PATHS[USER]["LOCAL_ARCHIVE_PATH_TO_PROJECT"] +
"/unit_description/MCSUR_NRW_CC_cell_lookup.csv")
def read_soil_properties(filename):
""
soil = defaultdict(list)
with open(filename) as _:
reader = csv.reader(_)
reader.next()
for row in reader:
row_col = (int(row[3]), int(row[2]))
#skip horizons with no properties defined
if float(row[45]) <= 0.00001:
continue
#avoid BD to be too low (soilT crashes)
bulk_d = max(600.0, float(row[32]) * 1000)
soil[row_col].append({
"depth": float(row[15]),
"thickness": float(row[16]),
"pwp": float(row[44]),
"fc": float(row[45]),
"sat": float(row[46]),
"corg": float(row[34]),
"bulk-density": bulk_d,
"sand": float(row[29]) / 100.0,
"clay": float(row[27]) / 100.0,
"ph": float(row[36]),
"sceleton": float(row[31]) / 100.0
})
return soil
soil = {
25:
read_soil_properties(PATHS[USER]["LOCAL_ARCHIVE_PATH_TO_PROJECT"] +
"Soil_data/climate_change_NRW_soil_r25.csv"),
50:
read_soil_properties(PATHS[USER]["LOCAL_ARCHIVE_PATH_TO_PROJECT"] +
"Soil_data/climate_change_NRW_soil_r50.csv"),
100:
read_soil_properties(PATHS[USER]["LOCAL_ARCHIVE_PATH_TO_PROJECT"] +
"Soil_data/climate_change_NRW_soil_r100.csv")
}
def update_soil(soil_res, row, col, crop_id):
"update function"
crop["cropRotation"][2] = crop_id
soil_layers = soil[soil_res][(row, col)]
site["Latitude"] = latitudes[(row, col)] if (
row, col) in latitudes else latitudes[(soil_res, -1)]
layers = []
profile_depth = 0
for iii, layer in enumerate(soil_layers):
lll = {
"SoilOrganicCarbon": [layer["corg"], "%"],
"SoilBulkDensity": [layer["bulk-density"], "kg m-3"],
"FieldCapacity": [layer["fc"], "m3 m-3"],
"PermanentWiltingPoint": [layer["pwp"], "m3 m-3"],
"PoreVolume": [layer["sat"], "m3 m-3"],
"SoilMoisturePercentFC":
[80.0 if crop_id == "M" else 50.0, "% [0-100]"],
"Sand":
layer["sand"],
"Clay":
layer["clay"],
"Sceleton":
layer["sceleton"],
"pH":
layer["ph"],
"Thickness":
layer["thickness"]
}
profile_depth = layer["depth"]
#!!!! danger, this is changing in memory sims events structure and events can only then be assigned to env
sims["output"][crop_id][1][6][1][1] = int(
min(math.ceil(profile_depth * 10), 20))
layers.append(lll)
site["SiteParameters"]["SoilProfileParameters"] = layers
return profile_depth
#print site["SiteParameters"]["SoilProfileParameters"]
#assert len(row_cols) == len(pheno["GM"].keys()) == len(pheno["WW"].keys())
#print "# of rowsCols = ", len(row_cols)
climate_to_soil = [{
"step": 1,
"climate": 25,
"soil": 25
}, {
"step": 1,
"climate": 25,
"soil": 50
}, {
"step": 1,
"climate": 25,
"soil": 100
}, {
"step": 2,
"climate": 50,
"soil": 25
}, {
"step": 2,
"climate": 100,
"soil": 25
}, {
"step": 3,
"climate": 50,
"soil": 50
}, {
"step": 3,
"climate": 100,
"soil": 100
}]
production_situations = {
"PLP": {
"water-response": False,
"nitrogen-response": False
},
"PLW": {
"water-response": True,
"nitrogen-response": False
},
"PLN": {
"water-response": True,
"nitrogen-response": True
}
}
i = 1
start_store = time.clock()
#start = config["start"] - 1
#end = config["end"] - 1
#row_cols_ = row_cols[start:end+1]
#print "running from ", start, "/", row_cols[start], " to ", end, "/", row_cols[end]
for c2s in climate_to_soil:
step = c2s["step"]
climate_resolution = c2s["climate"]
soil_resolution = c2s["soil"]
climate_to_soils = defaultdict(set)
for mmm in lookup:
climate_to_soils[mmm[climate_resolution]].add(mmm[soil_resolution])
print "step: ", step, " ", sum(
[len(s[1]) for s in climate_to_soils.iteritems()]
), " runs in climate_res: ", climate_resolution, " and soil_res: ", soil_resolution
for (climate_row,
climate_col), soil_coords in climate_to_soils.iteritems():
#if (climate_row, climate_col) != (6,6):
# continue
for soil_row, soil_col in soil_coords:
#if (soil_row, soil_col) != (11,11):
# continue
for crop_id in ["W", "M"]:
rootdepth_soillimited = update_soil(
soil_resolution, soil_row, soil_col, crop_id)
env = monica_io.create_env_json_from_json_config({
"crop":
crop,
"site":
site,
"sim":
sim,
"climate":
""
})
env["csvViaHeaderOptions"] = sim["climate.csv-options"]
env["events"] = sims["output"][crop_id]
for workstep in env["cropRotation"][0]["worksteps"]:
if workstep["type"] == "Seed":
current_rootdepth = float(
workstep["crop"]["cropParams"]["cultivar"]
["CropSpecificMaxRootingDepth"])
workstep["crop"]["cropParams"]["cultivar"][
"CropSpecificMaxRootingDepth"] = min(
current_rootdepth, rootdepth_soillimited)
break
for production_id, switches in production_situations.iteritems(
):
env["params"]["simulationParameters"][
"WaterDeficitResponseOn"] = switches[
"water-response"]
env["params"]["simulationParameters"][
"NitrogenResponseOn"] = switches[
"nitrogen-response"]
for period_gcm in period_gcms:
grcp = period_gcm["grcp"]
period = period_gcm["period"]
gcm = period_gcm["gcm-rcp"]
#if period != "0":
# continue
#if climate_resolution != 50:
# continue
#if soil_resolution != 25:
# continue
#if crop_id != "W":
# continue
if period != "0":
climate_filename = "daily_mean_P{}_GRCP_{}_RES{}_C0{}R{}.csv".format(
period, grcp, climate_resolution,
climate_col, climate_row)
else:
climate_filename = "daily_mean_P{}_RES{}_C0{}R{}.csv".format(
period, climate_resolution, climate_col,
climate_row)
if LOCAL_RUN:
env["pathToClimateCSV"] = PATHS[USER][
"LOCAL_ARCHIVE_PATH_TO_PROJECT"] + "Climate_data/NRW_weather_climate_change_v3/res_" + str(
climate_resolution
) + "/period_" + period + "/GRCP_" + grcp + "/" + climate_filename
else:
env["pathToClimateCSV"] = PATHS[USER][
"ARCHIVE_PATH_TO_PROJECT"] + "Climate_data/NRW_weather_climate_change_v3/res_" + str(
climate_resolution
) + "/period_" + period + "/GRCP_" + grcp + "/" + climate_filename
#initialize nitrate/ammonium in soil layers at start of simulation
#for i in range(3):
# env["cropRotation"][0]["worksteps"][i]["date"] = start_year[period] + "-01-01"
env["customId"] = crop_id \
+ "|" + str(climate_resolution) \
+ "|(" + str(climate_row) + "/" + str(climate_col) + ")" \
+ "|" + str(soil_resolution) \
+ "|(" + str(soil_row) + "/" + str(soil_col) + ")" \
+ "|" + period \
+ "|" + grcp \
+ "|" + gcm \
+ "|" + production_id
#with open("envs/env-"+str(i)+".json", "w") as _:
# _.write(json.dumps(env))
socket.send_json(env)
print "sent env ", i, " customId: ", env[
"customId"]
#exit()
i += 1
stop_store = time.clock()
print "sending ", (i - 1), " envs took ", (stop_store -
start_store), " seconds"
#print "ran from ", start, "/", row_cols[start], " to ", end, "/", row_cols[end]
return
def run_producer():
"main function"
#site_name = "FI"
site_name = "FI"
simulations = ["F"]
simulation_dir = "simulation/" + site_name + "/"
context = zmq.Context()
socket = context.socket(zmq.PUSH)
config = {"user": "******", "port": "66666", "server": "localhost"}
if len(sys.argv) > 1:
for arg in sys.argv[1:]:
k, v = arg.split("=")
if k in config:
config[k] = v
paths = PATHS[config["user"]]
socket.connect("tcp://" + config["server"] + ":" + str(config["port"]))
for sim_type in simulations:
c_files = climate_files[sim_type][site_name]
co2_values = climate_files[sim_type]["CO2"]
print c_files
for climate_file, co2 in zip(c_files, co2_values):
output_file = "O" + os.path.splitext(
os.path.basename(climate_file))[0] + ".txt"
if sim_type == "C":
output_file = "O" + os.path.splitext(
os.path.basename(climate_file))[0] + str(co2) + ".txt"
if sim_type == "F":
output_file = "MO_FU2050_YR_" + os.path.splitext(
os.path.basename(
climate_file))[0] + "_" + site_name + ".txt"
with open(simulation_dir + "sim-" + site_name + ".json") as _:
sim = json.load(_)
with open(simulation_dir + "site-" + site_name + ".json") as _:
site = json.load(_)
with open(simulation_dir + "crop-" + site_name + ".json") as _:
crop = json.load(_)
sim["include-file-base-path"] = paths["INCLUDE_FILE_BASE_PATH"]
if sim_type == "F":
sim["start-date"] = timeframes[site_name][climate_file][
"start-date"]
sim["end-date"] = timeframes[site_name][climate_file][
"end-date"]
sent_id = 0
start_send = time.clock()
env = monica_io.create_env_json_from_json_config({
"crop": crop,
"site": site,
"sim": sim,
"climate": ""
})
#monica_io.add_climate_data_to_env(env, sim)
env["csvViaHeaderOptions"] = sim["climate.csv-options"]
env["csvViaHeaderOptions"]["start-date"] = sim["start-date"]
env["csvViaHeaderOptions"]["end-date"] = sim["end-date"]
env["pathToClimateCSV"] = []
climate_path = paths[
"INCLUDE_FILE_BASE_PATH"] + simulation_dir + "climate/" + climate_file + ".csv"
print(climate_path)
env["pathToClimateCSV"].append(climate_path)
env["params"]["userEnvironmentParameters"]["AtmosphericCO2"] = co2
env["customId"] = {
"id": "BCD3",
"site": site_name,
"output_filename": output_file
}
#{
#"id": "BCD3",
#"site_name": site_name
#}
socket.send_json(env)
print("sent env ", sent_id, " customId: ", env["customId"])
sent_id += 1
stop_send = time.clock()
print("sending ", sent_id, " envs took ", (stop_send - start_send),
" seconds")
def __init__(self, exp_maps, host):
self.host = host
#for multi-experiment: create a M-2 relationship between exp_IDs and param files
self.IDs_paramspaths = {}
for exp_map in exp_maps:
self.IDs_paramspaths[exp_map["exp_ID"]] = {}
self.IDs_paramspaths[exp_map["exp_ID"]]["species"] = exp_map["species_file"]
self.IDs_paramspaths[exp_map["exp_ID"]]["cultivar"] = exp_map["cultivar_file"]
#load rules (for MONICA output)
with open("calibration_rules.json") as _:
self.calibration_rules = json.load(_)
#data structures for spotpy
self.expected_outcome = [] #former "observations"; stores success scores
self.simulation_outcome = [] #former "simulations"; stores outcome after checking rules
self.exp_ids = [] #mirrors spotpy data structure to provide the user with info on the exp id (success/failure)
self.checked_rules = [] #mirrors spotpy data structure to provide the user with info on the checked rule (success/failure)
self.species_params={} #map to store different species params sets avoiding repetition
self.cultivar_params={} #map to store different cultivar params sets avoiding repetition
#create envs
self.envs = []
for exp_map in exp_maps:
with open(exp_map["sim_file"]) as simfile:
sim = json.load(simfile)
sim["crop.json"] = exp_map["crop_file"]
sim["site.json"] = exp_map["site_file"]
#sim["climate.csv"] = exp_map["climate_file"]
with open(exp_map["site_file"]) as sitefile:
site = json.load(sitefile)
with open(exp_map["crop_file"]) as cropfile:
crop = json.load(cropfile)
mycrop = exp_map["crop_ID"]
crop["crops"][mycrop]["cropParams"]["species"][1] = exp_map["species_file"]
crop["crops"][mycrop]["cropParams"]["cultivar"][1] = exp_map["cultivar_file"]
env = monica_io.create_env_json_from_json_config({
"crop": crop,
"site": site,
"sim": sim,
"climate": ""
})
#customize monica output
env["events"] = []
for rule_id, rule_specs in self.calibration_rules.iteritems():
env["events"].append(rule_specs["custom event"][0]) #customize output
env["events"].append(rule_specs["custom event"][1])
#climate is read by the server
env["csvViaHeaderOptions"] = sim["climate.csv-options"]
env["csvViaHeaderOptions"]["start-date"] = sim["climate.csv-options"]["start-date"]
env["csvViaHeaderOptions"]["end-date"] = sim["climate.csv-options"]["end-date"]
env["pathToClimateCSV"] = []
env["pathToClimateCSV"].append(exp_map["climate_file"])
#monica_io.add_climate_data_to_env(env, sim) this might not be supported anymore
for ws in env["cropRotation"][0]["worksteps"]:
if ws["type"] == "Seed" or ws["type"] == "Sowing":
self.species_params[exp_map["species_file"]] = ws["crop"]["cropParams"]["species"]
self.cultivar_params[exp_map["cultivar_file"]] = ws["crop"]["cropParams"]["cultivar"]
break
env["customId"] = exp_map["exp_ID"]
self.envs.append(env)
self.context = zmq.Context()
self.socket_producer = self.context.socket(zmq.PUSH)
self.socket_producer.connect("tcp://" + self.host["server"] + ":" + self.host["push-port"])
def __init__(self, exp_maps, obslist):
#for multi-experiment: create a M-2 relationship between exp_IDs and param files
self.IDs_paramspaths = {}
for exp_map in exp_maps:
self.IDs_paramspaths[exp_map["exp_ID"]] = {}
self.IDs_paramspaths[
exp_map["exp_ID"]]["species"] = exp_map["species_file"]
self.IDs_paramspaths[
exp_map["exp_ID"]]["cultivar"] = exp_map["cultivar_file"]
#custom events (for MONICA output)
custom_events = defaultdict(list)
with open("template_events.json") as _:
template_events = json.load(_)
#observations data structures
self.observations = [] #for spotpy
for record in obslist:
my_event = deepcopy(
template_events[record["stage"]]) #deepcopy just to be sure :)
if record["stage"] == "Zadok12":
#special case, "Stage" (=2) instead of "DOY" is used
self.observations.append(2)
my_event[0] = unicode(record["date"].isoformat())
else:
self.observations.append(record["DOY"])
custom_events[record["exp_ID"]].append(my_event[0])
custom_events[record["exp_ID"]].append(my_event[1])
self.species_params = {
} #map to store different species params sets avoiding repetition
self.cultivar_params = {
} #map to store different cultivar params sets avoiding repetition
#create envs
self.envs = []
for exp_map in exp_maps:
with open(exp_map["sim_file"]) as simfile:
sim = json.load(simfile)
#sim["events"] = custom_events[exp_map["exp_ID"]] #customize output, if here monica.io will ignore it (why?)
#with open("test_sim_events.json", "w") as out_f:
# json.dump(sim, out_f, indent=4)
sim["crop.json"] = exp_map["crop_file"]
sim["site.json"] = exp_map["site_file"]
sim["EmergenceMoistureControlOn"] = True
sim["EmergenceFloodingControlOn"] = True
#sim["climate.csv"] = exp_map["climate_file"]
with open(exp_map["site_file"]) as sitefile:
site = json.load(sitefile)
with open(exp_map["crop_file"]) as cropfile:
crop = json.load(cropfile)
mycrop = exp_map["crop_ID"]
crop["crops"][mycrop]["cropParams"]["species"][1] = exp_map[
"species_file"]
crop["crops"][mycrop]["cropParams"]["cultivar"][1] = exp_map[
"cultivar_file"]
env = monica_io.create_env_json_from_json_config({
"crop": crop,
"site": site,
"sim": sim,
"climate": ""
})
env["events"] = custom_events[exp_map["exp_ID"]] #customize output
#climate is read by the server
env["csvViaHeaderOptions"] = sim["climate.csv-options"]
env["csvViaHeaderOptions"]["start-date"] = sim[
"climate.csv-options"]["start-date"]
env["csvViaHeaderOptions"]["end-date"] = sim[
"climate.csv-options"]["end-date"]
env["pathToClimateCSV"] = []
env["pathToClimateCSV"].append(exp_map["climate_file"])
position = int(exp_map["where_in_rotation"][0])
for position in exp_map["where_in_rotation"]:
for ws in env["cropRotation"][position]["worksteps"]:
if ws["type"] == "Seed" or ws["type"] == "Sowing":
self.species_params[exp_map["species_file"]] = ws[
"crop"]["cropParams"]["species"]
self.cultivar_params[exp_map["cultivar_file"]] = ws[
"crop"]["cropParams"]["cultivar"]
break
#monica_io.add_climate_data_to_env(env, sim) this does not work anymore properly
env["customId"] = exp_map["exp_ID"]
env["where_in_rotation"] = exp_map["where_in_rotation"]
self.envs.append(env)
self.context = zmq.Context()
self.socket_producer = self.context.socket(zmq.PUSH)
#self.socket_producer.connect("tcp://cluster2:6666")
self.socket_producer.connect("tcp://localhost:6666")
def main():
"main function"
config = {
"port": 6666,
}
if len(sys.argv) > 1:
for arg in sys.argv[1:]:
k,v = arg.split("=")
if k in config:
config[k] = int(v)
context = zmq.Context()
socket = context.socket(zmq.PUSH)
if LOCAL_RUN:
socket.connect("tcp://localhost:" + str(config["port"]))
else:
socket.connect("tcp://cluster" + str(RUN_ON_CLUSTER) + ":" + str(config["port"]))
soil_db_con = sqlite3.connect("soil.sqlite")
with open("sim-voce.json") as _:
sim = json.load(_)
with open("site-voce.json") as _:
site = json.load(_)
with open("crop-voce.json") as _:
crop = json.load(_)
with open("rotations-voce.json") as _:
rotations = json.load(_)
sim["include-file-base-path"] = PATHS[USER]["INCLUDE_FILE_BASE_PATH"]
def read_general_metadata(path_to_file):
"read metadata file"
with open(path_to_file) as file_:
data = {}
reader = csv.reader(file_, delimiter="\t")
reader.next()
for row in reader:
if int(row[1]) != 1:
continue
data[(int(row[2]), int(row[3]))] = {
"subpath-climate.csv": row[9],
"latitude": float(row[13]),
"elevation": float(row[14])
}
return data
general_metadata = read_general_metadata("NRW_General_Metadata.csv")
def load_mapping(row_offset=0, col_offset=0):
to_climate_index = {}
with(open("working_resolution_to_climate_lat_lon_indices.json")) as _:
l = json.load(_)
for i in xrange(0, len(l), 2):
cell = (row_offset + l[i][0], col_offset + l[i][1])
to_climate_index[cell] = tuple(l[i+1])
return to_climate_index
def read_orgN_kreise(path_to_file):
"read organic N info for kreise"
with open(path_to_file) as file_:
data = {}
reader = csv.reader(file_, delimiter=",")
reader.next()
reader.next()
for row in reader:
for kreis_code in row[1].split("|"):
if kreis_code != "":
data[int(kreis_code)] = float(row[8])
return data
orgN_kreise = read_orgN_kreise("NRW_orgN_balance.csv")
#print orgN_kreise.keys()
def update_soil_crop_dates(row, col):
"in place update the env"
#startDate = date(1980, 1, 1)# + timedelta(days = p["sowing-doy"])
#sim["start-date"] = startDate.isoformat()
#sim["end-date"] = date(2010, 12, 31).isoformat()
#sim["debug?"] = True
site["Latitude"] = general_metadata[(row, col)]["latitude"]
site["HeightNN"] = [general_metadata[(row, col)]["elevation"], "m"]
site["SiteParameters"]["SoilProfileParameters"] = soil_io.soil_parameters(soil_db_con, soil_ids[(row, col)])
for layer in site["SiteParameters"]["SoilProfileParameters"]:
layer["SoilBulkDensity"][0] = max(layer["SoilBulkDensity"][0], 600)
def read_ascii_grid(path_to_file, include_no_data=False, row_offset=0, col_offset=0):
"read an ascii grid into a map, without the no-data values"
def int_or_float(s):
try:
return int(s)
except ValueError:
return float(s)
with open(path_to_file) as file_:
data = {}
#skip the header (first 6 lines)
for _ in range(0, 6):
file_.next()
row = -1
for line in file_:
row += 1
col = -1
for col_str in line.strip().split(" "):
col += 1
if not include_no_data and int_or_float(col_str) == -9999:
continue
data[(row_offset+row, col_offset+col)] = int_or_float(col_str)
return data
#offset is used to match info in general metadata and soil database
soil_ids = read_ascii_grid("soil-profile-id_nrw_gk3.asc", row_offset=282)
bkr_ids = read_ascii_grid("bkr_nrw_gk3.asc", row_offset=282)
lu_ids = read_ascii_grid("lu_resampled.asc", row_offset=282)
kreise_ids = read_ascii_grid("kreise_matrix.asc", row_offset=282)
meteo_ids = load_mapping(row_offset=282)
#counter = 0
#for k, v in bkr_ids.iteritems():
# if v == 134:
# if k in lu_ids:
# counter += 1
#print counter
i = 0
start_send = time.clock()
def calculate_orgfert_amount(N_applied, fert_type):
"convert N applied in amount of fresh org fert"
AOM_DryMatterContent = fert_type["AOM_DryMatterContent"][0]
AOM_NH4Content = fert_type["AOM_NH4Content"][0]
AOM_NO3Content = fert_type["AOM_NO3Content"][0]
CN_Ratio_AOM_Fast = fert_type["CN_Ratio_AOM_Fast"][0]
CN_Ratio_AOM_Slow = fert_type["CN_Ratio_AOM_Slow"][0]
PartAOM_to_AOM_Fast = fert_type["PartAOM_to_AOM_Fast"][0]
PartAOM_to_AOM_Slow = fert_type["PartAOM_to_AOM_Slow"][0]
AOM_to_C = 0.45
AOM_fast_factor = 1/(CN_Ratio_AOM_Fast/(AOM_to_C * PartAOM_to_AOM_Fast))
AOM_slow_factor = 1/(CN_Ratio_AOM_Slow/(AOM_to_C * PartAOM_to_AOM_Slow))
conversion_coeff = AOM_NH4Content + AOM_NO3Content + AOM_fast_factor + AOM_slow_factor
AOM_dry = N_applied / conversion_coeff
AOM_fresh = AOM_dry / AOM_DryMatterContent
return AOM_fresh
simulated_cells = 0
no_kreis = 0
for (row, col), gmd in general_metadata.iteritems():
if (row, col) in soil_ids and (row, col) in bkr_ids and (row, col) in lu_ids:
bkr_id = bkr_ids[(row, col)]
#if bkr_id != 143:
# continue
soil_id = soil_ids[(row, col)]
meteo_id = meteo_ids[(row, col)]
if (row, col) in kreise_ids:
kreis_id = kreise_ids[(row, col)]
else:
no_kreis += 1
print "-----------------------------------------------------"
print "kreis not found for calculation of organic N" #TODO find out a solution
print "-----------------------------------------------------"
simulated_cells += 1
update_soil_crop_dates(row, col)
for rot_id, rotation in rotations.iteritems():
crop["cropRotation"] = rotation
env = monica_io.create_env_json_from_json_config({
"crop": crop,
"site": site,
"sim": sim,
"climate": ""
})
#assign amount of organic fertilizer
#for cultivation_method in env["cropRotation"]:
# for workstep in cultivation_method["worksteps"]:
# if workstep["type"] == "OrganicFertilization":
# workstep["amount"] = calculate_orgfert_amount(orgN_kreise[kreis_id], workstep["parameters"])
#climate is read by the server
env["csvViaHeaderOptions"] = sim["climate.csv-options"]
if LOCAL_RUN:
env["pathToClimateCSV"] = LOCAL_PATH_TO_CLIMATE_DATA_DIR + "row-" + str(meteo_id[0]) + "/col-" + str(meteo_id[1]) + ".csv"
else:
env["pathToClimateCSV"] = PATH_TO_CLIMATE_DATA_DIR + "row-" + str(meteo_id[0]) + "/col-" + str(meteo_id[1]) + ".csv"
#env["pathToClimateCSV"] = PATH_TO_CLIMATE_DATA_DIR + gmd["subpath-climate.csv"]
env["customId"] = rot_id \
+ "|" + str(soil_id) \
+ "|(" + str(row) + "/" + str(col) + ")" \
+ "|" + str(bkr_id)
socket.send_json(env)
print "sent env ", i, " customId: ", env["customId"]
#if i > 10:
# exit()
i += 1
stop_send = time.clock()
print "sending ", i, " envs took ", (stop_send - start_send), " seconds"
print "simulated cells: ", simulated_cells, "; not found kreise for org N: ", no_kreis
def __init__(self, exp_maps, observations, config, finalrun):
#read params - they will be modified in the run method
with open(config["soilorg_params_path"]) as _:
self.soil_organic_params = json.load(_)
with open(config["fertorg_params_path"]) as _:
self.fert_organic_params = json.load(_)
#following params were calibrated using exps 25 and 35 from V140 Muencheberg
self.fert_organic_params["PartAOM_to_AOM_Fast"][0] = 0.81
self.fert_organic_params["PartAOM_to_AOM_Slow"][0] = 0.19
#observations data structures
self.observations = observations
self.obslist = helper.create_spotpylist(self.observations)
#create envs
self.envs = []
for exp_map in exp_maps:
with open(exp_map["sim_file"]) as simfile:
sim = json.load(simfile)
sim["crop.json"] = exp_map["crop_file"]
sim["site.json"] = exp_map["site_file"]
sim["climate.csv"] = exp_map["climate_file"]
with open(exp_map["site_file"]) as sitefile:
site = json.load(sitefile)
with open(exp_map["crop_file"]) as cropfile:
crop = json.load(cropfile)
crop["cropRotations"] = []
env = monica_io.create_env_json_from_json_config({
"crop": crop,
"site": site,
"sim": sim
})
#create references to self.soil_organic_params and self.fert_organic_params
env["params"][
"userSoilOrganicParameters"] = self.soil_organic_params
for cm in env["cropRotation"]:
for ws in cm["worksteps"]:
if ws["type"] == "OrganicFertilization":
ws["parameters"] = self.fert_organic_params
#customize events section
env["events"] = []
with open(config["events-file"]) as _:
json_out = json.load(_)
env["events"] = json_out["events"]
#climate is read by the server
env["csvViaHeaderOptions"] = sim["climate.csv-options"]
env["pathToClimateCSV"] = []
if config["server"] == "localhost":
env["pathToClimateCSV"].append(sim["climate.csv"])
else:
local_path_to_climate = os.path.dirname(
os.path.abspath(__file__)) + "\\climate_files"
cluster_path_to_climate = "/archiv-daten/md/projects/sustag/climate_files_uncertainty_analysis"
env["pathToClimateCSV"].append(sim["climate.csv"].replace(
local_path_to_climate,
cluster_path_to_climate).replace("\\", "/"))
#print env["pathToClimateCSV"][0]
env["customId"] = str(exp_map["treatment"]) + "|" + str(
exp_map["parcel"]) + "|" + config["events-file"]
self.envs.append(env)
#open sockets
self.context = zmq.Context()
self.socket_push = self.context.socket(zmq.PUSH)
s_push = "tcp://" + config["server"] + ":" + config["push-port"]
self.socket_push.connect(s_push)
self.socket_pull = self.context.socket(zmq.PULL)
s_pull = "tcp://" + config["server"] + ":" + config["pull-port"]
self.socket_pull.connect(s_pull)
def __init__(self, exp_maps, obslist):
#for multi-experiment: create a M-2 relationship between exp_IDs and param files
self.IDs_paramspaths = {}
for exp_map in exp_maps:
self.IDs_paramspaths[exp_map["exp_ID"]] = {}
self.IDs_paramspaths[exp_map["exp_ID"]]["species"] = exp_map["species_file"]
self.IDs_paramspaths[exp_map["exp_ID"]]["cultivar"] = exp_map["cultivar_file"]
#observations data structures
self.observations = [] #for spotpy
for record in obslist:
self.observations.append(record["value"])
self.species_params={} #map to store different species params sets avoiding repetition
self.cultivar_params={} #map to store different cultivar params sets avoiding repetition
#create envs
self.envs = []
for exp_map in exp_maps:
with open(exp_map["sim_file"]) as simfile:
sim = json.load(simfile)
sim["crop.json"] = exp_map["crop_file"]
sim["site.json"] = exp_map["site_file"]
#sim["climate.csv"] = exp_map["climate_file"]
with open(exp_map["site_file"]) as sitefile:
site = json.load(sitefile)
with open(exp_map["crop_file"]) as cropfile:
crop = json.load(cropfile)
mycrop = exp_map["crop_ID"]
crop["crops"][mycrop]["cropParams"]["species"][1] = exp_map["species_file"]
crop["crops"][mycrop]["cropParams"]["cultivar"][1] = exp_map["cultivar_file"]
env = monica_io.create_env_json_from_json_config({
"crop": crop,
"site": site,
"sim": sim,
"climate": ""
})
#climate is read by the server
env["csvViaHeaderOptions"] = sim["climate.csv-options"]
env["csvViaHeaderOptions"]["start-date"] = sim["climate.csv-options"]["start-date"]
env["csvViaHeaderOptions"]["end-date"] = sim["climate.csv-options"]["end-date"]
env["pathToClimateCSV"] = []
env["pathToClimateCSV"].append(exp_map["climate_file"])
position = int(exp_map["where_in_rotation"][0])
for position in exp_map["where_in_rotation"]:
for ws in env["cropRotation"][position]["worksteps"]:
if ws["type"] == "Seed" or ws["type"] == "Sowing":
self.species_params[exp_map["species_file"]] = ws["crop"]["cropParams"]["species"]
self.cultivar_params[exp_map["cultivar_file"]] = ws["crop"]["cropParams"]["cultivar"]
break
#monica_io.add_climate_data_to_env(env, sim) this does not work anymore properly
env["customId"] = exp_map["exp_ID"]
env["where_in_rotation"] = exp_map["where_in_rotation"]
self.envs.append(env)
self.context = zmq.Context()
self.socket_producer = self.context.socket(zmq.PUSH)
#self.socket_producer.connect("tcp://cluster2:6666")
self.socket_producer.connect("tcp://localhost:6666")
def __init__(self, exp_maps, obslist, finalrun):
#for multi-experiment: create a M-2 relationship between exp_IDs and param files
self.IDs_paramspaths = {}
for exp_map in exp_maps:
self.IDs_paramspaths[exp_map["exp_ID"]] = {}
self.IDs_paramspaths[exp_map["exp_ID"]]["species"] = exp_map["species_file"]
self.IDs_paramspaths[exp_map["exp_ID"]]["cultivar"] = exp_map["cultivar_file"]
#observations data structures
self.observations = [] #for spotpy
self.evaluationdates = {} #for plotting outputs
self.obsdict = {} #for plotting outputs
i = 0 #i is a reference to match the element in result array (from spotpy)
for record in obslist:
var_name = record["variable"]
if "aggregation" in record.keys():
#for plotting purposes the variable name must be unique (and match variable in collect results method)
fromL = str(record["aggregation"][1][0])
toL = str(record["aggregation"][1][1])
var_name += " " + fromL + " to " + toL
self.observations.append(record["value"])
if var_name not in self.evaluationdates: #add the variable as a key
self.evaluationdates[var_name] = {}
self.obsdict[var_name] = {}
if record["exp_ID"] not in self.evaluationdates[var_name]: #add the experiment as a key
self.evaluationdates[var_name][record["exp_ID"]] = []
self.obsdict[var_name][record["exp_ID"]] = []
thisdate = record["date"].split("-")#self.evaluationdates needs a date type (not isoformat)
self.evaluationdates[var_name][record["exp_ID"]].append([i, date(int(thisdate[0]), int(thisdate[1]), int(thisdate[2]))])
self.obsdict[var_name][record["exp_ID"]].append([i, record["value"]])
i += 1
#normalization data structure
self.normalize = defaultdict(lambda: defaultdict(dict))
for var in self.obsdict:
var_indexes = []
self.normalize[var]["max_obs_value"] = 0.01
for exp_id in self.obsdict[var]:
inds_values = self.obsdict[var][exp_id]
for index_value in inds_values:
var_indexes.append(index_value[0]) #index in evallist
self.normalize[var]["max_obs_value"] = max(self.normalize[var]["max_obs_value"], index_value[1]) #max observed value
self.normalize[var]["where"] = var_indexes
#apply normalization to obs
for anorm in self.normalize:
n_factor = self.normalize[anorm]["max_obs_value"] / 100
for jjj in self.normalize[anorm]["where"]:
self.observations[jjj] /= n_factor
self.species_params={} #map to store different species params sets avoiding repetition
self.cultivar_params={} #map to store different cultivar params sets avoiding repetition
#create envs
self.envs = []
for exp_map in exp_maps:
with open(exp_map["sim_file"]) as simfile:
sim = json.load(simfile)
sim["crop.json"] = exp_map["crop_file"]
sim["site.json"] = exp_map["site_file"]
sim["climate.csv"] = exp_map["climate_file"]
with open(exp_map["site_file"]) as sitefile:
site = json.load(sitefile)
with open(exp_map["crop_file"]) as cropfile:
crop = json.load(cropfile)
mycrop = exp_map["crop_ID"]
crop["crops"][mycrop]["cropParams"]["species"][1] = exp_map["species_file"]
crop["crops"][mycrop]["cropParams"]["cultivar"][1] = exp_map["cultivar_file"]
env = monica_io.create_env_json_from_json_config({
"crop": crop,
"site": site,
"sim": sim
})
#add required outputs
for record in obslist:
if record["exp_ID"] == exp_map["exp_ID"]:
if not finalrun: #output only info required by spotpy
env["events"].append(unicode(record["date"]))
var = [unicode(record["variable"])]
if "aggregation" in record.keys():
var = []
var.append(record["aggregation"])
env["events"].append(var) #TODO:Add weight here?
elif finalrun: #daily output for plots
if "daily" not in env["events"]:
env["events"].append(unicode("daily"))
env["events"].append([]) #empty list of daily variables
env["events"][1].append(unicode("Date"))
var = [unicode(record["variable"])]
if "aggregation" in record.keys():
var = record["aggregation"]
if var not in env["events"][1]: #avoid to ask twice the same var as out
env["events"][1].append(var)
position = int(exp_map["where_in_rotation"][0])
for ws in env["cropRotation"][position]["worksteps"]:
if ws["type"] == "Seed" or ws["type"] == "Sowing":
self.species_params[exp_map["species_file"]] = ws["crop"]["cropParams"]["species"]
self.cultivar_params[exp_map["cultivar_file"]] = ws["crop"]["cropParams"]["cultivar"]
break
monica_io.add_climate_data_to_env(env, sim)
env["customId"] = exp_map["exp_ID"]
env["where_in_rotation"] = exp_map["where_in_rotation"]
self.envs.append(env)
self.context = zmq.Context()
self.socket_producer = self.context.socket(zmq.PUSH)
#self.socket_producer.connect("tcp://cluster2:6666")
self.socket_producer.connect("tcp://localhost:6666")
def main():
"main"
context = zmq.Context()
socket = context.socket(zmq.PUSH)
#port = 6666 if len(sys.argv) == 1 else sys.argv[1]
config = {"port": 6666, "start": 1, "end": 8157}
if len(sys.argv) > 1:
for arg in sys.argv[1:]:
kkk, vvv = arg.split("=")
if kkk in config:
config[kkk] = int(vvv)
#socket.bind("tcp://*:" + str(config["port"]))
#socket.connect("tcp://localhost:" + str(config["port"]))
socket.connect("tcp://cluster2:" + str(config["port"]))
with open("sim.json") as _:
sim = json.load(_)
with open("site.json") as _:
site = json.load(_)
with open("crop.json") as _:
crop = json.load(_)
with open("sims.json") as _:
sims = json.load(_)
sim["include-file-base-path"] = "C:/Users/berg.ZALF-AD/MONICA"
def read_pheno(path_to_file):
"read phenology data"
with open(path_to_file) as _:
ppp = {}
reader = csv.reader(_)
reader.next()
for row in reader:
ppp[(int(row[0]), int(row[1]))] = {
"sowing-doy": int(row[6]),
"flowering-doy": int(row[7]),
"harvest-doy": int(row[8])
}
return ppp
pheno = {
"GM": read_pheno("Maize_pheno_v3.csv"),
"WW": read_pheno("WW_pheno_v3.csv")
}
soil = {}
row_cols = []
with open("JRC_soil_macsur_v3.csv") as _:
reader = csv.reader(_)
reader.next()
for row in reader:
row_col = (int(row[1]), int(row[0]))
row_cols.append(row_col)
soil[row_col] = {
"elevation": float(row[4]),
"latitude": float(row[5]),
"depth": float(row[6]),
"pwp": float(row[7]),
"fc": float(row[8]),
"sat": float(row[9]),
"sw-init": float(row[10]),
"oc-topsoil": float(row[11]),
"oc-subsoil": float(row[12]),
"bd-topsoil": float(row[13]),
"bd-subsoil": float(row[14]),
"sand-topsoil": float(row[15]),
"sand-subsoil": float(row[18]),
"clay-topsoil": float(row[16]),
"clay-subsoil": float(row[19]),
}
def read_calibrated_tsums(path_to_file, crop_id):
"read calibrated tsums into dict"
with open(path_to_file) as _:
rrr = {}
reader = csv.reader(_)
reader.next()
for line in reader:
ddd = {}
row_, col_ = line[0].split("_")
row, col = (int(row_), int(col_))
ddd["tsums"] = [
int(line[1]),
int(line[2]),
int(line[3]),
int(line[4]),
int(line[5]),
int(line[6])
]
delta = 0
if crop_id == "GM":
delta = 1
ddd["tsums"].append(int(line[7]))
ddd["CriticalTemperatureHeatStress"] = float(line[10])
ddd["BeginSensitivePhaseHeatStress"] = 0
ddd["EndSensitivePhaseHeatStress"] = 0
ddd["HeatSumIrrigationStart"] = float(line[delta + delta + 9])
ddd["HeatSumIrrigationEnd"] = float(line[delta + delta + 10])
rrr[(row, col)] = ddd
return rrr
calib = {
"GM": read_calibrated_tsums("Calibrated_TSUM_Maize.csv", "GM"),
"WW": read_calibrated_tsums("Calibrated_TSUM_WW.csv", "WW")
}
def update_soil_crop_dates(row, col, crop_id):
"update function"
sss = soil[(row, col)]
ppp = pheno[crop_id][(row, col)]
extended_harvest_doy = ppp["harvest-doy"] + 10
start_date = date(START_YEAR, 1, 1)
sim["climate.csv-options"]["start-date"] = start_date.isoformat()
end_date = date(2010, 12, 31)
sim["climate.csv-options"]["end-date"] = end_date.isoformat()
#sim["debug?"] = True
pwp = sss["pwp"]
fc_ = sss["fc"]
sm_percent_fc = sss["sw-init"] / fc_ * 100.0
is_wintercrop = ppp["sowing-doy"] > ppp["harvest-doy"]
seeding_date = date(START_YEAR, 1,
1) + timedelta(days=ppp["sowing-doy"])
crop["cropRotation"][0]["worksteps"][0][
"date"] = seeding_date.strftime("0000-%m-%d")
crop["cropRotation"][0]["worksteps"][0][
"soilMoisturePercentFC"] = sm_percent_fc
crop["cropRotation"][0]["worksteps"][1][
"date"] = seeding_date.strftime("0000-%m-%d")
crop["cropRotation"][0]["worksteps"][1]["crop"][2] = crop_id
harvest_date = date(START_YEAR + (1 if is_wintercrop else 0), 1,
1) + timedelta(days=extended_harvest_doy)
#harvest_date = date(1980, 12, 31) if crop_id == "GM" else date(1980 + (1 if is_wintercrop else 0), 1, 1) + timedelta(days=ppp["harvest-doy"])
crop["cropRotation"][0]["worksteps"][2][
"date"] = harvest_date.strftime("000" +
("1" if is_wintercrop else "0") +
"-%m-%d")
site["Latitude"] = sss["latitude"]
top = {
"Thickness": [0.3, "m"],
"SoilOrganicCarbon": [sss["oc-topsoil"], "%"],
"SoilBulkDensity": [sss["bd-topsoil"] * 1000, "kg m-3"],
"FieldCapacity": [fc_, "m3 m-3"],
"PermanentWiltingPoint": [pwp, "m3 m-3"],
"PoreVolume": [sss["sat"], "m3 m-3"],
"SoilMoisturePercentFC": [sm_percent_fc, "% [0-100]"],
"Sand": sss["sand-topsoil"] / 100.0,
"Clay": sss["clay-topsoil"] / 100.0
}
sub = {
"Thickness": [1.7, "m"],
"SoilOrganicCarbon": [sss["oc-subsoil"], "%"],
"SoilBulkDensity": [sss["bd-subsoil"] * 1000, "kg m-3"],
"FieldCapacity": [fc_, "m3 m-3"],
"PermanentWiltingPoint": [pwp, "m3 m-3"],
"PoreVolume": [sss["sat"], "m3 m-3"],
"SoilMoisturePercentFC": [sm_percent_fc, "% [0-100]"],
"Sand": sss["sand-subsoil"] / 100.0,
"Clay": sss["clay-subsoil"] / 100.0
}
site["SiteParameters"]["SoilProfileParameters"] = [top, sub]
#print site["SiteParameters"]["SoilProfileParameters"]
print "# of rowsCols = ", len(row_cols)
i = 0
start_store = time.clock()
start = config["start"] - 1
end = config["end"] - 1
row_cols_ = row_cols[start:end + 1]
print "running from ", start, "/", row_cols[
start], " to ", end, "/", row_cols[end]
for row, col in row_cols_:
if soil[(row, col)]["bd-topsoil"] < 0.6: #avoid to simulate peat soils
continue
for crop_id in ["WW", "GM"]:
update_soil_crop_dates(row, col, crop_id)
env = monica_io.create_env_json_from_json_config({
"crop": crop,
"site": site,
"sim": sim,
"climate": ""
})
env["csvViaHeaderOptions"] = sim["climate.csv-options"]
env["params"]["userEnvironmentParameters"]["AtmosphericCO2"] = 360
climate_filename = "{}_{:03d}_v1.csv".format(row, col)
#read climate data on the server and send just the path to the climate data csv file
env["pathToClimateCSV"] = PATH_TO_CLIMATE_DATA_SERVER + climate_filename
cal = calib[crop_id][(row, col)]
cultivar = env["cropRotation"][0]["worksteps"][1]["crop"][
"cropParams"]["cultivar"]
cultivar["CropSpecificMaxRootingDepth"] = 1.5
cultivar["StageTemperatureSum"] = cal["tsums"]
cultivar["BeginSensitivePhaseHeatStress"] = 0
cultivar["EndSensitivePhaseHeatStress"] = 0
cultivar["HeatSumIrrigationStart"] = cal["HeatSumIrrigationStart"]
cultivar["HeatSumIrrigationEnd"] = cal["HeatSumIrrigationEnd"]
env["customId"] = crop_id + "|(" + str(row) + "/" + str(col) + ")"
socket.send_json(env)
print "sent env ", i, " customId: ", env["customId"]
i += 1
#break
#break
stop_store = time.clock()
print "sending ", i, " envs took ", (stop_store - start_store), " seconds"
print "ran from ", start, "/", row_cols[start], " to ", end, "/", row_cols[
end]
return
def run_producer(setup,
custom_crop,
server={
"server": None,
"port": None,
"nd-port": None
},
preprocessed_data=None):
"main"
config = {
"user": USER,
"port": server["port"] if server["port"] else "66663",
"no-data-port": server["nd-port"] if server["nd-port"] else "5555",
"server": server["server"] if server["server"] else "localhost"
}
if len(sys.argv) > 1:
for arg in sys.argv[1:]:
k, v = arg.split("=")
if k in config:
config[k] = v
context = zmq.Context()
socket = context.socket(zmq.PUSH)
prod_cons_socket = context.socket(zmq.PUSH)
soil_db_con = sqlite3.connect(paths["path-to-data-dir"] +
"germany/buek1000.sqlite")
#connect producer and consumer directly
prod_cons_socket.bind("tcp://*:" + str(config["no-data-port"]))
if LOCAL_RUN:
socket.connect("tcp://localhost:" + str(config["port"]))
else:
socket.connect("tcp://" + config["server"] + ":" + str(config["port"]))
with open("sim.json") as _:
sim_json = json.load(_)
sim_json["include-file-base-path"] = paths["include-file-base-path"]
with open("site.json") as _:
site_json = json.load(_)
with open("crop.json") as _:
crop_json = json.load(_)
def get_value(list_or_value):
return list_or_value[0] if isinstance(list_or_value,
list) else list_or_value
sent_env_count = 1
start_time = time.clock()
if preprocessed_data == None:
preprocessed_data = preprocess_data()
dem_gk5_interpolate = preprocessed_data["dem_gk5_interpolate"]
slope_gk5_interpolate = preprocessed_data["slope_gk5_interpolate"]
soil_gk5_interpolate = preprocessed_data["soil_gk5_interpolate"]
cdict = preprocessed_data["cdict"]
climate_gk5_interpolate = preprocessed_data["climate_gk5_interpolate"]
stat_infos = preprocessed_data["stat_infos"]
#put custom crop in place
for stat_info in stat_infos:
for cm in stat_info["rotation"]:
for ws in cm["worksteps"]:
if "crop" in ws.keys():
ws["crop"] = custom_crop
#send stations list to the consumer
sim_stats = []
for stat_info in stat_infos:
sim_stats.append(int(stat_info["station_id"]))
prod_cons_socket.send_json(sim_stats)
for stat_info in stat_infos:
station_id = stat_info["station_id"]
r_gk5, h_gk5 = stat_info["gk5_rh"]
env_template = monica_io.create_env_json_from_json_config({
"crop": crop_json,
"site": site_json,
"sim": sim_json,
"climate": ""
})
env_template["cropRotation"] = stat_info["rotation"]
env_template["events"] = stat_info["events"]
soil_id = soil_gk5_interpolate(r_gk5, h_gk5)
if soil_id == -9999:
continue
if soil_id < 1 or soil_id > 71:
#print "row/col:", row, "/", col, "has unknown soil_id:", soil_id
#unknown_soil_ids.add(soil_id)
continue
crow, ccol = climate_gk5_interpolate(r_gk5, h_gk5)
height_nn = dem_gk5_interpolate(r_gk5, h_gk5)
slope = slope_gk5_interpolate(r_gk5, h_gk5)
clat, clon = cdict[(crow, ccol)]
#slon, slat = transform(gk5, wgs84, r, h)
#print "srow:", srow, "scol:", scol, "h:", h, "r:", r, " inter:", inter, "crow:", crow, "ccol:", ccol, "slat:", slat, "slon:", slon, "clat:", clat, "clon:", clon
env_template["params"]["userCropParameters"][
"__enable_T_response_leaf_expansion__"] = setup[
"LeafExtensionModifier"]
# set soil-profile
sp_json = soil_io.soil_parameters(soil_db_con, soil_id)
soil_profile = monica_io.find_and_replace_references(sp_json,
sp_json)["result"]
#print "soil:", soil_profile
env_template["params"]["siteParameters"][
"SoilProfileParameters"] = soil_profile
# setting groundwater level
if setup["groundwater-level"]:
groundwaterlevel = 20
layer_depth = 0
for layer in soil_profile:
if layer.get("is_in_groundwater", False):
groundwaterlevel = layer_depth
#print "setting groundwaterlevel of soil_id:", str(soil_id), "to", groundwaterlevel, "m"
break
layer_depth += get_value(layer["Thickness"])
env_template["params"]["userEnvironmentParameters"][
"MinGroundwaterDepthMonth"] = 3
env_template["params"]["userEnvironmentParameters"][
"MinGroundwaterDepth"] = [max(0, groundwaterlevel - 0.2), "m"]
env_template["params"]["userEnvironmentParameters"][
"MaxGroundwaterDepth"] = [groundwaterlevel + 0.2, "m"]
# setting impenetrable layer
if setup["impenetrable-layer"]:
impenetrable_layer_depth = get_value(
env_template["params"]["userEnvironmentParameters"]
["LeachingDepth"])
layer_depth = 0
for layer in soil_profile:
if layer.get("is_impenetrable", False):
impenetrable_layer_depth = layer_depth
#print "setting leaching depth of soil_id:", str(soil_id), "to", impenetrable_layer_depth, "m"
break
layer_depth += get_value(layer["Thickness"])
env_template["params"]["userEnvironmentParameters"][
"LeachingDepth"] = [impenetrable_layer_depth, "m"]
env_template["params"]["siteParameters"][
"ImpenetrableLayerDepth"] = [impenetrable_layer_depth, "m"]
if setup["elevation"]:
env_template["params"]["siteParameters"]["heightNN"] = height_nn
if setup["slope"]:
env_template["params"]["siteParameters"]["slope"] = slope / 100.0
if setup["latitude"]:
env_template["params"]["siteParameters"]["Latitude"] = clat
env_template["params"]["simulationParameters"][
"UseNMinMineralFertilisingMethod"] = setup["fertilization"]
env_template["params"]["simulationParameters"][
"UseAutomaticIrrigation"] = setup["irrigation"]
env_template["params"]["simulationParameters"][
"NitrogenResponseOn"] = setup["NitrogenResponseOn"]
env_template["params"]["simulationParameters"][
"WaterDeficitResponseOn"] = setup["WaterDeficitResponseOn"]
env_template["params"]["simulationParameters"][
"EmergenceMoistureControlOn"] = setup["EmergenceMoistureControlOn"]
env_template["params"]["simulationParameters"][
"EmergenceFloodingControlOn"] = setup["EmergenceFloodingControlOn"]
env_template["csvViaHeaderOptions"] = sim_json["climate.csv-options"]
if LOCAL_RUN:
env_template["pathToClimateCSV"] = paths[
"path-to-climate-csvs-dir"] + "row-" + str(
crow) + "/col-" + str(ccol) + ".csv"
else:
env_template["pathToClimateCSV"] = paths[
"archive-path-to-climate-csvs-dir"] + "row-" + str(
crow) + "/col-" + str(ccol) + ".csv"
env_template["customId"] = {"station_id": int(station_id)}
#with open("envs/env-"+str(sent_env_count)+".json", "w") as _:
# _.write(json.dumps(env))
socket.send_json(env_template)
print "sent env ", sent_env_count, " customId: ", env_template[
"customId"]
#exit()
sent_env_count += 1
stop_time = time.clock()
print "sending ", (sent_env_count -
1), " envs took ", (stop_time - start_time), " seconds"
#print "ran from ", start, "/", row_cols[start], " to ", end, "/", row_cols[end]
print "exiting run_producer()"
def run_producer():
# some options and configurations
create_simulation_files = False
# calibration options
training_mode = False
calibrate_apache = False
# simulation options
activate_debug = False
run_via_simulation_files = False
output_dir = "runs/2018-07-16/"
# calibration -------------------------------------
# original "StageTemperatureSum": [[148, 284, 380, 180, 420, 25 ], "\u00b0C d"]
# overwrite some settings if creation of simulation files is active
if create_simulation_files:
run_via_simulation_files = False
training_mode = False
# technical initialisation
config = {"user": "******", "port": "66666", "server": "localhost"}
sent_id = 0
start_send = time.clock()
socket = initialise_sockets(config)
print(socket)
# output file
output_filename = "agmip_calibration_phase2_step1_evaluation.csv"
if training_mode:
output_filename = "agmip_calibration_phase2_step1_calibration.csv"
paths = PATHS[config["user"]]
# read in management information
management_file = paths[
"INCLUDE_FILE_BASE_PATH"] + "monica_simulation_setup/input_data/cal2_phenology_mgt_soil_data.txt"
management_df = pd.read_csv(management_file,
sep='\t',
header=0,
index_col=None,
keep_default_na=False,
encoding="latin1")
# iterate over every row/environment where row contains the setup of one simulation
for environment in management_df.iterrows():
simulation_row = environment[1]
sim_id = int(simulation_row["n"])
if training_mode:
# calibration mode
if simulation_row["Date_observee_Epi_1cm"] == "NA":
print(
"Skip evaluation simulation because testing mode is active"
)
continue
if calibrate_apache:
if simulation_row["Variete"] != "Apache":
# calibration mode active but sim_id refers to wrong cultivar
continue
else:
# Bermude
if simulation_row["Variete"] != "Bermude":
continue
# apache parameters
stage1_sum = 148
stage2_sum = 101 # 340
stage3_sum = 429 # 407
if simulation_row["Variete"] == "Bermude":
stage1_sum = 148
stage2_sum = 83 # 340
stage3_sum = 477 # 407
# ------------------------------------------------
# based on Christian Kersebaums assumptions
tsum_bbch55 = (stage2_sum + stage3_sum)
tsum_bbch30 = (stage3_sum) * 0.25 + stage2_sum
print("Run simulation " + str(sim_id))
sim_parameters = None
site_parameters = None
crop_parameters = None
if run_via_simulation_files:
# run from existing simulation files
print("Run from MONICA simulation files ...")
simulation_dir = "monica_simulation_setup/calibration/"
if simulation_row["Date_observee_Epi_1cm"] == "NA":
simulation_dir = "monica_simulation_setup/evaluation/"
with open(simulation_dir + "sim-" + str(sim_id) + ".json") as fp:
sim_parameters = json.load(fp)
with open(simulation_dir + "site-" + str(sim_id) + ".json") as fp:
site_parameters = json.load(fp)
with open(simulation_dir + "crop-" + str(sim_id) + ".json") as fp:
crop_parameters = json.load(fp)
else:
# dynamically create simulation objects
print(
"Dynamic run by creating simulation objects directly from AgMIP management file."
)
site_parameters = create_site_parameters(simulation_row)
crop_parameters = create_crop_parameters(simulation_row)
sim_parameters = create_sim_parameters(
simulation_row, paths["INCLUDE_FILE_BASE_PATH"], sim_id,
output_dir, activate_debug, create_simulation_files,
tsum_bbch30, tsum_bbch55)
if site_parameters is None:
print("ERROR: site_parameters == None")
if crop_parameters is None:
print("ERROR: crop_parameters == None")
if sim_parameters is None:
print("ERROR: sim_parameters == None")
# check if just MONICA simulation files should be created from the provided input files
# do'nt send any information if input files should be created
if create_simulation_files:
dir = "monica_simulation_setup/calibration/"
if simulation_row["Date_observee_Epi_1cm"] == "NA":
dir = "monica_simulation_setup/evaluation/"
with open(dir + "site-" + str(sim_id) + ".json", "w") as fp:
json.dump(site_parameters, fp=fp, indent=4)
with open(dir + "crop-" + str(sim_id) + ".json", "w") as fp:
json.dump(crop_parameters, fp=fp, indent=4)
with open(dir + "sim-" + str(sim_id) + ".json", "w") as fp:
json.dump(sim_parameters, fp=fp, indent=4)
continue
env_map = {
"crop": crop_parameters,
"site": site_parameters,
"sim": sim_parameters
}
env = monica_io.create_env_json_from_json_config(env_map)
# calibration
# for calibration overwrite stage temperature sum values
env["cropRotation"][0]["worksteps"][0]["crop"]["cropParams"][
"cultivar"]["StageTemperatureSum"][0][1] = stage2_sum
env["cropRotation"][0]["worksteps"][0]["crop"]["cropParams"][
"cultivar"]["StageTemperatureSum"][0][2] = stage3_sum
env["cropRotation"][0]["worksteps"][0]["crop"]["cropParams"][
"cultivar"]["StageTemperatureSum"][0][0] = stage1_sum
temp_sum = env["cropRotation"][0]["worksteps"][0]["crop"][
"cropParams"]["cultivar"]["StageTemperatureSum"][0]
bbch30_tempsum = temp_sum[0] + temp_sum[1] + 0.25 * (temp_sum[3])
# final env object with all necessary information
env["customId"] = {
"id": sim_id,
"calibration": training_mode,
"sim_files": output_dir,
"output_filename": output_filename,
"cultivar": simulation_row["Variete"],
"sowing_date": simulation_row["Date_Semis"],
"bbch30":
simulation_row["Date_observee_Epi_1cm"], # observation bbch 30
"bbch55":
simulation_row["Date_observee_Epiaison"], # observation bbch 55
"site": simulation_row["Libelle"]
}
with open("monica_simulation_setup/env.json", "w") as fp:
json.dump(env, fp=fp, indent=4)
# sending env object to MONICA ZMQ
print("sent env ", sent_id, " customId: ", env["customId"])
socket.send_json(env)
sent_id += 1
stop_send = time.clock()
# just tell the sending time if objects have really been sent
if not create_simulation_files:
print("sending ", sent_id, " envs took ", (stop_send - start_send),
" seconds")
def producer(setup=None):
"main function"
paths = PATHS[USER]
#Configure producer
if setup == None:
#playground
run_id = "custom"
PRODUCTION_LEVEL = 'WL.NL.rain' #options: "Pot", "WL.NL.rain"
TF = "historical"
FERT_STRATEGY = "BASE" #options: "NDEM", "NMIN", "BASE"
COVER_CROP_FREQ = {
#always use int for insert-cc-every and out-of
#keep out-of as small as possible (to ensure uniform spatial distribution)
"insert-cc-every": 1, #CM
"out-of": 4, #CM
"suffix": "25" #REMEMBER to set it (for output file name)
}
RESIDUES_EXPORTED = True
EXPORT_RATE = "base"
RESIDUES_HUMUS_BALANCE = False #mgt complying with humus balance approach of NRW: if true, RESIDUE_EXPORTED has no effect!
else:
run_id = setup["id"]
PRODUCTION_LEVEL = setup["PRODUCTION_LEVEL"]
TF = setup["TF"]
FERT_STRATEGY = setup["FERT_STRATEGY"]
COVER_CROP_FREQ = setup["COVER_CROP_FREQ"]
if setup["res_mgt"] == "RESIDUES_HUMUS_BALANCE":
RESIDUES_HUMUS_BALANCE = True
RESIDUES_EXPORTED = False
else:
RESIDUES_HUMUS_BALANCE = False
RESIDUES_EXPORTED = True
EXPORT_RATE = setup["res_mgt"]
HUMBAL_CORRECTION = setup["HUMBAL_CORRECTION"]
#end of user configuration
#assemble file name suffix for out
suffix = "_id" + run_id + "_"
suffix += TF + "_"
suffix += "fert-" + FERT_STRATEGY.lower() + "_"
if RESIDUES_HUMUS_BALANCE:
suffix += "res-humbal_"
else:
suffix += "res-" + EXPORT_RATE + "_"
suffix += "cc-" + COVER_CROP_FREQ["suffix"] + "_"
suffix += "pl-" + PRODUCTION_LEVEL.replace(".", "") + "_"
if FERT_STRATEGY == "NMIN":
rotations_file = "rotations_dynamic_harv.json"
elif FERT_STRATEGY == "NDEM":
rotations_file = "rotations_dynamic_harv_Ndem.json"
elif FERT_STRATEGY == "BASE":
rotations_file = "rotations_dynamic_harv_Nbaseline.json"
if LOCAL_RUN:
PATH_TO_CLIMATE_DATA_DIR = timeframes[TF]["local-path-to-climate"]
else:
PATH_TO_CLIMATE_DATA_DIR = timeframes[TF]["cluster-path-to-climate"]
context = zmq.Context()
socket = context.socket(zmq.PUSH)
port = 6666 if len(sys.argv) == 1 else sys.argv[1]
if LOCAL_RUN:
socket.connect("tcp://localhost:66663")
else:
socket.connect("tcp://cluster1:" + str(port))
soil_db_con = sqlite3.connect("soil.sqlite")
with open("sim.json") as _:
sim = json.load(_)
sim["start-date"] = timeframes[TF]["start-date"]
sim["end-date"] = timeframes[TF]["end-date"]
with open("site.json") as _:
site = json.load(_)
with open("crop.json") as _:
crop = json.load(_)
with open("cover-crop.json") as _:
cover_crop = json.load(_)["CM"]
if RESIDUES_HUMUS_BALANCE:
#inject additional harvest params
for ws in range(len(cover_crop["worksteps"])):
if cover_crop["worksteps"][ws]["type"] == "AutomaticHarvest":
cover_crop["worksteps"][ws][
"opt-carbon-conservation"] = True
cover_crop["worksteps"][ws][
"crop-impact-on-humus-balance"] = [
humus_equivalent["crop"]["CC"],
"Humus equivalent [Heq]"
]
cover_crop["worksteps"][ws]["residue-heq"] = [
humus_equivalent["material"]["green-manure"],
"Heq ton-1 DM"
]
cover_crop["worksteps"][ws][
"crop-usage"] = CROP_USAGE_HUMBAL["CC"]
cover_crop["worksteps"][ws][
"exported"] = True #Needed to fire the humus balance approach; if true and crop-usage="green-manure" --> the crop is anyway returned to the soil
with open("sims.json") as _:
sims = json.load(_)
if FERT_STRATEGY != "NMIN":
#turn off Nmin automatic fertilization
for setting in sims.iteritems():
setting[1]["UseNMinMineralFertilisingMethod"] = False
with open(rotations_file) as _:
rotations = json.load(_)
#identify rotations with codes
rots_info = {}
for bkr, rots in rotations.iteritems():
for rot in rots.iteritems():
rot_code = int(rot[0])
my_rot = []
for cm in rot[1]: #["worksteps"]:
for ws in range(len(cm["worksteps"])):
if cm["worksteps"][ws]["type"] == "Sowing":
my_rot.append(cm["worksteps"][ws]["crop"][2])
#for each crop, identify previous main one (needed to determine expected N availability)
rot_info = []
for i in range(len(my_rot)):
cm_info = {}
current_cp = my_rot[i]
if i != 0:
previous_cp = my_rot[i - 1]
else:
previous_cp = my_rot[-1]
cm_info["current"] = current_cp
cm_info["previous"] = previous_cp
rot_info.append(cm_info)
rots_info[rot_code] = rot_info
if RESIDUES_HUMUS_BALANCE:
#inject additional harvest params
for bkr, rots in rotations.iteritems():
for rot in rots.iteritems():
rot_code = int(rot[0])
rot_info = rots_info[rot_code]
cp_index = 0
for cm in rot[1]: #["worksteps"]:
for ws in range(len(cm["worksteps"])):
cp = rot_info[cp_index]["current"]
if cm["worksteps"][ws][
"type"] == "AutomaticHarvest":
cm["worksteps"][ws][
"opt-carbon-conservation"] = True
cm["worksteps"][ws][
"crop-impact-on-humus-balance"] = [
humus_equivalent["crop"][cp],
"Humus equivalent [Heq]"
]
if cp == "SBee":
cm["worksteps"][ws]["residue-heq"] = [
humus_equivalent["material"]
["green-manure"], "Heq ton-1 DM"
]
else:
cm["worksteps"][ws]["residue-heq"] = [
humus_equivalent["material"]["straw"],
"Heq ton-1 DM"
]
cm["worksteps"][ws][
"organic-fertilizer-heq"] = [
humus_equivalent["material"]
["pig-slurry"], "Heq ton-1 DM"
]
cm["worksteps"][ws][
"crop-usage"] = CROP_USAGE_HUMBAL[cp]
cm["worksteps"][ws][
"max-residue-recover-fraction"] = CROP_USAGE_HUMBAL[
"max-residue-recover-fraction"]
cp_index += 1
if FERT_STRATEGY == "BASE":
#read additional info required for baseline fert strategy:
#1. expected mineral N availability
expected_N_availability = defaultdict(lambda: defaultdict())
with open("expected_N_availability.csv") as _:
reader = csv.reader(_)
reader.next()
for row in reader:
cp_sequence = (row[0], row[1])
soil_type = row[2]
expected_N_value = float(row[3])
expected_N_availability[cp_sequence][
soil_type] = expected_N_value
expected_N_availability[cp_sequence]["target_depth"] = float(
row[4])
#2. rules to split mineral fertilization
mineralN_split = defaultdict(lambda: defaultdict())
with open("MineralN_topdressing.csv") as _:
reader = csv.reader(_)
reader.next()
for row in reader:
cp = row[0]
mineralN_split[cp]["target"] = float(row[2])
for i in range(4, 7):
if row[i] != "":
mineralN_split[cp][i - 4] = float(row[i])
sim["UseSecondaryYields"] = RESIDUES_EXPORTED
if RESIDUES_EXPORTED:
for cp in crop["crops"].iteritems():
for k in EXPORT_PRESETS[EXPORT_RATE].keys():
if cp[0] in k:
my_rate = EXPORT_PRESETS[EXPORT_RATE][k]
for organ in cp[1]["cropParams"]["cultivar"][
"OrganIdsForSecondaryYield"]:
organ["yieldPercentage"] *= my_rate
sim["include-file-base-path"] = paths["INCLUDE_FILE_BASE_PATH"]
def read_general_metadata(path_to_file):
"read metadata file"
with open(path_to_file) as file_:
data = {}
reader = csv.reader(file_, delimiter="\t")
reader.next()
for row in reader:
if int(row[1]) != 1:
continue
data[(int(row[2]), int(row[3]))] = {
"subpath-climate.csv": row[9],
"latitude": float(row[13]),
"elevation": float(row[14])
}
return data
general_metadata = read_general_metadata("NRW_General_Metadata.csv")
def load_mapping(row_offset=0, col_offset=0):
to_climate_index = {}
with (open("working_resolution_to_climate_lat_lon_indices.json")) as _:
l = json.load(_)
for i in xrange(0, len(l), 2):
cell = (row_offset + l[i][0], col_offset + l[i][1])
to_climate_index[cell] = tuple(l[i + 1])
return to_climate_index
def read_orgN_kreise(path_to_file):
"read organic N info for kreise"
with open(path_to_file) as file_:
data = {}
reader = csv.reader(file_, delimiter=",")
reader.next()
reader.next()
for row in reader:
for kreis_code in row[1].split("|"):
if kreis_code != "":
data[int(kreis_code)] = float(row[8])
return data
orgN_kreise = read_orgN_kreise("NRW_orgN_balance.csv")
def update_soil(row, col):
"in place update the env"
site["SiteParameters"]["Latitude"] = general_metadata[(
row, col)]["latitude"]
site["SiteParameters"]["HeightNN"] = [
general_metadata[(row, col)]["elevation"], "m"
]
site["SiteParameters"][
"SoilProfileParameters"] = soil_io.soil_parameters(
soil_db_con, soil_ids[(row, col)])
KA5_txt = soil_io.sand_and_clay_to_ka5_texture(
site["SiteParameters"]["SoilProfileParameters"][0]["Sand"][0],
site["SiteParameters"]["SoilProfileParameters"][0]["Clay"][0])
for layer in site["SiteParameters"]["SoilProfileParameters"]:
layer["SoilBulkDensity"][0] = max(layer["SoilBulkDensity"][0], 600)
layer["SoilOrganicCarbon"][0] = layer["SoilOrganicCarbon"][
0] * 0.6 #correction factor suggested by TGaiser
return KA5_txt
def read_header(path_to_ascii_grid_file):
"read metadata from esri ascii grid file"
metadata = {}
header_str = ""
with open(path_to_ascii_grid_file) as _:
for i in range(0, 6):
line = _.readline()
header_str += line
sline = [x for x in line.split() if len(x) > 0]
if len(sline) > 1:
metadata[sline[0].strip().lower()] = float(
sline[1].strip())
return metadata, header_str
def read_ascii_grid(path_to_file,
include_no_data=False,
row_offset=0,
col_offset=0):
"read an ascii grid into a map, without the no-data values"
def int_or_float(s):
try:
return int(s)
except ValueError:
return float(s)
with open(path_to_file) as file_:
data = {}
#skip the header (first 6 lines)
for _ in range(0, 6):
file_.next()
row = -1
for line in file_:
row += 1
col = -1
for col_str in line.strip().split(" "):
col += 1
if not include_no_data and int_or_float(col_str) == -9999:
continue
data[(row_offset + row,
col_offset + col)] = int_or_float(col_str)
return data
#offset is used to match info in general metadata and soil database
soil_ids = read_ascii_grid("soil-profile-id_nrw_gk3.asc", row_offset=282)
bkr_ids = read_ascii_grid("bkr_nrw_gk3.asc", row_offset=282)
lu_ids = read_ascii_grid("lu_resampled.asc", row_offset=282)
kreise_ids = read_ascii_grid("kreise_matrix.asc", row_offset=282)
meteo_ids = load_mapping(row_offset=282)
soil_metadata, _ = read_header("soil-profile-id_nrw_gk3.asc")
wgs84 = Proj(init="epsg:4326")
gk3 = Proj(init="epsg:3396")
gk5 = Proj(init="epsg:31469")
def create_ascii_grid_interpolator(arr, meta, ignore_nodata=True):
"read an ascii grid into a map, without the no-data values"
rows, cols = arr.shape
cellsize = int(meta["cellsize"])
xll = int(meta["xllcorner"])
yll = int(meta["yllcorner"])
nodata_value = meta["nodata_value"]
xll_center = xll + cellsize // 2
yll_center = yll + cellsize // 2
yul_center = yll_center + (rows - 1) * cellsize
points = []
values = []
for row in range(rows):
for col in range(cols):
value = arr[row, col]
if ignore_nodata and value == nodata_value:
continue
r = xll_center + col * cellsize
h = yul_center - row * cellsize
points.append([r, h])
values.append(value)
return NearestNDInterpolator(np.array(points), np.array(values))
path_to_slope_grid = paths[
"path-to-data-dir"] + "/germany/slope_1000_gk5.asc"
slope_metadata, _ = read_header(path_to_slope_grid)
slope_grid = np.loadtxt(path_to_slope_grid, dtype=float, skiprows=6)
slope_gk5_interpolate = create_ascii_grid_interpolator(
slope_grid, slope_metadata)
#counter = 0
#for k, v in bkr_ids.iteritems():
# if v == 134:
# if k in lu_ids:
# counter += 1
#print counter
def rotate(crop_rotation):
"rotate the crops in the rotation"
crop_rotation.insert(0, crop_rotation.pop())
def insert_cc(crop_rotation, cc_data):
"insert cover crops in the rotation"
insert_cover_here = []
for cultivation_method in range(len(crop_rotation)):
for workstep in crop_rotation[cultivation_method]["worksteps"]:
if workstep["type"] == "Sowing":
if workstep["crop"][2] in COVER_BEFORE:
insert_cover_here.append(
(cultivation_method, workstep["date"]))
break
for position, mydate in reversed(insert_cover_here):
mydate = mydate.split("-")
main_crop_sowing = date(2017, int(mydate[1]), int(mydate[2]))
latest_harvest_cc = main_crop_sowing - timedelta(days=10)
latest_harvest_cc = unicode("0001-" +
str(latest_harvest_cc.month).zfill(2) +
"-" +
str(latest_harvest_cc.day).zfill(2))
crop_rotation.insert(position, copy.deepcopy(cc_data))
crop_rotation[position]["worksteps"][1][
"latest-date"] = latest_harvest_cc
def calculate_orgfert_amount(N_applied, fert_type, soilCN=10):
"convert N applied in amount of fresh org fert"
AOM_DryMatterContent = fert_type["AOM_DryMatterContent"][0]
AOM_NH4Content = fert_type["AOM_NH4Content"][0]
AOM_NO3Content = fert_type["AOM_NO3Content"][0]
CN_Ratio_AOM_Fast = fert_type["CN_Ratio_AOM_Fast"][0]
CN_Ratio_AOM_Slow = fert_type["CN_Ratio_AOM_Slow"][0]
PartAOM_to_AOM_Fast = fert_type["PartAOM_to_AOM_Fast"][0]
PartAOM_to_AOM_Slow = fert_type["PartAOM_to_AOM_Slow"][0]
AOM_to_C = 0.45
AOM_fast_factor = (AOM_to_C * PartAOM_to_AOM_Fast) / CN_Ratio_AOM_Fast
AOM_slow_factor = (AOM_to_C * PartAOM_to_AOM_Slow) / CN_Ratio_AOM_Slow
AOM_SOM_factor = (
1 -
(PartAOM_to_AOM_Fast + PartAOM_to_AOM_Slow)) * AOM_to_C / soilCN
conversion_coeff = AOM_NH4Content + AOM_NO3Content + AOM_fast_factor + AOM_slow_factor + AOM_SOM_factor
AOM_dry = N_applied / conversion_coeff
AOM_fresh = AOM_dry / AOM_DryMatterContent
return AOM_fresh
def update_fert_values(rotation, rot_info, cc_in_cm,
expected_N_availability, mineralN_split, soil_type,
orgN_applied):
"function to mimic baseline fertilization"
cow_unit = 100
GVs = orgN_applied / cow_unit
orgN_effect = GVs * 10
#insert cc in rotation info
for cm in reversed(range(len(rot_info))):
rot_info[cm]["has_cover_before"] = False
if rot_info[cm]["current"] in COVER_BEFORE and cc_in_cm:
rot_info[cm]["has_cover_before"] = True
cc_info = {"current": "CC"}
rot_info.insert(cm, cc_info)
for cm in range(len(rotation)):
if rot_info[cm]["current"] == "CC":
#cover crops do not receive any fertilization
continue
current_cp = rot_info[cm]["current"]
previous_cp = rot_info[cm]["previous"]
has_cover = rot_info[cm]["has_cover_before"]
N_target = mineralN_split[current_cp]["target"]
expected_Nmin = expected_N_availability[(current_cp,
previous_cp)][soil_type]
#modify expected Nmin depending on livestock pressure and presence of cover crop
if has_cover:
expected_Nmin += 20
expected_Nmin += orgN_effect
#calculate N to be applied with mineral fertilization
sum_Nfert = max(N_target - expected_Nmin, 0)
#map the fertilization worksteps
ref_fert = 0
for ws in range(len(rotation[cm]["worksteps"])):
workstep = rotation[cm]["worksteps"][ws]
if workstep["type"] == "MineralFertilization" and workstep[
"amount"][0] == 0:
workstep["amount"][
0] = sum_Nfert * mineralN_split[current_cp][ref_fert]
ref_fert += 1
sent_id = 0
start_send = time.clock()
simulated_cells = 0
no_kreis = 0
#bkr2lk = defaultdict(set) #for additional info
#soilty2iniSOC = defaultdict(list) #for additional info
export_lat_lon_coords = False
export_lat_lon_file = None
srows = int(soil_metadata["nrows"])
scellsize = int(soil_metadata["cellsize"])
sxll = int(soil_metadata["xllcorner"])
syll = int(soil_metadata["yllcorner"])
sxll_center = sxll + scellsize // 2
syll_center = syll + scellsize // 2
syul_center = syll_center + (srows - 1) * scellsize
for (row, col), gmd in general_metadata.iteritems():
#test
#if int(row) != 505 or int(col) != 58:
# continue
if (row, col) in soil_ids and (row,
col) in bkr_ids and (row,
col) in lu_ids:
# get gk3 coordinates for soil row/col
sr_gk3 = sxll_center + col * scellsize
sh_gk3 = syul_center - row * scellsize
if export_lat_lon_coords:
if not export_lat_lon_file:
export_lat_lon_file = open(
"soil_row_col_to_lat_lon_coords.csv", "w")
export_lat_lon_file.write("row,col,lat,lon\n")
slon, slat = transform(gk3, wgs84, sr_gk3, sh_gk3)
export_lat_lon_file.write(
",".join(map(str, [row, col, slat, slon])) + "\n")
sr_gk5, sh_gk5 = transform(gk3, gk5, sr_gk3, sh_gk3)
site["SiteParameters"]["Slope"] = slope_gk5_interpolate(
sr_gk5, sh_gk5) / 100.0
#continue
bkr_id = bkr_ids[(row, col)]
########for testing
#if bkr_id != 129:
# continue
soil_id = soil_ids[(row, col)]
meteo_id = meteo_ids[(row, col)]
if (row, col) in kreise_ids:
kreis_id = kreise_ids[(row, col)]
#bkr2lk[bkr_id].add(kreis_id)
else:
no_kreis += 1
print "-----------------------------------------------------"
print "kreis not found for calculation of organic N"
print "-----------------------------------------------------"
simulated_cells += 1
KA5_txt = update_soil(row, col)
light_soils = ["Ss", "Su2", "Su3", "Su4", "St2", "Sl3", "Sl2"]
heavy_soils = ["Tu3", "Tu4", "Lt3", "Ts2", "Tl", "Tu2", "Tt"]
soil_type = "medium"
if KA5_txt in light_soils:
soil_type = "light"
elif KA5_txt in heavy_soils:
soil_type = "heavy"
site["SiteParameters"][
"SoilSpecificHumusBalanceCorrection"] = HUMBAL_CORRECTION[
soil_type]
#soilty2iniSOC[soil_type].append(site["SiteParameters"]["SoilProfileParameters"][0]["SoilOrganicCarbon"][0])
#continue
#row_col = "{}{:03d}".format(row, col)
#topsoil_carbon[row_col] = site["SiteParameters"]["SoilProfileParameters"][0]["SoilOrganicCarbon"][0]
#continue
for rot_id, rotation in rotations[str(bkr_id)].iteritems():
########for testing
#if rot_id not in ["9120", "7120", "7130", "8120", "6110", "6120", "5120", "1110", "1130", "3110", "3130", "2110", "2120", "2130", "4110", "4120"]:
# continue
#extend rotation
ext_rot = []
for i in range(COVER_CROP_FREQ["out-of"]):
ext_rot.append(copy.deepcopy(rotation))
#insert CC in a subset of CM
cc_in_cm = {}
for cm in range(len(ext_rot)):
cc_in_cm[cm] = False
if (cm + 1) <= COVER_CROP_FREQ["insert-cc-every"]:
insert_cc(ext_rot[cm], cover_crop)
cc_in_cm[cm] = True
#update mineral fert (baseline N scenario)
if FERT_STRATEGY == "BASE":
for rot in range(len(ext_rot)):
update_fert_values(
ext_rot[rot],
copy.deepcopy(rots_info[int(rot_id)]),
cc_in_cm[rot], expected_N_availability,
mineralN_split, soil_type, orgN_kreise[kreis_id])
#compose the rotation
composed_rot = []
for rot in ext_rot:
for cm in rot:
composed_rot.append(cm)
crop["cropRotation"] = composed_rot
env = monica_io.create_env_json_from_json_config({
"crop": crop,
"site": site,
"sim": sim,
"climate": ""
})
env["sharedId"] = "ts_sustag_nrw"
#assign amount of organic fertilizer
for cultivation_method in env["cropRotation"]:
for workstep in cultivation_method["worksteps"]:
if workstep["type"] == "OrganicFertilization":
workstep["amount"][0] = calculate_orgfert_amount(
orgN_kreise[kreis_id], workstep["parameters"]
) #TODO: assign soilCN param dynamically
#with open("test_crop.json", "w") as _:
# _.write(json.dumps(crop, indent=4))
#climate is read by the server
env["csvViaHeaderOptions"] = sim["climate.csv-options"]
env["csvViaHeaderOptions"]["start-date"] = sim["start-date"]
env["csvViaHeaderOptions"]["end-date"] = sim["end-date"]
env["pathToClimateCSV"] = []
for PATH in PATH_TO_CLIMATE_DATA_DIR:
env["pathToClimateCSV"].append(PATH + "row-" +
str(meteo_id[0]) + "/col-" +
str(meteo_id[1]) + ".csv")
for sim_id, sim_ in sims.iteritems():
if sim_id != PRODUCTION_LEVEL:
continue
env["events"] = sim_["output"]
env["params"]["simulationParameters"][
"NitrogenResponseOn"] = sim_["NitrogenResponseOn"]
env["params"]["simulationParameters"][
"WaterDeficitResponseOn"] = sim_[
"WaterDeficitResponseOn"]
env["params"]["simulationParameters"][
"UseAutomaticIrrigation"] = sim_[
"UseAutomaticIrrigation"]
env["params"]["simulationParameters"][
"UseNMinMineralFertilisingMethod"] = sim_[
"UseNMinMineralFertilisingMethod"]
env["params"]["simulationParameters"][
"FrostKillOn"] = sim_["FrostKillOn"]
for main_cp_iteration in range(
0, len(rots_info[int(rot_id)])):
#do not allow crop rotation to start with a CC
if "is-cover-crop" in env["cropRotation"][0].keys(
) and env["cropRotation"][0]["is-cover-crop"] == True:
rotate(env["cropRotation"])
env["customId"] = rot_id \
+ "|" + sim_id \
+ "|" + str(soil_id) \
+ "|(" + str(row) + "/" + str(col) + ")" \
+ "|" + str(bkr_id) \
+ "|" + str(main_cp_iteration) \
+ "|" + str(sim["UseSecondaryYields"]) \
+ "|" + str(timeframes[TF]["start-recording-out"]) \
+ "|" + str(RESIDUES_HUMUS_BALANCE) \
+ "|" + suffix \
+ "|" + KA5_txt \
+ "|" + soil_type
socket.send_json(env)
print "sent env ", sent_id, "customId:", env[
"customId"], "shared_id:", env["sharedId"]
sent_id += 1
rotate(env["cropRotation"])
if export_lat_lon_file:
export_lat_lon_file.close()
stop_send = time.clock()
print "sending ", sent_id, " envs took ", (stop_send -
start_send), " seconds"
print "simulated cells: ", simulated_cells, "; not found kreise for org N: ", no_kreis
def main():
"main"
context = zmq.Context()
socket = context.socket(zmq.PUSH)
#port = 6666 if len(sys.argv) == 1 else sys.argv[1]
config = {"port": 16666, "start": 1, "end": 8157}
if len(sys.argv) > 1:
for arg in sys.argv[1:]:
k, v = arg.split("=")
if k in config:
config[k] = int(v)
soil_db_con = sqlite3.connect(PATHS[USER]["PATH_TO_SOIL_DIR"] +
"soil.sqlite")
if LOCAL_RUN:
socket.connect("tcp://localhost:6666") # + str(config["port"]))
else:
socket.connect("tcp://cluster2:" + str(config["port"]))
with open("sim.json") as _:
sim = json.load(_)
with open("site.json") as _:
site = json.load(_)
with open("crop.json") as _:
crop = json.load(_)
#sim["include-file-base-path"] = PATHS[USER]["INCLUDE_FILE_BASE_PATH"]
crows = 938
ccols = 720
xll = 5137800
yll = 5562800
cellsize = 100
scols = 3653
srows = 5001
wgs84 = Proj(init="epsg:4326")
#gk3 = Proj(init="epsg:31467")
gk5 = Proj(init="epsg:31469")
#cdict = {}
def create_interpolator(path_to_file, wgs84, gk5):
"read an ascii grid into a map, without the no-data values"
with open(path_to_file) as file_:
# skip headerlines
file_.next()
file_.next()
nrows = 938
ncols = 720
points = np.zeros((ccols * crows, 2), np.int32)
values = np.zeros((ccols * crows), np.int32)
i = -1
row = -1
for line in file_:
row += 1
col = -1
for col_str in line.strip().split(" "):
col += 1
i += 1
clat, clon = col_str.split("|")
#cdict[(row, col)] = (clat, clon)
r, h = transform(wgs84, gk5, clon, clat)
points[i, 0] = h
points[i, 1] = r
values[i] = 1000 * row + col
#print "row:", row, "col:", col, "clat:", clat, "clon:", clon, "h:", h, "r:", r, "val:", values[i]
return NearestNDInterpolator(points, values)
interpol = create_interpolator(
PATHS[USER]["PATH_TO_CLIMATE_CSVS_DIR"] +
"germany-lat-lon-coordinates.grid", wgs84, gk5)
soil_profiles = {}
envs = {}
start = time.clock()
i = 1
srow = -1
with open(PATHS[USER]["PATH_TO_SOIL_DIR"] + "buek1000_100_gk5.asc") as _:
for i in range(0, 6):
_.next()
for line in _:
srow += 1
scol = -1
for col_str in line.strip().split(" "):
scol += 1
soil_id = int(col_str)
if soil_id == -9999:
continue
if soil_id not in envs:
site["SiteParameters"][
"SoilProfileParameters"] = soil_io.soil_parameters(
soil_db_con, soil_id)
envs[soil_id] = monica_io.create_env_json_from_json_config(
{
"crop": crop,
"site": site,
"sim": sim,
"climate": ""
})
envs[soil_id]["csvViaHeaderOptions"] = sim[
"climate.csv-options"]
env = envs[soil_id]
sh = yll + (cellsize / 2) + (srows - srow) * cellsize
sr = xll + (cellsize / 2) + scol * cellsize
inter = interpol(sh, sr)
crow = int(inter / 1000)
ccol = inter - (crow * 1000)
#clat, clon = cdict[(crow, ccol)]
#slon, slat = transform(gk5, wgs84, r, h)
#print "srow:", srow, "scol:", scol, "h:", sh, "r:", sr, " inter:", inter, "crow:", crow, "ccol:", ccol, "slat:", slat, "slon:", slon, "clat:", clat, "clon:", clon
if LOCAL_RUN:
env["pathToClimateCSV"] = PATHS[USER][
"PATH_TO_CLIMATE_CSVS_DIR"] + "germany/row-" + str(
crow) + "/col-" + str(ccol) + ".csv"
env["pathToClimateCSV"] = PATHS[USER][
"PATH_TO_CLIMATE_CSVS_DIR"] + "germany/row-851/col-543.csv"
else:
env["pathToClimateCSV"] = PATHS[USER][
"ARCHIVE_PATH_TO_CLIMATE_CSVS_DIR"] + "germany/row-" + str(
crow) + "/col-" + str(ccol) + ".csv"
env["pathToClimateCSV"] = PATHS[USER][
"ARCHIVE_PATH_TO_CLIMATE_CSVS_DIR"] + "germany/row-851/col-543.csv"
#print env["pathToClimateCSV"]
env["customId"] = "(" + str(crow) + "/" + str(ccol) + ")" \
+ "|(" + str(srow) + "/" + str(scol) + ")"
#with open("envs/env-"+str(i)+".json", "w") as _:
# _.write(json.dumps(env))
socket.send_json(env)
print "sent env ", i, " customId: ", env["customId"]
#if i > 100:
# exit()
i += 1
stop = time.clock()
print "sending ", (i - 1), " envs took ", (stop - start), " seconds"
#print "ran from ", start, "/", row_cols[start], " to ", end, "/", row_cols[end]
return
