def GetExpectedIncome(settings, console_output=None, logs_on=None):
"""
Either loading expected income if it was already calculated for these
settings, or calling the function to calculate the expected income.
Parameters
----------
settings : dict
Dictionary of settings as given by DefaultSettingsExcept().
console_output : boolean, optional
Specifying whether the progress should be documented thorugh console
outputs. The default is defined in ModelCode/GeneralSettings.
logs_on : boolean, optional
Specifying whether the progress should be documented in a log file.
The default is defined in ModelCode/GeneralSettings.
Returns
-------
expected_incomes : np.array of size (T, len(k_using))
The expected income of farmers in a scenario where the government is
not involved.
"""
# not all settings affect the expected income (as no government is
# included)
SettingsAffectingGuaranteedIncome = "k" + str(settings["k"]) + \
"Using" + '_'.join(str(n) for n in settings["k_using"]) + \
"Crops" + str(settings["num_crops"]) + \
"Start" + str(settings["sim_start"]) + \
"N" + str(settings["N"])
# open dict with all expected incomes that were calculated so far
with open("PenaltiesAndIncome/ExpectedIncomes.txt", "rb") as fp:
dict_incomes = pickle.load(fp)
# if expected income was already calculated for these settings, fetch it
if SettingsAffectingGuaranteedIncome in dict_incomes.keys():
_printing("\nFetching expected income",
console_output=console_output,
logs_on=logs_on)
expected_incomes = dict_incomes[SettingsAffectingGuaranteedIncome]
# else calculate (and save) it
else:
expected_incomes = _CalcExpectedIncome(settings, \
SettingsAffectingGuaranteedIncome, console_output = console_output, logs_on = logs_on)
dict_incomes[SettingsAffectingGuaranteedIncome] = expected_incomes
with open("PenaltiesAndIncome/ExpectedIncomes.txt", "wb") as fp:
pickle.dump(dict_incomes, fp)
# should not happen
if np.sum(expected_incomes < 0) > 0:
sys.exit("Negative expected income")
_printing(" Expected income per cluster in " + \
str(settings["sim_start"] - 1) + ": " + \
str(np.round(expected_incomes, 3)),
console_output = console_output, logs_on = logs_on)
return (expected_incomes)
def CropAreasDependingOnColaboration(panda_file="current_panda",
groupAim="Dissimilar",
groupMetric="medoids",
adjacent=False,
figsize=None,
cols=None,
console_output=None,
close_plots=None,
**kwargs):
"""
For each group size used for grouping, this makes plots to compare the
crop areas depending on the level of cooperation (i.e. group sizes)
Parameters
----------
panda_file : str, optional
Filename of panda csv to use for results.
groupAim : str, optional
The aim in grouping clusters, either "Similar" or "Dissimilar".
groupMetric : str, optional
The metric on which the grouping is based. The default is "medoids".
adjacent : boolean, optional
Whether clusters in a cluster group need to be adjacent. The default is False.
figsize : tuple, optional
The figure size. If None, the default as defined in ModelCode/GeneralSettings is used.
cols : list, optional
List of colors to use for plotting. If None, default values will
be used. The default is None.
close_plots : boolean or None
Whether plots should be closed after plotting (and saving). If None,
the default as defined in ModelCode/GeneralSettings is used.
**kwargs :
Settings specifiying for which model run results shall be returned
Returns
-------
None.
"""
# settings
if close_plots is None:
from ModelCode.GeneralSettings import close_plots
if console_output is None:
from ModelCode.GeneralSettings import console_output
if cols is None:
cols = ["royalblue", "darkred", "grey", "gold", \
"limegreen", "darkturquoise", "darkorchid", "seagreen",
"indigo"]
if adjacent:
add = "Adj"
add_title = ", adjacent"
else:
add = ""
add_title = ""
foldername = groupAim
if adjacent:
foldername = foldername + "Adjacent"
else:
foldername = foldername + "NonAdjacent"
# get cluster grouping for size 1 (i.e. all independent)
_printing("\nGroup size " + str(1),
console_output=console_output,
logs_on=False)
with open(
"InputData/Clusters/ClusterGroups/Grouping" +
groupMetric.capitalize() + "Size" + str(1) + groupAim + add +
".txt", "rb") as fp:
BestGrouping = pickle.load(fp)
# get results for the different cluster groups in this grouping
CropAllocs, MaxAreas, labels, Ns, Ms = _GetResultsToCompare(ResType="k_using",\
panda_file = panda_file,
console_output = console_output,
k_using = BestGrouping,
**kwargs)
total_areas = [sum([np.sum(cr, axis=(1, 2)) for cr in CropAllocs])]
# more settings
kwargs["N"] = min(Ns)
kwargs["validation_size"] = min(Ms)
settingsIterate = DefaultSettingsExcept(k_using=BestGrouping, **kwargs)
fnPlot = GetFilename(settingsIterate, groupSize = 1, groupAim = groupAim, \
adjacent = adjacent)
kwargs["N"] = None
kwargs["validation_size"] = None
# plot the crop areas for this grouping
_printing(" Plotting", console_output=console_output, logs_on=False)
_CompareCropAllocs(CropAllocs=CropAllocs,
MaxAreas=MaxAreas,
labels=labels,
title="Groups of size " + str(1) + " (" +
groupAim.lower() + add_title + ")",
legend_title="Cluster: ",
comparing="clusters",
filename=fnPlot,
foldername=foldername,
subplots=(3, 3),
close_plots=close_plots)
for size in [2, 3, 5, 9]:
# get cluster grouping for given group size
_printing("\nGroup size " + str(size),
console_output=console_output,
logs_on=False)
with open(
"InputData/Clusters/ClusterGroups/Grouping" +
groupMetric.capitalize() + "Size" + str(size) + groupAim +
add + ".txt", "rb") as fp:
BestGrouping = pickle.load(fp)
# settings
settingsIterate = DefaultSettingsExcept(k_using=BestGrouping, **kwargs)
fnPlot = GetFilename(settingsIterate, groupSize = str(size), groupAim = groupAim, \
adjacent = adjacent)
# get results for the different cluster groups in this grouping
CropAllocs_pooling, MaxAreas_pooling, labels_pooling, Ns, Ms = \
_GetResultsToCompare(ResType="k_using",\
panda_file = panda_file,
console_output = console_output,
k_using = BestGrouping,
**kwargs)
total_areas.append(
sum([np.sum(cr, axis=(1, 2)) for cr in CropAllocs_pooling]))
# more settings
kwargs["N"] = min(Ns)
kwargs["validation_size"] = min(Ms)
settingsIterate = DefaultSettingsExcept(k_using=BestGrouping, **kwargs)
fnPlot = GetFilename(settingsIterate, groupSize = str(size), groupAim = groupAim, \
adjacent = adjacent)
kwargs["N"] = None
kwargs["validation_size"] = None
# plot the crop areas for this grouping
_printing(" Plotting", console_output=console_output, logs_on=False)
_CompareCropAllocs(CropAllocs=CropAllocs_pooling,
MaxAreas=MaxAreas_pooling,
labels=labels_pooling,
title="Groups of size " + str(size) + " (" +
groupAim.lower() + add_title + ")",
legend_title="Cluster: ",
comparing="clusters",
filename=fnPlot,
foldername=foldername,
subplots=(3, 3),
close_plots=close_plots)
# plot the crop area comparison between independent and grouped for this grouping
_CompareCropAllocRiskPooling(CropAllocs_pooling,
CropAllocs,
MaxAreas_pooling,
MaxAreas,
labels_pooling,
labels,
filename=fnPlot,
foldername=foldername,
title=str(BestGrouping),
subplots=False,
close_plots=close_plots)
# plot total areas
settingsIterate["N"] = ""
settingsIterate["validation_size"] = ""
fnPlot = GetFilename(settingsIterate, groupAim=groupAim, adjacent=adjacent)
_PlotTotalAreas(total_areas,
groupAim,
adjacent,
fnPlot,
foldername=foldername,
close_plots=close_plots)
return (None)
def _GetResultsToCompare(ResType = "k_using", panda_file = "current_panda", \
console_output = None, **kwargs):
"""
Function that loads results from different model runs, with one setting
changing while the others stay the same (e.g. different clusters for same
settings).
Parameters
----------
ResType : str, optional
Which setting will be changed to compare different results. Needs to
be the exact name of that setting. The default is "k_using".
panda_file : str, optional
Filename of panda csv to use for results.
console_output : boolean, optional
Specifying whether the progress should be documented thorugh console
outputs. The default is defined in ModelCode/GeneralSettings.
**kwargs :
Settings specifiying for which model run results shall be returned
Returns
-------
CropAllocs : list
List of crop allocations for the different settings.
MaxAreas : list
List of maximum available agricultural areas for the different
settings.
labels : list
List of labels for plots (given information on the setting that is
changing).
Ns :
sample size of each model run for which results are returned
Ms :
validation sample size of each model run for which results are returned
"""
if console_output is None:
from ModelCode.GeneralSettings import console_output
if "PenMet" not in kwargs.keys():
kwargs["PenMet"] = "prob"
if kwargs["PenMet"] == "penalties":
kwargs["probF"] = None
kwargs["probS"] = None
# prepare cluster groups
if "k_using" in kwargs.keys():
k_using = kwargs["k_using"]
if ResType == "k_using":
if (type(k_using) is list) and (type(k_using[0]) is tuple):
k_using = [
sorted(list(k_using_tmp)) for k_using_tmp in k_using
]
elif (type(k_using) is list) and (type(k_using[0]) is int):
k_using = [[k_using_tmp] for k_using_tmp in k_using]
else:
if type(k_using) is tuple:
k_using = sorted(list(k_using))
elif type(k_using) is int:
k_using = [k_using]
kwargs["k_using"] = k_using
ToIterate = kwargs[ResType]
if type(ToIterate) is not list:
ToIterate = [ToIterate]
kwargs_tmp = kwargs.copy()
del kwargs_tmp["PenMet"]
# get filnenames of full results from panda
panda_filenames = pd.DataFrame()
for it in ToIterate:
kwargs_tmp[ResType] = it
panda_filenames = panda_filenames.append(\
_ReadFromPandaSingleClusterGroup(file = panda_file,
output_var = "Filename for full results",
**kwargs_tmp))
panda_filenames = panda_filenames.reset_index()
# get data
CropAllocs = []
MaxAreas = []
labels = []
Ns = []
Ms = []
_printing(" Fetching data", console_output=console_output, logs_on=False)
for idx, val in enumerate(ToIterate):
_printing(" " + ResType + ": " + str(val),
console_output=console_output,
logs_on=False)
settings, args, yield_information, population_information, \
status, all_durations, exp_incomes, crop_alloc, meta_sol, \
crop_allocF, meta_solF, crop_allocS, meta_solS, \
crop_alloc_vss, meta_sol_vss, VSS_value, validation_values = \
LoadModelResults(panda_filenames.at[idx, "Filename for full results"])
CropAllocs.append(crop_alloc)
MaxAreas.append(args["max_areas"])
labels.append(val)
Ns.append(settings["N"])
Ms.append(settings["validation_size"])
return (CropAllocs, MaxAreas, labels, Ns, Ms)
def _CalcExpectedIncome(settings,
SettingsAffectingGuaranteedIncome,
console_output=None,
logs_on=None):
"""
Calculating the expected income in the scenario corresponding to the
settings but without government.
Parameters
----------
settings : dict
Dictionary of settings as given by DefaultSettingsExcept().
SettingsAffectingGuaranteedIncome : str
Combining all settings that influence the expected income, used to
save the result for further runs.
console_output : boolean, optional
Specifying whether the progress should be documented thorugh console
outputs. The default is defined in ModelCode/GeneralSettings.
logs_on : boolean, optional
Specifying whether the progress should be documented in a log file.
The default is defined in ModelCode/GeneralSettings.
Returns
-------
expected_incomes : np.array of size (len(k_using),)
The expected income of farmers in a scenario where the government is
not involved.
"""
_printing("\nCalculating expected income ",
console_output=console_output,
logs_on=logs_on)
settings_ExpIn = settings.copy()
# change some settings: we are interested in the expected income in 2016
# (no need to change start year, as we set scenarios to fixed)
settings_ExpIn["yield_projection"] = "fixed"
settings_ExpIn["pop_scenario"] = "fixed"
settings_ExpIn["T"] = 1
probF = 0.99
# settings affecting the food demand penalty
SettingsBasics = "k" + str(settings_ExpIn["k"]) + \
"Using" + '_'.join(str(n) for n in settings_ExpIn["k_using"]) + \
"Crops" + str(settings_ExpIn["num_crops"]) + \
"Yield" + str(settings_ExpIn["yield_projection"]).capitalize() + \
"Start" + str(settings_ExpIn["sim_start"]) + \
"Pop" + str(settings_ExpIn["pop_scenario"]).capitalize() + \
"T" + str(settings_ExpIn["T"])
SettingsFirstGuess = SettingsBasics + "ProbF" + str(probF)
SettingsAffectingRhoF = SettingsFirstGuess + "N" + str(settings_ExpIn["N"])
# first guess
with open("PenaltiesAndIncome/RhoFs.txt", "rb") as fp:
dict_rhoFs = pickle.load(fp)
with open("PenaltiesAndIncome/crop_allocF.txt", "rb") as fp:
dict_crop_allocF = pickle.load(fp)
rhoFini, checkedGuess = _GetInitialGuess(dict_rhoFs, SettingsFirstGuess,
settings["N"])
# we assume that without government farmers aim for 99% probability of
# food security, therefore we find the right penalty for probF = 99%.
# As we want the income in a scenario without government, the resulting run
# of GetRhoF (with rohS = 0) automatically is the right run
args, yield_information, population_information = \
SetParameters(settings_ExpIn, console_output = False, logs_on = False)
rhoF, meta_solF, crop_allocF = _GetRhoWrapper(args,
probF,
rhoFini,
checkedGuess,
"F",
SettingsAffectingRhoF,
console_output=False,
logs_on=False)
dict_rhoFs[SettingsAffectingRhoF] = rhoF
dict_crop_allocF[SettingsAffectingRhoF] = crop_allocF
# saving updated dicts
with open("PenaltiesAndIncome/RhoFs.txt", "wb") as fp:
pickle.dump(dict_rhoFs, fp)
with open("PenaltiesAndIncome/crop_allocF.txt", "wb") as fp:
pickle.dump(dict_crop_allocF, fp)
return (meta_solF["avg_profits_preTax"].flatten())
def _OptimizeModel(settings, panda_file, console_output = None, logs_on = None, \
save = True, plotTitle = None):
"""
Function combines setting up and solving the model, calculating additional
information, and saving the results.
Parameters
----------
settings : dict
All input settings for the model framework.
panda_file : str or None
Name of the csv file used to append the results of the model run for
plotting and easy accessibility. If None, results are not saved to
panda csv. Default is "current_panda"
console_output : boolean, optional
Specifying whether the progress should be documented thorugh console
outputs. If None, the default as defined in ModelCode/GeneralSettings
is used.
logs_on : boolean, optional
Specifying whether the progress should be documented in a log file.
If None, the default as defined in ModelCode/GeneralSettings is used.
save : boolean, optional
whether the results should be saved (does not refer to the panda results).
The default is True.
plotTitle : str or None
If not None, a plot of the resulting crop allocations will be made
with that title and saved to Figures/CropAllocs.
Returns
-------
settings : dict
The model input settings that were given by user.
args : dict
Dictionary of arguments needed as direct model input.
yield_information : dict
Information on the yield distributions for the considered clusters.
population_information : dict
Information on the population in the considered area.
status : int
status of solver (optimal: 2)
all_durations : dict
Information on the duration of different steps of the model framework.
exp_incomes : dict
Expected income (on which guaranteed income is based) for the stochastic
and the deterministic setting.
crop_alloc : np.array
gives the optimal crop areas for all years, crops, clusters
meta_sol : dict
additional information about the model output ('exp_tot_costs',
'fix_costs', 'yearly_fixed_costs', 'fd_penalty', 'avg_fd_penalty',
'sol_penalty', 'shortcomings', 'exp_shortcomings', 'avg_profits_preTax',
'avg_profits_afterTax', 'food_supply', 'profits_preTax',
'profits_afterTax', 'num_years_with_losses', 'payouts', 'final_fund',
'probF', 'probS', 'avg_nec_import', 'avg_nec_debt', 'guaranteed_income')
crop_allocF : np.array
optimal crop allocation for scenario with only food security objective
meta_solF : dict
additional information on model output for scenario with only food
security objective
crop_allocS : np.array
optimal crop allocation for scenario with only solvency objective
meta_solS : dict
additional information on model output for scenario with only solvency
objective
crop_alloc_vss : np.array
deterministic solution for optimal crop areas
meta_sol_vss : dict
additional information on the deterministic solution
VSS_value : float
VSS calculated as the difference between total costs using
deterministic solution for crop allocation and stochastic solution
for crop allocation
validation_values : dict
total costs and penalty costs for the resulted crop areas but a higher
sample size of crop yields for validation ("sample_size",
"total_costs", "total_costs_val", "fd_penalty", "fd_penalty_val",
"sol_penalty", "sol_penalty_val", "total_penalties",
"total_penalties_val", "deviation_penalties")
"""
# timing
all_start = tm.time()
all_durations = {}
# file name used for log file and saving results
fn = GetFilename(settings)
# initialize log file
if logs_on is None:
from ModelCode.GeneralSettings import logs_on
if logs_on:
import ModelCode.GeneralSettings as GS
GS.fn_log = fn
if os.path.exists("ModelLogs/" + GS.fn_log + ".txt"):
i = 1
while os.path.exists("ModelLogs/" + GS.fn_log + "_" + str(i) +
".txt"):
i += 1
GS.fn_log = GS.fn_log + "_" + str(i)
log = open("ModelLogs/" + GS.fn_log + ".txt", "a")
log.write("Model started " +
str(datetime.now().strftime("%B %d, %Y, at %H:%M")) + "\n")
log.write("\nModel Settings: ")
for key in settings.keys():
log.write(key + " = " + str(settings[key]) + "\n ")
log.write("save = " + str(save) + "\n ")
log.write("plotTitle = " + str(plotTitle))
log.close()
# get parameters for the given settings
ex_income_start = tm.time()
exp_incomes = GetExpectedIncome(settings, console_output = console_output, \
logs_on = logs_on)
exp_incomes = {"sto. setting": exp_incomes}
ex_income_end = tm.time()
all_durations["GetExpectedIncome"] = ex_income_end - ex_income_start
_printing("\nGetting parameters", console_output = console_output, \
logs_on = logs_on)
args, yield_information, population_information = \
SetParameters(settings, exp_incomes["sto. setting"])
# get the right penalties
penalties_start = tm.time()
if settings["PenMet"] == "prob":
rhoF, rhoS, meta_solF, meta_solS, crop_allocF, crop_allocS = \
GetPenalties(settings, args, console_output = console_output, \
logs_on = logs_on)
args["rhoF"] = rhoF
args["rhoS"] = rhoS
else:
args["rhoF"] = settings["rhoF"]
if settings["solv_const"] == "on":
args["rhoS"] = settings["rhoS"]
elif settings["solv_const"] == "off":
args["rhoS"] = 0
meta_solF = None
meta_solS = None
crop_allocF = None
crop_allocS = None
penalties_end = tm.time()
all_durations["GetPenalties"] = penalties_end - penalties_start
# run the optimization
status, crop_alloc, meta_sol, prob, durations = \
SolveReducedLinearProblemGurobiPy(args, \
console_output = console_output, \
logs_on = logs_on)
all_durations["MainModelRun"] = durations[2]
_printing("\nResulting probabilities:\n" + \
" probF: " + str(np.round(meta_sol["probF"]*100, 2)) + "%\n" + \
" probS: " + str(np.round(meta_sol["probS"]*100, 2)) + "%",
console_output = console_output,
logs_on = logs_on)
# VSS
vss_start = tm.time()
_printing("\nCalculating VSS", console_output = console_output, \
logs_on = logs_on)
crop_alloc_vss, expected_incomes_vss, meta_sol_vss = VSS(settings, args)
exp_incomes["det. setting"] = expected_incomes_vss
VSS_value = meta_sol_vss["exp_tot_costs"] - meta_sol["exp_tot_costs"]
vss_end = tm.time()
all_durations["VSS"] = vss_end - vss_start
# out of sample validation
validation_start = tm.time()
if settings["validation_size"] is not None:
_printing("\nOut of sample validation", console_output = console_output, \
logs_on = logs_on)
validation_values = OutOfSampleVal(crop_alloc, settings,
exp_incomes["sto. setting"], args["rhoF"], args["rhoS"], \
meta_sol, console_output, logs_on = logs_on)
validation_end = tm.time()
all_durations["Validation"] = validation_end - validation_start
# add results to pandas overview
if panda_file is not None:
_WriteToPandas(settings, args, yield_information, population_information, \
status, all_durations, exp_incomes, crop_alloc, meta_sol, \
crop_allocF, meta_solF, crop_allocS, meta_solS, \
crop_alloc_vss, meta_sol_vss, VSS_value, validation_values, \
fn, console_output, logs_on, panda_file)
# timing
all_end = tm.time()
full_time = all_end - all_start
_printing("\nTotal time: " + str(np.round(full_time, 2)) + "s",
console_output=console_output,
logs_on=logs_on)
all_durations["TotalTime"] = full_time
# saving results
if save:
info = ["settings", "args", "yield_information", "population_information", \
"status", "durations", "crop_alloc", "crop_allocF", "crop_allocS", \
"crop_alloc_vss", "VSS_value", "validation_values"]
with open("ModelOutput/SavedRuns/" + fn + ".txt", "wb") as fp:
pickle.dump(info, fp)
pickle.dump(settings, fp)
pickle.dump(args, fp)
pickle.dump(yield_information, fp)
pickle.dump(population_information, fp)
pickle.dump(status, fp)
pickle.dump(all_durations, fp)
pickle.dump(exp_incomes, fp)
pickle.dump(crop_alloc, fp)
pickle.dump(crop_allocF, fp)
pickle.dump(crop_allocS, fp)
pickle.dump(crop_alloc_vss, fp)
pickle.dump(VSS_value, fp)
pickle.dump(validation_values, fp)
# remove the global variable fn_log
if logs_on:
del GS.fn_log
return(settings, args, yield_information, population_information, \
status, all_durations, exp_incomes, crop_alloc, meta_sol, \
crop_allocF, meta_solF, crop_allocS, meta_solS, \
crop_alloc_vss, meta_sol_vss, VSS_value, validation_values)
def FoodSecurityProblem(console_output = None, logs_on = None, \
save = True, plotTitle = None, close_plots = None,
panda_file = "current_panda", **kwargs):
"""
Setting up and solving the food security problem. Returns model output
and additional information on the solution, as well as the VSS and a
validation of the model output. The results are also saved. If the model
has already been solved for the exact settings, results are loaded instead
of recalculated.
Parameters
----------
console_output : boolean, optional
Specifying whether the progress should be documented thorugh console
outputs. If None, the default as defined in ModelCode/GeneralSettings
is used.
logs_on : boolean, optional
Specifying whether the progress should be documented in a log file.
If None, the default as defined in ModelCode/GeneralSettings is used.
save : boolean, optional
whether the results should be saved. The default is True.
plotTitle : str or None
If not None, a plot of the resulting crop allocations will be made
with that title and saved to Figures/CropAllocs/numclusters/.
close_plots : boolean or None
Whether plots should be closed after plotting (and saving). If None,
the default as defined in ModelCode/GeneralSettings is used.
panda_file : str
Name of the csv file used to append the results of the model run for
plotting and easy accessibility. If None, results are not saved to
panda csv. Default is "current_panda"
**kwargs
settings for the model, passed to DefaultSettingsExcept()
Returns
-------
settings : dict
The model input settings that were given by user.
args : dict
Dictionary of arguments needed as direct model input.
yield_information : dict
Information on the yield distributions for the considered clusters.
population_information : dict
Information on the population in the considered area.
status : int
status of solver (optimal: 2)
all_durations : dict
Information on the duration of different steps of the model framework.
exp_incomes : dict
Expected income (on which guaranteed income is based) for the stochastic
and the deterministic setting.
crop_alloc : np.array
gives the optimal crop areas for all years, crops, clusters
meta_sol : dict
additional information about the model output ('exp_tot_costs',
'fix_costs', 'yearly_fixed_costs', 'fd_penalty', 'avg_fd_penalty',
'sol_penalty', 'shortcomings', 'exp_shortcomings', 'avg_profits_preTax',
'avg_profits_afterTax', 'food_supply', 'profits_preTax',
'profits_afterTax', 'num_years_with_losses', 'payouts', 'final_fund',
'probF', 'probS', 'avg_nec_import', 'avg_nec_debt', 'guaranteed_income')
crop_allocF : np.array
optimal crop allocation for scenario with only food security objective
meta_solF : dict
additional information on model output for scenario with only food
security objective
crop_allocS : np.array
optimal crop allocation for scenario with only solvency objective
meta_solS : dict
additional information on model output for scenario with only solvency
objective
crop_alloc_vss : np.array
deterministic solution for optimal crop areas
meta_sol_vss : dict
additional information on the deterministic solution
VSS_value : float
VSS calculated as the difference between total costs using
deterministic solution for crop allocation and stochastic solution
for crop allocation
validation_values : dict
total costs and penalty costs for the resulted crop areas but a higher
sample size of crop yields for validation ("sample_size",
"total_costs", "total_costs_val", "fd_penalty", "fd_penalty_val",
"sol_penalty", "sol_penalty_val", "total_penalties",
"total_penalties_val", "deviation_penalties")
fn : str
all settings combined to a single file name to save/load results and
for log file
"""
# set up folder structure (if not already done)
CheckFolderStructure()
# defining settings
settings = DefaultSettingsExcept(**kwargs)
# get filename for model results
fn = GetFilename(settings)
# if model output does not exist yet it is calculated
if not os.path.isfile("ModelOutput/SavedRuns/" + fn + ".txt"):
try:
settings, args, yield_information, population_information, \
status, all_durations, exp_incomes, crop_alloc, meta_sol, \
crop_allocF, meta_solF, crop_allocS, meta_solS, \
crop_alloc_vss, meta_sol_vss, VSS_value, validation_values = \
_OptimizeModel(settings,
console_output = console_output,
save = save,
logs_on = logs_on,
plotTitle = plotTitle,
panda_file = panda_file)
except KeyboardInterrupt:
print(colored("\nThe model run was interrupted by the user.",
"red"),
flush=True)
import ModelCode.GeneralSettings as GS
if logs_on or (logs_on is None and GS.logs_on):
log = open("ModelLogs/" + GS.fn_log + ".txt", "a")
log.write("\n\nThe model run was interupted by the user.")
log.close()
del GS.fn_log
# if it does, it is loaded from output file
else:
_printing("Loading results",
console_output=console_output,
logs_on=False)
settings, args, yield_information, population_information, \
status, all_durations, exp_incomes, crop_alloc, meta_sol, \
crop_allocF, meta_solF, crop_allocS, meta_solS, \
crop_alloc_vss, meta_sol_vss, VSS_value, validation_values = \
LoadModelResults(fn)
# if a plottitle is provided, crop allocations over time are plotted
if plotTitle is not None:
_PlotCropAlloc(crop_alloc = crop_alloc, k = settings["k"], \
k_using = settings["k_using"], max_areas = args["max_areas"], \
close_plots = close_plots, title = plotTitle, file = fn)
return(settings, args, yield_information, population_information, \
status, all_durations, exp_incomes, crop_alloc, meta_sol, \
crop_allocF, meta_solF, crop_allocS, meta_solS, \
crop_alloc_vss, meta_sol_vss, VSS_value, validation_values, fn)
def OutOfSampleVal(crop_alloc, settings, expected_incomes, rhoF, rhoS, \
meta_sol, console_output = None, logs_on = None):
"""
For validation, the objective function is re-evaluate, using the optimal
crop allocation but a higher sample size.
Parameters
----------
crop_alloc : np.array
Optimal crop areas for all years, crops, clusters.
settings : dict
the model settings that were used
expected_incomes : np.array of size (len(k_using),)
The expected income of farmers in a scenario where the government is
not involved.
rhoF : float
The penalty for shortcomings of the food demand.
rhoS : float
The penalty for insolvency.
meta_sol : dict
additional information about the model output
console_output : boolean, optional
Specifying whether the progress should be documented thorugh console
outputs. The default is defined in ModelCode/GeneralSettings.
logs_on : boolean, optional
Specifying whether the progress should be documented in a log file.
The default is defined in ModelCode/GeneralSettings.
Returns
------
validation_values : dict
Dictionary of validation values
- sample_size : Validation sample size
- total_costs : expected total costs in model run
- total_costs_val : expected total costs in validation run
- fd_penalty : average total food demand penalty in model run
- fd_penalty_val : average total food demand penalty in validation run
- sol_penalty : average total solvency penalty in model run
- sol_penalty_val : average total solvency penalty in validation run
- total_penalties : fd_penalty + sol_penalty
- total_penalties_val : fd_penalty_val + sol_penalty_val
- deviation_penalties : 1 - (total_penalties / total_penalties_val)
"""
# higher sample size
settings_val = settings.copy()
settings_val["N"] = settings["validation_size"]
# get yield samples
_printing(" Getting parameters and yield samples",
console_output=console_output,
logs_on=logs_on)
args, yield_information, population_information = \
SetParameters(settings_val, expected_incomes, \
console_output = False, logs_on = False)
# run objective function for higher sample size
_printing(" Objective function",
console_output=console_output,
logs_on=logs_on)
meta_sol_val = GetMetaInformation(crop_alloc, args, rhoF, rhoS)
# create dictionary with validation information
validation_values = {
"sample_size": settings["validation_size"],
"total_costs": meta_sol["exp_tot_costs"],
"total_costs_val": meta_sol_val["exp_tot_costs"],
"fd_penalty": np.nanmean(np.sum(meta_sol["fd_penalty"], axis=1)),
"fd_penalty_val": np.nanmean(np.sum(meta_sol_val["fd_penalty"],
axis=1)),
"sol_penalty": np.nanmean(meta_sol["sol_penalty"]),
"sol_penalty_val": np.nanmean(meta_sol_val["sol_penalty"])
}
validation_values["total_penalties"] = validation_values["fd_penalty"] + \
validation_values["sol_penalty"]
validation_values["total_penalties_val"] = validation_values["fd_penalty_val"] + \
validation_values["sol_penalty_val"]
validation_values["deviation_penalties"] = 1 - \
(validation_values["total_penalties"]/validation_values["total_penalties_val"])
# add this dictionary to validation file
with open("ModelOutput/validation.txt", "rb") as fp:
val_dict = pickle.load(fp)
val_name = str(len(settings["k_using"])) + "of" + \
str(settings["k"]) + "_N" + str(settings["N"]) + \
"_M" + str(settings["validation_size"])
if val_name in val_dict.keys():
tmp = val_dict[val_name]
tmp.append(validation_values["deviation_penalties"])
val_dict[val_name] = tmp
else:
val_dict[val_name] = [validation_values["deviation_penalties"]]
with open("ModelOutput/validation.txt", "wb") as fp:
pickle.dump(val_dict, fp)
return (validation_values)
def SolveReducedLinearProblemGurobiPy(args, rhoF = None, rhoS = None, \
console_output = None, logs_on = None):
"""
Sets up and solves the linear form of the food security problem.
Parameters
----------
args : dict
Dictionary of arguments needed as model input (as given by
SetParameters()).
rhoF : float or None
The penalty for shortcomings of the food demand. If None the values in
args are used. (This is used from within the GetPenalties function to
easily change penalties while keeping other args the same.)
rhoS : float or None
The penalty for insolvency. If None the values in
args are used. (This is used from within the GetPenalties function to
easily change penalties while keeping other args the same.)
console_output : boolean, optional
Specifying whether the progress should be documented thorugh console
outputs. The default is defined in ModelCode/GeneralSettings.
logs_on : boolean, optional
Specifying whether the progress should be documented in a log document.
The default is defined in ModelCode/GeneralSettings.
Returns
-------
status : int
status of solver (optimal: 2)
crop_alloc : np.array
gives the optimal crop areas for all years, crops, clusters
meta_sol : dict
additional information about the model output
prob : gurobi model
The food security model that was set up.
durations : list
time for setting up the model, time for solving, and total time (in
sec.)
"""
def _flatten(ListOfLists):
return(list(it.chain(*ListOfLists)))
if rhoF is None:
rhoF = args["rhoF"]
if rhoS is None:
rhoS = args["rhoS"]
_printing("\nSolving Model", console_output = console_output, logs_on = logs_on)
start = tm.time()
# no output to console from solver
env = gp.Env(empty = True)
env.setParam('OutputFlag', 0)
env.start()
# intitialize stochastic optimization problem
prob = gp.Model("SustainableFoodSecurity", env = env)
# get dimensions
T = args["T"]
K = len(args["k_using"])
J = args["num_crops"]
N = args["N"]
termyear_p1 = args["terminal_years"] + 1
termyear_p1[termyear_p1 == 0] = T
termyear_p1 = termyear_p1.astype(int)
# index tupels for variables and constraints
indVfood = _flatten([[(t, s) for t in range(0, termyear_p1[s])] \
for s in range(0, N)])
indW = _flatten(_flatten([[[(t, k, s) for t in range(0, termyear_p1[s])] \
for k in range(0,K)] for s in range(0, N)]))
indCultCosts = _flatten([[(t, j, k) for (t,j,k) in \
it.product(range(0,termyear_p1[s]), range(0, J), range(0, K))] \
for s in range(0, N)])
indMaxArea = list(it.product(range(0, K), range(0, T)))
indCropsClusters = list(it.product(range(0, J), range(0, K)))
# variables
x = prob.addVars(range(0, T), range(0, J), range(0, K), name = "x")
Vfood = prob.addVars(indVfood, name = "Vfood")
Vsol = prob.addVars(range(0, N), name = "Vsol")
Wgov = prob.addVars(indW, name = "Wgov")
# objective function
obj = gp.quicksum([1/N * x[t,j,k] * args["costs"][j,k] \
for (t,j,k) in indCultCosts] + \
[1/N * rhoF * Vfood[t, s] for (t, s) in indVfood] + \
[1/N * rhoS * Vsol[s] for s in range(0, N)] + \
[0 * Wgov[t, k, s] for (t, k, s) in indW])
prob.setObjective(obj, gp.GRB.MINIMIZE)
# constraints 1
prob.addConstrs((gp.quicksum([x[t, j, k] for j in range(0, J)]) \
<= args["max_areas"][k] for (k, t) in indMaxArea), "c_marea")
# constraints 2
prob.addConstrs((gp.quicksum([Vfood[t, s]] + \
[args["ylds"][s, t, j, k] * x[t, j, k] * args["crop_cal"][j] \
for (j, k) in indCropsClusters]) \
>= (args["demand"][t] - args["import"]) \
for (t, s) in indVfood), "c_demand")
# constraints 3
prob.addConstrs((gp.quicksum([-Vsol[s]] + \
[- args["tax"] * (args["ylds"][s, t, j, k] * \
x[t, j, k] * args["prices"][j, k] - \
x[t, j, k] * args["costs"][j, k]) \
for (j, t, k) in it.product(range(0, J), \
range(0, termyear_p1[s]), range(0, K))] + \
[args["cat_clusters"][s, t, k] * (1 - args["tax"]) * Wgov[t, k, s] \
for (t, k) in it.product(range(0, termyear_p1[s]), \
range(0, K))]) \
<= args["ini_fund"] for s in range(0, N)), "c_sol")
# constraints 4
prob.addConstrs((gp.quicksum([- Wgov[t, k, s]] + \
[- args["ylds"][s, t, j, k] * x[t, j, k] * args["prices"][j, k] + \
x[t, j, k] * args["costs"][j, k] for j in range(0, J)]) \
<= - args["guaranteed_income"][t, k] for (t, k, s) in indW), "c_gov")
# solving
middle = tm.time()
prob.optimize()
end = tm.time()
status = prob.status
# calculate durations
durationBuild = middle - start
durationSolve = end - middle
durationTotal = end - start
durations = [durationBuild, durationSolve, durationTotal]
# get results
crop_alloc = np.zeros((T, J, K))
meta_sol = []
if status != 2:
warn.warn("Non-optimal status of solver")
return(status, crop_alloc, meta_sol, prob, durations)
else:
for t in range(0, T):
for j in range(0, J):
for k in range(0, K):
crop_alloc[t, j, k] = prob.getVarByName("x[" + str(t) + \
"," + str(j) + "," + str(k) + "]").X
meta_sol = GetMetaInformation(crop_alloc, args, rhoF, rhoS)
_printing(" Time Setting up model: " + \
str(np.round(durations[0], 2)) + "s", console_output = console_output, logs_on = logs_on)
_printing(" Solving model: " + \
str(np.round(durations[1], 2)) + "s", console_output = console_output, logs_on = logs_on)
_printing(" Total: " + \
str(np.round(durations[2], 2)) + "s", console_output = console_output, logs_on = logs_on)
return(status, crop_alloc, meta_sol, prob, durations)
def RemoveRun(file = "current_panda", **kwargs):
"""
Removes results of a specific model run (specified by **kwargs) from all
output files (panda csv, penaltiy dicts, guaranteed income, direct model
output).
Parameters
----------
file : str, optional
Name of the panda file from which the results should be removed.
The default is "current_panda".
**kwargs :
Settings specifiying for which model run results shall be removed.
Returns
-------
None.
"""
# get full settings
settings = DefaultSettingsExcept(**kwargs)
# get filename and name for dicts
fn, SettingsMaxProbF, SettingsAffectingRhoF, SettingsMaxProbS, \
SettingsAffectingRhoS = GetFilename(settings, allNames = True)
# remove line from panda
_printing("Removing from panda", logs_on = False)
current_panda = pd.read_csv("ModelOutput/Pandas/" + file + ".csv")
current_panda = current_panda[current_panda["Filename for full results"] != fn]
current_panda.to_csv("ModelOutput/Pandas/" + file + ".csv", index = False)
# remove from food demand penalty dicts
_printing("Removing from food security penalty dicts", logs_on = False)
with open("PenaltiesAndIncome/RhoFs.txt", "rb") as fp:
dict_rhoFs = pickle.load(fp)
with open("PenaltiesAndIncome/crop_allocF.txt", "rb") as fp:
dict_crop_allocF = pickle.load(fp)
dict_rhoFs.pop(SettingsAffectingRhoF, None)
dict_crop_allocF.pop(SettingsAffectingRhoF, None)
with open("PenaltiesAndIncome/RhoFs.txt", "wb") as fp:
pickle.dump(dict_rhoFs, fp)
with open("PenaltiesAndIncome/crop_allocF.txt", "wb") as fp:
pickle.dump(dict_crop_allocF, fp)
# remove from solvency penalty dicts
_printing("Removing from solvency penalty dicts", logs_on = False)
with open("PenaltiesAndIncome/RhoSs.txt", "rb") as fp:
dict_rhoSs = pickle.load(fp)
with open("PenaltiesAndIncome/crop_allocS.txt", "rb") as fp:
dict_crop_allocS = pickle.load(fp)
dict_rhoSs.pop(SettingsAffectingRhoS, None)
dict_crop_allocS.pop(SettingsAffectingRhoS, None)
with open("PenaltiesAndIncome/RhoSs.txt", "wb") as fp:
pickle.dump(dict_rhoSs, fp)
with open("PenaltiesAndIncome/crop_allocS.txt", "wb") as fp:
pickle.dump(dict_crop_allocS, fp)
# remove from income dict
_printing("Removing from income dict", logs_on = False)
SettingsAffectingGuaranteedIncome = "k" + str(settings["k"]) + \
"Using" + '_'.join(str(n) for n in settings["k_using"]) + \
"Crops" + str(settings["num_crops"]) + \
"Start" + str(settings["sim_start"]) + \
"N" + str(settings["N"])
with open("PenaltiesAndIncome/ExpectedIncomes.txt", "rb") as fp:
dict_incomes = pickle.load(fp)
dict_incomes.pop(SettingsAffectingGuaranteedIncome, None)
with open("PenaltiesAndIncome/ExpectedIncomes.txt", "wb") as fp:
pickle.dump(dict_incomes, fp)
# remove model outputs
_printing("Removing direct model outputs", logs_on = False)
if os.path.isfile("ModelOutput/SavedRuns/" + fn + ".txt"):
os.remove("ModelOutput/SavedRuns/" + fn + ".txt")
return(None)
def _WriteToPandas(settings, args, yield_information, population_information, \
status, all_durations, exp_incomes, crop_alloc, \
meta_sol, crop_allocF, meta_solF, crop_allocS, meta_solS, \
crop_alloc_vss, meta_sol_vss, VSS_value, validation_values, \
fn_fullresults, console_output = None, logs_on = None,
file = "current_panda"):
"""
Adds information on the model run to the given pandas csv.
!! If additional variables are included in the _WriteToPandas function,
they need to be added to all three dictionaries in _SetUpPandaDicts as well !!
Parameters
----------
settings : dict
The model input settings that were given by user.
args : dict
Dictionary of arguments needed as direct model input.
yield_information : dict
Information on the yield distributions for the considered clusters.
population_information : dict
Information on the population in the considered area.
status : int
status of solver (optimal: 2)
all_durations : dict
Information on the duration of different steps of the model framework.
exp_incomes : dict
Expected income (on which guaranteed income is based) for the stochastic
and the deterministic setting.
crop_alloc : np.array
gives the optimal crop areas for all years, crops, clusters
meta_sol : dict
additional information about the model output ('exp_tot_costs',
'fix_costs', 'yearly_fixed_costs', 'fd_penalty', 'avg_fd_penalty',
'sol_penalty', 'shortcomings', 'exp_shortcomings', 'avg_profits_preTax',
'avg_profits_afterTax', 'food_supply', 'profits_preTax',
'profits_afterTax', 'num_years_with_losses', 'payouts', 'final_fund',
'probF', 'probS', 'avg_nec_import', 'avg_nec_debt', 'guaranteed_income')
crop_allocF : np.array
optimal crop allocation for scenario with only food security objective
meta_solF : dict
additional information on model output for scenario with only food
security objective
crop_allocS : np.array
optimal crop allocation for scenario with only solvency objective
meta_solS : dict
additional information on model output for scenario with only solvency
objective
crop_alloc_vss : np.array
deterministic solution for optimal crop areas
meta_sol_vss : dict
additional information on the deterministic solution
VSS_value : float
VSS calculated as the difference between total costs using
deterministic solution for crop allocation and stochastic solution
for crop allocation
validation_values : dict
total costs and penalty costs for the resulted crop areas but a higher
sample size of crop yields for validation ("sample_size",
"total_costs", "total_costs_val", "fd_penalty", "fd_penalty_val",
"sol_penalty", "sol_penalty_val", "total_penalties",
"total_penalties_val", "deviation_penalties")
fn_fullresults : str
Filename used to save the full model results.
console_output : boolean, optional
Specifying whether the progress should be documented thorugh console
outputs. The default is defined in ModelCode/GeneralSettings.
logs_on : boolean, optional
Specifying whether the progress should be documented in a log file.
The default is defined in ModelCode/GeneralSettings.
file : str
filename of panda csv to which the information is to be added. The
default is "current_panda".
Returns
-------
None.
"""
_printing("\nAdding results to pandas",
console_output=console_output,
logs_on=logs_on)
# some pre-calculations
pop_per_cluster = np.outer(population_information["total_pop_scen"],
population_information["pop_cluster_ratio2015"])
pop_of_area = population_information["population"]
food_shortage_capita = meta_sol["shortcomings"] / pop_of_area
# meta_sol["shortcimings"] reports shortcomings as positive values, and
# surplus as zero
food_shortage_capita_only_shortage = food_shortage_capita.copy()
food_shortage_capita_only_shortage[food_shortage_capita_only_shortage ==
0] = np.nan
# to avoid getting nan as average: set years that never have shortcomings back to zero
food_shortage_capita_only_shortage[:,
np.sum(np.isnan(
food_shortage_capita_only_shortage),
axis=0) == settings["N"]] = 0
wh_neg_fund = np.where(meta_sol["final_fund"] < 0)[0]
ff_debt_neg = -meta_sol["final_fund"][wh_neg_fund]
ter_years_neg = args["terminal_years"][wh_neg_fund].astype(int)
wh_catastrophe = np.where(args["terminal_years"] != -1)[0]
ff_debt_cat = -meta_sol["final_fund"][wh_catastrophe]
ff_debt_cat[ff_debt_cat < 0] = 0
ter_years_cat = args["terminal_years"][wh_catastrophe].astype(int)
ff_debt_all = -meta_sol["final_fund"].copy()
ff_debt_all[ff_debt_all < 0] = 0
ter_years_all = args["terminal_years"].astype(int)
# finding cultivation costs by taking a sample w/o catastrophic yields
sample_no_cat = np.where(args["terminal_years"] == -1)[0][0]
cultivation_costs = np.sum(
meta_sol["yearly_fixed_costs"][sample_no_cat, :, :])
cultivation_costs_det = np.sum(
meta_sol_vss["yearly_fixed_costs"][sample_no_cat, :, :])
# setting up dictionary of parameters to add to the panda object as new row:
# 1 settings
panda = {
"Penalty method": settings["PenMet"],
"Input probability food security": settings["probF"],
"Input probability solvency": settings["probS"],
"Including solvency constraint": settings["solv_const"],
"Number of crops": settings["num_crops"],
"Number of clusters": settings["k"],
"Used clusters": settings["k_using"],
"Yield projection": settings["yield_projection"],
"Simulation start": settings["sim_start"],
"Population scenario": settings["pop_scenario"],
"Risk level covered": settings["risk"],
"Tax rate": settings["tax"],
"Share of income that is guaranteed": settings["perc_guaranteed"],
"Initial fund size": settings["ini_fund"],
"Sample size": settings["N"],
"Sample size for validation": settings["validation_size"],
"Number of covered years": settings["T"]
}
# 2 resulting penalties and probabilities
panda["Penalty for food shortage"] = args["rhoF"]
panda["Penalty for insolvency"] = args["rhoS"]
panda["Resulting probability for food security"] = meta_sol["probF"]
panda["Resulting probability for solvency"] = meta_sol["probS"]
if meta_solF is not None:
panda["Max. possible probability for food security (excluding solvency constraint)"] \
= meta_solF["probF"]
else:
panda["Max. possible probability for food security (excluding solvency constraint)"] \
= np.nan
if meta_solS is not None:
panda["Max. possible probability for solvency (excluding food security constraint)"] \
= meta_solS["probS"]
else:
panda["Max. possible probability for solvency (excluding food security constraint)"] \
= np.nan
# 3 yield information
panda["Probability for a catastrophic year"] = yield_information[
"prob_cat_year"]
panda["Share of samples with no catastrophe"] = yield_information[
"share_no_cat"]
panda[
"Share of years/clusters with unprofitable rice yields"] = yield_information[
"share_rice_np"]
panda[
"Share of years/clusters with unprofitable maize yields"] = yield_information[
"share_maize_np"]
# 4 population information
panda["Share of West Africa's population that is living in total considered region (2015)"] \
= np.sum(population_information["pop_cluster_ratio2015"])
panda["Share of West Africa's population that is living in considered clusters (2015)"] \
= list(population_information["pop_cluster_ratio2015"])
# 5 crop areas
panda["Available arable area"] = np.sum(args["max_areas"])
panda["On average cultivated area per cluster"] = list(
np.nanmean(crop_alloc, axis=(0, 1)))
panda["Average yearly total cultivated area"] = np.nanmean(
np.nansum(crop_alloc, axis=(1, 2)))
panda["Total cultivation costs (sto. solution)"] = cultivation_costs
# 6 food demand and needed import
panda["Import (given as model input)"] = args["import"]
panda["Average food demand"] = np.mean(args["demand"])
panda["Food demand per capita"] = args["demand"][
0] / population_information["population"][0]
panda["Average aggregate food shortage (without taking into account imports)"] \
= args["import"] + meta_sol["avg_nec_import"]
panda["Average aggregate food shortage"] \
= meta_sol["avg_nec_import"] # includes also caes that don't need import as zero
if meta_solF is not None:
panda["Average aggregate food shortage excluding solvency constraint"] \
= meta_solF["avg_nec_import"]
else:
panda["Average aggregate food shortage excluding solvency constraint"] \
= np.nan
panda["Average aggregate food shortage per capita"] \
= np.nanmean(np.nanmean(food_shortage_capita, axis = 0))*1e9
panda["Average aggregate food shortage per capita (including only samples that have shortage)"] \
= np.nanmean(np.nanmean(food_shortage_capita_only_shortage, axis = 0))*1e9
# 7 a priori expected income, resulting average income
panda["Expected income (stochastic setting)"] \
= list(exp_incomes["sto. setting"])
panda["Expected income (deterministic setting)"] \
= list(exp_incomes["det. setting"])
panda["Number of occurrences per cluster where farmers make losses"] \
= list(meta_sol["num_years_with_losses"])
panda["Average profits (pre tax) per cluster in final run (over samples and then years)"] \
= list(np.nanmean(np.nanmean(meta_sol["profits_preTax"], axis = 0), axis = 0)) # profits include both actual profits and losses
panda["Average profits (after tax) per cluster in final run (over samples and then years)"] \
= list(np.nanmean(np.nanmean(meta_sol["profits_afterTax"], axis = 0), axis = 0)) # profits include both actual profits and losses
panda["Average profits (pre tax) per cluster in final run scaled with capita (over samples and then years)"] \
= list(np.nanmean(np.nanmean(meta_sol["profits_preTax"], axis = 0)/(pop_per_cluster/1e9), axis = 0))
panda["Average profits (after tax) per cluster in final run scaled with capita (over samples and then years)"] \
= list(np.nanmean(np.nanmean(meta_sol["profits_afterTax"], axis = 0)/(pop_per_cluster/1e9), axis = 0))
panda["Aggregated average government payouts per cluster (over samples)"] \
= list(np.nansum(np.nanmean(meta_sol["payouts"], axis = 0), axis = 0))
# 8 final fund and needed debt
panda["Number of samples with negative final fund"] \
= np.nansum(meta_sol["final_fund"] < 0)
panda["Average final fund (over all samples)"] \
= np.nanmean(meta_sol["final_fund"])
panda["Average final fund (over samples with catastrophe)"] \
= np.nanmean(meta_sol["final_fund"][args["terminal_years"] != -1])
if meta_solS is not None:
panda["Average aggregate debt after payout (excluding food security constraint)"] \
= meta_solS["avg_nec_debt"]
else:
panda["Average aggregate debt after payout (excluding food security constraint)"] \
= np.nan
panda["Average aggregate debt after payout"] \
= meta_sol["avg_nec_debt"] # includes cases that don't need debt as zero
panda["Average aggregate debt after payout (including only samples with negative final fund)"] \
= np.nanmean(ff_debt_neg)
panda["Average aggregate debt after payout (including only samples with catastrophe)"] \
= np.nanmean(ff_debt_cat)
panda["Average aggregate debt after payout per capita (including only samples with catastrophe)"] \
= np.nanmean(ff_debt_cat / (pop_of_area[ter_years_cat]/1e9))
panda["Average aggregate debt after payout per capita (including only samples with negative final fund)"] \
= np.nanmean(ff_debt_neg / (pop_of_area[ter_years_neg]/1e9))
panda["Average aggregate debt after payout per capita"] \
= np.nanmean(ff_debt_all / (pop_of_area[ter_years_all]/1e9)) # negative debt set to zero
# 9 different cost items in objective function
panda["Average food demand penalty (over samples and then years)"] \
= np.nanmean(np.nanmean(meta_sol["fd_penalty"], axis = 0))
panda["Average total food demand penalty (over samples)"] \
=np.nanmean(np.nansum(meta_sol["fd_penalty"], axis = 1))
panda["Average solvency penalty (over samples)"] \
= np.mean(meta_sol["sol_penalty"])
panda["Average total cultivation costs"] \
= np.nanmean(np.nansum(meta_sol["yearly_fixed_costs"], axis = (1,2)))
panda["Expected total costs"] \
= meta_sol["exp_tot_costs"] # this includes varying cultivation costs (depending on catastrophic year)
# 10 VSS
panda[
"Value of stochastic solution"] = VSS_value # diff of total costs using det. solution and using
panda["Total cultivation costs (det. solution)"] = cultivation_costs_det
if meta_sol["exp_tot_costs"] == 0:
panda["VSS as share of total costs (sto. solution)"] = 0
else:
panda[
"VSS as share of total costs (sto. solution)"] = VSS_value / meta_sol[
"exp_tot_costs"]
panda[
"VSS as share of total costs (det. solution)"] = VSS_value / meta_sol_vss[
"exp_tot_costs"]
panda["VSS in terms of avg. nec. debt"] \
= meta_sol_vss["avg_nec_debt"] - meta_sol["avg_nec_debt"]
if meta_sol_vss["avg_nec_debt"] == 0:
panda[
"VSS in terms of avg. nec. debt as share of avg. nec. debt of det. solution"] = 0
else:
panda["VSS in terms of avg. nec. debt as share of avg. nec. debt of det. solution"] \
= (meta_sol_vss["avg_nec_debt"] - meta_sol["avg_nec_debt"])/meta_sol_vss["avg_nec_debt"]
panda["VSS in terms of avg. nec. debt as share of avg. nec. debt of sto. solution"] \
=(meta_sol_vss["avg_nec_debt"] - meta_sol["avg_nec_debt"])/meta_sol["avg_nec_debt"]
panda["VSS in terms of avg. nec. import"] \
= meta_sol_vss["avg_nec_import"] - meta_sol["avg_nec_import"]
panda["VSS in terms of avg. nec. import as share of avg. nec. import of det. solution"] \
= (meta_sol_vss["avg_nec_import"] - meta_sol["avg_nec_import"])/meta_sol_vss["avg_nec_import"]
if meta_sol["avg_nec_import"] == 0:
panda["VSS in terms of avg. nec. import as share of avg. nec. import of sto. solution"] \
= 0
else:
panda["VSS in terms of avg. nec. import as share of avg. nec. import of sto. solution"] \
= (meta_sol_vss["avg_nec_import"] - meta_sol["avg_nec_import"])/meta_sol["avg_nec_import"]
panda["Resulting probability for food security for VSS"] = meta_sol_vss[
"probF"]
panda["Resulting probability for solvency for VSS"] = meta_sol_vss["probS"]
# 11 technincal variables
panda[
"Validation value (deviation of total penalty costs)"] = validation_values[
"deviation_penalties"]
panda["Seed (for yield generation)"] = settings["seed"]
panda["Filename for full results"] = fn_fullresults
# load panda object
if not os.path.exists("ModelOutput/Pandas/" + file + ".csv"):
CreateEmptyPanda(file)
# add new row
current_panda = pd.read_csv("ModelOutput/Pandas/" + file + ".csv")
current_panda = current_panda.append(panda, ignore_index=True)
# try to sve updated panda object as csv
saved = False
while saved == False:
try:
current_panda.to_csv("ModelOutput/Pandas/" + file + ".csv",
index=False)
except PermissionError:
print(colored("Could not save updated panda.", "cyan"))
print(
colored(
"Please close the corresponding csv if currently open.",
"cyan"))
continue
saved = True
return (None)
def SetParameters(settings,
expected_incomes=None,
wo_yields=False,
VSS=False,
console_output=None,
logs_on=None):
"""
Based on the settings, this sets most of the parameters that are needed as
input to the model.
Parameters
----------
settings : dict
Dictionary of settings as given by DefaultSettingsExcept().
expected_incomes : np.array of size (len(k_using),)
The expected income of farmers in a scenario where the government is
not involved.
wo_yields : boolean, optional
If True, the function will do everything execept generating the yield
samples (and return an empty list as placeholder for the ylds
parameter). The default is False.
VSS : boolean, optional
If True, instead of yield samples only the average yields will be
returned and all clusters will be indicated as non-catastrophic by
cat_clusters, as needed to calculate the deterministic solution on
which the VSS is based. The default is False.
console_output : boolean, optional
Specifying whether the progress should be documented thorugh console
outputs. The default is defined in ModelCode/GeneralSettings.
logs_on : boolean, optional
Specifying whether the progress should be documented in a log document.
The default is defined in ModelCode/GeneralSettings.
Returns
-------
args : dict
Dictionary of arguments needed as model input.
- k: number of clusters in which the area is devided.
- k_using: specifies which of the clusters are to be considered in
the model.
- num_crops: the number of crops that are used
- N: number of yield samples to be used to approximate the
expected value in the original objective function
- cat_cluster: np.array of size (N, T, len(k_using)),
indicating clusters with yields labeled as catastrophic with 1,
clusters with "normal" yields with 0
- terminal years: np.array of size (N,) indicating the year in
which the simulation is terminated (i.e. the first year with a
catastrophic cluster) for each sample
- ylds: np.array of size (N, T, num_crops, len(k_using)) of
yield samples in 10^6t/10^6ha according to the presence of
catastrophes
- costs: np array of size (num_crops,) giving cultivation costs
for each crop in 10^9$/10^6ha
- demand: np.array of size (T,) giving the total food demand
for each year in 10^12kcal
- ini_fund: initial fund size in 10^9$
- import: given import that will be subtraced from demand in 10^12kcal
- tax: tax rate to be paied on farmers profits
- prices: np.array of size (num_crops,) giving farm gate prices
farmers earn in 10^9$/10^6t
- T: number of years covered in model
- guaranteed_income: np.array of size (T, len(k_using)) giving
the income guaranteed by the government for each year and cluster
in case of catastrophe in 10^9$, based on expected income
- crop_cal : np.array of size (num_crops,) giving the calorie content
of the crops in 10^12kcal/10^6t
- max_areas: np.array of size (len(k_using),) giving the upper
limit of area available for agricultural cultivation in each
cluster
- probF : float or None, gigving demanded probability of keeping the
food demand constraint.
- probS : float or None, giving demanded probability of keeping the
solvency constraint.
yield_information : dict
Dictionary giving additional information on the yields
- slopes: slopes of the yield trends
- constants: constants of the yield trends
- yld_means: average yields
- residual_stds: standard deviations of the residuals of the yield
trends
- prob_cat_year: probability for a catastrophic year given the
covered risk level
- share_no_cat: share of samples that don't have a catastrophe within
the considered timeframe
- y_profit: minimal yield per crop and cluster needed to not have
losses
- share_rice_np: share of cases where rice yields are too low to
provide profit
- share_maize_np: share of cases where maize yields are too low to
provide profit
- exp_profit: expected profits in 10^9$/10^6ha per year, crop, cluster
population_information: dict
Dictionary giving additional information on the population size in the
model.
- population : np.array of size (T,) giving estimates of population
in considered clusters from simulation start to end for given population
scenario (from UN PopDiv)
- total_pop_scen : np.array of size (T,) giving estimates of population
in West Africa from simulation start to end for given population
scenario (from UN PopDiv)
- pop_cluster_ratio2015 : np.array of size (len(k_using),) giving the
share of population of a cluster to total population of West Africa
in 2015.
"""
crops = ["Rice", "Maize"]
# 0. extract settings from dictionary
k = settings["k"]
k_using = settings["k_using"]
num_crops = settings["num_crops"]
yield_projection = settings["yield_projection"]
sim_start = settings["sim_start"]
pop_scenario = settings["pop_scenario"]
risk = settings["risk"]
if VSS:
N = 1
else:
N = settings["N"]
T = settings["T"]
seed = settings["seed"]
tax = settings["tax"]
perc_guaranteed = settings["perc_guaranteed"]
if expected_incomes is None:
expected_incomes = np.zeros(len(k_using))
# 1. get cluster information (clusters given by k-Medoids on SPEI data)
with open("InputData/Clusters/Clustering/kMediods" + \
str(k) + "_PearsonDistSPEI.txt", "rb") as fp:
clusters = pickle.load(fp)
costs = pickle.load(fp)
# 2. calculate total area and area proportions of the different clusters
with open("InputData/Population/land_area.txt", "rb") as fp:
land_area = pickle.load(fp)
cluster_areas = np.zeros(len(k_using))
for i, cl in enumerate(k_using):
cluster_areas[i] = np.nansum(land_area[clusters == cl])
cluster_areas = cluster_areas * 100 # convert sq km to ha
# 3. Share of population in the area we use (in 2015):
with open("InputData/Population/GPW_WA.txt", "rb") as fp:
gridded_pop = pickle.load(fp)[3, :, :]
with open("InputData/Population/" + \
"UN_PopTotal_Prospects_WesternAfrica.txt", "rb") as fp:
total_pop = pickle.load(fp)
scenarios = pickle.load(fp)
total_pop_est_past = total_pop[np.where(
scenarios == "Medium")[0][0], :][0:71]
if pop_scenario == "fixed":
total_pop_scen = np.repeat(
total_pop[np.where(
scenarios == "Medium")[0][0], :][(sim_start - 1) - 1950], T)
else:
total_pop_scen = total_pop[np.where(scenarios == pop_scenario)[0][0],:]\
[(sim_start - 1950):(sim_start + T - 1950)]
total_pop_year_before = total_pop[np.where(scenarios == pop_scenario)[0][0],:]\
[sim_start - 1 - 1950]
total_pop_UN_2015 = total_pop_est_past[2015 - 1950]
cluster_pop = np.zeros(len(k_using))
for i, cl in enumerate(k_using):
cluster_pop[i] = np.nansum(gridded_pop[clusters == cl])
total_pop_GPW = np.sum(cluster_pop)
cluster_pop_ratio_2015 = cluster_pop / total_pop_UN_2015
total_pop_ratio_2015 = total_pop_GPW / total_pop_UN_2015
considered_pop_scen = total_pop_scen * total_pop_ratio_2015 # use 2015 ratio to scale down population scneario to considered area
# 5. Per person/day demand
# based on country specific caloric demand from a paper on food waste,
# averaged based on area. For more detail see DataPreparation_DoNotRun.py
with open("InputData/Other/AvgCaloricDemand.txt", "rb") as fp:
ppdemand = pickle.load(fp)
# 6. cultivation costs of crops
# average cultivation costs based on literature data for some West
# African countries (see DataPreparation_DoNotRun for details and
# sources)
with open("InputData/Other/CultivationCosts.txt", "rb") as fp:
costs = pickle.load(fp)
costs = np.transpose(np.tile(costs, (len(k_using), 1)))
# 7. Energy value of crops
with open("InputData/Other/CalorieContentCrops.txt", "rb") as fp:
crop_cal = pickle.load(fp)
# 8. Food demand
# based on the demand per person and day (ppdemand) and assuming no change
# of per capita daily consumption we use UN population scenarios for West
# Africa and scale them down to the area we use, using the ratio from 2015
# (from gridded GPW data)
demand = ppdemand * 365 * total_pop_scen
demand = demand * total_pop_ratio_2015
# in 10^12 kcal
demand = 1e-12 * demand
# 9. guaranteed income as share of expected income w/o government
# if expected income is not given...
guaranteed_income = np.repeat(expected_incomes[np.newaxis, :], T, axis=0)
guaranteed_income = perc_guaranteed * guaranteed_income
# guaraneteed income per person assumed to be constant over time,
# therefore scale with population size
if not pop_scenario == "fixed":
total_pop_ratios = total_pop_scen / total_pop_year_before
guaranteed_income = (guaranteed_income.swapaxes(0, 1) *
total_pop_ratios).swapaxes(0, 1)
# 10. prices for selling crops, per crop and cluster (but same over all clusters)
with open("InputData//Prices/RegionFarmGatePrices.txt", "rb") as fp:
prices = pickle.load(fp)
prices = np.transpose(np.tile(prices, (len(k_using), 1)))
# 11. thresholds for yields being profitable
y_profit = costs / prices
# 12. Agricultural Areas:
# Landscapes of West Africa - A Window on a Changing World:
# "Between 1975 and 2013, the area covered by crops doubled in West
# Africa, reaching a total of 1,100,000 sq km, or 22.4 percent, of the
# land surface."
# (https://eros.usgs.gov/westafrica/agriculture-expansion)
# => Approximate max. agricultural area by 22.4% of total cluster land
# area (assuming agricultural area is evenly spread over West Africa)
# Their area extends a bit more north, where agricultural land is
# probably reduced due to proximity to desert, so in our region the
# percentage of agricultural might be a bit higher in reality. But
# assuming evenly spread agricultural area over the area we use is
# already a big simplification, hence this will not be that critical.
max_areas = cluster_areas * 0.224
# in 10^6ha
max_areas = 1e-6 * max_areas
# 13. generating yield samples
# using historic yield data from GDHY
with open("InputData/YieldTrends/DetrYieldAvg_k" + str(k) + ".txt",
"rb") as fp:
pickle.load(fp) # yields_avg
pickle.load(fp) # avg_pred
pickle.load(fp) # residuals
pickle.load(fp) # residual_means
residual_stds = pickle.load(fp)
pickle.load(fp) # fstat
constants = pickle.load(fp)
slopes = pickle.load(fp)
pickle.load(fp) # crops
pickle.load(fp) # years
residual_stds = residual_stds[:, [i - 1 for i in k_using]]
constants = constants[:, [i - 1 for i in k_using]]
slopes = slopes[:, [i - 1 for i in k_using]]
# get yield realizations:
# what is the probability of a catastrophic year for given settings?
_printing("\nOverview on yield samples",
console_output=console_output,
logs_on=logs_on)
prob_cat_year = RiskForCatastrophe(risk, len(k_using))
_printing(" Prob for catastrophic year: " + \
str(np.round(prob_cat_year*100, 2)) + "%", \
console_output = console_output, logs_on = logs_on)
# create realizations of presence of catastrophic yields and corresponding
# yield distributions
np.random.seed(seed)
cat_clusters, terminal_years, ylds, yld_means = \
_YieldRealisations(slopes, constants, residual_stds, sim_start, \
N, risk, T, len(k_using), num_crops, \
yield_projection, VSS, wo_yields)
# probability to not have a catastrophe
no_cat = np.sum(terminal_years == -1) / N
_printing(" Share of samples without catastrophe: " + str(np.round(no_cat*100, 2)), \
console_output = console_output, logs_on = logs_on)
# share of non-profitable crops
if wo_yields:
share_rice_np = 0
share_maize_np = 0
else:
share_rice_np = np.sum(ylds[:, :, 0, :] < y_profit[0, :]) / np.sum(
~np.isnan(ylds[:, :, 0, :]))
_printing(" Share of cases with rice yields too low to provide profit: " + \
str(np.round(share_rice_np * 100, 2)), console_output = console_output, \
logs_on = logs_on)
share_maize_np = np.sum(ylds[:, :, 1, :] < y_profit[1, :]) / np.sum(
~np.isnan(ylds[:, :, 1, :]))
_printing(" Share of cases with maize yields too low to provide profit: " + \
str(np.round(share_maize_np * 100, 2)), console_output = console_output, \
logs_on = logs_on)
# in average more profitable crop
exp_profit = yld_means * prices - costs
avg_time_profit = np.nanmean(exp_profit, axis=0)
more_profit = np.argmax(avg_time_profit, axis=0) # per cluster
_printing(" On average more profit (per cluster): " + \
str([crops[i] for i in more_profit]), \
console_output = console_output, logs_on = logs_on)
# in average more productive crop
avg_time_production = np.nanmean(yld_means, axis=0)
more_food = np.argmax(avg_time_production, axis=0)
_printing(" On average higher productivity (per cluster): " + \
str([crops[i] for i in more_food]) + "\n", \
console_output = console_output, logs_on = logs_on)
# 14. group output into different dictionaries
# arguments that are given to the objective function by the solver
args = {
"k": k,
"k_using": k_using,
"num_crops": num_crops,
"N": N,
"cat_clusters": cat_clusters,
"terminal_years": terminal_years,
"ylds": ylds,
"costs": costs,
"demand": demand,
"ini_fund": settings["ini_fund"],
"import": settings["import"],
"tax": tax,
"prices": prices,
"T": T,
"guaranteed_income": guaranteed_income,
"crop_cal": crop_cal,
"max_areas": max_areas,
"probF": settings["probF"],
"probS": settings["probS"]
}
# information not needed by the solver but potentially interesting
yield_information = {
"slopes": slopes,
"constants": constants,
"yld_means": yld_means,
"residual_stds": residual_stds,
"prob_cat_year": prob_cat_year,
"share_no_cat": no_cat,
"y_profit": y_profit,
"share_rice_np": share_rice_np,
"share_maize_np": share_maize_np,
"exp_profit": exp_profit
}
population_information = {
"population": considered_pop_scen,
"total_pop_scen": total_pop_scen, # full area WA
"pop_cluster_ratio2015": cluster_pop_ratio_2015
}
return (args, yield_information, population_information)