def save_new_scenario_data(self, request):
# Create initial scenario
# Create initial scenario
# Create initial scenario
adapter = EMOD_Adapter(request.user.username)
emod_scenario = EMODBaseline(
name='',
description='Made with representative workflow',
model=DimModel.objects.get(model='EMOD'),
user=DimUser.objects.get_or_create(username=request.user.username)[0]
)
template_object = DimTemplate.objects.get(template_name="Representative Location")
emod_scenario.model_version = template_object.model_version
emod_scenario.template_id = 21
emod_scenario.template = template_object
# Create and add a config file
config_json = json.loads(template_object.get_file_content('config.json'))
emod_scenario.add_file_from_string('config', 'config.json', json.dumps(config_json), description='SOMETHING')
# populate campaign
try:
campaign = json.loads(template_object.get_file_content('campaign.json'))
except KeyError:
# use empty campaign file
campaign = json.loads({"Events": []})
# Add the emod_scenario campaign file
emod_scenario.add_file_from_string('campaign', 'campaign.json', json.dumps(campaign), description='SOMETHING')
emod_scenario.save()
# Create initial scenario
# Create initial scenario
# Create initial scenario
# Hack until I know how to retrieve files from DimBaseline without funneling it through EMODBaseline
dim_scenario = DimBaseline.objects.get(id=emod_scenario.id)
# Add metadata
current_metadata = {}
current_metadata['representative'] = {}
# Fill data
current_metadata['representative']['steps_complete'] = step_number + 1
current_metadata['representative']['is_complete'] = False
current_metadata['representative']['is_editable'] = True
dim_scenario.metadata = json.dumps(current_metadata)
dim_scenario.save()
print "Scenario ID = " + str(emod_scenario.id)
return emod_scenario.id
def add_simulation(dim_user, model, version, simulation_group, baseline_id, input_file_metadata=None):
assert isinstance(simulation_group, SimulationGroup)
emod_scenario = EMODBaseline.from_dw(id=baseline_id)
# Check if this is the right user for this scenario. All three should be the same. If the dim_scenario user
# and the simulation_group user are the same, and if user coming in is the same dis_scenario user, then all
# three are the same user.
if emod_scenario.user != simulation_group.submitted_by or dim_user != emod_scenario.user:
raise PermissionDenied
# Create simulation
simulation = Simulation.objects.create(
group=simulation_group,
model=model,
version=version,
status=sim_status.READY_TO_RUN
)
# Create input files and put them into a list
simulation_input_files = create_and_return_input_files(dim_user, emod_scenario)
# Add simulation input files to simulation
for i in range(len(simulation_input_files)):
simulation.input_files.add(simulation_input_files[i])
simulation.save()
return simulation
def get_targets(request, scenario_id):
dim_scenario = DimBaseline.objects.get(id=scenario_id)
# Check to see if this user has permission
if dim_scenario.user != DimUser.objects.get_or_create(username=request.user.username)[0]:
raise PermissionDenied
# Hack until I know how to retrieve files from DimBaseline without funneling it through EMODBaseline
emod_scenario = EMODBaseline.from_dw(id=scenario_id)
# Populate config
config_json = json.loads(emod_scenario.get_config_file().content)
# Get species
species = config_json['parameters']['Vector_Species_Params']
print species
# Get EIR
current_metadata = json.loads(dim_scenario.metadata)
if 'parasite_parameters' in current_metadata['representative']:
eir = current_metadata['representative']['parasite_parameters']['EIR']
else:
raise Exception("parasite_parameters missing from metadata.")
print eir
# Set up the targets dictionary with the items we want
targets = {}
targets['sporozoite'] = []
targets['eir'] = []
for specie in species:
targets['sporozoite'].append(species[specie]['Sporozoite_Rate'])
targets['eir'].append(eir)
return targets
def save_parasite_data(request):
# Get data
data = json.loads(request.body)
# Hack until I know how to retrieve files from DimBaseline without funneling it through EMODBaseline
dim_scenario = DimBaseline.objects.get(id=data['scenario_id'])
# Check to see if this user has permission
if dim_scenario.user != DimUser.objects.get_or_create(username=request.user.username)[0]:
raise PermissionDenied
# Get the metadata
current_metadata = json.loads(dim_scenario.metadata)
# Print data
print current_metadata
print "Data = " + str(data)
# Float correction
for species in data['parasite_parameters']['species']:
print species
for parasite_parameter in data['parasite_parameters']['species'][species]:
data['parasite_parameters']['species'][species][parasite_parameter] = float(data['parasite_parameters']['species'][species][parasite_parameter])
# Fill data
if current_metadata['representative']['steps_complete'] <= step_number:
current_metadata['representative']['steps_complete'] = step_number + 1
current_metadata['representative']['is_complete'] = True
dim_scenario.metadata = json.dumps(current_metadata)
dim_scenario.save()
# Hack until I know how to retrieve files from DimBaseline without funneling it through EMODBaseline
emod_scenario = EMODBaseline.from_dw(id=data['scenario_id'])
# Add the parasites to the scenario
# Load parasite parameter data
# parasite_parameters = data['parasite_parameters']
# new_diagnostic_sensitivity = parasite_parameters['new_diagnostic_sensitivity']
# parasite_smear_sensitivity = parasite_parameters['parasite_smear_sensitivity']
# Populate config
config_json = json.loads(emod_scenario.get_config_file().content)
# Change config
# config_json['parameters']['New_Diagnostic_Sensitivity'] = new_diagnostic_sensitivity
# config_json['parameters']['Parasite_Smear_Sensitivity'] = parasite_smear_sensitivity
for species in config_json['parameters']['Vector_Species_Params']:
clean_species_name = species.replace(' ', '_')
config_json['parameters']['Vector_Species_Params'][species]['EIR'] = data['parasite_parameters']['species'][clean_species_name]['EIR']
# Attach the scenario's config file
emod_scenario.add_file_from_string('config', 'config.json', json.dumps(config_json), description='SOMETHING')
emod_scenario.save()
return HttpResponse()
def form_valid(self, form):
# This method is called when valid form data has been POSTed.
scenario = EMODBaseline.from_dw(pk=self.kwargs["scenario_id"])
my_type = self.kwargs["file_type"]
scenario.add_file_from_string(my_type, my_type + '.json', str(form.cleaned_data['json']),
description='Edited by User in JSON Editor')
# update kwargs to detail page goes to new version of the scenario
self.kwargs["scenario_id"] = scenario.id
return super(JsonEditView, self).form_valid(form)
def get_context_data(self, **kwargs):
context = super(JsonEditView, self).get_context_data(**kwargs)
if "scenario_id" in self.kwargs:
scenario = EMODBaseline.from_dw(pk=self.kwargs["scenario_id"])
context["scenario_id"] = scenario.id
context["file_type"] = self.kwargs["file_type"]
context["json"] = json.loads(json.dumps(scenario.get_file_by_type(self.kwargs["file_type"]).content.replace("u\'", "\'")))
return context
def submit_a_verification_job(request, scenario_id):
adapter = EMOD_Adapter(request.user.username)
emod_scenario = EMODBaseline.from_dw(id=scenario_id)
config_json = ast.literal_eval(emod_scenario.get_file_by_type('config').content)
config_json['parameters']['Simulation_Duration'] = 20 * 365 # 20 years
emod_scenario.add_file_from_string('config', 'config.json', json.dumps(config_json), description='SOMETHING')
my_simulation_duration = config_json['parameters']['Simulation_Duration']
## Set the run start_date based on location and config's start_time
my_start_date = \
datetime.datetime.strptime(emod_scenario.template.climate_start_date, '%Y-%m-%d').date() \
+ datetime.timedelta(days=config_json['parameters']['Start_Time'])
# Initialize the run
run = adapter.save_run(scenario_id=scenario_id,
template_id=int(emod_scenario.template.id),
start_date=my_start_date,
duration=my_simulation_duration,
name='calibration-verification',
description=str(datetime.datetime.now()),
location_ndx=emod_scenario.template.location_key.id,
timestep_interval=1,
run_id=-1,
as_object=True)
run.baseline_key_id = scenario_id
run.save()
# add in JCD for config file
changes = []
newdict = copy.deepcopy(config_json)
newdict['config.json/parameters'] = newdict.pop('parameters')
changes.append(Change.node(name='config.json', node=[newdict], mode='-'))
run.jcd = JCD.from_changes(changes)
run.save()
### Launch the run
# 5853 - hard-code to 1 for now
#reps_per_exec = int(self.get_cleaned_data_for_step('start')['reps_per_exec'])
reps_per_exec = 1
# submit returns tuple (success, message)
try:
status = submit(run, user=request.user, manifest=True, reps_per_exec=reps_per_exec)
set_notification('alert-success', '<strong>Success!</strong> Run launched.', request.session)
except (RuntimeError, TypeError, AssertionError), e:
set_notification('alert-error', '<strong>Error!</strong> ' + str(e.message), request.session)
def save_species_data(request):
# Get data
data = json.loads(request.body)
# Hack until I know how to retrieve files from DimBaseline without funneling it through EMODBaseline
dim_scenario = DimBaseline.objects.get(id=data['scenario_id'])
# Check to see if this user has permission
if dim_scenario.user != DimUser.objects.get_or_create(username=request.user.username)[0]:
raise PermissionDenied
# Get the metadata
current_metadata = json.loads(dim_scenario.metadata)
# Print data
print current_metadata
print "Data = " + str(data)
# Fill data
if current_metadata['representative']['steps_complete'] <= step_number:
current_metadata['representative']['steps_complete'] = step_number + 1
current_metadata['representative']['species'] = data['species']
dim_scenario.metadata = json.dumps(current_metadata)
dim_scenario.save()
# Hack until I know how to retrieve files from DimBaseline without funneling it through EMODBaseline
emod_scenario = EMODBaseline.from_dw(id=data['scenario_id'])
# Add the species to the scenario
# Load species data
species = get_species(data['species'])
# Populate config
config_json = json.loads(emod_scenario.get_config_file().content)
print config_json
# Change config
for i in range(len(species)):
species_name = species[i].emod_snippet.name
config_json['parameters']['Vector_Species_Names'].append(species_name)
config_json['parameters']['Vector_Species_Params'][species_name] = json.loads(species[i].emod_snippet.snippet)[species_name]
print config_json
# Attach the scenario's config file
emod_scenario.add_file_from_string('config', 'config.json', json.dumps(config_json), description='SOMETHING')
emod_scenario.save()
return HttpResponse()
def save_demographics_data(request):
# Get data
data = json.loads(request.body)
# Hack until I know how to retrieve files from DimBaseline without funneling it through EMODBaseline
dim_scenario = DimBaseline.objects.get(id=data['scenario_id'])
# Check to see if this user has permission
if dim_scenario.user != DimUser.objects.get_or_create(username=request.user.username)[0]:
raise PermissionDenied
# Get the metadata
current_metadata = json.loads(dim_scenario.metadata)
# Print data
print current_metadata
print "Demographics id = " + data['demographics_id']
# Fill data
if current_metadata['representative']['steps_complete'] <= step_number:
current_metadata['representative']['steps_complete'] = step_number + 1
current_metadata['representative']['demographics_id'] = data['demographics_id']
dim_scenario.metadata = json.dumps(current_metadata)
dim_scenario.save()
# Hack until I know how to retrieve files from DimBaseline without funneling it through EMODBaseline
emod_scenario = EMODBaseline.from_dw(id=data['scenario_id'])
# Add the demographics to the scenario
# Load demographics data
demographics_data = get_demographics(data['demographics_id'])
# Populate config
config_json = json.loads(emod_scenario.get_config_file().content)
print config_json
# Change config
config_json['parameters']['Demographics_Filename'] = demographics_data['file_location']
# Attach the scenario's config file
emod_scenario.add_file_from_string('config', 'config.json', json.dumps(config_json), description='SOMETHING')
# Add the demographics
emod_scenario.add_file_from_string('demographics', 'demographics.compiled.json', demographics_data['file_data'], description='SOMETHING')
emod_scenario.save()
return HttpResponse()
def approve_scenario(request, scenario_id):
""" Method to allow a user to approve a scenario (without going through the wizard) """
try:
# get the scenario
my_scenario = EMODBaseline.from_dw(pk=scenario_id)
# check for run results
# ToDo: check for run results
# set approved, save
my_scenario.is_approved = True
my_scenario.save()
set_notification('alert-success', 'The simulation was approved. ', request.session)
except:
set_notification('alert-error', 'An Error occurred: the simulation was not approved. ', request.session)
return redirect("ts_emod_scenario_details", scenario_id=scenario_id)
def duplicate_scenario(request, scenario_id):
""" Function to allow duplication of scenario DB Objects """
if not request.user.is_authenticated():
set_notification('alert-error',
'<strong>Error!</strong> Please log in before copying. Simulation was not copied.',
request.session)
else:
# Save a copy of the scenario
try:
old_scenario = EMODBaseline.from_dw(pk=int(scenario_id))
my_copy_id = old_scenario.dimbaseline.copy()
set_notification('alert-success', '<strong>Success!</strong> The simulation ' + old_scenario._name
+ ' has been copied. You are now viewing the copy.', request.session)
return redirect("ts_emod_scenario_details", scenario_id=my_copy_id)
except:
print 'Error copying the scenario.'
set_notification('alert-error', '<strong>Error!</strong> Error copying the scenario.', request.session)
return redirect("ts_emod_scenario_details", scenario_id=scenario_id)
def get_parasite_parameters(scenario_id):
parasite_parameters = {}
parasite_parameters['species'] = {}
emod_scenario = EMODBaseline.from_dw(id=scenario_id)
# Populate config
config_json = json.loads(emod_scenario.get_config_file().content)
vector_species_parameters = config_json['parameters']['Vector_Species_Params']
for species in vector_species_parameters:
clean_species_name = str(species).replace(' ', '_')
parasite_parameters['species'][clean_species_name] = {}
if 'EIR' in vector_species_parameters[species]:
parasite_parameters['species'][clean_species_name]['EIR'] = vector_species_parameters[species]['EIR']
else:
parasite_parameters['species'][clean_species_name]['EIR'] = 0
return parasite_parameters
def dispatch(self, request, *args, **kwargs):
""" The method that accepts a request argument plus arguments, and returns an HTTP response.
- Verify that the user owns the selected scenario
- if not, go to selection page with error message
"""
# add the storage engine to the current wizard view instance here so we can act on storage here
self.prefix = self.get_prefix(*args, **kwargs)
self.storage = get_storage(self.storage_name, self.prefix, request, getattr(self, 'file_storage', None))
# Maybe not need to set a flag to make this block work - if first time, we NEED an id
# - if revisit, should do nothing here but continue (example: user reloads page)
#if not kwargs['scenario_id']:
# set_notification('alert-error',
# 'Please select a simulation to work in.',
# request.session)
# return redirect("ts_emod_scenario_browse")
if 'scenario_id' in kwargs.keys() and kwargs['scenario_id'] > 0:
#todo: check for intervention_tool in self.storage.extra_data.keys() ??
# First hit in Edit mode - populate the steps with initial defaults
try:
self.storage.reset()
self.storage.extra_data['intervention_tool'] = kwargs['scenario_id']
except AttributeError:
pass
# get the scenario for editing
self.request.session['scenario'] = EMODBaseline.from_dw(pk=kwargs['scenario_id'])
if str(self.request.session['scenario'].user) == \
str(DimUser.objects.get_or_create(username=self.request.user.username)[0]):
return super(InterventionToolView, self).dispatch(request, *args, **kwargs)
else:
set_notification('alert-error',
'You cannot modify simulations unless you own them. '
'Please select a simulation to work in.',
request.session)
return redirect("ts_emod_scenario_browse")
def get_chart(request, scenario_id=None, file_type=None):
# verify that scenario_id is good
if scenario_id is None or file_type is None:
return
# get the scenario
#my_scenario = DimBaseline.objects.get(id=scenario_id) # Objects returned here don't have get_file_by_type
my_scenario = EMODBaseline.from_dw(pk=scenario_id)
# get the scenario's file of the given type
# 'air binary',
# 'humidity binary',
# 'rainfall binary',
my_file_json = my_scenario.get_file_by_type(file_type + ' json')
my_file_bin = my_scenario.get_file_by_type(file_type + ' binary')
my_chart = getChartData(my_file_json.content, my_file_bin.content)
# put series into dict
chart_dict = {"title": {"text": ""}, "series": [{"data": my_chart}]}
return HttpResponse(json.dumps(chart_dict), mimetype="application/json")
def delete_scenario(request, scenario_id):
"""Function to allow deletion of scenario DB Objects
"""
my_folder = None
try:
my_scenario = EMODBaseline.from_dw(pk=scenario_id,
user=DimUser.objects.get_or_create(username=request.user.username)[0])
my_folder = my_scenario.dimbaseline.folder_id
my_scenario.delete()
if my_scenario.dimbaseline.is_deleted:
set_notification('alert-success', '<strong>Success!</strong> You have successfully deleted the simulation '
+ my_scenario._name, request.session)
except:
print 'Error deleting scenario with id: %s' % scenario_id
set_notification('alert-error', '<strong>Error!</strong> Simulation was not deleted.', request.session)
if my_folder is None:
# return to the home/root folder
return redirect("ts_emod_scenario_browse")
else:
# return to the scenario's folder
return redirect("ts_emod_scenario_browse", folder_id=str(my_folder))
def use_precalibrated_values(scenario_id):
emod_scenario = EMODBaseline.from_dw(id=scenario_id)
dim_scenario = DimBaseline.objects.get(id=scenario_id)
current_metadata = json.loads(dim_scenario.metadata)
weather_id = current_metadata['representative']['weather_id']
demographics_id = current_metadata['representative']['demographics_id']
weather_precalibration_name = get_weather_precalibration_name(weather_id)
demographics_precalibration_name = get_demographics_precalibration_name(demographics_id)
# Populate config
config_json = json.loads(emod_scenario.get_config_file().content)
vector_species_parameters = config_json['parameters']['Vector_Species_Params']
print vector_species_parameters
for species in vector_species_parameters:
eir = int(vector_species_parameters[species]['EIR'])
new_species_parameters = json.loads(get_calibrated_species(weather_precalibration_name, demographics_precalibration_name, eir, species))
vector_species_parameters[species] = new_species_parameters
emod_scenario.add_file_from_string('config', 'config.json', json.dumps(config_json), description='SOMETHING')
print config_json
def post(self, request, scenario_id):
dim_user = DimUser.objects.get_or_create(username=self.request.user.username)[0]
emod_scenario = EMODBaseline.from_dw(id=scenario_id)
# Check permissions
if emod_scenario.user != dim_user:
raise PermissionDenied
campaign = {}
campaign['Events'] = []
campaign['Use_Defaults'] = 1
interventions = {}
# Loop through all of the stuff sent from the frontend.
for key in request.POST:
value = request.POST[key]
# Example key: SimpleBednet-0-cost_to_consumer. The left defines the intervention_type, the middle defines
# the index of the intervention list of this type, the right defines the field_name for the form of type
# intervention_type. If the key is not one that we are looking for, it will fail the try-except below and
# move to the next key (ie. __prefix__).
split_key = key.split('-')
# If this is a valid entry for our interventions and not things like csrf tokens, INITIAL_FORMS,
# MAX_NUM_FORMS, etc.
if len(split_key) == 3:
intervention_type = split_key[0]
try:
index = int(split_key[1])
except ValueError: # Read split_key description
continue
field_name = split_key[2]
if value == '':
value = 0
if field_name == 'demographic_coverage':
value = float(value) / 100
# Check if this intervention_type has already been added. If it has, then all of its intervention
# entries has also been added. If this intervention_type is not already in the list, then we need to
# add an entry for each intervention of this type right now.
if intervention_type not in interventions:
interventions[intervention_type] = []
for j in range(int(request.POST[intervention_type + '-TOTAL_FORMS'])):
# Create a new intervention with all the necessary dictionaries and common default values
interventions[intervention_type].append({})
last = len(interventions[intervention_type]) - 1
intervention = interventions[intervention_type][last]
intervention['Event_Coordinator_Config'] = {}
event_coordinator_config = intervention['Event_Coordinator_Config']
event_coordinator_config['Intervention_Config'] = {}
event_coordinator_config['Intervention_Config']['class'] = intervention_type
event_coordinator_config['Target_Demographic'] = 'Everyone'
event_coordinator_config['class'] = 'StandardInterventionDistributionEventCoordinator'
intervention['Nodeset_Config'] = {}
intervention['Nodeset_Config']['class'] = 'NodeSetAll'
intervention['class'] = 'CampaignEvent'
intervention = interventions[intervention_type][index]
add_entry_to_intervention(intervention, field_name, value)
# Compile all the interventions into a campaign
for intervention_type in interventions:
intervention_set = interventions[intervention_type]
for i in range(len(intervention_set)):
campaign['Events'].append(intervention_set[i])
print campaign
config = ast.literal_eval(emod_scenario.get_file_by_type('config').content)
if len(campaign['Events']) > 0:
# Enable in config
if config['parameters']['Enable_Interventions'] == 0:
config['parameters']['Enable_Interventions'] = 1
emod_scenario.add_file_from_string('config', 'config.json',
json.dumps(config),
description="Modified by Intervention Tool")
# Replace campaign file
emod_scenario.add_file_from_string('campaign', 'campaign.json',
json.dumps(campaign),
description="Modified by Intervention Tool")
emod_scenario.save()
set_notification('alert-success', '<strong>Success!</strong> The intervention(s) have been saved to ' +
emod_scenario.name + '.', self.request.session)
# Check if user wants to launch, or just save
if 'submission_type' in request.POST:
if request.POST['submission_type'] == 'launch':
return redirect("ts_emod_launch_tool", step='start', scenario_id=str(scenario_id))
elif request.POST['submission_type'] == 'save':
return redirect("ts_emod_scenario_details", scenario_id=str(scenario_id))
else:
raise Exception("Bad submission_type of " + str(request.POST['submission_type']))
else:
return redirect("ts_emod_scenario_details", scenario_id=str(scenario_id))
def done(self, form_list, **kwargs):
if 'error_invalid_id' in self.storage.extra_data:
self.storage.reset()
set_notification('alert-error', 'Error Launching Simulation: Unknown/Invalid run provided. ' +
'Please select a simulation below to launch.', self.request.session)
return redirect("ts_emod_scenario_browse")
if 'scenario_id' in self.storage.extra_data:
scenario_id = int(self.storage.extra_data['scenario_id'])
else:
Exception()
adapter = EMOD_Adapter(self.request.user.username)
if 'emod_launch_run_id' in self.storage.extra_data:
my_run_id = int(self.storage.extra_data['emod_launch_run_id'])
my_run = adapter.fetch_runs(scenario_id=-1, run_id=int(my_run_id))
# check for name changes from user
start_data = self.get_cleaned_data_for_step('start')
if self.storage.extra_data['old_run_name'] != start_data['name']:
my_run.name = start_data['name']
my_run.save()
else:
# get the scenario
self.storage.extra_data['scenario'] = EMODBaseline.from_dw(pk=scenario_id)
my_config = ast.literal_eval(self.storage.extra_data['scenario'].get_file_by_type('config').content)
try:
my_campaign = ast.literal_eval(self.storage.extra_data['scenario'].get_file_by_type('campaign').content)
except AttributeError:
# if file list was returned, use last file
file_list = self.request.session['scenario'].get_file_by_type('campaign')
my_campaign = ast.literal_eval(file_list[0].content)
my_campaign_length = len(my_campaign['Events'])
###
# create a new run instance
if my_campaign_length > 0:
with_interventions = "with interventions "
else:
with_interventions = ""
my_simulation_duration = my_config['parameters']['Simulation_Duration']
## Set the run start_date based on location and config's start_time
my_start_date = \
datetime.datetime.strptime(
self.storage.extra_data['scenario'].template.climate_start_date,
'%Y-%m-%d').date() \
+ datetime.timedelta(days=my_config['parameters']['Start_Time'])
# Initialize the run
my_run = adapter.save_run(scenario_id=scenario_id,
template_id=int(self.storage.extra_data['scenario'].template.id),
start_date=my_start_date,
duration=my_simulation_duration,
name=self.get_cleaned_data_for_step('start')['name'],
description='Launch Tool: ' + self.storage.extra_data['scenario'].name
+ ': Run ' + with_interventions + str(datetime.datetime.now()),
location_ndx=self.storage.extra_data['scenario'].template.location_key.id,
timestep_interval=1,
run_id=-1,
as_object=True)
my_run.baseline_key_id = scenario_id
my_run.save()
if settings.DEBUG:
print "DEBUG: LaunchTool: run id: ", my_run.id
# add in JCD for config file
changes = []
newdict = copy.deepcopy(my_config)
newdict['config.json/parameters'] = newdict.pop('parameters')
changes.append(Change.node(name='config.json', node=[newdict], mode='-'))
# add in JCD for campaign file
if my_campaign_length > 0:
newdict = copy.deepcopy(my_campaign)
newdict['campaign.json/Use_Defaults'] = newdict.pop('Use_Defaults')
newdict['campaign.json/Events'] = newdict.pop('Events')
changes.append(Change.node(name='campaign.json', node=[newdict], mode='-'))
my_run.jcd = JCD.from_changes(changes)
my_run.save()
### Launch the run
# 5853 - hard-code to 1 for now
#reps_per_exec = int(self.get_cleaned_data_for_step('start')['reps_per_exec'])
reps_per_exec = 1
# If this is a representative based scenario, then set it to non-editable
representative_scenario = RepresentativeScenario.objects.filter(emod_scenario_id=scenario_id)
if representative_scenario:
representative_scenario[0].is_editable = False
representative_scenario[0].save()
# submit returns tuple (success, message)
try:
status = submit(my_run, user=self.request.user, manifest=True, reps_per_exec=reps_per_exec)
set_notification('alert-success', '<strong>Success!</strong> Run launched.',
self.request.session)
except (RuntimeError, TypeError, AssertionError), e:
set_notification('alert-error', '<strong>Error!</strong> ' + str(e.message), self.request.session)
def get_context_data(self, **kwargs):
"""Extension of get_context_data
Add context data to drive menu nav highlights and breadcrumbs.
"""
context = super(ScenarioDetailView, self).get_context_data(**kwargs)
context['nav_button'] = "browse_scenario"
self.request.session['emod_nav_button'] = "browse_scenario"
# This is here so that Alex can run this later and change all production database entries
# all_runs = DimRun.objects.all()
# for run in all_runs:
# if run.start_date_key and run.start_date_key.timestamp:
# run.new_start_date_key = run.start_date_key.timestamp
# if run.end_date_key and run.end_date_key.timestamp:
# run.new_end_date_key = run.end_date_key.timestamp
# print run.new_start_date_key
# print run.new_end_date_key
# run.save()
scenario_id = kwargs['scenario_id']
scenario = EMODBaseline.from_dw(pk=scenario_id)
# Hack until DimBaseline/EMODBaseline issues are sorted out
dim_scenario = DimBaseline.objects.get(id=scenario_id)
# Hack for calibration
species_info = {}
the_config = ast.literal_eval(scenario.get_config_file().content)
the_species = the_config['parameters']['Vector_Species_Params']
for specie in the_species:
species_info[specie] = {}
if 'EIR' in the_species[specie]:
eir = the_species[specie]['EIR']
else:
eir = 20.0 / float(len(the_species))
if 'Sporozoite_Rate' in the_species[specie]:
sporozoite = the_species[specie]['Sporozoite_Rate']
else:
sporozoite = 0.003
species_info[specie]['eir'] = eir
species_info[specie]['sporozoite'] = sporozoite
context['species_info'] = species_info
print species_info
# data_to_send = {}
# data_to_send['eir'] = []
# data_to_send['sporozoite'] = []
# the_config = ast.literal_eval(scenario.get_config_file().content)
# the_species = the_config['parameters']['Vector_Species_Params']
# for i in range(len(the_species)):
# data_to_send['eir'].append(20)
# data_to_send['sporozoite'].append(0.003)
# context['data_to_send'] = data_to_send
if dim_scenario.metadata and 'calibration_status' in dim_scenario.metadata:
scenario.calibration_status = json.loads(dim_scenario.metadata)['calibration_status']
else:
scenario.calibration_status = "Ready"
if dim_scenario.metadata:
current_metadata = json.loads(dim_scenario.metadata)
else:
current_metadata = {}
if 'calibration_verification_run_id' in current_metadata:
calibration_verification_run = DimRun.objects.get(id=current_metadata['calibration_verification_run_id'])
if calibration_verification_run.status == '1/1/0':
vector_species_parameters = the_config['parameters']['Vector_Species_Params']
calibration_verification_info = {}
for species in vector_species_parameters:
calibration_verification_info[species] = {}
calibration_verification_info[species]['sporozoite_rate'] = {}
calibration_verification_info[species]['eir'] = {}
calibration_verification_info[species]['biting_rate'] = {}
target_sporozoite_rate = vector_species_parameters[species]['Sporozoite_Rate']
daily_sporozoite_rate_2_3 = get_average_daily_sporozoite_rate(calibration_verification_run, species, 2, 3)
daily_sporozoite_rate_4_9 = get_average_daily_sporozoite_rate(calibration_verification_run, species, 4, 9)
daily_sporozoite_rate_10_20 = get_average_daily_sporozoite_rate(calibration_verification_run, species, 10, 20)
calibration_verification_info[species]['sporozoite_rate']['target'] = target_sporozoite_rate
calibration_verification_info[species]['sporozoite_rate']['years_2_3'] = daily_sporozoite_rate_2_3
calibration_verification_info[species]['sporozoite_rate']['years_4_9'] = daily_sporozoite_rate_4_9
calibration_verification_info[species]['sporozoite_rate']['years_10_20'] = daily_sporozoite_rate_10_20
target_average_annual_eir = vector_species_parameters[species]['EIR']
annual_eir_2_3 = get_average_daily_eir(calibration_verification_run, species, 2, 3) * 365
annual_eir_4_9 = get_average_daily_eir(calibration_verification_run, species, 4, 9) * 365
annual_eir_10_20 = get_average_daily_eir(calibration_verification_run, species, 10, 20) * 365
calibration_verification_info[species]['eir']['target'] = target_average_annual_eir
calibration_verification_info[species]['eir']['years_2_3'] = annual_eir_2_3
calibration_verification_info[species]['eir']['years_4_9'] = annual_eir_4_9
calibration_verification_info[species]['eir']['years_10_20'] = annual_eir_10_20
target_average_daily_biting_rate = 'NA'
daily_biting_rate_2_3 = get_average_daily_biting_rate(calibration_verification_run, species, 2, 3)
daily_biting_rate_4_9 = get_average_daily_biting_rate(calibration_verification_run, species, 4, 9)
daily_biting_rate_10_20 = get_average_daily_biting_rate(calibration_verification_run, species, 10, 20)
calibration_verification_info[species]['biting_rate']['target'] = target_average_daily_biting_rate
calibration_verification_info[species]['biting_rate']['years_2_3'] = daily_biting_rate_2_3
calibration_verification_info[species]['biting_rate']['years_4_9'] = daily_biting_rate_4_9
calibration_verification_info[species]['biting_rate']['years_10_20'] = daily_biting_rate_10_20
context['calibration_verification_info'] = calibration_verification_info
context['scenario_id'] = kwargs['scenario_id']
context['scenario_userid'] = scenario.user.id
context['dim_user'] = DimUser.objects.get_or_create(username=self.request.user.username)[0]
scenario.last_edited = scenario._last_edited
scenario.is_public = scenario._is_public
# Hack until DimBaseline/EMODBaseline issues are sorted out
if dim_scenario.metadata:
metadata = json.loads(dim_scenario.metadata)
if 'representative' in metadata:
scenario.is_representative = True
if 'is_complete' in metadata['representative']:
scenario.representative_is_completed = True
else:
scenario.representative_is_completed = False
scenario.name = derive_autoname(dim_scenario)
else:
scenario.is_representative = False
context['scenario'] = scenario
file_list = []
for my_file in scenario.files:
if my_file['type'] == 'config':
try:
context['config'] = json.dumps(ast.literal_eval(scenario.get_file_by_type('config').content),
sort_keys=True, indent=4, separators=(',', ': '))
except AttributeError:
# if file list was returned, use last file
context['config'] = json.dumps(ast.literal_eval(
scenario.get_file_by_type('config')[scenario.get_file_by_type('config').count()
- 1].content), sort_keys=True, indent=4,
separators=(',', ': '))
elif my_file['type'] == 'campaign':
try:
scenario_campaign = json.dumps(ast.literal_eval(scenario.get_file_by_type('campaign').content),
sort_keys=True, indent=4, separators=(',', ': '))
if len(json.loads(scenario_campaign)['Events']) > 0:
context['campaign'] = scenario_campaign
else:
context['has_campaign'] = None
except AttributeError:
# if file list was returned, use last file
context['campaign'] = json.dumps(ast.literal_eval(
scenario.get_file_by_type('campaign')[scenario.get_file_by_type('campaign').count()
- 1].content), sort_keys=True, indent=4,
separators=(',', ': '))
else:
file_list.append(my_file['name'])
context['file_list'] = file_list
# Get a list of the runs for this scenario
adapter = EMOD_Adapter(self.request.user.username)
run_list = DimRun.objects.filter(baseline_key=scenario.id)
dim_scenario = DimBaseline.objects.get(id=scenario.id)
if dim_scenario.simulation_group:
simulation_group = dim_scenario.simulation_group
simulation_list = Simulation.objects.filter(group=simulation_group)
for simulation in simulation_list:
if simulation.status == sim_status.SCRIPT_DONE:
simulation.is_complete = True
elif simulation.status == sim_status.SCRIPT_ERROR:
simulation.has_failed = True
context['simulation_list'] = simulation_list
# sort descending by id (for those old runs that all have the same old time_created
context['run_list'] = sorted(run_list, key=lambda x: x.id, reverse=True)
# sort descending by time_saved
context['run_list'] = sorted(context['run_list'], key=lambda x: x.time_created, reverse=True)
note_list = []
for run in context['run_list']:
try:
run_diffs = []
# config diffs
run_config = run.jcd.jcdict['config.json']['Changes'][0]['config.json/parameters']
scenario_config = json.loads(context['config'])['parameters']
for key in scenario_config:
if scenario_config[key] != run_config[key]:
run_diffs.append({'key': key, 'run': run_config[key], 'scenario': scenario_config[key]})
# ToDo: split out each key difference of Vector Species Parameters in both
# campaign diffs
if 'campaign.json' not in run.jcd.jcdict:
run_diffs.append({'key': 'Interventions', 'run': 'None'})
else:
run_campaign = run.jcd.jcdict['campaign.json']['Changes'][0]['campaign.json/Events']
if json.loads(scenario_campaign)['Events'] != run_campaign:
run_diffs.append({'key': 'Interventions', 'run': run_campaign,
'scenario': str(json.loads(scenario_campaign)['Events'])})
# ToDo: split out each key difference of each Intervention in both
# else:
# run_diffs.append({'key': 'Interventions', 'run': 'See below',
# 'scenario': '<a class="btn tooltip_link" href="#campaign">see below</a>'})
run.diffs = run_diffs
except (AttributeError, KeyError):
# no JCD?
run.diffs = [{'Unknown Error Occurred:': 'no changes found.'}]
# get the status of each run
status = adapter.run_status(run.id)
if status is None:
run.my_completed = None
else:
run.my_completed = status[0]
run.my_total = status[1]
run.my_failed = status[2]
run.percent_complete = (run.my_completed + run.my_failed) / run.my_total
run.my_incomplete = int(run.my_total) - (int(run.my_completed) + int(run.my_failed))
if int(status[0]) > 0:
context['has_results'] = 1
note_list.extend(adapter.fetch_notes(run_id=int(run.id), as_object=True))
if note_list != []:
context['note_list'] = note_list
return context
def update_emod_config(request, scenario_id):
# Check if api key was supplied
try:
api_key = request.POST['api_key']
except KeyError:
return HttpResponse(content=json.dumps({'Error': 'api_key missing in POST request'}))
# Check if the supplied api key is valid
if api_key != EMOD_BASELINE_UPDATE_BASELINE_API_KEY:
return HttpResponse(content=json.dumps({'Error': 'The api_key that was provided is invalid'}))
# Check if the supplied scenario id is valid
try:
emod_scenario = EMODBaseline.from_dw(id=scenario_id)
except ObjectDoesNotExist:
return HttpResponse(content=json.dumps({'Error': 'Scenario with id ' + str(scenario_id) + ' does not exist.'}))
except Exception as exception:
return HttpResponse(content=json.dumps({'Error': 'Unhandled exception: ' + str(exception)}))
# Check if a message was supplied
try:
message = request.POST['message']
except KeyError:
return HttpResponse(content=json.dumps({'Error': 'message missing in POST request'}))
# This variable and if statement is just a place holder for what we could have later if we allow users to access this api
CALIBRATION_API_KEY = EMOD_BASELINE_UPDATE_BASELINE_API_KEY
# Handle the message
if message == 'success':
# Check if a config was supplied
if 'config' not in request.POST:
return HttpResponse(content=json.dumps({'Error': 'config missing in POST request'}))
# Check if the config is valid json
try:
calibrated_config_json = json.loads(request.POST['config'])
except:
return HttpResponse(content=json.dumps({'Error': 'The json that was provided is invalid'}))
merged_config_json = ast.literal_eval(emod_scenario.get_config_file().content)
merged_config_json['parameters']['Vector_Species_Params'] = calibrated_config_json['parameters']['Vector_Species_Params']
# Add the new config
emod_scenario.add_file_from_string('config', 'config.json', json.dumps(merged_config_json), description='SOMETHING')
emod_scenario.save()
# Set the calibration status if applicable
calibration_status = 'complete'
elif message == 'failure':
# Set the calibration status if applicable
calibration_status = 'failed'
else:
return HttpResponse(content=json.dumps({'Error': 'The message that was provided is invalid. Should be either "success" or "failure"'}))
if (api_key == CALIBRATION_API_KEY):
# Yet another hack until DimBaseline vs EMODBaseline issues are resolved
dim_scenario = DimBaseline.objects.get(id=scenario_id)
# This catches any simulations that don't have metadata set yet
try:
current_metadata = json.loads(dim_scenario.metadata) # Need to double check if the loading and dumping is necessary
except ValueError:
current_metadata = {}
current_metadata['calibration_status'] = calibration_status
dim_scenario.metadata = json.dumps(current_metadata)
dim_scenario.save()
return HttpResponse(content=json.dumps({'Success': 'The config json was successfully changed'}))
def dispatch(self, request, *args, **kwargs):
""" The method that accepts a request argument plus arguments, and returns an HTTP response.
Make sure user is logged in
"""
# add the storage engine to the current wizardview instance here so we can act on storage here
self.prefix = self.get_prefix(*args, **kwargs)
self.storage = get_storage(self.storage_name, self.prefix, request, getattr(self, 'file_storage', None))
if 'scenario_id' in kwargs:
if kwargs['scenario_id'] > 0:
# First hit in Edit mode - populate the steps with initial defaults
self.storage.reset()
self.storage.extra_data['edit_scenario'] = kwargs['scenario_id']
# get the scenario for editing
self.request.session['scenario'] = EMODBaseline.from_dw(pk=kwargs['scenario_id'])
try:
self.request.session['scenario_config'] = \
ast.literal_eval(self.request.session['scenario'].get_file_by_type('config').content)
except AttributeError:
# if query set returned, use first file in "list"
file_list = self.request.session['scenario'].get_file_by_type('config')
self.request.session['scenario_config'] = \
ast.literal_eval(file_list[0].content)
######
# config step data:
self.storage.set_step_data('config', {u'config-name': [self.request.session['scenario']._name],
u'config-description': [self.request.session['scenario'].description],
u'config-Start_Time': [self.request.session['scenario_config']['parameters']['Start_Time']],
u'config-Simulation_Duration':
[unicode(self.request.session['scenario_config']['parameters']['Simulation_Duration'])],
u'config-Simulation_Type':
[self.request.session['scenario_config']['parameters']['Simulation_Type']],
u'baseline_wizard_view-current_step': [u'config']
})
template_id = self.request.session['scenario'].template.id
my_template_obj = DimTemplate.objects.get(id=template_id)
location_id = my_template_obj.location_key_id
# climate step data:
self.storage.set_step_data('climate', {u'climate-location': [location_id]})
# demographic step data:
self.storage.set_step_data('demographic', {u'demographic-location': [location_id]})
# parasite
for step_name in ["vector", "parasite"]:
# get db config for the step
my_json = ConfigData.objects.filter(type='JSONConfig', name=step_name)[0].json
# wizard values based on config + template values
my_wiz = {u'orig_json_obj': [my_json]}
my_json = json.loads(my_json)
for key in my_json.keys():
try:
# = scenario value OR template value
# change into wizard format step name + -json_ + key name
my_wiz.update({u'' + step_name + '-json_'
+ key: [u'' + str(self.request.session['scenario_config']['parameters'][key])]})
except KeyError:
pass
# insert into wizard storage
self.storage.set_step_data(step_name, my_wiz)
# Scaling step data:
self.storage.set_step_data('scaling_factors',
{u'scaling_factors-x_Temporary_Larval_Habitat':
[self.request.session['scenario_config']['parameters']['x_Temporary_Larval_Habitat']]})
######
if 'step' in kwargs.keys() and kwargs['step'] == 'climate':
# 6230: redirect to Intervention Tool (if user clicked "Skip to Intervention Tool Button")
# check to see if the user wants to jump over to the Intervention tool
if 'jump_to_intervention_tool' in self.request.session.keys():
my_scenario_id = self.request.session['scenario'].id
# clear wizard storage
self.storage.reset()
del self.request.session['jump_to_intervention_tool']
# redirect to Intervention Tool
return redirect("ts_intervention_tool_step", scenario_id=my_scenario_id)
self.request.session['scenario'].save()
return super(EditWizardView, self).dispatch(request, *args, **kwargs)
def save_weather_data(request):
# Get data
data = json.loads(request.body)
# Hack until I know how to retrieve files from DimBaseline without funneling it through EMODBaseline
dim_scenario = DimBaseline.objects.get(id=data['scenario_id'])
# Check to see if this user has permission
if dim_scenario.user != DimUser.objects.get_or_create(username=request.user.username)[0]:
raise PermissionDenied
# Get the metadata
current_metadata = json.loads(dim_scenario.metadata)
# Print data
print current_metadata
print data
print "Weather id = " + data['weather_id']
# Fill data
if current_metadata['representative']['steps_complete'] <= step_number:
current_metadata['representative']['steps_complete'] = step_number + 1
current_metadata['representative']['weather_id'] = data['weather_id']
dim_scenario.metadata = json.dumps(current_metadata)
print dim_scenario.metadata
dim_scenario.save()
# Hack until I know how to retrieve files from DimBaseline without funneling it through EMODBaseline
emod_scenario = EMODBaseline.from_dw(id=data['scenario_id'])
# Add the weather to the scenario
# Load weather data
weather_data = get_weather(data['weather_id'])
# Populate config
config_json = json.loads(emod_scenario.get_config_file().content)
print config_json
# Change config
config_json['parameters']['Rainfall_Filename'] = weather_data['file_locations']['rainfall']
config_json['parameters']['Relative_Humidity_Filename'] = weather_data['file_locations']['humidity']
config_json['parameters']['Air_Temperature_Filename'] = weather_data['file_locations']['temperature']
config_json['parameters']['Land_Temperature_Filename'] = weather_data['file_locations']['temperature']
# Attach the emod_scenario config file
emod_scenario.add_file_from_string('config', 'config.json', json.dumps(config_json), description='SOMETHING')
# Add the weather
emod_scenario.add_file_from_string('rainfall json', 'rainfall.json', weather_data['file_data']['rainfall']['json'], description='SOMETHING')
emod_scenario.add_file_from_string('rainfall binary', 'rainfall.bin', weather_data['file_data']['rainfall']['bin'], description='SOMETHING')
emod_scenario.add_file_from_string('humidity json', 'humidity.json', weather_data['file_data']['humidity']['json'], description='SOMETHING')
emod_scenario.add_file_from_string('humidity binary', 'humidity.bin', weather_data['file_data']['humidity']['bin'], description='SOMETHING')
emod_scenario.add_file_from_string('air json', 'temperature.json', weather_data['file_data']['temperature']['json'], description='SOMETHING')
emod_scenario.add_file_from_string('air binary', 'temperature.bin', weather_data['file_data']['temperature']['bin'], description='SOMETHING')
# Add land temperature
emod_scenario.add_file_from_string('land_temp json', 'temperature.json', weather_data['file_data']['temperature']['json'], description='SOMETHING')
emod_scenario.add_file_from_string('land_temp binary', 'temperature.bin', weather_data['file_data']['temperature']['bin'], description='SOMETHING')
emod_scenario.save()
print dim_scenario.metadata
return HttpResponse()
def send_calibration_request(request, targets, scenario_id):
dim_scenario = DimBaseline.objects.get(id=scenario_id)
# Get user
user = DimUser.objects.get_or_create(username=request.user.username)[0]
# Check user
#if dim_scenario.user != user:
# raise PermissionDenied
# Create simulation group
simulation_group = SimulationGroup(submitted_by=user)
simulation_group.save()
# Create emod_scenario for helper functions to get files
emod_scenario = EMODBaseline.from_dw(id=scenario_id)
# Create input files and put them into a list
simulation_input_files = create_and_return_input_files(user, emod_scenario)
# Get version
try:
# Slices off the excess and leaves just the version number
version = emod_scenario.template.model_version.split('v')[1]
except:
version = 'unknown'
# Create simulation
simulation = Simulation.objects.create(
group=simulation_group,
model=sim_model.EMOD_CALIBRATION,
version=version,
status=sim_status.READY_TO_RUN
)
file_urls = {}
# Add simulation input files to simulation
for i in range(len(simulation_input_files)):
simulation.input_files.add(simulation_input_files[i])
file_urls[simulation_input_files[i].name] = SITE_ROOT_URL[:-1] + reverse(
'input_file_download',
kwargs={
'resource_name': 'input_files',
'pk': simulation_input_files[i].id,
'api_name': 'v1'
}
)
print "targets = " + json.dumps(targets)
print file_urls
data = {}
data['targets'] = json.dumps(targets)
data['files'] = json.dumps(file_urls)
data['url'] = SITE_ROOT_URL[:-1] + reverse('data_services.update_emod_config', kwargs={'scenario_id': scenario_id})
data['version'] = version
print data
url = CONFIG_CALIBRATION_URL
try:
send_request = requests.post(url, data=data)
except ConnectionError:
set_notification('alert-error', '<strong>Error!</strong> Connection error with calibration service.', request.session)
return
print send_request.status_code
try:
current_metadata = json.loads(dim_scenario.metadata)
except:
current_metadata = {}
if str(send_request.status_code) == "200":
current_metadata['calibration_status'] = 'sent'
else:
current_metadata['calibration_status'] = 'failed'
dim_scenario.metadata = json.dumps(current_metadata)
dim_scenario.save()
if current_metadata['calibration_status'] == 'failed':
set_notification('alert-error', '<strong>Error!</strong> Request error with calibration service.', request.session)
return
set_notification('alert-success', '<strong>Success!</strong> Calibration request sent.', request.session)
def convert_baseline_to_scenario(request, baseline_id):
dim_user = DimUser.objects.get(username=request.user.username)
emod_scenario = EMODBaseline.from_dw(id=baseline_id)
dim_scenario = DimBaseline.objects.get(id=baseline_id)
extra_metadata = {}
# This doesn't all need to be here, but it is simpler just to leave it
if dim_scenario.metadata:
metadata = json.loads(dim_scenario.metadata)
if 'representative' in metadata:
emod_scenario.is_representative = True
extra_metadata = metadata
if 'is_complete' in metadata['representative']:
emod_scenario.representative_is_completed = True
else:
emod_scenario.representative_is_completed = False
emod_scenario.name = derive_autoname(dim_scenario)
else:
emod_scenario.is_representative = False
# Create simulation group if one is not already made
if not dim_scenario.simulation_group:
simulation_group = SimulationGroup(submitted_by=dim_user)
simulation_group.save()
simulation = add_simulation(dim_user, sim_model.EMOD, dim_scenario.template.model_version.split('v')[1], simulation_group, baseline_id)
scenario = Scenario.objects.create(
name=dim_scenario.name,
description=dim_scenario.description,
user=dim_user,
simulation=simulation,
metadata={},
extra_metadata=extra_metadata
)
# Modify config
config_json = json.loads(scenario.config_file.get_contents())
config_json['parameters']['Land_Temperature_Filename'] = 'temperature.bin'
config_json['parameters']['Rainfall_Filename'] = 'rainfall.bin'
config_json['parameters']['Relative_Humidity_Filename'] = 'humidity.bin'
config_json['parameters']['Air_Temperature_Filename'] = 'temperature.bin'
config_json['parameters']['Campaign_Filename'] = 'campaign.json'
config_json['parameters']['Demographics_Filename'] = 'demographics.compiled.json'
contents = json.dumps(config_json)
scenario.set_file_by_type('config', contents)
set_notification('alert-success', '<strong>Success!</strong> Converted baseline to scenario.', request.session)
# This makes sure we don't try to convert the same baseline twice
dim_scenario.simulation_group = simulation_group
dim_scenario.save()
else:
simulation_group = dim_scenario.simulation_group
try:
simulation = simulation_group.simulations.all()[0]
scenario = simulation.emod_scenario_set.all()[0]
except Exception as exception:
set_notification(
'alert-error',
'<strong>Error!</strong> Simulation group exists but there is no matching scenario. ' + str(exception),
request.session
)
return redirect("ts.index")
set_notification('alert-success', '<strong>Success!</strong> This baseline has already been converted.', request.session)
return redirect("ts_emod2.details", scenario_id=scenario.id)
def get_context_data(self, **kwargs):
"""Extension of get_context_data
Add context data to drive menu nav highlights, breadcrumbs and pagers.
"""
context = super(ScenarioBrowseView, self).get_context_data(**kwargs)
the_request = self.request
context['nav_button'] = 'scenario_browse'
pager_size = the_request.POST.get('pager_size') or the_request.GET.get('pager_size')
if pager_size is None:
pager_size = 10
else:
pager_size = int(pager_size)
# Get list of folders in this folder (if no current folder, include folders with no parent)
if 'folder_id' in kwargs.keys():
current_folder = kwargs['folder_id'] or the_request.POST.get('folder_id') or the_request.GET.get('folder_id') or None
context['current_folder'] = Folder.objects.get(pk=current_folder)
else:
current_folder = None
# get a list of all scenarios in order to get count of sims in each folder
scenario_list_all = list(DimBaseline.objects.filter(user=DimUser.objects.get(username=self.request.user.username),
is_deleted=False))
my_count = {}
for scen in scenario_list_all:
if scen.folder_id in my_count.keys():
my_count[scen.folder_id] += 1
else:
my_count[scen.folder_id] = 1
# create a dictionary of children (keys are parents, value is list of children)
folder_list_all = list(Folder.objects.filter(user=DimUser.objects.get(username=self.request.user.username),
is_deleted=False))
my_dict = defaultdict(list)
for folder in folder_list_all:
if folder.id in my_count.keys():
counter = ' (' + str(my_count[folder.id]) + ')'
else:
counter = ''
if folder.parent is None:
my_dict[0].append({'id': folder.id, 'name': folder.name, 'child_count': counter})
else:
my_dict[folder.parent.id].append({'id': folder.id, 'name': folder.name, 'child_count': counter})
current_found = 0 # Set flag that deterimines if the current folder has been visited in the tree
if None in my_count.keys():
counter_root = ' (' + str(my_count[None]) + ')'
else:
counter_root = ''
folder_tree = '{title: "My Simulations' + counter_root + '", folder: true, expanded: true'
if current_folder is None:
folder_tree += ', active: true'
current_found = 1
# context['folder_tree'] = '{title: "My Tororo Folder full of Simulations (and folders) in home folder ", folder: true }, ' \
# '{title: "Folder 2", folder: true, ' \
# 'children: [ ' \
# '{ title: "Sub-item 2.1", folder: true }, ' \
# '{ title: "Sub-item 2.2" }] }, ' \
# '{ title: "Item 3" }'
# without root:
# context['folder_tree'] = '{title: "Solomon Islands", folder: true}, {title: "Second folder", folder: true, children: [ {title: "subfolder", folder: true}] }'
#folder_tree = add_children(0, my_dict)
# with root:
# context['folder_tree'] = '{title: "My Simulations", folder: true, children: [ {title: "Solomon Islands", folder: true}, {title: "Second folder", folder: true, children: [ {title: "subfolder", folder: true}] }] }'
folder_tree += add_children(0, my_dict, current_folder, current_found) + '}'
context['folder_tree'] = str(folder_tree)
# Get list of scenarios
scenario_list = list(DimBaseline.objects.filter(user=DimUser.objects.get(username=self.request.user.username),
folder=current_folder,
is_deleted=False))
if scenario_list is None:
return context
# sort descending by id (for those old scenarios that all have the same old time_created
scenario_list = sorted(scenario_list, key=lambda x: x.id, reverse=True)
# sort descending by time_saved (version)
scenario_list = sorted(scenario_list, key=lambda x: x.last_modified, reverse=True)
# merge folders into scenario list
# - show folders in file list
#scenario_list = folder_list + scenario_list
# Flag scenarios that are representative
for scenario in scenario_list:
# Hack for now. I don't remember if it is still needed, but I will fix it later. Not currently important.
if scenario.description == "Made with representative workflow" and not scenario.metadata:
scenario.metadata = json.dumps({'representative': ''})
scenario.save()
if scenario.metadata:
metadata = json.loads(scenario.metadata)
if 'representative' in metadata:
scenario.is_representative = True
if 'is_complete' in metadata['representative']:
scenario.representative_is_complete = metadata['representative']['is_complete']
else:
scenario.representative_is_complete = False
scenario.name = derive_autoname(scenario)
else:
scenario.is_representative = False
paginator = Paginator(scenario_list, pager_size)
page = int(the_request.GET.get('page') or 1)
try:
scenarios = paginator.page(page)
except PageNotAnInteger:
# If page is not an integer, deliver first page.
scenarios = paginator.page(1)
except EmptyPage:
# If page is out of range (e.g. 9999), deliver last page of results.
scenarios = paginator.page(paginator.num_pages)
scenario_count = len(scenario_list)
context['pager_size'] = pager_size
context['scenarios'] = scenarios
context['scenario_range'] = range(paginator.num_pages)
context['scenario_count'] = scenario_count
context['current_start'] = (page-1)*pager_size + 1
context['current_stop'] = min(scenario_count, (page-1)*pager_size + pager_size)
context['my_username'] = self.request.user.username
# get locations (need name from dict by id)
adapter = EMOD_Adapter(self.request.user.username)
template_list = adapter.fetch_template_list()
location_dict = adapter.fetch_locations() # gets list based on EMOD_buffer list in model_adapter.py
# determine which step edit buttons link to
step_list = [i[0] for i in named_baseline_forms]
for scenario in scenario_list:
if scenario._meta.object_name != 'Folder':
try:
# check to see if it has a campaign file (besides the empty default)
if scenario.is_approved:
my_scenario = EMODBaseline.from_dw(pk=scenario['id'])
try:
scenario_campaign = json.dumps(ast.literal_eval(my_scenario.get_file_by_type('campaign').content),
sort_keys=True, indent=4, separators=(',', ': '))
except (AttributeError, ObjectDoesNotExist):
try:
# if file list was returned, use last file
scenario_campaign = json.dumps(ast.literal_eval(
my_scenario.get_file_by_type('campaign')[my_scenario.get_file_by_type('campaign').count()
- 1].content), sort_keys=True, indent=4,
separators=(',', ': '))
except ObjectDoesNotExist:
pass
if len(json.loads(scenario_campaign)['Events']) > 0:
scenario['has_interventions'] = 1
if scenario.template:
scenario.location_name = str(DimTemplate.objects.get(id=scenario.template.id).location_key)
# add model version to scenario
scenario.model_version = DimTemplate.objects.get(id=scenario.template.id)\
.model_version.replace('emod ', '')
except:
pass
return context
def get_context_data(self, **kwargs):
context = super(InterventionView, self).get_context_data(**kwargs)
dim_user = DimUser.objects.get_or_create(username=self.request.user.username)[0]
context['dim_user'] = dim_user
#context['nav_button'] = 'browse_scenario'
scenario = EMODBaseline.from_dw(id=kwargs['scenario_id'])
# Check permissions
if scenario.user != dim_user:
raise PermissionDenied
# # Get all of the interventions from local DB (public OR created by user)
# for item in list(Intervention.objects.filter(Q(is_public=1) | Q(created_by=self.request.user))):
# interventions.append(item)
# Get the campaign file
campaign = ast.literal_eval(scenario.get_file_by_type('campaign').content) # fixme Temporarily using ast.literal_eval until stuff starts getting stored correctly
# Ensure the campaign is valid, otherwise make it a valid empty campaign.
if 'Events' not in campaign:
campaign = {"Events": []}
# Get all formsets and fill initial ones with ones already in the campaign file
formsets = self.get_emod_formsets(campaign)
# Make a list of templates based on the formsets. These aren't templates for the forms themselves, but instead
# are just references to them.
intervention_templates = []
for i in range(len(formsets)):
intervention_templates.append({})
form = formsets[i].empty_form
form_attributes = vars(form)
intervention_templates[i]['name'] = form_attributes['fields']['name'].initial
intervention_templates[i]['name_for_url'] = get_name_for_url(intervention_templates[i]['name'])
intervention_templates[i]['prefix'] = formsets[i].prefix
intervention_templates[i]['details'] = '<th>Parameter</th><th>Default Value</th>'
# Fill the details table on the intervention page. Loop through all the fields in the form and put them
# into a table, but skip start_day, number_of_repetitions, timesteps_between_repetitions,
# max_number_of_distributions, demographic_coverage, and DELETE since these is not applicable information
# to the details table. I know the format of this is confusing. Bear with it.
for attribute_name in form_attributes['fields']:
if attribute_name != "start_day" and attribute_name != "number_of_repetitions" and \
attribute_name != "timesteps_between_repetitions" and attribute_name != "max_number_of_distributions" and \
attribute_name != "demographic_coverage" and attribute_name != "DELETE":
attribute = form_attributes['fields'][attribute_name]
formatted_name = convert_underscore_lower_to_underscore_upper(attribute_name).replace('_', ' ')
non_camel_cased_value = convert_camel_case_to_human_readable(str(attribute.initial))
formatted_value = non_camel_cased_value.replace('_', ' ')
intervention_templates[i]['details'] += '<tr><td>' + formatted_name + \
'    </td><td>' + formatted_value + '    </td></tr>'
# Get the config file
config = ast.literal_eval(scenario.get_file_by_type('config').content) # fixme Temporarily using ast.literal_eval until stuff starts getting stored correctly
# Get date info to determine the maximum allowable start_date for interventions.
try:
start_day_max = config['parameters']['Simulation_Duration']
except TypeError:
start_day_max = 2147480000 # Based on EMOD documentation
context['start_day_max'] = start_day_max
context['campaign'] = campaign
context['scenario'] = scenario
context['formsets'] = formsets
context['intervention_templates'] = intervention_templates
return context
def dispatch(self, request, *args, **kwargs):
""" The method that accepts a request argument plus arguments, and returns an HTTP response.
Make sure user is logged in
"""
# add the storage engine to the current wizardview instance here so we can act on storage here
self.prefix = self.get_prefix(*args, **kwargs)
self.storage = get_storage(self.storage_name, self.prefix, request, getattr(self, 'file_storage', None))
if 'scenario_id' in kwargs:
if kwargs['scenario_id'] == '0':
# first visit to "Add a New Baseline"
# wipe out any previous session-scenario and wizard storage (including edit flag).
try:
self.storage.reset()
except AttributeError:
pass
if 'scenario' in self.request.session.keys():
del self.request.session['scenario']
if 'scenario_config' in self.request.session.keys():
del self.request.session['scenario_config']
# create empty scenario
self.request.session['scenario'] = EMODBaseline(
name='',
description='',
model=DimModel.objects.get(model='EMOD'),
user=DimUser.objects.get_or_create(username=self.request.user.username)[0]
)
self.request.session['scenario'].model_version = 'emod' # set dummy, will be overwritten
elif kwargs['scenario_id'] > 0:
# First hit in Edit mode - populate the steps with initial defaults
self.storage.reset()
self.storage.extra_data['edit_scenario'] = kwargs['scenario_id']
# get the scenario for editing
self.request.session['scenario'] = EMODBaseline.from_dw(pk=kwargs['scenario_id'])
else:
# Make sure there's a scenario in the session
if 'scenario' not in self.request.session.keys():
# create empty scenario
self.request.session['scenario'] = EMODBaseline(
name='',
description='',
model=DimModel.objects.get(model='EMOD'),
user=DimUser.objects.get_or_create(username=self.request.user.username)[0]
)
if 'step' in kwargs.keys() and kwargs['step'] == 'climate':
# 6230: redirect to Intervention Tool (if user clicked "Skip to Intervention Tool Button")
# check to see if the user wants to jump over to the Intervention tool
if 'jump_to_intervention_tool' in self.request.session.keys():
my_scenario_id = self.request.session['scenario'].id
# clear wizard storage
self.storage.reset()
del self.request.session['jump_to_intervention_tool']
# redirect to Intervention Tool
return redirect("ts_intervention_tool_step", scenario_id=my_scenario_id)
self.request.session['scenario'].save()
return super(BaselineWizardView, self).dispatch(request, *args, **kwargs)
def done(self, form_list, **kwargs):
"""A method that processes all the form data at the end of Wizard
Save collected data from multiple forms into one DB table (Baseline)
"""
adapter = EMOD_Adapter(self.request.user.username)
# populate new scenario with template data
template_id = self.storage.get_step_data("location")["location-template_id"]
my_name = self.storage.get_step_data("location")["location-name"]
my_description = self.storage.get_step_data("location")["location-description"]
# create a new scenario
my_scenario = EMODBaseline(
name=my_name,
description=my_description,
model=DimModel.objects.get(model="EMOD"),
user=DimUser.objects.get_or_create(username=self.request.user.username)[0],
)
my_template_obj = DimTemplate.objects.get(id=template_id)
my_scenario.model_version = my_template_obj.model_version
my_scenario.template_id = template_id
my_scenario.template = my_template_obj
my_scenario.save()
# populate config
template_config = ast.literal_eval(my_template_obj.get_file_content("config.json"))
my_scenario_config = copy.deepcopy(template_config)
# save the scenario config file
my_scenario.add_file_from_string(
"config", "config.json", str(my_scenario_config), description=self.steps.current
)
# populate campaign
try:
template_campaign = ast.literal_eval(my_template_obj.get_file_content("campaign.json"))
my_campaign = str(copy.deepcopy(template_campaign))
except KeyError:
# use empty campaign file
my_campaign = str({"Events": []})
# save the scenario config file
my_scenario.add_file_from_string("campaign", "campaign.json", my_campaign, description=self.steps.current)
#########################
### Add default files to scenario
# todo: stop using template zips, when climate files are available elsewhere
# print my_template_obj.climate_url
my_location_zip_link = my_template_obj.climate_url
# Do we cache zip to temp dir first time, and always access them from there (if they exist)
# FILE_UPLOAD_TEMP_DIR
# setup work directory
# my_directory = os.path.join(settings.MEDIA_ROOT, 'uploads/expert_emod/')
# my_directory = os.path.join(FILE_UPLOAD_TEMP_DIR, 'ts_emod/')
# if not os.path.exists(my_directory):
# os.makedirs(my_directory)
# inStream = urllib2.urlopen('http://dl-vecnet.crc.nd.edu/downloads/wh246s153')
# inStream = urllib2.urlopen('https://dl.vecnet.org/downloads/wh246s153')
if settings.DEBUG is True:
now = datetime.datetime.now()
print now, " DEBUG: getting zip file: ", my_location_zip_link
url = urlopen(my_location_zip_link)
zipfile = ZipFile(StringIO(url.read()))
for zip_file_name in zipfile.namelist():
if settings.DEBUG is True:
now = datetime.datetime.now()
print now, " DEBUG: handling file: ", zip_file_name
my_file = zipfile.open(zip_file_name)
my_name = ntpath.basename(zip_file_name)
# determine which file type this is
# my_name contains 'demographics' ? (use compiled, store both?)
# Honiara_demographics.static.compiled.json Honiara_demographics.json
"""
'air binary',
'air json',
'humidity binary',
'humidity json',
'land_temp binary',
'land_temp json',
'rainfall binary',
'rainfall json',
Honiara_demographics.compiled.json
Honiara_demographics.json
Honiara_demographics.static.compiled.json
Honiara_demographics.static.json
Honiara_humidity_daily10y.bin
Honiara_humidity_daily10y.bin.json
Kenya_Nyanza_2.5arcminhumid_1365118879_climateheader.json
Honiara_rainfall_daily10y.bin
Honiara_rainfall_daily10y.bin.json
Honiara_temperature_daily10y.bin
Honiara_temperature_daily10y.bin.json
"""
my_type = None
if ".md5" in my_name:
continue
elif "demographics" in my_name or "Demographics" in my_name:
if "compiled" in my_name:
# use in scenario
my_type = "demographics"
else:
# save to storage (to allow user to see contents)
self.storage.extra_data["demographics"] = my_file
elif ".json" in my_name and ".json.bin" not in my_name:
# 'json' files
if "humid" in my_name:
my_type = "humidity json"
elif "rain" in my_name:
my_type = "rainfall json"
elif "temp" in my_name or "tmean" in my_name:
my_type = "air json"
else:
# must be a binary file
if "humid" in my_name:
my_type = "humidity binary"
elif "rain" in my_name:
my_type = "rainfall binary"
elif "temp" in my_name or "tmean" in my_name:
my_type = "air binary"
if my_type is not None:
if "binary" in my_type:
my_scenario.add_file_from_string(my_type, my_name, my_file.read(), description=self.steps.current)
else:
my_scenario.add_file_from_string(my_type, my_name, my_file.read(), description=self.steps.current)
# todo: fix to allow separate air and land temp files
# for now: use same files for air temp and land temp
if my_type in ("air binary", "air json"):
my_type = my_type.replace("air", "land_temp")
my_scenario.add_file_from_string(
my_type, my_name, "no land temp file", description=self.steps.current
)
### END Zip/Input File handling
######################
try:
my_id, completed = my_scenario.save()
set_notification("alert-success", "The simulation was saved. ", self.request.session)
# todo: add folder stuff to adapter/EMODBaseline (instead, for now get/save object)
# #6574 - add the folder that owns this scenario
if self.storage.get_step_data("location")["folder_id"] != "":
# see todo: my_scenario.folder = int(self.storage.get_step_data('location')['folder_id'])
temp = DimBaseline.objects.get(pk=my_scenario.id)
temp.folder = Folder.objects.get(pk=int(self.storage.get_step_data("location")["folder_id"]))
temp.save()
except KeyError:
set_notification("alert-error", "Error: The simulation was not saved. ", self.request.session)
# check if user wants to launch, or just save
if (
"edit" in self.storage.data["step_data"]["location"]
and self.storage.data["step_data"]["location"]["edit"][0] == "1"
):
edit_flag = True
else:
edit_flag = False
# clear out saved form data.
self.storage.reset()
if edit_flag:
return redirect("ts_edit_step", step="config", scenario_id=str(my_id))
else:
return redirect("ts_emod_scenario_details", scenario_id=my_id)
