def convert_baseline_to_scenario(request, baseline_id):
dim_user = DimUser.objects.get(username=request.user.username)
emod_scenario = EMODBaseline.from_dw(id=baseline_id)
dim_scenario = DimBaseline.objects.get(id=baseline_id)
extra_metadata = {}
# This doesn't all need to be here, but it is simpler just to leave it
if dim_scenario.metadata:
metadata = json.loads(dim_scenario.metadata)
if 'representative' in metadata:
emod_scenario.is_representative = True
extra_metadata = metadata
if 'is_complete' in metadata['representative']:
emod_scenario.representative_is_completed = True
else:
emod_scenario.representative_is_completed = False
emod_scenario.name = derive_autoname(dim_scenario)
else:
emod_scenario.is_representative = False
# Create simulation group if one is not already made
if not dim_scenario.simulation_group:
simulation_group = SimulationGroup(submitted_by=dim_user)
simulation_group.save()
simulation = add_simulation(dim_user, sim_model.EMOD, dim_scenario.template.model_version.split('v')[1], simulation_group, baseline_id)
scenario = Scenario.objects.create(
name=dim_scenario.name,
description=dim_scenario.description,
user=dim_user,
simulation=simulation,
metadata={},
extra_metadata=extra_metadata
)
# Modify config
config_json = json.loads(scenario.config_file.get_contents())
config_json['parameters']['Land_Temperature_Filename'] = 'temperature.bin'
config_json['parameters']['Rainfall_Filename'] = 'rainfall.bin'
config_json['parameters']['Relative_Humidity_Filename'] = 'humidity.bin'
config_json['parameters']['Air_Temperature_Filename'] = 'temperature.bin'
config_json['parameters']['Campaign_Filename'] = 'campaign.json'
config_json['parameters']['Demographics_Filename'] = 'demographics.compiled.json'
contents = json.dumps(config_json)
scenario.set_file_by_type('config', contents)
set_notification('alert-success', '<strong>Success!</strong> Converted baseline to scenario.', request.session)
# This makes sure we don't try to convert the same baseline twice
dim_scenario.simulation_group = simulation_group
dim_scenario.save()
else:
simulation_group = dim_scenario.simulation_group
try:
simulation = simulation_group.simulations.all()[0]
scenario = simulation.emod_scenario_set.all()[0]
except Exception as exception:
set_notification(
'alert-error',
'<strong>Error!</strong> Simulation group exists but there is no matching scenario. ' + str(exception),
request.session
)
return redirect("ts.index")
set_notification('alert-success', '<strong>Success!</strong> This baseline has already been converted.', request.session)
return redirect("ts_emod2.details", scenario_id=scenario.id)
def get_context_data(self, **kwargs):
"""Extension of get_context_data
Add context data to drive menu nav highlights, breadcrumbs and pagers.
"""
context = super(ScenarioBrowseView, self).get_context_data(**kwargs)
the_request = self.request
context['nav_button'] = 'scenario_browse'
pager_size = the_request.POST.get('pager_size') or the_request.GET.get('pager_size')
if pager_size is None:
pager_size = 10
else:
pager_size = int(pager_size)
# Get list of folders in this folder (if no current folder, include folders with no parent)
if 'folder_id' in kwargs.keys():
current_folder = kwargs['folder_id'] or the_request.POST.get('folder_id') or the_request.GET.get('folder_id') or None
context['current_folder'] = Folder.objects.get(pk=current_folder)
else:
current_folder = None
# get a list of all scenarios in order to get count of sims in each folder
scenario_list_all = list(DimBaseline.objects.filter(user=DimUser.objects.get(username=self.request.user.username),
is_deleted=False))
my_count = {}
for scen in scenario_list_all:
if scen.folder_id in my_count.keys():
my_count[scen.folder_id] += 1
else:
my_count[scen.folder_id] = 1
# create a dictionary of children (keys are parents, value is list of children)
folder_list_all = list(Folder.objects.filter(user=DimUser.objects.get(username=self.request.user.username),
is_deleted=False))
my_dict = defaultdict(list)
for folder in folder_list_all:
if folder.id in my_count.keys():
counter = ' (' + str(my_count[folder.id]) + ')'
else:
counter = ''
if folder.parent is None:
my_dict[0].append({'id': folder.id, 'name': folder.name, 'child_count': counter})
else:
my_dict[folder.parent.id].append({'id': folder.id, 'name': folder.name, 'child_count': counter})
current_found = 0 # Set flag that deterimines if the current folder has been visited in the tree
if None in my_count.keys():
counter_root = ' (' + str(my_count[None]) + ')'
else:
counter_root = ''
folder_tree = '{title: "My Simulations' + counter_root + '", folder: true, expanded: true'
if current_folder is None:
folder_tree += ', active: true'
current_found = 1
# context['folder_tree'] = '{title: "My Tororo Folder full of Simulations (and folders) in home folder ", folder: true }, ' \
# '{title: "Folder 2", folder: true, ' \
# 'children: [ ' \
# '{ title: "Sub-item 2.1", folder: true }, ' \
# '{ title: "Sub-item 2.2" }] }, ' \
# '{ title: "Item 3" }'
# without root:
# context['folder_tree'] = '{title: "Solomon Islands", folder: true}, {title: "Second folder", folder: true, children: [ {title: "subfolder", folder: true}] }'
#folder_tree = add_children(0, my_dict)
# with root:
# context['folder_tree'] = '{title: "My Simulations", folder: true, children: [ {title: "Solomon Islands", folder: true}, {title: "Second folder", folder: true, children: [ {title: "subfolder", folder: true}] }] }'
folder_tree += add_children(0, my_dict, current_folder, current_found) + '}'
context['folder_tree'] = str(folder_tree)
# Get list of scenarios
scenario_list = list(DimBaseline.objects.filter(user=DimUser.objects.get(username=self.request.user.username),
folder=current_folder,
is_deleted=False))
if scenario_list is None:
return context
# sort descending by id (for those old scenarios that all have the same old time_created
scenario_list = sorted(scenario_list, key=lambda x: x.id, reverse=True)
# sort descending by time_saved (version)
scenario_list = sorted(scenario_list, key=lambda x: x.last_modified, reverse=True)
# merge folders into scenario list
# - show folders in file list
#scenario_list = folder_list + scenario_list
# Flag scenarios that are representative
for scenario in scenario_list:
# Hack for now. I don't remember if it is still needed, but I will fix it later. Not currently important.
if scenario.description == "Made with representative workflow" and not scenario.metadata:
scenario.metadata = json.dumps({'representative': ''})
scenario.save()
if scenario.metadata:
metadata = json.loads(scenario.metadata)
if 'representative' in metadata:
scenario.is_representative = True
if 'is_complete' in metadata['representative']:
scenario.representative_is_complete = metadata['representative']['is_complete']
else:
scenario.representative_is_complete = False
scenario.name = derive_autoname(scenario)
else:
scenario.is_representative = False
paginator = Paginator(scenario_list, pager_size)
page = int(the_request.GET.get('page') or 1)
try:
scenarios = paginator.page(page)
except PageNotAnInteger:
# If page is not an integer, deliver first page.
scenarios = paginator.page(1)
except EmptyPage:
# If page is out of range (e.g. 9999), deliver last page of results.
scenarios = paginator.page(paginator.num_pages)
scenario_count = len(scenario_list)
context['pager_size'] = pager_size
context['scenarios'] = scenarios
context['scenario_range'] = range(paginator.num_pages)
context['scenario_count'] = scenario_count
context['current_start'] = (page-1)*pager_size + 1
context['current_stop'] = min(scenario_count, (page-1)*pager_size + pager_size)
context['my_username'] = self.request.user.username
# get locations (need name from dict by id)
adapter = EMOD_Adapter(self.request.user.username)
template_list = adapter.fetch_template_list()
location_dict = adapter.fetch_locations() # gets list based on EMOD_buffer list in model_adapter.py
# determine which step edit buttons link to
step_list = [i[0] for i in named_baseline_forms]
for scenario in scenario_list:
if scenario._meta.object_name != 'Folder':
try:
# check to see if it has a campaign file (besides the empty default)
if scenario.is_approved:
my_scenario = EMODBaseline.from_dw(pk=scenario['id'])
try:
scenario_campaign = json.dumps(ast.literal_eval(my_scenario.get_file_by_type('campaign').content),
sort_keys=True, indent=4, separators=(',', ': '))
except (AttributeError, ObjectDoesNotExist):
try:
# if file list was returned, use last file
scenario_campaign = json.dumps(ast.literal_eval(
my_scenario.get_file_by_type('campaign')[my_scenario.get_file_by_type('campaign').count()
- 1].content), sort_keys=True, indent=4,
separators=(',', ': '))
except ObjectDoesNotExist:
pass
if len(json.loads(scenario_campaign)['Events']) > 0:
scenario['has_interventions'] = 1
if scenario.template:
scenario.location_name = str(DimTemplate.objects.get(id=scenario.template.id).location_key)
# add model version to scenario
scenario.model_version = DimTemplate.objects.get(id=scenario.template.id)\
.model_version.replace('emod ', '')
except:
pass
return context
def get_context_data(self, **kwargs):
"""Extension of get_context_data
Add context data to drive menu nav highlights and breadcrumbs.
"""
context = super(ScenarioDetailView, self).get_context_data(**kwargs)
context['nav_button'] = "browse_scenario"
self.request.session['emod_nav_button'] = "browse_scenario"
# This is here so that Alex can run this later and change all production database entries
# all_runs = DimRun.objects.all()
# for run in all_runs:
# if run.start_date_key and run.start_date_key.timestamp:
# run.new_start_date_key = run.start_date_key.timestamp
# if run.end_date_key and run.end_date_key.timestamp:
# run.new_end_date_key = run.end_date_key.timestamp
# print run.new_start_date_key
# print run.new_end_date_key
# run.save()
scenario_id = kwargs['scenario_id']
scenario = EMODBaseline.from_dw(pk=scenario_id)
# Hack until DimBaseline/EMODBaseline issues are sorted out
dim_scenario = DimBaseline.objects.get(id=scenario_id)
# Hack for calibration
species_info = {}
the_config = ast.literal_eval(scenario.get_config_file().content)
the_species = the_config['parameters']['Vector_Species_Params']
for specie in the_species:
species_info[specie] = {}
if 'EIR' in the_species[specie]:
eir = the_species[specie]['EIR']
else:
eir = 20.0 / float(len(the_species))
if 'Sporozoite_Rate' in the_species[specie]:
sporozoite = the_species[specie]['Sporozoite_Rate']
else:
sporozoite = 0.003
species_info[specie]['eir'] = eir
species_info[specie]['sporozoite'] = sporozoite
context['species_info'] = species_info
print species_info
# data_to_send = {}
# data_to_send['eir'] = []
# data_to_send['sporozoite'] = []
# the_config = ast.literal_eval(scenario.get_config_file().content)
# the_species = the_config['parameters']['Vector_Species_Params']
# for i in range(len(the_species)):
# data_to_send['eir'].append(20)
# data_to_send['sporozoite'].append(0.003)
# context['data_to_send'] = data_to_send
if dim_scenario.metadata and 'calibration_status' in dim_scenario.metadata:
scenario.calibration_status = json.loads(dim_scenario.metadata)['calibration_status']
else:
scenario.calibration_status = "Ready"
if dim_scenario.metadata:
current_metadata = json.loads(dim_scenario.metadata)
else:
current_metadata = {}
if 'calibration_verification_run_id' in current_metadata:
calibration_verification_run = DimRun.objects.get(id=current_metadata['calibration_verification_run_id'])
if calibration_verification_run.status == '1/1/0':
vector_species_parameters = the_config['parameters']['Vector_Species_Params']
calibration_verification_info = {}
for species in vector_species_parameters:
calibration_verification_info[species] = {}
calibration_verification_info[species]['sporozoite_rate'] = {}
calibration_verification_info[species]['eir'] = {}
calibration_verification_info[species]['biting_rate'] = {}
target_sporozoite_rate = vector_species_parameters[species]['Sporozoite_Rate']
daily_sporozoite_rate_2_3 = get_average_daily_sporozoite_rate(calibration_verification_run, species, 2, 3)
daily_sporozoite_rate_4_9 = get_average_daily_sporozoite_rate(calibration_verification_run, species, 4, 9)
daily_sporozoite_rate_10_20 = get_average_daily_sporozoite_rate(calibration_verification_run, species, 10, 20)
calibration_verification_info[species]['sporozoite_rate']['target'] = target_sporozoite_rate
calibration_verification_info[species]['sporozoite_rate']['years_2_3'] = daily_sporozoite_rate_2_3
calibration_verification_info[species]['sporozoite_rate']['years_4_9'] = daily_sporozoite_rate_4_9
calibration_verification_info[species]['sporozoite_rate']['years_10_20'] = daily_sporozoite_rate_10_20
target_average_annual_eir = vector_species_parameters[species]['EIR']
annual_eir_2_3 = get_average_daily_eir(calibration_verification_run, species, 2, 3) * 365
annual_eir_4_9 = get_average_daily_eir(calibration_verification_run, species, 4, 9) * 365
annual_eir_10_20 = get_average_daily_eir(calibration_verification_run, species, 10, 20) * 365
calibration_verification_info[species]['eir']['target'] = target_average_annual_eir
calibration_verification_info[species]['eir']['years_2_3'] = annual_eir_2_3
calibration_verification_info[species]['eir']['years_4_9'] = annual_eir_4_9
calibration_verification_info[species]['eir']['years_10_20'] = annual_eir_10_20
target_average_daily_biting_rate = 'NA'
daily_biting_rate_2_3 = get_average_daily_biting_rate(calibration_verification_run, species, 2, 3)
daily_biting_rate_4_9 = get_average_daily_biting_rate(calibration_verification_run, species, 4, 9)
daily_biting_rate_10_20 = get_average_daily_biting_rate(calibration_verification_run, species, 10, 20)
calibration_verification_info[species]['biting_rate']['target'] = target_average_daily_biting_rate
calibration_verification_info[species]['biting_rate']['years_2_3'] = daily_biting_rate_2_3
calibration_verification_info[species]['biting_rate']['years_4_9'] = daily_biting_rate_4_9
calibration_verification_info[species]['biting_rate']['years_10_20'] = daily_biting_rate_10_20
context['calibration_verification_info'] = calibration_verification_info
context['scenario_id'] = kwargs['scenario_id']
context['scenario_userid'] = scenario.user.id
context['dim_user'] = DimUser.objects.get_or_create(username=self.request.user.username)[0]
scenario.last_edited = scenario._last_edited
scenario.is_public = scenario._is_public
# Hack until DimBaseline/EMODBaseline issues are sorted out
if dim_scenario.metadata:
metadata = json.loads(dim_scenario.metadata)
if 'representative' in metadata:
scenario.is_representative = True
if 'is_complete' in metadata['representative']:
scenario.representative_is_completed = True
else:
scenario.representative_is_completed = False
scenario.name = derive_autoname(dim_scenario)
else:
scenario.is_representative = False
context['scenario'] = scenario
file_list = []
for my_file in scenario.files:
if my_file['type'] == 'config':
try:
context['config'] = json.dumps(ast.literal_eval(scenario.get_file_by_type('config').content),
sort_keys=True, indent=4, separators=(',', ': '))
except AttributeError:
# if file list was returned, use last file
context['config'] = json.dumps(ast.literal_eval(
scenario.get_file_by_type('config')[scenario.get_file_by_type('config').count()
- 1].content), sort_keys=True, indent=4,
separators=(',', ': '))
elif my_file['type'] == 'campaign':
try:
scenario_campaign = json.dumps(ast.literal_eval(scenario.get_file_by_type('campaign').content),
sort_keys=True, indent=4, separators=(',', ': '))
if len(json.loads(scenario_campaign)['Events']) > 0:
context['campaign'] = scenario_campaign
else:
context['has_campaign'] = None
except AttributeError:
# if file list was returned, use last file
context['campaign'] = json.dumps(ast.literal_eval(
scenario.get_file_by_type('campaign')[scenario.get_file_by_type('campaign').count()
- 1].content), sort_keys=True, indent=4,
separators=(',', ': '))
else:
file_list.append(my_file['name'])
context['file_list'] = file_list
# Get a list of the runs for this scenario
adapter = EMOD_Adapter(self.request.user.username)
run_list = DimRun.objects.filter(baseline_key=scenario.id)
dim_scenario = DimBaseline.objects.get(id=scenario.id)
if dim_scenario.simulation_group:
simulation_group = dim_scenario.simulation_group
simulation_list = Simulation.objects.filter(group=simulation_group)
for simulation in simulation_list:
if simulation.status == sim_status.SCRIPT_DONE:
simulation.is_complete = True
elif simulation.status == sim_status.SCRIPT_ERROR:
simulation.has_failed = True
context['simulation_list'] = simulation_list
# sort descending by id (for those old runs that all have the same old time_created
context['run_list'] = sorted(run_list, key=lambda x: x.id, reverse=True)
# sort descending by time_saved
context['run_list'] = sorted(context['run_list'], key=lambda x: x.time_created, reverse=True)
note_list = []
for run in context['run_list']:
try:
run_diffs = []
# config diffs
run_config = run.jcd.jcdict['config.json']['Changes'][0]['config.json/parameters']
scenario_config = json.loads(context['config'])['parameters']
for key in scenario_config:
if scenario_config[key] != run_config[key]:
run_diffs.append({'key': key, 'run': run_config[key], 'scenario': scenario_config[key]})
# ToDo: split out each key difference of Vector Species Parameters in both
# campaign diffs
if 'campaign.json' not in run.jcd.jcdict:
run_diffs.append({'key': 'Interventions', 'run': 'None'})
else:
run_campaign = run.jcd.jcdict['campaign.json']['Changes'][0]['campaign.json/Events']
if json.loads(scenario_campaign)['Events'] != run_campaign:
run_diffs.append({'key': 'Interventions', 'run': run_campaign,
'scenario': str(json.loads(scenario_campaign)['Events'])})
# ToDo: split out each key difference of each Intervention in both
# else:
# run_diffs.append({'key': 'Interventions', 'run': 'See below',
# 'scenario': '<a class="btn tooltip_link" href="#campaign">see below</a>'})
run.diffs = run_diffs
except (AttributeError, KeyError):
# no JCD?
run.diffs = [{'Unknown Error Occurred:': 'no changes found.'}]
# get the status of each run
status = adapter.run_status(run.id)
if status is None:
run.my_completed = None
else:
run.my_completed = status[0]
run.my_total = status[1]
run.my_failed = status[2]
run.percent_complete = (run.my_completed + run.my_failed) / run.my_total
run.my_incomplete = int(run.my_total) - (int(run.my_completed) + int(run.my_failed))
if int(status[0]) > 0:
context['has_results'] = 1
note_list.extend(adapter.fetch_notes(run_id=int(run.id), as_object=True))
if note_list != []:
context['note_list'] = note_list
return context
