def submit_a_verification_job(request, scenario_id):
adapter = EMOD_Adapter(request.user.username)
emod_scenario = EMODBaseline.from_dw(id=scenario_id)
config_json = ast.literal_eval(emod_scenario.get_file_by_type('config').content)
config_json['parameters']['Simulation_Duration'] = 20 * 365 # 20 years
emod_scenario.add_file_from_string('config', 'config.json', json.dumps(config_json), description='SOMETHING')
my_simulation_duration = config_json['parameters']['Simulation_Duration']
## Set the run start_date based on location and config's start_time
my_start_date = \
datetime.datetime.strptime(emod_scenario.template.climate_start_date, '%Y-%m-%d').date() \
+ datetime.timedelta(days=config_json['parameters']['Start_Time'])
# Initialize the run
run = adapter.save_run(scenario_id=scenario_id,
template_id=int(emod_scenario.template.id),
start_date=my_start_date,
duration=my_simulation_duration,
name='calibration-verification',
description=str(datetime.datetime.now()),
location_ndx=emod_scenario.template.location_key.id,
timestep_interval=1,
run_id=-1,
as_object=True)
run.baseline_key_id = scenario_id
run.save()
# add in JCD for config file
changes = []
newdict = copy.deepcopy(config_json)
newdict['config.json/parameters'] = newdict.pop('parameters')
changes.append(Change.node(name='config.json', node=[newdict], mode='-'))
run.jcd = JCD.from_changes(changes)
run.save()
### Launch the run
# 5853 - hard-code to 1 for now
#reps_per_exec = int(self.get_cleaned_data_for_step('start')['reps_per_exec'])
reps_per_exec = 1
# submit returns tuple (success, message)
try:
status = submit(run, user=request.user, manifest=True, reps_per_exec=reps_per_exec)
set_notification('alert-success', '<strong>Success!</strong> Run launched.', request.session)
except (RuntimeError, TypeError, AssertionError), e:
set_notification('alert-error', '<strong>Error!</strong> ' + str(e.message), request.session)
def done(self, form_list, **kwargs):
if 'error_invalid_id' in self.storage.extra_data:
self.storage.reset()
set_notification('alert-error', 'Error Launching Simulation: Unknown/Invalid run provided. ' +
'Please select a simulation below to launch.', self.request.session)
return redirect("ts_emod_scenario_browse")
if 'scenario_id' in self.storage.extra_data:
scenario_id = int(self.storage.extra_data['scenario_id'])
else:
Exception()
adapter = EMOD_Adapter(self.request.user.username)
if 'emod_launch_run_id' in self.storage.extra_data:
my_run_id = int(self.storage.extra_data['emod_launch_run_id'])
my_run = adapter.fetch_runs(scenario_id=-1, run_id=int(my_run_id))
# check for name changes from user
start_data = self.get_cleaned_data_for_step('start')
if self.storage.extra_data['old_run_name'] != start_data['name']:
my_run.name = start_data['name']
my_run.save()
else:
# get the scenario
self.storage.extra_data['scenario'] = EMODBaseline.from_dw(pk=scenario_id)
my_config = ast.literal_eval(self.storage.extra_data['scenario'].get_file_by_type('config').content)
try:
my_campaign = ast.literal_eval(self.storage.extra_data['scenario'].get_file_by_type('campaign').content)
except AttributeError:
# if file list was returned, use last file
file_list = self.request.session['scenario'].get_file_by_type('campaign')
my_campaign = ast.literal_eval(file_list[0].content)
my_campaign_length = len(my_campaign['Events'])
###
# create a new run instance
if my_campaign_length > 0:
with_interventions = "with interventions "
else:
with_interventions = ""
my_simulation_duration = my_config['parameters']['Simulation_Duration']
## Set the run start_date based on location and config's start_time
my_start_date = \
datetime.datetime.strptime(
self.storage.extra_data['scenario'].template.climate_start_date,
'%Y-%m-%d').date() \
+ datetime.timedelta(days=my_config['parameters']['Start_Time'])
# Initialize the run
my_run = adapter.save_run(scenario_id=scenario_id,
template_id=int(self.storage.extra_data['scenario'].template.id),
start_date=my_start_date,
duration=my_simulation_duration,
name=self.get_cleaned_data_for_step('start')['name'],
description='Launch Tool: ' + self.storage.extra_data['scenario'].name
+ ': Run ' + with_interventions + str(datetime.datetime.now()),
location_ndx=self.storage.extra_data['scenario'].template.location_key.id,
timestep_interval=1,
run_id=-1,
as_object=True)
my_run.baseline_key_id = scenario_id
my_run.save()
if settings.DEBUG:
print "DEBUG: LaunchTool: run id: ", my_run.id
# add in JCD for config file
changes = []
newdict = copy.deepcopy(my_config)
newdict['config.json/parameters'] = newdict.pop('parameters')
changes.append(Change.node(name='config.json', node=[newdict], mode='-'))
# add in JCD for campaign file
if my_campaign_length > 0:
newdict = copy.deepcopy(my_campaign)
newdict['campaign.json/Use_Defaults'] = newdict.pop('Use_Defaults')
newdict['campaign.json/Events'] = newdict.pop('Events')
changes.append(Change.node(name='campaign.json', node=[newdict], mode='-'))
my_run.jcd = JCD.from_changes(changes)
my_run.save()
### Launch the run
# 5853 - hard-code to 1 for now
#reps_per_exec = int(self.get_cleaned_data_for_step('start')['reps_per_exec'])
reps_per_exec = 1
# If this is a representative based scenario, then set it to non-editable
representative_scenario = RepresentativeScenario.objects.filter(emod_scenario_id=scenario_id)
if representative_scenario:
representative_scenario[0].is_editable = False
representative_scenario[0].save()
# submit returns tuple (success, message)
try:
status = submit(my_run, user=self.request.user, manifest=True, reps_per_exec=reps_per_exec)
set_notification('alert-success', '<strong>Success!</strong> Run launched.',
self.request.session)
except (RuntimeError, TypeError, AssertionError), e:
set_notification('alert-error', '<strong>Error!</strong> ' + str(e.message), self.request.session)
def process_step(self, form):
""" Method to handle "after Next-click" processing of steps """
# set the flag that determines when to save the config to the scenario
change_config = 0
# set the flag that determines when to save the scenario
change_scenario = 0
adapter = EMOD_Adapter(self.request.user.username)
if self.steps.current == 'config':
if self.request.session['scenario_config']['parameters']['Simulation_Duration'] != \
form.cleaned_data['Simulation_Duration']:
self.request.session['scenario_config']['parameters']['Simulation_Duration'] = \
form.cleaned_data['Simulation_Duration']
change_config = 1
if self.request.session['scenario_config']['parameters']['Simulation_Type'] != \
form.cleaned_data['Simulation_Type']:
self.request.session['scenario_config']['parameters']['Simulation_Type'] = \
form.cleaned_data['Simulation_Type']
change_config = 1
if self.request.session['scenario_config']['parameters']['Start_Time'] != form.cleaned_data['Start_Time']:
self.request.session['scenario_config']['parameters']['Start_Time'] = form.cleaned_data['Start_Time']
change_config = 1
if self.request.session['scenario'].name != form.cleaned_data['name']:
self.request.session['scenario'].name = form.cleaned_data['name']
change_scenario = 1
if self.request.session['scenario'].description != form.cleaned_data['description']:
self.request.session['scenario'].description = form.cleaned_data['description']
change_scenario = 1
# Workaround for bug #5626 config file is not saved if user doesn't change Simulation Duration on
# Step 2 of 9: Configure Simulation
change_config = 1
# 6230: redirect to Intervention Tool (if user clicked "Skip to Intervention Tool Button")
# Set session variable here, so dispatch at beginning of next step knows to redirect (wizard won't allow
# redirect here to work.
if 'jump_to_intervention_tool' in self.request.POST.keys():
self.request.session['jump_to_intervention_tool'] = 1
if self.steps.current == 'species':
species_list = form.cleaned_data['species']
# if the list NOT matches the old list, update it
if species_list != self.request.session['scenario_config']['parameters']['Vector_Species_Names']:
self.request.session['scenario_config']['parameters']['Vector_Species_Names'] = species_list
change_config == 1
# check also for changes in the Vector Parameters (vector list may be the same)
my_json = dict(form.data)
# Gather parameter sets for all species in names list.
parameter_dict = {}
for species in species_list:
# add each Vector to Params list
my_dict = {}
for key in sorted(my_json.keys()): # sorted so Habitat_Type/Required_Habitat_Factor order preserved
if str(species) + '__json_' in key:
if type(my_json[key]) == list:
my_value = my_json[key][0]
else:
my_value = my_json[key]
if type(my_value) not in (float, int):
try:
my_value = int(str(my_value))
except (ValueError, TypeError):
try:
my_value = float(str(my_value))
except (ValueError, TypeError):
pass
if 'Required_Habitat_Factor' in key.split('__json_')[1]:
if my_value == '':
continue
if 'Required_Habitat_Factor' in my_dict.keys():
my_dict['Required_Habitat_Factor'].append(my_value)
else:
my_dict.update({'Required_Habitat_Factor': [my_value]})
elif 'Habitat_Type' in key.split('__json_')[1]:
if 'Habitat_Type' in my_dict.keys():
my_dict['Habitat_Type'].append(my_value)
else:
my_dict.update({'Habitat_Type': [my_value]})
elif key.split('__json_')[1] != 'obj':
my_dict.update({key.split('__json_')[1]: my_value})
parameter_dict.update({species: my_dict})
if self.request.session['scenario_config']['parameters']['Vector_Species_Params'] != parameter_dict:
self.request.session['scenario_config']['parameters']['Vector_Species_Params'] = parameter_dict
change_config = 1
if self.steps.current == 'parasite':
my_fields = json.loads(form.cleaned_data['json'])
for key in my_fields:
my_value = my_fields[key]['value']
if type(my_value) in (str, unicode):
try:
my_value = int(my_value)
except ValueError:
try:
my_value = float(my_value)
except ValueError:
pass
if my_value != self.request.session['scenario_config']['parameters'][key]:
self.request.session['scenario_config']['parameters'][key] = my_value
change_config = 1
if self.steps.current == 'scaling_factors':
my_value = form.cleaned_data['x_Temporary_Larval_Habitat']
if type(my_value) in (str, unicode):
try:
my_value = int(my_value)
except ValueError:
try:
my_value = float(my_value)
except ValueError:
pass
if my_value != self.request.session['scenario_config']['parameters']['x_Temporary_Larval_Habitat']:
self.request.session['scenario_config']['parameters']['x_Temporary_Larval_Habitat'] = my_value
change_config = 1
# populate new scenario with template data
# or with existing scenario data if this call is for Editing a scenario
if (self.steps.current == 'location' and form.cleaned_data['template_id'])\
and ('template_id' not in self.storage.extra_data.keys()
or self.storage.extra_data['template_id'] != form.cleaned_data['template_id']):
# todo: some of this is either edit-mode or new-mode: sort out later (depending on availability of data)
template_id = int(form.cleaned_data['template_id'])
if 'template_id' in self.storage.extra_data \
and template_id != self.storage.extra_data['template_id']:
pass # location_change = 1
else:
# first time through wizard, not a location change
# location_change = 0
# save schema
self.request.session['schema.json'] = ConfigData.objects.filter(type='schema',
name='emod-v1.5-schema.txt')[0].json
# Set name here with dummy name as adding a file will attempt to save the scenario,
# which fails if name is null
if self.request.session['scenario'].name is None:
self.request.session['scenario'].name = ''
self.request.session['scenario'].description = ''
# check for campaign file...
try:
# if it exists, do nothing
self.request.session['scenario'].get_file_by_type('campaign')
except ObjectDoesNotExist:
# else: add in empty campaign file
self.request.session['scenario'].add_file_from_string('campaign',
'campaign.json',
str({"Events": []}),
description=self.steps.current)
my_template_obj = DimTemplate.objects.get(id=template_id)
self.request.session['scenario'].model_version = my_template_obj.model_version
self.request.session['scenario'].dimbaseline.model_version = my_template_obj.model_version
self.request.session['scenario'].template_id = template_id
self.request.session['scenario'].template = my_template_obj
self.request.session['scenario'].save()
location_id = my_template_obj.location_key_id
self.request.session['scenario'].template_id = template_id
self.request.session['scenario'].template = my_template_obj
# needed for editing
if self.request.session['scenario'].dimbaseline is not None:
self.request.session['scenario'].dimbaseline.template = my_template_obj
self.request.session['scenario'].dimbaseline.template_id = template_id
change_scenario = 1
# Populate the step data with the chosen location's data
self.storage.extra_data.update({'template_id': template_id})
# populate config to the session for multiple uses later
template_config = ast.literal_eval(my_template_obj.get_file_content('config.json'))
if 'edit_scenario' in self.storage.extra_data.keys() \
and 'config' not in self.request.session['scenario'].get_missing_files():
# use scenario's config file
try:
self.request.session['scenario_config'] = \
ast.literal_eval(self.request.session['scenario'].get_file_by_type('config').content)
except AttributeError:
# if query set returned, use first file in "list"
file_list = self.request.session['scenario'].get_file_by_type('config')
self.request.session['scenario_config'] = \
ast.literal_eval(file_list[0].content)
else:
# use template's (or parent scenario's) config file
self.request.session['scenario_config'] = copy.deepcopy(template_config)
self.storage.extra_data.update({'scenario_template_config': template_config})
change_config = 1
#########################
### Add default files to scenario
# todo: stop using template zips, when climate files are available elsewhere
print my_template_obj.climate_url
self.storage.extra_data.update({'location_zip_link': my_template_obj.climate_url})
# Do we cache zip to temp dir first time, and always access them from there (if they exist)
# FILE_UPLOAD_TEMP_DIR
# setup work directory
#my_directory = os.path.join(settings.MEDIA_ROOT, 'uploads/expert_emod/')
#my_directory = os.path.join(FILE_UPLOAD_TEMP_DIR, 'ts_emod/')
#if not os.path.exists(my_directory):
# os.makedirs(my_directory)
#inStream = urllib2.urlopen('http://dl-vecnet.crc.nd.edu/downloads/wh246s153')
#inStream = urllib2.urlopen('https://dl.vecnet.org/downloads/wh246s153')
if 'location_zip_link' in self.storage.extra_data:
if settings.DEBUG is True:
now = datetime.datetime.now()
print now, " DEBUG: getting zip file: ", self.storage.extra_data['location_zip_link']
url = urlopen(self.storage.extra_data['location_zip_link'])
zipfile = ZipFile(StringIO(url.read()))
for zip_file_name in zipfile.namelist():
if settings.DEBUG is True:
now = datetime.datetime.now()
print now, " DEBUG: handling file: ", zip_file_name
my_file = zipfile.open(zip_file_name)
my_name = ntpath.basename(zip_file_name)
# determine which file type this is
# my_name contains 'demographics' ? (use compiled, store both?) Honiara_demographics.static.compiled.json Honiara_demographics.json
"""
'air binary',
'air json',
'humidity binary',
'humidity json',
'land_temp binary',
'land_temp json',
'rainfall binary',
'rainfall json',
Honiara_demographics.compiled.json
Honiara_demographics.json
Honiara_demographics.static.compiled.json
Honiara_demographics.static.json
Honiara_humidity_daily10y.bin
Honiara_humidity_daily10y.bin.json
Kenya_Nyanza_2.5arcminhumid_1365118879_climateheader.json
Honiara_rainfall_daily10y.bin
Honiara_rainfall_daily10y.bin.json
Honiara_temperature_daily10y.bin
Honiara_temperature_daily10y.bin.json
"""
my_type = None
if '.md5' in my_name:
continue
elif 'demographics' in my_name or 'Demographics' in my_name:
if 'compiled' in my_name:
# use in scenario
my_type = 'demographics'
else:
# save to storage (to allow user to see contents)
self.storage.extra_data['demographics'] = my_file
elif '.json' in my_name and '.json.bin' not in my_name:
# 'json' files
if 'humid' in my_name:
my_type = 'humidity json'
elif 'rain' in my_name:
my_type = 'rainfall json'
elif 'temp' in my_name or 'tmean' in my_name:
my_type = 'air json'
else:
# must be a binary file
if 'humid' in my_name:
my_type = 'humidity binary'
elif 'rain' in my_name:
my_type = 'rainfall binary'
elif 'temp' in my_name or 'tmean' in my_name:
my_type = 'air binary'
if my_type is not None:
if 'binary' in my_type:
self.request.session['scenario'].add_file_from_string(my_type, my_name,
my_file.read(),
description=self.steps.current)
else:
self.request.session['scenario'].add_file_from_string(my_type, my_name,
my_file.read(),
description=self.steps.current)
change_scenario = 1
# todo: fix to allow separate air and land temp files
# for now: use same files for air temp and land temp
if my_type in ('air binary', 'air json'):
my_type = my_type.replace('air', 'land_temp')
self.request.session['scenario'].add_file_from_string(my_type, my_name,
"no land temp file",
description=self.steps.current)
### END if zip file is none
# could save here to free up memory from the object file storage
# but it fails without a name
#if change_scenario == 1:
# self.request.session['scenario'].save()
### END Zip/Input File handling
######################
# config step data:
self.storage.set_step_data('config', {u'config-name': [self.request.session['scenario']._name],
u'config-description': [self.request.session['scenario'].description],
u'config-Start_Time': [self.request.session['scenario_config']['parameters']['Start_Time']],
u'config-Simulation_Duration':
[self.request.session['scenario_config']['parameters']['Simulation_Duration']],
u'config-Simulation_Type':
[self.request.session['scenario_config']['parameters']['Simulation_Type']],
})
# climate step data:
self.storage.set_step_data('climate', {u'climate-location': [location_id]})
# demographic step data:
self.storage.set_step_data('demographic', {u'demographic-location': [location_id]})
# parasite
for step_name in ["vector", "parasite"]:
# get db config for the step
my_json = ConfigData.objects.filter(type='JSONConfig', name=step_name)[0].json
# wizard values based on config + template values
my_wiz = {u'orig_json_obj': [my_json]}
my_json = json.loads(my_json)
for key in my_json.keys():
try:
# = scenario value OR template value
# change into wizard format step name + -json_ + key name
my_wiz.update({u'' + step_name + '-json_'
+ key: [u'' + str(self.request.session['scenario_config']['parameters'][key])]})
except KeyError:
pass
# insert into wizard storage
self.storage.set_step_data(step_name, my_wiz)
# Scaling step data:
self.storage.set_step_data('scaling_factors',
{u'scaling_factors-x_Temporary_Larval_Habitat':
[self.request.session['scenario_config']['parameters']['x_Temporary_Larval_Habitat']]})
# save any changes made to the scenario config file
if change_config == 1 and self.steps.current != 'location':
self.request.session['scenario'].add_file_from_string('config', 'config.json',
str(self.request.session['scenario_config']),
description=self.steps.current)
change_scenario = 1
if change_scenario == 1 and self.steps.current != 'location':
# don't save until config step (name and description will be populated there.
try:
my_id, completed = self.request.session['scenario'].save()
set_notification('alert-success', 'The simulation was saved. ', self.request.session)
except KeyError:
set_notification('alert-error', 'Error: The simulation was not saved. ', self.request.session)
###########################
### Create a run with scenario settings
# - launch it
# - display status
# ??? Do we want to force/allow user to add name/description to these runs?
# ??? Cap duration for initial run? 5 years?
if self.steps.current == 'run':
# create a new run instance
adapter = EMOD_Adapter(self.request.user.username)
my_template = DimTemplate.objects.get(id=self.request.session['scenario'].template.id)
my_start_date = \
datetime.datetime.strptime(
my_template.climate_start_date,
'%Y-%m-%d').date() \
+ datetime.timedelta(days=self.request.session['scenario_config']['parameters']['Start_Time'])
my_run = adapter.save_run(
template_id=int(self.request.session['scenario'].template.id),
start_date=my_start_date,
duration=100, # 1825, 5 years
#=self.request.session['scenario_config']['parameters']['Simulation_Duration'],
name=form.cleaned_data['name'], # self.request.session['scenario'].name + ': Test Run '
description='Run by New Simulation Wizard',
location_ndx=my_template.location_key.id,
timestep_interval=1,
run_id=-1, # run_id,
as_object=True)
my_run.scenario_key_id = self.request.session['scenario'].id
my_run.save()
if settings.DEBUG:
print "DEBUG: line 890 BaselineWiz: run id: ", my_run.id
# add in JCD for it for config file
changes = []
newdict = copy.deepcopy(self.request.session['scenario_config'])
newdict['config.json/parameters'] = newdict.pop('parameters')
changes.append(Change.node(name='config.json', node=[newdict], mode='-'))
my_run.jcd = JCD.from_changes(changes)
my_run.save()
### Launch the run
# save launch info to extra data todo: this should eventually go into success block below
self.storage.extra_data['last_launched_run'] = my_run.id
# submit returns tuple (success, message)
status = submit(my_run, user=self.request.user, manifest=True, reps_per_exec=1)
try:
if status[0] is True:
set_notification('alert-success', '<strong>Success!</strong> Test Run launched.',
self.request.session)
else:
set_notification('alert-error', '<strong>Error!</strong> ' + status[1], self.request.session)
except TypeError:
pass
return self.get_form_step_data(form)