def delete_run(self, run_node, force=False):
'''
Remove a run both from the services database and from the model.
@param run_node (Element): the node to remove.
'''
# Prevent the user from removing base years
cache_directory = run_node.find('cache_directory').text
if cache_directory.endswith('base_year_data') and not force:
msg = ('Removing the base year data directory is restricted from '
'within OpusGUI since doing so will make it impossible to '
'run any simulations or estimations.')
MessageBox.warning(mainwindow = self.view,
text = 'Cannot remove base year data',
detailed_text = msg)
return
try:
run_manager = get_run_manager()
run_id = run_node.get('run_id')
try:
run_id = int(run_id)
except:
run_id = -1
run_manager.delete_everything_for_this_run(run_id, cache_directory)
run_manager.close()
self.project.delete_node(run_node)
# self.model.remove_node(run_node)
except Exception, ex: # TODO catch more specific error?
MessageBox.warning(self.view, 'Could not remove run', str(ex))
def _scanForRuns(self):
data_path = self.project.data_path()
if not os.path.exists(data_path):
MessageBox.warning(mainwindow = self.base_widget,
text="Project data path %s doesn't exist. " % data_path + \
"Simulation runs in the Results tab cannot be updated." )
return
run_manager = get_run_manager()
run_manager.clean_runs()
self._sync_base_year_data(run_manager)
run_manager.close()
added_runs, removed_runs = sync_available_runs(self.project)
added_msg = removed_msg = None
if len(added_runs) > 0:
added_msg = ('The following simulation runs have been '
'automatically added to the results manager:\n\n%s'
% '\n'.join(added_runs))
if len(removed_runs) > 0:
removed_msg = ('The following simulation runs have been '
'automatically removed from the results manager:\n\n%s'
% '\n'.join(removed_runs))
if added_msg or removed_msg:
## The idea is to leave the run information to services db & cache, and
## we don't need to save the newly added runs, once we set the temporary
# self.project.dirty = True
text = 'The list of simulation runs has been automatically updated.'
detailed_text = '%s\n%s' % (added_msg or '', removed_msg or '')
logger.log_status(text+'\n'+detailed_text)
def import_run(self, cache_directory, run_info={}):
run_manager = get_run_manager()
retval, retmsg = run_manager.import_run_from_cache(cache_directory)
if not retval:
MessageBox.warning(mainwindow = self.base_widget,
text = retmsg,
detailed_text = '')
else:
sync_available_runs(self.project)
def restart_run(self, run_id, config, restart_year, end_year=None, run_name=None):
run_manager = get_run_manager()
statusdir = run_manager.get_cache_directory(run_id)
self.statusfile = os.path.join(statusdir, 'status.txt')
self.currentLogfileYear = restart_year
self.currentLogfileKey = 0
self.config = config
config['status_file_for_gui'] = self.statusfile
self.commandfile = os.path.join(statusdir, 'command.txt')
config['command_file_for_gui'] = self.commandfile
# To test delay in writing the first log file entry...
# time.sleep(5)
self.running = True
self.run_name = run_name
self.run_manager = run_manager
self.startedCallback(run_id=run_id,
run_name=run_name,
scenario_name=self.scenariotorun,
run_resources = config,
status = 'running')
run_manager.restart_run(run_id, restart_year,
end_year=end_year,
project_name='',
skip_urbansim=False,
create_baseyear_cache_if_not_exists=False,
skip_cache_cleanup=False)
self.running = False
succeeded = True
if self.statusfile is not None:
gc.collect()
# adding try/except, windows sometimes has a lock on this file
# when it does, os.remove will cause the simulation to appear
# as if it has crashed, when in fact it simply could not delete
# status.txt
try:
os.remove(self.statusfile)
except:
pass
self.finishedCallback(succeeded, run_name = run_name)
def restart_run(self, run_id, config, restart_year, end_year=None, run_name=None):
run_manager = get_run_manager()
statusdir = run_manager.get_cache_directory(run_id)
self.statusfile = os.path.join(statusdir, 'status.txt')
self.currentLogfileYear = restart_year
self.currentLogfileKey = 0
self.config = config
config['status_file_for_gui'] = self.statusfile
self.commandfile = os.path.join(statusdir, 'command.txt')
config['command_file_for_gui'] = self.commandfile
# To test delay in writing the first log file entry...
# time.sleep(5)
self.running = True
self.run_name = run_name
self.run_manager = run_manager
self.startedCallback(run_id=run_id,
run_name=run_name,
scenario_name=self.scenariotorun,
run_resources = config,
status = 'running')
run_manager.restart_run(run_id, restart_year,
end_year=end_year,
project_name='',
skip_urbansim=False,
create_baseyear_cache_if_not_exists=False,
skip_cache_cleanup=False)
self.running = False
succeeded = True
if self.statusfile is not None:
gc.collect()
# adding try/except, windows sometimes has a lock on this file
# when it does, os.remove will cause the simulation to appear
# as if it has crashed, when in fact it simply could not delete
# status.txt
try:
os.remove(self.statusfile)
except:
pass
self.finishedCallback(succeeded, run_name = run_name)
def _scanForRuns(self):
run_manager = get_run_manager()
run_manager.clean_runs()
run_manager.close()
added_runs, removed_runs = update_available_runs(self.project)
added_msg = removed_msg = None
if len(added_runs) > 0:
added_msg = ('The following simulation runs have been '
'automatically added to the results manager:\n\n%s'
% '\n'.join(added_runs))
if len(removed_runs) > 0:
removed_msg = ('The following simulation runs have been '
'automatically removed from the results manager:\n\n%s'
% '\n'.join(removed_runs))
if added_msg or removed_msg:
self.project.dirty = True
text = 'The list of simulation runs has been automatically updated.'
detailed_text = '%s\n\n%s' % (added_msg or '', removed_msg or '')
MessageBox.information(mainwindow = self.base_widget,
text = text,
detailed_text = detailed_text)
def _restart_selected_run(self):
assert self.has_selected_item()
run_node = self.selected_item().node
# ask for start_year, end_year
# start_year default to the last year of years run
# need to avoid insert auto generate run directory again
# original_start_year = int(run_node.find('start_year').text)
# original_end_year = int(run_node.find('end_year').text)
# restart_year = original_end_year + 1
# end_year = restart_year + 1
run_name = run_node.get('name')
scenario_name = run_node.find('scenario_name').text
run_id = run_node.get('run_id')
try:
run_id = int(run_id)
except ValueError:
raise ValueError, "run_id for run %s is invalid: %s; the run cannot be restarted." % \
(run_name, run_id)
run_manager = get_run_manager()
config = run_manager.get_resources_for_run_id_from_history(run_id)
xml_config = self.project.xml_config
opusgui = get_mainwindow_instance()
scenario_manager = get_manager_instance('scenario_manager')
opusmodel = OpusModel(scenario_manager, xml_config, scenario_name)
tab_widget = SimulationGuiElementRestart(mainwindow=opusgui,
runManager=scenario_manager,
model=opusmodel,
xml_config=xml_config,
run_id=run_id,
run_name=run_name)
tab_widget.config = config
# tab_widget.start_year = start_year
# tab_widget.end_year = end_year
scenario_manager._attach_tab(tab_widget)
opusgui.update()
def _restart_selected_run(self):
assert self.has_selected_item()
run_node = self.selected_item().node
# ask for start_year, end_year
# start_year default to the last year of years run
# need to avoid insert auto generate run directory again
# original_start_year = int(run_node.find('start_year').text)
# original_end_year = int(run_node.find('end_year').text)
# restart_year = original_end_year + 1
# end_year = restart_year + 1
run_name = run_node.get('name')
scenario_name = run_node.find('scenario_name').text
run_id = run_node.get('run_id')
try:
run_id = int(run_id)
except ValueError:
raise ValueError, "run_id for run %s is invalid: %s; the run cannot be restarted." % \
(run_name, run_id)
run_manager = get_run_manager()
config = run_manager.get_resources_for_run_id_from_history(run_id)
xml_config = self.project.xml_config
opusgui = get_mainwindow_instance()
scenario_manager = get_manager_instance('scenario_manager')
opusmodel = OpusModel(scenario_manager, xml_config, scenario_name)
tab_widget = SimulationGuiElementRestart(mainwindow=opusgui,
runManager=scenario_manager,
model=opusmodel,
xml_config=xml_config,
run_id=run_id,
run_name=run_name)
tab_widget.config = config
# tab_widget.start_year = start_year
# tab_widget.end_year = end_year
scenario_manager._attach_tab(tab_widget)
opusgui.update()
def _scanForRuns(self):
run_manager = get_run_manager()
run_manager.clean_runs()
run_manager.close()
added_runs, removed_runs = update_available_runs(self.project)
added_msg = removed_msg = None
if len(added_runs) > 0:
added_msg = ('The following simulation runs have been '
'automatically added to the results manager:\n\n%s' %
'\n'.join(added_runs))
if len(removed_runs) > 0:
removed_msg = (
'The following simulation runs have been '
'automatically removed from the results manager:\n\n%s' %
'\n'.join(removed_runs))
if added_msg or removed_msg:
self.project.dirty = True
text = 'The list of simulation runs has been automatically updated.'
detailed_text = '%s\n\n%s' % (added_msg or '', removed_msg or '')
MessageBox.information(mainwindow=self.base_widget,
text=text,
detailed_text=detailed_text)
def run(self):
# Run the Eugene model using the XML version of the Eugene configuration.
# This code adapted from opus_core/tools/start_run.py
# statusdir is a temporary directory into which to write a status file
# regarding the progress of the simulation - the progress bar reads this file
statusdir = None
succeeded = False
run_id = None
try:
config = self.xml_config.get_run_configuration(str(self.scenariotorun))
# self.xmltreeobject.toolboxbase.opus_core_xml_configuration.get_run_configuration(str(self.scenariotorun))
cache_directory_root = config['creating_baseyear_cache_configuration'].cache_directory_root
run_name = self.get_run_name(config = config, run_name = self.run_name)
config['cache_directory'] = os.path.join(cache_directory_root, run_name)
#insert_auto_generated_cache_directory_if_needed(config)
(self.start_year, self.end_year) = config['years']
run_manager = get_run_manager()
run_manager.setup_new_run(cache_directory = config['cache_directory'],
configuration = config)
statusdir = run_manager.get_current_cache_directory()
self.statusfile = os.path.join(statusdir, 'status.txt')
self.currentLogfileYear = self.start_year
self.currentLogfileKey = 0
self.config = config
config['status_file_for_gui'] = self.statusfile
self.commandfile = os.path.join(statusdir, 'command.txt')
config['command_file_for_gui'] = self.commandfile
# To test delay in writing the first log file entry...
# time.sleep(5)
self.running = True
self.run_name = run_name
self.run_manager = run_manager
run_id = run_manager.run_id
self.startedCallback(run_id = run_id,
run_name = run_name,
scenario_name = self.scenariotorun,
run_resources = config,
status = 'running')
run_manager.run_run(config, run_name=run_name,
scenario_name=self.scenariotorun)
self.running = False
succeeded = True
except SimulationRunError:
self.running = False
succeeded = False
except:
self.running = False
succeeded = False
errorInfo = formatExceptionInfo(custom_message = 'Unexpected Error From Model')
self.errorCallback(errorInfo)
if self.statusfile is not None:
gc.collect()
# adding try/except, windows sometimes has a lock on this file
# when it does, os.remove will cause the simulation to appear
# as if it has crashed, when in fact it simply could not delete
# status.txt
try:
os.remove(self.statusfile)
except:
pass
self.finishedCallback(succeeded, run_name = run_name)
def _sync_base_year_data(self, run_manager=None):
"""
synchronize base_year_data information in xml_configuration with
run_activity table. Information in xml_configuration takes
precedent, because we assume users don't directly modify data in
serivecs.run_activity table
"""
# TODO baseyear_dir is somewhat hard-coded; it would be better to read
# from xml_configuration instead, but there is no such node currently
run_name = 'base_year_data'
baseyear_dir = os.path.join(self.project.data_path(), run_name)
baseyear_dir = os.path.normpath(baseyear_dir)
if not os.path.exists(baseyear_dir):
MessageBox.warning(mainwindow = self.base_widget,
text="base_year_data directory %s doesn't exist. " % baseyear_dir
)
return
import glob
years = [int(os.path.basename(year_dir)) for year_dir in
glob.glob(os.path.join(baseyear_dir, '[0-9][0-9][0-9][0-9]'))]
if not years:
MessageBox.warning(mainwindow = self.base_widget,
text="base_year_data directory %s doesn't contain any year sub-directory. " % baseyear_dir
)
return
start_year = min(years)
end_year = max(years)
base_year = end_year # default to the last year in baseyear_dir
# and update it with information found in scenario_manager
scenario_manager_node = self.project.find('scenario_manager')
for base_year_node in scenario_manager_node.findall('.//base_year'):
try:
base_year = int(base_year_node.text.strip())
break
except (TypeError, ValueError):
continue
resources = {
'cache_directory': baseyear_dir,
'description': 'base year data',
'base_year': base_year,
'years': (start_year, end_year)
}
if run_manager is None: run_manager = get_run_manager()
base_year_data_db = run_manager.get_runs(run_name=run_name,
process_name=get_host_name())
if len(base_year_data_db) == 0:
run_id = run_manager._get_new_run_id()
elif len(base_year_data_db) >= 1:
for idx, row in enumerate(base_year_data_db):
if idx==0:
run_id = row[0]
else:
run_manager.delete_everything_for_this_run(row[0])
resources_db = run_manager.get_resources_for_run_id_from_history(run_id)
if resources_db.get('cache_directory', '') == baseyear_dir and \
resources_db.get('base_year', -1) == base_year:
#all good, we don't need to do anything
return
else:
resources_db.merge(resources)
resources = resources_db
run_manager.add_row_to_history(run_id = run_id,
resources = resources,
status = 'done',
run_name = run_name)
def on_pbnStartModel_clicked(self):
self.diagnostic_go_button.setEnabled(True)
if self.running and not self.paused:
# Take care of pausing a run
success = self.runThread.pause()
if success:
self.paused = True
self.timer.stop()
self.pbnStartModel.setText(QString("Resume simulation run"))
elif self.running and self.paused:
# Need to resume a paused run
success = self.runThread.resume()
if success:
self.paused = False
self.timer.start(1000)
self.pbnStartModel.setText(QString("Pause simulation run"))
elif not self.running:
# this is to work around that results_manager_functions.add_simulation_run function
# isn't able to update existing runs and years information in simulation_runs/run
# node may not be correct
original_start_year, original_end_year = get_years_range_for_simulation_run(self.project, self.run_name)
# #ask for first and last year of the run to be restarted
run_manager = get_run_manager()
years_config = self.config["years"] # = (original_start_year, end_year)
base_year = self.config["base_year"]
dlg_years = InputRestartYearsDialog(self.mainwindow)
dlg_years.lblBaseYear.setText(str(base_year))
if original_end_year is not None:
dlg_years.lblEndYear.setText(str(original_end_year))
proposed_start_year = original_end_year + 1
else:
proposed_start_year = base_year + 1
proposed_end_year = years_config[1] if years_config[1] > proposed_start_year else proposed_start_year + 1
dlg_years.leFirstYear.setText(str(proposed_start_year))
dlg_years.leLastYear.setText(str(proposed_end_year))
if dlg_years.exec_() == QDialog.Rejected:
return
self.start_year = int(dlg_years.leFirstYear.text())
self.end_year = int(dlg_years.leLastYear.text())
assert self.end_year >= self.start_year
self.config["years"] = (self.start_year, self.end_year)
# run_manager._create_seed_dictionary(self.config)
# run_manager.add_row_to_history(self.run_id,
# self.config,
# status='restarted',
# run_name=self.run_name,
# scenario_name=self.scenario_name)
# Update the XML
self.project.update_xml_config()
# self.updateConfigAndGuiForRun()
# # Fire up a new thread and run the model
self._init_run(self.run_name)
self.runThread.setup_restart_run(self.run_id, self.config, self.start_year, self.end_year)
self.runThread.start()
else:
print "Unexpected state in the model run..."
def run(self):
# Run the Eugene model using the XML version of the Eugene configuration.
# This code adapted from opus_core/tools/start_run.py
# statusdir is a temporary directory into which to write a status file
# regarding the progress of the simulation - the progress bar reads this file
statusdir = None
succeeded = False
run_id = None
try:
config = self.xml_config.get_run_configuration(str(
self.modeltorun))
# self.xmltreeobject.toolboxbase.opus_core_xml_configuration.get_run_configuration(str(self.modeltorun))
cache_directory_root = config[
'creating_baseyear_cache_configuration'].cache_directory_root
run_name = self.get_run_name(config=config, run_name=self.run_name)
config['cache_directory'] = os.path.join(cache_directory_root,
run_name)
#insert_auto_generated_cache_directory_if_needed(config)
(self.start_year, self.end_year) = config['years']
run_manager = get_run_manager()
run_manager.setup_new_run(
cache_directory=config['cache_directory'],
configuration=config)
statusdir = run_manager.get_current_cache_directory()
self.statusfile = os.path.join(statusdir, 'status.txt')
self.currentLogfileYear = self.start_year
self.currentLogfileKey = 0
self.config = config
config['status_file_for_gui'] = self.statusfile
self.commandfile = os.path.join(statusdir, 'command.txt')
config['command_file_for_gui'] = self.commandfile
# To test delay in writing the first log file entry...
# time.sleep(5)
self.running = True
self.run_name = run_name
self.run_manager = run_manager
self.startedCallback(run_id=run_id,
run_name=run_name,
scenario_name=self.modeltorun,
run_resources=config)
run_id = run_manager.run_run(config, run_name=run_name)
self.running = False
succeeded = True
except SimulationRunError:
self.running = False
succeeded = False
except:
self.running = False
succeeded = False
errorInfo = formatExceptionInfo(
custom_message='Unexpected Error From Model')
self.errorCallback(errorInfo)
if self.statusfile is not None:
gc.collect()
# adding try/except, windows sometimes has a lock on this file
# when it does, os.remove will cause the simulation to appear
# as if it has crashed, when in fact it simply could not delete
# status.txt
try:
os.remove(self.statusfile)
except:
pass
self.finishedCallback(succeeded, run_name=run_name)
def on_pbnStartModel_clicked(self):
self.diagnostic_go_button.setEnabled(True)
if self.running and not self.paused:
# Take care of pausing a run
success = self.runThread.pause()
if success:
self.paused = True
self.timer.stop()
self.pbnStartModel.setText(QString("Resume simulation run"))
elif self.running and self.paused:
# Need to resume a paused run
success = self.runThread.resume()
if success:
self.paused = False
self.timer.start(1000)
self.pbnStartModel.setText(QString("Pause simulation run"))
elif not self.running:
#this is to work around that results_manager_functions.add_simulation_run function
#isn't able to update existing runs and years information in simulation_runs/run
#node may not be correct
original_start_year, original_end_year = get_years_range_for_simulation_run(self.project,
self.run_name)
# #ask for first and last year of the run to be restarted
run_manager = get_run_manager()
years_config = self.config['years'] # = (original_start_year, end_year)
base_year = self.config['base_year']
dlg_years = InputRestartYearsDialog(self.mainwindow)
dlg_years.lblBaseYear.setText(str(base_year))
if original_end_year is not None:
dlg_years.lblEndYear.setText(str(original_end_year))
proposed_start_year = original_end_year + 1
else:
proposed_start_year = base_year + 1
proposed_end_year = years_config[1] if years_config[1] > proposed_start_year \
else proposed_start_year + 1
dlg_years.leFirstYear.setText(str(proposed_start_year))
dlg_years.leLastYear.setText(str(proposed_end_year))
if dlg_years.exec_() == QDialog.Rejected:
return
self.start_year = int(dlg_years.leFirstYear.text())
self.end_year = int(dlg_years.leLastYear.text())
assert self.end_year >= self.start_year
self.config['years'] = (self.start_year, self.end_year)
# run_manager._create_seed_dictionary(self.config)
# run_manager.add_row_to_history(self.run_id,
# self.config,
# status='restarted',
# run_name=self.run_name,
# scenario_name=self.scenario_name)
# Update the XML
self.project.update_xml_config()
#self.updateConfigAndGuiForRun()
# # Fire up a new thread and run the model
self._init_run(self.run_name)
self.runThread.setup_restart_run(self.run_id,
self.config,
self.start_year,
self.end_year
)
self.runThread.start()
else:
print "Unexpected state in the model run..."
