def build_database(simdir=None):
"""
Iterate through the folder where simulations are and for each
directory, execute each python file found, adding the results to the
*db* dict.
:param simdir: The main simulation output directory, which holds the
directories of each simulation. If :keyword:None, then it will be
assumed to be the *sims* directory under the project root.
:return (db, simdir): ``db`` is the dict where the keys are the name of the
simulation folders that contain the various runs' output files. The
values of the dict are sub-dictionaries written to Python *'.py'* files
in those folders. ``simdir`` is the directory (either full path or
relative to *simdb.py*.
"""
db = {}
if simdir is None:
(project_path, path_to_src) = util.get_project_and_src_paths('simdb')
simdir = os.path.join(project_path,'sims')
for (dirpath, dirnames, filenames) in os.walk(simdir):
parent_dir = dirpath.rsplit('/')[-1] # Get the tail of the path
for the_file in filenames:
if the_file.endswith('.py'):
local_dict = {}
execfile(os.path.join(dirpath, the_file), globals(), \
local_dict)
if not parent_dir in db:
db[parent_dir] = {}
db[parent_dir].update(local_dict)
return db, simdir
def _initialize(argv):
"""
Load the ``sim_var`` dict and process arguments.
:param argv: List of strings, each of which are arguments that would be
specified on the command line.
"""
local_dict = {}
sim_var.clear()
util.load_sim_var(sim_var)
param_files, cmds = paraminit.parse_args(argv, globals(), local_dict)
if "sim_var" in local_dict:
# Replace default values with those in the local_dict
sim_var.update(local_dict["sim_var"])
if "prj_tmp" in local_dict:
prj_tmp.update(local_dict["prj_tmp"])
# Check that the ``prj_tmp`` and ``sim_var`` dicts are complete and fill in
# anything missing that wasn't already accounted for.
if not "prj_path" in prj_tmp:
(project_path, path_to_src) = util.get_project_and_src_paths()
prj_tmp["prj_path"] = project_path
prj_tmp["src_path"] = path_to_src
if not "sim_dir_id" in sim_var:
util.set_sim_dir_id(prj_tmp, sim_var)
prj_tmp["sim_out"] = os.path.join(prj_tmp["prj_path"], sim_var["sim_path"], sim_var["sim_dir_id"])
def run(argv=None):
"""
Run the program. This does the following:
1. Determines the file path of this module (no matter where in the
directory structure it was launched from) so that it can reference
output directories.
2. Processes the arguments.
3. Runs the simulation.
4. Writes the results.
:param argv: List of strings, each of which are arguments that would be
specified on the command line.
"""
(project_path, path_to_src) = util.get_project_and_src_paths("simrunner")
prj_tmp["prj_path"] = project_path
prj_tmp["src_path"] = path_to_src
try:
_initialize(argv)
run_sim() # Run the simulation
# run_sim_hoc() # Run with hoc. If uncommented, comment out the line
# above.
write_dicts()
except Exception, e:
raise
def run(argv = None):
"""
Run main program. This does the following:
1. Determines the file path of this module (no matter where in the
directory structure it was launched from) so that it can reference
output directories.
2. Checks that all source code has been committed.
3. Processes the arguments.
4. Runs the simulation in a subprocess, which writes the results.
5. If an error was encountered, it reverts to the last committed
changeset.
:param argv: List of strings, each of which are arguments that would be
specified on the command line.
"""
global replace_output, use_tip, use_hg
filtered_args = _preprocess_args(argv)
(project_path, path_to_src) = util.get_project_and_src_paths()
prj_tmp['prj_path'] = project_path
prj_tmp['src_path'] = path_to_src
if use_hg:
hg_obj = hgutil.HGUtil(os.path.join(project_path, path_to_src))
try:
hg_obj.check_status() # Check that we have committed all changes.
except Exception as _ex:
print _ex
return # If there is any error just return.
else:
hg_obj = None
try:
_initialize(hg_obj, filtered_args)
# We may have just reverted sources, so we launch simrunner as a
# subprocess, using the sim_var dict we have just assembled. This
# ensures that modules imported by simrunner will be the correct
# revision.
exec_file = os.path.join(project_path, path_to_src, 'simrunner.py')
child = subprocess.Popen(['python', exec_file, 'sim_var='+str(sim_var)])
child.wait()
if hg_obj:
hg_obj.revert_to_changeset()
except:
if hg_obj:
hg_obj.revert_to_changeset() # Revert to the last changeset
def __init__(self,sim_var):
self.sim_var=sim_var
(project_path, path_to_src) = util.get_project_and_src_paths()
self.project_path=project_path
self.path_to_src=path_to_src
dbpath=os.path.join(self.project_path,sim_var['sim_path'], sim_var['dbname'])
self.conn=db_connect(dbpath) #connection object
self.c = self.conn.cursor()
# Create a simulations table
create_sims_table='create table simulations \
(id INTEGER PRIMARY KEY, simulation_name TEXT,morphology TEXT\
,author TEXT);'
execute_query(self.conn,create_sims_table)
#To do:generate an error if exp_id already exists on the database
if 'exp_id' in sim_var: #Need a check to ensure key is unique! (WIP)
simulation_string='''INSERT INTO simulations(id,\
simulation_name, morphology,author) values (?,?,?,?);'''
values=[sim_var['exp_id'],sim_var['simulation_name'],sim_var['morphology'],\
sim_var['author']]
else:
simulation_string='''INSERT INTO simulations(simulation_name, \
morphology,author) values (?,?,?);'''
values=[sim_var['simulation_name'],sim_var['morphology'],\
sim_var['author']]
execute_query(self.conn,simulation_string,values)
exp_id_cursor=execute_query(self.conn,'SELECT last_insert_rowid();')
self.experiment_id=exp_id_cursor.fetchone()[0]
self.conn.commit()
def run():
"""
Retrieve the records of those simulations that finished in the past
24 hours and draw their traces into one matplotlib figure as a PDF and
PNG file.
"""
(project_path, path_to_src) = util.get_project_and_src_paths()
path_to_sims = os.path.join(project_path, 'sims')
db, simdir = simdb.build_database(path_to_sims) # Build the database
sub_dict = dict(dictdb.filter_dict(db, lambda k, v, p: \
k == 'end_time' and \
len(p) >= 2 and p[-2] == 'sim_info' and \
(datetime.datetime.now() - v).days <= 1))
# Initialize a figure
fig = pyplot.figure()
ax = fig.add_subplot(111)
for (k, v) in sub_dict.iteritems():
# k is the name of the simulation folder in the simdir directory.
simpath = os.path.join(simdir, k)
# Unpickle the vec.p files in the folder
with open(os.path.join(simpath, 'v_vec.p')) as vec_file:
v_vec = pickle.load(vec_file)
with open(os.path.join(simpath, 't_vec.p')) as vec_file:
t_vec = pickle.load(vec_file)
# Draw the results to the figure
ax.plot(t_vec, v_vec)
# Save as a pdf file and a png file.
outdir = path_to_sims = os.path.join(project_path, 'analysis')
fig.savefig(os.path.join(outdir, 'fig1.pdf'))
fig.savefig(os.path.join(outdir, 'fig1.png'))
api
Assuming that a Sphinx makefile exists in the *../doc/* directory from
running ``sphinx-quickstart``, then executing ``make html`` or ``make latex``
should compile the documentation.
AUTHORS:
- THOMAS MCTAVISH (2010-08-01): initial version
"""
__version__ = 0.1
import os
import util
(project_path, path_to_src) = util.get_project_and_src_paths('refbuilder')
# Set the directory where .rst files will go.
build_path=os.path.join(project_path, 'doc', 'source')
# Files to include in the API.
modules = ['main', 'simrunner', 'simrun', 'simdb', 'sqldbutils', 'fig1', \
'refbuilder', 'util']
# Write the api.rst file
filepath = os.path.join(build_path, 'api.rst')
with open(filepath, 'w') as api_file:
api_file.write('.. _api:\n\n')
api_file.write('################\n')
api_file.write('Source Reference\n')
api_file.write('################\n')