def memuse_batch(script_file,
times=[0],
analysis_fn=default_memuse_analysis_fn,
**params):
"""
Similar to run_batch, but analyzes the memory requirement of a simulation at different times.
First, the specified script file will be run using the specified parameters.
Then at each of the specified times, the given analysis_fn (which
calls allsizes_mb() by default) is run. The output is labeled
with the script file, time, and parameters so that results from
different runs can be compared.
"""
import os, re, __main__, topo
from topo.misc.commandline import global_params
# Construct simulation name, etc.
scriptbase = re.sub('.ty$', '', os.path.basename(script_file))
prefix = ""
#prefix += time.strftime("%Y%m%d%H%M") + "_"
prefix += scriptbase
simname = prefix
# Construct parameter-value portion of filename; should do more filtering
for a, val in params.iteritems():
# Special case to give reasonable filenames for lists
valstr = ("_".join([str(i) for i in val])
if isinstance(val, list) else str(val))
prefix += "," + a + "=" + valstr
# Set provided parameter values in main namespace
global_params.set_in_context(**params)
# Run script in main
try:
execfile(script_file, __main__.__dict__) #global_params.context
global_params.check_for_unused_names()
topo.sim.name = simname
# Run each segment, doing the analysis and saving the script state each time
for run_to in times:
topo.sim.run(run_to - topo.sim.time())
analysis_fn(prefix=prefix + ": ")
except:
import traceback, sys
traceback.print_exc(file=sys.stdout)
sys.stderr.write("Warning -- Error detected: execution halted.\n")
def memuse_batch(script_file,times=[0],analysis_fn=default_memuse_analysis_fn,**params):
"""
Similar to run_batch, but analyzes the memory requirement of a simulation at different times.
First, the specified script file will be run using the specified parameters.
Then at each of the specified times, the given analysis_fn (which
calls allsizes_mb() by default) is run. The output is labeled
with the script file, time, and parameters so that results from
different runs can be compared.
"""
import os,re,__main__,topo
from topo.misc.commandline import global_params
# Construct simulation name, etc.
scriptbase= re.sub('.ty$','',os.path.basename(script_file))
prefix = ""
#prefix += time.strftime("%Y%m%d%H%M") + "_"
prefix += scriptbase
simname = prefix
# Construct parameter-value portion of filename; should do more filtering
for a,val in params.iteritems():
# Special case to give reasonable filenames for lists
valstr= ("_".join([str(i) for i in val]) if isinstance(val,list)
else str(val))
prefix += "," + a + "=" + valstr
# Set provided parameter values in main namespace
global_params.set_in_context(**params)
# Run script in main
try:
execfile(script_file,__main__.__dict__) #global_params.context
global_params.check_for_unused_names()
topo.sim.name=simname
# Run each segment, doing the analysis and saving the script state each time
for run_to in times:
topo.sim.run(run_to - topo.sim.time())
analysis_fn(prefix=prefix+": ")
except:
import traceback,sys
traceback.print_exc(file=sys.stdout)
sys.stderr.write("Warning -- Error detected: execution halted.\n")
def __call__(self,script_file,**params_to_override):
p=ParamOverrides(self,params_to_override,allow_extra_keywords=True)
import os
import shutil
# Construct simulation name, etc.
scriptbase= re.sub('.ty$','',os.path.basename(script_file))
prefix = ""
if p.timestamp==(0,0): prefix += time.strftime(p.name_time_format)
else: prefix += time.strftime(p.name_time_format, p.timestamp)
prefix += "_" + scriptbase + "_" + p.tag
simname = prefix
# Construct parameter-value portion of filename; should do more filtering
# CBENHANCEMENT: should provide chance for user to specify a
# function (i.e. make this a function, and have a parameter to
# allow the function to be overridden).
# And sort by name by default? Skip ones that aren't different
# from default, or at least put them at the end?
prefix += p.dirname_params_filter(p.extra_keywords())
# Set provided parameter values in main namespace
from topo.misc.commandline import global_params
global_params.set_in_context(**p.extra_keywords())
# Create output directories
if not os.path.isdir(normalize_path(p['output_directory'])):
try: os.mkdir(normalize_path(p['output_directory']))
except OSError: pass # Catches potential race condition (simultaneous run_batch runs)
dirname = self._truncate(p,p.dirname_prefix+prefix)
normalize_path.prefix = normalize_path(os.path.join(p['output_directory'],dirname))
if os.path.isdir(normalize_path.prefix):
print "Batch run: Warning -- directory already exists!"
print "Run aborted; wait one minute before trying again, or else rename existing directory: \n" + \
normalize_path.prefix
sys.exit(-1)
else:
os.mkdir(normalize_path.prefix)
print "Batch run output will be in " + normalize_path.prefix
if p['vc_info']:
_print_vc_info(simname+".diffs")
hostinfo = "Host: " + " ".join(platform.uname())
topographicalocation = "Topographica: " + os.path.abspath(sys.argv[0])
topolocation = "topo package: " + os.path.abspath(topo.__file__)
scriptlocation = "script: " + os.path.abspath(script_file)
starttime=time.time()
startnote = "Batch run started at %s." % time.strftime("%a %d %b %Y %H:%M:%S +0000",
time.gmtime())
# store a re-runnable copy of the command used to start this batch run
try:
# pipes.quote is undocumented, so I'm not sure which
# versions of python include it (I checked python 2.6 and
# 2.7 on linux; they both have it).
import pipes
quotefn = pipes.quote
except (ImportError,AttributeError):
# command will need a human to insert quotes before it can be re-used
quotefn = lambda x: x
command_used_to_start = string.join([quotefn(arg) for arg in sys.argv])
# CBENHANCEMENT: would be nice to separately write out a
# runnable script that does everything necessary to
# re-generate results (applies diffs etc).
# Shadow stdout to a .out file in the output directory, so that
# print statements will go to both the file and to stdout.
batch_output = open(normalize_path(simname+".out"),'w')
batch_output.write(command_used_to_start+"\n")
sys.stdout = MultiFile(batch_output,sys.stdout)
print
print hostinfo
print topographicalocation
print topolocation
print scriptlocation
print
print startnote
from topo.misc.commandline import auto_import_commands
auto_import_commands()
# Ensure that saved state includes all parameter values
from topo.command import save_script_repr
param.parameterized.script_repr_suppress_defaults=False
# Save a copy of the script file for reference
shutil.copy2(script_file, normalize_path.prefix)
shutil.move(normalize_path(scriptbase+".ty"),
normalize_path(simname+".ty"))
# Default case: times is just a number that scales a standard list of times
times=p['times']
if not isinstance(times,list):
times=[t*times for t in [0,50,100,500,1000,2000,3000,4000,5000,10000]]
# Run script in main
error_count = 0
initial_warning_count = param.parameterized.warning_count
try:
execfile(script_file,__main__.__dict__) #global_params.context
global_params.check_for_unused_names()
if p.save_global_params:
_save_parameters(p.extra_keywords(),simname+".global_params.pickle")
print_sizes()
topo.sim.name=simname
# Run each segment, doing the analysis and saving the script state each time
for run_to in times:
topo.sim.run(run_to - topo.sim.time())
p['analysis_fn']()
save_script_repr()
elapsedtime=time.time()-starttime
param.Parameterized(name="run_batch").message(
"Elapsed real time %02d:%02d." % (int(elapsedtime/60),int(elapsedtime%60)))
if p['snapshot']:
save_snapshot()
except:
error_count+=1
import traceback
traceback.print_exc(file=sys.stdout)
sys.stderr.write("Warning -- Error detected: execution halted.\n")
print "\nBatch run completed at %s." % time.strftime("%a %d %b %Y %H:%M:%S +0000",
time.gmtime())
print "There were %d error(s) and %d warning(s)%s." % \
(error_count,(param.parameterized.warning_count-initial_warning_count),
((" (plus %d warning(s) prior to entering run_batch)"%initial_warning_count
if initial_warning_count>0 else "")))
# restore stdout
sys.stdout = sys.__stdout__
batch_output.close()
def __call__(self, script_file, **params_to_override):
p = ParamOverrides(self, params_to_override, allow_extra_keywords=True)
import os
import shutil
# Construct simulation name, etc.
scriptbase = re.sub('.ty$', '', os.path.basename(script_file))
prefix = ""
if p.timestamp == (0, 0): prefix += time.strftime(p.name_time_format)
else: prefix += time.strftime(p.name_time_format, p.timestamp)
prefix += "_" + scriptbase + "_" + p.tag
simname = prefix
# Construct parameter-value portion of filename; should do more filtering
# CBENHANCEMENT: should provide chance for user to specify a
# function (i.e. make this a function, and have a parameter to
# allow the function to be overridden).
# And sort by name by default? Skip ones that aren't different
# from default, or at least put them at the end?
prefix += p.dirname_params_filter(p.extra_keywords())
# Set provided parameter values in main namespace
from topo.misc.commandline import global_params
global_params.set_in_context(**p.extra_keywords())
# Create output directories
if not os.path.isdir(normalize_path(p['output_directory'])):
try:
os.mkdir(normalize_path(p['output_directory']))
except OSError:
pass # Catches potential race condition (simultaneous run_batch runs)
dirname = self._truncate(p, p.dirname_prefix + prefix)
normalize_path.prefix = normalize_path(
os.path.join(p['output_directory'], dirname))
if os.path.isdir(normalize_path.prefix):
print "Batch run: Warning -- directory already exists!"
print "Run aborted; wait one minute before trying again, or else rename existing directory: \n" + \
normalize_path.prefix
sys.exit(-1)
else:
os.mkdir(normalize_path.prefix)
print "Batch run output will be in " + normalize_path.prefix
if p['vc_info']:
_print_vc_info(simname + ".diffs")
hostinfo = "Host: " + " ".join(platform.uname())
topographicalocation = "Topographica: " + os.path.abspath(sys.argv[0])
topolocation = "topo package: " + os.path.abspath(topo.__file__)
scriptlocation = "script: " + os.path.abspath(script_file)
starttime = time.time()
startnote = "Batch run started at %s." % time.strftime(
"%a %d %b %Y %H:%M:%S +0000", time.gmtime())
# store a re-runnable copy of the command used to start this batch run
try:
# pipes.quote is undocumented, so I'm not sure which
# versions of python include it (I checked python 2.6 and
# 2.7 on linux; they both have it).
import pipes
quotefn = pipes.quote
except (ImportError, AttributeError):
# command will need a human to insert quotes before it can be re-used
quotefn = lambda x: x
command_used_to_start = string.join([quotefn(arg) for arg in sys.argv])
# CBENHANCEMENT: would be nice to separately write out a
# runnable script that does everything necessary to
# re-generate results (applies diffs etc).
# Shadow stdout to a .out file in the output directory, so that
# print statements will go to both the file and to stdout.
batch_output = open(normalize_path(simname + ".out"), 'w')
batch_output.write(command_used_to_start + "\n")
sys.stdout = MultiFile(batch_output, sys.stdout)
print
print hostinfo
print topographicalocation
print topolocation
print scriptlocation
print
print startnote
from topo.misc.commandline import auto_import_commands
auto_import_commands()
# Ensure that saved state includes all parameter values
from topo.command import save_script_repr
param.parameterized.script_repr_suppress_defaults = False
# Save a copy of the script file for reference
shutil.copy2(script_file, normalize_path.prefix)
shutil.move(normalize_path(scriptbase + ".ty"),
normalize_path(simname + ".ty"))
# Default case: times is just a number that scales a standard list of times
times = p['times']
if not isinstance(times, list):
times = [
t * times for t in
[0, 50, 100, 500, 1000, 2000, 3000, 4000, 5000, 10000]
]
# Run script in main
error_count = 0
initial_warning_count = param.parameterized.warning_count
try:
execfile(script_file, __main__.__dict__) #global_params.context
global_params.check_for_unused_names()
if p.save_global_params:
_save_parameters(p.extra_keywords(),
simname + ".global_params.pickle")
print_sizes()
topo.sim.name = simname
# Run each segment, doing the analysis and saving the script state each time
for run_to in times:
topo.sim.run(run_to - topo.sim.time())
p['analysis_fn']()
save_script_repr()
elapsedtime = time.time() - starttime
param.Parameterized(name="run_batch").message(
"Elapsed real time %02d:%02d." %
(int(elapsedtime / 60), int(elapsedtime % 60)))
if p['snapshot']:
save_snapshot()
except:
error_count += 1
import traceback
traceback.print_exc(file=sys.stdout)
sys.stderr.write("Warning -- Error detected: execution halted.\n")
print "\nBatch run completed at %s." % time.strftime(
"%a %d %b %Y %H:%M:%S +0000", time.gmtime())
print "There were %d error(s) and %d warning(s)%s." % \
(error_count,(param.parameterized.warning_count-initial_warning_count),
((" (plus %d warning(s) prior to entering run_batch)"%initial_warning_count
if initial_warning_count>0 else "")))
# restore stdout
sys.stdout = sys.__stdout__
batch_output.close()
