def cmd(self, cmd, *args, env={}):
_env = {'PATH': '{devtree_dir_abs}/{host}/bin:/usr/bin:/bin:/usr/sbin:/sbin'.format(**self.config),
'LANG': 'C'
}
_env.update(env)
for key in _env:
_env[key] = _env[key].format(**self.config)
args = [cmd] + list(args)
args = [a.format(**self.config) for a in args]
cmd = ' '.join(args) # FIXME: Not 100% accurate
logger.info('{} ENV={}\n'.format(cmd, _env))
with setenv(_env):
r = os.system(cmd)
if r != 0:
raise Exception("Error: {}".format(r))
def _set_pod_env_vars(self, verbose=0):
"""Private method called by :meth:`~shared_diagnostic.Diagnostic.setUp`.
Args:
verbose (:obj:`int`, optional): Logging verbosity level. Default 0.
Returns:
Dict of POD-specific environment variables that were set.
"""
pod_envvars = {}
# location of POD's code
setenv("POD_HOME", self.POD_CODE_DIR, pod_envvars, verbose=verbose)
# POD's observational data
setenv("OBS_DATA", self.POD_OBS_DATA, pod_envvars, verbose=verbose)
# POD's subdir within working directory
setenv("WK_DIR", self.POD_WK_DIR, pod_envvars, verbose=verbose)
# optional POD-specific env vars defined in settings.yml
for key, val in self.pod_env_vars.items():
setenv(key, val, pod_envvars, verbose=verbose)
return pod_envvars
def main():
parser = optparse.OptionParser()
parser.add_option(
"-g",
"--generation",
dest="generation",
type="int",
default=0,
help="generation to spawn (generation 0 is fully random)")
parser.add_option("-f",
"--folder",
dest="folder",
type="string",
default="run",
help="folder to create generation in")
parser.add_option("-n",
"--population",
dest="population",
type="int",
default=20,
help="population size")
parser.add_option("-p",
"--processes",
dest="processes",
type="int",
default=1,
help="number of processes to use")
parser.add_option("-d",
"--dry",
dest="dry",
action="store_true",
help="spawn generation without programming/testing")
options, args = parser.parse_args(sys.argv)
# make base folder for this generation
initial = os.getcwd()
base = ("%s/%s/" + GENERATION) % (initial, options.folder,
options.generation)
util.makepath(base, delete=True)
os.chdir(base)
# get or set environment variable for the scripts path
scripts = util.setenv("EVOLUTION_SCRIPTS", initial)
# get or set environment variable for verilog folder path
verilog = util.setenv("EVOLUTION_VERILOG", initial + "/verilog")
print "generation %d:" % options.generation, options.population, "individuals"
# create seeds from previous generation
if options.generation > 0:
prev = options.generation - 1
print "creating seeds from generation", prev
inputs = glob.glob(
"../%s/%s" %
(GENERATION % prev, re.sub("%.*", "*.csv", INDIVIDUAL)))
if len(inputs) == 0:
print "ERROR: no seeds available in generation", prev
sys.exit(1)
for i in range(options.population):
in1 = random.choice(inputs)
in2 = random.choice(inputs)
out = (INDIVIDUAL + ".seed") % i
util.execute("%s/merge_individuals.py %s %s %s" % \
(scripts, in1, in2, out), redirect="merge.log")
# don't test individuals in the generate script
os.unsetenv("EVOLUTION_RUN")
# run processes
if options.processes > 1:
# create that many processes
print "creating", options.processes, "parallel processes"
pids = list()
for i in range(options.processes):
pid = os.fork()
if pid > 0:
pids.append(pid)
else:
run_process(i, options.processes, options.population, base,
scripts)
sys.exit(0)
# wait for all processes to return
for pid in pids:
os.waitpid(pid, 0)
else:
# just run it directly if there's only one thread
run_process(0, 1, options.population, base, scripts)
outputs = glob.glob(re.sub("%.*", "*.sof", INDIVIDUAL))
print "%d/%d individuals were generated sucessfully" % \
(len(outputs), options.population)
if options.dry:
return
for sof in outputs:
test_individual(sof, 1, "/dev/ttyUSB0", scripts)
def _set_model_env_vars(self, config, verbose=0):
setenv("DATADIR",
self.MODEL_DATA_DIR,
config['envvars'],
verbose=verbose)
setenv("variab_dir",
self.MODEL_WK_DIR,
config['envvars'],
verbose=verbose)
setenv("CASENAME", self.case_name, config['envvars'], verbose=verbose)
setenv("model", self.model_name, config['envvars'], verbose=verbose)
setenv("FIRSTYR", self.firstyr, config['envvars'], verbose=verbose)
setenv("LASTYR", self.lastyr, config['envvars'], verbose=verbose)
translate = util.VariableTranslator()
# todo: set/unset for multiple models
# verify all vars requested by PODs have been set
assert self.convention in translate.field_dict, \
"Variable name translation doesn't recognize {}.".format(self.convention)
for key, val in translate.field_dict[self.convention].items():
setenv(key, val, config['envvars'], verbose=verbose)
def main():
parser = optparse.OptionParser()
parser.add_option("-g", "--generation", dest="generation", type="int",
default=0, help="generation to spawn (generation 0 is fully random)")
parser.add_option("-f", "--folder", dest="folder", type="string",
default="run", help="folder to create generation in")
parser.add_option("-n", "--population", dest="population", type="int",
default=20, help="population size")
parser.add_option("-p", "--processes", dest="processes", type="int",
default=1, help="number of processes to use")
parser.add_option("-d", "--dry", dest="dry", action="store_true",
help="spawn generation without programming/testing")
options, args = parser.parse_args(sys.argv)
# make base folder for this generation
initial = os.getcwd()
base = ("%s/%s/" + GENERATION) % (initial, options.folder, options.generation)
util.makepath(base, delete=True)
os.chdir(base)
# get or set environment variable for the scripts path
scripts = util.setenv("EVOLUTION_SCRIPTS", initial)
# get or set environment variable for verilog folder path
verilog = util.setenv("EVOLUTION_VERILOG", initial + "/verilog")
print "generation %d:" % options.generation, options.population, "individuals"
# create seeds from previous generation
if options.generation > 0:
prev = options.generation - 1
print "creating seeds from generation", prev
inputs = glob.glob("../%s/%s" % (GENERATION % prev, re.sub("%.*", "*.csv", INDIVIDUAL)))
if len(inputs) == 0:
print "ERROR: no seeds available in generation", prev
sys.exit(1)
for i in range(options.population):
in1 = random.choice(inputs)
in2 = random.choice(inputs)
out = (INDIVIDUAL + ".seed") % i
util.execute("%s/merge_individuals.py %s %s %s" % \
(scripts, in1, in2, out), redirect="merge.log")
# don't test individuals in the generate script
os.unsetenv("EVOLUTION_RUN")
# run processes
if options.processes > 1:
# create that many processes
print "creating", options.processes, "parallel processes"
pids = list()
for i in range(options.processes):
pid = os.fork()
if pid > 0:
pids.append(pid)
else:
run_process(i, options.processes, options.population, base, scripts)
sys.exit(0)
# wait for all processes to return
for pid in pids:
os.waitpid(pid, 0)
else:
# just run it directly if there's only one thread
run_process(0, 1, options.population, base, scripts)
outputs = glob.glob(re.sub("%.*", "*.sof", INDIVIDUAL))
print "%d/%d individuals were generated sucessfully" % \
(len(outputs), options.population)
if options.dry:
return
for sof in outputs:
test_individual(sof, 1, "/dev/ttyUSB0", scripts)