def __init__(self, *args, **kwargs):
super(NEATStrategy, self).__init__([], [])
##############################
# HyperNEAT Parameters
##############################
now = datetime.now()
#self.identifier = ''.join([random.choice('abcdef1234567890') for x in range(8)])
self.identifier = now.strftime('neat_%y%m%d_%H%M%S')
self.executable = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/Hypercube_NEAT'
self.motionFile = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/spiderJointAngles.txt'
#self.motionFile = 'spiderJointAngles.txt'
self.fitnessFile = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/fitness'
self.datFile = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/SpiderRobotExperiment.dat'
self.avgPoints = 4 # Average over this many points
self.junkPoints = 1000
# How many lines to expect from HyperNEAT file
self.expectedLines = self.junkPoints + 12 * 40 * self.avgPoints
#self.motorColumns = [0,1,4,5,2,3,6,7] # Order of motors in HyperNEAT file
self.motorColumns = [0, 1, 4, 5, 2, 3, 6, 7,
8] # Order of motors in HyperNEAT file
self.generationSize = 9
self.initNeatFile = kwargs.get(
'initNeatFile', None) # Pop file to start with, None for restart
self.prefix = 'delme'
self.nextNeatFile = '%s_pop.xml' % self.prefix
self.separateLogInfo = True
#self.proc = sp.Popen((self.executable,
# '-O', 'delme', '-R', '102', '-I', self.datFile),
# stdout=sp.PIPE, stderr=sp.PIPE, stdin=sp.PIPE)
#os.system('%s -O delme -R 102 -I %s' % (self.executable, self.datFile))
self.spawnProc = False
if self.spawnProc:
if self.initNeatFile is None:
self.proc = Process((self.executable, '-O', self.prefix, '-R',
'102', '-I', self.datFile))
else:
print 'Starting with neatfile', self.initNeatFile
self.proc = Process(
(self.executable, '-O', self.prefix, '-R', '102', '-I',
self.datFile, '-X', self.nextNeatFile, '-XG', '1'))
#'%s -O delme -R 102 -I %s' % (self.executable, self.datFile))
self.genId = 0
self.orgId = 0
def __wipe_thread(self, command):
process = Process(command, shell=True)
self.processes.append(process)
re_pct = re.compile(r"(\d+)%", re.M)
while True:
# check to see if process has ended
poll_exitcode = process.wait(os.WNOHANG)
if poll_exitcode != None:
break
# print any new output
stdout = process.read()
m = re_pct.search(stdout)
if m:
pct = int(m.group(1))
self.__progress = pct / 100.0
logger.info("Wipe progress: %d%%" % (self.__progress * 100))
# Wait another 2 seconds before polling
time.sleep(2.0)
stderr = process.readerr()
if poll_exitcode > 0:
self.state = protocol.FAIL
self.__fail_message = (
"Failed while wiping. Return code: %d, Stderr was: %s" %
(poll_exitcode, stderr))
logger.error(self.__fail_message)
return
self.__wipe_done = True
self.__progress = 1.0
logger.info("Wipe finished.")
self.state = protocol.IDLE
def run_command(self, command, seconds=None):
"""
Run a command on the system and keep the user updated on the completeness.
"""
proc = Process(command.strip().split(" "))
self.display_progress(0)
start_time = time.time()
while True:
time.sleep(.1)
sys.stdout.flush()
poll = proc.wait(os.WNOHANG)
if poll != None:
break
line = proc.read()
if "PERCENT COMPLETE:" in line:
self.display_progress(
float(
line.split("PERCENT COMPLETE:")[-1].replace("\n", "")))
if seconds:
current_time = time.time()
if current_time - start_time > seconds:
break
self.display_progress(
float(current_time - start_time) * 100. / seconds)
try:
proc.kill(signal.SIGTERM)
except OSError:
pass
self.display_progress(100)
print
def call_cmd(cmd, timeout=-1, output_filter=None, cwd=None, check_ret=None):
'''
Utility to call a command.
timeout is in seconds.
'''
print("call_cmd: calling " + " ".join(cmd))
p = Process(cmd, cwd=cwd, stderr=subprocess.STDOUT)
launch_time = time.clock()
output = ""
while True:
# check to see if process has ended
ret = p.wait(os.WNOHANG)
# print any new output
o = p.read()
if len(o) > 0:
print("output = %s" % o)
if output_filter:
output_filter(p, o, output)
output += o
time.sleep(1)
if ret is not None:
if check_ret is not None:
assert check_ret == ret
return ret, output
if timeout > 0 and time.clock() - launch_time > timeout:
p.kill(signal.SIGKILL)
if check_ret is not None:
assert check_ret == -1
return -1, output
def __init__(self, args, num=0, **kw):
"""Starts a set of processes, defined by num:
if num is an int:
> 0: that many procs
<= 0: getNumCPUs() - num
elif num is a float:
that percentage of cpus of this sys
Any additional kw args are passed to the initializer for Process().
(These are the same as the inputs to subprocess.Popen())
"""
from threading import Lock
from copy import deepcopy
from Queue import Queue, LifoQueue
from nkthreadutils import spawnWorkers
from nkutils import parseNProcs
self.nprocs = nprocs = parseNProcs(num)
self.args = deepcopy(args) # in case we alter them
# spawn processes and associated locks and working queues
self.procs = [Process(args, **kw) for i in range(nprocs)]
self.proclocks = [Lock() for p in self.procs]
self.working = [LifoQueue() for p in self.procs]
# spawn instance vars to track inputs and results
self.inq = Queue()
self.results = {}
self.resultlock = Lock()
# spawn worker threads
self.inloop = spawnWorkers(1, self.inputloop)[0]
self.outloop = spawnWorkers(1, self.outputloop)[0]
def __init__(self, name, command):
self.name = name
self.command = command
self.process = Process(self.command, env=os.environ)
self.out = ''
self.newOut = ''
self.isRunning = True
self.isTerminating = False
def __shell_exec_thread(self, command, shell_exec_id):
logger.info("Shell execution of: %s" % command)
process = Process(command, shell=True)
# Initiate process
process.wait(os.WNOHANG)
self.processes.append(process)
# Blocking wait
process.wait()
self.shell_exec_results[shell_exec_id] = (process.read(),
process.readerr())
self.state = protocol.IDLE
def __init__(self, parent=None): super(MainWindow, self).__init__(parent) self.setupUi(self) self.piece = ["P", "N", "B", "R", "Q", "K"] self.callList = [] self.timer = QTimer() self.initGui() cmdline = ["./genchess", ""] self.engine = Process(cmdline) self.newGame()
def run_routers(nodes, folder_name, initcost_prefix):
channels = {}
call("pkill {}".format(executable), shell=True)
call("rm -rf ./log", shell=True)
call("mkdir log", shell=True)
for node in nodes:
call("./{binary} {id} {folder}/{prefix}{id} ./log/log{id} &".format(
binary=executable,
id=node,
folder=folder_name,
prefix=initcost_prefix),
shell=True)
time.sleep(0.3)
for node in nodes:
sp = Process("tail -f ./log/log{}".format(node), shell=True)
channels[node] = sp
return channels
def get(self, url):
request = self.request
filename = 'scripts/' + url + '.input'
args = request.arguments
logging.debug('Debugging %s, %s' % (filename, str(args)))
if not os.path.exists(filename):
logging.warning('script does not exist!')
raise tornado.web.HTTPError(404)
if sessions.has_key(url):
logging.debug('Restart ' + url)
sessions[url].terminate()
script = open(filename, 'r')
decoded = base64.b64encode(repr(args))
param = '--param="%s"' % decoded
logging.debug(param)
p = Process(['python', 'trace.py', param], stdin=script)
sessions[url] = p
src = open(filename, 'r').read()
self.render('template.html', source=src, random=random.randint(1, 100000), fname=url)
def __scan_thread(self, commands):
# Execute a number of commands and return their output in the same order.
command_cnt = float(len(commands))
self.__progress = 0.0
for cnt, command in enumerate(commands, start=1):
logger.info("SCAN executing command: %s" % command)
self.__progress = cnt / command_cnt
logger.debug("Progress: %.2f" % self.__progress)
try:
process = Process(
command,
shell=True,
)
process.wait(os.WNOHANG)
self.processes.append(process)
process.wait()
self.scan_results[command] = (process.read(),
process.readerr())
except OSError:
self.scan_results[command] = ("", "Command does not exist")
self.state = protocol.IDLE
def __badblocks_thread(self, command):
process = Process(command, shell=True)
self.processes.append(process)
re_pct = re.compile(r"(\d+)\.\d+%", re.M)
while True:
# check to see if process has ended
poll_exitcode = process.wait(os.WNOHANG)
if poll_exitcode != None:
break
# Search stderr - yes, actually stderr is the pipe that badblocks
# uses :(
stderr = process.readerr()
m = re_pct.search(stderr)
if m:
pct = int(m.group(1))
self.__progress = pct / 100.0
logger.info("Badblocks progress: %d%%" %
(self.__progress * 100))
else:
logger.debug("Could not understand badblocks output: %s" %
stderr)
# Wait another 2 seconds before polling
time.sleep(2.0)
stderr = process.readerr()
if poll_exitcode > 0:
self.state = protocol.FAIL
self.__fail_message = (
"Failed executing badblocks. Return code: %d, Stderr was: %s" %
(poll_exitcode, stderr))
logger.error(self.__fail_message)
return
self.__progress = 1.0
logger.info("Badblocks finished.")
self.__badblocks_done = True
self.state = protocol.IDLE
def makeprocess(app, action):
return Process(["sudo", "control", app, action])
def _getNext(self):
'''Get the next point to try. This reads from the file
self.motionFile'''
#print 'TRYING READ STDOUT'
#stdout = self.proc.stdout.read()
#print 'TRYING READ STDERR'
#stderr = self.proc.stderr.read()
#print 'STDOUT:'
#print stdout
#print 'STDERR:'
#print stderr
if self.orgId == self.generationSize:
print 'Restarting process after %d runs, push enter when ready...' % self.generationSize
raw_input()
#sleep(10)
if self.spawnProc:
print 'Continuing with neatfile', self.nextNeatFile
self.proc = Process(
(self.executable, '-O', self.prefix, '-R', '102', '-I',
self.datFile, '-X', self.nextNeatFile, '-XG', '1'))
self.genId += 1
self.orgId = 0
#print 'On iteration', self.orgId+1
while True:
if self.spawnProc:
out = self.proc.read()
if out != '':
#print 'Got stdout:'
#print out
pass
out = self.proc.readerr()
if out != '':
#print 'Got stderr:'
#print out
pass
try:
ff = open(self.motionFile, 'r')
except IOError:
print 'File does not exist yet'
sleep(1)
continue
lines = ff.readlines()
nLines = len(lines)
if nLines < self.expectedLines:
print ' only %d of %d lines, waiting...' % (
nLines, self.expectedLines)
ff.close()
sleep(.5)
continue
break
self.orgId += 1
for ii, line in enumerate(lines[self.junkPoints:]):
#print 'line', ii, 'is', line
nums = [float(xx) for xx in line.split()]
if ii == 0:
rawPositions = array(nums)
else:
rawPositions = vstack((rawPositions, array(nums)))
# swap and scale rawPositions appropriately
rawPositions = rawPositions.T[ix_(
self.motorColumns)].T # remap to right columns
#print 'First few lines of rawPositions are:'
#for ii in range(10):
# print prettyVec(rawPositions[ii,:], prec=2)
# Average over every self.avgPoints
for ii in range(self.expectedLines / self.avgPoints):
temp = mean(rawPositions[ii:(ii + self.avgPoints), :], 0)
if ii == 0:
positions = temp
else:
positions = vstack((positions, temp))
#print 'First few lines of positions are:'
#for ii in range(10):
# print prettyVec(positions[ii,:], prec=2)
# scale from [-1,1] to [0,1]
positions += 1
positions *= .5
# scale from [0,1] to appropriate ranges
innerIdx = [0, 2, 4, 6]
outerIdx = [1, 3, 5, 7]
centerIdx = [8]
for ii in innerIdx:
positions[:, ii] *= (MAX_INNER - MIN_INNER)
positions[:, ii] += MIN_INNER
for ii in outerIdx:
positions[:, ii] *= (MAX_OUTER - MIN_OUTER)
positions[:, ii] += MIN_OUTER
for ii in centerIdx:
positions[:, ii] *= (MAX_CENTER - MIN_CENTER)
positions[:, ii] += MIN_CENTER
# append a column of 512s for center
#positions = hstack((positions,
# NORM_CENTER * ones((positions.shape[0],1))))
times = linspace(0, 12, positions.shape[0])
# Dump both raw positions and positions to file
thisIdentifier = '%s_%05d_%03d' % (self.identifier, self.genId,
self.orgId)
ff = open('%s_raw' % thisIdentifier, 'w')
writeArray(ff, rawPositions)
ff.close()
ff = open('%s_filt' % thisIdentifier, 'w')
writeArray(ff, positions)
ff.close()
# return function of time
return lambda time: matInterp(time, times, positions), thisIdentifier