def __wipe_thread(self, command):
process = Process(command, shell=True)
self.processes.append(process)
re_pct = re.compile(r"(\d+)%", re.M)
while True:
# check to see if process has ended
poll_exitcode = process.wait(os.WNOHANG)
if poll_exitcode != None:
break
# print any new output
stdout = process.read()
m = re_pct.search(stdout)
if m:
pct = int(m.group(1))
self.__progress = pct / 100.0
logger.info("Wipe progress: %d%%" % (self.__progress*100))
# Wait another 2 seconds before polling
time.sleep(2.0)
stderr = process.readerr()
if poll_exitcode > 0:
self.state = protocol.FAIL
self.__fail_message = ("Failed while wiping. Return code: %d, Stderr was: %s" %
(poll_exitcode, stderr))
logger.error(self.__fail_message)
return
self.__wipe_done = True
self.__progress = 1.0
logger.info("Wipe finished.")
self.state = protocol.IDLE
def run_command(self,command,seconds=None):
"""
Run a command on the system and keep the user updated on the completeness.
"""
proc = Process(command.strip().split(" "))
self.display_progress(0)
start_time = time.time()
while True:
time.sleep(.1)
sys.stdout.flush()
poll = proc.wait(os.WNOHANG)
if poll != None:
break
line = proc.read()
if "PERCENT COMPLETE:" in line:
try:
self.display_progress(float(line.split("PERCENT COMPLETE:")[-1].replace("\n","")))
except ValueError:
pass
if seconds:
current_time = time.time()
if current_time-start_time>seconds:
break
self.display_progress(float(current_time-start_time)*100./seconds)
try:
proc.kill(signal.SIGTERM)
except OSError:
pass
self.display_progress(100)
print
def call_cmd(cmd, timeout=-1, output_filter=None, cwd=None, check_ret=None):
'''
Utility to call a command.
timeout is in seconds.
'''
print("call_cmd: calling " + " ".join(cmd))
p = Process(cmd, cwd=cwd, stderr=subprocess.STDOUT)
launch_time = time.clock()
output = ""
while True:
# check to see if process has ended
ret = p.wait(os.WNOHANG)
# print any new output
o = p.read()
if len(o) > 0:
print("output = %s" % o)
if output_filter:
output_filter(p, o, output)
output += o
time.sleep(1)
if ret is not None:
if check_ret is not None:
assert check_ret == ret
return ret, output
if timeout > 0 and time.clock() - launch_time > timeout:
p.kill(signal.SIGKILL)
if check_ret is not None:
assert check_ret == -1
return -1, output
def __wipe_thread(self, command):
process = Process(command, shell=True)
self.processes.append(process)
re_pct = re.compile(r"(\d+)%", re.M)
while True:
# check to see if process has ended
poll_exitcode = process.wait(os.WNOHANG)
if poll_exitcode != None:
break
# print any new output
stdout = process.read()
m = re_pct.search(stdout)
if m:
pct = int(m.group(1))
self.__progress = pct / 100.0
logger.info("Wipe progress: %d%%" % (self.__progress * 100))
# Wait another 2 seconds before polling
time.sleep(2.0)
stderr = process.readerr()
if poll_exitcode > 0:
self.state = protocol.FAIL
self.__fail_message = (
"Failed while wiping. Return code: %d, Stderr was: %s" %
(poll_exitcode, stderr))
logger.error(self.__fail_message)
return
self.__wipe_done = True
self.__progress = 1.0
logger.info("Wipe finished.")
self.state = protocol.IDLE
def __badblocks_thread(self, command):
process = Process(command, shell=True)
self.processes.append(process)
re_pct = re.compile(r"(\d+)\.\d+%", re.M)
while True:
# check to see if process has ended
poll_exitcode = process.wait(os.WNOHANG)
if poll_exitcode != None:
break
# Search stderr - yes, actually stderr is the pipe that badblocks
# uses :(
stderr = process.readerr()
m = re_pct.search(stderr)
if m:
pct = int(m.group(1))
self.__progress = pct / 100.0
logger.info("Badblocks progress: %d%%" % (self.__progress * 100))
else:
logger.debug("Could not understand badblocks output: %s" % stderr)
# Wait another 2 seconds before polling
time.sleep(2.0)
stderr = process.readerr()
if poll_exitcode > 0:
self.state = protocol.FAIL
self.__fail_message = ("Failed executing badblocks. Return code: %d, Stderr was: %s" %
(poll_exitcode, stderr))
logger.error(self.__fail_message)
return
self.__progress = 1.0
logger.info("Badblocks finished.")
self.__badblocks_done = True
self.state = protocol.IDLE
def __init__(self, *args, **kwargs):
super(NEATStrategy, self).__init__([], [])
##############################
# HyperNEAT Parameters
##############################
now = datetime.now()
#self.identifier = ''.join([random.choice('abcdef1234567890') for x in range(8)])
self.identifier = now.strftime('neat_%y%m%d_%H%M%S')
self.executable = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/Hypercube_NEAT'
self.motionFile = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/spiderJointAngles.txt'
#self.motionFile = 'spiderJointAngles.txt'
self.fitnessFile = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/fitness'
self.datFile = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/SpiderRobotExperiment.dat'
self.avgPoints = 4 # Average over this many points
self.junkPoints = 1000
# How many lines to expect from HyperNEAT file
self.expectedLines = self.junkPoints + 12 * 40 * self.avgPoints
#self.motorColumns = [0,1,4,5,2,3,6,7] # Order of motors in HyperNEAT file
self.motorColumns = [0, 1, 4, 5, 2, 3, 6, 7,
8] # Order of motors in HyperNEAT file
self.generationSize = 9
self.initNeatFile = kwargs.get(
'initNeatFile', None) # Pop file to start with, None for restart
self.prefix = 'delme'
self.nextNeatFile = '%s_pop.xml' % self.prefix
self.separateLogInfo = True
#self.proc = sp.Popen((self.executable,
# '-O', 'delme', '-R', '102', '-I', self.datFile),
# stdout=sp.PIPE, stderr=sp.PIPE, stdin=sp.PIPE)
#os.system('%s -O delme -R 102 -I %s' % (self.executable, self.datFile))
self.spawnProc = False
if self.spawnProc:
if self.initNeatFile is None:
self.proc = Process((self.executable, '-O', self.prefix, '-R',
'102', '-I', self.datFile))
else:
print 'Starting with neatfile', self.initNeatFile
self.proc = Process(
(self.executable, '-O', self.prefix, '-R', '102', '-I',
self.datFile, '-X', self.nextNeatFile, '-XG', '1'))
#'%s -O delme -R 102 -I %s' % (self.executable, self.datFile))
self.genId = 0
self.orgId = 0
def __init__(self, *args, **kwargs):
super(NEATStrategy, self).__init__([], [])
##############################
# HyperNEAT Parameters
##############################
now = datetime.now()
#self.identifier = ''.join([random.choice('abcdef1234567890') for x in range(8)])
self.identifier = now.strftime('neat_%y%m%d_%H%M%S')
self.executable = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/Hypercube_NEAT'
self.motionFile = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/spiderJointAngles.txt'
#self.motionFile = 'spiderJointAngles.txt'
self.fitnessFile = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/fitness'
self.datFile = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/SpiderRobotExperiment.dat'
self.avgPoints = 4 # Average over this many points
self.junkPoints = 1000
# How many lines to expect from HyperNEAT file
self.expectedLines = self.junkPoints + 12 * 40 * self.avgPoints
#self.motorColumns = [0,1,4,5,2,3,6,7] # Order of motors in HyperNEAT file
self.motorColumns = [0,1,4,5,2,3,6,7,8] # Order of motors in HyperNEAT file
self.generationSize = 9
self.initNeatFile = kwargs.get('initNeatFile', None) # Pop file to start with, None for restart
self.prefix = 'delme'
self.nextNeatFile = '%s_pop.xml' % self.prefix
self.separateLogInfo = True
#self.proc = sp.Popen((self.executable,
# '-O', 'delme', '-R', '102', '-I', self.datFile),
# stdout=sp.PIPE, stderr=sp.PIPE, stdin=sp.PIPE)
#os.system('%s -O delme -R 102 -I %s' % (self.executable, self.datFile))
self.spawnProc = False
if self.spawnProc:
if self.initNeatFile is None:
self.proc = Process((self.executable,
'-O', self.prefix, '-R', '102', '-I',
self.datFile))
else:
print 'Starting with neatfile', self.initNeatFile
self.proc = Process((self.executable,
'-O', self.prefix, '-R', '102', '-I',
self.datFile, '-X', self.nextNeatFile, '-XG', '1'))
#'%s -O delme -R 102 -I %s' % (self.executable, self.datFile))
self.genId = 0
self.orgId = 0
def __init__(self, parent=None): super(MainWindow, self).__init__(parent) self.setupUi(self) self.piece = ["P", "N", "B", "R", "Q", "K"] self.callList = [] self.timer = QTimer() self.initGui() cmdline = ["./genchess", ""] self.engine = Process(cmdline) self.newGame()
def main(args=None):
timer = Timer(formatstr='{elapsed:3} {el_task:5.1f} ')
outf = sys.stdout
outf.write( timer.format('{elapsed:3} - run_tests.py\n') )
if args is None:
args = sys.argv[1:]
print args
boringp = re.compile(
'(' +
'|'.join(
'Trying Adding Checking Creat Destroy Install Loading Process Running'
.split()) +
'|\\+'
'|No \S+ fixture'
'|No fixtures found'
'|No custom'
'|.+/bin/psql -d test_geodelic'
# XX: weird:
'|rience model$'
'|.+geobuilder/field_att_mergers.sql'
')').match
warnpat = re.compile(
'^(.+\) \.\.\. )WARN.+$',
)
# "server.apps.geopoi.tests.t_banner.TestGeodelicImageBanner"
# => 'geopoi', 'TestGeodelicImageBanner'
simplifypat = re.compile('\S+\.apps\.(\S+)\.(\S+)\)')
myProc = Process(get_cmd(args),
shell=True,
)
outf.write( timer.format() + ' bin/run_tests\n' )
timer.sync()
for line in lineiter(blockiter(myProc)):
if boringp(line):
continue
m = warnpat.match(line)
if m:
line = m.group(1)
m = simplifypat.search(line)
if m:
line = re.sub(m.group(0), '%s.%s' % (m.group(1), m.group(2)), line)
if ' ... ' in line: # startswith('test'):
outf.write( timer.format() )
outf.write(line)
outf.write( timer.format('{elapsed:03}s end\n') )
sys.exit( myProc.wait() / 256 ) # XX
def __init__(self, args, num=0, **kw):
"""Starts a set of processes, defined by num:
if num is an int:
> 0: that many procs
<= 0: getNumCPUs() - num
elif num is a float:
that percentage of cpus of this sys
Any additional kw args are passed to the initializer for Process().
(These are the same as the inputs to subprocess.Popen())
"""
from threading import Lock
from copy import deepcopy
from Queue import Queue, LifoQueue
from nkthreadutils import spawnWorkers
from nkutils import parseNProcs
self.nprocs = nprocs = parseNProcs(num)
self.args = deepcopy(args) # in case we alter them
# spawn processes and associated locks and working queues
self.procs = [Process(args, **kw) for i in range(nprocs)]
self.proclocks = [Lock() for p in self.procs]
self.working = [LifoQueue() for p in self.procs]
# spawn instance vars to track inputs and results
self.inq = Queue()
self.results = {}
self.resultlock = Lock()
# spawn worker threads
self.inloop = spawnWorkers(1, self.inputloop)[0]
self.outloop = spawnWorkers(1, self.outputloop)[0]
def __init__(self, encrypt_or_decrypt, iterable = None, user = None, passphrase = None, test_branch = False):
self.test_branch = test_branch
print 'initing gpg'
if(encrypt_or_decrypt[0] == 'e'):
#print "user = "******""
self.gpg_closed = False
self.padded = False
self.first_read = True
else:
#cmd = ['gpg', '-d', '--batch', '--passphrase-fd', '0']
#cmd = ['gpg', '--no-tty', '--homedir', '/etc/swift/gnupg', '-d', '--batch', '--passphrase-fd', '0']
#cmd = ['tr', 'd', 'D']
#cmd = 'gpg -d --batch --passphrase-fd 0'
cmd = 'cat'
#cmd = '/home/amedeiro/bin/python MyCat.py'
#self.str_iter = stringIterate(self.buffer, read_size)
self.p = Process(cmd, shell=True, bufsize = 2)
# TODO send stderr output to the logs
#self.p = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE, bufsize = 1024)
#self.p = Popen(cmd, stdin=PIPE, stdout=PIPE, bufsize = 4096)
if(encrypt_or_decrypt[0] == 'd'):
self.p.write('test-swift' + '\n')
self.my_buffer = ''
def __init__(self, name, command):
self.name = name
self.command = command
self.process = Process(self.command, env=os.environ)
self.out = ''
self.newOut = ''
self.isRunning = True
self.isTerminating = False
def __shell_exec_thread(self, command, shell_exec_id):
logger.info("Shell execution of: %s" % command)
process = Process(command, shell=True)
# Initiate process
process.wait(os.WNOHANG)
self.processes.append(process)
# Blocking wait
process.wait()
self.shell_exec_results[shell_exec_id] = (process.read(),
process.readerr())
self.state = protocol.IDLE
def __shell_exec_thread(self, command, shell_exec_id):
logger.info("Shell execution of: %s" % command)
process = Process(command, shell=True)
# Initiate process
process.wait(os.WNOHANG)
self.processes.append(process)
# Blocking wait
process.wait()
self.shell_exec_results[shell_exec_id] = (process.read(), process.readerr())
self.state = protocol.IDLE
def run_command(self, command, seconds=None):
"""
Run a command on the system and keep the user updated on the completeness.
"""
proc = Process(command.strip().split(" "))
self.display_progress(0)
start_time = time.time()
while True:
time.sleep(.1)
sys.stdout.flush()
poll = proc.wait(os.WNOHANG)
if poll != None:
break
line = proc.read()
if "PERCENT COMPLETE:" in line:
self.display_progress(
float(
line.split("PERCENT COMPLETE:")[-1].replace("\n", "")))
if seconds:
current_time = time.time()
if current_time - start_time > seconds:
break
self.display_progress(
float(current_time - start_time) * 100. / seconds)
try:
proc.kill(signal.SIGTERM)
except OSError:
pass
self.display_progress(100)
print
def call_cmd(cmd, timeout=-1, output_filter=None, cwd=None, check_ret=None):
'''
Utility to call a command.
timeout is in seconds.
'''
print("call_cmd: calling " + " ".join(cmd))
p = Process(cmd, cwd=cwd, stderr=subprocess.STDOUT)
launch_time = time.clock()
output = ""
while True:
# check to see if process has ended
ret = p.wait(os.WNOHANG)
# print any new output
o = p.read()
if len(o) > 0:
print("output = %s" % o)
if output_filter:
output_filter(p, o, output)
output += o
time.sleep(1)
if ret is not None:
if check_ret is not None:
assert check_ret == ret
return ret, output
if timeout > 0 and time.clock() - launch_time > timeout:
p.kill(signal.SIGKILL)
if check_ret is not None:
assert check_ret == -1
return -1, output
def __badblocks_thread(self, command):
process = Process(command, shell=True)
self.processes.append(process)
re_pct = re.compile(r"(\d+)\.\d+%", re.M)
while True:
# check to see if process has ended
poll_exitcode = process.wait(os.WNOHANG)
if poll_exitcode != None:
break
# Search stderr - yes, actually stderr is the pipe that badblocks
# uses :(
stderr = process.readerr()
m = re_pct.search(stderr)
if m:
pct = int(m.group(1))
self.__progress = pct / 100.0
logger.info("Badblocks progress: %d%%" %
(self.__progress * 100))
else:
logger.debug("Could not understand badblocks output: %s" %
stderr)
# Wait another 2 seconds before polling
time.sleep(2.0)
stderr = process.readerr()
if poll_exitcode > 0:
self.state = protocol.FAIL
self.__fail_message = (
"Failed executing badblocks. Return code: %d, Stderr was: %s" %
(poll_exitcode, stderr))
logger.error(self.__fail_message)
return
self.__progress = 1.0
logger.info("Badblocks finished.")
self.__badblocks_done = True
self.state = protocol.IDLE
def run_routers(nodes, folder_name, initcost_prefix):
channels = {}
call("pkill {}".format(executable), shell=True)
call("rm -rf ./log", shell=True)
call("mkdir log", shell=True)
for node in nodes:
call("./{binary} {id} {folder}/{prefix}{id} ./log/log{id} &".format(
binary=executable,
id=node,
folder=folder_name,
prefix=initcost_prefix),
shell=True)
time.sleep(0.3)
for node in nodes:
sp = Process("tail -f ./log/log{}".format(node), shell=True)
channels[node] = sp
return channels
def get(self, url):
request = self.request
filename = 'scripts/' + url + '.input'
args = request.arguments
logging.debug('Debugging %s, %s' % (filename, str(args)))
if not os.path.exists(filename):
logging.warning('script does not exist!')
raise tornado.web.HTTPError(404)
if sessions.has_key(url):
logging.debug('Restart ' + url)
sessions[url].terminate()
script = open(filename, 'r')
decoded = base64.b64encode(repr(args))
param = '--param="%s"' % decoded
logging.debug(param)
p = Process(['python', 'trace.py', param], stdin=script)
sessions[url] = p
src = open(filename, 'r').read()
self.render('template.html', source=src, random=random.randint(1, 100000), fname=url)
def __scan_thread(self, commands):
# Execute a number of commands and return their output in the same order.
command_cnt = float(len(commands))
self.__progress = 0.0
for cnt, command in enumerate(commands, start=1):
logger.info("SCAN executing command: %s" % command)
self.__progress = cnt / command_cnt
logger.debug("Progress: %.2f" % self.__progress)
try:
process = Process(command, shell=True,)
process.wait(os.WNOHANG)
self.processes.append(process)
process.wait()
self.scan_results[command] = (process.read(), process.readerr())
except OSError:
self.scan_results[command] = ("", "Command does not exist")
self.state = protocol.IDLE
def __scan_thread(self, commands):
# Execute a number of commands and return their output in the same order.
command_cnt = float(len(commands))
self.__progress = 0.0
for cnt, command in enumerate(commands, start=1):
logger.info("SCAN executing command: %s" % command)
self.__progress = cnt / command_cnt
logger.debug("Progress: %.2f" % self.__progress)
try:
process = Process(
command,
shell=True,
)
process.wait(os.WNOHANG)
self.processes.append(process)
process.wait()
self.scan_results[command] = (process.read(),
process.readerr())
except OSError:
self.scan_results[command] = ("", "Command does not exist")
self.state = protocol.IDLE
def _getNext(self):
'''Get the next point to try. This reads from the file
self.motionFile'''
#print 'TRYING READ STDOUT'
#stdout = self.proc.stdout.read()
#print 'TRYING READ STDERR'
#stderr = self.proc.stderr.read()
#print 'STDOUT:'
#print stdout
#print 'STDERR:'
#print stderr
if self.orgId == self.generationSize:
print 'Restarting process after %d runs, push enter when ready...' % self.generationSize
raw_input()
#sleep(10)
if self.spawnProc:
print 'Continuing with neatfile', self.nextNeatFile
self.proc = Process(
(self.executable, '-O', self.prefix, '-R', '102', '-I',
self.datFile, '-X', self.nextNeatFile, '-XG', '1'))
self.genId += 1
self.orgId = 0
#print 'On iteration', self.orgId+1
while True:
if self.spawnProc:
out = self.proc.read()
if out != '':
#print 'Got stdout:'
#print out
pass
out = self.proc.readerr()
if out != '':
#print 'Got stderr:'
#print out
pass
try:
ff = open(self.motionFile, 'r')
except IOError:
print 'File does not exist yet'
sleep(1)
continue
lines = ff.readlines()
nLines = len(lines)
if nLines < self.expectedLines:
print ' only %d of %d lines, waiting...' % (
nLines, self.expectedLines)
ff.close()
sleep(.5)
continue
break
self.orgId += 1
for ii, line in enumerate(lines[self.junkPoints:]):
#print 'line', ii, 'is', line
nums = [float(xx) for xx in line.split()]
if ii == 0:
rawPositions = array(nums)
else:
rawPositions = vstack((rawPositions, array(nums)))
# swap and scale rawPositions appropriately
rawPositions = rawPositions.T[ix_(
self.motorColumns)].T # remap to right columns
#print 'First few lines of rawPositions are:'
#for ii in range(10):
# print prettyVec(rawPositions[ii,:], prec=2)
# Average over every self.avgPoints
for ii in range(self.expectedLines / self.avgPoints):
temp = mean(rawPositions[ii:(ii + self.avgPoints), :], 0)
if ii == 0:
positions = temp
else:
positions = vstack((positions, temp))
#print 'First few lines of positions are:'
#for ii in range(10):
# print prettyVec(positions[ii,:], prec=2)
# scale from [-1,1] to [0,1]
positions += 1
positions *= .5
# scale from [0,1] to appropriate ranges
innerIdx = [0, 2, 4, 6]
outerIdx = [1, 3, 5, 7]
centerIdx = [8]
for ii in innerIdx:
positions[:, ii] *= (MAX_INNER - MIN_INNER)
positions[:, ii] += MIN_INNER
for ii in outerIdx:
positions[:, ii] *= (MAX_OUTER - MIN_OUTER)
positions[:, ii] += MIN_OUTER
for ii in centerIdx:
positions[:, ii] *= (MAX_CENTER - MIN_CENTER)
positions[:, ii] += MIN_CENTER
# append a column of 512s for center
#positions = hstack((positions,
# NORM_CENTER * ones((positions.shape[0],1))))
times = linspace(0, 12, positions.shape[0])
# Dump both raw positions and positions to file
thisIdentifier = '%s_%05d_%03d' % (self.identifier, self.genId,
self.orgId)
ff = open('%s_raw' % thisIdentifier, 'w')
writeArray(ff, rawPositions)
ff.close()
ff = open('%s_filt' % thisIdentifier, 'w')
writeArray(ff, positions)
ff.close()
# return function of time
return lambda time: matInterp(time, times, positions), thisIdentifier
def _getNext(self):
'''Get the next point to try. This reads from the file
self.motionFile'''
#print 'TRYING READ STDOUT'
#stdout = self.proc.stdout.read()
#print 'TRYING READ STDERR'
#stderr = self.proc.stderr.read()
#print 'STDOUT:'
#print stdout
#print 'STDERR:'
#print stderr
if self.orgId == self.generationSize:
# print 'Restarting process after %d runs, push enter when ready...' % self.generationSize
if self.restartHN:
print 'Restarting process after %d runs' % self.generationSize
print 'waiting for exit code...'
code = self.proc.wait()
print 'got exit code: %d' % code
# raw_input()
#sleep(10)
if self.spawnProc:
print 'Continuing with neatfile', self.nextNeatFile
self.proc = Process((self.executable,
'-O', self.prefix, '-R', '102', '-I',
self.datFile, '-X', self.nextNeatFile, '-XG', '1'))
self.genId += 1
self.orgId = 0
#print 'On iteration', self.orgId+1
while True:
if self.spawnProc:
out = self.proc.read()
if out != '':
#print 'Got stdout:'
#print out
pass
out = self.proc.readerr()
if out != '':
#print 'Got stderr:'
#print out
pass
try:
ff = open(self.motionFile, 'r')
except IOError:
print 'File does not exist yet'
sleep(1)
if self.proc.wait(os.WNOHANG) is None:
continue
else:
raise Exception('HN died')
lines = ff.readlines()
nLines = len(lines)
if nLines < self.expectedLines:
print ' only %d of %d lines, waiting...' % (nLines, self.expectedLines)
ff.close()
sleep(.5)
continue
break
self.orgId += 1
for ii,line in enumerate(lines[self.junkPoints:]):
#print 'line', ii, 'is', line
nums = [float(xx) for xx in line.split()]
if ii == 0:
rawPositions = array(nums)
else:
rawPositions = vstack((rawPositions, array(nums)))
# swap and scale rawPositions appropriately
rawPositions = rawPositions.T[ix_(self.motorColumns)].T # remap to right columns
#print 'First few lines of rawPositions are:'
#for ii in range(10):
# print prettyVec(rawPositions[ii,:], prec=2)
# Average over every self.avgPoints
for ii in range((self.expectedLines-self.junkPoints)/self.avgPoints):
temp = mean(rawPositions[ii*self.avgPoints:(ii+1)*self.avgPoints,:], 0)
if ii == 0:
positions = temp
else:
positions = vstack((positions, temp))
#print 'First few lines of positions are:'
#for ii in range(10):
# print prettyVec(positions[ii,:], prec=2)
# scale from [-1,1] to [0,1]
positions += 1
positions *= .5
# scale from [0,1] to appropriate ranges
innerIdx = [0, 2, 4, 6]
outerIdx = [1, 3, 5, 7]
centerIdx = [8]
for ii in innerIdx:
positions[:,ii] *= (MAX_INNER - MIN_INNER)
positions[:,ii] += MIN_INNER
for ii in outerIdx:
positions[:,ii] *= (MAX_OUTER - MIN_OUTER)
positions[:,ii] += MIN_OUTER
for ii in centerIdx:
positions[:,ii] *= (MAX_CENTER - MIN_CENTER)
positions[:,ii] += MIN_CENTER
# append a column of 512s for center
#positions = hstack((positions,
# NORM_CENTER * ones((positions.shape[0],1))))
times = linspace(0,12,positions.shape[0])
# Dump both raw positions and positions to file
thisIdentifier = '%s_%05d_%03d' % (self.identifier, self.genId, self.orgId)
ff = open('%s_raw' % thisIdentifier, 'w')
writeArray(ff, rawPositions)
ff.close()
ff = open('%s_filt' % thisIdentifier, 'w')
writeArray(ff, positions)
ff.close()
# return function of time
return lambda time: matInterp(time, times, positions), thisIdentifier
class NEATStrategy(OneStepStrategy):
'''
A strategy that calls a NEAT executable to determine a gait
'''
def __init__(self, *args, **kwargs):
super(NEATStrategy, self).__init__([], [])
##############################
# HyperNEAT Parameters
##############################
now = datetime.now()
#self.identifier = ''.join([random.choice('abcdef1234567890') for x in range(8)])
self.identifier = now.strftime('neat_%y%m%d_%H%M%S')
# self.executable = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/Hypercube_NEAT'
self.executable = '/home/sean/hyperneat/HyperNEAT_v2_5/out/Hypercube_NEAT'
# self.motionFile = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/spiderJointAngles.txt'
self.motionFile = '/home/sean/simtest/spiderJointAngles.txt'
# self.fitnessFile = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/fitness'
self.fitnessFile = '/home/sean/simtest/fitness'
# self.datFile = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/SpiderRobotExperiment.dat'
self.datFile = '/home/sean/hyperneat/HyperNEAT_v2_5/out/SpiderRobotExperiment.dat'
self.avgPoints = 4 # Average over this many points
self.junkPoints = 1000
# How many lines to expect from HyperNEAT file
self.expectedLines = self.junkPoints + 12 * 40 * self.avgPoints
#self.motorColumns = [0,1,4,5,2,3,6,7] # Order of motors in HyperNEAT file
self.motorColumns = [0,1,4,5,2,3,6,7,8] # Order of motors in HyperNEAT file
self.generationSize = 200
self.initNeatFile = kwargs.get('initNeatFile', None) # Pop file to start with, None for restart
self.prefix = 'delme'
# self.nextNeatFile = '%s_pop.xml' % self.prefix
self.nextNeatFile = '/home/sean/simtest/delme_pop.xml'
self.separateLogInfo = True
self.restartHN = kwargs.get('restartHN', True)
# stdout=sp.PIPE, stderr=sp.PIPE, stdin=sp.PIPE)
#os.system('%s -O delme -R 102 -I %s' % (self.executable, self.datFile))
self.spawnProc = True
if self.spawnProc:
if self.initNeatFile is None:
hnout = open('hyperneat.out', 'w')
hnerr = open('hyperneat.err', 'w')
self.proc = Process((self.executable,
'-O', self.prefix, '-R', '102', '-I',
self.datFile),
#stdout = sys.stdout,
#stderr = sys.stderr,
stdout = hnout,
stderr = hnerr,
)
else:
print 'Starting with neatfile', self.initNeatFile
self.proc = Process((self.executable,
'-O', self.prefix, '-R', '102', '-I',
self.datFile, '-X', self.nextNeatFile, '-XG', '1'))
#'%s -O delme -R 102 -I %s' % (self.executable, self.datFile))
self.genId = 0
self.orgId = 0
def __del__(self):
if self.spawnProc:
print 'Waiting for %s to exit...' % self.executable,
code = self.proc.wait()
print 'done.'
print code
def _getNext(self):
'''Get the next point to try. This reads from the file
self.motionFile'''
#print 'TRYING READ STDOUT'
#stdout = self.proc.stdout.read()
#print 'TRYING READ STDERR'
#stderr = self.proc.stderr.read()
#print 'STDOUT:'
#print stdout
#print 'STDERR:'
#print stderr
if self.orgId == self.generationSize:
# print 'Restarting process after %d runs, push enter when ready...' % self.generationSize
if self.restartHN:
print 'Restarting process after %d runs' % self.generationSize
print 'waiting for exit code...'
code = self.proc.wait()
print 'got exit code: %d' % code
# raw_input()
#sleep(10)
if self.spawnProc:
print 'Continuing with neatfile', self.nextNeatFile
self.proc = Process((self.executable,
'-O', self.prefix, '-R', '102', '-I',
self.datFile, '-X', self.nextNeatFile, '-XG', '1'))
self.genId += 1
self.orgId = 0
#print 'On iteration', self.orgId+1
while True:
if self.spawnProc:
out = self.proc.read()
if out != '':
#print 'Got stdout:'
#print out
pass
out = self.proc.readerr()
if out != '':
#print 'Got stderr:'
#print out
pass
try:
ff = open(self.motionFile, 'r')
except IOError:
print 'File does not exist yet'
sleep(1)
if self.proc.wait(os.WNOHANG) is None:
continue
else:
raise Exception('HN died')
lines = ff.readlines()
nLines = len(lines)
if nLines < self.expectedLines:
print ' only %d of %d lines, waiting...' % (nLines, self.expectedLines)
ff.close()
sleep(.5)
continue
break
self.orgId += 1
for ii,line in enumerate(lines[self.junkPoints:]):
#print 'line', ii, 'is', line
nums = [float(xx) for xx in line.split()]
if ii == 0:
rawPositions = array(nums)
else:
rawPositions = vstack((rawPositions, array(nums)))
# swap and scale rawPositions appropriately
rawPositions = rawPositions.T[ix_(self.motorColumns)].T # remap to right columns
#print 'First few lines of rawPositions are:'
#for ii in range(10):
# print prettyVec(rawPositions[ii,:], prec=2)
# Average over every self.avgPoints
for ii in range((self.expectedLines-self.junkPoints)/self.avgPoints):
temp = mean(rawPositions[ii*self.avgPoints:(ii+1)*self.avgPoints,:], 0)
if ii == 0:
positions = temp
else:
positions = vstack((positions, temp))
#print 'First few lines of positions are:'
#for ii in range(10):
# print prettyVec(positions[ii,:], prec=2)
# scale from [-1,1] to [0,1]
positions += 1
positions *= .5
# scale from [0,1] to appropriate ranges
innerIdx = [0, 2, 4, 6]
outerIdx = [1, 3, 5, 7]
centerIdx = [8]
for ii in innerIdx:
positions[:,ii] *= (MAX_INNER - MIN_INNER)
positions[:,ii] += MIN_INNER
for ii in outerIdx:
positions[:,ii] *= (MAX_OUTER - MIN_OUTER)
positions[:,ii] += MIN_OUTER
for ii in centerIdx:
positions[:,ii] *= (MAX_CENTER - MIN_CENTER)
positions[:,ii] += MIN_CENTER
# append a column of 512s for center
#positions = hstack((positions,
# NORM_CENTER * ones((positions.shape[0],1))))
times = linspace(0,12,positions.shape[0])
# Dump both raw positions and positions to file
thisIdentifier = '%s_%05d_%03d' % (self.identifier, self.genId, self.orgId)
ff = open('%s_raw' % thisIdentifier, 'w')
writeArray(ff, rawPositions)
ff.close()
ff = open('%s_filt' % thisIdentifier, 'w')
writeArray(ff, positions)
ff.close()
# return function of time
return lambda time: matInterp(time, times, positions), thisIdentifier
def updateResults(self, dist):
'''This must be called for the last point that was handed out!
This communicates back to the running subprocess.
'''
dist = float(dist)
# MAKE SURE TO CALL super().updateResults!
super(NEATStrategy, self).updateResults(dist)
# Send fitness to process
#out,err = proc.communicate('%f\n' % dist)
print 'Deleting old motion file %s' % self.motionFile
os.remove(self.motionFile)
print 'Sending to process: %f' % dist,
if self.spawnProc:
print 'via process'
self.proc.write('%f\n' % dist)
else:
print 'via file'
# Instead, write this to a file
ff = open('.fitness', 'w')
ff.write('%s\n' % dist)
ff.close()
os.rename('.fitness', self.fitnessFile)
#print 'TRYING READ STDOUT'
#stdout = self.proc.stdout.read()
#print 'TRYING READ STDOUT'
#stderr = self.proc.stderr.read()
#
#print 'Got stdout:'
#print stdout
#print 'Got stderr:'
#print stderr
if self.spawnProc:
out = self.proc.read()
if out != '':
#print 'Got stdout:'
#print out
pass
out = self.proc.readerr()
if out != '':
#print 'Got stderr:'
#print out
pass
def logHeader(self):
return '# NEATStrategy starting, identifier %s\n' % self.identifier
class GPGEncryption():
def __init__(self, encrypt_or_decrypt, iterable = None, user = None, passphrase = None, test_branch = False):
self.test_branch = test_branch
print 'initing gpg'
if(encrypt_or_decrypt[0] == 'e'):
#print "user = "******""
self.gpg_closed = False
self.padded = False
self.first_read = True
else:
#cmd = ['gpg', '-d', '--batch', '--passphrase-fd', '0']
#cmd = ['gpg', '--no-tty', '--homedir', '/etc/swift/gnupg', '-d', '--batch', '--passphrase-fd', '0']
#cmd = ['tr', 'd', 'D']
#cmd = 'gpg -d --batch --passphrase-fd 0'
cmd = 'cat'
#cmd = '/home/amedeiro/bin/python MyCat.py'
#self.str_iter = stringIterate(self.buffer, read_size)
self.p = Process(cmd, shell=True, bufsize = 2)
# TODO send stderr output to the logs
#self.p = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE, bufsize = 1024)
#self.p = Popen(cmd, stdin=PIPE, stdout=PIPE, bufsize = 4096)
if(encrypt_or_decrypt[0] == 'd'):
self.p.write('test-swift' + '\n')
self.my_buffer = ''
def print_errors(self):
print self.p.readerr()
def stringIterate(self, text, chunk_size):
index = 0
while index < len(text):
yield text[index : index + chunk_size]
index += chunk_size
def readFromStream():
chunk = self.stream_iter_read(read_size)
while chunk:
self.p.stdin.write(chunk)
chunk = self.stream_iter_read(read_size)
self.p.stdin.close()
def read(self, read_size):
def padder(text, div, pad = chr(0)):
print 'padder reading ' + text
size = (div - len(text) % div) + len(text)
diff = size - len(text)
text += pad * diff
return text
if self.first_read:
self.first_read = False
t = Thread(target = readFromStream)
t.start()
return self.p.stdout.read(read_size)
def digest(self, chunk):
print "digesting chunk = " + chunk
self.p.write(chunk)
def dump_buffer(self):
buf = self.p.read()
print 'dump buffer returning: ' + buf
return buf
def close_and_dump(self, timeout = .2):
print 'close and dump called'
self.p.closeinput()
time.sleep(timeout)
#self.p.closeinput()
buf = ''
#as long as there was a non blank string digested before this is called,
#we can wait for the process to dump its last output... Might be a problem
#if output dumps
#poll = self.p.wait(os.WNOHANG)
print 'entering close and dump loop'
count = 0
while buf == '' and count < 100:
buf = self.dump_buffer()
time.sleep(.01)
print self.p.readerr()
poll = self.p.wait(os.WNOHANG)
count = count + 1
print 'leaving close and dump loop'
poll = self.p.wait(os.WNOHANG)
print poll
return buf
def _getNext(self):
"""Get the next point to try. This reads from the file
self.motionFile"""
# print 'TRYING READ STDOUT'
# stdout = self.proc.stdout.read()
# print 'TRYING READ STDERR'
# stderr = self.proc.stderr.read()
# print 'STDOUT:'
# print stdout
# print 'STDERR:'
# print stderr
if self.orgId == self.generationSize:
print "Restarting process after %d runs, push enter when ready..." % self.generationSize
raw_input()
# sleep(10)
if self.spawnProc:
print "Continuing with neatfile", self.nextNeatFile
self.proc = Process(
(
self.executable,
"-O",
self.prefix,
"-R",
"102",
"-I",
self.datFile,
"-X",
self.nextNeatFile,
"-XG",
"1",
)
)
self.genId += 1
self.orgId = 0
# print 'On iteration', self.orgId+1
while True:
if self.spawnProc:
out = self.proc.read()
if out != "":
# print 'Got stdout:'
# print out
pass
out = self.proc.readerr()
if out != "":
# print 'Got stderr:'
# print out
pass
try:
ff = open(self.motionFile, "r")
except IOError:
print "File does not exist yet"
sleep(1)
continue
lines = ff.readlines()
nLines = len(lines)
if nLines < self.expectedLines:
print " only %d of %d lines, waiting..." % (nLines, self.expectedLines)
ff.close()
sleep(0.5)
continue
break
self.orgId += 1
for ii, line in enumerate(lines[self.junkPoints :]):
# print 'line', ii, 'is', line
nums = [float(xx) for xx in line.split()]
if ii == 0:
rawPositions = array(nums)
else:
rawPositions = vstack((rawPositions, array(nums)))
# swap and scale rawPositions appropriately
rawPositions = rawPositions.T[ix_(self.motorColumns)].T # remap to right columns
# print 'First few lines of rawPositions are:'
# for ii in range(10):
# print prettyVec(rawPositions[ii,:], prec=2)
# Average over every self.avgPoints
for ii in range(self.expectedLines / self.avgPoints):
temp = mean(rawPositions[ii : (ii + self.avgPoints), :], 0)
if ii == 0:
positions = temp
else:
positions = vstack((positions, temp))
# print 'First few lines of positions are:'
# for ii in range(10):
# print prettyVec(positions[ii,:], prec=2)
# scale from [-1,1] to [0,1]
positions += 1
positions *= 0.5
# scale from [0,1] to appropriate ranges
innerIdx = [0, 2, 4, 6]
outerIdx = [1, 3, 5, 7]
centerIdx = [8]
for ii in innerIdx:
positions[:, ii] *= MAX_INNER - MIN_INNER
positions[:, ii] += MIN_INNER
for ii in outerIdx:
positions[:, ii] *= MAX_OUTER - MIN_OUTER
positions[:, ii] += MIN_OUTER
for ii in centerIdx:
positions[:, ii] *= MAX_CENTER - MIN_CENTER
positions[:, ii] += MIN_CENTER
# append a column of 512s for center
# positions = hstack((positions,
# NORM_CENTER * ones((positions.shape[0],1))))
times = linspace(0, 12, positions.shape[0])
# Dump both raw positions and positions to file
thisIdentifier = "%s_%05d_%03d" % (self.identifier, self.genId, self.orgId)
ff = open("%s_raw" % thisIdentifier, "w")
writeArray(ff, rawPositions)
ff.close()
ff = open("%s_filt" % thisIdentifier, "w")
writeArray(ff, positions)
ff.close()
# return function of time
return lambda time: matInterp(time, times, positions), thisIdentifier
class NEATStrategy(OneStepStrategy):
"""
A strategy that calls a NEAT executable to determine a gait
"""
def __init__(self, *args, **kwargs):
super(NEATStrategy, self).__init__([], [])
##############################
# HyperNEAT Parameters
##############################
now = datetime.now()
# self.identifier = ''.join([random.choice('abcdef1234567890') for x in range(8)])
self.identifier = now.strftime("neat_%y%m%d_%H%M%S")
self.executable = "/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/Hypercube_NEAT"
self.motionFile = "/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/spiderJointAngles.txt"
# self.motionFile = 'spiderJointAngles.txt'
self.fitnessFile = "/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/fitness"
self.datFile = "/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/SpiderRobotExperiment.dat"
self.avgPoints = 4 # Average over this many points
self.junkPoints = 1000
# How many lines to expect from HyperNEAT file
self.expectedLines = self.junkPoints + 12 * 40 * self.avgPoints
# self.motorColumns = [0,1,4,5,2,3,6,7] # Order of motors in HyperNEAT file
self.motorColumns = [0, 1, 4, 5, 2, 3, 6, 7, 8] # Order of motors in HyperNEAT file
self.generationSize = 9
self.initNeatFile = kwargs.get("initNeatFile", None) # Pop file to start with, None for restart
self.prefix = "delme"
self.nextNeatFile = "%s_pop.xml" % self.prefix
self.separateLogInfo = True
# self.proc = sp.Popen((self.executable,
# '-O', 'delme', '-R', '102', '-I', self.datFile),
# stdout=sp.PIPE, stderr=sp.PIPE, stdin=sp.PIPE)
# os.system('%s -O delme -R 102 -I %s' % (self.executable, self.datFile))
self.spawnProc = False
if self.spawnProc:
if self.initNeatFile is None:
self.proc = Process((self.executable, "-O", self.prefix, "-R", "102", "-I", self.datFile))
else:
print "Starting with neatfile", self.initNeatFile
self.proc = Process(
(
self.executable,
"-O",
self.prefix,
"-R",
"102",
"-I",
self.datFile,
"-X",
self.nextNeatFile,
"-XG",
"1",
)
)
#'%s -O delme -R 102 -I %s' % (self.executable, self.datFile))
self.genId = 0
self.orgId = 0
def __del__(self):
if self.spawnProc:
print "Waiting for %s to exit..." % self.executable,
code = self.proc.wait()
print "done."
def _getNext(self):
"""Get the next point to try. This reads from the file
self.motionFile"""
# print 'TRYING READ STDOUT'
# stdout = self.proc.stdout.read()
# print 'TRYING READ STDERR'
# stderr = self.proc.stderr.read()
# print 'STDOUT:'
# print stdout
# print 'STDERR:'
# print stderr
if self.orgId == self.generationSize:
print "Restarting process after %d runs, push enter when ready..." % self.generationSize
raw_input()
# sleep(10)
if self.spawnProc:
print "Continuing with neatfile", self.nextNeatFile
self.proc = Process(
(
self.executable,
"-O",
self.prefix,
"-R",
"102",
"-I",
self.datFile,
"-X",
self.nextNeatFile,
"-XG",
"1",
)
)
self.genId += 1
self.orgId = 0
# print 'On iteration', self.orgId+1
while True:
if self.spawnProc:
out = self.proc.read()
if out != "":
# print 'Got stdout:'
# print out
pass
out = self.proc.readerr()
if out != "":
# print 'Got stderr:'
# print out
pass
try:
ff = open(self.motionFile, "r")
except IOError:
print "File does not exist yet"
sleep(1)
continue
lines = ff.readlines()
nLines = len(lines)
if nLines < self.expectedLines:
print " only %d of %d lines, waiting..." % (nLines, self.expectedLines)
ff.close()
sleep(0.5)
continue
break
self.orgId += 1
for ii, line in enumerate(lines[self.junkPoints :]):
# print 'line', ii, 'is', line
nums = [float(xx) for xx in line.split()]
if ii == 0:
rawPositions = array(nums)
else:
rawPositions = vstack((rawPositions, array(nums)))
# swap and scale rawPositions appropriately
rawPositions = rawPositions.T[ix_(self.motorColumns)].T # remap to right columns
# print 'First few lines of rawPositions are:'
# for ii in range(10):
# print prettyVec(rawPositions[ii,:], prec=2)
# Average over every self.avgPoints
for ii in range(self.expectedLines / self.avgPoints):
temp = mean(rawPositions[ii : (ii + self.avgPoints), :], 0)
if ii == 0:
positions = temp
else:
positions = vstack((positions, temp))
# print 'First few lines of positions are:'
# for ii in range(10):
# print prettyVec(positions[ii,:], prec=2)
# scale from [-1,1] to [0,1]
positions += 1
positions *= 0.5
# scale from [0,1] to appropriate ranges
innerIdx = [0, 2, 4, 6]
outerIdx = [1, 3, 5, 7]
centerIdx = [8]
for ii in innerIdx:
positions[:, ii] *= MAX_INNER - MIN_INNER
positions[:, ii] += MIN_INNER
for ii in outerIdx:
positions[:, ii] *= MAX_OUTER - MIN_OUTER
positions[:, ii] += MIN_OUTER
for ii in centerIdx:
positions[:, ii] *= MAX_CENTER - MIN_CENTER
positions[:, ii] += MIN_CENTER
# append a column of 512s for center
# positions = hstack((positions,
# NORM_CENTER * ones((positions.shape[0],1))))
times = linspace(0, 12, positions.shape[0])
# Dump both raw positions and positions to file
thisIdentifier = "%s_%05d_%03d" % (self.identifier, self.genId, self.orgId)
ff = open("%s_raw" % thisIdentifier, "w")
writeArray(ff, rawPositions)
ff.close()
ff = open("%s_filt" % thisIdentifier, "w")
writeArray(ff, positions)
ff.close()
# return function of time
return lambda time: matInterp(time, times, positions), thisIdentifier
def updateResults(self, dist):
"""This must be called for the last point that was handed out!
This communicates back to the running subprocess.
"""
dist = float(dist)
# MAKE SURE TO CALL super().updateResults!
super(NEATStrategy, self).updateResults(dist)
# Send fitness to process
# out,err = proc.communicate('%f\n' % dist)
print "Deleting old motion file %s" % self.motionFile
os.remove(self.motionFile)
print "Sending to process: %f" % dist,
if self.spawnProc:
print "via process"
self.proc.write("%f\n" % dist)
else:
print "via file"
# Instead, write this to a file
ff = open(".fitness", "w")
ff.write("%s\n" % dist)
ff.close()
os.rename(".fitness", self.fitnessFile)
# print 'TRYING READ STDOUT'
# stdout = self.proc.stdout.read()
# print 'TRYING READ STDOUT'
# stderr = self.proc.stderr.read()
#
# print 'Got stdout:'
# print stdout
# print 'Got stderr:'
# print stderr
if self.spawnProc:
out = self.proc.read()
if out != "":
# print 'Got stdout:'
# print out
pass
out = self.proc.readerr()
if out != "":
# print 'Got stderr:'
# print out
pass
def logHeader(self):
return "# NEATStrategy starting, identifier %s\n" % self.identifier
class NEATStrategy(OneStepStrategy):
'''
A strategy that calls a NEAT executable to determine a gait
'''
def __init__(self, *args, **kwargs):
super(NEATStrategy, self).__init__([], [])
##############################
# HyperNEAT Parameters
##############################
now = datetime.now()
#self.identifier = ''.join([random.choice('abcdef1234567890') for x in range(8)])
self.identifier = now.strftime('neat_%y%m%d_%H%M%S')
self.executable = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/Hypercube_NEAT'
self.motionFile = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/spiderJointAngles.txt'
#self.motionFile = 'spiderJointAngles.txt'
self.fitnessFile = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/fitness'
self.datFile = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/SpiderRobotExperiment.dat'
self.avgPoints = 4 # Average over this many points
self.junkPoints = 1000
# How many lines to expect from HyperNEAT file
self.expectedLines = self.junkPoints + 12 * 40 * self.avgPoints
#self.motorColumns = [0,1,4,5,2,3,6,7] # Order of motors in HyperNEAT file
self.motorColumns = [0, 1, 4, 5, 2, 3, 6, 7,
8] # Order of motors in HyperNEAT file
self.generationSize = 9
self.initNeatFile = kwargs.get(
'initNeatFile', None) # Pop file to start with, None for restart
self.prefix = 'delme'
self.nextNeatFile = '%s_pop.xml' % self.prefix
self.separateLogInfo = True
#self.proc = sp.Popen((self.executable,
# '-O', 'delme', '-R', '102', '-I', self.datFile),
# stdout=sp.PIPE, stderr=sp.PIPE, stdin=sp.PIPE)
#os.system('%s -O delme -R 102 -I %s' % (self.executable, self.datFile))
self.spawnProc = False
if self.spawnProc:
if self.initNeatFile is None:
self.proc = Process((self.executable, '-O', self.prefix, '-R',
'102', '-I', self.datFile))
else:
print 'Starting with neatfile', self.initNeatFile
self.proc = Process(
(self.executable, '-O', self.prefix, '-R', '102', '-I',
self.datFile, '-X', self.nextNeatFile, '-XG', '1'))
#'%s -O delme -R 102 -I %s' % (self.executable, self.datFile))
self.genId = 0
self.orgId = 0
def __del__(self):
if self.spawnProc:
print 'Waiting for %s to exit...' % self.executable,
code = self.proc.wait()
print 'done.'
def _getNext(self):
'''Get the next point to try. This reads from the file
self.motionFile'''
#print 'TRYING READ STDOUT'
#stdout = self.proc.stdout.read()
#print 'TRYING READ STDERR'
#stderr = self.proc.stderr.read()
#print 'STDOUT:'
#print stdout
#print 'STDERR:'
#print stderr
if self.orgId == self.generationSize:
print 'Restarting process after %d runs, push enter when ready...' % self.generationSize
raw_input()
#sleep(10)
if self.spawnProc:
print 'Continuing with neatfile', self.nextNeatFile
self.proc = Process(
(self.executable, '-O', self.prefix, '-R', '102', '-I',
self.datFile, '-X', self.nextNeatFile, '-XG', '1'))
self.genId += 1
self.orgId = 0
#print 'On iteration', self.orgId+1
while True:
if self.spawnProc:
out = self.proc.read()
if out != '':
#print 'Got stdout:'
#print out
pass
out = self.proc.readerr()
if out != '':
#print 'Got stderr:'
#print out
pass
try:
ff = open(self.motionFile, 'r')
except IOError:
print 'File does not exist yet'
sleep(1)
continue
lines = ff.readlines()
nLines = len(lines)
if nLines < self.expectedLines:
print ' only %d of %d lines, waiting...' % (
nLines, self.expectedLines)
ff.close()
sleep(.5)
continue
break
self.orgId += 1
for ii, line in enumerate(lines[self.junkPoints:]):
#print 'line', ii, 'is', line
nums = [float(xx) for xx in line.split()]
if ii == 0:
rawPositions = array(nums)
else:
rawPositions = vstack((rawPositions, array(nums)))
# swap and scale rawPositions appropriately
rawPositions = rawPositions.T[ix_(
self.motorColumns)].T # remap to right columns
#print 'First few lines of rawPositions are:'
#for ii in range(10):
# print prettyVec(rawPositions[ii,:], prec=2)
# Average over every self.avgPoints
for ii in range(self.expectedLines / self.avgPoints):
temp = mean(rawPositions[ii:(ii + self.avgPoints), :], 0)
if ii == 0:
positions = temp
else:
positions = vstack((positions, temp))
#print 'First few lines of positions are:'
#for ii in range(10):
# print prettyVec(positions[ii,:], prec=2)
# scale from [-1,1] to [0,1]
positions += 1
positions *= .5
# scale from [0,1] to appropriate ranges
innerIdx = [0, 2, 4, 6]
outerIdx = [1, 3, 5, 7]
centerIdx = [8]
for ii in innerIdx:
positions[:, ii] *= (MAX_INNER - MIN_INNER)
positions[:, ii] += MIN_INNER
for ii in outerIdx:
positions[:, ii] *= (MAX_OUTER - MIN_OUTER)
positions[:, ii] += MIN_OUTER
for ii in centerIdx:
positions[:, ii] *= (MAX_CENTER - MIN_CENTER)
positions[:, ii] += MIN_CENTER
# append a column of 512s for center
#positions = hstack((positions,
# NORM_CENTER * ones((positions.shape[0],1))))
times = linspace(0, 12, positions.shape[0])
# Dump both raw positions and positions to file
thisIdentifier = '%s_%05d_%03d' % (self.identifier, self.genId,
self.orgId)
ff = open('%s_raw' % thisIdentifier, 'w')
writeArray(ff, rawPositions)
ff.close()
ff = open('%s_filt' % thisIdentifier, 'w')
writeArray(ff, positions)
ff.close()
# return function of time
return lambda time: matInterp(time, times, positions), thisIdentifier
def updateResults(self, dist):
'''This must be called for the last point that was handed out!
This communicates back to the running subprocess.
'''
dist = float(dist)
# MAKE SURE TO CALL super().updateResults!
super(NEATStrategy, self).updateResults(dist)
# Send fitness to process
#out,err = proc.communicate('%f\n' % dist)
print 'Deleting old motion file %s' % self.motionFile
os.remove(self.motionFile)
print 'Sending to process: %f' % dist,
if self.spawnProc:
print 'via process'
self.proc.write('%f\n' % dist)
else:
print 'via file'
# Instead, write this to a file
ff = open('.fitness', 'w')
ff.write('%s\n' % dist)
ff.close()
os.rename('.fitness', self.fitnessFile)
#print 'TRYING READ STDOUT'
#stdout = self.proc.stdout.read()
#print 'TRYING READ STDOUT'
#stderr = self.proc.stderr.read()
#
#print 'Got stdout:'
#print stdout
#print 'Got stderr:'
#print stderr
if self.spawnProc:
out = self.proc.read()
if out != '':
#print 'Got stdout:'
#print out
pass
out = self.proc.readerr()
if out != '':
#print 'Got stderr:'
#print out
pass
def logHeader(self):
return '# NEATStrategy starting, identifier %s\n' % self.identifier
def makeprocess(app, action):
return Process(["sudo", "control", app, action])
server_socket = socket(AF_INET, SOCK_STREAM)
try:
server_socket.bind(('', port))
finally:
force(playback.stop)
server_socket.listen(1)
print("Server listening on port %d"%port)
try:
while True:
client, addr = server_socket.accept()
try:
print("Accepted connection from %s:%d"%addr)
decode_process = Process(("./audio/decoder -c %d -r %d -w %d"%(\
channels, sample_rate, sample_width)).split(),\
env={"AUDIO_PATH":"./audio"})
print("Started decoder")
decode_pipe = decode_process
print("Opened decode pipe")
playback.set_source(decode_process)
play_thread = Thread(target=playback.start)
play_thread.start()
print("Started playback thread")
while True:
bts = client.recv(1024)
print(len(bts))
if len(bts) < 1024:
decode_pipe.write(bts)
break
while len(bts):
class MainWindow(QMainWindow, Ui_MainWindow):
def __init__(self, parent=None):
super(MainWindow, self).__init__(parent)
self.setupUi(self)
self.piece = ["P", "N", "B", "R", "Q", "K"]
self.callList = []
self.timer = QTimer()
self.initGui()
cmdline = ["./genchess", ""]
self.engine = Process(cmdline)
self.newGame()
def __del__(self):
pass
def initGui(self):
self.connect(self, SIGNAL('triggered()'), self.closeEvent)
self.connect(self.actionNew_Game_F2, SIGNAL("triggered()"), self.newGame)
for i in range(6):
self.connect(self.whiteButtons[i], SIGNAL("clicked()"), lambda n="w" + str(i): self.runButton(n))
self.connect(self.blackButtons[i], SIGNAL("clicked()"), lambda n="b" + str(i): self.runButton(n))
for i in range(64):
self.connect(self.boardButtons[i], SIGNAL("clicked()"), lambda n="n" + str(i): self.runButton(n))
self.iconMap = {}
list = listdir("./Images/")
for item in list:
icon = QIcon()
icon.addFile("./Images/" + item)
self.iconMap[item[0] + item[1]] = icon
for i in range(0, 6):
self.whiteButtons[i].setIcon(self.iconMap["W"+self.piece[i]])
self.blackButtons[i].setIcon(self.iconMap["B"+self.piece[i]])
def closeEvent(self, event):
self.engine.write("quit\n")
sleep(0.2)
self.engine.terminate()
def comExe(self, data, lines):
self.engine.write(data)
output = self.engine.read()
while len(output.splitlines()) < lines:
sleep(0.05)
output += self.engine.read()
return output
def displayMessage(self, msg):
self.statusbar.showMessage(msg)
def updateBoard(self, move):
color = ["W", "B"]
color = color[self.stm]
if move[0] in self.piece:
to = boardIndex(move[1:])
self.board[to] = color + move[0]
self.boardButtons[to].setIcon(self.iconMap[self.board[to]])
return
fr = boardIndex(move[0:2])
to = boardIndex(move[2:4])
self.board[to] = self.board[fr]
self.boardButtons[fr].setIcon(self.iconMap["EE"])
self.boardButtons[to].setIcon(self.iconMap[self.board[to]])
def runComputer(self):
data = self.comExe("go\n", 2).splitlines()
move = data[0].split()[1]
results = data[1].split()[0]
self.updateBoard(move)
msg = color[self.stm] + "'s move: " + move
if results == 'ok':
self.displayMessage(msg)
elif results == 'special':
self.displayMessage(msg + ' (' + dropWord(data[1]) + ')')
elif results == 'result':
self.displayMessage(msg + ' ' + dropWord(data[1], 2))
return
self.stm ^= True
self.callList = []
self.timer.singleShot(300, self.nextPlayer)
def nextPlayer(self):
if player[self.stm] == 'computer':
self.runComputer()
def newGame(self):
self.engine.write("newgame\n")
for i in range(64):
self.boardButtons[i].setIcon(self.iconMap["EE"])
self.stm = False
self.board = [''] * 64
self.nextPlayer()
def runButton(self, n):
if n[0] == "w" or n[0] == "b":
if len(self.callList) == 1 and self.callList[0] in self.piece:
self.callList = [ self.piece[int(n[1:])] ]
self.displayMessage("active button: " + self.callList[0])
return
self.callList.append(self.piece[int(n[1:])])
else:
self.callList.append(boardCord(int(n[1:])))
if len(self.callList) == 1:
self.displayMessage("active button: " + self.callList[0])
return
elif self.callList[1][0] in self.piece:
self.statusbar.clearMessage()
self.callList = []
return
elif self.callList[0] == self.callList[1]:
self.statusbar.clearMessage()
self.callList = []
return
act = ''.join(self.callList)
self.callList = []
data = self.comExe("move " + act + '\n', 1)
results = data.split()[0]
if results == 'ok':
self.displayMessage(color[self.stm] + "'s move: " + act)
self.updateBoard(act)
self.stm ^= True
elif results == 'special':
self.displayMessage(color[self.stm] + "'s move: " + act + " (" + dropWord(data) + ")")
self.updateBoard(act)
self.stm ^= True
elif results == 'illegal':
self.displayMessage(data)
elif results == 'result':
self.displayMessage(color[self.stm] + "'s move: " + act + " " + dropWord(data, 2))
self.updateBoard(act)
return
self.timer.singleShot(300, self.nextPlayer)