def __wipe_thread(self, command):
process = Process(command, shell=True)
self.processes.append(process)
re_pct = re.compile(r"(\d+)%", re.M)
while True:
# check to see if process has ended
poll_exitcode = process.wait(os.WNOHANG)
if poll_exitcode != None:
break
# print any new output
stdout = process.read()
m = re_pct.search(stdout)
if m:
pct = int(m.group(1))
self.__progress = pct / 100.0
logger.info("Wipe progress: %d%%" % (self.__progress * 100))
# Wait another 2 seconds before polling
time.sleep(2.0)
stderr = process.readerr()
if poll_exitcode > 0:
self.state = protocol.FAIL
self.__fail_message = (
"Failed while wiping. Return code: %d, Stderr was: %s" %
(poll_exitcode, stderr))
logger.error(self.__fail_message)
return
self.__wipe_done = True
self.__progress = 1.0
logger.info("Wipe finished.")
self.state = protocol.IDLE
def run_command(self, command, seconds=None):
"""
Run a command on the system and keep the user updated on the completeness.
"""
proc = Process(command.strip().split(" "))
self.display_progress(0)
start_time = time.time()
while True:
time.sleep(.1)
sys.stdout.flush()
poll = proc.wait(os.WNOHANG)
if poll != None:
break
line = proc.read()
if "PERCENT COMPLETE:" in line:
self.display_progress(
float(
line.split("PERCENT COMPLETE:")[-1].replace("\n", "")))
if seconds:
current_time = time.time()
if current_time - start_time > seconds:
break
self.display_progress(
float(current_time - start_time) * 100. / seconds)
try:
proc.kill(signal.SIGTERM)
except OSError:
pass
self.display_progress(100)
print
def call_cmd(cmd, timeout=-1, output_filter=None, cwd=None, check_ret=None):
'''
Utility to call a command.
timeout is in seconds.
'''
print("call_cmd: calling " + " ".join(cmd))
p = Process(cmd, cwd=cwd, stderr=subprocess.STDOUT)
launch_time = time.clock()
output = ""
while True:
# check to see if process has ended
ret = p.wait(os.WNOHANG)
# print any new output
o = p.read()
if len(o) > 0:
print("output = %s" % o)
if output_filter:
output_filter(p, o, output)
output += o
time.sleep(1)
if ret is not None:
if check_ret is not None:
assert check_ret == ret
return ret, output
if timeout > 0 and time.clock() - launch_time > timeout:
p.kill(signal.SIGKILL)
if check_ret is not None:
assert check_ret == -1
return -1, output
def call_cmd(cmd, timeout=-1, output_filter=None, cwd=None, check_ret=None):
'''
Utility to call a command.
timeout is in seconds.
'''
print("call_cmd: calling " + " ".join(cmd))
p = Process(cmd, cwd=cwd, stderr=subprocess.STDOUT)
launch_time = time.clock()
output = ""
while True:
# check to see if process has ended
ret = p.wait(os.WNOHANG)
# print any new output
o = p.read()
if len(o) > 0:
print("output = %s" % o)
if output_filter:
output_filter(p, o, output)
output += o
time.sleep(1)
if ret is not None:
if check_ret is not None:
assert check_ret == ret
return ret, output
if timeout > 0 and time.clock() - launch_time > timeout:
p.kill(signal.SIGKILL)
if check_ret is not None:
assert check_ret == -1
return -1, output
def run_command(self,command,seconds=None):
"""
Run a command on the system and keep the user updated on the completeness.
"""
proc = Process(command.strip().split(" "))
self.display_progress(0)
start_time = time.time()
while True:
time.sleep(.1)
sys.stdout.flush()
poll = proc.wait(os.WNOHANG)
if poll != None:
break
line = proc.read()
if "PERCENT COMPLETE:" in line:
try:
self.display_progress(float(line.split("PERCENT COMPLETE:")[-1].replace("\n","")))
except ValueError:
pass
if seconds:
current_time = time.time()
if current_time-start_time>seconds:
break
self.display_progress(float(current_time-start_time)*100./seconds)
try:
proc.kill(signal.SIGTERM)
except OSError:
pass
self.display_progress(100)
print
def __wipe_thread(self, command):
process = Process(command, shell=True)
self.processes.append(process)
re_pct = re.compile(r"(\d+)%", re.M)
while True:
# check to see if process has ended
poll_exitcode = process.wait(os.WNOHANG)
if poll_exitcode != None:
break
# print any new output
stdout = process.read()
m = re_pct.search(stdout)
if m:
pct = int(m.group(1))
self.__progress = pct / 100.0
logger.info("Wipe progress: %d%%" % (self.__progress*100))
# Wait another 2 seconds before polling
time.sleep(2.0)
stderr = process.readerr()
if poll_exitcode > 0:
self.state = protocol.FAIL
self.__fail_message = ("Failed while wiping. Return code: %d, Stderr was: %s" %
(poll_exitcode, stderr))
logger.error(self.__fail_message)
return
self.__wipe_done = True
self.__progress = 1.0
logger.info("Wipe finished.")
self.state = protocol.IDLE
def __shell_exec_thread(self, command, shell_exec_id):
logger.info("Shell execution of: %s" % command)
process = Process(command, shell=True)
# Initiate process
process.wait(os.WNOHANG)
self.processes.append(process)
# Blocking wait
process.wait()
self.shell_exec_results[shell_exec_id] = (process.read(), process.readerr())
self.state = protocol.IDLE
def __shell_exec_thread(self, command, shell_exec_id):
logger.info("Shell execution of: %s" % command)
process = Process(command, shell=True)
# Initiate process
process.wait(os.WNOHANG)
self.processes.append(process)
# Blocking wait
process.wait()
self.shell_exec_results[shell_exec_id] = (process.read(),
process.readerr())
self.state = protocol.IDLE
def __scan_thread(self, commands):
# Execute a number of commands and return their output in the same order.
command_cnt = float(len(commands))
self.__progress = 0.0
for cnt, command in enumerate(commands, start=1):
logger.info("SCAN executing command: %s" % command)
self.__progress = cnt / command_cnt
logger.debug("Progress: %.2f" % self.__progress)
try:
process = Process(command, shell=True,)
process.wait(os.WNOHANG)
self.processes.append(process)
process.wait()
self.scan_results[command] = (process.read(), process.readerr())
except OSError:
self.scan_results[command] = ("", "Command does not exist")
self.state = protocol.IDLE
def __scan_thread(self, commands):
# Execute a number of commands and return their output in the same order.
command_cnt = float(len(commands))
self.__progress = 0.0
for cnt, command in enumerate(commands, start=1):
logger.info("SCAN executing command: %s" % command)
self.__progress = cnt / command_cnt
logger.debug("Progress: %.2f" % self.__progress)
try:
process = Process(
command,
shell=True,
)
process.wait(os.WNOHANG)
self.processes.append(process)
process.wait()
self.scan_results[command] = (process.read(),
process.readerr())
except OSError:
self.scan_results[command] = ("", "Command does not exist")
self.state = protocol.IDLE
class NEATStrategy(OneStepStrategy):
'''
A strategy that calls a NEAT executable to determine a gait
'''
def __init__(self, *args, **kwargs):
super(NEATStrategy, self).__init__([], [])
##############################
# HyperNEAT Parameters
##############################
now = datetime.now()
#self.identifier = ''.join([random.choice('abcdef1234567890') for x in range(8)])
self.identifier = now.strftime('neat_%y%m%d_%H%M%S')
# self.executable = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/Hypercube_NEAT'
self.executable = '/home/sean/hyperneat/HyperNEAT_v2_5/out/Hypercube_NEAT'
# self.motionFile = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/spiderJointAngles.txt'
self.motionFile = '/home/sean/simtest/spiderJointAngles.txt'
# self.fitnessFile = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/fitness'
self.fitnessFile = '/home/sean/simtest/fitness'
# self.datFile = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/SpiderRobotExperiment.dat'
self.datFile = '/home/sean/hyperneat/HyperNEAT_v2_5/out/SpiderRobotExperiment.dat'
self.avgPoints = 4 # Average over this many points
self.junkPoints = 1000
# How many lines to expect from HyperNEAT file
self.expectedLines = self.junkPoints + 12 * 40 * self.avgPoints
#self.motorColumns = [0,1,4,5,2,3,6,7] # Order of motors in HyperNEAT file
self.motorColumns = [0,1,4,5,2,3,6,7,8] # Order of motors in HyperNEAT file
self.generationSize = 200
self.initNeatFile = kwargs.get('initNeatFile', None) # Pop file to start with, None for restart
self.prefix = 'delme'
# self.nextNeatFile = '%s_pop.xml' % self.prefix
self.nextNeatFile = '/home/sean/simtest/delme_pop.xml'
self.separateLogInfo = True
self.restartHN = kwargs.get('restartHN', True)
# stdout=sp.PIPE, stderr=sp.PIPE, stdin=sp.PIPE)
#os.system('%s -O delme -R 102 -I %s' % (self.executable, self.datFile))
self.spawnProc = True
if self.spawnProc:
if self.initNeatFile is None:
hnout = open('hyperneat.out', 'w')
hnerr = open('hyperneat.err', 'w')
self.proc = Process((self.executable,
'-O', self.prefix, '-R', '102', '-I',
self.datFile),
#stdout = sys.stdout,
#stderr = sys.stderr,
stdout = hnout,
stderr = hnerr,
)
else:
print 'Starting with neatfile', self.initNeatFile
self.proc = Process((self.executable,
'-O', self.prefix, '-R', '102', '-I',
self.datFile, '-X', self.nextNeatFile, '-XG', '1'))
#'%s -O delme -R 102 -I %s' % (self.executable, self.datFile))
self.genId = 0
self.orgId = 0
def __del__(self):
if self.spawnProc:
print 'Waiting for %s to exit...' % self.executable,
code = self.proc.wait()
print 'done.'
print code
def _getNext(self):
'''Get the next point to try. This reads from the file
self.motionFile'''
#print 'TRYING READ STDOUT'
#stdout = self.proc.stdout.read()
#print 'TRYING READ STDERR'
#stderr = self.proc.stderr.read()
#print 'STDOUT:'
#print stdout
#print 'STDERR:'
#print stderr
if self.orgId == self.generationSize:
# print 'Restarting process after %d runs, push enter when ready...' % self.generationSize
if self.restartHN:
print 'Restarting process after %d runs' % self.generationSize
print 'waiting for exit code...'
code = self.proc.wait()
print 'got exit code: %d' % code
# raw_input()
#sleep(10)
if self.spawnProc:
print 'Continuing with neatfile', self.nextNeatFile
self.proc = Process((self.executable,
'-O', self.prefix, '-R', '102', '-I',
self.datFile, '-X', self.nextNeatFile, '-XG', '1'))
self.genId += 1
self.orgId = 0
#print 'On iteration', self.orgId+1
while True:
if self.spawnProc:
out = self.proc.read()
if out != '':
#print 'Got stdout:'
#print out
pass
out = self.proc.readerr()
if out != '':
#print 'Got stderr:'
#print out
pass
try:
ff = open(self.motionFile, 'r')
except IOError:
print 'File does not exist yet'
sleep(1)
if self.proc.wait(os.WNOHANG) is None:
continue
else:
raise Exception('HN died')
lines = ff.readlines()
nLines = len(lines)
if nLines < self.expectedLines:
print ' only %d of %d lines, waiting...' % (nLines, self.expectedLines)
ff.close()
sleep(.5)
continue
break
self.orgId += 1
for ii,line in enumerate(lines[self.junkPoints:]):
#print 'line', ii, 'is', line
nums = [float(xx) for xx in line.split()]
if ii == 0:
rawPositions = array(nums)
else:
rawPositions = vstack((rawPositions, array(nums)))
# swap and scale rawPositions appropriately
rawPositions = rawPositions.T[ix_(self.motorColumns)].T # remap to right columns
#print 'First few lines of rawPositions are:'
#for ii in range(10):
# print prettyVec(rawPositions[ii,:], prec=2)
# Average over every self.avgPoints
for ii in range((self.expectedLines-self.junkPoints)/self.avgPoints):
temp = mean(rawPositions[ii*self.avgPoints:(ii+1)*self.avgPoints,:], 0)
if ii == 0:
positions = temp
else:
positions = vstack((positions, temp))
#print 'First few lines of positions are:'
#for ii in range(10):
# print prettyVec(positions[ii,:], prec=2)
# scale from [-1,1] to [0,1]
positions += 1
positions *= .5
# scale from [0,1] to appropriate ranges
innerIdx = [0, 2, 4, 6]
outerIdx = [1, 3, 5, 7]
centerIdx = [8]
for ii in innerIdx:
positions[:,ii] *= (MAX_INNER - MIN_INNER)
positions[:,ii] += MIN_INNER
for ii in outerIdx:
positions[:,ii] *= (MAX_OUTER - MIN_OUTER)
positions[:,ii] += MIN_OUTER
for ii in centerIdx:
positions[:,ii] *= (MAX_CENTER - MIN_CENTER)
positions[:,ii] += MIN_CENTER
# append a column of 512s for center
#positions = hstack((positions,
# NORM_CENTER * ones((positions.shape[0],1))))
times = linspace(0,12,positions.shape[0])
# Dump both raw positions and positions to file
thisIdentifier = '%s_%05d_%03d' % (self.identifier, self.genId, self.orgId)
ff = open('%s_raw' % thisIdentifier, 'w')
writeArray(ff, rawPositions)
ff.close()
ff = open('%s_filt' % thisIdentifier, 'w')
writeArray(ff, positions)
ff.close()
# return function of time
return lambda time: matInterp(time, times, positions), thisIdentifier
def updateResults(self, dist):
'''This must be called for the last point that was handed out!
This communicates back to the running subprocess.
'''
dist = float(dist)
# MAKE SURE TO CALL super().updateResults!
super(NEATStrategy, self).updateResults(dist)
# Send fitness to process
#out,err = proc.communicate('%f\n' % dist)
print 'Deleting old motion file %s' % self.motionFile
os.remove(self.motionFile)
print 'Sending to process: %f' % dist,
if self.spawnProc:
print 'via process'
self.proc.write('%f\n' % dist)
else:
print 'via file'
# Instead, write this to a file
ff = open('.fitness', 'w')
ff.write('%s\n' % dist)
ff.close()
os.rename('.fitness', self.fitnessFile)
#print 'TRYING READ STDOUT'
#stdout = self.proc.stdout.read()
#print 'TRYING READ STDOUT'
#stderr = self.proc.stderr.read()
#
#print 'Got stdout:'
#print stdout
#print 'Got stderr:'
#print stderr
if self.spawnProc:
out = self.proc.read()
if out != '':
#print 'Got stdout:'
#print out
pass
out = self.proc.readerr()
if out != '':
#print 'Got stderr:'
#print out
pass
def logHeader(self):
return '# NEATStrategy starting, identifier %s\n' % self.identifier
class NEATStrategy(OneStepStrategy):
'''
A strategy that calls a NEAT executable to determine a gait
'''
def __init__(self, *args, **kwargs):
super(NEATStrategy, self).__init__([], [])
##############################
# HyperNEAT Parameters
##############################
now = datetime.now()
#self.identifier = ''.join([random.choice('abcdef1234567890') for x in range(8)])
self.identifier = now.strftime('neat_%y%m%d_%H%M%S')
self.executable = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/Hypercube_NEAT'
self.motionFile = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/spiderJointAngles.txt'
#self.motionFile = 'spiderJointAngles.txt'
self.fitnessFile = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/fitness'
self.datFile = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/SpiderRobotExperiment.dat'
self.avgPoints = 4 # Average over this many points
self.junkPoints = 1000
# How many lines to expect from HyperNEAT file
self.expectedLines = self.junkPoints + 12 * 40 * self.avgPoints
#self.motorColumns = [0,1,4,5,2,3,6,7] # Order of motors in HyperNEAT file
self.motorColumns = [0, 1, 4, 5, 2, 3, 6, 7,
8] # Order of motors in HyperNEAT file
self.generationSize = 9
self.initNeatFile = kwargs.get(
'initNeatFile', None) # Pop file to start with, None for restart
self.prefix = 'delme'
self.nextNeatFile = '%s_pop.xml' % self.prefix
self.separateLogInfo = True
#self.proc = sp.Popen((self.executable,
# '-O', 'delme', '-R', '102', '-I', self.datFile),
# stdout=sp.PIPE, stderr=sp.PIPE, stdin=sp.PIPE)
#os.system('%s -O delme -R 102 -I %s' % (self.executable, self.datFile))
self.spawnProc = False
if self.spawnProc:
if self.initNeatFile is None:
self.proc = Process((self.executable, '-O', self.prefix, '-R',
'102', '-I', self.datFile))
else:
print 'Starting with neatfile', self.initNeatFile
self.proc = Process(
(self.executable, '-O', self.prefix, '-R', '102', '-I',
self.datFile, '-X', self.nextNeatFile, '-XG', '1'))
#'%s -O delme -R 102 -I %s' % (self.executable, self.datFile))
self.genId = 0
self.orgId = 0
def __del__(self):
if self.spawnProc:
print 'Waiting for %s to exit...' % self.executable,
code = self.proc.wait()
print 'done.'
def _getNext(self):
'''Get the next point to try. This reads from the file
self.motionFile'''
#print 'TRYING READ STDOUT'
#stdout = self.proc.stdout.read()
#print 'TRYING READ STDERR'
#stderr = self.proc.stderr.read()
#print 'STDOUT:'
#print stdout
#print 'STDERR:'
#print stderr
if self.orgId == self.generationSize:
print 'Restarting process after %d runs, push enter when ready...' % self.generationSize
raw_input()
#sleep(10)
if self.spawnProc:
print 'Continuing with neatfile', self.nextNeatFile
self.proc = Process(
(self.executable, '-O', self.prefix, '-R', '102', '-I',
self.datFile, '-X', self.nextNeatFile, '-XG', '1'))
self.genId += 1
self.orgId = 0
#print 'On iteration', self.orgId+1
while True:
if self.spawnProc:
out = self.proc.read()
if out != '':
#print 'Got stdout:'
#print out
pass
out = self.proc.readerr()
if out != '':
#print 'Got stderr:'
#print out
pass
try:
ff = open(self.motionFile, 'r')
except IOError:
print 'File does not exist yet'
sleep(1)
continue
lines = ff.readlines()
nLines = len(lines)
if nLines < self.expectedLines:
print ' only %d of %d lines, waiting...' % (
nLines, self.expectedLines)
ff.close()
sleep(.5)
continue
break
self.orgId += 1
for ii, line in enumerate(lines[self.junkPoints:]):
#print 'line', ii, 'is', line
nums = [float(xx) for xx in line.split()]
if ii == 0:
rawPositions = array(nums)
else:
rawPositions = vstack((rawPositions, array(nums)))
# swap and scale rawPositions appropriately
rawPositions = rawPositions.T[ix_(
self.motorColumns)].T # remap to right columns
#print 'First few lines of rawPositions are:'
#for ii in range(10):
# print prettyVec(rawPositions[ii,:], prec=2)
# Average over every self.avgPoints
for ii in range(self.expectedLines / self.avgPoints):
temp = mean(rawPositions[ii:(ii + self.avgPoints), :], 0)
if ii == 0:
positions = temp
else:
positions = vstack((positions, temp))
#print 'First few lines of positions are:'
#for ii in range(10):
# print prettyVec(positions[ii,:], prec=2)
# scale from [-1,1] to [0,1]
positions += 1
positions *= .5
# scale from [0,1] to appropriate ranges
innerIdx = [0, 2, 4, 6]
outerIdx = [1, 3, 5, 7]
centerIdx = [8]
for ii in innerIdx:
positions[:, ii] *= (MAX_INNER - MIN_INNER)
positions[:, ii] += MIN_INNER
for ii in outerIdx:
positions[:, ii] *= (MAX_OUTER - MIN_OUTER)
positions[:, ii] += MIN_OUTER
for ii in centerIdx:
positions[:, ii] *= (MAX_CENTER - MIN_CENTER)
positions[:, ii] += MIN_CENTER
# append a column of 512s for center
#positions = hstack((positions,
# NORM_CENTER * ones((positions.shape[0],1))))
times = linspace(0, 12, positions.shape[0])
# Dump both raw positions and positions to file
thisIdentifier = '%s_%05d_%03d' % (self.identifier, self.genId,
self.orgId)
ff = open('%s_raw' % thisIdentifier, 'w')
writeArray(ff, rawPositions)
ff.close()
ff = open('%s_filt' % thisIdentifier, 'w')
writeArray(ff, positions)
ff.close()
# return function of time
return lambda time: matInterp(time, times, positions), thisIdentifier
def updateResults(self, dist):
'''This must be called for the last point that was handed out!
This communicates back to the running subprocess.
'''
dist = float(dist)
# MAKE SURE TO CALL super().updateResults!
super(NEATStrategy, self).updateResults(dist)
# Send fitness to process
#out,err = proc.communicate('%f\n' % dist)
print 'Deleting old motion file %s' % self.motionFile
os.remove(self.motionFile)
print 'Sending to process: %f' % dist,
if self.spawnProc:
print 'via process'
self.proc.write('%f\n' % dist)
else:
print 'via file'
# Instead, write this to a file
ff = open('.fitness', 'w')
ff.write('%s\n' % dist)
ff.close()
os.rename('.fitness', self.fitnessFile)
#print 'TRYING READ STDOUT'
#stdout = self.proc.stdout.read()
#print 'TRYING READ STDOUT'
#stderr = self.proc.stderr.read()
#
#print 'Got stdout:'
#print stdout
#print 'Got stderr:'
#print stderr
if self.spawnProc:
out = self.proc.read()
if out != '':
#print 'Got stdout:'
#print out
pass
out = self.proc.readerr()
if out != '':
#print 'Got stderr:'
#print out
pass
def logHeader(self):
return '# NEATStrategy starting, identifier %s\n' % self.identifier
class GPGEncryption():
def __init__(self, encrypt_or_decrypt, iterable = None, user = None, passphrase = None, test_branch = False):
self.test_branch = test_branch
print 'initing gpg'
if(encrypt_or_decrypt[0] == 'e'):
#print "user = "******""
self.gpg_closed = False
self.padded = False
self.first_read = True
else:
#cmd = ['gpg', '-d', '--batch', '--passphrase-fd', '0']
#cmd = ['gpg', '--no-tty', '--homedir', '/etc/swift/gnupg', '-d', '--batch', '--passphrase-fd', '0']
#cmd = ['tr', 'd', 'D']
#cmd = 'gpg -d --batch --passphrase-fd 0'
cmd = 'cat'
#cmd = '/home/amedeiro/bin/python MyCat.py'
#self.str_iter = stringIterate(self.buffer, read_size)
self.p = Process(cmd, shell=True, bufsize = 2)
# TODO send stderr output to the logs
#self.p = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE, bufsize = 1024)
#self.p = Popen(cmd, stdin=PIPE, stdout=PIPE, bufsize = 4096)
if(encrypt_or_decrypt[0] == 'd'):
self.p.write('test-swift' + '\n')
self.my_buffer = ''
def print_errors(self):
print self.p.readerr()
def stringIterate(self, text, chunk_size):
index = 0
while index < len(text):
yield text[index : index + chunk_size]
index += chunk_size
def readFromStream():
chunk = self.stream_iter_read(read_size)
while chunk:
self.p.stdin.write(chunk)
chunk = self.stream_iter_read(read_size)
self.p.stdin.close()
def read(self, read_size):
def padder(text, div, pad = chr(0)):
print 'padder reading ' + text
size = (div - len(text) % div) + len(text)
diff = size - len(text)
text += pad * diff
return text
if self.first_read:
self.first_read = False
t = Thread(target = readFromStream)
t.start()
return self.p.stdout.read(read_size)
def digest(self, chunk):
print "digesting chunk = " + chunk
self.p.write(chunk)
def dump_buffer(self):
buf = self.p.read()
print 'dump buffer returning: ' + buf
return buf
def close_and_dump(self, timeout = .2):
print 'close and dump called'
self.p.closeinput()
time.sleep(timeout)
#self.p.closeinput()
buf = ''
#as long as there was a non blank string digested before this is called,
#we can wait for the process to dump its last output... Might be a problem
#if output dumps
#poll = self.p.wait(os.WNOHANG)
print 'entering close and dump loop'
count = 0
while buf == '' and count < 100:
buf = self.dump_buffer()
time.sleep(.01)
print self.p.readerr()
poll = self.p.wait(os.WNOHANG)
count = count + 1
print 'leaving close and dump loop'
poll = self.p.wait(os.WNOHANG)
print poll
return buf
class MainWindow(QMainWindow, Ui_MainWindow):
def __init__(self, parent=None):
super(MainWindow, self).__init__(parent)
self.setupUi(self)
self.piece = ["P", "N", "B", "R", "Q", "K"]
self.callList = []
self.timer = QTimer()
self.initGui()
cmdline = ["./genchess", ""]
self.engine = Process(cmdline)
self.newGame()
def __del__(self):
pass
def initGui(self):
self.connect(self, SIGNAL('triggered()'), self.closeEvent)
self.connect(self.actionNew_Game_F2, SIGNAL("triggered()"), self.newGame)
for i in range(6):
self.connect(self.whiteButtons[i], SIGNAL("clicked()"), lambda n="w" + str(i): self.runButton(n))
self.connect(self.blackButtons[i], SIGNAL("clicked()"), lambda n="b" + str(i): self.runButton(n))
for i in range(64):
self.connect(self.boardButtons[i], SIGNAL("clicked()"), lambda n="n" + str(i): self.runButton(n))
self.iconMap = {}
list = listdir("./Images/")
for item in list:
icon = QIcon()
icon.addFile("./Images/" + item)
self.iconMap[item[0] + item[1]] = icon
for i in range(0, 6):
self.whiteButtons[i].setIcon(self.iconMap["W"+self.piece[i]])
self.blackButtons[i].setIcon(self.iconMap["B"+self.piece[i]])
def closeEvent(self, event):
self.engine.write("quit\n")
sleep(0.2)
self.engine.terminate()
def comExe(self, data, lines):
self.engine.write(data)
output = self.engine.read()
while len(output.splitlines()) < lines:
sleep(0.05)
output += self.engine.read()
return output
def displayMessage(self, msg):
self.statusbar.showMessage(msg)
def updateBoard(self, move):
color = ["W", "B"]
color = color[self.stm]
if move[0] in self.piece:
to = boardIndex(move[1:])
self.board[to] = color + move[0]
self.boardButtons[to].setIcon(self.iconMap[self.board[to]])
return
fr = boardIndex(move[0:2])
to = boardIndex(move[2:4])
self.board[to] = self.board[fr]
self.boardButtons[fr].setIcon(self.iconMap["EE"])
self.boardButtons[to].setIcon(self.iconMap[self.board[to]])
def runComputer(self):
data = self.comExe("go\n", 2).splitlines()
move = data[0].split()[1]
results = data[1].split()[0]
self.updateBoard(move)
msg = color[self.stm] + "'s move: " + move
if results == 'ok':
self.displayMessage(msg)
elif results == 'special':
self.displayMessage(msg + ' (' + dropWord(data[1]) + ')')
elif results == 'result':
self.displayMessage(msg + ' ' + dropWord(data[1], 2))
return
self.stm ^= True
self.callList = []
self.timer.singleShot(300, self.nextPlayer)
def nextPlayer(self):
if player[self.stm] == 'computer':
self.runComputer()
def newGame(self):
self.engine.write("newgame\n")
for i in range(64):
self.boardButtons[i].setIcon(self.iconMap["EE"])
self.stm = False
self.board = [''] * 64
self.nextPlayer()
def runButton(self, n):
if n[0] == "w" or n[0] == "b":
if len(self.callList) == 1 and self.callList[0] in self.piece:
self.callList = [ self.piece[int(n[1:])] ]
self.displayMessage("active button: " + self.callList[0])
return
self.callList.append(self.piece[int(n[1:])])
else:
self.callList.append(boardCord(int(n[1:])))
if len(self.callList) == 1:
self.displayMessage("active button: " + self.callList[0])
return
elif self.callList[1][0] in self.piece:
self.statusbar.clearMessage()
self.callList = []
return
elif self.callList[0] == self.callList[1]:
self.statusbar.clearMessage()
self.callList = []
return
act = ''.join(self.callList)
self.callList = []
data = self.comExe("move " + act + '\n', 1)
results = data.split()[0]
if results == 'ok':
self.displayMessage(color[self.stm] + "'s move: " + act)
self.updateBoard(act)
self.stm ^= True
elif results == 'special':
self.displayMessage(color[self.stm] + "'s move: " + act + " (" + dropWord(data) + ")")
self.updateBoard(act)
self.stm ^= True
elif results == 'illegal':
self.displayMessage(data)
elif results == 'result':
self.displayMessage(color[self.stm] + "'s move: " + act + " " + dropWord(data, 2))
self.updateBoard(act)
return
self.timer.singleShot(300, self.nextPlayer)
class NEATStrategy(OneStepStrategy):
"""
A strategy that calls a NEAT executable to determine a gait
"""
def __init__(self, *args, **kwargs):
super(NEATStrategy, self).__init__([], [])
##############################
# HyperNEAT Parameters
##############################
now = datetime.now()
# self.identifier = ''.join([random.choice('abcdef1234567890') for x in range(8)])
self.identifier = now.strftime("neat_%y%m%d_%H%M%S")
self.executable = "/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/Hypercube_NEAT"
self.motionFile = "/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/spiderJointAngles.txt"
# self.motionFile = 'spiderJointAngles.txt'
self.fitnessFile = "/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/fitness"
self.datFile = "/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/SpiderRobotExperiment.dat"
self.avgPoints = 4 # Average over this many points
self.junkPoints = 1000
# How many lines to expect from HyperNEAT file
self.expectedLines = self.junkPoints + 12 * 40 * self.avgPoints
# self.motorColumns = [0,1,4,5,2,3,6,7] # Order of motors in HyperNEAT file
self.motorColumns = [0, 1, 4, 5, 2, 3, 6, 7, 8] # Order of motors in HyperNEAT file
self.generationSize = 9
self.initNeatFile = kwargs.get("initNeatFile", None) # Pop file to start with, None for restart
self.prefix = "delme"
self.nextNeatFile = "%s_pop.xml" % self.prefix
self.separateLogInfo = True
# self.proc = sp.Popen((self.executable,
# '-O', 'delme', '-R', '102', '-I', self.datFile),
# stdout=sp.PIPE, stderr=sp.PIPE, stdin=sp.PIPE)
# os.system('%s -O delme -R 102 -I %s' % (self.executable, self.datFile))
self.spawnProc = False
if self.spawnProc:
if self.initNeatFile is None:
self.proc = Process((self.executable, "-O", self.prefix, "-R", "102", "-I", self.datFile))
else:
print "Starting with neatfile", self.initNeatFile
self.proc = Process(
(
self.executable,
"-O",
self.prefix,
"-R",
"102",
"-I",
self.datFile,
"-X",
self.nextNeatFile,
"-XG",
"1",
)
)
#'%s -O delme -R 102 -I %s' % (self.executable, self.datFile))
self.genId = 0
self.orgId = 0
def __del__(self):
if self.spawnProc:
print "Waiting for %s to exit..." % self.executable,
code = self.proc.wait()
print "done."
def _getNext(self):
"""Get the next point to try. This reads from the file
self.motionFile"""
# print 'TRYING READ STDOUT'
# stdout = self.proc.stdout.read()
# print 'TRYING READ STDERR'
# stderr = self.proc.stderr.read()
# print 'STDOUT:'
# print stdout
# print 'STDERR:'
# print stderr
if self.orgId == self.generationSize:
print "Restarting process after %d runs, push enter when ready..." % self.generationSize
raw_input()
# sleep(10)
if self.spawnProc:
print "Continuing with neatfile", self.nextNeatFile
self.proc = Process(
(
self.executable,
"-O",
self.prefix,
"-R",
"102",
"-I",
self.datFile,
"-X",
self.nextNeatFile,
"-XG",
"1",
)
)
self.genId += 1
self.orgId = 0
# print 'On iteration', self.orgId+1
while True:
if self.spawnProc:
out = self.proc.read()
if out != "":
# print 'Got stdout:'
# print out
pass
out = self.proc.readerr()
if out != "":
# print 'Got stderr:'
# print out
pass
try:
ff = open(self.motionFile, "r")
except IOError:
print "File does not exist yet"
sleep(1)
continue
lines = ff.readlines()
nLines = len(lines)
if nLines < self.expectedLines:
print " only %d of %d lines, waiting..." % (nLines, self.expectedLines)
ff.close()
sleep(0.5)
continue
break
self.orgId += 1
for ii, line in enumerate(lines[self.junkPoints :]):
# print 'line', ii, 'is', line
nums = [float(xx) for xx in line.split()]
if ii == 0:
rawPositions = array(nums)
else:
rawPositions = vstack((rawPositions, array(nums)))
# swap and scale rawPositions appropriately
rawPositions = rawPositions.T[ix_(self.motorColumns)].T # remap to right columns
# print 'First few lines of rawPositions are:'
# for ii in range(10):
# print prettyVec(rawPositions[ii,:], prec=2)
# Average over every self.avgPoints
for ii in range(self.expectedLines / self.avgPoints):
temp = mean(rawPositions[ii : (ii + self.avgPoints), :], 0)
if ii == 0:
positions = temp
else:
positions = vstack((positions, temp))
# print 'First few lines of positions are:'
# for ii in range(10):
# print prettyVec(positions[ii,:], prec=2)
# scale from [-1,1] to [0,1]
positions += 1
positions *= 0.5
# scale from [0,1] to appropriate ranges
innerIdx = [0, 2, 4, 6]
outerIdx = [1, 3, 5, 7]
centerIdx = [8]
for ii in innerIdx:
positions[:, ii] *= MAX_INNER - MIN_INNER
positions[:, ii] += MIN_INNER
for ii in outerIdx:
positions[:, ii] *= MAX_OUTER - MIN_OUTER
positions[:, ii] += MIN_OUTER
for ii in centerIdx:
positions[:, ii] *= MAX_CENTER - MIN_CENTER
positions[:, ii] += MIN_CENTER
# append a column of 512s for center
# positions = hstack((positions,
# NORM_CENTER * ones((positions.shape[0],1))))
times = linspace(0, 12, positions.shape[0])
# Dump both raw positions and positions to file
thisIdentifier = "%s_%05d_%03d" % (self.identifier, self.genId, self.orgId)
ff = open("%s_raw" % thisIdentifier, "w")
writeArray(ff, rawPositions)
ff.close()
ff = open("%s_filt" % thisIdentifier, "w")
writeArray(ff, positions)
ff.close()
# return function of time
return lambda time: matInterp(time, times, positions), thisIdentifier
def updateResults(self, dist):
"""This must be called for the last point that was handed out!
This communicates back to the running subprocess.
"""
dist = float(dist)
# MAKE SURE TO CALL super().updateResults!
super(NEATStrategy, self).updateResults(dist)
# Send fitness to process
# out,err = proc.communicate('%f\n' % dist)
print "Deleting old motion file %s" % self.motionFile
os.remove(self.motionFile)
print "Sending to process: %f" % dist,
if self.spawnProc:
print "via process"
self.proc.write("%f\n" % dist)
else:
print "via file"
# Instead, write this to a file
ff = open(".fitness", "w")
ff.write("%s\n" % dist)
ff.close()
os.rename(".fitness", self.fitnessFile)
# print 'TRYING READ STDOUT'
# stdout = self.proc.stdout.read()
# print 'TRYING READ STDOUT'
# stderr = self.proc.stderr.read()
#
# print 'Got stdout:'
# print stdout
# print 'Got stderr:'
# print stderr
if self.spawnProc:
out = self.proc.read()
if out != "":
# print 'Got stdout:'
# print out
pass
out = self.proc.readerr()
if out != "":
# print 'Got stderr:'
# print out
pass
def logHeader(self):
return "# NEATStrategy starting, identifier %s\n" % self.identifier