def __badblocks_thread(self, command):
process = Process(command, shell=True)
self.processes.append(process)
re_pct = re.compile(r"(\d+)\.\d+%", re.M)
while True:
# check to see if process has ended
poll_exitcode = process.wait(os.WNOHANG)
if poll_exitcode != None:
break
# Search stderr - yes, actually stderr is the pipe that badblocks
# uses :(
stderr = process.readerr()
m = re_pct.search(stderr)
if m:
pct = int(m.group(1))
self.__progress = pct / 100.0
logger.info("Badblocks progress: %d%%" % (self.__progress * 100))
else:
logger.debug("Could not understand badblocks output: %s" % stderr)
# Wait another 2 seconds before polling
time.sleep(2.0)
stderr = process.readerr()
if poll_exitcode > 0:
self.state = protocol.FAIL
self.__fail_message = ("Failed executing badblocks. Return code: %d, Stderr was: %s" %
(poll_exitcode, stderr))
logger.error(self.__fail_message)
return
self.__progress = 1.0
logger.info("Badblocks finished.")
self.__badblocks_done = True
self.state = protocol.IDLE
def __wipe_thread(self, command):
process = Process(command, shell=True)
self.processes.append(process)
re_pct = re.compile(r"(\d+)%", re.M)
while True:
# check to see if process has ended
poll_exitcode = process.wait(os.WNOHANG)
if poll_exitcode != None:
break
# print any new output
stdout = process.read()
m = re_pct.search(stdout)
if m:
pct = int(m.group(1))
self.__progress = pct / 100.0
logger.info("Wipe progress: %d%%" % (self.__progress * 100))
# Wait another 2 seconds before polling
time.sleep(2.0)
stderr = process.readerr()
if poll_exitcode > 0:
self.state = protocol.FAIL
self.__fail_message = (
"Failed while wiping. Return code: %d, Stderr was: %s" %
(poll_exitcode, stderr))
logger.error(self.__fail_message)
return
self.__wipe_done = True
self.__progress = 1.0
logger.info("Wipe finished.")
self.state = protocol.IDLE
def __wipe_thread(self, command):
process = Process(command, shell=True)
self.processes.append(process)
re_pct = re.compile(r"(\d+)%", re.M)
while True:
# check to see if process has ended
poll_exitcode = process.wait(os.WNOHANG)
if poll_exitcode != None:
break
# print any new output
stdout = process.read()
m = re_pct.search(stdout)
if m:
pct = int(m.group(1))
self.__progress = pct / 100.0
logger.info("Wipe progress: %d%%" % (self.__progress*100))
# Wait another 2 seconds before polling
time.sleep(2.0)
stderr = process.readerr()
if poll_exitcode > 0:
self.state = protocol.FAIL
self.__fail_message = ("Failed while wiping. Return code: %d, Stderr was: %s" %
(poll_exitcode, stderr))
logger.error(self.__fail_message)
return
self.__wipe_done = True
self.__progress = 1.0
logger.info("Wipe finished.")
self.state = protocol.IDLE
def __shell_exec_thread(self, command, shell_exec_id):
logger.info("Shell execution of: %s" % command)
process = Process(command, shell=True)
# Initiate process
process.wait(os.WNOHANG)
self.processes.append(process)
# Blocking wait
process.wait()
self.shell_exec_results[shell_exec_id] = (process.read(), process.readerr())
self.state = protocol.IDLE
def __shell_exec_thread(self, command, shell_exec_id):
logger.info("Shell execution of: %s" % command)
process = Process(command, shell=True)
# Initiate process
process.wait(os.WNOHANG)
self.processes.append(process)
# Blocking wait
process.wait()
self.shell_exec_results[shell_exec_id] = (process.read(),
process.readerr())
self.state = protocol.IDLE
def __badblocks_thread(self, command):
process = Process(command, shell=True)
self.processes.append(process)
re_pct = re.compile(r"(\d+)\.\d+%", re.M)
while True:
# check to see if process has ended
poll_exitcode = process.wait(os.WNOHANG)
if poll_exitcode != None:
break
# Search stderr - yes, actually stderr is the pipe that badblocks
# uses :(
stderr = process.readerr()
m = re_pct.search(stderr)
if m:
pct = int(m.group(1))
self.__progress = pct / 100.0
logger.info("Badblocks progress: %d%%" %
(self.__progress * 100))
else:
logger.debug("Could not understand badblocks output: %s" %
stderr)
# Wait another 2 seconds before polling
time.sleep(2.0)
stderr = process.readerr()
if poll_exitcode > 0:
self.state = protocol.FAIL
self.__fail_message = (
"Failed executing badblocks. Return code: %d, Stderr was: %s" %
(poll_exitcode, stderr))
logger.error(self.__fail_message)
return
self.__progress = 1.0
logger.info("Badblocks finished.")
self.__badblocks_done = True
self.state = protocol.IDLE
def __scan_thread(self, commands):
# Execute a number of commands and return their output in the same order.
command_cnt = float(len(commands))
self.__progress = 0.0
for cnt, command in enumerate(commands, start=1):
logger.info("SCAN executing command: %s" % command)
self.__progress = cnt / command_cnt
logger.debug("Progress: %.2f" % self.__progress)
try:
process = Process(command, shell=True,)
process.wait(os.WNOHANG)
self.processes.append(process)
process.wait()
self.scan_results[command] = (process.read(), process.readerr())
except OSError:
self.scan_results[command] = ("", "Command does not exist")
self.state = protocol.IDLE
def __scan_thread(self, commands):
# Execute a number of commands and return their output in the same order.
command_cnt = float(len(commands))
self.__progress = 0.0
for cnt, command in enumerate(commands, start=1):
logger.info("SCAN executing command: %s" % command)
self.__progress = cnt / command_cnt
logger.debug("Progress: %.2f" % self.__progress)
try:
process = Process(
command,
shell=True,
)
process.wait(os.WNOHANG)
self.processes.append(process)
process.wait()
self.scan_results[command] = (process.read(),
process.readerr())
except OSError:
self.scan_results[command] = ("", "Command does not exist")
self.state = protocol.IDLE
class NEATStrategy(OneStepStrategy):
'''
A strategy that calls a NEAT executable to determine a gait
'''
def __init__(self, *args, **kwargs):
super(NEATStrategy, self).__init__([], [])
##############################
# HyperNEAT Parameters
##############################
now = datetime.now()
#self.identifier = ''.join([random.choice('abcdef1234567890') for x in range(8)])
self.identifier = now.strftime('neat_%y%m%d_%H%M%S')
# self.executable = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/Hypercube_NEAT'
self.executable = '/home/sean/hyperneat/HyperNEAT_v2_5/out/Hypercube_NEAT'
# self.motionFile = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/spiderJointAngles.txt'
self.motionFile = '/home/sean/simtest/spiderJointAngles.txt'
# self.fitnessFile = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/fitness'
self.fitnessFile = '/home/sean/simtest/fitness'
# self.datFile = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/SpiderRobotExperiment.dat'
self.datFile = '/home/sean/hyperneat/HyperNEAT_v2_5/out/SpiderRobotExperiment.dat'
self.avgPoints = 4 # Average over this many points
self.junkPoints = 1000
# How many lines to expect from HyperNEAT file
self.expectedLines = self.junkPoints + 12 * 40 * self.avgPoints
#self.motorColumns = [0,1,4,5,2,3,6,7] # Order of motors in HyperNEAT file
self.motorColumns = [0,1,4,5,2,3,6,7,8] # Order of motors in HyperNEAT file
self.generationSize = 200
self.initNeatFile = kwargs.get('initNeatFile', None) # Pop file to start with, None for restart
self.prefix = 'delme'
# self.nextNeatFile = '%s_pop.xml' % self.prefix
self.nextNeatFile = '/home/sean/simtest/delme_pop.xml'
self.separateLogInfo = True
self.restartHN = kwargs.get('restartHN', True)
# stdout=sp.PIPE, stderr=sp.PIPE, stdin=sp.PIPE)
#os.system('%s -O delme -R 102 -I %s' % (self.executable, self.datFile))
self.spawnProc = True
if self.spawnProc:
if self.initNeatFile is None:
hnout = open('hyperneat.out', 'w')
hnerr = open('hyperneat.err', 'w')
self.proc = Process((self.executable,
'-O', self.prefix, '-R', '102', '-I',
self.datFile),
#stdout = sys.stdout,
#stderr = sys.stderr,
stdout = hnout,
stderr = hnerr,
)
else:
print 'Starting with neatfile', self.initNeatFile
self.proc = Process((self.executable,
'-O', self.prefix, '-R', '102', '-I',
self.datFile, '-X', self.nextNeatFile, '-XG', '1'))
#'%s -O delme -R 102 -I %s' % (self.executable, self.datFile))
self.genId = 0
self.orgId = 0
def __del__(self):
if self.spawnProc:
print 'Waiting for %s to exit...' % self.executable,
code = self.proc.wait()
print 'done.'
print code
def _getNext(self):
'''Get the next point to try. This reads from the file
self.motionFile'''
#print 'TRYING READ STDOUT'
#stdout = self.proc.stdout.read()
#print 'TRYING READ STDERR'
#stderr = self.proc.stderr.read()
#print 'STDOUT:'
#print stdout
#print 'STDERR:'
#print stderr
if self.orgId == self.generationSize:
# print 'Restarting process after %d runs, push enter when ready...' % self.generationSize
if self.restartHN:
print 'Restarting process after %d runs' % self.generationSize
print 'waiting for exit code...'
code = self.proc.wait()
print 'got exit code: %d' % code
# raw_input()
#sleep(10)
if self.spawnProc:
print 'Continuing with neatfile', self.nextNeatFile
self.proc = Process((self.executable,
'-O', self.prefix, '-R', '102', '-I',
self.datFile, '-X', self.nextNeatFile, '-XG', '1'))
self.genId += 1
self.orgId = 0
#print 'On iteration', self.orgId+1
while True:
if self.spawnProc:
out = self.proc.read()
if out != '':
#print 'Got stdout:'
#print out
pass
out = self.proc.readerr()
if out != '':
#print 'Got stderr:'
#print out
pass
try:
ff = open(self.motionFile, 'r')
except IOError:
print 'File does not exist yet'
sleep(1)
if self.proc.wait(os.WNOHANG) is None:
continue
else:
raise Exception('HN died')
lines = ff.readlines()
nLines = len(lines)
if nLines < self.expectedLines:
print ' only %d of %d lines, waiting...' % (nLines, self.expectedLines)
ff.close()
sleep(.5)
continue
break
self.orgId += 1
for ii,line in enumerate(lines[self.junkPoints:]):
#print 'line', ii, 'is', line
nums = [float(xx) for xx in line.split()]
if ii == 0:
rawPositions = array(nums)
else:
rawPositions = vstack((rawPositions, array(nums)))
# swap and scale rawPositions appropriately
rawPositions = rawPositions.T[ix_(self.motorColumns)].T # remap to right columns
#print 'First few lines of rawPositions are:'
#for ii in range(10):
# print prettyVec(rawPositions[ii,:], prec=2)
# Average over every self.avgPoints
for ii in range((self.expectedLines-self.junkPoints)/self.avgPoints):
temp = mean(rawPositions[ii*self.avgPoints:(ii+1)*self.avgPoints,:], 0)
if ii == 0:
positions = temp
else:
positions = vstack((positions, temp))
#print 'First few lines of positions are:'
#for ii in range(10):
# print prettyVec(positions[ii,:], prec=2)
# scale from [-1,1] to [0,1]
positions += 1
positions *= .5
# scale from [0,1] to appropriate ranges
innerIdx = [0, 2, 4, 6]
outerIdx = [1, 3, 5, 7]
centerIdx = [8]
for ii in innerIdx:
positions[:,ii] *= (MAX_INNER - MIN_INNER)
positions[:,ii] += MIN_INNER
for ii in outerIdx:
positions[:,ii] *= (MAX_OUTER - MIN_OUTER)
positions[:,ii] += MIN_OUTER
for ii in centerIdx:
positions[:,ii] *= (MAX_CENTER - MIN_CENTER)
positions[:,ii] += MIN_CENTER
# append a column of 512s for center
#positions = hstack((positions,
# NORM_CENTER * ones((positions.shape[0],1))))
times = linspace(0,12,positions.shape[0])
# Dump both raw positions and positions to file
thisIdentifier = '%s_%05d_%03d' % (self.identifier, self.genId, self.orgId)
ff = open('%s_raw' % thisIdentifier, 'w')
writeArray(ff, rawPositions)
ff.close()
ff = open('%s_filt' % thisIdentifier, 'w')
writeArray(ff, positions)
ff.close()
# return function of time
return lambda time: matInterp(time, times, positions), thisIdentifier
def updateResults(self, dist):
'''This must be called for the last point that was handed out!
This communicates back to the running subprocess.
'''
dist = float(dist)
# MAKE SURE TO CALL super().updateResults!
super(NEATStrategy, self).updateResults(dist)
# Send fitness to process
#out,err = proc.communicate('%f\n' % dist)
print 'Deleting old motion file %s' % self.motionFile
os.remove(self.motionFile)
print 'Sending to process: %f' % dist,
if self.spawnProc:
print 'via process'
self.proc.write('%f\n' % dist)
else:
print 'via file'
# Instead, write this to a file
ff = open('.fitness', 'w')
ff.write('%s\n' % dist)
ff.close()
os.rename('.fitness', self.fitnessFile)
#print 'TRYING READ STDOUT'
#stdout = self.proc.stdout.read()
#print 'TRYING READ STDOUT'
#stderr = self.proc.stderr.read()
#
#print 'Got stdout:'
#print stdout
#print 'Got stderr:'
#print stderr
if self.spawnProc:
out = self.proc.read()
if out != '':
#print 'Got stdout:'
#print out
pass
out = self.proc.readerr()
if out != '':
#print 'Got stderr:'
#print out
pass
def logHeader(self):
return '# NEATStrategy starting, identifier %s\n' % self.identifier
class NEATStrategy(OneStepStrategy):
'''
A strategy that calls a NEAT executable to determine a gait
'''
def __init__(self, *args, **kwargs):
super(NEATStrategy, self).__init__([], [])
##############################
# HyperNEAT Parameters
##############################
now = datetime.now()
#self.identifier = ''.join([random.choice('abcdef1234567890') for x in range(8)])
self.identifier = now.strftime('neat_%y%m%d_%H%M%S')
self.executable = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/Hypercube_NEAT'
self.motionFile = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/spiderJointAngles.txt'
#self.motionFile = 'spiderJointAngles.txt'
self.fitnessFile = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/fitness'
self.datFile = '/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/SpiderRobotExperiment.dat'
self.avgPoints = 4 # Average over this many points
self.junkPoints = 1000
# How many lines to expect from HyperNEAT file
self.expectedLines = self.junkPoints + 12 * 40 * self.avgPoints
#self.motorColumns = [0,1,4,5,2,3,6,7] # Order of motors in HyperNEAT file
self.motorColumns = [0, 1, 4, 5, 2, 3, 6, 7,
8] # Order of motors in HyperNEAT file
self.generationSize = 9
self.initNeatFile = kwargs.get(
'initNeatFile', None) # Pop file to start with, None for restart
self.prefix = 'delme'
self.nextNeatFile = '%s_pop.xml' % self.prefix
self.separateLogInfo = True
#self.proc = sp.Popen((self.executable,
# '-O', 'delme', '-R', '102', '-I', self.datFile),
# stdout=sp.PIPE, stderr=sp.PIPE, stdin=sp.PIPE)
#os.system('%s -O delme -R 102 -I %s' % (self.executable, self.datFile))
self.spawnProc = False
if self.spawnProc:
if self.initNeatFile is None:
self.proc = Process((self.executable, '-O', self.prefix, '-R',
'102', '-I', self.datFile))
else:
print 'Starting with neatfile', self.initNeatFile
self.proc = Process(
(self.executable, '-O', self.prefix, '-R', '102', '-I',
self.datFile, '-X', self.nextNeatFile, '-XG', '1'))
#'%s -O delme -R 102 -I %s' % (self.executable, self.datFile))
self.genId = 0
self.orgId = 0
def __del__(self):
if self.spawnProc:
print 'Waiting for %s to exit...' % self.executable,
code = self.proc.wait()
print 'done.'
def _getNext(self):
'''Get the next point to try. This reads from the file
self.motionFile'''
#print 'TRYING READ STDOUT'
#stdout = self.proc.stdout.read()
#print 'TRYING READ STDERR'
#stderr = self.proc.stderr.read()
#print 'STDOUT:'
#print stdout
#print 'STDERR:'
#print stderr
if self.orgId == self.generationSize:
print 'Restarting process after %d runs, push enter when ready...' % self.generationSize
raw_input()
#sleep(10)
if self.spawnProc:
print 'Continuing with neatfile', self.nextNeatFile
self.proc = Process(
(self.executable, '-O', self.prefix, '-R', '102', '-I',
self.datFile, '-X', self.nextNeatFile, '-XG', '1'))
self.genId += 1
self.orgId = 0
#print 'On iteration', self.orgId+1
while True:
if self.spawnProc:
out = self.proc.read()
if out != '':
#print 'Got stdout:'
#print out
pass
out = self.proc.readerr()
if out != '':
#print 'Got stderr:'
#print out
pass
try:
ff = open(self.motionFile, 'r')
except IOError:
print 'File does not exist yet'
sleep(1)
continue
lines = ff.readlines()
nLines = len(lines)
if nLines < self.expectedLines:
print ' only %d of %d lines, waiting...' % (
nLines, self.expectedLines)
ff.close()
sleep(.5)
continue
break
self.orgId += 1
for ii, line in enumerate(lines[self.junkPoints:]):
#print 'line', ii, 'is', line
nums = [float(xx) for xx in line.split()]
if ii == 0:
rawPositions = array(nums)
else:
rawPositions = vstack((rawPositions, array(nums)))
# swap and scale rawPositions appropriately
rawPositions = rawPositions.T[ix_(
self.motorColumns)].T # remap to right columns
#print 'First few lines of rawPositions are:'
#for ii in range(10):
# print prettyVec(rawPositions[ii,:], prec=2)
# Average over every self.avgPoints
for ii in range(self.expectedLines / self.avgPoints):
temp = mean(rawPositions[ii:(ii + self.avgPoints), :], 0)
if ii == 0:
positions = temp
else:
positions = vstack((positions, temp))
#print 'First few lines of positions are:'
#for ii in range(10):
# print prettyVec(positions[ii,:], prec=2)
# scale from [-1,1] to [0,1]
positions += 1
positions *= .5
# scale from [0,1] to appropriate ranges
innerIdx = [0, 2, 4, 6]
outerIdx = [1, 3, 5, 7]
centerIdx = [8]
for ii in innerIdx:
positions[:, ii] *= (MAX_INNER - MIN_INNER)
positions[:, ii] += MIN_INNER
for ii in outerIdx:
positions[:, ii] *= (MAX_OUTER - MIN_OUTER)
positions[:, ii] += MIN_OUTER
for ii in centerIdx:
positions[:, ii] *= (MAX_CENTER - MIN_CENTER)
positions[:, ii] += MIN_CENTER
# append a column of 512s for center
#positions = hstack((positions,
# NORM_CENTER * ones((positions.shape[0],1))))
times = linspace(0, 12, positions.shape[0])
# Dump both raw positions and positions to file
thisIdentifier = '%s_%05d_%03d' % (self.identifier, self.genId,
self.orgId)
ff = open('%s_raw' % thisIdentifier, 'w')
writeArray(ff, rawPositions)
ff.close()
ff = open('%s_filt' % thisIdentifier, 'w')
writeArray(ff, positions)
ff.close()
# return function of time
return lambda time: matInterp(time, times, positions), thisIdentifier
def updateResults(self, dist):
'''This must be called for the last point that was handed out!
This communicates back to the running subprocess.
'''
dist = float(dist)
# MAKE SURE TO CALL super().updateResults!
super(NEATStrategy, self).updateResults(dist)
# Send fitness to process
#out,err = proc.communicate('%f\n' % dist)
print 'Deleting old motion file %s' % self.motionFile
os.remove(self.motionFile)
print 'Sending to process: %f' % dist,
if self.spawnProc:
print 'via process'
self.proc.write('%f\n' % dist)
else:
print 'via file'
# Instead, write this to a file
ff = open('.fitness', 'w')
ff.write('%s\n' % dist)
ff.close()
os.rename('.fitness', self.fitnessFile)
#print 'TRYING READ STDOUT'
#stdout = self.proc.stdout.read()
#print 'TRYING READ STDOUT'
#stderr = self.proc.stderr.read()
#
#print 'Got stdout:'
#print stdout
#print 'Got stderr:'
#print stderr
if self.spawnProc:
out = self.proc.read()
if out != '':
#print 'Got stdout:'
#print out
pass
out = self.proc.readerr()
if out != '':
#print 'Got stderr:'
#print out
pass
def logHeader(self):
return '# NEATStrategy starting, identifier %s\n' % self.identifier
class GPGEncryption():
def __init__(self, encrypt_or_decrypt, iterable = None, user = None, passphrase = None, test_branch = False):
self.test_branch = test_branch
print 'initing gpg'
if(encrypt_or_decrypt[0] == 'e'):
#print "user = "******""
self.gpg_closed = False
self.padded = False
self.first_read = True
else:
#cmd = ['gpg', '-d', '--batch', '--passphrase-fd', '0']
#cmd = ['gpg', '--no-tty', '--homedir', '/etc/swift/gnupg', '-d', '--batch', '--passphrase-fd', '0']
#cmd = ['tr', 'd', 'D']
#cmd = 'gpg -d --batch --passphrase-fd 0'
cmd = 'cat'
#cmd = '/home/amedeiro/bin/python MyCat.py'
#self.str_iter = stringIterate(self.buffer, read_size)
self.p = Process(cmd, shell=True, bufsize = 2)
# TODO send stderr output to the logs
#self.p = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE, bufsize = 1024)
#self.p = Popen(cmd, stdin=PIPE, stdout=PIPE, bufsize = 4096)
if(encrypt_or_decrypt[0] == 'd'):
self.p.write('test-swift' + '\n')
self.my_buffer = ''
def print_errors(self):
print self.p.readerr()
def stringIterate(self, text, chunk_size):
index = 0
while index < len(text):
yield text[index : index + chunk_size]
index += chunk_size
def readFromStream():
chunk = self.stream_iter_read(read_size)
while chunk:
self.p.stdin.write(chunk)
chunk = self.stream_iter_read(read_size)
self.p.stdin.close()
def read(self, read_size):
def padder(text, div, pad = chr(0)):
print 'padder reading ' + text
size = (div - len(text) % div) + len(text)
diff = size - len(text)
text += pad * diff
return text
if self.first_read:
self.first_read = False
t = Thread(target = readFromStream)
t.start()
return self.p.stdout.read(read_size)
def digest(self, chunk):
print "digesting chunk = " + chunk
self.p.write(chunk)
def dump_buffer(self):
buf = self.p.read()
print 'dump buffer returning: ' + buf
return buf
def close_and_dump(self, timeout = .2):
print 'close and dump called'
self.p.closeinput()
time.sleep(timeout)
#self.p.closeinput()
buf = ''
#as long as there was a non blank string digested before this is called,
#we can wait for the process to dump its last output... Might be a problem
#if output dumps
#poll = self.p.wait(os.WNOHANG)
print 'entering close and dump loop'
count = 0
while buf == '' and count < 100:
buf = self.dump_buffer()
time.sleep(.01)
print self.p.readerr()
poll = self.p.wait(os.WNOHANG)
count = count + 1
print 'leaving close and dump loop'
poll = self.p.wait(os.WNOHANG)
print poll
return buf
class NEATStrategy(OneStepStrategy):
"""
A strategy that calls a NEAT executable to determine a gait
"""
def __init__(self, *args, **kwargs):
super(NEATStrategy, self).__init__([], [])
##############################
# HyperNEAT Parameters
##############################
now = datetime.now()
# self.identifier = ''.join([random.choice('abcdef1234567890') for x in range(8)])
self.identifier = now.strftime("neat_%y%m%d_%H%M%S")
self.executable = "/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/Hypercube_NEAT"
self.motionFile = "/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/spiderJointAngles.txt"
# self.motionFile = 'spiderJointAngles.txt'
self.fitnessFile = "/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/fitness"
self.datFile = "/home/team/s/h2_synced/HyperNEAT_v2_5/out/hyperneatTo20gens_102/SpiderRobotExperiment.dat"
self.avgPoints = 4 # Average over this many points
self.junkPoints = 1000
# How many lines to expect from HyperNEAT file
self.expectedLines = self.junkPoints + 12 * 40 * self.avgPoints
# self.motorColumns = [0,1,4,5,2,3,6,7] # Order of motors in HyperNEAT file
self.motorColumns = [0, 1, 4, 5, 2, 3, 6, 7, 8] # Order of motors in HyperNEAT file
self.generationSize = 9
self.initNeatFile = kwargs.get("initNeatFile", None) # Pop file to start with, None for restart
self.prefix = "delme"
self.nextNeatFile = "%s_pop.xml" % self.prefix
self.separateLogInfo = True
# self.proc = sp.Popen((self.executable,
# '-O', 'delme', '-R', '102', '-I', self.datFile),
# stdout=sp.PIPE, stderr=sp.PIPE, stdin=sp.PIPE)
# os.system('%s -O delme -R 102 -I %s' % (self.executable, self.datFile))
self.spawnProc = False
if self.spawnProc:
if self.initNeatFile is None:
self.proc = Process((self.executable, "-O", self.prefix, "-R", "102", "-I", self.datFile))
else:
print "Starting with neatfile", self.initNeatFile
self.proc = Process(
(
self.executable,
"-O",
self.prefix,
"-R",
"102",
"-I",
self.datFile,
"-X",
self.nextNeatFile,
"-XG",
"1",
)
)
#'%s -O delme -R 102 -I %s' % (self.executable, self.datFile))
self.genId = 0
self.orgId = 0
def __del__(self):
if self.spawnProc:
print "Waiting for %s to exit..." % self.executable,
code = self.proc.wait()
print "done."
def _getNext(self):
"""Get the next point to try. This reads from the file
self.motionFile"""
# print 'TRYING READ STDOUT'
# stdout = self.proc.stdout.read()
# print 'TRYING READ STDERR'
# stderr = self.proc.stderr.read()
# print 'STDOUT:'
# print stdout
# print 'STDERR:'
# print stderr
if self.orgId == self.generationSize:
print "Restarting process after %d runs, push enter when ready..." % self.generationSize
raw_input()
# sleep(10)
if self.spawnProc:
print "Continuing with neatfile", self.nextNeatFile
self.proc = Process(
(
self.executable,
"-O",
self.prefix,
"-R",
"102",
"-I",
self.datFile,
"-X",
self.nextNeatFile,
"-XG",
"1",
)
)
self.genId += 1
self.orgId = 0
# print 'On iteration', self.orgId+1
while True:
if self.spawnProc:
out = self.proc.read()
if out != "":
# print 'Got stdout:'
# print out
pass
out = self.proc.readerr()
if out != "":
# print 'Got stderr:'
# print out
pass
try:
ff = open(self.motionFile, "r")
except IOError:
print "File does not exist yet"
sleep(1)
continue
lines = ff.readlines()
nLines = len(lines)
if nLines < self.expectedLines:
print " only %d of %d lines, waiting..." % (nLines, self.expectedLines)
ff.close()
sleep(0.5)
continue
break
self.orgId += 1
for ii, line in enumerate(lines[self.junkPoints :]):
# print 'line', ii, 'is', line
nums = [float(xx) for xx in line.split()]
if ii == 0:
rawPositions = array(nums)
else:
rawPositions = vstack((rawPositions, array(nums)))
# swap and scale rawPositions appropriately
rawPositions = rawPositions.T[ix_(self.motorColumns)].T # remap to right columns
# print 'First few lines of rawPositions are:'
# for ii in range(10):
# print prettyVec(rawPositions[ii,:], prec=2)
# Average over every self.avgPoints
for ii in range(self.expectedLines / self.avgPoints):
temp = mean(rawPositions[ii : (ii + self.avgPoints), :], 0)
if ii == 0:
positions = temp
else:
positions = vstack((positions, temp))
# print 'First few lines of positions are:'
# for ii in range(10):
# print prettyVec(positions[ii,:], prec=2)
# scale from [-1,1] to [0,1]
positions += 1
positions *= 0.5
# scale from [0,1] to appropriate ranges
innerIdx = [0, 2, 4, 6]
outerIdx = [1, 3, 5, 7]
centerIdx = [8]
for ii in innerIdx:
positions[:, ii] *= MAX_INNER - MIN_INNER
positions[:, ii] += MIN_INNER
for ii in outerIdx:
positions[:, ii] *= MAX_OUTER - MIN_OUTER
positions[:, ii] += MIN_OUTER
for ii in centerIdx:
positions[:, ii] *= MAX_CENTER - MIN_CENTER
positions[:, ii] += MIN_CENTER
# append a column of 512s for center
# positions = hstack((positions,
# NORM_CENTER * ones((positions.shape[0],1))))
times = linspace(0, 12, positions.shape[0])
# Dump both raw positions and positions to file
thisIdentifier = "%s_%05d_%03d" % (self.identifier, self.genId, self.orgId)
ff = open("%s_raw" % thisIdentifier, "w")
writeArray(ff, rawPositions)
ff.close()
ff = open("%s_filt" % thisIdentifier, "w")
writeArray(ff, positions)
ff.close()
# return function of time
return lambda time: matInterp(time, times, positions), thisIdentifier
def updateResults(self, dist):
"""This must be called for the last point that was handed out!
This communicates back to the running subprocess.
"""
dist = float(dist)
# MAKE SURE TO CALL super().updateResults!
super(NEATStrategy, self).updateResults(dist)
# Send fitness to process
# out,err = proc.communicate('%f\n' % dist)
print "Deleting old motion file %s" % self.motionFile
os.remove(self.motionFile)
print "Sending to process: %f" % dist,
if self.spawnProc:
print "via process"
self.proc.write("%f\n" % dist)
else:
print "via file"
# Instead, write this to a file
ff = open(".fitness", "w")
ff.write("%s\n" % dist)
ff.close()
os.rename(".fitness", self.fitnessFile)
# print 'TRYING READ STDOUT'
# stdout = self.proc.stdout.read()
# print 'TRYING READ STDOUT'
# stderr = self.proc.stderr.read()
#
# print 'Got stdout:'
# print stdout
# print 'Got stderr:'
# print stderr
if self.spawnProc:
out = self.proc.read()
if out != "":
# print 'Got stdout:'
# print out
pass
out = self.proc.readerr()
if out != "":
# print 'Got stderr:'
# print out
pass
def logHeader(self):
return "# NEATStrategy starting, identifier %s\n" % self.identifier
