def startVMCLIapp(self, isofile):
self.createAPISession()
self.stopVMCLIapp()
print(self.host + ' Starting VMCLI Service with ' + isofile)
time.sleep(1)
if 'win' in sys.platform:
cwd = os.getcwd()
# cmd = 'sc start VMCLI_' + self.host + ' -r [' + self.host + ']:443 -u ' + self.username + ' -p ' + self.password + ' -c "' + cwd + '\\' + isofile + '"'
cmd = "VMCLI.exe -r [" + self.host + "]:443 -u " + self.username + " -p " + self.password + " -c " + cwd + "/" + isofile
cmd = cmd.replace("\\", "/")
count = 0
while count < 10:
try:
# output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT)
# print(self.host + ' ' + output)
session = PopenSpawn(cmd,
timeout=30,
encoding='utf-8',
searchwindowsize=100)
self.VMCLISession = session
session.expect('Starting CD redirection', timeout=30)
except:
pass
time.sleep(30)
if self.statusVMCLIapp():
break
else:
print(self.host + " VMCLI Failed to start. Attempt #" +
str(count))
session.kill(signal.CTRL_C_EVENT)
count += 1
else:
self.startTunnel(['443', '5120'])
cmd = 'VMCLIEXE -r [' + self.host.split('%')[0].split(
'fe80'
)[1] + ']:443 -u ' + self.username + ' -p ' + self.password + ' -c ' + isofile
session = self.spawn(cmd, encoding='utf-8')
try:
session.expect('CD redirection in progress')
self.VMCLISession = session
time.sleep(1)
print(self.host + ' ' + ' CD redirection in progress')
except:
print(self.host + 'VMCLI Failed to start')
if not self.statusVMCLIapp():
print(self.host + ' VMCLI Failed to start')
else:
print(self.host + ' CD redirection in progress')
self.destroyAPISession()
class MMGenPexpect(object):
NL = '\r\n'
if g.platform == 'linux' and opt.popen_spawn:
import atexit
atexit.register(lambda: os.system('stty sane'))
NL = '\n'
def __init__(self,name,mmgen_cmd,cmd_args,desc,no_output=False,passthru_args=[],msg_only=False,no_msg=False):
cmd_args = ['--{}{}'.format(k.replace('_','-'),
'='+getattr(opt,k) if getattr(opt,k) != True else ''
) for k in passthru_args if getattr(opt,k)] \
+ ['--data-dir='+data_dir] + cmd_args
if g.platform == 'win': cmd,args = 'python',[mmgen_cmd]+cmd_args
else: cmd,args = mmgen_cmd,cmd_args
for i in args:
if type(i) not in (str,unicode):
m1 = 'Error: missing input files in cmd line?:'
m2 = '\nName: {}\nCmd: {}\nCmd args: {}'
die(2,(m1+m2).format(name,cmd,args))
if opt.popen_spawn:
args = [u'{q}{}{q}'.format(a,q="'" if ' ' in a else '') for a in args]
cmd_str = u'{} {}'.format(cmd,u' '.join(args)).replace('\\','/')
if opt.coverage:
fs = 'python -m trace --count --coverdir={} --file={} {c}'
cmd_str = fs.format(*init_coverage(),c=cmd_str)
if opt.log:
log_fd.write(cmd_str+'\n')
if not no_msg:
if opt.verbose or opt.print_cmdline or opt.exact_output:
clr1,clr2,eol = ((green,cyan,'\n'),(nocolor,nocolor,' '))[bool(opt.print_cmdline)]
sys.stderr.write(green('Testing: {}\n'.format(desc)))
if not msg_only:
sys.stderr.write(clr1(u'Executing {}{}'.format(clr2(cmd_str),eol)))
else:
m = 'Testing {}: '.format(desc)
msg_r(m)
if msg_only: return
if opt.direct_exec:
msg('')
from subprocess import call,check_output
f = (call,check_output)[bool(no_output)]
ret = f([cmd] + args)
if f == call and ret != 0:
die(1,red('ERROR: process returned a non-zero exit status ({})'.format(ret)))
else:
if opt.traceback:
cmd,args = g.traceback_cmd,[cmd]+args
cmd_str = g.traceback_cmd + ' ' + cmd_str
# Msg('\ncmd_str: {}'.format(cmd_str))
if opt.popen_spawn:
self.p = PopenSpawn(cmd_str.encode('utf8'))
else:
self.p = pexpect.spawn(cmd,args)
if opt.exact_output: self.p.logfile = sys.stdout
def ok(self,exit_val=0):
ret = self.p.wait()
# Msg('expect: {} got: {}'.format(exit_val,ret))
if ret != exit_val and not opt.coverage:
die(1,red('test.py: spawned program exited with value {}'.format(ret)))
if opt.profile: return
if opt.verbose or opt.exact_output:
sys.stderr.write(green('OK\n'))
else: msg(' OK')
def cmp_or_die(self,s,t,skip_ok=False,exit_val=0):
ret = self.p.wait()
if ret != exit_val:
rdie(1,'test.py: spawned program exited with value {}'.format(ret))
if s == t:
if not skip_ok: ok()
else:
fs = 'ERROR: recoded data:\n{}\ndiffers from original data:\n{}'
rdie(3,fs.format(repr(t),repr(s)))
def license(self):
if 'MMGEN_NO_LICENSE' in os.environ: return
p = "'w' for conditions and warranty info, or 'c' to continue: "
my_expect(self.p,p,'c')
def label(self,label='Test Label'):
p = 'Enter a wallet label, or hit ENTER for no label: '
my_expect(self.p,p,label+'\n')
def usr_rand_out(self,saved=False):
fs = 'Generating encryption key from OS random data plus {}user-supplied entropy'
my_expect(self.p,fs.format(('','saved ')[saved]))
def usr_rand(self,num_chars):
if opt.usr_random:
self.interactive()
my_send(self.p,'\n')
else:
rand_chars = list(getrandstr(num_chars,no_space=True))
my_expect(self.p,'symbols left: ','x')
try:
vmsg_r('SEND ')
while self.p.expect('left: ',0.1) == 0:
ch = rand_chars.pop(0)
msg_r(yellow(ch)+' ' if opt.verbose else '+')
self.p.send(ch)
except:
vmsg('EOT')
my_expect(self.p,'ENTER to continue: ','\n')
def passphrase_new(self,desc,passphrase):
my_expect(self.p,'Enter passphrase for {}: '.format(desc),passphrase+'\n')
my_expect(self.p,'Repeat passphrase: ',passphrase+'\n')
def passphrase(self,desc,passphrase,pwtype=''):
if pwtype: pwtype += ' '
my_expect(self.p,
'Enter {}passphrase for {}.*?: '.format(pwtype,desc),
passphrase+'\n',regex=True)
def hash_preset(self,desc,preset=''):
my_expect(self.p,'Enter hash preset for {}'.format(desc))
my_expect(self.p,'or hit ENTER .*?:',str(preset)+'\n',regex=True)
def written_to_file(self,desc,overwrite_unlikely=False,query='Overwrite? ',oo=False):
s1 = '{} written to file '.format(desc)
s2 = query + "Type uppercase 'YES' to confirm: "
ret = my_expect(self.p,([s1,s2],s1)[overwrite_unlikely])
if ret == 1:
my_send(self.p,'YES\n')
# if oo:
outfile = self.expect_getend("Overwriting file '").rstrip("'").decode('utf8')
return outfile
# else:
# ret = my_expect(self.p,s1)
self.expect(self.NL,nonl=True)
outfile = self.p.before.strip().strip("'").decode('utf8')
if opt.debug_pexpect: rmsg('Outfile [{}]'.format(outfile))
vmsg(u'{} file: {}'.format(desc,cyan(outfile.replace("'",''))))
return outfile
def no_overwrite(self):
self.expect("Overwrite? Type uppercase 'YES' to confirm: ",'\n')
self.expect('Exiting at user request')
def tx_view(self,view=None):
repl = { 'terse':'t', 'full':'v' }[view] if view else 'n'
my_expect(self.p,r'View .*?transaction.*? \(y\)es, \(N\)o, pager \(v\)iew.*?: ',repl,regex=True)
if repl == 't':
my_expect(self.p,r'any key to continue: ','\n')
def expect_getend(self,s,regex=False):
ret = self.expect(s,regex=regex,nonl=True)
debug_pexpect_msg(self.p)
# end = self.readline().strip()
# readline() of partial lines doesn't work with PopenSpawn, so do this instead:
self.expect(self.NL,nonl=True,silent=True)
debug_pexpect_msg(self.p)
end = self.p.before
if not g.debug:
vmsg(' ==> {}'.format(cyan(end)))
return end
def interactive(self):
return self.p.interact() # interact() not available with popen_spawn
def kill(self,signal):
return self.p.kill(signal)
def logfile(self,arg):
self.p.logfile = arg
def expect(self,*args,**kwargs):
return my_expect(self.p,*args,**kwargs)
def send(self,*args,**kwargs):
return my_send(self.p,*args,**kwargs)
# def readline(self):
# return self.p.readline()
# def readlines(self):
# return [l.rstrip()+'\n' for l in self.p.readlines()]
def read(self,n=None):
return self.p.read(n)
def close(self):
if not opt.popen_spawn:
self.p.close()
class SerAPI:
def __init__(self, timeout, debug=False):
'Initialize the SerAPI subprocess'
self.debug = debug
try:
self.proc = PopenSpawn('sertop --implicit --omit_loc --print0',
encoding='utf-8',
timeout=timeout,
maxread=10000000)
except FileNotFoundError:
log(
'Please make sure the "sertop" program is in the PATH.\nYou may have to run "eval $(opam env)".',
'ERROR')
sys.exit(1)
self.proc.expect_exact(
'(Feedback((doc_id 0)(span_id 1)(route 0)(contents Processed)))\0')
self.send('Noop')
self.states_stack = []
# global printing options
self.execute('Unset Printing Notations.')
self.execute('Unset Printing Wildcard.')
self.execute('Set Printing Coercions.')
self.execute('Unset Printing Allow Match Default Clause.')
self.execute('Unset Printing Factorizable Match Patterns.')
self.execute('Unset Printing Compact Contexts.')
self.execute('Set Printing Implicit.')
self.execute('Set Printing Depth 999999.')
self.execute('Unset Printing Records.')
# initialize the state stack
self.push()
self.ast_cache = {}
self.dead = False
def set_timeout(self, timeout):
self.proc.timeout = timeout
def get_timeout(self):
return proc.timeout
def send(self, cmd):
'Send a command to SerAPI and retrieve the responses'
#print(cmd)
assert '\n' not in cmd
self.proc.sendline(cmd)
try:
self.proc.expect([
'\(Answer \d+ Ack\)\x00.*\(Answer \d+ Completed\)\x00',
'\(Answer \d+ Ack\)\x00.*\(Answer \d+\(CoqExn.*\)\x00'
])
except pexpect.TIMEOUT as ex:
print(self.proc.before)
raise CoqTimeout
raw_responses = self.proc.after
#print(raw_responses)
ack_num = int(
re.search(r'^\(Answer (?P<num>\d+)', raw_responses)['num'])
for num in re.findall(r'(?<=\(Answer) \d+', raw_responses):
assert int(num) == ack_num
responses = []
msg_str = []
for item in raw_responses.split('\x00'):
item = item.strip()
if item == '':
continue
if not item.startswith('(Feedback') and not item.startswith(
'(Answer'):
m = re.search(r'\(Feedback|\(Answer', item)
if m is None:
continue
item = item[m.span()[0]:]
assert item.endswith(')')
parsed_item = sexpdata.loads(item, nil=None, true=None)
if 'CoqExn' in item: # an error occured in Coq
assert parsed_item[2][0] == Symbol('CoqExn')
raise CoqExn(sexpdata.dumps(parsed_item[2][4]),
sexpdata.dumps(parsed_item[2]))
if item.startswith('(Feedback'): # ignore Feedback for now
try:
msg = parsed_item[1][3][1]
if isinstance(msg,
list) and msg != [] and msg[0] == Symbol(
'Message'):
msg_sexp, _ = self.send(
'(Print ((pp_format PpStr)) (CoqPp %s))' %
sexpdata.dumps(msg[3]))
msg_str.extend(
[symbol2str(x[1]) for x in msg_sexp[1][2][1]])
except IndexError:
pass
continue
responses.append(parsed_item)
msg_str = '\n'.join(msg_str)
return responses, raw_responses
def send_add(self, cmd, return_ast):
'Send a (Add () "XXX") command to SerAPI, return the state id and optionally the AST'
responses, raw_responses = self.send('(Add () "%s")' % escape(cmd))
state_ids = [
int(sid) for sid in ADDED_STATE_PATTERN.findall(raw_responses)
]
state_id = state_ids[-1]
if self.states_stack != []:
self.states_stack[-1].append(state_id)
if return_ast:
if cmd not in self.ast_cache:
self.ast_cache[cmd] = self.query_ast(cmd)
ast = self.ast_cache[cmd]
else:
ast = None
return state_id, ast
def query_ast(self, cmd):
'Query the AST of the vernac command just added'
responses, _ = self.send('(Parse () "%s")' % escape(cmd))
ast = responses[1][2][1][0]
assert ast[0] == Symbol('CoqAst')
return ast
def query_library(self, lib):
responses, _ = self.send('(Query () (LocateLibrary "%s"))' % lib)
physical_path = symbol2str(responses[1][2][1][0][3])
return physical_path
def query_qualid(self, qualid):
responses, _ = self.send('(Query () (Locate "%s"))' % qualid)
if responses[1][2][1] == [] and qualid.startswith('SerTop.'):
qualid = qualid[len('SerTop.'):]
responses, _ = self.send('(Query () (Locate "%s"))' % qualid)
assert len(responses[1][2][1]) == 1
short_responses = responses[1][2][1][0][1][0][1]
assert short_responses[1][0] == Symbol('DirPath')
short_ident = '.'.join(
[symbol2str(x[1]) for x in short_responses[1][1][::-1]] +
[symbol2str(short_responses[2][1])])
return short_ident
def query_env(self, current_file):
'Query the global environment'
responses, _ = self.send('(Query () Env)')
env = responses[1][2][1][0]
# store the constants
constants = []
for const in env[1][0][1][0][1]:
# identifier
qualid = print_mod_path(const[0][1]) + '.' + \
'.'.join([symbol2str(x[1]) for x in const[0][2][1][::-1]] + [symbol2str(const[0][3][1])])
if qualid.startswith('SerTop.'):
logical_path = 'SerTop'
physical_path = current_file
else:
logical_path = mod_path_file(const[0][1])
assert qualid.startswith(logical_path)
physical_path = os.path.relpath(
self.query_library(logical_path))
physical_path += ':' + qualid[len(logical_path) + 1:]
short_ident = self.query_qualid(qualid)
# term
assert const[1][0][1][0] == Symbol('const_body')
if const[1][0][1][1][0] == Symbol('Undef'): # delaration
opaque = None
term = None
elif const[1][0][1][1][0] == Symbol(
'Def'): # transparent definition
opaque = False
term = None
else:
assert const[1][0][1][1][0] == Symbol(
'OpaqueDef') # opaque definition
opaque = True
term = None
# type
assert const[1][0][2][0] == Symbol('const_type')
type_sexp = sexpdata.dumps(const[1][0][2][1])
type = self.print_constr(type_sexp)
sort = self.query_type(type_sexp, return_str=True)
constants.append({
'physical_path': physical_path,
'short_ident': short_ident,
'qualid': qualid,
'term': term,
'type': type,
'sort': sort,
'opaque': opaque,
'sexp': sexpdata.dumps(const[1][0][2][1])
})
# store the inductives
inductives = []
for induct in env[1][0][1][1][1]:
# identifier
qualid = print_mod_path(induct[0][1]) + '.' + \
'.'.join([symbol2str(x[1]) for x in induct[0][2][1][::-1]] + [symbol2str(induct[0][3][1])])
short_ident = self.query_qualid(qualid)
if qualid.startswith('SerTop.'):
logical_path = 'SerTop'
physical_path = current_file
else:
logical_path = mod_path_file(induct[0][1])
physical_path = os.path.relpath(
self.query_library(logical_path))
assert qualid.startswith(logical_path)
physical_path += ':' + qualid[len(logical_path) + 1:]
# blocks
blocks = []
for blk in induct[1][0][0][1]:
blk_qualid = '.'.join(
qualid.split('.')[:-1] + [symbol2str(blk[0][1][1])])
blk_short_ident = self.query_qualid(blk_qualid)
# constructors
constructors = []
for c_name, c_type in zip(blk[3][1], blk[4][1]):
c_name = symbol2str(c_name[1])
c_type = self.print_constr(sexpdata.dumps(c_type))
#if c_type is not None:
# c_type = UNBOUND_REL_PATTERN.sub(short_ident, c_type)
constructors.append((c_name, c_type))
blocks.append({
'short_ident': blk_short_ident,
'qualid': blk_qualid,
'constructors': constructors
})
inductives.append({
'physical_path':
physical_path,
'blocks':
blocks,
'is_record':
induct[1][0][1][1] != Symbol('NotRecord'),
'sexp':
sexpdata.dumps(induct)
})
return constants, inductives
def query_goals(self):
'Retrieve a list of open goals'
responses, _ = self.send('(Query () Goals)')
assert responses[1][2][0] == Symbol('ObjList')
if responses[1][2][1] == []: # no goals
return [], [], [], []
else:
assert len(responses[1][2][1]) == 1
def store_goals(goals_sexp):
goals = []
for g in goals_sexp:
hypotheses = []
for h in g[2][1]:
h_sexp = sexpdata.dumps(h[2])
hypotheses.append({
'idents':
[symbol2str(ident[1]) for ident in h[0][::-1]],
'term': [
None if t == [] else self.print_constr(
sexpdata.dumps(t)) for t in h[1]
],
'type':
self.print_constr(h_sexp),
'sexp':
h_sexp
})
type_sexp = sexpdata.dumps(g[1][1])
goals.append({
'id': int(g[0][1]),
'type': self.print_constr(type_sexp),
'sexp': type_sexp,
'hypotheses': hypotheses[::-1]
})
return goals
fg_goals = store_goals(responses[1][2][1][0][1][0][1])
bg_goals = store_goals(
list(
chain.from_iterable(
chain.from_iterable(responses[1][2][1][0][1][1][1]))))
shelved_goals = store_goals(responses[1][2][1][0][1][2][1])
given_up_goals = store_goals(responses[1][2][1][0][1][3][1])
return fg_goals, bg_goals, shelved_goals, given_up_goals
def has_open_goals(self):
responses, _ = self.send('(Query () Goals)')
assert responses[1][2][0] == Symbol('ObjList')
return responses[1][2][1] != []
def print_constr(self, sexp_str):
if not hasattr(self, 'constr_cache'):
self.constr_cache = {}
if sexp_str not in self.constr_cache:
try:
responses, _ = self.send(
'(Print ((pp_format PpStr)) (CoqConstr %s))' % sexp_str)
self.constr_cache[sexp_str] = normalize_spaces(
symbol2str(responses[1][2][1][0][1]))
except CoqExn as ex:
if ex.err_msg == 'Not_found':
return None
else:
raise ex
except TypeError as ex:
self.constr_cache[sexp_str] = normalize_spaces(
symbol2str(responses[0][2][1][0][1]))
return self.constr_cache[sexp_str]
def query_vernac(self, cmd):
return self.send('(Query () (Vernac "%s"))' % escape(cmd))
def query_type(self, term_sexp, return_str=False):
try:
responses, _ = self.send('(Query () (Type %s))' % term_sexp)
except CoqExn as ex:
if ex.err_msg == 'Not_found':
return None
else:
raise ex
assert responses[1][2][1][0][0] == Symbol('CoqConstr')
type_sexp = responses[1][2][1][0][1]
if return_str:
return self.print_constr(sexpdata.dumps(type_sexp))
else:
return type_sexp
def execute(self, cmd, return_ast=False):
'Execute a vernac command'
state_id, ast = self.send_add(cmd, return_ast)
responses, _ = self.send('(Exec %d)' % state_id)
return responses, sexpdata.dumps(ast)
def push(self):
'push a new frame on the state stack (a checkpoint), which can be used to roll back to the current state'
self.states_stack.append([])
def cancel(self, states):
self.send('(Cancel (%s))' % ' '.join([str(s) for s in states]))
def pull(self):
'remove a checkpoint created by push'
states = self.states_stack.pop()
self.states_stack[-1].extend(states)
def pop(self):
'rollback to a checkpoint created by push'
self.cancel(self.states_stack.pop())
def clean(self):
self.proc.sendeof()
self.proc.wait()
self.dead = True
def shutdown(self):
self.proc.kill(signal.SIGKILL)
self.dead = True
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
self.clean()
def launch_wasabi(wallet,
launch_path='',
destination='',
keepalive=True,
pwd=''):
"""
Launch Wasabi daemon.
Return pexpect child if successful or return False if Wasabi didn't
started.
Wasabi daemon's documentation:
https://docs.wasabiwallet.io/using-wasabi/Daemon.html#headless-wasabi-daemon
"""
command = ''
# If no launch_path provided, assume deb package is installed.
if not launch_path:
command += 'wassabee mix'
# If launch_path is provided, if WalletWasabi.Gui in it,
# assume source code.
elif 'WalletWasabi.Gui' in launch_path:
command += 'dotnet run --mix'
# If launch_path is provided, if WalletWasabi.Gui not in it,
# assume targz.
else:
command += './wassabee mix'
command += ' --wallet:{}'.format(wallet)
if destination:
command += ' --destination:{}'.format(destination)
if keepalive:
command += ' --keepalive'
print('Starting Wasabi')
if launch_path:
try:
wasabi_proc = PopenSpawn(command, cwd=launch_path)
except FileNotFoundError as e:
raise FileNotFoundError(
str(e) + ', check that your ' +
'"launch_path" setting is correct')
else:
wasabi_proc = PopenSpawn(command)
index = wasabi_proc.expect_exact(
['Password:'******'selected wallet does not exist', TIMEOUT, EOF],
timeout=30)
if index == 0:
wasabi_proc.sendline(pwd)
elif index == 1:
raise MyExceptions.WalletMissing(name + '.json wallet does not exist')
elif index == 2:
raise MyExceptions.ProcessTimeout('Wasabi process TIMEOUT')
elif index == 3:
raise EOFError('Pexpect EOF')
index = wasabi_proc.expect_exact(
['Correct password', 'Wrong password', TIMEOUT, EOF], timeout=30)
if index == 0:
print('Wasabi daemon started')
elif index == 1:
wasabi_proc.kill(SIGTERM)
wasabi_proc.wait()
raise MyExceptions.WrongPassword('Wrong password')
elif index == 2:
raise MyExceptions.ProcessTimeout('Wasabi process TIMEOUT')
elif index == 3:
raise EOFError('Pexpect EOF')
index = wasabi_proc.expect_exact(['Starting Wallet', TIMEOUT, EOF],
timeout=30)
if index == 0:
print('Wallet starting')
elif index == 1:
raise MyExceptions.ProcessTimeout('Wasabi process TIMEOUT')
elif index == 2:
raise EOFError('Pexpect EOF')
if is_wasabi_running():
print('Wasabi started')
return wasabi_proc
else:
return False
class MMGenPexpect(object):
def __init__(self,args,no_output=False):
if opt.direct_exec:
msg('')
from subprocess import call,check_output
f = (call,check_output)[bool(no_output)]
ret = f([args[0]] + args[1:])
if f == call and ret != 0:
die(1,red('ERROR: process returned a non-zero exit status ({})'.format(ret)))
else:
if opt.pexpect_spawn:
self.p = pexpect.spawn(args[0],args[1:],encoding='utf8')
self.p.delaybeforesend = 0
else:
self.p = PopenSpawn(args,encoding='utf8')
# self.p.delaybeforesend = 0 # TODO: try this here too
if opt.exact_output: self.p.logfile = sys.stdout
self.req_exit_val = 0
self.skip_ok = False
self.timeout = int(opt.pexpect_timeout or 0) or (60,5)[bool(opt.debug_pexpect)]
self.sent_value = None
def do_decrypt_ka_data(self,hp,pw,desc='key-address data',check=True,have_yes_opt=False):
# self.hash_preset(desc,hp)
self.passphrase(desc,pw)
if not have_yes_opt:
self.expect('Check key-to-address validity? (y/N): ',('n','y')[check])
def view_tx(self,view):
self.expect('View.* transaction.*\? .*: ',view,regex=True)
if view not in 'n\n':
self.expect('to continue: ','\n')
def do_comment(self,add_comment,has_label=False):
p = ('Add a comment to transaction','Edit transaction comment')[has_label]
self.expect('{}? (y/N): '.format(p),('n','y')[bool(add_comment)])
if add_comment:
self.expect('Comment: ',add_comment+'\n')
def ok(self):
ret = self.p.wait()
if ret != self.req_exit_val and not opt.coverage:
die(1,red('test.py: spawned program exited with value {}'.format(ret)))
if opt.profile: return
if not self.skip_ok:
sys.stderr.write(green('OK\n') if opt.exact_output or opt.verbose else (' OK\n'))
return self
def license(self):
if 'MMGEN_NO_LICENSE' in os.environ: return
self.expect("'w' for conditions and warranty info, or 'c' to continue: ",'c')
def label(self,label='Test Label (UTF-8) α'):
self.expect('Enter a wallet label, or hit ENTER for no label: ',label+'\n')
def usr_rand_out(self,saved=False):
fs = 'Generating encryption key from OS random data plus {}user-supplied entropy'
self.expect(fs.format(('','saved ')[saved]))
def usr_rand(self,num_chars):
if opt.usr_random:
self.interactive()
self.send('\n')
else:
rand_chars = list(getrandstr(num_chars,no_space=True))
vmsg_r('SEND ')
while rand_chars:
ch = rand_chars.pop(0)
msg_r(yellow(ch)+' ' if opt.verbose else '+')
ret = self.expect('left: ',ch,delay=0.005)
self.expect('ENTER to continue: ','\n')
def passphrase_new(self,desc,passphrase):
self.expect('Enter passphrase for {}: '.format(desc),passphrase+'\n')
self.expect('Repeat passphrase: ',passphrase+'\n')
def passphrase(self,desc,passphrase,pwtype=''):
if pwtype: pwtype += ' '
self.expect('Enter {}passphrase for {}.*?: '.format(pwtype,desc),passphrase+'\n',regex=True)
def hash_preset(self,desc,preset=''):
self.expect('Enter hash preset for {}'.format(desc))
self.expect('or hit ENTER .*?:',str(preset)+'\n',regex=True)
def written_to_file(self,desc,overwrite_unlikely=False,query='Overwrite? ',oo=False):
s1 = '{} written to file '.format(desc)
s2 = query + "Type uppercase 'YES' to confirm: "
ret = self.expect(([s1,s2],s1)[overwrite_unlikely])
if ret == 1:
self.send('YES\n')
return self.expect_getend("Overwriting file '").rstrip("'")
self.expect(NL,nonl=True)
outfile = self.p.before.strip().strip("'")
if opt.debug_pexpect:
rmsg('Outfile [{}]'.format(outfile))
vmsg('{} file: {}'.format(desc,cyan(outfile.replace("'",''))))
return outfile
def no_overwrite(self):
self.expect("Overwrite? Type uppercase 'YES' to confirm: ",'\n')
self.expect('Exiting at user request')
def expect_getend(self,s,regex=False):
ret = self.expect(s,regex=regex,nonl=True)
debug_pexpect_msg(self.p)
# readline() of partial lines doesn't work with PopenSpawn, so do this instead:
self.expect(NL,nonl=True,silent=True)
debug_pexpect_msg(self.p)
end = self.p.before.rstrip()
if not g.debug:
vmsg(' ==> {}'.format(cyan(end)))
return end
def interactive(self):
return self.p.interact() # interact() not available with popen_spawn
def kill(self,signal):
return self.p.kill(signal)
def expect(self,s,t='',delay=None,regex=False,nonl=False,silent=False):
delay = delay or (0,0.3)[bool(opt.buf_keypress)]
if not silent:
if opt.verbose:
msg_r('EXPECT ' + yellow(str(s)))
elif not opt.exact_output: msg_r('+')
try:
if s == '':
ret = 0
else:
f = (self.p.expect_exact,self.p.expect)[bool(regex)]
ret = f(s,self.timeout)
except pexpect.TIMEOUT:
if opt.debug_pexpect: raise
m1 = red('\nERROR. Expect {!r} timed out. Exiting\n'.format(s))
m2 = 'before: [{}]\n'.format(self.p.before)
m3 = 'sent value: [{}]'.format(self.sent_value) if self.sent_value != None else ''
rdie(1,m1+m2+m3)
debug_pexpect_msg(self.p)
if opt.verbose and type(s) != str:
msg_r(' ==> {} '.format(ret))
if ret == -1:
rdie(1,'Error. Expect returned {}'.format(ret))
else:
if t == '':
if not nonl and not silent: vmsg('')
else:
self.send(t,delay,s)
return ret
def send(self,t,delay=None,s=False):
self.sent_value = None
delay = delay or (0,0.3)[bool(opt.buf_keypress)]
if delay: time.sleep(delay)
ret = self.p.send(t) # returns num bytes written
if ret:
self.sent_value = t
if delay: time.sleep(delay)
if opt.verbose:
ls = (' ','')[bool(opt.debug or not s)]
es = (' ','')[bool(s)]
msg('{}SEND {}{}'.format(ls,es,yellow("'{}'".format(t.replace('\n',r'\n')))))
return ret
def read(self,n=-1):
return self.p.read(n)
def close(self):
if opt.pexpect_spawn:
self.p.close()
class MMGenPexpect(object):
def __init__(self, args, no_output=False):
if opt.direct_exec:
msg('')
from subprocess import call, check_output
f = (call, check_output)[bool(no_output)]
ret = f([args[0]] + args[1:])
if f == call and ret != 0:
die(
1,
red('ERROR: process returned a non-zero exit status ({})'.
format(ret)))
else:
if opt.pexpect_spawn:
self.p = pexpect.spawn(args[0], args[1:], encoding='utf8')
self.p.delaybeforesend = 0
else:
self.p = PopenSpawn(args, encoding='utf8')
# self.p.delaybeforesend = 0 # TODO: try this here too
if opt.exact_output: self.p.logfile = sys.stdout
self.req_exit_val = 0
self.skip_ok = False
self.timeout = int(opt.pexpect_timeout or 0) or (60, 5)[bool(
opt.debug_pexpect)]
self.sent_value = None
def do_decrypt_ka_data(self,
hp,
pw,
desc='key-address data',
check=True,
have_yes_opt=False):
# self.hash_preset(desc,hp)
self.passphrase(desc, pw)
if not have_yes_opt:
self.expect('Check key-to-address validity? (y/N): ',
('n', 'y')[check])
def view_tx(self, view):
self.expect('View.* transaction.*\? .*: ', view, regex=True)
if view not in 'n\n':
self.expect('to continue: ', '\n')
def do_comment(self, add_comment, has_label=False):
p = ('Add a comment to transaction',
'Edit transaction comment')[has_label]
self.expect('{}? (y/N): '.format(p), ('n', 'y')[bool(add_comment)])
if add_comment:
self.expect('Comment: ', add_comment + '\n')
def ok(self):
ret = self.p.wait()
if ret != self.req_exit_val and not opt.coverage:
die(
1,
red('test.py: spawned program exited with value {}'.format(
ret)))
if opt.profile: return
if not self.skip_ok:
sys.stderr.write(
green('OK\n') if opt.exact_output or opt.verbose else (
' OK\n'))
return self
def license(self):
if 'MMGEN_NO_LICENSE' in os.environ: return
self.expect(
"'w' for conditions and warranty info, or 'c' to continue: ", 'c')
def label(self, label='Test Label (UTF-8) α'):
self.expect('Enter a wallet label, or hit ENTER for no label: ',
label + '\n')
def usr_rand_out(self, saved=False):
fs = 'Generating encryption key from OS random data plus {}user-supplied entropy'
self.expect(fs.format(('', 'saved ')[saved]))
def usr_rand(self, num_chars):
if opt.usr_random:
self.interactive()
self.send('\n')
else:
rand_chars = list(getrandstr(num_chars, no_space=True))
vmsg_r('SEND ')
while rand_chars:
ch = rand_chars.pop(0)
msg_r(yellow(ch) + ' ' if opt.verbose else '+')
ret = self.expect('left: ', ch, delay=0.005)
self.expect('ENTER to continue: ', '\n')
def passphrase_new(self, desc, passphrase):
self.expect('Enter passphrase for {}: '.format(desc),
passphrase + '\n')
self.expect('Repeat passphrase: ', passphrase + '\n')
def passphrase(self, desc, passphrase, pwtype=''):
if pwtype: pwtype += ' '
self.expect('Enter {}passphrase for {}.*?: '.format(pwtype, desc),
passphrase + '\n',
regex=True)
def hash_preset(self, desc, preset=''):
self.expect('Enter hash preset for {}'.format(desc))
self.expect('or hit ENTER .*?:', str(preset) + '\n', regex=True)
def written_to_file(self,
desc,
overwrite_unlikely=False,
query='Overwrite? ',
oo=False):
s1 = '{} written to file '.format(desc)
s2 = query + "Type uppercase 'YES' to confirm: "
ret = self.expect(([s1, s2], s1)[overwrite_unlikely])
if ret == 1:
self.send('YES\n')
return self.expect_getend("Overwriting file '").rstrip("'")
self.expect(NL, nonl=True)
outfile = self.p.before.strip().strip("'")
if opt.debug_pexpect:
rmsg('Outfile [{}]'.format(outfile))
vmsg('{} file: {}'.format(desc, cyan(outfile.replace("'", ''))))
return outfile
def no_overwrite(self):
self.expect("Overwrite? Type uppercase 'YES' to confirm: ", '\n')
self.expect('Exiting at user request')
def expect_getend(self, s, regex=False):
ret = self.expect(s, regex=regex, nonl=True)
debug_pexpect_msg(self.p)
# readline() of partial lines doesn't work with PopenSpawn, so do this instead:
self.expect(NL, nonl=True, silent=True)
debug_pexpect_msg(self.p)
end = self.p.before.rstrip()
if not g.debug:
vmsg(' ==> {}'.format(cyan(end)))
return end
def interactive(self):
return self.p.interact() # interact() not available with popen_spawn
def kill(self, signal):
return self.p.kill(signal)
def expect(self,
s,
t='',
delay=None,
regex=False,
nonl=False,
silent=False):
delay = delay or (0, 0.3)[bool(opt.buf_keypress)]
if not silent:
if opt.verbose:
msg_r('EXPECT ' + yellow(str(s)))
elif not opt.exact_output:
msg_r('+')
try:
if s == '':
ret = 0
else:
f = (self.p.expect_exact, self.p.expect)[bool(regex)]
ret = f(s, self.timeout)
except pexpect.TIMEOUT:
if opt.debug_pexpect: raise
m1 = red('\nERROR. Expect {!r} timed out. Exiting\n'.format(s))
m2 = 'before: [{}]\n'.format(self.p.before)
m3 = 'sent value: [{}]'.format(
self.sent_value) if self.sent_value != None else ''
rdie(1, m1 + m2 + m3)
debug_pexpect_msg(self.p)
if opt.verbose and type(s) != str:
msg_r(' ==> {} '.format(ret))
if ret == -1:
rdie(1, 'Error. Expect returned {}'.format(ret))
else:
if t == '':
if not nonl and not silent: vmsg('')
else:
self.send(t, delay, s)
return ret
def send(self, t, delay=None, s=False):
self.sent_value = None
delay = delay or (0, 0.3)[bool(opt.buf_keypress)]
if delay: time.sleep(delay)
ret = self.p.send(t) # returns num bytes written
if ret:
self.sent_value = t
if delay: time.sleep(delay)
if opt.verbose:
ls = (' ', '')[bool(opt.debug or not s)]
es = (' ', '')[bool(s)]
msg('{}SEND {}{}'.format(
ls, es, yellow("'{}'".format(t.replace('\n', r'\n')))))
return ret
def read(self, n=-1):
return self.p.read(n)
def close(self):
if opt.pexpect_spawn:
self.p.close()
class MMGenPexpect(object):
NL = '\r\n'
if g.platform == 'linux' and opt.popen_spawn:
import atexit
atexit.register(lambda: os.system('stty sane'))
NL = '\n'
def __init__(self,name,mmgen_cmd,cmd_args,desc,no_output=False,passthru_args=[],msg_only=False):
cmd_args = ['--{}{}'.format(k.replace('_','-'),
'='+getattr(opt,k) if getattr(opt,k) != True else ''
) for k in passthru_args if getattr(opt,k)] \
+ ['--data-dir='+os.path.join('test','data_dir')] + cmd_args
if g.platform == 'win': cmd,args = 'python',[mmgen_cmd]+cmd_args
else: cmd,args = mmgen_cmd,cmd_args
for i in args:
if type(i) not in (str,unicode):
m1 = 'Error: missing input files in cmd line?:'
m2 = '\nName: {}\nCmd: {}\nCmd args: {}'
die(2,(m1+m2).format(name,cmd,args))
if opt.popen_spawn:
args = [(a,"'{}'".format(a))[' ' in a] for a in args]
cmd_str = '{} {}'.format(cmd,' '.join(args)).replace('\\','/')
if opt.log:
log_fd.write(cmd_str+'\n')
if opt.verbose or opt.print_cmdline or opt.exact_output:
clr1,clr2,eol = ((green,cyan,'\n'),(nocolor,nocolor,' '))[bool(opt.print_cmdline)]
sys.stderr.write(green('Testing: {}\n'.format(desc)))
if not msg_only:
sys.stderr.write(clr1('Executing {}{}'.format(clr2(cmd_str),eol)))
else:
m = 'Testing %s: ' % desc
msg_r(m)
if msg_only: return
if opt.direct_exec:
msg('')
from subprocess import call,check_output
f = (call,check_output)[bool(no_output)]
ret = f([cmd] + args)
if f == call and ret != 0:
m = 'ERROR: process returned a non-zero exit status (%s)'
die(1,red(m % ret))
else:
if opt.traceback:
cmd,args = g.traceback_cmd,[cmd]+args
cmd_str = g.traceback_cmd + ' ' + cmd_str
# Msg('\ncmd_str: {}'.format(cmd_str))
if opt.popen_spawn:
self.p = PopenSpawn(cmd_str)
else:
self.p = pexpect.spawn(cmd,args)
if opt.exact_output: self.p.logfile = sys.stdout
def ok(self,exit_val=0):
ret = self.p.wait()
# Msg('expect: {} got: {}'.format(exit_val,ret))
if ret != exit_val:
die(1,red('test.py: spawned program exited with value {}'.format(ret)))
if opt.profile: return
if opt.verbose or opt.exact_output:
sys.stderr.write(green('OK\n'))
else: msg(' OK')
def cmp_or_die(self,s,t,skip_ok=False,exit_val=0):
ret = self.p.wait()
if ret != exit_val:
die(1,red('test.py: spawned program exited with value {}'.format(ret)))
if s == t:
if not skip_ok: ok()
else:
sys.stderr.write(red(
'ERROR: recoded data:\n%s\ndiffers from original data:\n%s\n' %
(repr(t),repr(s))))
sys.exit(3)
def license(self):
if 'MMGEN_NO_LICENSE' in os.environ: return
p = "'w' for conditions and warranty info, or 'c' to continue: "
my_expect(self.p,p,'c')
def label(self,label='Test Label'):
p = 'Enter a wallet label, or hit ENTER for no label: '
my_expect(self.p,p,label+'\n')
def usr_rand_out(self,saved=False):
m = '%suser-supplied entropy' % (('','saved ')[saved])
my_expect(self.p,'Generating encryption key from OS random data plus ' + m)
def usr_rand(self,num_chars):
if opt.usr_random:
self.interactive()
my_send(self.p,'\n')
else:
rand_chars = list(getrandstr(num_chars,no_space=True))
my_expect(self.p,'symbols left: ','x')
try:
vmsg_r('SEND ')
while self.p.expect('left: ',0.1) == 0:
ch = rand_chars.pop(0)
msg_r(yellow(ch)+' ' if opt.verbose else '+')
self.p.send(ch)
except:
vmsg('EOT')
my_expect(self.p,'ENTER to continue: ','\n')
def passphrase_new(self,desc,passphrase):
my_expect(self.p,('Enter passphrase for %s: ' % desc), passphrase+'\n')
my_expect(self.p,'Repeat passphrase: ', passphrase+'\n')
def passphrase(self,desc,passphrase,pwtype=''):
if pwtype: pwtype += ' '
my_expect(self.p,('Enter %spassphrase for %s.*?: ' % (pwtype,desc)),
passphrase+'\n',regex=True)
def hash_preset(self,desc,preset=''):
my_expect(self.p,('Enter hash preset for %s' % desc))
my_expect(self.p,('or hit ENTER .*?:'), str(preset)+'\n',regex=True)
def written_to_file(self,desc,overwrite_unlikely=False,query='Overwrite? ',oo=False):
s1 = '%s written to file ' % desc
s2 = query + "Type uppercase 'YES' to confirm: "
ret = my_expect(self.p,([s1,s2],s1)[overwrite_unlikely])
if ret == 1:
my_send(self.p,'YES\n')
# if oo:
outfile = self.expect_getend("Overwriting file '").rstrip("'")
return outfile
# else:
# ret = my_expect(self.p,s1)
self.expect(self.NL,nonl=True)
outfile = self.p.before.strip().strip("'")
if opt.debug_pexpect: msgred('Outfile [%s]' % outfile)
vmsg('%s file: %s' % (desc,cyan(outfile.replace("'",''))))
return outfile
def no_overwrite(self):
self.expect("Overwrite? Type uppercase 'YES' to confirm: ",'\n')
self.expect('Exiting at user request')
def tx_view(self):
my_expect(self.p,r'View .*?transaction.*? \(y\)es, \(N\)o, pager \(v\)iew.*?: ','\n',regex=True)
def expect_getend(self,s,regex=False):
ret = self.expect(s,regex=regex,nonl=True)
debug_pexpect_msg(self.p)
# end = self.readline().strip()
# readline() of partial lines doesn't work with PopenSpawn, so do this instead:
self.expect(self.NL,nonl=True,silent=True)
debug_pexpect_msg(self.p)
end = self.p.before
vmsg(' ==> %s' % cyan(end))
return end
def interactive(self):
return self.p.interact() # interact() not available with popen_spawn
def kill(self,signal):
return self.p.kill(signal)
def logfile(self,arg):
self.p.logfile = arg
def expect(self,*args,**kwargs):
return my_expect(self.p,*args,**kwargs)
def send(self,*args,**kwargs):
return my_send(self.p,*args,**kwargs)
# def readline(self):
# return self.p.readline()
# def readlines(self):
# return [l.rstrip()+'\n' for l in self.p.readlines()]
def read(self,n=None):
return self.p.read(n)
def close(self):
if not opt.popen_spawn:
self.p.close()
class WeatherSimulation(threading.Thread):
def __init__(self,
id,
path,
options,
weather,
redo=False,
deactivate_window_mode=True,
bin_folder=os.path.join(
os.getcwd(), "3rdparty", "weather-particle-simulator",
"{}_{}".format(bin_platform, bin_bitness))):
threading.Thread.__init__(self)
self.id = id
self.simtime = 0. # Time (s)
self.simdur = 0. # Duration (s)
self.options = options
if "preset" in self.options:
self.preset = self.options["preset"]
else:
self.preset = None
self.path = path
self.weather = weather
self.deactivate_window_mode = deactivate_window_mode
self.redo = redo
self.bin_folder = bin_folder
self.output_dir = os.path.join(self.path, weather["weather"],
"{}mm".format(weather["fallrate"]))
print("Create thread", self.output_dir)
self.assert_validity()
def assert_validity(self):
if self.preset is not None:
assert self.preset[0] != "kitti" or self.preset[1] in [
"0000", "0032", "0056", "0071", "0117"
], "Kitti preset is invalid"
assert self.preset[0] != "cityscapes" or self.preset[1] in [
"0000"
], "Cityscapes preset is invalid"
def _print(self, *argv):
print("\r #{} ".format(self.id), end='')
print.__call__(*argv)
def interact(self, wait_for, send_str):
# print('wait for: {}'.format(wait_for))
self.child.expect(wait_for)
# print('send: {}'.format(send_str))
self.child.sendline(send_str.encode('ascii'))
def interact_step_menu(self, menu):
self.interact('Steps: What do you want to do \?', menu)
def set_sim_steps_times(self, start, dur, last):
self.interact_step_menu('2')
self.interact('Enter new duration', '{}'.format(start))
self.interact_step_menu('3')
self.interact('Enter new duration', '{}'.format(dur))
self.interact_step_menu('4')
self.interact('Enter new duration', '{}'.format(last))
def set_sim_steps_camera_focal(self, values):
self.interact_step_menu('12')
self.interact('What do you want to do \?', '{}'.format(3))
self.interact('Separator', '{}'.format(";"))
self.interact('Enter all steps values',
'{}'.format(";".join(["{}".format(v) for v in values])))
self.interact('Continue \?', '{}'.format("y"))
def set_sim_steps_camera_exposure(self, values):
self.interact_step_menu('13')
self.interact('What do you want to do \?', '{}'.format(3))
self.interact('Separator', '{}'.format(";"))
self.interact('Enter all steps values',
'{}'.format(";".join(["{}".format(v) for v in values])))
self.interact('Continue \?', '{}'.format("y"))
def set_sim_steps_camera_motion(self, values):
self.interact_step_menu('18')
self.interact('What do you want to do \?', '{}'.format(3))
self.interact('Separator', '{}'.format(";"))
self.interact('Enter all steps values',
'{}'.format(";".join(["{}".format(v) for v in values])))
self.interact('Continue \?', '{}'.format("y"))
def set_sim_steps_rain_fallrate(self, values):
self.interact_step_menu('41')
self.interact('What do you want to do \?', '{}'.format(3))
self.interact('Separator', '{}'.format(";"))
self.interact('Enter all steps values',
'{}'.format(";".join(["{}".format(v) for v in values])))
self.interact('Continue \?', '{}'.format("y"))
def interact_main_menu(self, menu):
self.interact('What do you want to do \?', menu)
def set_sim_Duration(self, val):
self.interact_main_menu('6')
self.interact('Enter new duration', '{}'.format(val))
def set_sim_Hz(self, val):
self.interact_main_menu('7')
self.interact('Enter new frequency', '{}'.format(val))
def set_sim_ParticlesDetectionLatencyFrames(self, val):
self.interact_main_menu('61')
self.interact('Enter new particles detection latency',
'{}'.format(val))
def set_sim_ParticlesDetectionErrorMargin(self, val):
self.interact_main_menu('62')
self.interact('Enter new particles detection error', '{}'.format(val))
def set_sim_Camera0_Hz(self, val):
self.interact_main_menu('10')
self.interact('Enter new frequency', '{}'.format(val))
def set_sim_Camera0_ViewMatrixIC(self, pos, lookat, up):
self.interact_main_menu('15')
self.interact('Enter new IC pos x', '{}'.format(pos[0]))
self.interact('Enter new IC pos y', '{}'.format(pos[1]))
self.interact('Enter new IC pos z', '{}'.format(pos[2]))
self.interact('Enter new IC lookat x', '{}'.format(lookat[0]))
self.interact('Enter new IC lookat y', '{}'.format(lookat[1]))
self.interact('Enter new IC lookat z', '{}'.format(lookat[2]))
self.interact('Enter new IC up x', '{}'.format(up[0]))
self.interact('Enter new IC up y', '{}'.format(up[1]))
self.interact('Enter new IC up z', '{}'.format(up[2]))
def set_sim_Camera0_CcdSpecs(self, width, height, pixsize):
self.interact_main_menu('11')
self.interact('Camera 0 CCD pxl size', '{}'.format(pixsize))
self.interact('Camera 0 CCD width', '{}'.format(width))
self.interact('Camera 0 CCD height', '{}'.format(height))
def set_sim_Camera0_Focal(self, val):
self.interact_main_menu('12')
self.interact('Enter new focal', '{}'.format(val))
def set_sim_Camera0_Resolution(self, width, height):
self.interact_main_menu('14')
self.interact('Camera 0 Resolution WIDTH', '{}'.format(width))
self.interact('Camera 0 Resolution HEIGHT', '{}'.format(height))
def set_sim_Camera0_ExposureTime(self, val):
self.interact_main_menu('13')
self.interact('Enter new exposure time', '{}'.format(val))
def set_sim_Camera0_VisibilityMappingAuto(self):
self.interact_main_menu('17')
self.interact('Enter new visibility mapping MIN', '{}'.format(0))
self.interact('Enter new visibility mapping MAX', '{}'.format(0))
def set_sim_Camera0_MotionSpeedIC(self, val):
self.interact_main_menu('18')
self.interact('Enter new initial motion speed', '{}'.format(val))
def set_sim_Projector0_Hz(self, val):
self.interact_main_menu('21')
assert (val == "AHL_HZ_MAX")
# No need to do anything
def set_sim_Projector0_Res(self, width, height):
self.interact_main_menu('22')
# No need to do anything
def set_sim_Projector0_MinPixelOverlay(self, val):
self.interact_main_menu('24')
self.interact('Enter new minimum pixel overlay', '{}'.format(val))
def set_sim_Projector0_DbgSaveLightmap(self, val):
while True:
self.interact_main_menu('28')
obs = self.child.expect([
"Projector 0 save light maps \(OFF\)",
"Projector 0 save light maps \(ON\)"
])
if (val and obs == 1) or (not val and obs == 0):
break
def set_sim_Stats_Active(self, val):
while True:
self.interact_main_menu('70')
obs = self.child.expect([
"Output simulation stats \(OFF\)",
"Output simulation stats \(ON\)"
])
if (val and obs == 1) or (not val and obs == 0):
break
def set_sim_Stats_StatsLevel(self, val):
while True:
self.interact_main_menu('72')
obs = self.child.expect(
["Stats level \(HIERARCHY\)", "Stats level \(NO HIERARCHY\)"])
if (val == "HIERARCHY" and obs == 0) or (val == "NO HIERARCHY"
and obs == 1):
break
def set_sim_Stats_StartTime(self, val):
self.interact_main_menu('71')
self.interact('Enter start time', '{}'.format(val))
def apply_options(self):
# Series of hacks to avoid warning due to incompatible parameters
self.set_sim_Camera0_CcdSpecs(99999, 99999, 1.0)
# Apply settings
self.set_sim_Duration(stats_start_time * 1000 +
self.options["sim_duration"] * 1000)
self.set_sim_Hz(str(self.options["sim_hz"]))
self.set_sim_ParticlesDetectionLatencyFrames(0)
self.set_sim_ParticlesDetectionErrorMargin(0)
# Camera parameters
self.set_sim_Camera0_Hz(self.options["cam_hz"])
self.set_sim_Camera0_ViewMatrixIC(self.options["cam_pos"],
self.options["cam_lookat"],
self.options["cam_up"])
self.set_sim_Camera0_Resolution(
self.options["cam_WH"][0],
self.options["cam_WH"][1]) # Order matters
self.set_sim_Camera0_CcdSpecs(
self.options["cam_CCD_WH"][0], self.options["cam_CCD_WH"][1],
self.options["cam_CCD_pixsize"]) # Order matters
self.set_sim_Camera0_Focal(self.options["cam_focal"])
self.set_sim_Camera0_ExposureTime(self.options["cam_exposure"])
self.set_sim_Camera0_VisibilityMappingAuto()
# self.set_sim_Camera0_motionSpeedIC = 30
# Projector parameters
self.set_sim_Projector0_Hz("AHL_HZ_MAX")
self.set_sim_Projector0_Res(self.options["cam_WH"][0],
self.options["cam_WH"][1])
self.set_sim_Projector0_MinPixelOverlay(50)
self.set_sim_Projector0_DbgSaveLightmap(True)
# Debug
# parameters
# self.set_sim_Render()->Active(false)
# self.set_sim_Render()->Hz(200)
# self.set_sim_Render()->DbgCameraManip(true)
# self.set_sim_Render()->DbgCameraManipViewMatrix(self.set_sim_Camera0_Pos(), self.set_sim_Camera0_Lookat(), self.set_sim_Camera(
# 0)->Up())
self.set_sim_Stats_Active(True)
self.set_sim_Stats_StartTime("{}".format(stats_start_time * 1000))
self.set_sim_Stats_StatsLevel("HIERARCHY")
def run(self):
weather = self.weather
os.makedirs(self.output_dir, exist_ok=True)
if not self.redo:
files = os.listdir(self.output_dir)
results_computed = np.any(["camera0.xml" in f for f in files])
if results_computed:
self._print("Simulation file exits {}, next!".format(
self.output_dir))
return
# Save options as json dumps
try:
fp = open(os.path.join(self.output_dir, "sim_options.json"), 'w')
options_ = {
k: v.tolist() if isinstance(v, np.ndarray) else v
for k, v in self.options.items()
} # Convert to native types
if "sim_steps" in options_:
options_["sim_steps"] = {
k: v.tolist() if isinstance(v, np.ndarray) else v
for k, v in options_["sim_steps"].items()
}
json.dump(options_, fp)
fp.close()
except Exception as e:
print(e)
print("Failed saving JSON {}... Not crucial, continuing")
log_path = os.path.join(self.output_dir, 'automate_log.txt')
log_fp = open(log_path, 'a+')
# self._print(self.output_dir)
self.child = PopenSpawn(os.path.join(self.bin_folder, 'AHLSimulation'),
cwd=self.output_dir,
logfile=logwriter(log_fp))
try:
self._print("In main menu")
self.interact_main_menu('9')
self.interact('Set the seed for random generator', '0')
if self.preset is None:
self._print("Apply options:", self.options)
self.apply_options()
else:
self._print("Apply preset (ignoring options):",
self.preset[:2])
if self.preset[0] in sequence_code:
self.interact_main_menu('99')
time.sleep(0.5)
self._print("Setting system")
self.child.expect('Which system to run ?')
seq_code = sequence_code[self.preset[0]] + self.preset[1]
self._print(' System code: ', seq_code)
self.child.sendline(seq_code.encode('ascii'))
elif "nuscenes" in self.preset[0].lower():
self.interact_main_menu('99')
time.sleep(0.5)
self._print("Setting system")
self.child.expect('Which system to run ?')
if "2Hz" in self.preset[0]:
seq_code = '1000'
else:
seq_code = '100'
self._print(' System code: ', seq_code)
self.child.sendline(seq_code.encode('ascii'))
else:
raise NotImplementedError(
"No settings for this set {}".format(self.preset[0]))
# Deactivate windows AND save light map option
if self.deactivate_window_mode:
self._print("In main menu")
self.interact_main_menu('28')
time.sleep(0.5)
self._print(" Save light map")
# Deactivate rain particles
self._print("Deactivating rain particles")
self.interact_main_menu('410')
self.child.expect('410. Rain \(OFF\)')
if weather["weather"] == "rain":
self._print("Activating rain particles")
# Activate rain particles
self.interact_main_menu('410')
self.child.expect('410. Rain \(ON\)')
self._print("Setting rain fallrate")
# Set rain fallrate
self.interact_main_menu('414')
self.child.expect('Enter new Rain fall rate')
code = str(weather["fallrate"])
self.child.sendline(code.encode('ascii'))
self._print("In main menu")
_steps_menu = False
if self.preset is None:
assert self.options["sim_mode"] in ["normal", "steps"]
# Actions in main menu
if self.options["sim_mode"] == "steps":
assert (
"sim_steps" in self.options
and type(self.options["sim_steps"]) == dict
and len(self.options["sim_steps"].keys()) != 0
), "[ERROR]: sim_steps must be provided and be a non-empty dictionary."
assert np.all(
[
type(v) == list or type(v) == np.ndarray
for k, v in self.options["sim_steps"].items()
]
), "[ERROR]: values in sim_steps should all be list/arrays"
# Apply initial cam speed
if "cam_motion" in self.options["sim_steps"]:
speedIC = self.options["sim_steps"]["cam_motion"][0]
self.set_sim_Camera0_MotionSpeedIC(speedIC)
# Actions in step menu
if self.options["sim_mode"] == "steps":
# Enter step menu
self.interact_main_menu('102')
_steps_menu = True
# Set step durations
step_dur = 1. / self.options["cam_hz"]
self.set_sim_steps_times(stats_start_time * 1000,
step_dur * 1000, step_dur * 1000)
# Attempt to compute the total simulation time
max_steps = max(
[len(v) for k, v in self.options["sim_steps"].items()])
self.simdur = stats_start_time + (1 + max_steps) * step_dur
if "cam_motion" in self.options["sim_steps"]:
self._print(" Apply {} steps cam motion: {}".format(
len(self.options["sim_steps"]["cam_motion"]),
";".join([
"{}".format(v) for v in
self.options["sim_steps"]["cam_motion"]
])))
self.set_sim_steps_camera_motion(
self.options["sim_steps"]["cam_motion"])
if "cam_focal" in self.options["sim_steps"]:
self._print(" Apply {} steps cam focal: {}".format(
len(self.options["sim_steps"]["cam_focal"]),
";".join([
"{}".format(v)
for v in self.options["sim_steps"]["cam_focal"]
])))
self.set_sim_steps_camera_focal(
self.options["sim_steps"]["cam_focal"])
if "cam_exposure" in self.options["sim_steps"]:
self._print(" Apply {} steps cam exposure: {}".format(
len(self.options["sim_steps"]["cam_exposure"]),
";".join([
"{}".format(v) for v in
self.options["sim_steps"]["cam_exposure"]
])))
self.set_sim_steps_camera_exposure(
self.options["sim_steps"]["cam_exposure"])
if "rain_fallrate" in self.options["sim_steps"]:
self._print(" Apply {} steps rain fallrate: {}".format(
len(self.options["sim_steps"]["rain_fallrate"]),
";".join([
"{}".format(v) for v in
self.options["sim_steps"]["rain_fallrate"]
])))
self.set_sim_steps_rain_fallrate(
self.options["sim_steps"]["rain_fallrate"])
if self.options["sim_mode"] == "normal":
self.simdur = stats_start_time + self.options[
"sim_duration"]
else:
# Enter step menu
self.interact_main_menu('102')
_steps_menu = True
# self._print("Going to step menu")
# self.interact_main_menu('102')
# if "nuscenes" in preset[0].lower():
# self.child.expect('Steps: What do you want to do \?')
# self._print("In Step menu")
# self.child.sendline('18'.encode('ascii'))
# self._print("Camera 0 motion speed choices")
# self.child.expect("What do you want to do")
# self.child.sendline('3'.encode('ascii'))
# self._print(" Camera 0 motion speed choices (3 -> all at once)")
#
# speed = np.linalg.norm(np.array(preset[3]), axis=1) / 1000 * 3600 / (np.array(preset[4]) * 1e-6)
# self.child.expect("Separator")
# self.child.sendline(';'.encode('ascii'))
# self.child.expect("Enter all steps values")
# self._print(" Camera 0 motion speeds (min, max): ({}, {})".format(np.min(speed), np.max(speed)))
# self.child.sendline(';'.join([str(s) for s in speed.tolist()]).encode('ascii'))
# self.child.expect("Continue")
# self.child.sendline('y'.encode('ascii'))
if _steps_menu:
self._print("In Step menu")
self._print("Starting simulation")
self.interact_step_menu('1')
index = None
while index != 1:
index = self.child.expect([
'[0-9]+:[0-9]+:[0-9]+\.[0-9]* +\(p#[0-9]*\)',
'\[Simulation stopped\]'
],
timeout=None)
if index == 0:
self.simtime += 0.5 # Each time a time is displayed the simulation advanced of 0.5 seconds
self._print("Simulation stopped")
time.sleep(5.) # Wait for the "Press any key to continue"
self.child.sendline(b'\n')
if _steps_menu:
self.child.expect('Steps: What do you want to do \?')
self._print("In Step menu")
self._print("Going to main menu")
self.child.sendline('0'.encode('ascii'))
_steps_menu = False
self._print("In main menu")
self._print("Stopping process")
self.child.expect('What do you want to do \?')
self.child.sendline('0'.encode('ascii'))
self.child.expect('Press any key to continue . . .')
self.child.sendline(b'\n')
self.child.expect('Press any key to continue . . .')
self.child.sendline(b'\n')
except ExceptionPexpect as e:
self._print(
"ERROR occured. Check the content of the log file to solve: {}"
.format(log_path))
self._print(e)
self._print(
"If starting python from an IDE, check LD_LIBRARY_PATH environment is accessible (https://youtrack.jetbrains.com/issue/PY-29580)"
)
self.child.kill(signal.SIGINT)
exit()
except Exception as e:
# Might happens at the end of the simulation, if the process closes slightly before last key strike
self._print(e)
pass
# Kill process in case it's not dead (cruel world)
try:
self.child.wait()
self.child.kill(signal.SIGINT)
except Exception:
pass
log_fp.close()
class SerAPI:
def __init__(self, timeout, debug=False):
"Initialize the SerAPI subprocess"
self.debug = debug
try:
self.proc = PopenSpawn(
"sertop --implicit --omit_loc --print0",
encoding="utf-8",
timeout=timeout,
maxread=10000000,
)
except FileNotFoundError:
log(
'Please make sure the "sertop" program is in the PATH.\nYou may have to run "eval $(opam env)".',
"ERROR",
)
sys.exit(1)
self.proc.expect_exact(
"(Feedback((doc_id 0)(span_id 1)(route 0)(contents Processed)))\0")
self.send("Noop")
self.states_stack = []
# global printing options
self.execute("Unset Printing Notations.")
self.execute("Unset Printing Wildcard.")
self.execute("Set Printing Coercions.")
self.execute("Unset Printing Allow Match Default Clause.")
self.execute("Unset Printing Factorizable Match Patterns.")
self.execute("Unset Printing Compact Contexts.")
self.execute("Set Printing Implicit.")
self.execute("Set Printing Depth 999999.")
self.execute("Unset Printing Records.")
# initialize the state stack
self.push()
self.ast_cache = {}
self.dead = False
def set_timeout(self, timeout):
self.proc.timeout = timeout
def get_timeout(self):
return proc.timeout
def send(self, cmd):
"Send a command to SerAPI and retrieve the responses"
# print(cmd)
assert "\n" not in cmd
self.proc.sendline(cmd)
try:
self.proc.expect([
"\(Answer \d+ Ack\)\x00.*\(Answer \d+ Completed\)\x00",
"\(Answer \d+ Ack\)\x00.*\(Answer \d+\(CoqExn.*\)\x00",
])
except pexpect.TIMEOUT as ex:
print(self.proc.before)
raise CoqTimeout
raw_responses = self.proc.after
# print(raw_responses)
ack_num = int(
re.search(r"^\(Answer (?P<num>\d+)", raw_responses)["num"])
for num in re.findall(r"(?<=\(Answer) \d+", raw_responses):
assert int(num) == ack_num
responses = []
msg_str = []
for item in raw_responses.split("\x00"):
item = item.strip()
if item == "":
continue
if not item.startswith("(Feedback") and not item.startswith(
"(Answer"):
m = re.search(r"\(Feedback|\(Answer", item)
if m is None:
continue
item = item[m.span()[0]:]
assert item.endswith(")")
parsed_item = sexpdata.loads(item, nil=None, true=None)
if "CoqExn" in item: # an error occured in Coq
assert parsed_item[2][0] == Symbol("CoqExn")
raise CoqExn(sexpdata.dumps(parsed_item[2][4]),
sexpdata.dumps(parsed_item[2]))
if item.startswith("(Feedback"): # ignore Feedback for now
try:
msg = parsed_item[1][3][1]
if (isinstance(msg, list) and msg != []
and msg[0] == Symbol("Message")):
msg_sexp, _ = self.send(
"(Print ((pp_format PpStr)) (CoqPp %s))" %
sexpdata.dumps(msg[3]))
msg_str.extend(
[symbol2str(x[1]) for x in msg_sexp[1][2][1]])
except IndexError:
pass
continue
responses.append(parsed_item)
msg_str = "\n".join(msg_str)
return responses, raw_responses
def send_add(self, cmd, return_ast):
'Send a (Add () "XXX") command to SerAPI, return the state id and optionally the AST'
responses, raw_responses = self.send('(Add () "%s")' % escape(cmd))
state_ids = [
int(sid) for sid in ADDED_STATE_PATTERN.findall(raw_responses)
]
state_id = state_ids[-1]
if self.states_stack != []:
self.states_stack[-1].append(state_id)
if return_ast:
if cmd not in self.ast_cache:
self.ast_cache[cmd] = self.query_ast(cmd)
ast = self.ast_cache[cmd]
else:
ast = None
return state_id, ast
def query_ast(self, cmd):
"Query the AST of the vernac command just added"
responses, _ = self.send('(Parse () "%s")' % escape(cmd))
ast = responses[1][2][1][0]
assert ast[0] == Symbol("CoqAst")
return ast
def query_library(self, lib):
responses, _ = self.send('(Query () (LocateLibrary "%s"))' % lib)
physical_path = symbol2str(responses[1][2][1][0][3])
return physical_path
def query_qualid(self, qualid):
responses, _ = self.send('(Query () (Locate "%s"))' % qualid)
if responses[1][2][1] == [] and qualid.startswith("SerTop."):
qualid = qualid[len("SerTop."):]
responses, _ = self.send('(Query () (Locate "%s"))' % qualid)
assert len(responses[1][2][1]) == 1
short_responses = responses[1][2][1][0][1][0][1]
assert short_responses[1][0] == Symbol("DirPath")
short_ident = ".".join(
[symbol2str(x[1]) for x in short_responses[1][1][::-1]] +
[symbol2str(short_responses[2][1])])
return short_ident
def query_env(self, current_file):
"Query the global environment"
responses, _ = self.send("(Query () Env)")
env = responses[1][2][1][0]
# store the constants
constants = []
for const in env[1][0][1][0][1]:
# identifier
qualid = (
print_mod_path(const[0][1]) + "." +
".".join([symbol2str(x[1]) for x in const[0][2][1][::-1]] +
[symbol2str(const[0][3][1])]))
if qualid.startswith("SerTop."):
logical_path = "SerTop"
physical_path = current_file
else:
logical_path = mod_path_file(const[0][1])
assert qualid.startswith(logical_path)
physical_path = os.path.relpath(
self.query_library(logical_path))
physical_path += ":" + qualid[len(logical_path) + 1:]
short_ident = self.query_qualid(qualid)
# term
assert const[1][0][1][0] == Symbol("const_body")
if const[1][0][1][1][0] == Symbol("Undef"): # delaration
opaque = None
term = None
elif const[1][0][1][1][0] == Symbol(
"Def"): # transparent definition
opaque = False
term = None
else:
assert const[1][0][1][1][0] == Symbol(
"OpaqueDef") # opaque definition
opaque = True
term = None
# type
assert const[1][0][2][0] == Symbol("const_type")
type_sexp = sexpdata.dumps(const[1][0][2][1])
type = self.print_constr(type_sexp)
sort = self.query_type(type_sexp, return_str=True)
constants.append({
"physical_path": physical_path,
"short_ident": short_ident,
"qualid": qualid,
"term": term,
"type": type,
"sort": sort,
"opaque": opaque,
"sexp": sexpdata.dumps(const[1][0][2][1]),
})
# store the inductives
inductives = []
for induct in env[1][0][1][1][1]:
# identifier
qualid = (
print_mod_path(induct[0][1]) + "." +
".".join([symbol2str(x[1]) for x in induct[0][2][1][::-1]] +
[symbol2str(induct[0][3][1])]))
short_ident = self.query_qualid(qualid)
if qualid.startswith("SerTop."):
logical_path = "SerTop"
physical_path = current_file
else:
logical_path = mod_path_file(induct[0][1])
physical_path = os.path.relpath(
self.query_library(logical_path))
assert qualid.startswith(logical_path)
physical_path += ":" + qualid[len(logical_path) + 1:]
# blocks
blocks = []
for blk in induct[1][0][0][1]:
blk_qualid = ".".join(
qualid.split(".")[:-1] + [symbol2str(blk[0][1][1])])
blk_short_ident = self.query_qualid(blk_qualid)
# constructors
constructors = []
for c_name, c_type in zip(blk[3][1], blk[4][1]):
c_name = symbol2str(c_name[1])
c_type = self.print_constr(sexpdata.dumps(c_type))
# if c_type is not None:
# c_type = UNBOUND_REL_PATTERN.sub(short_ident, c_type)
constructors.append((c_name, c_type))
blocks.append({
"short_ident": blk_short_ident,
"qualid": blk_qualid,
"constructors": constructors,
})
inductives.append({
"physical_path":
physical_path,
"blocks":
blocks,
"is_record":
induct[1][0][1][1] != Symbol("NotRecord"),
"sexp":
sexpdata.dumps(induct),
})
return constants, inductives
def query_goals(self):
"Retrieve a list of open goals"
responses, _ = self.send("(Query () Goals)")
assert responses[1][2][0] == Symbol("ObjList")
if responses[1][2][1] == []: # no goals
return [], [], [], []
else:
assert len(responses[1][2][1]) == 1
def store_goals(goals_sexp):
goals = []
for g in goals_sexp:
hypotheses = []
for h in g[2][1]:
h_sexp = sexpdata.dumps(h[2])
hypotheses.append({
"idents":
[symbol2str(ident[1]) for ident in h[0][::-1]],
"term": [
None if t == [] else self.print_constr(
sexpdata.dumps(t)) for t in h[1]
],
"type":
self.print_constr(h_sexp),
"sexp":
h_sexp,
})
type_sexp = sexpdata.dumps(g[1][1])
goals.append({
"id": int(g[0][1]),
"type": self.print_constr(type_sexp),
"sexp": type_sexp,
"hypotheses": hypotheses[::-1],
})
return goals
fg_goals = store_goals(responses[1][2][1][0][1][0][1])
bg_goals = store_goals(
list(
chain.from_iterable(
chain.from_iterable(responses[1][2][1][0][1][1][1]))))
shelved_goals = store_goals(responses[1][2][1][0][1][2][1])
given_up_goals = store_goals(responses[1][2][1][0][1][3][1])
return fg_goals, bg_goals, shelved_goals, given_up_goals
def has_open_goals(self):
responses, _ = self.send("(Query () Goals)")
assert responses[1][2][0] == Symbol("ObjList")
return responses[1][2][1] != []
def print_constr(self, sexp_str):
if not hasattr(self, "constr_cache"):
self.constr_cache = {}
if sexp_str not in self.constr_cache:
try:
responses, _ = self.send(
"(Print ((pp_format PpStr)) (CoqConstr %s))" % sexp_str)
self.constr_cache[sexp_str] = normalize_spaces(
symbol2str(responses[1][2][1][0][1]))
except CoqExn as ex:
if ex.err_msg == "Not_found":
return None
else:
raise ex
except TypeError as ex:
self.constr_cache[sexp_str] = normalize_spaces(
symbol2str(responses[0][2][1][0][1]))
return self.constr_cache[sexp_str]
def query_vernac(self, cmd):
return self.send('(Query () (Vernac "%s"))' % escape(cmd))
def query_type(self, term_sexp, return_str=False):
try:
responses, _ = self.send("(Query () (Type %s))" % term_sexp)
except CoqExn as ex:
if ex.err_msg == "Not_found":
return None
else:
raise ex
assert responses[1][2][1][0][0] == Symbol("CoqConstr")
type_sexp = responses[1][2][1][0][1]
if return_str:
return self.print_constr(sexpdata.dumps(type_sexp))
else:
return type_sexp
def execute(self, cmd, return_ast=False):
"Execute a vernac command"
state_id, ast = self.send_add(cmd, return_ast)
responses, _ = self.send("(Exec %d)" % state_id)
return responses, sexpdata.dumps(ast)
def push(self):
"push a new frame on the state stack (a checkpoint), which can be used to roll back to the current state"
self.states_stack.append([])
def cancel(self, states):
self.send("(Cancel (%s))" % " ".join([str(s) for s in states]))
def pull(self):
"remove a checkpoint created by push"
states = self.states_stack.pop()
self.states_stack[-1].extend(states)
return len(states)
def pop(self):
"rollback to a checkpoint created by push"
self.cancel(self.states_stack.pop())
def pop_n(self, cnt):
states = []
for i in range(cnt):
states.append(self.states_stack[-1].pop())
self.cancel(states)
def clean(self):
self.proc.sendeof()
self.proc.wait()
self.dead = True
def shutdown(self):
self.proc.kill(signal.SIGKILL)
self.dead = True
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
self.clean()