def run_indirect_validation(config):
s = 'validation'
cmd = 'python fit_predictor.py '
cmd += build_cmd(config.items('prediction'), prefix='batch/')
cmd += build_cmd(config.items(s), prefix='indirect/')
cmd_dir = os.path.join(ROOTDIR, 'prediction/batch')
run_cmd(cmd, config.get(s, 'indirect/logfile'), cmd_dir=cmd_dir, dry_run=config.dry_run)
def present(self, resource, command, bot):
offset = int(command.get_var("offset", 0))
kb_items = [self.build_button(item) for item in resource[1]]
# TODO inefficient af
next_cmd = utils.build_cmd(
self.get_command(), {
'p': command.get_var("p"),
'offset': int(command.get_var("offset", 0)) + ITEMS_PER_PAGE,
'rep': 1
})
prev_cmd = utils.build_cmd(
self.get_command(), {
'p': command.get_var("p"),
'offset': int(command.get_var("offset", 0)) - ITEMS_PER_PAGE,
'rep': 1
})
if offset >= ITEMS_PER_PAGE:
kb_items = [[{
"text": lang.msg('previous_page'),
"callback_data": prev_cmd
}]] + kb_items
if offset + ITEMS_PER_PAGE < resource[0]:
kb_items.append([{
"text": lang.msg('next_page'),
"callback_data": next_cmd
}])
command.msg(lang.msg('list_items',
floor(offset / ITEMS_PER_PAGE) + 1,
ceil(resource[0] / ITEMS_PER_PAGE)),
reply_markup=utils.build_keyboard(kb_items))
def _keyboard_buttons(self, data):
return [{
"text": lang.msg('view_tracks'),
"callback_data": utils.build_cmd('track', {'p': data['id']})
}, {
"text":
lang.msg('start_game_with'),
"callback_data":
utils.build_cmd('game', {'i': data['id']}, ['album'])
}] + super()._keyboard_buttons(data)
def get_afl_command(self, idx):
''' Build command '''
if self._resume:
input_opt = '-i-'
else:
input_opt = '-i {}/input'.format(self._basedir)
output_opt = '-o {}/output_afl/{}/'.format(self._basedir, self._fid)
if self._timeout != 0:
timeout_opt = '-t {} '.format(self._timeout)
else:
timeout_opt = ''
# binary command
cmd_file = '{}/command.json'.format(self._basedir)
bin_cmd = ' '.join(build_cmd(cmd_file, basedir=self._basedir))
node = self.get_node(idx)
command = ('/tmp/afl-{_ARCH_}/afl-fuzz {_OUTPUT_} '
'{_INPUT_} {_TIMEOUT_} {_MODE_} {_NODE_} '
'-m none -- {_CMD_} > /dev/null 2>&1')
return command.format(_ARCH_=self._arch,
_OUTPUT_=output_opt,
_INPUT_=input_opt,
_TIMEOUT_=timeout_opt,
_MODE_=self._mode,
_NODE_=node,
_CMD_=bin_cmd)
def run_heuristic_collection(config):
s = 'collection'
cmd = 'julia -p {} '.format(config.get(s, 'nprocs'))
cmd += 'run_collect_dataset.jl '
cmd += build_cmd(config.items(s), prefix='col/')
cmd_dir = os.path.join(ROOTDIR, 'collection')
run_cmd(cmd, config.get(s, 'logfile'), cmd_dir=cmd_dir,
dry_run=config.dry_run)
def _keyboard_buttons(self, data):
ret = []
for choice in self.session.get_choices():
ret.append([{
"text": choice,
"callback_data": utils.build_cmd('guess', {}, [choice])
}])
return ret
def run_td_prediction(config):
s = 'prediction'
cmd = 'python train.py '
cmd += build_cmd(config.items(s), prefix='td/')
cmd_dir = os.path.join(ROOTDIR, 'prediction/td/scripts/training')
run_cmd(cmd, config.get(s, 'logfile'), cmd_dir=cmd_dir, dry_run=config.dry_run)
print('Async prediction running in the background...')
print("Enter 'tmux attach -t a3c' to attach")
print("Enter 'tmux kill-session -t a3c' to end training manually")
def run_visualization(config):
s = 'visualization'
print_intro(s)
cmd = 'python visualization_utilities.py '
cmd += build_cmd(config.items(s), prefix='viz/')
cmd_dir = os.path.join(ROOTDIR, 'visualization')
run_cmd(cmd, config.get(s, 'logfile'), cmd_dir=cmd_dir,
dry_run=config.dry_run, blocking=False)
def fit_proposal_bayes_net(config):
s = 'generation'
cmd = 'julia -p {} run_fit_proposal_bayes_net.jl '.format(config.get(s, 'nprocs'))
# want to use the same configuration for fitting the proposal
# BN as will be used for data generation, so add the data gen flags
# first, and then add the explictly proposal-BN training flags second
# this lets the explict ones override, but uses the data gen flags
# as defaults
cmd += build_cmd(config.items(s), prefix='gen/')
cmd += '--dataset_filepath {} '.format(
config.get('collection', 'col/output_filepath'))
cmd += '--base_bn_filepath {} '.format(
config.get(s, 'base_bn_filepath'))
cmd += '--output_filepath {} '.format(
config.get(s, 'prop_bn_filepath'))
cmd += build_cmd(config.items(s), prefix='prop/')
cmd_dir = os.path.join(ROOTDIR, 'scene_generation')
run_cmd(cmd, config.get(s, 'prop_bn_logfile'), cmd_dir=cmd_dir,
dry_run=config.dry_run)
def generate_prediction_data(config):
s = 'generation'
cmd = 'julia -p {} run_collect_dataset.jl '.format(config.get(s, 'nprocs'))
cmd += '--base_bn_filepath {} '.format(
config.get(s, 'base_bn_filepath'))
cmd += '--prop_bn_filepath {} '.format(
config.get(s, 'prop_bn_filepath'))
cmd += build_cmd(config.items(s), prefix='gen/')
cmd_dir = os.path.join(ROOTDIR, 'collection')
run_cmd(cmd, config.get(s, 'generation_logfile'), cmd_dir=cmd_dir,
dry_run=config.dry_run)
def __init__(self, db_config, qemu, basedir, project_id):
""" init """
self._qemu = qemu
self.basedir = basedir
self._project_id = project_id
self._fifo_name = ''
self._tmpdir = ''
self._mmap_tb = dict()
self._debug = False
self._mmap_edges = dict()
self._command = build_cmd('{}/command.json'.format(self.basedir))
self._db = DB(host=db_config['host'],
port=db_config['port'],
database=db_config['database'],
user=db_config['user'],
password=db_config['password'])
def __init__(self, argv):
# docker + s2e variables
self._basedir = argv.get('basedir')
self._project_id = argv.get('project_id')
self._db_config = argv.get('db_config')
self._max_process = argv.get('process' , 1) #number of s2e instances
self._timeout = argv.get('timeout' , 300)
self._debug = argv.get('debug' , False)
self._arch = argv.get('arch' , 'i386')
self._interval = argv.get('interval' , 10)
self._threshold = argv.get('threshold' , 4)
self._mem_limit = argv.get('mem_limit' , 10 * 1024 * 1024 * 1024)
self._command = build_cmd('{}/command.json'.format(self._basedir), basedir='./')
needles = ['.//file/', './/binary/']
for idx, ele in enumerate(self._command):
for needle in needles:
if ele.startswith(needle):
self._command[idx] = ele.replace(needle, '', 1)
self._covered_count = 0
self._S2EDIR = '{}/s2e'.format(os.path.dirname(os.path.realpath(__file__)))
def prepare_template(input_name, input_hash, dir_struct, argv):
''' prepare the lua config for s2e as well as bootstrap shell script '''
current_dir = os.path.dirname(os.path.realpath(__file__))
basedir = dir_struct['.']
paths = dict()
paths['library'] = '{}/library'.format(basedir)
paths['config'] = '{}/output_s2e/config/{}'.format(basedir, input_hash)
paths['testcases'] = '{}/output_s2e/testcases'.format(basedir)
paths['template'] = os.path.join(current_dir, 'templates')
if argv['arch'] == 'i386':
paths['tools'] = '{}/build/bin/guest-tools32'.format(S2EDIR)
elif argv['arch'] == 'x86_64':
paths['tools'] = '{}/build/bin/guest-tools64'.format(S2EDIR)
check_dir(paths['config'])
check_dir(paths['testcases'])
with open(os.path.join(paths['template'], 'analyze.lua.template.driller')) as f_lua_temp:
lua_temp = f_lua_temp.read()
lua_mod = lua_temp.format(PATH_BINARY = '{}/binary'.format(basedir),
PATH_INPUT = dir_struct['in_s2e'],
PATH_LIB = paths['library'],
PATH_TOOLS = paths['tools'],
PATH_CONFIG = paths['config'],
PATH_TESTGEN = paths['testcases'])
with open('{}/analyze.lua'.format(paths['config']), 'w') as f_lua:
f_lua.write(lua_mod)
# build the execute command according to
# whether the input is from file or stdin ('@@' in command.json)
command = build_cmd(os.path.join(basedir, 'command.json'))
command[0] = os.path.basename(command[0])
# replace the first occurance of '@@' in the command from right
# if no '@@' found, meaning that input should be piped to stdin
if '@@' in command:
command[command.index('@@')] = '${SYMB_FILE}'
else:
command.append('< ${SYMB_FILE}')
exec_cmd = ' '.join(command)
libs = []
for _, _, files in os.walk(paths['library']):
for lib in files:
libs.append('s2eget "{}"'.format(lib))
break
with open(paths['template']+'/bootstrap.sh.template') as f_bootstrap_temp:
bs_temp = f_bootstrap_temp.read()
bs_mod = bs_temp.format(_CB = command[0],
_INPUT = '{}'.format(input_hash),
_CMD = exec_cmd,
_LIBS = '\n'.join(libs))
with open(paths['config']+'/bootstrap.sh', 'w') as f_bs:
f_bs.write(bs_mod)
return paths['config']
def build_button(self, item):
return [{
"text": item['name'],
"callback_data": utils.build_cmd('album', {'i': item['id']})
}]