def load_functions():
"""Load functions data to database."""
print "Functions"
# delete modules before data gets added to avoid duplicate info
# Function.query.delete()
# insert data from seed_function
with open("seed_data/seed_function") as function_data:
for row in function_data:
name, description, additional_info, sample_code, output, user_id, module_id = row.rstrip(
).split("|")
function = Function(name=name,
description=description,
additional_info=additional_info,
sample_code=sample_code,
output=output,
user_id=user_id,
module_id=module_id)
# add function to session
db.session.add(function)
# commit changes
db.session.commit()
print "Function loaded."
def make_function_at(static, addr):
if static[addr]['function'] != None:
# already function
return
rc = static.r2core
rc.cmd("af @ %d" % (addr, ))
this_function = Function(addr)
static['functions'].add(this_function)
info = rc.cmd_json("afj %d" % (addr, ))[0]
callrefs = info['callrefs']
for ref in callrefs:
if ref["type"] == "J":
static[ref['addr']]['crefs'].add(addr)
if ref["type"] == "C":
static[ref['addr']]['xrefs'].add(addr)
function_details = rc.cmd_json("pdfj @ %d" % addr)
if function_details['addr'] == addr:
for opcode in function_details['ops']:
static[opcode['offset']]['function'] = this_function
i = static[opcode['offset']]['instruction']
addr_re = re.compile(r'\| (0x[a-f0-9]+) ')
blocks = rc.cmd_json("agj %d" % addr)[0]['blocks']
for block in blocks:
this_block = Block(block['offset'])
this_function.add_block(this_block)
for op in block['ops']:
address = op['offset']
this_block.add(address)
static[address]['block'] = this_block
static['blocks'].add(this_block)
def test_function_definition():
scope = Scope()
func = Function(['a', 'b'],
[BinaryOperation(Reference('a'), '+', Reference('b'))])
func_def = FunctionDefinition('sum', func)
assert func_def.evaluate(scope) == func
assert scope['sum'] == func
def apply_unifier(a, unifier):
"""Apply unifier to atom/function."""
if bool(unifier):
new_terms = []
for term in a.terms:
if isinstance(term, Constant):
new_terms.append(term)
elif isinstance(term, Variable):
if term in unifier:
new_terms.append(unifier.get(term))
else:
return False
elif isinstance(term, Function):
new_terms.append(apply_unifier(term, unifier))
else:
return False
if all(new_terms):
return Function(name=a.name, terms=new_terms) if isinstance(
a, Function) else Atom(
name=a.name, terms=new_terms, neg=a.neg,
naf=a.naf) if isinstance(a, Atom) else False
else:
return False
else:
return False
def add_modules(username):
"""Add function/module information"""
if not verify_user(username):
return redirect("/login")
mname = request.form.get("mname")
mdesc = request.form.get("mdesc")
maddinfo = request.form.get("maddinfo")
fname = request.form.get("fname")
fdesc = request.form.get("fdesc")
faddinfo = request.form.get("faddinfo")
samplecode = request.form.get("samplecode")
output = request.form.get("output")
if fname == "":
flash("Please input a function name.")
return redirect("/{}/addmodules".format(username))
# fetch user to get user_id
user = User.query.filter_by(username=username).first()
existing_mod = Module.query.filter(
(Module.user_id == user.user_id) | (Module.user_id == 1),
Module.name == mname).first()
if mname == "":
module = Module.query.filter_by(module_id=1).first()
elif existing_mod:
module = existing_mod
else:
module = Module(name=mname,
description=mdesc,
additional_info=maddinfo,
user_id=user.user_id)
db.session.add(module)
db.session.commit()
function = Function(name=fname,
description=fdesc,
additional_info=faddinfo,
sample_code=samplecode,
output=output,
user_id=user.user_id,
module_id=module.module_id)
db.session.add(function)
db.session.commit()
flash("Your notes have been added.")
return redirect("/{}/studynotes".format(username))
def create_reduce_function(function_id, registry, run_registry=None):
function_name = "red-{}".format(function_id)
deploy_info = create_deploy_info(
settings.REDUCE_PATH,
settings.REDUCE_CONFIG_TEMPLATE_PATH,
registry,
run_registry
)
# create function
function = Function(function_name, deploy_info, debug=settings.DEBUG)
# load config data
config_data = function.get_config_data()
spec = config_data['spec']
if spec['env'] == None:
spec['env'] = []
# update env vars
update_env_var(spec['env'], name="ID", value=str(function_id))
update_env_var(spec['env'], name=settings.HDFS_HOST_KEY, value=settings.HDFS_HOST_VALUE)
output_dir = "" if settings.HDFS_OUTPUT_DIR_VALUE == "/" else settings.HDFS_OUTPUT_DIR_VALUE
update_env_var(spec['env'], name='REDUCER_OUTPUT_FILENAME', value="{}/out_{}.txt".format(output_dir, function_id))
update_env_var(spec['env'], name=settings.HDFS_CHUNK_COUNT_KEY, value=settings.HDFS_CHUNK_COUNT_VALUE)
update_env_var(spec['env'], name=settings.NUM_MAPPERS_KEY, value=settings.NUM_MAPPERS_VALUE)
update_env_var(spec['env'], name=settings.HDFS_USER_KEY, value=settings.HDFS_USER_VALUE)
update_env_var(spec['env'], name=settings.DONE_TOPIC_KEY, value=settings.DONE_TOPIC_VALUE)
update_env_var(spec['env'], name=settings.RMQ_HOST_KEY, value=settings.RMQ_HOST_VALUE)
update_env_var(spec['env'], name=settings.RMQ_PORT_KEY, value=settings.RMQ_PORT_VALUE)
update_env_var(spec['env'], name=settings.RMQ_USER_KEY, value=settings.RMQ_USER_VALUE)
update_env_var(spec['env'], name=settings.RMQ_PASS_KEY, value=settings.RMQ_PASS_VALUE)
update_env_var(spec['env'], name=settings.EXCHANGE_NAME_KEY, value=settings.EXCHANGE_NAME_VALUE)
# update rmq trigger info
update_rmq_trigger(
spec['triggers'],
settings.RMQ_URL_VALUE,
settings.EXCHANGE_NAME_VALUE,
["{}{}".format(settings.REDUCE_TOPIC_PREFIX_VALUE, function_id)],
max_workers=3,
)
# update http trigger
update_http_trigger(
spec['triggers'],
settings.REDUCE_PORT_START + function_id,
max_workers=3,
)
# apply config changes
function.apply_config(config_data)
return function
def create_map_function(function_id, registry, run_registry=None):
function_name = "map-{}".format(function_id)
deploy_info = create_deploy_info(
settings.MAP_PATH,
settings.MAP_CONFIG_TEMPLATE_PATH,
registry,
run_registry
)
# create function
function = Function(function_name, deploy_info, debug=settings.DEBUG)
# load config data
config_data = function.get_config_data()
spec = config_data['spec']
if spec['env'] == None:
spec['env'] = []
# update env vars
update_env_var(spec['env'], name="ID", value=str(function_id))
update_env_var(spec['env'], name=settings.RMQ_HOST_KEY, value=settings.RMQ_HOST_VALUE)
update_env_var(spec['env'], name=settings.RMQ_PORT_KEY, value=settings.RMQ_PORT_VALUE)
update_env_var(spec['env'], name=settings.RMQ_USER_KEY, value=settings.RMQ_USER_VALUE)
update_env_var(spec['env'], name=settings.RMQ_PASS_KEY, value=settings.RMQ_PASS_VALUE)
update_env_var(spec['env'], name=settings.NUM_REDUCERS_KEY, value=settings.NUM_REDUCERS_VALUE)
update_env_var(spec['env'], name=settings.REDUCE_TOPIC_PREFIX_KEY, value=settings.REDUCE_TOPIC_PREFIX_VALUE)
update_env_var(spec['env'], name=settings.EXCHANGE_NAME_KEY, value=settings.EXCHANGE_NAME_VALUE)
update_env_var(spec['env'], name=settings.HDFS_HOST_KEY, value=settings.HDFS_HOST_VALUE)
update_env_var(spec['env'], name=settings.DONE_TOPIC_KEY, value=settings.DONE_TOPIC_VALUE)
update_env_var(spec['env'], name=settings.HDFS_USER_KEY, value=settings.HDFS_USER_VALUE)
# update rmq trigger info
update_rmq_trigger(
spec['triggers'],
settings.RMQ_URL_VALUE,
settings.EXCHANGE_NAME_VALUE,
["{}{}".format(settings.MAP_TOPIC_PREFIX_VALUE, function_id)],
max_workers=3
)
# update http trigger
update_http_trigger(
spec['triggers'],
settings.MAP_PORT_START + function_id,
max_workers=3,
)
# apply config changes
function.apply_config(config_data)
return function
def test_end_to_end(capsys):
yat_fac = FunctionDefinition(
'fac',
Function(['n'], [
Conditional(BinaryOperation(
Reference('n'), '==', Number(0)), [Number(1)], [
BinaryOperation(
Reference('n'), '*',
FunctionCall(
Reference('fac'),
[BinaryOperation(Reference('n'), '-', Number(1))]))
])
]))
s = Scope()
yat_fac.evaluate(s)
a = Print(FunctionCall(Reference('fac'), [Number(5)])).evaluate(s)
out, err = capsys.readouterr()
assert a == Number(120)
assert out == '120\n'
assert err == ''
def make_function_at(static, address, recurse=True):
if static[address]['function'] != None:
# already function
return
start = time.time()
block_starts = set([address])
function_starts = set()
this_function = Function(address)
static['functions'].add(this_function)
def disassemble(address):
raw = static.memory(address, 0x10)
d = static[address]['instruction']
static[address]['function'] = this_function
for (c, flag) in d.dests():
if flag == DESTTYPE.call:
static._auto_update_name(c, "sub_%x" % (c))
function_starts.add(c)
#print "%s %x is in %x xrefs" % (d,address, c)
static[c]['xrefs'].add(address)
# add this to the potential function boundary starts
continue
if c != address + d.size():
#print "%s %x is in %x crefs" % (d,address, c)
static[c]['crefs'].add(address)
static._auto_update_name(c, "loc_%x" % (c))
block_starts.add(c)
#if we come after a jump and are an implicit xref, we are the start
#of a new block
elif d.is_jump() and not d.is_call():
static._auto_update_name(c, "loc_%x" % (c))
block_starts.add(c)
return d.dests()
# recursive descent pass
pending = Queue.Queue()
done = set()
pending.put(address)
while not pending.empty():
dests = disassemble(pending.get())
for (d, flag) in dests:
if flag == DESTTYPE.call:
#this will get handled in the function pass
continue
if d not in done:
pending.put(d)
done.add(d)
if (time.time() - start) > 0.01:
time.sleep(0.01)
start = time.time()
#print map(hex, done)
# block finding pass
for b in block_starts:
this_block = Block(b)
this_function.add_block(this_block)
address = b
i = static[address]['instruction']
while not i.is_ending() and i.size() != 0:
if address + i.size() in block_starts:
break
address += i.size()
i = static[address]['instruction']
this_block.add(address)
static[address]['block'] = this_block
if (time.time() - start) > 0.01:
time.sleep(0.01)
start = time.time()
static['blocks'].add(this_block)
# find more functions
if recurse:
for f in function_starts:
if static[f]['function'] == None:
make_function_at(static, f)
def addfnc(self, name, params, rettype):
if self.fnc:
self.fncsl.append((self.fnc, self.loc))
self.fnc = Function(name, params, rettype)
self.fncs[name] = self.fnc
self.prog.addfnc(self.fnc)
def test_function_call():
scope = Scope()
function = Function(['a'], [Reference('a')])
scope['function'] = function
function_call = FunctionCall(Reference('function'), [Number(8)])
assert function_call.evaluate(scope) == Number(8)
elif isinstance(term, Function):
new_terms.append(apply_unifier(term, unifier))
else:
return False
if all(new_terms):
return Function(name=a.name, terms=new_terms) if isinstance(
a, Function) else Atom(
name=a.name, terms=new_terms, neg=a.neg,
naf=a.naf) if isinstance(a, Atom) else False
else:
return False
else:
return False
# Test
if __name__ == '__main__':
from model import *
a = Atom(name='a',
terms=[
Function('f1',
[Function('f2', [Variable('X')]),
Variable('Y')])
])
b = Atom(name='a',
terms=[
Function('f1',
[Function('f2', [Constant('a')]),
Constant('b')])
])
print(apply_unifier(a, unify_atom(a, b)))
