def do_save(self):
for hexsha in self.cached_data:
val = self.cached_data[hexsha]
try:
actor = Actor.objects.get(full_name = val['name'])
except Actor.DoesNotExist:
actor = Actor(full_name = val['name'])
actor.save()
#Create the actor
try:
commit = Commit.objects.get(hexsha = hexsha)
except Commit.DoesNotExist:
commit = Commit(hexsha = hexsha, repo = self.repo_model, actor = actor)
commit.save()
for path, fun in val['funcs']:
if not Function.objects.filter(name = fun, path = path).exists():
fmodel = Function(name = fun, commit = commit, path = path)
fmodel.save()
print "Saved `%s` : `%s`" % (path[-16:], fun)
for file_name in val['files_changed']:
FileChange(path = file_name, actor = actor, commit = commit).save()
self.cached_data.clear()
def main():
func = Function()
fp = FunctionParameters(func)
best = fp.run()
print(func)
print(best)
fig, ax = plt.subplots()
target = ax.plot(*func())
trial, = ax.plot([], [], 'k.')
time_template = 'generation = %d\nchild = %d\nfitness = %.2f'
time_text = ax.text(0.05, 0.85, '', transform=ax.transAxes)
dt = 50
def animate(i):
g = i # i // fp.population
f = 0 # i % fp.population
gen = fp.generation[g]
fit = fp.fitnesses[g]
data = [g for _, g in sorted(zip(fit, gen), key=lambda pair: pair[0])][f]
trial.set_data(*data())
time_text.set_text(time_template % (g, f, fit[f]))
return trial, time_text
line_ani = animation.FuncAnimation(fig, animate,
range(0, len(fp.generation)),
interval=dt, blit=True, repeat=False)
plt.show()
def delete_function(function_id):
try:
f = Function.get(id=function_id)
except NotFoundError as error:
log.error(error.message)
return 404
else:
zip_file = f.zip_location[-36:]
delete_zip(zip_file)
f.delete()
def new_function():
function = request.json.get("function")
if function is None:
return jsonify({"error": "function lost"}), 400
try:
a = Function.query.filter_by(name=function).first()
if a is None:
a = Function.from_json({"function": function})
db.session.add(a)
db.session.commit()
return jsonify({"function_id": a.to_json().get("function_id")}), 201
except:
return "", 500
def generate_function(attributes):
function = Function()
function.initialize(
attributes["func_signature"],
attributes["exe_signature"],
attributes["args_size"],
attributes["vars_size"],
attributes["regs_size"],
attributes["frame_size"],
attributes["num_of_strings"],
attributes["num_of_calls"],
attributes["num_of_imms"],
attributes["num_of_insns"],
attributes["func_name"],
attributes["exe_name"],
attributes["immediates"],
attributes["strings"],
attributes["itypes"],
attributes["calls"],
attributes["block_bounds"],
attributes["edges"],
)
return function
def create_function(tenant_id, user_id, function_id, image_id, zip_url, tag, meta=None):
try:
f = Function()
f.meta.id = function_id # make this an uuid
f.function_id = function_id # make this an uuid
f.user_id = user_id
f.tenant_id = tenant_id
f.image_id = image_id
f.zip_location = zip_url
f.type = 'async'
f.event = 'webhook'
f.description = meta.get('description', None)
f.memory = meta.get('memory', None)
f.name = meta.get('name', None)
f.runtime = meta.get('runtime', None)
f.endpoint = "{0}/functions/{1}".format(endpoint, function_id)
f.tag = tag
return f.save()
except Exception as error:
log.error(error.message)
return None
def request_handler(request):
"""
Handles a Request for descriptions.
"""
print "REDB: request_handler called"
unpickled_request = redb_server_com.Request()
unpickled_request.from_json(request.FILES['pickled_request'].read())
# Adding more graph attributes:
unpickled_request.matching_grade_attributes = \
_process_matching_grade_attrs(unpickled_request.\
matching_grade_attributes)
# Building a temporary function
second_function = Function()
second_function.primary_attributes = \
json.dumps(unpickled_request.primary_attributes)
second_function.filtering_attributes = \
json.dumps(unpickled_request.filtering_attributes)
second_function.matching_grade_attributes = \
json.dumps(unpickled_request.matching_grade_attributes)
second_function.func_md5 = \
unpickled_request.primary_attributes["_func_md5_attr"]
second_function.exe_md5 = \
unpickled_request.primary_attributes["_exe_md5_attr"]
second_function.exe_name = \
unpickled_request.primary_attributes["_exe_name_attr"]
second_function.first_addr = \
unpickled_request.primary_attributes["_first_addr_attr"]
second_function.ins_num = \
unpickled_request.primary_attributes["_ins_num_attr"]
num_req_descs = unpickled_request.num_of_returned_comments
if num_req_descs <= 0 or num_req_descs > MAX_REQ_CMTS:
return _error_http_response("Error: required descriptions" +
"number not in range")
filtered_functions = []
for function_obj in Function.objects.all():
filter_grade = redb_similarity_grading.similarity_grading().\
filter_grade(function_obj, second_function)
if filter_grade >= redb_similarity_grading.FILTERING_THRESHOLD:
filtered_functions.append(function_obj)
function_grade_pairs = []
for filtered_function in filtered_functions:
matching_grade = redb_similarity_grading.similarity_grading().\
matching_grade(filtered_function, second_function)
if matching_grade >= redb_similarity_grading.MATCHING_THRESHOLD:
pair = (filtered_function, matching_grade)
function_grade_pairs.append(pair)
sorted_functions = sorted(function_grade_pairs, key=lambda func: func[1])
sorted_functions.reverse()
suggested_descriptions = []
for func, grade in sorted_functions:
if len(suggested_descriptions) == num_req_descs:
break
fitting_descriptions = Description.objects.filter(function=func)
for desc in fitting_descriptions:
func_name_and_cmts = json.loads(desc.func_name_and_cmts,
object_hook=_decode_dict)
suggested_description = \
SuggestedDecsription(func_name_and_cmts=func_name_and_cmts,
matching_grade=grade,
can_be_embedded=(func.ins_num ==
second_function.ins_num),
date=desc.date)
suggested_description_dict = suggested_description.to_dict()
suggested_descriptions.append(suggested_description_dict)
if len(suggested_descriptions) == num_req_descs:
break
response = redb_server_com.Response(\
suggested_descriptions_list=suggested_descriptions)
response = response.to_json()
http_response = HttpResponse(response)
print "REDB: request_handler finished"
return http_response
def submit_handler(request):
"""
Handles a Submitted descriptions.
"""
print "REDB: submit_handler called"
unpickled_submit = redb_server_com.Submit()
unpickled_submit.from_json(request.FILES['pickled_submit'].read())
# Adding more graph attributes:
unpickled_submit.matching_grade_attributes = \
_process_matching_grade_attrs(unpickled_submit.\
matching_grade_attributes)
primary_attributes_unjsoned = unpickled_submit.primary_attributes
primary_attributes = json.dumps(unpickled_submit.primary_attributes)
filtering_attributes = json.dumps(unpickled_submit.filtering_attributes)
matching_grade_attributes = \
json.dumps(unpickled_submit.matching_grade_attributes)
func_name_and_cmts = json.dumps(unpickled_submit.comments_and_name)
first_addr = primary_attributes_unjsoned["_first_addr_attr"]
exe_name = primary_attributes_unjsoned["_exe_name_attr"]
exe_md5 = primary_attributes_unjsoned["_exe_md5_attr"]
func_md5 = primary_attributes_unjsoned["_func_md5_attr"]
ins_num = primary_attributes_unjsoned["_ins_num_attr"]
func_set = Function.\
objects.filter(first_addr=first_addr,
exe_name=exe_name,
exe_md5=exe_md5,
func_md5=func_md5,
ins_num=ins_num,
primary_attributes=primary_attributes,
filtering_attributes=filtering_attributes,
matching_grade_attributes=matching_grade_attributes
)
func = None
duplicate = False
if len(func_set):
func = func_set[0]
dup_descriptions = \
Description.objects.filter(func_name_and_cmts=func_name_and_cmts)
if len(dup_descriptions):
duplicate = True
else:
func = Function(first_addr=first_addr,
exe_name=exe_name,
exe_md5=exe_md5,
func_md5=func_md5,
ins_num=ins_num,
primary_attributes=primary_attributes,
filtering_attributes=filtering_attributes,
matching_grade_attributes=matching_grade_attributes)
func.save()
print "REDB: Added new Function!"
if not duplicate:
func.description_set.create(func_name_and_cmts=func_name_and_cmts,
date=str(timezone.now()))
print "REDB: Added new Description!"
else:
print "REB: Duplicate Description!"
print "DEBUG: submit_handler finished"
return HttpResponse("Success")
def request_handler(request):
"""
Handles a Request for descriptions.
"""
print "REDB: request_handler called"
unpickled_request = redb_server_com.Request()
unpickled_request.from_json(request.FILES['pickled_request'].read())
# Adding more graph attributes:
unpickled_request.matching_grade_attributes = \
_process_matching_grade_attrs(unpickled_request.\
matching_grade_attributes)
# Building a temporary function
second_function = Function()
second_function.primary_attributes = \
json.dumps(unpickled_request.primary_attributes)
second_function.filtering_attributes = \
json.dumps(unpickled_request.filtering_attributes)
second_function.matching_grade_attributes = \
json.dumps(unpickled_request.matching_grade_attributes)
second_function.func_md5 = \
unpickled_request.primary_attributes["_func_md5_attr"]
second_function.exe_md5 = \
unpickled_request.primary_attributes["_exe_md5_attr"]
second_function.exe_name = \
unpickled_request.primary_attributes["_exe_name_attr"]
second_function.first_addr = \
unpickled_request.primary_attributes["_first_addr_attr"]
second_function.ins_num = \
unpickled_request.primary_attributes["_ins_num_attr"]
num_req_descs = unpickled_request.num_of_returned_comments
if num_req_descs <= 0 or num_req_descs > MAX_REQ_CMTS:
return _error_http_response("Error: required descriptions" +
"number not in range")
filtered_functions = []
for function_obj in Function.objects.all():
filter_grade = redb_similarity_grading.similarity_grading().\
filter_grade(function_obj, second_function)
if filter_grade >= redb_similarity_grading.FILTERING_THRESHOLD:
filtered_functions.append(function_obj)
function_grade_pairs = []
for filtered_function in filtered_functions:
matching_grade = redb_similarity_grading.similarity_grading().\
matching_grade(filtered_function, second_function)
if matching_grade >= redb_similarity_grading.MATCHING_THRESHOLD:
pair = (filtered_function, matching_grade)
function_grade_pairs.append(pair)
sorted_functions = sorted(function_grade_pairs, key=lambda func: func[1])
sorted_functions.reverse()
suggested_descriptions = []
for func, grade in sorted_functions:
if len(suggested_descriptions) == num_req_descs:
break
fitting_descriptions = Description.objects.filter(function=func)
for desc in fitting_descriptions:
func_name_and_cmts = json.loads(desc.func_name_and_cmts,
object_hook=_decode_dict)
suggested_description = \
SuggestedDecsription(func_name_and_cmts=func_name_and_cmts,
matching_grade=grade,
can_be_embedded=(func.ins_num ==
second_function.ins_num),
date=desc.date)
suggested_description_dict = suggested_description.to_dict()
suggested_descriptions.append(suggested_description_dict)
if len(suggested_descriptions) == num_req_descs:
break
response = redb_server_com.Response(\
suggested_descriptions_list=suggested_descriptions)
response = response.to_json()
http_response = HttpResponse(response)
print "REDB: request_handler finished"
return http_response
def submit_handler(request):
"""
Handles a Submitted descriptions.
"""
print "REDB: submit_handler called"
unpickled_submit = redb_server_com.Submit()
unpickled_submit.from_json(request.FILES['pickled_submit'].read())
# Adding more graph attributes:
unpickled_submit.matching_grade_attributes = \
_process_matching_grade_attrs(unpickled_submit.\
matching_grade_attributes)
primary_attributes_unjsoned = unpickled_submit.primary_attributes
primary_attributes = json.dumps(unpickled_submit.primary_attributes)
filtering_attributes = json.dumps(unpickled_submit.filtering_attributes)
matching_grade_attributes = \
json.dumps(unpickled_submit.matching_grade_attributes)
func_name_and_cmts = json.dumps(unpickled_submit.comments_and_name)
first_addr = primary_attributes_unjsoned["_first_addr_attr"]
exe_name = primary_attributes_unjsoned["_exe_name_attr"]
exe_md5 = primary_attributes_unjsoned["_exe_md5_attr"]
func_md5 = primary_attributes_unjsoned["_func_md5_attr"]
ins_num = primary_attributes_unjsoned["_ins_num_attr"]
func_set = Function.\
objects.filter(first_addr=first_addr,
exe_name=exe_name,
exe_md5=exe_md5,
func_md5=func_md5,
ins_num=ins_num,
primary_attributes=primary_attributes,
filtering_attributes=filtering_attributes,
matching_grade_attributes=matching_grade_attributes
)
func = None
duplicate = False
if len(func_set):
func = func_set[0]
dup_descriptions = \
Description.objects.filter(func_name_and_cmts=func_name_and_cmts)
if len(dup_descriptions):
duplicate = True
else:
func = Function(first_addr=first_addr,
exe_name=exe_name,
exe_md5=exe_md5,
func_md5=func_md5,
ins_num=ins_num,
primary_attributes=primary_attributes,
filtering_attributes=filtering_attributes,
matching_grade_attributes=matching_grade_attributes)
func.save()
print "REDB: Added new Function!"
if not duplicate:
func.description_set.create(func_name_and_cmts=func_name_and_cmts,
date=str(timezone.now()))
print "REDB: Added new Description!"
else:
print "REB: Duplicate Description!"
print "DEBUG: submit_handler finished"
return HttpResponse("Success")
def create_mappings():
log.info('Creating mappings')
Function.init()
Log.init()
Daemon.init()
Runtime.init()
