def proc_build(tarbz2_file, work_dir, save_dir, config):
tarbz2_lib.untar_file_to_path(tarbz2_file, work_dir)
#untar_one_pickle_file(tarbz2_file, work_dir)
pickle_file = work_dir + os.path.basename(tarbz2_file).replace(
'.tar.bz2', '')
pickle_file_content = pickle_lib.get_pickle_file_content(pickle_file)
#pickle_file_content = get_pickle_file_content(work_dir + os.path.basename(pickle_file).replace('.tar.bz2', ''))
binaries = set()
functions = set()
for elem in pickle_file_content:
binaries.add(elem[7])
functions.add(elem[2])
print(f'binaries >{binaries}<')
counter = 0
dataset_list = list()
## 1. get one binary
## 2. get one function of this binary
## 3. get disassembly of this function
## 4. check if this disassembly calls another function
## 4.1 filter @plt
## 5. if yes: get disassembly of caller function
## 6. save caller, callee, func_signature
## 7. check again, if it calls another function
## 8. if yes: get disassembly of caller function
## 9. save caller, calle, func_signature
##10. get disassembly of next function of this binary
##11. check if ....
for bin in binaries:
for func in functions:
## search for bin and func
for elem in pickle_file_content:
### if we found bin and func
if elem[7] == bin and elem[2] == func:
## get att disassembly
att_dis = elem[4]
## check every line if there is a call
for item in att_dis:
## find call in disas
if disassembly_lib.find_call_in_disassembly_line(item):
## if found, get callee name
callee_name = disassembly_lib.get_callee_name_from_disassembly_line(
item)
#print(f'callee_name >{callee_name}<')
## search for same bin, but callee func
for elem2 in pickle_file_content:
### if we found it, get return type and disassembly
if elem2[7] == bin and elem2[2] == callee_name:
return_type_func_sign = return_type_lib.get_return_type_from_function_signature(
elem2[0])
return_type = return_type_lib.get_return_type_from_gdb_ptype(
elem2[1])
###for debugging, what string is still unknown ?? should show nothing
if return_type == 'unknown':
print(
f'string_before_func_name: {string_before_func_name}'
)
if return_type == 'unknown':
#print('unknown found')
#breaker = True
#break
pass
elif return_type == 'delete':
#print('delete found')
### no return type found, so delete this item
pass
elif return_type == 'process_further':
print(f'ERRROOOORRRR---------------')
else:
dis1_str = ' '.join(att_dis)
dis2_str = ' '.join(elem2[4])
dis1_str = disassembly_lib.split_disassembly(
dis1_str)
dis2_str = disassembly_lib.split_disassembly(
dis2_str)
#dis1_str = dis_split(dis1_str)
#dis2_str = dis_split(dis2_str)
dis_str = dis1_str + dis2_str
#print(f'dis_str >{dis_str}<')
dataset_list.append(
(dis_str, return_type))
counter += 1
break
if dataset_list:
if config['save_file_type'] == 'pickle':
ret_file = open(
config['save_dir'] +
os.path.basename(pickle_file).replace('.tar.bz2', ''), 'wb+')
pickle_list = pickle.dump(dataset_list, ret_file)
ret_file.close()
else:
## save as tfrecord
dis_list = list()
ret_list = list()
for item in dataset_list:
dis_list.append(item[0])
ret_list.append(item[1])
raw_dataset = tf.data.Dataset.from_tensor_slices(
(dis_list, ret_list))
serialized_features_dataset = raw_dataset.map(tf_serialize_example)
filename = config['save_dir'] + os.path.basename(
tarbz2_file).replace('.pickle.tar.bz2', '') + '.tfrecord'
writer = tf.data.experimental.TFRecordWriter(filename)
writer.write(serialized_features_dataset)
return counter
def proc_build(tarbz2_file, work_dir, save_dir, config):
tarbz2_lib.untar_file_to_path(tarbz2_file, work_dir)
#untar_one_pickle_file(tarbz2_file, work_dir)
pickle_file = work_dir + os.path.basename(tarbz2_file).replace(
'.tar.bz2', '')
pickle_file_content = pickle_lib.get_pickle_file_content(pickle_file)
#pickle_file_content = get_pickle_file_content(work_dir + os.path.basename(pickle_file).replace('.tar.bz2', ''))
binaries = set()
functions = set()
for elem in pickle_file_content:
binaries.add(elem[7])
functions.add(elem[2])
print(f'binaries >{binaries}<')
counter = 0
dataset_list = list()
## 1. get one binary
## 2. get one function of this binary
## 3. get disassembly of this function
## 4. check if this disassembly calls another function
## 4.1 filter @plt
## 5. if yes: get disassembly of caller function
## 6. save caller, callee, nr_of_args
## 7. check again, if it calls another function
## 8. if yes: get disassembly of caller function
## 9. save caller, calle, func_signature
##10. get disassembly of next function of this binary
##11. check if ....
for bin in binaries:
for func in functions:
## search for bin and func
for elem in pickle_file_content:
### if we found bin and func
if elem[7] == bin and elem[2] == func:
## get att disassembly
att_dis = elem[4]
#print(f'att-dis >{att_dis}<')
## check every line if there is a call
for item in att_dis:
## find call in disas
if disassembly_lib.find_call_in_disassembly_line(item):
## if found, get callee name
callee_name = disassembly_lib.get_callee_name_from_disassembly_line(
item)
#print(f'callee_name >{callee_name}<')
## search for same bin, but callee func
for elem2 in pickle_file_content:
### if we found it, get return type and disassembly
if elem2[7] == bin and elem2[2] == callee_name:
#return_type_func_sign = return_type_lib.get_return_type_from_function_signature(elem2[0])
#return_type = return_type_lib.get_return_type_from_gdb_ptype(elem2[1])
nr_of_args = return_type_lib.get_nr_of_args_from_function_signature(
elem2[0])
arg_nr_we_want = 3
if nr_of_args < arg_nr_we_want:
#print(f'func got to less args for us')
break
arg_two = return_type_lib.get_arg_two_name_from_function_signature(
elem2[0])
result = common_stuff_lib.is_type_known(
arg_two)
if result == False:
#print(f'arg_two not a known type')
pass
else:
tmp_att_dis = att_dis
#print(f'len att-dis 1 >{len(tmp_att_dis)}<')
tmp_att_dis = disassembly_lib.clean_att_disassembly_from_comment(
tmp_att_dis)
callee_dis = disassembly_lib.clean_att_disassembly_from_comment(
elem2[4])
#print(f'len att-dis 1 >{len(tmp_att_dis)}<')
#print(f'att-dis >{tmp_att_dis}<')
dis1_str = ' '.join(tmp_att_dis)
#dis2_str = ' '.join(elem2[4])
dis2_str = ' '.join(callee_dis)
dis1_str = disassembly_lib.split_disassembly(
dis1_str)
dis2_str = disassembly_lib.split_disassembly(
dis2_str)
#dis1_str = dis_split(dis1_str)
#dis2_str = dis_split(dis2_str)
##the max-seq-length blows memory (>160GB ram) with model.fit() if e.g. over 6million
if (len(dis1_str) > 100000) or (
len(dis2_str) > 100000) or (
len(dis1_str) <
1) or (len(dis2_str) < 1):
print(
f'dis1_str >{len(dis1_str)}<')
print(
f'dis2_str >{len(dis2_str)}<')
#print(f"package >{elem[2]}< bin >{elem[3]}< file >{elem[6]}< func >{elem[7]}<")
#print(f"package >{elem2[2]}< bin >{elem2[3]}< file >{elem2[6]}< func >{elem2[7]}<")
else:
dis_str = dis1_str + dis2_str
#print(f'dis_str >{dis_str}<')
dataset_list.append(
(dis_str, arg_two))
counter += 1
break
if dataset_list:
if config['save_file_type'] == 'pickle':
ret_file = open(
config['save_dir'] +
os.path.basename(pickle_file).replace('.tar.bz2', ''), 'wb+')
pickle_list = pickle.dump(dataset_list, ret_file)
ret_file.close()
else:
## save as tfrecord
dis_list = list()
ret_list = list()
for item in dataset_list:
dis_list.append(item[0])
ret_list.append(item[1])
raw_dataset = tf.data.Dataset.from_tensor_slices(
(dis_list, ret_list))
serialized_features_dataset = raw_dataset.map(tf_serialize_example)
filename = config['save_dir'] + os.path.basename(
tarbz2_file).replace('.pickle.tar.bz2', '') + '.tfrecord'
writer = tf.data.experimental.TFRecordWriter(filename)
writer.write(serialized_features_dataset)
return counter
def proc_build(tarbz2_file, work_dir, save_dir, config):
tarbz2_lib.untar_file_to_path(tarbz2_file, work_dir)
#untar_one_pickle_file(tarbz2_file, work_dir)
pickle_file = work_dir + os.path.basename(tarbz2_file).replace(
'.tar.bz2', '')
pickle_file_content = pickle_lib.get_pickle_file_content(pickle_file)
#pickle_file_content = get_pickle_file_content(work_dir + os.path.basename(pickle_file).replace('.tar.bz2', ''))
binaries = set()
functions = set()
for elem in pickle_file_content:
binaries.add(elem[7])
functions.add(elem[2])
print(f'binaries >{binaries}<')
counter = 0
dataset_list = list()
## 1. get one binary
## 2. get one function of this binary
## 3. get disassembly of this function
## 4. check if this disassembly calls another function
## 4.1 filter @plt
## 5. if yes: get disassembly of caller function
## 6. save caller, callee, nr_of_args
## 7. check again, if it calls another function
## 8. if yes: get disassembly of caller function
## 9. save caller, calle, func_signature
##10. get disassembly of next function of this binary
##11. check if ....
for bin in binaries:
for func in functions:
## search for bin and func
for elem in pickle_file_content:
### if we found bin and func
if elem[7] == bin and elem[2] == func:
## get att disassembly
att_dis = elem[4]
#print(f'att-dis >{att_dis}<')
## check every line if there is a call
for item in att_dis:
## find call in disas
if disassembly_lib.find_call_in_disassembly_line(item):
## if found, get callee name
callee_name = disassembly_lib.get_callee_name_from_disassembly_line(
item)
#print(f'callee_name >{callee_name}<')
## search for same bin, but callee func
for elem2 in pickle_file_content:
### if we found it, get return type and disassembly
if elem2[7] == bin and elem2[2] == callee_name:
if (len(elem2[4]) >
(int(config[
'tokenized_disassembly_length']) /
2)) or (len(att_dis) > (int(config[
'tokenized_disassembly_length']) /
2)
) or (len(elem2[4]) < 1) or (
len(att_dis) < 1):
continue
#return_type_func_sign = return_type_lib.get_return_type_from_function_signature(elem2[0])
#return_type = return_type_lib.get_return_type_from_gdb_ptype(elem2[1])
nr_of_args = return_type_lib.get_nr_of_args_from_function_signature(
elem2[0])
###for debugging, what string is still unknown ?? should show nothing
# if return_type == 'unknown':
# print(f'string_before_func_name: {return_type_func_sign}')
#
# if return_type == 'unknown':
# #print('unknown found')
# #breaker = True
# #break
# pass
# elif return_type == 'delete':
# #print('delete found')
# ### no return type found, so delete this item
# pass
# elif return_type == 'process_further':
# print(f'ERRROOOORRRR---------------')
if nr_of_args == -1:
print(f'Error nr_of_args')
else:
print(f'nr_of_args >{nr_of_args}<',
end='\r')
tmp_att_dis = att_dis
#print(f'len att-dis 1 >{len(tmp_att_dis)}<')
tmp_att_dis = disassembly_lib.clean_att_disassembly_from_comment(
tmp_att_dis)
callee_dis = disassembly_lib.clean_att_disassembly_from_comment(
elem2[4])
#print(f'len att-dis 1 >{len(tmp_att_dis)}<')
#print(f'att-dis >{tmp_att_dis}<')
dis1_str = ' '.join(tmp_att_dis)
#dis2_str = ' '.join(elem2[4])
dis2_str = ' '.join(callee_dis)
dis1_str = disassembly_lib.split_disassembly(
dis1_str)
dis2_str = disassembly_lib.split_disassembly(
dis2_str)
#dis1_str = dis_split(dis1_str)
#dis2_str = dis_split(dis2_str)
#print(f'dis1_str >{dis1_str}<')
##the max-seq-length blows memory (>160GB ram) with model.fit() if e.g. over 6million
if (len(dis1_str) > (int(config[
'tokenized_disassembly_length']
) / 2)
) or (len(dis2_str) > (int(config[
'tokenized_disassembly_length']
) / 2)
) or (len(dis1_str) < 1) or (
len(dis2_str) < 1):
print(
f'tokenized_disassembly_length caller >{len(dis1_str)}<'
)
print(
f'tokenized_disassembly_length callee >{len(dis2_str)}<'
)
#print(f"package >{elem[2]}< bin >{elem[3]}< file >{elem[6]}< func >{elem[7]}<")
#print(f"package >{elem2[2]}< bin >{elem2[3]}< file >{elem2[6]}< func >{elem2[7]}<")
else:
dis_str = dis1_str + dis2_str
#print(f'dis_str >{dis_str}<')
dataset_list.append(
(dis_str, nr_of_args))
counter += 1
break
if dataset_list:
if config['save_file_type'] == 'pickle':
ret_file = open(
config['save_dir'] +
os.path.basename(pickle_file).replace('.tar.bz2', ''), 'wb+')
pickle_list = pickle.dump(dataset_list, ret_file)
ret_file.close()
else:
## save as tfrecord
dis_list = list()
ret_list = list()
for item in dataset_list:
dis_list.append(item[0])
ret_list.append(item[1])
raw_dataset = tf.data.Dataset.from_tensor_slices(
(dis_list, ret_list))
serialized_features_dataset = raw_dataset.map(tf_serialize_example)
filename = config['save_dir'] + os.path.basename(
tarbz2_file).replace('.pickle.tar.bz2', '') + '.tfrecord'
writer = tf.data.experimental.TFRecordWriter(filename)
writer.write(serialized_features_dataset)
return counter