def __init__(self,remote,db_name,exclude_pattern=None):
# Keep remote address:
self._remote = remote
# Keep remote db name:
self._db_name = db_name
# A thread executor. Allows only one task to be run every time.
self._te = ThreadExecutor()
# A regexp pattern that identifies functions that are not named, and
# should be ignored.
self._exclude_pattern = exclude_pattern
# A thread safe print function. I am not sure if this is rquired. It is
# done to be one the safe side:
self._print = print
def __init__(self,remote,db_name,exclude_pattern=None):
# Keep remote address:
self._remote = remote
# Keep remote db name:
self._db_name = db_name
# A thread executor. Allows only one task to be run every time.
self._te = ThreadExecutor()
# A regexp pattern that identifies functions that are not named, and
# should be ignored.
self._exclude_pattern = exclude_pattern
# A thread safe print function. I am not sure if this is rquired. It is
# done to be one the safe side:
self._print = print
def calculate(self, wait=False):
_debug("Calculating")
self.__freeze_changes()
parent = None
have_error = False
executor = None
for chunk in self.iterate_chunks():
if isinstance(chunk, StatementChunk):
changed = False
if chunk.needs_compile or chunk.needs_execute:
if not executor:
executor = ThreadExecutor(parent)
if executor:
statement = chunk.get_statement(self)
executor.add_statement(statement)
parent = chunk.statement
if executor:
if wait:
loop = gobject.MainLoop()
def on_statement_execution_state_changed(executor, statement):
if (statement.state == Statement.COMPILE_ERROR or
statement.state == Statement.EXECUTE_ERROR or
statement.state == Statement.INTERRUPTED):
self.__executor_error = True
statement.chunk.update_statement()
if self.__freeze_changes_count == 0:
self.__freeze_changes()
self.__chunk_changed(statement.chunk)
self.__thaw_changes()
else:
self.__chunk_changed(statement.chunk)
def on_complete(executor):
self.__executor = None
self.__set_state(NotebookFile.ERROR if self.__executor_error else NotebookFile.EXECUTE_SUCCESS)
if wait:
loop.quit()
self.__executor = executor
self.__executor_error = False
self.__set_state(NotebookFile.EXECUTING)
executor.connect('statement-executing', on_statement_execution_state_changed)
executor.connect('statement-complete', on_statement_execution_state_changed)
executor.connect('complete', on_complete)
if executor.compile():
executor.execute()
if wait:
loop.run()
else:
# Nothing to execute, we could have been in a non-success state if statements were deleted
# at the end of the file.
self.__set_state(NotebookFile.EXECUTE_SUCCESS)
self.__thaw_changes()
class FCatalogClient(object):
def __init__(self,remote,db_name,exclude_pattern=None):
# Keep remote address:
self._remote = remote
# Keep remote db name:
self._db_name = db_name
# A thread executor. Allows only one task to be run every time.
self._te = ThreadExecutor()
# A regexp pattern that identifies functions that are not named, and
# should be ignored.
self._exclude_pattern = exclude_pattern
# A thread safe print function. I am not sure if this is rquired. It is
# done to be one the safe side:
self._print = print
def _is_func_named(self,func_addr):
"""
Check if a function was ever named by the user.
"""
logger.debug('_is_func_named {}'.format(func_addr))
func_name = GetFunctionName(func_addr)
# Avoid functions like sub_409f498:
if func_name.startswith('sub_'):
return False
# If exclude_pattern was provided, make sure that the function
# name does not match it:
if self._exclude_pattern is not None:
mt = re.match(self._exclude_pattern,func_name)
if mt is not None:
return False
# Avoid reindexing FCATALOG functions:
if is_func_fcatalog(func_addr):
return False
return True
def _is_func_commit_candidate(self,func_addr):
"""
Is this function a candidate for committing?
"""
# Don't commit if chunked:
if is_func_chunked(func_addr):
return False
if not self._is_func_named(func_addr):
return False
if not is_func_long_enough(func_addr):
return False
return True
def _is_func_find_candidate(self,func_addr):
"""
Is this function a candidate for finding from database (Finding similars
for this function?)
"""
if is_func_chunked(func_addr):
return False
if self._is_func_named(func_addr):
return False
if not is_func_long_enough(func_addr):
return False
return True
def _iter_func_find_candidates(self):
"""
Iterate over all functions that are candidates for finding similars from
the remote database.
This function is IDA read thread safe.
"""
for func_addr in Functions():
if self._is_func_find_candidate(func_addr):
yield func_addr
def _commit_funcs_thread(self):
"""
Commit all the named functions from this idb to the server.
This is an IDA read thread safe function.
"""
self._print('Commiting functions...')
# Set up a connection to remote db:
frame_endpoint = TCPFrameClient(self._remote)
fdb = DBEndpoint(frame_endpoint,self._db_name)
for func_addr in Functions():
logger.debug('Iterating over func_addr: {}'.format(func_addr))
if not self._is_func_commit_candidate(func_addr):
continue
func_name = GetFunctionName(func_addr)
func_comment = strip_comment_fcatalog(get_func_comment(func_addr))
func_data = get_func_data(func_addr)
# If we had problems reading the function data, we skip it.
if func_data is None:
self._print('!> Skipping {}'.format(func_name))
continue
fdb.add_function(func_name,func_comment,func_data)
self._print(func_name)
# Close db:
fdb.close()
self._print('Done commiting functions.')
def commit_funcs(self):
"""
Commit all functions from this IDB to the server.
"""
try:
t = self._te.execute(self._commit_funcs_thread)
except ThreadExecutorError:
print('Another operation is currently running. Please wait.')
def _batch_similars(self,fdb,l_func_addr):
"""
Given a list of function addresses, request similars for each of those
functions. Then wait for all the responses, and return a list of tuples
of the form: (func_addr,similars)
This function is IDA read thread safe.
"""
# Send requests for similars for every function in l_func_addr list:
for func_addr in l_func_addr:
func_data = get_func_data(func_addr)
fdb.request_similars(func_data,1)
# Collect responses from remote server:
lres = []
for func_addr in l_func_addr:
similars = fdb.response_similars()
lres.append((func_addr,similars))
return lres
def _find_similars_thread(self,similarity_cut,batch_size):
"""
For each unnamed function in this database find a similar functions
from the fcatalog remote db, and rename appropriately.
This thread is IDA write thread safe.
"""
self._print('Finding similars...')
# Set up a connection to remote db:
frame_endpoint = TCPFrameClient(self._remote)
fdb = DBEndpoint(frame_endpoint,self._db_name)
# Iterate over blocks of candidate functions addresses:
for l_func_addr in blockify(self._iter_func_find_candidates(),\
batch_size):
# Send block to remote server and get results:
bsimilars = self._batch_similars(fdb,l_func_addr)
# Iterate over functions and results:
for func_addr,similars in bsimilars:
if len(similars) == 0:
# No similars found.
continue
# Get the first entry (Highest similarity):
fsim = similars[0]
# Discard if doesn't pass the similarity cut:
if fsim.sim_grade < similarity_cut:
continue
old_name = GetFunctionName(func_addr)
# Generate new name:
new_name = make_fcatalog_name(fsim.name,fsim.sim_grade,func_addr)
# If name matches old name, skip:
if new_name == old_name:
continue
# Set function to have the new name:
make_name(func_addr,new_name)
# Add the comments from the fcatalog entry:
func_comment = get_func_comment(func_addr)
func_comment_new = \
add_comment_fcatalog(func_comment,fsim.comment)
set_func_comment(func_addr,func_comment_new)
self._print('{} --> {}'.format(old_name,new_name))
# Close db:
fdb.close()
self._print('Done finding similars.')
def find_similars(self,similarity_cut,batch_size=GET_SIMILARS_BATCH_SIZE):
"""
For each unnamed function in this database find a similar functions
from the fcatalog remote db, and rename appropriately.
"""
try:
t = self._te.execute(self._find_similars_thread,\
similarity_cut,batch_size)
except ThreadExecutorError:
print('Another operation is currently running. Please wait.')
def calculate(self, wait=False, end_line=None):
_debug("Calculating")
self.__freeze_changes()
parent = None
have_error = False
executor = None
for chunk in self.iterate_chunks(end_line=end_line):
if isinstance(chunk, StatementChunk):
if chunk.needs_compile or chunk.needs_execute:
if not executor:
executor = ThreadExecutor(parent)
if executor:
statement = chunk.get_clean_statement(self)
executor.add_statement(statement)
parent = chunk.statement
# See if there are any more statements after the ones we are executing
more_statements = (end_line is not None) and \
any(isinstance(chunk, StatementChunk) for chunk in self.iterate_chunks(start_line=end_line))
if executor:
if wait:
loop = executor.event_loop
def on_statement_execution_state_changed(executor, statement):
if (statement.state == Statement.COMPILE_ERROR or
statement.state == Statement.EXECUTE_ERROR or
statement.state == Statement.INTERRUPTED):
self.__executor_error = True
statement.chunk.update_statement()
if self.__freeze_changes_count == 0:
self.__freeze_changes()
self.__chunk_changed(statement.chunk)
self.__thaw_changes()
else:
self.__chunk_changed(statement.chunk)
def on_complete(executor):
self.__executor.destroy()
self.__executor = None
if self.__executor_error:
self.__set_state(NotebookFile.ERROR)
elif more_statements:
self.__set_state(NotebookFile.NEEDS_EXECUTE)
else:
self.__set_state(NotebookFile.EXECUTE_SUCCESS)
if wait:
loop.quit()
self.__executor = executor
self.__executor_error = False
self.__set_state(NotebookFile.EXECUTING)
executor.sig_statement_executing.connect(on_statement_execution_state_changed)
executor.sig_statement_complete.connect(on_statement_execution_state_changed)
executor.sig_complete.connect(on_complete)
if executor.compile():
executor.execute()
if wait:
loop.run()
else:
# Nothing to execute, we could have been in a non-success state if statements were deleted
# at the end of the file.
if not more_statements:
self.__set_state(NotebookFile.EXECUTE_SUCCESS)
self.__thaw_changes()
class FCatalogClient(object):
def __init__(self,remote,db_name,exclude_pattern=None):
# Keep remote address:
self._remote = remote
# Keep remote db name:
self._db_name = db_name
# A thread executor. Allows only one task to be run every time.
self._te = ThreadExecutor()
# A regexp pattern that identifies functions that are not named, and
# should be ignored.
self._exclude_pattern = exclude_pattern
# A thread safe print function. I am not sure if this is rquired. It is
# done to be one the safe side:
self._print = print
def _is_func_named(self,func_addr):
"""
Check if a function was ever named by the user.
"""
logger.debug('_is_func_named {}'.format(func_addr))
func_name = GetFunctionName(func_addr)
# Avoid functions like sub_409f498:
if func_name.startswith('sub_'):
return False
# If exclude_pattern was provided, make sure that the function
# name does not match it:
if self._exclude_pattern is not None:
mt = re.match(self._exclude_pattern,func_name)
if mt is not None:
return False
# Avoid reindexing FCATALOG functions:
if is_func_fcatalog(func_addr):
return False
return True
def _is_func_commit_candidate(self,func_addr):
"""
Is this function a candidate for committing?
"""
# Don't commit if chunked:
if is_func_chunked(func_addr):
return False
if not self._is_func_named(func_addr):
return False
if not is_func_long_enough(func_addr):
return False
return True
def _is_func_find_candidate(self,func_addr):
"""
Is this function a candidate for finding from database (Finding similars
for this function?)
"""
if is_func_chunked(func_addr):
return False
if self._is_func_named(func_addr):
return False
if not is_func_long_enough(func_addr):
return False
return True
def _iter_func_find_candidates(self):
"""
Iterate over all functions that are candidates for finding similars from
the remote database.
This function is IDA read thread safe.
"""
for func_addr in Functions():
if self._is_func_find_candidate(func_addr):
yield func_addr
def _commit_funcs_thread(self):
"""
Commit all the named functions from this idb to the server.
This is an IDA read thread safe function.
"""
self._print('Commiting functions...')
# Set up a connection to remote db:
frame_endpoint = TCPFrameClient(self._remote)
fdb = DBEndpoint(frame_endpoint,self._db_name)
for func_addr in Functions():
logger.debug('Iterating over func_addr: {}'.format(func_addr))
if not self._is_func_commit_candidate(func_addr):
continue
func_name = GetFunctionName(func_addr)
func_comment = strip_comment_fcatalog(get_func_comment(func_addr))
func_data = get_func_data(func_addr)
# If we had problems reading the function data, we skip it.
if func_data is None:
self._print('!> Skipping {}'.format(func_name))
continue
fdb.add_function(func_name,func_comment,func_data)
self._print(func_name)
# Close db:
fdb.close()
self._print('Done commiting functions.')
def commit_funcs(self):
"""
Commit all functions from this IDB to the server.
"""
try:
t = self._te.execute(self._commit_funcs_thread)
except ThreadExecutorError:
print('Another operation is currently running. Please wait.')
def _batch_similars(self,fdb,l_func_addr):
"""
Given a list of function addresses, request similars for each of those
functions. Then wait for all the responses, and return a list of tuples
of the form: (func_addr,similars)
This function is IDA read thread safe.
"""
# Send requests for similars for every function in l_func_addr list:
for func_addr in l_func_addr:
func_data = get_func_data(func_addr)
fdb.request_similars(func_data,1)
# Collect responses from remote server:
lres = []
for func_addr in l_func_addr:
similars = fdb.response_similars()
lres.append((func_addr,similars))
return lres
def _find_similars_thread(self,similarity_cut,batch_size):
"""
For each unnamed function in this database find a similar functions
from the fcatalog remote db, and rename appropriately.
This thread is IDA write thread safe.
"""
self._print('Finding similars...')
# Set up a connection to remote db:
frame_endpoint = TCPFrameClient(self._remote)
fdb = DBEndpoint(frame_endpoint,self._db_name)
# Iterate over blocks of candidate functions addresses:
for l_func_addr in blockify(self._iter_func_find_candidates(),\
batch_size):
# Send block to remote server and get results:
bsimilars = self._batch_similars(fdb,l_func_addr)
# Iterate over functions and results:
for func_addr,similars in bsimilars:
if len(similars) == 0:
# No similars found.
continue
# Get the first entry (Highest similarity):
fsim = similars[0]
# Discard if doesn't pass the similarity cut:
if fsim.sim_grade < similarity_cut:
continue
old_name = GetFunctionName(func_addr)
# Generate new name:
new_name = make_fcatalog_name(fsim.name,fsim.sim_grade,func_addr)
# If name matches old name, skip:
if new_name == old_name:
continue
# Set function to have the new name:
make_name(func_addr,new_name)
# Add the comments from the fcatalog entry:
func_comment = get_func_comment(func_addr)
func_comment_new = \
add_comment_fcatalog(func_comment,fsim.comment)
set_func_comment(func_addr,func_comment_new)
self._print('{} --> {}'.format(old_name,new_name))
# Close db:
fdb.close()
self._print('Done finding similars.')
def find_similars(self,similarity_cut,batch_size=GET_SIMILARS_BATCH_SIZE):
"""
For each unnamed function in this database find a similar functions
from the fcatalog remote db, and rename appropriately.
"""
try:
t = self._te.execute(self._find_similars_thread,\
similarity_cut,batch_size)
except ThreadExecutorError:
print('Another operation is currently running. Please wait.')
def calculate(self, wait=False, end_line=None):
_debug("Calculating")
self.__freeze_changes()
parent = None
have_error = False
executor = None
for chunk in self.iterate_chunks(end_line=end_line):
if isinstance(chunk, StatementChunk):
if chunk.needs_compile or chunk.needs_execute:
if not executor:
executor = ThreadExecutor(parent)
if executor:
statement = chunk.get_clean_statement(self)
executor.add_statement(statement)
parent = chunk.statement
# See if there are any more statements after the ones we are executing
more_statements = (end_line is not None) and \
any(isinstance(chunk, StatementChunk) for chunk in self.iterate_chunks(start_line=end_line))
if executor:
if wait:
loop = executor.event_loop
def on_statement_execution_state_changed(executor, statement):
if (statement.state == Statement.COMPILE_ERROR
or statement.state == Statement.EXECUTE_ERROR
or statement.state == Statement.INTERRUPTED):
self.__executor_error = True
statement.chunk.update_statement()
if self.__freeze_changes_count == 0:
self.__freeze_changes()
self.__chunk_changed(statement.chunk)
self.__thaw_changes()
else:
self.__chunk_changed(statement.chunk)
def on_complete(executor):
self.__executor.destroy()
self.__executor = None
if self.__executor_error:
self.__set_state(NotebookFile.ERROR)
elif more_statements:
self.__set_state(NotebookFile.NEEDS_EXECUTE)
else:
self.__set_state(NotebookFile.EXECUTE_SUCCESS)
if wait:
loop.quit()
self.__executor = executor
self.__executor_error = False
self.__set_state(NotebookFile.EXECUTING)
executor.sig_statement_executing.connect(
on_statement_execution_state_changed)
executor.sig_statement_complete.connect(
on_statement_execution_state_changed)
executor.sig_complete.connect(on_complete)
if executor.compile():
executor.execute()
if wait:
loop.run()
else:
# Nothing to execute, we could have been in a non-success state if statements were deleted
# at the end of the file.
if not more_statements:
self.__set_state(NotebookFile.EXECUTE_SUCCESS)
self.__thaw_changes()