def cli_check_n_exec(func, prompt_prefix = "", circumvent_check = False, args=(), kwargs={}):
''' Check if the user really want's to continue and exec `func`.
Otherwise quit the program with exit code 1 and log some message.
Returns the result of the `func` or None.
'''
BASE_PROMPT = 'Continue and execute? (y/n): '
prompt = "%s\n\t\t%s" % (prompt_prefix, BASE_PROMPT) if prompt_prefix else BASE_PROMPT
if circumvent_check or raw_input(prompt).lower() == 'y':
return func(*args, **kwargs)
else:
log.critical("aborted ...")
sys.exit(1)
def cli_check_n_exec(func,
prompt_prefix="",
circumvent_check=False,
args=(),
kwargs={}):
''' Check if the user really want's to continue and exec `func`.
Otherwise quit the program with exit code 1 and log some message.
Returns the result of the `func` or None.
'''
BASE_PROMPT = 'Continue and execute? (y/n): '
prompt = "%s\n\t\t%s" % (prompt_prefix,
BASE_PROMPT) if prompt_prefix else BASE_PROMPT
if circumvent_check or raw_input(prompt).lower() == 'y':
return func(*args, **kwargs)
else:
log.critical("aborted ...")
sys.exit(1)
def __recreate_collections(self, gridfs = False, res_collection = False):
'''
Drop and recreate collections.
Parameters
----------
gridfs : bool, optional (default is False)
Recreate gridfs.
res_collection, bool, optional (default is False)
Recreate results collection.
'''
try:
if gridfs:
log.debug("dropping collection %s", GRIDFS_COLLS_PREFIX)
log.debug("dropping collection %s", FILES_COLL_NAME)
self.db.drop_collection(FILES_COLL_NAME)
log.debug("dropping collection %s", CHUNKS_COLL_NAME)
self.db.drop_collection(CHUNKS_COLL_NAME)
log.debug("recreating collection %s", GRIDFS_COLLS_PREFIX)
self._open_gridfs()
self._create_idx_for_colls()
except PyMongoError as e:
log.critical(e)
try:
if res_collection:
log.debug("dropping collection %s", RESULT_DOCUMENTS_COLLECTION_NAME)
self.db.drop_collection(RESULT_DOCUMENTS_COLLECTION_NAME)
self._open_res_coll()
log.debug("recreating collection %s", RESULT_DOCUMENTS_COLLECTION_NAME)
except PyMongoError as e:
log.critical(e)
def _analyze(self):
''' See doc of :py:method:`.BaseAnalyzer.analyze`. '''
# try to get registered workers
# it network fails at this point -> stop analysis
try:
clilog.info(CeleryUtil.get_workers_and_check_network())
except NetworkError as e:
log.critical(e)
return 0
# storage objects
storage = self.storage
clilog.info("Number of apks to analyze: %d", self._cnt_apks)
try:
# get analyze task
analyze_task = tasks[CeleryConstants.get_analyze_task_name()]
# create storage
storage.create_or_open_sub_storages()
# send tasks
start = time()
# apk generator over .apk or apk hashes
apk_gen = AnalyzeUtil.apk_id_or_raw_data_gen(
self.apks, force_raw_data=self.serialize_apks)
clilog.info("Task publishing progress:")
# send and serialize .apks
# if analysis via path serialize them!
if self.serialize_apks:
log.info("sending .apks to message broker")
self.group_result = group_result = GroupResult(results=[])
for args in self.send_apk_args_generator(apk_gen):
task = analyze_task.delay(*args)
group_result.add(task)
# send only apk id and let fetch via mongodb
else:
log.info("sending ids of apks")
task_group = group(
(analyze_task.s(*args)
for args in self.send_id_args_generator(apk_gen)))
# publish tasks
self.group_result = task_group()
log.info("sending took %ss", (time() - start))
sys.stderr.write("\nAnalysis progress:\n")
# start showing analysis progress
self.analyze_stats_view.start()
# wait for results
log.debug("joining on ResultGroup ... ")
# setup callback
callback_func = self.get_callback_func(self.success_handler,
self.error_handler)
CeleryUtil.join_native(self.group_result,
propagate=False,
callback=callback_func)
clilog.info("\nanalysis done ... ")
log.info("distributed analysis took %ss", (time() - start))
return self.stop_analysis_view()
except DatabaseOpenError as e:
log.critical(e)
return 0
except (KeyboardInterrupt, Exception) as e:
if not isinstance(e, KeyboardInterrupt):
log.exception(e)
log.warn(
"Interrupting distributed analysis ... Please wait a moment!")
log.warn("revoking tasks on all workers ...")
if celerysettings.CELERY_TASK_REVOCATION_ENABLED:
# revoke tasks
if self.group_result is None:
# revoke via task ids
log.debug("revoking while publishing tasks ...")
self.task_collection.revoke_all(terminate=True,
signal='SIGKILL')
else:
# revoke via GroupResult if yet available/created
# first available after all tasks have been send
self.group_result.revoke(terminate=True, signal='SIGKILL')
log.warn("revoked tasks and killed workers ...")
#return number of analyzed apks
return self.stop_analysis_view()
def _analyze(self):
''' See doc of :py:method:`.BaseAnalyzer.analyze`. '''
# try to get registered workers
# it network fails at this point -> stop analysis
try:
clilog.info(CeleryUtil.get_workers_and_check_network())
except NetworkError as e:
log.critical(e)
return 0
# storage objects
storage = self.storage
clilog.info("Number of apks to analyze: %d", self._cnt_apks)
try:
# get analyze task
analyze_task = tasks[CeleryConstants.get_analyze_task_name()]
# create storage
storage.create_or_open_sub_storages()
# send tasks
start = time()
# apk generator over .apk or apk hashes
apk_gen = AnalyzeUtil.apk_id_or_raw_data_gen(self.apks, force_raw_data = self.serialize_apks)
clilog.info("Task publishing progress:")
# send and serialize .apks
# if analysis via path serialize them!
if self.serialize_apks:
log.info("sending .apks to message broker")
self.group_result = group_result = GroupResult(results = [])
for args in self.send_apk_args_generator(apk_gen):
task = analyze_task.delay(*args)
group_result.add(task)
# send only apk id and let fetch via mongodb
else:
log.info("sending ids of apks")
task_group = group((analyze_task.s(*args) for args in self.send_id_args_generator(apk_gen)))
# publish tasks
self.group_result = task_group()
log.info("sending took %ss", (time() - start))
sys.stderr.write("\nAnalysis progress:\n")
# start showing analysis progress
self.analyze_stats_view.start()
# wait for results
log.debug("joining on ResultGroup ... ")
# setup callback
callback_func = self.get_callback_func(self.success_handler, self.error_handler)
CeleryUtil.join_native(self.group_result, propagate = False, callback = callback_func)
clilog.info("\nanalysis done ... ")
log.info("distributed analysis took %ss", (time() - start))
return self.stop_analysis_view()
except DatabaseOpenError as e:
log.critical(e)
return 0
except (KeyboardInterrupt, Exception) as e:
if not isinstance(e, KeyboardInterrupt):
log.exception(e)
log.warn("Interrupting distributed analysis ... Please wait a moment!")
log.warn("revoking tasks on all workers ...")
if celerysettings.CELERY_TASK_REVOCATION_ENABLED:
# revoke tasks
if self.group_result is None:
# revoke via task ids
log.debug("revoking while publishing tasks ...")
self.task_collection.revoke_all(terminate = True, signal = 'SIGKILL')
else:
# revoke via GroupResult if yet available/created
# first available after all tasks have been send
self.group_result.revoke(terminate = True, signal = 'SIGKILL')
log.warn("revoked tasks and killed workers ...")
#return number of analyzed apks
return self.stop_analysis_view()