def __init__(self, mainConfiguration, reportingConfiguration):
self._logger = Logger.getLogger(__name__)
self.mainConfiguration = mainConfiguration
self.reportingConfiguration = reportingConfiguration
self.reporter = Reporter(self.reportingConfiguration)
def __init__(self, mainConfiguration, reportingConfiguration):
self._logger = Logger.getLogger(__name__)
self.mainConfiguration = mainConfiguration
self.reportingConfiguration = reportingConfiguration
self.reporter = Reporter(self.reportingConfiguration)
class Analysis(object):
"""The mother of any kind of analysis (manual, automated, ....)
It provides usefull common methods to manage the emulators, create reports,
reports events to optional databases...
"""
def __init__(self, mainConfiguration, reportingConfiguration):
self._logger = Logger.getLogger(__name__)
self.mainConfiguration = mainConfiguration
self.reportingConfiguration = reportingConfiguration
self.reporter = Reporter(self.reportingConfiguration)
def _createReport(self, idXp, emulatorName, packageName, filename, typeAnalysis, descAnalysis):
"""Creates a new report to collect events produced while executing experiment IdXp"""
if idXp is None:
raise Exception("Cannot create a report if no IdXP is provided.")
Analysis.createReport(self.reporter, idXp, emulatorName, packageName, filename, typeAnalysis, descAnalysis)
@staticmethod
def createReport(reporter, idXp, emulatorName, packageName, filename, typeAnalysis, descAnalysis):
"""Creates a new report to collect events produced while executing experiment IdXp"""
if idXp is None:
raise Exception("Cannot create a report if no IdXP is provided.")
if reporter is None:
raise Exception("Reporter is null.")
# TODO: one day, authors will be stored ;)
author = None
# compute file sha1
fileSha1 = Analysis._computeSha1(filename)
reporter.createReport(idXp, emulatorName, author, packageName, filename, fileSha1, typeAnalysis, descAnalysis)
def _reportEvent(self, idXp, sourceEvent, actionEvent, paramsEvent=None):
"""Insert in the report a new event."""
if idXp is None:
raise Exception("Cannot create a report if no IdXP is provided.")
self.reporter.reportEvent(idXp, sourceEvent, actionEvent, paramsEvent)
@staticmethod
def reportEvent(reporter, idXp, sourceEvent, actionEvent, paramsEvent=None):
"""Insert in the report a new event."""
if reporter is None:
raise Exception("Reporter is null.")
reporter.reportEvent(idXp, sourceEvent, actionEvent, paramsEvent)
def _generateIdXp(self, apkFiles):
"""Generates and returns the identifier of the experiment
based on the provided list of apk following schema:
md5("".join(sorted(apkFileNames))+currentDate)
"""
return Analysis.generateIdXp(apkFiles)
@staticmethod
def generateIdXp(apkFiles):
"""Generates and returns the identifier of the experiment
based on the provided list of apk following schema:
md5("".join(sorted(apkFileNames))+currentDate)
"""
logger = Logger.getLogger(__name__)
apkNames = [os.path.basename(apkFile) for apkFile in apkFiles]
strApkNames = ''.join(sorted(apkNames))
logger.warning(strApkNames)
currentDate = str(int(round(time.time() * 1000)))
# builds an IDXP (md5(apkFile+currentDate))
idXp = str(hashlib.md5(strApkNames+currentDate).hexdigest())
logger.debug("ID Experiment: {0} (based on {1} and {2})".format(idXp, strApkNames, currentDate))
return idXp
def _createEmulator(self, emulatorNumber, emulatorName):
"""Creates a new emulator and returns it"""
return Analysis.createDevice(emulatorNumber, emulatorName, self.mainConfiguration, None)
@staticmethod
def createDevice(adbNumber, name, mainConfiguration, backupDirectory):
logger = Logger.getLogger(__name__)
if adbNumber is None or int(adbNumber)<0:
raise Exception("Cannot create a device with an invalid adb number, must be >0")
if name is None or len(name)==0:
raise Exception("Cannot create a device if no name is provided.")
logger.debug("Creation of new device named '{0}'.".format(name))
if mainConfiguration.typeOfDevice=='real':
return PhysicalDevice(adbNumber, name, mainConfiguration, backupDirectory)
else:
return AVDEmulator(adbNumber, name, mainConfiguration)
def _writeConfigurationOnEmulator(self, emulator, idXP):
"""Creates a configuration for the current analysis
and deploys it on the provided emulator."""
Analysis.writeConfigurationOnEmulator(emulator, idXP, self.reportingConfiguration)
@staticmethod
def writeConfigurationOnEmulator(emulator, idXP, reportingConfiguration):
logger = Logger.getLogger(__name__)
configurationContent = """# Hooker Analysis Configuration File
# Network configuration
[elasticsearch]
elasticsearch_mode={0}
elasticsearch_nb_thread={1}
elasticsearch_ip=10.0.2.2
elasticsearch_port={3}
elasticsearch_index={4}
elasticsearch_doctype={5}
# File configuration
[file]
file_mode={6}
file_name={7}
[analysis]
idXP={8}
""".format(reportingConfiguration.elasticsearchMode,
1,
"10.0.2.2",
reportingConfiguration.elasticsearchPort,
reportingConfiguration.elasticsearchIndex,
reportingConfiguration.elasticsearchDoctype,
reportingConfiguration.fileMode,
"events.logs",
idXP)
logger.debug("Deploy the following configuration on emulator {0}: \n{1}".format(emulator.name, configurationContent))
# Write on the emulator
emulator.writeContentOnSdCard("experiment.conf", configurationContent)
@staticmethod
def _computeSha1(filepath):
BLOCKSIZE = 65536
hasher = hashlib.sha1()
with open(filepath, 'rb') as afile:
buf = afile.read(BLOCKSIZE)
while len(buf) > 0:
hasher.update(buf)
buf = afile.read(BLOCKSIZE)
return hasher.hexdigest()
@property
def mainConfiguration(self):
"""The main configuration
"""
return self.__mainConfiguration
@mainConfiguration.setter
def mainConfiguration(self, configuration):
if configuration is None:
raise Exception("Main configuration cannot be None")
self.__mainConfiguration = configuration
@property
def reportingConfiguration(self):
"""The reporting configuration
"""
return self.__reportingConfiguration
@reportingConfiguration.setter
def reportingConfiguration(self, configuration):
if configuration is None:
raise Exception("Reporing configuration cannot be None")
self.__reportingConfiguration = configuration
@property
def reporter(self):
"""The reporter
"""
return self.__reporter
@reporter.setter
def reporter(self, reporter):
self.__reporter = reporter
def executeExperiment(args):
"""
Executes the analysis for an application.
The analysis depends of the scenario the user as defined in configuration.
"""
logger = Logger.getLogger(__name__)
(listOfAPKs, iEmulator, mainConfiguration, analysisConfiguration,
reportingConfiguration) = args
reporter = Reporter(reportingConfiguration)
logger.warning("Execute experiment with scenario: [{0}]".format(', '.join(
analysisConfiguration.scenario)))
# When you play with emulators, you can modify this variable to sleep a bit longer.
# Default has to be >1
SLEEP_FACTOR = 1
if mainConfiguration.typeOfDevice == 'emulated':
SLEEP_FACTOR = 5
if SLEEP_FACTOR < 1:
raise Exception("SLEEP_FACTOR variable cannot be less than 1.")
while True:
# Pick one APK to analyse
try:
apkToAnalyze = listOfAPKs.get() # 0xFFFF
logger.info("APK to analyze : {0}".format(apkToAnalyze))
# Build the identifier experiment
idXp = Analysis.generateIdXp([apkToAnalyze])
# Build the emulator name
if mainConfiguration.typeOfDevice == 'emulated':
emulatorName = "Emulator_{0}".format(iEmulator)
else:
emulatorName = mainConfiguration.deviceId
# Create a new report for this analysis
logger.debug("Create report.")
Analysis.createReport(reporter, idXp, emulatorName, "unknown",
apkToAnalyze, "automatic",
mainConfiguration.name)
# Execute static analysis
logger.debug("Executing the Static Analysis")
staticAnalysis = StaticAnalysis(apkToAnalyze, mainConfiguration,
reporter, idXp)
staticAnalysis.execute()
logger.info(staticAnalysis)
# Create the emulator
if mainConfiguration.typeOfDevice == 'emulated':
device = Analysis.createEmulator(iEmulator,
emulatorName,
mainConfiguration,
analysisType="automatic")
else:
device = Analysis.createDevice(
iEmulator,
mainConfiguration.deviceId,
mainConfiguration,
analysisConfiguration.backupDirectory,
analysisType="automatic")
if device is None:
raise Exception(
"Something has prevented the creation of an device.")
# Starts the device
Analysis.reportEvent(reporter, idXp, emulatorName, "Start device")
device.start()
# Install et execute preparation applications
if mainConfiguration.typeOfDevice == 'emulated':
for prepareAPK in analysisConfiguration.prepareAPKs:
Analysis.reportEvent(reporter, idXp, emulatorName,
"Install preparation APK", prepareAPK)
device.installAPK(prepareAPK)
# Execute preparation applications
for prepareAPK in analysisConfiguration.prepareAPKs:
Analysis.reportEvent(reporter, idXp, emulatorName,
"Start activity",
os.path.basename(prepareAPK)[:-4])
device.startActivity(os.path.basename(prepareAPK)[:-4])
# Writes the experiment configuration on the device
Analysis.reportEvent(reporter, idXp, emulatorName,
"Write configuration file")
Analysis.writeConfigurationOnEmulator(device, idXp,
reportingConfiguration)
if mainConfiguration.typeOfDevice == 'emulated':
sleepDuration = 30
logger.debug(
"Waiting {0} seconds for the device to prepare...".format(
sleepDuration))
time.sleep(sleepDuration)
# Install the targeted application
Analysis.reportEvent(reporter, idXp, emulatorName,
"Install target APK", apkToAnalyze)
device.installAPK(apkToAnalyze)
time.sleep(5 * SLEEP_FACTOR / 5)
# We then follow the scenario user has filled in configuration
for order in analysisConfiguration.scenario:
if "execute" == order:
Analysis.reportEvent(reporter, idXp, emulatorName,
"Launching main activity",
staticAnalysis.mainActivity)
logger.info("Starting main activity: {0}".format(
staticAnalysis.mainActivity))
device.startActivityFromPackage(
staticAnalysis.packageName,
staticAnalysis.mainActivity)
time.sleep(5 * SLEEP_FACTOR)
elif "stimulate" == order:
Analysis.reportEvent(reporter, idXp, emulatorName,
"Stimulating package with monkey",
staticAnalysis.packageName)
logger.info("Stimulating with monkey: {0}".format(
staticAnalysis.packageName))
device.stimulateWithMonkey(staticAnalysis.packageName)
time.sleep(10 * SLEEP_FACTOR)
elif "externalStimulation" == order:
Analysis.reportEvent(
reporter, idXp, emulatorName,
"Stimulating phone with external conditions")
logger.info(
"Stimulating phone with external conditions...")
externalStimulation = TelnetEmulation(
reporter, idXp, device)
externalStimulation.start()
time.sleep(10 * SLEEP_FACTOR)
elif "reboot" == order:
Analysis.reportEvent(reporter, idXp, emulatorName,
"Rebooting device")
logger.info("Rebooting device.")
device.reboot()
time.sleep(5 * SLEEP_FACTOR)
Analysis.reportEvent(reporter, idXp, emulatorName,
"Analysis has finished", apkToAnalyze)
logger.info("Analysis of APK {0} has been finished.")
Analysis.reportEvent(reporter, idXp, emulatorName,
"Emulator closed")
device.stop()
except KeyboardInterrupt:
logger.debug("Keyboard interrupt caught\n")
# Try to stop device if necessary
if device is not None:
device.stop()
break
except Exception, e:
logger.error(
"Exception while executing an experiment : {0}".format(e))
tb = traceback.format_exc()
logger.error(tb)
try:
device.stop()
except Exception:
logger.error("Cannot stop the AVD, quitting experience.")
break
def executeExperiment(args):
"""
Executes the analysis for an application.
The analysis depends of the scenario the user as defined in configuration.
"""
logger = Logger.getLogger(__name__)
(listOfAPKs, iEmulator, mainConfiguration, analysisConfiguration,
reportingConfiguration) = args
reporter = Reporter(reportingConfiguration)
logger.warning("Execute experiment with scenario: [{0}]".format(', '.join(
analysisConfiguration.scenario)))
while True:
# Pick one APK to analyse
try:
apkToAnalyze = listOfAPKs.get() # 0xFFFF
logger.info("APK to analyze : {0}".format(apkToAnalyze))
# Build the identifier experiment
idXp = Analysis.generateIdXp([apkToAnalyze])
# Build the emulator name
emulatorName = "Emulator_{0}".format(iEmulator)
# Create a new report for this analysis
logger.debug("Create report.")
Analysis.createReport(reporter, idXp, emulatorName, "unknown",
apkToAnalyze, "automatic", None)
# Execute static analysis
logger.debug("Executing the Static Analysis")
staticAnalysis = StaticAnalysis(apkToAnalyze, mainConfiguration,
reporter, idXp)
staticAnalysis.execute()
logger.info(staticAnalysis)
# Create the emulator
emulator = Analysis.createEmulator(iEmulator, emulatorName,
mainConfiguration)
if emulator is None:
raise Exception(
"Something has prevented the creation of an emulator.")
# Starts the emulator
Analysis.reportEvent(reporter, idXp, "Emulator", "start")
emulator.start()
# Install et execute preparation applications
for prepareAPK in analysisConfiguration.prepareAPKs:
Analysis.reportEvent(reporter, idXp, "Emulator", "installAPK",
prepareAPK)
emulator.installAPK(prepareAPK)
# Execute preparation applications
for prepareAPK in analysisConfiguration.prepareAPKs:
Analysis.reportEvent(reporter, idXp, "Emulator",
"startActivity",
os.path.basename(prepareAPK)[:-4])
emulator.startActivity(os.path.basename(prepareAPK)[:-4])
# Writes the experiment configuration on the emulator
Analysis.reportEvent(reporter, idXp, "Emulator",
"writeConfiguration")
Analysis.writeConfigurationOnEmulator(emulator, idXp,
reportingConfiguration)
sleepDuration = 30
logger.debug(
"Waiting {0} seconds for the emulator to prepare...".format(
sleepDuration))
time.sleep(sleepDuration)
# Install the targeted application
Analysis.reportEvent(reporter, idXp, "Emulator", "installAPK",
apkToAnalyze)
emulator.installAPK(apkToAnalyze)
time.sleep(5)
# We then follow the scenario user has filled in configuration
for order in analysisConfiguration.scenario:
if "execute" == order:
Analysis.reportEvent(reporter, idXp, "Emulator",
"Launching main activity",
staticAnalysis.mainActivity)
logger.info("Starting main activity: {0}".format(
staticAnalysis.mainActivity))
emulator.startActivityFromPackage(
staticAnalysis.packageName,
staticAnalysis.mainActivity)
time.sleep(60)
elif "stimulate" == order:
Analysis.reportEvent(reporter, idXp, "Emulator",
"Stimulating package with monkey",
staticAnalysis.packageName)
logger.info("Stimulating with monkey: {0}".format(
staticAnalysis.packageName))
emulator.stimulateWithMonkey(staticAnalysis.packageName)
time.sleep(80)
elif "externalStimulation" == order:
Analysis.reportEvent(
reporter, idXp, "Emulator",
"Stimulating phone with external conditions")
logger.info(
"Stimulating phone with external conditions...")
externalStimulation = TelnetEmulation(
reporter, idXp, emulator, iEmulator)
externalStimulation.start()
time.sleep(30)
elif "reboot" == order:
Analysis.reportEvent(reporter, idXp, "Emulator",
"Rebooting emulator")
logger.info("Rebooting emulator.")
emulator.rebootAVD()
time.sleep(100)
Analysis.reportEvent(reporter, idXp, "Emulator",
"Finished analysis", apkToAnalyze)
logger.info("Analysis of APK {0} has been finished.")
Analysis.reportEvent(reporter, idXp, "Emulator", "closed")
emulator.stopAVD()
except KeyboardInterrupt:
pass
except Exception, e:
logger.error(
"Exception while executing an experiment : {0}".format(e))
tb = traceback.format_exc()
logger.error(tb)
try:
emulator.stopAVD()
except Exception:
logger.error("Cannot stop the AVD.")
class Analysis(object):
"""The mother of any kind of analysis (manual, automated, ....)
It provides usefull common methods to manage the emulators, create reports,
reports events to optional databases...
"""
def __init__(self, mainConfiguration, reportingConfiguration):
self._logger = Logger.getLogger(__name__)
self.mainConfiguration = mainConfiguration
self.reportingConfiguration = reportingConfiguration
self.reporter = Reporter(self.reportingConfiguration)
def _createReport(self, idXp, emulatorName, packageName, filename,
typeAnalysis, descAnalysis):
"""Creates a new report to collect events produced while executing experiment IdXp"""
if idXp is None:
raise Exception("Cannot create a report if no IdXP is provided.")
Analysis.createReport(self.reporter, idXp, emulatorName, packageName,
filename, typeAnalysis, descAnalysis)
@staticmethod
def createReport(reporter, idXp, emulatorName, packageName, filename,
typeAnalysis, descAnalysis):
"""Creates a new report to collect events produced while executing experiment IdXp"""
if idXp is None:
raise Exception("Cannot create a report if no IdXP is provided.")
if reporter is None:
raise Exception("Reporter is null.")
# TODO: one day, authors will be stored ;)
author = None
# compute file sha1
fileSha1 = Analysis._computeSha1(filename)
reporter.createReport(idXp, emulatorName, author, packageName,
filename, fileSha1, typeAnalysis, descAnalysis)
def _reportEvent(self, idXp, sourceEvent, actionEvent, paramsEvent=None):
"""Insert in the report a new event."""
if idXp is None:
raise Exception("Cannot create a report if no IdXP is provided.")
self.reporter.reportEvent(idXp, sourceEvent, actionEvent, paramsEvent)
@staticmethod
def reportEvent(reporter,
idXp,
sourceEvent,
actionEvent,
paramsEvent=None):
"""Insert in the report a new event."""
if reporter is None:
raise Exception("Reporter is null.")
reporter.reportEvent(idXp, sourceEvent, actionEvent, paramsEvent)
def _generateIdXp(self, apkFiles):
"""Generates and returns the identifier of the experiment
based on the provided list of apk following schema:
md5("".join(sorted(apkFileNames))+currentDate)
"""
return Analysis.generateIdXp(apkFiles)
@staticmethod
def generateIdXp(apkFiles):
"""Generates and returns the identifier of the experiment
based on the provided list of apk following schema:
md5("".join(sorted(apkFileNames))+currentDate)
"""
logger = Logger.getLogger(__name__)
apkNames = [os.path.basename(apkFile) for apkFile in apkFiles]
strApkNames = ''.join(sorted(apkNames))
logger.warning(strApkNames)
currentDate = str(int(round(time.time() * 1000)))
# builds an IDXP (md5(apkFile+currentDate))
idXp = str(hashlib.md5(strApkNames + currentDate).hexdigest())
logger.debug("ID Experiment: {0} (based on {1} and {2})".format(
idXp, strApkNames, currentDate))
return idXp
def _createEmulator(self, emulatorNumber, emulatorName):
"""Creates a new emulator and returns it"""
return Analysis.createDevice(emulatorNumber, emulatorName,
self.mainConfiguration, None)
@staticmethod
def createDevice(adbNumber, name, mainConfiguration, backupDirectory):
logger = Logger.getLogger(__name__)
if adbNumber is None or int(adbNumber) < 0:
raise Exception(
"Cannot create a device with an invalid adb number, must be >0"
)
if name is None or len(name) == 0:
raise Exception("Cannot create a device if no name is provided.")
logger.debug("Creation of new device named '{0}'.".format(name))
if mainConfiguration.typeOfDevice == 'real':
return PhysicalDevice(adbNumber, name, mainConfiguration,
backupDirectory)
else:
return AVDEmulator(adbNumber, name, mainConfiguration)
def _writeConfigurationOnEmulator(self, emulator, idXP):
"""Creates a configuration for the current analysis
and deploys it on the provided emulator."""
Analysis.writeConfigurationOnEmulator(emulator, idXP,
self.reportingConfiguration)
@staticmethod
def writeConfigurationOnEmulator(emulator, idXP, reportingConfiguration):
logger = Logger.getLogger(__name__)
configurationContent = """# Hooker Analysis Configuration File
# Network configuration
[elasticsearch]
elasticsearch_mode={0}
elasticsearch_nb_thread={1}
elasticsearch_ip=10.0.2.2
elasticsearch_port={3}
elasticsearch_index={4}
elasticsearch_doctype={5}
# File configuration
[file]
file_mode={6}
file_name={7}
[analysis]
idXP={8}
""".format(reportingConfiguration.elasticsearchMode, 1, "10.0.2.2",
reportingConfiguration.elasticsearchPort,
reportingConfiguration.elasticsearchIndex,
reportingConfiguration.elasticsearchDoctype,
reportingConfiguration.fileMode, "events.logs", idXP)
logger.debug(
"Deploy the following configuration on emulator {0}: \n{1}".format(
emulator.name, configurationContent))
# Write on the emulator
emulator.writeContentOnSdCard("experiment.conf", configurationContent)
@staticmethod
def _computeSha1(filepath):
BLOCKSIZE = 65536
hasher = hashlib.sha1()
with open(filepath, 'rb') as afile:
buf = afile.read(BLOCKSIZE)
while len(buf) > 0:
hasher.update(buf)
buf = afile.read(BLOCKSIZE)
return hasher.hexdigest()
@property
def mainConfiguration(self):
"""The main configuration
"""
return self.__mainConfiguration
@mainConfiguration.setter
def mainConfiguration(self, configuration):
if configuration is None:
raise Exception("Main configuration cannot be None")
self.__mainConfiguration = configuration
@property
def reportingConfiguration(self):
"""The reporting configuration
"""
return self.__reportingConfiguration
@reportingConfiguration.setter
def reportingConfiguration(self, configuration):
if configuration is None:
raise Exception("Reporing configuration cannot be None")
self.__reportingConfiguration = configuration
@property
def reporter(self):
"""The reporter
"""
return self.__reporter
@reporter.setter
def reporter(self, reporter):
self.__reporter = reporter
