def __init__(self, options: ModelOptions, n_classes: int):
self.name = 'simple_nn'
self.width = options.image_width
self.height = options.image_height
self.learning_rate = options.learning_rate
self.n_epochs = options.n_epochs
self.batch_size = options.batch_size
self.n_classes = n_classes
self.model = None
if not options.flatten:
Printer.error(
"Images has to be flattened before sending to Small VGGNet")
exit()
def build(self):
"""https://arxiv.org/abs/1602.07261"""
if self.height != 299 or self.width != 299:
Printer.error("ImageNet requires image sizes to be 299x299")
exit()
input_shape = (self.height, self.width, self.depth)
model = InceptionV4(num_classes=self.n_classes, dropout_prob=0.2)
optimizer = SGD(lr=self.learning_rate,
decay=self.learning_rate / self.n_epochs)
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
self.model = model
def from_json(json):
options = ModelOptions()
try:
options.model_name = json['model_name']
options.random_seed = int(json['random_seed'])
options.dataset_directory = json['dataset_directory']
options.image_extensions = set(json['image_extensions'].split())
options.image_width = int(json['image_width'])
options.image_height = int(json['image_height']) if json['image_height'] \
else options.image_width
options.flatten = (str(json['flatten']) == 'True')
options.test_percentage = float(json['test_percentage'])
options.learning_rate = float(json['learning_rate'])
options.n_epochs = int(json['n_epochs'])
options.batch_size = int(json['batch_size'])
options.output_dir = json['output_dir']
except ValueError:
Printer.error("Input data parsing failed.")
return options
neuralNet: SmallVGGNet = SmallVGGNet(options=options,
n_classes=len(classes))
neuralNet.build()
history = neuralNet.fit(train_x, train_y, test_x, test_y)
report = Report(neuralNet,
test_x,
test_y,
classes,
output_dir=options.output_dir)
report.evaluation_report()
report.chart(history)
model_output_path = os.path.join(
options.output_dir, '[{}]model.hdf5'.format(neuralNet.name))
classes_output_path = os.path.join(
options.output_dir, '[{}]classes.txt'.format(neuralNet.name))
answers_output_path = os.path.join(
options.output_dir, '[{}]options.json'.format(neuralNet.name))
neuralNet.save(model_output_path)
with open(classes_output_path, 'w') as fw:
fw.write('\n'.join(classes))
with open(answers_output_path, 'w') as fw:
fw.write(str(options.to_json()).replace("'", '"'))
except KeyboardInterrupt:
Printer.error("Exiting (Ctrl+C)")
exit()
def predict(self, x):
if self.model is None:
Printer.error("Model not built nor trained yet. Cannot predict.")
return self.model.predict(x, batch_size=self.batch_size)
def ask():
def is_digit_and_positive(_, x):
return x.isdigit() and 0 < int(x)
def is_digit_and_bigger_than_one(_, x):
return x.isdigit() and 1 < int(x)
def float_and_between_zero_one(_, x):
try:
x = float(x)
return 0 < x < 1
except ValueError:
return False
def is_directory(_, x):
return os.path.isdir(x)
questions = [
inquirer.List(
'model_name',
message='What model to use?',
choices=['SimpleNN', 'Small VGGNet', 'DenseNet', 'Inception']),
inquirer.Text('random_seed',
message='Set the random seed',
default='170081',
validate=is_digit_and_positive),
inquirer.Text('dataset_directory',
message='What is the dataset directory?',
default='../images',
validate=is_directory),
inquirer.Text(
'image_extensions',
message='What are image extensions?(Separate with space)',
default='jpg',
validate=lambda _, x: len(x) > 1),
inquirer.Text(
'image_width',
message='What is the image height? (Use 299 for Inception)',
default='64',
validate=is_digit_and_positive),
inquirer.Text(
'image_height',
message='What is the image weight? (Leave empty to use width)',
validate=lambda _, x: len(x) == 0 or is_digit_and_positive(
_, x)),
inquirer.Confirm(
'flatten',
message=
'Do images need to be flattened before feeding to the network?'
),
inquirer.Text('test_percentage',
message='How much to split for test data?',
default='0.15',
validate=float_and_between_zero_one),
inquirer.Text('learning_rate',
message='Set learning rate',
default='0.01',
validate=float_and_between_zero_one),
inquirer.Text('n_epochs',
message='Set number of epochs',
default='50',
validate=is_digit_and_bigger_than_one),
inquirer.Text('batch_size',
message='Set batch size',
default='64',
validate=is_digit_and_bigger_than_one),
inquirer.Text('output_dir',
message='What is the output directory?',
default='../output',
validate=is_directory)
]
try:
answers = inquirer.prompt(questions)
return ModelOptions.from_json(answers)
except KeyboardInterrupt:
Printer.error("User exit (Ctrl+C)")
return ModelOptions()
class Device(object):
# ==================================================================================================================
# FRAMEWORK ATTRIBUTES
# ==================================================================================================================
# Connection Parameters
_ip = Constants.GLOBAL_IP
_port = Constants.GLOBAL_PORT
_username = Constants.GLOBAL_USERNAME
_password = Constants.GLOBAL_PASSWORD
_pub_key_auth = bool(Constants.GLOBAL_PUB_KEY_AUTH)
_tools_local = Constants.PATH_TOOLS_LOCAL
_portforward = None
_frida_server = None
_debug_server = None
# App specific
_is_iOS8 = False
_is_iOS9 = False
_is_iOS7_or_less = False
_applist = None
_device_not_ready = bool(Constants.GLOBAL_SETUP_DEVICE)
# On-Device Paths
TEMP_FOLDER = Constants.DEVICE_PATH_TEMP_FOLDER
DEVICE_TOOLS = Constants.DEVICE_TOOLS
# Reference to External Objects
conn = None
app = None
installer = None
local_op = None
remote_op = None
printer = None
# ==================================================================================================================
# INIT
# ==================================================================================================================
def __init__(self):
# Init related objects
self.app = App(self)
self.installer = Installer(self)
self.local_op = LocalOperations()
self.remote_op = RemoteOperations(self)
self.printer = Printer()
self.connect()
self.setup()
# ==================================================================================================================
# UTILS - USB
# ==================================================================================================================
def _portforward_usb_start(self):
"""Setup USB port forwarding with TCPRelay."""
# Check if the user chose a valid port
if str(self._port) == '22':
raise Exception(
'Chosen port must be different from 22 in order to use USB over SSH'
)
# Setup the forwarding
self.printer.verbose('Setting up USB port forwarding on port %s' %
self._port)
cmd = '{app} -t 22:{port}'.format(app=self._tools_local['TCPRELAY'],
port=self._port)
self._portforward = self.local_op.command_subproc_start(cmd)
def _portforward_usb_stop(self):
"""Stop USB port forwarding."""
self.printer.verbose('Stopping USB port forwarding')
self.local_op.command_subproc_stop(self._portforward)
# ==================================================================================================================
# UTILS - SSH
# ==================================================================================================================
def _connect_ssh(self):
"""Open a new connection using Paramiko."""
try:
path = os.path.join(os.environ['HOME'], '.ssh', 'id_rsa')
key = paramiko.RSAKey.from_private_key_file(path)
self.printer.verbose('Setting up SSH connection...')
self.conn = paramiko.SSHClient()
self.conn.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.conn.connect(self._ip,
port=self._port,
username=self._username,
password=self._password,
allow_agent=self._pub_key_auth,
pkey=key)
except paramiko.AuthenticationException as e:
raise Exception(
'Authentication failed when connecting to %s. %s: %s' %
(self._ip, type(e).__name__, e.message))
except paramiko.SSHException as e:
raise Exception(
'Connection dropped. Please check your connection with the device, '
'and reload the module. %s: %s' %
(type(e).__name__, e.message))
except Exception as e:
raise Exception('Could not open a connection to %s. %s - %s' %
(self._ip, type(e).__name__, e.message))
def _disconnect_ssh(self):
"""Close the connection, if available."""
if self.conn:
self.conn.close()
def _exec_command_ssh(self, cmd, internal):
"""Execute a shell command on the device, then parse/print output."""
# Paramiko Exec Command
stdin, stdout, stderr = self.conn.exec_command(cmd)
# Parse STDOUT/ERR
out = stdout.readlines()
err = stderr.readlines()
if internal:
# For processing, don't display output
if err:
# Show error and abort run
err_str = ''.join(err)
raise Exception(err_str)
else:
# Display output
if out: map(lambda x: print('\t%s' % x, end=''), out)
if err:
map(
lambda x: print('\t%s%s%s' % (Colors.R, x, Colors.N),
end=''), err)
return out, err
# ==================================================================================================================
# FRIDA PORT FORWARDING
# ==================================================================================================================
def _portforward_frida_start(self):
"""Setup local port forward to enable communication with the Frida server running on the device"""
localhost = '127.0.0.1'
self._frida_server = SSHTunnelForwarder(
(self._ip, int(self._port)),
ssh_username=self._username,
ssh_password=self._password,
local_bind_address=(localhost, Constants.FRIDA_PORT),
remote_bind_address=(localhost, Constants.FRIDA_PORT),
)
self._frida_server.start()
def _portforward_frida_stop(self):
"""Stop local port forwarding"""
if self._frida_server:
self._frida_server.stop()
# ==================================================================================================================
# LLDB PORT FORWARDING
# ==================================================================================================================
def _portforward_debug_start(self):
"""Setup local port forward to enable communication with the debug server running on the device"""
localhost = '127.0.0.1'
self._debug_server = SSHTunnelForwarder(
(self._ip, int(self._port)),
ssh_username=self._username,
ssh_password=self._password,
local_bind_address=(localhost, Constants.DEBUG_PORT),
remote_bind_address=(localhost, Constants.DEBUG_PORT),
)
self._debug_server.start()
def _portforward_debug_stop(self):
"""Stop local port forwarding"""
if self._debug_server:
self._debug_server.stop()
# ==================================================================================================================
# UTILS - OS
# ==================================================================================================================
def _detect_ios_version(self):
"""Detect the iOS version running on the device."""
if self.remote_op.file_exist(Constants.DEVICE_PATH_APPLIST_iOS8):
self._is_iOS8 = True
elif self.remote_op.file_exist(Constants.DEVICE_PATH_APPLIST_iOS9):
self._is_iOS9 = True
else:
self._is_iOS7_or_less = True
def _list_apps(self):
"""List all the 3rd party apps installed on the device."""
def list_iOS_7():
raise Exception('Support for iOS < 8 not yet implemented')
def list_iOS_89(applist):
# Refresh UICache in case an app was installed after the last reboot
self.printer.verbose("Refreshing list of installed apps...")
self.remote_op.command_blocking(
'/bin/su mobile -c /usr/bin/uicache', internal=True)
# Parse plist file
pl = self.remote_op.parse_plist(applist)
self._applist = pl["User"]
# Dispatch
self._detect_ios_version()
if self._is_iOS8: list_iOS_89(Constants.DEVICE_PATH_APPLIST_iOS8)
elif self._is_iOS9: list_iOS_89(Constants.DEVICE_PATH_APPLIST_iOS9)
else: list_iOS_7()
# ==================================================================================================================
# EXPOSED COMMANDS
# ==================================================================================================================
def is_usb(self):
"""Returns true if using SSH over USB."""
return self._ip == '127.0.0.1' or self._ip == 'localhost'
def connect(self):
"""Connect to the device."""
if self.is_usb():
# Using SSH over USB, setup port forwarding first
self._portforward_usb_start()
# Connect
self._connect_ssh()
def disconnect(self):
"""Disconnect from the device."""
if self._portforward:
# Using SSH over USB, stop port forwarding
self._portforward_usb_stop()
self._disconnect_ssh()
def setup(self):
"""Create temp folder, and check if all tools are available"""
# Setup temp folder
self.printer.verbose("Creating temp folder: %s" % self.TEMP_FOLDER)
self.remote_op.dir_create(self.TEMP_FOLDER)
# Install tools
if self._device_not_ready:
self.printer.info("Configuring device...")
self._device_not_ready = self.installer.configure()
def cleanup(self):
"""Remove temp folder from device."""
self.printer.verbose("Cleaning up temp folder: %s" % self.TEMP_FOLDER)
self.remote_op.dir_delete(self.TEMP_FOLDER)
def shell(self):
"""Spawn a system shell on the device."""
cmd = 'sshpass -p "{password}" ssh {hostverification} -p {port} {username}@{ip}'.format(
password=self._password,
hostverification=Constants.DISABLE_HOST_VERIFICATION,
port=self._port,
username=self._username,
ip=self._ip)
self.local_op.command_interactive(cmd)
def pull(self, src, dst):
"""Pull a file from the device."""
self.printer.info("Pulling: %s -> %s" % (src, dst))
self.remote_op.download(src, dst)
def push(self, src, dst):
"""Push a file on the device."""
self.printer.info("Pushing: %s -> %s" % (src, dst))
self.remote_op.upload(src, dst)
def sync_files(self, src, dst):
"""sync files with device."""
device_ip = self.remote_op.get_ip()
device_ip = str(device_ip[0].strip())
self.printer.verbose("The Device IP address is: %s" % device_ip)
remote_dir = self._username + "@" + device_ip + ":" + src
self.printer.verbose("Start to sync data from %s >> %s" %
(remote_dir, dst))
subprocess.check_call(["rsync", "-avz", "--delete", remote_dir, dst])
def install_ipa(self, src):
"""Install app with ipa file."""
self.printer.verbose("Start to install %s to device" % src)
dst = self.remote_op.build_temp_path_for_file("app.ipa")
# Upload binary to device
self.printer.verbose("Uploading binary: %s" % src)
self.remote_op.upload(src, dst)
# Install
self.printer.verbose("Installing binary...")
cmd = "{bin} {app}".format(bin=self.DEVICE_TOOLS['IPAINSTALLER'],
app=dst)
self.remote_op.command_interactive_tty(cmd)
def uninstall_app(self, identifier):
self.printer.verbose("Uninstall binary...")
cmd = "{bin} -u {app}".format(bin=self.DEVICE_TOOLS['IPAINSTALLER'],
app=identifier)
self.remote_op.command_interactive_tty(cmd)
def have_installed(self, app_name):
self.printer.verbose("Start to check App whether have been installed")
self._list_apps()
if app_name in self._applist.keys():
self.printer.verbose("The %s App have been installed" % app_name)
return True
else:
self.printer.verbose("The %s App not been installed" % app_name)
return False
def get_keyboard_cache(self, LOCAL_KeyboardCache_DIR):
"""get keyboard cache from device."""
self.printer.verbose("Start to get Keyobard cache data from device.")
try:
self.remote_op.download(Constants.KEYBOARD_CACHE +
"en-dynamic.lm/",
LOCAL_KeyboardCache_DIR,
recursive=True)
self.pull(Constants.KEYBOARD_CACHE + "dynamic-text.dat",
LOCAL_KeyboardCache_DIR + ".")
except:
self.printer.error("Cannot sync the keyboard cache data.")
def get_cookies(self, LOCAL_COOKIES_DIR):
"""get Cookies file from device."""
self.printer.verbose("Start to get cookies files from device.")
try:
self.remote_op.download(Constants.COOKIES_PATH,
LOCAL_COOKIES_DIR,
recursive=True)
except:
self.printer.error("Cannot sync the cookies files.")
def analyze_cookies(self, fname):
cmd = 'python {bin} {temp_file}'.format(
bin=self._tools_local['BINARYCOOKIEREADER'], temp_file=fname)
out = self.local_op.command_interactive(cmd)
self.printer.verbose("COOKIES's value %s" % out)
return out
def dump_keychain(self):
self.printer.verbose("Start to dump keychain data from device.")
keychaindata = ''
cmd = '{} --action dump'.format(self.DEVICE_TOOLS['KEYCHAIN_DUMP'])
stdin, stdout, stderr = self.conn.exec_command(cmd)
out = stdout.read()
data = json.loads(out)
for key in data:
keychaindata += "," + (json.dumps(data[key]))
return keychaindata[1:]
def dump_head_memory(self, app_name, local_head_folder):
try:
self._list_apps()
metadata = self.app.get_metadata(app_name)
self.printer.info("Launching the app...")
self.app.open(metadata['bundle_id'])
pid = self.app.search_pid(metadata['name'])
# Create temp files/folders
dir_dumps = self.remote_op.build_temp_path_for_file("gdb_dumps")
fname_mach = self.remote_op.build_temp_path_for_file("gdb_mach")
fname_ranges = self.remote_op.build_temp_path_for_file(
"gdb_ranges")
self.remote_op.write_file(fname_mach, "info mach-regions")
if self.remote_op.dir_exist(dir_dumps):
self.remote_op.dir_delete(dir_dumps)
self.remote_op.dir_create(dir_dumps)
# Enumerate Mach Regions
self.printer.info("Enumerating mach regions...")
cmd = '''\
gdb --pid="%s" --batch --command=%s 2>/dev/null | grep sub-regions | awk '{print $3,$5}' | while read range; do
echo "mach-regions: $range"
cmd="dump binary memory %s/dump`echo $range| awk '{print $1}'`.dmp $range"
echo "$cmd" >> %s
done ''' % (pid, fname_mach, dir_dumps, fname_ranges)
self.remote_op.command_blocking(cmd)
# Dump memory
self.printer.info("Dumping memory (it might take a while)...")
cmd = 'gdb --pid="%s" --batch --command=%s &>>/dev/null' % (
pid, fname_ranges)
self.remote_op.command_blocking(cmd)
self.printer.info("Dump memory done.be stored under %s" %
dir_dumps)
self.remote_op.download(dir_dumps, local_head_folder, True)
#return dir_dumps
'''
# Check if we have dumps
self.printer.verbose("Checking if we have dumps...")
file_list = self.remote_op.dir_list(dir_dumps, recursive=True)
failure = filter(lambda x: 'total 0' in x, file_list)
if failure:
self.printer.error('It was not possible to attach to the process (known issue in iOS9. A Fix is coming soon)')
return
'''
except:
self.printer.error("Can't dump head memory data, Plese retry!!! ")
