def __init__(self, config):
"""Initialize Trainer
Args:
config (dict): Configuration dictionary
"""
super(Trainer, self).__init__()
# Define multi-task setting
dataset = config['dataset']
dataset_name = dataset['dataset_name']
self.tasks_weighting = dataset['tasks_weighting']
self.tasks = [k for k, v in self.tasks_weighting.items()]
# Setup network
model_config = config['model']
self.model = get_module(model_config, dataset_name, self.tasks)
print('Model constructed for {}'.format(' '.join(self.tasks)))
if 'grouping' in model_config:
print('groups = {}'.format(model_config['grouping']['groups']))
print('grouping method = {}'.format(model_config['grouping']['method']))
self.model = update_module(config, self.model, self.tasks)
# Setup for a task-conditional setting
model_params = config['model']['parameters']
if 'common_mt_params' in model_params:
self.task_conditional = not model_params['common_mt_params']
else:
self.task_conditional = False
# Setup optimizers
optimizer_config = config['optimizer']
optimizer_cls = get_optimizer(optimizer_config['algorithm'])
model_params = get_params(self.model, optimizer_config['parameters']['lr'], len(self.tasks),
self.task_conditional, self.tasks)
self.optimizer = optimizer_cls(model_params, **optimizer_config['parameters'])
# Setup schedulers
scheduler_config = config['scheduler']
scheduler_cls = get_scheduler(scheduler_config['lr_policy'])
self.scheduler = scheduler_cls(self.optimizer, **scheduler_config['parameters'])
# Setup loss function
losses_config = config['loss']
self.criterions = get_loss_functions(self.tasks, losses_config)
# Initialise performance meters
self.best_val_loss = 1e9
self.train_loss = {}
self.val_loss = {}
for task in self.tasks:
self.train_loss[task] = get_running_meter()
self.val_loss[task] = get_running_meter()
# Initialize img logging for visualization
self.img_logging = get_img_logging(dataset_name, self.tasks)
self.pred_decoder = get_pred_decoder(dataset_name, self.tasks)
def add_module( self, module_name ):
"""Load a module"""
if module_name in self.modules:
return 'Module already available'
try:
module = modules.get_module( module_name )
if module:
self.modules[ module_name ] = module
return 'Module added'
except Exception as e:
return 'Error loading module: {0}'.format( e )
return 'Module not available'
def main(backdoor=True):
# Open the back door, if we want it.
if backdoor:
eventlet.spawn(eventlet.backdoor.backdoor_server, eventlet.listen(('localhost', 6666)), {
'load_module': lambda x: modules.load_module(x),
'unload_module': lambda x: modules.unload_module(x),
'm': lambda x: modules.get_module(x),
})
# Create our server.
listener = core.listener.Listener()
modules.set_server(listener)
# Load modules as appropriate (manually for now)
modules.load_module('user_manager')
modules.load_module('channel_manager')
modules.load_module('motd')
modules.load_module('ping')
modules.load_module('whois')
modules.load_module('idle')
modules.load_module('privmsg')
# Go go go!
listener.run()
def database_extract(output, database, label_path, pcap_path, keep):
host_map = {}
tls_map = {}
for db_file in database:
print("Extracting data from {} ...".format(db_file))
try:
open(db_file, "r")
dbh = sqlite3.connect(db_file)
except:
print("error: Failed opening database '{}'.".format(db_file))
sys.exit(1)
dbh.row_factory = sqlite3.Row
curse = dbh.cursor()
curse.execute("SELECT COUNT(*) FROM Probe;")
total_rows = curse.fetchone()[0]
curse.execute("SELECT * FROM Probe;")
processed_rows = 0
while True:
row = curse.fetchone()
print_progress(processed_rows, total_rows)
processed_rows += 1
if not row:
break
ip = row["ip"]
uuid = row["uuid"]
if not host_map.get(ip):
host_map[ip] = modules.host.Host(ip, uuid)
if keep != "both" and host_map[ip].uuid != uuid:
if keep == "old":
# don't use the probe data that comes from newer scan
continue
elif keep == "new":
# keep the newer scan , trash the older probe data
host_map[ip] = modules.host.Host(ip, uuid)
if ip in tls_map:
del tls_map[ip]
module_name = row["name"]
port = row["port"]
if port == 0:
mod_obj = modules.get_module(module_name)
if not mod_obj:
continue
# ip module stuff
mod_obj.add_data(row)
if mod_obj.name == "geoip":
host_map[ip].geoip = mod_obj
elif mod_obj.name == "rdns":
host_map[ip].rdns = mod_obj
else:
# module stuff
if module_name == "tls":
if ip not in tls_map:
tls_map[ip] = {}
port_obj = tls_map[ip].get(port)
if not port_obj:
port_obj = modules.get_port("tls", port)
tls_map[ip][port] = port_obj
else:
port_obj = host_map[ip].ports.get(port)
if not port_obj:
port_obj = modules.get_port(module_name, port)
host_map[ip].insert_port(port_obj)
try:
port_obj.add_data(row)
except Exception as e:
print("Error adding data for {}:{}".format(ip, port))
import traceback
traceback.print_exc()
sys.exit(1)
curse.close()
print("")
# adding tls module to ports
for ip, port_map in tls_map.items():
for port, tls in port_map.items():
port_obj = host_map[ip].ports.get(port)
if not port_obj:
port_obj = modules.get_port("generic", port)
host_map[ip].insert_port(port_obj)
port_obj.tls = tls
# remove ip that doesn't have any ports open, or none gives any response
print("Filtering hosts without any ports open")
remove_ip = set()
for ip in host_map:
if len(host_map[ip].ports) == 0:
# TODO: add a flag that decides whether to exclude this or not
#print("{}: No ports open, omitting".format(ip))
remove_ip.add(ip)
continue
"""if len(host_map[ip].responsive_ports()) == 0:
# TODO: add a flag that decides whether to exclude this or not
print("{}: No ports responded, omitting".format(ip))
remove_ip.append(ip)
continue"""
for ip in remove_ip:
del host_map[ip]
print("Filtered {} hosts".format(len(remove_ip)))
# add labels to hosts
if label_path:
print("Adding labels to hosts")
with open(label_path, "r") as f:
line = f.readline()
while line != "":
csv = line.strip().split(",")
line = f.readline()
if len(csv) != 4:
continue
mwdb_id, ip, port, family = csv
if ip in host_map:
try:
port = int(port)
except:
# some c2 doesn't have port specified in label
port = None
pass
host_map[ip].add_label(mwdb_id, family, port)
# remove labels where label port is not open
# and remove the ip if it loses all label, since it means the relevant (C2 acting) port is closed
print("Filtering hosts without any label ports open")
remove_ip = set()
for ip in host_map:
if host_map[ip].filter_labels():
remove_ip.add(ip)
for ip in remove_ip:
del host_map[ip]
print("Filtered {} hosts".format(len(remove_ip)))
if pcap_path:
print("Adding pcap data...")
pcap_extract(pcap_path, host_map)
# TODO: serialize host object
print("{} hosts processed".format(len(host_map)))
print("Saving data to file {} ...".format(output))
joblib.dump(host_map, output)
dbh.close()
AP.add_argument('options',
nargs='*',
help='Module options, or --help to get help on that module')
help = False
raw_args = list()
for arg in sys.argv[1:]:
if arg.lower() in ('--help', '-h'):
help = True
else:
raw_args.append(arg)
args = AP.parse_args(raw_args)
if not args.module:
AP.print_help()
exit()
import modules
import db
db.load_config(args.config)
if modules.has_module(args.module):
module = modules.get_module(args.module)
if help:
module.START(db, '--help', *args.options)
else:
module.START(db, *args.options)
else:
print 'Module named "%s" not found.' % (args.module)
