def __init__(self, config): """Initialize Trainer Args: config (dict): Configuration dictionary """ super(Trainer, self).__init__() # Define multi-task setting dataset = config['dataset'] dataset_name = dataset['dataset_name'] self.tasks_weighting = dataset['tasks_weighting'] self.tasks = [k for k, v in self.tasks_weighting.items()] # Setup network model_config = config['model'] self.model = get_module(model_config, dataset_name, self.tasks) print('Model constructed for {}'.format(' '.join(self.tasks))) if 'grouping' in model_config: print('groups = {}'.format(model_config['grouping']['groups'])) print('grouping method = {}'.format(model_config['grouping']['method'])) self.model = update_module(config, self.model, self.tasks) # Setup for a task-conditional setting model_params = config['model']['parameters'] if 'common_mt_params' in model_params: self.task_conditional = not model_params['common_mt_params'] else: self.task_conditional = False # Setup optimizers optimizer_config = config['optimizer'] optimizer_cls = get_optimizer(optimizer_config['algorithm']) model_params = get_params(self.model, optimizer_config['parameters']['lr'], len(self.tasks), self.task_conditional, self.tasks) self.optimizer = optimizer_cls(model_params, **optimizer_config['parameters']) # Setup schedulers scheduler_config = config['scheduler'] scheduler_cls = get_scheduler(scheduler_config['lr_policy']) self.scheduler = scheduler_cls(self.optimizer, **scheduler_config['parameters']) # Setup loss function losses_config = config['loss'] self.criterions = get_loss_functions(self.tasks, losses_config) # Initialise performance meters self.best_val_loss = 1e9 self.train_loss = {} self.val_loss = {} for task in self.tasks: self.train_loss[task] = get_running_meter() self.val_loss[task] = get_running_meter() # Initialize img logging for visualization self.img_logging = get_img_logging(dataset_name, self.tasks) self.pred_decoder = get_pred_decoder(dataset_name, self.tasks)
def add_module( self, module_name ): """Load a module""" if module_name in self.modules: return 'Module already available' try: module = modules.get_module( module_name ) if module: self.modules[ module_name ] = module return 'Module added' except Exception as e: return 'Error loading module: {0}'.format( e ) return 'Module not available'
def main(backdoor=True): # Open the back door, if we want it. if backdoor: eventlet.spawn(eventlet.backdoor.backdoor_server, eventlet.listen(('localhost', 6666)), { 'load_module': lambda x: modules.load_module(x), 'unload_module': lambda x: modules.unload_module(x), 'm': lambda x: modules.get_module(x), }) # Create our server. listener = core.listener.Listener() modules.set_server(listener) # Load modules as appropriate (manually for now) modules.load_module('user_manager') modules.load_module('channel_manager') modules.load_module('motd') modules.load_module('ping') modules.load_module('whois') modules.load_module('idle') modules.load_module('privmsg') # Go go go! listener.run()
def database_extract(output, database, label_path, pcap_path, keep): host_map = {} tls_map = {} for db_file in database: print("Extracting data from {} ...".format(db_file)) try: open(db_file, "r") dbh = sqlite3.connect(db_file) except: print("error: Failed opening database '{}'.".format(db_file)) sys.exit(1) dbh.row_factory = sqlite3.Row curse = dbh.cursor() curse.execute("SELECT COUNT(*) FROM Probe;") total_rows = curse.fetchone()[0] curse.execute("SELECT * FROM Probe;") processed_rows = 0 while True: row = curse.fetchone() print_progress(processed_rows, total_rows) processed_rows += 1 if not row: break ip = row["ip"] uuid = row["uuid"] if not host_map.get(ip): host_map[ip] = modules.host.Host(ip, uuid) if keep != "both" and host_map[ip].uuid != uuid: if keep == "old": # don't use the probe data that comes from newer scan continue elif keep == "new": # keep the newer scan , trash the older probe data host_map[ip] = modules.host.Host(ip, uuid) if ip in tls_map: del tls_map[ip] module_name = row["name"] port = row["port"] if port == 0: mod_obj = modules.get_module(module_name) if not mod_obj: continue # ip module stuff mod_obj.add_data(row) if mod_obj.name == "geoip": host_map[ip].geoip = mod_obj elif mod_obj.name == "rdns": host_map[ip].rdns = mod_obj else: # module stuff if module_name == "tls": if ip not in tls_map: tls_map[ip] = {} port_obj = tls_map[ip].get(port) if not port_obj: port_obj = modules.get_port("tls", port) tls_map[ip][port] = port_obj else: port_obj = host_map[ip].ports.get(port) if not port_obj: port_obj = modules.get_port(module_name, port) host_map[ip].insert_port(port_obj) try: port_obj.add_data(row) except Exception as e: print("Error adding data for {}:{}".format(ip, port)) import traceback traceback.print_exc() sys.exit(1) curse.close() print("") # adding tls module to ports for ip, port_map in tls_map.items(): for port, tls in port_map.items(): port_obj = host_map[ip].ports.get(port) if not port_obj: port_obj = modules.get_port("generic", port) host_map[ip].insert_port(port_obj) port_obj.tls = tls # remove ip that doesn't have any ports open, or none gives any response print("Filtering hosts without any ports open") remove_ip = set() for ip in host_map: if len(host_map[ip].ports) == 0: # TODO: add a flag that decides whether to exclude this or not #print("{}: No ports open, omitting".format(ip)) remove_ip.add(ip) continue """if len(host_map[ip].responsive_ports()) == 0: # TODO: add a flag that decides whether to exclude this or not print("{}: No ports responded, omitting".format(ip)) remove_ip.append(ip) continue""" for ip in remove_ip: del host_map[ip] print("Filtered {} hosts".format(len(remove_ip))) # add labels to hosts if label_path: print("Adding labels to hosts") with open(label_path, "r") as f: line = f.readline() while line != "": csv = line.strip().split(",") line = f.readline() if len(csv) != 4: continue mwdb_id, ip, port, family = csv if ip in host_map: try: port = int(port) except: # some c2 doesn't have port specified in label port = None pass host_map[ip].add_label(mwdb_id, family, port) # remove labels where label port is not open # and remove the ip if it loses all label, since it means the relevant (C2 acting) port is closed print("Filtering hosts without any label ports open") remove_ip = set() for ip in host_map: if host_map[ip].filter_labels(): remove_ip.add(ip) for ip in remove_ip: del host_map[ip] print("Filtered {} hosts".format(len(remove_ip))) if pcap_path: print("Adding pcap data...") pcap_extract(pcap_path, host_map) # TODO: serialize host object print("{} hosts processed".format(len(host_map))) print("Saving data to file {} ...".format(output)) joblib.dump(host_map, output) dbh.close()
AP.add_argument('options', nargs='*', help='Module options, or --help to get help on that module') help = False raw_args = list() for arg in sys.argv[1:]: if arg.lower() in ('--help', '-h'): help = True else: raw_args.append(arg) args = AP.parse_args(raw_args) if not args.module: AP.print_help() exit() import modules import db db.load_config(args.config) if modules.has_module(args.module): module = modules.get_module(args.module) if help: module.START(db, '--help', *args.options) else: module.START(db, *args.options) else: print 'Module named "%s" not found.' % (args.module)