def setup_logger(): """Setup our logging instance. Returns: A logger for writing to two seperate files depending on error or debug messages """ logger = logging.getLogger() logger.setLevel(logging.NOTSET) formatter = logging.Formatter( '%(asctime)s {%(pathname)s:%(lineno)d}: \t%(message)s') handler = logging.FileHandler( filename=join_abs( ROOT_DIR, "logs", "drrobot.err")) handler.setLevel(logging.ERROR) handler.setFormatter(formatter) logger.addHandler(handler) handler = logging.FileHandler( filename=join_abs( ROOT_DIR, "logs", "drrobot.dbg")) handler.setLevel(logging.DEBUG) handler.setFormatter(formatter) logger.addHandler(handler) logging.getLogger("urllib3.connectionpool").disabled = True return logger
def create_dirs(parser): """Create directories for domain under output folder """ args = parser.parse_args() if getattr(args, 'domain', None): if not path.exists( join_abs(ROOT_DIR, "output", getattr(args, 'domain'))): makedirs( join_abs(ROOT_DIR, "output", getattr(args, 'domain'))) if not path.exists( join_abs(ROOT_DIR, "output", getattr(args, 'domain'), "aggregated")): makedirs( join_abs(ROOT_DIR, "output", getattr(args, 'domain'), "aggregated"))
def generate_output(self, _format, output_file): """Dumps contents of sqlite3 file into an alternative text format Args: _format: format of output file [xml, json] output_file: (Optional) filename to dump contents too Returns: (None) """ if not output_file: output_file = join_abs(self.OUTPUT_DIR, f"output.{_format}") file_index = self.aggregation.gen_output() if 'json' in _format: print("Generating JSON") try: """ need to dump json file here, error checking as well """ with open(output_file, 'w') as _file: json.dump(file_index, _file, indent="\t") except json.JSONDecodeError as error: self._print(str(error)) elif "xml" in _format: try: with open(output_file, 'w') as _file: xmlout = dicttoxml.dicttoxml(file_index) dom = parseString(xmlout) _file.write(dom.toprettyxml()) except TypeError: self._print("Error in generate_output check logs") LOG.exception("Error in generate output") except AttributeError: self._print("Error in generate_output check logs") LOG.exception("Error in generate output")
def start_dumpdb(drrobot, parser): """Dump database to output folder for given domain Generates all header text files and aggregated files under $HOME/.drrobot/output/<domain>/aggregated """ args = parser.parse_args() dbpath = getattr(args, "dbfile") if path.exists(dbpath): if not path.exists( join_abs(ROOT_DIR, "output", getattr(args, 'domain'), "headers")): makedirs(join_abs(ROOT_DIR, "output", getattr(args, 'domain'), "headers")) drrobot.dumpdb() else: print("[!] DB file does not exists, try running gather first")
def __init__(self, **kwargs): """Initialize Robot object. Args: dns (str): Added DNS for host configuration proxy (str): Proxy url of format "http://proxy.foo.bar:port domain (str): Target domain root_dir (str): Base directory containing config.json and template folders verbose (bool): verbose output on/off dbfile (str): Alternative database file to use Returns: None """ self.domain = kwargs.get("domain", None) self.ROOT_DIR = kwargs.get("root_dir") if self.domain: self.OUTPUT_DIR = join_abs(self.ROOT_DIR, "output", self.domain) self.aggregation = Aggregation(kwargs.get("dbfile"), self.domain, self.OUTPUT_DIR) self.dns = kwargs.get("dns", None) self.proxy = kwargs.get("proxy", None) self.verbose = kwargs.get("verbose", False) self.dbfile = kwargs.get("dbfile") # Disable warnings for insecure requests requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
def upload(self, **kwargs): """File upload Args: filepath (str): optional filepath to check for files to upload Returns: """ print("[*] doing post message") try: self.inst.login() team_id = self.inst.teams.get_team_by_name(self.team_name)['id'] channel_id = self.inst.channels.get_channel_by_name( channel_name=self.channel_name, team_id=team_id)['id'] except exceptions.NoAccessTokenProvided as er: print(f"[!] NoAccessTokenProvided {er}") LOG.exception() except exceptions.InvalidOrMissingParameters as er: print(f"[!] InvalidOrMissingParameters {er}") LOG.exception() try: if isfile(self.filepath): file_ids = [ self.inst.files.upload_file( channel_id=channel_id, files={ 'files': (basename(self.filepath), open(join_abs(self.filepath), 'rb')) })['file_infos'][0]['id'] ] self.inst.posts.create_post( options={ 'channel_id': channel_id, 'message': f"Recon Data {datetime.datetime.now()}", 'file_ids': file_ids }) elif isdir(self.filepath): file_location = abspath(self.filepath) self._upload_files(file_location, channel_id) except exceptions.ContentTooLarge: print(f"[!] ContentTooLarge in upload") LOG.exception() except exceptions.ResourceNotFound: print(f"[!] ResourceNotFound in upload") LOG.exception() except OSError: print(f"[!] File not found in upload") LOG.exception()
def _upload_files(self, file_location, channel_id): file_ids = [] for root, dirs, files in walk(file_location): for filename in files: # TODO add optional parameters for adjusting size. Implement # file splitting print(f"[...] Uploading {filename}") if stat(join_abs(root, filename)).st_size / 1024**2 > 49: print(f"[!]\tFile {filename} is to big, ignoring for now") continue else: file_ids += [ self.inst.files.upload_file( channel_id=channel_id, files={ 'files': (filename, open(join_abs(root, filename), 'rb')) })['file_infos'][0]['id'] ] if len(file_ids) >= 5: self.inst.posts.create_post( options={ 'channel_id': channel_id, 'message': f"Recon Data {datetime.datetime.now()}", 'file_ids': file_ids }) file_ids = [] if len(file_ids) > 0: self.inst.posts.create_post( options={ 'channel_id': channel_id, 'message': f"Recon Data {datetime.datetime.now()}", 'file_ids': file_ids })
def _get_files(self, file_location, max_filesize=50): """Generator to grab individual files for upload Args: file_location (str): Location of file(s) to upload max_filesize (int): Max allowed file size, in megabytes Returns: (Tuple) (filename, path to file) """ if not exists(file_location): return None for root, _, files in walk(file_location): for filename in files: # TODO add optional parameters for adjusting size. Implement # file splitting print(f"[...] Uploading {filename}") if stat(join_abs(root, filename)).st_size / \ 1024 ** 2 > max_filesize: print(f"[!]\tFile {filename} is to big, ignoring for now") continue else: yield (filename, join_abs(root, filename))
def __init__(self, **kwargs): """ Build Ansible object Args: **kwargs: { "ansible_arguments" : { "config" : "$config/httpscreenshot_play.yml", "flags": "-e '$extra' -i configs/ansible_inventory", "extra_flags":{ "1" : "variable_host=localhost", "2" : "variable_user=user", "3" : "infile=$infile", "4" : "outfile=$outfile/httpscreenshots.tar", "5" : "outfolder=$outfile/httpscreenshots" } }, "ansible_file_location" : "location", "verbose" : True, "domain" : "target.domain" } Returns: """ self.ansible_base = "ansible-playbook $flags $config" self.ansible_arguments = kwargs.get('ansible_arguments') self.ansible_file = kwargs.get('ansible_file_location', None) if not self.ansible_file: raise TypeError( "argument ansible_file must be of type string, not 'NoneType'") self.domain = kwargs.get('domain', None) if not self.domain: raise TypeError( "argument domain must be of type string, not 'NoneType'") self.output_dir = kwargs.get('output_dir') self.infile = kwargs.get('infile', None) if not self.infile: self.infile = join_abs(self.output_dir, "aggregated", "aggregated_protocol_hostnames.txt") self.verbose = kwargs.get('verbose', False) self.final_command = None
def rebuild(self, **kwargs): """Rebuilds sqlite3 database by parsing output files found under HOME directory Args: files (List): list of files to include in this rebuild. Returns: """ print("[*] Rebuilding DB") filenames = kwargs.get("files", None) output_files = [] output_files += [f for f in filenames if isfile(f)] for root, dirs, files in walk(self.OUTPUT_DIR, topdown=True): dirs = [d for d in filenames if isdir(d)] for _file in files: output_files += [join_abs(root, _file)] self.aggregation.aggregate(output_files=output_files) if kwargs.get("headers", False): self.aggregation.headers() print("[*] Rebuilding complete")
def gather(self, **kwargs): """Starts domain reconnaisance of target domain using the supplied tools Args: webtools (Dict): webtool dict scanners_dockers (Dict): scanners that use docker as their base scanners_ansible (Dict): scanners that use ansible as their base. headers (Boolean): if headers should be gathered Returns: """ _threads = [] output_folders = [] output_files = [] webtools = kwargs.get('webtools', {}) output_files += [v['output_file'] for _, v in webtools.items()] if webtools: _threads += self._run_webtools(webtools) scanners_dockers = kwargs.get('scanners_dockers', {}) output_folders += [ v.get('output_folder') for _, v in scanners_dockers.items() if v.get("output_folder") ] output_files += [ v.get('output_file') for _, v in scanners_dockers.items() if v.get('output_file') ] scanners_ansible = kwargs.get('scanners_ansible', {}) output_folders += [ v.get('output_folder', None) for _, v in scanners_ansible.items() if v.get("output_folder") ] output_files += [ v.get('output_file', None) for _, v in scanners_ansible.items() if v.get("output_file") ] for folder in output_folders: if not exists(join_abs(self.OUTPUT_DIR, folder)): makedirs(join_abs(self.OUTPUT_DIR, folder)) if scanners_dockers: scanner_threads, scanners = self._run_dockers(scanners_dockers) _threads += scanner_threads if scanners_ansible: _threads += self._run_ansible(scanners_ansible, None) if _threads: try: [thread.join() for thread in _threads if thread] except KeyboardInterrupt: self._print("Keyboard Interrupt sending kill signal to docker") try: _ = [scanner.kill() for scanner in scanners] except: pass raise KeyboardInterrupt verify = kwargs.get('verify', None) if verify and webtools: for k in webtools: if verify.lower() in k.lower(): verify = webtools[k].get('output_folder', None) if not verify: verify = webtools[k].get('output_file', None) break if verify: print(f"[*] Omit addresses gathered from web tool: {verify}") self.aggregation.aggregate(output_folders=output_folders, output_files=output_files) self.aggregation.dump_to_file() if kwargs.get("headers", False): self.aggregation.headers() print("[*] Gather complete")
def run(): """Main method for running Dr.ROBOT. Returns: Nothing. """ try: if not path.exists(join_abs(ROOT_DIR, "logs")): makedirs(join_abs(ROOT_DIR, "logs")) log = setup_logger() tools = load_config(get_config()) parser = parse_args(**tools, root_dir=ROOT_DIR) if len(sys.argv) <= 1: parser.print_help() sys.exit(1) args = parser.parse_args() log.debug(args) tool_check() drrobot = Robot(root_dir=ROOT_DIR, user_config=get_config(), **tools, dns=getattr(args, 'dns', None), proxy=getattr(args, 'proxy', None), domain=getattr(args, 'domain', None), verbose=getattr(args, 'verbose'), dbfile=getattr(args, 'dbfile'), verify=getattr(args, 'verify', None)) if not path.exists(join_abs(ROOT_DIR, "dbs")): makedirs(join_abs(ROOT_DIR, "dbs")) create_dirs(parser) log.debug(f"Dumping tools for run :{tools}") if args.actions in "gather": start_gather(drrobot, tools, parser) if args.actions in 'inspect': start_inspect(drrobot, tools, parser) if args.actions in "upload": start_upload(drrobot, tools, parser) if args.actions in "rebuild": start_rebuild(drrobot, tools, parser) if args.actions in "output": start_output(drrobot, parser) if args.actions in "dumpdb": start_dumpdb(drrobot, parser) except json.JSONDecodeError as error: print(f"[!] JSON load error, configuration file is bad.\n {error}") log.exception(error) except DatabaseError as error: print(f"[!] Something went wrong with SQLite\n {error}") log.exception(error) except KeyboardInterrupt: print("[!] KeyboardInterrup, exiting...") except OSError as error: log.exception(error) print(f"[!] OSError {error}") except TypeError as error: log.exception(error) print(f"[!] {error}")
def dump_to_file(self, dump_ips=True, dump_hostnames=True, dump_headers=False): """Dump database to file Dumps contents of database to three seperate files: 1. File with protocol (http/https) 2. File with hostname only 3. File with ip only Args: dump_ips (bool): dump ips dump_hostnames (bool): dump hostnames to file dump_headers (bool): dump headers to output firectory under headers Returns: """ dbconn = sqlite3.connect(self.dbfile) try: dbcurs = dbconn.cursor() ips = dbcurs.execute(f"""SELECT DISTINCT ip FROM data WHERE domain='{self.domain.replace('.', '_')}' AND ip IS NOT NULL""").fetchall() hostnames = dbcurs.execute(f"""SELECT DISTINCT hostname FROM data WHERE domain='{self.domain.replace('.', '_')}' AND hostname IS NOT NULL""").fetchall() """ Header options require there to have been a scan otherwise there will be no output but that should be expected. Might change db to a dataframe later... possible """ headers = dbcurs.execute( f"""SELECT DISTINCT ip, hostname, http_headers, https_headers FROM data WHERE domain='{self.domain.replace('.', '_')}' AND (http_headers IS NOT NULL AND https_headers IS NOT NULL)""").fetchall() if dump_ips: with open( join_abs(self.output_dir, 'aggregated', 'aggregated_ips.txt'), 'w') as _file: _file.writelines("\n".join(list(ip[0] for ip in ips))) if dump_hostnames: with open( join_abs(self.output_dir, 'aggregated', 'aggregated_hostnames.txt'), 'w') as _file: _file.writelines("\n".join( list(f"{host[0]}" for host in hostnames))) with open( join_abs(self.output_dir, 'aggregated', 'aggregated_protocol_hostnames.txt'), 'w') as _file: _file.writelines("\n".join( list(f"https://{host[0]}\nhttp://{host[0]}" for host in hostnames))) if dump_headers: keys = ["Ip", "Hostname", "Http", "Https"] for row in headers: _rows = dict(zip(keys, row)) with open( join_abs(self.output_dir, "headers", f"{_rows['Hostname']}_headers.txt"), 'w') as _file: _file.write(json.dumps(_rows, indent=2)) except sqlite3.Error: print("Failed to write to files in aggregated directory, exiting") self.logger.exception("Error in dump to file") return except OSError: print("Failed to write to files in aggregated directory, exiting") self.logger.exception("Error in dump to file") return finally: dbconn.close()
def _run_webtools(self, webtools): """Create custom WebTool object from dictionary containing WebTools Args: webtools (Dict): WebTool modules to build and run Example: { "Shodan" : { "short_name": "shodan", "class_name": "Shodan", "output_file" : "shodan.txt", "api_key" : null, "endpoint" : null, "username" : null, "password" : null }, } Returns: (List) Threads of WebTool objects that are running """ threads = [] for tool, tool_dict in webtools.items(): try: output_file_loc = join_abs(self.ROOT_DIR, "output", self.domain, tool_dict.get('output_file', tool)) attr = { "proxies": { 'http': self.proxy, 'https': self.proxy }, "api_key": tool_dict.get('api_key', None), "domain": self.domain, "output_file": output_file_loc, "username": tool_dict.get('username', None), "password": tool_dict.get('password', None), "endpoint": tool_dict.get('endpoint', None), "verbose": self.verbose, } self._print(f"Building webtool {tool} with options \n\t{attr}") """ module contains the modules loaded in from web_resources relative to __main__ tool_class contains the class object with the name specified in the default/user config file. tool_class_obj contains the instantiated object using the tool_class init method. """ module = importlib.import_module('robot_api.api.web_resources', __name__) tool_class = getattr(module, tool_dict.get('class_name')) tool_class_obj = tool_class(**attr) threads += [ multiprocessing.Process(target=tool_class_obj.do_query, daemon=True) ] except KeyError: print("[!] Error locating key for tool. " + "Check error log for details") LOG.exception("Key Error in run_webtools method") except json.JSONDecodeError: print("[!] Failure authenticating to service. " + "Check error log for details") LOG.exception("Failure authenticating to service.") except ValueError: print("[!] Value Error thrown. Check error log for details") LOG.exception("Value error on init") for thread in threads: thread.start() return threads
def _run_ansible(self, ansible_mods, infile): """Create ansible objects from dictionary containing the configurations. Args: ansible_mods (Dict): Dictionary of ansible modules to build and run infile (Strings): Path to file for upload to ansible server Example: { "Eyewitness": { "name" : "Eyewitness", "short_name" : "eye", "default" : 2, "ansible_arguments" : { "config" : "$config/ansible_plays/eyewitness_play.yml", "flags": "-e '$extra'", "extra_flags":{ "1" : "variable_host=localhost", "2" : "infile=$infile/aggregated_ips.txt", "3" : "outfile=$outfile/Eyewitness.tar", "4" : "outfolder=$outfile/Eyewitness" } }, "output" : "/tmp/output", "enabled" : false } } Returns: """ for ansible, ansible_json in ansible_mods.items(): try: attr = {} print(f"[*] Running {ansible} as ansible Module") if infile is None: if ansible_json.get('ansible_arguments', None) is not None: if ansible_json.get('ansible_arguments').get( 'infile', None) is not None: infile = join_abs( self.OUTPUT_DIR, ansible_json.get('ansible_arguments').get( 'infile', None)) elif infile is None: print("[*] No file provided, dumping db for input") if getsize(self.dbfile) > 0: self.aggregation.dump_to_file() else: print( "[!] \tDatabase file is empty. Have you ran gather?" ) if not isfile(infile): print("[!] file provided does not exist, terminating") return attr['infile'] = infile attr['domain'] = self.domain attr['ansible_file_location'] = join_abs( self.ROOT_DIR, "ansible_plays") attr['output_dir'] = self.OUTPUT_DIR attr['ansible_arguments'] = ansible_json.get( "ansible_arguments") attr['verbose'] = self.verbose self._print( f"Creating ansible {ansible} with attributes\n\t {attr}") ansible_mod = Ansible(**attr) ansible_mod.build() ansible_mod.run() except OSError: print(f"[!] Something went wrong. Check error log for details") LOG.exception("Error in ansible method") except TypeError: print(f"[!] Something went wrong. Check error log for details") LOG.exception("Error in ansible method")
def gen_tarfile(self): tarname = join_abs(self._docker_options['tarfiles'], basename(self._active_config_path) + ".tar.gz") with tarfile.open(name=tarname, mode="w:gz") as tar: tar.add(self._active_config_path, "Dockerfile") tar.add(self._docker_options['certs'], 'certs') return tarname
def gen_output(self): """Generate dictionary containing all data from the database Returns: A dictionary containing all data from database """ if not path.exists(self.dbfile): print("No database file found. Exiting") return None dbconn = sqlite3.connect(self.dbfile) dbcurs = dbconn.cursor() db_headers = dbcurs.execute(f"""SELECT * FROM data WHERE domain='{self.domain.replace('.','_')}' AND (http_headers IS NOT NULL OR https_headers IS NOT NULL)""" ).fetchall() db_ips = dbcurs.execute(f"""SELECT DISTINCT ip, hostname FROM data WHERE domain='{self.domain.replace('.', '_')}'""" ).fetchall() """ (IP, HOSTNAME, HTTP, HTTPS) """ file_index = {} """ Need to be smarter about this: Multiple different sql queries 1. Grabs all those with headers: most likely that if they have headers they have a screenshot glob can run and take it's time. 2. Grab all unique ips 2a. Grab all unique hostnames 3. Update json with all documents """ for _, ipv4, hostname, http, https, _ in db_headers: ip_screenshots = glob.glob("**/*{}*".format(ipv4), recursive=True) hostname_screeshots = glob.glob("**/*{}*".format(hostname), recursive=True) image_files = [] for _file in ip_screenshots: image_files += [join_abs(getcwd(), _file)] for _file in hostname_screeshots: image_files += [join_abs(getcwd(), _file)] file_index[ipv4] = { "hostnames": [hostname], "http_header": http, "https_header": https, "images": image_files } for ipv4, hostname in db_ips: if ipv4 not in file_index: file_index[ipv4] = { "hostnames": [hostname], "http_header": "", "https_header": "", "images": [] } elif hostname not in file_index[ipv4]['hostnames']: file_index[ipv4]['hostnames'] += [hostname] return file_index
def _run_dockers(self, dockers): """Build Docker containers provided dictionary of arguments for building Dockerize is a wrapper around the docker module. This module allows Dr.ROBOT to specify required arguments for building its containers Args: dockers (Dict): dictionary with all docker objects to build Example: { "Aquatone" : { "name": "Aquatone", "docker_name": "aqua", "default_conf": "docker_buildfiles/Dockerfile.Aquatone.tmp", "active_conf": "docker_buildfiles/Dockerfile.Aquatone", "description": "AQUATONE is a set of tools for performing reconnaissance on domain names", "src": "https://github.com/michenriksen/aquatone", "output": "/aqua", "output_dir" : "aquatone" } } Returns: A tuple containing the threads and the scanners being ran """ scanners = [] self._print(f"Creating scanners{dockers.keys()}") for scan, scan_dict in dockers.items(): options = scan_dict options.update({"proxy": self.proxy or None}) options.update({"dns": self.dns or None}) options.update({"target": self.domain}) options.update({"verbose": self.verbose}) options.update({"tarfiles": join_abs(self.ROOT_DIR, "tarfiles")}) options.update({"certs": join_abs(self.ROOT_DIR, "certs")}) output_dir = self.OUTPUT_DIR if options.get("output_folder", None): output_dir = join_abs(self.OUTPUT_DIR, options.get("output_folder")) self._print(f"Creating scanner for {scan} with options: " + "{json.dumps(options, indent=4)}") scanners += [ Docker(active_config_path=join_abs(self.ROOT_DIR, scan_dict['active_conf']), default_config_path=join_abs(self.ROOT_DIR, scan_dict['default_conf']), docker_options=options, output_dir=output_dir) ] self._print("Threading builds") build_threads = [ threading.Thread(target=scanner.build, daemon=True) for scanner in scanners ] for build in build_threads: build.start() build_monitor_threads = [ threading.Thread(target=scanner.monitor_build, daemon=True) for scanner in scanners ] for thread in build_monitor_threads: thread.start() for build in build_monitor_threads: build.join() for scanner in scanners: if scanner.error or scanner.image is None: print(f"[!] Error building {scanner.name}. Check logs") self._print("Images built, running containers") for scanner in scanners: scanner.run() status_threads = [ threading.Thread(target=scanner.update_status, daemon=True) for scanner in scanners ] for stat in status_threads: stat.start() return (status_threads, scanners)
def aggregate(self, output_files=[], output_folders=[]): """Aggregates all output from scanners into the database Args: output_files: list of output files referenced in config.json output_folders: list of folders to for aggregation Returns: """ try: dbconn = sqlite3.connect(self.dbfile) dbcurs = dbconn.cursor() # Enable foreign key support dbcurs.execute("PRAGMA foreign_keys=1") # Simple database that contains list of domains to run against dbcurs.execute(""" CREATE TABLE IF NOT EXISTS domains ( domain VARCHAR PRIMARY KEY, UNIQUE(domain) ) """) # Setup database to keep all data from all targets. This allows us # to use a single model for hosting with Django dbcurs.execute(""" CREATE TABLE IF NOT EXISTS data ( domainid INTEGER PRIMARY KEY, ip VARCHAR, hostname VARCHAR, http_headers TEXT, https_headers TEXT, domain VARCHAR, found TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL, FOREIGN KEY(domain) REFERENCES domains(domain), UNIQUE(hostname) ) """) # Quickly create entry in domains table. dbcurs.execute( f"INSERT OR IGNORE INTO domains(domain) VALUES ('{self.domain.replace('.', '_')}')" ) dbconn.commit() all_files = [] for name in output_files: if path.isfile(join_abs(self.output_dir, name)): all_files += [join_abs(self.output_dir, name)] elif path.isfile(name): all_files += [name] else: print( f"[!] File {name} does not exist, verify scan results") for folder in output_folders: for root, _, files in walk(join_abs(self.output_dir, folder)): for _file in files: if path.isfile(join_abs(root, _file)): all_files += [join_abs(root, _file)] # multi_queue = multiprocessing.Queue() qu_manager = multiprocessing.Manager() pool = multiprocessing.Pool(5) queue = qu_manager.Queue() reverse_partial = partial(self._reverse_ip_lookup, queue) pool.map(reverse_partial, all_files) pool.close() self._build_db(queue, dbcurs) dbconn.commit() except sqlite3.Error: self.logger.exception("Error in aggregation") finally: dbconn.close()