def activate(self, stoq): self.stoq = stoq parser = argparse.ArgumentParser() parser = StoqArgs(parser) worker_opts = parser.add_argument_group("Plugin Options") worker_opts.add_argument("-O", "--comment", dest='user_comments', default="", help="Comment associated with sample \ submission") worker_opts.add_argument("-w", "--worker", dest='submission_list', action='append', help="Worker queues that should process \ sample. May be used multiple times") options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options) # Activate the appropriate plugin so we can publish messages, if needed. self.publish_connector = self.stoq.load_plugin(self.publisher, 'source') return True
def activate(self, stoq): self.stoq = stoq parser = argparse.ArgumentParser() parser = StoqArgs(parser) worker_opts = parser.add_argument_group("Plugin Options") worker_opts.add_argument("-p", "--priority", dest='priority', type=int, choices=range(0,11), help="Priority of message") worker_opts.add_argument("-w", "--worker", dest='submission_list', action='append', help="Worker queues that should process \ sample. May be used multiple times") worker_opts.add_argument("-q", "--publisher", dest='publisher', help="Source plugin to publish samples to") options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options) # Activate the appropriate plugin so we can publish messages, # if needed. self.publish_connector = self.stoq.load_plugin(self.publisher, 'source') if not hasattr(self.publish_connector, 'publish'): self.log.warn("Plugin does not support publishing to queues") return False return True
def activate(self, stoq): self.stoq = stoq parser = argparse.ArgumentParser() parser = StoqArgs(parser) options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options) # Make sure our options are the proper type self.timeout = float(self.timeout) self.port = int(self.port) self.interval = int(self.interval) self._connect() # Start the ping threads so we can make sure we always have # a valid connection threads = threading.Thread(target=self._ping, args=()) threads.daemon = True threads.start() if not self.clamd: return False return True
def activate(self, stoq): # Ensure the stoq object is available throughout our class self.stoq = stoq # Instantiate our workers command line argument parser parser = argparse.ArgumentParser() # Initialize the default requirements for a worker, if needed. parser = StoqArgs(parser) # Define the argparse group for this plugin worker_opts = parser.add_argument_group("Plugin Options") # Define the command line arguments for the worker worker_opts.add_argument("-r", "--rules", dest="rulepath", help="Path to rules file.") # The first command line argument is reserved for the framework. # The work should only parse everything after the first command # line argument. We must always use stoQ's argv object to ensure # the plugin is properly instantied whether it is imported or # used via a command line script options = parser.parse_args(self.stoq.argv[2:]) # If we need to handle command line argument, let's pass them # to super().activate so they can be instantied within the worker super().activate(options=options) # Must return true, otherwise the framework believes something # went wrong return True
def activate(self, stoq): self.stoq = stoq parser = argparse.ArgumentParser() parser = StoqArgs(parser) worker_opts = parser.add_argument_group("Plugin Options") worker_opts.add_argument("-q", "--query", dest='query', default=False, help="String to query for") worker_opts.add_argument("-e", "--endpoint", dest='endpoint', default=False, help="Endpoint to query: search, view, report, or data") worker_opts.add_argument("-i", "--index", dest='index', default=False, help="Index to query: ipv4, websites, or certificates") worker_opts.add_argument("-f", "--field", dest='field', default=False, help="Field(s) for query in dot notation (i.e., location.country_code)") options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options) return True
def activate(self, stoq): self.stoq = stoq parser = argparse.ArgumentParser() parser = StoqArgs(parser) worker_opts = parser.add_argument_group("Plugin Options") worker_opts.add_argument("-r", "--yararules", dest='yararules', help="Path to yara rules file") options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options) # Thread the watchdog observer so we can reload yara rules on the fly. rule_thread = threading.Thread(target=self._monitor_yara_rules, args=()) rule_thread.daemon = True rule_thread.start() # Make sure we compile our rules when activated. self._load_yara_rules() return True
def activate(self, stoq): self.stoq = stoq parser = argparse.ArgumentParser() parser = StoqArgs(parser) options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options)
def activate(self, stoq): self.stoq = stoq # DomainIQ API calls are not consistent, so let's map them out so the # code can be as simple as possible. # https://www.domainiq.com/api_docs self.api_calls = {'domain_report': {'key': 'domain', 'method': 'get', 'type': False}, 'email_report': {'key': 'email', 'method': 'get', 'type': False}, 'registrant_report': {'key': 'name', 'method': 'get', 'type': False}, 'ip_report': {'key': 'ip', 'method': 'get', 'type': False}, 'domain_search': {'key': 'keyword', 'method': 'get', 'type': False}, 'reverse_dns': {'key': 'domain', 'method': 'get', 'type': False}, 'reverse_ip': {'key': 'data', 'method': 'get', 'type': True}, 'reverse_mx': {'key': 'data', 'method': 'get', 'type': True}, 'reverse_analytics': {'key': 'data', 'method': 'get', 'type': 'id'}, } parser = argparse.ArgumentParser() parser = StoqArgs(parser) worker_opts = parser.add_argument_group("Plugin Options") worker_opts.add_argument("-a", "--apikey", dest='apikey', help="DomainIQ API Key") worker_opts.add_argument("-r", "--resource", dest='api_resource', default=False, help="DomainIQ API Resource to interact with") worker_opts.add_argument("-q", "--query", dest='query_value', default=False, help="Value to query using the specified API resource") worker_opts.add_argument("-l", "--list", dest='list_resources', default=False, action='store_true', help="List all DomainIQ API resources available") options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options) if self.list_resources: print("DomainIQ API Resources Available:") for key, value in self.api_calls.items(): print("\t- {}".format(key)) print("\nUsage: stoq-cli.py domainiq -r domain_report -q google.com") print(" OR stoq-cli.py domainiq -r reverse_ip -q 8.8.8.8") exit(0) return True
def activate(self, stoq): self.stoq = stoq parser = argparse.ArgumentParser() parser = StoqArgs(parser) worker_opts = parser.add_argument_group("Plugin Options") worker_opts.add_argument("-e", "--exiftool", dest='exiftool', help="Path to exiftool script") options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options) return True
def activate(self, stoq): self.stoq = stoq parser = argparse.ArgumentParser() parser = StoqArgs(parser) worker_opts = parser.add_argument_group("Plugin Options") worker_opts.add_argument("-o", "--url", dest='url', default=None, help="Metascan API url") options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options) return True
def activate(self, stoq): self.stoq = stoq parser = argparse.ArgumentParser() parser = StoqArgs(parser) worker_opts = parser.add_argument_group("Plugin Options") worker_opts.add_argument("--defs", dest='defs', help="Path to definition database") options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options) return True
def activate(self, stoq): self.stoq = stoq parser = argparse.ArgumentParser() parser = StoqArgs(parser) worker_opts = parser.add_argument_group("Plugin Options") worker_opts.add_argument("-q", "--query", dest='query', default=False, help="Connector to use for queries") options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options) return True
def activate(self, stoq): self.stoq = stoq parser = argparse.ArgumentParser() parser = StoqArgs(parser) worker_opts = parser.add_argument_group("Plugin Options") worker_opts.add_argument("-p", "--password", dest='password', default=False, help="Password for encrypted file") options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options) self.load_reader('iocregex') return True
def activate(self, stoq): self.stoq = stoq parser = argparse.ArgumentParser() parser = StoqArgs(parser) worker_opts = parser.add_argument_group("Plugin Options") worker_opts.add_argument( "--dont-abstract", dest='abstract', default=None, action='store_false', help="Don't output the abstracted information") options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options) return True
def activate(self, stoq): self.stoq = stoq parser = argparse.ArgumentParser() parser = StoqArgs(parser) worker_opts = parser.add_argument_group("Plugin Options") worker_opts.add_argument("-r", "--yararules", dest='yararules', help="Path to yara rules file") options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options) # Make sure we compile our rules when activated. self._load_yara_rules() return True
def activate(self, stoq): self.stoq = stoq parser = argparse.ArgumentParser() parser = StoqArgs(parser) worker_opts = parser.add_argument_group("Plugin Options") worker_opts.add_argument("-l", "--limit", dest='limit', help="Maximum number of results to return") options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options) # We want to save each result individually, so let's save the output # when we scan it rather than having the framework handle it self.load_connector(self.output_connector) return True
def activate(self, stoq): self.stoq = stoq parser = argparse.ArgumentParser() parser = StoqArgs(parser) worker_opts = parser.add_argument_group("Plugin Options") worker_opts.add_argument("-r", "--peidrules", dest="peidrules", help="Path to peid rules file") options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options) self.signatures = peutils.SignatureDatabase(self.peidrules) return True
def activate(self, stoq): self.stoq = stoq parser = argparse.ArgumentParser() parser = StoqArgs(parser) options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options) # Make sure our options are the proper type self.timeout = float(self.timeout) self.port = int(self.port) self.interval = int(self.interval) self._connect() if not self.clamd: return False return True
def activate(self, stoq): self.stoq = stoq parser = argparse.ArgumentParser() parser = StoqArgs(parser) worker_opts = parser.add_argument_group("Plugin Options") worker_opts.add_argument("-x", "--xorsearch", dest='bin', help="Filename to scan") worker_opts.add_argument("--terms", dest='terms', help="Path to the xorsearch terms file") options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options) return True
def activate(self, stoq): self.stoq = stoq parser = argparse.ArgumentParser() parser = StoqArgs(parser) worker_opts = parser.add_argument_group("Plugin Options") worker_opts.add_argument("-a", "--apikey", dest='apikey', help="TotalHash API Key") worker_opts.add_argument("-s", "--sha1", dest='analysis_hash', default=False, help="Retrieve analysis from TotalHash for a SHA1 hash") options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options) return True
def activate(self, stoq): self.stoq = stoq parser = argparse.ArgumentParser() parser = StoqArgs(parser) worker_opts = parser.add_argument_group("Plugin Options") worker_opts.add_argument("-p", "--path", dest='floss_path', help="Path to FLOSS executable") worker_opts.add_argument("-n", "--minimum-length", dest='string_length', help="Minimum length for FLOSS string search") options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options) return True
def activate(self, stoq): self.stoq = stoq parser = argparse.ArgumentParser() parser = StoqArgs(parser) options = parser.parse_args(self.stoq.argv[2:]) options.log_level = 'critical' options.max_recursion = 1 options.max_processes = 0 options.archive_connector = 'test_connector' options.decorator_plugin = 'test_decorator' options.source_plugin = 'test_source' options.outfile = 'stoq_test_outfile' options.ingest_metadata = ['metatest:1', 'metatest:1', 'metatest2:abc'] super().activate(options=options) return True
def activate(self, stoq): self.stoq = stoq parser = argparse.ArgumentParser() parser = StoqArgs(parser) worker_opts = parser.add_argument_group("Plugin Options") worker_opts.add_argument("-q", "--query", dest='query', default=False, help="String to query for") worker_opts.add_argument("-e", "--endpoint", dest='endpoint', default=False, help="Endpoint to query: domain, email, ip, or antivirus") options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options) return True
def activate(self, stoq): self.stoq = stoq parser = argparse.ArgumentParser() parser = StoqArgs(parser) worker_opts = parser.add_argument_group("Plugin Options") worker_opts.add_argument("-r", "--root", dest='root', help="Root path Fireeye shares are located") worker_opts.add_argument("-i", "--images", dest='images_list', action='append', help="Fireeye images that should be used. May" " be used more than once.") options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options) return True
def activate(self, stoq): self.stoq = stoq parser = argparse.ArgumentParser() parser = StoqArgs(parser) worker_opts = parser.add_argument_group("Plugin Options") worker_opts.add_argument("--api_host", dest='api_host', help="ThreatGrid API host") worker_opts.add_argument("--vm", dest='vm', default=None, help="Specified virtual machine to use") worker_opts.add_argument( "--private", dest='private', default='false', help="Specify if sample should be marked private or not") worker_opts.add_argument("--api_key", dest='api_key', help="ThreatGrid API Key") worker_opts.add_argument("--tags", dest='tags', default=[], help="Tags applied to samples") worker_opts.add_argument( "--playbook", dest='playbook', default='none', help="Name of playbook to apply to sample run") worker_opts.add_argument("--network_exit", dest='network_exit', default='phl-ven', help="ThreatGrid network exit to use") options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options) return True
def activate(self, stoq): self.stoq = stoq parser = argparse.ArgumentParser() parser = StoqArgs(parser) worker_opts = parser.add_argument_group("Plugin Options") worker_opts.add_argument("-p", "--password", dest='password', default=False, help="Password for encrypted file") worker_opts.add_argument("-t", "--force-tika", dest='force_tika', action='store_true', help="Force the use of tika for all files") options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options) self.load_reader('iocregex') return True
def activate(self, stoq): self.stoq = stoq parser = argparse.ArgumentParser() parser = StoqArgs(parser) worker_opts = parser.add_argument_group("Plugin Options") worker_opts.add_argument("-p", "--priority", dest='priority', type=int, choices=range(0, 11), help="Priority of message") worker_opts.add_argument("-w", "--worker", dest='submission_list', action='append', help="Worker queues that should process \ sample. May be used multiple times") worker_opts.add_argument("-q", "--publisher", dest='publisher', help="Source plugin to publish samples to") options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options) # Activate the appropriate plugin so we can publish messages, # if needed. self.publish_connector = self.stoq.load_plugin(self.publisher, 'source') if not hasattr(self.publish_connector, 'publish'): self.log.warn("Plugin does not support publishing to queues") return False return True
def activate(self, stoq): self.stoq = stoq parser = argparse.ArgumentParser() parser = StoqArgs(parser) worker_opts = parser.add_argument_group("Plugin Options") worker_opts.add_argument("-a", "--apikey", dest='apikey', help="TotalHash API Key") worker_opts.add_argument( "-s", "--sha1", dest='analysis_hash', default=False, help="Retrieve analysis from TotalHash for a SHA1 hash") options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options) return True
def activate(self, stoq): self.stoq = stoq parser = argparse.ArgumentParser() parser = StoqArgs(parser) worker_opts = parser.add_argument_group("Plugin Options") worker_opts.add_argument("-q", "--query", dest='query', default=False, help="String to query for") worker_opts.add_argument( "-e", "--endpoint", dest='endpoint', default=False, help="Endpoint to query: domain, email, ip, or antivirus") options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options) return True
def activate(self, stoq): self.stoq = stoq # VTMIS API calls are not consistent, so let's map them out so the # code can be as simple as possible. The primary key below will # be appened to the root VTMIS API URI and the underscores ("_") # replaced with a "/". As an example, the key "ip-address_report" # will be translated as: # https://www.virustotal.com/vtapi/v2/ip-address/report/ self.api_calls = { 'file_report': { 'key': 'resource', 'allinfo': True, 'method': 'get', 'private': False }, 'file_behaviour': { 'key': 'hash', 'allinfo': True, 'method': 'get', 'private': True }, 'file_network-traffic': { 'key': 'hash', 'allinfo': False, 'method': 'get', 'private': True }, 'file_feed': { 'key': 'package', 'allinfo': False, 'method': 'get', 'private': True }, 'file_download': { 'key': 'hash', 'allinfo': False, 'method': 'get', 'private': True }, 'file_scan': { 'key': False, 'allinfo': False, 'method': 'post', 'private': False }, 'file_rescan': { 'key': 'resource', 'allinfo': False, 'method': 'post', 'private': False }, 'file_search': { 'key': 'query', 'allinfo': False, 'method': 'get', 'private': True }, 'file_clusters': { 'key': 'date', 'allinfo': False, 'method': 'get', 'private': True }, 'url_report': { 'key': 'resource', 'allinfo': True, 'method': 'get', 'private': False }, 'url_scan': { 'key': 'url', 'allinfo': False, 'method': 'post', 'private': False }, 'url_feed': { 'key': 'package', 'allinfo': False, 'method': 'get', 'private': True }, 'ip-address_report': { 'key': 'ip', 'allinfo': False, 'method': 'get', 'private': False }, 'domain_report': { 'key': 'domain', 'allinfo': False, 'method': 'get', 'private': False }, 'comments_get': { 'key': 'resource', 'allinfo': False, 'method': 'get', 'private': True } } parser = argparse.ArgumentParser() parser = StoqArgs(parser) worker_opts = parser.add_argument_group("Plugin Options") worker_opts.add_argument("-a", "--apikey", dest='apikey', help="VTMIS API Key") worker_opts.add_argument("-r", "--resource", dest='api_resource', default=False, help="VTMIS API Resource to interact with") worker_opts.add_argument( "-q", "--query", dest='query_value', default=False, help="Value to query using the specified API resource") worker_opts.add_argument("-l", "--list", dest='list_resources', default=False, action='store_true', help="List all VTMIS API resources available") worker_opts.add_argument("-s", "--alerts", dest='do_alerts', default=False, action='store_true', help="Check for alerts via the API") worker_opts.add_argument( "-d", "--download", dest='download_samples', default=self.download_samples, action='store_true', help="Download samples from alerts and file feed") worker_opts.add_argument( "--download-path", dest='download_path', default=False, help="Directory to save download samples, if supported") worker_opts.add_argument( "-c", "--feed-connector", dest='feed_connector', help="Connector to utilize to save original JSON feed content") worker_opts.add_argument("-f", "--save-feed", dest='feed_save', default=self.feed_save, action='store_true', help="Save original JSON feed content") worker_opts.add_argument( "-p", "--feed-path", dest='feed_path', help="Directory where the feed content is saved, if supported") worker_opts.add_argument( "-m", "--max-threads", dest='max_threads', help="Max number of threads when processing feeds") options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options) if self.list_resources: print("VTMIS API Resources Available:") for key, value in self.api_calls.items(): print("\t- {}".format(key)) print( "\nUsage: stoq-cli.py vtmis -r file_search -q 7896b9b34bdbedbe7bdc6d446ecb09d5" ) print( " OR stoq-cli.py vtmis -r domain_report -q www.google.com") exit(0) return True
def activate(self, stoq): self.stoq = stoq parser = argparse.ArgumentParser() parser = StoqArgs(parser) subs = parser.add_subparsers(dest='query_resource') pdns = subs.add_parser('pdns', help="Query passive DNS data") pdns.add_argument('--query', '-q', dest='query', default=False, help="Query for a domain, IP address or wildcard") pdns.add_argument('--sources', dest='sources', type=str, default=False, nargs='+', help="CSV string of passive DNS sources") pdns.add_argument('--end', '-e', dest='end_time', default=False, type=valid_date, help="Filter records up to this end date (YYYY-MM-DD)") pdns.add_argument('--start', '-s', dest='start_time', default=False, type=valid_date, help="Filter records from this start date (YYYY-MM-DD)") pdns.add_argument('--timeout', '-t', dest='timeout', default=3, help="Timeout to use for passive DNS source queries") pdns.add_argument('--unique', dest='unique', action="store_true", default=False, help="Use this to only get back unique resolutons") whois = subs.add_parser('whois', help="Query WHOIS data") whois.add_argument('--query', '-q', dest='query', default=False, help="Query for a domain or IP address") whois.add_argument('--field', '-f', dest='field', type=str, default=False, help="Run a specific query against a WHOIS field") whois.add_argument('--compact', dest='compact', action="store_true", default=False, help="Show WHOIS record in a compact way") ssl = subs.add_parser('ssl', help="Query SSL certificate data") ssl.add_argument('--query', '-q', dest='query', default=False, help="Query for an IP address or SHA-1") ssl.add_argument('--field', '-f', dest='field', type=str, default=False, help="Run a specific query against a certificate field") ssl.add_argument('--type', '-t', dest='type', choices=['search', 'history'], default=False, help="Perform a plain search or get history") ssl.add_argument('--compact', dest='compact', action="store_true", default=False, help="Show SSL record in a compact way") attribute = subs.add_parser('attribute', help="Query host attribute data") attribute.add_argument('--query', '-q', dest='query', default=False, help="Query for a domain or IP address") attribute.add_argument('--type', '-t', dest='type', default=False, choices=['tracker', 'component'], help="Query tracker data or component data") action = subs.add_parser('action', help="Query and input feedback") action.add_argument('--query', '-q', dest='query', default=False, help="Domain, IP address, Email, SSL certificate") action.add_argument('--metadata', dest='metadata', action="store_true", default=False, help="Get metadata associated with a query") action.add_argument('--tags', dest='tags', type=str, default=False, help="Tag values to use in conjunction with an action") action.add_argument('--add-tags', dest='add_tags', action="store_true", default=False, help="Add tag values") action.add_argument('--remove-tags', dest='remove_tags', action="store_true", default=False, help="Remove tag values") action.add_argument('--set-tags', dest='set_tags', action="store_true", default=False, help="Set tag values") action.add_argument('--classification', dest='classification', choices=['malicious', 'non-malicious', 'suspicious', 'unknown'], default=False, help="Classification to apply to the query") action.add_argument('--monitor', dest='monitor', choices=['true', 'false'], default=False, help="Read or write a monitor value") action.add_argument('--sinkhole', dest='sinkhole', choices=['true', 'false'], default=False, help="Read or write a sinkhole value") action.add_argument('--dynamic-dns', dest='dynamic_dns', choices=['true', 'false'], default=False, help="Read or write a dynamic DNS value") action.add_argument('--ever-compromised', dest='ever_compromised', choices=['true', 'false'], default=False, help="Read or write a compromised value") options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options) return True
def activate(self, stoq): self.stoq = stoq parser = argparse.ArgumentParser() parser = StoqArgs(parser) worker_opts = parser.add_argument_group("Plugin Options") worker_opts.add_argument( "-c", "--attachment_connector", dest='attachment_connector', help="Connector plugin to save attachments with") options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options) self.publisher = None try: # Initialize the IOC extract regex self.load_reader('iocregex') self.extract_iocs = True except AttributeError: self.log.warn( "iocregex reader plugin is not installed. IOC's will not be extracted" ) self.extract_iocs = False if self.use_bloom: # Initialize bloomfilters self.bloomfilters = {} for bloomfield in self.bloomfield_list: self.bloomfilters[bloomfield] = StoqBloomFilter() bloomfield_path = os.path.join(self.bloom_directory, bloomfield) # Ensure path exists directory = os.path.dirname(bloomfield_path) if not os.path.exists(directory): os.makedirs(directory) # Check whether a given filter exists if os.path.isfile(bloomfield_path): # Filter already exists, import it self.bloomfilters[bloomfield].import_filter( bloomfield_path) else: # Unable to find filter, create brand new one self.bloomfilters[bloomfield].create_filter( bloomfield_path, self.bloom_size, self.bloom_fp_rate) # Set backup schedule at regular intervals self.bloomfilters[bloomfield].backup_scheduler( self.bloom_update) # Load the attachment connector plugin self.load_connector(self.attachment_connector) # If we are using a queue to handle attachments, load the plugin now # otherwise, load the worker plugins we will be using and set the # output connector if self.use_queue: self.publisher = self.stoq.load_plugin('publisher', 'worker') else: for worker in self.workers_list: self.load_worker(worker) self.workers[worker].output_connector = self.output_connector return True
def activate(self, stoq): self.stoq = stoq parser = argparse.ArgumentParser() parser = StoqArgs(parser) worker_opts = parser.add_argument_group("Plugin Options") worker_opts.add_argument("-m", "--method", dest="method", choices=('API', "Fileshare"), help="Method to use to submit files. Can be " "API or Fileshare") worker_opts.add_argument("-i", "--images", dest='images_list', nargs="+", type=list, help="Fireeye images that should be used.") worker_opts.add_argument("-n", "--name", dest="keep_name", type=bool, required=False, help="preserve the original name of the file" " in the stoq framework. If set to false" " stoq will automatically generate a " " UUID for the name") fileshare_opts = worker_opts.add_argument_group( "Fileshare", "Fileshare options") fileshare_opts.add_argument( "-r", "--root", dest='root', required=False, help="Root path Fireeye shares are located") api_opts = worker_opts.add_argument_group("API", "API options") api_opts.add_argument("-a", "--address", dest="address", required=False, help="IP address or name of MAS/AX/CMS" " server.") api_opts.add_argument("-u", "--username", dest="username", required=False, help="Username to use to log into MAS API") api_opts.add_argument("-p", "--password", dest="password", required=False, help="Password to use to log into the " "MAS API") api_opts.add_argument("-s", "--ssl", dest="verify_ssl", required=False, help="Verify SSL. Some MAS units use a " "self-signed certificate, which fails " "SSL validation. Use this option to " "turn off SSL validation.") api_opts.add_argument("-v", "--version", dest="api_version", required=False, choices=("1.1.0", ), help="API Version to use. This is only " " used when using the API. Supports" " version 1.1.0 (default)") options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options) return True
def activate(self, stoq): self.stoq = stoq parser = argparse.ArgumentParser() parser = StoqArgs(parser) worker_opts = parser.add_argument_group("Plugin Options") worker_opts.add_argument("-c", "--attachment_connector", dest='attachment_connector', help="Connector plugin to save attachments") worker_opts.add_argument("-b", "--omit_body", dest='omit_body', default=False, action='store_true', help="Omit body from the results") options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options) # We want to save each result individually, so let's save the output # when we scan it rather than having the framework handle it self.load_connector(self.output_connector) self.publisher = None try: # Initialize the IOC extract regex self.load_reader('iocregex') self.extract_iocs = True except AttributeError: self.log.warn( "iocregex reader plugin is not installed. IOC's will not be extracted" ) self.extract_iocs = False if self.use_bloom and BLOOM_ENABLED: # Initialize bloomfilters self.bloomfilters = {} for bloomfield in self.bloomfield_list: self.bloomfilters[bloomfield] = StoqBloomFilter() bloomfield_path = os.path.join(self.bloom_directory, bloomfield) # Ensure path exists directory = os.path.dirname(bloomfield_path) if not os.path.exists(directory): os.makedirs(directory) # Check whether a given filter exists if os.path.isfile(bloomfield_path): # Filter already exists, import it self.bloomfilters[bloomfield].import_filter( bloomfield_path) else: # Unable to find filter, create brand new one self.bloomfilters[bloomfield].create_filter( bloomfield_path, self.bloom_size, self.bloom_fp_rate) # Set backup schedule at regular intervals self.bloomfilters[bloomfield].backup_scheduler( self.bloom_update) # Load the attachment connector plugin self.load_connector(self.attachment_connector) # If we are using a queue to handle attachments, load the plugin now # otherwise, load the worker plugins we will be using and set the # output connector if self.use_queue: self.publisher = self.stoq.load_plugin('publisher', 'worker') else: # Make sure the list isn't empty before we attempt to load plugins if self.workers_list: for worker in self.workers_list: try: self.load_worker(worker) self.workers[ worker].output_connector = self.output_connector except: self.log.error("Failed to load worker", exc_info=True) # Make sure we remove it from the list, otherwise an # exception will be raised when we go to deactivate # all of the loaded plugins self.workers_list.remove(worker) # No reason to log anything here, the framework will handle that. pass return True
def activate(self, stoq): self.stoq = stoq parser = argparse.ArgumentParser() parser = StoqArgs(parser) worker_opts = parser.add_argument_group("Plugin Options") worker_opts.add_argument("-c", "--attachment_connector", dest='attachment_connector', help="Connector plugin to save attachments with") options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options) self.publisher = None try: # Initialize the IOC extract regex self.load_reader('iocregex') self.extract_iocs = True except AttributeError: self.log.warn("iocregex reader plugin is not installed. IOC's will not be extracted") self.extract_iocs = False if self.use_bloom: # Initialize bloomfilters self.bloomfilters = {} for bloomfield in self.bloomfield_list: self.bloomfilters[bloomfield] = StoqBloomFilter() bloomfield_path = os.path.join(self.bloom_directory, bloomfield) # Ensure path exists directory = os.path.dirname(bloomfield_path) if not os.path.exists(directory): os.makedirs(directory) # Check whether a given filter exists if os.path.isfile(bloomfield_path): # Filter already exists, import it self.bloomfilters[bloomfield].import_filter(bloomfield_path) else: # Unable to find filter, create brand new one self.bloomfilters[bloomfield].create_filter(bloomfield_path, self.bloom_size, self.bloom_fp_rate) # Set backup schedule at regular intervals self.bloomfilters[bloomfield].backup_scheduler(self.bloom_update) # Load the attachment connector plugin self.load_connector(self.attachment_connector) # If we are using a queue to handle attachments, load the plugin now # otherwise, load the worker plugins we will be using and set the # output connector if self.use_queue: self.publisher = self.stoq.load_plugin('publisher', 'worker') else: for worker in self.workers_list: self.load_worker(worker) self.workers[worker].output_connector = self.output_connector return True
def activate(self, stoq): self.stoq = stoq parser = argparse.ArgumentParser() parser = StoqArgs(parser) worker_opts = parser.add_argument_group("Plugin Options") worker_opts.add_argument("-m", "--method", dest="method", choices=('API', "Fileshare"), help="Method to use to submit files. Can be " "API or Fileshare") worker_opts.add_argument("-i", "--images", dest='images_list', nargs="+", type=list, help="Fireeye images that should be used.") worker_opts.add_argument("-n", "--name", dest="keep_name", type=bool, required=False, help="preserve the original name of the file" " in the stoq framework. If set to false" " stoq will automatically generate a " " UUID for the name") fileshare_opts = worker_opts.add_argument_group("Fileshare", "Fileshare options") fileshare_opts.add_argument("-r", "--root", dest='root', required=False, help="Root path Fireeye shares are located") api_opts = worker_opts.add_argument_group("API", "API options") api_opts.add_argument("-a", "--address", dest="address", required=False, help="IP address or name of MAS/AX/CMS" " server.") api_opts.add_argument("-u", "--username", dest="username", required=False, help="Username to use to log into MAS API") api_opts.add_argument("-p", "--password", dest="password", required=False, help="Password to use to log into the " "MAS API") api_opts.add_argument("-s", "--ssl", dest="verify_ssl", required=False, help="Verify SSL. Some MAS units use a " "self-signed certificate, which fails " "SSL validation. Use this option to " "turn off SSL validation.") api_opts.add_argument("-v", "--version", dest="api_version", required=False, choices=("1.1.0",), help="API Version to use. This is only " " used when using the API. Supports" " version 1.1.0 (default)") options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options) return True
def activate(self, stoq): self.stoq = stoq # DomainIQ API calls are not consistent, so let's map them out so the # code can be as simple as possible. # https://www.domainiq.com/api_docs self.api_calls = { 'domain_report': { 'key': 'domain', 'method': 'get', 'type': False }, 'email_report': { 'key': 'email', 'method': 'get', 'type': False }, 'registrant_report': { 'key': 'name', 'method': 'get', 'type': False }, 'ip_report': { 'key': 'ip', 'method': 'get', 'type': False }, 'domain_search': { 'key': 'keyword', 'method': 'get', 'type': False }, 'reverse_dns': { 'key': 'domain', 'method': 'get', 'type': False }, 'reverse_ip': { 'key': 'data', 'method': 'get', 'type': True }, 'reverse_mx': { 'key': 'data', 'method': 'get', 'type': True }, 'reverse_analytics': { 'key': 'data', 'method': 'get', 'type': 'id' }, } parser = argparse.ArgumentParser() parser = StoqArgs(parser) worker_opts = parser.add_argument_group("Plugin Options") worker_opts.add_argument("-a", "--apikey", dest='apikey', help="DomainIQ API Key") worker_opts.add_argument("-r", "--resource", dest='api_resource', default=False, help="DomainIQ API Resource to interact with") worker_opts.add_argument( "-q", "--query", dest='query_value', default=False, help="Value to query using the specified API resource") worker_opts.add_argument( "-l", "--list", dest='list_resources', default=False, action='store_true', help="List all DomainIQ API resources available") options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options) if self.list_resources: print("DomainIQ API Resources Available:") for key, value in self.api_calls.items(): print("\t- {}".format(key)) print( "\nUsage: stoq-cli.py domainiq -r domain_report -q google.com") print(" OR stoq-cli.py domainiq -r reverse_ip -q 8.8.8.8") exit(0) return True
def activate(self, stoq): self.stoq = stoq # VTMIS API calls are not consistent, so let's map them out so the # code can be as simple as possible. The primary key below will # be appened to the root VTMIS API URI and the underscores ("_") # replaced with a "/". As an example, the key "ip-address_report" # will be translated as: # https://www.virustotal.com/vtapi/v2/ip-address/report/ self.api_calls = {'file_report': {'key': 'resource', 'allinfo': True, 'method': 'get', 'private': False}, 'file_behaviour': {'key': 'hash', 'allinfo': True, 'method': 'get', 'private': True}, 'file_network-traffic': {'key': 'hash', 'allinfo': False, 'method': 'get', 'private': True}, 'file_feed': {'key': 'package', 'allinfo': False, 'method': 'get', 'private': True}, 'file_download': {'key': 'hash', 'allinfo': False, 'method': 'get', 'private': True}, 'file_scan': {'key': False, 'allinfo': False, 'method': 'post', 'private': False}, 'file_rescan': {'key': 'resource', 'allinfo': False, 'method': 'post', 'private': False}, 'file_search': {'key': 'query', 'allinfo': False, 'method': 'get', 'private': True}, 'file_clusters': {'key': 'date', 'allinfo': False, 'method': 'get', 'private': True}, 'url_report': {'key': 'resource', 'allinfo': True, 'method': 'get', 'private': False}, 'url_scan': {'key': 'url', 'allinfo': False, 'method': 'post', 'private': False}, 'url_feed': {'key': 'package', 'allinfo': False, 'method': 'get', 'private': True}, 'ip-address_report': {'key': 'ip', 'allinfo': False, 'method': 'get', 'private': False}, 'domain_report': {'key': 'domain', 'allinfo': False, 'method': 'get', 'private': False}, 'comments_get': {'key': 'resource', 'allinfo': False, 'method': 'get', 'private': True} } parser = argparse.ArgumentParser() parser = StoqArgs(parser) worker_opts = parser.add_argument_group("Plugin Options") worker_opts.add_argument("-a", "--apikey", dest='apikey', help="VTMIS API Key") worker_opts.add_argument("-r", "--resource", dest='api_resource', default=False, help="VTMIS API Resource to interact with") worker_opts.add_argument("-q", "--query", dest='query_value', default=False, help="Value to query using the specified API resource") worker_opts.add_argument("-l", "--list", dest='list_resources', default=False, action='store_true', help="List all VTMIS API resources available") worker_opts.add_argument("-s", "--alerts", dest='do_alerts', default=False, action='store_true', help="Check for alerts via the API") worker_opts.add_argument("-d", "--download", dest='download_samples', default=self.download_samples, action='store_true', help="Download samples from alerts and file feed") worker_opts.add_argument("--download-path", dest='download_path', default=False, help="Directory to save download samples, if supported") worker_opts.add_argument("-c", "--feed-connector", dest='feed_connector', help="Connector to utilize to save original JSON feed content") worker_opts.add_argument("-f", "--save-feed", dest='feed_save', default=self.feed_save, action='store_true', help="Save original JSON feed content") worker_opts.add_argument("-p", "--feed-path", dest='feed_path', help="Directory where the feed content is saved, if supported") worker_opts.add_argument("-m", "--max-threads", dest='max_threads', help="Max number of threads when processing feeds") options = parser.parse_args(self.stoq.argv[2:]) super().activate(options=options) if self.list_resources: print("VTMIS API Resources Available:") for key, value in self.api_calls.items(): print("\t- {}".format(key)) print("\nUsage: stoq-cli.py vtmis -r file_search -q 7896b9b34bdbedbe7bdc6d446ecb09d5") print(" OR stoq-cli.py vtmis -r domain_report -q www.google.com") exit(0) return True