def _validate(cls, config): hostname = config.get("hostname", "").strip() keyfile = config.get("keyfile", "").strip() certfile = config.get("certfile", "").strip() data_feed = config.get("data_feed", "").strip() certfiles = config.get("certfiles", "") if not hostname: raise ServiceConfigError("You must specify a TAXII Server.") if not keyfile: raise ServiceConfigError("You must specify a keyfile location.") if not os.path.isfile(keyfile): raise ServiceConfigError("keyfile does not exist.") if not certfile: raise ServiceConfigError("You must specify a certfile location.") if not os.path.isfile(certfile): raise ServiceConfigError("certfile does not exist.") if not data_feed: raise ServiceConfigError("You must specify a TAXII Data Feed.") if not certfiles: raise ServiceConfigError("You must specify at least one certfile.") for crtfile in certfiles: try: (source, feed, filepath) = crtfile.split(',') except ValueError: raise ServiceConfigError( ("You must specify a source, feed name" ", and certificate path for each source.")) source.strip() feed.strip() filepath.strip() if not does_source_exist(source): raise ServiceConfigError("Invalid source: %s" % source) if not os.path.isfile(filepath): raise ServiceConfigError("certfile does not exist: %s" % filepath)
def _compile_rules(sigdir, sigfiles): if not sigfiles or not sigdir: raise ServiceConfigError("No signature files specified.") sigsets = [] for sigfile in sigfiles: sigfile = os.path.abspath(os.path.join(sigdir, sigfile.strip())) logger.debug("Full path to file file: %s" % sigfile) filename = os.path.basename(sigfile) dirname = os.path.dirname(sigfile) old = os.getcwd() try: with open(sigfile, "rt") as f: data = f.read() os.chdir(dirname) except Exception as e: logger.exception("File cannot be opened: %s" % sigfile) raise ServiceConfigError(str(e)) try: rules = yara.compile(source=data) except yara.SyntaxError as e: message = "Yara rules file: %s: %s" % (sigfile, str(e)) logger.exception(message) os.chdir(old) raise ServiceConfigError(message) sigsets.append({'name': filename, 'rules': rules}) os.chdir(old) logger.debug(str(sigsets)) return sigsets
def _compile_rules(sigdir, sigfiles): if not sigfiles: raise ServiceConfigError("No signature files specified.") logger.debug("Sigdir: %s" % sigdir) sigsets = [] for sigfile in sigfiles: sigfile = sigfile.strip() logger.debug("Sigfile: %s" % sigfile) if sigdir: abspath = os.path.abspath(os.path.join(sigdir, sigfile)) else: abspath = sigfile logger.debug("Full path to file file: %s" % abspath) filename = os.path.basename(abspath) version = sigfile.split('.')[0] try: with open(abspath, "rt") as f: data = f.read() except: logger.exception("File cannot be opened: %s" % abspath) raise sig_md5 = md5(data).hexdigest() try: rules = yara.compile(source=data) except yara.SyntaxError: message = "Not a valid yara rules file: %s" % abspath logger.exception(message) raise ServiceConfigError(message) sigsets.append({'name': filename, 'md5': sig_md5, 'rules': rules, 'version': version}) logger.debug(str(sigsets)) return sigsets
def valid_for(obj): valid_types = ('Domain', 'File Name', 'IPv4 Address', 'URI') if isinstance(obj, Indicator) and obj.ind_type not in valid_types: raise ServiceConfigError("Invalid Indicator Type: %s" % obj.ind_type) if isinstance(obj, Sample) and obj.filedata.grid_id is None: raise ServiceConfigError("Invalid Sample Data: Missing File Data")
def parse_config(config): api_key = config.get("api_key", "") if not api_key: raise ServiceConfigError("You must specify a valid API key for OTX.") otx_server = config.get("otx_server", "") if not otx_server: raise ServiceConfigError("You must specify a valid URL for OTX serveur.")
def validate_runtime(config, db_config): # To run, this service _MUST_ have sigfiles and if distribution_url # is set it must have api_key. if 'sigfiles' not in config: raise ServiceConfigError("Need sigfiles to run.") if db_config['distribution_url'] and 'api_key' not in config: raise ServiceConfigError("Need API key to run.")
def parse_config(config): sfa_api = config.get("sfa_api", "") if not sfa_api: raise ServiceConfigError("You must specify a URI for SFA API.") key_api = config.get("key_api", "") if not key_api: raise ServiceConfigError( "You must specify a valid key api for SFA.") tlp_value = config.get("tlp_value", "")
def parse_config(config): clamd_sock_path = config.get('clamd_sock_path', '') clamd_host_name = config.get('clamd_host_name', '') # Must have one of socket or host. if not clamd_sock_path and not clamd_host_name: raise ServiceConfigError("Socket path or hostname required.") # If socket is provided check it exists. if not os.path.exists(clamd_sock_path): raise ServiceConfigError('Socket path not found.')
def valid_for(obj): if obj.filedata.grid_id == None: raise ServiceConfigError("Missing filedata.") data = obj.filedata.read() if len(data) < 4: raise ServiceConfigError("Need at least 4 bytes.") # Reset the read pointer. obj.filedata.seek(0) if not data[0:4] == b'\x90\x12\x00\x00': raise ServiceConfigError("Not a SEP Local Quarantine file")
def valid_for(obj): if obj.filedata.grid_id == None: raise ServiceConfigError("Missing filedata.") data = obj.filedata.read(4) if len(data) < 4: raise ServiceConfigError("Need at least 4 bytes.") # Reset the read pointer. obj.filedata.seek(0) 'We only care about the compressed flash files' if not data[:3] in ['CWS','ZWS']: raise ServiceConfigError("Not a valid compressed Flash file.")
def parse_config(config): if not config['bit9_api_key']: raise ServiceConfigError("API key required.") if not config['bit9_server']: raise ServiceConfigError("Bit9 Server required.") else: if 'https://' not in config.get( 'bit9_server', '') and 'http://' not in config.get( 'bit9_server', ''): raise ServiceConfigError( "Bit9 Server required, include the http:// or https://")
def parse_config(config): # Make sure basedir exists. basedir = config.get('basedir', '') if basedir: shop_path = "%s/shop" % basedir if not os.path.exists(basedir): raise ServiceConfigError("Base directory does not exist.") elif not os.path.exists(shop_path): raise ServiceConfigError("'shop' does not exist in base.") else: raise ServiceConfigError("Base directory must be defined.")
def validate_runtime(config, db_config): # To run, this service _MUST_ have sigfiles. If no sigfiles are # specified at runtime (config) use the ones in db_config. If none # exist there, raise an error. # # If distribution_url is set it must have api_key. if 'sigfiles' not in config: config['sigfiles'] = db_config.get('sigfiles', []) if not config['sigfiles']: raise ServiceConfigError("Need sigfiles to run.") if db_config['distribution_url'] and 'api_key' not in config: raise ServiceConfigError("Need API key to run.")
def parse_config(config): upx_path = config.get("upx_path", "") if not upx_path: raise ServiceConfigError("Must specify UPX path.") if not os.path.isfile(upx_path): raise ServiceConfigError("UPX path does not exist.") if not os.access(upx_path, os.X_OK): raise ServiceConfigError("UPX path is not executable.") if not 'upx' in upx_path.lower(): raise ServiceConfigError("Executable does not appear to be UPX.")
def valid_for(obj): # Only run on zip files if obj.filedata.grid_id == None: raise ServiceConfigError("Missing filedata.") data = obj.filedata.read() # Reset the read pointer. obj.filedata.seek(0) if len(data) < 4: raise ServiceConfigError("Not enough filedata.") if data[:4] not in [ZipParser.zipLDMagic, ZipParser.zipCDMagic]: raise ServiceConfigError("Not a zip file.")
def parse_config(config): pdf2txt_path = config.get("pdf2txt_path", "") if not pdf2txt_path: raise ServiceConfigError( "You must specify a valid path for pdftotext.") antiword_path = config.get("antiword_path", "") if not antiword_path: raise ServiceConfigError( "You must specify a valid path for antiword.") if not os.path.isfile(pdf2txt_path): raise ServiceConfigError("pdftotext path does not exist.") if not os.access(pdf2txt_path, os.X_OK): raise ServiceConfigError("pdftotext is not executable.") if not 'pdftotext' in pdf2txt_path.lower(): raise ServiceConfigError( "Executable does not appear to be pdftotext.") if not os.path.isfile(antiword_path): raise ServiceConfigError("antiword path does not exist.") if not os.access(antiword_path, os.X_OK): raise ServiceConfigError("antiword is not executable.") if not 'antiword' in antiword_path.lower(): raise ServiceConfigError( "Executable does not appear to be antiword.")
def parse_config(config): exiftool_path = config.get("exiftool_path", "") if not exiftool_path: raise ServiceConfigError("Must specify exiftool path.") if not os.path.isfile(exiftool_path): raise ServiceConfigError("exiftool path does not exist.") if not os.access(exiftool_path, os.X_OK): raise ServiceConfigError("exiftool path is not executable.") if not 'exiftool' in exiftool_path.lower(): raise ServiceConfigError( "Executable does not appear to be exiftool.")
def valid_for(obj): # Check if already running in case of triage re-run rezs = AnalysisResult.objects(object_id=str(obj.id), status='started', service_name='VirusTotal_Download') if rezs: raise ServiceConfigError("Service is already running")
def parse_server_config(config): name = config.get("servername", "").strip() hostname = config.get("hostname", "").strip() ppath = config.get("ppath", "").strip() ipath = config.get("ipath", "").strip() keyfile = config.get("keyfile", "").strip() lcert = config.get("lcert", "").strip() errors = [] if not name: errors.append("You must specify a name for the TAXII Server.") if not re.match("^[\w ]+$", name): errors.append("Server name can only contain letters, " "numbers, and spaces.") if not hostname: errors.append("You must specify a TAXII Server hostname.") if not ppath: errors.append("You must specify a TAXII Server Poll Path.") if not ipath: errors.append("You must specify a TAXII Server Inbox Path.") if keyfile and not lcert: errors.append( "If you provide a keyfile, you must also provide a certificate." ) if not keyfile and lcert: errors.append( "If you provide a certificate, you must also provide a keyfile." ) if keyfile and not os.path.isfile(keyfile): errors.append("Keyfile does not exist at given location.") if lcert and not os.path.isfile(lcert): errors.append("Local cert file does not exist " "at given location.") if errors: raise ServiceConfigError("<br>".join(errors))
def parse_config(config): # When editing a config we are given a string. # When validating an existing config it will be a list. # Convert it to a list of strings. if not config['misp_url']: raise ServiceConfigError("MISP URL Required.") if not config['misp_key']: raise ServiceConfigError("MISP API Key Required.") default_tags = config.get('default_tags', []) if default_tags: if isinstance(default_tags, basestring): config['default_tags'] = [ default_tag.strip() for default_tag in default_tags.split(',') ] return config['default_tags']
def parse_feed_config(config): srv_name = config.get("srv_name", "").strip() feedname = config.get("feedname", "").strip() source = config.get("source", "").strip() subID = config.get("subID", "").strip() fcert = config.get("fcert", "").strip() fkey = config.get("fkey", "").strip() errors = [] if not srv_name: errors.append("No server name to which to relate this feed") if not re.match("^[\w ]+$", srv_name): errors.append("Provided server name is invalid") if not feedname: errors.append("You must specify a Feed Name") if not source: errors.append("You must specify a CRITs source") else: if not does_source_exist(source): errors.append("Provided CRITs source is invalid") if fcert and not os.path.isfile(fcert): errors.append( "Encryption Certificate does not exist at given location") if fkey and not os.path.isfile(fkey): errors.append("Decryption Key does not exist at given location") if errors: raise ServiceConfigError("<br>".join(errors))
def validate_runtime(config, db_config): errors = [] if "ssl" in config and "inet" not in config: errors.append("HTTPS inspection requires internet access.") if errors: raise ServiceConfigError(errors)
def valid_for(obj): if obj.filedata.grid_id == None: raise ServiceConfigError("Missing filedata.") data = obj.filedata.read() if len(data) < 4: raise ServiceConfigError("Need at least 4 bytes.") # Reset the read pointer. obj.filedata.seek(0) if not struct.unpack('@I', data[:4])[0] in [ MachOEntity.FAT_MAGIC, MachOEntity.FAT_CIGAM, MachOEntity.MH_MAGIC, MachOEntity.MH_CIGAM, MachOEntity.MH_MAGIC_64, MachOEntity.MH_CIGAM_64 ]: raise ServiceConfigError("Bad magic.")
def valid_for(obj): chm_magic = '\x49\x54\x53\x46\x03\x00\x00\x00\x60\x00\x00\x00' if obj.filedata != None: data = obj.filedata.read() # Need to reset the read pointer. obj.filedata.seek(0) if data.startswith(chm_magic): return raise ServiceConfigError("Not a valid ITSF (CHM) file.")
def valid_for(obj): office_magic = "\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1" if obj.filedata != None: data = obj.filedata.read() # Need to reset the read pointer. obj.filedata.seek(0) if data.startswith(office_magic): return raise ServiceConfigError("Not a valid office document.")
def valid_for(obj): rtf_magic = "{\\rt" if obj.filedata != None: data = obj.filedata.read() # Need to reset the read pointer. obj.filedata.seek(0) if data.startswith(rtf_magic): return raise ServiceConfigError("Not a valid rtf document.")
def _scan(self, context): logger.debug("Setting up shop...") shop_path = "%s/shop" % self.base_dir if not os.path.exists(self.base_dir): raise ServiceConfigError("ChopShop path does not exist") elif not os.path.exists(shop_path): raise ServiceConfigError("ChopShop shop path does not exist") else: sys.path.append(shop_path) from ChopLib import ChopLib from ChopUi import ChopUi logger.debug("Scanning...") choplib = ChopLib() chopui = ChopUi() choplib.base_dir = self.base_dir # XXX: Convert from unicode to str... choplib.modules = str(self.modules) chopui.jsonout = jsonhandler choplib.jsonout = True # ChopShop (because of pynids) needs to read a file off disk. # The services framework forces you to use 'with' here. It's not # possible to just get a path to a file on disk. with self._write_to_file() as pcap_file: choplib.filename = pcap_file chopui.bind(choplib) chopui.start() chopui.jsonclass.set_service(self) choplib.start() while chopui.is_alive(): time.sleep(.1) chopui.join() choplib.finish() choplib.join()
def valid_for(obj): if not obj.filedata: return False data = obj.filedata.read(8) obj.filedata.seek(0) if obj.is_pdf(): return True elif data.startswith("\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"): # M$ Word document return True else: raise ServiceConfigError("Not a valid PDF or a Word document.")
def parse_config(config): # When editing a config we are given a string. # When validating an existing config it will be a list. # Convert it to a list of strings. sigfiles = config.get('sigfiles', []) if not sigfiles: raise ServiceConfigError("Must specify signature files.") if isinstance(sigfiles, basestring): config['sigfiles'] = [sigfile for sigfile in sigfiles.split('\r\n')] # This will raise ServiceConfigError YaraService._compile_rules(config['sigdir'], config['sigfiles'])
def parse_service_config(config): namespace = config.get("namespace", "").strip() ns_prefix = config.get("ns_prefix", "").strip() max_rels = config.get("max_rels", "") errors = [] if not namespace: errors.append("You must specify a XML Namespace.") if not ns_prefix: errors.append("You must specify a XML Namespace Prefix.") if not max_rels or max_rels > 5000 or max_rels < 0: errors.append("Maximum Related must be in the range 0-5000.") if errors: raise ServiceConfigError("<br>".join(errors))