def _validate(cls, config):
hostname = config.get("hostname", "").strip()
keyfile = config.get("keyfile", "").strip()
certfile = config.get("certfile", "").strip()
data_feed = config.get("data_feed", "").strip()
certfiles = config.get("certfiles", "")
if not hostname:
raise ServiceConfigError("You must specify a TAXII Server.")
if not keyfile:
raise ServiceConfigError("You must specify a keyfile location.")
if not os.path.isfile(keyfile):
raise ServiceConfigError("keyfile does not exist.")
if not certfile:
raise ServiceConfigError("You must specify a certfile location.")
if not os.path.isfile(certfile):
raise ServiceConfigError("certfile does not exist.")
if not data_feed:
raise ServiceConfigError("You must specify a TAXII Data Feed.")
if not certfiles:
raise ServiceConfigError("You must specify at least one certfile.")
for crtfile in certfiles:
try:
(source, feed, filepath) = crtfile.split(',')
except ValueError:
raise ServiceConfigError(
("You must specify a source, feed name"
", and certificate path for each source."))
source.strip()
feed.strip()
filepath.strip()
if not does_source_exist(source):
raise ServiceConfigError("Invalid source: %s" % source)
if not os.path.isfile(filepath):
raise ServiceConfigError("certfile does not exist: %s" %
filepath)
def _compile_rules(sigdir, sigfiles):
if not sigfiles or not sigdir:
raise ServiceConfigError("No signature files specified.")
sigsets = []
for sigfile in sigfiles:
sigfile = os.path.abspath(os.path.join(sigdir, sigfile.strip()))
logger.debug("Full path to file file: %s" % sigfile)
filename = os.path.basename(sigfile)
dirname = os.path.dirname(sigfile)
old = os.getcwd()
try:
with open(sigfile, "rt") as f:
data = f.read()
os.chdir(dirname)
except Exception as e:
logger.exception("File cannot be opened: %s" % sigfile)
raise ServiceConfigError(str(e))
try:
rules = yara.compile(source=data)
except yara.SyntaxError as e:
message = "Yara rules file: %s: %s" % (sigfile, str(e))
logger.exception(message)
os.chdir(old)
raise ServiceConfigError(message)
sigsets.append({'name': filename, 'rules': rules})
os.chdir(old)
logger.debug(str(sigsets))
return sigsets
def _compile_rules(sigdir, sigfiles):
if not sigfiles:
raise ServiceConfigError("No signature files specified.")
logger.debug("Sigdir: %s" % sigdir)
sigsets = []
for sigfile in sigfiles:
sigfile = sigfile.strip()
logger.debug("Sigfile: %s" % sigfile)
if sigdir:
abspath = os.path.abspath(os.path.join(sigdir, sigfile))
else:
abspath = sigfile
logger.debug("Full path to file file: %s" % abspath)
filename = os.path.basename(abspath)
version = sigfile.split('.')[0]
try:
with open(abspath, "rt") as f:
data = f.read()
except:
logger.exception("File cannot be opened: %s" % abspath)
raise
sig_md5 = md5(data).hexdigest()
try:
rules = yara.compile(source=data)
except yara.SyntaxError:
message = "Not a valid yara rules file: %s" % abspath
logger.exception(message)
raise ServiceConfigError(message)
sigsets.append({'name': filename,
'md5': sig_md5,
'rules': rules,
'version': version})
logger.debug(str(sigsets))
return sigsets
def valid_for(obj):
valid_types = ('Domain', 'File Name', 'IPv4 Address', 'URI')
if isinstance(obj, Indicator) and obj.ind_type not in valid_types:
raise ServiceConfigError("Invalid Indicator Type: %s" %
obj.ind_type)
if isinstance(obj, Sample) and obj.filedata.grid_id is None:
raise ServiceConfigError("Invalid Sample Data: Missing File Data")
def parse_config(config):
api_key = config.get("api_key", "")
if not api_key:
raise ServiceConfigError("You must specify a valid API key for OTX.")
otx_server = config.get("otx_server", "")
if not otx_server:
raise ServiceConfigError("You must specify a valid URL for OTX serveur.")
def validate_runtime(config, db_config):
# To run, this service _MUST_ have sigfiles and if distribution_url
# is set it must have api_key.
if 'sigfiles' not in config:
raise ServiceConfigError("Need sigfiles to run.")
if db_config['distribution_url'] and 'api_key' not in config:
raise ServiceConfigError("Need API key to run.")
def parse_config(config):
sfa_api = config.get("sfa_api", "")
if not sfa_api:
raise ServiceConfigError("You must specify a URI for SFA API.")
key_api = config.get("key_api", "")
if not key_api:
raise ServiceConfigError(
"You must specify a valid key api for SFA.")
tlp_value = config.get("tlp_value", "")
def parse_config(config):
clamd_sock_path = config.get('clamd_sock_path', '')
clamd_host_name = config.get('clamd_host_name', '')
# Must have one of socket or host.
if not clamd_sock_path and not clamd_host_name:
raise ServiceConfigError("Socket path or hostname required.")
# If socket is provided check it exists.
if not os.path.exists(clamd_sock_path):
raise ServiceConfigError('Socket path not found.')
def valid_for(obj):
if obj.filedata.grid_id == None:
raise ServiceConfigError("Missing filedata.")
data = obj.filedata.read()
if len(data) < 4:
raise ServiceConfigError("Need at least 4 bytes.")
# Reset the read pointer.
obj.filedata.seek(0)
if not data[0:4] == b'\x90\x12\x00\x00':
raise ServiceConfigError("Not a SEP Local Quarantine file")
def valid_for(obj):
if obj.filedata.grid_id == None:
raise ServiceConfigError("Missing filedata.")
data = obj.filedata.read(4)
if len(data) < 4:
raise ServiceConfigError("Need at least 4 bytes.")
# Reset the read pointer.
obj.filedata.seek(0)
'We only care about the compressed flash files'
if not data[:3] in ['CWS','ZWS']:
raise ServiceConfigError("Not a valid compressed Flash file.")
def parse_config(config):
if not config['bit9_api_key']:
raise ServiceConfigError("API key required.")
if not config['bit9_server']:
raise ServiceConfigError("Bit9 Server required.")
else:
if 'https://' not in config.get(
'bit9_server', '') and 'http://' not in config.get(
'bit9_server', ''):
raise ServiceConfigError(
"Bit9 Server required, include the http:// or https://")
def parse_config(config):
# Make sure basedir exists.
basedir = config.get('basedir', '')
if basedir:
shop_path = "%s/shop" % basedir
if not os.path.exists(basedir):
raise ServiceConfigError("Base directory does not exist.")
elif not os.path.exists(shop_path):
raise ServiceConfigError("'shop' does not exist in base.")
else:
raise ServiceConfigError("Base directory must be defined.")
def validate_runtime(config, db_config):
# To run, this service _MUST_ have sigfiles. If no sigfiles are
# specified at runtime (config) use the ones in db_config. If none
# exist there, raise an error.
#
# If distribution_url is set it must have api_key.
if 'sigfiles' not in config:
config['sigfiles'] = db_config.get('sigfiles', [])
if not config['sigfiles']:
raise ServiceConfigError("Need sigfiles to run.")
if db_config['distribution_url'] and 'api_key' not in config:
raise ServiceConfigError("Need API key to run.")
def parse_config(config):
upx_path = config.get("upx_path", "")
if not upx_path:
raise ServiceConfigError("Must specify UPX path.")
if not os.path.isfile(upx_path):
raise ServiceConfigError("UPX path does not exist.")
if not os.access(upx_path, os.X_OK):
raise ServiceConfigError("UPX path is not executable.")
if not 'upx' in upx_path.lower():
raise ServiceConfigError("Executable does not appear to be UPX.")
def valid_for(obj):
# Only run on zip files
if obj.filedata.grid_id == None:
raise ServiceConfigError("Missing filedata.")
data = obj.filedata.read()
# Reset the read pointer.
obj.filedata.seek(0)
if len(data) < 4:
raise ServiceConfigError("Not enough filedata.")
if data[:4] not in [ZipParser.zipLDMagic, ZipParser.zipCDMagic]:
raise ServiceConfigError("Not a zip file.")
def parse_config(config):
pdf2txt_path = config.get("pdf2txt_path", "")
if not pdf2txt_path:
raise ServiceConfigError(
"You must specify a valid path for pdftotext.")
antiword_path = config.get("antiword_path", "")
if not antiword_path:
raise ServiceConfigError(
"You must specify a valid path for antiword.")
if not os.path.isfile(pdf2txt_path):
raise ServiceConfigError("pdftotext path does not exist.")
if not os.access(pdf2txt_path, os.X_OK):
raise ServiceConfigError("pdftotext is not executable.")
if not 'pdftotext' in pdf2txt_path.lower():
raise ServiceConfigError(
"Executable does not appear to be pdftotext.")
if not os.path.isfile(antiword_path):
raise ServiceConfigError("antiword path does not exist.")
if not os.access(antiword_path, os.X_OK):
raise ServiceConfigError("antiword is not executable.")
if not 'antiword' in antiword_path.lower():
raise ServiceConfigError(
"Executable does not appear to be antiword.")
def parse_config(config):
exiftool_path = config.get("exiftool_path", "")
if not exiftool_path:
raise ServiceConfigError("Must specify exiftool path.")
if not os.path.isfile(exiftool_path):
raise ServiceConfigError("exiftool path does not exist.")
if not os.access(exiftool_path, os.X_OK):
raise ServiceConfigError("exiftool path is not executable.")
if not 'exiftool' in exiftool_path.lower():
raise ServiceConfigError(
"Executable does not appear to be exiftool.")
def valid_for(obj):
# Check if already running in case of triage re-run
rezs = AnalysisResult.objects(object_id=str(obj.id),
status='started',
service_name='VirusTotal_Download')
if rezs:
raise ServiceConfigError("Service is already running")
def parse_server_config(config):
name = config.get("servername", "").strip()
hostname = config.get("hostname", "").strip()
ppath = config.get("ppath", "").strip()
ipath = config.get("ipath", "").strip()
keyfile = config.get("keyfile", "").strip()
lcert = config.get("lcert", "").strip()
errors = []
if not name:
errors.append("You must specify a name for the TAXII Server.")
if not re.match("^[\w ]+$", name):
errors.append("Server name can only contain letters, "
"numbers, and spaces.")
if not hostname:
errors.append("You must specify a TAXII Server hostname.")
if not ppath:
errors.append("You must specify a TAXII Server Poll Path.")
if not ipath:
errors.append("You must specify a TAXII Server Inbox Path.")
if keyfile and not lcert:
errors.append(
"If you provide a keyfile, you must also provide a certificate."
)
if not keyfile and lcert:
errors.append(
"If you provide a certificate, you must also provide a keyfile."
)
if keyfile and not os.path.isfile(keyfile):
errors.append("Keyfile does not exist at given location.")
if lcert and not os.path.isfile(lcert):
errors.append("Local cert file does not exist "
"at given location.")
if errors:
raise ServiceConfigError("<br>".join(errors))
def parse_config(config):
# When editing a config we are given a string.
# When validating an existing config it will be a list.
# Convert it to a list of strings.
if not config['misp_url']:
raise ServiceConfigError("MISP URL Required.")
if not config['misp_key']:
raise ServiceConfigError("MISP API Key Required.")
default_tags = config.get('default_tags', [])
if default_tags:
if isinstance(default_tags, basestring):
config['default_tags'] = [
default_tag.strip()
for default_tag in default_tags.split(',')
]
return config['default_tags']
def parse_feed_config(config):
srv_name = config.get("srv_name", "").strip()
feedname = config.get("feedname", "").strip()
source = config.get("source", "").strip()
subID = config.get("subID", "").strip()
fcert = config.get("fcert", "").strip()
fkey = config.get("fkey", "").strip()
errors = []
if not srv_name:
errors.append("No server name to which to relate this feed")
if not re.match("^[\w ]+$", srv_name):
errors.append("Provided server name is invalid")
if not feedname:
errors.append("You must specify a Feed Name")
if not source:
errors.append("You must specify a CRITs source")
else:
if not does_source_exist(source):
errors.append("Provided CRITs source is invalid")
if fcert and not os.path.isfile(fcert):
errors.append(
"Encryption Certificate does not exist at given location")
if fkey and not os.path.isfile(fkey):
errors.append("Decryption Key does not exist at given location")
if errors:
raise ServiceConfigError("<br>".join(errors))
def validate_runtime(config, db_config):
errors = []
if "ssl" in config and "inet" not in config:
errors.append("HTTPS inspection requires internet access.")
if errors:
raise ServiceConfigError(errors)
def valid_for(obj):
if obj.filedata.grid_id == None:
raise ServiceConfigError("Missing filedata.")
data = obj.filedata.read()
if len(data) < 4:
raise ServiceConfigError("Need at least 4 bytes.")
# Reset the read pointer.
obj.filedata.seek(0)
if not struct.unpack('@I', data[:4])[0] in [
MachOEntity.FAT_MAGIC, MachOEntity.FAT_CIGAM,
MachOEntity.MH_MAGIC, MachOEntity.MH_CIGAM,
MachOEntity.MH_MAGIC_64, MachOEntity.MH_CIGAM_64
]:
raise ServiceConfigError("Bad magic.")
def valid_for(obj):
chm_magic = '\x49\x54\x53\x46\x03\x00\x00\x00\x60\x00\x00\x00'
if obj.filedata != None:
data = obj.filedata.read()
# Need to reset the read pointer.
obj.filedata.seek(0)
if data.startswith(chm_magic):
return
raise ServiceConfigError("Not a valid ITSF (CHM) file.")
def valid_for(obj):
office_magic = "\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"
if obj.filedata != None:
data = obj.filedata.read()
# Need to reset the read pointer.
obj.filedata.seek(0)
if data.startswith(office_magic):
return
raise ServiceConfigError("Not a valid office document.")
def valid_for(obj):
rtf_magic = "{\\rt"
if obj.filedata != None:
data = obj.filedata.read()
# Need to reset the read pointer.
obj.filedata.seek(0)
if data.startswith(rtf_magic):
return
raise ServiceConfigError("Not a valid rtf document.")
def _scan(self, context):
logger.debug("Setting up shop...")
shop_path = "%s/shop" % self.base_dir
if not os.path.exists(self.base_dir):
raise ServiceConfigError("ChopShop path does not exist")
elif not os.path.exists(shop_path):
raise ServiceConfigError("ChopShop shop path does not exist")
else:
sys.path.append(shop_path)
from ChopLib import ChopLib
from ChopUi import ChopUi
logger.debug("Scanning...")
choplib = ChopLib()
chopui = ChopUi()
choplib.base_dir = self.base_dir
# XXX: Convert from unicode to str...
choplib.modules = str(self.modules)
chopui.jsonout = jsonhandler
choplib.jsonout = True
# ChopShop (because of pynids) needs to read a file off disk.
# The services framework forces you to use 'with' here. It's not
# possible to just get a path to a file on disk.
with self._write_to_file() as pcap_file:
choplib.filename = pcap_file
chopui.bind(choplib)
chopui.start()
chopui.jsonclass.set_service(self)
choplib.start()
while chopui.is_alive():
time.sleep(.1)
chopui.join()
choplib.finish()
choplib.join()
def valid_for(obj):
if not obj.filedata:
return False
data = obj.filedata.read(8)
obj.filedata.seek(0)
if obj.is_pdf():
return True
elif data.startswith("\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"):
# M$ Word document
return True
else:
raise ServiceConfigError("Not a valid PDF or a Word document.")
def parse_config(config):
# When editing a config we are given a string.
# When validating an existing config it will be a list.
# Convert it to a list of strings.
sigfiles = config.get('sigfiles', [])
if not sigfiles:
raise ServiceConfigError("Must specify signature files.")
if isinstance(sigfiles, basestring):
config['sigfiles'] = [sigfile for sigfile in sigfiles.split('\r\n')]
# This will raise ServiceConfigError
YaraService._compile_rules(config['sigdir'], config['sigfiles'])
def parse_service_config(config):
namespace = config.get("namespace", "").strip()
ns_prefix = config.get("ns_prefix", "").strip()
max_rels = config.get("max_rels", "")
errors = []
if not namespace:
errors.append("You must specify a XML Namespace.")
if not ns_prefix:
errors.append("You must specify a XML Namespace Prefix.")
if not max_rels or max_rels > 5000 or max_rels < 0:
errors.append("Maximum Related must be in the range 0-5000.")
if errors:
raise ServiceConfigError("<br>".join(errors))
