def xor_string(md5=None, data=None, key=0, null=0):
"""
XOR data.
:param md5: The MD5 of the Sample to parse.
:type md5: str
:param data: The data to parse.
:type data: str
:param key: The XOR key to use.
:type key: int
:param null: Whether or not to skip nulls.
:type null: int (0 or 1)
:returns: str
"""
if md5:
data = get_file(md5)
out = ''
for c in data:
if ord(c) == 0 and null == 1:
out += c
elif ord(c) == key and null == 1:
out += c
else:
out += chr(ord(c) ^ key)
return out
def run(self, argv):
parser = OptionParser()
parser.add_option("-c", "--collection", action="store", dest="collection",
type="string", help="collection (if not samples)")
parser.add_option("-m", "--md5", action="store", dest="md5",
type="string", help="md5 of file")
parser.add_option("-o", "--output", action="store", dest="filename",
type="string", help="output filename")
(opts, args) = parser.parse_args(argv)
data = None
collection = opts.collection or Sample._meta['collection']
if opts.md5:
try:
data = get_file(opts.md5, collection=collection)
except:
print "[+] could not read data for %s" % opts.md5
return
if data:
if opts.filename:
filename = opts.filename
else:
filename = get_filename(opts.md5, collection=collection)
if filename == None:
filename = opts.md5
print "[+] writing %d bytes to %s" % (len(data), filename)
try:
fin = open(filename, 'wb')
fin.write(data)
fin.close()
except:
print "[+] error writing %s to disk" % filename
def run(self, argv):
parser = OptionParser()
parser.add_option("-m",
"--md5",
action="store",
dest="md5",
type="string",
help="MD5 of file to retrieve")
parser.add_option("-f",
"--file",
action="store",
dest="file",
type="string",
help="File to analyze")
parser.add_option("-v",
"--verbose",
action="store_true",
dest="verbose",
default=False,
help="Be verbose")
(opts, args) = parser.parse_args(argv)
if opts.md5:
if opts.verbose:
print "[+] attempting to call get_file on %s" % opts.md5
data = get_file(opts.md5)
elif opts.file:
try:
fin = open(opts.file, 'rb')
except IOError, e:
print str(e)
return
data = fin.read()
fin.close()
def xor_string(md5=None, data=None, key=0, null=0):
"""
XOR data.
:param md5: The MD5 of the Sample to parse.
:type md5: str
:param data: The data to parse.
:type data: str
:param key: The XOR key to use.
:type key: int
:param null: Whether or not to skip nulls.
:type null: int (0 or 1)
:returns: str
"""
if md5:
data = get_file(md5)
out = ''
for c in data:
if ord(c) == 0 and null == 1:
out += c
elif ord(c) == key and null == 1:
out += c
else:
out += chr(ord(c) ^ key)
return out
def run(self, argv):
samples = mongo_connector(settings.COL_SAMPLES)
today = datetime.datetime.fromordinal(datetime.datetime.now().toordinal())
md5s = samples.find({"source.instances.date": {"$gte": today}})
filename = "%s/%s.tar.bz2" % ("/tmp/samples", today.strftime("%Y-%m-%d"))
tar = tarfile.open(filename, "w:bz2")
for md5 in md5s:
m = md5['md5']
f = md5['filename']
s = get_file(m)
info = tarfile.TarInfo(name="%s" % f)
info.mtime = time.time()
info.size = len(s)
tar.addfile(info, StringIO(s))
tar.close()
def run(self, argv):
samples = mongo_connector(settings.COL_SAMPLES)
today = datetime.datetime.fromordinal(datetime.datetime.now().toordinal())
md5s = samples.find({"source.instances.date": {"$gte": today}})
filename = "%s/%s.tar.bz2" % ("/tmp/samples", today.strftime("%Y-%m-%d"))
tar = tarfile.open(filename, "w:bz2")
for md5 in md5s:
m = md5['md5']
f = md5['filename']
s = get_file(m)
info = tarfile.TarInfo(name="%s" % f)
info.mtime = time.time()
info.size = len(s)
tar.addfile(info, BytesIO(s))
tar.close()
def xor_search(md5=None, data=None, string=None, skip_nulls=0):
"""
Search a string for potential XOR keys. Uses a small list of common
plaintext terms, XORs those terms using keys 0-255 and searches the data for
any match. If there is a match, that key is included in the results.
:param md5: The MD5 of the Sample to parse.
:type md5: str
:param data: The data to parse.
:type data: str
:param string: The custom string to XOR and search for.
:type string: str
:param skip_nulls: Whether or not to skip nulls.
:type skip_nulls: int (0 or 1)
:returns: list
"""
if md5:
data = get_file(md5)
if string is None or string == '':
plaintext_list = [
'This program',
'kernel32',
'KERNEL32',
'http',
'svchost',
'Microsoft',
'PE for WIN32',
'startxref',
'!This program cannot be run in DOS mode',
'\xD0\xCF\x11\xE0\xA1\xB1\x1a\xE1',
'D\x00o\x00c\x00u\x00m\x00e\x00n\x00t\x00 \x00S\x00u\x00m\x00m\x00a\x00r\x00y\x00 \x00I\x00n\x00f\x00o\x00r\x00m\x00a\x00t\x00i\x00o\x00n',
]
else:
plaintext_list = ["%s" % string]
results = []
for plaintext in plaintext_list:
for i in range(0, 255):
xord_string = xor_string(data=plaintext,
key=i,
null=skip_nulls)
if xord_string in data:
if i not in results:
results.append(i)
results.sort()
return results
def xor_search(md5=None, data=None, string=None, skip_nulls=0):
"""
Search a string for potential XOR keys. Uses a small list of common
plaintext terms, XORs those terms using keys 0-255 and searches the data for
any match. If there is a match, that key is included in the results.
:param md5: The MD5 of the Sample to parse.
:type md5: str
:param data: The data to parse.
:type data: str
:param string: The custom string to XOR and search for.
:type string: str
:param skip_nulls: Whether or not to skip nulls.
:type skip_nulls: int (0 or 1)
:returns: list
"""
if md5:
data = get_file(md5)
if string is None or string == '':
plaintext_list = [
'This program',
'kernel32',
'KERNEL32',
'http',
'svchost',
'Microsoft',
'PE for WIN32',
'startxref',
'!This program cannot be run in DOS mode',
'\xD0\xCF\x11\xE0\xA1\xB1\x1a\xE1',
'D\x00o\x00c\x00u\x00m\x00e\x00n\x00t\x00 \x00S\x00u\x00m\x00m\x00a\x00r\x00y\x00 \x00I\x00n\x00f\x00o\x00r\x00m\x00a\x00t\x00i\x00o\x00n',
]
else:
plaintext_list = ["%s" % string]
results = []
for plaintext in plaintext_list:
for i in range(0, 255):
xord_string = xor_string(data=plaintext,
key=i,
null=skip_nulls)
if xord_string in data:
if i not in results:
results.append(i)
results.sort()
return results
def run(self, argv):
parser = OptionParser()
parser.add_option("-c",
"--collection",
action="store",
dest="collection",
type="string",
help="collection (if not samples)")
parser.add_option("-m",
"--md5",
action="store",
dest="md5",
type="string",
help="md5 of file")
parser.add_option("-o",
"--output",
action="store",
dest="filename",
type="string",
help="output filename")
(opts, args) = parser.parse_args(argv)
data = None
collection = opts.collection or Sample._meta['collection']
if opts.md5:
try:
data = get_file(opts.md5, collection=collection)
except:
print "[+] could not read data for %s" % opts.md5
return
if data:
if opts.filename:
filename = opts.filename
else:
filename = get_filename(opts.md5, collection=collection)
if filename == None:
filename = opts.md5
print "[+] writing %d bytes to %s" % (len(data), filename)
try:
fin = open(filename, 'wb')
fin.write(data)
fin.close()
except:
print "[+] error writing %s to disk" % filename
def make_unicode_strings(md5=None, data=None):
"""
Find and return all printable Unicode strings in a string.
:param md5: The MD5 of the Sample to parse.
:type md5: str
:param data: The data to parse.
:type data: str
:returns: str
"""
if md5:
data = get_file(md5)
strings_data = 'Unicode Strings\n'
strings_data += "-" * 30
strings_data += "\n"
unicode_regex = re.compile('(([%s]\x00){4,})' % string.printable)
matches = unicode_regex.findall(data)
strings_data += '\n'.join([x[0].replace('\x00', '') for x in matches])
return strings_data + "\n\n\n\n"
def make_ascii_strings(md5=None, data=None):
"""
Find and return all printable ASCII strings in a string.
:param md5: The MD5 of the Sample to parse.
:type md5: str
:param data: The data to parse.
:type data: str
:returns: str
"""
if md5:
data = get_file(md5)
strings_data = 'ASCII Strings\n'
strings_data += "-" * 30
strings_data += "\n"
ascii_regex = re.compile('([ -~]{4,})')
matches = ascii_regex.findall(data)
strings_data += '\n'.join([x for x in matches])
return strings_data + "\n\n\n\n"
def make_unicode_strings(md5=None, data=None):
"""
Find and return all printable Unicode strings in a string.
:param md5: The MD5 of the Sample to parse.
:type md5: str
:param data: The data to parse.
:type data: str
:returns: str
"""
if md5:
data = get_file(md5)
strings_data = 'Unicode Strings\n'
strings_data += "-" * 30
strings_data += "\n"
unicode_regex = re.compile('(([%s]\x00){4,})' % string.printable)
matches = unicode_regex.findall(data)
strings_data += '\n'.join([x[0].replace('\x00', '') for x in matches])
return strings_data + "\n\n\n\n"
def make_ascii_strings(md5=None, data=None):
"""
Find and return all printable ASCII strings in a string.
:param md5: The MD5 of the Sample to parse.
:type md5: str
:param data: The data to parse.
:type data: str
:returns: str
"""
if md5:
data = get_file(md5)
strings_data = 'ASCII Strings\n'
strings_data += "-" * 30
strings_data += "\n"
ascii_regex = re.compile('([ -~]{4,})')
matches = ascii_regex.findall(data)
strings_data += '\n'.join([x for x in matches])
return strings_data + "\n\n\n\n"
def make_stackstrings(md5=None, data=None):
"""
Find and return all stack strings in a string.
:param md5: The MD5 of the Sample to parse.
:type md5: str
:param data: The data to parse.
:type data: str
:returns: str
"""
if md5:
data = get_file(md5)
x = 0
prev = 0
strings = ""
while x < len(data):
if (data[x] == "\xc6") and ((data[x + 1] == "\x45") or (data[x + 1] == "\x84")):
a = ord(data[x + 3])
if (a <= 126 and a >= 32) or (a == 9):
strings += data[x + 3]
prev = x
x += 4
elif (data[x] == "\xc6") and (data[x + 1] == "\x44"):
a = ord(data[x + 4])
if (a <= 126 and a >= 32) or (a == 9):
strings += data[x + 4]
prev = x
x += 5
elif (data[x] == "\xc6") and ((data[x + 1] == "\x05") or (data[x + 1] == "\x85")):
a = ord(data[x + 6])
if (a <= 126 and a >= 32) or (a == 9):
strings += data[x + 6]
prev = x
x += 7
else:
if (x - prev) == 12:
strings += "\n"
x += 1
strings = strings.replace("\x00", "\r")
return strings
def _scan(self, context):
data = get_file(context.md5)
zipdata = create_zip([("samples", data)])
url = self.config.get('OPSWAT_url', '')
req = urllib2.Request(url)
req.add_header("Content-Type", "application/zip")
req.add_data(bytearray(zipdata))
out = urllib2.urlopen(req)
text_out = out.read()
# Parse XML output
handler = XMLTagHandler()
parser = xml.parsers.expat.ParserCreate()
parser.StartElementHandler = handler.StartElement
parser.EndElementHandler = handler.EndElement
parser.CharacterDataHandler = handler.CharData
parser.Parse(text_out)
for threat in handler.threatList:
self._add_result('av_result', threat["threat_name"], {"engine":threat["engine_name"], "date":datetime.now().isoformat()})
def make_hex(md5=None, data=None):
"""
Convert data into hex formatted output.
:param md5: The MD5 of the Sample to parse.
:type md5: str
:param data: The data to parse.
:type data: str
:returns: str
"""
if md5:
data = get_file(md5)
length = 16
hex_data = ''
digits = 4 if isinstance(data, unicode) else 2
for i in xrange(0, len(data), length):
s = data[i:i+length]
hexa = ' '.join(["%0*X" % (digits, ord(x)) for x in s])
text = ' '.join([x if 0x20 <= ord(x) < 0x7F else '.' for x in s])
hex_data += "%04X %-*s %s\r\n" % (i, length*(digits + 1), hexa, text)
return hex_data
def make_stackstrings(md5=None, data=None):
"""
Find and return all stack strings in a string.
:param md5: The MD5 of the Sample to parse.
:type md5: str
:param data: The data to parse.
:type data: str
:returns: str
"""
if md5:
data = get_file(md5)
x = 0
prev = 0
strings = ''
while x < len(data):
if (data[x] == '\xc6') and ((data[x + 1] == '\x45') or
(data[x + 1] == '\x84')):
a = ord(data[x + 3])
if (a <= 126 and a >= 32) or (a == 9): strings += data[x + 3]
prev = x
x += 4
elif (data[x] == '\xc6') and (data[x + 1] == '\x44'):
a = ord(data[x + 4])
if (a <= 126 and a >= 32) or (a == 9): strings += data[x + 4]
prev = x
x += 5
elif (data[x] == '\xc6') and ((data[x + 1] == '\x05') or
(data[x + 1] == '\x85')):
a = ord(data[x + 6])
if (a <= 126 and a >= 32) or (a == 9): strings += data[x + 6]
prev = x
x += 7
else:
if ((x - prev) == 12): strings += '\n'
x += 1
strings = strings.replace('\x00', '\r')
return strings
def make_hex(md5=None, data=None):
"""
Convert data into hex formatted output.
:param md5: The MD5 of the Sample to parse.
:type md5: str
:param data: The data to parse.
:type data: str
:returns: str
"""
if md5:
data = get_file(md5)
length = 16
hex_data = ''
digits = 4 if isinstance(data, unicode) else 2
for i in xrange(0, len(data), length):
s = data[i:i+length]
hexa = ' '.join(["%0*X" % (digits, ord(x)) for x in s])
text = ' '.join([x if 0x20 <= ord(x) < 0x7F else '.' for x in s])
hex_data += "%04X %-*s %s\r\n" % (i, length*(digits + 1), hexa, text)
return hex_data
def run(self, argv):
parser = OptionParser()
parser.add_option("-m", "--md5", action="store", dest="md5",
type="string", help="MD5 of file to retrieve")
parser.add_option("-f", "--file", action="store", dest="file",
type="string", help="File to analyze")
parser.add_option("-v", "--verbose", action="store_true", dest="verbose",
default=False, help="Be verbose")
(opts, args) = parser.parse_args(argv)
if opts.md5:
if opts.verbose:
print "[+] attempting to call get_file on %s" % opts.md5
data = get_file(opts.md5)
elif opts.file:
try:
fin = open(opts.file, 'rb')
except IOError, e:
print str(e)
return
data = fin.read()
fin.close()
def run(self, argv):
parser = OptionParser()
parser.add_option("-m", "--md5", action="store", dest="md5",
type="string", help="MD5 of file to retrieve")
parser.add_option("-f", "--file", action="store", dest="file",
type="string", help="File to analyze")
parser.add_option("-v", "--verbose", action="store_true", dest="verbose",
default=False, help="Be verbose")
(opts, args) = parser.parse_args(argv)
if opts.md5:
if opts.verbose:
print "[+] attempting to call get_file on %s" % opts.md5
data = get_file(opts.md5)
elif opts.file:
fin = open(opts.file, 'rb')
data = fin.read()
fin.close()
if opts.verbose:
print "[+] parsing %d bytes" % len(data)
if len(data) > 512:
oparser = OfficeParser(data, opts.verbose)
oparser.parse_office_doc()
def make_stackstrings(md5=None, data=None):
"""
Find and return all stack strings in a string.
:param md5: The MD5 of the Sample to parse.
:type md5: str
:param data: The data to parse.
:type data: str
:returns: str
"""
if md5:
data = get_file(md5)
x = 0
prev = 0
strings = ''
while x < len(data):
if (data[x] == '\xc6') and ((data[x+1] == '\x45') or (data[x+1] == '\x84')):
a = ord(data[x+3])
if (a <= 126 and a >= 32) or (a==9): strings += data[x+3]
prev = x
x += 4
elif (data[x] == '\xc6') and (data[x+1] == '\x44'):
a = ord(data[x+4])
if (a <= 126 and a >= 32) or (a==9): strings += data[x+4]
prev = x
x += 5
elif (data[x] == '\xc6') and ((data[x+1] == '\x05') or (data[x+1] == '\x85')):
a = ord(data[x+6])
if (a <= 126 and a >= 32) or (a==9): strings += data[x+6]
prev = x
x += 7
else:
if ((x - prev) ==12): strings += '\n'
x += 1
strings = strings.replace('\x00', '\r')
return strings
sys.exit(1)
filename = None
error_count = 0
if opts.gridfs:
print "Copying GridFS data matching md5s to new collection..."
filename = opts.gridfs
elif opts.griddelete:
print "Deleting matching md5s from GridFS..."
filename = opts.griddelete
count = 0
no_data = 0
try:
with open(filename) as o:
for line in o:
md5 = line.strip()
data = get_file(md5, opts.collection)
if data:
if opts.gridfs:
put_file(md5, data)
s = Sample.objects(md5=md5).first()
if s:
s.discover_binary()
try:
s.save()
except:
error_count += 1
err.write(
"Error saving sample for discover binary: %s"
% md5)
elif opts.griddelete:
delete_file(md5, opts.collection)
sys.exit(1)
filename = None
error_count = 0
if opts.gridfs:
print "Copying GridFS data matching md5s to new collection..."
filename = opts.gridfs
elif opts.griddelete:
print "Deleting matching md5s from GridFS..."
filename = opts.griddelete
count = 0
no_data = 0
try:
with open(filename) as o:
for line in o:
md5 = line.strip()
data = get_file(md5, opts.collection)
if data:
if opts.gridfs:
put_file(md5, data)
s = Sample.objects(md5=md5).first()
if s:
s.discover_binary()
try:
s.save()
except:
error_count += 1
err.write("Error saving sample for discover binary: %s" % md5)
elif opts.griddelete:
delete_file(md5, opts.collection)
count += 1
else: