class DatabaseMySQL():
def __init__(self):
self.__cfgReporting = Config(os.path.join(RAGPICKER_ROOT, 'config', 'reporting.conf'))
self.__mysqlEnabled = self.__cfgReporting.getOption("mysql", "enabled")
if self.__mysqlEnabled:
#Anbindung an Datenbank MySQL herstellen
try:
mysqlHost = self.__cfgReporting.getOption("mysql", "host")
mysqlPort = self.__cfgReporting.getOption("mysql", "port")
mysqlDatabase = self.__cfgReporting.getOption("mysql", "database")
mysqlUser = self.__cfgReporting.getOption("mysql", "user")
mysqlPassword = self.__cfgReporting.getOption("mysql", "password")
self.__mysqlConnection = MySQLdb.Connect(host=mysqlHost, port=mysqlPort, db=mysqlDatabase, user=mysqlUser, passwd=mysqlPassword)
except (Exception) as e:
raise Exception("Cannot connect to MySQL (ragpicker): %s" % e)
def __del__(self):
if self.__mysqlEnabled:
self.__mysqlConnection.close()
# ------------------------------------------------------------------------------
# Ragpicker Database (MySQL)
# ------------------------------------------------------------------------------
def isRagpickerDBEnabledMySQL(self):
return self.__mysqlEnabled
class DatabaseMySQL():
def __init__(self):
self.__cfgReporting = Config(
os.path.join(RAGPICKER_ROOT, 'config', 'reporting.conf'))
self.__mysqlEnabled = self.__cfgReporting.getOption("mysql", "enabled")
if self.__mysqlEnabled:
#Anbindung an Datenbank MySQL herstellen
try:
mysqlHost = self.__cfgReporting.getOption("mysql", "host")
mysqlPort = self.__cfgReporting.getOption("mysql", "port")
mysqlDatabase = self.__cfgReporting.getOption(
"mysql", "database")
mysqlUser = self.__cfgReporting.getOption("mysql", "user")
mysqlPassword = self.__cfgReporting.getOption(
"mysql", "password")
self.__mysqlConnection = MySQLdb.Connect(host=mysqlHost,
port=mysqlPort,
db=mysqlDatabase,
user=mysqlUser,
passwd=mysqlPassword)
except (Exception) as e:
raise Exception("Cannot connect to MySQL (ragpicker): %s" % e)
def __del__(self):
if self.__mysqlEnabled:
self.__mysqlConnection.close()
# ------------------------------------------------------------------------------
# Ragpicker Database (MySQL)
# ------------------------------------------------------------------------------
def isRagpickerDBEnabledMySQL(self):
return self.__mysqlEnabled
class UPSDModule_testserv(UPSDModule):
modname = 'testserv'
def start(self):
if not UPSDModule.start(self):
return False
self.conf = Config()
self.userconf = {}
### Load info for pseudo client from config ###
self.userconf['nick'] = self.conf.getOption(self.modname, 'nick')
self.userconf['ident'] = self.conf.getOption(self.modname, 'ident')
self.userconf['host'] = self.conf.getOption(self.modname, 'host')
self.userconf['modes'] = self.conf.getOption(self.modname, 'modes')
self.userconf['realname'] = self.conf.getOption(self.modname, 'realname')
self.userconf['mainchan'] = self.conf.getOption(self.modname, 'mainchan')
self.proto.sendNickCreate(self.userconf['nick'], self.userconf['ident'], self.userconf['host'], self.userconf['modes'], self.userconf['realname'])
self.proto.sendJoin(self.userconf['nick'], self.userconf['mainchan'])
return True
def stop(self):
UPSDModule.stop(self)
self.proto.sendUserQuit(self.userconf['nick'])
def testserv_ctcp(self, user, command, params):
if command.lower() == "\x01version\x01":
self.proto.sendNotice(self.userconf['nick'], user, '\x01VERSION %s\x01' % (self.getVersion()))
return True
def testserv_channel(self, user, command, params):
nick = params[0].split(':')[1]
if (len(params) > 4):
args = params[4].split(' ')
else:
args = None
if (command == ".kick"):
self.proto.sendKick(self.userconf['nick'], user, args[0], args[1])
def getModHooks(self):
return (('privmsg', self.testserv_ctcp),
('privmsg', self.testserv_channel))
def getVersion(self):
return 'UnrealPSD TestServ Module'
class Database:
def __init__(self):
self.__cfgReporting = Config(os.path.join(RAGPICKER_ROOT, "config", "reporting.conf"))
self.__cfgProcessing = Config(os.path.join(RAGPICKER_ROOT, "config", "processing.conf"))
self.__mongodbEnabled = self.__cfgReporting.getOption("mongodb", "enabled")
self.__codedbEnabled = self.__cfgReporting.getOption("codeDB", "enabled")
self.__bluecoatEnabled = self.__cfgProcessing.getOption("all_bluecoatMalwareAnalysisAppliance", "enabled")
if self.__mongodbEnabled:
# Anbindung an Datenbank MongoDB Collection Ragpicker herstellen
try:
mongodbHost = self.__cfgReporting.getOption("mongodb", "host")
mongodbPort = self.__cfgReporting.getOption("mongodb", "port")
self.__mongodbConnection = MongoClient(mongodbHost, mongodbPort)
self.__mongodbCollectionRagpicker = self.__mongodbConnection.MalwareAnalyse.ragpicker
self.__mongodbCollectionFamilies = self.__mongodbConnection.MalwareAnalyse.families
self.__mongodbCollectionSandboxTaskQueue = self.__mongodbConnection.MalwareAnalyse.sandboxTaskQueue
except TypeError:
raise Exception("MongoDB connection port in report.config must be integer")
except ConnectionFailure:
raise Exception("Cannot connect to MongoDB (ragpicker)")
if self.__codedbEnabled:
# Anbindung an Datenbank MongoDB Collection CodeDB herstellen
try:
codedbHost = self.__cfgReporting.getOption("codeDB", "mongo_db_host")
codedbPort = self.__cfgReporting.getOption("codeDB", "mongo_db_port")
self.__codedbConnection = MongoClient(codedbHost, codedbPort)
self.__codedbCollectionCodedb = self.__codedbConnection.MalwareAnalyse.codeDB
except TypeError:
raise Exception("MongoDB connection port for CodeDB in report.config must be integer")
except ConnectionFailure:
raise Exception("Cannot connect to MongoDB (codeDB)")
def __del__(self):
if self.__mongodbEnabled:
self.__mongodbConnection.disconnect()
if self.__codedbEnabled:
self.__codedbConnection.disconnect()
# ------------------------------------------------------------------------------
# Ragpicker Database (MongoDB)
# ------------------------------------------------------------------------------
def isRagpickerDBEnabled(self):
return self.__mongodbEnabled
def getStatisticsAntivirus(self):
queries = []
ret = []
queries.append(
{
"product": "Avast Antivirus",
"findStr1": "AntivirusScanAvast",
"findStr2": "AntivirusScanAvast.avast",
"ok": "OK",
}
)
queries.append(
{"product": "AVG Antivirus", "findStr1": "AntivirusScanAvg", "findStr2": "AntivirusScanAvg.Avg", "ok": "OK"}
)
queries.append(
{
"product": "Avira",
"findStr1": "AntivirusScanAvira",
"findStr2": "AntivirusScanAvira.Avira.scan",
"ok": "OK",
}
)
queries.append(
{
"product": "BitDefender",
"findStr1": "AntivirusScanBitDefender",
"findStr2": "AntivirusScanBitDefender.BitDefender",
"ok": "OK",
}
)
queries.append(
{
"product": "ClamAV",
"findStr1": "AntivirusScanClamAv",
"findStr2": "AntivirusScanClamAv.ClamAv",
"ok": " OK",
}
)
queries.append(
{
"product": "COMODO",
"findStr1": "AntivirusScanCOMODO",
"findStr2": "AntivirusScanCOMODO.COMODO",
"ok": "OK",
}
)
queries.append(
{"product": "ESET", "findStr1": "AntivirusScanESET", "findStr2": "AntivirusScanESET.ESET", "ok": "OK"}
)
queries.append(
{"product": "F-Prot", "findStr1": "AntivirusScanFProt", "findStr2": "AntivirusScanFProt.FProt", "ok": "OK"}
)
queries.append(
{
"product": "F-Secure",
"findStr1": "AntivirusScanF-Secure",
"findStr2": "AntivirusScanF-Secure.F-Secure",
"ok": "OK",
}
)
for q in queries:
av = {}
av["product"] = q.get("product")
av["analyzed"] = str(self.__mongodbCollectionRagpicker.find({q.get("findStr1"): {"$ne": None}}).count())
av["notanalyzed"] = str(self.__mongodbCollectionRagpicker.find({q.get("findStr1"): None}).count())
av["malware"] = str(
self.__mongodbCollectionRagpicker.find(
{"$and": [{q.get("findStr1"): {"$ne": None}}, {q.get("findStr2"): {"$ne": q.get("ok")}}]}
).count()
)
av["nonemalware"] = str(self.__mongodbCollectionRagpicker.find({q.get("findStr2"): q.get("ok")}).count())
if av.get("analyzed") != "0":
av["rate"] = "{:.2f} %".format((float(av.get("malware")) / float(av.get("analyzed")) * 100))
else:
av["rate"] = "--"
ret.append(av)
return ret
def getStatisticsNoneMalwareByAV(self):
return self.__mongodbCollectionRagpicker.find(
{
"$and": [
{"$or": [{"AntivirusScanAvast.avast": "OK"}, {"AntivirusScanAvast": None}]},
{"$or": [{"AntivirusScanAvg.Avg": "OK"}, {"AntivirusScanAvg": None}]},
{"$or": [{"AntivirusScanAvira.Avira.scan": "OK"}, {"AntivirusScanAvira": None}]},
{"$or": [{"AntivirusScanBitDefender.BitDefender": "OK"}, {"AntivirusScanBitDefender": None}]},
{"$or": [{"AntivirusScanClamAv.ClamAv": " OK"}, {"AntivirusScanClamAv": None}]},
{"$or": [{"AntivirusScanCOMODO.COMODO": "OK"}, {"AntivirusScanCOMODO": None}]},
{"$or": [{"AntivirusScanESET.ESET": "OK"}, {"AntivirusScanESET": None}]},
{"$or": [{"AntivirusScanFProt.FProt": "OK"}, {"AntivirusScanFProt": None}]},
{"$or": [{"AntivirusScanF-Secure.F-Secure": "OK"}, {"AntivirusScanF-Secure": None}]},
{"VirusTotal.file.verbose_msg": {"$ne": None}},
]
}
).count()
def getSamplesNotFoundByAV(self):
return self.__mongodbCollectionRagpicker.find(
{
"$and": [
{"$or": [{"AntivirusScanAvast.avast": "OK"}, {"AntivirusScanAvast": None}]},
{"$or": [{"AntivirusScanAvg.Avg": "OK"}, {"AntivirusScanAvg": None}]},
{"$or": [{"AntivirusScanAvira.Avira.scan": "OK"}, {"AntivirusScanAvira": None}]},
{"$or": [{"AntivirusScanBitDefender.BitDefender": "OK"}, {"AntivirusScanBitDefender": None}]},
{"$or": [{"AntivirusScanClamAv.ClamAv": " OK"}, {"AntivirusScanClamAv": None}]},
{"$or": [{"AntivirusScanCOMODO.COMODO": "OK"}, {"AntivirusScanCOMODO": None}]},
{"$or": [{"AntivirusScanESET.ESET": "OK"}, {"AntivirusScanESET": None}]},
{"$or": [{"AntivirusScanFProt.FProt": "OK"}, {"AntivirusScanFProt": None}]},
{"$or": [{"AntivirusScanF-Secure.F-Secure": "OK"}, {"AntivirusScanF-Secure": None}]},
{"VirusTotal.file.verbose_msg": {"$ne": None}},
]
},
{"Info.file.sha256": True, "Info.analyse.started": True},
)
def getSamplesNotFoundByVT(self):
return self.__mongodbCollectionRagpicker.find(
{"VirusTotal.file.verbose_msg": {"$ne": None}}, {"Info.file.sha256": True, "Info.analyse.started": True}
)
def getSamplesNotFoundByLocalAV(self):
return self.__mongodbCollectionRagpicker.find(
{
"$and": [
{"$or": [{"AntivirusScanAvast.avast": "OK"}, {"AntivirusScanAvast": None}]},
{"$or": [{"AntivirusScanAvg.Avg": "OK"}, {"AntivirusScanAvg": None}]},
{"$or": [{"AntivirusScanAvira.Avira.scan": "OK"}, {"AntivirusScanAvira": None}]},
{"$or": [{"AntivirusScanBitDefender.BitDefender": "OK"}, {"AntivirusScanBitDefender": None}]},
{"$or": [{"AntivirusScanClamAv.ClamAv": " OK"}, {"AntivirusScanClamAv": None}]},
{"$or": [{"AntivirusScanCOMODO.COMODO": "OK"}, {"AntivirusScanCOMODO": None}]},
{"$or": [{"AntivirusScanESET.ESET": "OK"}, {"AntivirusScanESET": None}]},
{"$or": [{"AntivirusScanFProt.FProt": "OK"}, {"AntivirusScanFProt": None}]},
{"$or": [{"AntivirusScanF-Secure.F-Secure": "OK"}, {"AntivirusScanF-Secure": None}]},
]
},
{"Info.file.sha256": True, "Info.analyse.started": True},
)
def getStatisticsVirusTotal(self):
ret = {}
ret["analyzed"] = self.__mongodbCollectionRagpicker.find({"VirusTotal": {"$ne": None}}).count()
ret["notAnalyzed"] = self.__mongodbCollectionRagpicker.find({"VirusTotal": None}).count()
ret["samplesFound"] = self.__mongodbCollectionRagpicker.find(
{"VirusTotal.file.positives": {"$ne": None}}
).count()
ret["SamplesNotFound"] = self.__mongodbCollectionRagpicker.find(
{"VirusTotal.file.verbose_msg": {"$ne": None}}
).count()
return ret
def getStatisticsPackerSignatures(self):
return self.__mongodbCollectionRagpicker.aggregate(
[
{
"$group": {
"_id": {"PublisherO": "$VerifySigs.PublisherO", "Issuer": "$VerifySigs.Issuer"},
"count": {"$sum": 1},
}
},
{"$sort": {"count": -1}},
]
)
def getStatisticsPackerCompiler(self):
return self.__mongodbCollectionRagpicker.aggregate(
[{"$group": {"_id": "$PEID", "count": {"$sum": 1}}}, {"$sort": {"count": -1}}]
)
def getStatisticsPeCharacteristics(self):
ret = {}
ret["exe"] = str(
self.__mongodbCollectionRagpicker.find(
{"$and": [{"Info.file.EXE": True}, {"Info.file.DLL": False}, {"Info.file.DRIVER": False}]}
).count()
)
ret["dll"] = str(
self.__mongodbCollectionRagpicker.find(
{"$and": [{"Info.file.EXE": False}, {"Info.file.DLL": True}, {"Info.file.DRIVER": False}]}
).count()
)
ret["driver"] = str(
self.__mongodbCollectionRagpicker.find(
{"$and": [{"Info.file.EXE": False}, {"Info.file.DLL": False}, {"Info.file.DRIVER": True}]}
).count()
)
ret["noPe"] = str(
self.__mongodbCollectionRagpicker.find(
{"$and": [{"Info.file.EXE": None}, {"Info.file.DLL": None}, {"Info.file.DRIVER": None}]}
).count()
)
ret["dllDriver"] = str(
self.__mongodbCollectionRagpicker.find(
{"$and": [{"Info.file.EXE": False}, {"Info.file.DLL": True}, {"Info.file.DRIVER": True}]}
).count()
)
return ret
def getFiletypes(self):
return self.__mongodbCollectionRagpicker.aggregate(
[{"$group": {"_id": "$Info.file.type", "count": {"$sum": 1}}}, {"$sort": {"count": -1}}]
)
def countReportsRagpickerDB(self):
return self.__mongodbCollectionRagpicker.find().count()
def iterateRagpickerReports(self, sha256):
for report in self.__mongodbCollectionRagpicker.find({"Info.file.sha256": sha256}, {"_id": 0}):
yield report
# Attention deletes the whole Ragpicker-Database!!!
# returns number of deleted reports
def deleteRagpickerDB(self):
count = self.__mongodbCollectionRagpicker.find().count()
# Alle Ragpicker-Daten aus der MongoDB loeschen
self.__mongodbCollectionRagpicker.remove()
return count
# Insert Ragpicker-Report in MongoDB
def insertRagpickerDB(self, report):
# Store the report
try:
self.__mongodbCollectionRagpicker.insert(report)
except InvalidDocument as e:
log.exception("Error InvalidDocument: %s", report)
raise Exception("Error InvalidDocument: {0}".format(e))
except InvalidStringData:
self.__mongodbCollectionRagpicker.insert(convertDirtyDict2ASCII(report))
# Count Ragpicker-Reports by file (and url)
def countRagpickerDB(self, file_md5, url_md5=None):
if url_md5:
query = {"$and": [{"Info.url.md5": {"$in": [url_md5]}}, {"Info.file.md5": {"$in": [file_md5]}}]}
else:
query = {"$and": [{"Info.file.md5": {"$in": [file_md5]}}]}
return self.__mongodbCollectionRagpicker.find(query).count()
# ------------------------------------------------------------------------------
# Ragpicker SandboxTaskQueue database (MongoDB)
# ------------------------------------------------------------------------------
def insertSandboxTaskStatus(self, sandboxName, sha256, taskID, sampleID, taskState=None):
statusReport = {
"sandbox": sandboxName,
"sha256": sha256,
"sample_id": sampleID,
"task_id": taskID,
"task_state": taskState,
}
# Store the SandboxTaskQueue-Status-report
self.__mongodbCollectionSandboxTaskQueue.insert(statusReport)
# Attention deletes the whole Ragpicker-SandboxTaskQueue-Database!!!
# returns number of deleted reports
def deleteSandboxTaskQueueDB(self):
count = self.__mongodbCollectionSandboxTaskQueue.find().count()
# Alle Daten aus der MongoDB loeschen
self.__mongodbCollectionSandboxTaskQueue.remove()
return count
# ------------------------------------------------------------------------------
# Ragpicker families database (MongoDB)
# ------------------------------------------------------------------------------
def insertFamily(self, familyReport):
# Store the family-report
self.__mongodbCollectionFamilies.insert(familyReport)
# Count Ragpicker-Reports by file (and url)
def countFamilyDB(self, parentObjectSHA256):
query = {"$and": [{"parentObjectSHA256": {"$in": [parentObjectSHA256]}}]}
return self.__mongodbCollectionFamilies.find(query).count()
def iterateFamilyReports(self, sha256):
for report in self.__mongodbCollectionFamilies.find({"parentObjectSHA256": sha256}, {"_id": 0}):
yield report
# Attention deletes the whole Ragpicker-Family-Database!!!
# returns number of deleted reports
def deleteFamilyDB(self):
count = self.__mongodbCollectionFamilies.find().count()
# Alle Ragpicker-Daten aus der MongoDB loeschen
self.__mongodbCollectionFamilies.remove()
return count
class Statistics():
def __init__(self):
# Datenbank
self.__database = Database()
# Kofiguration aus der reporting.conf holen
self.__cfgReporting = Config(os.path.join(RAGPICKER_ROOT, 'config', 'reporting.conf'))
self.__vxcageEnabled = self.__cfgReporting.getOption("vxcage", "enabled")
self.__vxcageHost = self.__cfgReporting.getOption("vxcage", "host")
self.__vxcagePort = self.__cfgReporting.getOption("vxcage", "port")
def runStatisticsLong(self):
#Pruefen ob VxCage und MongoDB aktiviert sind
if self.__database.isRagpickerDBEnabled():
if self.__vxcageEnabled:
self.__runStatisticsMongodbLong()
if self.__database.isCodeDBEnabled():
self.__runStatisticsCodedb()
else:
print("vxcage in reporting.conf is not enabled")
sys.stdout.flush()
else:
print("mongodb in reporting.conf is not enabled")
sys.stdout.flush()
def runStatisticsShort(self):
#Pruefen ob VxCage und MongoDB aktiviert sind
if self.__database.isRagpickerDBEnabled():
if self.__vxcageEnabled:
self.__runStatisticsMongodbShort()
if self.__database.isCodeDBEnabled():
self.__runStatisticsCodedb()
else:
print("vxcage in reporting.conf is not enabled")
sys.stdout.flush()
else:
print("mongodb in reporting.conf is not enabled")
sys.stdout.flush()
def runStatisticsAV(self):
#Pruefen ob VxCage und MongoDB aktiviert sind
if self.__database.isRagpickerDBEnabled():
if self.__vxcageEnabled:
self.__runStatisticsAV()
else:
print("vxcage in reporting.conf is not enabled")
sys.stdout.flush()
else:
print("mongodb in reporting.conf is not enabled")
sys.stdout.flush()
def __runStatisticsMongodbLong(self):
print "**************************************"
print "*** Statistics MongoDB (Ragpicker) ***"
print "**************************************"
print ""
print "Number of malware samples in database:", self.__database.countReportsRagpickerDB()
print ""
#Statistiken der eingesetzten AV-Produkte
self.__runStatisticsAVProducts()
#Liste der letzen 20 Samples, die weder auf VT noch von einem lokalen AV gefunden wurden
self.__runStatisticsLast20SamplesNotFoundByAV()
#Liste der letzen 20 Samples, die nicht auf VT gefunden wurden
self.__runStatisticsLast20SamplesNotFoundByVT()
#Liste der letzen 20 Samples, die nicht von einem lokalen AV-Produkt gefunden wurden
self.__runStatisticsLast20SamplesNotFoundByLocalAV()
#Liste und Haeufigkeit der Filetypes
self.__runStatisticsFiletypes()
#Haeufigkeit der PE Charakteristiken
self.__runStatisticsPeCharacteristics()
#Liste und Haeufigkeit der verwendeten Packer/Compiler in der Malware
self.__runStatisticsPackerCompiler()
#Liste der verwendeten digitalen Signaturen
self.__runStatisticsPackerSignatures()
sys.stdout.flush()
def __runStatisticsMongodbShort(self):
print "**************************************"
print "*** Statistics MongoDB (Ragpicker) ***"
print "**************************************"
print ""
print "Number of malware samples in database:", self.__database.countReportsRagpickerDB()
print ""
#Liste und Haeufigkeit der Filetypes
self.__runStatisticsFiletypes()
#Haeufigkeit der PE Charakteristiken
self.__runStatisticsPeCharacteristics()
sys.stdout.flush()
def __runStatisticsAV(self):
print "**************************************"
print "*** Statistics MongoDB (Ragpicker) ***"
print "**************************************"
print ""
print "Number of malware samples in database:", self.__database.countReportsRagpickerDB()
print ""
#Statistiken der eingesetzten AV-Produkte
self.__runStatisticsAVProducts()
#Liste der letzen 20 Samples, die weder auf VT noch von einem lokalen AV gefunden wurden
self.__runStatisticsLast20SamplesNotFoundByAV()
#Liste der letzen 20 Samples, die nicht auf VT gefunden wurden
self.__runStatisticsLast20SamplesNotFoundByVT()
#Liste der letzen 20 Samples, die nicht von einem lokalen AV-Produkt gefunden wurden
self.__runStatisticsLast20SamplesNotFoundByLocalAV()
sys.stdout.flush()
def __runStatisticsFiletypes(self):
#Liste und Haeufigkeit der Filetypes
print "Filetypes of malware"
res = self.__database.getFiletypes()
table = PrettyTable(["filetype", "count"])
table.align["filetype"] = "l"
table.align["count"] = "c"
table.padding_width = 1
try:
for values in res['result']:
if values.get("_id"):
outputPacker = values.get("_id")
outputCount = str(values.get("count"))
table.add_row([outputPacker, outputCount])
print(table)
except KeyError:
raise Exception("Dict has no key 'result' ")
print ""
def __runStatisticsPeCharacteristics(self):
#Haeufigkeit der PE Charakteristiken
print "PE-Characteristics of malware"
peC = self.__database.getStatisticsPeCharacteristics()
table = PrettyTable(["pe-characteristics", "count"])
table.align["pe-characteristics"] = "l"
table.align["count"] = "c"
table.padding_width = 1
table.add_row(["EXE", peC.get("exe")])
table.add_row(["DLL", peC.get("dll")])
table.add_row(["Driver", peC.get("driver")])
table.add_row(["DLL/Driver", peC.get("dllDriver")])
table.add_row(["No PE File", peC.get("noPe")])
print (table)
print ""
def __runStatisticsPackerCompiler(self):
#Liste und Haeufigkeit der verwendeten Packer/Compiler in der Malware
print "Packer/compiler used in malware"
res = self.__database.getStatisticsPackerCompiler()
table = PrettyTable(["packer/compiler", "count"])
table.align["packer/compiler"] = "l"
table.align["count"] = "c"
table.padding_width = 1
try:
for values in res['result']:
if values.get("_id"):
outputPacker = values.get("_id")[0]
outputCount = str(values.get("count"))
table.add_row([outputPacker, outputCount])
print(table)
except KeyError:
raise Exception("Dict has no key 'result' ")
print " "
def __runStatisticsPackerSignatures(self):
#Liste der verwendeten digitalen Signaturen
print "Signatures used by malware"
res = self.__database.getStatisticsPackerSignatures()
table = PrettyTable(["publisher", "issuer", "count"])
table.align["publisher"] = "l"
table.align["issuer"] = "l"
table.align["count"] = "c"
table.padding_width = 1
try:
for values in res['result']:
if values.get("_id"):
outputPublisher = values.get("_id").get("PublisherO")
if values.get("_id").get("Issuer"):
outputIssuer = values.get("_id").get("Issuer")
else:
outputIssuer = " "
outputCount = str(values.get("count"))
table.add_row([outputPublisher, outputIssuer, outputCount])
print(table)
except KeyError:
raise Exception("Dict has no key 'result' ")
print ""
def __runStatisticsLast20SamplesNotFoundByAV(self):
#Liste der letzen 20 Samples, die weder auf VT noch von einem lokalen AV gefunden wurden
print "Last 20 samples not found by VirusTotal and local AV-Products"
res = sorted(self.__database.getSamplesNotFoundByAV(), reverse=True)
table = PrettyTable(["timestamp of crawling", "sha256"])
table.align["timestamp of crawling"] = "c"
table.align["sha256"] = "c"
table.padding_width = 1
try:
for values in res:
sha256 = values.get("Info").get("file").get("sha256")
timestamp = values.get("Info").get("analyse").get("started")
table.add_row([timestamp, sha256])
print(table.get_string(start=0, end=20))
except KeyError:
raise Exception("Dict has no key 'Info' ")
print ""
def __runStatisticsLast20SamplesNotFoundByVT(self):
#Liste der letzen 20 Samples, die nicht auf VirusTotal gefunden wurden
print "Last 20 samples not found by VirusTotal"
res = sorted(self.__database.getSamplesNotFoundByVT(), reverse=True)
table = PrettyTable(["timestamp of crawling", "sha256"])
table.align["timestamp of crawling"] = "c"
table.align["sha256"] = "c"
table.padding_width = 1
try:
for values in res:
sha256 = values.get("Info").get("file").get("sha256")
timestamp = values.get("Info").get("analyse").get("started")
table.add_row([timestamp, sha256])
print(table.get_string(start=0, end=20))
except KeyError:
raise Exception("Dict has no key 'Info' ")
print ""
def __runStatisticsLast20SamplesNotFoundByLocalAV(self):
#Liste der letzen 20 Samples, die nicht von einem lokalen AV-Produkt gefunden wurden
print "Last 20 samples not found by local AV-Products"
res = sorted(self.__database.getSamplesNotFoundByLocalAV(), reverse=True)
table = PrettyTable(["timestamp of crawling", "sha256"])
table.align["timestamp of crawling"] = "c"
table.align["sha256"] = "c"
table.padding_width = 1
try:
for values in res:
sha256 = values.get("Info").get("file").get("sha256")
timestamp = values.get("Info").get("analyse").get("started")
table.add_row([timestamp, sha256])
print(table.get_string(start=0, end=20))
except KeyError:
raise Exception("Dict has no key 'Info' ")
print ""
def __runStatisticsAVProducts(self):
#Statistiken der eingesetzten AV-Produkte
#VirusTotal und lokale AV-Produkte
print "VirusTotal and local AV-Products"
print " Samples rated as none-malware by all AV-Products at time of crawling:", \
self.__database.getStatisticsNoneMalwareByAV()
print ""
#VirusTotal
ret = self.__database.getStatisticsVirusTotal()
print "VirusTotal"
print " Samples analyzed at time of crawling:", ret.get("analyzed")
print " Samples not analyzed at time of crawling:", ret.get("notAnalyzed")
print " Samples found at time of crawling:", ret.get("samplesFound")
print " Samples not found at time of crawling:", ret.get("SamplesNotFound")
print ""
#Lokale AV-Produkte
print "Local AV-Products"
print " analyzed => Samples analyzed at time of crawling"
print " not analyzed => Samples not analyzed at time of crawling"
print " malware => Samples rated as malware at time of crawling"
print " none-malware => Samples rated as none-malware at time of crawling"
table = PrettyTable(["product", "analyzed", "not analyzed", "malware", "none-malware", "detection rate"])
table.align["product"] = "l"
table.align["analyzed"] = "r"
table.align["not analyzed"] = "r"
table.align["malware"] = "r"
table.align["none-malware"] = "r"
table.align["detection rate"] = "r"
table.padding_width = 1
# Statistik Daten holen
ret = self.__database.getStatisticsAntivirus()
# Table-Body zusammenbauen
for av in ret:
table.add_row([av.get("product"), av.get("analyzed"), av.get("notanalyzed"), av.get("malware"), av.get("nonemalware"), av.get("rate")])
print(table)
print ""
class ViperHandler():
def __init__(self):
self.cfgReporting = Config(
os.path.join(RAGPICKER_ROOT, 'config', 'reporting.conf'))
self.viperEnabled = self.cfgReporting.getOption("viper", "enabled")
self.host = self.cfgReporting.getOption("viper", "host")
self.port = self.cfgReporting.getOption("viper", "port")
if not self.host or not self.port:
raise Exception("Viper REST API server not configurated")
def upload(self, filePath, fileName, tags):
rawFile = open(filePath, 'rb')
log.debug(VIPER_URL_ADD % (self.host, self.port) + " file=" + fileName)
try:
form = MultiPartForm()
form.add_file('file', fileName, fileHandle=rawFile)
form.add_field('tags', tags)
request = urllib2.Request(VIPER_URL_ADD % (self.host, self.port))
body = str(form)
request.add_header('Content-type', form.get_content_type())
request.add_header('Content-length', len(body))
request.add_data(body)
response_data = urllib2.urlopen(request, timeout=60).read()
reponsejson = json.loads(response_data)
log.info("Submitted to Viper, message: %s", reponsejson["message"])
except urllib2.URLError as e:
raise Exception(
"Unable to establish connection to Viper REST API server: %s" %
e)
except urllib2.HTTPError as e:
raise Exception(
"Unable to perform HTTP request to Viper REST API server (http code=%s)"
% e)
except ValueError as e:
raise Exception("Unable to convert response to JSON: %s" % e)
if reponsejson["message"] != 'added':
raise Exception("Failed to store file in Viper: %s" %
reponsejson["message"])
# Exports malware file from the Viper using the sha256-hash
def exportViper(self, sha256, exportDir):
if os.path.isfile(exportDir + sha256):
raise Exception("File %s already exists.")
cmd = "wget -q --tries=1 --directory-prefix=%s 'http://%s:%s/file/get/%s'" % (
exportDir, self.host, self.port, sha256)
os.system(cmd)
if not os.path.isfile(exportDir + sha256):
raise Exception("Download %s failed." % sha256)
def isFileInCage(self, md5=None, sha256=None):
if md5:
param = {'md5': md5}
elif sha256:
param = {'sha256': sha256}
param['project'] = 'default'
request_data = urllib.urlencode(param)
try:
request = urllib2.Request(VIPER_URL_FIND % (self.host, self.port),
request_data)
response = urllib2.urlopen(request, timeout=60)
response_data = response.read()
except urllib2.HTTPError as e:
if e.code == 400:
# Error: 404 Not Found
log.info('400 Invalid Search Term (' + str(param) + ')')
return False
else:
raise Exception(
"Unable to perform HTTP request to Viper (http code=%s)" %
e)
except urllib2.URLError as e:
raise Exception("Unable to establish connection to Viper: %s" % e)
try:
check = json.loads(response_data)
check = check['results']
except ValueError as e:
raise Exception("Unable to convert response to JSON: %s" % e)
if md5:
for i in check:
if str(i) == "../":
return False
if str(i) == "default":
for v in check[i]:
if 'md5' in v:
if v['md5'] == md5:
log.info("File " + md5 + " is in Viper")
return True
log.info("File " + sha256 + " is not in Viper")
return False
elif sha256:
for i in check:
if str(i) == "../":
return False
if str(i) == "default":
for v in check[i]:
log.info(v)
if 'sha256' in v:
if v['sha256'] == sha256:
log.info("File " + sha256 + " is in Viper")
return True
log.info("File " + sha256 + " is not in Viper")
return False
else:
raise Exception(
"isFileInCage() invoked without md5 or sha256. Can't do anything."
)
class UPSDModule_adminserv(UPSDModule):
modname = 'adminserv'
modperm = 'master'
def start(self):
if not UPSDModule.start(self):
return False
self.conf = Config()
self.userconf = {}
### Load info for pseudo client from config ###
self.userconf['nick'] = self.conf.getOption(self.modname, 'nick')
self.userconf['ident'] = self.conf.getOption(self.modname, 'ident')
self.userconf['host'] = self.conf.getOption(self.modname, 'host')
self.userconf['modes'] = self.conf.getOption(self.modname, 'modes')
self.userconf['realname'] = self.conf.getOption(self.modname, 'realname')
self.userconf['mainchan'] = self.conf.getOption(self.modname, 'mainchan')
self.proto.sendNickCreate(self.userconf['nick'], self.userconf['ident'], self.userconf['host'], self.userconf['modes'], self.userconf['realname'])
self.proto.sendJoin(self.userconf['nick'], self.userconf['mainchan'])
return True
def stop(self):
UPSDModule.stop(self)
self.proto.sendUserQuit(self.userconf['nick'])
def loadMod(self, destination, params):
if (self.perm.checkPerm(destination, self.modperm)):
if self.main.modhandler.loadModule(params[0]):
self.proto.sendNotice(self.userconf['nick'], destination, 'Loaded module %s!' % (params[0]))
else:
self.proto.sendNotice(self.userconf['nick'], destination, 'Unable to load module %s.' % (params[0]))
return True
def unloadMod(self, destination, params):
if (self.perm.checkPerm(destination, self.modperm)) & (params[0] != self.modname):
if self.main.modhandler.unloadModule(params[0]):
self.proto.sendNotice(self.userconf['nick'], destination, 'Unloaded module %s!' % (params[0]))
else:
self.proto.sendNotice(self.userconf['nick'], destination, 'Unable to unload module %s.' % (params[0]))
else:
self.proto.sendNotice(self.userconf['nick'], destination, 'You aren\'t allowed to use this command!')
return False
return True
def reload(self, destination, params=None):
if (self.perm.checkPerm(destination, self.modperm)):
self.proto.sendNotice(self.userconf['nick'], destination, 'Reloading %s module!' % (self.modname))
self.main.modhandler.reloadModule(self.modname)
else:
self.proto.sendNotice(self.userconf['nick'], destination, 'You aren\'t allowed to use this command!')
return False
return True
def sendHelp(self, destination, params=None):
if (self.perm.checkPerm(destination, self.modperm)):
self.proto.sendNotice(self.userconf['nick'], destination, '\x02AdminServ Commands\x02:')
self.proto.sendNotice(self.userconf['nick'], destination, '\x02LOADMOD <module>\x02 - Loads the specified module.')
self.proto.sendNotice(self.userconf['nick'], destination, '\x02UNLOADMOD <module>\x02 - Unloads the specified module.')
self.proto.sendNotice(self.userconf['nick'], destination, '\x02RELOAD\x02 - Reloads the AdminServ module.')
else:
self.proto.sendNotice(self.userconf['nick'], destination, 'You aren\'t allowed to use this command!')
return False
return True
def adminserv_ctcp(self, user, command, params):
if command.lower() == "\x01version\x01":
self.proto.sendNotice(self.userconf['nick'], user, '\x01VERSION %s\x01' % (self.getVersion()))
return True
def adminserv_quit(self, user, command, params):
print 'QUIT Detected!'
return True
def adminserv_channel(self, user, command, params):
nick = params[0].split(':')[1]
if (command == '.help'):
if (self.perm.checkPerm(nick, self.modperm)):
self.proto.sendMsg(self.userconf['nick'], user, 'Okay I\'ll help you ' + nick + '.')
self.sendHelp(nick)
elif (command == '.reload'):
if (self.perm.checkPerm(nick, self.modperm)):
self.proto.sendMsg(self.userconf['nick'], user, 'Okay, I\'ll only reload because I love you ' + nick + ".")
self.reload(nick)
return True
def getModCommands(self):
return (
('loadmod', {
'function': self.loadMod,
'usage': '\x02Syntax\x02: LOADMOD <module>'
}),
('unloadmod', {
'function': self.unloadMod,
'usage': '\x02Syntax\x02: UNLOADMOD <module>'
}),
('reload', {
'function': self.reload,
'usage': '\x02Syntax\x02: RELOAD'
}),
('help', {
'function': self.sendHelp,
'usage': '\x02Syntax\x02: HELP'
})
)
def getModHooks(self):
return (('quit', self.adminserv_quit),
('privmsg', self.adminserv_ctcp),
('privmsg', self.adminserv_channel))
def getVersion(self):
return 'UnrealPSD AdminServ Module'
class Database():
def __init__(self):
self.__cfgReporting = Config(os.path.join(RAGPICKER_ROOT, 'config', 'reporting.conf'))
self.__cfgProcessing = Config(os.path.join(RAGPICKER_ROOT, 'config', 'processing.conf'))
self.__mongodbEnabled = self.__cfgReporting.getOption("mongodb", "enabled")
self.__codedbEnabled = self.__cfgReporting.getOption("codeDB", "enabled")
self.__bluecoatEnabled = self.__cfgProcessing.getOption("all_bluecoatMalwareAnalysisAppliance", "enabled")
if self.__mongodbEnabled:
#Anbindung an Datenbank MongoDB Collection Ragpicker herstellen
try:
mongodbHost = self.__cfgReporting.getOption("mongodb", "host")
mongodbPort = self.__cfgReporting.getOption("mongodb", "port")
self.__mongodbConnection = MongoClient(mongodbHost, mongodbPort)
self.__mongodbCollectionRagpicker = self.__mongodbConnection.MalwareAnalyse.ragpicker
self.__mongodbCollectionFamilies = self.__mongodbConnection.MalwareAnalyse.families
self.__mongodbCollectionSandboxTaskQueue = self.__mongodbConnection.MalwareAnalyse.sandboxTaskQueue
except TypeError:
raise Exception("MongoDB connection port in report.config must be integer")
except ConnectionFailure:
raise Exception("Cannot connect to MongoDB (ragpicker)")
if self.__codedbEnabled:
#Anbindung an Datenbank MongoDB Collection CodeDB herstellen
try:
codedbHost = self.__cfgReporting.getOption("codeDB", "mongo_db_host")
codedbPort = self.__cfgReporting.getOption("codeDB", "mongo_db_port")
self.__codedbConnection = MongoClient(codedbHost, codedbPort)
self.__codedbCollectionCodedb = self.__codedbConnection.MalwareAnalyse.codeDB
except TypeError:
raise Exception("MongoDB connection port for CodeDB in report.config must be integer")
except ConnectionFailure:
raise Exception("Cannot connect to MongoDB (codeDB)")
def __del__(self):
if self.__mongodbEnabled:
self.__mongodbConnection.disconnect()
if self.__codedbEnabled:
self.__codedbConnection.disconnect()
# ------------------------------------------------------------------------------
# Ragpicker Database (MongoDB)
# ------------------------------------------------------------------------------
def isRagpickerDBEnabled(self):
return self.__mongodbEnabled
def getStatisticsAntivirus(self):
queries = []
ret = []
queries.append({"product" : "Avast Antivirus", "findStr1" : "AntivirusScanAvast", "findStr2" : "AntivirusScanAvast.avast", "ok" : "OK"})
queries.append({"product" : "AVG Antivirus", "findStr1" : "AntivirusScanAvg", "findStr2" : "AntivirusScanAvg.Avg", "ok" : "OK"})
queries.append({"product" : "Avira", "findStr1" : "AntivirusScanAvira", "findStr2" : "AntivirusScanAvira.Avira.scan", "ok" : "OK"})
queries.append({"product" : "BitDefender", "findStr1" : "AntivirusScanBitDefender", "findStr2" : "AntivirusScanBitDefender.BitDefender", "ok" : "OK"})
queries.append({"product" : "ClamAV", "findStr1" : "AntivirusScanClamAv", "findStr2" : "AntivirusScanClamAv.ClamAv", "ok" : " OK"})
queries.append({"product" : "COMODO", "findStr1" : "AntivirusScanCOMODO", "findStr2" : "AntivirusScanCOMODO.COMODO", "ok" : "OK"})
queries.append({"product" : "ESET", "findStr1" : "AntivirusScanESET", "findStr2" : "AntivirusScanESET.ESET", "ok" : "OK"})
queries.append({"product" : "F-Prot", "findStr1" : "AntivirusScanFProt", "findStr2" : "AntivirusScanFProt.FProt", "ok" : "OK"})
queries.append({"product" : "F-Secure", "findStr1" : "AntivirusScanF-Secure", "findStr2" : "AntivirusScanF-Secure.F-Secure", "ok" : "OK"})
for q in queries:
av = {}
av["product"] = q.get("product")
av["analyzed"] = str(self.__mongodbCollectionRagpicker.find({q.get("findStr1"): {"$ne":None}}).count())
av["notanalyzed"] = str(self.__mongodbCollectionRagpicker.find({q.get("findStr1") : None}).count())
av["malware"] = str(self.__mongodbCollectionRagpicker.find({ "$and": [{q.get("findStr1") : {"$ne":None}}, {q.get("findStr2") : {"$ne": q.get("ok")}}]}).count())
av["nonemalware"] = str(self.__mongodbCollectionRagpicker.find({q.get("findStr2"): q.get("ok")}).count())
if av.get("analyzed") != "0":
av["rate"] = "{:.2f} %".format((float(av.get("malware"))/float(av.get("analyzed"))*100))
else:
av["rate"] = "--"
ret.append(av)
return ret
def getStatisticsNoneMalwareByAV(self):
return self.__mongodbCollectionRagpicker.find({ "$and": [{ "$or": [{"AntivirusScanAvast.avast" : "OK"}, {"AntivirusScanAvast" : None}]},
{ "$or": [{"AntivirusScanAvg.Avg" : "OK"}, {"AntivirusScanAvg" : None}]},
{ "$or": [{"AntivirusScanAvira.Avira.scan" : "OK"}, {"AntivirusScanAvira" : None}]},
{ "$or": [{"AntivirusScanBitDefender.BitDefender" : "OK"}, {"AntivirusScanBitDefender" : None}]},
{ "$or": [{"AntivirusScanClamAv.ClamAv" : " OK"}, {"AntivirusScanClamAv" : None}]},
{ "$or": [{"AntivirusScanCOMODO.COMODO" : "OK"}, {"AntivirusScanCOMODO" : None}]},
{ "$or": [{"AntivirusScanESET.ESET" : "OK"}, {"AntivirusScanESET" : None}]},
{ "$or": [{"AntivirusScanFProt.FProt" : "OK"}, {"AntivirusScanFProt" : None}]},
{ "$or": [{"AntivirusScanF-Secure.F-Secure" : "OK"}, {"AntivirusScanF-Secure" : None}]},
{"VirusTotal.file.verbose_msg" : {"$ne":None}}]}).count()
def getSamplesNotFoundByAV(self):
return self.__mongodbCollectionRagpicker.find({ "$and": [{ "$or": [{"AntivirusScanAvast.avast" : "OK"}, {"AntivirusScanAvast" : None}]},
{ "$or": [{"AntivirusScanAvg.Avg" : "OK"}, {"AntivirusScanAvg" : None}]},
{ "$or": [{"AntivirusScanAvira.Avira.scan" : "OK"}, {"AntivirusScanAvira" : None}]},
{ "$or": [{"AntivirusScanBitDefender.BitDefender" : "OK"}, {"AntivirusScanBitDefender" : None}]},
{ "$or": [{"AntivirusScanClamAv.ClamAv" : " OK"}, {"AntivirusScanClamAv" : None}]},
{ "$or": [{"AntivirusScanCOMODO.COMODO" : "OK"}, {"AntivirusScanCOMODO" : None}]},
{ "$or": [{"AntivirusScanESET.ESET" : "OK"}, {"AntivirusScanESET" : None}]},
{ "$or": [{"AntivirusScanFProt.FProt" : "OK"}, {"AntivirusScanFProt" : None}]},
{ "$or": [{"AntivirusScanF-Secure.F-Secure" : "OK"}, {"AntivirusScanF-Secure" : None}]},
{"VirusTotal.file.verbose_msg" : {"$ne":None}}
]}, {"Info.file.sha256": True, "Info.analyse.started": True })
def getSamplesNotFoundByVT(self):
return self.__mongodbCollectionRagpicker.find({"VirusTotal.file.verbose_msg" : {"$ne":None}}, {"Info.file.sha256": True, "Info.analyse.started": True })
def getSamplesNotFoundByLocalAV(self):
return self.__mongodbCollectionRagpicker.find({ "$and": [{ "$or": [{"AntivirusScanAvast.avast" : "OK"}, {"AntivirusScanAvast" : None}]},
{ "$or": [{"AntivirusScanAvg.Avg" : "OK"}, {"AntivirusScanAvg" : None}]},
{ "$or": [{"AntivirusScanAvira.Avira.scan" : "OK"}, {"AntivirusScanAvira" : None}]},
{ "$or": [{"AntivirusScanBitDefender.BitDefender" : "OK"}, {"AntivirusScanBitDefender" : None}]},
{ "$or": [{"AntivirusScanClamAv.ClamAv" : " OK"}, {"AntivirusScanClamAv" : None}]},
{ "$or": [{"AntivirusScanCOMODO.COMODO" : "OK"}, {"AntivirusScanCOMODO" : None}]},
{ "$or": [{"AntivirusScanESET.ESET" : "OK"}, {"AntivirusScanESET" : None}]},
{ "$or": [{"AntivirusScanFProt.FProt" : "OK"}, {"AntivirusScanFProt" : None}]},
{ "$or": [{"AntivirusScanF-Secure.F-Secure" : "OK"}, {"AntivirusScanF-Secure" : None}]}
]}, {"Info.file.sha256": True, "Info.analyse.started": True })
def getStatisticsVirusTotal(self):
ret = {}
ret["analyzed"] = self.__mongodbCollectionRagpicker.find({"VirusTotal" : {"$ne":None}}).count()
ret["notAnalyzed"] = self.__mongodbCollectionRagpicker.find({"VirusTotal" : None}).count()
ret["samplesFound"] = self.__mongodbCollectionRagpicker.find({"VirusTotal.file.positives" : {"$ne":None}}).count()
ret["SamplesNotFound"] = self.__mongodbCollectionRagpicker.find({"VirusTotal.file.verbose_msg" : {"$ne":None}}).count()
return ret
def getStatisticsPackerSignatures(self):
return self.__mongodbCollectionRagpicker.aggregate([{ '$group' : {'_id' : { 'PublisherO': "$VerifySigs.PublisherO", 'Issuer': "$VerifySigs.Issuer" }, 'count' : { '$sum': 1 }}},{'$sort':{"count": -1}}])
def getStatisticsPackerCompiler(self):
return self.__mongodbCollectionRagpicker.aggregate([{ '$group' : {'_id' : '$PEID', 'count' : { '$sum': 1 } }}, {'$sort':{'count': -1}}])
def getStatisticsPeCharacteristics(self):
ret = {}
ret["exe"] = str(self.__mongodbCollectionRagpicker.find({ "$and": [{"Info.file.EXE" : True}, {"Info.file.DLL" : False}, {"Info.file.DRIVER" : False}]}).count())
ret["dll"] = str(self.__mongodbCollectionRagpicker.find({ "$and": [{"Info.file.EXE" : False}, {"Info.file.DLL" : True}, {"Info.file.DRIVER" : False}]}).count())
ret["driver"] = str(self.__mongodbCollectionRagpicker.find({ "$and": [{"Info.file.EXE" : False}, {"Info.file.DLL" : False}, {"Info.file.DRIVER" : True}]}).count())
ret["noPe"] = str(self.__mongodbCollectionRagpicker.find({ "$and": [{"Info.file.EXE" : None}, {"Info.file.DLL" : None}, {"Info.file.DRIVER" : None}]}).count())
ret["dllDriver"] = str(self.__mongodbCollectionRagpicker.find({ "$and": [{"Info.file.EXE" : False}, {"Info.file.DLL" : True}, {"Info.file.DRIVER" : True}]}).count())
return ret
def getFiletypes(self):
return self.__mongodbCollectionRagpicker.aggregate([{ '$group' : {'_id' : '$Info.file.type', 'count' : { '$sum': 1 } }}, {'$sort':{'count': -1}}])
def countReportsRagpickerDB(self):
return self.__mongodbCollectionRagpicker.find().count()
def iterateRagpickerReports(self, sha256):
for report in self.__mongodbCollectionRagpicker.find({'Info.file.sha256' : sha256}, {"_id" : 0}):
yield report
# Attention deletes the whole Ragpicker-Database!!!
# returns number of deleted reports
def deleteRagpickerDB(self):
count = self.__mongodbCollectionRagpicker.find().count()
# Alle Ragpicker-Daten aus der MongoDB loeschen
self.__mongodbCollectionRagpicker.remove()
return count
#Insert Ragpicker-Report in MongoDB
def insertRagpickerDB(self, report):
# Store the report
try:
self.__mongodbCollectionRagpicker.insert(report)
except InvalidDocument as e:
log.exception("Error InvalidDocument: %s", report)
raise Exception("Error InvalidDocument: {0}".format(e))
except InvalidStringData:
self.__mongodbCollectionRagpicker.insert(convertDirtyDict2ASCII(report))
#Count Ragpicker-Reports by file (and url)
def countRagpickerDB(self, file_md5, url_md5=None):
if url_md5:
query = { "$and" : [{ "Info.url.md5": { "$in": [url_md5] } }, { "Info.file.md5": { "$in": [file_md5] } }]}
else:
query = { "$and" : [{ "Info.file.md5": { "$in": [file_md5] } }]}
return self.__mongodbCollectionRagpicker.find(query).count()
# ------------------------------------------------------------------------------
# Ragpicker SandboxTaskQueue database (MongoDB)
# ------------------------------------------------------------------------------
def insertSandboxTaskStatus(self, sandboxName, sha256, taskID, sampleID, taskState=None):
statusReport = {"sandbox":sandboxName, "sha256":sha256, "sample_id":sampleID,
"task_id":taskID, "task_state":taskState}
# Store the SandboxTaskQueue-Status-report
self.__mongodbCollectionSandboxTaskQueue.insert(statusReport)
# Attention deletes the whole Ragpicker-SandboxTaskQueue-Database!!!
# returns number of deleted reports
def deleteSandboxTaskQueueDB(self):
count = self.__mongodbCollectionSandboxTaskQueue.find().count()
# Alle Daten aus der MongoDB loeschen
self.__mongodbCollectionSandboxTaskQueue.remove()
return count
# ------------------------------------------------------------------------------
# Ragpicker families database (MongoDB)
# ------------------------------------------------------------------------------
def insertFamily(self, familyReport):
# Store the family-report
self.__mongodbCollectionFamilies.insert(familyReport)
#Count Ragpicker-Reports by file (and url)
def countFamilyDB(self, parentObjectSHA256):
query = { "$and" : [{ "parentObjectSHA256": { "$in": [parentObjectSHA256] } }]}
return self.__mongodbCollectionFamilies.find(query).count()
def iterateFamilyReports(self, sha256):
for report in self.__mongodbCollectionFamilies.find({'parentObjectSHA256' : sha256}, {"_id" : 0}):
yield report
# Attention deletes the whole Ragpicker-Family-Database!!!
# returns number of deleted reports
def deleteFamilyDB(self):
count = self.__mongodbCollectionFamilies.find().count()
# Alle Ragpicker-Daten aus der MongoDB loeschen
self.__mongodbCollectionFamilies.remove()
return count
class Statistics():
def __init__(self):
# Datenbank
self.__database = Database()
# Kofiguration aus der reporting.conf holen
self.__cfgReporting = Config(
os.path.join(RAGPICKER_ROOT, 'config', 'reporting.conf'))
self.__vxcageEnabled = self.__cfgReporting.getOption(
"vxcage", "enabled")
self.__vxcageHost = self.__cfgReporting.getOption("vxcage", "host")
self.__vxcagePort = self.__cfgReporting.getOption("vxcage", "port")
def runStatisticsLong(self):
#Pruefen ob VxCage und MongoDB aktiviert sind
if self.__database.isRagpickerDBEnabled():
if self.__vxcageEnabled:
self.__runStatisticsMongodbLong()
if self.__database.isCodeDBEnabled():
self.__runStatisticsCodedb()
else:
print("vxcage in reporting.conf is not enabled")
sys.stdout.flush()
else:
print("mongodb in reporting.conf is not enabled")
sys.stdout.flush()
def runStatisticsShort(self):
#Pruefen ob VxCage und MongoDB aktiviert sind
if self.__database.isRagpickerDBEnabled():
if self.__vxcageEnabled:
self.__runStatisticsMongodbShort()
if self.__database.isCodeDBEnabled():
self.__runStatisticsCodedb()
else:
print("vxcage in reporting.conf is not enabled")
sys.stdout.flush()
else:
print("mongodb in reporting.conf is not enabled")
sys.stdout.flush()
def runStatisticsAV(self):
#Pruefen ob VxCage und MongoDB aktiviert sind
if self.__database.isRagpickerDBEnabled():
if self.__vxcageEnabled:
self.__runStatisticsAV()
else:
print("vxcage in reporting.conf is not enabled")
sys.stdout.flush()
else:
print("mongodb in reporting.conf is not enabled")
sys.stdout.flush()
def __runStatisticsMongodbLong(self):
print "**************************************"
print "*** Statistics MongoDB (Ragpicker) ***"
print "**************************************"
print ""
print "Number of malware samples in database:", self.__database.countReportsRagpickerDB(
)
print ""
#Statistiken der eingesetzten AV-Produkte
self.__runStatisticsAVProducts()
#Liste der letzen 20 Samples, die weder auf VT noch von einem lokalen AV gefunden wurden
self.__runStatisticsLast20SamplesNotFoundByAV()
#Liste der letzen 20 Samples, die nicht auf VT gefunden wurden
self.__runStatisticsLast20SamplesNotFoundByVT()
#Liste der letzen 20 Samples, die nicht von einem lokalen AV-Produkt gefunden wurden
self.__runStatisticsLast20SamplesNotFoundByLocalAV()
#Liste und Haeufigkeit der Filetypes
self.__runStatisticsFiletypes()
#Haeufigkeit der PE Charakteristiken
self.__runStatisticsPeCharacteristics()
#Liste und Haeufigkeit der verwendeten Packer/Compiler in der Malware
self.__runStatisticsPackerCompiler()
#Liste der verwendeten digitalen Signaturen
self.__runStatisticsPackerSignatures()
sys.stdout.flush()
def __runStatisticsMongodbShort(self):
print "**************************************"
print "*** Statistics MongoDB (Ragpicker) ***"
print "**************************************"
print ""
print "Number of malware samples in database:", self.__database.countReportsRagpickerDB(
)
print ""
#Liste und Haeufigkeit der Filetypes
self.__runStatisticsFiletypes()
#Haeufigkeit der PE Charakteristiken
self.__runStatisticsPeCharacteristics()
sys.stdout.flush()
def __runStatisticsAV(self):
print "**************************************"
print "*** Statistics MongoDB (Ragpicker) ***"
print "**************************************"
print ""
print "Number of malware samples in database:", self.__database.countReportsRagpickerDB(
)
print ""
#Statistiken der eingesetzten AV-Produkte
self.__runStatisticsAVProducts()
#Liste der letzen 20 Samples, die weder auf VT noch von einem lokalen AV gefunden wurden
self.__runStatisticsLast20SamplesNotFoundByAV()
#Liste der letzen 20 Samples, die nicht auf VT gefunden wurden
self.__runStatisticsLast20SamplesNotFoundByVT()
#Liste der letzen 20 Samples, die nicht von einem lokalen AV-Produkt gefunden wurden
self.__runStatisticsLast20SamplesNotFoundByLocalAV()
sys.stdout.flush()
def __runStatisticsCodedb(self):
print "***********************************"
print "*** Statistics MongoDB (CodeDB) ***"
print "***********************************"
print ""
print "Number of malware samples in database:", self.__database.countReportsCodeDB(
)
print ""
sys.stdout.flush()
def __runStatisticsFiletypes(self):
#Liste und Haeufigkeit der Filetypes
print "Filetypes of malware"
res = self.__database.getFiletypes()
table = PrettyTable(["filetype", "count"])
table.align["filetype"] = "l"
table.align["count"] = "c"
table.padding_width = 1
try:
for values in res['result']:
if values.get("_id"):
outputPacker = values.get("_id")
outputCount = str(values.get("count"))
table.add_row([outputPacker, outputCount])
print(table)
except KeyError:
raise Exception("Dict has no key 'result' ")
print ""
def __runStatisticsPeCharacteristics(self):
#Haeufigkeit der PE Charakteristiken
print "PE-Characteristics of malware"
peC = self.__database.getStatisticsPeCharacteristics()
table = PrettyTable(["pe-characteristics", "count"])
table.align["pe-characteristics"] = "l"
table.align["count"] = "c"
table.padding_width = 1
table.add_row(["EXE", peC.get("exe")])
table.add_row(["DLL", peC.get("dll")])
table.add_row(["Driver", peC.get("driver")])
table.add_row(["DLL/Driver", peC.get("dllDriver")])
table.add_row(["No PE File", peC.get("noPe")])
print(table)
print ""
def __runStatisticsPackerCompiler(self):
#Liste und Haeufigkeit der verwendeten Packer/Compiler in der Malware
print "Packer/compiler used in malware"
res = self.__database.getStatisticsPackerCompiler()
table = PrettyTable(["packer/compiler", "count"])
table.align["packer/compiler"] = "l"
table.align["count"] = "c"
table.padding_width = 1
try:
for values in res['result']:
if values.get("_id"):
outputPacker = values.get("_id")[0]
outputCount = str(values.get("count"))
table.add_row([outputPacker, outputCount])
print(table)
except KeyError:
raise Exception("Dict has no key 'result' ")
print " "
def __runStatisticsPackerSignatures(self):
#Liste der verwendeten digitalen Signaturen
print "Signatures used by malware"
res = self.__database.getStatisticsPackerSignatures()
table = PrettyTable(["publisher", "issuer", "count"])
table.align["publisher"] = "l"
table.align["issuer"] = "l"
table.align["count"] = "c"
table.padding_width = 1
try:
for values in res['result']:
if values.get("_id"):
outputPublisher = values.get("_id").get("PublisherO")
if values.get("_id").get("Issuer"):
outputIssuer = values.get("_id").get("Issuer")
else:
outputIssuer = " "
outputCount = str(values.get("count"))
table.add_row([outputPublisher, outputIssuer, outputCount])
print(table)
except KeyError:
raise Exception("Dict has no key 'result' ")
print ""
def __runStatisticsLast20SamplesNotFoundByAV(self):
#Liste der letzen 20 Samples, die weder auf VT noch von einem lokalen AV gefunden wurden
print "Last 20 samples not found by VirusTotal and local AV-Products"
res = sorted(self.__database.getSamplesNotFoundByAV(), reverse=True)
table = PrettyTable(["timestamp of crawling", "sha256"])
table.align["timestamp of crawling"] = "c"
table.align["sha256"] = "c"
table.padding_width = 1
try:
for values in res:
sha256 = values.get("Info").get("file").get("sha256")
timestamp = values.get("Info").get("analyse").get("started")
table.add_row([timestamp, sha256])
print(table.get_string(start=0, end=20))
except KeyError:
raise Exception("Dict has no key 'Info' ")
print ""
def __runStatisticsLast20SamplesNotFoundByVT(self):
#Liste der letzen 20 Samples, die nicht auf VirusTotal gefunden wurden
print "Last 20 samples not found by VirusTotal"
res = sorted(self.__database.getSamplesNotFoundByVT(), reverse=True)
table = PrettyTable(["timestamp of crawling", "sha256"])
table.align["timestamp of crawling"] = "c"
table.align["sha256"] = "c"
table.padding_width = 1
try:
for values in res:
sha256 = values.get("Info").get("file").get("sha256")
timestamp = values.get("Info").get("analyse").get("started")
table.add_row([timestamp, sha256])
print(table.get_string(start=0, end=20))
except KeyError:
raise Exception("Dict has no key 'Info' ")
print ""
def __runStatisticsLast20SamplesNotFoundByLocalAV(self):
#Liste der letzen 20 Samples, die nicht von einem lokalen AV-Produkt gefunden wurden
print "Last 20 samples not found by local AV-Products"
res = sorted(self.__database.getSamplesNotFoundByLocalAV(),
reverse=True)
table = PrettyTable(["timestamp of crawling", "sha256"])
table.align["timestamp of crawling"] = "c"
table.align["sha256"] = "c"
table.padding_width = 1
try:
for values in res:
sha256 = values.get("Info").get("file").get("sha256")
timestamp = values.get("Info").get("analyse").get("started")
table.add_row([timestamp, sha256])
print(table.get_string(start=0, end=20))
except KeyError:
raise Exception("Dict has no key 'Info' ")
print ""
def __runStatisticsAVProducts(self):
#Statistiken der eingesetzten AV-Produkte
#VirusTotal und lokale AV-Produkte
print "VirusTotal and local AV-Products"
print " Samples rated as none-malware by all AV-Products at time of crawling:", \
self.__database.getStatisticsNoneMalwareByAV()
print ""
#VirusTotal
ret = self.__database.getStatisticsVirusTotal()
print "VirusTotal"
print " Samples analyzed at time of crawling:", ret.get("analyzed")
print " Samples not analyzed at time of crawling:", ret.get(
"notAnalyzed")
print " Samples found at time of crawling:", ret.get("samplesFound")
print " Samples not found at time of crawling:", ret.get(
"SamplesNotFound")
print ""
#Lokale AV-Produkte
print "Local AV-Products"
print " analyzed => Samples analyzed at time of crawling"
print " not analyzed => Samples not analyzed at time of crawling"
print " malware => Samples rated as malware at time of crawling"
print " none-malware => Samples rated as none-malware at time of crawling"
table = PrettyTable([
"product", "analyzed", "not analyzed", "malware", "none-malware",
"detection rate"
])
table.align["product"] = "l"
table.align["analyzed"] = "r"
table.align["not analyzed"] = "r"
table.align["malware"] = "r"
table.align["none-malware"] = "r"
table.align["detection rate"] = "r"
table.padding_width = 1
# Statistik Daten holen
ret = self.__database.getStatisticsAntivirus()
# Table-Body zusammenbauen
for av in ret:
table.add_row([
av.get("product"),
av.get("analyzed"),
av.get("notanalyzed"),
av.get("malware"),
av.get("nonemalware"),
av.get("rate")
])
print(table)
print ""
class VxCageHandler():
def __init__(self):
self.cfgReporting = Config(os.path.join(RAGPICKER_ROOT, 'config', 'reporting.conf'))
self.vxcageEnabled = self.cfgReporting.getOption("vxcage", "enabled")
self.host = self.cfgReporting.getOption("vxcage", "host")
self.port = self.cfgReporting.getOption("vxcage", "port")
if not self.host or not self.port:
raise Exception("VxCage REST API server not configurated")
def upload(self, filePath, fileName, tags):
rawFile = open(filePath, 'rb')
log.debug(VXCAGE_URL_ADD % (self.host, self.port) + " file=" + fileName)
try:
form = MultiPartForm()
form.add_file('file', fileName, fileHandle=rawFile)
form.add_field('tags', tags)
request = urllib2.Request(VXCAGE_URL_ADD % (self.host, self.port))
body = str(form)
request.add_header('Content-type', form.get_content_type())
request.add_header('Content-length', len(body))
request.add_data(body)
response_data = urllib2.urlopen(request, timeout=60).read()
reponsejson = json.loads(response_data)
log.info("Submitted to vxcage, message: %s", reponsejson["message"])
except urllib2.URLError as e:
raise Exception("Unable to establish connection to VxCage REST API server: %s" % e)
except urllib2.HTTPError as e:
raise Exception("Unable to perform HTTP request to VxCage REST API server (http code=%s)" % e)
except ValueError as e:
raise Exception("Unable to convert response to JSON: %s" % e)
if reponsejson["message"] != 'added':
raise Exception("Failed to store file in VxCage: %s" % reponsejson["message"])
# Exports malware file from the VxCage using the sha256-hash
def exportVxCage(self, sha256, exportDir):
if os.path.isfile(exportDir + sha256):
raise Exception("File %s already exists.")
cmd = "wget -q --tries=1 --directory-prefix=%s 'http://%s:%s/malware/get/%s'" % (exportDir, self.host, self.port, sha256)
os.system(cmd)
if not os.path.isfile(exportDir + sha256):
raise Exception("Download %s failed." % sha256)
def isFileInCage(self, md5=None, sha256=None):
if md5:
param = { 'md5': md5 }
elif sha256:
param = { 'sha256': sha256 }
request_data = urllib.urlencode(param)
try:
request = urllib2.Request(VXCAGE_URL_FIND % (self.host, self.port), request_data)
response = urllib2.urlopen(request, timeout=60)
response_data = response.read()
except urllib2.HTTPError as e:
if e.code == 404:
# Error: 404 Not Found
log.info('404 Not Found (' + str(param) + ')')
return False
else:
raise Exception("Unable to perform HTTP request to VxCage (http code=%s)" % e)
except urllib2.URLError as e:
raise Exception("Unable to establish connection to VxCage: %s" % e)
try:
check = json.loads(response_data)
except ValueError as e:
raise Exception("Unable to convert response to JSON: %s" % e)
if md5 and check["md5"] == md5:
log.info("File " + md5 + " is in VxCage")
return True
elif sha256 and check["sha256"] == sha256:
log.info("File " + sha256 + " is in VxCage")
return True
return False
h = httplib2.Http(".cache", disable_ssl_certificate_validation=True)
response, content = h.request(CODE_DB_URL_TAG % (config.get("host"),
config.get("port")), "POST", body=urlencode(data), headers=headers)
if not "'status': '200'" in str(response) :
log.error("%s --> %s = %s" % (sha256, tags, str(content)))
data = json.loads(content)
log.info("%s --> %s = %s" % (sha256, tags, data.get("Status")))
if __name__ == '__main__':
# Datenbank
database = Database()
# VxCage-Handler
vxCage = VxCageHandler()
vxcageEnabled = cfgReporting.getOption("vxcage", "enabled")
parser = argparse.ArgumentParser(description='Ragpicker Manager')
subparsers = parser.add_subparsers(title='subcommands', description='valid subcommands', help='additional help')
parser_stop = subparsers.add_parser('stop', help='Stops a running Ragpicker instance')
parser_stop.set_defaults(which='stop')
parser_export = subparsers.add_parser('export', help='Export Ragpicker-Data')
parser_export.set_defaults(which='export')
parser_export.add_argument('-d','--dirname', required=True, help='Export-Directory')
parser_export.add_argument('-f','--sha256_file', required=True, help='SHA256-File')
parser_export.add_argument('--json', default=False, help='File in json-format? Default=False')
parser_vxcage = subparsers.add_parser('vxcage', help='Exports only the malware files from the VxCage')
parser_vxcage.set_defaults(which='vxcage')
parser_vxcage.add_argument('-d','--dirname', required=True, help='Export-Directory')
parser_vxcage.add_argument('-f','--sha256_file', required=True, help='SHA256-File')
parser_vxcage.add_argument('--json', default=False, help='File in json-format? Default=False')
if isJson:
data = json.load(file)
for sha256 in data.itervalues():
yield sha256.rstrip()
else:
for sha256 in file:
yield sha256.rstrip()
file.close()
if __name__ == '__main__':
# Datenbank
database = Database()
# VxCage-Handler
vxCage = VxCageHandler()
vxcageEnabled = cfgReporting.getOption("vxcage", "enabled")
parser = argparse.ArgumentParser(description='Ragpicker Manager')
subparsers = parser.add_subparsers(title='subcommands', description='valid subcommands', help='additional help')
parser_stop = subparsers.add_parser('stop', help='Stops a running Ragpicker instance')
parser_stop.set_defaults(which='stop')
parser_export = subparsers.add_parser('export', help='Export Ragpicker-Data')
parser_export.set_defaults(which='export')
parser_export.add_argument('-d','--dirname', required=True, help='Export-Directory')
parser_export.add_argument('-f','--sha256_file', required=True, help='SHA256-File')
parser_export.add_argument('--json', default=False, help='File in json-format? Default=False')
parser_vxcage = subparsers.add_parser('vxcage', help='Exports only the malware files from the VxCage')
parser_vxcage.set_defaults(which='vxcage')
parser_vxcage.add_argument('-d','--dirname', required=True, help='Export-Directory')
parser_vxcage.add_argument('-f','--sha256_file', required=True, help='SHA256-File')
parser_vxcage.add_argument('--json', default=False, help='File in json-format? Default=False')
h = httplib2.Http(".cache", disable_ssl_certificate_validation=True)
response, content = h.request(CODE_DB_URL_TAG % (config.get("host"),
config.get("port")), "POST", body=urlencode(data), headers=headers)
if not "'status': '200'" in str(response) :
log.error("%s --> %s = %s" % (sha256, tags, str(content)))
data = json.loads(content)
log.info("%s --> %s = %s" % (sha256, tags, data.get("Status")))
if __name__ == '__main__':
# Datenbank
database = Database()
# Viper-Handler
viper = ViperHandler()
viperEnabled = cfgReporting.getOption("viper", "enabled")
parser = argparse.ArgumentParser(description='Ragpicker Manager')
subparsers = parser.add_subparsers(title='subcommands', description='valid subcommands', help='additional help')
parser_stop = subparsers.add_parser('stop', help='Stops a running Ragpicker instance')
parser_stop.set_defaults(which='stop')
parser_export = subparsers.add_parser('export', help='Export Ragpicker-Data')
parser_export.set_defaults(which='export')
parser_export.add_argument('-d','--dirname', required=True, help='Export-Directory')
parser_export.add_argument('-f','--sha256_file', required=True, help='SHA256-File')
parser_export.add_argument('--json', default=False, help='File in json-format? Default=False')
parser_viper = subparsers.add_parser('viper', help='Exports only the malware files from the Viper')
parser_viper.set_defaults(which='viper')
parser_viper.add_argument('-d','--dirname', required=True, help='Export-Directory')
parser_viper.add_argument('-f','--sha256_file', required=True, help='SHA256-File')
parser_viper.add_argument('--json', default=False, help='File in json-format? Default=False')
