class DatabaseMySQL(): def __init__(self): self.__cfgReporting = Config(os.path.join(RAGPICKER_ROOT, 'config', 'reporting.conf')) self.__mysqlEnabled = self.__cfgReporting.getOption("mysql", "enabled") if self.__mysqlEnabled: #Anbindung an Datenbank MySQL herstellen try: mysqlHost = self.__cfgReporting.getOption("mysql", "host") mysqlPort = self.__cfgReporting.getOption("mysql", "port") mysqlDatabase = self.__cfgReporting.getOption("mysql", "database") mysqlUser = self.__cfgReporting.getOption("mysql", "user") mysqlPassword = self.__cfgReporting.getOption("mysql", "password") self.__mysqlConnection = MySQLdb.Connect(host=mysqlHost, port=mysqlPort, db=mysqlDatabase, user=mysqlUser, passwd=mysqlPassword) except (Exception) as e: raise Exception("Cannot connect to MySQL (ragpicker): %s" % e) def __del__(self): if self.__mysqlEnabled: self.__mysqlConnection.close() # ------------------------------------------------------------------------------ # Ragpicker Database (MySQL) # ------------------------------------------------------------------------------ def isRagpickerDBEnabledMySQL(self): return self.__mysqlEnabled
class DatabaseMySQL(): def __init__(self): self.__cfgReporting = Config( os.path.join(RAGPICKER_ROOT, 'config', 'reporting.conf')) self.__mysqlEnabled = self.__cfgReporting.getOption("mysql", "enabled") if self.__mysqlEnabled: #Anbindung an Datenbank MySQL herstellen try: mysqlHost = self.__cfgReporting.getOption("mysql", "host") mysqlPort = self.__cfgReporting.getOption("mysql", "port") mysqlDatabase = self.__cfgReporting.getOption( "mysql", "database") mysqlUser = self.__cfgReporting.getOption("mysql", "user") mysqlPassword = self.__cfgReporting.getOption( "mysql", "password") self.__mysqlConnection = MySQLdb.Connect(host=mysqlHost, port=mysqlPort, db=mysqlDatabase, user=mysqlUser, passwd=mysqlPassword) except (Exception) as e: raise Exception("Cannot connect to MySQL (ragpicker): %s" % e) def __del__(self): if self.__mysqlEnabled: self.__mysqlConnection.close() # ------------------------------------------------------------------------------ # Ragpicker Database (MySQL) # ------------------------------------------------------------------------------ def isRagpickerDBEnabledMySQL(self): return self.__mysqlEnabled
class UPSDModule_testserv(UPSDModule): modname = 'testserv' def start(self): if not UPSDModule.start(self): return False self.conf = Config() self.userconf = {} ### Load info for pseudo client from config ### self.userconf['nick'] = self.conf.getOption(self.modname, 'nick') self.userconf['ident'] = self.conf.getOption(self.modname, 'ident') self.userconf['host'] = self.conf.getOption(self.modname, 'host') self.userconf['modes'] = self.conf.getOption(self.modname, 'modes') self.userconf['realname'] = self.conf.getOption(self.modname, 'realname') self.userconf['mainchan'] = self.conf.getOption(self.modname, 'mainchan') self.proto.sendNickCreate(self.userconf['nick'], self.userconf['ident'], self.userconf['host'], self.userconf['modes'], self.userconf['realname']) self.proto.sendJoin(self.userconf['nick'], self.userconf['mainchan']) return True def stop(self): UPSDModule.stop(self) self.proto.sendUserQuit(self.userconf['nick']) def testserv_ctcp(self, user, command, params): if command.lower() == "\x01version\x01": self.proto.sendNotice(self.userconf['nick'], user, '\x01VERSION %s\x01' % (self.getVersion())) return True def testserv_channel(self, user, command, params): nick = params[0].split(':')[1] if (len(params) > 4): args = params[4].split(' ') else: args = None if (command == ".kick"): self.proto.sendKick(self.userconf['nick'], user, args[0], args[1]) def getModHooks(self): return (('privmsg', self.testserv_ctcp), ('privmsg', self.testserv_channel)) def getVersion(self): return 'UnrealPSD TestServ Module'
class Database: def __init__(self): self.__cfgReporting = Config(os.path.join(RAGPICKER_ROOT, "config", "reporting.conf")) self.__cfgProcessing = Config(os.path.join(RAGPICKER_ROOT, "config", "processing.conf")) self.__mongodbEnabled = self.__cfgReporting.getOption("mongodb", "enabled") self.__codedbEnabled = self.__cfgReporting.getOption("codeDB", "enabled") self.__bluecoatEnabled = self.__cfgProcessing.getOption("all_bluecoatMalwareAnalysisAppliance", "enabled") if self.__mongodbEnabled: # Anbindung an Datenbank MongoDB Collection Ragpicker herstellen try: mongodbHost = self.__cfgReporting.getOption("mongodb", "host") mongodbPort = self.__cfgReporting.getOption("mongodb", "port") self.__mongodbConnection = MongoClient(mongodbHost, mongodbPort) self.__mongodbCollectionRagpicker = self.__mongodbConnection.MalwareAnalyse.ragpicker self.__mongodbCollectionFamilies = self.__mongodbConnection.MalwareAnalyse.families self.__mongodbCollectionSandboxTaskQueue = self.__mongodbConnection.MalwareAnalyse.sandboxTaskQueue except TypeError: raise Exception("MongoDB connection port in report.config must be integer") except ConnectionFailure: raise Exception("Cannot connect to MongoDB (ragpicker)") if self.__codedbEnabled: # Anbindung an Datenbank MongoDB Collection CodeDB herstellen try: codedbHost = self.__cfgReporting.getOption("codeDB", "mongo_db_host") codedbPort = self.__cfgReporting.getOption("codeDB", "mongo_db_port") self.__codedbConnection = MongoClient(codedbHost, codedbPort) self.__codedbCollectionCodedb = self.__codedbConnection.MalwareAnalyse.codeDB except TypeError: raise Exception("MongoDB connection port for CodeDB in report.config must be integer") except ConnectionFailure: raise Exception("Cannot connect to MongoDB (codeDB)") def __del__(self): if self.__mongodbEnabled: self.__mongodbConnection.disconnect() if self.__codedbEnabled: self.__codedbConnection.disconnect() # ------------------------------------------------------------------------------ # Ragpicker Database (MongoDB) # ------------------------------------------------------------------------------ def isRagpickerDBEnabled(self): return self.__mongodbEnabled def getStatisticsAntivirus(self): queries = [] ret = [] queries.append( { "product": "Avast Antivirus", "findStr1": "AntivirusScanAvast", "findStr2": "AntivirusScanAvast.avast", "ok": "OK", } ) queries.append( {"product": "AVG Antivirus", "findStr1": "AntivirusScanAvg", "findStr2": "AntivirusScanAvg.Avg", "ok": "OK"} ) queries.append( { "product": "Avira", "findStr1": "AntivirusScanAvira", "findStr2": "AntivirusScanAvira.Avira.scan", "ok": "OK", } ) queries.append( { "product": "BitDefender", "findStr1": "AntivirusScanBitDefender", "findStr2": "AntivirusScanBitDefender.BitDefender", "ok": "OK", } ) queries.append( { "product": "ClamAV", "findStr1": "AntivirusScanClamAv", "findStr2": "AntivirusScanClamAv.ClamAv", "ok": " OK", } ) queries.append( { "product": "COMODO", "findStr1": "AntivirusScanCOMODO", "findStr2": "AntivirusScanCOMODO.COMODO", "ok": "OK", } ) queries.append( {"product": "ESET", "findStr1": "AntivirusScanESET", "findStr2": "AntivirusScanESET.ESET", "ok": "OK"} ) queries.append( {"product": "F-Prot", "findStr1": "AntivirusScanFProt", "findStr2": "AntivirusScanFProt.FProt", "ok": "OK"} ) queries.append( { "product": "F-Secure", "findStr1": "AntivirusScanF-Secure", "findStr2": "AntivirusScanF-Secure.F-Secure", "ok": "OK", } ) for q in queries: av = {} av["product"] = q.get("product") av["analyzed"] = str(self.__mongodbCollectionRagpicker.find({q.get("findStr1"): {"$ne": None}}).count()) av["notanalyzed"] = str(self.__mongodbCollectionRagpicker.find({q.get("findStr1"): None}).count()) av["malware"] = str( self.__mongodbCollectionRagpicker.find( {"$and": [{q.get("findStr1"): {"$ne": None}}, {q.get("findStr2"): {"$ne": q.get("ok")}}]} ).count() ) av["nonemalware"] = str(self.__mongodbCollectionRagpicker.find({q.get("findStr2"): q.get("ok")}).count()) if av.get("analyzed") != "0": av["rate"] = "{:.2f} %".format((float(av.get("malware")) / float(av.get("analyzed")) * 100)) else: av["rate"] = "--" ret.append(av) return ret def getStatisticsNoneMalwareByAV(self): return self.__mongodbCollectionRagpicker.find( { "$and": [ {"$or": [{"AntivirusScanAvast.avast": "OK"}, {"AntivirusScanAvast": None}]}, {"$or": [{"AntivirusScanAvg.Avg": "OK"}, {"AntivirusScanAvg": None}]}, {"$or": [{"AntivirusScanAvira.Avira.scan": "OK"}, {"AntivirusScanAvira": None}]}, {"$or": [{"AntivirusScanBitDefender.BitDefender": "OK"}, {"AntivirusScanBitDefender": None}]}, {"$or": [{"AntivirusScanClamAv.ClamAv": " OK"}, {"AntivirusScanClamAv": None}]}, {"$or": [{"AntivirusScanCOMODO.COMODO": "OK"}, {"AntivirusScanCOMODO": None}]}, {"$or": [{"AntivirusScanESET.ESET": "OK"}, {"AntivirusScanESET": None}]}, {"$or": [{"AntivirusScanFProt.FProt": "OK"}, {"AntivirusScanFProt": None}]}, {"$or": [{"AntivirusScanF-Secure.F-Secure": "OK"}, {"AntivirusScanF-Secure": None}]}, {"VirusTotal.file.verbose_msg": {"$ne": None}}, ] } ).count() def getSamplesNotFoundByAV(self): return self.__mongodbCollectionRagpicker.find( { "$and": [ {"$or": [{"AntivirusScanAvast.avast": "OK"}, {"AntivirusScanAvast": None}]}, {"$or": [{"AntivirusScanAvg.Avg": "OK"}, {"AntivirusScanAvg": None}]}, {"$or": [{"AntivirusScanAvira.Avira.scan": "OK"}, {"AntivirusScanAvira": None}]}, {"$or": [{"AntivirusScanBitDefender.BitDefender": "OK"}, {"AntivirusScanBitDefender": None}]}, {"$or": [{"AntivirusScanClamAv.ClamAv": " OK"}, {"AntivirusScanClamAv": None}]}, {"$or": [{"AntivirusScanCOMODO.COMODO": "OK"}, {"AntivirusScanCOMODO": None}]}, {"$or": [{"AntivirusScanESET.ESET": "OK"}, {"AntivirusScanESET": None}]}, {"$or": [{"AntivirusScanFProt.FProt": "OK"}, {"AntivirusScanFProt": None}]}, {"$or": [{"AntivirusScanF-Secure.F-Secure": "OK"}, {"AntivirusScanF-Secure": None}]}, {"VirusTotal.file.verbose_msg": {"$ne": None}}, ] }, {"Info.file.sha256": True, "Info.analyse.started": True}, ) def getSamplesNotFoundByVT(self): return self.__mongodbCollectionRagpicker.find( {"VirusTotal.file.verbose_msg": {"$ne": None}}, {"Info.file.sha256": True, "Info.analyse.started": True} ) def getSamplesNotFoundByLocalAV(self): return self.__mongodbCollectionRagpicker.find( { "$and": [ {"$or": [{"AntivirusScanAvast.avast": "OK"}, {"AntivirusScanAvast": None}]}, {"$or": [{"AntivirusScanAvg.Avg": "OK"}, {"AntivirusScanAvg": None}]}, {"$or": [{"AntivirusScanAvira.Avira.scan": "OK"}, {"AntivirusScanAvira": None}]}, {"$or": [{"AntivirusScanBitDefender.BitDefender": "OK"}, {"AntivirusScanBitDefender": None}]}, {"$or": [{"AntivirusScanClamAv.ClamAv": " OK"}, {"AntivirusScanClamAv": None}]}, {"$or": [{"AntivirusScanCOMODO.COMODO": "OK"}, {"AntivirusScanCOMODO": None}]}, {"$or": [{"AntivirusScanESET.ESET": "OK"}, {"AntivirusScanESET": None}]}, {"$or": [{"AntivirusScanFProt.FProt": "OK"}, {"AntivirusScanFProt": None}]}, {"$or": [{"AntivirusScanF-Secure.F-Secure": "OK"}, {"AntivirusScanF-Secure": None}]}, ] }, {"Info.file.sha256": True, "Info.analyse.started": True}, ) def getStatisticsVirusTotal(self): ret = {} ret["analyzed"] = self.__mongodbCollectionRagpicker.find({"VirusTotal": {"$ne": None}}).count() ret["notAnalyzed"] = self.__mongodbCollectionRagpicker.find({"VirusTotal": None}).count() ret["samplesFound"] = self.__mongodbCollectionRagpicker.find( {"VirusTotal.file.positives": {"$ne": None}} ).count() ret["SamplesNotFound"] = self.__mongodbCollectionRagpicker.find( {"VirusTotal.file.verbose_msg": {"$ne": None}} ).count() return ret def getStatisticsPackerSignatures(self): return self.__mongodbCollectionRagpicker.aggregate( [ { "$group": { "_id": {"PublisherO": "$VerifySigs.PublisherO", "Issuer": "$VerifySigs.Issuer"}, "count": {"$sum": 1}, } }, {"$sort": {"count": -1}}, ] ) def getStatisticsPackerCompiler(self): return self.__mongodbCollectionRagpicker.aggregate( [{"$group": {"_id": "$PEID", "count": {"$sum": 1}}}, {"$sort": {"count": -1}}] ) def getStatisticsPeCharacteristics(self): ret = {} ret["exe"] = str( self.__mongodbCollectionRagpicker.find( {"$and": [{"Info.file.EXE": True}, {"Info.file.DLL": False}, {"Info.file.DRIVER": False}]} ).count() ) ret["dll"] = str( self.__mongodbCollectionRagpicker.find( {"$and": [{"Info.file.EXE": False}, {"Info.file.DLL": True}, {"Info.file.DRIVER": False}]} ).count() ) ret["driver"] = str( self.__mongodbCollectionRagpicker.find( {"$and": [{"Info.file.EXE": False}, {"Info.file.DLL": False}, {"Info.file.DRIVER": True}]} ).count() ) ret["noPe"] = str( self.__mongodbCollectionRagpicker.find( {"$and": [{"Info.file.EXE": None}, {"Info.file.DLL": None}, {"Info.file.DRIVER": None}]} ).count() ) ret["dllDriver"] = str( self.__mongodbCollectionRagpicker.find( {"$and": [{"Info.file.EXE": False}, {"Info.file.DLL": True}, {"Info.file.DRIVER": True}]} ).count() ) return ret def getFiletypes(self): return self.__mongodbCollectionRagpicker.aggregate( [{"$group": {"_id": "$Info.file.type", "count": {"$sum": 1}}}, {"$sort": {"count": -1}}] ) def countReportsRagpickerDB(self): return self.__mongodbCollectionRagpicker.find().count() def iterateRagpickerReports(self, sha256): for report in self.__mongodbCollectionRagpicker.find({"Info.file.sha256": sha256}, {"_id": 0}): yield report # Attention deletes the whole Ragpicker-Database!!! # returns number of deleted reports def deleteRagpickerDB(self): count = self.__mongodbCollectionRagpicker.find().count() # Alle Ragpicker-Daten aus der MongoDB loeschen self.__mongodbCollectionRagpicker.remove() return count # Insert Ragpicker-Report in MongoDB def insertRagpickerDB(self, report): # Store the report try: self.__mongodbCollectionRagpicker.insert(report) except InvalidDocument as e: log.exception("Error InvalidDocument: %s", report) raise Exception("Error InvalidDocument: {0}".format(e)) except InvalidStringData: self.__mongodbCollectionRagpicker.insert(convertDirtyDict2ASCII(report)) # Count Ragpicker-Reports by file (and url) def countRagpickerDB(self, file_md5, url_md5=None): if url_md5: query = {"$and": [{"Info.url.md5": {"$in": [url_md5]}}, {"Info.file.md5": {"$in": [file_md5]}}]} else: query = {"$and": [{"Info.file.md5": {"$in": [file_md5]}}]} return self.__mongodbCollectionRagpicker.find(query).count() # ------------------------------------------------------------------------------ # Ragpicker SandboxTaskQueue database (MongoDB) # ------------------------------------------------------------------------------ def insertSandboxTaskStatus(self, sandboxName, sha256, taskID, sampleID, taskState=None): statusReport = { "sandbox": sandboxName, "sha256": sha256, "sample_id": sampleID, "task_id": taskID, "task_state": taskState, } # Store the SandboxTaskQueue-Status-report self.__mongodbCollectionSandboxTaskQueue.insert(statusReport) # Attention deletes the whole Ragpicker-SandboxTaskQueue-Database!!! # returns number of deleted reports def deleteSandboxTaskQueueDB(self): count = self.__mongodbCollectionSandboxTaskQueue.find().count() # Alle Daten aus der MongoDB loeschen self.__mongodbCollectionSandboxTaskQueue.remove() return count # ------------------------------------------------------------------------------ # Ragpicker families database (MongoDB) # ------------------------------------------------------------------------------ def insertFamily(self, familyReport): # Store the family-report self.__mongodbCollectionFamilies.insert(familyReport) # Count Ragpicker-Reports by file (and url) def countFamilyDB(self, parentObjectSHA256): query = {"$and": [{"parentObjectSHA256": {"$in": [parentObjectSHA256]}}]} return self.__mongodbCollectionFamilies.find(query).count() def iterateFamilyReports(self, sha256): for report in self.__mongodbCollectionFamilies.find({"parentObjectSHA256": sha256}, {"_id": 0}): yield report # Attention deletes the whole Ragpicker-Family-Database!!! # returns number of deleted reports def deleteFamilyDB(self): count = self.__mongodbCollectionFamilies.find().count() # Alle Ragpicker-Daten aus der MongoDB loeschen self.__mongodbCollectionFamilies.remove() return count
class Statistics(): def __init__(self): # Datenbank self.__database = Database() # Kofiguration aus der reporting.conf holen self.__cfgReporting = Config(os.path.join(RAGPICKER_ROOT, 'config', 'reporting.conf')) self.__vxcageEnabled = self.__cfgReporting.getOption("vxcage", "enabled") self.__vxcageHost = self.__cfgReporting.getOption("vxcage", "host") self.__vxcagePort = self.__cfgReporting.getOption("vxcage", "port") def runStatisticsLong(self): #Pruefen ob VxCage und MongoDB aktiviert sind if self.__database.isRagpickerDBEnabled(): if self.__vxcageEnabled: self.__runStatisticsMongodbLong() if self.__database.isCodeDBEnabled(): self.__runStatisticsCodedb() else: print("vxcage in reporting.conf is not enabled") sys.stdout.flush() else: print("mongodb in reporting.conf is not enabled") sys.stdout.flush() def runStatisticsShort(self): #Pruefen ob VxCage und MongoDB aktiviert sind if self.__database.isRagpickerDBEnabled(): if self.__vxcageEnabled: self.__runStatisticsMongodbShort() if self.__database.isCodeDBEnabled(): self.__runStatisticsCodedb() else: print("vxcage in reporting.conf is not enabled") sys.stdout.flush() else: print("mongodb in reporting.conf is not enabled") sys.stdout.flush() def runStatisticsAV(self): #Pruefen ob VxCage und MongoDB aktiviert sind if self.__database.isRagpickerDBEnabled(): if self.__vxcageEnabled: self.__runStatisticsAV() else: print("vxcage in reporting.conf is not enabled") sys.stdout.flush() else: print("mongodb in reporting.conf is not enabled") sys.stdout.flush() def __runStatisticsMongodbLong(self): print "**************************************" print "*** Statistics MongoDB (Ragpicker) ***" print "**************************************" print "" print "Number of malware samples in database:", self.__database.countReportsRagpickerDB() print "" #Statistiken der eingesetzten AV-Produkte self.__runStatisticsAVProducts() #Liste der letzen 20 Samples, die weder auf VT noch von einem lokalen AV gefunden wurden self.__runStatisticsLast20SamplesNotFoundByAV() #Liste der letzen 20 Samples, die nicht auf VT gefunden wurden self.__runStatisticsLast20SamplesNotFoundByVT() #Liste der letzen 20 Samples, die nicht von einem lokalen AV-Produkt gefunden wurden self.__runStatisticsLast20SamplesNotFoundByLocalAV() #Liste und Haeufigkeit der Filetypes self.__runStatisticsFiletypes() #Haeufigkeit der PE Charakteristiken self.__runStatisticsPeCharacteristics() #Liste und Haeufigkeit der verwendeten Packer/Compiler in der Malware self.__runStatisticsPackerCompiler() #Liste der verwendeten digitalen Signaturen self.__runStatisticsPackerSignatures() sys.stdout.flush() def __runStatisticsMongodbShort(self): print "**************************************" print "*** Statistics MongoDB (Ragpicker) ***" print "**************************************" print "" print "Number of malware samples in database:", self.__database.countReportsRagpickerDB() print "" #Liste und Haeufigkeit der Filetypes self.__runStatisticsFiletypes() #Haeufigkeit der PE Charakteristiken self.__runStatisticsPeCharacteristics() sys.stdout.flush() def __runStatisticsAV(self): print "**************************************" print "*** Statistics MongoDB (Ragpicker) ***" print "**************************************" print "" print "Number of malware samples in database:", self.__database.countReportsRagpickerDB() print "" #Statistiken der eingesetzten AV-Produkte self.__runStatisticsAVProducts() #Liste der letzen 20 Samples, die weder auf VT noch von einem lokalen AV gefunden wurden self.__runStatisticsLast20SamplesNotFoundByAV() #Liste der letzen 20 Samples, die nicht auf VT gefunden wurden self.__runStatisticsLast20SamplesNotFoundByVT() #Liste der letzen 20 Samples, die nicht von einem lokalen AV-Produkt gefunden wurden self.__runStatisticsLast20SamplesNotFoundByLocalAV() sys.stdout.flush() def __runStatisticsFiletypes(self): #Liste und Haeufigkeit der Filetypes print "Filetypes of malware" res = self.__database.getFiletypes() table = PrettyTable(["filetype", "count"]) table.align["filetype"] = "l" table.align["count"] = "c" table.padding_width = 1 try: for values in res['result']: if values.get("_id"): outputPacker = values.get("_id") outputCount = str(values.get("count")) table.add_row([outputPacker, outputCount]) print(table) except KeyError: raise Exception("Dict has no key 'result' ") print "" def __runStatisticsPeCharacteristics(self): #Haeufigkeit der PE Charakteristiken print "PE-Characteristics of malware" peC = self.__database.getStatisticsPeCharacteristics() table = PrettyTable(["pe-characteristics", "count"]) table.align["pe-characteristics"] = "l" table.align["count"] = "c" table.padding_width = 1 table.add_row(["EXE", peC.get("exe")]) table.add_row(["DLL", peC.get("dll")]) table.add_row(["Driver", peC.get("driver")]) table.add_row(["DLL/Driver", peC.get("dllDriver")]) table.add_row(["No PE File", peC.get("noPe")]) print (table) print "" def __runStatisticsPackerCompiler(self): #Liste und Haeufigkeit der verwendeten Packer/Compiler in der Malware print "Packer/compiler used in malware" res = self.__database.getStatisticsPackerCompiler() table = PrettyTable(["packer/compiler", "count"]) table.align["packer/compiler"] = "l" table.align["count"] = "c" table.padding_width = 1 try: for values in res['result']: if values.get("_id"): outputPacker = values.get("_id")[0] outputCount = str(values.get("count")) table.add_row([outputPacker, outputCount]) print(table) except KeyError: raise Exception("Dict has no key 'result' ") print " " def __runStatisticsPackerSignatures(self): #Liste der verwendeten digitalen Signaturen print "Signatures used by malware" res = self.__database.getStatisticsPackerSignatures() table = PrettyTable(["publisher", "issuer", "count"]) table.align["publisher"] = "l" table.align["issuer"] = "l" table.align["count"] = "c" table.padding_width = 1 try: for values in res['result']: if values.get("_id"): outputPublisher = values.get("_id").get("PublisherO") if values.get("_id").get("Issuer"): outputIssuer = values.get("_id").get("Issuer") else: outputIssuer = " " outputCount = str(values.get("count")) table.add_row([outputPublisher, outputIssuer, outputCount]) print(table) except KeyError: raise Exception("Dict has no key 'result' ") print "" def __runStatisticsLast20SamplesNotFoundByAV(self): #Liste der letzen 20 Samples, die weder auf VT noch von einem lokalen AV gefunden wurden print "Last 20 samples not found by VirusTotal and local AV-Products" res = sorted(self.__database.getSamplesNotFoundByAV(), reverse=True) table = PrettyTable(["timestamp of crawling", "sha256"]) table.align["timestamp of crawling"] = "c" table.align["sha256"] = "c" table.padding_width = 1 try: for values in res: sha256 = values.get("Info").get("file").get("sha256") timestamp = values.get("Info").get("analyse").get("started") table.add_row([timestamp, sha256]) print(table.get_string(start=0, end=20)) except KeyError: raise Exception("Dict has no key 'Info' ") print "" def __runStatisticsLast20SamplesNotFoundByVT(self): #Liste der letzen 20 Samples, die nicht auf VirusTotal gefunden wurden print "Last 20 samples not found by VirusTotal" res = sorted(self.__database.getSamplesNotFoundByVT(), reverse=True) table = PrettyTable(["timestamp of crawling", "sha256"]) table.align["timestamp of crawling"] = "c" table.align["sha256"] = "c" table.padding_width = 1 try: for values in res: sha256 = values.get("Info").get("file").get("sha256") timestamp = values.get("Info").get("analyse").get("started") table.add_row([timestamp, sha256]) print(table.get_string(start=0, end=20)) except KeyError: raise Exception("Dict has no key 'Info' ") print "" def __runStatisticsLast20SamplesNotFoundByLocalAV(self): #Liste der letzen 20 Samples, die nicht von einem lokalen AV-Produkt gefunden wurden print "Last 20 samples not found by local AV-Products" res = sorted(self.__database.getSamplesNotFoundByLocalAV(), reverse=True) table = PrettyTable(["timestamp of crawling", "sha256"]) table.align["timestamp of crawling"] = "c" table.align["sha256"] = "c" table.padding_width = 1 try: for values in res: sha256 = values.get("Info").get("file").get("sha256") timestamp = values.get("Info").get("analyse").get("started") table.add_row([timestamp, sha256]) print(table.get_string(start=0, end=20)) except KeyError: raise Exception("Dict has no key 'Info' ") print "" def __runStatisticsAVProducts(self): #Statistiken der eingesetzten AV-Produkte #VirusTotal und lokale AV-Produkte print "VirusTotal and local AV-Products" print " Samples rated as none-malware by all AV-Products at time of crawling:", \ self.__database.getStatisticsNoneMalwareByAV() print "" #VirusTotal ret = self.__database.getStatisticsVirusTotal() print "VirusTotal" print " Samples analyzed at time of crawling:", ret.get("analyzed") print " Samples not analyzed at time of crawling:", ret.get("notAnalyzed") print " Samples found at time of crawling:", ret.get("samplesFound") print " Samples not found at time of crawling:", ret.get("SamplesNotFound") print "" #Lokale AV-Produkte print "Local AV-Products" print " analyzed => Samples analyzed at time of crawling" print " not analyzed => Samples not analyzed at time of crawling" print " malware => Samples rated as malware at time of crawling" print " none-malware => Samples rated as none-malware at time of crawling" table = PrettyTable(["product", "analyzed", "not analyzed", "malware", "none-malware", "detection rate"]) table.align["product"] = "l" table.align["analyzed"] = "r" table.align["not analyzed"] = "r" table.align["malware"] = "r" table.align["none-malware"] = "r" table.align["detection rate"] = "r" table.padding_width = 1 # Statistik Daten holen ret = self.__database.getStatisticsAntivirus() # Table-Body zusammenbauen for av in ret: table.add_row([av.get("product"), av.get("analyzed"), av.get("notanalyzed"), av.get("malware"), av.get("nonemalware"), av.get("rate")]) print(table) print ""
class ViperHandler(): def __init__(self): self.cfgReporting = Config( os.path.join(RAGPICKER_ROOT, 'config', 'reporting.conf')) self.viperEnabled = self.cfgReporting.getOption("viper", "enabled") self.host = self.cfgReporting.getOption("viper", "host") self.port = self.cfgReporting.getOption("viper", "port") if not self.host or not self.port: raise Exception("Viper REST API server not configurated") def upload(self, filePath, fileName, tags): rawFile = open(filePath, 'rb') log.debug(VIPER_URL_ADD % (self.host, self.port) + " file=" + fileName) try: form = MultiPartForm() form.add_file('file', fileName, fileHandle=rawFile) form.add_field('tags', tags) request = urllib2.Request(VIPER_URL_ADD % (self.host, self.port)) body = str(form) request.add_header('Content-type', form.get_content_type()) request.add_header('Content-length', len(body)) request.add_data(body) response_data = urllib2.urlopen(request, timeout=60).read() reponsejson = json.loads(response_data) log.info("Submitted to Viper, message: %s", reponsejson["message"]) except urllib2.URLError as e: raise Exception( "Unable to establish connection to Viper REST API server: %s" % e) except urllib2.HTTPError as e: raise Exception( "Unable to perform HTTP request to Viper REST API server (http code=%s)" % e) except ValueError as e: raise Exception("Unable to convert response to JSON: %s" % e) if reponsejson["message"] != 'added': raise Exception("Failed to store file in Viper: %s" % reponsejson["message"]) # Exports malware file from the Viper using the sha256-hash def exportViper(self, sha256, exportDir): if os.path.isfile(exportDir + sha256): raise Exception("File %s already exists.") cmd = "wget -q --tries=1 --directory-prefix=%s 'http://%s:%s/file/get/%s'" % ( exportDir, self.host, self.port, sha256) os.system(cmd) if not os.path.isfile(exportDir + sha256): raise Exception("Download %s failed." % sha256) def isFileInCage(self, md5=None, sha256=None): if md5: param = {'md5': md5} elif sha256: param = {'sha256': sha256} param['project'] = 'default' request_data = urllib.urlencode(param) try: request = urllib2.Request(VIPER_URL_FIND % (self.host, self.port), request_data) response = urllib2.urlopen(request, timeout=60) response_data = response.read() except urllib2.HTTPError as e: if e.code == 400: # Error: 404 Not Found log.info('400 Invalid Search Term (' + str(param) + ')') return False else: raise Exception( "Unable to perform HTTP request to Viper (http code=%s)" % e) except urllib2.URLError as e: raise Exception("Unable to establish connection to Viper: %s" % e) try: check = json.loads(response_data) check = check['results'] except ValueError as e: raise Exception("Unable to convert response to JSON: %s" % e) if md5: for i in check: if str(i) == "../": return False if str(i) == "default": for v in check[i]: if 'md5' in v: if v['md5'] == md5: log.info("File " + md5 + " is in Viper") return True log.info("File " + sha256 + " is not in Viper") return False elif sha256: for i in check: if str(i) == "../": return False if str(i) == "default": for v in check[i]: log.info(v) if 'sha256' in v: if v['sha256'] == sha256: log.info("File " + sha256 + " is in Viper") return True log.info("File " + sha256 + " is not in Viper") return False else: raise Exception( "isFileInCage() invoked without md5 or sha256. Can't do anything." )
class UPSDModule_adminserv(UPSDModule): modname = 'adminserv' modperm = 'master' def start(self): if not UPSDModule.start(self): return False self.conf = Config() self.userconf = {} ### Load info for pseudo client from config ### self.userconf['nick'] = self.conf.getOption(self.modname, 'nick') self.userconf['ident'] = self.conf.getOption(self.modname, 'ident') self.userconf['host'] = self.conf.getOption(self.modname, 'host') self.userconf['modes'] = self.conf.getOption(self.modname, 'modes') self.userconf['realname'] = self.conf.getOption(self.modname, 'realname') self.userconf['mainchan'] = self.conf.getOption(self.modname, 'mainchan') self.proto.sendNickCreate(self.userconf['nick'], self.userconf['ident'], self.userconf['host'], self.userconf['modes'], self.userconf['realname']) self.proto.sendJoin(self.userconf['nick'], self.userconf['mainchan']) return True def stop(self): UPSDModule.stop(self) self.proto.sendUserQuit(self.userconf['nick']) def loadMod(self, destination, params): if (self.perm.checkPerm(destination, self.modperm)): if self.main.modhandler.loadModule(params[0]): self.proto.sendNotice(self.userconf['nick'], destination, 'Loaded module %s!' % (params[0])) else: self.proto.sendNotice(self.userconf['nick'], destination, 'Unable to load module %s.' % (params[0])) return True def unloadMod(self, destination, params): if (self.perm.checkPerm(destination, self.modperm)) & (params[0] != self.modname): if self.main.modhandler.unloadModule(params[0]): self.proto.sendNotice(self.userconf['nick'], destination, 'Unloaded module %s!' % (params[0])) else: self.proto.sendNotice(self.userconf['nick'], destination, 'Unable to unload module %s.' % (params[0])) else: self.proto.sendNotice(self.userconf['nick'], destination, 'You aren\'t allowed to use this command!') return False return True def reload(self, destination, params=None): if (self.perm.checkPerm(destination, self.modperm)): self.proto.sendNotice(self.userconf['nick'], destination, 'Reloading %s module!' % (self.modname)) self.main.modhandler.reloadModule(self.modname) else: self.proto.sendNotice(self.userconf['nick'], destination, 'You aren\'t allowed to use this command!') return False return True def sendHelp(self, destination, params=None): if (self.perm.checkPerm(destination, self.modperm)): self.proto.sendNotice(self.userconf['nick'], destination, '\x02AdminServ Commands\x02:') self.proto.sendNotice(self.userconf['nick'], destination, '\x02LOADMOD <module>\x02 - Loads the specified module.') self.proto.sendNotice(self.userconf['nick'], destination, '\x02UNLOADMOD <module>\x02 - Unloads the specified module.') self.proto.sendNotice(self.userconf['nick'], destination, '\x02RELOAD\x02 - Reloads the AdminServ module.') else: self.proto.sendNotice(self.userconf['nick'], destination, 'You aren\'t allowed to use this command!') return False return True def adminserv_ctcp(self, user, command, params): if command.lower() == "\x01version\x01": self.proto.sendNotice(self.userconf['nick'], user, '\x01VERSION %s\x01' % (self.getVersion())) return True def adminserv_quit(self, user, command, params): print 'QUIT Detected!' return True def adminserv_channel(self, user, command, params): nick = params[0].split(':')[1] if (command == '.help'): if (self.perm.checkPerm(nick, self.modperm)): self.proto.sendMsg(self.userconf['nick'], user, 'Okay I\'ll help you ' + nick + '.') self.sendHelp(nick) elif (command == '.reload'): if (self.perm.checkPerm(nick, self.modperm)): self.proto.sendMsg(self.userconf['nick'], user, 'Okay, I\'ll only reload because I love you ' + nick + ".") self.reload(nick) return True def getModCommands(self): return ( ('loadmod', { 'function': self.loadMod, 'usage': '\x02Syntax\x02: LOADMOD <module>' }), ('unloadmod', { 'function': self.unloadMod, 'usage': '\x02Syntax\x02: UNLOADMOD <module>' }), ('reload', { 'function': self.reload, 'usage': '\x02Syntax\x02: RELOAD' }), ('help', { 'function': self.sendHelp, 'usage': '\x02Syntax\x02: HELP' }) ) def getModHooks(self): return (('quit', self.adminserv_quit), ('privmsg', self.adminserv_ctcp), ('privmsg', self.adminserv_channel)) def getVersion(self): return 'UnrealPSD AdminServ Module'
class Database(): def __init__(self): self.__cfgReporting = Config(os.path.join(RAGPICKER_ROOT, 'config', 'reporting.conf')) self.__cfgProcessing = Config(os.path.join(RAGPICKER_ROOT, 'config', 'processing.conf')) self.__mongodbEnabled = self.__cfgReporting.getOption("mongodb", "enabled") self.__codedbEnabled = self.__cfgReporting.getOption("codeDB", "enabled") self.__bluecoatEnabled = self.__cfgProcessing.getOption("all_bluecoatMalwareAnalysisAppliance", "enabled") if self.__mongodbEnabled: #Anbindung an Datenbank MongoDB Collection Ragpicker herstellen try: mongodbHost = self.__cfgReporting.getOption("mongodb", "host") mongodbPort = self.__cfgReporting.getOption("mongodb", "port") self.__mongodbConnection = MongoClient(mongodbHost, mongodbPort) self.__mongodbCollectionRagpicker = self.__mongodbConnection.MalwareAnalyse.ragpicker self.__mongodbCollectionFamilies = self.__mongodbConnection.MalwareAnalyse.families self.__mongodbCollectionSandboxTaskQueue = self.__mongodbConnection.MalwareAnalyse.sandboxTaskQueue except TypeError: raise Exception("MongoDB connection port in report.config must be integer") except ConnectionFailure: raise Exception("Cannot connect to MongoDB (ragpicker)") if self.__codedbEnabled: #Anbindung an Datenbank MongoDB Collection CodeDB herstellen try: codedbHost = self.__cfgReporting.getOption("codeDB", "mongo_db_host") codedbPort = self.__cfgReporting.getOption("codeDB", "mongo_db_port") self.__codedbConnection = MongoClient(codedbHost, codedbPort) self.__codedbCollectionCodedb = self.__codedbConnection.MalwareAnalyse.codeDB except TypeError: raise Exception("MongoDB connection port for CodeDB in report.config must be integer") except ConnectionFailure: raise Exception("Cannot connect to MongoDB (codeDB)") def __del__(self): if self.__mongodbEnabled: self.__mongodbConnection.disconnect() if self.__codedbEnabled: self.__codedbConnection.disconnect() # ------------------------------------------------------------------------------ # Ragpicker Database (MongoDB) # ------------------------------------------------------------------------------ def isRagpickerDBEnabled(self): return self.__mongodbEnabled def getStatisticsAntivirus(self): queries = [] ret = [] queries.append({"product" : "Avast Antivirus", "findStr1" : "AntivirusScanAvast", "findStr2" : "AntivirusScanAvast.avast", "ok" : "OK"}) queries.append({"product" : "AVG Antivirus", "findStr1" : "AntivirusScanAvg", "findStr2" : "AntivirusScanAvg.Avg", "ok" : "OK"}) queries.append({"product" : "Avira", "findStr1" : "AntivirusScanAvira", "findStr2" : "AntivirusScanAvira.Avira.scan", "ok" : "OK"}) queries.append({"product" : "BitDefender", "findStr1" : "AntivirusScanBitDefender", "findStr2" : "AntivirusScanBitDefender.BitDefender", "ok" : "OK"}) queries.append({"product" : "ClamAV", "findStr1" : "AntivirusScanClamAv", "findStr2" : "AntivirusScanClamAv.ClamAv", "ok" : " OK"}) queries.append({"product" : "COMODO", "findStr1" : "AntivirusScanCOMODO", "findStr2" : "AntivirusScanCOMODO.COMODO", "ok" : "OK"}) queries.append({"product" : "ESET", "findStr1" : "AntivirusScanESET", "findStr2" : "AntivirusScanESET.ESET", "ok" : "OK"}) queries.append({"product" : "F-Prot", "findStr1" : "AntivirusScanFProt", "findStr2" : "AntivirusScanFProt.FProt", "ok" : "OK"}) queries.append({"product" : "F-Secure", "findStr1" : "AntivirusScanF-Secure", "findStr2" : "AntivirusScanF-Secure.F-Secure", "ok" : "OK"}) for q in queries: av = {} av["product"] = q.get("product") av["analyzed"] = str(self.__mongodbCollectionRagpicker.find({q.get("findStr1"): {"$ne":None}}).count()) av["notanalyzed"] = str(self.__mongodbCollectionRagpicker.find({q.get("findStr1") : None}).count()) av["malware"] = str(self.__mongodbCollectionRagpicker.find({ "$and": [{q.get("findStr1") : {"$ne":None}}, {q.get("findStr2") : {"$ne": q.get("ok")}}]}).count()) av["nonemalware"] = str(self.__mongodbCollectionRagpicker.find({q.get("findStr2"): q.get("ok")}).count()) if av.get("analyzed") != "0": av["rate"] = "{:.2f} %".format((float(av.get("malware"))/float(av.get("analyzed"))*100)) else: av["rate"] = "--" ret.append(av) return ret def getStatisticsNoneMalwareByAV(self): return self.__mongodbCollectionRagpicker.find({ "$and": [{ "$or": [{"AntivirusScanAvast.avast" : "OK"}, {"AntivirusScanAvast" : None}]}, { "$or": [{"AntivirusScanAvg.Avg" : "OK"}, {"AntivirusScanAvg" : None}]}, { "$or": [{"AntivirusScanAvira.Avira.scan" : "OK"}, {"AntivirusScanAvira" : None}]}, { "$or": [{"AntivirusScanBitDefender.BitDefender" : "OK"}, {"AntivirusScanBitDefender" : None}]}, { "$or": [{"AntivirusScanClamAv.ClamAv" : " OK"}, {"AntivirusScanClamAv" : None}]}, { "$or": [{"AntivirusScanCOMODO.COMODO" : "OK"}, {"AntivirusScanCOMODO" : None}]}, { "$or": [{"AntivirusScanESET.ESET" : "OK"}, {"AntivirusScanESET" : None}]}, { "$or": [{"AntivirusScanFProt.FProt" : "OK"}, {"AntivirusScanFProt" : None}]}, { "$or": [{"AntivirusScanF-Secure.F-Secure" : "OK"}, {"AntivirusScanF-Secure" : None}]}, {"VirusTotal.file.verbose_msg" : {"$ne":None}}]}).count() def getSamplesNotFoundByAV(self): return self.__mongodbCollectionRagpicker.find({ "$and": [{ "$or": [{"AntivirusScanAvast.avast" : "OK"}, {"AntivirusScanAvast" : None}]}, { "$or": [{"AntivirusScanAvg.Avg" : "OK"}, {"AntivirusScanAvg" : None}]}, { "$or": [{"AntivirusScanAvira.Avira.scan" : "OK"}, {"AntivirusScanAvira" : None}]}, { "$or": [{"AntivirusScanBitDefender.BitDefender" : "OK"}, {"AntivirusScanBitDefender" : None}]}, { "$or": [{"AntivirusScanClamAv.ClamAv" : " OK"}, {"AntivirusScanClamAv" : None}]}, { "$or": [{"AntivirusScanCOMODO.COMODO" : "OK"}, {"AntivirusScanCOMODO" : None}]}, { "$or": [{"AntivirusScanESET.ESET" : "OK"}, {"AntivirusScanESET" : None}]}, { "$or": [{"AntivirusScanFProt.FProt" : "OK"}, {"AntivirusScanFProt" : None}]}, { "$or": [{"AntivirusScanF-Secure.F-Secure" : "OK"}, {"AntivirusScanF-Secure" : None}]}, {"VirusTotal.file.verbose_msg" : {"$ne":None}} ]}, {"Info.file.sha256": True, "Info.analyse.started": True }) def getSamplesNotFoundByVT(self): return self.__mongodbCollectionRagpicker.find({"VirusTotal.file.verbose_msg" : {"$ne":None}}, {"Info.file.sha256": True, "Info.analyse.started": True }) def getSamplesNotFoundByLocalAV(self): return self.__mongodbCollectionRagpicker.find({ "$and": [{ "$or": [{"AntivirusScanAvast.avast" : "OK"}, {"AntivirusScanAvast" : None}]}, { "$or": [{"AntivirusScanAvg.Avg" : "OK"}, {"AntivirusScanAvg" : None}]}, { "$or": [{"AntivirusScanAvira.Avira.scan" : "OK"}, {"AntivirusScanAvira" : None}]}, { "$or": [{"AntivirusScanBitDefender.BitDefender" : "OK"}, {"AntivirusScanBitDefender" : None}]}, { "$or": [{"AntivirusScanClamAv.ClamAv" : " OK"}, {"AntivirusScanClamAv" : None}]}, { "$or": [{"AntivirusScanCOMODO.COMODO" : "OK"}, {"AntivirusScanCOMODO" : None}]}, { "$or": [{"AntivirusScanESET.ESET" : "OK"}, {"AntivirusScanESET" : None}]}, { "$or": [{"AntivirusScanFProt.FProt" : "OK"}, {"AntivirusScanFProt" : None}]}, { "$or": [{"AntivirusScanF-Secure.F-Secure" : "OK"}, {"AntivirusScanF-Secure" : None}]} ]}, {"Info.file.sha256": True, "Info.analyse.started": True }) def getStatisticsVirusTotal(self): ret = {} ret["analyzed"] = self.__mongodbCollectionRagpicker.find({"VirusTotal" : {"$ne":None}}).count() ret["notAnalyzed"] = self.__mongodbCollectionRagpicker.find({"VirusTotal" : None}).count() ret["samplesFound"] = self.__mongodbCollectionRagpicker.find({"VirusTotal.file.positives" : {"$ne":None}}).count() ret["SamplesNotFound"] = self.__mongodbCollectionRagpicker.find({"VirusTotal.file.verbose_msg" : {"$ne":None}}).count() return ret def getStatisticsPackerSignatures(self): return self.__mongodbCollectionRagpicker.aggregate([{ '$group' : {'_id' : { 'PublisherO': "$VerifySigs.PublisherO", 'Issuer': "$VerifySigs.Issuer" }, 'count' : { '$sum': 1 }}},{'$sort':{"count": -1}}]) def getStatisticsPackerCompiler(self): return self.__mongodbCollectionRagpicker.aggregate([{ '$group' : {'_id' : '$PEID', 'count' : { '$sum': 1 } }}, {'$sort':{'count': -1}}]) def getStatisticsPeCharacteristics(self): ret = {} ret["exe"] = str(self.__mongodbCollectionRagpicker.find({ "$and": [{"Info.file.EXE" : True}, {"Info.file.DLL" : False}, {"Info.file.DRIVER" : False}]}).count()) ret["dll"] = str(self.__mongodbCollectionRagpicker.find({ "$and": [{"Info.file.EXE" : False}, {"Info.file.DLL" : True}, {"Info.file.DRIVER" : False}]}).count()) ret["driver"] = str(self.__mongodbCollectionRagpicker.find({ "$and": [{"Info.file.EXE" : False}, {"Info.file.DLL" : False}, {"Info.file.DRIVER" : True}]}).count()) ret["noPe"] = str(self.__mongodbCollectionRagpicker.find({ "$and": [{"Info.file.EXE" : None}, {"Info.file.DLL" : None}, {"Info.file.DRIVER" : None}]}).count()) ret["dllDriver"] = str(self.__mongodbCollectionRagpicker.find({ "$and": [{"Info.file.EXE" : False}, {"Info.file.DLL" : True}, {"Info.file.DRIVER" : True}]}).count()) return ret def getFiletypes(self): return self.__mongodbCollectionRagpicker.aggregate([{ '$group' : {'_id' : '$Info.file.type', 'count' : { '$sum': 1 } }}, {'$sort':{'count': -1}}]) def countReportsRagpickerDB(self): return self.__mongodbCollectionRagpicker.find().count() def iterateRagpickerReports(self, sha256): for report in self.__mongodbCollectionRagpicker.find({'Info.file.sha256' : sha256}, {"_id" : 0}): yield report # Attention deletes the whole Ragpicker-Database!!! # returns number of deleted reports def deleteRagpickerDB(self): count = self.__mongodbCollectionRagpicker.find().count() # Alle Ragpicker-Daten aus der MongoDB loeschen self.__mongodbCollectionRagpicker.remove() return count #Insert Ragpicker-Report in MongoDB def insertRagpickerDB(self, report): # Store the report try: self.__mongodbCollectionRagpicker.insert(report) except InvalidDocument as e: log.exception("Error InvalidDocument: %s", report) raise Exception("Error InvalidDocument: {0}".format(e)) except InvalidStringData: self.__mongodbCollectionRagpicker.insert(convertDirtyDict2ASCII(report)) #Count Ragpicker-Reports by file (and url) def countRagpickerDB(self, file_md5, url_md5=None): if url_md5: query = { "$and" : [{ "Info.url.md5": { "$in": [url_md5] } }, { "Info.file.md5": { "$in": [file_md5] } }]} else: query = { "$and" : [{ "Info.file.md5": { "$in": [file_md5] } }]} return self.__mongodbCollectionRagpicker.find(query).count() # ------------------------------------------------------------------------------ # Ragpicker SandboxTaskQueue database (MongoDB) # ------------------------------------------------------------------------------ def insertSandboxTaskStatus(self, sandboxName, sha256, taskID, sampleID, taskState=None): statusReport = {"sandbox":sandboxName, "sha256":sha256, "sample_id":sampleID, "task_id":taskID, "task_state":taskState} # Store the SandboxTaskQueue-Status-report self.__mongodbCollectionSandboxTaskQueue.insert(statusReport) # Attention deletes the whole Ragpicker-SandboxTaskQueue-Database!!! # returns number of deleted reports def deleteSandboxTaskQueueDB(self): count = self.__mongodbCollectionSandboxTaskQueue.find().count() # Alle Daten aus der MongoDB loeschen self.__mongodbCollectionSandboxTaskQueue.remove() return count # ------------------------------------------------------------------------------ # Ragpicker families database (MongoDB) # ------------------------------------------------------------------------------ def insertFamily(self, familyReport): # Store the family-report self.__mongodbCollectionFamilies.insert(familyReport) #Count Ragpicker-Reports by file (and url) def countFamilyDB(self, parentObjectSHA256): query = { "$and" : [{ "parentObjectSHA256": { "$in": [parentObjectSHA256] } }]} return self.__mongodbCollectionFamilies.find(query).count() def iterateFamilyReports(self, sha256): for report in self.__mongodbCollectionFamilies.find({'parentObjectSHA256' : sha256}, {"_id" : 0}): yield report # Attention deletes the whole Ragpicker-Family-Database!!! # returns number of deleted reports def deleteFamilyDB(self): count = self.__mongodbCollectionFamilies.find().count() # Alle Ragpicker-Daten aus der MongoDB loeschen self.__mongodbCollectionFamilies.remove() return count
class Statistics(): def __init__(self): # Datenbank self.__database = Database() # Kofiguration aus der reporting.conf holen self.__cfgReporting = Config( os.path.join(RAGPICKER_ROOT, 'config', 'reporting.conf')) self.__vxcageEnabled = self.__cfgReporting.getOption( "vxcage", "enabled") self.__vxcageHost = self.__cfgReporting.getOption("vxcage", "host") self.__vxcagePort = self.__cfgReporting.getOption("vxcage", "port") def runStatisticsLong(self): #Pruefen ob VxCage und MongoDB aktiviert sind if self.__database.isRagpickerDBEnabled(): if self.__vxcageEnabled: self.__runStatisticsMongodbLong() if self.__database.isCodeDBEnabled(): self.__runStatisticsCodedb() else: print("vxcage in reporting.conf is not enabled") sys.stdout.flush() else: print("mongodb in reporting.conf is not enabled") sys.stdout.flush() def runStatisticsShort(self): #Pruefen ob VxCage und MongoDB aktiviert sind if self.__database.isRagpickerDBEnabled(): if self.__vxcageEnabled: self.__runStatisticsMongodbShort() if self.__database.isCodeDBEnabled(): self.__runStatisticsCodedb() else: print("vxcage in reporting.conf is not enabled") sys.stdout.flush() else: print("mongodb in reporting.conf is not enabled") sys.stdout.flush() def runStatisticsAV(self): #Pruefen ob VxCage und MongoDB aktiviert sind if self.__database.isRagpickerDBEnabled(): if self.__vxcageEnabled: self.__runStatisticsAV() else: print("vxcage in reporting.conf is not enabled") sys.stdout.flush() else: print("mongodb in reporting.conf is not enabled") sys.stdout.flush() def __runStatisticsMongodbLong(self): print "**************************************" print "*** Statistics MongoDB (Ragpicker) ***" print "**************************************" print "" print "Number of malware samples in database:", self.__database.countReportsRagpickerDB( ) print "" #Statistiken der eingesetzten AV-Produkte self.__runStatisticsAVProducts() #Liste der letzen 20 Samples, die weder auf VT noch von einem lokalen AV gefunden wurden self.__runStatisticsLast20SamplesNotFoundByAV() #Liste der letzen 20 Samples, die nicht auf VT gefunden wurden self.__runStatisticsLast20SamplesNotFoundByVT() #Liste der letzen 20 Samples, die nicht von einem lokalen AV-Produkt gefunden wurden self.__runStatisticsLast20SamplesNotFoundByLocalAV() #Liste und Haeufigkeit der Filetypes self.__runStatisticsFiletypes() #Haeufigkeit der PE Charakteristiken self.__runStatisticsPeCharacteristics() #Liste und Haeufigkeit der verwendeten Packer/Compiler in der Malware self.__runStatisticsPackerCompiler() #Liste der verwendeten digitalen Signaturen self.__runStatisticsPackerSignatures() sys.stdout.flush() def __runStatisticsMongodbShort(self): print "**************************************" print "*** Statistics MongoDB (Ragpicker) ***" print "**************************************" print "" print "Number of malware samples in database:", self.__database.countReportsRagpickerDB( ) print "" #Liste und Haeufigkeit der Filetypes self.__runStatisticsFiletypes() #Haeufigkeit der PE Charakteristiken self.__runStatisticsPeCharacteristics() sys.stdout.flush() def __runStatisticsAV(self): print "**************************************" print "*** Statistics MongoDB (Ragpicker) ***" print "**************************************" print "" print "Number of malware samples in database:", self.__database.countReportsRagpickerDB( ) print "" #Statistiken der eingesetzten AV-Produkte self.__runStatisticsAVProducts() #Liste der letzen 20 Samples, die weder auf VT noch von einem lokalen AV gefunden wurden self.__runStatisticsLast20SamplesNotFoundByAV() #Liste der letzen 20 Samples, die nicht auf VT gefunden wurden self.__runStatisticsLast20SamplesNotFoundByVT() #Liste der letzen 20 Samples, die nicht von einem lokalen AV-Produkt gefunden wurden self.__runStatisticsLast20SamplesNotFoundByLocalAV() sys.stdout.flush() def __runStatisticsCodedb(self): print "***********************************" print "*** Statistics MongoDB (CodeDB) ***" print "***********************************" print "" print "Number of malware samples in database:", self.__database.countReportsCodeDB( ) print "" sys.stdout.flush() def __runStatisticsFiletypes(self): #Liste und Haeufigkeit der Filetypes print "Filetypes of malware" res = self.__database.getFiletypes() table = PrettyTable(["filetype", "count"]) table.align["filetype"] = "l" table.align["count"] = "c" table.padding_width = 1 try: for values in res['result']: if values.get("_id"): outputPacker = values.get("_id") outputCount = str(values.get("count")) table.add_row([outputPacker, outputCount]) print(table) except KeyError: raise Exception("Dict has no key 'result' ") print "" def __runStatisticsPeCharacteristics(self): #Haeufigkeit der PE Charakteristiken print "PE-Characteristics of malware" peC = self.__database.getStatisticsPeCharacteristics() table = PrettyTable(["pe-characteristics", "count"]) table.align["pe-characteristics"] = "l" table.align["count"] = "c" table.padding_width = 1 table.add_row(["EXE", peC.get("exe")]) table.add_row(["DLL", peC.get("dll")]) table.add_row(["Driver", peC.get("driver")]) table.add_row(["DLL/Driver", peC.get("dllDriver")]) table.add_row(["No PE File", peC.get("noPe")]) print(table) print "" def __runStatisticsPackerCompiler(self): #Liste und Haeufigkeit der verwendeten Packer/Compiler in der Malware print "Packer/compiler used in malware" res = self.__database.getStatisticsPackerCompiler() table = PrettyTable(["packer/compiler", "count"]) table.align["packer/compiler"] = "l" table.align["count"] = "c" table.padding_width = 1 try: for values in res['result']: if values.get("_id"): outputPacker = values.get("_id")[0] outputCount = str(values.get("count")) table.add_row([outputPacker, outputCount]) print(table) except KeyError: raise Exception("Dict has no key 'result' ") print " " def __runStatisticsPackerSignatures(self): #Liste der verwendeten digitalen Signaturen print "Signatures used by malware" res = self.__database.getStatisticsPackerSignatures() table = PrettyTable(["publisher", "issuer", "count"]) table.align["publisher"] = "l" table.align["issuer"] = "l" table.align["count"] = "c" table.padding_width = 1 try: for values in res['result']: if values.get("_id"): outputPublisher = values.get("_id").get("PublisherO") if values.get("_id").get("Issuer"): outputIssuer = values.get("_id").get("Issuer") else: outputIssuer = " " outputCount = str(values.get("count")) table.add_row([outputPublisher, outputIssuer, outputCount]) print(table) except KeyError: raise Exception("Dict has no key 'result' ") print "" def __runStatisticsLast20SamplesNotFoundByAV(self): #Liste der letzen 20 Samples, die weder auf VT noch von einem lokalen AV gefunden wurden print "Last 20 samples not found by VirusTotal and local AV-Products" res = sorted(self.__database.getSamplesNotFoundByAV(), reverse=True) table = PrettyTable(["timestamp of crawling", "sha256"]) table.align["timestamp of crawling"] = "c" table.align["sha256"] = "c" table.padding_width = 1 try: for values in res: sha256 = values.get("Info").get("file").get("sha256") timestamp = values.get("Info").get("analyse").get("started") table.add_row([timestamp, sha256]) print(table.get_string(start=0, end=20)) except KeyError: raise Exception("Dict has no key 'Info' ") print "" def __runStatisticsLast20SamplesNotFoundByVT(self): #Liste der letzen 20 Samples, die nicht auf VirusTotal gefunden wurden print "Last 20 samples not found by VirusTotal" res = sorted(self.__database.getSamplesNotFoundByVT(), reverse=True) table = PrettyTable(["timestamp of crawling", "sha256"]) table.align["timestamp of crawling"] = "c" table.align["sha256"] = "c" table.padding_width = 1 try: for values in res: sha256 = values.get("Info").get("file").get("sha256") timestamp = values.get("Info").get("analyse").get("started") table.add_row([timestamp, sha256]) print(table.get_string(start=0, end=20)) except KeyError: raise Exception("Dict has no key 'Info' ") print "" def __runStatisticsLast20SamplesNotFoundByLocalAV(self): #Liste der letzen 20 Samples, die nicht von einem lokalen AV-Produkt gefunden wurden print "Last 20 samples not found by local AV-Products" res = sorted(self.__database.getSamplesNotFoundByLocalAV(), reverse=True) table = PrettyTable(["timestamp of crawling", "sha256"]) table.align["timestamp of crawling"] = "c" table.align["sha256"] = "c" table.padding_width = 1 try: for values in res: sha256 = values.get("Info").get("file").get("sha256") timestamp = values.get("Info").get("analyse").get("started") table.add_row([timestamp, sha256]) print(table.get_string(start=0, end=20)) except KeyError: raise Exception("Dict has no key 'Info' ") print "" def __runStatisticsAVProducts(self): #Statistiken der eingesetzten AV-Produkte #VirusTotal und lokale AV-Produkte print "VirusTotal and local AV-Products" print " Samples rated as none-malware by all AV-Products at time of crawling:", \ self.__database.getStatisticsNoneMalwareByAV() print "" #VirusTotal ret = self.__database.getStatisticsVirusTotal() print "VirusTotal" print " Samples analyzed at time of crawling:", ret.get("analyzed") print " Samples not analyzed at time of crawling:", ret.get( "notAnalyzed") print " Samples found at time of crawling:", ret.get("samplesFound") print " Samples not found at time of crawling:", ret.get( "SamplesNotFound") print "" #Lokale AV-Produkte print "Local AV-Products" print " analyzed => Samples analyzed at time of crawling" print " not analyzed => Samples not analyzed at time of crawling" print " malware => Samples rated as malware at time of crawling" print " none-malware => Samples rated as none-malware at time of crawling" table = PrettyTable([ "product", "analyzed", "not analyzed", "malware", "none-malware", "detection rate" ]) table.align["product"] = "l" table.align["analyzed"] = "r" table.align["not analyzed"] = "r" table.align["malware"] = "r" table.align["none-malware"] = "r" table.align["detection rate"] = "r" table.padding_width = 1 # Statistik Daten holen ret = self.__database.getStatisticsAntivirus() # Table-Body zusammenbauen for av in ret: table.add_row([ av.get("product"), av.get("analyzed"), av.get("notanalyzed"), av.get("malware"), av.get("nonemalware"), av.get("rate") ]) print(table) print ""
class VxCageHandler(): def __init__(self): self.cfgReporting = Config(os.path.join(RAGPICKER_ROOT, 'config', 'reporting.conf')) self.vxcageEnabled = self.cfgReporting.getOption("vxcage", "enabled") self.host = self.cfgReporting.getOption("vxcage", "host") self.port = self.cfgReporting.getOption("vxcage", "port") if not self.host or not self.port: raise Exception("VxCage REST API server not configurated") def upload(self, filePath, fileName, tags): rawFile = open(filePath, 'rb') log.debug(VXCAGE_URL_ADD % (self.host, self.port) + " file=" + fileName) try: form = MultiPartForm() form.add_file('file', fileName, fileHandle=rawFile) form.add_field('tags', tags) request = urllib2.Request(VXCAGE_URL_ADD % (self.host, self.port)) body = str(form) request.add_header('Content-type', form.get_content_type()) request.add_header('Content-length', len(body)) request.add_data(body) response_data = urllib2.urlopen(request, timeout=60).read() reponsejson = json.loads(response_data) log.info("Submitted to vxcage, message: %s", reponsejson["message"]) except urllib2.URLError as e: raise Exception("Unable to establish connection to VxCage REST API server: %s" % e) except urllib2.HTTPError as e: raise Exception("Unable to perform HTTP request to VxCage REST API server (http code=%s)" % e) except ValueError as e: raise Exception("Unable to convert response to JSON: %s" % e) if reponsejson["message"] != 'added': raise Exception("Failed to store file in VxCage: %s" % reponsejson["message"]) # Exports malware file from the VxCage using the sha256-hash def exportVxCage(self, sha256, exportDir): if os.path.isfile(exportDir + sha256): raise Exception("File %s already exists.") cmd = "wget -q --tries=1 --directory-prefix=%s 'http://%s:%s/malware/get/%s'" % (exportDir, self.host, self.port, sha256) os.system(cmd) if not os.path.isfile(exportDir + sha256): raise Exception("Download %s failed." % sha256) def isFileInCage(self, md5=None, sha256=None): if md5: param = { 'md5': md5 } elif sha256: param = { 'sha256': sha256 } request_data = urllib.urlencode(param) try: request = urllib2.Request(VXCAGE_URL_FIND % (self.host, self.port), request_data) response = urllib2.urlopen(request, timeout=60) response_data = response.read() except urllib2.HTTPError as e: if e.code == 404: # Error: 404 Not Found log.info('404 Not Found (' + str(param) + ')') return False else: raise Exception("Unable to perform HTTP request to VxCage (http code=%s)" % e) except urllib2.URLError as e: raise Exception("Unable to establish connection to VxCage: %s" % e) try: check = json.loads(response_data) except ValueError as e: raise Exception("Unable to convert response to JSON: %s" % e) if md5 and check["md5"] == md5: log.info("File " + md5 + " is in VxCage") return True elif sha256 and check["sha256"] == sha256: log.info("File " + sha256 + " is in VxCage") return True return False
h = httplib2.Http(".cache", disable_ssl_certificate_validation=True) response, content = h.request(CODE_DB_URL_TAG % (config.get("host"), config.get("port")), "POST", body=urlencode(data), headers=headers) if not "'status': '200'" in str(response) : log.error("%s --> %s = %s" % (sha256, tags, str(content))) data = json.loads(content) log.info("%s --> %s = %s" % (sha256, tags, data.get("Status"))) if __name__ == '__main__': # Datenbank database = Database() # VxCage-Handler vxCage = VxCageHandler() vxcageEnabled = cfgReporting.getOption("vxcage", "enabled") parser = argparse.ArgumentParser(description='Ragpicker Manager') subparsers = parser.add_subparsers(title='subcommands', description='valid subcommands', help='additional help') parser_stop = subparsers.add_parser('stop', help='Stops a running Ragpicker instance') parser_stop.set_defaults(which='stop') parser_export = subparsers.add_parser('export', help='Export Ragpicker-Data') parser_export.set_defaults(which='export') parser_export.add_argument('-d','--dirname', required=True, help='Export-Directory') parser_export.add_argument('-f','--sha256_file', required=True, help='SHA256-File') parser_export.add_argument('--json', default=False, help='File in json-format? Default=False') parser_vxcage = subparsers.add_parser('vxcage', help='Exports only the malware files from the VxCage') parser_vxcage.set_defaults(which='vxcage') parser_vxcage.add_argument('-d','--dirname', required=True, help='Export-Directory') parser_vxcage.add_argument('-f','--sha256_file', required=True, help='SHA256-File') parser_vxcage.add_argument('--json', default=False, help='File in json-format? Default=False')
if isJson: data = json.load(file) for sha256 in data.itervalues(): yield sha256.rstrip() else: for sha256 in file: yield sha256.rstrip() file.close() if __name__ == '__main__': # Datenbank database = Database() # VxCage-Handler vxCage = VxCageHandler() vxcageEnabled = cfgReporting.getOption("vxcage", "enabled") parser = argparse.ArgumentParser(description='Ragpicker Manager') subparsers = parser.add_subparsers(title='subcommands', description='valid subcommands', help='additional help') parser_stop = subparsers.add_parser('stop', help='Stops a running Ragpicker instance') parser_stop.set_defaults(which='stop') parser_export = subparsers.add_parser('export', help='Export Ragpicker-Data') parser_export.set_defaults(which='export') parser_export.add_argument('-d','--dirname', required=True, help='Export-Directory') parser_export.add_argument('-f','--sha256_file', required=True, help='SHA256-File') parser_export.add_argument('--json', default=False, help='File in json-format? Default=False') parser_vxcage = subparsers.add_parser('vxcage', help='Exports only the malware files from the VxCage') parser_vxcage.set_defaults(which='vxcage') parser_vxcage.add_argument('-d','--dirname', required=True, help='Export-Directory') parser_vxcage.add_argument('-f','--sha256_file', required=True, help='SHA256-File') parser_vxcage.add_argument('--json', default=False, help='File in json-format? Default=False')
h = httplib2.Http(".cache", disable_ssl_certificate_validation=True) response, content = h.request(CODE_DB_URL_TAG % (config.get("host"), config.get("port")), "POST", body=urlencode(data), headers=headers) if not "'status': '200'" in str(response) : log.error("%s --> %s = %s" % (sha256, tags, str(content))) data = json.loads(content) log.info("%s --> %s = %s" % (sha256, tags, data.get("Status"))) if __name__ == '__main__': # Datenbank database = Database() # Viper-Handler viper = ViperHandler() viperEnabled = cfgReporting.getOption("viper", "enabled") parser = argparse.ArgumentParser(description='Ragpicker Manager') subparsers = parser.add_subparsers(title='subcommands', description='valid subcommands', help='additional help') parser_stop = subparsers.add_parser('stop', help='Stops a running Ragpicker instance') parser_stop.set_defaults(which='stop') parser_export = subparsers.add_parser('export', help='Export Ragpicker-Data') parser_export.set_defaults(which='export') parser_export.add_argument('-d','--dirname', required=True, help='Export-Directory') parser_export.add_argument('-f','--sha256_file', required=True, help='SHA256-File') parser_export.add_argument('--json', default=False, help='File in json-format? Default=False') parser_viper = subparsers.add_parser('viper', help='Exports only the malware files from the Viper') parser_viper.set_defaults(which='viper') parser_viper.add_argument('-d','--dirname', required=True, help='Export-Directory') parser_viper.add_argument('-f','--sha256_file', required=True, help='SHA256-File') parser_viper.add_argument('--json', default=False, help='File in json-format? Default=False')