class DatabaseMySQL(): def __init__(self): self.__cfgReporting = Config(os.path.join(RAGPICKER_ROOT, 'config', 'reporting.conf')) self.__mysqlEnabled = self.__cfgReporting.getOption("mysql", "enabled") if self.__mysqlEnabled: #Anbindung an Datenbank MySQL herstellen try: mysqlHost = self.__cfgReporting.getOption("mysql", "host") mysqlPort = self.__cfgReporting.getOption("mysql", "port") mysqlDatabase = self.__cfgReporting.getOption("mysql", "database") mysqlUser = self.__cfgReporting.getOption("mysql", "user") mysqlPassword = self.__cfgReporting.getOption("mysql", "password") self.__mysqlConnection = MySQLdb.Connect(host=mysqlHost, port=mysqlPort, db=mysqlDatabase, user=mysqlUser, passwd=mysqlPassword) except (Exception) as e: raise Exception("Cannot connect to MySQL (ragpicker): %s" % e) def __del__(self): if self.__mysqlEnabled: self.__mysqlConnection.close() # ------------------------------------------------------------------------------ # Ragpicker Database (MySQL) # ------------------------------------------------------------------------------ def isRagpickerDBEnabledMySQL(self): return self.__mysqlEnabled
def __init__(self): self.__cfgReporting = Config(os.path.join(RAGPICKER_ROOT, "config", "reporting.conf")) self.__cfgProcessing = Config(os.path.join(RAGPICKER_ROOT, "config", "processing.conf")) self.__mongodbEnabled = self.__cfgReporting.getOption("mongodb", "enabled") self.__codedbEnabled = self.__cfgReporting.getOption("codeDB", "enabled") self.__bluecoatEnabled = self.__cfgProcessing.getOption("all_bluecoatMalwareAnalysisAppliance", "enabled") if self.__mongodbEnabled: # Anbindung an Datenbank MongoDB Collection Ragpicker herstellen try: mongodbHost = self.__cfgReporting.getOption("mongodb", "host") mongodbPort = self.__cfgReporting.getOption("mongodb", "port") self.__mongodbConnection = MongoClient(mongodbHost, mongodbPort) self.__mongodbCollectionRagpicker = self.__mongodbConnection.MalwareAnalyse.ragpicker self.__mongodbCollectionFamilies = self.__mongodbConnection.MalwareAnalyse.families self.__mongodbCollectionSandboxTaskQueue = self.__mongodbConnection.MalwareAnalyse.sandboxTaskQueue except TypeError: raise Exception("MongoDB connection port in report.config must be integer") except ConnectionFailure: raise Exception("Cannot connect to MongoDB (ragpicker)") if self.__codedbEnabled: # Anbindung an Datenbank MongoDB Collection CodeDB herstellen try: codedbHost = self.__cfgReporting.getOption("codeDB", "mongo_db_host") codedbPort = self.__cfgReporting.getOption("codeDB", "mongo_db_port") self.__codedbConnection = MongoClient(codedbHost, codedbPort) self.__codedbCollectionCodedb = self.__codedbConnection.MalwareAnalyse.codeDB except TypeError: raise Exception("MongoDB connection port for CodeDB in report.config must be integer") except ConnectionFailure: raise Exception("Cannot connect to MongoDB (codeDB)")
def __process_url(self, url): # Crawler config load cfgCrawler = Config(os.path.join(RAGPICKER_ROOT, 'config', 'crawler.conf')).get("clientConfig") data = None headers = { 'User-Agent': cfgCrawler.get("browser_user_agent", "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)"), 'Accept-Language': cfgCrawler.get("browser_accept_language", "en-US"), } # Save original socket originalSocket = socket.socket # Set TOR Socks proxy commonutils.setTorProxy() request = urllib2.Request(url, data, headers) try: url_dl = urllib2.urlopen(request, timeout=30).read() except urllib2.HTTPError as e: raise e except urllib2.URLError as e: raise e except Exception, e: raise IOError("Thread(" + self.processName + ") - %s - Error parsing %s" % (e, url))
def active_only( self, value ): if not isinstance( value, bool ): raise TypeError( "Expected bool, not {}".format( type( value ).__name__ ) ) if self._active_only != value: self._active_only = value Config.set( "Interface", "active_only", self._active_only ) GLib.idle_add( self.refilter )
def getHtmlContent(target_url, header_type): config = Config() try: if header_type == 'sougou': send_headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 6.2; rv:16.0) Gecko/20100101 Firefox/16.0', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Connection': 'keep-alive', 'Cookie': 'com_sohu_websearch_ITEM_PER_PAGE='+str(config.getValue("pagesize", "sougou")) } else: send_headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 6.2; rv:16.0) Gecko/20100101 Firefox/16.0', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Connection': 'keep-alive' } if sys.version > '3': req = urllib.request.Request(target_url, headers=send_headers) response = urllib.request.urlopen(req, timeout=10) else: req = urllib2.Request(target_url, headers=send_headers) response = urllib2.urlopen(req, timeout=30) # print get_request.info() return response.read().decode('utf-8') except Exception as e: print("Get html page content error:" + e.message)
def runCrawler(): mapURL = {} cfgCrawler = Config(os.path.join(RAGPICKER_ROOT, 'config', 'crawler.conf')) # Build the PluginManager crawlerPluginManager = PluginManager() crawlerPluginManager.setPluginPlaces(["crawler"]) crawlerPluginManager.collectPlugins() # Trigger run from the "Crawler" plugins for pluginInfo in sorted(crawlerPluginManager.getAllPlugins(), key=lambda PluginInfo: PluginInfo.name): crawlerModul = pluginInfo.plugin_object # Config for crawler module try: options = cfgCrawler.get(pluginInfo.name) crawlerModul.set_options(options) except Exception: log.error("Crawler module %s not found in configuration file", pluginInfo.name) # If the crawler module is disabled in the config, skip it. if not options.enabled: continue try: log.debug("Run Crawler: " + pluginInfo.name) returnMap = crawlerModul.run() mapURL.update(returnMap) except Exception as e: log.error('Error (%s) in %s', e, pluginInfo.name) return mapURL
def response_cb( self, dialog, response ): """Response Callback of the Gtk.Dialog""" if response == 1: # Save new config values Config.set( "Autocompletion", "bic_file_de", self.builder.get_object( 'general_bic_open_entry' ).get_text() ) Config.set( "Autocompletion", "zipcode_file_de", self.builder.get_object( 'general_zipcode_open_entry' ).get_text() ) self._session.commit() else: # Discard new config values self._session.rollback() self.hide()
def init(): path = os.path.split(os.path.realpath(__file__))[0] config_path = "%s/config.ini" % path Config.init(config_path) CACHEFOLDER = os.getenv("HOME") + "/.cache/autohome/" if not os.path.exists(CACHEFOLDER): os.makedirs(CACHEFOLDER) PIDLOG = "%sautohome.pid" % CACHEFOLDER Config.set("global", "PID", PIDLOG) init_log()
def __init__(self, persistence): valid = [ "help_url", "twitter_url", "info_url", "ts_address", "ts_port", "ts_password" ] Config.__init__("module-admin", persistence, valid);
def __init__(self, module, page, keyword): config = Config() pagesize = config.getValue("pagesize", module) print("\033[1;37;40m[*]Search Engine [%s] starting!The number of display bars per page is %s" % (module, pagesize)) myps = multiprocessing.Process(target=Collect, args=(module, page, pagesize, keyword,)) myps.start() processList = gol.get_value("process") processList.append(myps) gol.set_value("process", processList)
class UPSDModule_testserv(UPSDModule): modname = 'testserv' def start(self): if not UPSDModule.start(self): return False self.conf = Config() self.userconf = {} ### Load info for pseudo client from config ### self.userconf['nick'] = self.conf.getOption(self.modname, 'nick') self.userconf['ident'] = self.conf.getOption(self.modname, 'ident') self.userconf['host'] = self.conf.getOption(self.modname, 'host') self.userconf['modes'] = self.conf.getOption(self.modname, 'modes') self.userconf['realname'] = self.conf.getOption(self.modname, 'realname') self.userconf['mainchan'] = self.conf.getOption(self.modname, 'mainchan') self.proto.sendNickCreate(self.userconf['nick'], self.userconf['ident'], self.userconf['host'], self.userconf['modes'], self.userconf['realname']) self.proto.sendJoin(self.userconf['nick'], self.userconf['mainchan']) return True def stop(self): UPSDModule.stop(self) self.proto.sendUserQuit(self.userconf['nick']) def testserv_ctcp(self, user, command, params): if command.lower() == "\x01version\x01": self.proto.sendNotice(self.userconf['nick'], user, '\x01VERSION %s\x01' % (self.getVersion())) return True def testserv_channel(self, user, command, params): nick = params[0].split(':')[1] if (len(params) > 4): args = params[4].split(' ') else: args = None if (command == ".kick"): self.proto.sendKick(self.userconf['nick'], user, args[0], args[1]) def getModHooks(self): return (('privmsg', self.testserv_ctcp), ('privmsg', self.testserv_channel)) def getVersion(self): return 'UnrealPSD TestServ Module'
def reload_data(): global Banks global Cities for option in Config.options( 'Autocompletion' ): if option.startswith( 'bic_file_' ) or option.startswith( 'zipcode_file_' ): filename = Config.getfilepath( 'Autocompletion', option ) if os.path.exists( filename ): if option.startswith( 'bic_file_' ): tag = option[9:].upper() Banks.clear( tag ) Banks.load( tag, filename, 'latin1' ) elif option.startswith( 'zipcode_file_' ): tag = option[13:].upper() Cities.clear( tag ) Cities.load( tag, filename, 'utf-8' )
def __init__(self, name, root="wrapper-data/json", pickle=True): # type: (str, str, bool) -> None """ :param name: Name of Storage :param root: Path on disk to storage data :param pickle: Boolean; Pickle (True) or not (False, use Json) """ self.Data = {} self.name = name self.root = root self.pickle = pickle self.configManager = Config() self.configManager.loadconfig() self.log = logging.getLogger('Storage.py') self.encoding = self.configManager.config["General"]["encoding"] self.paused_saving = False self.periodic_save_timer = 60 if self.pickle: self.file_ext = "pkl" else: self.file_ext = "json" self.load() self.timer = time.time() self.abort = False t = threading.Thread(target=self._periodicsave, args=()) t.daemon = True t.start()
def __init__(self, app): super(HashmalMain, self).__init__() self.app = app self.app.setStyleSheet(hashmal_style) self.changes_saved = True self.config = Config() self.config.load() QtCore.QCoreApplication.setOrganizationName('mazaclub') QtCore.QCoreApplication.setApplicationName('hashmal') self.qt_settings = QtCore.QSettings() self.setDockNestingEnabled(True) self.dock_handler = DockHandler(self) self.dock_handler.create_docks() self.dock_handler.do_default_layout() self.script_editor = ScriptEditor(self) self.script_editor.changesSaved.connect(self.on_changes_saved) self.setCentralWidget(self.script_editor) self.create_menubar() self.create_default_script() self.statusBar().setVisible(True) self.statusBar().messageChanged.connect(self.change_status_bar) self.restoreState(self.qt_settings.value('toolLayout/default').toByteArray()) if self.qt_settings.value('quickTipsOnStart', defaultValue=QtCore.QVariant(True)).toBool(): QtCore.QTimer.singleShot(500, self.do_quick_tips)
def _process_url(self, url): #Crawler config load cfgCrawler = Config(os.path.join(RAGPICKER_ROOT, 'config', 'crawler.conf')).get("clientConfig") data = None headers = { 'User-Agent': cfgCrawler.get("browser_user_agent", "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)"), 'Accept-Language': cfgCrawler.get("browser_accept_language", "en-US"), } request = urllib2.Request(url, data, headers) try: url_dl = urllib2.urlopen(request, timeout=30).read() except Exception, e: raise IOError("Thread("+self.threadName+") - %s - Error parsing %s" % (e, url))
def __init__(self): self.cfgReporting = Config(os.path.join(RAGPICKER_ROOT, 'config', 'reporting.conf')) self.vxcageEnabled = self.cfgReporting.getOption("vxcage", "enabled") self.host = self.cfgReporting.getOption("vxcage", "host") self.port = self.cfgReporting.getOption("vxcage", "port") if not self.host or not self.port: raise Exception("VxCage REST API server not configurated")
def __init__(self): # Datenbank self.__database = Database() # Kofiguration aus der reporting.conf holen self.__cfgReporting = Config(os.path.join(RAGPICKER_ROOT, 'config', 'reporting.conf')) self.__vxcageEnabled = self.__cfgReporting.getOption("vxcage", "enabled") self.__vxcageHost = self.__cfgReporting.getOption("vxcage", "host") self.__vxcagePort = self.__cfgReporting.getOption("vxcage", "port")
def show( self ): """ Shows the dialog. """ # DB info name, driver, version = core.database.Database.get_engine_info() version = '.'.join( [str( x ) for x in version] ) data = "%i Magazine\n%i Vertragstypen\n%i Ausgaben\n%i Kunden\n%i Adressen\n%i Bankkonten\n%i Verträge" % ( core.database.Magazine.count(), core.database.Subscription.count(), core.database.Issue.count(), core.database.Customer.count(), core.database.Address.count(), core.database.Bankaccount.count(), core.database.Contract.count() ) self.builder.get_object( 'general_database_name_label' ).set_text( name ) self.builder.get_object( 'general_database_driver_label' ).set_text( driver ) self.builder.get_object( 'general_database_version_label' ).set_text( version ) self.builder.get_object( 'general_database_data_label' ).set_text( data ) self.builder.get_object( 'general_bic_open_entry' ).set_text( Config.get( "Autocompletion", "bic_file_de" ) ) self.builder.get_object( 'general_zipcode_open_entry' ).set_text( Config.get( "Autocompletion", "zipcode_file_de" ) ) self._magazinemanager.start_edit() self.builder.get_object( "content" ).show_all()
def runCrawler(): mapURL = {} cfgCrawler = Config(os.path.join(RAGPICKER_ROOT, 'config', 'crawler.conf')) #TOR Socks proxy isTorEnabled = cfgCrawler.get("clientConfig").get("tor_enabled", False) if isTorEnabled: torProxyAdress = cfgCrawler.get("clientConfig").get("tor_proxyadress", "localhost") torProxyPort = cfgCrawler.get("clientConfig").get("tor_proxyport", 9050) # Route an HTTP request through the SOCKS proxy socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, torProxyAdress, torProxyPort) socket.socket = socks.socksocket # Build the PluginManager crawlerPluginManager = PluginManager() crawlerPluginManager.setPluginPlaces(["crawler"]) crawlerPluginManager.collectPlugins() # Trigger run from the "Crawler" plugins for pluginInfo in crawlerPluginManager.getAllPlugins(): crawlerModul = pluginInfo.plugin_object #Config for crawler module try: options = cfgCrawler.get(pluginInfo.name) crawlerModul.set_options(options) except Exception: log.error("Crawler module %s not found in configuration file", pluginInfo.name) # If the crawler module is disabled in the config, skip it. if not options.enabled: continue log.info("Run Crawler: " + pluginInfo.name) try: returnMap = crawlerModul.run() mapURL.update(returnMap) except Exception as e: log.error('Error (%s) in %s', e, pluginInfo.name) return mapURL
class TestConfig(unittest2.TestCase): def setUp(self): self.config = Config("tests/test_config.ini") def test_sort_sections(self): self.assertEqual(list(self.config.sort_sections()), ["lvrj.com/blogs", "lvrj.com"]) def test_domain_lookup(self): config = self.config["lvrj.com"] self.assertEqual(config, {"date": "today"}) config = self.config["not here"] self.assertEqual(config, {})
def printRagpickerInfos(print_options=False): infolog = logging.getLogger("Info") infolog.info(color("RAGPICKER_VERSION: " + RAGPICKER_VERSION, RED)) infolog.info(color("RAGPICKER_BUILD_DATE: " + RAGPICKER_BUILD_DATE, RED)) infolog.info(color("RAGPICKER_ROOT: " + RAGPICKER_ROOT, RED)) infolog.info("") pluginPlaces = ["crawler", "preProcessing", "processing", "reporting"] for place in pluginPlaces: infolog.info(color("%s| " % (place + " ").upper().ljust(14, '-'), MAGENTA)) cfg = Config(os.path.join(RAGPICKER_ROOT, 'config', place + '.conf')) ragpickerPluginManager = PluginManager() ragpickerPluginManager.setPluginPlaces([place]) ragpickerPluginManager.collectPlugins() for pluginInfo in sorted(ragpickerPluginManager.getAllPlugins(), key=lambda PluginInfo: PluginInfo.name): options = cfg.get(pluginInfo.name) if options.enabled: infolog.info(color("%s V%s %s", MAGENTA) % (" |----[+] " + pluginInfo.name.ljust(25), pluginInfo.version, pluginInfo.description)) if print_options: for key, value in options.iteritems(): if key != "enabled": infolog.info(color(" |-- %s = %s", MAGENTA) % (key, str(value))) else: infolog.info(color("%s V%s %s", BLUE) % (" |----[-] " + pluginInfo.name.ljust(25), pluginInfo.version, pluginInfo.description)) infolog.info("") infolog.info(" %s %s" % (color("([+] = enabled)", MAGENTA), color("([-] = disabled)", BLUE))) infolog.info("") checkVersion() sys.stdout.flush()
def __init__(self, module, page, pagesize, keyword): self.config = Config() self.module = module self.page = int(page) self.keyword = keyword self.pageSize = int(pagesize) self.saveFile = self.config.getValue("global", "savefile") if self.saveFile == 'True': self.outfile = OutFile(unquote(self.keyword)) else: self.outfile = None self.collection()
def on_finished( self, rendering_results ): template = self._template_env.get_template( "{}.template".format( "base" ) ) rendered_letters = [rendered_letter for rendered_letter in rendering_results if rendered_letter is not None] if len( rendered_letters ) == 0: raise ValueError( "No rendered letters" ) lco_template = Config.get("LetterRenderer", "lco_template") if not lco_template: lco_template = "a4paper" rendered_document = template.render( {'lco_template': lco_template, 'prerendered_letters':rendered_letters} ) dirs = [paths.config('templates', 'latex'), paths.config('images'), paths.data('templates', 'latex'), paths.data('images')] textinputs = [dir for dir in dirs if os.path.exists(dir)] pdflatex.compile_str(rendered_document, self._output_file, textinputs)
class Collect(object): def __init__(self, module, page, pagesize, keyword): self.config = Config() self.module = module self.page = int(page) self.keyword = keyword self.pageSize = int(pagesize) self.saveFile = self.config.getValue("global", "savefile") if self.saveFile == 'True': self.outfile = OutFile(unquote(self.keyword)) else: self.outfile = None self.collection() def collection(self): for i in range(self.page): print("\033[1;37;40m[*]Search Engine [%s],Page [%s] Start collecting." % (self.module, i+1)) page_pn = (i * self.pageSize) if self.module == "baidu": my_baidu = Baidu(self.outfile) my_baidu.search(self.keyword, self.pageSize, page_pn) elif self.module == "so": my_so = So(self.outfile) my_so.search(self.keyword, i+1) elif self.module == "sougou": my_sougou = Sougou(self.outfile) my_sougou.search(self.keyword, i+1) if self.config.sleeptime > 0: time.sleep(self.config.sleeptime) if self.outfile == 'True': self.outfile.closeFile()
def start(self): if not UPSDModule.start(self): return False self.conf = Config() self.userconf = {} ### Load info for pseudo client from config ### self.userconf['nick'] = self.conf.getOption(self.modname, 'nick') self.userconf['ident'] = self.conf.getOption(self.modname, 'ident') self.userconf['host'] = self.conf.getOption(self.modname, 'host') self.userconf['modes'] = self.conf.getOption(self.modname, 'modes') self.userconf['realname'] = self.conf.getOption(self.modname, 'realname') self.userconf['mainchan'] = self.conf.getOption(self.modname, 'mainchan') self.proto.sendNickCreate(self.userconf['nick'], self.userconf['ident'], self.userconf['host'], self.userconf['modes'], self.userconf['realname']) self.proto.sendJoin(self.userconf['nick'], self.userconf['mainchan']) return True
def __init__( self ): self._invoicetable = msmgui.widgets.invoicetable.InvoiceTable() msmgui.widgets.base.RefreshableWindow.__init__( self, [self._invoicetable] ) self.builder = Gtk.Builder() self.builder.add_from_file( paths.data("ui","widgets","invoicewindow", "invoicewindow.glade")) self.builder.get_object( "content" ).reparent( self ) self.builder.connect_signals( self ) self.builder.get_object( "tablebox" ).add( self._invoicetable ) self._invoicetable.connect( "selection-changed", self.invoicetable_selection_changed_cb ) self._invoicingassistant = msmgui.assistants.invoicing.InvoicingAssistant() self._invoicingassistant.connect( "saved", self.invoicingassistant_saved_cb ) self._letterexportassistant = msmgui.assistants.letterexport.LetterExportAssistant() active_only = Config.getboolean( "Interface", "active_only" ) if not active_only: self.builder.get_object( "invoices_showall_switch" ).set_active( not active_only )
def __init__( self ): self._customertable = msmgui.widgets.customertable.CustomerTable() msmgui.widgets.base.RefreshableWindow.__init__( self, [self._customertable] ) self.builder = Gtk.Builder() self.builder.add_from_file( paths.data("ui","widgets","customerwindow", "customerwindow.glade")) self.builder.add_from_file( "data/ui/widgets/customerwindow/customerwindow.glade" ) self.builder.get_object( "content" ).reparent( self ) self.builder.connect_signals( self ) self._customereditor = msmgui.widgets.customereditor.CustomerEditor() self.builder.get_object( "editorbox" ).add( self._customereditor ) self._customereditor.connect( "edit-started", self.customereditor_edit_started_cb ) self._customereditor.connect( "edit-ended", self.customereditor_edit_ended_cb ) self._customereditor.connect( "changed", self.customereditor_changed_cb ) self._customereditor.connect( "save", self.customereditor_saved_cb ) self._customereditor.connect( "expanded-changed", self.customereditor_expanded_changed_cb ) self.builder.get_object( "tablebox" ).add( self._customertable ) self._customertable.connect( "selection-changed", self.customertable_selection_changed_cb ) active_only = Config.getboolean( "Interface", "active_only" ) if not active_only: self.builder.get_object( "customers_showall_switch" ).set_active( not active_only )
class Storage(object): def __init__(self, name, root="wrapper-data/json", encoding="default", pickle=True): self.Data = {} self.name = name self.root = root self.pickle = pickle self.configManager = Config() self.configManager.loadconfig() self.log = logging.getLogger('Storage.py') if encoding == "default": self.encoding = self.configManager.config["General"]["encoding"] else: self.encoding = encoding if self.pickle: self.file_ext = "pkl" else: self.file_ext = "json" self.load() self.timer = time.time() self.abort = False t = threading.Thread(target=self.periodicsave, args=()) t.daemon = True t.start() def periodicsave(self): # doing it this way (versus just sleeping for 60 seconds), # allows faster shutdown response while not self.abort: if time.time() - self.timer > 60: self.save() self.timer = time.time() time.sleep(1) def load(self): mkdir_p(self.root) if not os.path.exists("%s/%s.%s" % (self.root, self.name, self.file_ext)): # load old json storages if there is no pickled # file (and if storage is using pickle) if self.pickle: self.Data = self.json_load() # save to the selected file mode (json or pkl) self.save() if self.pickle: self.Data = self.pickle_load() else: self.Data = self.json_load() def save(self): if not os.path.exists(self.root): mkdir_p(self.root) if self.pickle: self.pickle_save() else: self.json_save() def pickle_save(self): if "human" in self.encoding.lower(): _protocol = 0 else: # using something less than HIGHEST allows both Pythons 2/3 # to use the files interchangeably. It should also allow # moving files between machines with different configurations # with fewer issues. # # Python 2 cPickle does not have a DEFAULT_PROTOCOL # constant like Python 3 pickle (else I would just # use the Default (currently 3, I believe). # # This will probably use either 1 or 2 depending on # which python you use. # # We imported either pickle (Py3) or cPickle (Py2) depending # on what wrapper detected. Both are implemented in C for # speed. # # The MAIN POINT: # I wanted the code to use something better/faster than # Human-readable (unless that is what you specify), while # still permitting some portability of the final files _protocol = Pickle.HIGHEST_PROTOCOL // 2 with open("%s/%s.%s" % (self.root, self.name, self.file_ext), "wb") as f: Pickle.dump(self.Data, f, protocol=_protocol) def json_save(self): putcode = putjsonfile(self.Data, self.name, self.root) if not putcode: self.log.exception( "Error encoutered while saving json data:\n'%s/%s.%s'" "\nData Dump:\n%s" % (self.root, self.name, self.file_ext, self.Data)) def pickle_load(self): with open("%s/%s.%s" % (self.root, self.name, self.file_ext), "rb") as f: return Pickle.load(f) def json_load(self): try_load = getjsonfile(self.name, self.root, encodedas=self.encoding) if try_load in (None, False): self.log.exception("bad/non-existent file or data '%s/%s.%s'" % (self.root, self.name, self.file_ext)) return {} else: return try_load def close(self): self.abort = True self.save()
class Statistics(): def __init__(self): # Datenbank self.__database = Database() # Kofiguration aus der reporting.conf holen self.__cfgReporting = Config(os.path.join(RAGPICKER_ROOT, 'config', 'reporting.conf')) self.__vxcageEnabled = self.__cfgReporting.getOption("vxcage", "enabled") self.__vxcageHost = self.__cfgReporting.getOption("vxcage", "host") self.__vxcagePort = self.__cfgReporting.getOption("vxcage", "port") def runStatisticsLong(self): #Pruefen ob VxCage und MongoDB aktiviert sind if self.__database.isRagpickerDBEnabled(): if self.__vxcageEnabled: self.__runStatisticsMongodbLong() if self.__database.isCodeDBEnabled(): self.__runStatisticsCodedb() else: print("vxcage in reporting.conf is not enabled") sys.stdout.flush() else: print("mongodb in reporting.conf is not enabled") sys.stdout.flush() def runStatisticsShort(self): #Pruefen ob VxCage und MongoDB aktiviert sind if self.__database.isRagpickerDBEnabled(): if self.__vxcageEnabled: self.__runStatisticsMongodbShort() if self.__database.isCodeDBEnabled(): self.__runStatisticsCodedb() else: print("vxcage in reporting.conf is not enabled") sys.stdout.flush() else: print("mongodb in reporting.conf is not enabled") sys.stdout.flush() def runStatisticsAV(self): #Pruefen ob VxCage und MongoDB aktiviert sind if self.__database.isRagpickerDBEnabled(): if self.__vxcageEnabled: self.__runStatisticsAV() else: print("vxcage in reporting.conf is not enabled") sys.stdout.flush() else: print("mongodb in reporting.conf is not enabled") sys.stdout.flush() def __runStatisticsMongodbLong(self): print "**************************************" print "*** Statistics MongoDB (Ragpicker) ***" print "**************************************" print "" print "Number of malware samples in database:", self.__database.countReportsRagpickerDB() print "" #Statistiken der eingesetzten AV-Produkte self.__runStatisticsAVProducts() #Liste der letzen 20 Samples, die weder auf VT noch von einem lokalen AV gefunden wurden self.__runStatisticsLast20SamplesNotFoundByAV() #Liste der letzen 20 Samples, die nicht auf VT gefunden wurden self.__runStatisticsLast20SamplesNotFoundByVT() #Liste der letzen 20 Samples, die nicht von einem lokalen AV-Produkt gefunden wurden self.__runStatisticsLast20SamplesNotFoundByLocalAV() #Liste und Haeufigkeit der Filetypes self.__runStatisticsFiletypes() #Haeufigkeit der PE Charakteristiken self.__runStatisticsPeCharacteristics() #Liste und Haeufigkeit der verwendeten Packer/Compiler in der Malware self.__runStatisticsPackerCompiler() #Liste der verwendeten digitalen Signaturen self.__runStatisticsPackerSignatures() sys.stdout.flush() def __runStatisticsMongodbShort(self): print "**************************************" print "*** Statistics MongoDB (Ragpicker) ***" print "**************************************" print "" print "Number of malware samples in database:", self.__database.countReportsRagpickerDB() print "" #Liste und Haeufigkeit der Filetypes self.__runStatisticsFiletypes() #Haeufigkeit der PE Charakteristiken self.__runStatisticsPeCharacteristics() sys.stdout.flush() def __runStatisticsAV(self): print "**************************************" print "*** Statistics MongoDB (Ragpicker) ***" print "**************************************" print "" print "Number of malware samples in database:", self.__database.countReportsRagpickerDB() print "" #Statistiken der eingesetzten AV-Produkte self.__runStatisticsAVProducts() #Liste der letzen 20 Samples, die weder auf VT noch von einem lokalen AV gefunden wurden self.__runStatisticsLast20SamplesNotFoundByAV() #Liste der letzen 20 Samples, die nicht auf VT gefunden wurden self.__runStatisticsLast20SamplesNotFoundByVT() #Liste der letzen 20 Samples, die nicht von einem lokalen AV-Produkt gefunden wurden self.__runStatisticsLast20SamplesNotFoundByLocalAV() sys.stdout.flush() def __runStatisticsFiletypes(self): #Liste und Haeufigkeit der Filetypes print "Filetypes of malware" res = self.__database.getFiletypes() table = PrettyTable(["filetype", "count"]) table.align["filetype"] = "l" table.align["count"] = "c" table.padding_width = 1 try: for values in res['result']: if values.get("_id"): outputPacker = values.get("_id") outputCount = str(values.get("count")) table.add_row([outputPacker, outputCount]) print(table) except KeyError: raise Exception("Dict has no key 'result' ") print "" def __runStatisticsPeCharacteristics(self): #Haeufigkeit der PE Charakteristiken print "PE-Characteristics of malware" peC = self.__database.getStatisticsPeCharacteristics() table = PrettyTable(["pe-characteristics", "count"]) table.align["pe-characteristics"] = "l" table.align["count"] = "c" table.padding_width = 1 table.add_row(["EXE", peC.get("exe")]) table.add_row(["DLL", peC.get("dll")]) table.add_row(["Driver", peC.get("driver")]) table.add_row(["DLL/Driver", peC.get("dllDriver")]) table.add_row(["No PE File", peC.get("noPe")]) print (table) print "" def __runStatisticsPackerCompiler(self): #Liste und Haeufigkeit der verwendeten Packer/Compiler in der Malware print "Packer/compiler used in malware" res = self.__database.getStatisticsPackerCompiler() table = PrettyTable(["packer/compiler", "count"]) table.align["packer/compiler"] = "l" table.align["count"] = "c" table.padding_width = 1 try: for values in res['result']: if values.get("_id"): outputPacker = values.get("_id")[0] outputCount = str(values.get("count")) table.add_row([outputPacker, outputCount]) print(table) except KeyError: raise Exception("Dict has no key 'result' ") print " " def __runStatisticsPackerSignatures(self): #Liste der verwendeten digitalen Signaturen print "Signatures used by malware" res = self.__database.getStatisticsPackerSignatures() table = PrettyTable(["publisher", "issuer", "count"]) table.align["publisher"] = "l" table.align["issuer"] = "l" table.align["count"] = "c" table.padding_width = 1 try: for values in res['result']: if values.get("_id"): outputPublisher = values.get("_id").get("PublisherO") if values.get("_id").get("Issuer"): outputIssuer = values.get("_id").get("Issuer") else: outputIssuer = " " outputCount = str(values.get("count")) table.add_row([outputPublisher, outputIssuer, outputCount]) print(table) except KeyError: raise Exception("Dict has no key 'result' ") print "" def __runStatisticsLast20SamplesNotFoundByAV(self): #Liste der letzen 20 Samples, die weder auf VT noch von einem lokalen AV gefunden wurden print "Last 20 samples not found by VirusTotal and local AV-Products" res = sorted(self.__database.getSamplesNotFoundByAV(), reverse=True) table = PrettyTable(["timestamp of crawling", "sha256"]) table.align["timestamp of crawling"] = "c" table.align["sha256"] = "c" table.padding_width = 1 try: for values in res: sha256 = values.get("Info").get("file").get("sha256") timestamp = values.get("Info").get("analyse").get("started") table.add_row([timestamp, sha256]) print(table.get_string(start=0, end=20)) except KeyError: raise Exception("Dict has no key 'Info' ") print "" def __runStatisticsLast20SamplesNotFoundByVT(self): #Liste der letzen 20 Samples, die nicht auf VirusTotal gefunden wurden print "Last 20 samples not found by VirusTotal" res = sorted(self.__database.getSamplesNotFoundByVT(), reverse=True) table = PrettyTable(["timestamp of crawling", "sha256"]) table.align["timestamp of crawling"] = "c" table.align["sha256"] = "c" table.padding_width = 1 try: for values in res: sha256 = values.get("Info").get("file").get("sha256") timestamp = values.get("Info").get("analyse").get("started") table.add_row([timestamp, sha256]) print(table.get_string(start=0, end=20)) except KeyError: raise Exception("Dict has no key 'Info' ") print "" def __runStatisticsLast20SamplesNotFoundByLocalAV(self): #Liste der letzen 20 Samples, die nicht von einem lokalen AV-Produkt gefunden wurden print "Last 20 samples not found by local AV-Products" res = sorted(self.__database.getSamplesNotFoundByLocalAV(), reverse=True) table = PrettyTable(["timestamp of crawling", "sha256"]) table.align["timestamp of crawling"] = "c" table.align["sha256"] = "c" table.padding_width = 1 try: for values in res: sha256 = values.get("Info").get("file").get("sha256") timestamp = values.get("Info").get("analyse").get("started") table.add_row([timestamp, sha256]) print(table.get_string(start=0, end=20)) except KeyError: raise Exception("Dict has no key 'Info' ") print "" def __runStatisticsAVProducts(self): #Statistiken der eingesetzten AV-Produkte #VirusTotal und lokale AV-Produkte print "VirusTotal and local AV-Products" print " Samples rated as none-malware by all AV-Products at time of crawling:", \ self.__database.getStatisticsNoneMalwareByAV() print "" #VirusTotal ret = self.__database.getStatisticsVirusTotal() print "VirusTotal" print " Samples analyzed at time of crawling:", ret.get("analyzed") print " Samples not analyzed at time of crawling:", ret.get("notAnalyzed") print " Samples found at time of crawling:", ret.get("samplesFound") print " Samples not found at time of crawling:", ret.get("SamplesNotFound") print "" #Lokale AV-Produkte print "Local AV-Products" print " analyzed => Samples analyzed at time of crawling" print " not analyzed => Samples not analyzed at time of crawling" print " malware => Samples rated as malware at time of crawling" print " none-malware => Samples rated as none-malware at time of crawling" table = PrettyTable(["product", "analyzed", "not analyzed", "malware", "none-malware", "detection rate"]) table.align["product"] = "l" table.align["analyzed"] = "r" table.align["not analyzed"] = "r" table.align["malware"] = "r" table.align["none-malware"] = "r" table.align["detection rate"] = "r" table.padding_width = 1 # Statistik Daten holen ret = self.__database.getStatisticsAntivirus() # Table-Body zusammenbauen for av in ret: table.add_row([av.get("product"), av.get("analyzed"), av.get("notanalyzed"), av.get("malware"), av.get("nonemalware"), av.get("rate")]) print(table) print ""
async def post(self): code = self.post_data.get('code', None) if not code: raise HTTPError(400, "code为空?", reason="code为空?") # fetch userinfo from wechat server wx_pb_userinfo = await WxCCServer().open_app_login( code, self.request_id) if wx_pb_userinfo is None: raise HTTPError(500, "登录微信服务器出错", reason="登录微信服务器出错") wx_userinfo_dict = MessageToDict(wx_pb_userinfo, including_default_value_fields=True, preserving_proto_field_name=True, use_integers_for_enums=True) # get account exists_wechat = await AccountServer().exists_partner_wechat( identifier=wx_pb_userinfo.unionid, app_id=self.app_id_int64, request_id=self.request_id) if not exists_wechat: # create account pb_account = await AccountServer().create_account_by_partner( partner=pbaccount_pb2.PARTNER_WECHAT, identifier=wx_pb_userinfo.unionid, origin_data=wx_userinfo_dict, app_id=self.app_id_int64, request_id=self.request_id) # 分配空间 await MembershipServer().set_total_storage_change( account_id=pb_account.account_id, changed_value=Config().membership.get("register_default_size"), title=Config().membership.get("register_default_title"), details=Config().membership.get("register_default_details"), request_id=self.request_id) else: # auth the wechat account pb_account = await AccountServer().auth_partner_account( wx_pb_userinfo.unionid, app_id=self.app_id_int64, request_id=self.request_id) # update partner info await AccountServer().update_partner_account_origin_data( pb_account.auth_token, pbaccount_pb2.PARTNER_WECHAT, wx_pb_userinfo.unionid, wx_userinfo_dict, self.app_id_int64, self.request_id) # set account pair to wxcc server await WxCCServer().open_app_set_account_pair( openid=wx_pb_userinfo.openid, account_id=pb_account.account_id, request_id=self.request_id) # get userinfo pb_current_userinfo = await AccountServer().get_userinfo( pb_account.account_id, self.app_id_int64, self.request_id) # update user info params_people = dict() if not pb_current_userinfo.nickname: params_people["nickname"] = wx_pb_userinfo.nickname if not pb_current_userinfo.gender: params_people["gender"] = str(wx_pb_userinfo.gender) if params_people: await AccountServer().update_user_info( account_id=pb_account.account_id, params=params_people, app_id=self.app_id_int64, request_id=self.request_id) if not pb_current_userinfo.avatar: # generate avatar filename zid = await ZIDServer().generate(self.request_id) # upload filename, err = await backend.backend_user_media.transmit_wechat_avatar( zid=zid, remote_url=wx_pb_userinfo.avatar) if not err and filename: # update avatar await AccountServer().update_user_avatar( account_id=pb_account.account_id, app_id=self.app_id_int64, request_id=self.request_id, avatar=filename) account = Account(pb_account) pb_userinfo = await AccountServer().get_userinfo( pb_account.account_id, self.app_id_int64, self.request_id) await account.set_userinfo(pb_userinfo) self.send_to_client(0, message='登录成功', response=account.dump())
async def post(self): try: # upload file avatar_files = self.request.files.get('file') if not avatar_files: self.send_to_client(error_id=400, message='请上传头像') return avatar_file = avatar_files[0] # save to temp file tempfile = Config().uploads_temp_path + str(uuid.uuid4()) fh = open(tempfile, 'wb') fh.write(avatar_file.get('body')) fh.close() # upload zid = await ZIDServer().generate(self.request_id) filename, err = await backend.backend_user_media.upload_avatar( zid, tempfile, "square") # remove temp file try: os.remove(tempfile) except: pass if err: self.send_to_client(error_id=500, message='头像上传失败。错误信息:' + err) return if not filename: self.send_to_client(error_id=500, message='头像上传失败(未知错误),请联系管理员解决') return # 更新记录 pb_userinfo = await AccountServer().get_userinfo( account_id=self.current_user.account_id, app_id=self.app_id_int64, request_id=self.request_id) # 删除原头像 if pb_userinfo: if pb_userinfo.avatar: await backend.backend_user_media.delete_object( pb_userinfo.avatar) pb_userinfo.avatar = filename # update avatar await AccountServer().update_user_avatar( account_id=self.current_user.account_id, app_id=self.app_id_int64, request_id=self.request_id, avatar=filename) account = self.current_user await account.set_userinfo(pb_userinfo) self.send_to_client(0, message='头像上传成功', response=account.dump()) return except grpc.RpcError as e: # except grpc._channel._Rendezvous as e: msg = "code: {}, details:{}".format(e.code(), e.details()) raise HTTPError(500, msg, reason=str(e.details()))
async def post(self): mobile = self.post_data.get('mobile', '') smscode = self.post_data.get('smscode', '') errors = self._check_input({"mobile": mobile, "smscode": smscode}) if errors: err = '' for item, er in errors.items(): err = er break self.send_to_client(error_id=error.input_error.id, message=err) return # Check smscode mobile_smscode = rd.redis_get('verify:{}'.format(mobile)) if mobile_smscode: mobile_smscode = mobile_smscode.decode() if mobile_smscode != smscode: self.send_to_client(-1, '登陆验证码错误') return # Exists Mobile exists_account_already = False try: exists_account_already = await AccountServer( ).exists_account_by_mobile(mobile, request_id=self.request_id) except grpc.RpcError as e: self.send_to_client(error_id=500, message='服务器内部错误,请联系管理员确认') return pb_account = None if not exists_account_already: # not exists, create account pb_account = await AccountServer().create_account( mobile=mobile, country_code="86", app_id=self.app_id_int64, request_id=self.request_id) # 分配空间 result = await MembershipServer().set_total_storage_change( account_id=pb_account.account_id, changed_value=Config().membership.get("register_default_size"), title=Config().membership.get("register_default_title"), details=Config().membership.get("register_default_details"), request_id=self.request_id) else: # exists, get the account pb_account = await AccountServer().get_account_by_mobile( mobile=mobile, country_code="86", app_id=self.app_id_int64, request_id=self.request_id) # make Account Model if pb_account is None: self.send_to_client(error_id=500, message='服务器内部错误,请联系管理员确认') return account = Account(pb_account) # get account userinfo if account.status == pbaccount_pb2.STATUS_ACTIVATED: rd.redis_del('verify:{}'.format(mobile)) pb_userinfo = await AccountServer().get_userinfo( account.account_id, app_id=self.app_id_int64, request_id=self.request_id) await account.set_userinfo(pb_userinfo) self.send_to_client(0, message='登陆成功', response=account.dump()) return status_error = {0: "未知状态的账号", 2: "未激活账号", 3: "锁定并暂停账号或封号", 4: "账号已注销"} self.send_to_client(-1, message=status_error.get(account.status))
class HashmalMain(QMainWindow): def __init__(self, app): super(HashmalMain, self).__init__() self.app = app self.app.setStyleSheet(hashmal_style) self.changes_saved = True self.config = Config() self.config.load() QtCore.QCoreApplication.setOrganizationName('mazaclub') QtCore.QCoreApplication.setApplicationName('hashmal') self.qt_settings = QtCore.QSettings() self.setDockNestingEnabled(True) self.dock_handler = DockHandler(self) self.dock_handler.create_docks() self.dock_handler.do_default_layout() self.script_editor = ScriptEditor(self) self.script_editor.changesSaved.connect(self.on_changes_saved) self.setCentralWidget(self.script_editor) self.create_menubar() self.create_default_script() self.statusBar().setVisible(True) self.statusBar().messageChanged.connect(self.change_status_bar) self.restoreState(self.qt_settings.value('toolLayout/default').toByteArray()) if self.qt_settings.value('quickTipsOnStart', defaultValue=QtCore.QVariant(True)).toBool(): QtCore.QTimer.singleShot(500, self.do_quick_tips) def sizeHint(self): return QtCore.QSize(800, 500) def create_default_script(self): filename = os.path.expanduser('Untitled.coinscript') self.load_script(filename) def create_menubar(self): menubar = QMenuBar() file_menu = menubar.addMenu('&File') file_menu.addAction('&New', self.new_script).setShortcut(QKeySequence.New) file_menu.addAction('Save As...', self.save_script_as).setShortcut(QKeySequence.SaveAs) file_menu.addAction('&Open', self.open_script).setShortcut(QKeySequence.Open) file_menu.addAction('&Save', self.save_script).setShortcut(QKeySequence.Save) file_menu.addAction('&Quit', self.close) # Script actions script_menu = menubar.addMenu('&Script') script_menu.addAction('&Evaluate', self.dock_handler.evaluate_current_script) # Settings and tool toggling tools_menu = menubar.addMenu('&Tools') tools_menu.addAction('&Settings', lambda: SettingsDialog(self).exec_()) tools_menu.addSeparator() for i in sorted(self.dock_handler.dock_widgets): tools_menu.addAction(i.toggleViewAction()) help_menu = menubar.addMenu('&Help') help_menu.addAction('&About', self.do_about) help_menu.addAction('&Tool Info', lambda: ToolInfo(self).exec_()) help_menu.addAction('&Quick Tips', self.do_quick_tips) self.setMenuBar(menubar) def show_status_message(self, msg, error=False): self.statusBar().showMessage(msg, 3000) if error: self.statusBar().setProperty('hasError', True) else: self.statusBar().setProperty('hasError', False) self.style().polish(self.statusBar()) def change_status_bar(self, new_msg): # Unset hasError if an error is removed. if not new_msg and self.statusBar().property('hasError'): self.statusBar().setProperty('hasError', False) self.style().polish(self.statusBar()) def on_changes_saved(self, saved): title = ''.join(['Hashmal - ', self.script_editor.filename]) if not saved: title = ''.join([title, ' *']) self.setWindowTitle(title) self.changes_saved = saved def closeEvent(self, event): # Save layout if configured to. if self.qt_settings.value('saveLayoutOnExit', defaultValue=QtCore.QVariant(False)).toBool(): self.qt_settings.setValue('toolLayout/default', self.saveState()) if self.changes_saved or (not self.script_editor.filename and not str(self.script_editor.script_edit.toPlainText())): event.accept() return result = QMessageBox.question(self, 'Save Changes', 'Do you want to save your changes to ' + self.script_editor.filename + ' before closing?', QMessageBox.Yes | QMessageBox.No) if result == QMessageBox.Yes: self.save_script() event.accept() def new_script(self, filename=''): if not filename: base_name = ''.join(['Untitled-', str(time.time()), '.coinscript']) filename = os.path.expanduser(base_name) self.load_script(filename) def save_script(self): filename = self.script_editor.filename if not filename: filename = str(QFileDialog.getSaveFileName(self, 'Save script', filter=script_file_filter)) if not filename: return if not filename.endswith('.coinscript'): filename += '.coinscript' self.script_editor.filename = filename self.script_editor.save() def save_script_as(self): filename = str(QFileDialog.getSaveFileName(self, 'Save script as', filter=script_file_filter)) if not filename: return if not filename.endswith('.coinscript'): filename += '.coinscript' self.script_editor.filename = filename self.script_editor.save() def open_script(self): filename = str(QFileDialog.getOpenFileName(self, 'Open script', '.', filter=script_file_filter)) if not filename: return # Confirm discarding changes if an unsaved file is open. if (self.script_editor.filename and str(self.script_editor.script_edit.toPlainText()) and filename != self.script_editor.filename and not self.changes_saved): result = QMessageBox.question(self, 'Save Changes', 'Do you want to save your changes to ' + self.script_editor.filename + ' before closing?', QMessageBox.Yes | QMessageBox.No) if result == QMessageBox.Yes: self.save_script() self.load_script(filename) def load_script(self, filename): self.setWindowTitle('Hashmal - ' + filename) self.script_editor.load(filename) def do_about(self): d = QDialog(self) vbox = QVBoxLayout() about_label = QLabel() about_label.setWordWrap(True) txt = [] txt.append(' '.join([ 'Hashmal is an IDE for Bitcoin transaction scripts.', 'Its purpose is to make it easier to write, evaluate, and learn about transaction scripts.' ])) txt.append('Hashmal is intended for cryptocurrency developers and power users.') txt.append('Use at own risk!') txt = '\n\n'.join(txt) about_label.setText(txt) close_button = QPushButton('Close') close_button.clicked.connect(d.close) btn_box = floated_buttons([close_button]) vbox.addWidget(about_label) vbox.addLayout(btn_box) d.setLayout(vbox) d.setWindowTitle('About Hashmal') d.exec_() def do_quick_tips(self): QuickTips(self).exec_()
class Database: def __init__(self): self.__cfgReporting = Config(os.path.join(RAGPICKER_ROOT, "config", "reporting.conf")) self.__cfgProcessing = Config(os.path.join(RAGPICKER_ROOT, "config", "processing.conf")) self.__mongodbEnabled = self.__cfgReporting.getOption("mongodb", "enabled") self.__codedbEnabled = self.__cfgReporting.getOption("codeDB", "enabled") self.__bluecoatEnabled = self.__cfgProcessing.getOption("all_bluecoatMalwareAnalysisAppliance", "enabled") if self.__mongodbEnabled: # Anbindung an Datenbank MongoDB Collection Ragpicker herstellen try: mongodbHost = self.__cfgReporting.getOption("mongodb", "host") mongodbPort = self.__cfgReporting.getOption("mongodb", "port") self.__mongodbConnection = MongoClient(mongodbHost, mongodbPort) self.__mongodbCollectionRagpicker = self.__mongodbConnection.MalwareAnalyse.ragpicker self.__mongodbCollectionFamilies = self.__mongodbConnection.MalwareAnalyse.families self.__mongodbCollectionSandboxTaskQueue = self.__mongodbConnection.MalwareAnalyse.sandboxTaskQueue except TypeError: raise Exception("MongoDB connection port in report.config must be integer") except ConnectionFailure: raise Exception("Cannot connect to MongoDB (ragpicker)") if self.__codedbEnabled: # Anbindung an Datenbank MongoDB Collection CodeDB herstellen try: codedbHost = self.__cfgReporting.getOption("codeDB", "mongo_db_host") codedbPort = self.__cfgReporting.getOption("codeDB", "mongo_db_port") self.__codedbConnection = MongoClient(codedbHost, codedbPort) self.__codedbCollectionCodedb = self.__codedbConnection.MalwareAnalyse.codeDB except TypeError: raise Exception("MongoDB connection port for CodeDB in report.config must be integer") except ConnectionFailure: raise Exception("Cannot connect to MongoDB (codeDB)") def __del__(self): if self.__mongodbEnabled: self.__mongodbConnection.disconnect() if self.__codedbEnabled: self.__codedbConnection.disconnect() # ------------------------------------------------------------------------------ # Ragpicker Database (MongoDB) # ------------------------------------------------------------------------------ def isRagpickerDBEnabled(self): return self.__mongodbEnabled def getStatisticsAntivirus(self): queries = [] ret = [] queries.append( { "product": "Avast Antivirus", "findStr1": "AntivirusScanAvast", "findStr2": "AntivirusScanAvast.avast", "ok": "OK", } ) queries.append( {"product": "AVG Antivirus", "findStr1": "AntivirusScanAvg", "findStr2": "AntivirusScanAvg.Avg", "ok": "OK"} ) queries.append( { "product": "Avira", "findStr1": "AntivirusScanAvira", "findStr2": "AntivirusScanAvira.Avira.scan", "ok": "OK", } ) queries.append( { "product": "BitDefender", "findStr1": "AntivirusScanBitDefender", "findStr2": "AntivirusScanBitDefender.BitDefender", "ok": "OK", } ) queries.append( { "product": "ClamAV", "findStr1": "AntivirusScanClamAv", "findStr2": "AntivirusScanClamAv.ClamAv", "ok": " OK", } ) queries.append( { "product": "COMODO", "findStr1": "AntivirusScanCOMODO", "findStr2": "AntivirusScanCOMODO.COMODO", "ok": "OK", } ) queries.append( {"product": "ESET", "findStr1": "AntivirusScanESET", "findStr2": "AntivirusScanESET.ESET", "ok": "OK"} ) queries.append( {"product": "F-Prot", "findStr1": "AntivirusScanFProt", "findStr2": "AntivirusScanFProt.FProt", "ok": "OK"} ) queries.append( { "product": "F-Secure", "findStr1": "AntivirusScanF-Secure", "findStr2": "AntivirusScanF-Secure.F-Secure", "ok": "OK", } ) for q in queries: av = {} av["product"] = q.get("product") av["analyzed"] = str(self.__mongodbCollectionRagpicker.find({q.get("findStr1"): {"$ne": None}}).count()) av["notanalyzed"] = str(self.__mongodbCollectionRagpicker.find({q.get("findStr1"): None}).count()) av["malware"] = str( self.__mongodbCollectionRagpicker.find( {"$and": [{q.get("findStr1"): {"$ne": None}}, {q.get("findStr2"): {"$ne": q.get("ok")}}]} ).count() ) av["nonemalware"] = str(self.__mongodbCollectionRagpicker.find({q.get("findStr2"): q.get("ok")}).count()) if av.get("analyzed") != "0": av["rate"] = "{:.2f} %".format((float(av.get("malware")) / float(av.get("analyzed")) * 100)) else: av["rate"] = "--" ret.append(av) return ret def getStatisticsNoneMalwareByAV(self): return self.__mongodbCollectionRagpicker.find( { "$and": [ {"$or": [{"AntivirusScanAvast.avast": "OK"}, {"AntivirusScanAvast": None}]}, {"$or": [{"AntivirusScanAvg.Avg": "OK"}, {"AntivirusScanAvg": None}]}, {"$or": [{"AntivirusScanAvira.Avira.scan": "OK"}, {"AntivirusScanAvira": None}]}, {"$or": [{"AntivirusScanBitDefender.BitDefender": "OK"}, {"AntivirusScanBitDefender": None}]}, {"$or": [{"AntivirusScanClamAv.ClamAv": " OK"}, {"AntivirusScanClamAv": None}]}, {"$or": [{"AntivirusScanCOMODO.COMODO": "OK"}, {"AntivirusScanCOMODO": None}]}, {"$or": [{"AntivirusScanESET.ESET": "OK"}, {"AntivirusScanESET": None}]}, {"$or": [{"AntivirusScanFProt.FProt": "OK"}, {"AntivirusScanFProt": None}]}, {"$or": [{"AntivirusScanF-Secure.F-Secure": "OK"}, {"AntivirusScanF-Secure": None}]}, {"VirusTotal.file.verbose_msg": {"$ne": None}}, ] } ).count() def getSamplesNotFoundByAV(self): return self.__mongodbCollectionRagpicker.find( { "$and": [ {"$or": [{"AntivirusScanAvast.avast": "OK"}, {"AntivirusScanAvast": None}]}, {"$or": [{"AntivirusScanAvg.Avg": "OK"}, {"AntivirusScanAvg": None}]}, {"$or": [{"AntivirusScanAvira.Avira.scan": "OK"}, {"AntivirusScanAvira": None}]}, {"$or": [{"AntivirusScanBitDefender.BitDefender": "OK"}, {"AntivirusScanBitDefender": None}]}, {"$or": [{"AntivirusScanClamAv.ClamAv": " OK"}, {"AntivirusScanClamAv": None}]}, {"$or": [{"AntivirusScanCOMODO.COMODO": "OK"}, {"AntivirusScanCOMODO": None}]}, {"$or": [{"AntivirusScanESET.ESET": "OK"}, {"AntivirusScanESET": None}]}, {"$or": [{"AntivirusScanFProt.FProt": "OK"}, {"AntivirusScanFProt": None}]}, {"$or": [{"AntivirusScanF-Secure.F-Secure": "OK"}, {"AntivirusScanF-Secure": None}]}, {"VirusTotal.file.verbose_msg": {"$ne": None}}, ] }, {"Info.file.sha256": True, "Info.analyse.started": True}, ) def getSamplesNotFoundByVT(self): return self.__mongodbCollectionRagpicker.find( {"VirusTotal.file.verbose_msg": {"$ne": None}}, {"Info.file.sha256": True, "Info.analyse.started": True} ) def getSamplesNotFoundByLocalAV(self): return self.__mongodbCollectionRagpicker.find( { "$and": [ {"$or": [{"AntivirusScanAvast.avast": "OK"}, {"AntivirusScanAvast": None}]}, {"$or": [{"AntivirusScanAvg.Avg": "OK"}, {"AntivirusScanAvg": None}]}, {"$or": [{"AntivirusScanAvira.Avira.scan": "OK"}, {"AntivirusScanAvira": None}]}, {"$or": [{"AntivirusScanBitDefender.BitDefender": "OK"}, {"AntivirusScanBitDefender": None}]}, {"$or": [{"AntivirusScanClamAv.ClamAv": " OK"}, {"AntivirusScanClamAv": None}]}, {"$or": [{"AntivirusScanCOMODO.COMODO": "OK"}, {"AntivirusScanCOMODO": None}]}, {"$or": [{"AntivirusScanESET.ESET": "OK"}, {"AntivirusScanESET": None}]}, {"$or": [{"AntivirusScanFProt.FProt": "OK"}, {"AntivirusScanFProt": None}]}, {"$or": [{"AntivirusScanF-Secure.F-Secure": "OK"}, {"AntivirusScanF-Secure": None}]}, ] }, {"Info.file.sha256": True, "Info.analyse.started": True}, ) def getStatisticsVirusTotal(self): ret = {} ret["analyzed"] = self.__mongodbCollectionRagpicker.find({"VirusTotal": {"$ne": None}}).count() ret["notAnalyzed"] = self.__mongodbCollectionRagpicker.find({"VirusTotal": None}).count() ret["samplesFound"] = self.__mongodbCollectionRagpicker.find( {"VirusTotal.file.positives": {"$ne": None}} ).count() ret["SamplesNotFound"] = self.__mongodbCollectionRagpicker.find( {"VirusTotal.file.verbose_msg": {"$ne": None}} ).count() return ret def getStatisticsPackerSignatures(self): return self.__mongodbCollectionRagpicker.aggregate( [ { "$group": { "_id": {"PublisherO": "$VerifySigs.PublisherO", "Issuer": "$VerifySigs.Issuer"}, "count": {"$sum": 1}, } }, {"$sort": {"count": -1}}, ] ) def getStatisticsPackerCompiler(self): return self.__mongodbCollectionRagpicker.aggregate( [{"$group": {"_id": "$PEID", "count": {"$sum": 1}}}, {"$sort": {"count": -1}}] ) def getStatisticsPeCharacteristics(self): ret = {} ret["exe"] = str( self.__mongodbCollectionRagpicker.find( {"$and": [{"Info.file.EXE": True}, {"Info.file.DLL": False}, {"Info.file.DRIVER": False}]} ).count() ) ret["dll"] = str( self.__mongodbCollectionRagpicker.find( {"$and": [{"Info.file.EXE": False}, {"Info.file.DLL": True}, {"Info.file.DRIVER": False}]} ).count() ) ret["driver"] = str( self.__mongodbCollectionRagpicker.find( {"$and": [{"Info.file.EXE": False}, {"Info.file.DLL": False}, {"Info.file.DRIVER": True}]} ).count() ) ret["noPe"] = str( self.__mongodbCollectionRagpicker.find( {"$and": [{"Info.file.EXE": None}, {"Info.file.DLL": None}, {"Info.file.DRIVER": None}]} ).count() ) ret["dllDriver"] = str( self.__mongodbCollectionRagpicker.find( {"$and": [{"Info.file.EXE": False}, {"Info.file.DLL": True}, {"Info.file.DRIVER": True}]} ).count() ) return ret def getFiletypes(self): return self.__mongodbCollectionRagpicker.aggregate( [{"$group": {"_id": "$Info.file.type", "count": {"$sum": 1}}}, {"$sort": {"count": -1}}] ) def countReportsRagpickerDB(self): return self.__mongodbCollectionRagpicker.find().count() def iterateRagpickerReports(self, sha256): for report in self.__mongodbCollectionRagpicker.find({"Info.file.sha256": sha256}, {"_id": 0}): yield report # Attention deletes the whole Ragpicker-Database!!! # returns number of deleted reports def deleteRagpickerDB(self): count = self.__mongodbCollectionRagpicker.find().count() # Alle Ragpicker-Daten aus der MongoDB loeschen self.__mongodbCollectionRagpicker.remove() return count # Insert Ragpicker-Report in MongoDB def insertRagpickerDB(self, report): # Store the report try: self.__mongodbCollectionRagpicker.insert(report) except InvalidDocument as e: log.exception("Error InvalidDocument: %s", report) raise Exception("Error InvalidDocument: {0}".format(e)) except InvalidStringData: self.__mongodbCollectionRagpicker.insert(convertDirtyDict2ASCII(report)) # Count Ragpicker-Reports by file (and url) def countRagpickerDB(self, file_md5, url_md5=None): if url_md5: query = {"$and": [{"Info.url.md5": {"$in": [url_md5]}}, {"Info.file.md5": {"$in": [file_md5]}}]} else: query = {"$and": [{"Info.file.md5": {"$in": [file_md5]}}]} return self.__mongodbCollectionRagpicker.find(query).count() # ------------------------------------------------------------------------------ # Ragpicker SandboxTaskQueue database (MongoDB) # ------------------------------------------------------------------------------ def insertSandboxTaskStatus(self, sandboxName, sha256, taskID, sampleID, taskState=None): statusReport = { "sandbox": sandboxName, "sha256": sha256, "sample_id": sampleID, "task_id": taskID, "task_state": taskState, } # Store the SandboxTaskQueue-Status-report self.__mongodbCollectionSandboxTaskQueue.insert(statusReport) # Attention deletes the whole Ragpicker-SandboxTaskQueue-Database!!! # returns number of deleted reports def deleteSandboxTaskQueueDB(self): count = self.__mongodbCollectionSandboxTaskQueue.find().count() # Alle Daten aus der MongoDB loeschen self.__mongodbCollectionSandboxTaskQueue.remove() return count # ------------------------------------------------------------------------------ # Ragpicker families database (MongoDB) # ------------------------------------------------------------------------------ def insertFamily(self, familyReport): # Store the family-report self.__mongodbCollectionFamilies.insert(familyReport) # Count Ragpicker-Reports by file (and url) def countFamilyDB(self, parentObjectSHA256): query = {"$and": [{"parentObjectSHA256": {"$in": [parentObjectSHA256]}}]} return self.__mongodbCollectionFamilies.find(query).count() def iterateFamilyReports(self, sha256): for report in self.__mongodbCollectionFamilies.find({"parentObjectSHA256": sha256}, {"_id": 0}): yield report # Attention deletes the whole Ragpicker-Family-Database!!! # returns number of deleted reports def deleteFamilyDB(self): count = self.__mongodbCollectionFamilies.find().count() # Alle Ragpicker-Daten aus der MongoDB loeschen self.__mongodbCollectionFamilies.remove() return count
def create(cls, name="default"): config = Config.get(name, "database") if config["driver"] == "mysql": return MySQL(config)
class Storage(object): """ The Storage class underpins the API.base.getStorage(). The object is available once you call storage = api.getStorage(). :init() arguments: :name: Storage name on disk. :root="wrapper-data/json": File path of the storage. :pickle=True: True to use Binary storage, False to use json. :Methods: :load(): :save(): :close(): :Properties/variables: :periodic_save_timer: Default is 60 seconds :paused_saving: Set to True to pause the periodic save. """ def __init__(self, name, root="wrapper-data/json", pickle=True): # type: (str, str, bool) -> None """ :param name: Name of Storage :param root: Path on disk to storage data :param pickle: Boolean; Pickle (True) or not (False, use Json) """ self.Data = {} self.name = name self.root = root self.pickle = pickle self.configManager = Config() self.configManager.loadconfig() self.log = logging.getLogger('Storage.py') self.encoding = self.configManager.config["General"]["encoding"] self.paused_saving = False self.periodic_save_timer = 60 if self.pickle: self.file_ext = "pkl" else: self.file_ext = "json" self.load() self.timer = time.time() self.abort = False t = threading.Thread(target=self._periodicsave, args=()) t.daemon = True t.start() def _periodicsave(self): # doing it this way (versus just sleep() for certain number of seconds), # allows faster shutdown response while not self.abort: if self.paused_saving: time.sleep(1) continue if time.time() - self.timer > self.periodic_save_timer: self.save() self.timer = time.time() time.sleep(1) def load(self): """ Loads the Storage data from disk. In conjunction with setting the storages paused_saving to true, this allows you to make changes to a wrapper storage on disk while wrapper is running. :return: Nothing """ mkdir_p(self.root) if not os.path.exists( "%s/%s.%s" % (self.root, self.name, self.file_ext)): # load old json storages if there is no pickled # file (and if storage is using pickle) if self.pickle: self.Data = self._json_load() # save to the selected file mode (json or pkl) self.save() if self.pickle: filenameis = "%s.pkl" % self.name self.Data = pickle_load(self.root, filenameis) else: self.Data = self._json_load() def save(self): """ Force a save of the Storage to disk. Saves are also done periodically and when the storage is closed. :return: Nothing """ if not os.path.exists(self.root): mkdir_p(self.root) if self.pickle: filenameis = "%s.pkl" % self.name pickle_save(self.root, filenameis, self.Data, self.encoding) else: self._json_save() def _json_save(self): putcode = putjsonfile(self.Data, self.name, self.root) if not putcode: self.log.exception( "Error encoutered while saving json data:\n'%s/%s.%s'" "\nData Dump:\n%s" % ( self.root, self.name, self.file_ext, self.Data)) def _json_load(self): try_load = getjsonfile(self.name, self.root, encodedas=self.encoding) if try_load is None: self.log.exception("bad file or data '%s/%s.json'" % (self.root, self.name)) return {} if try_load is False: # file just does not exist (yet); return without comments/errors. return {} else: return try_load def close(self): """ Close the Storage and save it's Data to disk. :return: Nothing """ self.abort = True self.save()