class DatabaseMySQL():
def __init__(self):
self.__cfgReporting = Config(os.path.join(RAGPICKER_ROOT, 'config', 'reporting.conf'))
self.__mysqlEnabled = self.__cfgReporting.getOption("mysql", "enabled")
if self.__mysqlEnabled:
#Anbindung an Datenbank MySQL herstellen
try:
mysqlHost = self.__cfgReporting.getOption("mysql", "host")
mysqlPort = self.__cfgReporting.getOption("mysql", "port")
mysqlDatabase = self.__cfgReporting.getOption("mysql", "database")
mysqlUser = self.__cfgReporting.getOption("mysql", "user")
mysqlPassword = self.__cfgReporting.getOption("mysql", "password")
self.__mysqlConnection = MySQLdb.Connect(host=mysqlHost, port=mysqlPort, db=mysqlDatabase, user=mysqlUser, passwd=mysqlPassword)
except (Exception) as e:
raise Exception("Cannot connect to MySQL (ragpicker): %s" % e)
def __del__(self):
if self.__mysqlEnabled:
self.__mysqlConnection.close()
# ------------------------------------------------------------------------------
# Ragpicker Database (MySQL)
# ------------------------------------------------------------------------------
def isRagpickerDBEnabledMySQL(self):
return self.__mysqlEnabled
def __init__(self):
self.__cfgReporting = Config(os.path.join(RAGPICKER_ROOT, "config", "reporting.conf"))
self.__cfgProcessing = Config(os.path.join(RAGPICKER_ROOT, "config", "processing.conf"))
self.__mongodbEnabled = self.__cfgReporting.getOption("mongodb", "enabled")
self.__codedbEnabled = self.__cfgReporting.getOption("codeDB", "enabled")
self.__bluecoatEnabled = self.__cfgProcessing.getOption("all_bluecoatMalwareAnalysisAppliance", "enabled")
if self.__mongodbEnabled:
# Anbindung an Datenbank MongoDB Collection Ragpicker herstellen
try:
mongodbHost = self.__cfgReporting.getOption("mongodb", "host")
mongodbPort = self.__cfgReporting.getOption("mongodb", "port")
self.__mongodbConnection = MongoClient(mongodbHost, mongodbPort)
self.__mongodbCollectionRagpicker = self.__mongodbConnection.MalwareAnalyse.ragpicker
self.__mongodbCollectionFamilies = self.__mongodbConnection.MalwareAnalyse.families
self.__mongodbCollectionSandboxTaskQueue = self.__mongodbConnection.MalwareAnalyse.sandboxTaskQueue
except TypeError:
raise Exception("MongoDB connection port in report.config must be integer")
except ConnectionFailure:
raise Exception("Cannot connect to MongoDB (ragpicker)")
if self.__codedbEnabled:
# Anbindung an Datenbank MongoDB Collection CodeDB herstellen
try:
codedbHost = self.__cfgReporting.getOption("codeDB", "mongo_db_host")
codedbPort = self.__cfgReporting.getOption("codeDB", "mongo_db_port")
self.__codedbConnection = MongoClient(codedbHost, codedbPort)
self.__codedbCollectionCodedb = self.__codedbConnection.MalwareAnalyse.codeDB
except TypeError:
raise Exception("MongoDB connection port for CodeDB in report.config must be integer")
except ConnectionFailure:
raise Exception("Cannot connect to MongoDB (codeDB)")
def __process_url(self, url):
# Crawler config load
cfgCrawler = Config(os.path.join(RAGPICKER_ROOT, 'config', 'crawler.conf')).get("clientConfig")
data = None
headers = {
'User-Agent': cfgCrawler.get("browser_user_agent", "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)"),
'Accept-Language': cfgCrawler.get("browser_accept_language", "en-US"),
}
# Save original socket
originalSocket = socket.socket
# Set TOR Socks proxy
commonutils.setTorProxy()
request = urllib2.Request(url, data, headers)
try:
url_dl = urllib2.urlopen(request, timeout=30).read()
except urllib2.HTTPError as e:
raise e
except urllib2.URLError as e:
raise e
except Exception, e:
raise IOError("Thread(" + self.processName + ") - %s - Error parsing %s" % (e, url))
def active_only( self, value ):
if not isinstance( value, bool ):
raise TypeError( "Expected bool, not {}".format( type( value ).__name__ ) )
if self._active_only != value:
self._active_only = value
Config.set( "Interface", "active_only", self._active_only )
GLib.idle_add( self.refilter )
def getHtmlContent(target_url, header_type):
config = Config()
try:
if header_type == 'sougou':
send_headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 6.2; rv:16.0) Gecko/20100101 Firefox/16.0',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Connection': 'keep-alive',
'Cookie': 'com_sohu_websearch_ITEM_PER_PAGE='+str(config.getValue("pagesize", "sougou"))
}
else:
send_headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 6.2; rv:16.0) Gecko/20100101 Firefox/16.0',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Connection': 'keep-alive'
}
if sys.version > '3':
req = urllib.request.Request(target_url, headers=send_headers)
response = urllib.request.urlopen(req, timeout=10)
else:
req = urllib2.Request(target_url, headers=send_headers)
response = urllib2.urlopen(req, timeout=30)
# print get_request.info()
return response.read().decode('utf-8')
except Exception as e:
print("Get html page content error:" + e.message)
def runCrawler():
mapURL = {}
cfgCrawler = Config(os.path.join(RAGPICKER_ROOT, 'config', 'crawler.conf'))
# Build the PluginManager
crawlerPluginManager = PluginManager()
crawlerPluginManager.setPluginPlaces(["crawler"])
crawlerPluginManager.collectPlugins()
# Trigger run from the "Crawler" plugins
for pluginInfo in sorted(crawlerPluginManager.getAllPlugins(), key=lambda PluginInfo: PluginInfo.name):
crawlerModul = pluginInfo.plugin_object
# Config for crawler module
try:
options = cfgCrawler.get(pluginInfo.name)
crawlerModul.set_options(options)
except Exception:
log.error("Crawler module %s not found in configuration file", pluginInfo.name)
# If the crawler module is disabled in the config, skip it.
if not options.enabled:
continue
try:
log.debug("Run Crawler: " + pluginInfo.name)
returnMap = crawlerModul.run()
mapURL.update(returnMap)
except Exception as e:
log.error('Error (%s) in %s', e, pluginInfo.name)
return mapURL
def response_cb( self, dialog, response ):
"""Response Callback of the Gtk.Dialog"""
if response == 1: # Save new config values
Config.set( "Autocompletion", "bic_file_de", self.builder.get_object( 'general_bic_open_entry' ).get_text() )
Config.set( "Autocompletion", "zipcode_file_de", self.builder.get_object( 'general_zipcode_open_entry' ).get_text() )
self._session.commit()
else: # Discard new config values
self._session.rollback()
self.hide()
def init():
path = os.path.split(os.path.realpath(__file__))[0]
config_path = "%s/config.ini" % path
Config.init(config_path)
CACHEFOLDER = os.getenv("HOME") + "/.cache/autohome/"
if not os.path.exists(CACHEFOLDER):
os.makedirs(CACHEFOLDER)
PIDLOG = "%sautohome.pid" % CACHEFOLDER
Config.set("global", "PID", PIDLOG)
init_log()
def __init__(self, persistence):
valid = [
"help_url",
"twitter_url",
"info_url",
"ts_address",
"ts_port",
"ts_password"
]
Config.__init__("module-admin", persistence, valid);
def __init__(self, module, page, keyword):
config = Config()
pagesize = config.getValue("pagesize", module)
print("\033[1;37;40m[*]Search Engine [%s] starting！The number of display bars per page is %s" % (module, pagesize))
myps = multiprocessing.Process(target=Collect, args=(module, page, pagesize, keyword,))
myps.start()
processList = gol.get_value("process")
processList.append(myps)
gol.set_value("process", processList)
class UPSDModule_testserv(UPSDModule):
modname = 'testserv'
def start(self):
if not UPSDModule.start(self):
return False
self.conf = Config()
self.userconf = {}
### Load info for pseudo client from config ###
self.userconf['nick'] = self.conf.getOption(self.modname, 'nick')
self.userconf['ident'] = self.conf.getOption(self.modname, 'ident')
self.userconf['host'] = self.conf.getOption(self.modname, 'host')
self.userconf['modes'] = self.conf.getOption(self.modname, 'modes')
self.userconf['realname'] = self.conf.getOption(self.modname, 'realname')
self.userconf['mainchan'] = self.conf.getOption(self.modname, 'mainchan')
self.proto.sendNickCreate(self.userconf['nick'], self.userconf['ident'], self.userconf['host'], self.userconf['modes'], self.userconf['realname'])
self.proto.sendJoin(self.userconf['nick'], self.userconf['mainchan'])
return True
def stop(self):
UPSDModule.stop(self)
self.proto.sendUserQuit(self.userconf['nick'])
def testserv_ctcp(self, user, command, params):
if command.lower() == "\x01version\x01":
self.proto.sendNotice(self.userconf['nick'], user, '\x01VERSION %s\x01' % (self.getVersion()))
return True
def testserv_channel(self, user, command, params):
nick = params[0].split(':')[1]
if (len(params) > 4):
args = params[4].split(' ')
else:
args = None
if (command == ".kick"):
self.proto.sendKick(self.userconf['nick'], user, args[0], args[1])
def getModHooks(self):
return (('privmsg', self.testserv_ctcp),
('privmsg', self.testserv_channel))
def getVersion(self):
return 'UnrealPSD TestServ Module'
def reload_data():
global Banks
global Cities
for option in Config.options( 'Autocompletion' ):
if option.startswith( 'bic_file_' ) or option.startswith( 'zipcode_file_' ):
filename = Config.getfilepath( 'Autocompletion', option )
if os.path.exists( filename ):
if option.startswith( 'bic_file_' ):
tag = option[9:].upper()
Banks.clear( tag )
Banks.load( tag, filename, 'latin1' )
elif option.startswith( 'zipcode_file_' ):
tag = option[13:].upper()
Cities.clear( tag )
Cities.load( tag, filename, 'utf-8' )
def __init__(self, name, root="wrapper-data/json", pickle=True):
# type: (str, str, bool) -> None
"""
:param name: Name of Storage
:param root: Path on disk to storage data
:param pickle: Boolean; Pickle (True) or not (False, use Json)
"""
self.Data = {}
self.name = name
self.root = root
self.pickle = pickle
self.configManager = Config()
self.configManager.loadconfig()
self.log = logging.getLogger('Storage.py')
self.encoding = self.configManager.config["General"]["encoding"]
self.paused_saving = False
self.periodic_save_timer = 60
if self.pickle:
self.file_ext = "pkl"
else:
self.file_ext = "json"
self.load()
self.timer = time.time()
self.abort = False
t = threading.Thread(target=self._periodicsave, args=())
t.daemon = True
t.start()
def __init__(self, app):
super(HashmalMain, self).__init__()
self.app = app
self.app.setStyleSheet(hashmal_style)
self.changes_saved = True
self.config = Config()
self.config.load()
QtCore.QCoreApplication.setOrganizationName('mazaclub')
QtCore.QCoreApplication.setApplicationName('hashmal')
self.qt_settings = QtCore.QSettings()
self.setDockNestingEnabled(True)
self.dock_handler = DockHandler(self)
self.dock_handler.create_docks()
self.dock_handler.do_default_layout()
self.script_editor = ScriptEditor(self)
self.script_editor.changesSaved.connect(self.on_changes_saved)
self.setCentralWidget(self.script_editor)
self.create_menubar()
self.create_default_script()
self.statusBar().setVisible(True)
self.statusBar().messageChanged.connect(self.change_status_bar)
self.restoreState(self.qt_settings.value('toolLayout/default').toByteArray())
if self.qt_settings.value('quickTipsOnStart', defaultValue=QtCore.QVariant(True)).toBool():
QtCore.QTimer.singleShot(500, self.do_quick_tips)
def _process_url(self, url):
#Crawler config load
cfgCrawler = Config(os.path.join(RAGPICKER_ROOT, 'config', 'crawler.conf')).get("clientConfig")
data = None
headers = {
'User-Agent': cfgCrawler.get("browser_user_agent", "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)"),
'Accept-Language': cfgCrawler.get("browser_accept_language", "en-US"),
}
request = urllib2.Request(url, data, headers)
try:
url_dl = urllib2.urlopen(request, timeout=30).read()
except Exception, e:
raise IOError("Thread("+self.threadName+") - %s - Error parsing %s" % (e, url))
def __init__(self):
self.cfgReporting = Config(os.path.join(RAGPICKER_ROOT, 'config', 'reporting.conf'))
self.vxcageEnabled = self.cfgReporting.getOption("vxcage", "enabled")
self.host = self.cfgReporting.getOption("vxcage", "host")
self.port = self.cfgReporting.getOption("vxcage", "port")
if not self.host or not self.port:
raise Exception("VxCage REST API server not configurated")
def __init__(self):
# Datenbank
self.__database = Database()
# Kofiguration aus der reporting.conf holen
self.__cfgReporting = Config(os.path.join(RAGPICKER_ROOT, 'config', 'reporting.conf'))
self.__vxcageEnabled = self.__cfgReporting.getOption("vxcage", "enabled")
self.__vxcageHost = self.__cfgReporting.getOption("vxcage", "host")
self.__vxcagePort = self.__cfgReporting.getOption("vxcage", "port")
def show( self ):
"""
Shows the dialog.
"""
# DB info
name, driver, version = core.database.Database.get_engine_info()
version = '.'.join( [str( x ) for x in version] )
data = "%i Magazine\n%i Vertragstypen\n%i Ausgaben\n%i Kunden\n%i Adressen\n%i Bankkonten\n%i Verträge" % ( core.database.Magazine.count(), core.database.Subscription.count(), core.database.Issue.count(), core.database.Customer.count(), core.database.Address.count(), core.database.Bankaccount.count(), core.database.Contract.count() )
self.builder.get_object( 'general_database_name_label' ).set_text( name )
self.builder.get_object( 'general_database_driver_label' ).set_text( driver )
self.builder.get_object( 'general_database_version_label' ).set_text( version )
self.builder.get_object( 'general_database_data_label' ).set_text( data )
self.builder.get_object( 'general_bic_open_entry' ).set_text( Config.get( "Autocompletion", "bic_file_de" ) )
self.builder.get_object( 'general_zipcode_open_entry' ).set_text( Config.get( "Autocompletion", "zipcode_file_de" ) )
self._magazinemanager.start_edit()
self.builder.get_object( "content" ).show_all()
def runCrawler():
mapURL = {}
cfgCrawler = Config(os.path.join(RAGPICKER_ROOT, 'config', 'crawler.conf'))
#TOR Socks proxy
isTorEnabled = cfgCrawler.get("clientConfig").get("tor_enabled", False)
if isTorEnabled:
torProxyAdress = cfgCrawler.get("clientConfig").get("tor_proxyadress", "localhost")
torProxyPort = cfgCrawler.get("clientConfig").get("tor_proxyport", 9050)
# Route an HTTP request through the SOCKS proxy
socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, torProxyAdress, torProxyPort)
socket.socket = socks.socksocket
# Build the PluginManager
crawlerPluginManager = PluginManager()
crawlerPluginManager.setPluginPlaces(["crawler"])
crawlerPluginManager.collectPlugins()
# Trigger run from the "Crawler" plugins
for pluginInfo in crawlerPluginManager.getAllPlugins():
crawlerModul = pluginInfo.plugin_object
#Config for crawler module
try:
options = cfgCrawler.get(pluginInfo.name)
crawlerModul.set_options(options)
except Exception:
log.error("Crawler module %s not found in configuration file", pluginInfo.name)
# If the crawler module is disabled in the config, skip it.
if not options.enabled:
continue
log.info("Run Crawler: " + pluginInfo.name)
try:
returnMap = crawlerModul.run()
mapURL.update(returnMap)
except Exception as e:
log.error('Error (%s) in %s', e, pluginInfo.name)
return mapURL
class TestConfig(unittest2.TestCase):
def setUp(self):
self.config = Config("tests/test_config.ini")
def test_sort_sections(self):
self.assertEqual(list(self.config.sort_sections()), ["lvrj.com/blogs", "lvrj.com"])
def test_domain_lookup(self):
config = self.config["lvrj.com"]
self.assertEqual(config, {"date": "today"})
config = self.config["not here"]
self.assertEqual(config, {})
def printRagpickerInfos(print_options=False):
infolog = logging.getLogger("Info")
infolog.info(color("RAGPICKER_VERSION: " + RAGPICKER_VERSION, RED))
infolog.info(color("RAGPICKER_BUILD_DATE: " + RAGPICKER_BUILD_DATE, RED))
infolog.info(color("RAGPICKER_ROOT: " + RAGPICKER_ROOT, RED))
infolog.info("")
pluginPlaces = ["crawler", "preProcessing", "processing", "reporting"]
for place in pluginPlaces:
infolog.info(color("%s| " % (place + " ").upper().ljust(14, '-'), MAGENTA))
cfg = Config(os.path.join(RAGPICKER_ROOT, 'config', place + '.conf'))
ragpickerPluginManager = PluginManager()
ragpickerPluginManager.setPluginPlaces([place])
ragpickerPluginManager.collectPlugins()
for pluginInfo in sorted(ragpickerPluginManager.getAllPlugins(), key=lambda PluginInfo: PluginInfo.name):
options = cfg.get(pluginInfo.name)
if options.enabled:
infolog.info(color("%s V%s %s", MAGENTA) % (" |----[+] " +
pluginInfo.name.ljust(25), pluginInfo.version, pluginInfo.description))
if print_options:
for key, value in options.iteritems():
if key != "enabled":
infolog.info(color(" |-- %s = %s", MAGENTA) % (key, str(value)))
else:
infolog.info(color("%s V%s %s", BLUE) % (" |----[-] " +
pluginInfo.name.ljust(25), pluginInfo.version, pluginInfo.description))
infolog.info("")
infolog.info(" %s %s" % (color("([+] = enabled)", MAGENTA), color("([-] = disabled)", BLUE)))
infolog.info("")
checkVersion()
sys.stdout.flush()
def __init__(self, module, page, pagesize, keyword):
self.config = Config()
self.module = module
self.page = int(page)
self.keyword = keyword
self.pageSize = int(pagesize)
self.saveFile = self.config.getValue("global", "savefile")
if self.saveFile == 'True':
self.outfile = OutFile(unquote(self.keyword))
else:
self.outfile = None
self.collection()
def on_finished( self, rendering_results ):
template = self._template_env.get_template( "{}.template".format( "base" ) )
rendered_letters = [rendered_letter for rendered_letter in rendering_results if rendered_letter is not None]
if len( rendered_letters ) == 0:
raise ValueError( "No rendered letters" )
lco_template = Config.get("LetterRenderer", "lco_template")
if not lco_template:
lco_template = "a4paper"
rendered_document = template.render( {'lco_template': lco_template, 'prerendered_letters':rendered_letters} )
dirs = [paths.config('templates', 'latex'),
paths.config('images'),
paths.data('templates', 'latex'),
paths.data('images')]
textinputs = [dir for dir in dirs if os.path.exists(dir)]
pdflatex.compile_str(rendered_document, self._output_file, textinputs)
class Collect(object):
def __init__(self, module, page, pagesize, keyword):
self.config = Config()
self.module = module
self.page = int(page)
self.keyword = keyword
self.pageSize = int(pagesize)
self.saveFile = self.config.getValue("global", "savefile")
if self.saveFile == 'True':
self.outfile = OutFile(unquote(self.keyword))
else:
self.outfile = None
self.collection()
def collection(self):
for i in range(self.page):
print("\033[1;37;40m[*]Search Engine [%s],Page [%s] Start collecting." % (self.module, i+1))
page_pn = (i * self.pageSize)
if self.module == "baidu":
my_baidu = Baidu(self.outfile)
my_baidu.search(self.keyword, self.pageSize, page_pn)
elif self.module == "so":
my_so = So(self.outfile)
my_so.search(self.keyword, i+1)
elif self.module == "sougou":
my_sougou = Sougou(self.outfile)
my_sougou.search(self.keyword, i+1)
if self.config.sleeptime > 0:
time.sleep(self.config.sleeptime)
if self.outfile == 'True':
self.outfile.closeFile()
def start(self):
if not UPSDModule.start(self):
return False
self.conf = Config()
self.userconf = {}
### Load info for pseudo client from config ###
self.userconf['nick'] = self.conf.getOption(self.modname, 'nick')
self.userconf['ident'] = self.conf.getOption(self.modname, 'ident')
self.userconf['host'] = self.conf.getOption(self.modname, 'host')
self.userconf['modes'] = self.conf.getOption(self.modname, 'modes')
self.userconf['realname'] = self.conf.getOption(self.modname, 'realname')
self.userconf['mainchan'] = self.conf.getOption(self.modname, 'mainchan')
self.proto.sendNickCreate(self.userconf['nick'], self.userconf['ident'], self.userconf['host'], self.userconf['modes'], self.userconf['realname'])
self.proto.sendJoin(self.userconf['nick'], self.userconf['mainchan'])
return True
def __init__( self ):
self._invoicetable = msmgui.widgets.invoicetable.InvoiceTable()
msmgui.widgets.base.RefreshableWindow.__init__( self, [self._invoicetable] )
self.builder = Gtk.Builder()
self.builder.add_from_file( paths.data("ui","widgets","invoicewindow", "invoicewindow.glade"))
self.builder.get_object( "content" ).reparent( self )
self.builder.connect_signals( self )
self.builder.get_object( "tablebox" ).add( self._invoicetable )
self._invoicetable.connect( "selection-changed", self.invoicetable_selection_changed_cb )
self._invoicingassistant = msmgui.assistants.invoicing.InvoicingAssistant()
self._invoicingassistant.connect( "saved", self.invoicingassistant_saved_cb )
self._letterexportassistant = msmgui.assistants.letterexport.LetterExportAssistant()
active_only = Config.getboolean( "Interface", "active_only" )
if not active_only:
self.builder.get_object( "invoices_showall_switch" ).set_active( not active_only )
def __init__( self ):
self._customertable = msmgui.widgets.customertable.CustomerTable()
msmgui.widgets.base.RefreshableWindow.__init__( self, [self._customertable] )
self.builder = Gtk.Builder()
self.builder.add_from_file( paths.data("ui","widgets","customerwindow", "customerwindow.glade"))
self.builder.add_from_file( "data/ui/widgets/customerwindow/customerwindow.glade" )
self.builder.get_object( "content" ).reparent( self )
self.builder.connect_signals( self )
self._customereditor = msmgui.widgets.customereditor.CustomerEditor()
self.builder.get_object( "editorbox" ).add( self._customereditor )
self._customereditor.connect( "edit-started", self.customereditor_edit_started_cb )
self._customereditor.connect( "edit-ended", self.customereditor_edit_ended_cb )
self._customereditor.connect( "changed", self.customereditor_changed_cb )
self._customereditor.connect( "save", self.customereditor_saved_cb )
self._customereditor.connect( "expanded-changed", self.customereditor_expanded_changed_cb )
self.builder.get_object( "tablebox" ).add( self._customertable )
self._customertable.connect( "selection-changed", self.customertable_selection_changed_cb )
active_only = Config.getboolean( "Interface", "active_only" )
if not active_only:
self.builder.get_object( "customers_showall_switch" ).set_active( not active_only )
class Storage(object):
def __init__(self,
name,
root="wrapper-data/json",
encoding="default",
pickle=True):
self.Data = {}
self.name = name
self.root = root
self.pickle = pickle
self.configManager = Config()
self.configManager.loadconfig()
self.log = logging.getLogger('Storage.py')
if encoding == "default":
self.encoding = self.configManager.config["General"]["encoding"]
else:
self.encoding = encoding
if self.pickle:
self.file_ext = "pkl"
else:
self.file_ext = "json"
self.load()
self.timer = time.time()
self.abort = False
t = threading.Thread(target=self.periodicsave, args=())
t.daemon = True
t.start()
def periodicsave(self):
# doing it this way (versus just sleeping for 60 seconds),
# allows faster shutdown response
while not self.abort:
if time.time() - self.timer > 60:
self.save()
self.timer = time.time()
time.sleep(1)
def load(self):
mkdir_p(self.root)
if not os.path.exists("%s/%s.%s" %
(self.root, self.name, self.file_ext)):
# load old json storages if there is no pickled
# file (and if storage is using pickle)
if self.pickle:
self.Data = self.json_load()
# save to the selected file mode (json or pkl)
self.save()
if self.pickle:
self.Data = self.pickle_load()
else:
self.Data = self.json_load()
def save(self):
if not os.path.exists(self.root):
mkdir_p(self.root)
if self.pickle:
self.pickle_save()
else:
self.json_save()
def pickle_save(self):
if "human" in self.encoding.lower():
_protocol = 0
else:
# using something less than HIGHEST allows both Pythons 2/3
# to use the files interchangeably. It should also allow
# moving files between machines with different configurations
# with fewer issues.
#
# Python 2 cPickle does not have a DEFAULT_PROTOCOL
# constant like Python 3 pickle (else I would just
# use the Default (currently 3, I believe).
#
# This will probably use either 1 or 2 depending on
# which python you use.
#
# We imported either pickle (Py3) or cPickle (Py2) depending
# on what wrapper detected. Both are implemented in C for
# speed.
#
# The MAIN POINT:
# I wanted the code to use something better/faster than
# Human-readable (unless that is what you specify), while
# still permitting some portability of the final files
_protocol = Pickle.HIGHEST_PROTOCOL // 2
with open("%s/%s.%s" % (self.root, self.name, self.file_ext),
"wb") as f:
Pickle.dump(self.Data, f, protocol=_protocol)
def json_save(self):
putcode = putjsonfile(self.Data, self.name, self.root)
if not putcode:
self.log.exception(
"Error encoutered while saving json data:\n'%s/%s.%s'"
"\nData Dump:\n%s" %
(self.root, self.name, self.file_ext, self.Data))
def pickle_load(self):
with open("%s/%s.%s" % (self.root, self.name, self.file_ext),
"rb") as f:
return Pickle.load(f)
def json_load(self):
try_load = getjsonfile(self.name, self.root, encodedas=self.encoding)
if try_load in (None, False):
self.log.exception("bad/non-existent file or data '%s/%s.%s'" %
(self.root, self.name, self.file_ext))
return {}
else:
return try_load
def close(self):
self.abort = True
self.save()
class Statistics():
def __init__(self):
# Datenbank
self.__database = Database()
# Kofiguration aus der reporting.conf holen
self.__cfgReporting = Config(os.path.join(RAGPICKER_ROOT, 'config', 'reporting.conf'))
self.__vxcageEnabled = self.__cfgReporting.getOption("vxcage", "enabled")
self.__vxcageHost = self.__cfgReporting.getOption("vxcage", "host")
self.__vxcagePort = self.__cfgReporting.getOption("vxcage", "port")
def runStatisticsLong(self):
#Pruefen ob VxCage und MongoDB aktiviert sind
if self.__database.isRagpickerDBEnabled():
if self.__vxcageEnabled:
self.__runStatisticsMongodbLong()
if self.__database.isCodeDBEnabled():
self.__runStatisticsCodedb()
else:
print("vxcage in reporting.conf is not enabled")
sys.stdout.flush()
else:
print("mongodb in reporting.conf is not enabled")
sys.stdout.flush()
def runStatisticsShort(self):
#Pruefen ob VxCage und MongoDB aktiviert sind
if self.__database.isRagpickerDBEnabled():
if self.__vxcageEnabled:
self.__runStatisticsMongodbShort()
if self.__database.isCodeDBEnabled():
self.__runStatisticsCodedb()
else:
print("vxcage in reporting.conf is not enabled")
sys.stdout.flush()
else:
print("mongodb in reporting.conf is not enabled")
sys.stdout.flush()
def runStatisticsAV(self):
#Pruefen ob VxCage und MongoDB aktiviert sind
if self.__database.isRagpickerDBEnabled():
if self.__vxcageEnabled:
self.__runStatisticsAV()
else:
print("vxcage in reporting.conf is not enabled")
sys.stdout.flush()
else:
print("mongodb in reporting.conf is not enabled")
sys.stdout.flush()
def __runStatisticsMongodbLong(self):
print "**************************************"
print "*** Statistics MongoDB (Ragpicker) ***"
print "**************************************"
print ""
print "Number of malware samples in database:", self.__database.countReportsRagpickerDB()
print ""
#Statistiken der eingesetzten AV-Produkte
self.__runStatisticsAVProducts()
#Liste der letzen 20 Samples, die weder auf VT noch von einem lokalen AV gefunden wurden
self.__runStatisticsLast20SamplesNotFoundByAV()
#Liste der letzen 20 Samples, die nicht auf VT gefunden wurden
self.__runStatisticsLast20SamplesNotFoundByVT()
#Liste der letzen 20 Samples, die nicht von einem lokalen AV-Produkt gefunden wurden
self.__runStatisticsLast20SamplesNotFoundByLocalAV()
#Liste und Haeufigkeit der Filetypes
self.__runStatisticsFiletypes()
#Haeufigkeit der PE Charakteristiken
self.__runStatisticsPeCharacteristics()
#Liste und Haeufigkeit der verwendeten Packer/Compiler in der Malware
self.__runStatisticsPackerCompiler()
#Liste der verwendeten digitalen Signaturen
self.__runStatisticsPackerSignatures()
sys.stdout.flush()
def __runStatisticsMongodbShort(self):
print "**************************************"
print "*** Statistics MongoDB (Ragpicker) ***"
print "**************************************"
print ""
print "Number of malware samples in database:", self.__database.countReportsRagpickerDB()
print ""
#Liste und Haeufigkeit der Filetypes
self.__runStatisticsFiletypes()
#Haeufigkeit der PE Charakteristiken
self.__runStatisticsPeCharacteristics()
sys.stdout.flush()
def __runStatisticsAV(self):
print "**************************************"
print "*** Statistics MongoDB (Ragpicker) ***"
print "**************************************"
print ""
print "Number of malware samples in database:", self.__database.countReportsRagpickerDB()
print ""
#Statistiken der eingesetzten AV-Produkte
self.__runStatisticsAVProducts()
#Liste der letzen 20 Samples, die weder auf VT noch von einem lokalen AV gefunden wurden
self.__runStatisticsLast20SamplesNotFoundByAV()
#Liste der letzen 20 Samples, die nicht auf VT gefunden wurden
self.__runStatisticsLast20SamplesNotFoundByVT()
#Liste der letzen 20 Samples, die nicht von einem lokalen AV-Produkt gefunden wurden
self.__runStatisticsLast20SamplesNotFoundByLocalAV()
sys.stdout.flush()
def __runStatisticsFiletypes(self):
#Liste und Haeufigkeit der Filetypes
print "Filetypes of malware"
res = self.__database.getFiletypes()
table = PrettyTable(["filetype", "count"])
table.align["filetype"] = "l"
table.align["count"] = "c"
table.padding_width = 1
try:
for values in res['result']:
if values.get("_id"):
outputPacker = values.get("_id")
outputCount = str(values.get("count"))
table.add_row([outputPacker, outputCount])
print(table)
except KeyError:
raise Exception("Dict has no key 'result' ")
print ""
def __runStatisticsPeCharacteristics(self):
#Haeufigkeit der PE Charakteristiken
print "PE-Characteristics of malware"
peC = self.__database.getStatisticsPeCharacteristics()
table = PrettyTable(["pe-characteristics", "count"])
table.align["pe-characteristics"] = "l"
table.align["count"] = "c"
table.padding_width = 1
table.add_row(["EXE", peC.get("exe")])
table.add_row(["DLL", peC.get("dll")])
table.add_row(["Driver", peC.get("driver")])
table.add_row(["DLL/Driver", peC.get("dllDriver")])
table.add_row(["No PE File", peC.get("noPe")])
print (table)
print ""
def __runStatisticsPackerCompiler(self):
#Liste und Haeufigkeit der verwendeten Packer/Compiler in der Malware
print "Packer/compiler used in malware"
res = self.__database.getStatisticsPackerCompiler()
table = PrettyTable(["packer/compiler", "count"])
table.align["packer/compiler"] = "l"
table.align["count"] = "c"
table.padding_width = 1
try:
for values in res['result']:
if values.get("_id"):
outputPacker = values.get("_id")[0]
outputCount = str(values.get("count"))
table.add_row([outputPacker, outputCount])
print(table)
except KeyError:
raise Exception("Dict has no key 'result' ")
print " "
def __runStatisticsPackerSignatures(self):
#Liste der verwendeten digitalen Signaturen
print "Signatures used by malware"
res = self.__database.getStatisticsPackerSignatures()
table = PrettyTable(["publisher", "issuer", "count"])
table.align["publisher"] = "l"
table.align["issuer"] = "l"
table.align["count"] = "c"
table.padding_width = 1
try:
for values in res['result']:
if values.get("_id"):
outputPublisher = values.get("_id").get("PublisherO")
if values.get("_id").get("Issuer"):
outputIssuer = values.get("_id").get("Issuer")
else:
outputIssuer = " "
outputCount = str(values.get("count"))
table.add_row([outputPublisher, outputIssuer, outputCount])
print(table)
except KeyError:
raise Exception("Dict has no key 'result' ")
print ""
def __runStatisticsLast20SamplesNotFoundByAV(self):
#Liste der letzen 20 Samples, die weder auf VT noch von einem lokalen AV gefunden wurden
print "Last 20 samples not found by VirusTotal and local AV-Products"
res = sorted(self.__database.getSamplesNotFoundByAV(), reverse=True)
table = PrettyTable(["timestamp of crawling", "sha256"])
table.align["timestamp of crawling"] = "c"
table.align["sha256"] = "c"
table.padding_width = 1
try:
for values in res:
sha256 = values.get("Info").get("file").get("sha256")
timestamp = values.get("Info").get("analyse").get("started")
table.add_row([timestamp, sha256])
print(table.get_string(start=0, end=20))
except KeyError:
raise Exception("Dict has no key 'Info' ")
print ""
def __runStatisticsLast20SamplesNotFoundByVT(self):
#Liste der letzen 20 Samples, die nicht auf VirusTotal gefunden wurden
print "Last 20 samples not found by VirusTotal"
res = sorted(self.__database.getSamplesNotFoundByVT(), reverse=True)
table = PrettyTable(["timestamp of crawling", "sha256"])
table.align["timestamp of crawling"] = "c"
table.align["sha256"] = "c"
table.padding_width = 1
try:
for values in res:
sha256 = values.get("Info").get("file").get("sha256")
timestamp = values.get("Info").get("analyse").get("started")
table.add_row([timestamp, sha256])
print(table.get_string(start=0, end=20))
except KeyError:
raise Exception("Dict has no key 'Info' ")
print ""
def __runStatisticsLast20SamplesNotFoundByLocalAV(self):
#Liste der letzen 20 Samples, die nicht von einem lokalen AV-Produkt gefunden wurden
print "Last 20 samples not found by local AV-Products"
res = sorted(self.__database.getSamplesNotFoundByLocalAV(), reverse=True)
table = PrettyTable(["timestamp of crawling", "sha256"])
table.align["timestamp of crawling"] = "c"
table.align["sha256"] = "c"
table.padding_width = 1
try:
for values in res:
sha256 = values.get("Info").get("file").get("sha256")
timestamp = values.get("Info").get("analyse").get("started")
table.add_row([timestamp, sha256])
print(table.get_string(start=0, end=20))
except KeyError:
raise Exception("Dict has no key 'Info' ")
print ""
def __runStatisticsAVProducts(self):
#Statistiken der eingesetzten AV-Produkte
#VirusTotal und lokale AV-Produkte
print "VirusTotal and local AV-Products"
print " Samples rated as none-malware by all AV-Products at time of crawling:", \
self.__database.getStatisticsNoneMalwareByAV()
print ""
#VirusTotal
ret = self.__database.getStatisticsVirusTotal()
print "VirusTotal"
print " Samples analyzed at time of crawling:", ret.get("analyzed")
print " Samples not analyzed at time of crawling:", ret.get("notAnalyzed")
print " Samples found at time of crawling:", ret.get("samplesFound")
print " Samples not found at time of crawling:", ret.get("SamplesNotFound")
print ""
#Lokale AV-Produkte
print "Local AV-Products"
print " analyzed => Samples analyzed at time of crawling"
print " not analyzed => Samples not analyzed at time of crawling"
print " malware => Samples rated as malware at time of crawling"
print " none-malware => Samples rated as none-malware at time of crawling"
table = PrettyTable(["product", "analyzed", "not analyzed", "malware", "none-malware", "detection rate"])
table.align["product"] = "l"
table.align["analyzed"] = "r"
table.align["not analyzed"] = "r"
table.align["malware"] = "r"
table.align["none-malware"] = "r"
table.align["detection rate"] = "r"
table.padding_width = 1
# Statistik Daten holen
ret = self.__database.getStatisticsAntivirus()
# Table-Body zusammenbauen
for av in ret:
table.add_row([av.get("product"), av.get("analyzed"), av.get("notanalyzed"), av.get("malware"), av.get("nonemalware"), av.get("rate")])
print(table)
print ""
async def post(self):
code = self.post_data.get('code', None)
if not code:
raise HTTPError(400, "code为空？", reason="code为空？")
# fetch userinfo from wechat server
wx_pb_userinfo = await WxCCServer().open_app_login(
code, self.request_id)
if wx_pb_userinfo is None:
raise HTTPError(500, "登录微信服务器出错", reason="登录微信服务器出错")
wx_userinfo_dict = MessageToDict(wx_pb_userinfo,
including_default_value_fields=True,
preserving_proto_field_name=True,
use_integers_for_enums=True)
# get account
exists_wechat = await AccountServer().exists_partner_wechat(
identifier=wx_pb_userinfo.unionid,
app_id=self.app_id_int64,
request_id=self.request_id)
if not exists_wechat:
# create account
pb_account = await AccountServer().create_account_by_partner(
partner=pbaccount_pb2.PARTNER_WECHAT,
identifier=wx_pb_userinfo.unionid,
origin_data=wx_userinfo_dict,
app_id=self.app_id_int64,
request_id=self.request_id)
# 分配空间
await MembershipServer().set_total_storage_change(
account_id=pb_account.account_id,
changed_value=Config().membership.get("register_default_size"),
title=Config().membership.get("register_default_title"),
details=Config().membership.get("register_default_details"),
request_id=self.request_id)
else:
# auth the wechat account
pb_account = await AccountServer().auth_partner_account(
wx_pb_userinfo.unionid,
app_id=self.app_id_int64,
request_id=self.request_id)
# update partner info
await AccountServer().update_partner_account_origin_data(
pb_account.auth_token, pbaccount_pb2.PARTNER_WECHAT,
wx_pb_userinfo.unionid, wx_userinfo_dict, self.app_id_int64,
self.request_id)
# set account pair to wxcc server
await WxCCServer().open_app_set_account_pair(
openid=wx_pb_userinfo.openid,
account_id=pb_account.account_id,
request_id=self.request_id)
# get userinfo
pb_current_userinfo = await AccountServer().get_userinfo(
pb_account.account_id, self.app_id_int64, self.request_id)
# update user info
params_people = dict()
if not pb_current_userinfo.nickname:
params_people["nickname"] = wx_pb_userinfo.nickname
if not pb_current_userinfo.gender:
params_people["gender"] = str(wx_pb_userinfo.gender)
if params_people:
await AccountServer().update_user_info(
account_id=pb_account.account_id,
params=params_people,
app_id=self.app_id_int64,
request_id=self.request_id)
if not pb_current_userinfo.avatar:
# generate avatar filename
zid = await ZIDServer().generate(self.request_id)
# upload
filename, err = await backend.backend_user_media.transmit_wechat_avatar(
zid=zid, remote_url=wx_pb_userinfo.avatar)
if not err and filename:
# update avatar
await AccountServer().update_user_avatar(
account_id=pb_account.account_id,
app_id=self.app_id_int64,
request_id=self.request_id,
avatar=filename)
account = Account(pb_account)
pb_userinfo = await AccountServer().get_userinfo(
pb_account.account_id, self.app_id_int64, self.request_id)
await account.set_userinfo(pb_userinfo)
self.send_to_client(0, message='登录成功', response=account.dump())
async def post(self):
try:
# upload file
avatar_files = self.request.files.get('file')
if not avatar_files:
self.send_to_client(error_id=400, message='请上传头像')
return
avatar_file = avatar_files[0]
# save to temp file
tempfile = Config().uploads_temp_path + str(uuid.uuid4())
fh = open(tempfile, 'wb')
fh.write(avatar_file.get('body'))
fh.close()
# upload
zid = await ZIDServer().generate(self.request_id)
filename, err = await backend.backend_user_media.upload_avatar(
zid, tempfile, "square")
# remove temp file
try:
os.remove(tempfile)
except:
pass
if err:
self.send_to_client(error_id=500, message='头像上传失败。错误信息：' + err)
return
if not filename:
self.send_to_client(error_id=500,
message='头像上传失败（未知错误），请联系管理员解决')
return
# 更新记录
pb_userinfo = await AccountServer().get_userinfo(
account_id=self.current_user.account_id,
app_id=self.app_id_int64,
request_id=self.request_id)
# 删除原头像
if pb_userinfo:
if pb_userinfo.avatar:
await backend.backend_user_media.delete_object(
pb_userinfo.avatar)
pb_userinfo.avatar = filename
# update avatar
await AccountServer().update_user_avatar(
account_id=self.current_user.account_id,
app_id=self.app_id_int64,
request_id=self.request_id,
avatar=filename)
account = self.current_user
await account.set_userinfo(pb_userinfo)
self.send_to_client(0, message='头像上传成功', response=account.dump())
return
except grpc.RpcError as e:
# except grpc._channel._Rendezvous as e:
msg = "code: {}, details:{}".format(e.code(), e.details())
raise HTTPError(500, msg, reason=str(e.details()))
async def post(self):
mobile = self.post_data.get('mobile', '')
smscode = self.post_data.get('smscode', '')
errors = self._check_input({"mobile": mobile, "smscode": smscode})
if errors:
err = ''
for item, er in errors.items():
err = er
break
self.send_to_client(error_id=error.input_error.id, message=err)
return
# Check smscode
mobile_smscode = rd.redis_get('verify:{}'.format(mobile))
if mobile_smscode:
mobile_smscode = mobile_smscode.decode()
if mobile_smscode != smscode:
self.send_to_client(-1, '登陆验证码错误')
return
# Exists Mobile
exists_account_already = False
try:
exists_account_already = await AccountServer(
).exists_account_by_mobile(mobile, request_id=self.request_id)
except grpc.RpcError as e:
self.send_to_client(error_id=500, message='服务器内部错误，请联系管理员确认')
return
pb_account = None
if not exists_account_already:
# not exists, create account
pb_account = await AccountServer().create_account(
mobile=mobile,
country_code="86",
app_id=self.app_id_int64,
request_id=self.request_id)
# 分配空间
result = await MembershipServer().set_total_storage_change(
account_id=pb_account.account_id,
changed_value=Config().membership.get("register_default_size"),
title=Config().membership.get("register_default_title"),
details=Config().membership.get("register_default_details"),
request_id=self.request_id)
else:
# exists, get the account
pb_account = await AccountServer().get_account_by_mobile(
mobile=mobile,
country_code="86",
app_id=self.app_id_int64,
request_id=self.request_id)
# make Account Model
if pb_account is None:
self.send_to_client(error_id=500, message='服务器内部错误，请联系管理员确认')
return
account = Account(pb_account)
# get account userinfo
if account.status == pbaccount_pb2.STATUS_ACTIVATED:
rd.redis_del('verify:{}'.format(mobile))
pb_userinfo = await AccountServer().get_userinfo(
account.account_id,
app_id=self.app_id_int64,
request_id=self.request_id)
await account.set_userinfo(pb_userinfo)
self.send_to_client(0, message='登陆成功', response=account.dump())
return
status_error = {0: "未知状态的账号", 2: "未激活账号", 3: "锁定并暂停账号或封号", 4: "账号已注销"}
self.send_to_client(-1, message=status_error.get(account.status))
class HashmalMain(QMainWindow):
def __init__(self, app):
super(HashmalMain, self).__init__()
self.app = app
self.app.setStyleSheet(hashmal_style)
self.changes_saved = True
self.config = Config()
self.config.load()
QtCore.QCoreApplication.setOrganizationName('mazaclub')
QtCore.QCoreApplication.setApplicationName('hashmal')
self.qt_settings = QtCore.QSettings()
self.setDockNestingEnabled(True)
self.dock_handler = DockHandler(self)
self.dock_handler.create_docks()
self.dock_handler.do_default_layout()
self.script_editor = ScriptEditor(self)
self.script_editor.changesSaved.connect(self.on_changes_saved)
self.setCentralWidget(self.script_editor)
self.create_menubar()
self.create_default_script()
self.statusBar().setVisible(True)
self.statusBar().messageChanged.connect(self.change_status_bar)
self.restoreState(self.qt_settings.value('toolLayout/default').toByteArray())
if self.qt_settings.value('quickTipsOnStart', defaultValue=QtCore.QVariant(True)).toBool():
QtCore.QTimer.singleShot(500, self.do_quick_tips)
def sizeHint(self):
return QtCore.QSize(800, 500)
def create_default_script(self):
filename = os.path.expanduser('Untitled.coinscript')
self.load_script(filename)
def create_menubar(self):
menubar = QMenuBar()
file_menu = menubar.addMenu('&File')
file_menu.addAction('&New', self.new_script).setShortcut(QKeySequence.New)
file_menu.addAction('Save As...', self.save_script_as).setShortcut(QKeySequence.SaveAs)
file_menu.addAction('&Open', self.open_script).setShortcut(QKeySequence.Open)
file_menu.addAction('&Save', self.save_script).setShortcut(QKeySequence.Save)
file_menu.addAction('&Quit', self.close)
# Script actions
script_menu = menubar.addMenu('&Script')
script_menu.addAction('&Evaluate', self.dock_handler.evaluate_current_script)
# Settings and tool toggling
tools_menu = menubar.addMenu('&Tools')
tools_menu.addAction('&Settings', lambda: SettingsDialog(self).exec_())
tools_menu.addSeparator()
for i in sorted(self.dock_handler.dock_widgets):
tools_menu.addAction(i.toggleViewAction())
help_menu = menubar.addMenu('&Help')
help_menu.addAction('&About', self.do_about)
help_menu.addAction('&Tool Info', lambda: ToolInfo(self).exec_())
help_menu.addAction('&Quick Tips', self.do_quick_tips)
self.setMenuBar(menubar)
def show_status_message(self, msg, error=False):
self.statusBar().showMessage(msg, 3000)
if error:
self.statusBar().setProperty('hasError', True)
else:
self.statusBar().setProperty('hasError', False)
self.style().polish(self.statusBar())
def change_status_bar(self, new_msg):
# Unset hasError if an error is removed.
if not new_msg and self.statusBar().property('hasError'):
self.statusBar().setProperty('hasError', False)
self.style().polish(self.statusBar())
def on_changes_saved(self, saved):
title = ''.join(['Hashmal - ', self.script_editor.filename])
if not saved:
title = ''.join([title, ' *'])
self.setWindowTitle(title)
self.changes_saved = saved
def closeEvent(self, event):
# Save layout if configured to.
if self.qt_settings.value('saveLayoutOnExit', defaultValue=QtCore.QVariant(False)).toBool():
self.qt_settings.setValue('toolLayout/default', self.saveState())
if self.changes_saved or (not self.script_editor.filename and not str(self.script_editor.script_edit.toPlainText())):
event.accept()
return
result = QMessageBox.question(self, 'Save Changes',
'Do you want to save your changes to ' + self.script_editor.filename + ' before closing?',
QMessageBox.Yes | QMessageBox.No)
if result == QMessageBox.Yes:
self.save_script()
event.accept()
def new_script(self, filename=''):
if not filename:
base_name = ''.join(['Untitled-', str(time.time()), '.coinscript'])
filename = os.path.expanduser(base_name)
self.load_script(filename)
def save_script(self):
filename = self.script_editor.filename
if not filename:
filename = str(QFileDialog.getSaveFileName(self, 'Save script', filter=script_file_filter))
if not filename: return
if not filename.endswith('.coinscript'):
filename += '.coinscript'
self.script_editor.filename = filename
self.script_editor.save()
def save_script_as(self):
filename = str(QFileDialog.getSaveFileName(self, 'Save script as', filter=script_file_filter))
if not filename: return
if not filename.endswith('.coinscript'):
filename += '.coinscript'
self.script_editor.filename = filename
self.script_editor.save()
def open_script(self):
filename = str(QFileDialog.getOpenFileName(self, 'Open script', '.', filter=script_file_filter))
if not filename:
return
# Confirm discarding changes if an unsaved file is open.
if (self.script_editor.filename
and str(self.script_editor.script_edit.toPlainText())
and filename != self.script_editor.filename
and not self.changes_saved):
result = QMessageBox.question(self, 'Save Changes',
'Do you want to save your changes to ' + self.script_editor.filename + ' before closing?',
QMessageBox.Yes | QMessageBox.No)
if result == QMessageBox.Yes:
self.save_script()
self.load_script(filename)
def load_script(self, filename):
self.setWindowTitle('Hashmal - ' + filename)
self.script_editor.load(filename)
def do_about(self):
d = QDialog(self)
vbox = QVBoxLayout()
about_label = QLabel()
about_label.setWordWrap(True)
txt = []
txt.append(' '.join([
'Hashmal is an IDE for Bitcoin transaction scripts.',
'Its purpose is to make it easier to write, evaluate, and learn about transaction scripts.'
]))
txt.append('Hashmal is intended for cryptocurrency developers and power users.')
txt.append('Use at own risk!')
txt = '\n\n'.join(txt)
about_label.setText(txt)
close_button = QPushButton('Close')
close_button.clicked.connect(d.close)
btn_box = floated_buttons([close_button])
vbox.addWidget(about_label)
vbox.addLayout(btn_box)
d.setLayout(vbox)
d.setWindowTitle('About Hashmal')
d.exec_()
def do_quick_tips(self):
QuickTips(self).exec_()
class Database:
def __init__(self):
self.__cfgReporting = Config(os.path.join(RAGPICKER_ROOT, "config", "reporting.conf"))
self.__cfgProcessing = Config(os.path.join(RAGPICKER_ROOT, "config", "processing.conf"))
self.__mongodbEnabled = self.__cfgReporting.getOption("mongodb", "enabled")
self.__codedbEnabled = self.__cfgReporting.getOption("codeDB", "enabled")
self.__bluecoatEnabled = self.__cfgProcessing.getOption("all_bluecoatMalwareAnalysisAppliance", "enabled")
if self.__mongodbEnabled:
# Anbindung an Datenbank MongoDB Collection Ragpicker herstellen
try:
mongodbHost = self.__cfgReporting.getOption("mongodb", "host")
mongodbPort = self.__cfgReporting.getOption("mongodb", "port")
self.__mongodbConnection = MongoClient(mongodbHost, mongodbPort)
self.__mongodbCollectionRagpicker = self.__mongodbConnection.MalwareAnalyse.ragpicker
self.__mongodbCollectionFamilies = self.__mongodbConnection.MalwareAnalyse.families
self.__mongodbCollectionSandboxTaskQueue = self.__mongodbConnection.MalwareAnalyse.sandboxTaskQueue
except TypeError:
raise Exception("MongoDB connection port in report.config must be integer")
except ConnectionFailure:
raise Exception("Cannot connect to MongoDB (ragpicker)")
if self.__codedbEnabled:
# Anbindung an Datenbank MongoDB Collection CodeDB herstellen
try:
codedbHost = self.__cfgReporting.getOption("codeDB", "mongo_db_host")
codedbPort = self.__cfgReporting.getOption("codeDB", "mongo_db_port")
self.__codedbConnection = MongoClient(codedbHost, codedbPort)
self.__codedbCollectionCodedb = self.__codedbConnection.MalwareAnalyse.codeDB
except TypeError:
raise Exception("MongoDB connection port for CodeDB in report.config must be integer")
except ConnectionFailure:
raise Exception("Cannot connect to MongoDB (codeDB)")
def __del__(self):
if self.__mongodbEnabled:
self.__mongodbConnection.disconnect()
if self.__codedbEnabled:
self.__codedbConnection.disconnect()
# ------------------------------------------------------------------------------
# Ragpicker Database (MongoDB)
# ------------------------------------------------------------------------------
def isRagpickerDBEnabled(self):
return self.__mongodbEnabled
def getStatisticsAntivirus(self):
queries = []
ret = []
queries.append(
{
"product": "Avast Antivirus",
"findStr1": "AntivirusScanAvast",
"findStr2": "AntivirusScanAvast.avast",
"ok": "OK",
}
)
queries.append(
{"product": "AVG Antivirus", "findStr1": "AntivirusScanAvg", "findStr2": "AntivirusScanAvg.Avg", "ok": "OK"}
)
queries.append(
{
"product": "Avira",
"findStr1": "AntivirusScanAvira",
"findStr2": "AntivirusScanAvira.Avira.scan",
"ok": "OK",
}
)
queries.append(
{
"product": "BitDefender",
"findStr1": "AntivirusScanBitDefender",
"findStr2": "AntivirusScanBitDefender.BitDefender",
"ok": "OK",
}
)
queries.append(
{
"product": "ClamAV",
"findStr1": "AntivirusScanClamAv",
"findStr2": "AntivirusScanClamAv.ClamAv",
"ok": " OK",
}
)
queries.append(
{
"product": "COMODO",
"findStr1": "AntivirusScanCOMODO",
"findStr2": "AntivirusScanCOMODO.COMODO",
"ok": "OK",
}
)
queries.append(
{"product": "ESET", "findStr1": "AntivirusScanESET", "findStr2": "AntivirusScanESET.ESET", "ok": "OK"}
)
queries.append(
{"product": "F-Prot", "findStr1": "AntivirusScanFProt", "findStr2": "AntivirusScanFProt.FProt", "ok": "OK"}
)
queries.append(
{
"product": "F-Secure",
"findStr1": "AntivirusScanF-Secure",
"findStr2": "AntivirusScanF-Secure.F-Secure",
"ok": "OK",
}
)
for q in queries:
av = {}
av["product"] = q.get("product")
av["analyzed"] = str(self.__mongodbCollectionRagpicker.find({q.get("findStr1"): {"$ne": None}}).count())
av["notanalyzed"] = str(self.__mongodbCollectionRagpicker.find({q.get("findStr1"): None}).count())
av["malware"] = str(
self.__mongodbCollectionRagpicker.find(
{"$and": [{q.get("findStr1"): {"$ne": None}}, {q.get("findStr2"): {"$ne": q.get("ok")}}]}
).count()
)
av["nonemalware"] = str(self.__mongodbCollectionRagpicker.find({q.get("findStr2"): q.get("ok")}).count())
if av.get("analyzed") != "0":
av["rate"] = "{:.2f} %".format((float(av.get("malware")) / float(av.get("analyzed")) * 100))
else:
av["rate"] = "--"
ret.append(av)
return ret
def getStatisticsNoneMalwareByAV(self):
return self.__mongodbCollectionRagpicker.find(
{
"$and": [
{"$or": [{"AntivirusScanAvast.avast": "OK"}, {"AntivirusScanAvast": None}]},
{"$or": [{"AntivirusScanAvg.Avg": "OK"}, {"AntivirusScanAvg": None}]},
{"$or": [{"AntivirusScanAvira.Avira.scan": "OK"}, {"AntivirusScanAvira": None}]},
{"$or": [{"AntivirusScanBitDefender.BitDefender": "OK"}, {"AntivirusScanBitDefender": None}]},
{"$or": [{"AntivirusScanClamAv.ClamAv": " OK"}, {"AntivirusScanClamAv": None}]},
{"$or": [{"AntivirusScanCOMODO.COMODO": "OK"}, {"AntivirusScanCOMODO": None}]},
{"$or": [{"AntivirusScanESET.ESET": "OK"}, {"AntivirusScanESET": None}]},
{"$or": [{"AntivirusScanFProt.FProt": "OK"}, {"AntivirusScanFProt": None}]},
{"$or": [{"AntivirusScanF-Secure.F-Secure": "OK"}, {"AntivirusScanF-Secure": None}]},
{"VirusTotal.file.verbose_msg": {"$ne": None}},
]
}
).count()
def getSamplesNotFoundByAV(self):
return self.__mongodbCollectionRagpicker.find(
{
"$and": [
{"$or": [{"AntivirusScanAvast.avast": "OK"}, {"AntivirusScanAvast": None}]},
{"$or": [{"AntivirusScanAvg.Avg": "OK"}, {"AntivirusScanAvg": None}]},
{"$or": [{"AntivirusScanAvira.Avira.scan": "OK"}, {"AntivirusScanAvira": None}]},
{"$or": [{"AntivirusScanBitDefender.BitDefender": "OK"}, {"AntivirusScanBitDefender": None}]},
{"$or": [{"AntivirusScanClamAv.ClamAv": " OK"}, {"AntivirusScanClamAv": None}]},
{"$or": [{"AntivirusScanCOMODO.COMODO": "OK"}, {"AntivirusScanCOMODO": None}]},
{"$or": [{"AntivirusScanESET.ESET": "OK"}, {"AntivirusScanESET": None}]},
{"$or": [{"AntivirusScanFProt.FProt": "OK"}, {"AntivirusScanFProt": None}]},
{"$or": [{"AntivirusScanF-Secure.F-Secure": "OK"}, {"AntivirusScanF-Secure": None}]},
{"VirusTotal.file.verbose_msg": {"$ne": None}},
]
},
{"Info.file.sha256": True, "Info.analyse.started": True},
)
def getSamplesNotFoundByVT(self):
return self.__mongodbCollectionRagpicker.find(
{"VirusTotal.file.verbose_msg": {"$ne": None}}, {"Info.file.sha256": True, "Info.analyse.started": True}
)
def getSamplesNotFoundByLocalAV(self):
return self.__mongodbCollectionRagpicker.find(
{
"$and": [
{"$or": [{"AntivirusScanAvast.avast": "OK"}, {"AntivirusScanAvast": None}]},
{"$or": [{"AntivirusScanAvg.Avg": "OK"}, {"AntivirusScanAvg": None}]},
{"$or": [{"AntivirusScanAvira.Avira.scan": "OK"}, {"AntivirusScanAvira": None}]},
{"$or": [{"AntivirusScanBitDefender.BitDefender": "OK"}, {"AntivirusScanBitDefender": None}]},
{"$or": [{"AntivirusScanClamAv.ClamAv": " OK"}, {"AntivirusScanClamAv": None}]},
{"$or": [{"AntivirusScanCOMODO.COMODO": "OK"}, {"AntivirusScanCOMODO": None}]},
{"$or": [{"AntivirusScanESET.ESET": "OK"}, {"AntivirusScanESET": None}]},
{"$or": [{"AntivirusScanFProt.FProt": "OK"}, {"AntivirusScanFProt": None}]},
{"$or": [{"AntivirusScanF-Secure.F-Secure": "OK"}, {"AntivirusScanF-Secure": None}]},
]
},
{"Info.file.sha256": True, "Info.analyse.started": True},
)
def getStatisticsVirusTotal(self):
ret = {}
ret["analyzed"] = self.__mongodbCollectionRagpicker.find({"VirusTotal": {"$ne": None}}).count()
ret["notAnalyzed"] = self.__mongodbCollectionRagpicker.find({"VirusTotal": None}).count()
ret["samplesFound"] = self.__mongodbCollectionRagpicker.find(
{"VirusTotal.file.positives": {"$ne": None}}
).count()
ret["SamplesNotFound"] = self.__mongodbCollectionRagpicker.find(
{"VirusTotal.file.verbose_msg": {"$ne": None}}
).count()
return ret
def getStatisticsPackerSignatures(self):
return self.__mongodbCollectionRagpicker.aggregate(
[
{
"$group": {
"_id": {"PublisherO": "$VerifySigs.PublisherO", "Issuer": "$VerifySigs.Issuer"},
"count": {"$sum": 1},
}
},
{"$sort": {"count": -1}},
]
)
def getStatisticsPackerCompiler(self):
return self.__mongodbCollectionRagpicker.aggregate(
[{"$group": {"_id": "$PEID", "count": {"$sum": 1}}}, {"$sort": {"count": -1}}]
)
def getStatisticsPeCharacteristics(self):
ret = {}
ret["exe"] = str(
self.__mongodbCollectionRagpicker.find(
{"$and": [{"Info.file.EXE": True}, {"Info.file.DLL": False}, {"Info.file.DRIVER": False}]}
).count()
)
ret["dll"] = str(
self.__mongodbCollectionRagpicker.find(
{"$and": [{"Info.file.EXE": False}, {"Info.file.DLL": True}, {"Info.file.DRIVER": False}]}
).count()
)
ret["driver"] = str(
self.__mongodbCollectionRagpicker.find(
{"$and": [{"Info.file.EXE": False}, {"Info.file.DLL": False}, {"Info.file.DRIVER": True}]}
).count()
)
ret["noPe"] = str(
self.__mongodbCollectionRagpicker.find(
{"$and": [{"Info.file.EXE": None}, {"Info.file.DLL": None}, {"Info.file.DRIVER": None}]}
).count()
)
ret["dllDriver"] = str(
self.__mongodbCollectionRagpicker.find(
{"$and": [{"Info.file.EXE": False}, {"Info.file.DLL": True}, {"Info.file.DRIVER": True}]}
).count()
)
return ret
def getFiletypes(self):
return self.__mongodbCollectionRagpicker.aggregate(
[{"$group": {"_id": "$Info.file.type", "count": {"$sum": 1}}}, {"$sort": {"count": -1}}]
)
def countReportsRagpickerDB(self):
return self.__mongodbCollectionRagpicker.find().count()
def iterateRagpickerReports(self, sha256):
for report in self.__mongodbCollectionRagpicker.find({"Info.file.sha256": sha256}, {"_id": 0}):
yield report
# Attention deletes the whole Ragpicker-Database!!!
# returns number of deleted reports
def deleteRagpickerDB(self):
count = self.__mongodbCollectionRagpicker.find().count()
# Alle Ragpicker-Daten aus der MongoDB loeschen
self.__mongodbCollectionRagpicker.remove()
return count
# Insert Ragpicker-Report in MongoDB
def insertRagpickerDB(self, report):
# Store the report
try:
self.__mongodbCollectionRagpicker.insert(report)
except InvalidDocument as e:
log.exception("Error InvalidDocument: %s", report)
raise Exception("Error InvalidDocument: {0}".format(e))
except InvalidStringData:
self.__mongodbCollectionRagpicker.insert(convertDirtyDict2ASCII(report))
# Count Ragpicker-Reports by file (and url)
def countRagpickerDB(self, file_md5, url_md5=None):
if url_md5:
query = {"$and": [{"Info.url.md5": {"$in": [url_md5]}}, {"Info.file.md5": {"$in": [file_md5]}}]}
else:
query = {"$and": [{"Info.file.md5": {"$in": [file_md5]}}]}
return self.__mongodbCollectionRagpicker.find(query).count()
# ------------------------------------------------------------------------------
# Ragpicker SandboxTaskQueue database (MongoDB)
# ------------------------------------------------------------------------------
def insertSandboxTaskStatus(self, sandboxName, sha256, taskID, sampleID, taskState=None):
statusReport = {
"sandbox": sandboxName,
"sha256": sha256,
"sample_id": sampleID,
"task_id": taskID,
"task_state": taskState,
}
# Store the SandboxTaskQueue-Status-report
self.__mongodbCollectionSandboxTaskQueue.insert(statusReport)
# Attention deletes the whole Ragpicker-SandboxTaskQueue-Database!!!
# returns number of deleted reports
def deleteSandboxTaskQueueDB(self):
count = self.__mongodbCollectionSandboxTaskQueue.find().count()
# Alle Daten aus der MongoDB loeschen
self.__mongodbCollectionSandboxTaskQueue.remove()
return count
# ------------------------------------------------------------------------------
# Ragpicker families database (MongoDB)
# ------------------------------------------------------------------------------
def insertFamily(self, familyReport):
# Store the family-report
self.__mongodbCollectionFamilies.insert(familyReport)
# Count Ragpicker-Reports by file (and url)
def countFamilyDB(self, parentObjectSHA256):
query = {"$and": [{"parentObjectSHA256": {"$in": [parentObjectSHA256]}}]}
return self.__mongodbCollectionFamilies.find(query).count()
def iterateFamilyReports(self, sha256):
for report in self.__mongodbCollectionFamilies.find({"parentObjectSHA256": sha256}, {"_id": 0}):
yield report
# Attention deletes the whole Ragpicker-Family-Database!!!
# returns number of deleted reports
def deleteFamilyDB(self):
count = self.__mongodbCollectionFamilies.find().count()
# Alle Ragpicker-Daten aus der MongoDB loeschen
self.__mongodbCollectionFamilies.remove()
return count
def create(cls, name="default"):
config = Config.get(name, "database")
if config["driver"] == "mysql":
return MySQL(config)
class Storage(object):
"""
The Storage class underpins the API.base.getStorage(). The
object is available once you call storage = api.getStorage().
:init() arguments:
:name: Storage name on disk.
:root="wrapper-data/json": File path of the storage.
:pickle=True: True to use Binary storage, False to use json.
:Methods:
:load():
:save():
:close():
:Properties/variables:
:periodic_save_timer: Default is 60 seconds
:paused_saving: Set to True to pause the periodic save.
"""
def __init__(self, name, root="wrapper-data/json", pickle=True):
# type: (str, str, bool) -> None
"""
:param name: Name of Storage
:param root: Path on disk to storage data
:param pickle: Boolean; Pickle (True) or not (False, use Json)
"""
self.Data = {}
self.name = name
self.root = root
self.pickle = pickle
self.configManager = Config()
self.configManager.loadconfig()
self.log = logging.getLogger('Storage.py')
self.encoding = self.configManager.config["General"]["encoding"]
self.paused_saving = False
self.periodic_save_timer = 60
if self.pickle:
self.file_ext = "pkl"
else:
self.file_ext = "json"
self.load()
self.timer = time.time()
self.abort = False
t = threading.Thread(target=self._periodicsave, args=())
t.daemon = True
t.start()
def _periodicsave(self):
# doing it this way (versus just sleep() for certain number of seconds),
# allows faster shutdown response
while not self.abort:
if self.paused_saving:
time.sleep(1)
continue
if time.time() - self.timer > self.periodic_save_timer:
self.save()
self.timer = time.time()
time.sleep(1)
def load(self):
"""
Loads the Storage data from disk.
In conjunction with setting the storages paused_saving to true,
this allows you to make changes to a wrapper storage on disk
while wrapper is running.
:return: Nothing
"""
mkdir_p(self.root)
if not os.path.exists(
"%s/%s.%s" % (self.root, self.name, self.file_ext)):
# load old json storages if there is no pickled
# file (and if storage is using pickle)
if self.pickle:
self.Data = self._json_load()
# save to the selected file mode (json or pkl)
self.save()
if self.pickle:
filenameis = "%s.pkl" % self.name
self.Data = pickle_load(self.root, filenameis)
else:
self.Data = self._json_load()
def save(self):
"""
Force a save of the Storage to disk. Saves are also done
periodically and when the storage is closed.
:return: Nothing
"""
if not os.path.exists(self.root):
mkdir_p(self.root)
if self.pickle:
filenameis = "%s.pkl" % self.name
pickle_save(self.root, filenameis, self.Data, self.encoding)
else:
self._json_save()
def _json_save(self):
putcode = putjsonfile(self.Data, self.name, self.root)
if not putcode:
self.log.exception(
"Error encoutered while saving json data:\n'%s/%s.%s'"
"\nData Dump:\n%s" % (
self.root, self.name, self.file_ext, self.Data))
def _json_load(self):
try_load = getjsonfile(self.name, self.root, encodedas=self.encoding)
if try_load is None:
self.log.exception("bad file or data '%s/%s.json'" %
(self.root, self.name))
return {}
if try_load is False:
# file just does not exist (yet); return without comments/errors.
return {}
else:
return try_load
def close(self):
"""
Close the Storage and save it's Data to disk.
:return: Nothing
"""
self.abort = True
self.save()
