Esempio n. 1
0
    def _addFile(self, results, objfile, file, unpacked, extracted):
        # rawFile = open(file.temp_file, 'rb')
        log.info("_addFile: " + CODE_DB_URL_ADD % (self.cfg_host, self.cfg_port))
        voCodeDB = VOMalwareSample()
        
        try:                            
            voCodeDB.setsha256(file.get_fileSha256())
            
            if unpacked:
                # Bei entpackten Samples wird der Orighash gespeichert
                voCodeDB.setOrighash(objfile.file.get_fileSha256())
            
            voCodeDB.setVertraulich(VERTRAULICH_FREIGEGEBEN)
            voCodeDB.setFileName(convertDirtyDict2ASCII(objfile.get_url_filename()))
            
            info = results.get("Info")
            
            # Ist Datei exe, dll, sys?
            if(info.get("file").get("EXE") == True):
                voCodeDB.setbinType("exe")
            elif(info.get("file").get("DLL") == True):
                voCodeDB.setbinType("dll")
            elif(info.get("file").get("DRIVER") == True):
                voCodeDB.setbinType("sys")
                
            voCodeDB.setDownloadDatestamp(info.get("analyse").get("started").strftime("%Y-%m-%d %H:%M"))
            voCodeDB.setDownloadHostname(convertDirtyDict2ASCII(info.get("url").get("hostname")))
            
            # Felder nicht zwingend vorhanden
            if "OwnLocation" in results:    
                ownLocation = results.get('OwnLocation')
                voCodeDB.setGeolocationSelf(ownLocation.get("country"))
                
            # GeolocationHost nur vorhanden wenn InetSourceAnalysis benutzt wird
            if results.has_key('InetSourceAnalysis') and results.get('InetSourceAnalysis').has_key('URLVoid'):
                urlResult = results.get('InetSourceAnalysis').get('URLVoid').get('urlResult')
                voCodeDB.setGeolocationHost(urlResult.get('CountryCode'))
                voCodeDB.setDownloadIP(urlResult.get("IP"))                

            voCodeDB.setTags(self._getTags(results, objfile, unpacked, extracted))
            # Debug Ausgabe
            voCodeDB.prints()
            
            # Upload File to CodeDB
            uploadStatus = self._upload(convertDirtyDict2ASCII(objfile.get_url_filename()), file.file_data, voCodeDB)
        
            return uploadStatus
                
        except urllib2.URLError as e:
            raise Exception("Unable to establish connection to CodeDB REST API-Server server: %s" % e)
        except urllib2.HTTPError as e:
            raise Exception("Unable to perform HTTP request to CodeDB REST API-Server (http code=%s)" % e)        
Esempio n. 2
0
    def _addFile(self, results, objfile, file, unpacked, extracted):
        # rawFile = open(file.temp_file, 'rb')
        log.info("_addFile: " + CODE_DB_URL_ADD % (self.cfg_host, self.cfg_port))
        voCodeDB = VOMalwareSample()

        try:
            voCodeDB.setsha256(file.get_fileSha256())

            if unpacked:
                # Bei entpackten Samples wird der Orighash gespeichert
                voCodeDB.setOrighash(objfile.file.get_fileSha256())

            voCodeDB.setVertraulich(VERTRAULICH_FREIGEGEBEN)
            voCodeDB.setFileName(convertDirtyDict2ASCII(objfile.get_url_filename()))

            info = results.get("Info")

            # Ist Datei exe, dll, sys?
            if info.get("file").get("EXE") == True:
                voCodeDB.setbinType("exe")
            elif info.get("file").get("DLL") == True:
                voCodeDB.setbinType("dll")
            elif info.get("file").get("DRIVER") == True:
                voCodeDB.setbinType("sys")

            voCodeDB.setDownloadDatestamp(info.get("analyse").get("started").strftime("%Y-%m-%d %H:%M"))
            voCodeDB.setDownloadHostname(convertDirtyDict2ASCII(info.get("url").get("hostname")))

            # Felder nicht zwingend vorhanden
            if "OwnLocation" in results:
                ownLocation = results.get("OwnLocation")
                voCodeDB.setGeolocationSelf(ownLocation.get("country"))

            # GeolocationHost nur vorhanden wenn InetSourceAnalysis benutzt wird
            if results.has_key("InetSourceAnalysis") and results.get("InetSourceAnalysis").has_key("URLVoid"):
                urlResult = results.get("InetSourceAnalysis").get("URLVoid").get("urlResult")
                voCodeDB.setGeolocationHost(urlResult.get("CountryCode"))
                voCodeDB.setDownloadIP(urlResult.get("IP"))

            voCodeDB.setTags(self._getTags(results, objfile, unpacked, extracted))
            # Debug Ausgabe
            voCodeDB.prints()

            # Upload File to CodeDB
            uploadStatus = self._upload(convertDirtyDict2ASCII(objfile.get_url_filename()), file.file_data, voCodeDB)

            return uploadStatus

        except urllib2.URLError as e:
            raise Exception("Unable to establish connection to CodeDB REST API-Server server: %s" % e)
        except urllib2.HTTPError as e:
            raise Exception("Unable to perform HTTP request to CodeDB REST API-Server (http code=%s)" % e)
Esempio n. 3
0
 def insertCodeDB(self, report):
     # Store the report
     try:
         self.__codedbCollectionCodedb.insert(report)
     except InvalidStringData:
         self.__codedbCollectionCodedb.insert(
             convertDirtyDict2ASCII(report))
Esempio n. 4
0
    def run(self, results, objfile):
        dumpdir = self.options.get("dumpdir", None)

        if not dumpdir:
            raise Exception("dumpdir not configured, skip")
        
        try:
            if not os.path.exists(dumpdir):
                os.makedirs(dumpdir)  
            d = tempfile.mkdtemp(dir=dumpdir)
        except Exception as e:
            raise Exception('Could not open %s for writing (%s)', dumpdir, e)
        else:
            os.rmdir(d)
            
            url_md5 = results["Info"]["url"]["md5"]
            file_md5 = results["Info"]["file"]["md5"]
            jfile = url_md5 + "_" + file_md5 + ".xml"
            
            if not os.path.exists(dumpdir + jfile):
                try:
                    reportxml = dict2xml.dicttoxml(results)
                except UnicodeDecodeError:
                    reportxml = dict2xml.dicttoxml(convertDirtyDict2ASCII(results))
                except Exception as e:
                    raise Exception("Failed to generate XML report: %s" % e)
                        
                try:
                    
                    reportfile = codecs.open(os.path.join(dumpdir, jfile), "w", "utf-8")
                    reportfile.write(reportxml)
                    reportfile.close()
                except (TypeError, IOError) as e:
                    raise Exception("Failed to write XML report: %s" % e)   
Esempio n. 5
0
 def add_file_data(self, fieldname, filename, file_data, mimetype=None):
     filename = convertDirtyDict2ASCII(filename)
     
     """Add a file to be uploaded."""
     if mimetype is None:
         mimetype = mimetypes.guess_type(filename)[0] or 'application/octet-stream'
     self.files.append((fieldname, filename, mimetype, file_data))
     return
Esempio n. 6
0
 def add_file_data(self, fieldname, filename, file_data, mimetype=None):
     filename = convertDirtyDict2ASCII(filename)
     """Add a file to be uploaded."""
     if mimetype is None:
         mimetype = mimetypes.guess_type(
             filename)[0] or 'application/octet-stream'
     self.files.append((fieldname, filename, mimetype, file_data))
     return
    def _urlQuery(self, urlInput):
        httplib2.debuglevel = 4

        url = "http://urlquery.net/%s"
        action_search = url % "search.php?q=%s" % urlInput

        conn = urllib2.urlopen(action_search, timeout=60)
        content2String = conn.read()

        rpd = re.compile('.*  0\sresults\sreturned*', re.IGNORECASE)
        rpdFind = re.findall(rpd, content2String)

        if not rpdFind:
            # Reports found
            log.debug('urlquery Reports found')
            self.hitcount += 1
            urlqueryResults = []

            rpd = re.compile("\shref='(.*?)'\>", re.IGNORECASE)
            rpdFindReport = re.findall(rpd, content2String)

            rpd = re.compile("\<td\>\<a\stitle='(.*?)'\shref='report.php",
                             re.IGNORECASE)
            rpdFindReportUrl = re.findall(rpd, content2String)

            rpd = re.compile("\<td\salign='center'\>\<b\>(.*?)\<\/b\>\<\/td\>",
                             re.IGNORECASE)
            rpdFindAlertsIDS = re.findall(rpd, content2String)

            rpd = re.compile(
                "\<td\>\<nobr\>\<center\>(.*?)\<\/center\>\<\/nobr\>\<\/td\>",
                re.IGNORECASE)
            rpdFindDatum = re.findall(rpd, content2String)

            rpd = re.compile(
                "align='left'\stitle='(.*?)'\swidth='\d{2}'\sheight='\d{2}'\s/>",
                re.IGNORECASE)
            rpdFindLand = re.findall(rpd, content2String)

            i = 0
            datum = ''
            for datum in rpdFindDatum:
                result = {}
                result["datum"] = datum
                result["alerts_ids"] = rpdFindAlertsIDS[i]
                result["country"] = rpdFindLand[i]
                result["reportUrl"] = convertDirtyDict2ASCII(
                    rpdFindReportUrl[i])
                result["report"] = url % rpdFindReport[i]
                urlqueryResults.append(result)
                i += 1

            urlquery = {'url': urlInput, 'urlResult': urlqueryResults}
        else:
            log.debug('urlquery Reports NOT found')
            urlquery = {'url': urlInput, 'urlResult': 'NOT listed'}

        return urlquery
Esempio n. 8
0
 def insertRagpickerDB(self, report):
     # Store the report
     try:
         self.__mongodbCollectionRagpicker.insert(report)
     except InvalidDocument as e:
         log.exception("Error InvalidDocument: %s", report)
         raise Exception("Error InvalidDocument: {0}".format(e)) 
     except InvalidStringData:
         self.__mongodbCollectionRagpicker.insert(convertDirtyDict2ASCII(report))
Esempio n. 9
0
 def insertRagpickerDB(self, report):
     # Store the report
     try:
         self.__mongodbCollectionRagpicker.insert(report)
     except InvalidDocument as e:
         log.exception("Error InvalidDocument: %s", report)
         raise Exception("Error InvalidDocument: {0}".format(e))
     except InvalidStringData:
         self.__mongodbCollectionRagpicker.insert(convertDirtyDict2ASCII(report))
Esempio n. 10
0
    def get_url_filename(self):
        unknownFilename = "unknown_" + self.file.get_fileMd5(
        ) + "." + self.file.file_extension()
        urlpath, urlFileName = os.path.split(self.url)

        if ("&%" in urlFileName):
            urlFileName = urlFileName[urlFileName.rfind("&%") + 4:]
        if ("=%" in urlFileName):
            urlFileName = urlFileName[urlFileName.rfind("=%") + 4:]

        if ("&" in urlFileName):
            urlFileName = urlFileName[urlFileName.rfind("&") + 1:]
        if ("+" in urlFileName):
            urlFileName = urlFileName.replace("+", "_")
        if ("=" in urlFileName):
            urlFileName = urlFileName[urlFileName.rfind("=") + 1:]
        if (":" in urlFileName):
            urlFileName = urlFileName[urlFileName.rfind(":") + 1:]
        if ("#" in urlFileName):
            urlFileName = urlFileName.replace("#", "")
        if ("%" in urlFileName):
            urlFileName = urlFileName.replace("%", "#")
        if ("?" in urlFileName):
            urlFileName = urlFileName[urlFileName.rfind("?") + 1:]
        if ("!" in urlFileName):
            urlFileName = urlFileName[urlFileName.rfind("!") + 1:]
        if ("$" in urlFileName):
            urlFileName = urlFileName[urlFileName.rfind("$") + 1:]
        if ("'" in urlFileName):
            urlFileName = urlFileName.replace("'", "")
        if ("," in urlFileName):
            urlFileName = urlFileName[urlFileName.rfind(",") + 1:]
        if (";" in urlFileName):
            urlFileName = urlFileName[urlFileName.rfind(";") + 1:]

        if (len(urlFileName) > 40):

            extension = ""
            if "." in urlFileName:
                extension = urlFileName[urlFileName.rfind(".") + 1:]
                urlFileName = urlFileName[:urlFileName.rfind(".")]
            urlFileName = "%s[...].%s" % (urlFileName[:40], extension)

        # Verstueckelter Name, bringt nix.
        if (len(urlFileName) < 3): return unknownFilename

        if (urlFileName[0] == "_") or (urlFileName[0]
                                       == "-") or (urlFileName[0] == "#"):
            urlFileName = urlFileName[1:]

        if not urlFileName or urlFileName == "":
            urlFileName = unknownFilename

        return convertDirtyDict2ASCII(urlFileName).strip()
Esempio n. 11
0
 def get_url_filename(self):
     unknownFilename = "unknown_" + self.file.get_fileMd5() + "." + self.file.file_extension()
     urlpath, urlFileName = os.path.split(self.url)
 
     if ("&%" in urlFileName):
         urlFileName = urlFileName[urlFileName.rfind("&%") + 4:]
     if ("=%" in urlFileName):
         urlFileName = urlFileName[urlFileName.rfind("=%") + 4:]
         
     if ("&" in urlFileName):
         urlFileName = urlFileName[urlFileName.rfind("&") + 1:]
     if ("+" in urlFileName):
         urlFileName = urlFileName.replace("+", "_")
     if ("=" in urlFileName):
         urlFileName = urlFileName[urlFileName.rfind("=") + 1:]
     if (":" in urlFileName):
         urlFileName = urlFileName[urlFileName.rfind(":") + 1:]
     if ("#" in urlFileName):
         urlFileName = urlFileName.replace("#", "")
     if ("%" in urlFileName):
         urlFileName = urlFileName.replace("%", "#")
     if ("?" in urlFileName):
         urlFileName = urlFileName[urlFileName.rfind("?") + 1:]
     if ("!" in urlFileName):
         urlFileName = urlFileName[urlFileName.rfind("!") + 1:]
     if ("$" in urlFileName):
         urlFileName = urlFileName[urlFileName.rfind("$") + 1:]
     if ("'" in urlFileName):
         urlFileName = urlFileName.replace("'", "")
     if ("," in urlFileName):
         urlFileName = urlFileName[urlFileName.rfind(",") + 1:]
     if (";" in urlFileName):
         urlFileName = urlFileName[urlFileName.rfind(";") + 1:]
     
     if (len(urlFileName) > 40):
         
         extension = ""
         if "." in urlFileName:
             extension = urlFileName[urlFileName.rfind(".") + 1:]
             urlFileName = urlFileName[:urlFileName.rfind(".")]
         urlFileName = "%s[...].%s" % (urlFileName[:40], extension)
 
     # Verstueckelter Name, bringt nix.    
     if (len(urlFileName) < 3): return unknownFilename
     
     if (urlFileName[0] == "_") or (urlFileName[0] == "-") or (urlFileName[0] == "#"):
         urlFileName = urlFileName[1:]
         
     if not urlFileName or urlFileName == "":
         urlFileName = unknownFilename   
         
     return convertDirtyDict2ASCII(urlFileName).strip()
    def _urlQuery(self, urlInput):
        httplib2.debuglevel = 4

        url = "http://urlquery.net/%s"
        action_search = url % "search.php?q=%s" % urlInput

        conn = urllib2.urlopen(action_search, timeout=60)
        content2String = conn.read()

        rpd = re.compile(".*&nbsp;&nbsp;0\sresults\sreturned*", re.IGNORECASE)
        rpdFind = re.findall(rpd, content2String)

        if not rpdFind:
            # Reports found
            log.debug("urlquery Reports found")
            self.hitcount += 1
            urlqueryResults = []

            rpd = re.compile("\shref='(.*?)'\>", re.IGNORECASE)
            rpdFindReport = re.findall(rpd, content2String)

            rpd = re.compile("\<td\>\<a\stitle='(.*?)'\shref='report.php", re.IGNORECASE)
            rpdFindReportUrl = re.findall(rpd, content2String)

            rpd = re.compile("\<td\salign='center'\>\<b\>(.*?)\<\/b\>\<\/td\>", re.IGNORECASE)
            rpdFindAlertsIDS = re.findall(rpd, content2String)

            rpd = re.compile("\<td\>\<nobr\>\<center\>(.*?)\<\/center\>\<\/nobr\>\<\/td\>", re.IGNORECASE)
            rpdFindDatum = re.findall(rpd, content2String)

            rpd = re.compile("align='left'\stitle='(.*?)'\swidth='\d{2}'\sheight='\d{2}'\s/>", re.IGNORECASE)
            rpdFindLand = re.findall(rpd, content2String)

            i = 0
            datum = ""
            for datum in rpdFindDatum:
                result = {}
                result["datum"] = datum
                result["alerts_ids"] = rpdFindAlertsIDS[i]
                result["country"] = rpdFindLand[i]
                result["reportUrl"] = convertDirtyDict2ASCII(rpdFindReportUrl[i])
                result["report"] = url % rpdFindReport[i]
                urlqueryResults.append(result)
                i += 1

            urlquery = {"url": urlInput, "urlResult": urlqueryResults}
        else:
            log.debug("urlquery Reports NOT found")
            urlquery = {"url": urlInput, "urlResult": "NOT listed"}

        return urlquery
Esempio n. 13
0
def writeJsonReportFile(exportDir, dbresults, fileName):
    try:
        jsonpickle.set_encoder_options('simplejson', indent=4)
        jsonpickle.handlers.registry.register(datetime.datetime, DatetimeHandler)
        jsonpickle.handlers.registry.register(uuid.UUID, UUIDHandler)
        jsonReport = jsonpickle.encode(dbresults)
    except (UnicodeError, TypeError):
        jsonReport = jsonpickle.encode(convertDirtyDict2ASCII(dbresults))
    try:
        if not os.path.exists(exportDir + fileName):
            report = codecs.open(os.path.join(exportDir, fileName), "w", "utf-8")
            report.write(jsonReport)
            report.close()
    except (TypeError, IOError) as e:
        raise Exception("Failed to generate JSON report: %s" % e)
Esempio n. 14
0
def writeJsonReportFile(exportDir, dbresults, fileName):
    try:
        jsonpickle.set_encoder_options('simplejson', indent=4)
        jsonpickle.handlers.registry.register(datetime.datetime, DatetimeHandler)
        jsonpickle.handlers.registry.register(uuid.UUID, UUIDHandler)
        jsonReport = jsonpickle.encode(dbresults)
    except (UnicodeError, TypeError):
        jsonReport = jsonpickle.encode(convertDirtyDict2ASCII(dbresults))
    try:
        if not os.path.exists(exportDir + fileName):
            report = codecs.open(os.path.join(exportDir, fileName), "w", "utf-8")
            report.write(jsonReport)
            report.close()
    except (TypeError, IOError) as e:
        raise Exception("Failed to generate JSON report: %s" % e)
Esempio n. 15
0
 def _getTags(self, results, objfile, unpacked, extracted):
     tags = {}
     
     tags["Collector"] = "Ragpicker"
     # Analyse-UUID
     tags["Ragpicker-uuid"] = results.get("Info").get("analyse").get("uuid")
     
     if extracted:
         tags["OrigFileType"] = objfile.file.get_type()
         tags["ExtractedFrom"] = objfile.file.get_fileSha256()
     if unpacked:
         tags["OrigFileType"] = objfile.file.get_type()
         
     #clean tags
     for k in tags: 
         tags[k] = convertDirtyDict2ASCII(tags[k])
     
     log.debug(tags)
     
     return tags    
Esempio n. 16
0
    def run(self, results, objfile):
        """Writes report.
        @param results: results dict.
        @param objfile: file object
        @raise Exception: if fails to write report.
        """
        dumpdir = self.options.get("dumpdir", None)

        if not dumpdir:
            raise Exception("dumpdir not configured, skip")

        try:
            if not os.path.exists(dumpdir):
                os.makedirs(dumpdir)
            d = tempfile.mkdtemp(dir=dumpdir)
        except Exception as e:
            raise Exception('Could not open %s for writing (%s)', dumpdir, e)
        else:
            os.rmdir(d)

        url_md5 = results["Info"]["url"]["md5"]
        file_md5 = results["Info"]["file"]["md5"]
        jfile = url_md5 + "_" + file_md5 + ".json"

        try:
            jsonpickle.set_encoder_options('simplejson', indent=4)
            jsonpickle.handlers.registry.register(datetime.datetime,
                                                  DatetimeHandler)
            jsonpickle.handlers.registry.register(uuid.UUID, UUIDHandler)
            jsonReport = jsonpickle.encode(results)
        except (UnicodeError, TypeError):
            jsonReport = jsonpickle.encode(convertDirtyDict2ASCII(results))

        try:
            if not os.path.exists(dumpdir + jfile):
                report = codecs.open(os.path.join(dumpdir, jfile), "w",
                                     "utf-8")
                report.write(jsonReport)
                report.close()
        except (TypeError, IOError) as e:
            raise Exception("Failed to generate JSON report: %s" % e)
Esempio n. 17
0
    def run(self, results, objfile):
        dumpdir = self.options.get("dumpdir", None)

        if not dumpdir:
            raise Exception("dumpdir not configured, skip")

        try:
            if not os.path.exists(dumpdir):
                os.makedirs(dumpdir)
            d = tempfile.mkdtemp(dir=dumpdir)
        except Exception as e:
            raise Exception('Could not open %s for writing (%s)', dumpdir, e)
        else:
            os.rmdir(d)

            url_md5 = results["Info"]["url"]["md5"]
            file_md5 = results["Info"]["file"]["md5"]
            jfile = url_md5 + "_" + file_md5 + ".html"

            if not os.path.exists(dumpdir + jfile):
                try:
                    env = Environment(autoescape=True)
                    env.loader = FileSystemLoader(
                        os.path.join(RAGPICKER_ROOT, "data", "html"))
                    template = env.get_template("report.html")
                    reporthtml = template.render({"results": results})
                except UnicodeDecodeError:
                    reporthtml = template.render(
                        {"results": convertDirtyDict2ASCII(results)})
                except Exception as e:
                    raise Exception("Failed to generate HTML report: %s" % e)

                try:
                    reportfile = codecs.open(os.path.join(dumpdir, jfile), "w",
                                             "utf-8")
                    reportfile.write(reporthtml)
                    reportfile.close()
                except (TypeError, IOError) as e:
                    raise Exception("Failed to write HTML report: %s" % e)
Esempio n. 18
0
    def run(self, results, objfile):
        """Writes report.
        @param results: results dict.
        @param objfile: file object
        @raise Exception: if fails to write report.
        """
        dumpdir = self.options.get("dumpdir", None)

        if not dumpdir:
            raise Exception("dumpdir not configured, skip")
        
        try:
            if not os.path.exists(dumpdir):
                os.makedirs(dumpdir)  
            d = tempfile.mkdtemp(dir=dumpdir)
        except Exception as e:
            raise Exception('Could not open %s for writing (%s)', dumpdir, e)
        else:
            os.rmdir(d)
        
        url_md5 = results["Info"]["url"]["md5"]
        file_md5 = results["Info"]["file"]["md5"]
        jfile = url_md5 + "_" + file_md5 + ".json"

        try:
            jsonpickle.set_encoder_options('simplejson', indent=4) 
            jsonpickle.handlers.registry.register(datetime.datetime, DatetimeHandler)
            jsonpickle.handlers.registry.register(uuid.UUID, UUIDHandler)
            jsonReport = jsonpickle.encode(results)
        except (UnicodeError, TypeError):
            jsonReport = jsonpickle.encode(convertDirtyDict2ASCII(results))
        
        try:  
            if not os.path.exists(dumpdir + jfile):
                report = codecs.open(os.path.join(dumpdir, jfile), "w", "utf-8")      
                report.write(jsonReport)
                report.close()
        except (TypeError, IOError) as e:
            raise Exception("Failed to generate JSON report: %s" % e)    
Esempio n. 19
0
    def run(self, results, objfile):
        dumpdir = self.options.get("dumpdir", None)

        if not dumpdir:
            raise Exception("dumpdir not configured, skip")

        try:
            if not os.path.exists(dumpdir):
                os.makedirs(dumpdir)
            d = tempfile.mkdtemp(dir=dumpdir)
        except Exception as e:
            raise Exception('Could not open %s for writing (%s)', dumpdir, e)
        else:
            os.rmdir(d)

            url_md5 = results["Info"]["url"]["md5"]
            file_md5 = results["Info"]["file"]["md5"]
            jfile = url_md5 + "_" + file_md5 + ".xml"

            if not os.path.exists(dumpdir + jfile):
                try:
                    reportxml = dict2xml.dicttoxml(results)
                except UnicodeDecodeError:
                    reportxml = dict2xml.dicttoxml(
                        convertDirtyDict2ASCII(results))
                except Exception as e:
                    raise Exception("Failed to generate XML report: %s" % e)

                try:

                    reportfile = codecs.open(os.path.join(dumpdir, jfile), "w",
                                             "utf-8")
                    reportfile.write(reportxml)
                    reportfile.close()
                except (TypeError, IOError) as e:
                    raise Exception("Failed to write XML report: %s" % e)
Esempio n. 20
0
    def run(self, results, objfile):
        dumpdir = self.options.get("dumpdir", None)

        if not dumpdir:
            raise Exception("dumpdir not configured, skip")
        
        try:
            if not os.path.exists(dumpdir):
                os.makedirs(dumpdir)  
            d = tempfile.mkdtemp(dir=dumpdir)
        except Exception as e:
            raise Exception('Could not open %s for writing (%s)', dumpdir, e)
        else:
            os.rmdir(d)
            
            url_md5 = results["Info"]["url"]["md5"]
            file_md5 = results["Info"]["file"]["md5"]
            jfile = url_md5 + "_" + file_md5 + ".html"
            
            if not os.path.exists(dumpdir + jfile):
                try:
                    env = Environment(autoescape=True)
                    env.loader = FileSystemLoader(os.path.join(RAGPICKER_ROOT, "data", "html"))
                    template = env.get_template("report.html")
                    reporthtml = template.render({"results" : results})
                except UnicodeDecodeError:
                    reporthtml = template.render({"results" : convertDirtyDict2ASCII(results)})
                except Exception as e:
                    raise Exception("Failed to generate HTML report: %s" % e)
                        
                try:
                    reportfile = codecs.open(os.path.join(dumpdir, jfile), "w", "utf-8")
                    reportfile.write(reporthtml)
                    reportfile.close()
                except (TypeError, IOError) as e:
                    raise Exception("Failed to write HTML report: %s" % e)      
Esempio n. 21
0
    def _getTags(self, results, objfile, unpacked, extracted):
        tags = {}
        resultsFile = results.get("Info").get("file")

        tags["Collector"] = "Ragpicker"

        # Antivirus is no longer used
        # for k, v in results.items():
        #    if "Antivirus" in k:
        #        tags.update(flatten_dict(v))

        # Analyse-UUID
        tags["Ragpicker-uuid"] = results.get("Info").get("analyse").get("uuid")

        # Special hashes
        if resultsFile.get("pehash"):
            tags["PEHash"] = resultsFile.get("pehash")
        if resultsFile.get("imphash"):
            tags["ImpHash"] = resultsFile.get("imphash")

        if extracted:
            tags["OrigFileType"] = objfile.file.get_type()
            tags["ExtractedFrom"] = objfile.file.get_fileSha256()
        if unpacked:
            tags["OrigFileType"] = objfile.file.get_type()

        # PE-File CPU, Subsystem, Architecture
        if resultsFile.get("Subsystem"):
            tags["Subsystem"] = resultsFile.get("Subsystem")
        if resultsFile.get("Architecture"):
            tags["Architecture"] = resultsFile.get("Architecture")
        if resultsFile.get("CPU"):
            tags["CPU"] = resultsFile.get("CPU")

        if not unpacked or not extracted:
            # Digital Signature
            try:
                if resultsFile.has_key("digitalSignature"):
                    tags["DigitalSignature"] = results.get("Info").get("file").get("digitalSignature")
            except KeyError:
                # Key is not present
                pass

            try:
                if results.has_key("VerifySigs"):
                    if "ValidationError" in results.get("VerifySigs"):
                        tags["ValidationError"] = results.get("VerifySigs").get("ValidationError")
                    else:
                        tags.update(flatten_dict(results.get("VerifySigs")))
            except KeyError:
                # Key is not present
                pass

            if results.has_key("PEID"):
                tags["PEID"] = results.get("PEID")[0]

            if results.has_key("Teamcymru"):
                tags["Teamcymru"] = "malwarepercent=%s" % results.get("Teamcymru").get("malwarepercent")

            # VirusTotal
            try:
                if results.has_key("VirusTotal") and results.get("VirusTotal").has_key("file"):
                    vtFile = results.get("VirusTotal").get("file")
                    s = "%s/%s" % (vtFile.get("positives"), vtFile.get("total"))
                    tags["VirusTotal"] = s

                    if vtFile.has_key("scannerMalwareFamily"):
                        family = vtFile.get("scannerMalwareFamily")
                        tags["AvScannerMalwareFamily"] = "%s (count=%s)" % (family.get("family"), family.get("count"))

            except KeyError:
                # Key is not present
                pass

        # clean tags
        for k in tags:
            tags[k] = convertDirtyDict2ASCII(tags[k])

        log.debug(tags)

        return tags
Esempio n. 22
0
 def insertCodeDB(self, report):
     # Store the report
     try:
         self.__codedbCollectionCodedb.insert(report)
     except InvalidStringData:
         self.__codedbCollectionCodedb.insert(convertDirtyDict2ASCII(report))         
Esempio n. 23
0
    def _getTags(self, results, objfile, unpacked, extracted):
        tags = {}
        resultsFile = results.get("Info").get("file")

        tags["Collector"] = "Ragpicker"

        # Antivirus is no longer used
        #for k, v in results.items():
        #    if "Antivirus" in k:
        #        tags.update(flatten_dict(v))

        # Analyse-UUID
        tags["Ragpicker-uuid"] = results.get("Info").get("analyse").get("uuid")

        # Special hashes
        if resultsFile.get("pehash"):
            tags["PEHash"] = resultsFile.get("pehash")
        if resultsFile.get("imphash"):
            tags["ImpHash"] = resultsFile.get("imphash")

        if extracted:
            tags["OrigFileType"] = objfile.file.get_type()
            tags["ExtractedFrom"] = objfile.file.get_fileSha256()
        if unpacked:
            tags["OrigFileType"] = objfile.file.get_type()

        # PE-File CPU, Subsystem, Architecture
        if resultsFile.get("Subsystem"):
            tags["Subsystem"] = resultsFile.get("Subsystem")
        if resultsFile.get("Architecture"):
            tags["Architecture"] = resultsFile.get("Architecture")
        if resultsFile.get("CPU"):
            tags["CPU"] = resultsFile.get("CPU")

        if not unpacked or not extracted:
            # Digital Signature
            try:
                if resultsFile.has_key("digitalSignature"):
                    tags["DigitalSignature"] = results.get("Info").get(
                        "file").get("digitalSignature")
            except KeyError:
                # Key is not present
                pass

            try:
                if results.has_key("VerifySigs"):
                    if "ValidationError" in results.get("VerifySigs"):
                        tags["ValidationError"] = results.get(
                            "VerifySigs").get("ValidationError")
                    else:
                        tags.update(flatten_dict(results.get("VerifySigs")))
            except KeyError:
                # Key is not present
                pass

            if results.has_key("PEID"):
                tags["PEID"] = results.get("PEID")[0]

            if results.has_key("Teamcymru"):
                tags["Teamcymru"] = "malwarepercent=%s" % results.get(
                    "Teamcymru").get("malwarepercent")

            # VirusTotal
            try:
                if results.has_key("VirusTotal") and results.get(
                        "VirusTotal").has_key("file"):
                    vtFile = results.get("VirusTotal").get("file")
                    s = "%s/%s" % (vtFile.get("positives"),
                                   vtFile.get("total"))
                    tags["VirusTotal"] = s

                    if vtFile.has_key("scannerMalwareFamily"):
                        family = vtFile.get("scannerMalwareFamily")
                        tags["AvScannerMalwareFamily"] = "%s (count=%s)" % (
                            family.get("family"), family.get("count"))

            except KeyError:
                # Key is not present
                pass

        #clean tags
        for k in tags:
            tags[k] = convertDirtyDict2ASCII(tags[k])

        log.debug(tags)

        return tags
Esempio n. 24
0
 def add_field(self, name, value):
     value = convertDirtyDict2ASCII(value)
     """Add a simple field to the form data."""
     self.form_fields.append((name, value))
     return
Esempio n. 25
0
 def add_field(self, name, value):
     value = convertDirtyDict2ASCII(value)
     
     """Add a simple field to the form data."""
     self.form_fields.append((name, value))
     return