Example #1
0
def upload_view():
    upload_file = request.files['file']
    file_stream = upload_file.stream.read()
    if file_stream:
        #: Collect upload file data
        sample = {
            'filename': secure_filename(upload_file.filename),
            'sha1': hashlib.sha1(file_stream).hexdigest().upper(),
            'sha256': hashlib.sha256(file_stream).hexdigest().upper(),
            'md5': hashlib.md5(file_stream).hexdigest().upper(),
            'ssdeep': pydeep.hash_buf(file_stream),
            'filesize': len(file_stream),
            'filetype': magic.from_buffer(file_stream),
            'filemime': upload_file.mimetype,
            'upload_date': r.now(),
            'uploaded_by': "api",  # g.user
            'detection_ratio': dict(infected=0, count=0),
            'filestatus': "Processing"
        }
        insert_in_samples_db(sample)
        update_upload_file_metadata(sample)
        #: Run all configured scanners
        sample['detection_ratio'] = scan_upload(file_stream, sample)
        #: Done Processing File
        sample['filestatus'] = 'Complete'
        sample['scancomplete'] = r.now()
        update_sample_in_db(sample)
        found = is_hash_in_db(sample['md5'])
        if found:
            return jsonify(found)
        else:
            return jsonify(
                dict(error='Not a valid API end point.', response=404)), 404
    else:
        return jsonify(dict(error='Missing Parameters', response=400)), 400
Example #2
0
    def _pydeep_page(self):
        """Run pydeep and return the hash"""

        page_sig = None

        try:
            if self._config.SSDEEP_SIG:
                # s = self._config.YARA_RULES
                ## Don't wrap hex or regex rules in quotes
                # if s[0] not in ("{", "/"): s = '"' + s + '"'
                ## Scan for unicode strings
                # if self._config.WIDE: s += "wide"
                # rules = yara.compile(sources = {
                #'n' : 'rule r1 {strings: $a = ' + s + ' condition: $a}'
                # })
                pass
            elif self._config.SSDEEP_FILE:
                # rules = yara.compile(self._config.YARA_FILE)
                pass
            elif self._config.SSDEEP_PIDOFF:
                (pid, base) = self._config.SSDEEP_PIDOFF.split(":")
                for proc in tasks.pslist(self._addr_space):
                    if proc.UniqueProcessId == int(pid):
                        process_space = proc.get_process_address_space()
                        page_data = process_space.zread(int(base, 16), 0x1000)
                        page_sig = pydeep.hash_buf(page_data)
                if page_sig == "3::":
                    debug.error("PID XXX and OFFSET YYY null or not found")
            else:
                debug.error("You must specify an ssdeep hash (-Y), a file to hash (-y), or a PID:BASE pair (-T)")
        except Exception as why:
            debug.error("Cannot compile rules: {0}".format(str(why)))

        return page_sig
Example #3
0
 def _pydeep_page(self):
     """Run pydeep and return the hash"""
 
     page_sig = None
 
     try:
         if self._config.SSDEEP_SIG:
             #s = self._config.YARA_RULES
             ## Don't wrap hex or regex rules in quotes 
             #if s[0] not in ("{", "/"): s = '"' + s + '"'
             ## Scan for unicode strings 
             #if self._config.WIDE: s += "wide"
             #rules = yara.compile(sources = {
                         #'n' : 'rule r1 {strings: $a = ' + s + ' condition: $a}'
                         #})
             pass
         elif self._config.SSDEEP_FILE:
             #rules = yara.compile(self._config.YARA_FILE)
             pass
         elif self._config.SSDEEP_PIDOFF:
             (pid, base) = self._config.SSDEEP_PIDOFF.split(':')
             for proc in tasks.pslist(self._addr_space):
                 if proc.UniqueProcessId == int(pid):
                     process_space = proc.get_process_address_space()
                     page_data = process_space.zread(int(base, 16), 0x1000)
                     page_sig = pydeep.hash_buf(page_data)
             if page_sig == "3::":
                 debug.error('PID XXX and OFFSET YYY null or not found')
         else:
             debug.error("You must specify an ssdeep hash (-Y), a file to hash (-y), or a PID:BASE pair (-T)")
     except Exception as why:
         debug.error("Cannot compile rules: {0}".format(str(why)))
         
     return page_sig
Example #4
0
def data_hashes(data, algo="sha256"):
    if not data:
        return None
    algo = algo.lower()
    if algo == "crc32":
        return int("%d" % (zlib.crc32(data) & 0xffffffff))
    elif algo == "adler32":
        return "%d" % (zlib.adler32(data) & 0xffffffff)
    elif algo == "md5":
        hasher = hashlib.md5()
    elif algo == "sha1":
        hasher = hashlib.sha1()
    elif algo == "sha224":
        hasher = hashlib.sha224()
    elif algo == "sha256":
        hasher = hashlib.sha256()
    elif algo == "sha384":
        hasher = hashlib.sha384()
    elif algo == "sha512":
        hasher = hashlib.sha512()
    elif algo == "ssdeep":
        if hasattr(pydeep, "hash_data"):
            return pydeep.hash_data(data)
        elif hasattr(pydeep, "hash_buf"):
            return pydeep.hash_buf(data)
        else:
            return None
    else:
        return None
    hasher.update(data)
    return hasher.hexdigest()
Example #5
0
def import_dir(directory, r):
    p = r.pipeline(False)
    md5s = []
    sha1s = []
    sha256s = []
    for (dirpath, dirnames, filenames) in os.walk(args.dir):
        for filename in filenames:
            path = os.path.join(dirpath, filename)
            content = open(path, 'rb').read()
            md5 = hashlib.md5(content).hexdigest()
            sha1 = hashlib.sha1(content).hexdigest()
            sha256 = hashlib.sha256(content).hexdigest()
            ssdeep = pydeep.hash_buf(content)
            md5s.append(md5)
            sha1s.append(sha1)
            sha256s.append(sha256)
            p.hmset(
                sha256, {
                    'md5': md5,
                    'sha1': sha1,
                    'filename': filename,
                    'path': path,
                    'ssdeep': ssdeep
                })
    p.execute()
    return md5s, sha1s, sha256s
Example #6
0
def upload_view():
    upload_file = request.files['file']
    file_stream = upload_file.stream.read()
    if file_stream:
        #: Collect upload file data
        sample = {'filename': secure_filename(upload_file.filename),
                  'sha1': hashlib.sha1(file_stream).hexdigest().upper(),
                  'sha256': hashlib.sha256(file_stream).hexdigest().upper(),
                  'md5': hashlib.md5(file_stream).hexdigest().upper(),
                  'ssdeep': pydeep.hash_buf(file_stream),
                  'filesize': len(file_stream),
                  'filetype': magic.from_buffer(file_stream),
                  'filemime': upload_file.mimetype,
                  'upload_date': r.now(),
                  'uploaded_by': "api",  # g.user
                  'detection_ratio': dict(infected=0, count=0),
                  'filestatus': "Processing"}
        insert_in_samples_db(sample)
        update_upload_file_metadata(sample)
        #: Run all configured scanners
        sample['detection_ratio'] = scan_upload(file_stream, sample)
        #: Done Processing File
        sample['filestatus'] = 'Complete'
        sample['scancomplete'] = r.now()
        update_sample_in_db(sample)
        found = is_hash_in_db(sample['md5'])
        if found:
            return jsonify(found)
        else:
            return jsonify(dict(error='Not a valid API end point.',
                                response=404)), 404
    else:
        return jsonify(dict(error='Missing Parameters', response=400)), 400
Example #7
0
    def _load_meta(self, db, _object):
        content = base64.b64decode(_object["content"])
        entry = {
            "magic":  magic.from_buffer(content),
            "ssdeep": pydeep.hash_buf(content),
            "md5":    hashlib.md5(content).hexdigest(),
            "sha1":   hashlib.sha1(content).hexdigest(),
            "sha256": hashlib.sha256(content).hexdigest()
        }

        if entry["magic"] == "MS-DOS executable":
            ### This is a weak application of magic
            try: 
                pe_data = self._get_pe(content)
                for k, v in pe_data.iteritems(): entry[k] = v
            except Exception, e: print e; pass 
            pass
        #entry_copy = copy.deepcopy(entry)
        #del entry
        #del content
        #gc.collect()

        db.table("content").get(_object["id"]).update({"load_meta": entry}).run()
        print "Loaded meta for object (%s) %s." % (_object["firmware_id"], _object["id"])
        pass
Example #8
0
 def generate_attributes(self):
     self.add_attribute('name', value=self.__section.name)
     size = self.add_attribute('size-in-bytes', value=self.__section.size)
     if int(size.value) > 0:
         # zero-filled sections can create too many correlations
         to_ids = float(self.__section.entropy) > 0
         disable_correlation = not to_ids
         self.add_attribute('entropy', value=self.__section.entropy)
         self.add_attribute('md5',
                            value=md5(self.__data).hexdigest(),
                            disable_correlation=disable_correlation,
                            to_ids=to_ids)
         self.add_attribute('sha1',
                            value=sha1(self.__data).hexdigest(),
                            disable_correlation=disable_correlation,
                            to_ids=to_ids)
         self.add_attribute('sha256',
                            value=sha256(self.__data).hexdigest(),
                            disable_correlation=disable_correlation,
                            to_ids=to_ids)
         self.add_attribute('sha512',
                            value=sha512(self.__data).hexdigest(),
                            disable_correlation=disable_correlation,
                            to_ids=to_ids)
         if HAS_PYDEEP and float(self.__section.entropy) > 0:
             if self.__section.name == '.rsrc':
                 # ssdeep of .rsrc creates too many correlations
                 disable_correlation = True
                 to_ids = False
             self.add_attribute('ssdeep',
                                value=pydeep.hash_buf(self.__data).decode(),
                                disable_correlation=disable_correlation,
                                to_ids=to_ids)
Example #9
0
def parse_dl(fname):
    try:
        filetype=magic.from_file(fname)
        filecontent=open(fname,'rb').read()
        (mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime) = os.stat(fname)
        dl_file={'filename':fname, 'mimetype':filetype,'size':size,'atime':time.strftime("%a, %d %b %Y %X GMT", time.gmtime(atime)),'ctime':time.strftime("%a, %d %b %Y %X GMT", time.gmtime(ctime)),'mtime':time.strftime("%a, %d %b %Y %X GMT", time.gmtime(mtime))}
        if filetype=='HTML document, ASCII text, with CRLF line terminators' or filetype=='XML document text':
            dl_file['jar_href']=re.findall(r'\<jar\ href\=\"(.*?)\"',filecontent)[0]
            main_class_arr=re.findall('\<applet\-desc.*main\-class\=\"(.*?)\"',filecontent)
            if main_class_arr:
                dl_file['main_class']=main_class_arr[0]
            dl_file['parameters']={}
            for param,value in re.findall(r'<param name="(.*?)" value="(.*?)"',filecontent):
                dl_file['parameters'][param]=value
            
        m = hashlib.md5()
        m.update(filecontent)
        dl_file['md5']=m.digest().encode('hex')
    
        # computing sha1
        s = hashlib.sha1()               
        s.update(filecontent)
        dl_file['sha1']=s.digest().encode('hex')
    
        # computing ssdeep
        dl_file['ssdeep']=pydeep.hash_buf(filecontent)
        return dl_file
    except:
        print "Unable to stat the downloaded file"
Example #10
0
def get_fuzzy(data):
    """
    Uses SSDeep's fuzzy.dll to return a fuzzy hash for a block of data
    Based off of http://codepaste.ru/13245/

    Arguments:
        data: binary data to perform hash of
    """
    error_code = ''
    try:
        import pydeep
        return pydeep.hash_buf(data)
    except ImportError:  # Oh man, this is going to be ugly
        fuzzy_dll = os.path.join(SCRIPT_PATH, 'fuzzy.dll')
        if not file_exists(fuzzy_dll):
            root_fuzzy_dll = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), fuzzy_dll)
            if file_exists(root_fuzzy_dll):
                fuzzy_dll = root_fuzzy_dll        
            
        try:
            fuzzy = ctypes.CDLL(fuzzy_dll)
        except WindowsError, error:
            error = str(error)
            if '[Error ' in error:
                error_code = error.split()[1].split(']')[0]
        if error_code:
            if error_code == '193':
                py_bits = struct.calcsize('P') * 8
                return '[!] %s incompatible. Needs to be same as Python: %d-bits' % (fuzzy_dll, py_bits)
            elif error_code == '126':
                return '[!] %s not found' % fuzzy_dll
            else:
                return '[!] %s not loaded. Unknown error.'
            return
Example #11
0
def upload():
    form = SearchForm(request.form)
    if request.method == 'POST':
        # TODO: use secure_filename
        for upload_file in request.files.getlist('files[]'):
            file_stream = upload_file.stream.read()
            file_md5 = hashlib.md5(file_stream).hexdigest().upper()
            #: Add file hash to Bloomfilter unless it is already there
            #: Check if user wishes to force a sample rescan
            if file_md5 not in bf or form.force.data:
                bf.add(file_md5)
                #: Collect upload file data
                sample = {'filename': secure_filename(upload_file.filename.encode('utf-8')),
                          'sha1': hashlib.sha1(file_stream).hexdigest().upper(),
                          'sha256': hashlib.sha256(file_stream).hexdigest().upper(),
                          'md5': file_md5,
                          'ssdeep': pydeep.hash_buf(file_stream),
                          'filesize': len(file_stream),
                          'filetype': magic.from_buffer(file_stream),
                          'filemime': upload_file.mimetype,
                          'upload_date': r.now(),
                          'uploaded_by': "jmaine", # g.user
                          'detection_ratio': dict(infected=0, count=0),
                          'filestatus': 'Processing'}
                insert_in_samples_db(sample)
                update_upload_file_metadata(sample)
                #: Run all configured scanners
                sample['detection_ratio'] = scan_upload(file_stream, sample)
                #: Done Processing File
                sample['filestatus'] = 'Complete'
                sample['scancomplete'] = r.now()
                update_sample_in_db(sample)
        #: Once Finished redirect user to the samples page
        return redirect(url_for('.samples'))
    return render_template('samples.html')
 def __init__(self, jsonObj):
     self.address = jsonObj['address']
     self.name = jsonObj['contract_name']
     self.bytecode = jsonObj['bytecode']
     self.hash = pydeep.hash_buf(self.bytecode)
     contract.contract_list.append(self)
     contract.address_list.append(self.address)
     contract.name_list.append(self.name)
     contract.hash_list.append(self.hash)
Example #13
0
def test_pydeep():
    for test in testL:
        filename, filelen, filehash = test
        data = io.open(filename, 'rb').read()
        hash_buf = pydeep.hash_buf(data)
        hash_file = pydeep.hash_file(filename)
        assert len(data) == filelen, "File length error"
        assert hash_buf == filehash, "Error hashing %s using hash_buf"%filename
        assert hash_file == filehash, "Error hashing %s using hash_file"%filename
Example #14
0
def upload():
    form = SearchForm(request.form)
    if request.method == 'POST':
        # TODO: use secure_filename
        for upload_file in request.files.getlist('files[]'):
            file_stream = upload_file.stream.read()
            file_md5 = hashlib.md5(file_stream).hexdigest().upper()
            #: Add file hash to Bloomfilter unless it is already there
            #: Check if user wishes to force a sample rescan
            if file_md5 not in bf or form.force.data:
                bf.add(file_md5)
                #: Collect upload file data
                sample = {
                    'filename':
                    secure_filename(upload_file.filename.encode('utf-8')),
                    'sha1':
                    hashlib.sha1(file_stream).hexdigest().upper(),
                    'sha256':
                    hashlib.sha256(file_stream).hexdigest().upper(),
                    'md5':
                    file_md5,
                    'ssdeep':
                    pydeep.hash_buf(file_stream),
                    'filesize':
                    len(file_stream),
                    'filetype':
                    magic.from_buffer(file_stream),
                    'filemime':
                    upload_file.mimetype,
                    'upload_date':
                    r.now(),
                    'uploaded_by':
                    "anonymous",  # g.user
                    'detection_ratio':
                    dict(infected=0, count=0),
                    'filestatus':
                    'Processing'
                }
                insert_in_samples_db(sample)
                update_upload_file_metadata(sample)
                #: Run all configured scanners
                sample['detection_ratio'] = scan_upload(file_stream, sample)
                #: Done Processing File
                sample['filestatus'] = 'Complete'
                sample['scancomplete'] = r.now()
                update_sample_in_db(sample)
            else:
                # flash('File {0} already submitted. Visit: {1}'.format(secure_filename(upload_file.filename.encode('utf-8')), file_md5), 'error')
                flash(
                    'File {} already submitted.'.format(
                        secure_filename(upload_file.filename.encode('utf-8'))),
                    'error')
                return redirect(url_for('.index'))
        #: Once Finished redirect user to the samples page
        return redirect(url_for('.samples'))
        # return render_template('samples.html')
    return render_template('samples.html')
Example #15
0
 def get_ssdeep(self):
     if not HAVE_SSDEEP:
         return None
     try:
         return pydeep.hash_buf(self.file_data)
     except MemoryError:
         logging.exception("Out of memory")
         sys.exit("Out of memory error")
     except Exception:
         return None
Example #16
0
 def generate_attributes(self):
     self.name = self.section_info['Name']['Value']
     self.size = self.section_info['SizeOfRawData']['Value']
     if self.size > 0:
         self.entropy = self.section_info['Entropy']
         self.md5 = self.section_info['MD5']
         self.sha1 = self.section_info['SHA1']
         self.sha256 = self.section_info['SHA256']
         self.sha512 = self.section_info['SHA512']
         self.ssdeep = pydeep.hash_buf(self.data).decode()
Example #17
0
File: lib.py Project: CIRCL/ssdc
def prepare_hashes(r, buf, path):
    deephash = pydeep.hash_buf(buf)
    sha256 = hashlib.sha256(file(path, 'rb').read()).hexdigest()
    p = r.pipeline(False)
    p.hmset(sha256, {'path': path, 'ssdeep': deephash})
    p.sadd('hashes', sha256)

    block_size, chunk, double_chunk = preprocess_hash(deephash)
    add_chunks_db(p, block_size, chunk, sha256)
    add_chunks_db(p, block_size, double_chunk, sha256)
    p.execute()
Example #18
0
 def generate_attributes(self):
     self.add_attribute('name', value=self.__section.name)
     size = self.add_attribute('size-in-bytes', value=self.__section.size)
     if int(size.value) > 0:
         self.add_attribute('entropy', value=self.__section.entropy)
         self.add_attribute('md5', value=md5(self.__data).hexdigest())
         self.add_attribute('sha1', value=sha1(self.__data).hexdigest())
         self.add_attribute('sha256', value=sha256(self.__data).hexdigest())
         self.add_attribute('sha512', value=sha512(self.__data).hexdigest())
         if HAS_PYDEEP:
             self.add_attribute('ssdeep', value=pydeep.hash_buf(self.__data).decode())
Example #19
0
 def generate_attributes(self):
     self.add_attribute('name', value=self.__section.name)
     size = self.add_attribute('size-in-bytes', value=self.__section.size)
     if int(size.value) > 0:
         self.add_attribute('entropy', value=self.__section.entropy)
         self.add_attribute('md5', value=md5(self.__data).hexdigest())
         self.add_attribute('sha1', value=sha1(self.__data).hexdigest())
         self.add_attribute('sha256', value=sha256(self.__data).hexdigest())
         self.add_attribute('sha512', value=sha512(self.__data).hexdigest())
         if HAS_PYDEEP:
             self.add_attribute('ssdeep', value=pydeep.hash_buf(self.__data).decode())
Example #20
0
 def generate_attributes(self):
     self.filename = os.path.basename(self.filepath)
     self.size = os.path.getsize(self.filepath)
     if self.size > 0:
         self.filetype = magic.from_buffer(self.data)
         self.entropy = self.__entropy_H(self.data)
         self.md5 = md5(self.data).hexdigest()
         self.sha1 = sha1(self.data).hexdigest()
         self.sha256 = sha256(self.data).hexdigest()
         self.sha512 = sha512(self.data).hexdigest()
         self.ssdeep = pydeep.hash_buf(self.data).decode()
Example #21
0
def hash_mem_file(file):
    mem_file = open(file, "r")

    mem_hashes = {}
    pos = 0
    mem_file.seek(pos)
    mem_buf = mem_file.read(4096)
    while len(mem_buf) > 0:
        mem_hashes[pos] = pydeep.hash_buf(mem_buf)
        pos += 4096
        mem_file.seek(pos)
        mem_buf = mem_file.read(4096)
    return mem_hashes
Example #22
0
 def generate_attributes(self):
     self.add_attribute('name', value=self.__section.name)
     self.add_attribute('type', value=str(self.__section.type).split('.')[1])
     for flag in self.__section.flags_list:
         self.add_attribute('flag', value=str(flag).split('.')[1])
     size = self.add_attribute('size-in-bytes', value=self.__section.size)
     if int(size.value) > 0:
         self.add_attribute('entropy', value=self.__section.entropy)
         self.add_attribute('md5', value=md5(self.__data).hexdigest())
         self.add_attribute('sha1', value=sha1(self.__data).hexdigest())
         self.add_attribute('sha256', value=sha256(self.__data).hexdigest())
         self.add_attribute('sha512', value=sha512(self.__data).hexdigest())
         if HAS_PYDEEP:
             self.add_attribute('ssdeep', value=pydeep.hash_buf(self.__data).decode())
Example #23
0
 def generate_attributes(self):
     self.add_attribute('filename', value=self.__filename)
     size = self.add_attribute('size-in-bytes', value=len(self.__data))
     if int(size.value) > 0:
         self.add_attribute('entropy', value=self.__entropy_H(self.__data))
         self.add_attribute('md5', value=md5(self.__data).hexdigest())
         self.add_attribute('sha1', value=sha1(self.__data).hexdigest())
         self.add_attribute('sha256', value=sha256(self.__data).hexdigest())
         self.add_attribute('sha512', value=sha512(self.__data).hexdigest())
         self.add_attribute('malware-sample', value=self.__filename, data=self.__pseudofile)
         if HAS_MAGIC:
             self.add_attribute('mimetype', value=magic.from_buffer(self.__data))
         if HAS_PYDEEP:
             self.add_attribute('ssdeep', value=pydeep.hash_buf(self.__data).decode())
Example #24
0
 def generate_attributes(self):
     self.add_attribute('filename', value=self.__filename)
     size = self.add_attribute('size-in-bytes', value=len(self.__data))
     if int(size.value) > 0:
         self.add_attribute('entropy', value=self.__entropy_H(self.__data))
         self.add_attribute('md5', value=md5(self.__data).hexdigest())
         self.add_attribute('sha1', value=sha1(self.__data).hexdigest())
         self.add_attribute('sha256', value=sha256(self.__data).hexdigest())
         self.add_attribute('sha512', value=sha512(self.__data).hexdigest())
         self.add_attribute('malware-sample', value=self.__filename, data=self.__pseudofile)
         if HAS_MAGIC:
             self.add_attribute('mimetype', value=magic.from_buffer(self.__data))
         if HAS_PYDEEP:
             self.add_attribute('ssdeep', value=pydeep.hash_buf(self.__data).decode())
Example #25
0
    def scan(self, offset, maxlen):
        # Start scanning from offset until maxlen:
        i = offset

        pydeep_hash = self.pydeep_hash

        while i < offset + maxlen:
            # Read some data and match it.
            data = self.address_space.zread(i, 0x1000)
            if data:
                data_hash = pydeep.hash_buf(data)
                alike = pydeep.compare(pydeep_hash, data_hash)
                if alike > 10:
                    yield data_hash, i, alike

            i += 0x1000
    def scan(self, offset, maxlen):
        # Start scanning from offset until maxlen:
        i = offset

        pydeep_hash = self.pydeep_hash

        while i < offset + maxlen:
            # Read some data and match it.
            data = self.address_space.zread(i, 0x1000)
            if data:
                data_hash = pydeep.hash_buf(data)
                alike = pydeep.compare(pydeep_hash, data_hash)
                if alike > 10:
                    yield data_hash, i, alike

            i += 0x1000
Example #27
0
def upload_form(request):

    
    if request.method == 'POST':
        form = UploadUrlForm(request.POST)
        if form.is_valid():
        
            uri = request.POST['uri']
            newurl = URL(
                uri = uri,
                ticket = request.POST['ticket'],
                md5 = hashlib.md5(uri).hexdigest(),
                fuzzy = pydeep.hash_buf(uri),
                #html = handler.get_html(uri),
            )
            ua = request.POST['UserAgent']
            results = handler.get_thug(uri, ua, request.POST['ticket'])
            
            #newurl.ssdeep_compare = unicode(handler.ssdeep_compare(newurl.fuzzy, newurl.md5), 'utf-8', errors="replace")
            newurl.ssdeep_compare = handler.ssdeep_compare(newurl.fuzzy, newurl.md5)
            newurl.html = unicode(results['html'], 'utf-8', errors="replace")
            newurl.thug = unicode(results['thug_res'], 'utf-8', errors="replace")
            newurl.js = unicode(results['js'], 'utf-8', errors="replace")
            newurl.js_didier = unicode(results['js_didier'], 'utf-8', errors="replace")
            
            #newurl.js = handler.get_js(newurl.html)
        
            #If VirusTotal is activated, get vt results
            #URL['vt']=handler.get_vt(url)
            
            newurl.save()

            newpage = "/uanalysis/url/" + newurl.md5

            return HttpResponseRedirect(newpage)
        else:
            form = UploadUrlForm()
            url = URL.objects.filter(created__lte=timezone.now()).order_by('-id')[:25]
            return render(request, 'uanalysis/upload_form.html', {'form': form, 'url': url})

    else:
        form = UploadUrlForm()
        url = URL.objects.filter(created__lte=timezone.now()).order_by('-id')[:25]
        return render(request, 'uanalysis/upload_form.html', {'form': form, 'url': url})
Example #28
0
def import_dir(directory, r):
    p = r.pipeline(False)
    md5s = []
    sha1s = []
    sha256s = []
    for (dirpath, dirnames, filenames) in os.walk(args.dir):
        for filename in filenames:
            path = os.path.join(dirpath, filename)
            content = open(path, 'rb').read()
            md5 = hashlib.md5(content).hexdigest()
            sha1 = hashlib.sha1(content).hexdigest()
            sha256 = hashlib.sha256(content).hexdigest()
            ssdeep = pydeep.hash_buf(content)
            md5s.append(md5)
            sha1s.append(sha1)
            sha256s.append(sha256)
            p.hmset(sha256, {'md5': md5, 'sha1': sha1, 'filename': filename, 'path': path, 'ssdeep': ssdeep})
    p.execute()
    return md5s, sha1s, sha256s
Example #29
0
    def get_hashes(self):
        crc = 0
        md5 = hashlib.md5()
        sha1 = hashlib.sha1()
        sha256 = hashlib.sha256()
        sha512 = hashlib.sha512()

        crc = binascii.crc32(self.artifact, crc)
        md5.update(self.artifact)
        sha1.update(self.artifact)
        sha256.update(self.artifact)
        sha512.update(self.artifact)

        self.hashes['crc32'] = ''.join('%02X' % ((crc>>i)&0xff) for i in [24, 16, 8, 0])
        self.hashes['md5'] = md5.hexdigest()
        self.hashes['sha1'] = sha1.hexdigest()
        self.hashes['sha256'] = sha256.hexdigest()
        self.hashes['sha512'] = sha512.hexdigest()
        self.hashes['ssdeep'] = pydeep.hash_buf(self.artifact)
Example #30
0
def get_fuzzy(data):
    """
    Uses SSDeep's fuzzy.dll to return a fuzzy hash for a block of data
    Based off of http://codepaste.ru/13245/

    Arguments:
        data: binary data to perform hash of
    """
    error_code = ''
    #try:
    if True:
        try:
            import pydeep
            return pydeep.hash_buf(data)
        except ImportError:
            return None
    #except ImportError:  # Oh man, this is going to be ugly
        fuzzy_dll = os.path.join(SCRIPT_PATH, 'fuzzy.dll')
        if not file_exists(fuzzy_dll):
            root_fuzzy_dll = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), fuzzy_dll)
            if file_exists(root_fuzzy_dll):
                fuzzy_dll = root_fuzzy_dll

        try:
            fuzzy = ctypes.CDLL(fuzzy_dll)
        except:
            return

        if error_code:
            if error_code == '193':
                py_bits = struct.calcsize('P') * 8
                return '[!] %s incompatible. Needs to be same as Python: %d-bits' % (fuzzy_dll, py_bits)
            elif error_code == '126':
                return '[!] %s not found' % fuzzy_dll
            else:
                return '[!] %s not loaded. Unknown error.'
            return


    out_buf = ctypes.create_string_buffer(b'\x00' * 1024)
    fuzzy.fuzzy_hash_buf(data, len(data), out_buf)
    return out_buf.value
Example #31
0
    def _load_meta(self, db, _object):
        content = base64.b64decode(_object["content"])
        entry = {
            "magic": magic.from_buffer(content),
            "ssdeep": pydeep.hash_buf(content),
            "md5": hashlib.md5(content).hexdigest(),
            "sha1": hashlib.sha1(content).hexdigest(),
            "sha256": hashlib.sha256(content).hexdigest()
        }

        if entry["magic"] == "MS-DOS executable":
            ### This is a weak application of magic
            try:
                pe_data = self._get_pe(content)
                for k, v in pe_data.iteritems():
                    entry[k] = v
            except Exception, e:
                print e
                pass
            pass
Example #32
0
    def get_hashes(self):
        crc = 0
        md5 = hashlib.md5()
        sha1 = hashlib.sha1()
        sha256 = hashlib.sha256()
        sha512 = hashlib.sha512()

        crc = binascii.crc32(self.artifact, crc)
        md5.update(self.artifact)
        sha1.update(self.artifact)
        sha256.update(self.artifact)
        sha512.update(self.artifact)

        self.hashes['crc32'] = ''.join('%02X' % ((crc >> i) & 0xff)
                                       for i in [24, 16, 8, 0])
        self.hashes['md5'] = md5.hexdigest()
        self.hashes['sha1'] = sha1.hexdigest()
        self.hashes['sha256'] = sha256.hexdigest()
        self.hashes['sha512'] = sha512.hexdigest()
        self.hashes['ssdeep'] = pydeep.hash_buf(self.artifact)
Example #33
0
                key = {'md5': sample['md5']}

                metadata = {}
                logger.debug('[%s] Downloading data' % sampleno)
                data = get_file(db, md5=sample['md5'])

                # Do analysis

                logger.debug('[%s] Analysing' % sampleno)

                # metadata['md5'] = hashlib.md5(data).hexdigest()

                metadata['sha1'] = hashlib.sha1(data).hexdigest()
                metadata['sha256'] = hashlib.sha256(data).hexdigest()
                metadata['sha512'] = hashlib.sha512(data).hexdigest()
                metadata['ssdeep'] = pydeep.hash_buf(data)

                # Store results

                logger.debug('[%s] Storing results into MongoDB'
                             % sampleno)
                for (metakey, metaval) in metadata.iteritems():
                    db.fs.files.update(key,
                            {'$set': {metakey: metaval}}, upsert=True)

                logger.debug('[%s] Removing temporary data' % sampleno)
                del key
                del metadata
                del data

                logger.info('[%s] Metadata updated' % sampleno)
Example #34
0
    def render_text(self, outfd, data):

        if not has_distorm3:
            debug.warning("For best results please install distorm3")

        if self._config.DUMP_DIR and not os.path.isdir(self._config.DUMP_DIR):
            debug.error(self._config.DUMP_DIR + " is not a directory")

        for task in data:
            for vad, address_space in task.get_vads(vad_filter = task._injection_filter):

                if self._is_vad_empty(vad, address_space):
                    continue

                if self._config.SSDEEP and has_pydeep:
                    skip = False
                    # read the first page of the VAD then hash it
                    inject_buf = address_space.zread(vad.Start, 0x1000)
                    inject_hash = pydeep.hash_buf(inject_buf)
                    # loop through all the whitelist hashes and compare
                    for (whitelist_name, whitelist_hash) in whitelist_ssdeep:
                        alike = pydeep.compare(inject_hash, whitelist_hash)
                        # the comparison is greater than the threshold so display an informational message
                        # then skip the rest of the output in normal malfind
                        if alike > self._config.THRESHOLD:
                            outfd.write("Process: {0} Pid: {1} Address: {2:#x}\n".format(
                                task.ImageFileName, task.UniqueProcessId, vad.Start))
                            outfd.write("Injection is {0}% similar to whitelist hook {1}\n".format(alike, whitelist_name))
                            #outfd.write("  hook: {0}\n".format(inject_hash))
                            #outfd.write("  whitelist: {0}\n".format(whitelist_hash))
                            skip = True
                            continue
                    if skip:
                        continue

                content = address_space.zread(vad.Start, 64)

                outfd.write("Process: {0} Pid: {1} Address: {2:#x}\n".format(
                    task.ImageFileName, task.UniqueProcessId, vad.Start))

                outfd.write("Vad Tag: {0} Protection: {1}\n".format(
                    vad.Tag, vadinfo.PROTECT_FLAGS.get(vad.u.VadFlags.Protection.v(), "")))

                outfd.write("Flags: {0}\n".format(str(vad.u.VadFlags)))
                outfd.write("\n")

                outfd.write("{0}\n".format("\n".join(
                    ["{0:#010x}  {1:<48}  {2}".format(vad.Start + o, h, ''.join(c))
                    for o, h, c in utils.Hexdump(content)
                    ])))

                outfd.write("\n")
                outfd.write("\n".join(
                    ["{0:#x} {1:<16} {2}".format(o, h, i)
                    for o, i, h in malfind.Disassemble(content, vad.Start)
                    ]))

                # Dump the data if --dump-dir was supplied
                if self._config.DUMP_DIR:

                    filename = os.path.join(self._config.DUMP_DIR,
                        "process.{0:#x}.{1:#x}.dmp".format(
                        task.obj_offset, vad.Start))

                    self.dump_vad(filename, vad, address_space)

                outfd.write("\n\n")
Example #35
0
 def _get_ssdeep(self):
     try:
         return pydeep.hash_buf(self._file_data)
     except Exception as e:
         print e
         return None
Example #36
0
 def get_fuzzy(fh):
     fuzzy = pydeep.hash_buf(fh.read())
     position = fh.seek(0, 0)
     return fuzzy
Example #37
0
def parse_idx(fname):
    data=open(fname,'rb')
    filecontent=open(fname,'rb').read()
    filesize = os.path.getsize(fname)
    busy_byte = data.read(1)
    complete_byte = data.read(1)
    cache_ver = struct.unpack(">i", data.read(4))[0]
    idx_file={'filename':fname,'mimetype':"IDX file: %s (IDX File Version %d.%02d)" % (fname, cache_ver / 100, cache_ver - 600)}
    if cache_ver not in (602, 603, 604, 605, 606):
        print "Invalid IDX header found"
        print "Found:    0x%s" % cache_ver

    m = hashlib.md5()
    m.update(filecontent)
    idx_file['md5']=m.digest().encode('hex')
    
    # computing sha1
    s = hashlib.sha1()               
    s.update(filecontent)
    idx_file['sha1']=s.digest().encode('hex')

    # computing ssdeep
    idx_file['ssdeep']=pydeep.hash_buf(filecontent)
    
    # Different IDX cache versions have data in different offsets
    if cache_ver in [602, 603, 604, 605]:
        if cache_ver in [602, 603, 604]:
            data.seek(8)
        elif cache_ver == 605:
            data.seek(6)
        is_shortcut_img = data.read(1)
        content_len = struct.unpack(">l", data.read(4))[0] 
        last_modified_date = struct.unpack(">q", data.read(8))[0]/1000
        expiration_date = struct.unpack(">q", data.read(8))[0]/1000
        validation_date = struct.unpack(">q", data.read(8))[0]/1000
        sec1={}
        sec1['last_modified_date']=time.strftime("%a, %d %b %Y %X GMT", time.gmtime(last_modified_date))
        if expiration_date:
            sec1['expiration_date']=time.strftime("%a, %d %b %Y %X GMT", time.gmtime(expiration_date))
        if validation_date and cache_ver > 602: #While 6.02 technically supports this, every sample I've seen just has 3 null bytes and skips to Section 2
            sec1['validation_date']=time.strftime("%a, %d %b %Y %X GMT", time.gmtime(validation_date))
        
        if cache_ver == 602:
            sec2_len = 1
            sec3_len = 0
            sec4_len = 0
            sec5_len = 0
        elif cache_ver in [603, 604, 605]:
            known_to_be_signed = data.read(1)
            sec2_len = struct.unpack(">i", data.read(4))[0]
            sec3_len = struct.unpack(">i", data.read(4))[0]
            sec4_len = struct.unpack(">i", data.read(4))[0]
            sec5_len = struct.unpack(">i", data.read(4))[0]
            
            blacklist_timestamp = struct.unpack(">q", data.read(8))[0]/1000
            cert_expiration_date = struct.unpack(">q", data.read(8))[0]/1000
            class_verification_status = data.read(1)
            reduced_manifest_length = struct.unpack(">l", data.read(4))[0]
            
            #print "Section 2 length: %d" % sec2_len
            if sec3_len: print "Section 3 length: %d" % sec3_len
            if sec4_len: print "Section 4 length: %d" % sec4_len
            if sec5_len: print "Section 4 length: %d" % sec5_len
            if expiration_date:
                sec1['blacklist_date']=time.strftime("%a, %d %b %Y %X GMT", time.gmtime(blacklist_timestamp))
            if cert_expiration_date:
                sec1['cert_expiration_date']=time.strftime("%a, %d %b %Y %X GMT", time.gmtime(cert_expiration_date))
    else:
        print "Current file version, %d, is not supported at this time." % cache_ver

    if sec2_len:
        if cache_ver == 602: idx_file['sec2']=sec2_parse_old(data)
        else: idx_file['sec2']=sec2_parse(data)

    if sec3_len:
        #print "\n[*] Section 3 (Jar Manifest) found:" 
        idx_file['sec3']=sec3_parse(data,sec2_len,sec3_len,filesize)

    if sec4_len:
        #print "\n[*] Section 4 (Code Signer) found:"
        idx_file['sec4']=sec4_parse(data,sec2_len,sec3_len,filesize)
                
    if sec5_len:
        print "\n[*] Section 5 found (offset 0x%X, length %d bytes)" % (128 + sec2_len + sec3_len + sec4_len, sec5_len)
    
    return idx_file
Example #38
0
 def _get_ssdeep(self):
     try:
         return pydeep.hash_buf(self._file_data).decode()
     except Exception as e:
         logging.warn(f"Error: {e}")
         return None
Example #39
0
    def get_ssdeep(file_bytes):
        """Returns the SSDEEP."""

        return pydeep.hash_buf(file_bytes).decode()
    def render_text(self, outfd, data):
        for process, module, hook, addr_space in data:

            if not self._config.NO_WHITELIST:

                if process:
                    process_name = str(process.ImageFileName)
                else:
                    process_name = ''

                if self.whitelist(hook.hook_mode | hook.hook_type,
                                    process_name, hook.VictimModule,
                                    hook.HookModule, hook.Function):
                    #debug.debug("Skipping whitelisted function: {0} {1} {2} {3}".format(
                    #    process_name, hook.VictimModule, hook.HookModule, 
                    #    hook.Function))
                    continue

                if self._config.SSDEEP and has_pydeep:
                    skip = False
                    # read from the start of the page containing the hook, then hash it
                    page_address = hook.hook_address & 0xFFFFF000
                    hook_buf = addr_space.zread(page_address, 0x1000)
                    hook_hash = pydeep.hash_buf(hook_buf)
                    # loop through all the whitelist hashes and compare
                    for (whitelist_name, whitelist_hash) in whitelist_ssdeep:
                        alike = pydeep.compare(hook_hash, whitelist_hash)
                        # the comparison is greater than the threshold so display an informational message
                        # then skip the rest of the output in normal malfind
                        if alike > self._config.THRESHOLD:
                            if process:
                                outfd.write('Process: {0} ({1})\n'.format(
                                    process.UniqueProcessId, process.ImageFileName))
                            outfd.write("Hook at 0x{0:x} in page 0x{1:x} is {2}% similar to whitelist hook {3}\n".format(hook.hook_address, page_address, alike, whitelist_name))
                            #outfd.write("  hook: {0}\n".format(hook_hash))
                            #outfd.write("  whitelist: {0}\n".format(whitelist_hash))
                            outfd.write("\n")
                            skip = True
                            continue
                    if skip:
                        continue

            outfd.write("*" * 72 + "\n")
            outfd.write("Hook mode: {0}\n".format(hook.Mode))
            outfd.write("Hook type: {0}\n".format(hook.Type))

            if process:
                outfd.write('Process: {0} ({1})\n'.format(
                    process.UniqueProcessId, process.ImageFileName))

            outfd.write("Victim module: {0} ({1:#x} - {2:#x})\n".format(
                str(module.BaseDllName or '') or ntpath.basename(str(module.FullDllName or '')),
                module.DllBase, module.DllBase + module.SizeOfImage))

            outfd.write("Function: {0}\n".format(hook.Detail))
            outfd.write("Hook address: {0:#x}\n".format(hook.hook_address))
            outfd.write("Hooking module: {0}\n\n".format(hook.HookModule))

            for n, info in enumerate(hook.disassembled_hops):
                (address, data) = info
                s = ["{0:#x} {1:<16} {2}".format(o, h, i)
                        for o, i, h in
                        malfind.Disassemble(data, int(address), bits = "32bit" if hook.decode_bits == distorm3.Decode32Bits else "64bit")
                    ]
                outfd.write("Disassembly({0}):\n{1}".format(n, "\n".join(s)))
                outfd.write("\n\n")
Example #41
0
File: objects.py Project: 18z/abc
 def _get_ssdeep(self):
     try:
         return pydeep.hash_buf(self._file_data)
     except Exception as e:
         print e
         return None
Example #42
0
def get_fuzzy(f):

    fuzzy = pydeep.hash_buf(f.read())
    position = f.seek(0, 0)

    return fuzzy
Example #43
0
email: [email protected]
------------------------------

This takes a file and a name as input and will provide a list of tuples,
where each tuple is that name and the ssdeep hash of every 4096 byte chunk of the file.
These chunks correspond to a page of memory.  This script is intended for use with 
my volatility plugins malfinddeep and apihooksdeep.
'''

import argparse
import pydeep

def cliargs():
    '''Parse CLI args'''
    parser = argparse.ArgumentParser(description="hash_by_page.py -- return SSDeep hash of each 4096 byte chunk of a file in whitelist format for Volatility")
    parser.add_argument('-n', '--name', required=True, action='store', dest='name', help='Name associated with file')
    parser.add_argument('-f', '--file', required=True, action='store', dest='filename', help='File to hash')
    args = parser.parse_args()
    return args

args = cliargs()
fh = open(args.filename,"r")

buff = fh.read(0x1000)
while len(buff) > 0:
    hash_buff = pydeep.hash_buf(buff)
    if hash_buff != "3::":
        print "('" + args.name + "', '" + hash_buff + "'),"
    buff = fh.read(0x1000)

Example #44
0
import pydeep
file1 = 'calc.exe'
file2 = 'notepad.exe'
file1hash = '1536:JEl14rQcWAkN7GAlqbkfAGQGV8aMbrNyrf1w+noPvLV6eBsCXKc:JYmZWXyaiedMbrN6pnoXL1BsC'
file2hash = '1536:0awOnbNQKLjWDyy1o5RefYMJUEbooPRrKKRl1P3:0YNQKPWDyDRefVJltZrpRl1P3'
data1 = open(file1).read()
data2 = open(file2).read()
assert len(data1) == 114688, "File length error"
assert len(data2) == 69120, "File lendth error"
hash01 = pydeep.hash_buf(data1)
hash02 = pydeep.hash_buf(data2)
assert hash01 == file1hash, "Error hashing file1"
assert hash02 == file2hash, "Error hashing file2"
hash1 = pydeep.hash_file(file1)
hash2 = pydeep.hash_file(file2)
assert hash1 == file1hash, "Error hashing file1"
assert hash2 == file2hash, "Error hashing file2"
assert pydeep.compare(hash1, hash2) == 0, "Error fuzzy compare value"
print 'Stuff looks fine..'
Example #45
0
    def render_text(self, outfd, data):
        for process, module, hook, addr_space in data:

            if not self._config.NO_WHITELIST:

                if process:
                    process_name = str(process.ImageFileName)
                else:
                    process_name = ''

                if self.whitelist(hook.hook_mode | hook.hook_type,
                                  process_name, hook.VictimModule,
                                  hook.HookModule, hook.Function):
                    #debug.debug("Skipping whitelisted function: {0} {1} {2} {3}".format(
                    #    process_name, hook.VictimModule, hook.HookModule,
                    #    hook.Function))
                    continue

                if self._config.SSDEEP and has_pydeep:
                    skip = False
                    # read from the start of the page containing the hook, then hash it
                    page_address = hook.hook_address & 0xFFFFF000
                    hook_buf = addr_space.zread(page_address, 0x1000)
                    hook_hash = pydeep.hash_buf(hook_buf)
                    # loop through all the whitelist hashes and compare
                    for (whitelist_name, whitelist_hash) in whitelist_ssdeep:
                        alike = pydeep.compare(hook_hash, whitelist_hash)
                        # the comparison is greater than the threshold so display an informational message
                        # then skip the rest of the output in normal malfind
                        if alike > self._config.THRESHOLD:
                            if process:
                                outfd.write('Process: {0} ({1})\n'.format(
                                    process.UniqueProcessId,
                                    process.ImageFileName))
                            outfd.write(
                                "Hook at 0x{0:x} in page 0x{1:x} is {2}% similar to whitelist hook {3}\n"
                                .format(hook.hook_address, page_address, alike,
                                        whitelist_name))
                            #outfd.write("  hook: {0}\n".format(hook_hash))
                            #outfd.write("  whitelist: {0}\n".format(whitelist_hash))
                            outfd.write("\n")
                            skip = True
                            continue
                    if skip:
                        continue

            outfd.write("*" * 72 + "\n")
            outfd.write("Hook mode: {0}\n".format(hook.Mode))
            outfd.write("Hook type: {0}\n".format(hook.Type))

            if process:
                outfd.write('Process: {0} ({1})\n'.format(
                    process.UniqueProcessId, process.ImageFileName))

            outfd.write("Victim module: {0} ({1:#x} - {2:#x})\n".format(
                str(module.BaseDllName or '')
                or ntpath.basename(str(module.FullDllName or '')),
                module.DllBase, module.DllBase + module.SizeOfImage))

            outfd.write("Function: {0}\n".format(hook.Detail))
            outfd.write("Hook address: {0:#x}\n".format(hook.hook_address))
            outfd.write("Hooking module: {0}\n\n".format(hook.HookModule))

            for n, info in enumerate(hook.disassembled_hops):
                (address, data) = info
                s = [
                    "{0:#x} {1:<16} {2}".format(o, h, i) for o, i, h in
                    malfind.Disassemble(data,
                                        int(address),
                                        bits="32bit" if hook.decode_bits ==
                                        distorm3.Decode32Bits else "64bit")
                ]
                outfd.write("Disassembly({0}):\n{1}".format(n, "\n".join(s)))
                outfd.write("\n\n")
Example #46
0
import pydeep
file1 = 'calc.exe'
file2 = 'notepad.exe'
file3 = 'bc'
file1hash = '1536:JEl14rQcWAkN7GAlqbkfAGQGV8aMbrNyrf1w+noPvLV6eBsCXKc:JYmZWXyaiedMbrN6pnoXL1BsC'
file2hash = '1536:0awOnbNQKLjWDyy1o5RefYMJUEbooPRrKKRl1P3:0YNQKPWDyDRefVJltZrpRl1P3'
file3hash = '1536:MsjYdR3Bul8hcURWhEcg4/btZzDcQflbCUPEBEh8wkcGDioxMYeo7:TYf8l8htRWA4ztZsGlWUPEBEh8wmxMYe'
data1 = open(file1).read()
data2 = open(file2).read()
data3 = open(file3).read()
assert len(data1) == 114688, "File length error"
assert len(data2) ==  69120, "File length error"
assert len(data3) ==  77168, "File length error"
hash01 = pydeep.hash_buf(data1)
hash02 = pydeep.hash_buf(data2)
hash03 = pydeep.hash_buf(data3)
assert hash01 == file1hash, "Error hashing file1"
assert hash02 == file2hash, "Error hashing file2"
assert hash03 == file3hash, "Error hashing file2"
hash1 = pydeep.hash_file(file1)
hash2 = pydeep.hash_file(file2)
hash3 = pydeep.hash_file(file3)
assert hash1 == file1hash, "Error hashing file1"
assert hash2 == file2hash, "Error hashing file2"
assert hash3 == file3hash, "Error hashing file3"
assert pydeep.compare(hash1,hash2) == 0, "Error fuzzy compare value"
print 'Stuff looks fine..' 
Example #47
0
def hash_mem_block(block):
    return pydeep.hash_buf(block)