Пример #1
0
def find_malware():
    def details(row):
        tags = []
        for tag in row.tag:
            tags.append(tag.tag)

        entry = {
            "id" : row.id,
            "file_name" : row.file_name,
            "file_type" : row.file_type,
            "file_size" : row.file_size,
            "md5" : row.md5,
            "sha1" : row.sha1,
            "sha256" : row.sha256,
            "sha512" : row.sha512,
            "crc32" : row.crc32,
            "ssdeep": row.ssdeep,
            "created_at": row.created_at.__str__(),
            "tags" : tags
        }

        return entry

    md5 = request.forms.get("md5")
    sha256 = request.forms.get("sha256")
    ssdeep = request.forms.get("ssdeep")
    tag = request.forms.get("tag")
    date = request.forms.get("date")

    if md5:
        row = db.find_md5(md5)
        if row:
            return jsonize(details(row))
        else:
            raise HTTPError(404, "File not found")
    elif sha256:
        row = db.find_sha256(sha256)
        if row:
            return jsonize(details(row))
        else:
            raise HTTPError(404, "File not found")
    else:
        if ssdeep:
            rows = db.find_ssdeep(ssdeep)
        elif tag:
            rows = db.find_tag(tag)
        elif date:
            rows = db.find_date(date)
        else:
            return HTTPError(400, "Invalid search term")

        if not rows:
            return HTTPError(404, "File not found")

        results = []
        for row in rows:
            entry = details(row)
            results.append(entry)

        return jsonize(results)
Пример #2
0
def del_malware(filehash):
    '''
    Delete sample object by hash
    @md5 : md5 hash
    @sha1 : sha1 hash
    @sha256 : sha256 hash
    @sha512 : sha512 hash
    @filehash : any of the above hash methods, the method will try to
                identify the hash algorithm.
    returns : JSON status message 
    '''

    if not filehash:
        md5 = request.forms.get('md5')
        sha1 = request.forms.get('sha1')
        sha256 = request.forms.get('sha256')
        sha512 = request.forms.get('sha512')

        if md5 != None:
            filehash = md5
        if sha1 != None:
            filehash = sha1
        if sha256 != None:
            filehash = sha256
        if sha512 != None:
            filehash = sha512

    if re.findall(r"([a-fA-F\d]{32})", filehash):

        # MD5

        sampleData = del_file(db, md5=filehash)
    elif re.findall(r"([a-fA-F\d]{40})", filehash):

        # SHA1

        sampleData = del_file(db, sha1=filehash)
    elif re.findall(r"([a-fA-F\d]{64})", filehash):

        # SHA256

        sampleData = del_file(db, sha256=filehash)
    elif re.findall(r"([a-fA-F\d]{128})", filehash):

        # SHA512

        sampleData = del_file(db, sha512=filehash)
    else:

        # Hash not recognized

        response.content_type = 'application/json'
        return (jsonize({'error': 'unknown_hash'}), 400)

    if sampleData:
        response.content_type = 'application/json'
        return (jsonize({'ok': 'sample_deleted'}), 200)
    else:
        response.content_type = 'application/json'
        return (jsonize({'error': 'sample_not_found'}), 404)
Пример #3
0
def find_malware():
    def details(row):
        tags = []
        for tag in row.tag:
            tags.append(tag.tag)

        entry = {
            "id": row.id,
            "file_name": row.file_name,
            "file_type": row.file_type,
            "file_size": row.file_size,
            "md5": row.md5,
            "sha1": row.sha1,
            "sha256": row.sha256,
            "sha512": row.sha512,
            "crc32": row.crc32,
            "ssdeep": row.ssdeep,
            "created_at": row.created_at.__str__(),
            "tags": tags
        }

        return entry

    md5 = request.forms.get("md5")
    sha256 = request.forms.get("sha256")
    ssdeep = request.forms.get("ssdeep")
    tag = request.forms.get("tag")
    date = request.forms.get("date")

    if md5:
        row = db.find_md5(md5)
        if row:
            return jsonize(details(row))
        else:
            raise HTTPError(404, "File not found")
    elif sha256:
        row = db.find_sha256(sha256)
        if row:
            return jsonize(details(row))
        else:
            raise HTTPError(404, "File not found")
    else:
        if ssdeep:
            rows = db.find_ssdeep(ssdeep)
        elif tag:
            rows = db.find_tag(tag)
        elif date:
            rows = db.find_date(date)
        else:
            return HTTPError(400, "Invalid search term")

        if not rows:
            return HTTPError(404, "File not found")

        results = []
        for row in rows:
            entry = details(row)
            results.append(entry)

        return jsonize(results)
Пример #4
0
def list_tags():
    rows = db.list_tags()

    results = []
    for row in rows:
        results.append(row.tag)

    return jsonize(results)
Пример #5
0
def add_malware():
    tags = request.forms.get("tags")
    data = request.files.file
    info = File(file_path=store_sample(data.file.read()))

    db.add(obj=info, file_name=data.filename, tags=tags)

    return jsonize({"message" : "added"})
Пример #6
0
def add_malware():
    tags = request.forms.get("tags")
    data = request.files.file
    info = File(file_path=store_sample(data.file.read()))

    db.add(obj=info, file_name=data.filename, tags=tags)

    return jsonize({"message": "added"})
Пример #7
0
def list_tags():
    '''
    List all tags
    returns : list of tags in JSON format
    '''

    response.content_type = 'application/json'
    return jsonize(db.fs.files.distinct('tags'))
Пример #8
0
def list_tags():
    rows = db.list_tags()

    results = []
    for row in rows:
        results.append(row.tag)

    return jsonize(results)
Пример #9
0
def list_tags(request):
    if request.method == 'GET':
        rows = tag.objects.all()

        results = []
        for row in rows:
            results.append(row.tag)

        return HttpResponse(jsonize(results))
    else:
        return HttpResponse('METHOD not supported for URL')
Пример #10
0
def list_tags(request):
    if request.method == 'GET':
        rows = tag.objects.all()

        results = []
        for row in rows:
            results.append(row.tag)

        return HttpResponse(jsonize(results))
    else:
        return HttpResponse('METHOD not supported for URL')
Пример #11
0
def get_malware(sha256):
    path = get_sample_path(sha256)
    if not path:
        raise HTTPError(404, jsonize({"error": "file_not_found"}))

    response.content_type = "application/octet-stream; charset=UTF-8"
    code, data = get_sample_content(sha256)
    response.content_length = len(data)
    if code == 200:
        return data
    else:
        raise HTTPError(code, data)
Пример #12
0
def add_malware(request):
    if request.method == 'POST':
        form = AddMalwareForm(request.POST, request.FILES)
        if form.is_valid():
            file_path = store_sample(request.FILES['file'].read())
            obj = File(file_path=file_path)
            tags = request.POST.get('tags')
            orig_url = request.POST.get('url')
            file_name = request.FILES['file'].name

        if isinstance(obj, File):
            malware_entry = malware(md5=obj.get_md5(),
                                    crc32=obj.get_crc32(),
                                    sha1=obj.get_sha1(),
                                    sha256=obj.get_sha256(),
                                    sha512=obj.get_sha512(),
                                    file_size=obj.get_size(),
                                    file_type=obj.get_type(),
                                    ssdeep=obj.get_ssdeep(),
                                    orig_url=orig_url,
                                    file_name=file_name)
        malware_entry.save()

        if store_encoded:
            encode_sample(file_path)
            delete_file(file_path)

        if tags:
            tags = tags.strip()
            if "," in tags:
                tags = tags.split(",")
            else:
                tags = tags.split(" ")

            for t in tags:
                t = t.strip().lower()
                if t == "":
                    continue

                if tag.objects.filter(tag=t).exists():
                    malware_entry.tags.add(tag.objects.get(tag=t))
                    continue

                malware_entry.tags.add(tag.objects.create(tag=t))

        return HttpResponse(
            jsonize({"message": file_name + " added to repository"}))

    else:
        return HttpResponse('METHOD not supported for URL')
Пример #13
0
def add_malware(request):
    if request.method == 'POST':
        form = AddMalwareForm(request.POST, request.FILES)
        if form.is_valid():
            file_path=store_sample(request.FILES['file'].read())
            obj = File(file_path=file_path)
            tags = request.POST.get('tags')
            orig_url = request.POST.get('url')
            file_name = request.FILES['file'].name

        if isinstance(obj, File):
            malware_entry = malware(md5=obj.get_md5(),
                                    crc32=obj.get_crc32(),
                                    sha1=obj.get_sha1(),
                                    sha256=obj.get_sha256(),
                                    sha512=obj.get_sha512(),
                                    file_size=obj.get_size(),
                                    file_type=obj.get_type(),
                                    ssdeep=obj.get_ssdeep(),
                                    orig_url=orig_url,
                                    file_name=file_name)
        malware_entry.save()
        
        if store_encoded:
            encode_sample(file_path)
            delete_file(file_path)

        if tags:
            tags = tags.strip()
            if "," in tags:
                tags = tags.split(",")
            else:
                tags = tags.split(" ")

            for t in tags:
                t = t.strip().lower()
                if t == "":
                    continue
            
                if tag.objects.filter(tag=t).exists():
                    malware_entry.tags.add(tag.objects.get(tag=t))
                    continue

                malware_entry.tags.add(tag.objects.create(tag=t))

        return HttpResponse(jsonize({"message" : file_name + " added to repository"}))

    else:
        return HttpResponse('METHOD not supported for URL')
Пример #14
0
def search_malware():
    '''
    search the database using user-supplied key
    returns : JSON data or JSON status message
    '''

    dblist = list()
    for (key, val) in request.forms.iteritems():
        logging.debug('Looking for %s : %s' % (key, val))
        dblist.extend(db.fs.files.find({key: {'$regex': val}}))

    logging.debug('Found %s items in total' % len(dblist))

    for index in xrange(len(dblist)):
        del dblist[index]['_id']
        dblist[index]['uploadDate'] = str(dblist[index]['uploadDate'])

    return jsonize(dblist)
Пример #15
0
def add_tags(key=None, tags=None):
    '''
    @key  : dict to identify what sample to update
    @tags : list of tags to add to the sample

    returns : JSON status message 
    '''

    if not key:
        key  = request.forms.get('key')
    if not tags:
        tags = request.forms.get('tags').split(',')

    if key and tags:
        logging.info('Adding tags: %s' % ','.join(tags))
        db.fs.files.update(key, {'$addToSet': {'tags': {'$each': tags}}})

        return jsonize({'message': 'added'})
    else:
        logging.info('No key (%s) or no tags (%s) to add' % (key, tags))
Пример #16
0
def add_malware():
    tags = request.forms.get("tags")
    data = request.files.file
    info = File(path=store_sample(data.file.read()))

    if tags:
        tags = tags.strip()
        if "," in tags:
            tags = tags.split(",")
        else:
            tags = tags.split(" ")
        try:
            tags.append("User:"******"message": "added"})
Пример #17
0
def test(request):
    if request.method == 'GET':
        return HttpResponse(jsonize({"message": "test"}))
    else:
        return HttpResponse('METHOD not supported for URL')
Пример #18
0
def find_malware(request):
    def details(row):
        tags = []
        for tag in row.tags.all():
            tags.append(tag.tag)

        entry = {
            "id": row.id,
            "file_name": row.file_name,
            "orig_url": row.orig_url,
            "file_type": row.file_type,
            "file_size": row.file_size,
            "md5": row.md5,
            "sha1": row.sha1,
            "sha256": row.sha256,
            "sha512": row.sha512,
            "crc32": row.crc32,
            "ssdeep": row.ssdeep,
            "created_at": row.created_at.__str__(),
            "modified_at": row.modified_at.__str__(),
            "tags": tags
        }

        return entry

    if request.method == 'POST':
        form = FindMalwareForm(request.POST, request.FILES)
        if form.is_valid():
            md5 = request.POST.get('md5')
            sha1 = request.POST.get('sha1')
            sha256 = request.POST.get('sha256')
            ssdeep = request.POST.get('ssdeep')
            qtag = request.POST.get('tag')
            date = request.POST.get('date')

        if md5:
            row = malware.objects.get(md5=md5)
            if row:
                return HttpResponse(jsonize(details(row)))
            else:
                return HttpResponseNotFound('<h1>Page not found</h1>')
        elif sha1:
            row = malware.objects.get(sha1=sha1)
            if row:
                return HttpResponse(jsonize(details(row)))
            else:
                return HttpResponseNotFound('<h1>Page not found</h1>')
        elif sha256:
            row = malware.objects.get(sha256=sha256)
            if row:
                return HttpResponse(jsonize(details(row)))
            else:
                return HttpResponseNotFound('<h1>Page not found</h1>')
        else:
            if ssdeep:
                rows = malware.objects.filter(ssdeep=ssdeep)
            elif qtag:
                rows = tag.objects.get(tag=qtag).malware_set.all()
            elif date:
                rows = malware.objects.filter(date=date)
            else:
                return HttpResponseBadRequest("Invalid search term")

            if not rows:
                return HttpResponseNotFound('<h1>Page not found</h1>')

            results = []
            for row in rows:
                entry = details(row)
                results.append(entry)

            return HttpResponse(jsonize(results))
    else:
        return HttpResponse('METHOD not supported for URL')
Пример #19
0
def find_malware(request):
    def details(row):
        tags = []
        for tag in row.tags.all():
            tags.append(tag.tag)

        entry = {
            "id" : row.id,
            "file_name" : row.file_name,
            "orig_url" : row.orig_url,
            "file_type" : row.file_type,
            "file_size" : row.file_size,
            "md5" : row.md5,
            "sha1" : row.sha1,
            "sha256" : row.sha256,
            "sha512" : row.sha512,
            "crc32" : row.crc32,
            "ssdeep": row.ssdeep,
            "created_at": row.created_at.__str__(),
            "modified_at": row.modified_at.__str__(),
            "tags" : tags
        }

        return entry

    if request.method == 'POST':
        form = FindMalwareForm(request.POST, request.FILES)
        if form.is_valid():
            md5 = request.POST.get('md5')
            sha1 = request.POST.get('sha1')
            sha256 = request.POST.get('sha256')
            ssdeep = request.POST.get('ssdeep')
            qtag = request.POST.get('tag')
            date = request.POST.get('date')

        if md5:
            row = malware.objects.get(md5=md5)
            if row:
                return HttpResponse(jsonize(details(row)))
            else:
                return HttpResponseNotFound('<h1>Page not found</h1>') 
        elif sha1:
            row = malware.objects.get(sha1=sha1)
            if row:
                return HttpResponse(jsonize(details(row)))
            else:
                return HttpResponseNotFound('<h1>Page not found</h1>') 
        elif sha256:
            row = malware.objects.get(sha256=sha256)
            if row:
                return HttpResponse(jsonize(details(row)))
            else:
                return HttpResponseNotFound('<h1>Page not found</h1>') 
        else:
            if ssdeep:
                rows = malware.objects.filter(ssdeep=ssdeep)
            elif qtag:
                rows = tag.objects.get(tag=qtag).malware_set.all()
            elif date:
                rows = malware.objects.filter(date=date)
            else:
                return HttpResponseBadRequest("Invalid search term")

            if not rows:
                return HttpResponseNotFound('<h1>Page not found</h1>') 

            results = []
            for row in rows:
                entry = details(row)
                results.append(entry)

            return HttpResponse(jsonize(results))
    else:
        return HttpResponse('METHOD not supported for URL')
Пример #20
0
def test():
    return jsonize({"message" : "test"})
Пример #21
0
def test(request):
    if request.method == 'GET':
        return HttpResponse(jsonize({"message" : "test"}))
    else:
        return HttpResponse('METHOD not supported for URL')
Пример #22
0
def add_malware():
    '''
    Adds a sample to the repository. Performs hashing and filemagic
    analysis of the uploaded sample.

    @tags : comma seperated tags list
    @file : binary sample stream

    returns : JSON status message 
    '''

    try:
        with timeout(Config().api.timeout * 60, exception=RuntimeError):
            tags = request.forms.get('tags').split(',')
            data = request.files.file
            data.file.seek(0)

            filename = data.filename
            sampleData = data.file.read()

            logging.debug('[%s] Generating hashes' % sampleEntry)
            md5 = hashlib.md5(sampleData).hexdigest()
            sha1 = hashlib.sha1(sampleData).hexdigest()
            sha256 = hashlib.sha256(sampleData).hexdigest()
            sha512 = hashlib.sha512(sampleData).hexdigest()
            filetype = get_type(sampleData)

            key = {'md5': md5}

            logging.debug('Quering database for already existing file (hash=%s)'
                           % md5)
            existing = db.fs.files.find_one({'md5': md5})

            upload_sample = True
            if existing:
                logging.info('Sample already exists')
                logging.info('Verifying contents')
                if not md5 == existing['md5']:
                    logging.warning('Checksum not matching')
                    upload_sample = True
                else:
                    logging.info('Checksum matching')
                    upload_sample = False
            else:
                upload_sample = True

            if upload_sample:
                logging.debug('Uploading sample')
                new = fs.new_file(filename=filename, sha1=sha1,
                                  sha256=sha256, sha512=sha512,
                                  filetype=filetype)
                for chunk in get_chunks(sampleData):
                    logging.debug('writing chunk')
                    new.write(chunk)
                new.close()
                logging.info('Uploaded sample')

            add_tags(key=key, tags=tags)

            logging.debug('Reclaiming memory')
            del sampleData

            response.content_type = 'application/json'
            return jsonize({'message': 'added'})
    except RuntimeError:
        response.content_type = 'application/json'
        return (jsonize({'error': 'timeout'}), 504)
Пример #23
0
def test():
    return jsonize({"message": "test"})
Пример #24
0
def find_malware(
    md5=None,
    sha1=None,
    sha256=None,
    sha512=None,
    ssdeep=None,
    tag=None,
    created=None,
    filehash='',
    ):
    '''
    search the database using hash, tag or date uploaded
    returns : JSON metadata or JSON status message
    '''

    if not filehash:
        md5 = request.forms.get('md5')
        sha1 = request.forms.get('sha1')
        sha256 = request.forms.get('sha256')
        sha512 = request.forms.get('sha512')
        if md5 != None:
            filehash = md5
        if sha1 != None:
            filehash = sha1
        if sha256 != None:
            filehash = sha256
        if sha512 != None:
            filehash = sha512
        if filehash == None:
            filehash = ''

    ssdeep = request.forms.get('ssdeep')
    tag = request.forms.get('tag')
    created = request.forms.get('created')
    filehash = request.forms.get('filehash')

    if re.search(r"([a-fA-F\d]{128})", filehash):

        # SHA512

        logging.debug('Looking for SHA512 hash %s' % filehash)
        metadata = db.fs.files.find_one({'sha512': filehash})
    elif re.search(r"([a-fA-F\d]{64})", filehash):

        # SHA256

        logging.debug('Looking for SHA256 hash %s' % filehash)
        metadata = db.fs.files.find_one({'sha256': filehash})
    elif re.search(r"([a-fA-F\d]{40})", filehash):

        # SHA1

        logging.debug('Looking for SHA1 hash %s' % filehash)
        metadata = db.fs.files.find_one({'sha1': filehash})
    elif re.search(r"([a-fA-F\d]{32})", filehash):

        # MD5

        logging.debug('Looking for MD5 hash %s' % filehash)
        metadata = db.fs.files.find_one({'md5': filehash})
    else:
        if ssdeep:
            metadata = db.fs.files.find({'ssdeep': {'$regex': '.*'
                    + ssdeep + '.*'}})
        elif tag:
            metadata = db.fs.files.find({'tags': tag})
        elif created:
            start = datetime.datetime.strptime(created, '%Y-%m-%d')
            end = datetime.datetime.strptime(created, '%Y-%m-%d') \
                + datetime.timedelta(days=1)
            metadata = db.fs.files.find({'uploadDate': {'$gte': start,
                    '$lt': end}})
        else:
            response.content_type = 'application/json'
            return jsonize({'error': 'invalid_search_term'})

    if not metadata:
        response.content_type = 'application/json'
        return (jsonize({'error': 'file_not_found'}), 404)

    response.content_type = 'application/json'
    del metadata['_id']
    metadata['uploadDate'] = str(metadata['uploadDate'])
    return jsonize(metadata)