def find_malware():
def details(row):
tags = []
for tag in row.tag:
tags.append(tag.tag)
entry = {
"id" : row.id,
"file_name" : row.file_name,
"file_type" : row.file_type,
"file_size" : row.file_size,
"md5" : row.md5,
"sha1" : row.sha1,
"sha256" : row.sha256,
"sha512" : row.sha512,
"crc32" : row.crc32,
"ssdeep": row.ssdeep,
"created_at": row.created_at.__str__(),
"tags" : tags
}
return entry
md5 = request.forms.get("md5")
sha256 = request.forms.get("sha256")
ssdeep = request.forms.get("ssdeep")
tag = request.forms.get("tag")
date = request.forms.get("date")
if md5:
row = db.find_md5(md5)
if row:
return jsonize(details(row))
else:
raise HTTPError(404, "File not found")
elif sha256:
row = db.find_sha256(sha256)
if row:
return jsonize(details(row))
else:
raise HTTPError(404, "File not found")
else:
if ssdeep:
rows = db.find_ssdeep(ssdeep)
elif tag:
rows = db.find_tag(tag)
elif date:
rows = db.find_date(date)
else:
return HTTPError(400, "Invalid search term")
if not rows:
return HTTPError(404, "File not found")
results = []
for row in rows:
entry = details(row)
results.append(entry)
return jsonize(results)
def del_malware(filehash):
'''
Delete sample object by hash
@md5 : md5 hash
@sha1 : sha1 hash
@sha256 : sha256 hash
@sha512 : sha512 hash
@filehash : any of the above hash methods, the method will try to
identify the hash algorithm.
returns : JSON status message
'''
if not filehash:
md5 = request.forms.get('md5')
sha1 = request.forms.get('sha1')
sha256 = request.forms.get('sha256')
sha512 = request.forms.get('sha512')
if md5 != None:
filehash = md5
if sha1 != None:
filehash = sha1
if sha256 != None:
filehash = sha256
if sha512 != None:
filehash = sha512
if re.findall(r"([a-fA-F\d]{32})", filehash):
# MD5
sampleData = del_file(db, md5=filehash)
elif re.findall(r"([a-fA-F\d]{40})", filehash):
# SHA1
sampleData = del_file(db, sha1=filehash)
elif re.findall(r"([a-fA-F\d]{64})", filehash):
# SHA256
sampleData = del_file(db, sha256=filehash)
elif re.findall(r"([a-fA-F\d]{128})", filehash):
# SHA512
sampleData = del_file(db, sha512=filehash)
else:
# Hash not recognized
response.content_type = 'application/json'
return (jsonize({'error': 'unknown_hash'}), 400)
if sampleData:
response.content_type = 'application/json'
return (jsonize({'ok': 'sample_deleted'}), 200)
else:
response.content_type = 'application/json'
return (jsonize({'error': 'sample_not_found'}), 404)
def find_malware():
def details(row):
tags = []
for tag in row.tag:
tags.append(tag.tag)
entry = {
"id": row.id,
"file_name": row.file_name,
"file_type": row.file_type,
"file_size": row.file_size,
"md5": row.md5,
"sha1": row.sha1,
"sha256": row.sha256,
"sha512": row.sha512,
"crc32": row.crc32,
"ssdeep": row.ssdeep,
"created_at": row.created_at.__str__(),
"tags": tags
}
return entry
md5 = request.forms.get("md5")
sha256 = request.forms.get("sha256")
ssdeep = request.forms.get("ssdeep")
tag = request.forms.get("tag")
date = request.forms.get("date")
if md5:
row = db.find_md5(md5)
if row:
return jsonize(details(row))
else:
raise HTTPError(404, "File not found")
elif sha256:
row = db.find_sha256(sha256)
if row:
return jsonize(details(row))
else:
raise HTTPError(404, "File not found")
else:
if ssdeep:
rows = db.find_ssdeep(ssdeep)
elif tag:
rows = db.find_tag(tag)
elif date:
rows = db.find_date(date)
else:
return HTTPError(400, "Invalid search term")
if not rows:
return HTTPError(404, "File not found")
results = []
for row in rows:
entry = details(row)
results.append(entry)
return jsonize(results)
def list_tags():
rows = db.list_tags()
results = []
for row in rows:
results.append(row.tag)
return jsonize(results)
def add_malware():
tags = request.forms.get("tags")
data = request.files.file
info = File(file_path=store_sample(data.file.read()))
db.add(obj=info, file_name=data.filename, tags=tags)
return jsonize({"message" : "added"})
def add_malware():
tags = request.forms.get("tags")
data = request.files.file
info = File(file_path=store_sample(data.file.read()))
db.add(obj=info, file_name=data.filename, tags=tags)
return jsonize({"message": "added"})
def list_tags():
'''
List all tags
returns : list of tags in JSON format
'''
response.content_type = 'application/json'
return jsonize(db.fs.files.distinct('tags'))
def list_tags():
rows = db.list_tags()
results = []
for row in rows:
results.append(row.tag)
return jsonize(results)
def list_tags(request):
if request.method == 'GET':
rows = tag.objects.all()
results = []
for row in rows:
results.append(row.tag)
return HttpResponse(jsonize(results))
else:
return HttpResponse('METHOD not supported for URL')
def list_tags(request):
if request.method == 'GET':
rows = tag.objects.all()
results = []
for row in rows:
results.append(row.tag)
return HttpResponse(jsonize(results))
else:
return HttpResponse('METHOD not supported for URL')
def get_malware(sha256):
path = get_sample_path(sha256)
if not path:
raise HTTPError(404, jsonize({"error": "file_not_found"}))
response.content_type = "application/octet-stream; charset=UTF-8"
code, data = get_sample_content(sha256)
response.content_length = len(data)
if code == 200:
return data
else:
raise HTTPError(code, data)
def add_malware(request):
if request.method == 'POST':
form = AddMalwareForm(request.POST, request.FILES)
if form.is_valid():
file_path = store_sample(request.FILES['file'].read())
obj = File(file_path=file_path)
tags = request.POST.get('tags')
orig_url = request.POST.get('url')
file_name = request.FILES['file'].name
if isinstance(obj, File):
malware_entry = malware(md5=obj.get_md5(),
crc32=obj.get_crc32(),
sha1=obj.get_sha1(),
sha256=obj.get_sha256(),
sha512=obj.get_sha512(),
file_size=obj.get_size(),
file_type=obj.get_type(),
ssdeep=obj.get_ssdeep(),
orig_url=orig_url,
file_name=file_name)
malware_entry.save()
if store_encoded:
encode_sample(file_path)
delete_file(file_path)
if tags:
tags = tags.strip()
if "," in tags:
tags = tags.split(",")
else:
tags = tags.split(" ")
for t in tags:
t = t.strip().lower()
if t == "":
continue
if tag.objects.filter(tag=t).exists():
malware_entry.tags.add(tag.objects.get(tag=t))
continue
malware_entry.tags.add(tag.objects.create(tag=t))
return HttpResponse(
jsonize({"message": file_name + " added to repository"}))
else:
return HttpResponse('METHOD not supported for URL')
def add_malware(request):
if request.method == 'POST':
form = AddMalwareForm(request.POST, request.FILES)
if form.is_valid():
file_path=store_sample(request.FILES['file'].read())
obj = File(file_path=file_path)
tags = request.POST.get('tags')
orig_url = request.POST.get('url')
file_name = request.FILES['file'].name
if isinstance(obj, File):
malware_entry = malware(md5=obj.get_md5(),
crc32=obj.get_crc32(),
sha1=obj.get_sha1(),
sha256=obj.get_sha256(),
sha512=obj.get_sha512(),
file_size=obj.get_size(),
file_type=obj.get_type(),
ssdeep=obj.get_ssdeep(),
orig_url=orig_url,
file_name=file_name)
malware_entry.save()
if store_encoded:
encode_sample(file_path)
delete_file(file_path)
if tags:
tags = tags.strip()
if "," in tags:
tags = tags.split(",")
else:
tags = tags.split(" ")
for t in tags:
t = t.strip().lower()
if t == "":
continue
if tag.objects.filter(tag=t).exists():
malware_entry.tags.add(tag.objects.get(tag=t))
continue
malware_entry.tags.add(tag.objects.create(tag=t))
return HttpResponse(jsonize({"message" : file_name + " added to repository"}))
else:
return HttpResponse('METHOD not supported for URL')
def search_malware():
'''
search the database using user-supplied key
returns : JSON data or JSON status message
'''
dblist = list()
for (key, val) in request.forms.iteritems():
logging.debug('Looking for %s : %s' % (key, val))
dblist.extend(db.fs.files.find({key: {'$regex': val}}))
logging.debug('Found %s items in total' % len(dblist))
for index in xrange(len(dblist)):
del dblist[index]['_id']
dblist[index]['uploadDate'] = str(dblist[index]['uploadDate'])
return jsonize(dblist)
def add_tags(key=None, tags=None):
'''
@key : dict to identify what sample to update
@tags : list of tags to add to the sample
returns : JSON status message
'''
if not key:
key = request.forms.get('key')
if not tags:
tags = request.forms.get('tags').split(',')
if key and tags:
logging.info('Adding tags: %s' % ','.join(tags))
db.fs.files.update(key, {'$addToSet': {'tags': {'$each': tags}}})
return jsonize({'message': 'added'})
else:
logging.info('No key (%s) or no tags (%s) to add' % (key, tags))
def add_malware():
tags = request.forms.get("tags")
data = request.files.file
info = File(path=store_sample(data.file.read()))
if tags:
tags = tags.strip()
if "," in tags:
tags = tags.split(",")
else:
tags = tags.split(" ")
try:
tags.append("User:"******"message": "added"})
def test(request):
if request.method == 'GET':
return HttpResponse(jsonize({"message": "test"}))
else:
return HttpResponse('METHOD not supported for URL')
def find_malware(request):
def details(row):
tags = []
for tag in row.tags.all():
tags.append(tag.tag)
entry = {
"id": row.id,
"file_name": row.file_name,
"orig_url": row.orig_url,
"file_type": row.file_type,
"file_size": row.file_size,
"md5": row.md5,
"sha1": row.sha1,
"sha256": row.sha256,
"sha512": row.sha512,
"crc32": row.crc32,
"ssdeep": row.ssdeep,
"created_at": row.created_at.__str__(),
"modified_at": row.modified_at.__str__(),
"tags": tags
}
return entry
if request.method == 'POST':
form = FindMalwareForm(request.POST, request.FILES)
if form.is_valid():
md5 = request.POST.get('md5')
sha1 = request.POST.get('sha1')
sha256 = request.POST.get('sha256')
ssdeep = request.POST.get('ssdeep')
qtag = request.POST.get('tag')
date = request.POST.get('date')
if md5:
row = malware.objects.get(md5=md5)
if row:
return HttpResponse(jsonize(details(row)))
else:
return HttpResponseNotFound('<h1>Page not found</h1>')
elif sha1:
row = malware.objects.get(sha1=sha1)
if row:
return HttpResponse(jsonize(details(row)))
else:
return HttpResponseNotFound('<h1>Page not found</h1>')
elif sha256:
row = malware.objects.get(sha256=sha256)
if row:
return HttpResponse(jsonize(details(row)))
else:
return HttpResponseNotFound('<h1>Page not found</h1>')
else:
if ssdeep:
rows = malware.objects.filter(ssdeep=ssdeep)
elif qtag:
rows = tag.objects.get(tag=qtag).malware_set.all()
elif date:
rows = malware.objects.filter(date=date)
else:
return HttpResponseBadRequest("Invalid search term")
if not rows:
return HttpResponseNotFound('<h1>Page not found</h1>')
results = []
for row in rows:
entry = details(row)
results.append(entry)
return HttpResponse(jsonize(results))
else:
return HttpResponse('METHOD not supported for URL')
def find_malware(request):
def details(row):
tags = []
for tag in row.tags.all():
tags.append(tag.tag)
entry = {
"id" : row.id,
"file_name" : row.file_name,
"orig_url" : row.orig_url,
"file_type" : row.file_type,
"file_size" : row.file_size,
"md5" : row.md5,
"sha1" : row.sha1,
"sha256" : row.sha256,
"sha512" : row.sha512,
"crc32" : row.crc32,
"ssdeep": row.ssdeep,
"created_at": row.created_at.__str__(),
"modified_at": row.modified_at.__str__(),
"tags" : tags
}
return entry
if request.method == 'POST':
form = FindMalwareForm(request.POST, request.FILES)
if form.is_valid():
md5 = request.POST.get('md5')
sha1 = request.POST.get('sha1')
sha256 = request.POST.get('sha256')
ssdeep = request.POST.get('ssdeep')
qtag = request.POST.get('tag')
date = request.POST.get('date')
if md5:
row = malware.objects.get(md5=md5)
if row:
return HttpResponse(jsonize(details(row)))
else:
return HttpResponseNotFound('<h1>Page not found</h1>')
elif sha1:
row = malware.objects.get(sha1=sha1)
if row:
return HttpResponse(jsonize(details(row)))
else:
return HttpResponseNotFound('<h1>Page not found</h1>')
elif sha256:
row = malware.objects.get(sha256=sha256)
if row:
return HttpResponse(jsonize(details(row)))
else:
return HttpResponseNotFound('<h1>Page not found</h1>')
else:
if ssdeep:
rows = malware.objects.filter(ssdeep=ssdeep)
elif qtag:
rows = tag.objects.get(tag=qtag).malware_set.all()
elif date:
rows = malware.objects.filter(date=date)
else:
return HttpResponseBadRequest("Invalid search term")
if not rows:
return HttpResponseNotFound('<h1>Page not found</h1>')
results = []
for row in rows:
entry = details(row)
results.append(entry)
return HttpResponse(jsonize(results))
else:
return HttpResponse('METHOD not supported for URL')
def test():
return jsonize({"message" : "test"})
def test(request):
if request.method == 'GET':
return HttpResponse(jsonize({"message" : "test"}))
else:
return HttpResponse('METHOD not supported for URL')
def add_malware():
'''
Adds a sample to the repository. Performs hashing and filemagic
analysis of the uploaded sample.
@tags : comma seperated tags list
@file : binary sample stream
returns : JSON status message
'''
try:
with timeout(Config().api.timeout * 60, exception=RuntimeError):
tags = request.forms.get('tags').split(',')
data = request.files.file
data.file.seek(0)
filename = data.filename
sampleData = data.file.read()
logging.debug('[%s] Generating hashes' % sampleEntry)
md5 = hashlib.md5(sampleData).hexdigest()
sha1 = hashlib.sha1(sampleData).hexdigest()
sha256 = hashlib.sha256(sampleData).hexdigest()
sha512 = hashlib.sha512(sampleData).hexdigest()
filetype = get_type(sampleData)
key = {'md5': md5}
logging.debug('Quering database for already existing file (hash=%s)'
% md5)
existing = db.fs.files.find_one({'md5': md5})
upload_sample = True
if existing:
logging.info('Sample already exists')
logging.info('Verifying contents')
if not md5 == existing['md5']:
logging.warning('Checksum not matching')
upload_sample = True
else:
logging.info('Checksum matching')
upload_sample = False
else:
upload_sample = True
if upload_sample:
logging.debug('Uploading sample')
new = fs.new_file(filename=filename, sha1=sha1,
sha256=sha256, sha512=sha512,
filetype=filetype)
for chunk in get_chunks(sampleData):
logging.debug('writing chunk')
new.write(chunk)
new.close()
logging.info('Uploaded sample')
add_tags(key=key, tags=tags)
logging.debug('Reclaiming memory')
del sampleData
response.content_type = 'application/json'
return jsonize({'message': 'added'})
except RuntimeError:
response.content_type = 'application/json'
return (jsonize({'error': 'timeout'}), 504)
def test():
return jsonize({"message": "test"})
def find_malware(
md5=None,
sha1=None,
sha256=None,
sha512=None,
ssdeep=None,
tag=None,
created=None,
filehash='',
):
'''
search the database using hash, tag or date uploaded
returns : JSON metadata or JSON status message
'''
if not filehash:
md5 = request.forms.get('md5')
sha1 = request.forms.get('sha1')
sha256 = request.forms.get('sha256')
sha512 = request.forms.get('sha512')
if md5 != None:
filehash = md5
if sha1 != None:
filehash = sha1
if sha256 != None:
filehash = sha256
if sha512 != None:
filehash = sha512
if filehash == None:
filehash = ''
ssdeep = request.forms.get('ssdeep')
tag = request.forms.get('tag')
created = request.forms.get('created')
filehash = request.forms.get('filehash')
if re.search(r"([a-fA-F\d]{128})", filehash):
# SHA512
logging.debug('Looking for SHA512 hash %s' % filehash)
metadata = db.fs.files.find_one({'sha512': filehash})
elif re.search(r"([a-fA-F\d]{64})", filehash):
# SHA256
logging.debug('Looking for SHA256 hash %s' % filehash)
metadata = db.fs.files.find_one({'sha256': filehash})
elif re.search(r"([a-fA-F\d]{40})", filehash):
# SHA1
logging.debug('Looking for SHA1 hash %s' % filehash)
metadata = db.fs.files.find_one({'sha1': filehash})
elif re.search(r"([a-fA-F\d]{32})", filehash):
# MD5
logging.debug('Looking for MD5 hash %s' % filehash)
metadata = db.fs.files.find_one({'md5': filehash})
else:
if ssdeep:
metadata = db.fs.files.find({'ssdeep': {'$regex': '.*'
+ ssdeep + '.*'}})
elif tag:
metadata = db.fs.files.find({'tags': tag})
elif created:
start = datetime.datetime.strptime(created, '%Y-%m-%d')
end = datetime.datetime.strptime(created, '%Y-%m-%d') \
+ datetime.timedelta(days=1)
metadata = db.fs.files.find({'uploadDate': {'$gte': start,
'$lt': end}})
else:
response.content_type = 'application/json'
return jsonize({'error': 'invalid_search_term'})
if not metadata:
response.content_type = 'application/json'
return (jsonize({'error': 'file_not_found'}), 404)
response.content_type = 'application/json'
del metadata['_id']
metadata['uploadDate'] = str(metadata['uploadDate'])
return jsonize(metadata)