def metadata_history(self, metadata):
results = {}
metadata_ids, engine_metadata = separate_metadata(metadata)
e_comment = ('Generated by Engine: {0.name}\n{0.description}\n\n'
'Developer: {0.developer.user_handle}')
for _id, metadata in Metadata.objects.in_bulk(metadata_ids).items():
data = metadata.dump(True)
result_key = make_id(0, metadata=_id)
results[result_key] = { 'creator' : data['creator'],
'history' : data['history']}
# Provide information for engine created metadata...
for flag, engine_id, _id in engine_metadata:
engine = self.get_engine(engine_id)
if not engine:
continue
data = {'creator' : engine.name,
'history' : [{'committed' : '',
'name' : 'N/A',
'prototype' : 'N/A',
'comment' : e_comment.format(engine)}]}
result_key = make_id(flag, engine=engine_id, metadata=_id)
results[result_key] = data
return results
def _init(self, **kwargs):
self._data = None
self._metadata = 0
if 'data' in kwargs:
self._data = kwargs['data']
self._data['id'] = make_id(1, self._metadata, self.id)
def get_metadata_list(self, metadata):
results = []
metadata_ids, engine_metadata = separate_metadata(metadata)
for _id, metadata in Metadata.objects.in_bulk(metadata_ids).items():
data = metadata.dump()
data['id'] = make_id(0, metadata=metadata.id)
results.append(data)
for flag, _id, metadata_id in engine_metadata:
engines = Engine.objects.get(pk=_id)
# TODO: Send metadata_id to engine for more info
if (not engines) or (len(engines) > 1):
continue
data = {'id' : make_id(flag, metadata_id, _id),
'engine' : engine.name,
'description' : engine.description}
results.append(data)
return results
def _get_metadata(self, db):
if not hasattr(self, '_metadata'):
self._metadata = list(db.get_function_metadata(self.id))
self._metadata.sort(key=lambda x: x.rank)
data = None
if len(self._metadata) > 0:
metadata = self._metadata.pop()
data = metadata.dump()
data['id'] = make_id(0, metadata=metadata.id)
return data
def created(self, user, page, max_metadata=20):
pages = 0
results = []
if (page < 1) or (not isinstance(user, User)):
return (results, pages)
p = Paginator(Metadata.objects.filter(user=user), max_metadata)
pages = p.num_pages
if page > pages:
return (results, pages)
for metadata in p.page(page):
temp = metadata.dump()
temp['id'] = make_id(0, metadata=metadata.id)
results.append(temp)
return (results, pages)
def metadata_add(request, md5_hash, crc32, user):
'''
Adds/Updates metadata for a given function to the db.
POST request, expects:
{
# Required - Sample
'md5' : /^[a-fA-F\d]{32}$/
'crc32' : <32 bit int>
'functions' : Dictionary of json-ed Dictionaries (max_length = 20)
{
'client_id' :
{
'opcodes' : String (base64 encoded)
'architecture' : String (max_length = 64)
'name' : String (max_length = 128)
'prototype' : String (max_length = 256)
'comment' : String (max_length = 512)
'apis' : List of Strings (max_string_length = 64)
# Optional
'id' : String
}
}
}
'''
# Check if required keys are provided
if not request.POST.get('functions'):
return render(request, 'rest/error_json.html',
{'msg' : 'All required data was not provided'})
try:
functions = json.loads(request.POST.get('functions'))
except ValueError:
return render(request, 'rest/error_json.html',
{'msg' : 'Invalid json object'})
if (dict != type(functions)) or (MAX_FUNCTIONS < len(functions)):
return render(request, 'rest/error_json.html', {'msg' : 'Invalid function list'})
# Iterate through functions to validate input, fail if something is wrong
required_keys = { 'opcodes', 'architecture', 'name',
'prototype', 'comment', 'apis'}
for client_key in functions:
f = functions[client_key]
if not required_keys.issubset(list(f.keys())):
return render(request, 'rest/error_json.html',
{'msg' : 'Invalid function list'})
try:
f['opcodes'] = codecs.decode(f['opcodes'].encode(), 'base64')
except binascii.Error as e:
return render(request, 'rest/error_json.html',
{'msg' : 'Unable to decode opcodes'})
# TODO: Normailize architecture
# Ensure string lengths are enforced
string_restrictions = { 'architecture' : 64, 'name' : 128,
'prototype' : 256, 'comment' : 512}
for key, max_length in string_restrictions.items():
if max_length < len(f[key]):
return render(request, 'rest/error_json.html',
{'msg' : ('Data for "{}" exceeds the maximum '
'length ({})').format(key, max_length)})
# Ensure list of API strings are within the enforced length
for api in f['apis']:
if 128 < len(api):
return render(request, 'rest/error_json.html',
{'msg' : ('API {} is longer than 128 bytes. '
'Report issue is this is a valid '
'API').format(api)})
if not re.match('^[a-zA-Z\d_:@\?\$i\.]+$', api):
return render(request, 'rest/error_json.html',
{'msg' : ('Invalid characters in API, supported'
'characters match the regex /^[a-zA-Z'
'\\d_:@\\?\\$\\.]+$/. Report issue if'
'the submitted API valid is valid.')})
# All input has been validated
db = DBManager.first_db
if not db:
return render(request, 'rest/error_json.html',
{'msg' : 'Unable to connect to FIRST DB'})
# Get sample
sample = db.get_sample(md5_hash, crc32)
if not sample:
return render(request, 'rest/error_json.html',
{'msg' : 'Sample does not exist in FIRST'})
db.sample_seen_by_user(sample, user)
results = {}
for client_key in functions:
f = functions[client_key]
# Check if the id sent back is from an engine, if so skip it
if (('id' in f) and (f['id']) and is_engine_metadata(f['id'])):
continue;
function = db.get_function(create=True, **f)
if not function:
return render(request, 'rest/error_json.html',
{'msg' : 'Function does not exist in FIRST'})
if not db.add_function_to_sample(sample, function):
return render(request, 'rest/error_json.html',
{'msg' : ('Unable to associate function with '
'sample in FIRST')})
metadata_id = db.add_metadata_to_function(user, function, **f)
if not metadata_id:
return render(request, 'rest/error_json.html',
{'msg' : ('Unable to associate metadata with '
'function in FIRST')})
# The '0' indicated the metadata_id is from a user.
_id = make_id(0, metadata=metadata_id)
results[client_key] = _id
# Set the user as applying the metadata
db.applied(sample, user, _id)
# Send opcode to EngineManager
EngineManager.add(function.dump(True))
return HttpResponse(json.dumps({'failed' : False, 'results' : results}))