async def post(self, data):
if data == []:
self.write_warning("upload/file - no request body found", 422)
self.finish()
return
if 'file' not in self.request.files:
self.write_warning("upload/file - no 'file' in part", 422)
self.finish()
return
# Set name if missing
if 'name' not in data:
data['name'] = self.request.files['file'][0]['filename']
# Get the files offset and size
f_path = self.request.files['file'][0]['body'].decode('utf-8')
# Extract if required, zip only
if data['extract']:
try:
f_path = await route_support.unzip_file(f_path, data['password'])
except error.SnakeError as err:
self.write_warning("upload/file - {}".format(err), 422)
self.finish()
return
# Update name if not overriden
if data['name'] == self.request.files['file'][0]['filename']:
data['name'] = path.basename(f_path)
# Set submission type
data['submission_type'] = 'upload:file'
# Hash the file
sha2 = hashlib.sha256()
with open(f_path, 'rb') as f:
chunk = f.read(4096)
while chunk:
sha2.update(chunk)
chunk = f.read(4096)
sha256_digest = sha2.hexdigest()
# Check if the file already exists
document = await db.async_file_collection.select(sha256_digest)
if document:
document = schema.FileSchema().dump(schema.FileSchema().load(document))
self.write_warning("upload/file - file already exists for given sha256 digest", 409, {'sample': document})
self.finish()
return
# Save the file and add it to the database
document = await route_support.store_file(sha256_digest, f_path, enums.FileType.FILE, data)
document = schema.FileSchema().dump(schema.FileSchema().load(document))
self.jsonify({'sample': document})
self.finish()
def extract(self, args, file, opts):
samples = []
with tempfile.TemporaryDirectory(dir=path.abspath(
path.expanduser(
config.snake_config['cache_dir']))) as temp_dir:
# Extract the samples
proc = subprocess.run(
[self.binwalk_path, file.file_path, '-e', '-C', temp_dir],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if not proc:
raise error.CommandError(
"failed to successfully extract from sample")
# Get file name
document = db.file_collection.select(file.sha256_digest)
if not document:
raise error.SnakeError("failed to get sample's metadata")
# There will be one output directory connataining files with the offsets as names
contents = os.listdir(temp_dir)
if not contents:
return []
directory = path.join(temp_dir, contents[0])
for i in os.listdir(directory):
file_path = path.join(directory, i)
name = '{}.{}'.format(document['name'], i)
file_schema = schema.FileSchema().load({
'name':
name,
'description':
'extracted with binwalk'
})
new_file = fs.FileStorage()
new_file.create(file_path)
new_document = submitter.submit(file_schema,
enums.FileType.FILE, new_file,
file, NAME)
new_document = schema.FileSchema().dump(
schema.FileSchema().load(
new_document)) # Required to clean the above
samples += [new_document]
return samples
async def patch(self, sha256_digest):
document = await db.async_file_collection.select(sha256_digest)
if not document or document['file_type'] != enums.FileType.MEMORY:
self.write_warning("memory - no sample for given sha256 digest",
404, sha256_digest)
self.finish()
return
if not self.request.body:
self.write_warning("memory - no request body found", 422,
sha256_digest)
self.finish()
return
data = escape.json_decode(self.request.body)
data = schema.FileSchema(only=('description', 'name', 'tags'),
partial=True).load(data)
data = schema.FileSchema(only=('description', 'name',
'tags')).dump(data)
if data.keys():
await db.async_file_collection.update(sha256_digest, data)
document = await db.async_file_collection.select(sha256_digest)
document = schema.FileSchema().dump(schema.FileSchema().load(document))
self.jsonify({'memory': document})
self.finish()
async def get(self, data):
documents = []
filter_ = self.create_filter(self.request.arguments, data['operator'])
if filter_:
filter_ = {'$and': [filter_]}
if data['file_type']:
filter_['$and'] += [{'file_type': data['file_type']}]
elif data['file_type']:
filter_ = {'file_type': data['file_type']}
# NOTE: With async (motor) there is no count() on cursor so we have to work around that
total = await db.async_file_collection.db.files.count_documents(
filter_ if filter_ else {})
cursor = db.async_file_collection.select_all(filter_, data['order'],
data['sort'],
data['limit'],
data['from'])
while await cursor.fetch_next:
documents += [cursor.next_object()]
documents = schema.FileSchema(many=True).dump(
schema.FileSchema(many=True).load(documents))
self.jsonify({'samples': documents, 'total': total})
self.finish()
async def store_file(sha256_digest, file_path, file_type, data):
"""Store a file to disk.
Uses file storage to store the new file to disk. Upon success insert the
metadata into the database.
Args:
sha256_digest (str): The has of the file to store.
file_path (str): The location of the file to move into the store.
file_type (:obj:`FileType`): The type of the file being stored.
data (:obj:`CommandSchema`): The metadata for the file.
Returns:
:obj:`CommandSchema`: The updated document metadata.
Raises:
SnakeError: When the metadata cannot be inserted into the database.
"""
# Save the file to the 'filedb' and add it to the database
file_storage = utils.FileStorage()
file_storage.create(file_path, sha256_digest)
if not file_storage.save(move=True):
raise error.SnakeError("Failed to store file on disk")
data.update(file_storage.to_dict())
data['name'] = strip_extensions(data['name'])
data['timestamp'] = datetime.utcnow()
data = schema.FileSchema().dump(data)
data['file_type'] = file_type # load_only=True
document = await db.async_file_collection.insert(data)
if not document:
file_storage.delete()
raise error.SnakeError("Failed to insert document")
document = await db.async_file_collection.select(file_storage.sha256_digest
)
# Run any autoruns, if allowed
await execute_autoruns(sha256_digest, file_type, file_storage.mime)
return document
async def post(self, scale): # pylint: disable=too-many-locals
if not self.request.body:
self.write_warning("scale/upload - no request body found", 422,
scale)
self.finish()
return
data = escape.json_decode(self.request.body)
# Validate args
try:
data = self.UploadSchema().dump(self.UploadSchema().load(data))
except exceptions.ValidationError as err:
self.write_warning(
self.json_decode(('%s' % err.messages).replace("'", '"')), 422)
self.finish()
return
scale_ = scale_manager.get_scale(scale)
upload = scale_manager.get_component(scale_,
enums.ScaleComponent.UPLOAD)
# Validate arguments and update
upld_args = upload.arguments()
try:
if upld_args:
data['args'] = schema.Schema(fields=upld_args).load(
data['args'])
except exceptions.ValidationError as err:
self.write_warning(
self.json_decode(
('{"args": %s}' % err.messages).replace("'", '"')), 422)
self.finish()
return
# Get the file
with tempfile.TemporaryDirectory(dir=path.abspath(
path.expanduser(snake_config['cache_dir']))) as temp_dir:
loop = asyncio.get_event_loop()
f_name = await loop.run_in_executor(None, upload.upload,
data['args'], temp_dir)
f_path = path.join(temp_dir, f_name)
# Extract if required, zip only
if data['extract']:
f_path = await route_support.unzip_file(
f_path, data['password'])
f_name = path.basename(f_path)
# Update name if not overriden
if not data['name']:
data['name'] = f_name
# Set submission type
data['submission_type'] = 'upload:{}'.format(scale)
# Check that the file is not empty
if path.getsize(f_path) == 0:
self.write_warning("scale/upload - sample is empty", 422)
self.finish()
return
# Hash the file
sha2 = hashlib.sha256()
with open(f_path, 'rb') as f:
chunk = f.read(4096)
while chunk:
sha2.update(chunk)
chunk = f.read(4096)
sha256_digest = sha2.hexdigest()
# Check if the file already exists
document = await db.async_file_collection.select(sha256_digest)
if document:
document = schema.FileSchema().dump(
schema.FileSchema().load(document))
self.write_warning(
"scale/upload - sample already exists for given sha256 digest",
409, {'sample': document})
self.finish()
return
# Save the file and add it to the database
document = await route_support.store_file(sha256_digest, f_path,
data['file_type'], data)
document = schema.FileSchema().dump(
schema.FileSchema().load(document))
self.jsonify({'sample': document})
self.finish()
def oleobj(self, args, file, opts):
# Monkeypatch 1 - This is to force the script argument to the appropriate file locaiton for analysis
def temp_args(_a, _b, _c):
return [file.file_path]
# Deploy Monkeypatch 1
import optparse
get_args = optparse.OptionParser._get_args
optparse.OptionParser._get_args = temp_args
output = []
fname_prefix = oleobj.sanitize_filename(file.file_path)
index = 1
DUMP_CHUNK_SIZE = 4096
result = dict()
for ole in oleobj.find_ole(file.file_path, None):
if ole is None: # no ole file found
continue
for path_parts in ole.listdir():
stream_path = '/'.join(path_parts)
if path_parts[-1] == '\x01Ole10Native':
stream = None
try:
stream = ole.openstream(path_parts)
#print('extract file embedded in OLE object from stream %r:'
# % stream_path)
#print('Parsing OLE Package')
opkg = oleobj.OleNativeStream(stream)
# leave stream open until dumping is finished
except Exception:
raise error.CommandWarning("*** Not an OLE 1.0 Object")
if stream is not None:
stream.close()
continue
if opkg.is_link:
raise error.CommandWarning('Object is not embedded but only linked to '
'- skip')
continue
result['SHA256_AnalyzedFile'] = fname_prefix
result['Extracted_file.NAME'] = opkg.filename
result['Source_path'] = opkg.src_path
result['Temp_path'] = opkg.temp_path
if opkg.filename:
fname = '%s_%s' % (fname_prefix,
oleobj.sanitize_filename(opkg.filename))
else:
fname = '%s_object_%03d.noname' % (fname_prefix, index)
try:
result['Saved_Filename'] = fname
samples = []
with tempfile.TemporaryDirectory(dir=path.abspath(path.expanduser(config.snake_config['cache_dir']))) as temp_dir:
file_path = path.join(temp_dir, fname)
with open(file_path, 'wb') as writer:
n_dumped = 0
next_size = min(DUMP_CHUNK_SIZE, opkg.actual_size)
while next_size:
data = stream.read(next_size)
writer.write(data)
n_dumped += len(data)
if len(data) != next_size:
raise error.CommandWarning('Wanted to read {0}, got {1}'
.format(next_size, len(data)))
break
next_size = min(DUMP_CHUNK_SIZE,
opkg.actual_size - n_dumped)
file_schema = schema.FileSchema().load({
'name': fname,
'description': 'extracted with oleobj from ' + fname
})
new_file = fs.FileStorage()
new_file.create(file_path)
new_document = submitter.submit(file_schema, enums.FileType.FILE, new_file, file, NAME)
new_document = schema.FileSchema().dump(schema.FileSchema().load(new_document))
samples += [new_document]
for i in samples:
i['name'] = fname
result['samples'] = samples
except Exception as exc:
raise error.CommandWarning('error dumping to {0} ({1})'
.format(fname, exc))
finally:
stream.close()
index += 1
output.append(result)
if not output:
raise error.CommandWarning("No ole object was found")
return output
async def post(self):
# XXX: Does not support extraction atm
# curl 'http://127.0.0.1:5000/upload/files' -F '0=@./file1' -F '1=@./file2' -F 'data={0:{"name": "file1"}, 1:{"name": "file2"}}'
#
data = {}
try:
data = self.get_argument('data')
except Exception: # noqa
data = {}
if data == {}:
missing_fields = {}
missing_fields['data'] = ["Missing data for required field."]
self.write_warning(missing_fields, 422)
self.finish()
return
try:
data = escape.json_decode(data)
except Exception: # noqa
self.write_warning("upload/files - must be content type application/json", 422, data)
self.finish()
return
# Data is optional we do not check that it keys correctly, to avoid
# some errors later down the line prevalidate the data dictionaries
data_arrays = []
for k, v in data.items():
if 'name' not in v:
v['name'] = self.request.files[k][0]['filename']
data_arrays += [v]
# Validate with discard we need the keys
data_arrays = schema.FileSchema(many=True).load(data_arrays)
schema.FileSchema(many=True).dump(data_arrays)
# Upload the files
documents = []
for k, v in data.items():
# Set submission type
v['submission_type'] = 'upload:file'
# Get the files offset and size
f_path = self.request.files[k][0]['body'].decode('utf-8')
# Hash the file
sha2 = hashlib.sha256()
with open(f_path, 'rb') as f:
chunk = f.read(4096)
while chunk:
sha2.update(chunk)
chunk = f.read(4096)
sha256_digest = sha2.hexdigest()
# Check if the file already exists, if so add to documents, but there is no need to upload it
document = await db.async_file_collection.select(sha256_digest)
if document:
documents += [document]
else:
documents += [await route_support.store_file(sha256_digest, f_path, enums.FileType.FILE, v)]
documents = schema.FileSchema(many=True).dump(schema.FileSchema(many=True).load(documents))
self.jsonify({'samples': documents})
self.finish()
def submit(file_schema, file_type, file, parent, scale_name): # pylint: disable=too-many-branches
"""Submit a new file to Snake.
This is used generally by the command component of scales to submit a new
file into snake.
Args:
"""
# We need to be safe here so instance check the above
if not isinstance(file_schema, dict):
raise TypeError("file_schema must be of type dict")
if not isinstance(file, fs.FileStorage):
raise TypeError("file must be of type FileSchema")
if not isinstance(parent, fs.FileStorage):
raise TypeError("parent must be of type FileStorage")
# If the hashes are the same, just stop
if file.sha256_digest == parent.sha256_digest:
return db.file_collection.select(file.sha256_digest)
# Create submission type
submission_type = 'scale:{}'.format(scale_name)
# Check if the file to submit is already in Snake, if not lets add it
document = db.file_collection.select(file.sha256_digest)
if not document:
# Validate
data = schema.FileSchema().dump(schema.FileSchema().load(file_schema))
# Save the file
if not file.save(move=True):
raise error.SnakeError("could not save new file to disk for hash {}".format(file.sha256_digest))
data.update(file.to_dict())
# NOTE: Don't set the parent we will do this later, so blank them out
# if the scale tried to be smart
data['children'] = {}
data['parents'] = {}
data['submission_type'] = submission_type
data['timestamp'] = datetime.utcnow()
data = schema.FileSchema().dump(data)
data['file_type'] = file_type # load_only=True
# Save
db.file_collection.insert(data)
# Update the parent child relationships
document = db.file_collection.select(file.sha256_digest)
if document:
# HACK: This is needed to get submission_type of parent
p = db.file_collection.select(parent.sha256_digest)
# Check if the parent and type already exist
if 'parents' not in document:
document['parents'] = {}
if parent.sha256_digest in document['parents']:
if submission_type in document['parents'][parent.sha256_digest]:
return document
else:
document['parents'][parent.sha256_digest] += [p["submission_type"]]
else:
document['parents'][parent.sha256_digest] = [p["submission_type"]]
# Validate
document = schema.FileSchema().dump(schema.FileSchema().load(document))
# Update
db.file_collection.update(file.sha256_digest, document)
# Update the parents children
document = db.file_collection.select(parent.sha256_digest)
if not document: # Parent does not exist it has been delete, don't update it
return db.file_collection.select(file.sha256_digest)
if 'children' not in document:
document['children'] = {}
if file.sha256_digest in document['children']:
if submission_type in document['children'][file.sha256_digest]:
return db.file_collection.select(file.sha256_digest)
else:
document['children'][file.sha256_digest] += [submission_type]
else:
document['children'][file.sha256_digest] = [submission_type]
# Validate
document = schema.FileSchema().dump(schema.FileSchema().load(document))
# Update
db.file_collection.update(parent.sha256_digest, document)
else:
raise error.SnakeError("could not submit new file for hash {}".format(file.sha256_digest))
return db.file_collection.select(file.sha256_digest)