def preprocess(sample):
"""Preprocess files after upload.
:param sample: :class:`~app.models.Sample`
:return:
"""
hash_path = os.path.join(
current_app.config['APP_UPLOADS_SAMPLES'],
sample.sha256
)
if zipfile.is_zipfile(hash_path):
mt = magic.from_file(hash_path, mime=True)
if mt in skip_mimes:
return None
current_app.log.debug('Extracting {}'.format(hash_path))
zfile = zipfile.ZipFile(hash_path)
for zipfo in zfile.namelist():
cfg = current_app.config
if zfile.getinfo(zipfo).compress_type == 99: # PK compat. v5.1
pwd = '-p{}'.format(cfg['INFECTED_PASSWD'])
with popen('7z', 'e', '-so', pwd, hash_path) as zproc:
buf, stderr = zproc.communicate()
else:
buf = zfile.read(zipfo,
pwd=bytes(cfg['INFECTED_PASSWD'], 'utf-8'))
digests = get_hashes(buf)
hash_path = os.path.join(cfg['APP_UPLOADS_SAMPLES'],
digests.sha256)
if not os.path.isfile(hash_path):
with open(hash_path, 'wb') as wf:
wf.write(buf)
s = Sample(user_id=sample.user_id, filename=zipfo,
parent_id=sample.id,
md5=digests.md5, sha1=digests.sha1,
sha256=digests.sha256, sha512=digests.sha512,
ctph=digests.ctph)
db.session.add(s)
db.session.commit()
def add_sample():
"""Upload untrusted files, E.i. malware samples, files for analysis.
Uploaded files are save in :attr:`config.Config.APP_UPLOADS_SAMPLES` using
the SHA-256 of the content as file name. Existing files are not
overwritten. After upload MD5, SHA1, SHA256, SHA512 and CTPH hashes
are calculated.
**Example request**:
.. sourcecode:: http
POST /api/1.0/samples HTTP/1.1
Host: do.cert.europa.eu
Accept: application/json
Content-Length: 317825
Content-Type: multipart/form-data; boundary=-----------3fa8efc8eb2a42e7
Content-Disposition: form-data; name="files[0]"; filename="zepto.exe"
**Example response**:
.. sourcecode:: http
HTTP/1.0 201 CREATED
Content-Type: application/json
{
"files": [
{
"created": "2016-08-02T14:26:15",
"ctph": "6144:dOWrXkMZWMKsLXiyLgDf1tedfmqmqeGGAV//CNGa1FPi:d3rV",
"filename": "zepto.exe",
"id": 32,
"md5": "a8188e964bc1f9cb1e905ce8f309e086",
"sha1": "c3db12e0ffc4b4b090e32679c95aaa76e07150f7",
"sha256": "7768d4e54b066a567bed1456077025ba7eb56a88aed1bc8cb207",
"sha512": "1fa1ea4a72be8adc9257185a9d71d889fbea2360cee3f6102302e"
}
],
"message": "Files uploaded"
}
:reqheader Accept: Content type(s) accepted by the client
:reqheader Content-Type: multipart/form-data required
:resheader Content-Type: this depends on `Accept` header or request
:form files: Files to be uploaded
:>json array files: List of files saved to disk
:>jsonarr integer id: Sample unique ID
:>jsonarr string created: Time of upload
:>jsonarr string sha256: SHA256 of file
:>jsonarr string ctph: CTPH (a.k.a. fuzzy hash) of file
:>jsonarr string filename: Filename (as provided by the client)
:>json string message: Status message
:statuscode 201: Files successfully saved
"""
uploaded_samples = []
for idx, file in request.files.items():
buf = file.stream.read()
digests = get_hashes(buf)
hash_path = os.path.join(current_app.config['APP_UPLOADS_SAMPLES'],
digests.sha256)
if not os.path.isfile(hash_path):
file.stream.seek(0)
file.save(hash_path)
s = Sample(user_id=g.user.id,
filename=file.filename,
md5=digests.md5,
sha1=digests.sha1,
sha256=digests.sha256,
sha512=digests.sha512,
ctph=digests.ctph)
db.session.add(s)
try:
db.session.commit()
analysis.preprocess(s)
except Exception as e:
db.session.rollback()
db.session.flush()
current_app.log.error(e.args[0])
uploaded_samples.append(s.serialize())
return ApiResponse({
'message': 'Files uploaded',
'files': uploaded_samples
}, 201)
def add_cp_sample():
"""Upload untrusted files, E.i. malware samples, files for analysis.
After upload MD5, SHA1, SHA256, SHA512 and CTPH hashes are calculated.
**Example request**:
.. sourcecode:: http
POST /api/1.0/samples HTTP/1.1
Host: cp.cert.europa.eu
Accept: application/json
Content-Type: multipart/form-data; boundary=----FormBoundaryrflTTZA0oE
------FormBoundaryrflTTZA0oE
Content-Disposition: form-data; name="files[0]"; filename="stux.zip"
Content-Type: application/zip
------FormBoundaryrflTTZA0oE--
**Example response**:
.. sourcecode:: http
HTTP/1.0 201 CREATED
Content-Type: application/json
{
"files": [
{
"created": "2016-03-21T16:09:47",
"ctph": "49152:77qzLl6EKvwkdB7qzLl6EKvwkTY40GfAHw7qzLl6EKvwk...",
"filename": "stux.zip",
"id": 2,
"sha256": "1eedab2b09a4bf6c87b273305c096fa2f597ff9e4bdd39bc4..."
}
],
"message": "Files uploaded"
}
:reqheader Accept: Content type(s) accepted by the client
:reqheader Content-Type: multipart/form-data required
:resheader Content-Type: this depends on `Accept` header or request
:form files: Files to be uploaded
:>json array files: List of files saved to disk
:>jsonarr integer id: Sample unique ID
:>jsonarr string created: Time of upload
:>jsonarr string sha256: SHA256 of file
:>jsonarr string ctph: CTPH (a.k.a. fuzzy hash) of file
:>jsonarr string filename: Filename (as provided by the client)
:>json string message: Status message
:statuscode 201: Files successfully saved
"""
uploaded_samples = []
for idx, file_ in request.files.items():
buf = file_.stream.read()
hashes = get_hashes(buf)
hash_path = os.path.join(current_app.config['APP_UPLOADS_SAMPLES'],
hashes.sha256)
if not os.path.isfile(hash_path):
file_.stream.seek(0)
file_.save(hash_path)
s = Sample(user_id=g.user.id,
filename=file_.filename,
md5=hashes.md5,
sha1=hashes.sha1,
sha256=hashes.sha256,
sha512=hashes.sha512,
ctph=hashes.ctph)
db.session.add(s)
try:
db.session.commit()
analysis.preprocess(s)
except Exception as e:
db.session.rollback()
db.session.flush()
current_app.log.error(e.args[0])
uploaded_samples.append(s.serialize())
return ApiResponse({
'message': 'Files uploaded',
'files': uploaded_samples
}, 201)
def test_calc_hashes():
digests = utils.get_hashes(b'42')
assert digests.md5 == 'a1d0c6e83f027327d8461063f4ac58a6'
assert digests.sha1 == '92cfceb39d57d914ed8b14d0e37643de0797ae56'
assert digests.sha256 == \
'73475cb40a568e8da8a045ced110137e159f890ac4da883b6b17dc651b3a8049'