def preprocess(sample): """Preprocess files after upload. :param sample: :class:`~app.models.Sample` :return: """ hash_path = os.path.join( current_app.config['APP_UPLOADS_SAMPLES'], sample.sha256 ) if zipfile.is_zipfile(hash_path): mt = magic.from_file(hash_path, mime=True) if mt in skip_mimes: return None current_app.log.debug('Extracting {}'.format(hash_path)) zfile = zipfile.ZipFile(hash_path) for zipfo in zfile.namelist(): cfg = current_app.config if zfile.getinfo(zipfo).compress_type == 99: # PK compat. v5.1 pwd = '-p{}'.format(cfg['INFECTED_PASSWD']) with popen('7z', 'e', '-so', pwd, hash_path) as zproc: buf, stderr = zproc.communicate() else: buf = zfile.read(zipfo, pwd=bytes(cfg['INFECTED_PASSWD'], 'utf-8')) digests = get_hashes(buf) hash_path = os.path.join(cfg['APP_UPLOADS_SAMPLES'], digests.sha256) if not os.path.isfile(hash_path): with open(hash_path, 'wb') as wf: wf.write(buf) s = Sample(user_id=sample.user_id, filename=zipfo, parent_id=sample.id, md5=digests.md5, sha1=digests.sha1, sha256=digests.sha256, sha512=digests.sha512, ctph=digests.ctph) db.session.add(s) db.session.commit()
def add_sample(): """Upload untrusted files, E.i. malware samples, files for analysis. Uploaded files are save in :attr:`config.Config.APP_UPLOADS_SAMPLES` using the SHA-256 of the content as file name. Existing files are not overwritten. After upload MD5, SHA1, SHA256, SHA512 and CTPH hashes are calculated. **Example request**: .. sourcecode:: http POST /api/1.0/samples HTTP/1.1 Host: do.cert.europa.eu Accept: application/json Content-Length: 317825 Content-Type: multipart/form-data; boundary=-----------3fa8efc8eb2a42e7 Content-Disposition: form-data; name="files[0]"; filename="zepto.exe" **Example response**: .. sourcecode:: http HTTP/1.0 201 CREATED Content-Type: application/json { "files": [ { "created": "2016-08-02T14:26:15", "ctph": "6144:dOWrXkMZWMKsLXiyLgDf1tedfmqmqeGGAV//CNGa1FPi:d3rV", "filename": "zepto.exe", "id": 32, "md5": "a8188e964bc1f9cb1e905ce8f309e086", "sha1": "c3db12e0ffc4b4b090e32679c95aaa76e07150f7", "sha256": "7768d4e54b066a567bed1456077025ba7eb56a88aed1bc8cb207", "sha512": "1fa1ea4a72be8adc9257185a9d71d889fbea2360cee3f6102302e" } ], "message": "Files uploaded" } :reqheader Accept: Content type(s) accepted by the client :reqheader Content-Type: multipart/form-data required :resheader Content-Type: this depends on `Accept` header or request :form files: Files to be uploaded :>json array files: List of files saved to disk :>jsonarr integer id: Sample unique ID :>jsonarr string created: Time of upload :>jsonarr string sha256: SHA256 of file :>jsonarr string ctph: CTPH (a.k.a. fuzzy hash) of file :>jsonarr string filename: Filename (as provided by the client) :>json string message: Status message :statuscode 201: Files successfully saved """ uploaded_samples = [] for idx, file in request.files.items(): buf = file.stream.read() digests = get_hashes(buf) hash_path = os.path.join(current_app.config['APP_UPLOADS_SAMPLES'], digests.sha256) if not os.path.isfile(hash_path): file.stream.seek(0) file.save(hash_path) s = Sample(user_id=g.user.id, filename=file.filename, md5=digests.md5, sha1=digests.sha1, sha256=digests.sha256, sha512=digests.sha512, ctph=digests.ctph) db.session.add(s) try: db.session.commit() analysis.preprocess(s) except Exception as e: db.session.rollback() db.session.flush() current_app.log.error(e.args[0]) uploaded_samples.append(s.serialize()) return ApiResponse({ 'message': 'Files uploaded', 'files': uploaded_samples }, 201)
def add_cp_sample(): """Upload untrusted files, E.i. malware samples, files for analysis. After upload MD5, SHA1, SHA256, SHA512 and CTPH hashes are calculated. **Example request**: .. sourcecode:: http POST /api/1.0/samples HTTP/1.1 Host: cp.cert.europa.eu Accept: application/json Content-Type: multipart/form-data; boundary=----FormBoundaryrflTTZA0oE ------FormBoundaryrflTTZA0oE Content-Disposition: form-data; name="files[0]"; filename="stux.zip" Content-Type: application/zip ------FormBoundaryrflTTZA0oE-- **Example response**: .. sourcecode:: http HTTP/1.0 201 CREATED Content-Type: application/json { "files": [ { "created": "2016-03-21T16:09:47", "ctph": "49152:77qzLl6EKvwkdB7qzLl6EKvwkTY40GfAHw7qzLl6EKvwk...", "filename": "stux.zip", "id": 2, "sha256": "1eedab2b09a4bf6c87b273305c096fa2f597ff9e4bdd39bc4..." } ], "message": "Files uploaded" } :reqheader Accept: Content type(s) accepted by the client :reqheader Content-Type: multipart/form-data required :resheader Content-Type: this depends on `Accept` header or request :form files: Files to be uploaded :>json array files: List of files saved to disk :>jsonarr integer id: Sample unique ID :>jsonarr string created: Time of upload :>jsonarr string sha256: SHA256 of file :>jsonarr string ctph: CTPH (a.k.a. fuzzy hash) of file :>jsonarr string filename: Filename (as provided by the client) :>json string message: Status message :statuscode 201: Files successfully saved """ uploaded_samples = [] for idx, file_ in request.files.items(): buf = file_.stream.read() hashes = get_hashes(buf) hash_path = os.path.join(current_app.config['APP_UPLOADS_SAMPLES'], hashes.sha256) if not os.path.isfile(hash_path): file_.stream.seek(0) file_.save(hash_path) s = Sample(user_id=g.user.id, filename=file_.filename, md5=hashes.md5, sha1=hashes.sha1, sha256=hashes.sha256, sha512=hashes.sha512, ctph=hashes.ctph) db.session.add(s) try: db.session.commit() analysis.preprocess(s) except Exception as e: db.session.rollback() db.session.flush() current_app.log.error(e.args[0]) uploaded_samples.append(s.serialize()) return ApiResponse({ 'message': 'Files uploaded', 'files': uploaded_samples }, 201)
def test_calc_hashes(): digests = utils.get_hashes(b'42') assert digests.md5 == 'a1d0c6e83f027327d8461063f4ac58a6' assert digests.sha1 == '92cfceb39d57d914ed8b14d0e37643de0797ae56' assert digests.sha256 == \ '73475cb40a568e8da8a045ced110137e159f890ac4da883b6b17dc651b3a8049'