class SaveAttachmentTest(MailgunTestBase): def setUp(self): super(SaveAttachmentTest, self).setUp() (filename, file_stream) = get_attachment() self.attachment = FileStorage(stream=file_stream, filename=filename) def tearDown(self): super(SaveAttachmentTest, self).tearDown() self.attachment.close() def test_fileallowed(self): self.assertTrue(self.mailgun.file_allowed('test.txt')) self.assertFalse(self.mailgun.file_allowed('bob')) def test_save_attachments(self): testdir = tempfile.mkdtemp() self.attachment.seek(0) filenames = self.mailgun.save_attachments([self.attachment], testdir) filenames = [os.path.basename(filename) for filename in filenames] self.assertTrue(set(os.listdir(testdir)) == set(filenames)) self.assertEqual(len(os.listdir(testdir)), 1) shutil.rmtree(testdir) with self.assertRaises(OSError): os.listdir(testdir)
def __init__(self, stream=None, filename=None, name=None, content_type='application/octet-stream', content_length=-1, headers=None): FileStorage.__init__(self, stream, filename, name=name, content_type=content_type, content_length=content_length, headers=None) self.saved = None
def test_read_csv_file_to_dict(self): csvstream = open("testdata/test.csv", "rb") csvfile = FileStorage(filename="test.csv", stream=csvstream) csvfile.mode = "rb" (success, result_dict, header_list) = read_csv_file_to_dict(csvfile) self.assertEqual(success, 'test') self.assertEqual(len(result_dict.keys()), 29) self.assertEqual(header_list,["__id__", "__filename__", "Text", "Success"])
def test_Transfer_preprocess(transf): @transf.preprocessor def to_upper(filehandle, meta): filehandle.stream = BytesIO(filehandle.stream.read().lower()) return filehandle source = FileStorage(stream=BytesIO(b'Hello World')) transf._preprocess(source, {}) source.seek(0) assert source.read() == b'hello world'
def test_jfu_server(self): self._test_view_post('frontend.jfu_server', status='200 OK', data=['{', '}'], form=dict(data_file=FileStorage( StringIO("Hello, world"), filename='C:\\fakepath\\DoesntExist.txt', content_type='text/plain; charset=utf-8'), ), viewopts=dict(item_name='WillBeCreated'), content_types=[ 'application/json', ])
def test_api_extensions_custom_fields_file(self): """Test rest api custom file field""" class SampleSchema(ma.Schema): """Sample custom file field schema""" class Meta: strict = True input_file = FileField(allowed_file_extensions=('.pdf', )) file_storage = FileStorage(filename='test.pdf') file_datas = SampleSchema().load({'input_file': file_storage}).data assert file_datas['input_file'] == file_storage # Errors # validation: wrong file extension with pytest.raises(ma.ValidationError): SampleSchema().load( {'input_file': FileStorage(filename='not_valid.zip')}) # validation: wrong value type with pytest.raises(ma.ValidationError): SampleSchema().load({'input_file': 'wrong_type'})
def rsa_encrypt(): if request.method == 'GET': return render_template('rsa/encrypt/index.html') if request.files['pubkey-file']: rsaPublicKey = RSAPublicKey.from_json( request.files['pubkey-file'].read()) else: rsaPublicKey = RSAPublicKey(int(request.form['n']), int(request.form['e'])) start = default_timer() n_byte_length = byte_length(rsaPublicKey.n) chunk_byte_length = n_byte_length - 1 if chunk_byte_length <= 0: return 'Modulus too small' message = request.files['message'].read() plaintext = len(message).to_bytes(4, byteorder) + message plaintext_chunks = [ plaintext[i:i + chunk_byte_length] for i in range(0, len(plaintext), chunk_byte_length) ] ciphertext = bytes() for chunk in plaintext_chunks: ciphertext += rsaPublicKey.encrypt(int.from_bytes( chunk, byteorder)).to_bytes(n_byte_length, byteorder) end = default_timer() filename = 'encrypted_' + request.files['message'].filename ciphertext_file = FileStorage(stream=BytesIO(ciphertext), filename=filename) ciphertext_file.seek(0, SEEK_END) size = ciphertext_file.tell() ciphertext_file.seek(0) ciphertext_file.save('static/' + filename) return render_template('rsa/encrypt/result.html', plaintext=message, ciphertext=ciphertext, time=end - start, filename=filename, size=size)
def create(self, user_id: str, file_: FileStorage) -> Optional[File]: # Save file in local filename = secure_filename(file_.filename) temporary_path = FileHelper().get_temporary_path(filename) file_.save(temporary_path) # Update file to S3 path = S3().create_file(temporary_path, file_.content_type) fields = { 'user_id': user_id, 'name': filename, 'media_type': file_.content_type, 'path': path, 'size': os.path.getsize(temporary_path) } file_model = self.file_repository.create(fields) # Delete directory shutil.rmtree(os.path.dirname(temporary_path)) return file_model
def ecc_decrypt(): if request.method == 'GET': return render_template('ecc/decrypt/index.html') message = request.files['message'].read() kob = int(request.form['koblitz']) coef = int(request.form['x-coef']) const = int(request.form['const']) modulo = int(request.form['modulo']) if not is_prime(modulo): return "Error: Modulo must be prime" curve = EllipticCurve(coef, const, modulo) prikey = request.files.get('prikey-file', None) if prikey == None: prikey = int(request.form['prikey']) else: prikey = int(prikey.read()) raw_ciphertext = message.decode().split(' ') while raw_ciphertext[-1] == '': del raw_ciphertext[-1] ciphertext = [Point(0, 0, modulo, point) for point in raw_ciphertext] start = default_timer() point_neg = curve.multiply(prikey, ciphertext[-1]) del ciphertext[-1] point_neg.negate() plaintext = [curve.add(point, point_neg) for point in ciphertext] end = default_timer() string = decode(plaintext, kob) filename = request.form['filename'] file = FileStorage(stream=BytesIO(string), filename=filename) file.seek(0, SEEK_END) size = file.tell() file.seek(0) file.save('static/' + filename) return render_template('ecc/decrypt/result.html', plaintext=string, ciphertext=message.decode(), time=end - start, filename=filename, size=size)
def test_file(self): stream = StringIO.StringIO() stream.write("""\ Content-Type: application/octet-stream Content-Length: 1 X-AppEngine-Upload-Creation: 2010-10-01 05:34:00.000000 """) stream.seek(0) headers = {} headers['Content-Type'] = 'image/png; blob-key=foo' f = FileStorage(stream=stream, headers=headers) self.assertNotEqual(parse_blob_info(f, 'my_field_name'), None)
def test_invalid_CREATION(self): stream = StringIO.StringIO() stream.write("""\ Content-Type: application/octet-stream Content-Length: 1 X-AppEngine-Upload-Creation: XXX """) stream.seek(0) headers = {} headers['Content-Type'] = 'image/png; blob-key=foo' f = FileStorage(stream=stream, headers=headers) self.assertRaises(blobstore.BlobInfoParseError, parse_blob_info,f, 'my_field_name')
def test_Transfer_validate_raises_with_falsey(transf): source = FileStorage(stream=BytesIO(), filename='test.conf') @transf.validator def bad_validator(fh, m): False with pytest.raises(UploadError) as excinfo: transf.save(source, metadata={}, catch_all_errors=False, destination=lambda *a, **k: None) expected = "{0!r}({1!r}, {2!r}) returned False".format(bad_validator, source, {}) assert str(excinfo.value) == expected
def __resource_update_with_upload(cls, context, filename, resource_name, resource_id): file_path = os.path.join(os.path.dirname(__file__), filename) with open(file_path) as f: file_upload = FileStorage(f) resource_dict = _get_action('resource_update')( context, { 'id': resource_id, 'name': resource_name, 'url_type': 'upload', 'resource_type': 'file.upload', 'upload': file_upload }) return resource_dict
def test_boto3_upload_file_func(): conn = boto3.resource('s3', region_name='us-east-1') conn.create_bucket(Bucket=s3_bucket) file = FileStorage(filename='test.txt', stream=io.BytesIO(b'test'), content_type='text') s3_url, unique_secure_filename = upload_file_to_s3(file) body = conn.Object( s3_bucket, unique_secure_filename).get()['Body'].read().decode("utf-8") assert (body == 'test')
def rsa_decrypt(): if request.method == 'GET': return render_template('rsa/decrypt/index.html') if request.files['prikey-file']: rsaPrivateKey = RSAPrivateKey.from_json( request.files['prikey-file'].read()) else: rsaPrivateKey = RSAPrivateKey(int(request.form['n']), int(request.form['d'])) start = default_timer() n_byte_length = byte_length(rsaPrivateKey.n) if n_byte_length <= 1: return 'Modulus too small' ciphertext = request.files['message'].read() ciphertext_chunks = [ ciphertext[i:i + n_byte_length] for i in range(0, len(ciphertext), n_byte_length) ] plaintext = bytes() for chunk in ciphertext_chunks: plaintext += rsaPrivateKey.decrypt(int.from_bytes( chunk, byteorder)).to_bytes(n_byte_length, byteorder)[:-1] plaintext = plaintext[4:int.from_bytes(plaintext[:4], byteorder) + 4] end = default_timer() filename = 'decrypted_' + request.files['message'].filename plaintext_file = FileStorage(stream=BytesIO(plaintext), filename=filename) plaintext_file.seek(0, SEEK_END) size = plaintext_file.tell() plaintext_file.seek(0) plaintext_file.save('static/' + filename) return render_template('rsa/decrypt/result.html', plaintext=plaintext, ciphertext=ciphertext, time=end - start, filename=filename, size=size)
def upload_file(): file = request.files["user_file"] temp_name = file.filename file.save(temp_name) to_convert = AudioSegment.from_file(temp_name) new_name = temp_name.rsplit(".", 1)[0]+".wav" os.remove(temp_name) to_convert.export(new_name, format="wav") converted = open(new_name, "rb") file = FileStorage(converted, content_type="audio/wav") output = upload_file_to_s3(file, app.config["S3_BUCKET"]) return (file.filename)
class SaveAttachmentTest(MailgunTestBase): def setUp(self): super(SaveAttachmentTest, self).setUp() (filename, file_stream) = get_attachment() self.attachment = FileStorage(stream=file_stream, filename=filename) def tearDown(self): super(SaveAttachmentTest, self).tearDown() self.attachment.close() def test_fileallowed(self): self.assertTrue(self.mailgun._is_file_allowed('test.txt')) self.assertFalse(self.mailgun._is_file_allowed('bob')) def test_save_attachments(self): testdir = tempfile.mkdtemp() self.attachment.seek(0) filenames = save_attachments([self.attachment], testdir) filenames = [os.path.basename(filename) for filename in filenames] self.assertTrue(set(os.listdir(testdir)) == set(filenames)) self.assertEqual(len(os.listdir(testdir)), 1) shutil.rmtree(testdir) with self.assertRaises(OSError): os.listdir(testdir)
def rsa_generate(): if request.method == 'GET': return render_template('rsa/generate/index.html') rsaKeyPair = RSAKeyPair(int(request.form['p']), int(request.form['q']), int(request.form['e'])) filename = request.form['filename'] public_file = FileStorage(stream=BytesIO( rsaKeyPair.public.to_json().encode()), filename=filename + '.pub') public_file.save('static/' + public_file.filename) private_file = FileStorage(stream=BytesIO( rsaKeyPair.private.to_json().encode()), filename=filename + '.pri') private_file.save('static/' + private_file.filename) return render_template('rsa/generate/result.html')
def return_werkzeug_filestorage(request, filename): extension = request.headers['content-type'].split('/')[-1] if extension not in current_app.config['ALLOWED_EXTENSIONS']: raise UploadNotAllowed("Unsupported file format") new_filename = secure_filename(filename + '.' + extension) try: tempfile = StringIO(buf=request.content) except AttributeError: if hasattr(request.stream, 'getvalue'): tempfile = StringIO(buf=request.stream.getvalue()) elif hasattr(request.stream, 'read'): tempfile = StringIO(buf=request.stream.read()) tempfile.name = new_filename filestorage = FileStorage(tempfile, filename=new_filename, content_type=request.headers['content-type']) return filestorage
def test_handle_post_request_file_other(tmpdir): filename = "test.pdf" filepath = os.path.join(tmpdir, filename) with open(filepath, "w") as f: f.write("mock data") m = mock.MagicMock() files = { "file": FileStorage(open(filepath, "rb"), content_type='poulet/pdf') } m.files = files upload_folder = os.path.join(tmpdir, "upload_folder") with pytest.raises(NotImplementedError): with mock.patch("mot.serving.inference.request", m): output = handle_post_request(upload_folder=upload_folder) assert os.path.isdir( upload_folder ) # the upload_folder should be created by handle post request
def bulkupload(): form = BulkUploadForm() if form.validate_on_submit(): # print(form.file) _, file_extension = os.path.splitext( secure_filename(form.file.data.filename)) temp = ''.join([uuid.uuid4().hex, file_extension]) filename = files.save(FileStorage(form.file.data), name=secure_filename(temp)) file_url = files.url(filename) print(filename) utils.process_bulkupload('C:\\test.xlsx', file_url) return render_template('base.html') return render_template('bulkupload.html', title='New Hire - Bulk Upload', form=form)
def process_image(requestfile, maxsize=(170, 130)): fileext = splitext(requestfile.filename)[1].lower() if fileext not in ['.jpg', '.jpeg', '.png', '.gif']: raise UploadNotAllowed("Unsupported file format") img = Image.open(requestfile) img.load() if img.size[0] > maxsize[0] or img.size[1] > maxsize[1]: img.thumbnail(maxsize, Image.ANTIALIAS) boximg = Image.new('RGBA', maxsize, (255, 255, 255, 0)) boximg.paste(img, ((boximg.size[0] - img.size[0]) / 2, (boximg.size[1] - img.size[1]) / 2)) savefile = StringIO() savefile.name = requestfile.filename boximg.save(savefile) savefile.seek(0) return FileStorage(savefile, filename=requestfile.filename, content_type=requestfile.content_type)
def test_handle_post_request_file_zip(mock_server_result, tmpdir): m = mock.MagicMock() files = { "file": FileStorage(open(PATH_TO_TEST_ZIP, "rb"), content_type='application/zip') } m.files = files m.form = {"fps": 2, "foo": "bar"} with mock.patch("mot.serving.inference.request", m): output = handle_post_request(upload_folder=str(tmpdir)) assert len(output["detected_trash"]) == 2 assert "id" in output["detected_trash"][0] assert "frame_to_box" in output["detected_trash"][1] for frame, box in output["detected_trash"][1]["frame_to_box"].items(): assert isinstance(frame, int) assert isinstance(box, list) assert len(box) == 4 assert output["video_length"] == 6 or output["video_length"] == 7 assert output["fps"] == 2 assert "video_id" in output
def resize_image(requestfile, maxsize=(320, 240)): fileext = requestfile.filename.split('.')[-1].lower() if fileext not in current_app.config['ALLOWED_EXTENSIONS']: raise UploadNotAllowed("Unsupported file format") img = Image.open(requestfile) img.load() if img.size[0] > maxsize[0] or img.size[1] > maxsize[1]: img.thumbnail(maxsize, Image.ANTIALIAS) boximg = Image.new('RGBA', (img.size[0], img.size[1]), (255, 255, 255, 0)) boximg.paste(img, (0, 0)) savefile = StringIO() if fileext in ['jpg', 'jpeg']: savefile.name = secure_filename(".".join(requestfile.filename.split('.')[:-1]) + ".png") boximg.save(savefile, format="PNG") content_type = "image/png" else: savefile.name = secure_filename(requestfile.filename) boximg.save(savefile) content_type = requestfile.content_type savefile.seek(0) return FileStorage(savefile, filename=savefile.name, content_type=content_type)
def upload_stream(self, stream: FileStorage, upload_dir: str = None, chunk_size: int = 5 * 1024 * 1024, progress: bool = False) -> str: if not upload_dir: upload_dir = uuid4().hex ext = os.path.splitext(secure_filename(stream.name))[1] s3_file_path = os.path.join(upload_dir, uuid4().hex + ext) bucket = self.resource.Object(self.bucket, s3_file_path) mp = bucket.initiate_multipart_upload(ACL='public-read') #'bucket-owner-full-control') parts_etag = {} chunk_idx = 1 while True: chunk = stream.read(chunk_size) if not chunk: break part = mp.Part(chunk_idx) result = part.upload(Body=chunk) parts_etag[str(chunk_idx)] = result['ETag'] chunk_idx += 1 yield chunk_idx - 1, self._endpoint(s3_file_path) mp.complete( MultipartUpload={ 'Parts': [{ 'ETag': parts_etag[str(idx)], 'PartNumber': idx } for idx in range(1, chunk_idx)] }) yield chunk_idx - 1, self._endpoint(s3_file_path)
def test_handle_post_request_file_video(mock_server_result, tmpdir): m = mock.MagicMock() files = { "file": FileStorage(open(PATH_TO_TEST_VIDEO, "rb"), content_type='video/mkv') } m.files = files split_frames_folder = os.path.join( tmpdir, "{}_split".format(os.path.basename(PATH_TO_TEST_VIDEO))) os.mkdir(split_frames_folder ) # this folder should be deleted by handle post request with mock.patch("mot.serving.inference.request", m): output = handle_post_request(upload_folder=str(tmpdir), fps=2) assert len(output["detected_trash"]) == 2 assert "id" in output["detected_trash"][0] assert "frame_to_box" in output["detected_trash"][1] for frame, box in output["detected_trash"][1]["frame_to_box"].items(): assert isinstance(frame, int) assert isinstance(box, list) assert len(box) == 4 assert output["video_length"] == 6 or output["video_length"] == 7 assert output["fps"] == 2 assert "video_id" in output
def handle_file(file: FileStorage, upload_folder: str = UPLOAD_FOLDER, fps: int = FPS, resolution: Tuple[int, int] = RESOLUTION, **kwargs) -> Dict[str, np.array]: """Make the prediction if the data is coming from an uploaded file. Arguments: - *file*: The file, can be either an image or a video, or a zipped folder - *upload_folder*: Where the files are temporarly stored Returns: - for an image: a json of format ```json { "image": filename, "detected_trash": [ { "box": [1, 1, 2, 20], "label": "fragments", "score": 0.92 }, { "box": [10, 10, 25, 20], "label": "bottles", "score": 0.75 } ] } ``` - for a video or a zipped file: a json of format ```json { "video_length": 132, "fps": 2, "video_id": "GOPRO1234.mp4", "detected_trash": [ { "label": "bottles", "id": 0, "frame_to_box": { 23: [0, 0, 1, 10], 24: [1, 1, 4, 13] } }, { "label": "fragments", "id": 1, "frame_to_box": { 12: [10, 8, 9, 15] } } ] } ``` Raises: - *NotImplementedError*: If the format of data isn't handled yet """ if kwargs: logger.warning("Unused kwargs: {}".format(kwargs)) filename = secure_filename(file.filename) full_filepath = os.path.join(upload_folder, filename) if not os.path.isdir(upload_folder): os.mkdir(upload_folder) if os.path.isfile(full_filepath): os.remove(full_filepath) file.save(full_filepath) file_type = file.mimetype.split("/")[0] # mimetype is for example 'image/png' and we only want the image if file_type == "image": image = cv2.imread(full_filepath) # cv2 opens in BGR os.remove(full_filepath) # remove it as we don't need it anymore try: detected_trash = predict_and_format_image(image) except ValueError as e: return {"error": str(e)} return {"image": filename, "detected_trash": detected_trash} elif file_type in ["video", "application"]: folder = None if file.mimetype == "application/zip": # zip case ZipFile(full_filepath).extractall(upload_folder) dirname = None with ZipFile(full_filepath, 'r') as zipObj: listOfFileNames = zipObj.namelist() for fileName in listOfFileNames: dirname = os.path.dirname(fileName) zipObj.extract(fileName, upload_folder) folder = os.path.join(upload_folder, dirname) else: # video case: splitting video and saving frames folder = os.path.join(upload_folder, "{}_split".format(filename)) if os.path.isdir(folder): shutil.rmtree(folder) os.mkdir(folder) logger.info("Splitting video {} to {}.".format( full_filepath, folder)) split_video(full_filepath, folder, fps=fps, resolution=resolution) print("folder:", folder, "uplaod_folder:", upload_folder, "file.filename:", file.filename) image_paths = read_folder(folder) if len(image_paths) == 0: raise ValueError("No output image") # making inference on frames logger.info("{} images to analyze on {} CPUs.".format( len(image_paths), CPU_COUNT)) try: with multiprocessing.Pool(CPU_COUNT) as p: inference_outputs = list( tqdm( p.imap(process_image, image_paths), total=len(image_paths), )) except ValueError as e: return {"error": str(e)} logger.info("Finish analyzing video {}.".format(full_filepath)) # tracking objects logger.info("Starting tracking.") object_tracker = ObjectTracking(filename, image_paths, inference_outputs, fps=fps) tracks = object_tracker.compute_tracks() logger.info("Tracking finished.") return object_tracker.json_result(tracks) else: raise NotImplementedError(file_type)
def requests(req=None): if req == 'comment' and request.method == 'POST': #COMMENT REQUESTS error = False r = request.json n = r['name'] e = r['email'] s = r['text'] l = r['level'] + 1 if 'level' in r else '0' p = r['parent'] if 'parent' in r else '0' d = r['id'] if n.replace(" ", "") == "": error = True if s.replace(" ", "") == "": error = True if e.split('@')[1] == 'undefined' or e.split('@')[1].split( '.')[1] == 'undefined': error = True if error == True: return 'failed: requests not valid' #processing and limiting text s = ("<p>%s</p>" % cut(Markup.escape(s), 1500)).replace( "\n", "</p><p>") n = cut(n, 60) e = cut(e, 100) query_db( "INSERT INTO comments (Name, Date, Email, Parent, Level, Comment, ArticleID) VALUES (?,DATETIME('NOW'),?,?,?,?,?)", [n, e, p, l, s, d]) get_db().commit() return "Nice! Request recieved! And look at you looking into the console. Well here's your reward: raw request data { Name: %s, Email: %s, Comment: %s, Level: %s, Parent: %s, ArticleID: %s }" % ( n, e, s, l, p, d) if req == 'profile' and request.method == 'POST': if not session.get('user'): return "Fail.." r = request.form['name'] d = request.form['desc'] h = request.form['hasimage'] n = session.get('name') query_db("UPDATE users SET Description=? WHERE Username = ?", [d, session.get('user')]) get_db().commit() g = 'image uploaded!' if h == 'true': s = FileStorage(stream=request.files['image']).save( os.path.join(app.config['AUTHOR_FOLDER'], secure_filename(''.join([n, '.jpg'])))) else: g = "no image uploaded." return "Nice! Author Updated! Description changed to '%s', and %s" % ( d, g) if req == 'article' and request.method == 'POST': if not session.get('user'): return "Fail.." a = request.form['article'] h = request.form['html'] id = request.form['id'] t = request.form['title'].lstrip().rstrip() st = secure_filename(t) c = request.form['category'] ta = request.form['tags'] hc = request.form['hascover'] cd = request.form['coverdesc'] d = request.form['desc'] i = request.form[ 'image'] #DO NOT I REPEAT DO NOT Markup.escape THIS U F*****G DUMB PIECE OF SHIT. titles = query_db('SELECT ID FROM articles WHERE Title=?', [t]) hoc = request.form['hoc'] p = None ta = ','.join([e.strip() for e in ta.split(',')]) if id == 'false': #new article if (len(titles) > 0): return "ERROR: The title ur trying to use has been used %s time before!" % ( len(titles)) id = query_db('SELECT ID FROM users WHERE Username=?', [session.get('user') ])[0][0] #set ID from article id to author id query_db( "INSERT INTO articles (Title, SafeTitle, AuthorID, Date, Category, Tags, Description, CoverInfo, Pictures, Content, Text) VALUES (?,?,?,DATETIME('NOW'),?,?,?,?,?,?,?)", [t, st, id, c, ta, d, cd, i, h, a]) else: #update article if (len(titles) > 0 and id != str(titles[0][0])): return "ERROR: The renaming of this article will cause a conflict with (%s) other articles!" % ( len(titles)) query_db( "UPDATE articles SET Title=?, SafeTitle=?, Category=?, Tags=?, Description=?, CoverInfo=?, Pictures=?, Content=?, Text=? WHERE id = ?", [t, st, c, ta, d, cd, i, h, a, id]) #UPLOAD COVER if hc == 'true': pid = str( query_db("SELECT ID FROM articles WHERE Title=?", [t])[0][0]) p = secure_filename(''.join([pid, '.jpeg'])) pic = os.path.join(app.config['COVER_FOLDER'], p) s = FileStorage(stream=request.files['cover']).save(pic) optimize(pic) query_db("UPDATE articles SET CoverURL=?, CoverInfo=? WHERE id=?", [p, cd, pid]) #DELETE cover?!?!? D: if hoc == 'false': pic = query_db("SELECT CoverURL FROM articles WHERE ID=?", [str(id)]) try: os.remove( os.path.join(app.config['COVER_FOLDER'], secure_filename(str(pic[0][0])))) except: pass query_db( "UPDATE articles SET CoverURL='', CoverInfo='' WHERE id=?", [id]) get_db().commit() return "Nice! everything went smoothly!" if req == 'upload' and request.method == 'POST': if not session.get('user'): return "Fail.." t = request.form['type'] n = secure_filename(request.form['name']) folder = app.config['PICTURE_FOLDER'] allfiles = [f for f in listdir(folder) if isfile(join(folder, f))] for file in allfiles: if file.split('.')[0] == n: return "FAIL:, Already a picture named that!" if t == 'file': ext = request.form['ext'] if allowed_file(ext): d = request.files['stuff'] filename = secure_filename(''.join([n, ext])) pic = os.path.join(folder, filename) s = FileStorage(stream=d).save(pic) optimize(pic) n = filename else: return "FAIL:, not right file type!" elif t == 'url': d = request.form['stuff'] name = urlparse(d) notusing, ext = splitext(basename(name.path)) filename = secure_filename(''.join([n, ext])) if allowed_file(filename): finalp = os.path.join(folder, filename) urllib.urlretrieve(d, finalp) optimize(finalp) n = filename else: return "FAIL:, not right file type!" else: return "FAIL:, no images!!" return "%s" % url_for('pics', set='pictures', pic=n) if req == 'category' and request.method == 'POST': if not session.get('user'): return "Fail.." n = request.form['name'].lstrip().rstrip() d = request.form['desc'] c = request.form['color'] h = request.form['hasimage'] fname = secure_filename(''.join([n.lower(), '.jpg'])) all = query_db('SELECT Name FROM category') done = False for a in all: if n.lower() == str(a[0]).lower(): done = True query_db( "UPDATE category SET Name = ?, Description = ?, Color = ?, Picture = ? WHERE Name = ?", [n.lower(), d, c, fname, n.lower()]) if done == False: query_db( "INSERT INTO category(Name, Description, Color, Picture) VALUES (?,?,?,?)", [n.lower(), d, c, fname]) get_db().commit() g = 'image uploaded!' if h == 'true': file = os.path.join(app.config['TOPIC_FOLDER'], fname) s = FileStorage(stream=request.files['image']).save(file) optimize(file, resize=False) else: g = 'no image uploaded.' return "DONE my good sir! NEW CATEGORY! Name: %s, Color: %s, Description: %s and %s" % ( n, c, d, g) if req == 'delete_article' and request.method == 'POST': if not session.get('user'): return "Fail.." id = request.form['id'] pic = str( query_db("SELECT CoverURL FROM articles WHERE ID=?", [str(id)])[0][0]) try: os.remove( os.path.join(app.config['COVER_FOLDER'], secure_filename(pic))) except: pass query_db("DELETE FROM articles WHERE ID = ?", [id]) query_db("DELETE FROM comments WHERE ArticleID = ?", [id]) get_db().commit() return "article with id #%s deleted :(" % (id) if req == 'posts' and request.method == 'POST': na = request.form['numthere'] #number of articles already there n = request.form['numnew'] #number of articles to request args = request.form['args'].split(',') #get main info if args[0] == 'None': results = postmaker( query_db( "SELECT ID, Title, AuthorID, Date, Category, Tags, CoverURL, CoverInfo, Description, Content FROM articles ORDER BY ID DESC LIMIT ?,?", [na, n])) elif args[0] == 'Category': results = postmaker( query_db( "SELECT ID, Title, AuthorID, Date, Category, Tags, CoverURL, CoverInfo, Description, Content FROM articles WHERE Category = ? ORDER BY ID DESC LIMIT ?,?", [args[1], na, n])) elif args[0] == 'Tags': results = postmaker( query_db( "SELECT ID, Title, AuthorID, Date, Category, Tags, CoverURL, CoverInfo, Description, Content FROM articles WHERE Tags = ? ORDER BY ID DESC LIMIT ?,?", [args[1], na, n])) if len(results) == 0: return "FAIL: NO RESULTS" #output everything return render_template('post.html', results=results) return abort(404)
def image1t(): if os.path.exists('testpic.png'): os.remove('testpic.jpg') FileStorage(stream=request.files['image']).save('testpic.jpg') return jsonify({'done':'done'})
def _save_request_content(destination, stream): file_storage = FileStorage(stream) file_storage.save(destination)
def storage_service(job_id, file_name=None): # def storage_service(job_id, file_name=None): """Endpoint serving as the gateway to storage bucket""" if file_name: object_name = os.path.join(job_id, file_name) # print('%s %s' % (request.method, object_name)) if request.method == 'GET': return_json = False if 'json' in request.args.keys(): if request.args['json'].lower() == 'true': return_json = True if not return_json: '''send_file_from_directory''' # file_path_in_cache = storageClient.fget_object(JOB_BUCKET_NAME, object_name) # file_dir = os.path.dirname(file_path_in_cache) # return send_from_directory(file_dir, file_path_in_cache.split('/')[-1]) try: file_path_in_cache = storageClient.fget_object(JOB_BUCKET_NAME, object_name) file_dir = os.path.dirname(file_path_in_cache) return send_from_directory(file_dir, file_path_in_cache.split('/')[-1]) except MaxRetryError: return 'Error in retrieving file\n', 500 except: return 'File %s does not exist\n' % file_name, 404 else: try: file_str = storageClient.get_object(JOB_BUCKET_NAME, object_name) file_str_json = { object_name: file_str.decode('utf-8') } # response = make_response(JSONEncoder().encode(file_str_json)) response = make_response( dumps(file_str_json) ) response.headers['Content-Type'] = 'application/json' http_response_code = 200 # return response, http_response_code except MaxRetryError: json_string = {object_name: None} response = make_response(dumps(json_string)) response.headers['Content-Type'] = 'application/json' http_response_code = 500 # return response, 500 except Exception as e: # import traceback # json_string = {object_name: None, 'error': str(e), 'traceback': traceback.format_exc()} json_string = {object_name: None} response = make_response(dumps(json_string)) response.headers['Content-Type'] = 'application/json' http_response_code = 500 # return response, 500 finally: return response, http_response_code elif request.method == 'PUT': try: payload = loads(request.data) except: payload = request.data elif request.method == 'POST': EXTENSION_WHITELIST = set(['pqr', 'pdb', 'in', 'p']) # pp.pprint(dict(request.files)) # pp.pprint(request.form['job_id']) # pp.pprint(request.files.keys()) print('request.files keys:') for key in request.files.keys(): print(' ', key) try: file_data = request.files['file_data'] # print(type(file_data), flush=True) except: # file_data = BytesIO(request.data) # print(request.data.decode('utf-8')) file_data = FileStorage( stream=BytesIO(request.data), filename=file_name, ) # print(type(file_data)) if file_data.filename: file_name = secure_filename(file_data.filename) if file_data.filename and file_name: storageClient.put_object(JOB_BUCKET_NAME, object_name, file_data) # if file_data.filename and allowed_file(file_name, EXTENSION_WHITELIST): # # print('uploading to bucket') # storageClient.put_object(JOB_BUCKET_NAME, object_name, file_data) # elif not allowed_file(file_name, EXTENSION_WHITELIST): # return 'Unsupported media type', 415 # time.sleep(1) return 'Success', 201 elif request.method == 'DELETE': object_list = [] if file_name is None: # get list of objects with prefix # for each object, delete from bucket job_objects = storageClient.list_objects(JOB_BUCKET_NAME, prefix=job_id+'/') for obj in job_objects: object_list.append(obj.object_name) else: # delete single object from bucket object_list.append(object_name) storageClient.remove_objects(JOB_BUCKET_NAME, object_list) return 'Success', 204 elif request.method == 'OPTIONS': options = ['GET', 'PUT', 'POST', 'DELETE'] response = make_response() response = storage_utils.get_request_options(response, options) http_response_code = 204 return response, http_response_code
def _wrap_file(filename, content_type): """Open the file *filename* and wrap in a FileStorage object.""" assert os.path.isfile(filename), "File does not exist." return FileStorage(stream=open(filename, 'rb'), filename=os.path.basename(filename), content_type=content_type)
def upload_file(): ##connect to DB conn = mysql.connector.Connect(database=db, **conn_kwargs) c = conn.cursor() timestamp = time.time() with open("log.txt", "a") as f: f.write(f"{timestamp}\n") # A if "user_file" not in request.files: #DEMO #Get last submitted File c.execute(""" SELECT filename FROM conversations ORDER BY idconversations DESC LIMIT 1""") last_audio = c.fetchall()[0][0] file = File(last_audio) #get bullets ,transcription text, image , audio_path c.execute(f""" SELECT bullet FROM summary_bullets WHERE origin = "{file.filename}" ORDER BY bullet_pos""") bullets = [entry[0] for entry in c.fetchall()] c.execute(f""" SELECT full_text , bigram_cloud_url , audio_path FROM conversations WHERE filename = '{file.filename}'""") transcription, image, audio_path = c.fetchall()[0] conn.close() return render_template("success.html", bullets=bullets, image=image, transcription=transcription, file_name=file.filename, audio_path=audio_path) #KEEP Below after Debugging return "No user_file key in request.files" # B file = request.files["user_file"] # C. if file.filename == "": return "Please select a file" # D. if file and allowed_file(file.filename): #convert file if necessary if file.filename[-4:] != ".wav": #temporarily store file to avoid No Header bug temp_name = file.filename file.save(temp_name) #Read and convert bytes then store temp file to_convert = AudioSegment.from_file(temp_name) new_name = temp_name.rsplit(".", 1)[0] + ".wav" os.remove(temp_name) to_convert.export(new_name, format="wav") #Open temp file and wrap in werkzeug FileStorage Object, so Boto3 can properly handle it converted = open(new_name, "rb") file = FileStorage(converted, content_type="audio/wav") os.remove(new_name) file.filename = secure_filename(file.filename) output = upload_file_to_s3(file, app.config["S3_BUCKET"]) #run the pipelines os.system("cd .. && python3 driver.py >> log.txt") ##connect to DB #conn = mysql.connector.Connect(database = db, **conn_kwargs) #c = conn.cursor() #get bullets ,transcription text, image , audio_path c.execute(f""" SELECT bullet FROM summary_bullets WHERE origin = "{file.filename}" ORDER BY bullet_pos""") bullets = [entry[0] for entry in c.fetchall()] c.execute(f""" SELECT full_text , bigram_cloud_url , audio_path FROM conversations WHERE filename = '{file.filename}'""") transcription, image, audio_path = c.fetchall()[0] conn.close() return render_template("success.html", bullets=bullets, image=image, transcription=transcription, file_name=file.filename, audio_path=audio_path) #return str(output) else: return redirect("/")
def handle_img_tags(text, entry_id=None, timestamp=None): """Get image tags from the text. Extract embedded images and save them as attachments""" attachments = [] timestamp = timestamp or datetime.now() try: doc = html.document_fromstring(text) except etree.ParserError: return text, attachments for i, element in enumerate(doc.xpath("//*[@src]")): src = element.attrib['src'].split("?", 1)[0] if src.startswith("data:"): header, data = src[5:].split(",", 1) # TODO: find a safer way filetype, encoding = header.split(";") try: raw_image = decode_base64(data.encode("ascii")) except binascii.Error as e: print("failed to decode image!", e) continue try: # TODO: possible to be more clever about the filename? filename = "inline-{}-{}.{}".format( len(raw_image), i, filetype.split("/")[1].lower()) except IndexError: print("weird filetype!?", filetype) continue file_ = FileStorage(io.BytesIO(raw_image), filename=filename, content_type=filetype) attachment = save_attachment(file_, timestamp, entry_id, embedded=True) # TODO: maybe it would be a better idea to use a URL like # "/attachments/<id>" here, and then just have a redirect # to the real URI? That way we could change the way the files # are stored under the hood without bothering about having # to keep URLs unchanged... src = element.attrib["src"] = url_for("get_attachment", path=attachment.path) if element.getparent().tag == "a": element.getparent().attrib["href"] = src else: parent = element.getparent() wrapper = etree.Element('a') wrapper.attrib['href'] = src loc = parent.index(element) parent.insert(loc, wrapper) parent.remove(element) wrapper.append(element) attachments.append(attachment) # throw in a cleanup here since we've already parsed the HTML html_clean(doc) # remove some evil tags content = '\n'.join((etree.tostring(stree, pretty_print=True, method="xml").decode("utf-8").strip()) for stree in doc[0].iterchildren()) return content, attachments
def handle_file(file: FileStorage, upload_folder=UPLOAD_FOLDER, fps=FPS, resolution=RESOLUTION) -> Dict[str, np.array]: """Make the prediction if the data is coming from an uploaded file. Arguments: - *file*: The file, can be either an image or a video - *upload_folder*: Where the files are temporarly stored Returns: - for an image: a json of format ```json { "image": filename, "detected_trash": [ { "box": [1, 1, 2, 20], "label": "fragments", "score": 0.92 }, { "box": [10, 10, 25, 20], "label": "bottles", "score": 0.75 } ] } ``` - for a video: a json of format ```json { "video_length": 132, "fps": 2, "video_id": "GOPRO1234.mp4", "detected_trash": [ { "label": "bottles", "id": 0, "frame_to_box": { 23: [0, 0, 1, 10], 24: [1, 1, 4, 13] } }, { "label": "fragments", "id": 1, "frame_to_box": { 12: [10, 8, 9, 15] } } ] } ``` Raises: - *NotImplementedError*: If the format of data isn't handled yet """ filename = secure_filename(file.filename) full_filepath = os.path.join(upload_folder, filename) if not os.path.isdir(upload_folder): os.mkdir(upload_folder) if os.path.isfile(full_filepath): os.remove(full_filepath) file.save(full_filepath) file_type = file.mimetype.split("/")[ 0] # mimetype is for example 'image/png' and we only want the image if file_type == "image": image = cv2.imread(full_filepath) # cv2 opens in BGR os.remove(full_filepath) # remove it as we don't need it anymore return { "image": filename, "detected_trash": predict_and_format_image(image) } elif file_type in ["video", "application"]: # splitting video and saving frames folder = os.path.join(upload_folder, "{}_split".format(filename)) if os.path.isdir(folder): shutil.rmtree(folder) os.mkdir(folder) logger.info("Splitting video {} to {}.".format(full_filepath, folder)) split_video(full_filepath, folder, fps=fps, resolution=resolution) image_paths = read_folder(folder) if len(image_paths) == 0: raise ValueError("No output image") # making inference on frames logger.info("{} images to analyze on {} CPUs.".format( len(image_paths), CPU_COUNT)) with multiprocessing.Pool(CPU_COUNT) as p: inference_outputs = list( tqdm( p.imap(process_image, image_paths), total=len(image_paths), )) logger.info("Finish analyzing video {}.".format(full_filepath)) # tracking objects logger.info("Starting tracking.") object_tracker = ObjectTracking(filename, image_paths, inference_outputs, fps=fps) logger.info("Tracking finished.") return object_tracker.json_result() else: raise NotImplementedError(file_type)
def setUp(self): super(SaveAttachmentTest, self).setUp() (filename, file_stream) = get_attachment() self.attachment = FileStorage(stream=file_stream, filename=filename)