class SaveAttachmentTest(MailgunTestBase):
def setUp(self):
super(SaveAttachmentTest, self).setUp()
(filename, file_stream) = get_attachment()
self.attachment = FileStorage(stream=file_stream,
filename=filename)
def tearDown(self):
super(SaveAttachmentTest, self).tearDown()
self.attachment.close()
def test_fileallowed(self):
self.assertTrue(self.mailgun.file_allowed('test.txt'))
self.assertFalse(self.mailgun.file_allowed('bob'))
def test_save_attachments(self):
testdir = tempfile.mkdtemp()
self.attachment.seek(0)
filenames = self.mailgun.save_attachments([self.attachment], testdir)
filenames = [os.path.basename(filename) for filename in filenames]
self.assertTrue(set(os.listdir(testdir)) == set(filenames))
self.assertEqual(len(os.listdir(testdir)), 1)
shutil.rmtree(testdir)
with self.assertRaises(OSError):
os.listdir(testdir)
def __init__(self, stream=None, filename=None, name=None,
content_type='application/octet-stream', content_length=-1,
headers=None):
FileStorage.__init__(self, stream, filename, name=name,
content_type=content_type,
content_length=content_length, headers=None)
self.saved = None
def test_read_csv_file_to_dict(self):
csvstream = open("testdata/test.csv", "rb")
csvfile = FileStorage(filename="test.csv", stream=csvstream)
csvfile.mode = "rb"
(success, result_dict, header_list) = read_csv_file_to_dict(csvfile)
self.assertEqual(success, 'test')
self.assertEqual(len(result_dict.keys()), 29)
self.assertEqual(header_list,["__id__", "__filename__", "Text", "Success"])
def test_Transfer_preprocess(transf):
@transf.preprocessor
def to_upper(filehandle, meta):
filehandle.stream = BytesIO(filehandle.stream.read().lower())
return filehandle
source = FileStorage(stream=BytesIO(b'Hello World'))
transf._preprocess(source, {})
source.seek(0)
assert source.read() == b'hello world'
def test_jfu_server(self):
self._test_view_post('frontend.jfu_server',
status='200 OK',
data=['{', '}'],
form=dict(data_file=FileStorage(
StringIO("Hello, world"),
filename='C:\\fakepath\\DoesntExist.txt',
content_type='text/plain; charset=utf-8'), ),
viewopts=dict(item_name='WillBeCreated'),
content_types=[
'application/json',
])
def test_api_extensions_custom_fields_file(self):
"""Test rest api custom file field"""
class SampleSchema(ma.Schema):
"""Sample custom file field schema"""
class Meta:
strict = True
input_file = FileField(allowed_file_extensions=('.pdf', ))
file_storage = FileStorage(filename='test.pdf')
file_datas = SampleSchema().load({'input_file': file_storage}).data
assert file_datas['input_file'] == file_storage
# Errors
# validation: wrong file extension
with pytest.raises(ma.ValidationError):
SampleSchema().load(
{'input_file': FileStorage(filename='not_valid.zip')})
# validation: wrong value type
with pytest.raises(ma.ValidationError):
SampleSchema().load({'input_file': 'wrong_type'})
def rsa_encrypt():
if request.method == 'GET':
return render_template('rsa/encrypt/index.html')
if request.files['pubkey-file']:
rsaPublicKey = RSAPublicKey.from_json(
request.files['pubkey-file'].read())
else:
rsaPublicKey = RSAPublicKey(int(request.form['n']),
int(request.form['e']))
start = default_timer()
n_byte_length = byte_length(rsaPublicKey.n)
chunk_byte_length = n_byte_length - 1
if chunk_byte_length <= 0:
return 'Modulus too small'
message = request.files['message'].read()
plaintext = len(message).to_bytes(4, byteorder) + message
plaintext_chunks = [
plaintext[i:i + chunk_byte_length]
for i in range(0, len(plaintext), chunk_byte_length)
]
ciphertext = bytes()
for chunk in plaintext_chunks:
ciphertext += rsaPublicKey.encrypt(int.from_bytes(
chunk, byteorder)).to_bytes(n_byte_length, byteorder)
end = default_timer()
filename = 'encrypted_' + request.files['message'].filename
ciphertext_file = FileStorage(stream=BytesIO(ciphertext),
filename=filename)
ciphertext_file.seek(0, SEEK_END)
size = ciphertext_file.tell()
ciphertext_file.seek(0)
ciphertext_file.save('static/' + filename)
return render_template('rsa/encrypt/result.html',
plaintext=message,
ciphertext=ciphertext,
time=end - start,
filename=filename,
size=size)
def create(self, user_id: str, file_: FileStorage) -> Optional[File]:
# Save file in local
filename = secure_filename(file_.filename)
temporary_path = FileHelper().get_temporary_path(filename)
file_.save(temporary_path)
# Update file to S3
path = S3().create_file(temporary_path, file_.content_type)
fields = {
'user_id': user_id,
'name': filename,
'media_type': file_.content_type,
'path': path,
'size': os.path.getsize(temporary_path)
}
file_model = self.file_repository.create(fields)
# Delete directory
shutil.rmtree(os.path.dirname(temporary_path))
return file_model
def ecc_decrypt():
if request.method == 'GET':
return render_template('ecc/decrypt/index.html')
message = request.files['message'].read()
kob = int(request.form['koblitz'])
coef = int(request.form['x-coef'])
const = int(request.form['const'])
modulo = int(request.form['modulo'])
if not is_prime(modulo):
return "Error: Modulo must be prime"
curve = EllipticCurve(coef, const, modulo)
prikey = request.files.get('prikey-file', None)
if prikey == None:
prikey = int(request.form['prikey'])
else:
prikey = int(prikey.read())
raw_ciphertext = message.decode().split(' ')
while raw_ciphertext[-1] == '':
del raw_ciphertext[-1]
ciphertext = [Point(0, 0, modulo, point) for point in raw_ciphertext]
start = default_timer()
point_neg = curve.multiply(prikey, ciphertext[-1])
del ciphertext[-1]
point_neg.negate()
plaintext = [curve.add(point, point_neg) for point in ciphertext]
end = default_timer()
string = decode(plaintext, kob)
filename = request.form['filename']
file = FileStorage(stream=BytesIO(string), filename=filename)
file.seek(0, SEEK_END)
size = file.tell()
file.seek(0)
file.save('static/' + filename)
return render_template('ecc/decrypt/result.html',
plaintext=string,
ciphertext=message.decode(),
time=end - start,
filename=filename,
size=size)
def test_file(self):
stream = StringIO.StringIO()
stream.write("""\
Content-Type: application/octet-stream
Content-Length: 1
X-AppEngine-Upload-Creation: 2010-10-01 05:34:00.000000
""")
stream.seek(0)
headers = {}
headers['Content-Type'] = 'image/png; blob-key=foo'
f = FileStorage(stream=stream, headers=headers)
self.assertNotEqual(parse_blob_info(f, 'my_field_name'), None)
def test_invalid_CREATION(self):
stream = StringIO.StringIO()
stream.write("""\
Content-Type: application/octet-stream
Content-Length: 1
X-AppEngine-Upload-Creation: XXX
""")
stream.seek(0)
headers = {}
headers['Content-Type'] = 'image/png; blob-key=foo'
f = FileStorage(stream=stream, headers=headers)
self.assertRaises(blobstore.BlobInfoParseError, parse_blob_info,f, 'my_field_name')
def test_Transfer_validate_raises_with_falsey(transf):
source = FileStorage(stream=BytesIO(), filename='test.conf')
@transf.validator
def bad_validator(fh, m): False
with pytest.raises(UploadError) as excinfo:
transf.save(source, metadata={}, catch_all_errors=False,
destination=lambda *a, **k: None)
expected = "{0!r}({1!r}, {2!r}) returned False".format(bad_validator,
source, {})
assert str(excinfo.value) == expected
def __resource_update_with_upload(cls, context, filename, resource_name,
resource_id):
file_path = os.path.join(os.path.dirname(__file__), filename)
with open(file_path) as f:
file_upload = FileStorage(f)
resource_dict = _get_action('resource_update')(
context, {
'id': resource_id,
'name': resource_name,
'url_type': 'upload',
'resource_type': 'file.upload',
'upload': file_upload
})
return resource_dict
def test_boto3_upload_file_func():
conn = boto3.resource('s3', region_name='us-east-1')
conn.create_bucket(Bucket=s3_bucket)
file = FileStorage(filename='test.txt',
stream=io.BytesIO(b'test'),
content_type='text')
s3_url, unique_secure_filename = upload_file_to_s3(file)
body = conn.Object(
s3_bucket, unique_secure_filename).get()['Body'].read().decode("utf-8")
assert (body == 'test')
def rsa_decrypt():
if request.method == 'GET':
return render_template('rsa/decrypt/index.html')
if request.files['prikey-file']:
rsaPrivateKey = RSAPrivateKey.from_json(
request.files['prikey-file'].read())
else:
rsaPrivateKey = RSAPrivateKey(int(request.form['n']),
int(request.form['d']))
start = default_timer()
n_byte_length = byte_length(rsaPrivateKey.n)
if n_byte_length <= 1:
return 'Modulus too small'
ciphertext = request.files['message'].read()
ciphertext_chunks = [
ciphertext[i:i + n_byte_length]
for i in range(0, len(ciphertext), n_byte_length)
]
plaintext = bytes()
for chunk in ciphertext_chunks:
plaintext += rsaPrivateKey.decrypt(int.from_bytes(
chunk, byteorder)).to_bytes(n_byte_length, byteorder)[:-1]
plaintext = plaintext[4:int.from_bytes(plaintext[:4], byteorder) + 4]
end = default_timer()
filename = 'decrypted_' + request.files['message'].filename
plaintext_file = FileStorage(stream=BytesIO(plaintext), filename=filename)
plaintext_file.seek(0, SEEK_END)
size = plaintext_file.tell()
plaintext_file.seek(0)
plaintext_file.save('static/' + filename)
return render_template('rsa/decrypt/result.html',
plaintext=plaintext,
ciphertext=ciphertext,
time=end - start,
filename=filename,
size=size)
def upload_file():
file = request.files["user_file"]
temp_name = file.filename
file.save(temp_name)
to_convert = AudioSegment.from_file(temp_name)
new_name = temp_name.rsplit(".", 1)[0]+".wav"
os.remove(temp_name)
to_convert.export(new_name, format="wav")
converted = open(new_name, "rb")
file = FileStorage(converted, content_type="audio/wav")
output = upload_file_to_s3(file, app.config["S3_BUCKET"])
return (file.filename)
class SaveAttachmentTest(MailgunTestBase):
def setUp(self):
super(SaveAttachmentTest, self).setUp()
(filename, file_stream) = get_attachment()
self.attachment = FileStorage(stream=file_stream, filename=filename)
def tearDown(self):
super(SaveAttachmentTest, self).tearDown()
self.attachment.close()
def test_fileallowed(self):
self.assertTrue(self.mailgun._is_file_allowed('test.txt'))
self.assertFalse(self.mailgun._is_file_allowed('bob'))
def test_save_attachments(self):
testdir = tempfile.mkdtemp()
self.attachment.seek(0)
filenames = save_attachments([self.attachment], testdir)
filenames = [os.path.basename(filename) for filename in filenames]
self.assertTrue(set(os.listdir(testdir)) == set(filenames))
self.assertEqual(len(os.listdir(testdir)), 1)
shutil.rmtree(testdir)
with self.assertRaises(OSError):
os.listdir(testdir)
def rsa_generate():
if request.method == 'GET':
return render_template('rsa/generate/index.html')
rsaKeyPair = RSAKeyPair(int(request.form['p']), int(request.form['q']),
int(request.form['e']))
filename = request.form['filename']
public_file = FileStorage(stream=BytesIO(
rsaKeyPair.public.to_json().encode()),
filename=filename + '.pub')
public_file.save('static/' + public_file.filename)
private_file = FileStorage(stream=BytesIO(
rsaKeyPair.private.to_json().encode()),
filename=filename + '.pri')
private_file.save('static/' + private_file.filename)
return render_template('rsa/generate/result.html')
def return_werkzeug_filestorage(request, filename):
extension = request.headers['content-type'].split('/')[-1]
if extension not in current_app.config['ALLOWED_EXTENSIONS']:
raise UploadNotAllowed("Unsupported file format")
new_filename = secure_filename(filename + '.' + extension)
try:
tempfile = StringIO(buf=request.content)
except AttributeError:
if hasattr(request.stream, 'getvalue'):
tempfile = StringIO(buf=request.stream.getvalue())
elif hasattr(request.stream, 'read'):
tempfile = StringIO(buf=request.stream.read())
tempfile.name = new_filename
filestorage = FileStorage(tempfile,
filename=new_filename,
content_type=request.headers['content-type'])
return filestorage
def test_handle_post_request_file_other(tmpdir):
filename = "test.pdf"
filepath = os.path.join(tmpdir, filename)
with open(filepath, "w") as f:
f.write("mock data")
m = mock.MagicMock()
files = {
"file": FileStorage(open(filepath, "rb"), content_type='poulet/pdf')
}
m.files = files
upload_folder = os.path.join(tmpdir, "upload_folder")
with pytest.raises(NotImplementedError):
with mock.patch("mot.serving.inference.request", m):
output = handle_post_request(upload_folder=upload_folder)
assert os.path.isdir(
upload_folder
) # the upload_folder should be created by handle post request
def bulkupload():
form = BulkUploadForm()
if form.validate_on_submit():
# print(form.file)
_, file_extension = os.path.splitext(
secure_filename(form.file.data.filename))
temp = ''.join([uuid.uuid4().hex, file_extension])
filename = files.save(FileStorage(form.file.data),
name=secure_filename(temp))
file_url = files.url(filename)
print(filename)
utils.process_bulkupload('C:\\test.xlsx', file_url)
return render_template('base.html')
return render_template('bulkupload.html',
title='New Hire - Bulk Upload',
form=form)
def process_image(requestfile, maxsize=(170, 130)):
fileext = splitext(requestfile.filename)[1].lower()
if fileext not in ['.jpg', '.jpeg', '.png', '.gif']:
raise UploadNotAllowed("Unsupported file format")
img = Image.open(requestfile)
img.load()
if img.size[0] > maxsize[0] or img.size[1] > maxsize[1]:
img.thumbnail(maxsize, Image.ANTIALIAS)
boximg = Image.new('RGBA', maxsize, (255, 255, 255, 0))
boximg.paste(img, ((boximg.size[0] - img.size[0]) / 2,
(boximg.size[1] - img.size[1]) / 2))
savefile = StringIO()
savefile.name = requestfile.filename
boximg.save(savefile)
savefile.seek(0)
return FileStorage(savefile,
filename=requestfile.filename,
content_type=requestfile.content_type)
def test_handle_post_request_file_zip(mock_server_result, tmpdir):
m = mock.MagicMock()
files = {
"file":
FileStorage(open(PATH_TO_TEST_ZIP, "rb"),
content_type='application/zip')
}
m.files = files
m.form = {"fps": 2, "foo": "bar"}
with mock.patch("mot.serving.inference.request", m):
output = handle_post_request(upload_folder=str(tmpdir))
assert len(output["detected_trash"]) == 2
assert "id" in output["detected_trash"][0]
assert "frame_to_box" in output["detected_trash"][1]
for frame, box in output["detected_trash"][1]["frame_to_box"].items():
assert isinstance(frame, int)
assert isinstance(box, list)
assert len(box) == 4
assert output["video_length"] == 6 or output["video_length"] == 7
assert output["fps"] == 2
assert "video_id" in output
def resize_image(requestfile, maxsize=(320, 240)):
fileext = requestfile.filename.split('.')[-1].lower()
if fileext not in current_app.config['ALLOWED_EXTENSIONS']:
raise UploadNotAllowed("Unsupported file format")
img = Image.open(requestfile)
img.load()
if img.size[0] > maxsize[0] or img.size[1] > maxsize[1]:
img.thumbnail(maxsize, Image.ANTIALIAS)
boximg = Image.new('RGBA', (img.size[0], img.size[1]), (255, 255, 255, 0))
boximg.paste(img, (0, 0))
savefile = StringIO()
if fileext in ['jpg', 'jpeg']:
savefile.name = secure_filename(".".join(requestfile.filename.split('.')[:-1]) + ".png")
boximg.save(savefile, format="PNG")
content_type = "image/png"
else:
savefile.name = secure_filename(requestfile.filename)
boximg.save(savefile)
content_type = requestfile.content_type
savefile.seek(0)
return FileStorage(savefile,
filename=savefile.name,
content_type=content_type)
def upload_stream(self,
stream: FileStorage,
upload_dir: str = None,
chunk_size: int = 5 * 1024 * 1024,
progress: bool = False) -> str:
if not upload_dir:
upload_dir = uuid4().hex
ext = os.path.splitext(secure_filename(stream.name))[1]
s3_file_path = os.path.join(upload_dir, uuid4().hex + ext)
bucket = self.resource.Object(self.bucket, s3_file_path)
mp = bucket.initiate_multipart_upload(ACL='public-read')
#'bucket-owner-full-control')
parts_etag = {}
chunk_idx = 1
while True:
chunk = stream.read(chunk_size)
if not chunk:
break
part = mp.Part(chunk_idx)
result = part.upload(Body=chunk)
parts_etag[str(chunk_idx)] = result['ETag']
chunk_idx += 1
yield chunk_idx - 1, self._endpoint(s3_file_path)
mp.complete(
MultipartUpload={
'Parts': [{
'ETag': parts_etag[str(idx)],
'PartNumber': idx
} for idx in range(1, chunk_idx)]
})
yield chunk_idx - 1, self._endpoint(s3_file_path)
def test_handle_post_request_file_video(mock_server_result, tmpdir):
m = mock.MagicMock()
files = {
"file":
FileStorage(open(PATH_TO_TEST_VIDEO, "rb"), content_type='video/mkv')
}
m.files = files
split_frames_folder = os.path.join(
tmpdir, "{}_split".format(os.path.basename(PATH_TO_TEST_VIDEO)))
os.mkdir(split_frames_folder
) # this folder should be deleted by handle post request
with mock.patch("mot.serving.inference.request", m):
output = handle_post_request(upload_folder=str(tmpdir), fps=2)
assert len(output["detected_trash"]) == 2
assert "id" in output["detected_trash"][0]
assert "frame_to_box" in output["detected_trash"][1]
for frame, box in output["detected_trash"][1]["frame_to_box"].items():
assert isinstance(frame, int)
assert isinstance(box, list)
assert len(box) == 4
assert output["video_length"] == 6 or output["video_length"] == 7
assert output["fps"] == 2
assert "video_id" in output
def handle_file(file: FileStorage,
upload_folder: str = UPLOAD_FOLDER,
fps: int = FPS,
resolution: Tuple[int, int] = RESOLUTION,
**kwargs) -> Dict[str, np.array]:
"""Make the prediction if the data is coming from an uploaded file.
Arguments:
- *file*: The file, can be either an image or a video, or a zipped folder
- *upload_folder*: Where the files are temporarly stored
Returns:
- for an image: a json of format
```json
{
"image": filename,
"detected_trash":
[
{
"box": [1, 1, 2, 20],
"label": "fragments",
"score": 0.92
}, {
"box": [10, 10, 25, 20],
"label": "bottles",
"score": 0.75
}
]
}
```
- for a video or a zipped file: a json of format
```json
{
"video_length": 132,
"fps": 2,
"video_id": "GOPRO1234.mp4",
"detected_trash":
[
{
"label": "bottles",
"id": 0,
"frame_to_box": {
23: [0, 0, 1, 10],
24: [1, 1, 4, 13]
}
}, {
"label": "fragments",
"id": 1,
"frame_to_box": {
12: [10, 8, 9, 15]
}
}
]
}
```
Raises:
- *NotImplementedError*: If the format of data isn't handled yet
"""
if kwargs:
logger.warning("Unused kwargs: {}".format(kwargs))
filename = secure_filename(file.filename)
full_filepath = os.path.join(upload_folder, filename)
if not os.path.isdir(upload_folder):
os.mkdir(upload_folder)
if os.path.isfile(full_filepath):
os.remove(full_filepath)
file.save(full_filepath)
file_type = file.mimetype.split("/")[0]
# mimetype is for example 'image/png' and we only want the image
if file_type == "image":
image = cv2.imread(full_filepath) # cv2 opens in BGR
os.remove(full_filepath) # remove it as we don't need it anymore
try:
detected_trash = predict_and_format_image(image)
except ValueError as e:
return {"error": str(e)}
return {"image": filename, "detected_trash": detected_trash}
elif file_type in ["video", "application"]:
folder = None
if file.mimetype == "application/zip":
# zip case
ZipFile(full_filepath).extractall(upload_folder)
dirname = None
with ZipFile(full_filepath, 'r') as zipObj:
listOfFileNames = zipObj.namelist()
for fileName in listOfFileNames:
dirname = os.path.dirname(fileName)
zipObj.extract(fileName, upload_folder)
folder = os.path.join(upload_folder, dirname)
else:
# video case: splitting video and saving frames
folder = os.path.join(upload_folder, "{}_split".format(filename))
if os.path.isdir(folder):
shutil.rmtree(folder)
os.mkdir(folder)
logger.info("Splitting video {} to {}.".format(
full_filepath, folder))
split_video(full_filepath, folder, fps=fps, resolution=resolution)
print("folder:", folder, "uplaod_folder:", upload_folder,
"file.filename:", file.filename)
image_paths = read_folder(folder)
if len(image_paths) == 0:
raise ValueError("No output image")
# making inference on frames
logger.info("{} images to analyze on {} CPUs.".format(
len(image_paths), CPU_COUNT))
try:
with multiprocessing.Pool(CPU_COUNT) as p:
inference_outputs = list(
tqdm(
p.imap(process_image, image_paths),
total=len(image_paths),
))
except ValueError as e:
return {"error": str(e)}
logger.info("Finish analyzing video {}.".format(full_filepath))
# tracking objects
logger.info("Starting tracking.")
object_tracker = ObjectTracking(filename,
image_paths,
inference_outputs,
fps=fps)
tracks = object_tracker.compute_tracks()
logger.info("Tracking finished.")
return object_tracker.json_result(tracks)
else:
raise NotImplementedError(file_type)
def requests(req=None):
if req == 'comment' and request.method == 'POST': #COMMENT REQUESTS
error = False
r = request.json
n = r['name']
e = r['email']
s = r['text']
l = r['level'] + 1 if 'level' in r else '0'
p = r['parent'] if 'parent' in r else '0'
d = r['id']
if n.replace(" ", "") == "": error = True
if s.replace(" ", "") == "": error = True
if e.split('@')[1] == 'undefined' or e.split('@')[1].split(
'.')[1] == 'undefined':
error = True
if error == True: return 'failed: requests not valid'
#processing and limiting text
s = ("<p>%s</p>" % cut(Markup.escape(s), 1500)).replace(
"\n", "</p><p>")
n = cut(n, 60)
e = cut(e, 100)
query_db(
"INSERT INTO comments (Name, Date, Email, Parent, Level, Comment, ArticleID) VALUES (?,DATETIME('NOW'),?,?,?,?,?)",
[n, e, p, l, s, d])
get_db().commit()
return "Nice! Request recieved! And look at you looking into the console. Well here's your reward: raw request data { Name: %s, Email: %s, Comment: %s, Level: %s, Parent: %s, ArticleID: %s }" % (
n, e, s, l, p, d)
if req == 'profile' and request.method == 'POST':
if not session.get('user'): return "Fail.."
r = request.form['name']
d = request.form['desc']
h = request.form['hasimage']
n = session.get('name')
query_db("UPDATE users SET Description=? WHERE Username = ?",
[d, session.get('user')])
get_db().commit()
g = 'image uploaded!'
if h == 'true':
s = FileStorage(stream=request.files['image']).save(
os.path.join(app.config['AUTHOR_FOLDER'],
secure_filename(''.join([n, '.jpg']))))
else:
g = "no image uploaded."
return "Nice! Author Updated! Description changed to '%s', and %s" % (
d, g)
if req == 'article' and request.method == 'POST':
if not session.get('user'): return "Fail.."
a = request.form['article']
h = request.form['html']
id = request.form['id']
t = request.form['title'].lstrip().rstrip()
st = secure_filename(t)
c = request.form['category']
ta = request.form['tags']
hc = request.form['hascover']
cd = request.form['coverdesc']
d = request.form['desc']
i = request.form[
'image'] #DO NOT I REPEAT DO NOT Markup.escape THIS U F*****G DUMB PIECE OF SHIT.
titles = query_db('SELECT ID FROM articles WHERE Title=?', [t])
hoc = request.form['hoc']
p = None
ta = ','.join([e.strip() for e in ta.split(',')])
if id == 'false': #new article
if (len(titles) > 0):
return "ERROR: The title ur trying to use has been used %s time before!" % (
len(titles))
id = query_db('SELECT ID FROM users WHERE Username=?',
[session.get('user')
])[0][0] #set ID from article id to author id
query_db(
"INSERT INTO articles (Title, SafeTitle, AuthorID, Date, Category, Tags, Description, CoverInfo, Pictures, Content, Text) VALUES (?,?,?,DATETIME('NOW'),?,?,?,?,?,?,?)",
[t, st, id, c, ta, d, cd, i, h, a])
else: #update article
if (len(titles) > 0 and id != str(titles[0][0])):
return "ERROR: The renaming of this article will cause a conflict with (%s) other articles!" % (
len(titles))
query_db(
"UPDATE articles SET Title=?, SafeTitle=?, Category=?, Tags=?, Description=?, CoverInfo=?, Pictures=?, Content=?, Text=? WHERE id = ?",
[t, st, c, ta, d, cd, i, h, a, id])
#UPLOAD COVER
if hc == 'true':
pid = str(
query_db("SELECT ID FROM articles WHERE Title=?", [t])[0][0])
p = secure_filename(''.join([pid, '.jpeg']))
pic = os.path.join(app.config['COVER_FOLDER'], p)
s = FileStorage(stream=request.files['cover']).save(pic)
optimize(pic)
query_db("UPDATE articles SET CoverURL=?, CoverInfo=? WHERE id=?",
[p, cd, pid])
#DELETE cover?!?!? D:
if hoc == 'false':
pic = query_db("SELECT CoverURL FROM articles WHERE ID=?",
[str(id)])
try:
os.remove(
os.path.join(app.config['COVER_FOLDER'],
secure_filename(str(pic[0][0]))))
except:
pass
query_db(
"UPDATE articles SET CoverURL='', CoverInfo='' WHERE id=?",
[id])
get_db().commit()
return "Nice! everything went smoothly!"
if req == 'upload' and request.method == 'POST':
if not session.get('user'): return "Fail.."
t = request.form['type']
n = secure_filename(request.form['name'])
folder = app.config['PICTURE_FOLDER']
allfiles = [f for f in listdir(folder) if isfile(join(folder, f))]
for file in allfiles:
if file.split('.')[0] == n:
return "FAIL:, Already a picture named that!"
if t == 'file':
ext = request.form['ext']
if allowed_file(ext):
d = request.files['stuff']
filename = secure_filename(''.join([n, ext]))
pic = os.path.join(folder, filename)
s = FileStorage(stream=d).save(pic)
optimize(pic)
n = filename
else:
return "FAIL:, not right file type!"
elif t == 'url':
d = request.form['stuff']
name = urlparse(d)
notusing, ext = splitext(basename(name.path))
filename = secure_filename(''.join([n, ext]))
if allowed_file(filename):
finalp = os.path.join(folder, filename)
urllib.urlretrieve(d, finalp)
optimize(finalp)
n = filename
else:
return "FAIL:, not right file type!"
else:
return "FAIL:, no images!!"
return "%s" % url_for('pics', set='pictures', pic=n)
if req == 'category' and request.method == 'POST':
if not session.get('user'): return "Fail.."
n = request.form['name'].lstrip().rstrip()
d = request.form['desc']
c = request.form['color']
h = request.form['hasimage']
fname = secure_filename(''.join([n.lower(), '.jpg']))
all = query_db('SELECT Name FROM category')
done = False
for a in all:
if n.lower() == str(a[0]).lower():
done = True
query_db(
"UPDATE category SET Name = ?, Description = ?, Color = ?, Picture = ? WHERE Name = ?",
[n.lower(), d, c, fname, n.lower()])
if done == False:
query_db(
"INSERT INTO category(Name, Description, Color, Picture) VALUES (?,?,?,?)",
[n.lower(), d, c, fname])
get_db().commit()
g = 'image uploaded!'
if h == 'true':
file = os.path.join(app.config['TOPIC_FOLDER'], fname)
s = FileStorage(stream=request.files['image']).save(file)
optimize(file, resize=False)
else:
g = 'no image uploaded.'
return "DONE my good sir! NEW CATEGORY! Name: %s, Color: %s, Description: %s and %s" % (
n, c, d, g)
if req == 'delete_article' and request.method == 'POST':
if not session.get('user'): return "Fail.."
id = request.form['id']
pic = str(
query_db("SELECT CoverURL FROM articles WHERE ID=?",
[str(id)])[0][0])
try:
os.remove(
os.path.join(app.config['COVER_FOLDER'], secure_filename(pic)))
except:
pass
query_db("DELETE FROM articles WHERE ID = ?", [id])
query_db("DELETE FROM comments WHERE ArticleID = ?", [id])
get_db().commit()
return "article with id #%s deleted :(" % (id)
if req == 'posts' and request.method == 'POST':
na = request.form['numthere'] #number of articles already there
n = request.form['numnew'] #number of articles to request
args = request.form['args'].split(',')
#get main info
if args[0] == 'None':
results = postmaker(
query_db(
"SELECT ID, Title, AuthorID, Date, Category, Tags, CoverURL, CoverInfo, Description, Content FROM articles ORDER BY ID DESC LIMIT ?,?",
[na, n]))
elif args[0] == 'Category':
results = postmaker(
query_db(
"SELECT ID, Title, AuthorID, Date, Category, Tags, CoverURL, CoverInfo, Description, Content FROM articles WHERE Category = ? ORDER BY ID DESC LIMIT ?,?",
[args[1], na, n]))
elif args[0] == 'Tags':
results = postmaker(
query_db(
"SELECT ID, Title, AuthorID, Date, Category, Tags, CoverURL, CoverInfo, Description, Content FROM articles WHERE Tags = ? ORDER BY ID DESC LIMIT ?,?",
[args[1], na, n]))
if len(results) == 0:
return "FAIL: NO RESULTS"
#output everything
return render_template('post.html', results=results)
return abort(404)
def image1t():
if os.path.exists('testpic.png'):
os.remove('testpic.jpg')
FileStorage(stream=request.files['image']).save('testpic.jpg')
return jsonify({'done':'done'})
def _save_request_content(destination, stream):
file_storage = FileStorage(stream)
file_storage.save(destination)
def storage_service(job_id, file_name=None):
# def storage_service(job_id, file_name=None):
"""Endpoint serving as the gateway to storage bucket"""
if file_name:
object_name = os.path.join(job_id, file_name)
# print('%s %s' % (request.method, object_name))
if request.method == 'GET':
return_json = False
if 'json' in request.args.keys():
if request.args['json'].lower() == 'true':
return_json = True
if not return_json:
'''send_file_from_directory'''
# file_path_in_cache = storageClient.fget_object(JOB_BUCKET_NAME, object_name)
# file_dir = os.path.dirname(file_path_in_cache)
# return send_from_directory(file_dir, file_path_in_cache.split('/')[-1])
try:
file_path_in_cache = storageClient.fget_object(JOB_BUCKET_NAME, object_name)
file_dir = os.path.dirname(file_path_in_cache)
return send_from_directory(file_dir, file_path_in_cache.split('/')[-1])
except MaxRetryError:
return 'Error in retrieving file\n', 500
except:
return 'File %s does not exist\n' % file_name, 404
else:
try:
file_str = storageClient.get_object(JOB_BUCKET_NAME, object_name)
file_str_json = { object_name: file_str.decode('utf-8') }
# response = make_response(JSONEncoder().encode(file_str_json))
response = make_response( dumps(file_str_json) )
response.headers['Content-Type'] = 'application/json'
http_response_code = 200
# return response, http_response_code
except MaxRetryError:
json_string = {object_name: None}
response = make_response(dumps(json_string))
response.headers['Content-Type'] = 'application/json'
http_response_code = 500
# return response, 500
except Exception as e:
# import traceback
# json_string = {object_name: None, 'error': str(e), 'traceback': traceback.format_exc()}
json_string = {object_name: None}
response = make_response(dumps(json_string))
response.headers['Content-Type'] = 'application/json'
http_response_code = 500
# return response, 500
finally:
return response, http_response_code
elif request.method == 'PUT':
try:
payload = loads(request.data)
except:
payload = request.data
elif request.method == 'POST':
EXTENSION_WHITELIST = set(['pqr', 'pdb', 'in', 'p'])
# pp.pprint(dict(request.files))
# pp.pprint(request.form['job_id'])
# pp.pprint(request.files.keys())
print('request.files keys:')
for key in request.files.keys():
print(' ', key)
try:
file_data = request.files['file_data']
# print(type(file_data), flush=True)
except:
# file_data = BytesIO(request.data)
# print(request.data.decode('utf-8'))
file_data = FileStorage(
stream=BytesIO(request.data),
filename=file_name,
)
# print(type(file_data))
if file_data.filename:
file_name = secure_filename(file_data.filename)
if file_data.filename and file_name:
storageClient.put_object(JOB_BUCKET_NAME, object_name, file_data)
# if file_data.filename and allowed_file(file_name, EXTENSION_WHITELIST):
# # print('uploading to bucket')
# storageClient.put_object(JOB_BUCKET_NAME, object_name, file_data)
# elif not allowed_file(file_name, EXTENSION_WHITELIST):
# return 'Unsupported media type', 415
# time.sleep(1)
return 'Success', 201
elif request.method == 'DELETE':
object_list = []
if file_name is None:
# get list of objects with prefix
# for each object, delete from bucket
job_objects = storageClient.list_objects(JOB_BUCKET_NAME, prefix=job_id+'/')
for obj in job_objects:
object_list.append(obj.object_name)
else:
# delete single object from bucket
object_list.append(object_name)
storageClient.remove_objects(JOB_BUCKET_NAME, object_list)
return 'Success', 204
elif request.method == 'OPTIONS':
options = ['GET', 'PUT', 'POST', 'DELETE']
response = make_response()
response = storage_utils.get_request_options(response, options)
http_response_code = 204
return response, http_response_code
def _wrap_file(filename, content_type):
"""Open the file *filename* and wrap in a FileStorage object."""
assert os.path.isfile(filename), "File does not exist."
return FileStorage(stream=open(filename, 'rb'),
filename=os.path.basename(filename),
content_type=content_type)
def upload_file():
##connect to DB
conn = mysql.connector.Connect(database=db, **conn_kwargs)
c = conn.cursor()
timestamp = time.time()
with open("log.txt", "a") as f:
f.write(f"{timestamp}\n")
# A
if "user_file" not in request.files:
#DEMO
#Get last submitted File
c.execute(""" SELECT filename
FROM conversations
ORDER BY idconversations DESC
LIMIT 1""")
last_audio = c.fetchall()[0][0]
file = File(last_audio)
#get bullets ,transcription text, image , audio_path
c.execute(f""" SELECT bullet
FROM summary_bullets
WHERE origin = "{file.filename}"
ORDER BY bullet_pos""")
bullets = [entry[0] for entry in c.fetchall()]
c.execute(f""" SELECT full_text , bigram_cloud_url , audio_path
FROM conversations
WHERE filename = '{file.filename}'""")
transcription, image, audio_path = c.fetchall()[0]
conn.close()
return render_template("success.html",
bullets=bullets,
image=image,
transcription=transcription,
file_name=file.filename,
audio_path=audio_path)
#KEEP Below after Debugging
return "No user_file key in request.files"
# B
file = request.files["user_file"]
# C.
if file.filename == "":
return "Please select a file"
# D.
if file and allowed_file(file.filename):
#convert file if necessary
if file.filename[-4:] != ".wav":
#temporarily store file to avoid No Header bug
temp_name = file.filename
file.save(temp_name)
#Read and convert bytes then store temp file
to_convert = AudioSegment.from_file(temp_name)
new_name = temp_name.rsplit(".", 1)[0] + ".wav"
os.remove(temp_name)
to_convert.export(new_name, format="wav")
#Open temp file and wrap in werkzeug FileStorage Object, so Boto3 can properly handle it
converted = open(new_name, "rb")
file = FileStorage(converted, content_type="audio/wav")
os.remove(new_name)
file.filename = secure_filename(file.filename)
output = upload_file_to_s3(file, app.config["S3_BUCKET"])
#run the pipelines
os.system("cd .. && python3 driver.py >> log.txt")
##connect to DB
#conn = mysql.connector.Connect(database = db, **conn_kwargs)
#c = conn.cursor()
#get bullets ,transcription text, image , audio_path
c.execute(f""" SELECT bullet
FROM summary_bullets
WHERE origin = "{file.filename}"
ORDER BY bullet_pos""")
bullets = [entry[0] for entry in c.fetchall()]
c.execute(f""" SELECT full_text , bigram_cloud_url , audio_path
FROM conversations
WHERE filename = '{file.filename}'""")
transcription, image, audio_path = c.fetchall()[0]
conn.close()
return render_template("success.html",
bullets=bullets,
image=image,
transcription=transcription,
file_name=file.filename,
audio_path=audio_path)
#return str(output)
else:
return redirect("/")
def handle_img_tags(text, entry_id=None, timestamp=None):
"""Get image tags from the text. Extract embedded images and save
them as attachments"""
attachments = []
timestamp = timestamp or datetime.now()
try:
doc = html.document_fromstring(text)
except etree.ParserError:
return text, attachments
for i, element in enumerate(doc.xpath("//*[@src]")):
src = element.attrib['src'].split("?", 1)[0]
if src.startswith("data:"):
header, data = src[5:].split(",", 1) # TODO: find a safer way
filetype, encoding = header.split(";")
try:
raw_image = decode_base64(data.encode("ascii"))
except binascii.Error as e:
print("failed to decode image!", e)
continue
try:
# TODO: possible to be more clever about the filename?
filename = "inline-{}-{}.{}".format(
len(raw_image), i,
filetype.split("/")[1].lower())
except IndexError:
print("weird filetype!?", filetype)
continue
file_ = FileStorage(io.BytesIO(raw_image),
filename=filename,
content_type=filetype)
attachment = save_attachment(file_,
timestamp,
entry_id,
embedded=True)
# TODO: maybe it would be a better idea to use a URL like
# "/attachments/<id>" here, and then just have a redirect
# to the real URI? That way we could change the way the files
# are stored under the hood without bothering about having
# to keep URLs unchanged...
src = element.attrib["src"] = url_for("get_attachment",
path=attachment.path)
if element.getparent().tag == "a":
element.getparent().attrib["href"] = src
else:
parent = element.getparent()
wrapper = etree.Element('a')
wrapper.attrib['href'] = src
loc = parent.index(element)
parent.insert(loc, wrapper)
parent.remove(element)
wrapper.append(element)
attachments.append(attachment)
# throw in a cleanup here since we've already parsed the HTML
html_clean(doc) # remove some evil tags
content = '\n'.join((etree.tostring(stree, pretty_print=True,
method="xml").decode("utf-8").strip())
for stree in doc[0].iterchildren())
return content, attachments
def handle_file(file: FileStorage,
upload_folder=UPLOAD_FOLDER,
fps=FPS,
resolution=RESOLUTION) -> Dict[str, np.array]:
"""Make the prediction if the data is coming from an uploaded file.
Arguments:
- *file*: The file, can be either an image or a video
- *upload_folder*: Where the files are temporarly stored
Returns:
- for an image: a json of format
```json
{
"image": filename,
"detected_trash":
[
{
"box": [1, 1, 2, 20],
"label": "fragments",
"score": 0.92
}, {
"box": [10, 10, 25, 20],
"label": "bottles",
"score": 0.75
}
]
}
```
- for a video: a json of format
```json
{
"video_length": 132,
"fps": 2,
"video_id": "GOPRO1234.mp4",
"detected_trash":
[
{
"label": "bottles",
"id": 0,
"frame_to_box": {
23: [0, 0, 1, 10],
24: [1, 1, 4, 13]
}
}, {
"label": "fragments",
"id": 1,
"frame_to_box": {
12: [10, 8, 9, 15]
}
}
]
}
```
Raises:
- *NotImplementedError*: If the format of data isn't handled yet
"""
filename = secure_filename(file.filename)
full_filepath = os.path.join(upload_folder, filename)
if not os.path.isdir(upload_folder):
os.mkdir(upload_folder)
if os.path.isfile(full_filepath):
os.remove(full_filepath)
file.save(full_filepath)
file_type = file.mimetype.split("/")[
0] # mimetype is for example 'image/png' and we only want the image
if file_type == "image":
image = cv2.imread(full_filepath) # cv2 opens in BGR
os.remove(full_filepath) # remove it as we don't need it anymore
return {
"image": filename,
"detected_trash": predict_and_format_image(image)
}
elif file_type in ["video", "application"]:
# splitting video and saving frames
folder = os.path.join(upload_folder, "{}_split".format(filename))
if os.path.isdir(folder):
shutil.rmtree(folder)
os.mkdir(folder)
logger.info("Splitting video {} to {}.".format(full_filepath, folder))
split_video(full_filepath, folder, fps=fps, resolution=resolution)
image_paths = read_folder(folder)
if len(image_paths) == 0:
raise ValueError("No output image")
# making inference on frames
logger.info("{} images to analyze on {} CPUs.".format(
len(image_paths), CPU_COUNT))
with multiprocessing.Pool(CPU_COUNT) as p:
inference_outputs = list(
tqdm(
p.imap(process_image, image_paths),
total=len(image_paths),
))
logger.info("Finish analyzing video {}.".format(full_filepath))
# tracking objects
logger.info("Starting tracking.")
object_tracker = ObjectTracking(filename,
image_paths,
inference_outputs,
fps=fps)
logger.info("Tracking finished.")
return object_tracker.json_result()
else:
raise NotImplementedError(file_type)
def setUp(self):
super(SaveAttachmentTest, self).setUp()
(filename, file_stream) = get_attachment()
self.attachment = FileStorage(stream=file_stream,
filename=filename)
