def test_attachment_from_file_storage():
blob = randStream()
filename = randUnicode()
content_type = "y/x"
metadata = AttachmentMetadata(filename, content_type)
_file = FileStorage(stream=blob, filename=filename, content_type=content_type)
input_file = copy.deepcopy(_file)
attachment = Attachment.fromFileStorage(input_file, metadata)
assert attachment.content.content == _file.read()
assert attachment.metadata.filename == filename
assert attachment.metadata.content_type == content_type
assert attachment.metadata.content_disposition == AttachmentType.ATTACHMENT
# test content_type correction
blob = randStream()
filename = randUnicode()
content_type = "badcontenttype"
metadata = AttachmentMetadata(filename, content_type)
_file = FileStorage(stream=blob, filename=filename, content_type=content_type)
input_file = copy.deepcopy(_file)
attachment = Attachment.fromFileStorage(input_file, metadata)
assert attachment.content.content == _file.read()
assert attachment.metadata.filename == filename
assert attachment.metadata.content_type == "application/octet-stream"
assert attachment.metadata.content_disposition == AttachmentType.ATTACHMENT
def save_image(file: FileStorage):
thumbnail_size = 240, 240
directory = app.config["IMAGE_DIRECTORY"]
# Compute hash
file_hash = hashlib.md5(file.read()).hexdigest()
file.seek(0)
file_name = file_hash + ".jpg"
file_path = os.path.join(directory, file_name)
thumbnail_name = file_hash + "_thumb.jpg"
thumbnail_path = os.path.join(directory, thumbnail_name)
image = Image.open(file)
size = image.size
if not os.path.isfile(file_path):
# save as jpg for size efficiency
image.save(file_path)
print("Saved image as", file_path)
if not os.path.isfile(thumbnail_path):
# create thumbnail
image = Image.open(os.path.join(directory, file_name))
image.thumbnail(thumbnail_size, Image.ANTIALIAS)
image.save(os.path.join(directory, thumbnail_name))
print("saved thumbnail as", thumbnail_path)
store_image(file_hash, size)
def resumable_executor(self, data, base):
stream = binascii.a2b_base64(data['stream'])
file_ = base64.b64edecode(stream)
file_ = io.BytesIO(file_)
file_ = FileStorage(file_)
file_.seek(0, os.SEEK_END)
file_length = file_.tell()
file_.seek(0)
data = file_.read(1024)
count = 0
filename = data['stream']
with open(base + "/" + filename, "a") as f:
while data:
count += 1
f.write(data)
self.update_status(state="PROGRESS",
meta={
'current': count * 1024,
'total': file_length
})
return {
'current': 100,
'total': 100,
'status': 'Task completed!',
'result': "Uploaded"
}
def save_image(file: FileStorage):
thumbnail_size = 240, 240
directory = app.config["IMAGE_DIRECTORY"]
# Compute hash
file_hash = hashlib.md5(file.read()).hexdigest()
file.seek(0)
file_name = file_hash + ".jpg"
file_path = os.path.join(directory, file_name)
thumbnail_name = file_hash + "_thumb.jpg"
thumbnail_path = os.path.join(directory, thumbnail_name)
image = Image.open(file)
size = image.size
if not os.path.isfile(file_path):
# save as jpg for size efficiency
image.save(file_path)
print("Saved image as", file_path)
if not os.path.isfile(thumbnail_path):
# create thumbnail
image = Image.open(os.path.join(directory, file_name))
image.thumbnail(thumbnail_size, Image.ANTIALIAS)
image.save(os.path.join(directory, thumbnail_name))
print("saved thumbnail as", thumbnail_path)
store_image(file_hash, size)
def do_image(self, path):
common.opening_label('# IMAGE ANALYSIS')
# Instantiate a image client
ii = ImageIntelligence()
client = ii.get_client()
resp = None
# Get file by URI, else upload local file
if path.startswith('http') or path.startswith('gs:'):
resp = ii.analyze_image(path, client)
else:
file_exists = common.check_file_exists(path)
if not file_exists:
return
with open(path, 'rb') as fp:
img = FileStorage(fp)
url = common.upload_file(img.read(), img.filename,
img.content_type,
config.IMAGE_STORAGE_BUCKET)
if url is not '':
resp = ii.analyze_image(url, client)
else:
return
# Run all ImageIntelligence modules and print results
if resp is not None:
ii.run_all(resp)
def object_import(request: dict, file: FileStorage) -> str:
if request["replace"]:
delete_all("Device")
result = "Topology successfully imported."
if allowed_file(secure_filename(file.filename), {"xls", "xlsx"}):
book = open_workbook(file_contents=file.read())
for obj_type in ("Device", "Link"):
try:
sheet = book.sheet_by_name(obj_type)
except XLRDError:
continue
properties = sheet.row_values(0)
for row_index in range(1, sheet.nrows):
prop = dict(zip(properties, sheet.row_values(row_index)))
try:
factory(obj_type, **prop).serialized
except Exception as e:
info(f"{str(prop)} could not be imported ({str(e)})")
result = "Partial import (see logs)."
db.session.commit()
if request["update_pools"]:
for pool in fetch_all("Pool"):
pool.compute_pool()
db.session.commit()
return result
def process_file_for_upload(self, _user_id: str, _book_id: str,
f_obj: FileStorage, _privacy_scope: str):
print(
f"INFO: user: {_user_id} for book: {_book_id} for file: {f_obj} and privacy: {_privacy_scope}"
)
self._file_extension = self.verify_file_extension(f_obj.filename)
if isinstance(self._file_extension, list):
return self._file_extension
self._file_stream = f_obj.read()
_file_size_in_mb = BookUtils.convert_bytes_to_mb(self._get_file_size())
if _file_size_in_mb > BookEnums.MAX_FILE_SIZE_ALLOWED_IN_MB_FOR_DOC.value:
return [ErrorEnums.MAX_SIZE_EXCEED_ERROR_FOR_DOC.value, 413]
self._book = BookDatabaseService.find_active_book_by_id(_book_id)
if not self._book:
return [ErrorEnums.NO_BOOK_FOUND_ERROR.value, 404]
if self._book.created_by != _user_id:
return [ErrorEnums.BOOK_OWNER_NOT_MATCH_ERROR.value, 404]
if not self._book:
return [ErrorEnums.NO_BOOK_FOUND_ERROR.value, 404]
if not self._book.is_active:
return [ErrorEnums.INACTIVE_BOOK_ERROR.value, 400]
self._privacy_scope = _privacy_scope.upper()
if self._privacy_scope not in BookEnums.PRIVACY_SCOPES_FOR_DOCUMENT.value:
return [{'error': 'Please provide a valid privacy type.'}, 400]
self._user_id = _user_id
self._file_obj = self.bytes_to_obj()
self._file_name = self.get_secure_filename(f_obj.filename,
self._user_id,
self._file_extension)
self._repo_key = self.make_repo_key()
_upload_response = self.upload_file()
response = self.update_file_record(_upload_response)
if isinstance(response, list):
return response
return [{'response': {'updatedBookDetails': book_dto(response)}}, 200]
def do_image(self, path):
# Instantiate a speech client
client = ImageAnnotatorClient()
resp = None
# Get file by URI, else upload local file
if path.startswith('http') or path.startswith('gs:'):
resp = analyze_image(path, client)
else:
# TODO: Check if the file exists
with open(path, 'rb') as fp:
img = FileStorage(fp)
url = upload_file(img.read(), img.filename, img.content_type, IMAGE_STORAGE_BUCKET)
if url is not '':
resp = analyze_image(url, client)
# Process and extract as much data as possible. We upload to limit the
# number of API calls.
if resp is not None:
print_face_details(resp)
print_logo_details(resp)
print_text_details(resp)
print_label_details(resp)
print_landmark_details(resp)
print_web_details(resp)
print_safe_search_details(resp)
else:
print('[!] Error processing image...')
def saml2_file_dict():
with open('fixtures/saml2_trace.xml') as xml:
file = FileStorage(content_type='application/xml',
name='saml_file',
stream=xml)
data = file.read()
return dict(xmltodict.parse(data))
def test_open_image():
f = open('..\\hue.jpg', 'rb')
file = FileStorage(f)
file.save("file_storage_image.jpg")
content = file.read()
print(">> Content of file storgae : " + str(content))
f.close()
def make_from_file_in_memory(image_data: FileStorage):
image_bytes = image_data.read()
image = PIL.Image.open(BytesIO(image_bytes))
image_array = np.array(image)
return {
"image": image,
"image_array": image_array,
"image_bytes": image_bytes,
}
def __init__(self, *, metadata_url: str, name: str, logo: FileStorage,
description: str) -> None:
super().__init__(
metadata_url=metadata_url,
name=name,
description=description,
logo=logo.read(),
)
self._generate_keys()
def test_request_multipart():
f = open('..\\hue.jpg', 'rb')
file = FileStorage(f)
temp = file.read()
print(">> Client content : " + str(temp))
response = requests.post('http://127.0.0.1:5000/roomtemp',
files=dict(file=file))
print(">> Request response : " + str(response))
f.close()
def __init__(self, fs: FileStorage) -> None:
# Initialize the file proxy object from a Werkzeug FileStorage instance,
# cf. https://werkzeug.palletsprojects.com/en/1.0.x/datastructures/#werkzeug.datastructures.FileStorage
self._mimetype = fs.mimetype
self._mimetype_params = fs.mimetype_params
self._content_type = fs.content_type
# !!! Note: this reads the entire file stream into memory.
# !!! A fancier method using temporary files could be applied here
# !!! when and if needed.
self._bytes = fs.read()
def file2image(file: FileStorage) -> Image.Image:
logger.info(f'Got file {file.filename} of {file.mimetype}')
file.stream.seek(0)
byte_stream = io.BytesIO(file.read())
logger.info(f'File Size: {byte_stream.getbuffer().nbytes}')
file.close()
image = Image.open(byte_stream)
image = image.convert('RGB')
return image
def read_image(image: FileStorage) -> np.ndarray:
"""
preprocessed the image into designated size tensor
:param image: origin image (FileStorage type)
:return: preprocessed ndarray of image
"""
stream = image.read()
image = cv2.imdecode(np.asarray(bytearray(stream)), 0)
resized = cv2.resize(image, TARGET_SIZE, interpolation=cv2.INTER_AREA)
return resized
def test_image2nuparray():
f = open('..\\hue.jpg', 'rb')
file = FileStorage(f)
binary_array = file.read()
print(">> Content in test : " + str(binary_array))
img = Image.open(io.BytesIO(binary_array))
arr = np.asarray(img)
plotData(arr)
print(">> NP array " + str(arr))
f.close()
def test_file_object_to_string_base64(self):
# Given a file
with open(TEST_FILE_LOCATION, "rb") as io:
file = FileStorage(stream=io, filename="tests.xlsx")
# When it is converted to a string
f = file.read()
file_as_string = convert_file_object_to_string_base64(f)
# Then it is a string
self.assertEqual(type(file_as_string), str)
def set_file(file_item: FileStorage) -> str:
"""Set temporary file to database and returns link."""
data = file_item.read()
file = {
'file': base64.b64encode(data),
'filename': file_item.filename,
'link': uuid.uuid4()
}
file_model = _get_file_model()
created_file = file_model.create(vals_list=[file])
return format_link(created_file.link)
def calculate_checksum(fs: FileStorage) -> str:
checksum = hashlib.md5()
# Process in chunks to avoid reading the whole file into memory at once.
while True:
data = fs.read(1024)
if not data:
break
checksum.update(data)
# Rewind the file before returning.
fs.seek(0)
return checksum.hexdigest()
def do_audio(self, arg):
common.opening_label('# AUDIO TRANSCRIPTION')
# Split the arg variable
x = arg.split(' ')
code = x[0]
path = x[1]
# Instantiate a speech client
ai = AudioIntelligence()
client = ai.get_client()
# Determine if the file is remote or local. If the file is large,
# then use try_long_run. File must be flac if remote, and in the
# bucket gs://bucket_name/file.flac
if path.startswith('gs:') and path.endswith('flac'):
try:
ai.analyze_audio(path, client, code)
except:
ai.try_long_run(path, client, code)
else:
fileExists = common.check_file_exists(path)
if not fileExists:
return
# Convert the audio to FLAC and upload to audio bucket. Assuming
# the file is not FLAC here. Save the file in the same path.
base = os.path.splitext(path)[0]
new_path = base + '.flac'
common.convert_to_flac(path, new_path)
# Open and upload the file to the storage bucket. Split resulting
# URL to get the filename and use the same gs:// method to analyze
# the FLAC audio file.
with open(new_path, 'rb') as fp:
audio = FileStorage(fp)
url = common.upload_file(audio.read(), audio.filename,
audio.content_type,
config.AUDIO_STORAGE_BUCKET)
if url is not '':
gs_file = url.split("/")[-1]
try:
ai.analyze_audio(
'gs://' + Config.AUDIO_STORAGE_BUCKET + '/' +
gs_file, client, code)
except:
ai.try_long_run(
'gs://' + Config.AUDIO_STORAGE_BUCKET + '/' +
gs_file, client, code)
else:
return
def read(file_handler: FileStorage):
"""
read contents from handler
@Returns:
* [bytes] the content of file in bytes
"""
if file_handler is None:
return None
if isinstance(file_handler, FileStorage):
return file_handler.read()
# else
return None
def upload_pba_file(file_storage: FileStorage, is_linux=True):
"""
Uploads PBA file to island's file system
:param request_: Request object containing PBA file
:param is_linux: Boolean indicating if this file is for windows or for linux
:return: filename string
"""
filename = secure_filename(file_storage.filename)
file_contents = file_storage.read()
PostBreachFilesService.save_file(filename, file_contents)
ConfigService.set_config_value(
(PBA_LINUX_FILENAME_PATH
if is_linux else PBA_WINDOWS_FILENAME_PATH), filename)
return filename
def file2image(file: FileStorage) -> Image.Image:
"""
Args:
file (FileStorage): a flask file object received from the request
Returns:
(PIL.Image.Image)
"""
logger.info(f"Got file {file.filename} of {file.mimetype}")
file.stream.seek(0)
byte_stream = io.BytesIO(file.read())
logger.info(f"File Size: {byte_stream.getbuffer().nbytes}")
file.close()
image: Image.Image = Image.open(byte_stream)
image: Image.Image = image.convert("RGB")
return image
def post(self, utt_id: str, file: FileStorage):
# first checking the audio file
payload: bytes = file.read()
mime_type = magic.from_buffer(payload, mime=True)
mime_main_type = mime_type.split("/")[0]
buffer = BytesIO(payload)
if mime_main_type == "audio":
try:
try:
AudioSegment.from_file(buffer)
except MemoryError:
AudioSegment.from_file_using_temporary_files(buffer)
except Exception as e:
logging.warning(f"Error while decoding audio file : {str(e)}")
return abort("Invalid audio file")
else:
logging.warning(f"Invalid submission with mime type : {mime_type}")
raise abort("Invalid mime type for audio file")
def file2audiotensor(file: FileStorage) -> Tensor:
"""
Reads the FileStorage object into Bytes, then converts it to a normalized torch tensor
Args:
file: the FileStorage object from flask Request
Returns:
(Tensor): the normalized tensor representation of the audio
"""
logger.info(f"Got file {file.filename} of {file.mimetype}")
file.stream.seek(0)
byte_stream = io.BytesIO(file.read())
logger.info(f"File Size: {byte_stream.getbuffer().nbytes}")
file.close()
# convert webm to wav
process = (
ffmpeg
.input('pipe:0')
.output('pipe:1', format='wav')
.run_async(pipe_stdin=True, pipe_stdout=True, pipe_stderr=True, quiet=True, overwrite_output=True)
)
output, err = process.communicate(input=byte_stream.read())
riff_chunk_size = len(output) - 8
# Break up the chunk size into four bytes, held in b.
q = riff_chunk_size
b = []
for i in range(4):
q, r = divmod(q, 256)
b.append(r)
# Replace bytes 4:8 in proc.stdout with the actual size of the RIFF chunk.
riff = output[:4] + bytes(b) + output[8:]
data, _ = sf.read(io.BytesIO(riff))
waveform: Tensor = torch.from_numpy(data).float()
return waveform.unsqueeze(0)
def decode_csv_file(file: FileStorage) -> str:
user_error = BadRequest(
"Please submit a valid CSV."
" If you are working with an Excel spreadsheet,"
" make sure you export it as a .csv file before uploading")
# In Windows, CSVs have mimetype application/vnd.ms-excel
if file.mimetype not in ["text/csv", "application/vnd.ms-excel"]:
raise user_error
try:
file_bytes = file.read()
return str(file_bytes.decode("utf-8-sig"))
except UnicodeDecodeError as err:
try:
detect_result = chardet.detect(file_bytes)
if not detect_result["encoding"]:
raise user_error from err
return str(file_bytes.decode(detect_result["encoding"]))
except Exception:
raise user_error from err
def save_image(uploaded_image: FileStorage, categorised_tags: Dict):
image_hex_bytes = uploaded_image.read()
image = _hex_to_image(image_hex_bytes)
uploaded_image.seek(0)
exif_data = _format_exif_data(image.getexif())
updated_categorised_tags = update_categorised_tags_with_exif_data(exif_data, categorised_tags)
image_name = generate_hashed_image_name(uploaded_image.filename, exif_data)
if is_production():
save_image_file_to_s3_bucket(uploaded_image, image_name)
else:
save_image_file_locally(image, image_name)
db_image = Image(name=image_name, exif_data=exif_data)
db_image.add_tags(updated_categorised_tags)
db.session.add(db_image)
db.session.commit()
def create_file(cls, uploaded_file: FileStorage, user_id: int):
filename = secure_filename(f'{user_id}_{uploaded_file.filename}')
if db.session.query(File).filter_by(
owner_id=user_id, title=filename).first() is not None:
raise FileAlreadyExistsError(filename)
blob = uploaded_file.read()
# FileStorage class (which is the class to handle uploaded file in Flask)
# points to end of file after every action (saving or reading).
uploaded_file.stream.seek(0)
size = len(blob)
f_hash = sha256(blob).hexdigest()
# A way of transactional insert
try:
cls.__save_file_db(f_hash, filename, size, user_id)
except Exception as e:
st.logger.exception(e)
raise FileInsertionError(filename)
else:
cls.__save_file_disk(uploaded_file, filename)
db.session.commit()
def filestorage2ndarray(file: FileStorage) -> np.ndarray:
filestr = file.read()
img_np = np.fromstring(filestr, np.uint8)
img_np = cv2.imdecode(img_np, cv2.IMREAD_UNCHANGED)
return img_np
def __init__(self, to_extract: FileStorage):
self.to_extract = to_extract
cursor_position = to_extract.tell()
self.file_content = to_extract.read()
to_extract.seek(cursor_position)
self.filename = to_extract.filename
def do_video(self, arg):
common.opening_label('# VIDEO ANALYSIS')
# Split the arg variable
x = arg.split(' ')
code = x[0]
path = x[1]
# Instantiate a videointelligence client
vi = VideoIntelligence()
client = vi.get_client()
# If the video is already uploaded, process the file. If the file is
# local, upload the video and process.
if path.startswith('gs:'):
vi.analyze_video(path, client)
else:
file_exists = common.check_file_exists(path)
if not file_exists:
return
with open(path, 'rb') as fp:
video = FileStorage(fp)
url = common.upload_file(video.read(), video.filename,
video.content_type,
config.VIDEO_STORAGE_BUCKET)
if url is not '':
gs_file = url.split("/")[-1]
vi.analyze_video(
'gs://' + config.VIDEO_STORAGE_BUCKET + '/' + gs_file,
client)
else:
return
# Convert the video file to a mp4, then convert the video file
# to FLAC. Unable to directly convert some formats to FLAC, so
# first convert to mp4, then FLAC.
base = os.path.splitext(path)[0]
# There is a bug with some video to audio conversions. This primarily
# occurs when converting avi to mp4, then from mp4 to FLAC.
# TODO: Fix alternative video formats to FLAC
if path.endswith('.mp4') or path.endswith('.MP4'):
new_path = path
else:
new_path = base + '.mp4'
common.convert_to_mp4(path, new_path)
# Convert the mp4 to FLAC
new_path_audio = base + '.flac'
common.convert_to_audio(new_path, new_path_audio)
# Analyze the FLAC and transcribe.
with open(new_path_audio, 'rb') as fp:
audio = FileStorage(fp)
url = common.upload_file(audio.read(), audio.filename,
audio.content_type,
config.AUDIO_STORAGE_BUCKET)
if url is not '':
ai = AudioIntelligence()
client = ai.get_client()
gs_file = url.split("/")[-1]
try:
ai.analyze_audio(
'gs://' + config.AUDIO_STORAGE_BUCKET + '/' +
gs_file, client, code)
except:
ai.try_long_run(
'gs://' + config.AUDIO_STORAGE_BUCKET + '/' +
gs_file, client, code)
else:
return
