def get_or_create(cls,
file_name,
file_stream,
parent=None,
metakeys=None,
share_with=None):
file_stream.seek(0, os.SEEK_END)
file_size = file_stream.tell()
if file_size == 0:
raise EmptyFileError
sha256 = calc_hash(file_stream, hashlib.sha256(),
lambda h: h.hexdigest())
file_obj = File(
dhash=sha256,
file_name=secure_filename(file_name),
file_size=file_size,
file_type=calc_magic(file_stream),
crc32=calc_crc32(file_stream),
md5=calc_hash(file_stream, hashlib.md5(), lambda h: h.hexdigest()),
sha1=calc_hash(file_stream, hashlib.sha1(),
lambda h: h.hexdigest()),
sha256=sha256,
sha512=calc_hash(file_stream, hashlib.sha512(),
lambda h: h.hexdigest()),
ssdeep=calc_ssdeep(file_stream),
)
file_obj, is_new = cls._get_or_create(file_obj,
parent=parent,
metakeys=metakeys,
share_with=share_with)
if is_new:
file_stream.seek(0, os.SEEK_SET)
if app_config.mwdb.storage_provider == StorageProviderType.S3:
get_minio_client(
app_config.mwdb.s3_storage_endpoint,
app_config.mwdb.s3_storage_access_key,
app_config.mwdb.s3_storage_secret_key,
app_config.mwdb.s3_storage_region_name,
app_config.mwdb.s3_storage_secure,
).put_object(
app_config.mwdb.s3_storage_bucket_name,
file_obj._calculate_path(),
file_stream,
file_size,
)
else:
with open(file_obj._calculate_path(), "wb") as f:
shutil.copyfileobj(file_stream, f)
file_obj.upload_stream = file_stream
return file_obj, is_new
def open(self):
"""
Opens the file stream with contents.
File stream must be closed using File.close.
"""
if self.upload_stream is not None:
# If file contents are uploaded in this request,
# try to reuse the existing file instead of downloading it from Minio.
if isinstance(self.upload_stream, io.BytesIO):
return io.BytesIO(self.upload_stream.getbuffer())
else:
dupfd = os.dup(self.upload_stream.fileno())
stream = os.fdopen(dupfd, "rb")
stream.seek(0, os.SEEK_SET)
return stream
if app_config.mwdb.storage_provider == StorageProviderType.S3:
return get_minio_client(
app_config.mwdb.s3_storage_endpoint,
app_config.mwdb.s3_storage_access_key,
app_config.mwdb.s3_storage_secret_key,
app_config.mwdb.s3_storage_region_name,
app_config.mwdb.s3_storage_secure,
app_config.mwdb.s3_storage_iam_auth,
).get_object(app_config.mwdb.s3_storage_bucket_name, self._calculate_path())
elif app_config.mwdb.storage_provider == StorageProviderType.DISK:
return open(self._calculate_path(), "rb")
else:
raise RuntimeError(
f"StorageProvider {app_config.mwdb.storage_provider} is not supported"
)
def get_or_create(
cls,
file_name,
file_stream,
parent=None,
attributes=None,
share_with=None,
analysis_id=None,
tags=None,
):
file_stream.seek(0, os.SEEK_END)
file_size = file_stream.tell()
if file_size == 0:
raise EmptyFileError
sha256 = calc_hash(file_stream, hashlib.sha256(), lambda h: h.hexdigest())
file_obj = File(
dhash=sha256,
file_name=secure_filename(file_name),
file_size=file_size,
file_type=calc_magic(file_stream),
crc32=calc_crc32(file_stream),
md5=calc_hash(file_stream, hashlib.md5(), lambda h: h.hexdigest()),
sha1=calc_hash(file_stream, hashlib.sha1(), lambda h: h.hexdigest()),
sha256=sha256,
sha512=calc_hash(file_stream, hashlib.sha512(), lambda h: h.hexdigest()),
ssdeep=calc_ssdeep(file_stream),
)
file_obj, is_new = cls._get_or_create(
file_obj,
parent=parent,
attributes=attributes,
share_with=share_with,
analysis_id=analysis_id,
tags=tags,
)
# Check if add new alternative file name
if not is_new:
original_filename = secure_filename(file_name)
if (
file_obj.file_name != original_filename
and original_filename not in file_obj.alt_names
):
file_obj.alt_names.append(original_filename)
if is_new:
file_stream.seek(0, os.SEEK_SET)
if app_config.mwdb.storage_provider == StorageProviderType.S3:
get_minio_client(
app_config.mwdb.s3_storage_endpoint,
app_config.mwdb.s3_storage_access_key,
app_config.mwdb.s3_storage_secret_key,
app_config.mwdb.s3_storage_region_name,
app_config.mwdb.s3_storage_secure,
app_config.mwdb.s3_storage_iam_auth,
).put_object(
app_config.mwdb.s3_storage_bucket_name,
file_obj._calculate_path(),
file_stream,
file_size,
)
else:
with open(file_obj._calculate_path(), "wb") as f:
shutil.copyfileobj(file_stream, f)
file_obj.upload_stream = file_stream
return file_obj, is_new