def get_or_create(cls, file, parent=None, metakeys=None, share_with=None):
file.stream.seek(0, os.SEEK_END)
file_size = file.tell()
if file_size == 0:
raise EmptyFileError
sha256 = calc_hash(file.stream, hashlib.sha256(),
lambda h: h.hexdigest())
file_obj = File(dhash=sha256,
file_name=secure_filename(file.filename),
file_size=file_size,
file_type=calc_magic(file.stream),
crc32=calc_crc32(file.stream),
md5=calc_hash(file.stream, hashlib.md5(),
lambda h: h.hexdigest()),
sha1=calc_hash(file.stream, hashlib.sha1(),
lambda h: h.hexdigest()),
sha256=sha256,
sha512=calc_hash(file.stream, hashlib.sha512(),
lambda h: h.hexdigest()),
ssdeep=calc_ssdeep(file.stream))
file_obj, is_new = cls._get_or_create(file_obj,
parent=parent,
metakeys=metakeys,
share_with=share_with)
if is_new:
file.stream.seek(0, os.SEEK_SET)
file.save(file_obj.get_path())
return file_obj, is_new
def get_or_create(cls,
file_name,
file_stream,
parent=None,
metakeys=None,
share_with=None):
file_stream.seek(0, os.SEEK_END)
file_size = file_stream.tell()
if file_size == 0:
raise EmptyFileError
sha256 = calc_hash(file_stream, hashlib.sha256(),
lambda h: h.hexdigest())
file_obj = File(
dhash=sha256,
file_name=secure_filename(file_name),
file_size=file_size,
file_type=calc_magic(file_stream),
crc32=calc_crc32(file_stream),
md5=calc_hash(file_stream, hashlib.md5(), lambda h: h.hexdigest()),
sha1=calc_hash(file_stream, hashlib.sha1(),
lambda h: h.hexdigest()),
sha256=sha256,
sha512=calc_hash(file_stream, hashlib.sha512(),
lambda h: h.hexdigest()),
ssdeep=calc_ssdeep(file_stream),
)
file_obj, is_new = cls._get_or_create(file_obj,
parent=parent,
metakeys=metakeys,
share_with=share_with)
if is_new:
file_stream.seek(0, os.SEEK_SET)
if app_config.mwdb.storage_provider == StorageProviderType.S3:
get_minio_client(
app_config.mwdb.s3_storage_endpoint,
app_config.mwdb.s3_storage_access_key,
app_config.mwdb.s3_storage_secret_key,
app_config.mwdb.s3_storage_region_name,
app_config.mwdb.s3_storage_secure,
).put_object(
app_config.mwdb.s3_storage_bucket_name,
file_obj._calculate_path(),
file_stream,
file_size,
)
else:
with open(file_obj._calculate_path(), "wb") as f:
shutil.copyfileobj(file_stream, f)
file_obj.upload_stream = file_stream
return file_obj, is_new
def get_or_create(
cls,
file_name,
file_stream,
parent=None,
attributes=None,
share_with=None,
analysis_id=None,
tags=None,
):
file_stream.seek(0, os.SEEK_END)
file_size = file_stream.tell()
if file_size == 0:
raise EmptyFileError
sha256 = calc_hash(file_stream, hashlib.sha256(), lambda h: h.hexdigest())
file_obj = File(
dhash=sha256,
file_name=secure_filename(file_name),
file_size=file_size,
file_type=calc_magic(file_stream),
crc32=calc_crc32(file_stream),
md5=calc_hash(file_stream, hashlib.md5(), lambda h: h.hexdigest()),
sha1=calc_hash(file_stream, hashlib.sha1(), lambda h: h.hexdigest()),
sha256=sha256,
sha512=calc_hash(file_stream, hashlib.sha512(), lambda h: h.hexdigest()),
ssdeep=calc_ssdeep(file_stream),
)
file_obj, is_new = cls._get_or_create(
file_obj,
parent=parent,
attributes=attributes,
share_with=share_with,
analysis_id=analysis_id,
tags=tags,
)
# Check if add new alternative file name
if not is_new:
original_filename = secure_filename(file_name)
if (
file_obj.file_name != original_filename
and original_filename not in file_obj.alt_names
):
file_obj.alt_names.append(original_filename)
if is_new:
file_stream.seek(0, os.SEEK_SET)
if app_config.mwdb.storage_provider == StorageProviderType.S3:
get_minio_client(
app_config.mwdb.s3_storage_endpoint,
app_config.mwdb.s3_storage_access_key,
app_config.mwdb.s3_storage_secret_key,
app_config.mwdb.s3_storage_region_name,
app_config.mwdb.s3_storage_secure,
app_config.mwdb.s3_storage_iam_auth,
).put_object(
app_config.mwdb.s3_storage_bucket_name,
file_obj._calculate_path(),
file_stream,
file_size,
)
else:
with open(file_obj._calculate_path(), "wb") as f:
shutil.copyfileobj(file_stream, f)
file_obj.upload_stream = file_stream
return file_obj, is_new