def test_property_to_organize_sha512sums_from_the_dataset_table_files(
self):
table_file_1 = baker.make(TableFile,
filename="sample_02.csv",
table=self.tables[0])
table_file_2 = baker.make(TableFile,
filename="csv_data_02.csv",
table=self.tables[1])
hasher = hashlib.sha512()
expected_content = ""
for table_file in [table_file_2, table_file_1]:
hasher.update(expected_content.encode())
expected_content += f"{table_file.sha512sum} {table_file.filename}\n"
expected_url = f"{settings.AWS_S3_ENDPOINT_URL}{settings.MINIO_STORAGE_DATASETS_BUCKET_NAME}/{self.dataset.slug}/{settings.MINIO_DATASET_SHA512SUMS_FILENAME}"
sha512sums = self.dataset.sha512sums
assert settings.MINIO_DATASET_SHA512SUMS_FILENAME == sha512sums.filename
assert expected_content == sha512sums.content
assert hasher.hexdigest() == sha512sums.sha512sum
assert expected_url == sha512sums.file_url
assert human_readable_size(len(
expected_content.encode())) == sha512sums.readable_size
def sha512sums(self):
"""
Return a TableFile-like object with the SHA512SUMS from all tables
"""
FileInfo = namedtuple(
"FileInfo",
("filename", "file_url", "readable_size", "sha512sum", "content"))
sha_sum = hashlib.sha512()
content = ""
for table_file in self.tables_files:
sha_sum.update(content.encode())
content += f"{table_file.sha512sum} {table_file.filename}\n"
fname = settings.MINIO_DATASET_SHA512SUMS_FILENAME
url = f"{settings.AWS_S3_ENDPOINT_URL}{settings.MINIO_STORAGE_DATASETS_BUCKET_NAME}/{self.slug}/{fname}"
return FileInfo(
filename=fname,
file_url=url,
readable_size=human_readable_size(len(content.encode())),
sha512sum=sha_sum.hexdigest(),
content=content,
)
def readable_size(self):
return human_readable_size(int(self.size))