def save_csv_files(csv_files: list, using: str, path: str = None) -> None:
"""
Writes the CSV files to the configured storage backend
This could either be the SwiftStorage or a local FileSystemStorage
:param csv_files:
:param using:
:returns None:
"""
storage = _get_storage_backend(using=using)
stored_csv = list()
for csv_file_path in csv_files:
with open(csv_file_path, 'rb') as opened_csv_file:
file_name = os.path.basename(opened_csv_file.name)
file_path = None
if isinstance(storage, SwiftStorage):
file_path = f'{path}{file_name}' if path else file_name
storage.save(name=file_path, content=opened_csv_file)
else:
# Saves the file in a folder structure like "Y/m/d/file_name" for local storage
now = timezone.now()
file_path = f'{now:%Y}/{now:%m}/{now:%d}/{now:%H%M%S%Z}_{file_name}'
storage.save(name=file_path, content=opened_csv_file)
stored_csv.append(os.path.basename(file_path))
return stored_csv
def rotate_zip_files(using: str, max_csv_amount: int = 30) -> None:
"""
rotate csv zip file in the {now:%Y}/{now:%m}/{now:%d} folder
:returns:
"""
storage = _get_storage_backend(using=using)
now = timezone.now()
src_folder = f'{storage.location}/{now:%Y}/{now:%m}/{now:%d}'
if os.path.exists(src_folder):
list_of_files = glob(f'{src_folder}/*.zip', recursive=True)
if len(list_of_files) > max_csv_amount:
list_of_files.sort(key=os.path.getmtime)
for file_to_be_deleted in list_of_files[:len(list_of_files) - max_csv_amount]:
os.remove(file_to_be_deleted)
def test_get_storage_backend(self, mocked_swift_storage):
mocked_swift_storage_instance = mock.Mock()
mocked_swift_storage.return_value = mocked_swift_storage_instance
result = _get_storage_backend(using='datawarehouse')
self.assertEqual(result, mocked_swift_storage_instance)
mocked_swift_storage.assert_called_once_with(
api_auth_url='dwh_auth_url',
api_username='******',
api_key='dwh_password',
tenant_name='dwh_tenant_name',
tenant_id='dwh_tenant_id',
region_name='dwh_region_name',
container_name='dwh_container_name',
auto_overwrite=True)
def zip_csv_files(files_to_zip: list, using: str) -> None:
"""
Writes zip file of the generated csv files in the {now:%Y}/{now:%m}/{now:%d} folder
:returns:
"""
storage = _get_storage_backend(using=using)
now = timezone.now()
src_folder = f'{storage.location}/{now:%Y}/{now:%m}/{now:%d}'
dst_file = os.path.join(src_folder, f'{now:%Y%m%d_%H%M%S%Z}')
with zipfile.ZipFile(f'{dst_file}.zip', 'w') as zipper:
for file in files_to_zip:
if file:
base_file = os.path.basename(file)
zipper.write(filename=os.path.join(src_folder, base_file),
arcname=base_file,
compress_type=zipfile.ZIP_DEFLATED)
# SPDX-License-Identifier: MPL-2.0
# Copyright (C) 2019 - 2021 Gemeente Amsterdam
from django.contrib.gis.db import models
from signals.apps.reporting.utils import _get_storage_backend
storage = _get_storage_backend(using='horeca')
class HorecaCSVExport(models.Model):
class Meta:
ordering = ('-isoyear', '-isoweek', '-created_at')
created_by = models.EmailField(blank=True, null=True)
created_at = models.DateTimeField(auto_now_add=True)
isoweek = models.IntegerField()
isoyear = models.IntegerField()
uploaded_file = models.FileField(upload_to='exports/%Y', storage=storage)
