def save_csv_files(csv_files: list, using: str, path: str = None) -> None: """ Writes the CSV files to the configured storage backend This could either be the SwiftStorage or a local FileSystemStorage :param csv_files: :param using: :returns None: """ storage = _get_storage_backend(using=using) stored_csv = list() for csv_file_path in csv_files: with open(csv_file_path, 'rb') as opened_csv_file: file_name = os.path.basename(opened_csv_file.name) file_path = None if isinstance(storage, SwiftStorage): file_path = f'{path}{file_name}' if path else file_name storage.save(name=file_path, content=opened_csv_file) else: # Saves the file in a folder structure like "Y/m/d/file_name" for local storage now = timezone.now() file_path = f'{now:%Y}/{now:%m}/{now:%d}/{now:%H%M%S%Z}_{file_name}' storage.save(name=file_path, content=opened_csv_file) stored_csv.append(os.path.basename(file_path)) return stored_csv
def rotate_zip_files(using: str, max_csv_amount: int = 30) -> None: """ rotate csv zip file in the {now:%Y}/{now:%m}/{now:%d} folder :returns: """ storage = _get_storage_backend(using=using) now = timezone.now() src_folder = f'{storage.location}/{now:%Y}/{now:%m}/{now:%d}' if os.path.exists(src_folder): list_of_files = glob(f'{src_folder}/*.zip', recursive=True) if len(list_of_files) > max_csv_amount: list_of_files.sort(key=os.path.getmtime) for file_to_be_deleted in list_of_files[:len(list_of_files) - max_csv_amount]: os.remove(file_to_be_deleted)
def test_get_storage_backend(self, mocked_swift_storage): mocked_swift_storage_instance = mock.Mock() mocked_swift_storage.return_value = mocked_swift_storage_instance result = _get_storage_backend(using='datawarehouse') self.assertEqual(result, mocked_swift_storage_instance) mocked_swift_storage.assert_called_once_with( api_auth_url='dwh_auth_url', api_username='******', api_key='dwh_password', tenant_name='dwh_tenant_name', tenant_id='dwh_tenant_id', region_name='dwh_region_name', container_name='dwh_container_name', auto_overwrite=True)
def zip_csv_files(files_to_zip: list, using: str) -> None: """ Writes zip file of the generated csv files in the {now:%Y}/{now:%m}/{now:%d} folder :returns: """ storage = _get_storage_backend(using=using) now = timezone.now() src_folder = f'{storage.location}/{now:%Y}/{now:%m}/{now:%d}' dst_file = os.path.join(src_folder, f'{now:%Y%m%d_%H%M%S%Z}') with zipfile.ZipFile(f'{dst_file}.zip', 'w') as zipper: for file in files_to_zip: if file: base_file = os.path.basename(file) zipper.write(filename=os.path.join(src_folder, base_file), arcname=base_file, compress_type=zipfile.ZIP_DEFLATED)
# SPDX-License-Identifier: MPL-2.0 # Copyright (C) 2019 - 2021 Gemeente Amsterdam from django.contrib.gis.db import models from signals.apps.reporting.utils import _get_storage_backend storage = _get_storage_backend(using='horeca') class HorecaCSVExport(models.Model): class Meta: ordering = ('-isoyear', '-isoweek', '-created_at') created_by = models.EmailField(blank=True, null=True) created_at = models.DateTimeField(auto_now_add=True) isoweek = models.IntegerField() isoyear = models.IntegerField() uploaded_file = models.FileField(upload_to='exports/%Y', storage=storage)