def _on_after_delete(self, **kwargs): """_after_delete() hook. """ # Remove file from the storage storage_path = self.f_get('storage_path') if _path.exists(storage_path): _unlink(storage_path)
def rm(self, pool: str, key: str): """Remove a value from the pool """ try: _unlink(self._get_key_path(pool, key)) except FileNotFoundError: pass
def cleanup_files(root_path: str, ttl: int) -> tuple: """Remove obsolete files and empty directories """ now = _current_time() success = [] failed = [] for root, d_names, f_names in _walk(root_path): for f_name in f_names: f_path = _path.join(root, f_name) m_time = _path.getmtime(f_path) if m_time + ttl < now: try: _unlink(f_path) success.append(f_path) except Exception as e: failed.append((f_path, e)) # Remove empty directories for d_name in d_names: d_path = _path.join(root, d_name) if not _listdir(d_path): _rmdir(d_path) return success, failed
def _on_pre_save(self, **kwargs): """Hook. """ super()._on_pre_save(**kwargs) # Read EXIF from file with open(self.f_get('storage_path'), 'rb') as f: exif = _exifread.process_file(f, details=False) entity_exif = {} for k, v in exif.items(): if not k.startswith('Thumbnail'): entity_exif[k] = str(v) self.f_set('exif', entity_exif) # Open image for processing image = _PILImage.open( self.f_get('storage_path')) # type: _PILImage.Image # Rotate image if 'Image Orientation' in exif: orientation = str(exif['Image Orientation']) rotated = None if orientation == 'Rotated 90 CCW': rotated = image.rotate(90, _PILImage.BICUBIC, True) elif orientation == 'Rotated 90 CW': rotated = image.rotate(-90, _PILImage.BICUBIC, True) elif orientation == 'Rotated 180': rotated = image.rotate(180, _PILImage.BICUBIC, True) if rotated: rotated.save(self.f_get('storage_path')) image = rotated # Convert BMP and JPEG2000 to JPEG if image.format in ('BMP', 'JPEG2000'): current_storage_path = self.f_get('storage_path') # Change path and MIME info new_path = _re.sub('\.\w+$', '.jpg', self.f_get('path')) if not new_path.endswith('.jpg'): new_path += '.jpg' self.f_set('path', new_path) self.f_set('mime', 'image/jpeg') image.save(self.f_get('storage_path')) _unlink(current_storage_path) self.f_set('width', image.size[0]) self.f_set('height', image.size[1]) image.close()
def cleanup(self, pool: str): """Cleanup outdated items from the cache """ for root, dirs, files in _walk(_path.join(self._path, self._server_name, pool), topdown=False): for name in files: f_path = _path.join(root, name) with open(f_path, 'rb') as f: expires = _pickle_load(f.read())['e'] if expires and expires <= _time(): try: _unlink(f_path) except FileNotFoundError: pass
def _load(self, pool: str, key: str) -> dict: """Load an item from the pool """ f_path = self._get_key_path(pool, key) try: with open(f_path, 'rb') as f: return _pickle_load(f.read()) except FileNotFoundError: raise _error.KeyNotExist(pool, key) except (EOFError, _UnpicklingError): _unlink(f_path) raise _error.KeyNotExist(pool, key)
def data(): path = CONFIG['paths']['registry'] # downloads from https://iati-data-dump.codeforiati.org data_url = 'https://gitlab.com/codeforIATI/iati-data/-/archive/main/iati-data-main.zip' shutil.rmtree(path, ignore_errors=True) makedirs(path) zip_filepath = join(path, 'iati_dump.zip') logging.getLogger(__name__).info('Downloading all IATI registry data...') request = requests.get(data_url, stream=True) with open(zip_filepath, 'wb') as handler: shutil.copyfileobj(request.raw, handler) logging.getLogger(__name__).info('Unzipping data...') with zipfile.ZipFile(zip_filepath, 'r') as zip_ref: zip_ref.extractall(path) logging.getLogger(__name__).info('Cleaning up...') zip_output_path = join(path, 'iati-data-main') for f in listdir(zip_output_path): shutil.move(join(zip_output_path, f), path) _unlink(zip_filepath)
def os_unlink(path): return os._unlink(longpathify(uni(path)))