def process_ndarray( array: np.ndarray, path: PurePosixPath) -> Iterable[Tuple[Any, PurePosixPath]]: """ Yields 1D :class:`numpy.ndarray` and path object for each higher dimension index combination w.r.t the first dimension. e.g. .. code-block:: python a = np.array([[1, 2, 3] [4, 5, 6] [7, 8, 9]]) p = PurePosixPath("name") for obj, path in process_ndarray(a, p): print(path, obj) Output .. code-block:: none name_0 [1 4 7] name_1 [2 5 8] name_2 [3 6 9] :param array: Numpy array of N dimensions. :param path: Path object with the path to the numpy array in the object hierarchy. """ # If it only has one dimension, return the array and path without any processing. if array.ndim == 1: yield array, path else: # Loop through all combinations of higher dimension indices. for idx in product(*map(range, array.shape[1:])): # Yield all elements for this higher dimension index w.r.t. to the first dimension. # e.g. array[:, 1, 2, 3, 4] yield array[(slice(None), *idx)], path.with_name( f"{path.name}_{'_'.join(map(str, idx))}")
def _actual_upload_to(cls, instance: models.Model, filename: str, *, upload_to: Union[str, Callable[[models.Model, str], str]]): """This method should only be used by ``get_pk_prefixed_filename_func()``; do not use this method directly.""" if isinstance(upload_to, str): base_path = PurePosixPath(upload_to) / filename else: base_path = PurePosixPath(upload_to(instance, filename)) base_filename = base_path.name # Remove token if the filename already contains it (for whatever reason) if cls.REPLACEABLE_TOKEN_REGEX.search(base_filename): first_part, _token, last_part = cls.REPLACEABLE_TOKEN_REGEX.split( base_filename) base_filename = f"{first_part}{last_part}" # Remove the PK prefix if the filename already has it if instance.pk: base_filename = base_filename.removeprefix(f"{instance.pk}_") prefix = instance.pk or cls.generate_replaceable_token() prefixed_filename_path = base_path.with_name( f"{prefix}_{base_filename}") return str(prefixed_filename_path)
def open_remote_sqlite_database(adb, database): database = PurePosixPath(database) with TemporaryDirectory() as temp_dir: temp_dir = Path(temp_dir) for suffix in ['', '-journal', '-wal', '-shm']: remote_file = database.with_name(database.name + suffix) try: contents = adb.read_file(remote_file) except FileNotFoundError as e: # Throw the original exception if the actual db file cannot be read if suffix == '': raise e else: (temp_dir / remote_file.name).write_bytes(contents.read()) db_path = str(temp_dir / database.name) with contextlib.closing(sqlite3.connect(db_path)) as connection: yield connection
def open_remote_sqlite_database(adb, database): database = PurePosixPath(database) with TemporaryDirectory() as temp_dir: temp_dir = Path(temp_dir) for suffix in ['', '-journal', '-wal', '-shm']: remote_file = database.with_name(database.name + suffix) try: contents = adb.read_file(remote_file) except FileNotFoundError as e: if suffix != '': continue # Throw the original exception if the db file cannot be read raise e with (temp_dir / remote_file.name).open('wb') as local_file: local_file.write(contents.read()) connection = sqlite3.connect(str(temp_dir / database.name)) yield connection connection.close()