def process_ndarray(
array: np.ndarray,
path: PurePosixPath) -> Iterable[Tuple[Any, PurePosixPath]]:
"""
Yields 1D :class:`numpy.ndarray` and path object for each higher dimension index
combination w.r.t the first dimension.
e.g.
.. code-block:: python
a = np.array([[1, 2, 3]
[4, 5, 6]
[7, 8, 9]])
p = PurePosixPath("name")
for obj, path in process_ndarray(a, p):
print(path, obj)
Output
.. code-block:: none
name_0 [1 4 7]
name_1 [2 5 8]
name_2 [3 6 9]
:param array: Numpy array of N dimensions.
:param path: Path object with the path to the numpy array in the object hierarchy.
"""
# If it only has one dimension, return the array and path without any processing.
if array.ndim == 1:
yield array, path
else:
# Loop through all combinations of higher dimension indices.
for idx in product(*map(range, array.shape[1:])):
# Yield all elements for this higher dimension index w.r.t. to the first dimension.
# e.g. array[:, 1, 2, 3, 4]
yield array[(slice(None), *idx)], path.with_name(
f"{path.name}_{'_'.join(map(str, idx))}")
def _actual_upload_to(cls, instance: models.Model, filename: str, *,
upload_to: Union[str, Callable[[models.Model, str],
str]]):
"""This method should only be used by ``get_pk_prefixed_filename_func()``; do not use this method directly."""
if isinstance(upload_to, str):
base_path = PurePosixPath(upload_to) / filename
else:
base_path = PurePosixPath(upload_to(instance, filename))
base_filename = base_path.name
# Remove token if the filename already contains it (for whatever reason)
if cls.REPLACEABLE_TOKEN_REGEX.search(base_filename):
first_part, _token, last_part = cls.REPLACEABLE_TOKEN_REGEX.split(
base_filename)
base_filename = f"{first_part}{last_part}"
# Remove the PK prefix if the filename already has it
if instance.pk:
base_filename = base_filename.removeprefix(f"{instance.pk}_")
prefix = instance.pk or cls.generate_replaceable_token()
prefixed_filename_path = base_path.with_name(
f"{prefix}_{base_filename}")
return str(prefixed_filename_path)
def open_remote_sqlite_database(adb, database):
database = PurePosixPath(database)
with TemporaryDirectory() as temp_dir:
temp_dir = Path(temp_dir)
for suffix in ['', '-journal', '-wal', '-shm']:
remote_file = database.with_name(database.name + suffix)
try:
contents = adb.read_file(remote_file)
except FileNotFoundError as e:
# Throw the original exception if the actual db file cannot be read
if suffix == '':
raise e
else:
(temp_dir / remote_file.name).write_bytes(contents.read())
db_path = str(temp_dir / database.name)
with contextlib.closing(sqlite3.connect(db_path)) as connection:
yield connection
def open_remote_sqlite_database(adb, database):
database = PurePosixPath(database)
with TemporaryDirectory() as temp_dir:
temp_dir = Path(temp_dir)
for suffix in ['', '-journal', '-wal', '-shm']:
remote_file = database.with_name(database.name + suffix)
try:
contents = adb.read_file(remote_file)
except FileNotFoundError as e:
if suffix != '':
continue
# Throw the original exception if the db file cannot be read
raise e
with (temp_dir / remote_file.name).open('wb') as local_file:
local_file.write(contents.read())
connection = sqlite3.connect(str(temp_dir / database.name))
yield connection
connection.close()
