def _ut_set_up(self) -> None:
'''
Ensure our unit-testing dir doesn't exist, and then create it.
'''
# Make sure our root /does/ exist...
if (not self.root() or not self.root().exists()
or not self.root().is_dir()):
msg = ("Invalid root directory for repo data! It must exist "
"and be a directory.")
self._log_data_processing(self.dotted, msg, success=False)
error = UnitTestError(msg,
data={
'meta': self._bg,
'root': paths.to_str(self.root()),
'exists?': paths.exists(self.root()),
'file?': paths.is_file(self.root()),
'dir?': paths.is_dir(self.root()),
})
raise self._log_exception(error, msg)
# Make sure temp path doesn't exist first... Don't want to accidentally
# use data from a previous test.
path = self._path_temp()
if path.exists():
# Ideally each test would clean up after itself without fail,
# but...
self._log_data_processing(
self.dotted, "_ut_set_up found pre-existinng temp path... "
"Have to delete it first, which is sub-optimal.",
success=False)
# Some unit test failed? Try to clean up first.
self._ut_tear_down(note=("_ut_set_up() needs to "
"clean up dirty state."))
# Give it a bit to actually be deleted so we can check.
# Running a unit test so I'm ok with a sleep time here.
import time
time.sleep(0.1)
# /Now/ it's gone.... right?
if path.exists():
msg = "Temp Dir Path for Unit-Testing already exists!"
self._log_data_processing(self.dotted, msg, success=False)
error = UnitTestError(msg,
data={
'meta': self._bg,
'temp-path': paths.to_str(path),
'exists?': paths.exists(path),
'file?': paths.is_file(path),
'dir?': paths.is_dir(path),
})
raise self._log_exception(error, msg)
# And now we can create it.
path.mkdir(parents=True)
self._log_data_processing(self.dotted,
"_ut_set_up complete; temp dir has "
"been created: {}",
paths.to_str(path),
success=True)
def _path(self,
*unsafe: paths.PathType,
context: BaseDataContext,
ensure: bool = True,
glob: bool = False) -> paths.Path:
'''
Returns a path based on the Repository's root and `unsafe`.
If `glob` is True, adds `_ext_glob()` to the end of the returned path.
If `ensure` is False, skip (possible) parent directory creation. No
need to set for load vs save; that is handled automatically.
`context` is used for `context.action` and for errors.
Returned path is safe according to `_path_safed()`.
'''
# Make it into a safe path.
safe = self._path_safed(*unsafe, context=context)
path = None
if context.temp:
path = self._path_temp(safe, context=context)
else:
path = self.root() / safe
if glob:
if context.action == DataAction.SAVE:
msg = "Cannot glob filename when saving!"
self._log_data_processing(self.dotted,
msg,
context=context,
success=False)
error = self._error_type(context)(msg,
data={
'unsafe': unsafe,
'action': context.action,
'ensure': ensure,
'glob': glob,
'path': path,
})
raise self._log_exception(error, msg, context=context)
path = self._ext_glob(path)
# Make sure the directory exists?
if ensure and context.action == DataAction.SAVE:
self._path_ensure(path, context)
self._log_data_processing(self.dotted,
"Created path: {}",
paths.to_str(path),
context=context,
success=True)
return path
def _path_ensure(self, path: paths.Path, context: BaseDataContext) -> None:
'''
Creates path's parent's path if it does not exist.
NOTE: Currently will /not/ create path as I do not know if it is a dir
or file component name.
'''
path.parent.mkdir(parents=True, exist_ok=True)
self._log_data_processing(self.dotted,
"Ensured path exists: {}",
paths.to_str(path),
context=context,
success=True)
def _save(self,
save_path: paths.Path,
data: TextIOBase,
context: DataBareContext) -> bool:
'''
Save `data` to `save_path`. If it already exists, overwrites that file.
'''
self._log_data_processing(self.dotted,
"Saving '{}'...",
paths.to_str(save_path),
context=context)
super()._save(save_path, data, context)
success = False
with save_path.open('w') as file_stream:
self._log_data_processing(self.dotted,
"Writing...",
context=context)
# Can raise an error - we'll let it.
try:
# Make sure we're at the beginning of the data stream...
data.seek(0)
# ...and use shutils to copy the data to disk.
shutil.copyfileobj(data, file_stream)
# We don't have anything to easily check to return
# success/failure...
success = True
except SaveError:
self._log_data_processing(self.dotted,
"Got SaveError trying to "
"write file: {}",
paths.to_str(save_path),
context=context,
success=False)
# Let this one bubble up as-is.
raise
except Exception as error:
self._log_data_processing(self.dotted,
"Got an exception trying to "
"write file: {}",
paths.to_str(save_path),
context=context,
success=False)
# Complain that we found an exception we don't handle.
# ...then let it bubble up.
raise self._log_exception(
self._error_type(context),
"Error saving data to file. context: {}",
context=context) from error
self._log_data_processing(self.dotted,
"Saved file '{}'!",
paths.to_str(save_path),
context=context,
success=True)
return success
def _load(self,
load_path: paths.Path,
context: DataBareContext) -> TextIOBase:
'''
Looks for file at load_path. If it exists, loads that file.
'''
self._log_data_processing(self.dotted,
"Loading '{}'...",
paths.to_str(load_path),
context=context)
super()._load(load_path, context)
# load_path should be exact - no globbing.
if not load_path.exists():
msg = "Cannot load file. Path/file does not exist: {}"
self._log_data_processing(self.dotted,
msg,
paths.to_str(load_path),
context=context,
success=False)
raise self._log_exception(
self._error_type(context),
msg,
str(load_path),
context=context)
data_stream = None
with load_path.open('r') as file_stream:
self._log_data_processing(self.dotted,
"Reading...",
context=context)
# Can raise an error - we'll let it.
try:
data_stream = StringIO(file_stream.read(None))
except LoadError:
self._log_data_processing(self.dotted,
"Got LoadError trying to "
"read file: {}",
paths.to_str(load_path),
context=context,
success=False)
# Let this one bubble up as-is.
if data_stream and not data_stream.closed:
data_stream.close()
data_stream = None
raise
except Exception as error:
self._log_data_processing(self.dotted,
"Got an exception trying to "
"read file: {}",
paths.to_str(load_path),
context=context,
success=False)
# Complain that we found an exception we don't handle.
# ...then let it bubble up.
if data_stream and not data_stream.closed:
data_stream.close()
data_stream = None
raise self._log_exception(
self._error_type(context),
"Error loading data from file. context: {}",
context=context) from error
self._log_data_processing(self.dotted,
"Loaded file '{}'!",
paths.to_str(load_path),
context=context,
success=True)
return data_stream
def _ut_tear_down(self, note: str = None) -> None:
'''
Deletes our temp directory and all files in it.
'''
# ---
# Make sure our root /does/ exist...
# ---
if (not self.root() or not self.root().exists()
or not self.root().is_dir()):
msg = ("Invalid root directory for repo data! It must exist "
"and be a directory.")
self._log_data_processing(self.dotted, msg, success=False)
error = UnitTestError(msg,
data={
'meta': self._bg,
'note': note,
'root': paths.to_str(self.root()),
'exists?': paths.exists(self.root()),
'file?': paths.is_file(self.root()),
'dir?': paths.is_dir(self.root()),
})
raise self._log_exception(error, msg)
# ---
# Does temp path exist?
# ---
path = self._path_temp()
if path.exists():
# Is it a dir?
if path.is_dir():
self._log_data_processing(self.dotted,
"_ut_tear_down: deleting "
"temp directory: {}...",
paths.to_str(path),
success=False)
# Yeah - ok; delete it and it's files now.
shutil.rmtree(path)
# This is reporting false negatives - says "Failed to delete",
# but it's deleted when I go check. So don't check I guess.
# success = path.exists()
# success_str = "Deleted" if success else "Failed to delete"
# self._log_data_processing(self.dotted,
# "_ut_tear_down: {} "
# "temp directory: {}",
# success_str,
# paths.to_str(path),
# success=success)
# Not a dir - error.
else:
msg = ("Cannot delete temp directory - _path_temp() "
"is not a directory!")
self._log_data_processing(self.dotted,
"_ut_tear_down: {} {}",
msg,
paths.to_str(path),
success=False)
error = UnitTestError(msg,
data={
'meta': self._bg,
'note': note,
'temp-path': paths.to_str(path),
'exists?': path.exists(),
'file?': path.is_file(),
'dir?': path.is_dir(),
})
raise self._log_exception(error, msg)
def _path_temp(
self,
path_non_temp: Optional[paths.PathType] = None,
context: Optional['VerediContext'] = None,
raise_errors: bool = True,
) -> Nullable[paths.Path]:
'''
Returns a path to either our temp directory, a path /in/ our temp
directory, or Null().
'''
self._log_data_processing(self.dotted,
"Get temp path for non-temp: {}...",
paths.to_str(path_non_temp),
context=context)
path_non_temp = (paths.cast(path_non_temp) if path_non_temp else None)
path_temp = None
# ------------------------------
# No `self.root` Cases:
# ------------------------------
# No root is possible for some FileRepositories...
# FileBareRepository was like that for a long time.
if not self.root():
# ------------------------------
# Invalid.
# ------------------------------
# No root and no input? Gonna have a bad time.
if not path_non_temp:
msg = "Cannot make a temp path: no root and no path provided."
self._log_data_processing(self.dotted,
msg + "root: {}, path: {}",
paths.to_str(self.root()),
paths.to_str(path_non_temp),
context=context,
success=False)
if raise_errors:
error = self._error_type(context)(
msg,
data={
'root': paths.to_str(self.root()),
'path': paths.to_str(path_non_temp),
})
raise self._log_exception(error, msg, context=context)
self._log_warning(msg + "root: {}, path: {}",
paths.to_str(self.root()),
paths.to_str(path_non_temp),
context=context)
return Null()
# No root and input is relative? Can't be sure it's valid so don't
# return anything.
if not path_non_temp.is_absolute():
msg = ("Cannot make a temp path: no root and provided path "
"is not absolute.")
self._log_data_processing(self.dotted,
msg + "root: {}, path: {}",
str(self.root()),
paths.to_str(path_non_temp),
context=context,
success=False)
if raise_errors:
error = self._error_type(context)(
msg,
data={
'root': self.root(),
'path': paths.to_str(path_non_temp),
'absolute?': path_non_temp.is_absolute(),
})
raise self._log_exception(error, msg, context=context)
else:
self._log_warning(msg + "root: {}, path: {}",
str(self.root()),
paths.to_str(path_non_temp),
context=context)
return Null()
# Otherwise, we have no root and an absolute path. Best we can do
# is make sure the temp dir is in there somewhere? So... complain
# as well.
if self._TEMP_PATH not in path_non_temp.parts:
msg = ("Cannot create a temp path when we have no repository "
f"root and '{self._TEMP_PATH}' is not in input: "
f"{path_non_temp}")
self._log_data_processing(self.dotted,
msg,
context=context,
success=False)
if raise_errors:
error = self._error_type(context)(
msg,
data={
'root': self.root(),
'path': paths.to_str(path_non_temp),
'absolute?': path_non_temp.is_absolute(),
})
raise self._log_exception(error, msg, context=context)
else:
self._log_warning(msg, context=context)
return Null()
# ------------------------------
# Valid?
# ------------------------------
# Ok; We have:
# 1) No root.
# 2) Absolute input path.
# 3) Input path with one `parts` being `self._TEMP_PATH`.
# So... just send it back?
path_temp = path_non_temp
self._log_data_processing(self.dotted,
"Temp Path is (root-less): {}",
paths.to_str(path_temp),
context=context,
success=False)
# ------------------------------
# Normal/Expected Cases (w/ `self.root()`):
# ------------------------------
# We do have a root. Use it if the provided path is relative.
# Nothing requested?
elif not path_non_temp:
# Provide the temp dir itself...
path_temp = self.root() / self._TEMP_PATH
self._log_data_processing(self.dotted,
"Temp Path is (rooted default): {}",
paths.to_str(path_temp),
context=context,
success=False)
# Specific path requested.
else:
path = path_non_temp
# Make sure it's relative so it can be in our repo.
if path.is_absolute():
# Let this raise a ValueError if path isn't relative to root.
path = path.relative_to(self.root())
# It should have our `_TEMP_PATH` in it.
path = (path if self._TEMP_PATH in path.parts else
(self._TEMP_PATH / path))
# And it should be rooted in the repo.
path_temp = self.root() / path
self._log_data_processing(self.dotted,
"Temp Path is (rooted specific): {}",
paths.to_str(path_temp),
context=context,
success=False)
# ------------------------------
# Done!
# ------------------------------
return path_temp
def _load(self,
load_path: paths.PathType,
context: DataLoadContext) -> TextIOBase:
'''
Looks for a match to `load_path` by splitting into parent dir and
glob/file name. If only one match, loads that file.
'''
self._log_data_processing(self.dotted,
"Loading requested path '{}'...",
paths.to_str(load_path),
context=context)
# ------------------------------
# Search...
# ------------------------------
# Use load_path to find all file matchs...
directory = load_path.parent
glob = load_path.name
matches = []
for match in directory.glob(glob):
matches.append(match)
match_word = "match" if len(matches) == 1 else "matches"
self._log_data_processing(self.dotted,
f"Found {len(matches)} {match_word} files for "
f"loading '{load_path.name}': {matches}",
context=context)
# ------------------------------
# Sanity
# ------------------------------
# Error if we found not-exactly-one match.
if not matches:
# We found nothing.
self._context_data(context, matches)
msg = (f"No matches for loading file: "
f"directory: {directory}, glob: {glob}, "
f"matches: {matches}")
self._log_data_processing(self.dotted,
msg,
context=context,
success=False)
raise self._log_exception(
self._error_type(context),
msg,
context=context)
elif len(matches) > 1:
# Throw all matches into context for error.
self._context_data(context, matches)
msg = (f"Too many matches for loading file: "
f"directory: {directory}, glob: {glob}, "
f"matches: {sorted(matches)}")
self._log_data_processing(self.dotted,
msg,
context=context,
success=False)
raise self._log_exception(
self._error_type(context),
msg,
context=context)
# ------------------------------
# Set-Up...
# ------------------------------
self._log_data_processing(self.dotted,
f"Loading '{matches[0]}' file for "
f"load path '{load_path}'...",
context=context)
load_path = matches[0]
super()._load(load_path, context)
# ------------------------------
# Load!
# ------------------------------
data_stream = None
with load_path.open('r') as file_stream:
self._log_data_processing(self.dotted,
"Reading...",
context=context)
# Can raise an error - we'll let it.
try:
# print("\n\nfile tell:", file_stream.tell())
data_stream = StringIO(file_stream.read(None))
# print("string tell:", data_stream.tell(), "\n\n")
# print("\ndata_stream:")
# print(data_stream.read(None))
# print("\n")
except LoadError:
self._log_data_processing(self.dotted,
"Got LoadError trying to "
"read file: {}",
paths.to_str(load_path),
context=context,
success=False)
# Let this one bubble up as-is.
if data_stream and not data_stream.closed:
data_stream.close()
data_stream = None
raise
except Exception as error:
self._log_data_processing(self.dotted,
"Got an exception trying to "
"read file: {}",
paths.to_str(load_path),
context=context,
success=False)
# Complain that we found an exception we don't handle.
# ...then let it bubble up.
if data_stream and not data_stream.closed:
data_stream.close()
data_stream = None
raise self._log_exception(
self._error_type(context),
"Error loading data from file. context: {}",
context=context) from error
# ------------------------------
# Done.
# ------------------------------
self._log_data_processing(self.dotted,
"Loaded file '{}'!",
paths.to_str(load_path),
context=context,
success=True)
return data_stream