def test_make_dir_fails_on_input_files(self, tmp_path: pathlib.Path):
file_path = tmp_path / "test.txt"
file_path.write_text("test\ntest")
with pytest.raises(IOError):
make_dir(file_path)
def test_make_dir_creates_folder_on_input_files(self,
tmp_path: pathlib.Path):
file_path = tmp_path / "testing"
assert file_path.exists() is False
make_dir(file_path)
assert file_path.exists()
def _generate_output_path(self, save_dir: pathlib.Path) -> pathlib.Path:
"""
Generates a valid filename, appending an incrementing counter if
file with same name already exists
Parameters
----------
save_dir: pathlib.Path
Directory where logs are to be saved
Returns
-------
pathlib.Path
Path to output_file
"""
save_dir = make_dir(save_dir)
now = datetime.now()
iteration = 0
output_file = f'{self.name}_{now.strftime("%H%M%S")}_{iteration}.yaml'
output_path = save_dir.joinpath(output_file)
while output_path.exists():
output_file = f'{self.name}_{now.strftime("%H%M%S")}_{iteration}.yaml'
output_path = save_dir.joinpath(output_file)
iteration += 1
return output_path
def save(self,
estimator: Estimator,
filename: str,
prod: bool = False) -> "ArtifactoryPath":
"""
Save a pickled estimator to artifactory.
Parameters
----------
estimator: Estimator
The estimator object
filename: str
filename of estimator pickle file
prod: bool
Production variable, set to True if saving a production-ready estimator
Example
-------
To save your trained estimator:
storage = ArtifactoryStorage('http://artifactory.com', 'path/to/repo')
artifactory_path = storage.save(estimator, 'estimator.pkl')
We now have saved an estimator to a pickle file.
Returns
-------
ArtifactoryPath
File path to stored estimator
"""
if prod:
raise NotImplementedError(
"Artifactory Storage doesn't currently implement production storage. "
"Use FileStorage instead")
artifactory_path = make_dir(self.artifactory_path) / filename
with TemporaryDirectory() as tmpdir:
file_path = Path(tmpdir).joinpath(filename)
joblib.dump(estimator, file_path)
artifactory_path.deploy_file(file_path)
return artifactory_path
def save(self,
estimator: Estimator,
filename: str,
prod: bool = False) -> Path:
"""
Save a joblib pickled estimator.
Parameters
----------
estimator: obj
The estimator object
filename: str
filename of estimator pickle file
prod: bool
Whether or not to save in "production mode" -
Production mode saves to /src/<projectname>/ regardless
of what FileStorage was instantiated with
Example
-------
To save your trained estimator, use the FileStorage context manager.
storage = FileStorage('/path/to/save/dir/')
file_path = storage.save(estimator, 'filename')
We now have saved an estimator to a pickle file.
Returns
-------
Path
Path to the saved object
"""
if prod:
file_path = _find_src_dir() / filename
else:
file_path = make_dir(self.dir_path) / filename
joblib.dump(estimator, file_path)
return file_path