def test_download_artifact_passing_path(self):
filepath = os.path.join(self.archive_run_path, "1")
assert os.path.exists(filepath) is False
response = self.client.get(self.base_url + "?path=1")
assert response.status_code == 200
assert os.path.exists(filepath) is True
assert response.headers["Content-Type"] == ""
assert response.headers["X-Accel-Redirect"] == os.path.join(
self.archive_run_path, "1"
)
assert response.headers[
"Content-Disposition"
] == 'attachment; filename="{}"'.format("1")
# Nested dirs
nested_path = os.path.join(self.run_path, "foo")
create_path(nested_path)
create_tmp_files(nested_path)
filepath = os.path.join(self.archive_run_path, "foo", "1")
assert os.path.exists(filepath) is False
response = self.client.get(self.base_url + "?path=foo/1")
assert response.status_code == 200
assert os.path.exists(filepath) is True
assert response.headers["Content-Type"] == ""
assert response.headers["X-Accel-Redirect"] == os.path.join(
self.archive_run_path, "foo/1"
)
assert response.headers[
"Content-Disposition"
] == 'attachment; filename="{}"'.format("1")
def test_log_model_dir(self):
assert (os.path.exists(
get_event_path(self.run_path, kind=V1ArtifactKind.MODEL)) is False)
assert (os.path.exists(
get_asset_path(self.run_path, kind=V1ArtifactKind.MODEL)) is False)
model_dir = tempfile.mkdtemp() + "/model"
create_path(model_dir)
model_file = model_dir + "/model.pkl"
self.touch(model_file)
weights_file = model_dir + "/weights"
self.touch(weights_file)
configs_file = model_dir + "/configs"
self.touch(configs_file)
self.run.log_model(name="my_model",
path=model_dir,
framework="tensorflow")
self.event_logger.flush()
assert (os.path.exists(
get_asset_path(self.run_path, kind=V1ArtifactKind.MODEL)) is True)
assert (os.path.exists(
get_event_path(self.run_path, kind=V1ArtifactKind.MODEL)) is True)
events_file = get_event_path(self.run_path,
kind=V1ArtifactKind.MODEL,
name="my_model")
assert os.path.exists(events_file) is True
results = V1Events.read(kind="model",
name="my_model",
data=events_file)
assert len(results.df.values) == 1
asset_file = get_asset_path(self.run_path,
kind=V1ArtifactKind.MODEL,
name="my_model")
assert os.path.exists(asset_file) is True
def setUp(self):
super().setUp()
self.store_root = set_store()
self.run_path = os.path.join(self.store_root, "uuid")
# Create run artifacts path and some files
create_path(self.run_path)
create_tmp_files(self.run_path)
self.client = get_streams_client()
self.base_url = STREAMS_URL + "/namespace/owner/project/runs/uuid/artifacts"
def setUp(self):
super().setUp()
self.store_root = set_store()
self.run_path = os.path.join(self.store_root, "uuid")
self.run_events = os.path.join(self.run_path, "events")
# Create run artifacts path
create_path(self.run_path)
# Create run artifacts events path
create_path(self.run_events)
# Create run events
self.create_tmp_events()
self.client = get_streams_client()
self.base_url = STREAMS_URL + "/namespace/owner/project/runs/uuid/events"
def test_delete_artifacts(self):
# Created nested path
nested_path = os.path.join(self.run_path, "foo")
create_path(nested_path)
create_tmp_files(nested_path)
subpath = os.path.join(self.run_path, "foo")
assert os.path.exists(self.run_path) is True
assert os.path.exists(subpath) is True
response = self.client.delete(self.base_url + "?path=foo")
assert response.status_code == 204
assert os.path.exists(self.run_path) is True
assert os.path.exists(subpath) is False
response = self.client.delete(self.base_url)
assert response.status_code == 204
assert os.path.exists(self.run_path) is False
def test_get_artifacts_tree(self):
response = self.client.get(self.base_url)
assert response.status_code == 200
assert response.json() == {
"dirs": [],
"files": {
"0": 0,
"1": 0,
"2": 0,
"3": 0
},
}
# add nested
nested_path = os.path.join(self.run_path, "foo")
create_path(nested_path)
create_tmp_files(nested_path)
response = self.client.get(self.base_url)
assert response.status_code == 200
assert response.json() == {
"dirs": ["foo"],
"files": {
"0": 0,
"1": 0,
"2": 0,
"3": 0
},
}
response = self.client.get(self.base_url + "?path=foo")
assert response.status_code == 200
assert response.json() == {
"dirs": [],
"files": {
"0": 0,
"1": 0,
"2": 0,
"3": 0
},
}
response = self.client.get(self.base_url + "?path=foo/1")
assert response.status_code == 200
assert response.json() == {"dirs": [], "files": {}}
def test_delete_artifact_passing_path(self):
filepath = os.path.join(self.run_path, "1")
assert os.path.exists(filepath) is True
response = self.client.delete(self.base_url + "?path=1")
assert response.status_code == 204
assert os.path.exists(filepath) is False
# Nested dirs
nested_path = os.path.join(self.run_path, "foo")
create_path(nested_path)
create_tmp_files(nested_path)
filepath = os.path.join(self.run_path, "foo", "1")
assert os.path.exists(filepath) is True
response = self.client.delete(self.base_url + "?path=foo/1")
assert response.status_code == 204
assert os.path.exists(filepath) is False
# Deleting same file
response = self.client.delete(self.base_url + "?path=foo/1")
assert response.status_code == 400
def test_stream_artifact_passing_path_with_force(self):
# Nested dirs
nested_path = os.path.join(self.run_path, "foo")
create_path(nested_path)
create_tmp_files(nested_path)
filepath = os.path.join(self.archive_run_path, "foo", "1")
assert os.path.exists(filepath) is False
response = self.client.get(self.base_url + "?stream=true&path=foo/1")
assert response.status_code == 200
assert os.path.exists(filepath) is True
assert response.headers["Content-Type"] == "text/plain; charset=utf-8"
assert response.headers["content-length"] == "0"
assert response.headers["last-modified"] is not None
assert response.headers["etag"] is not None
# The file is cached but we force check
shutil.rmtree(self.run_path)
assert os.path.exists(filepath) is True
response = self.client.get(self.base_url + "?stream=true&path=foo/1&force=true")
assert response.status_code == 404
def create_kind_events(self, name, kind, events):
event_kind_path = "{}/{}".format(self.run_events, kind)
create_path(event_kind_path)
with open("{}/{}.plx".format(event_kind_path, name), "+w") as f:
f.write(events.get_csv_header())
f.write(events.get_csv_events())
