def test_update_configuration(self, *args):
with self.project_dir:
ws_dir = "/Shared/dbx/projects/%s" % self.project_name
invoke_cli_runner(
configure,
[
"--environment",
"test",
"--profile",
self.profile_name,
"--workspace-dir",
ws_dir,
],
)
new_ws_dir = ws_dir + "/updated"
invoke_cli_runner(
configure,
[
"--environment",
"test",
"--profile",
self.profile_name,
"--workspace-dir",
new_ws_dir,
],
)
env = InfoFile.get("environments").get("test")
self.assertIsNotNone(env)
self.assertEqual(env["profile"], self.profile_name)
self.assertEqual(env["workspace_dir"], new_ws_dir)
def _find_deployment_run(filter_string: str, tags: Dict[str, str],
as_run_submit: bool,
environment: str) -> Dict[str, Any]:
runs = mlflow.search_runs(filter_string=filter_string,
order_by=["start_time DESC"])
filter_conditions = []
if tags:
dbx_echo("Filtering deployments with set of additional tags")
for tag_name, tag_value in tags.items():
tag_column_name = f"tags.{tag_name}"
if tag_column_name not in runs.columns:
raise Exception(
f"Tag {tag_name} not found in underlying MLflow experiment, please verify tag existence in the UI"
)
tag_condition = runs[tag_column_name] == tag_value
filter_conditions.append(tag_condition)
full_filter = pd.DataFrame(filter_conditions).T.all(axis=1) # noqa
_runs = runs[full_filter]
else:
dbx_echo("No additional tags provided")
_runs = runs
if as_run_submit:
if "tags.dbx_deploy_type" not in _runs.columns:
raise Exception(""""
Run Submit API is available only when deployment was done with --files-only flag.
Currently there is no deployments with such flag under given filters.
Please re-deploy with --files-only flag and then re-run this launch command.
""")
_runs = _runs[_runs["tags.dbx_deploy_type"] == "files_only"]
if _runs.empty:
exception_string = """
No runs provided per given set of filters:
{filter_string}
"""
if tags:
exception_string = (exception_string + f"""
With additional tags: {tags}
""")
if as_run_submit:
exception_string = (exception_string + """
With file-based deployments (dbx_deployment_type='files_only').
""")
experiment_location = InfoFile.get(environment).get("workspace_dir")
exception_string = (exception_string + f"""
To verify current status of deployments please check experiment UI in workspace dir: {experiment_location}
""")
raise Exception(exception_string)
run_info = _runs.iloc[0].to_dict()
dbx_echo("Successfully found deployment per given job name")
return run_info
def test_non_empty_behaviour(self):
with self.project_dir:
logging.info(self.project_dir)
InfoFile.initialize()
self.assertTrue(Path(".dbx").exists())
InfoFile.update({"environments": {"new-env": "payload"}})
InfoFile.initialize()
def configure(
environment: str, workspace_dir: str, artifact_location: str, profile: str
):
dbx_echo("Configuring new environment with name %s" % environment)
if not workspace_dir:
workspace_dir = f'/Shared/dbx/projects/{Path(".").absolute().name}'
dbx_echo(
f"Workspace directory argument is not provided, using the following directory: {workspace_dir}"
)
if not Path(INFO_FILE_PATH).exists():
InfoFile.initialize()
if InfoFile.get("environments").get(environment):
dbx_echo(f"Environment {environment} will be overridden with new properties")
if not artifact_location:
artifact_location = f'dbfs:/dbx/{Path(".").absolute().name}'
environment_info = {
environment: {
"profile": profile,
"workspace_dir": workspace_dir,
"artifact_location": artifact_location,
}
}
environments = InfoFile.get("environments")
environments.update(environment_info)
InfoFile.update({"environments": environments})
dbx_echo("Environment configuration successfully finished")
def test_configure(self, *args) -> None:
with self.project_dir:
ws_dir = "/Shared/dbx/projects/%s" % self.project_name
first_result = invoke_cli_runner(
configure,
[
"--environment", "test", "--profile", self.profile_name,
"--workspace-dir", ws_dir, "--artifact-location",
f'dbfs:/dbx/custom-project-location/{self.project_name}'
],
)
self.assertEqual(first_result.exit_code, 0)
env = InfoFile.get("environments").get("test")
self.assertIsNotNone(env)
self.assertEqual(env["profile"], self.profile_name)
self.assertEqual(env["workspace_dir"], ws_dir)
def test_configure_default(self, *args) -> None:
with self.project_dir:
Path(INFO_FILE_PATH).unlink()
first_result = invoke_cli_runner(
configure,
[
"--environment",
"test",
"--profile",
self.profile_name,
],
)
self.assertEqual(first_result.exit_code, 0)
env = InfoFile.get("environments").get("test")
self.assertIsNotNone(env)
self.assertEqual(env["profile"], self.profile_name)
self.assertEqual(env["artifact_location"],
f"dbfs:/dbx/{self.project_name}")
self.assertEqual(env["workspace_dir"],
f"/Shared/dbx/projects/{self.project_name}")
def test_default_initialize(self):
with self.project_dir:
InfoFile.initialize()
init_content = read_json(INFO_FILE_PATH)
self.assertEqual(init_content.get("environments"), {})