def check_predictor_connection(predictor_url):
url = url_join(predictor_url, "isAlive/ping")
try:
with get_http_session(predictor_url) as session:
session.get(url)
return True
except Exception:
LOGGER.error("Error checking predictor connectivity", exc_info=True)
return False
def check_rest_api_connection(rest_api_url):
url = url_join(rest_api_url, "account-details")
try:
with get_http_session(rest_api_url) as session:
session.get(url)
return True
except Exception:
LOGGER.error("Error checking rest api connectivity", exc_info=True)
return False
def check_optimizer_connection(optimizer_url):
url = url_join(optimizer_url, "spec")
try:
with get_http_session(optimizer_url) as session:
params = {"algorithmName": "bayes"}
session.get(url, params=params)
return True
except Exception:
LOGGER.error("Error checking optimizer connectivity", exc_info=True)
return False
def test_conversion(tmp_path, monkeypatch):
path = tmp_path.resolve().as_posix()
os.chdir(path)
# Run the MLFlow example
mlflow_example()
# Check that MLFlow have created its on-disk content
assert os.path.isdir(os.path.join(path, "mlruns"))
# Monkey-patch HTTP interactions
backend_version_body = {
"msg": "1.2.131",
"name": "Python-Backend",
"ip": "",
"hostname": "",
"version": "1.2.131",
}
url = url_join(SERVER_ADDRESS, "isAlive/ver")
responses.add(
responses.GET,
url,
json=backend_version_body,
status=200,
)
# Check that comet_for_mlflow have created an offline experiment
api_key = "XXX"
monkeypatch.setenv("COMET_WORKSPACE", "WORKSPACE")
conv = comet_for_mlflow.Translator(False, api_key, path, None, None, "no",
"*****@*****.**")
conv.prepare()
assert len(list(tmp_path.glob("*.zip"))) == 1
def upload(self, prepared_data):
LOGGER.info("# Start uploading data to Comet.ml")
all_project_names = []
with tqdm(total=self.summary["runs"]) as pbar:
for experiment_data in prepared_data:
experiment = experiment_data["experiment"]
project_name = self.get_or_create_comet_project(experiment)
# Sync the experiment note
project_note = experiment.tags.get("mlflow.note.content", None)
if project_note:
note_template = (
u"/!\\ This project notes has been copied from MLFlow. It might be overwritten if you run comet_for_mlflow again/!\\ \n%s"
% project_note)
# We don't support Unicode project notes yet
self.api_client.set_project_notes(
self.workspace,
project_name,
note_template,
)
all_project_names.append(project_name)
runs = experiment_data["runs"]
for mlflow_run, archive_path in runs:
write_comet_experiment_metadata_file(
mlflow_run, project_name, archive_path, self.workspace)
upload_single_offline_experiment(
archive_path,
self.api_key,
force_reupload=self.force_reupload,
display_level="debug",
)
pbar.update(1)
LOGGER.info("")
LOGGER.info(
"Explore your experiment data on Comet.ml with the following links:",
)
if len(all_project_names) < 6:
for project_name in all_project_names:
project_url = url_join(
self.api_client._get_url_server(),
self.workspace + "/",
project_name,
loginToken=self.token,
)
LOGGER.info("\t- %s", project_url)
else:
url = url_join(
self.api_client._get_url_server(),
self.workspace,
query="mlflow",
loginToken=self.token,
)
LOGGER.info("\t- %s", url)
LOGGER.info(
"Get deeper instrumentation by adding Comet SDK to your project: https://comet.ml/docs/python-sdk/mlflow/"
)
LOGGER.info("")
def prepare_single_mlflow_run(self, run, original_experiment_name):
self.tmpdir = tempfile.mkdtemp()
if not run.info.end_time:
# Seems to be the case when using the optimizer, some runs doesn't have an end_time
LOGGER.warning("### Skipping run, no end time")
return False
run_start_time = run.info.start_time
messages_file_path = os.path.join(self.tmpdir, "messages.json")
with JsonLinesFile(messages_file_path, self.tmpdir) as json_writer:
# Get mlflow tags
tags = run.data.tags
if not tags:
tags = {}
LOGGER.debug("### Preparing env details")
json_writer.write_filename_msg(tags["mlflow.source.name"],
run_start_time)
json_writer.write_user_msg(tags["mlflow.user"], run_start_time)
LOGGER.debug("### Preparing git details")
json_writer.write_git_meta_msg(
tags.get("mlflow.source.git.commit"),
tags.get("mlflow.source.git.repoURL"),
run_start_time,
)
# Import any custom name
if tags.get("mlflow.runName"):
tags["Name"] = tags["mlflow.runName"]
# Save the run id as tag too as Experiment id can be different in case
# of multiple uploads
tags["mlflow.runId"] = run.info.run_id
if tags.get("mlflow.parentRunId"):
base_url = url_join(self.api_client.server_url,
"/api/experiment/redirect")
tags["mlflow.parentRunUrl"] = format_url(
base_url, experimentKey=tags["mlflow.parentRunId"])
# Save the original MLFlow experiment name too as Comet.ml project might
# get renamed
tags["mlflow.experimentName"] = original_experiment_name
LOGGER.debug("### Importing tags")
for tag_name, tag_value in tags.items():
LOGGER.debug("#### Tag %r: %r", tag_name, tag_value)
json_writer.write_log_other_msg(tag_name, tag_value,
run_start_time)
self.summary["tags"] += 1
# Mark the experiments has being uploaded from MLFlow
json_writer.write_log_other_msg("Uploaded from", "MLFlow",
run_start_time)
LOGGER.debug("### Importing params")
for param_key, param_value in run.data.params.items():
LOGGER.debug("#### Param %r: %r", param_key, param_value)
json_writer.write_param_msg(param_key, param_value,
run_start_time)
self.summary["params"] += 1
LOGGER.debug("### Importing metrics")
for metric in run.data._metric_objs:
metric_history = self.store.get_metric_history(
run.info.run_id, metric.key)
# Check if all steps are uniques, if not we don't pass any so the backend
# fallback to the unique timestamp
steps = [mh.step for mh in metric_history]
use_steps = True
if len(set(steps)) != len(metric_history):
LOGGER.warning(
"Non-unique steps detected, importing metrics with wall time instead"
)
use_steps = False
for mh in metric_history:
if use_steps:
step = mh.step
else:
step = None
json_writer.write_metric_msg(mh.key, step, mh.timestamp,
mh.value)
self.summary["metrics"] += 1
LOGGER.debug("#### Metric %r: %r", metric.key, metric_history)
LOGGER.debug("### Importing artifacts")
artifact_store = get_artifact_repository(run.info.artifact_uri)
# List all the registered models if possible
models_prefixes = {}
if self.model_registry_store:
query = "run_id='%s'" % run.info.run_id
registered_models = self.model_registry_store.search_model_versions(
query)
for model in registered_models:
model_relpath = os.path.relpath(model.source,
run.info.artifact_uri)
models_prefixes[model_relpath] = model
for artifact in walk_run_artifacts(artifact_store):
artifact_path = artifact.path
LOGGER.debug("### Artifact %r: %r", artifact, artifact_path)
# Check if the file is an visualization or not
_, extension = os.path.splitext(artifact_path)
local_artifact_path = artifact_store.download_artifacts(
artifact_path)
self.summary["artifacts"] += 1
# Check if it's belonging to one of the registered model
matching_model = None
for model_prefix, model in models_prefixes.items():
if artifact_path.startswith(model_prefix):
matching_model = model
# We should match at most one model
break
if matching_model:
json_writer.log_artifact_as_model(
local_artifact_path,
artifact_path,
run_start_time,
matching_model.registered_model.name,
)
else:
json_writer.log_artifact_as_asset(
local_artifact_path,
artifact_path,
run_start_time,
)
return self.compress_archive(run.info.run_id)
def check(args, rest=None):
# Called via `comet upload EXP.zip`
if args.debug:
activate_debug()
config = get_config()
LOGGER.info("Comet Check")
LOGGER.info("=" * 80)
print("")
LOGGER.info("Checking connectivity to server...")
print("")
# Clientlib
server_address = sanitize_url(config["comet.url_override"])
server_address_config_origin = config_source(
config.get_config_origin("comet.url_override"))
LOGGER.info("Configured server address %r", server_address)
if server_address_config_origin:
LOGGER.info("Server address was configured in %s",
server_address_config_origin)
else:
LOGGER.info("Server address is the default one")
print("")
server_connected = check_server_connection(server_address)
print("")
if server_connected:
LOGGER.info("Server connection is ok")
else:
LOGGER.warning("Server connection is not ok")
# Rest API
LOGGER.info("=" * 80)
LOGGER.info("Checking connectivity to Rest API...")
LOGGER.info("=" * 80)
root_url = sanitize_url(get_root_url(config["comet.url_override"]))
rest_api_url = url_join(root_url, *["api/rest/", "v2" + "/"])
LOGGER.info("Configured Rest API address %r", rest_api_url)
if server_address_config_origin:
LOGGER.info("Rest API address was configured in %s",
server_address_config_origin)
else:
LOGGER.info("Rest API address is the default one")
print("")
rest_api_connected = check_rest_api_connection(rest_api_url)
print("")
if rest_api_connected:
LOGGER.info("REST API connection is ok")
else:
LOGGER.warning("REST API connection is not ok")
# Websocket
LOGGER.info("=" * 80)
LOGGER.info("Checking connectivity to Websocket Server")
LOGGER.info("=" * 80)
websocket_url = config["comet.ws_url_override"]
if websocket_url is None:
websocket_url = get_default_ws_url(server_address)
LOGGER.warning(
"No WS address configured on client side, fallbacking on default WS address %r, if that's incorrect set the WS url through the `comet.ws_url_override` config key",
websocket_url,
)
websocket_url_config_origin = None
else:
websocket_url = websocket_url
websocket_url_config_origin = config_source(
config.get_config_origin("comet.ws_url_override"))
LOGGER.info(
"Configured WS address %r",
websocket_url,
)
if websocket_url_config_origin:
LOGGER.info("WS address was configured in %s",
websocket_url_config_origin)
print("")
ws_connected = check_ws_connection(websocket_url, args.debug)
print("")
if ws_connected:
LOGGER.info("Websocket connection is ok")
else:
LOGGER.warning("Websocket connection is not ok")
# Optimizer
LOGGER.info("=" * 80)
LOGGER.info("Checking connectivity to Optimizer Server")
LOGGER.info("=" * 80)
optimizer_url = sanitize_url(config["comet.optimizer_url"])
optimizer_url_config_origin = config_source(
config.get_config_origin("comet.optimizer_url"))
LOGGER.info(
"Configured Optimizer address %r",
optimizer_url,
)
if optimizer_url_config_origin:
LOGGER.info("Optimizer address was configured in %s",
optimizer_url_config_origin)
else:
LOGGER.info("Optimizer address is the default one")
print("")
optimizer_connected = check_optimizer_connection(optimizer_url)
print("")
if optimizer_connected:
LOGGER.info("Optimizer connection is ok")
else:
LOGGER.warning("Optimizer connection is not ok")
# Predictor
LOGGER.info("=" * 80)
LOGGER.info("Checking connectivity to Predictor Server")
LOGGER.info("=" * 80)
predictor_url = sanitize_url(config["comet.predictor_url"])
predictor_url_config_origin = config_source(
config.get_config_origin("comet.predictor_url"))
LOGGER.info(
"Configured Predictor address %r",
predictor_url,
)
if predictor_url_config_origin:
LOGGER.info("Predictor address was configured in %s",
predictor_url_config_origin)
else:
LOGGER.info("Predictor address is the default one")
print("")
predictor_connected = check_predictor_connection(predictor_url)
print("")
if predictor_connected:
LOGGER.info("Predictor connection is ok")
else:
LOGGER.warning("Predictor connection is not ok")
print("")
print("")
LOGGER.info("Summary")
LOGGER.info("-" * 80)
LOGGER.info("Server connectivity\t\t\t%s", server_connected)
LOGGER.info("Rest API connectivity\t\t%r", rest_api_connected)
LOGGER.info("WS server connectivity\t\t%r", ws_connected)
LOGGER.info("Optimizer server connectivity\t%r", optimizer_connected)
LOGGER.info("Predictor server connectivity\t%r", predictor_connected)