def test_get_stackdriver_group_url_raises(sd_client):
sd_client.list_groups.side_effect = Exception("fuuuuu")
with pytest.raises(Exception):
sd_utils.get_stackdriver_group_url(
"test-gcp-project", "test-job-name-1", "test-region"
)
def _create_stackdriver_group_if_missing(project, job_name, region):
dashboard_url = None
# first see if dashboard is already created for job; otherwise it will
# create a new one of the same name but a new ID
try:
dashboard_url = sd_utils.get_stackdriver_group_url(
project, job_name, region
)
except Exception as e:
msg = (
"Error encountered while trying to determine whether a "
"Stackdriver dashboard already exists for this job: {}. "
"Skipping...".format(e)
)
logging.warning(msg)
return
if dashboard_url:
msg = (
"Reusing existing dashboard previously created for "
"'{}': {}.".format(job_name, dashboard_url)
)
logging.info(msg)
else:
# nothing already exists; we'll create one now
dashboard_url = sd_utils.create_stackdriver_group(
project, job_name, region
)
return dashboard_url
def _verify_stackdriver_dashboard(self):
logging.info("Verifying Stackdriver Dashboard...")
dashboard_url = None
try:
dashboard_url = sd_utils.get_stackdriver_group_url(
self.project, self.job_name, self.region)
# Raising a general exception (caught in verify_job) since we're unsure
# of how to proceed
except Exception as e:
msg = ("Error encountered while trying to determine whether a "
"Stackdriver dashboard already exists for this job: . "
"{}".format(e))
logging.error(msg)
raise e
if dashboard_url:
logging.info(
"Stackdriver Dashboard exists at {}".format(dashboard_url))
return True
elif self.create_resources:
logging.info("Creating Stackdriver Dashboard...")
# nothing already exists; we'll create one now
dashboard_url = sd_utils.create_stackdriver_group(
self.project, self.job_name, self.region)
if dashboard_url: # could still be none
logging.info("Stackdriver Dashboard created at {}".format(
dashboard_url))
return True
# reason why dashboard couldn't be created will already be logged
logging.error("Could not create Stackdriver Dashboard.")
else:
logging.info("No Stackdriver Dashboard exists.")
return False
def _run_docker_container(self, runflags):
container = self._docker_client.containers.run(**runflags)
if self.run_job_config.direct_runner:
atexit.register(self._try_container_kill, container)
# TODO: container.logs(stream=True) redirects stderr to stdout.
# We should use appropriate streams so it's obvious to the use.
# (@jpvelez)
for line in container.logs(stream=True):
self._docker_logger.info(line.decode("utf-8").strip("\n"))
exit_status = container.wait()["StatusCode"]
if exit_status == 0 and not self.run_job_config.direct_runner:
dashboard_name = sd_utils.DASHBOARD_NAME_TPL.format(
job_name=self.klio_config.job_name,
region=self.klio_config.pipeline_options.region,
)
base_err_msg = (
"Could not find a Stackdriver dashboard for job '%s' that "
"matched the name %s"
% (self.klio_config.job_name, dashboard_name)
)
try:
dashboard_url = sd_utils.get_stackdriver_group_url(
self.klio_config.pipeline_options.project,
self.klio_config.job_name,
self.klio_config.pipeline_options.region,
)
except Exception as e:
logging.warning("%s: %s" % (base_err_msg, e))
else:
if dashboard_url:
logging.info(
"View the job's dashboard on Stackdriver: %s"
% dashboard_url
)
else:
logging.warning(base_err_msg)
return exit_status
def test_get_stackdriver_group_url(sd_client, groups, job_name, exp_ret):
sd_client.list_groups.return_value = groups
ret_url = sd_utils.get_stackdriver_group_url(
"test-gcp-project", job_name, "test-region"
)
exp_url = None
if exp_ret:
exp_url = (
"https://app.google.stackdriver.com/groups/1234567891/"
"test-job-name-1-test-region-klio-dashboard?"
"project=test-gcp-project"
)
assert exp_url == ret_url
sd_client.list_groups.assert_called_once_with(
request={"name": "projects/test-gcp-project"}
)