def _provision_cluster(cluster_id):
ctx, cluster, plugin = _prepare_provisioning(cluster_id)
if CONF.use_identity_api_v3 and cluster.is_transient:
trusts.create_trust_for_cluster(cluster)
# updating cluster infra
cluster = g.change_cluster_status(cluster, "InfraUpdating")
plugin.update_infra(cluster)
# creating instances and configuring them
cluster = conductor.cluster_get(ctx, cluster_id)
INFRA.create_cluster(cluster)
if not g.check_cluster_exists(cluster):
LOG.info(g.format_cluster_deleted_message(cluster))
return
# configure cluster
cluster = g.change_cluster_status(cluster, "Configuring")
try:
plugin.configure_cluster(cluster)
except Exception as ex:
if not g.check_cluster_exists(cluster):
LOG.info(g.format_cluster_deleted_message(cluster))
return
LOG.exception(
_LE("Can't configure cluster '%(name)s' (reason: %(reason)s)"),
{'name': cluster.name, 'reason': ex})
g.change_cluster_status(cluster, "Error")
return
if not g.check_cluster_exists(cluster):
LOG.info(g.format_cluster_deleted_message(cluster))
return
# starting prepared and configured cluster
cluster = g.change_cluster_status(cluster, "Starting")
try:
plugin.start_cluster(cluster)
except Exception as ex:
if not g.check_cluster_exists(cluster):
LOG.info(g.format_cluster_deleted_message(cluster))
return
LOG.exception(
_LE("Can't start services for cluster '%(name)s' (reason: "
"%(reason)s)"), {'name': cluster.name, 'reason': ex})
g.change_cluster_status(cluster, "Error")
return
if not g.check_cluster_exists(cluster):
LOG.info(g.format_cluster_deleted_message(cluster))
return
# cluster is now up and ready
cluster = g.change_cluster_status(cluster, "Active")
# schedule execution pending job for cluster
for je in conductor.job_execution_get_all(ctx, cluster_id=cluster.id):
job_manager.run_job(je.id)
def _provision_cluster(cluster_id):
ctx, cluster, plugin = _prepare_provisioning(cluster_id)
cluster = _update_sahara_info(ctx, cluster)
if CONF.use_identity_api_v3 and cluster.is_transient:
trusts.create_trust_for_cluster(cluster)
# updating cluster infra
cluster = g.change_cluster_status(cluster, "InfraUpdating")
plugin.update_infra(cluster)
# creating instances and configuring them
cluster = conductor.cluster_get(ctx, cluster_id)
INFRA.create_cluster(cluster)
# configure cluster
cluster = g.change_cluster_status(cluster, "Configuring")
plugin.configure_cluster(cluster)
# starting prepared and configured cluster
cluster = g.change_cluster_status(cluster, "Starting")
plugin.start_cluster(cluster)
# cluster is now up and ready
cluster = g.change_cluster_status(cluster, "Active")
# schedule execution pending job for cluster
for je in conductor.job_execution_get_all(ctx, cluster_id=cluster.id):
job_manager.run_job(je.id)
def _provision_cluster(cluster_id):
ctx, cluster, plugin = _prepare_provisioning(cluster_id)
cluster = _update_sahara_info(ctx, cluster)
if CONF.use_identity_api_v3 and cluster.is_transient:
trusts.create_trust_for_cluster(cluster)
# updating cluster infra
cluster = g.change_cluster_status(cluster, "InfraUpdating")
plugin.update_infra(cluster)
# creating instances and configuring them
cluster = conductor.cluster_get(ctx, cluster_id)
INFRA.create_cluster(cluster)
# configure cluster
cluster = g.change_cluster_status(cluster, "Configuring")
plugin.configure_cluster(cluster)
# starting prepared and configured cluster
cluster = g.change_cluster_status(cluster, "Starting")
plugin.start_cluster(cluster)
# cluster is now up and ready
cluster = g.change_cluster_status(cluster, "Active")
# schedule execution pending job for cluster
for je in conductor.job_execution_get_all(ctx, cluster_id=cluster.id):
job_manager.run_job(je.id)
def test_run_job_handles_exceptions(self, runjob, job_ex_upd):
runjob.side_effect = ex.SwiftClientException("Unauthorised")
job, job_exec = u.create_job_exec(edp.JOB_TYPE_PIG)
job_manager.run_job(job_exec.id)
self.assertEqual(1, job_ex_upd.call_count)
new_status = job_ex_upd.call_args[0][2]["info"]["status"]
self.assertEqual(edp.JOB_STATUS_FAILED, new_status)
def test_run_job_handles_exceptions(self, runjob, job_ex_upd):
runjob.side_effect = ex.SwiftClientException("Unauthorised")
job, job_exec = u.create_job_exec(edp.JOB_TYPE_PIG)
job_manager.run_job(job_exec.id)
self.assertEqual(1, job_ex_upd.call_count)
new_status = job_ex_upd.call_args[0][2]["info"]["status"]
self.assertEqual(edp.JOB_STATUS_FAILED, new_status)
def test_run_job_handles_exceptions_with_run_job(self, canceljob, runjob, job_ex_get, job_ex_upd):
runjob.side_effect = ex.OozieException("run_job failed")
job, job_exec = u.create_job_exec(edp.JOB_TYPE_PIG)
job_exec.engine_job_id = "fake_oozie_id"
job_ex_get.return_value = job_exec
job_manager.run_job(job_exec.id)
self.assertEqual(1, job_ex_get.call_count)
self.assertEqual(1, job_ex_upd.call_count)
new_status = job_ex_upd.call_args[0][2]["info"]["status"]
self.assertEqual(edp.JOB_STATUS_FAILED, new_status)
self.assertEqual(1, canceljob.call_count)
def test_run_job_handles_exceptions_with_run_job(self, canceljob, runjob,
job_ex_get, job_ex_upd):
runjob.side_effect = ex.OozieException("run_job failed")
job, job_exec = u.create_job_exec(edp.JOB_TYPE_PIG)
job_exec.engine_job_id = "fake_oozie_id"
job_ex_get.return_value = job_exec
job_manager.run_job(job_exec.id)
self.assertEqual(1, job_ex_get.call_count)
self.assertEqual(1, job_ex_upd.call_count)
new_status = job_ex_upd.call_args[0][2]["info"]["status"]
self.assertEqual(edp.JOB_STATUS_FAILED, new_status)
self.assertEqual(1, canceljob.call_count)
def _provision_cluster(cluster_id):
ctx, cluster, plugin = _prepare_provisioning(cluster_id)
if CONF.use_identity_api_v3 and cluster.is_transient:
trusts.create_trust(cluster)
# updating cluster infra
cluster = conductor.cluster_update(ctx, cluster,
{"status": "InfraUpdating"})
LOG.info(g.format_cluster_status(cluster))
plugin.update_infra(cluster)
# creating instances and configuring them
cluster = conductor.cluster_get(ctx, cluster_id)
INFRA.create_cluster(cluster)
# configure cluster
cluster = conductor.cluster_update(ctx, cluster, {"status": "Configuring"})
LOG.info(g.format_cluster_status(cluster))
try:
plugin.configure_cluster(cluster)
except Exception as ex:
LOG.exception("Can't configure cluster '%s' (reason: %s)",
cluster.name, ex)
cluster = conductor.cluster_update(ctx, cluster, {"status": "Error"})
LOG.info(g.format_cluster_status(cluster))
return
# starting prepared and configured cluster
cluster = conductor.cluster_update(ctx, cluster, {"status": "Starting"})
LOG.info(g.format_cluster_status(cluster))
try:
plugin.start_cluster(cluster)
except Exception as ex:
LOG.exception("Can't start services for cluster '%s' (reason: %s)",
cluster.name, ex)
cluster = conductor.cluster_update(ctx, cluster, {"status": "Error"})
LOG.info(g.format_cluster_status(cluster))
return
# cluster is now up and ready
cluster = conductor.cluster_update(ctx, cluster, {"status": "Active"})
LOG.info(g.format_cluster_status(cluster))
# schedule execution pending job for cluster
for je in conductor.job_execution_get_all(ctx, cluster_id=cluster.id):
job_manager.run_job(je.id)
def _provision_cluster(cluster_id):
ctx, cluster, plugin = _prepare_provisioning(cluster_id)
if CONF.use_identity_api_v3 and cluster.is_transient:
trusts.create_trust(cluster)
# updating cluster infra
cluster = conductor.cluster_update(ctx, cluster,
{"status": "InfraUpdating"})
LOG.info(g.format_cluster_status(cluster))
plugin.update_infra(cluster)
# creating instances and configuring them
cluster = conductor.cluster_get(ctx, cluster_id)
INFRA.create_cluster(cluster)
# configure cluster
cluster = conductor.cluster_update(ctx, cluster, {"status": "Configuring"})
LOG.info(g.format_cluster_status(cluster))
try:
plugin.configure_cluster(cluster)
except Exception as ex:
LOG.exception("Can't configure cluster '%s' (reason: %s)",
cluster.name, ex)
cluster = conductor.cluster_update(ctx, cluster, {"status": "Error"})
LOG.info(g.format_cluster_status(cluster))
return
# starting prepared and configured cluster
cluster = conductor.cluster_update(ctx, cluster, {"status": "Starting"})
LOG.info(g.format_cluster_status(cluster))
try:
plugin.start_cluster(cluster)
except Exception as ex:
LOG.exception("Can't start services for cluster '%s' (reason: %s)",
cluster.name, ex)
cluster = conductor.cluster_update(ctx, cluster, {"status": "Error"})
LOG.info(g.format_cluster_status(cluster))
return
# cluster is now up and ready
cluster = conductor.cluster_update(ctx, cluster, {"status": "Active"})
LOG.info(g.format_cluster_status(cluster))
# schedule execution pending job for cluster
for je in conductor.job_execution_get_all(ctx, cluster_id=cluster.id):
job_manager.run_job(je.id)
def _provision_cluster(cluster_id):
ctx = context.ctx()
cluster = conductor.cluster_get(ctx, cluster_id)
plugin = plugin_base.PLUGINS.get_plugin(cluster.plugin_name)
# updating cluster infra
cluster = conductor.cluster_update(ctx, cluster,
{"status": "InfraUpdating"})
LOG.info(g.format_cluster_status(cluster))
plugin.update_infra(cluster)
# creating instances and configuring them
cluster = conductor.cluster_get(ctx, cluster_id)
INFRA.create_cluster(cluster)
# configure cluster
cluster = conductor.cluster_update(ctx, cluster, {"status": "Configuring"})
LOG.info(g.format_cluster_status(cluster))
try:
plugin.configure_cluster(cluster)
except Exception as ex:
LOG.exception("Can't configure cluster '%s' (reason: %s)",
cluster.name, ex)
cluster = conductor.cluster_update(ctx, cluster, {"status": "Error"})
LOG.info(g.format_cluster_status(cluster))
return
# starting prepared and configured cluster
cluster = conductor.cluster_update(ctx, cluster, {"status": "Starting"})
LOG.info(g.format_cluster_status(cluster))
try:
plugin.start_cluster(cluster)
except Exception as ex:
LOG.exception("Can't start services for cluster '%s' (reason: %s)",
cluster.name, ex)
cluster = conductor.cluster_update(ctx, cluster, {"status": "Error"})
LOG.info(g.format_cluster_status(cluster))
return
# cluster is now up and ready
cluster = conductor.cluster_update(ctx, cluster, {"status": "Active"})
LOG.info(g.format_cluster_status(cluster))
# schedule execution pending job for cluster
for je in conductor.job_execution_get_all(ctx, cluster_id=cluster.id):
jm.run_job(je)
def _provision_cluster(cluster_id):
ctx = context.ctx()
cluster = conductor.cluster_get(ctx, cluster_id)
plugin = plugin_base.PLUGINS.get_plugin(cluster.plugin_name)
# updating cluster infra
cluster = conductor.cluster_update(ctx, cluster,
{"status": "InfraUpdating"})
LOG.info(g.format_cluster_status(cluster))
plugin.update_infra(cluster)
# creating instances and configuring them
cluster = conductor.cluster_get(ctx, cluster_id)
INFRA.create_cluster(cluster)
# configure cluster
cluster = conductor.cluster_update(ctx, cluster, {"status": "Configuring"})
LOG.info(g.format_cluster_status(cluster))
try:
plugin.configure_cluster(cluster)
except Exception as ex:
LOG.exception("Can't configure cluster '%s' (reason: %s)",
cluster.name, ex)
cluster = conductor.cluster_update(ctx, cluster, {"status": "Error"})
LOG.info(g.format_cluster_status(cluster))
return
# starting prepared and configured cluster
cluster = conductor.cluster_update(ctx, cluster, {"status": "Starting"})
LOG.info(g.format_cluster_status(cluster))
try:
plugin.start_cluster(cluster)
except Exception as ex:
LOG.exception("Can't start services for cluster '%s' (reason: %s)",
cluster.name, ex)
cluster = conductor.cluster_update(ctx, cluster, {"status": "Error"})
LOG.info(g.format_cluster_status(cluster))
return
# cluster is now up and ready
cluster = conductor.cluster_update(ctx, cluster, {"status": "Active"})
LOG.info(g.format_cluster_status(cluster))
# schedule execution pending job for cluster
for je in conductor.job_execution_get_all(ctx, cluster_id=cluster.id):
jm.run_job(je)
def _provision_cluster(cluster_id):
ctx, cluster, plugin = _prepare_provisioning(cluster_id)
try:
cluster = _update_sahara_info(ctx, cluster)
# updating cluster infra
cluster = c_u.change_cluster_status(
cluster, c_u.CLUSTER_STATUS_INFRAUPDATING)
plugin.update_infra(cluster)
# creating instances and configuring them
cluster = conductor.cluster_get(ctx, cluster_id)
context.set_step_type(_("Engine: create cluster"))
INFRA.create_cluster(cluster)
# configure cluster
cluster = c_u.change_cluster_status(
cluster, c_u.CLUSTER_STATUS_CONFIGURING)
shares.mount_shares(cluster)
context.set_step_type(_("Plugin: configure cluster"))
plugin.configure_cluster(cluster)
# starting prepared and configured cluster
ntp_service.configure_ntp(cluster_id)
cluster = c_u.change_cluster_status(
cluster, c_u.CLUSTER_STATUS_STARTING)
context.set_step_type(_("Plugin: start cluster"))
plugin.start_cluster(cluster)
# cluster is now up and ready
cluster = c_u.change_cluster_status(
cluster, c_u.CLUSTER_STATUS_ACTIVE)
# schedule execution pending job for cluster
for je in conductor.job_execution_get_all(ctx, cluster_id=cluster.id):
job_manager.run_job(je.id)
finally:
if CONF.use_identity_api_v3 and not cluster.is_transient:
trusts.delete_trust_from_cluster(cluster)
def _provision_cluster(cluster_id):
ctx, cluster, plugin = _prepare_provisioning(cluster_id)
try:
cluster = _update_sahara_info(ctx, cluster)
# updating cluster infra
cluster = c_u.change_cluster_status(cluster,
c_u.CLUSTER_STATUS_INFRAUPDATING)
plugin.update_infra(cluster)
# creating instances and configuring them
cluster = conductor.cluster_get(ctx, cluster_id)
context.set_step_type(_("Engine: create cluster"))
INFRA.create_cluster(cluster)
# configure cluster
cluster = c_u.change_cluster_status(cluster,
c_u.CLUSTER_STATUS_CONFIGURING)
shares.mount_shares(cluster)
context.set_step_type(_("Plugin: configure cluster"))
plugin.configure_cluster(cluster)
# starting prepared and configured cluster
ntp_service.configure_ntp(cluster_id)
cluster = c_u.change_cluster_status(cluster,
c_u.CLUSTER_STATUS_STARTING)
context.set_step_type(_("Plugin: start cluster"))
plugin.start_cluster(cluster)
# cluster is now up and ready
cluster = c_u.change_cluster_status(cluster, c_u.CLUSTER_STATUS_ACTIVE)
# schedule execution pending job for cluster
for je in conductor.job_execution_get_all(ctx, cluster_id=cluster.id):
job_manager.run_job(je.id)
finally:
if CONF.use_identity_api_v3 and not cluster.is_transient:
trusts.delete_trust_from_cluster(cluster)
def _provision_cluster(cluster_id):
ctx, cluster, plugin = _prepare_provisioning(cluster_id)
cluster = _update_sahara_info(ctx, cluster)
# updating cluster infra
cluster = c_u.change_cluster_status(
cluster, c_u.CLUSTER_STATUS_INFRAUPDATING)
plugin.update_infra(cluster)
# creating instances and configuring them
cluster = conductor.cluster_get(ctx, cluster_id)
context.set_step_type(_("Engine: create cluster"))
INFRA.create_cluster(cluster)
ntp_service.configure_ntp(cluster_id)
# configure cluster
cluster = c_u.change_cluster_status(
cluster, c_u.CLUSTER_STATUS_CONFIGURING)
context.set_step_type(_("Plugin: configure cluster"))
if hasattr(plugin, 'validate_images'):
plugin.validate_images(cluster, test_only=False)
shares.mount_shares(cluster)
plugin.configure_cluster(cluster)
# starting prepared and configured cluster
cluster = c_u.change_cluster_status(
cluster, c_u.CLUSTER_STATUS_STARTING)
context.set_step_type(_("Plugin: start cluster"))
plugin.start_cluster(cluster)
# cluster is now up and ready
cluster = c_u.change_cluster_status(
cluster, c_u.CLUSTER_STATUS_ACTIVE)
# schedule execution pending job for cluster
for je in conductor.job_execution_get_all(ctx, cluster_id=cluster.id):
job_manager.run_job(je.id)
_refresh_health_for_cluster(cluster_id)
def _provision_cluster(cluster_id):
ctx, cluster, plugin = _prepare_provisioning(cluster_id)
cluster = _update_sahara_info(ctx, cluster)
# updating cluster infra
cluster = c_u.change_cluster_status(
cluster, c_u.CLUSTER_STATUS_INFRAUPDATING)
plugin.update_infra(cluster)
# creating instances and configuring them
cluster = conductor.cluster_get(ctx, cluster_id)
context.set_step_type(_("Engine: create cluster"))
INFRA.create_cluster(cluster)
# configure cluster
cluster = c_u.change_cluster_status(
cluster, c_u.CLUSTER_STATUS_CONFIGURING)
context.set_step_type(_("Plugin: configure cluster"))
if hasattr(plugin, 'validate_images'):
plugin.validate_images(cluster, test_only=False)
shares.mount_shares(cluster)
plugin.configure_cluster(cluster)
# starting prepared and configured cluster
ntp_service.configure_ntp(cluster_id)
cluster = c_u.change_cluster_status(
cluster, c_u.CLUSTER_STATUS_STARTING)
context.set_step_type(_("Plugin: start cluster"))
plugin.start_cluster(cluster)
# cluster is now up and ready
cluster = c_u.change_cluster_status(
cluster, c_u.CLUSTER_STATUS_ACTIVE)
# schedule execution pending job for cluster
for je in conductor.job_execution_get_all(ctx, cluster_id=cluster.id):
job_manager.run_job(je.id)
_refresh_health_for_cluster(cluster_id)
def _run_edp_job(job_execution_id):
job_manager.run_job(job_execution_id)
def _run_edp_job(job_execution_id):
job_manager.run_job(job_execution_id)