def _provision_cluster(cluster_id): ctx, cluster, plugin = _prepare_provisioning(cluster_id) if CONF.use_identity_api_v3 and cluster.is_transient: trusts.create_trust_for_cluster(cluster) # updating cluster infra cluster = g.change_cluster_status(cluster, "InfraUpdating") plugin.update_infra(cluster) # creating instances and configuring them cluster = conductor.cluster_get(ctx, cluster_id) INFRA.create_cluster(cluster) if not g.check_cluster_exists(cluster): LOG.info(g.format_cluster_deleted_message(cluster)) return # configure cluster cluster = g.change_cluster_status(cluster, "Configuring") try: plugin.configure_cluster(cluster) except Exception as ex: if not g.check_cluster_exists(cluster): LOG.info(g.format_cluster_deleted_message(cluster)) return LOG.exception( _LE("Can't configure cluster '%(name)s' (reason: %(reason)s)"), {'name': cluster.name, 'reason': ex}) g.change_cluster_status(cluster, "Error") return if not g.check_cluster_exists(cluster): LOG.info(g.format_cluster_deleted_message(cluster)) return # starting prepared and configured cluster cluster = g.change_cluster_status(cluster, "Starting") try: plugin.start_cluster(cluster) except Exception as ex: if not g.check_cluster_exists(cluster): LOG.info(g.format_cluster_deleted_message(cluster)) return LOG.exception( _LE("Can't start services for cluster '%(name)s' (reason: " "%(reason)s)"), {'name': cluster.name, 'reason': ex}) g.change_cluster_status(cluster, "Error") return if not g.check_cluster_exists(cluster): LOG.info(g.format_cluster_deleted_message(cluster)) return # cluster is now up and ready cluster = g.change_cluster_status(cluster, "Active") # schedule execution pending job for cluster for je in conductor.job_execution_get_all(ctx, cluster_id=cluster.id): job_manager.run_job(je.id)
def _provision_cluster(cluster_id): ctx, cluster, plugin = _prepare_provisioning(cluster_id) cluster = _update_sahara_info(ctx, cluster) if CONF.use_identity_api_v3 and cluster.is_transient: trusts.create_trust_for_cluster(cluster) # updating cluster infra cluster = g.change_cluster_status(cluster, "InfraUpdating") plugin.update_infra(cluster) # creating instances and configuring them cluster = conductor.cluster_get(ctx, cluster_id) INFRA.create_cluster(cluster) # configure cluster cluster = g.change_cluster_status(cluster, "Configuring") plugin.configure_cluster(cluster) # starting prepared and configured cluster cluster = g.change_cluster_status(cluster, "Starting") plugin.start_cluster(cluster) # cluster is now up and ready cluster = g.change_cluster_status(cluster, "Active") # schedule execution pending job for cluster for je in conductor.job_execution_get_all(ctx, cluster_id=cluster.id): job_manager.run_job(je.id)
def _provision_cluster(cluster_id): ctx, cluster, plugin = _prepare_provisioning(cluster_id) cluster = _update_sahara_info(ctx, cluster) if CONF.use_identity_api_v3 and cluster.is_transient: trusts.create_trust_for_cluster(cluster) # updating cluster infra cluster = g.change_cluster_status(cluster, "InfraUpdating") plugin.update_infra(cluster) # creating instances and configuring them cluster = conductor.cluster_get(ctx, cluster_id) INFRA.create_cluster(cluster) # configure cluster cluster = g.change_cluster_status(cluster, "Configuring") plugin.configure_cluster(cluster) # starting prepared and configured cluster cluster = g.change_cluster_status(cluster, "Starting") plugin.start_cluster(cluster) # cluster is now up and ready cluster = g.change_cluster_status(cluster, "Active") # schedule execution pending job for cluster for je in conductor.job_execution_get_all(ctx, cluster_id=cluster.id): job_manager.run_job(je.id)
def test_run_job_handles_exceptions(self, runjob, job_ex_upd): runjob.side_effect = ex.SwiftClientException("Unauthorised") job, job_exec = u.create_job_exec(edp.JOB_TYPE_PIG) job_manager.run_job(job_exec.id) self.assertEqual(1, job_ex_upd.call_count) new_status = job_ex_upd.call_args[0][2]["info"]["status"] self.assertEqual(edp.JOB_STATUS_FAILED, new_status)
def test_run_job_handles_exceptions(self, runjob, job_ex_upd): runjob.side_effect = ex.SwiftClientException("Unauthorised") job, job_exec = u.create_job_exec(edp.JOB_TYPE_PIG) job_manager.run_job(job_exec.id) self.assertEqual(1, job_ex_upd.call_count) new_status = job_ex_upd.call_args[0][2]["info"]["status"] self.assertEqual(edp.JOB_STATUS_FAILED, new_status)
def test_run_job_handles_exceptions_with_run_job(self, canceljob, runjob, job_ex_get, job_ex_upd): runjob.side_effect = ex.OozieException("run_job failed") job, job_exec = u.create_job_exec(edp.JOB_TYPE_PIG) job_exec.engine_job_id = "fake_oozie_id" job_ex_get.return_value = job_exec job_manager.run_job(job_exec.id) self.assertEqual(1, job_ex_get.call_count) self.assertEqual(1, job_ex_upd.call_count) new_status = job_ex_upd.call_args[0][2]["info"]["status"] self.assertEqual(edp.JOB_STATUS_FAILED, new_status) self.assertEqual(1, canceljob.call_count)
def test_run_job_handles_exceptions_with_run_job(self, canceljob, runjob, job_ex_get, job_ex_upd): runjob.side_effect = ex.OozieException("run_job failed") job, job_exec = u.create_job_exec(edp.JOB_TYPE_PIG) job_exec.engine_job_id = "fake_oozie_id" job_ex_get.return_value = job_exec job_manager.run_job(job_exec.id) self.assertEqual(1, job_ex_get.call_count) self.assertEqual(1, job_ex_upd.call_count) new_status = job_ex_upd.call_args[0][2]["info"]["status"] self.assertEqual(edp.JOB_STATUS_FAILED, new_status) self.assertEqual(1, canceljob.call_count)
def _provision_cluster(cluster_id): ctx, cluster, plugin = _prepare_provisioning(cluster_id) if CONF.use_identity_api_v3 and cluster.is_transient: trusts.create_trust(cluster) # updating cluster infra cluster = conductor.cluster_update(ctx, cluster, {"status": "InfraUpdating"}) LOG.info(g.format_cluster_status(cluster)) plugin.update_infra(cluster) # creating instances and configuring them cluster = conductor.cluster_get(ctx, cluster_id) INFRA.create_cluster(cluster) # configure cluster cluster = conductor.cluster_update(ctx, cluster, {"status": "Configuring"}) LOG.info(g.format_cluster_status(cluster)) try: plugin.configure_cluster(cluster) except Exception as ex: LOG.exception("Can't configure cluster '%s' (reason: %s)", cluster.name, ex) cluster = conductor.cluster_update(ctx, cluster, {"status": "Error"}) LOG.info(g.format_cluster_status(cluster)) return # starting prepared and configured cluster cluster = conductor.cluster_update(ctx, cluster, {"status": "Starting"}) LOG.info(g.format_cluster_status(cluster)) try: plugin.start_cluster(cluster) except Exception as ex: LOG.exception("Can't start services for cluster '%s' (reason: %s)", cluster.name, ex) cluster = conductor.cluster_update(ctx, cluster, {"status": "Error"}) LOG.info(g.format_cluster_status(cluster)) return # cluster is now up and ready cluster = conductor.cluster_update(ctx, cluster, {"status": "Active"}) LOG.info(g.format_cluster_status(cluster)) # schedule execution pending job for cluster for je in conductor.job_execution_get_all(ctx, cluster_id=cluster.id): job_manager.run_job(je.id)
def _provision_cluster(cluster_id): ctx, cluster, plugin = _prepare_provisioning(cluster_id) if CONF.use_identity_api_v3 and cluster.is_transient: trusts.create_trust(cluster) # updating cluster infra cluster = conductor.cluster_update(ctx, cluster, {"status": "InfraUpdating"}) LOG.info(g.format_cluster_status(cluster)) plugin.update_infra(cluster) # creating instances and configuring them cluster = conductor.cluster_get(ctx, cluster_id) INFRA.create_cluster(cluster) # configure cluster cluster = conductor.cluster_update(ctx, cluster, {"status": "Configuring"}) LOG.info(g.format_cluster_status(cluster)) try: plugin.configure_cluster(cluster) except Exception as ex: LOG.exception("Can't configure cluster '%s' (reason: %s)", cluster.name, ex) cluster = conductor.cluster_update(ctx, cluster, {"status": "Error"}) LOG.info(g.format_cluster_status(cluster)) return # starting prepared and configured cluster cluster = conductor.cluster_update(ctx, cluster, {"status": "Starting"}) LOG.info(g.format_cluster_status(cluster)) try: plugin.start_cluster(cluster) except Exception as ex: LOG.exception("Can't start services for cluster '%s' (reason: %s)", cluster.name, ex) cluster = conductor.cluster_update(ctx, cluster, {"status": "Error"}) LOG.info(g.format_cluster_status(cluster)) return # cluster is now up and ready cluster = conductor.cluster_update(ctx, cluster, {"status": "Active"}) LOG.info(g.format_cluster_status(cluster)) # schedule execution pending job for cluster for je in conductor.job_execution_get_all(ctx, cluster_id=cluster.id): job_manager.run_job(je.id)
def _provision_cluster(cluster_id): ctx = context.ctx() cluster = conductor.cluster_get(ctx, cluster_id) plugin = plugin_base.PLUGINS.get_plugin(cluster.plugin_name) # updating cluster infra cluster = conductor.cluster_update(ctx, cluster, {"status": "InfraUpdating"}) LOG.info(g.format_cluster_status(cluster)) plugin.update_infra(cluster) # creating instances and configuring them cluster = conductor.cluster_get(ctx, cluster_id) INFRA.create_cluster(cluster) # configure cluster cluster = conductor.cluster_update(ctx, cluster, {"status": "Configuring"}) LOG.info(g.format_cluster_status(cluster)) try: plugin.configure_cluster(cluster) except Exception as ex: LOG.exception("Can't configure cluster '%s' (reason: %s)", cluster.name, ex) cluster = conductor.cluster_update(ctx, cluster, {"status": "Error"}) LOG.info(g.format_cluster_status(cluster)) return # starting prepared and configured cluster cluster = conductor.cluster_update(ctx, cluster, {"status": "Starting"}) LOG.info(g.format_cluster_status(cluster)) try: plugin.start_cluster(cluster) except Exception as ex: LOG.exception("Can't start services for cluster '%s' (reason: %s)", cluster.name, ex) cluster = conductor.cluster_update(ctx, cluster, {"status": "Error"}) LOG.info(g.format_cluster_status(cluster)) return # cluster is now up and ready cluster = conductor.cluster_update(ctx, cluster, {"status": "Active"}) LOG.info(g.format_cluster_status(cluster)) # schedule execution pending job for cluster for je in conductor.job_execution_get_all(ctx, cluster_id=cluster.id): jm.run_job(je)
def _provision_cluster(cluster_id): ctx = context.ctx() cluster = conductor.cluster_get(ctx, cluster_id) plugin = plugin_base.PLUGINS.get_plugin(cluster.plugin_name) # updating cluster infra cluster = conductor.cluster_update(ctx, cluster, {"status": "InfraUpdating"}) LOG.info(g.format_cluster_status(cluster)) plugin.update_infra(cluster) # creating instances and configuring them cluster = conductor.cluster_get(ctx, cluster_id) INFRA.create_cluster(cluster) # configure cluster cluster = conductor.cluster_update(ctx, cluster, {"status": "Configuring"}) LOG.info(g.format_cluster_status(cluster)) try: plugin.configure_cluster(cluster) except Exception as ex: LOG.exception("Can't configure cluster '%s' (reason: %s)", cluster.name, ex) cluster = conductor.cluster_update(ctx, cluster, {"status": "Error"}) LOG.info(g.format_cluster_status(cluster)) return # starting prepared and configured cluster cluster = conductor.cluster_update(ctx, cluster, {"status": "Starting"}) LOG.info(g.format_cluster_status(cluster)) try: plugin.start_cluster(cluster) except Exception as ex: LOG.exception("Can't start services for cluster '%s' (reason: %s)", cluster.name, ex) cluster = conductor.cluster_update(ctx, cluster, {"status": "Error"}) LOG.info(g.format_cluster_status(cluster)) return # cluster is now up and ready cluster = conductor.cluster_update(ctx, cluster, {"status": "Active"}) LOG.info(g.format_cluster_status(cluster)) # schedule execution pending job for cluster for je in conductor.job_execution_get_all(ctx, cluster_id=cluster.id): jm.run_job(je)
def _provision_cluster(cluster_id): ctx, cluster, plugin = _prepare_provisioning(cluster_id) try: cluster = _update_sahara_info(ctx, cluster) # updating cluster infra cluster = c_u.change_cluster_status( cluster, c_u.CLUSTER_STATUS_INFRAUPDATING) plugin.update_infra(cluster) # creating instances and configuring them cluster = conductor.cluster_get(ctx, cluster_id) context.set_step_type(_("Engine: create cluster")) INFRA.create_cluster(cluster) # configure cluster cluster = c_u.change_cluster_status( cluster, c_u.CLUSTER_STATUS_CONFIGURING) shares.mount_shares(cluster) context.set_step_type(_("Plugin: configure cluster")) plugin.configure_cluster(cluster) # starting prepared and configured cluster ntp_service.configure_ntp(cluster_id) cluster = c_u.change_cluster_status( cluster, c_u.CLUSTER_STATUS_STARTING) context.set_step_type(_("Plugin: start cluster")) plugin.start_cluster(cluster) # cluster is now up and ready cluster = c_u.change_cluster_status( cluster, c_u.CLUSTER_STATUS_ACTIVE) # schedule execution pending job for cluster for je in conductor.job_execution_get_all(ctx, cluster_id=cluster.id): job_manager.run_job(je.id) finally: if CONF.use_identity_api_v3 and not cluster.is_transient: trusts.delete_trust_from_cluster(cluster)
def _provision_cluster(cluster_id): ctx, cluster, plugin = _prepare_provisioning(cluster_id) try: cluster = _update_sahara_info(ctx, cluster) # updating cluster infra cluster = c_u.change_cluster_status(cluster, c_u.CLUSTER_STATUS_INFRAUPDATING) plugin.update_infra(cluster) # creating instances and configuring them cluster = conductor.cluster_get(ctx, cluster_id) context.set_step_type(_("Engine: create cluster")) INFRA.create_cluster(cluster) # configure cluster cluster = c_u.change_cluster_status(cluster, c_u.CLUSTER_STATUS_CONFIGURING) shares.mount_shares(cluster) context.set_step_type(_("Plugin: configure cluster")) plugin.configure_cluster(cluster) # starting prepared and configured cluster ntp_service.configure_ntp(cluster_id) cluster = c_u.change_cluster_status(cluster, c_u.CLUSTER_STATUS_STARTING) context.set_step_type(_("Plugin: start cluster")) plugin.start_cluster(cluster) # cluster is now up and ready cluster = c_u.change_cluster_status(cluster, c_u.CLUSTER_STATUS_ACTIVE) # schedule execution pending job for cluster for je in conductor.job_execution_get_all(ctx, cluster_id=cluster.id): job_manager.run_job(je.id) finally: if CONF.use_identity_api_v3 and not cluster.is_transient: trusts.delete_trust_from_cluster(cluster)
def _provision_cluster(cluster_id): ctx, cluster, plugin = _prepare_provisioning(cluster_id) cluster = _update_sahara_info(ctx, cluster) # updating cluster infra cluster = c_u.change_cluster_status( cluster, c_u.CLUSTER_STATUS_INFRAUPDATING) plugin.update_infra(cluster) # creating instances and configuring them cluster = conductor.cluster_get(ctx, cluster_id) context.set_step_type(_("Engine: create cluster")) INFRA.create_cluster(cluster) ntp_service.configure_ntp(cluster_id) # configure cluster cluster = c_u.change_cluster_status( cluster, c_u.CLUSTER_STATUS_CONFIGURING) context.set_step_type(_("Plugin: configure cluster")) if hasattr(plugin, 'validate_images'): plugin.validate_images(cluster, test_only=False) shares.mount_shares(cluster) plugin.configure_cluster(cluster) # starting prepared and configured cluster cluster = c_u.change_cluster_status( cluster, c_u.CLUSTER_STATUS_STARTING) context.set_step_type(_("Plugin: start cluster")) plugin.start_cluster(cluster) # cluster is now up and ready cluster = c_u.change_cluster_status( cluster, c_u.CLUSTER_STATUS_ACTIVE) # schedule execution pending job for cluster for je in conductor.job_execution_get_all(ctx, cluster_id=cluster.id): job_manager.run_job(je.id) _refresh_health_for_cluster(cluster_id)
def _provision_cluster(cluster_id): ctx, cluster, plugin = _prepare_provisioning(cluster_id) cluster = _update_sahara_info(ctx, cluster) # updating cluster infra cluster = c_u.change_cluster_status( cluster, c_u.CLUSTER_STATUS_INFRAUPDATING) plugin.update_infra(cluster) # creating instances and configuring them cluster = conductor.cluster_get(ctx, cluster_id) context.set_step_type(_("Engine: create cluster")) INFRA.create_cluster(cluster) # configure cluster cluster = c_u.change_cluster_status( cluster, c_u.CLUSTER_STATUS_CONFIGURING) context.set_step_type(_("Plugin: configure cluster")) if hasattr(plugin, 'validate_images'): plugin.validate_images(cluster, test_only=False) shares.mount_shares(cluster) plugin.configure_cluster(cluster) # starting prepared and configured cluster ntp_service.configure_ntp(cluster_id) cluster = c_u.change_cluster_status( cluster, c_u.CLUSTER_STATUS_STARTING) context.set_step_type(_("Plugin: start cluster")) plugin.start_cluster(cluster) # cluster is now up and ready cluster = c_u.change_cluster_status( cluster, c_u.CLUSTER_STATUS_ACTIVE) # schedule execution pending job for cluster for je in conductor.job_execution_get_all(ctx, cluster_id=cluster.id): job_manager.run_job(je.id) _refresh_health_for_cluster(cluster_id)
def _run_edp_job(job_execution_id): job_manager.run_job(job_execution_id)
def _run_edp_job(job_execution_id): job_manager.run_job(job_execution_id)