def install(self) -> Dict[str, Any]: if sdk_marathon.app_exists(self.app_definition["id"]): if self._persist: log.info("Found installed KDC app, reusing it") return _get_kdc_task(self.app_definition["id"]) log.info("Found installed KDC app, destroying it first") sdk_marathon.destroy_app(self.app_definition["id"]) # (re-)create a service account for the KDC service sdk_security.create_service_account( service_account_name=KDC_SERVICE_ACCOUNT, service_account_secret=KDC_SERVICE_ACCOUNT_SECRET, ) sdk_security._grant( KDC_SERVICE_ACCOUNT, "dcos:secrets:default:%252F*", "Create any secret in the root path", "create", ) sdk_security._grant( KDC_SERVICE_ACCOUNT, "dcos:secrets:default:%252F*", "Update any secret in the root path", "update", ) log.info("Installing KDC Marathon app") sdk_marathon.install_app(self.app_definition) log.info("KDC app installed successfully") log.info("Waiting for KDC web API endpoint to become available") self.__wait_for_kdc_api() log.info("KDC web API is now available") return _get_kdc_task(self.app_definition["id"])
def install( package_name: str, service_name: str, expected_running_tasks: int, additional_options: dict = {}, package_version: PackageVersion = PackageVersion.STUB_UNIVERSE, timeout_seconds: int = TIMEOUT_SECONDS, wait_for_deployment: bool = True, insert_strict_options: bool = True, wait_for_all_conditions: bool = True, ) -> None: start = time.time() # If the package is already installed at this point, fail immediately. if sdk_marathon.app_exists(service_name): raise Exception( "Service is already installed: {}".format(service_name)) if insert_strict_options and sdk_utils.is_strict_mode(): # strict mode requires correct principal and secret to perform install. # see also: sdk_security.py options = sdk_utils.merge_dictionaries( { "service": { "service_account": "service-acct", "principal": "service-acct", "service_account_secret": "secret", "secret_name": "secret", } }, additional_options, ) else: options = additional_options # 1. Install package, wait for tasks, wait for marathon deployment _retried_install_impl( package_name, service_name, expected_running_tasks, package_version.value if isinstance( package_version, PackageVersion) else package_version, options, timeout_seconds, wait_for_all_conditions) # 2. Wait for the scheduler to be idle (as implied by deploy plan completion and suppressed bit) # This should be skipped ONLY when it's known that the scheduler will be stuck in an incomplete # state, or if the thing being installed doesn't have a deployment plan (e.g. standalone app) if wait_for_deployment: # this can take a while, default is 15 minutes. for example with HDFS, we can hit the expected # total task count via FINISHED tasks, without actually completing deployment log.info( "Waiting for package={} service={} to finish deployment plan...". format(package_name, service_name)) sdk_plan.wait_for_completed_deployment(service_name, timeout_seconds) log.info("Installed package={} service={} after {}".format( package_name, service_name, sdk_utils.pretty_duration(time.time() - start))) global _installed_service_names _installed_service_names.add(service_name)
def install( package_name, service_name, expected_running_tasks, additional_options={}, package_version=None, timeout_seconds=TIMEOUT_SECONDS, wait_for_deployment=True, insert_strict_options=True, install_cli=True): start = time.time() # If the package is already installed at this point, fail immediately. if sdk_marathon.app_exists(service_name): raise dcos.errors.DCOSException('Service is already installed: {}'.format(service_name)) if insert_strict_options and sdk_utils.is_strict_mode(): # strict mode requires correct principal and secret to perform install. # see also: sdk_security.py options = merge_dictionaries({ 'service': { 'service_account': 'service-acct', 'principal': 'service-acct', 'service_account_secret': 'secret', 'secret_name': 'secret' } }, additional_options) else: options = additional_options # 1. Install package, wait for tasks, wait for marathon deployment _retried_install_impl( package_name, service_name, expected_running_tasks, options, package_version, timeout_seconds, install_cli) # 2. Wait for the scheduler to be idle (as implied by deploy plan completion and suppressed bit) # This should be skipped ONLY when it's known that the scheduler will be stuck in an incomplete # state, or if the thing being installed doesn't have a deployment plan (e.g. standalone app) if wait_for_deployment: # this can take a while, default is 15 minutes. for example with HDFS, we can hit the expected # total task count via FINISHED tasks, without actually completing deployment log.info('Waiting for package={} service={} to finish deployment plan...'.format( package_name, service_name)) sdk_plan.wait_for_completed_deployment(service_name, timeout_seconds) log.info('Installed package={} service={} after {}'.format( package_name, service_name, shakedown.pretty_duration(time.time() - start))) global _installed_service_names _installed_service_names.add(service_name)
def install( package_name, service_name, expected_running_tasks, additional_options={}, package_version=None, timeout_seconds=TIMEOUT_SECONDS, wait_for_deployment=True, insert_strict_options=True): start = time.time() # If the package is already installed at this point, fail immediately. if sdk_marathon.app_exists(service_name): raise dcos.errors.DCOSException('Service is already installed: {}'.format(service_name)) if insert_strict_options and sdk_utils.is_strict_mode(): # strict mode requires correct principal and secret to perform install. # see also: sdk_security.py options = merge_dictionaries({ 'service': { 'service_account': 'service-acct', 'principal': 'service-acct', 'service_account_secret': 'secret', 'secret_name': 'secret' } }, additional_options) else: options = additional_options # 1. Install package, wait for tasks, wait for marathon deployment _retried_install_impl( package_name, service_name, expected_running_tasks, options, package_version, timeout_seconds) # 2. Wait for the scheduler to be idle (as implied by deploy plan completion and suppressed bit) # This should be skipped ONLY when it's known that the scheduler will be stuck in an incomplete # state, or if the thing being installed doesn't have a deployment plan (e.g. standalone app) if wait_for_deployment: # this can take a while, default is 15 minutes. for example with HDFS, we can hit the expected # total task count via FINISHED tasks, without actually completing deployment log.info('Waiting for package={} service={} to finish deployment plan...'.format( package_name, service_name)) sdk_plan.wait_for_completed_deployment(service_name, timeout_seconds) log.info('Installed package={} service={} after {}'.format( package_name, service_name, shakedown.pretty_duration(time.time() - start))) global _installed_service_names _installed_service_names.add(service_name)
def install(self) -> dict: if sdk_marathon.app_exists(self.app_definition["id"]): if self._persist: log.info("Found installed KDC app, reusing it") return _get_kdc_task(self.app_definition["id"]) log.info("Found installed KDC app, destroying it first") sdk_marathon.destroy_app(self.app_definition["id"]) log.info("Installing KDC Marathon app") sdk_marathon.install_app(self.app_definition) log.info("KDC app installed successfully") return _get_kdc_task(self.app_definition["id"])
def wait_for_removal_log_deploy_plan(): if not sdk_marathon.app_exists(service_name): return True # App still exists, print the deploy plan. Best effort: It is expected for the scheduler # to become unavailable once uninstall completes. try: log.info( sdk_plan.plan_string("deploy", sdk_plan.get_plan_once(service_name, "deploy")) ) except Exception: pass # best effort attempt at logging plan content return False
def cleanup(self): sdk_security.install_enterprise_cli() log.info("Removing the marathon KDC app") if sdk_marathon.app_exists(self.app_definition["id"]): sdk_marathon.destroy_app(self.app_definition["id"]) if self._temp_working_dir and isinstance( self._temp_working_dir, tempfile.TemporaryDirectory): log.info("Deleting temporary working directory") self._temp_working_dir.cleanup() # TODO: separate secrets handling into another module log.info("Deleting keytab secret") sdk_security.delete_secret(self.keytab_secret_path) else: log.info("KDC app doesn't exist, skipping cleanup")
def _retried_install_impl( package_name: str, service_name: str, expected_running_tasks: int, package_version: Optional[str], options: Dict[str, Any], timeout_seconds: int, wait_for_all_conditions: bool, ) -> None: log.info( "Installing package={} service={} with options={} version={}".format( package_name, service_name, options, package_version)) # Trigger package install, but only if it's not already installed. # We expect upstream to have confirmed that it wasn't already installed beforehand. install_cmd = ["package", "install", package_name, "--yes"] if package_version: install_cmd.append("--package-version={}".format(package_version)) if sdk_marathon.app_exists(service_name): log.info( "Marathon app={} exists, ensuring CLI for package={} is installed". format(service_name, package_name)) install_cmd.append("--cli") elif options: # Write options to a temporary json file to be accessed by the CLI: options_file = tempfile.NamedTemporaryFile("w") json.dump(options, options_file) options_file.flush( ) # ensure content is available for the CLI to read below install_cmd.append("--options={}".format(options_file.name)) sdk_cmd.run_cli(" ".join(install_cmd), check=True) # Wait for expected tasks to come up if expected_running_tasks > 0 and wait_for_all_conditions: sdk_tasks.check_running( service_name=service_name, expected_task_count=expected_running_tasks, timeout_seconds=timeout_seconds, ) # Wait for completed marathon deployment if wait_for_all_conditions: sdk_marathon.wait_for_deployment(service_name, timeout_seconds, None)
def _retried_install_impl(package_name, service_name, expected_running_tasks, options={}, package_version=None, timeout_seconds=TIMEOUT_SECONDS): '''Cleaned up version of shakedown's package_install().''' package_manager = dcos.packagemanager.PackageManager( dcos.cosmos.get_cosmos_url()) pkg = package_manager.get_package_version(package_name, package_version) if package_version is None: # Get the resolved version for logging below package_version = 'auto:{}'.format(pkg.version()) log.info( 'Installing package={} service={} with options={} version={}'.format( package_name, service_name, options, package_version)) # Trigger package install, but only if it's not already installed. # We expect upstream to have confirmed that it wasn't already installed beforehand. if sdk_marathon.app_exists(service_name): log.info( 'Marathon app={} exists, skipping package install call'.format( service_name)) else: package_manager.install_app(pkg, options) # Install CLI while package starts to install if pkg.cli_definition(): log.info('Installing CLI for package={}'.format(package_name)) dcos.subcommand.install(pkg) # Wait for expected tasks to come up if expected_running_tasks > 0: shakedown.wait_for_service_tasks_running(service_name, expected_running_tasks, timeout_seconds) # Wait for completed marathon deployment app_id = pkg.marathon_json(options).get('id') # TODO: initial wait time here? jenkins install is usually 90-120s # TODO: randomize the sleep_seconds shakedown.time_wait(lambda: shakedown.deployment_predicate(app_id), timeout_seconds, sleep_seconds=20)
def install(self) -> dict: @retrying.retry(wait_exponential_multiplier=1000, wait_exponential_max=120 * 1000, retry_on_result=lambda result: not result) def _install_marathon_app(app_definition): success, _ = sdk_marathon.install_app(app_definition) return success if sdk_marathon.app_exists(self.app_definition["id"]): log.info("Found installed KDC app") return _get_kdc_task(self.app_definition["id"]) log.info("Installing KDC Marathon app") _install_marathon_app(self.app_definition) log.info("KDC app installed successfully") kdc_task_info = _get_kdc_task(self.app_definition["id"]) return kdc_task_info
def install(self) -> dict: @retrying.retry(stop_max_delay=3 * 60 * 1000, wait_exponential_multiplier=1000, wait_exponential_max=120 * 1000, retry_on_result=lambda result: not result) def _install_marathon_app(app_definition): success, _ = sdk_marathon.install_app(app_definition) return success if sdk_marathon.app_exists(self.app_definition["id"]): if self._persist: log.info("Found installed KDC app, reusing it") return _get_kdc_task(self.app_definition["id"]) log.info("Found installed KDC app, destroying it first") sdk_marathon.destroy_app(self.app_definition["id"]) log.info("Installing KDC Marathon app") _install_marathon_app(self.app_definition) log.info("KDC app installed successfully") return _get_kdc_task(self.app_definition["id"])
def install(self) -> dict: @retrying.retry(stop_max_delay=3*60*1000, wait_exponential_multiplier=1000, wait_exponential_max=120 * 1000, retry_on_result=lambda result: not result) def _install_marathon_app(app_definition): success, _ = sdk_marathon.install_app(app_definition) return success if sdk_marathon.app_exists(self.app_definition["id"]): log.info("Found installed KDC app") return _get_kdc_task(self.app_definition["id"]) log.info("Installing KDC Marathon app") _install_marathon_app(self.app_definition) log.info("KDC app installed successfully") kdc_task_info = _get_kdc_task(self.app_definition["id"]) return kdc_task_info
def _retried_install_impl( package_name, service_name, expected_running_tasks, options={}, package_version=None, timeout_seconds=TIMEOUT_SECONDS): '''Cleaned up version of shakedown's package_install().''' package_manager = dcos.packagemanager.PackageManager(dcos.cosmos.get_cosmos_url()) pkg = package_manager.get_package_version(package_name, package_version) if package_version is None: # Get the resolved version for logging below package_version = 'auto:{}'.format(pkg.version()) log.info('Installing package={} service={} with options={} version={}'.format( package_name, service_name, options, package_version)) # Trigger package install, but only if it's not already installed. # We expect upstream to have confirmed that it wasn't already installed beforehand. if sdk_marathon.app_exists(service_name): log.info('Marathon app={} exists, skipping package install call'.format(service_name)) else: package_manager.install_app(pkg, options) # Install CLI while package starts to install if pkg.cli_definition(): log.info('Installing CLI for package={}'.format(package_name)) dcos.subcommand.install(pkg) # Wait for expected tasks to come up if expected_running_tasks > 0: shakedown.wait_for_service_tasks_running( service_name, expected_running_tasks, timeout_seconds) # Wait for completed marathon deployment app_id = pkg.marathon_json(options).get('id') shakedown.deployment_wait(timeout_seconds, app_id)
def _retried_uninstall_package_and_wait(package_name: str, service_name: str) -> None: if sdk_marathon.app_exists(service_name): log.info("Uninstalling package {} with service name {}".format( package_name, service_name)) sdk_cmd.run_cli("package uninstall {} --app-id={} --yes".format( package_name, service_name), check=True) # Wait on the app no longer being listed in Marathon, at which point it is uninstalled. # At the same time, log the deploy plan state as we wait for the app to finish uninstalling. @retrying.retry( stop_max_delay=TIMEOUT_SECONDS * 1000, wait_fixed=5000, retry_on_result=lambda result: not result, ) def wait_for_removal_log_deploy_plan() -> bool: if not sdk_marathon.app_exists(service_name): return True # App still exists, print the deploy plan. Best effort: It is expected for the scheduler # to become unavailable once uninstall completes. try: log.info( sdk_plan.plan_string( "deploy", sdk_plan.get_plan_once(service_name, "deploy"))) except Exception: pass # best effort attempt at logging plan content return False log.info("Waiting for {} to be removed".format(service_name)) wait_for_removal_log_deploy_plan() else: log.info( 'Skipping uninstall of package {}/service {}: App named "{}" doesn\'t exist' .format(package_name, service_name, service_name))