def test_gen(self, mocker: MockFixture) -> None: mocker.patch("opta.layer.os.path.exists") mocker.patch("opta.layer.validate_yaml") test_gen_file_path = "pytest_main.tf.json" mocker.patch("opta.core.generator.TF_FILE_PATH", test_gen_file_path) # Opta configs and terraform files are 1-1 mappings. # Make sure the expected terraform file contents are generated # from the opta config opta_config, gen_tf_file = BASIC_APPLY mocked_file = mocker.mock_open(read_data=yaml.dump(opta_config)) mocker.patch("opta.layer.open", mocked_file) opta_config = opta_config.copy() mocker.patch("opta.layer.yaml.load", return_value=opta_config) layer = Layer.load_from_yaml("", None) mocked_state_storage = mocker.patch("opta.layer.Layer.state_storage") mocked_state_storage.return_value = "opta-tf-state-test-dev1-98f2" gen_all(layer) with open(test_gen_file_path) as f: real_output = json.load(f) assert gen_tf_file == real_output # Make sure generation does not work without org name. opta_config, _ = BASIC_APPLY opta_config = opta_config.copy() del opta_config["org_name"] mocker.patch("opta.layer.yaml.load", return_value=opta_config) with pytest.raises(UserErrors): Layer.load_from_yaml("", None) os.remove(test_gen_file_path)
def get_secret_name_and_namespace( layer: Layer, module_name: Optional[str]) -> Tuple[str, str]: k8s_services = layer.get_module_by_type("k8s-service") helm_charts = layer.get_module_by_type("helm-chart") total_modules = k8s_services + helm_charts if not total_modules: raise UserErrors("No helm/k8s-service modules were configured") if module_name is None and len(total_modules) > 1: module_name = click.prompt( "Multiple k8s-service/helm chart modules found. Please specify which one do you want the secret for.", type=click.Choice([x.name for x in total_modules]), ) if module_name is None: module: Module = total_modules[0] else: try: module = next(module for module in total_modules if module.name == module_name) except StopIteration: raise UserErrors( f"Could not find helm/k8s-service module with name {module_name}" ) from None if module.type == "k8s-service": return "manual-secrets", layer.name else: return ( f"opta-{layer.name}-{module.name}-secret", module.data.get("namespace", "default"), )
def test_service_persistent_storage(self, mocker: MockFixture): mocker.patch("opta.core.terraform.nice_run") mocker.patch("opta.core.terraform.AWS") mocked_delete_persistent_volume_claims = mocker.patch( "opta.core.kubernetes.delete_persistent_volume_claims") layer = Layer.load_from_yaml( os.path.join( os.path.dirname(os.path.dirname(__file__)), "tests", "fixtures", "sample_opta_files", "service.yaml", ), None, ) layer.post_delete(0) # check if delete pvc was NOT called mocked_delete_persistent_volume_claims.assert_not_called() layer = Layer.load_from_yaml( os.path.join( os.path.dirname(os.path.dirname(__file__)), "tests", "fixtures", "sample_opta_files", "service_persistent_storage.yaml", ), None, ) layer.post_delete(0) # check if delete pvc was called mocked_delete_persistent_volume_claims.assert_called_once()
def test_same_name_as_parent(self): with pytest.raises(UserErrors): Layer.load_from_yaml( os.path.join( os.path.dirname(os.path.dirname(__file__)), "same_name_as_parent.yaml", ), None, )
def test_infinite_loop_prevention(self): with pytest.raises(UserErrors): Layer.load_from_yaml( os.path.join( os.path.dirname(os.path.dirname(__file__)), "infinite_loop.yaml", ), None, )
def validate(config: str, json_schema: bool, env: Optional[str], var: Dict[str, str]) -> None: config = check_opta_file_exists(config) Layer.load_from_yaml(config, env, json_schema, input_variables=var, strict_input_variables=False)
def test_service_missing_env_file(self): with pytest.raises(UserErrors) as exception: Layer.load_from_yaml( os.path.join( os.path.dirname(os.path.dirname(__file__)), "tests", "fixtures", "sample_opta_files", "service_missing_env_file.yaml", ), None, ) assert "Could not find file" in str(exception.value) assert "opta-not-found.yml" in str(exception.value)
def test_service_github_repo_env(self, mocker: MockFixture): mocker.patch("git.Repo.clone_from") git_repo_mocked = mocker.patch("git.Repo.clone_from") service_github_repo_env = os.path.join( os.path.dirname(os.path.dirname(__file__)), "tests", "fixtures", "sample_opta_files", "service_github_repo_env.yaml", ) # use local file for parent instead of cloning a github repo dummy_config_parent = os.path.join( os.path.dirname(os.path.dirname(__file__)), "tests", "fixtures", "dummy_data", "dummy_config_parent.yaml", ) mocker.patch( "opta.layer.check_opta_file_exists", side_effect=[service_github_repo_env, dummy_config_parent], ) layer = Layer.load_from_yaml( service_github_repo_env, None, ) git_repo_mocked.assert_called_once_with( "[email protected]:run-x/runx-infra.git", mocker.ANY, branch="main", depth=1) assert layer.name == "app" assert layer.parent.name == "dummy-parent"
def test_handle_link(self, mocker: MockFixture): layer = Layer.load_from_yaml( os.path.join(os.getcwd(), "tests", "fixtures", "dummy_data", "local_dummy_config.yaml"), None, ) idx = len(layer.modules) app_module = layer.get_module("app", idx) atlas_module = layer.get_module("mongodbatlas", idx) atlas_module.data["link_secrets"] = [] app_module.data["link_secrets"] = [] link_permissions = [] for link_data in app_module.data.get("links", []): if type(link_data) is str: link_permissions = [] elif type(link_data) is dict: link_permissions = list(link_data.values())[0] print(link_permissions) LinkerHelper.handle_link( app_module, atlas_module, link_permissions, required_vars=[ "db_password", "db_user", "mongodb_atlas_connection_string" ], ) link_secret_keys = [x["name"] for x in app_module.data["link_secrets"]] assert "DB_USER" in link_secret_keys
def test_validate_dns_mismatch(self, mocker: MockFixture) -> None: layer = Layer.load_from_yaml( os.path.join(os.getcwd(), "tests", "fixtures", "dummy_data", "dummy_config_parent.yaml"), None, ) dns_module = layer.get_module("awsdns", 6) if dns_module is None: raise Exception("did not find dns module") del dns_module.data["upload_cert"] dns_module.data["delegated"] = True processor = DNSModuleProcessor(dns_module, layer) mocked_get_terraform_outputs = mocker.patch( "modules.base.get_terraform_outputs") mocked_get_terraform_outputs.return_value = { "name_servers": ["blah.com.", "baloney.com"] } mocked_query = mocker.patch("modules.base.query") mocked_1 = mocker.Mock() mocked_1.target = mocker.Mock() mocked_1.target.to_text.return_value = "baloney.com" mocked_2 = mocker.Mock() mocked_2.target = mocker.Mock() mocked_2.target.to_text.return_value = "bla.com" mocked_query.return_value = [mocked_1, mocked_2] with pytest.raises(UserErrors): processor.validate_dns()
def test_pre_hook_pending_upgrade_service(self, mocker: MockFixture) -> None: layer = Layer.load_from_yaml( os.path.join( os.getcwd(), "tests", "fixtures", "dummy_data", "aws_service_getting_started.yaml", ), None, ) idx = len(layer.modules) module_name: str = "hello" module: Optional[Module] = layer.get_module(module_name, idx) mocked_helm_list = mocker.patch( "modules.base.Helm.get_helm_list", return_value=[{ "name": f"{layer.name}-{module_name}" }], ) mocked_check_byok_ready = mocker.patch( "modules.base.K8sServiceModuleProcessor._check_byok_ready") with pytest.raises(UserErrors): K8sServiceModuleProcessor(module, layer).pre_hook(idx) mocked_helm_list.assert_called_once_with( kube_context=mocker.ANY, release=f"{layer.name}-{module_name}", status="pending-upgrade", ) mocked_check_byok_ready.assert_not_called()
def output( config: str, env: Optional[str], local: Optional[bool], var: Dict[str, str], ) -> None: """Print TF outputs""" disable_version_upgrade() config = check_opta_file_exists(config) if local: config = local_setup(config, var, None) layer = Layer.load_from_yaml(config, env, input_variables=var, strict_input_variables=False) amplitude_client.send_event( amplitude_client.VIEW_OUTPUT_EVENT, event_properties={ "org_name": layer.org_name, "layer_name": layer.name }, ) layer.verify_cloud_credentials() gen_all(layer) outputs = get_terraform_outputs(layer) # Adding extra outputs if layer.cloud == "aws": outputs = _load_extra_aws_outputs(outputs) elif layer.cloud == "google": outputs = _load_extra_gcp_outputs(outputs) outputs_formatted = json.dumps(outputs, indent=4) print(outputs_formatted)
def update( secret: str, value: str, env: Optional[str], config: str, no_restart: bool, local: Optional[bool], var: Dict[str, str], module: Optional[str], ) -> None: """Update a given secret of a k8s service with a new value Examples: opta secret update -c my-service.yaml "MY_SECRET_1" "value" """ config = check_opta_file_exists(config) if local: config = local_setup(config, input_variables=var) env = "localopta" layer = Layer.load_from_yaml(config, env, input_variables=var, strict_input_variables=False) secret_name, namespace = get_secret_name_and_namespace(layer, module) set_kube_config(layer) create_namespace_if_not_exists(namespace) amplitude_client.send_event(amplitude_client.UPDATE_SECRET_EVENT) update_secrets(namespace, secret_name, {secret: str(value)}) __restart_deployments(no_restart, namespace) logger.info("Success")
def setUp(self) -> None: self.layer = Layer( name="testname", org_name="testorg", providers={"local": { "path": "/tmp" }}, modules_data=[], path="/tmp", parent=None, ) self.local = Local(self.layer) self.local.tf_file = "/tmp/tfconfig" self.local.config_file_path = "/tmp/localconfig" with open(self.local.config_file_path, "w") as f: json.dump( { "opta_version": "dev", "date": "2021-11-15T18:26:47.553097", "original_spec": "", "defaults": {}, }, f, ) with open(self.local.tf_file, "w") as f: f.write("Some tf state for testing") return super().setUp()
def test_fetch_cert_body(self, mocker: MockFixture): layer = Layer.load_from_yaml( os.path.join( os.getcwd(), "tests", "fixtures", "dummy_data", "dummy_config_parent.yaml" ), None, ) dns_module = layer.get_module("aws-dns", 6) patched_prompt = mocker.patch("modules.aws_dns.aws_dns.prompt") patched_prompt.side_effect = [ "", os.path.join( os.getcwd(), "tests", "fixtures", "dummy_data", "dummy_config1_parent.yaml", ), os.path.join( os.getcwd(), "tests", "fixtures", "dummy_data", "nonexistent_file" ), os.path.join( os.getcwd(), "tests", "fixtures", "dummy_data", "dummy_cert.pem" ), ] AwsDnsProcessor(dns_module, layer).fetch_cert_body()
def test_post_hook_error(self, mocker: MockFixture): layer = Layer.load_from_yaml( os.path.join( os.getcwd(), "tests", "fixtures", "dummy_data", "dummy_config_parent.yaml" ), None, ) aws_email_module = layer.get_module("awsses", 8) patched_logger = mocker.patch("modules.aws_ses.aws_ses.logger") mocked_sesv2 = mocker.Mock() mocked_boto3 = mocker.patch("modules.aws_ses.aws_ses.boto3") mocked_boto3.client.return_value = mocked_sesv2 mocked_sesv2.get_account.return_value = { "ProductionAccessEnabled": False, "Details": {"ReviewDetails": {"Status": "FAILED", "CaseId": "123"}}, } aws_dns_modules = layer.get_module("awsdns", 8) aws_dns_modules.data["delegated"] = True mocked_state_storage = mocker.patch("opta.layer.Layer.state_storage") mocked_state_storage.return_value = "whatever" AwsEmailProcessor(aws_email_module, layer).post_hook(8, None) mocked_boto3.client.assert_called_once() mocked_sesv2.get_account.assert_called_once() patched_logger.warning.assert_has_calls([mocker.call(mocker.ANY)]) patched_logger.info.assert_not_called()
def events( env: Optional[str], config: str, seconds: Optional[int], local: Optional[bool], var: Dict[str, str], ) -> None: """ List the events for a service Examples: opta events -c my-service.yaml """ if local: config = local_setup(config, input_variables=var) # Configure kubectl layer = Layer.load_from_yaml(config, env, strict_input_variables=False) amplitude_client.send_event( amplitude_client.SHELL_EVENT, event_properties={ "org_name": layer.org_name, "layer_name": layer.name }, ) start_time = None if seconds: start_time = pytz.utc.localize( datetime.datetime.min) - datetime.timedelta(seconds=seconds) layer.verify_cloud_credentials() gen_all(layer) set_kube_config(layer) load_opta_kube_config() tail_namespace_events(layer, start_time)
def push_image(image: str, config: str, env: Optional[str], tag: Optional[str], input_variables: Dict) -> Tuple[str, str]: ensure_installed("docker") layer = Layer.load_from_yaml(config, env, input_variables=input_variables) amplitude_client.send_event( amplitude_client.PUSH_EVENT, event_properties={ "org_name": layer.org_name, "layer_name": layer.name }, ) layer.verify_cloud_credentials() gen_all(layer) registry_url = get_registry_url(layer) if layer.cloud == "aws": username, password = get_ecr_auth_info(layer) elif layer.cloud == "google": username, password = get_gcr_auth_info(layer) elif layer.cloud == "azurerm": username, password = get_acr_auth_info(layer) else: if layer.cloud == "local": return push_to_docker_local(image, registry_url, tag) raise Exception( f"No support for pushing image to provider {layer.cloud}") return push_to_docker(username, password, image, registry_url, tag)
def configure_kubectl(config: str, env: Optional[str], local: Optional[bool], var: Dict[str, str]) -> None: """ Configure kubectl so you can connect to the cluster This command constructs a configuration with prepopulated server and certificate authority data values for the managed cluster. If you have the KUBECONFIG environment variable set, then the resulting configuration file is created at that location. Otherwise, by default, the resulting configuration file is created at the default kubeconfig path (.kube/config) in your home directory. """ try: opta_acquire_lock() config = check_opta_file_exists(config) if local: config = local_setup(config, input_variables=var) layer = Layer.load_from_yaml(config, env, input_variables=var, strict_input_variables=False) amplitude_client.send_event( amplitude_client.CONFIGURE_KUBECTL_EVENT, event_properties={ "org_name": layer.org_name, "layer_name": layer.name }, ) layer.verify_cloud_credentials() purge_opta_kube_config(layer) configure(layer) load_opta_kube_config_to_default(layer) finally: opta_release_lock()
def test_gen_resource_tags(self, mocker: MockFixture) -> None: gen_tags_file = "pytest-override.tf.json" mocker.patch("opta.module.TAGS_OVERRIDE_FILE", gen_tags_file) gen_tags_file_path = os.path.join(tf_modules_path, "aws_base/tf_module", gen_tags_file) mocker.patch("opta.layer.open") mocker.patch("opta.layer.os.path.exists") mocker.patch("opta.layer.validate_yaml") opta_config, gen_tf_file = BASIC_APPLY opta_config = opta_config.copy() mocker.patch("opta.layer.yaml.load", return_value=opta_config) layer = Layer.load_from_yaml("", None) gen_opta_resource_tags(layer) with open(gen_tags_file_path) as f: tags_config = json.load(f) has_vpc = False for resource in tags_config["resource"]: resource_type = list(resource.keys())[0] if resource_type == "aws_vpc": has_vpc = True assert resource[resource_type]["vpc"]["tags"]["opta"] == "true" assert has_vpc os.remove(gen_tags_file_path)
def delete( secret: str, env: Optional[str], config: str, no_restart: bool, local: Optional[bool], var: Dict[str, str], module: Optional[str], ) -> None: """Delete a secret key from a k8s service Examples: opta secret delete -c my-service.yaml "MY_SECRET_1" """ config = check_opta_file_exists(config) if local: config = local_setup(config, input_variables=var) env = "localopta" layer = Layer.load_from_yaml(config, env, input_variables=var, strict_input_variables=False) secret_name, namespace = get_secret_name_and_namespace(layer, module) set_kube_config(layer) if check_if_namespace_exists(namespace): delete_secret_key(namespace, secret_name, secret) __restart_deployments(no_restart, namespace) amplitude_client.send_event(amplitude_client.UPDATE_SECRET_EVENT) logger.info("Success")
def tf_state(config: str, env: Optional[str]) -> None: """ Show terraform state """ layer = Layer.load_from_yaml(config, env, strict_input_variables=False) cloud_client = layer.get_cloud_client() x = cloud_client.get_remote_state() print(x)
def test_sensitive_formatter_on_gcp_yaml() -> None: layer = Layer.load_from_yaml( os.path.join( os.getcwd(), "tests", "fixtures", "dummy_data", "gcp_dummy_config_parent.yaml" ), None, ) original_spec = layer.original_spec formatted_spec = SensitiveFormatter.filter(original_spec) assert "my-gcp-project" not in formatted_spec
def shell(env: Optional[str], config: str, type: str, local: Optional[bool], var: Dict[str, str]) -> None: """ Get a shell into one of the pods in a service Examples: opta shell -c my-service.yaml """ config = check_opta_file_exists(config) if local: config = local_setup(config, input_variables=var) # Configure kubectl layer = Layer.load_from_yaml(config, env, input_variables=var, strict_input_variables=False) amplitude_client.send_event( amplitude_client.SHELL_EVENT, event_properties={ "org_name": layer.org_name, "layer_name": layer.name }, ) layer.verify_cloud_credentials() gen_all(layer) set_kube_config(layer) load_opta_kube_config() context_name = layer.get_cloud_client().get_kube_context_name() # Get a random pod in the service v1 = CoreV1Api() pod_list = v1.list_namespaced_pod(layer.name).items if len(pod_list) == 0: raise UserErrors("This service is not yet deployed") nice_run([ "kubectl", "exec", "-n", layer.name, "-c", "k8s-service", "--kubeconfig", constants.GENERATED_KUBE_CONFIG or constants.DEFAULT_KUBECONFIG, "--context", context_name, pod_list[0].metadata.name, "-it", "--", type, "-il", ])
def test_bad_gcs_permission(self): layer = Layer.load_from_yaml( os.path.join(os.getcwd(), "tests", "fixtures", "dummy_data", "gcp_dummy_config.yaml"), None, ) idx = len(layer.modules) app_module = layer.get_module("app", idx) app_module.data["links"] = [] app_module.data["links"].append({"bucket1": "blah"}) with pytest.raises(Exception): GcpServiceAccountProcessor(app_module, layer).process(idx)
def test_bad_docdb_permission(self): layer = Layer.load_from_yaml( os.path.join(os.getcwd(), "tests", "fixtures", "dummy_data", "dummy_config1.yaml"), None, ) idx = len(layer.modules) app_module = layer.get_module("app", idx) app_module.data["links"] = [] app_module.data["links"].append({"docdb": "read"}) with pytest.raises(Exception): AwsK8sServiceProcessor(app_module, layer).process(idx)
def test_state_storage(self, mocker: MockFixture): mocked_bucket_exists = mocker.patch("opta.layer.Layer.bucket_exists") layer = Layer.load_from_yaml( os.path.join(os.getcwd(), "tests", "fixtures", "dummy_data", "dummy_config1.yaml"), None, ) mocked_bucket_exists.return_value = True assert layer.state_storage() == "opta-tf-state-opta-tests-dummy-parent" mocked_bucket_exists.return_value = False assert layer.state_storage( ) == "opta-tf-state-opta-tests-dummy-parent-195d"
def test_get_event_properties(self): layer = Layer.load_from_yaml( os.path.join(os.getcwd(), "tests", "fixtures", "dummy_data", "dummy_config1.yaml"), None, ) idx = len(layer.modules) app_module = layer.get_module("app", idx) assert AwsK8sServiceProcessor(app_module, layer).get_event_properties() == { "module_aws_k8s_service": 2 }
def test_process(self, mocker: MockFixture): layer = Layer.load_from_yaml( os.path.join(os.getcwd(), "tests", "fixtures", "dummy_data", "dummy_config_parent.yaml"), None, ) aws_eks_module = layer.get_module("awseks", 8) AwsEksProcessor(aws_eks_module, layer).process(8) assert (aws_eks_module.data["private_subnet_ids"] == "${{module.awsbase.private_subnet_ids}}") assert (aws_eks_module.data["kms_account_key_arn"] == "${{module.awsbase.kms_account_key_arn}}")
def logs( env: Optional[str], config: str, seconds: Optional[int], local: Optional[bool], var: Dict[str, str], ) -> None: """ Get stream of logs for a service Examples: opta logs -c my-service.yaml """ config = check_opta_file_exists(config) if local: config = local_setup(config, input_variables=var) # Configure kubectl layer = Layer.load_from_yaml(config, env, input_variables=var, strict_input_variables=False) amplitude_client.send_event( amplitude_client.SHELL_EVENT, event_properties={ "org_name": layer.org_name, "layer_name": layer.name }, ) layer.verify_cloud_credentials() gen_all(layer) set_kube_config(layer) load_opta_kube_config() if layer.cloud == "aws": modules = layer.get_module_by_type("k8s-service") elif layer.cloud == "google": modules = layer.get_module_by_type("gcp-k8s-service") elif layer.cloud == "local": modules = layer.get_module_by_type("local-k8s-service") elif layer.cloud == "helm": modules = layer.get_module_by_type("local-k8s-service") else: raise Exception(f"Currently not handling logs for cloud {layer.cloud}") if len(modules) == 0: raise UserErrors("No module of type k8s-service in the yaml file") elif len(modules) > 1: raise UserErrors( "Don't put more than one k8s-service module file per opta file") module_name = modules[0].name tail_module_log(layer, module_name, seconds)