def __init__(self) -> None: self.consul = pulumi.StackReference( f"grapl/consul/{config.STACK_NAME}") self.nomad_server = pulumi.StackReference( f"grapl/nomad/{config.STACK_NAME}") self.networking = pulumi.StackReference( f"grapl/networking/{config.STACK_NAME}") self.nomad_agents = pulumi.StackReference( f"grapl/nomad-agents/{config.STACK_NAME}")
def define_resources(): mycomponent = MyComponent("mycomponent", inprop="hello") myinstance = Instance("instance", name="myvm", value=pulumi.Output.secret("secret_value")) mycustom = MyCustom("mycustom", {"instance": myinstance}) invoke_result = do_invoke() myremotecomponent = MyRemoteComponent( "myremotecomponent", inprop=myinstance.id.apply(lambda v: f"hello: {v}")) # Pass myinstance several more times to ensure deserialization of the resource reference # works on other asyncio threads. for x in range(5): MyCustom(f"mycustom{x}", {"instance": myinstance}) dns_ref = pulumi.StackReference("dns") pulumi.export("hello", "world") pulumi.export("outprop", mycomponent.outprop) pulumi.export("public_ip", myinstance.public_ip) return { 'mycomponent': mycomponent, 'myinstance': myinstance, 'mycustom': mycustom, 'dns_ref': dns_ref, 'invoke_result': invoke_result, 'myremotecomponent': myremotecomponent, }
def __init__(self, stack_name: str) -> None: self.upstream_stack_name = ( "local-grapl" if config.LOCAL_GRAPL else f"grapl/grapl/{stack_name}" ) ref = pulumi.StackReference(self.upstream_stack_name) def require_str(key: str) -> str: return cast(str, ref.require_output(key)) self.aws_env_vars_for_local = require_str("aws-env-vars-for-local") self.analyzer_bucket = require_str("analyzers-bucket") self.redis_endpoint = require_str("redis-endpoint") self.schema_properties_table_name = require_str("schema-properties-table") self.schema_table_name = require_str("schema-table") self.sysmon_generator_queue = require_str("sysmon-generator-queue") self.sysmon_log_bucket = require_str("sysmon-log-bucket") self.test_user_name = require_str("test-user-name") self.plugin_work_queue_db_hostname = require_str( "plugin-work-queue-db-hostname" ) self.plugin_work_queue_db_port = require_str("plugin-work-queue-db-port") self.plugin_work_queue_db_username = require_str( "plugin-work-queue-db-username" ) self.plugin_work_queue_db_password = require_str( "plugin-work-queue-db-password" ) self.organization_management_db_hostname = require_str( "organization-management-db-hostname" ) self.organization_management_db_port = require_str( "organization-management-db-port" ) self.organization_management_db_username = require_str( "organization-management-db-username" ) self.organization_management_db_password = require_str( "organization-management-db-password" ) self.kafka_bootstrap_servers = require_str("kafka-bootstrap-servers") self.kafka_e2e_sasl_username = require_str("kafka-e2e-sasl-username") self.kafka_e2e_sasl_password = require_str("kafka-e2e-sasl-password") self.kafka_e2e_consumer_group_name = require_str( "kafka-e2e-consumer-group-name" ) self.test_user_password_secret_id = require_str("test-user-password-secret-id")
import pulumi config = pulumi.Config() exporterStackName = config.require('exporter_stack_name') org = config.require('org') a = pulumi.StackReference(f'{org}/exporter/{exporterStackName}') pulumi.export('val1', a.require_output('val')) pulumi.export('val2', pulumi.Output.secret(['d', 'x']))
import pulumi from pulumi_azure_nextgen.resources import latest as resources from pulumi_azure_nextgen.network import latest as network from pulumi_azure_nextgen.compute import latest as compute from pprint import pprint import json import os import random import string stackName = pulumi.get_stack() infrastructure = pulumi.StackReference(f"DevSecNinja/infrastructure/{stackName}") # Check if the stackName is either develop or production if stackName == "develop": print("Pulumi Stack is develop") elif stackName == "production": print("Pulumi Stack is production - be careful") else: raise AssertionError("We can only serve the develop and production Pulumi stack") # Import generic variables from JSON with open("../../generic/json/generic.json") as generic_params_file: generic_params_data = json.load(generic_params_file) print("Generic parameter file contents:") pprint(generic_params_data) # Import environment-specific variables from JSON env_params_file_location = str("../../generic/json/compute-" + stackName + ".json")
"""A Python Pulumi program""" import pulumi # get the current stack we're in stack = pulumi.get_stack() # get the Pulumi organization we're in org = 'my-org' # get a stack reference stack_ref = pulumi.StackReference(f"{org}/stack-1/{stack}") # retrieve a value from that stack reference exported_value_from_other_stack = stack_ref.get_output("exported_value") pulumi.export("exported_value", exported_value_from_other_stack)
import pulumi import pulumi_kubernetes as k8s import pulumi_kubernetes.helm.v3 as helm from pulumi_kubernetes.core.v1 import Namespace from pulumi_kubernetes.storage.v1 import StorageClass # Get the stack stack = pulumi.get_stack() sr = "jaxxstorm/cluster/{}".format(stack) stack_ref = pulumi.StackReference(sr) # Get the kubeconfig from the stack kubeconfig = stack_ref.get_output("kubeConfig") # Get configuration options config = pulumi.Config() namespace = config.require("namespace") # Set up the provider provider = k8s.Provider("home.lbrlabs", kubeconfig=kubeconfig) # Create the namespace ns = Namespace( "ns", metadata={ "name": namespace, }, opts=pulumi.ResourceOptions(provider=provider), ) # Install the helm chart helm.Chart(
## Exercise 1: Add stack references to get the kubeconfig from the "base_cluster" stack. # Doc: https://www.pulumi.com/docs/intro/concepts/stack/#stackreferences import pulumi from pulumi import ResourceOptions import pulumi_kubernetes as k8s from pulumi_kubernetes.helm.v3 import Chart, ChartOpts ## Exercise 1 # Using config data to get the name of the base stack. # The Pulumi automation API or other methods could be used to automate the config value. config = pulumi.Config() base_cluster_stack_name = config.require("base_cluster_stack") base_cluster_stack = pulumi.StackReference(base_cluster_stack_name) kubeconfig = base_cluster_stack.get_output("kubeconfig") # The K8s provider which supplies the helm chart resource needs to know how to talk to the K8s cluster. # So, instantiate a K8s provider using the retrieved kubeconfig. k8s_provider = k8s.Provider('k8s-provider', kubeconfig=kubeconfig) # Create a chart resource to deploy apache using the k8s provider instantiated above. apache = Chart('apache-chart', ChartOpts( chart='apache', version='8.3.2', fetch_opts={'repo': 'https://charts.bitnami.com/bitnami'}), opts=ResourceOptions(provider=k8s_provider)) # Get the helm-deployed apache service IP which isn't known until the chart is deployed. apache_service_ip = apache.get_resource('v1/Service', 'apache-chart').apply( lambda res: res.status.load_balancer.ingress[0].ip)
import pulumi import pulumi_aws as aws # read local config settings config = pulumi.Config() node_type = config.require("node_type") node_port = config.require("node_port") vpc_stack = config.require("vpc_stack") ec2_stack = config.require("ec2_stack") # get stack reference from vpc and ec2 vpc = pulumi.StackReference(vpc_stack) ec2 = pulumi.StackReference(ec2_stack) # get vpc/subnet/securitygroup vpc_id = vpc.get_output("vpc_id") vpc_azs = vpc.get_output("vpc_azs") private_subnets = vpc.get_output("private_subnets") private_subnet_ids = vpc.get_output("private_subnet_ids") api_sg_id = ec2.get_output("api_sg_id") # create redis security group redis_sg = aws.ec2.SecurityGroup( resource_name="redis access - {}".format(node_port), vpc_id=vpc_id, description="Enable Redis access", ingress=[{ "protocol": "tcp", "from_port": node_port, "to_port": node_port, "security_groups": [api_sg_id],
import pulumi stack_ref = pulumi.StackReference("stackRef", stack_name="foo/bar/dev")
def main(): # pylint: disable=too-many-locals,too-many-branches """Main entry point.""" # Fetch configuration. config = pulumi.Config() enable_release = config.get_bool('enable_release') archive_age = config.get_int('archive_age') or 30 dataset = pulumi.get_stack() organization = gcp.organizations.get_organization(domain=DOMAIN) project_id = gcp.organizations.get_project().project_id dependency_stacks = {} for dependency in config.get_object('depends_on') or (): dependency_stacks[dependency] = pulumi.StackReference(dependency) def org_role_id(id_suffix: str) -> str: return f'{organization.id}/roles/{id_suffix}' lister_role_id = org_role_id('StorageLister') viewer_creator_role_id = org_role_id('StorageViewerAndCreator') viewer_role_id = org_role_id('StorageObjectAndBucketViewer') # The Cloud Resource Manager API is required for the Cloud Identity API. cloudresourcemanager = gcp.projects.Service( 'cloudresourcemanager-service', service='cloudresourcemanager.googleapis.com', disable_on_destroy=False, ) # The Cloud Identity API is required for creating access groups and service accounts. cloudidentity = gcp.projects.Service( 'cloudidentity-service', service='cloudidentity.googleapis.com', disable_on_destroy=False, opts=pulumi.resource.ResourceOptions( depends_on=[cloudresourcemanager]), ) # Enable Dataproc until the Hail Query Service is ready. _ = gcp.projects.Service( 'dataproc-service', service='dataproc.googleapis.com', disable_on_destroy=False, ) service_accounts = defaultdict(list) for kind in 'hail', 'deployment': for access_level in ACCESS_LEVELS: service_account = config.get( f'{kind}_service_account_{access_level}') if service_account: service_accounts[kind].append((access_level, service_account)) # Create Dataproc and Cromwell service accounts. for kind in 'dataproc', 'cromwell': service_accounts[kind] = [] for access_level in ACCESS_LEVELS: account = gcp.serviceaccount.Account( f'{kind}-service-account-{access_level}', account_id=f'{kind}-{access_level}', opts=pulumi.resource.ResourceOptions( depends_on=[cloudidentity]), ) service_accounts[kind].append((access_level, account.email)) def service_accounts_gen(): for kind, values in service_accounts.items(): for access_level, service_account in values: yield kind, access_level, service_account def bucket_name(kind: str) -> str: """Returns the bucket name for the given dataset.""" return f'cpg-{dataset}-{kind}' def create_bucket(name: str, enable_versioning=True, **kwargs) -> gcp.storage.Bucket: """Returns a new GCS bucket.""" return gcp.storage.Bucket( name, name=name, location=REGION, uniform_bucket_level_access=True, versioning=gcp.storage.BucketVersioningArgs( enabled=enable_versioning), labels={'bucket': name}, **kwargs, ) def bucket_member(*args, **kwargs): """Wraps gcp.storage.BucketIAMMember. When resources are renamed, it can be useful to explicitly apply changes in two phases: delete followed by create; that's opposite of the default create followed by delete, which can end up with missing permissions. To implement the first phase (delete), simply change this implementation to a no-op temporarily. """ gcp.storage.BucketIAMMember(*args, **kwargs) undelete_rule = gcp.storage.BucketLifecycleRuleArgs( action=gcp.storage.BucketLifecycleRuleActionArgs(type='Delete'), condition=gcp.storage.BucketLifecycleRuleConditionArgs( age=30, with_state='ARCHIVED'), ) main_upload_account = gcp.serviceaccount.Account( 'main-upload-service-account', account_id='main-upload', display_name='main-upload', opts=pulumi.resource.ResourceOptions(depends_on=[cloudidentity]), ) main_upload_buckets = { 'main-upload': create_bucket(bucket_name('main-upload'), lifecycle_rules=[undelete_rule]) } for additional_upload_bucket in ( config.get_object('additional_upload_buckets') or ()): main_upload_buckets[additional_upload_bucket] = create_bucket( additional_upload_bucket, lifecycle_rules=[undelete_rule]) test_upload_bucket = create_bucket(bucket_name('test-upload'), lifecycle_rules=[undelete_rule]) # Grant admin permissions as composite uploads need to delete temporary files. for bname, upload_bucket in main_upload_buckets.items(): bucket_member( f'main-upload-service-account-{bname}-bucket-creator', bucket=upload_bucket.name, role='roles/storage.admin', member=pulumi.Output.concat('serviceAccount:', main_upload_account.email), ) archive_bucket = create_bucket( bucket_name('archive'), lifecycle_rules=[ gcp.storage.BucketLifecycleRuleArgs( action=gcp.storage.BucketLifecycleRuleActionArgs( type='SetStorageClass', storage_class='ARCHIVE'), condition=gcp.storage.BucketLifecycleRuleConditionArgs( age=archive_age), ), undelete_rule, ], ) test_bucket = create_bucket(bucket_name('test'), lifecycle_rules=[undelete_rule]) # tmp buckets don't have an undelete lifecycle rule, to avoid paying for # intermediate results that get cleaned up immediately after workflow runs. test_tmp_bucket = create_bucket( bucket_name('test-tmp'), enable_versioning=False, lifecycle_rules=[ gcp.storage.BucketLifecycleRuleArgs( action=gcp.storage.BucketLifecycleRuleActionArgs( type='Delete'), condition=gcp.storage.BucketLifecycleRuleConditionArgs( age=TMP_BUCKET_PERIOD_IN_DAYS), ) ], ) test_analysis_bucket = create_bucket(bucket_name('test-analysis'), lifecycle_rules=[undelete_rule]) test_web_bucket = create_bucket(bucket_name('test-web'), lifecycle_rules=[undelete_rule]) main_bucket = create_bucket(bucket_name('main'), lifecycle_rules=[undelete_rule]) # tmp buckets don't have an undelete lifecycle rule, to avoid paying for # intermediate results that get cleaned up immediately after workflow runs. main_tmp_bucket = create_bucket( bucket_name('main-tmp'), enable_versioning=False, lifecycle_rules=[ gcp.storage.BucketLifecycleRuleArgs( action=gcp.storage.BucketLifecycleRuleActionArgs( type='Delete'), condition=gcp.storage.BucketLifecycleRuleConditionArgs( age=TMP_BUCKET_PERIOD_IN_DAYS), ) ], ) main_analysis_bucket = create_bucket(bucket_name('main-analysis'), lifecycle_rules=[undelete_rule]) main_web_bucket = create_bucket(bucket_name('main-web'), lifecycle_rules=[undelete_rule]) def group_mail(dataset: str, kind: str) -> str: """Returns the email address of a permissions group.""" return f'{dataset}-{kind}@{DOMAIN}' def create_group(mail: str) -> gcp.cloudidentity.Group: """Returns a new Cloud Identity group for the given email address.""" name = mail.split('@')[0] return gcp.cloudidentity.Group( name, display_name=name, group_key=gcp.cloudidentity.GroupGroupKeyArgs(id=mail), labels={ 'cloudidentity.googleapis.com/groups.discussion_forum': '' }, parent=f'customers/{CUSTOMER_ID}', opts=pulumi.resource.ResourceOptions(depends_on=[cloudidentity]), ) def create_secret(resource_name: str, secret_id: str, **kwargs): return gcp.secretmanager.Secret( resource_name, secret_id=secret_id, replication=gcp.secretmanager.SecretReplicationArgs( user_managed=gcp.secretmanager. SecretReplicationUserManagedArgs(replicas=[ gcp.secretmanager.SecretReplicationUserManagedReplicaArgs( location='australia-southeast1', ), ], ), ), opts=pulumi.resource.ResourceOptions(depends_on=[secretmanager]), **kwargs, ) def add_access_group_cache_as_secret_member(secret, resource_prefix: str): gcp.secretmanager.SecretIamMember( f'{resource_prefix}-group-cache-secret-accessor', secret_id=secret.id, role='roles/secretmanager.secretAccessor', member=f'serviceAccount:{ACCESS_GROUP_CACHE_SERVICE_ACCOUNT}', ) gcp.secretmanager.SecretIamMember( f'{resource_prefix}-group-cache-secret-version-manager', secret_id=secret.id, role='roles/secretmanager.secretVersionManager', member=f'serviceAccount:{ACCESS_GROUP_CACHE_SERVICE_ACCOUNT}', ) access_group_mail = group_mail(dataset, 'access') access_group = create_group(access_group_mail) web_access_group = create_group(group_mail(dataset, 'web-access')) # other stacks require the access group to exist pulumi.export( access_group_mail.split('@')[0] + '-group-id', access_group.id) # Create groups for each access level. access_level_groups = {} for access_level in ACCESS_LEVELS: group = create_group(group_mail(dataset, access_level)) access_level_groups[access_level] = group # The group provider ID is used by other stacks that depend on this one. group_provider_id_name = f'{access_level}-access-group-id' pulumi.export(group_provider_id_name, group.id) # Allow the access group cache to list memberships. gcp.cloudidentity.GroupMembership( f'access-group-cache-{access_level}-access-level-group-membership', group=group.id, preferred_member_key=gcp.cloudidentity. GroupMembershipPreferredMemberKeyArgs( id=ACCESS_GROUP_CACHE_SERVICE_ACCOUNT), roles=[gcp.cloudidentity.GroupMembershipRoleArgs(name='MEMBER')], opts=pulumi.resource.ResourceOptions(depends_on=[cloudidentity]), ) # Provide access transitively to datasets we depend on for dependency in config.get_object('depends_on') or (): dependency_group_id = dependency_stacks[dependency].get_output( group_provider_id_name, ) dependency_group = gcp.cloudidentity.Group.get( f'{dependency}-{access_level}-access-level-group', dependency_group_id, ) gcp.cloudidentity.GroupMembership( f'{dependency}-{access_level}-access-level-group-membership', group=dependency_group.id, preferred_member_key=group.group_key, roles=[ gcp.cloudidentity.GroupMembershipRoleArgs(name='MEMBER') ], opts=pulumi.resource.ResourceOptions( depends_on=[cloudidentity]), ) for dependency, dstack in dependency_stacks.items(): # add the {dataset}-access group to the dependency depends_on_access_group_name = ( group_mail(dependency, 'access').split('@')[0] + '-group-id') depends_on_access_group_id = dstack.get_output( depends_on_access_group_name, ) depends_on_access_group = gcp.cloudidentity.Group.get( depends_on_access_group_name, depends_on_access_group_id) gcp.cloudidentity.GroupMembership( f'{dataset}-{dependency}-access', group=depends_on_access_group, preferred_member_key=access_group.group_key, roles=[gcp.cloudidentity.GroupMembershipRoleArgs(name='MEMBER')], opts=pulumi.resource.ResourceOptions(depends_on=[cloudidentity]), ) for kind, access_level, service_account in service_accounts_gen(): gcp.cloudidentity.GroupMembership( f'{kind}-{access_level}-access-level-group-membership', group=access_level_groups[access_level], preferred_member_key=gcp.cloudidentity. GroupMembershipPreferredMemberKeyArgs(id=service_account), roles=[gcp.cloudidentity.GroupMembershipRoleArgs(name='MEMBER')], opts=pulumi.resource.ResourceOptions(depends_on=[cloudidentity]), ) secretmanager = gcp.projects.Service( 'secretmanager-service', service='secretmanager.googleapis.com', disable_on_destroy=False, ) # These secrets are used as a fast cache for checking memberships in the above groups. access_group_cache_secrets = {} for group_prefix in ('access', 'web-access') + ACCESS_LEVELS: access_secret = create_secret( f'{group_prefix}-group-cache-secret', secret_id=f'{dataset}-{group_prefix}-members-cache', ) add_access_group_cache_as_secret_member(access_secret, group_prefix) access_group_cache_secrets[group_prefix] = access_secret gcp.secretmanager.SecretIamMember( 'analyis-runner-access-group-cache-secret-accessor', secret_id=access_group_cache_secrets['access'].id, role='roles/secretmanager.secretAccessor', member=f'serviceAccount:{ANALYSIS_RUNNER_SERVICE_ACCOUNT}', ) gcp.secretmanager.SecretIamMember( 'web-server-web-access-group-cache-secret-accessor', secret_id=access_group_cache_secrets['web-access'].id, role='roles/secretmanager.secretAccessor', member=f'serviceAccount:{WEB_SERVER_SERVICE_ACCOUNT}', ) # Sample metadata access # permissions for read / write # - 4 secrets, main-read, main-write, test-read, test-write sm_groups = {} for env in ('main', 'test'): for rs in ('read', 'write'): key = f'sample-metadata-{env}-{rs}' group = create_group(group_mail(dataset, key)) sm_groups[f'{env}-{rs}'] = group gcp.cloudidentity.GroupMembership( f'sample-metadata-group-cache-{env}-{rs}-group-membership', group=group, preferred_member_key=gcp.cloudidentity. GroupMembershipPreferredMemberKeyArgs( id=ACCESS_GROUP_CACHE_SERVICE_ACCOUNT), roles=[ gcp.cloudidentity.GroupMembershipRoleArgs(name='MEMBER') ], opts=pulumi.resource.ResourceOptions( depends_on=[cloudidentity]), ) secret = create_secret( f'{key}-group-cache-secret', secret_id=f'{dataset}-{key}-members-cache', ) add_access_group_cache_as_secret_member(secret, resource_prefix=key) gcp.secretmanager.SecretIamMember( f'{key}-api-secret-accessor', secret_id=secret.id, role='roles/secretmanager.secretAccessor', member=f'serviceAccount:{SAMPLE_METADATA_API_SERVICE_ACCOUNT}', ) # Add cloud run invoker to analysis-runner for the access-group gcp.cloudrun.IamMember( f'analysis-runner-access-invoker', location=REGION, project=ANALYSIS_RUNNER_PROJECT, service='server', role='roles/run.invoker', member=pulumi.Output.concat('group:', access_group.group_key.id), ) # Declare access to sample-metadata API of format ({env}-{read,write}) sm_access_levels: List[SampleMetadataAccessorMembership] = [ SampleMetadataAccessorMembership( name='human', member_key=access_group.group_key.id, permissions=('main-read', 'test-read', 'test-write'), ), SampleMetadataAccessorMembership( name='test', member_key=access_level_groups['test'].group_key.id, permissions=('main-read', 'test-read', 'test-write'), ), SampleMetadataAccessorMembership( name='standard', member_key=access_level_groups['standard'].group_key.id, permissions=('main-read', 'main-write'), ), SampleMetadataAccessorMembership( name='full', member_key=access_level_groups['full'].group_key.id, permissions=sm_groups.keys(), ), # allow the analysis-runner logging cloud function to update the sample-metadata project SampleMetadataAccessorMembership( name='analysis-runner-logger', member_key=ANALYSIS_RUNNER_LOGGER_SERVICE_ACCOUNT, permissions=sm_groups.keys(), ), ] # give access to sample_metadata groups (and hence sample-metadata API through secrets) for name, service_account, permission in sm_access_levels: for kind in permission: gcp.cloudidentity.GroupMembership( f'sample-metadata-{kind}-{name}-access-level-group-membership', group=sm_groups[kind], preferred_member_key=gcp.cloudidentity. GroupMembershipPreferredMemberKeyArgs(id=service_account), roles=[ gcp.cloudidentity.GroupMembershipRoleArgs(name='MEMBER') ], opts=pulumi.resource.ResourceOptions( depends_on=[cloudidentity]), ) gcp.projects.IAMMember( 'project-buckets-lister', role=lister_role_id, member=pulumi.Output.concat('group:', access_group.group_key.id), ) # Grant visibility to Dataproc utilization metrics etc. gcp.projects.IAMMember( 'project-monitoring-viewer', role='roles/monitoring.viewer', member=pulumi.Output.concat('group:', access_group.group_key.id), ) bucket_member( 'access-group-test-bucket-admin', bucket=test_bucket.name, role='roles/storage.admin', member=pulumi.Output.concat('group:', access_group.group_key.id), ) bucket_member( 'access-group-test-upload-bucket-admin', bucket=test_upload_bucket.name, role='roles/storage.admin', member=pulumi.Output.concat('group:', access_group.group_key.id), ) bucket_member( 'access-group-test-tmp-bucket-admin', bucket=test_tmp_bucket.name, role='roles/storage.admin', member=pulumi.Output.concat('group:', access_group.group_key.id), ) bucket_member( 'access-group-test-analysis-bucket-admin', bucket=test_analysis_bucket.name, role='roles/storage.admin', member=pulumi.Output.concat('group:', access_group.group_key.id), ) bucket_member( 'access-group-test-web-bucket-admin', bucket=test_web_bucket.name, role='roles/storage.admin', member=pulumi.Output.concat('group:', access_group.group_key.id), ) for bname, upload_bucket in main_upload_buckets.items(): bucket_member( f'access-group-{bname}-bucket-viewer', bucket=upload_bucket.name, role=viewer_role_id, member=pulumi.Output.concat('group:', access_group.group_key.id), ) bucket_member( 'access-group-main-analysis-bucket-viewer', bucket=main_analysis_bucket.name, role=viewer_role_id, member=pulumi.Output.concat('group:', access_group.group_key.id), ) bucket_member( 'access-group-main-web-bucket-viewer', bucket=main_web_bucket.name, role=viewer_role_id, member=pulumi.Output.concat('group:', access_group.group_key.id), ) if enable_release: release_bucket = create_bucket( bucket_name('release-requester-pays'), lifecycle_rules=[undelete_rule], requester_pays=True, ) bucket_member( 'access-group-release-bucket-viewer', bucket=release_bucket.name, role=viewer_role_id, member=pulumi.Output.concat('group:', access_group.group_key.id), ) release_access_group = create_group( group_mail(dataset, 'release-access')) bucket_member( 'release-access-group-release-bucket-viewer', bucket=release_bucket.name, role=viewer_role_id, member=pulumi.Output.concat('group:', release_access_group.group_key.id), ) bucket_member( 'web-server-test-web-bucket-viewer', bucket=test_web_bucket.name, role=viewer_role_id, member=pulumi.Output.concat('serviceAccount:', WEB_SERVER_SERVICE_ACCOUNT), ) bucket_member( 'web-server-main-web-bucket-viewer', bucket=main_web_bucket.name, role=viewer_role_id, member=pulumi.Output.concat('serviceAccount:', WEB_SERVER_SERVICE_ACCOUNT), ) # Allow the usage of requester-pays buckets. gcp.projects.IAMMember( f'access-group-serviceusage-consumer', role='roles/serviceusage.serviceUsageConsumer', member=pulumi.Output.concat('group:', access_group.group_key.id), ) # Allow the access group cache to list memberships. gcp.cloudidentity.GroupMembership( 'access-group-cache-membership', group=access_group.id, preferred_member_key=gcp.cloudidentity. GroupMembershipPreferredMemberKeyArgs( id=ACCESS_GROUP_CACHE_SERVICE_ACCOUNT), roles=[gcp.cloudidentity.GroupMembershipRoleArgs(name='MEMBER')], opts=pulumi.resource.ResourceOptions(depends_on=[cloudidentity]), ) gcp.cloudidentity.GroupMembership( 'web-access-group-cache-membership', group=web_access_group.id, preferred_member_key=gcp.cloudidentity. GroupMembershipPreferredMemberKeyArgs( id=ACCESS_GROUP_CACHE_SERVICE_ACCOUNT), roles=[gcp.cloudidentity.GroupMembershipRoleArgs(name='MEMBER')], opts=pulumi.resource.ResourceOptions(depends_on=[cloudidentity]), ) # All members of the access group have web access automatically. gcp.cloudidentity.GroupMembership( 'web-access-group-access-group-membership', group=web_access_group.id, preferred_member_key=access_group.group_key, roles=[gcp.cloudidentity.GroupMembershipRoleArgs(name='MEMBER')], opts=pulumi.resource.ResourceOptions(depends_on=[cloudidentity]), ) for access_level, group in access_level_groups.items(): # Allow the service accounts to pull images. Note that the global project will # refer to the dataset, but the Docker images are stored in the "analysis-runner" # and "cpg-common" projects' Artifact Registry repositories. for project in [ANALYSIS_RUNNER_PROJECT, CPG_COMMON_PROJECT]: gcp.artifactregistry.RepositoryIamMember( f'{access_level}-images-reader-in-{project}', project=project, location=REGION, repository='images', role='roles/artifactregistry.reader', member=pulumi.Output.concat('group:', group.group_key.id), ) # Allow non-test service accounts to write images to the "cpg-common" Artifact # Registry repository. if access_level != 'test': gcp.artifactregistry.RepositoryIamMember( f'{access_level}-images-writer-in-cpg-common', project=CPG_COMMON_PROJECT, location=REGION, repository='images', role='roles/artifactregistry.writer', member=pulumi.Output.concat('group:', group.group_key.id), ) # Read access to reference data. bucket_member( f'{access_level}-reference-bucket-viewer', bucket=REFERENCE_BUCKET_NAME, role=viewer_role_id, member=pulumi.Output.concat('group:', group.group_key.id), ) # Allow the usage of requester-pays buckets. gcp.projects.IAMMember( f'{access_level}-serviceusage-consumer', role='roles/serviceusage.serviceUsageConsumer', member=pulumi.Output.concat('group:', group.group_key.id), ) # The bucket used for Hail Batch pipelines. hail_bucket = create_bucket(bucket_name('hail'), lifecycle_rules=[undelete_rule]) for access_level, service_account in service_accounts['hail']: # Full access to the Hail Batch bucket. bucket_member( f'hail-service-account-{access_level}-hail-bucket-admin', bucket=hail_bucket.name, role='roles/storage.admin', member=pulumi.Output.concat('serviceAccount:', service_account), ) # Permissions increase by access level: # - test: view / create on any "test" bucket # - standard: view / create on any "test" or "main" bucket # - full: view / create / delete anywhere for access_level, group in access_level_groups.items(): # test bucket bucket_member( f'{access_level}-test-bucket-admin', bucket=test_bucket.name, role='roles/storage.admin', member=pulumi.Output.concat('group:', group.group_key.id), ) # test-upload bucket bucket_member( f'{access_level}-test-upload-bucket-admin', bucket=test_upload_bucket.name, role='roles/storage.admin', member=pulumi.Output.concat('group:', group.group_key.id), ) # test-tmp bucket bucket_member( f'{access_level}-test-tmp-bucket-admin', bucket=test_tmp_bucket.name, role='roles/storage.admin', member=pulumi.Output.concat('group:', group.group_key.id), ) # test-analysis bucket bucket_member( f'{access_level}-test-analysis-bucket-admin', bucket=test_analysis_bucket.name, role='roles/storage.admin', member=pulumi.Output.concat('group:', group.group_key.id), ) # test-web bucket bucket_member( f'{access_level}-test-web-bucket-admin', bucket=test_web_bucket.name, role='roles/storage.admin', member=pulumi.Output.concat('group:', group.group_key.id), ) if access_level == 'standard': # main bucket bucket_member( f'standard-main-bucket-view-create', bucket=main_bucket.name, role=viewer_creator_role_id, member=pulumi.Output.concat('group:', group.group_key.id), ) # main-upload bucket for bname, upload_bucket in main_upload_buckets.items(): bucket_member( f'standard-{bname}-bucket-viewer', bucket=upload_bucket.name, role=viewer_role_id, member=pulumi.Output.concat('group:', group.group_key.id), ) # main-tmp bucket bucket_member( f'standard-main-tmp-bucket-view-create', bucket=main_tmp_bucket.name, role=viewer_creator_role_id, member=pulumi.Output.concat('group:', group.group_key.id), ) # main-analysis bucket bucket_member( f'standard-main-analysis-bucket-view-create', bucket=main_analysis_bucket.name, role=viewer_creator_role_id, member=pulumi.Output.concat('group:', group.group_key.id), ) # main-web bucket bucket_member( f'standard-main-web-bucket-view-create', bucket=main_web_bucket.name, role=viewer_creator_role_id, member=pulumi.Output.concat('group:', group.group_key.id), ) if access_level == 'full': # main bucket bucket_member( f'full-main-bucket-admin', bucket=main_bucket.name, role='roles/storage.admin', member=pulumi.Output.concat('group:', group.group_key.id), ) # main-upload bucket for bname, upload_bucket in main_upload_buckets.items(): bucket_member( f'full-{bname}-bucket-admin', bucket=upload_bucket.name, role='roles/storage.admin', member=pulumi.Output.concat('group:', group.group_key.id), ) # main-tmp bucket bucket_member( f'full-main-tmp-bucket-admin', bucket=main_tmp_bucket.name, role='roles/storage.admin', member=pulumi.Output.concat('group:', group.group_key.id), ) # main-analysis bucket bucket_member( f'full-main-analysis-bucket-admin', bucket=main_analysis_bucket.name, role='roles/storage.admin', member=pulumi.Output.concat('group:', group.group_key.id), ) # main-web bucket bucket_member( f'full-main-web-bucket-admin', bucket=main_web_bucket.name, role='roles/storage.admin', member=pulumi.Output.concat('group:', group.group_key.id), ) # archive bucket bucket_member( f'full-archive-bucket-admin', bucket=archive_bucket.name, role='roles/storage.admin', member=pulumi.Output.concat('group:', group.group_key.id), ) # release bucket if enable_release: bucket_member( f'full-release-bucket-admin', bucket=release_bucket.name, role='roles/storage.admin', member=pulumi.Output.concat('group:', group.group_key.id), ) # Notebook permissions notebook_account = gcp.serviceaccount.Account( 'notebook-account', project=NOTEBOOKS_PROJECT, account_id=f'notebook-{dataset}', display_name=f'Notebook service account for dataset {dataset}', opts=pulumi.resource.ResourceOptions(depends_on=[cloudidentity]), ) gcp.projects.IAMMember( 'notebook-account-compute-admin', project=NOTEBOOKS_PROJECT, role='roles/compute.admin', member=pulumi.Output.concat('serviceAccount:', notebook_account.email), ) gcp.serviceaccount.IAMMember( 'notebook-account-users', service_account_id=notebook_account, role='roles/iam.serviceAccountUser', member=pulumi.Output.concat('group:', access_group.group_key.id), ) # Grant the notebook account the same permissions as the access group members. gcp.cloudidentity.GroupMembership( 'notebook-service-account-access-group-member', group=access_group.id, preferred_member_key=gcp.cloudidentity. GroupMembershipPreferredMemberKeyArgs(id=notebook_account.email), roles=[gcp.cloudidentity.GroupMembershipRoleArgs(name='MEMBER')], opts=pulumi.resource.ResourceOptions(depends_on=[cloudidentity]), ) def find_service_account(kind: str, access_level: str) -> Optional[str]: for local_access_level, service_account in service_accounts[kind]: if access_level == local_access_level: return service_account return None for access_level, service_account in service_accounts['dataproc']: # Hail Batch service accounts need to be able to act as Dataproc service # accounts to start Dataproc clusters. gcp.serviceaccount.IAMMember( f'hail-service-account-{access_level}-dataproc-service-account-user', service_account_id=pulumi.Output.concat('projects/', project_id, '/serviceAccounts/', service_account), role='roles/iam.serviceAccountUser', member=pulumi.Output.concat( 'serviceAccount:', find_service_account('hail', access_level)), ) gcp.projects.IAMMember( f'dataproc-service-account-{access_level}-dataproc-worker', role='roles/dataproc.worker', member=pulumi.Output.concat('serviceAccount:', service_account), ) for access_level, service_account in service_accounts['hail']: # The Hail service account creates the cluster, specifying the Dataproc service # account as the worker. gcp.projects.IAMMember( f'hail-service-account-{access_level}-dataproc-admin', role='roles/dataproc.admin', member=pulumi.Output.concat('serviceAccount:', service_account), ) # Worker permissions are necessary to submit jobs. gcp.projects.IAMMember( f'hail-service-account-{access_level}-dataproc-worker', role='roles/dataproc.worker', member=pulumi.Output.concat('serviceAccount:', service_account), ) # Add Hail service accounts to Cromwell access group. gcp.cloudidentity.GroupMembership( f'hail-service-account-{access_level}-cromwell-access', group=CROMWELL_ACCESS_GROUP_ID, preferred_member_key=gcp.cloudidentity. GroupMembershipPreferredMemberKeyArgs(id=service_account, ), roles=[gcp.cloudidentity.GroupMembershipRoleArgs(name='MEMBER')], opts=pulumi.resource.ResourceOptions(depends_on=[cloudidentity]), ) for access_level, service_account in service_accounts['cromwell']: # Allow the Cromwell server to run worker VMs using the Cromwell service # accounts. gcp.serviceaccount.IAMMember( f'cromwell-runner-{access_level}-service-account-user', service_account_id=pulumi.Output.concat('projects/', project_id, '/serviceAccounts/', service_account), role='roles/iam.serviceAccountUser', member=f'serviceAccount:{CROMWELL_RUNNER_ACCOUNT}', ) # To use a service account for VMs, Cromwell accounts need to be allowed to act # on their own behalf ;). gcp.serviceaccount.IAMMember( f'cromwell-service-account-{access_level}-service-account-user', service_account_id=pulumi.Output.concat('projects/', project_id, '/serviceAccounts/', service_account), role='roles/iam.serviceAccountUser', member=pulumi.Output.concat('serviceAccount:', service_account), ) # Allow the Cromwell service accounts to run workflows. gcp.projects.IAMMember( f'cromwell-service-account-{access_level}-workflows-runner', role='roles/lifesciences.workflowsRunner', member=pulumi.Output.concat('serviceAccount:', service_account), ) # Store the service account key as a secret that's readable by the # analysis-runner. key = gcp.serviceaccount.Key( f'cromwell-service-account-{access_level}-key', service_account_id=service_account, ) secret = create_secret( f'cromwell-service-account-{access_level}-secret', secret_id=f'{dataset}-cromwell-{access_level}-key', project=ANALYSIS_RUNNER_PROJECT, ) gcp.secretmanager.SecretVersion( f'cromwell-service-account-{access_level}-secret-version', secret=secret.id, secret_data=key.private_key.apply( lambda s: base64.b64decode(s).decode('utf-8')), ) gcp.secretmanager.SecretIamMember( f'cromwell-service-account-{access_level}-secret-accessor', project=ANALYSIS_RUNNER_PROJECT, secret_id=secret.id, role='roles/secretmanager.secretAccessor', member=f'serviceAccount:{ANALYSIS_RUNNER_SERVICE_ACCOUNT}', ) # Allow the Hail service account to access its corresponding cromwell key hail_service_account = find_service_account('hail', access_level) gcp.secretmanager.SecretIamMember( f'cromwell-service-account-{access_level}-self-accessor', project=ANALYSIS_RUNNER_PROJECT, secret_id=secret.id, role='roles/secretmanager.secretAccessor', member=f'serviceAccount:{hail_service_account}', ) for access_level, group in access_level_groups.items(): # Give hail / dataproc / cromwell access to sample-metadata cloud run service gcp.cloudrun.IamMember( f'sample-metadata-service-account-{access_level}-invoker', location=REGION, project=SAMPLE_METADATA_PROJECT, service='sample-metadata-api', role='roles/run.invoker', member=pulumi.Output.concat('group:', group.group_key.id), )
# Copyright 2020, Pulumi Corporation. All rights reserved. import pulumi config = pulumi.Config() org = config.require('org') slug = f"{org}/{pulumi.get_project()}/{pulumi.get_stack()}" a = pulumi.StackReference(slug) oldVal = a.get_output('val') if len(oldVal) != 2 or oldVal[0] != 'a' or oldVal[1] != 'b': raise Exception('Invalid result') pulumi.export('val2', pulumi.Output.secret(['a', 'b']))
) # pulumi.export('IMAGE', my_image) # image_to_use = docker.RemoteImage("my-nginx", # name="pulumi-demo:0.0.1", # keep_locally=True # ) container = docker.Container("my-nginx", name="my-nginx", image=my_image.base_image_name, ports=[{"internal":80, "external":80}] ) # pulumi.export('CONTAINER', container) repo = pulumi.StackReference('aldenso/github_example/'+env) with open("Dockerfile", 'r') as f: filedata = f.read() dockerfile = github.RepositoryFile("Dockerfile", content=filedata, file="Dockerfile", repository=repo.get_output("repo")["id"], branch="master" ) # pulumi.export('DOCKERFILE', dockerfile)
def main() -> None: ##### Preamble stack_name = config.STACK_NAME pulumi_config = pulumi.Config() artifacts = ArtifactGetter.from_config(pulumi_config) # These tags will be added to all provisioned infrastructure # objects. register_auto_tags( {"pulumi:project": pulumi.get_project(), "pulumi:stack": stack_name} ) nomad_provider: Optional[pulumi.ProviderResource] = None if not config.LOCAL_GRAPL: nomad_server_stack = pulumi.StackReference(f"grapl/nomad/{stack_name}") nomad_provider = get_nomad_provider_address(nomad_server_stack) ##### Business Logic grapl_stack = GraplStack(stack_name) e2e_test_job_vars: NomadVars = { "analyzer_bucket": grapl_stack.analyzer_bucket, "aws_env_vars_for_local": grapl_stack.aws_env_vars_for_local, "aws_region": aws.get_region().name, "container_images": _e2e_container_images(artifacts), # Used by graplctl to determine if it should manual-event or not "stack_name": grapl_stack.upstream_stack_name, "kafka_bootstrap_servers": grapl_stack.kafka_bootstrap_servers, "kafka_sasl_username": grapl_stack.kafka_e2e_sasl_username, "kafka_sasl_password": grapl_stack.kafka_e2e_sasl_password, "kafka_consumer_group_name": grapl_stack.kafka_e2e_consumer_group_name, "schema_properties_table_name": grapl_stack.schema_properties_table_name, "sysmon_log_bucket": grapl_stack.sysmon_log_bucket, "schema_table_name": grapl_stack.schema_table_name, "sysmon_generator_queue": grapl_stack.sysmon_generator_queue, "test_user_name": grapl_stack.test_user_name, "test_user_password_secret_id": grapl_stack.test_user_password_secret_id, } e2e_tests = NomadJob( "e2e-tests", jobspec=path_from_root("nomad/e2e-tests.nomad").resolve(), vars=e2e_test_job_vars, opts=pulumi.ResourceOptions(provider=nomad_provider), ) if config.LOCAL_GRAPL: # We don't do integration tests in AWS yet, mostly because the current # Python Pants integration test setup is funky and requires an on-disk # Grapl repo. integration_test_job_vars: NomadVars = { "aws_env_vars_for_local": grapl_stack.aws_env_vars_for_local, "aws_region": aws.get_region().name, "container_images": _integration_container_images(artifacts), "docker_user": os.environ["DOCKER_USER"], "grapl_root": os.environ["GRAPL_ROOT"], "kafka_bootstrap_servers": grapl_stack.kafka_bootstrap_servers, "kafka_sasl_username": grapl_stack.kafka_e2e_sasl_username, "kafka_sasl_password": grapl_stack.kafka_e2e_sasl_password, "redis_endpoint": grapl_stack.redis_endpoint, "schema_properties_table_name": grapl_stack.schema_properties_table_name, "test_user_name": grapl_stack.test_user_name, "test_user_password_secret_id": grapl_stack.test_user_password_secret_id, "plugin_work_queue_db_hostname": grapl_stack.plugin_work_queue_db_hostname, "plugin_work_queue_db_port": grapl_stack.plugin_work_queue_db_port, "plugin_work_queue_db_username": grapl_stack.plugin_work_queue_db_username, "plugin_work_queue_db_password": grapl_stack.plugin_work_queue_db_password, "organization_management_db_hostname": grapl_stack.organization_management_db_hostname, "organization_management_db_port": grapl_stack.organization_management_db_port, "organization_management_db_username": grapl_stack.organization_management_db_username, "organization_management_db_password": grapl_stack.organization_management_db_password, } integration_tests = NomadJob( "integration-tests", jobspec=path_from_root("nomad/local/integration-tests.nomad").resolve(), vars=integration_test_job_vars, opts=pulumi.ResourceOptions(provider=nomad_provider), )
def __init__(self, resource_name, props: Optional[dict] = None, opts=None): super(MyCustom, self).__init__("pkg:index:MyCustom", resource_name, props, opts) def do_invoke(): value = pulumi.runtime.invoke("test:index:MyFunction", props={ "value": 41 }).value return value["out_value"] mycomponent = MyComponent("mycomponent", inprop="hello") myinstance = Instance("instance", name="myvm", value=pulumi.Output.secret("secret_value")) mycustom = MyCustom("mycustom", {"instance": myinstance}) invoke_result = do_invoke() # Pass myinstance several more times to ensure deserialization of the resource reference # works on other asyncio threads. for x in range(5): MyCustom(f"mycustom{x}", {"instance": myinstance}) dns_ref = pulumi.StackReference("dns") pulumi.export("hello", "world") pulumi.export("outprop", mycomponent.outprop) pulumi.export("public_ip", myinstance.public_ip)
def stack_reference(self, stack_name: str): stack = pulumi.StackReference(stack_name) return stack
"""An AWS Python Pulumi program""" # REF: https://github.com/pulumi/examples/blob/master/aws-py-fargate/__main__.py import json import pulumi import pulumi_aws as aws # get configs defined in our yaml files config = pulumi.Config() network_layer_stack = config.require("network-layer-stack") pulumi_account = config.require("user-account") # get settings from stack references env = pulumi.get_stack() network_layer = pulumi.StackReference(f"{pulumi_account}/network-layer/{env}") # Read back the project VPC and subnets id's that were set up in the network-layer-{env}, which we will use. vpc_id = network_layer.require_output("vcp_id") vpc_azs = network_layer.require_output("vpc_azs") private_subnet_1_id = network_layer.require_output("private_subnet_1_id") private_subnet_2_id = network_layer.require_output("private_subnet_2_id") public_subnet_1_id = network_layer.require_output("public_subnet_1_id") public_subnet_2_id = network_layer.require_output("public_subnet_2_id") # # un-stringify the lists # private_subnets = json.loads(private_subnets) # public_subnets = json.loads(public_subnets) # Create an ECS cluster to run a container-based service. cluster = aws.ecs.Cluster("dask-ml-workflows")
import pulumi import pulumi_aws as aws import pulumi_random as random # read local config settings config = pulumi.Config() certificate_arn = config.require("certificate_arn") web_type = config.require("web_type") api_type = config.require("api_type") web_port = config.require_int("web_port") api_port = config.require_int("api_port") alb_port = config.require_int("alb_port") vpc_stack = config.require("vpc_stack") # get stack reference from vpc vpc = pulumi.StackReference(vpc_stack) # get vpc/subnet vpc_id = vpc.get_output("vpc_id") private_subnets = vpc.get_output("private_subnets") private_subnet_ids = vpc.get_output("private_subnet_ids") public_subnets = vpc.get_output("public_subnets") public_subnet_ids = vpc.get_output("public_subnet_ids") # get user data script for web with open('web_user_data.tpl', 'r') as tpl_file: web_user_data = tpl_file.read() # get user data script for api with open('api_user_data.tpl', 'r') as tpl_file: api_user_data = tpl_file.read()