def Run(self, args): """Run 'variantsets list'. Args: args: argparse.Namespace, The arguments that this command was invoked with. Returns: The list of variant sets for this dataset. """ apitools_client = genomics_util.GetGenomicsClient() req_class = genomics_util.GetGenomicsMessages().SearchVariantSetsRequest request = req_class(datasetIds=[args.dataset_id]) return list_pager.YieldFromList( apitools_client.variantsets, request, method='Search', limit=args.limit, batch_size_attribute='pageSize', batch_size=args.limit, # Use limit if any, else server default. field='variantSets')
def Run(self, args): apitools_client = genomics_util.GetGenomicsClient() messages = genomics_util.GetGenomicsMessages() dataset_resource = resources.REGISTRY.Parse( args.id, collection='genomics.datasets') policy_request = messages.GenomicsDatasetsGetIamPolicyRequest( resource='datasets/{0}'.format(dataset_resource.Name()), getIamPolicyRequest=messages.GetIamPolicyRequest(), ) policy = apitools_client.datasets.GetIamPolicy(policy_request) iam_util.AddBindingToIamPolicy(messages.Binding, policy, args.member, args.role) policy_request = messages.GenomicsDatasetsSetIamPolicyRequest( resource='datasets/{0}'.format(dataset_resource.Name()), setIamPolicyRequest=messages.SetIamPolicyRequest(policy=policy), ) return apitools_client.datasets.SetIamPolicy(policy_request)
def Run(self, args): """This is what gets called when the user runs this command. Args: args: argparse.Namespace, All the arguments that were provided to this command invocation. Raises: HttpException: An http error response was received while executing api request. Returns: None """ apitools_client = genomics_util.GetGenomicsClient() genomics_messages = genomics_util.GetGenomicsMessages() call_set = genomics_messages.CallSet( name=args.name, variantSetIds=[args.variant_set_id], ) return apitools_client.callsets.Create(call_set)
def Run(self, args): """Run 'callsets list'. Args: args: argparse.Namespace, The arguments that this command was invoked with. Returns: The list of callsets matching the given variant set ids. """ apitools_client = genomics_util.GetGenomicsClient() req_class = genomics_util.GetGenomicsMessages().SearchCallSetsRequest request = req_class(name=args.name, variantSetIds=args.variant_set_ids) return list_pager.YieldFromList( apitools_client.callsets, request, method='Search', limit=args.limit, batch_size_attribute='pageSize', batch_size=args.limit, # Use limit if any, else server default. field='callSets')
def Run(self, args): """This is what gets called when the user runs this command. Args: args: an argparse namespace, All the arguments that were provided to this command invocation. Returns: an ImportVariantsResponse message """ apitools_client = genomics_util.GetGenomicsClient() genomics_messages = genomics_util.GetGenomicsMessages() format_enum = genomics_messages.ImportVariantsRequest.FormatValueValuesEnum file_format = format_enum.FORMAT_VCF if args.file_format == 'COMPLETE_GENOMICS': file_format = format_enum.FORMAT_COMPLETE_GENOMICS imc = genomics_messages.ImportVariantsRequest.InfoMergeConfigValue ops_enum = imc.AdditionalProperty.ValueValueValuesEnum info_merge_config = None if args.info_merge_config is not None: additional_properties = [] for k, v in sorted(args.info_merge_config.items()): op = ops_enum.INFO_MERGE_OPERATION_UNSPECIFIED if v == 'IGNORE_NEW': op = ops_enum.IGNORE_NEW elif v == 'MOVE_TO_CALLS': op = ops_enum.MOVE_TO_CALLS additional_properties.append(imc.AdditionalProperty(key=k, value=op)) info_merge_config = imc(additionalProperties=additional_properties) request = genomics_messages.ImportVariantsRequest( variantSetId=args.variantset_id, sourceUris=args.source_uris, format=file_format, normalizeReferenceNames=args.normalize_reference_names, infoMergeConfig=info_merge_config) return apitools_client.variants.Import(request)
def Run(self, args): """This is what gets called when the user runs this command. Args: args: an argparse namespace, All the arguments that were provided to this command invocation. Returns: A list of variants that meet the search criteria. """ apitools_client = genomics_util.GetGenomicsClient() genomics_messages = genomics_util.GetGenomicsMessages() fields = genomics_util.GetQueryFields(self.GetReferencedKeyNames(args), 'variants') if fields: global_params = genomics_messages.StandardQueryParameters( fields=fields) else: global_params = None variant_set_id = [args.variant_set_id] if args.variant_set_id else [] pager = list_pager.YieldFromList( apitools_client.variants, genomics_messages.SearchVariantsRequest( variantSetIds=variant_set_id, callSetIds=args.call_set_ids, referenceName=args.reference_name, start=args.start, end=args.end, maxCalls=args.limit_calls), global_params=global_params, limit=args.limit, method='Search', batch_size_attribute='pageSize', batch_size=args.page_size, field='variants') return genomics_util.ReraiseHttpExceptionPager(pager, self.RewriteError)
def Run(self, args): """This is what gets called when the user runs this command. Args: args: an argparse namespace, All the arguments that were provided to this command invocation. Raises: HttpException: An http error response was received while executing api request. Returns: None """ apitools_client = genomics_util.GetGenomicsClient() genomics_messages = genomics_util.GetGenomicsMessages() request = genomics_messages.GenomicsDatasetsPatchRequest( dataset=genomics_messages.Dataset(name=args.name, ), datasetId=args.id, ) return apitools_client.datasets.Patch(request)
def Run(self, args): """Run 'readgroupsets list'. Args: args: argparse.Namespace, The arguments that this command was invoked with. Returns: The list of read group sets matching the given dataset ids. Raises: HttpException: An http error response was received while executing api request. """ apitools_client = genomics_util.GetGenomicsClient() messages = genomics_util.GetGenomicsMessages() req_class = messages.SearchReadGroupSetsRequest fields = genomics_util.GetQueryFields(self.GetReferencedKeyNames(args), 'readGroupSets') if fields: global_params = messages.StandardQueryParameters(fields=fields) else: global_params = None page_size = args.page_size if args.limit and args.limit < page_size: page_size = args.limit pager = list_pager.YieldFromList(apitools_client.readgroupsets, req_class( name=args.name, datasetIds=args.dataset_ids), method='Search', global_params=global_params, limit=args.limit, batch_size_attribute='pageSize', batch_size=page_size, field='readGroupSets') return genomics_util.ReraiseHttpExceptionPager(pager)
def Run(self, args): """Run 'operations list'. Args: args: argparse.Namespace, The arguments that this command was invoked with. Yields: The list of operations for this project. Raises: HttpException: An http error response was received while executing api request. """ apitools_client = genomics_util.GetGenomicsClient() genomics_messages = genomics_util.GetGenomicsMessages() if args.where: args.where += ' AND ' args.where += 'projectId=%s' % genomics_util.GetProjectId() request = genomics_messages.GenomicsOperationsListRequest( name='operations', filter=args.where) try: for resource in list_pager.YieldFromList( apitools_client.operations, request, limit=args.limit, batch_size_attribute='pageSize', batch_size=args. limit, # Use limit if any, else server default. field='operations'): yield resource except apitools_exceptions.HttpError as error: raise exceptions.HttpException( genomics_util.GetErrorMessage(error))
def Run(self, args): """This is what gets called when the user runs this command. Args: args: an argparse namespace, All the arguments that were provided to this command invocation. Raises: HttpException: An http error response was received while executing api request. GenomicsError: if canceled by the user. Returns: None """ apitools_client = genomics_util.GetGenomicsClient() genomics_messages = genomics_util.GetGenomicsMessages() get_request = genomics_messages.GenomicsVariantsetsGetRequest( variantSetId=str(args.variant_set_id)) variant_set = apitools_client.variantsets.Get(get_request) prompt_message = ( 'Deleting variant set {0}: "{1}" will also delete all its contents ' '(variants, callsets, and calls).').format(args.variant_set_id, variant_set.name) if not console_io.PromptContinue(message=prompt_message): raise GenomicsError('Deletion aborted by user.') req = genomics_messages.GenomicsVariantsetsDeleteRequest( variantSetId=args.variant_set_id, ) ret = apitools_client.variantsets.Delete(req) log.DeletedResource('{0}: "{1}"'.format(args.variant_set_id, variant_set.name)) return ret
def Run(self, args): """This is what gets called when the user runs this command. Args: args: an argparse namespace, All the arguments that were provided to this command invocation. Raises: HttpException: An http error response was received while executing api request. Returns: None """ apitools_client = genomics_util.GetGenomicsClient() genomics_messages = genomics_util.GetGenomicsMessages() variantset = genomics_messages.VariantSet( datasetId=args.dataset_id, referenceSetId=args.reference_set_id, name=args.name, description=args.description) return apitools_client.variantsets.Create(variantset)
def Run(self, args): """This is what gets called when the user runs this command. Args: args: an argparse namespace, All the arguments that were provided to this command invocation. Raises: HttpException: An http error response was received while executing api request. Returns: None """ apitools_client = genomics_util.GetGenomicsClient() genomics_messages = genomics_util.GetGenomicsMessages() if not (args.name or args.reference_set_id): raise GenomicsError( 'Must specify --name and/or --reference-set-id') updated = genomics_messages.ReadGroupSet() mask = [] # TODO(b/26871577): Consider using resource parser here if args.name: updated.name = args.name mask.append('name') if args.reference_set_id: updated.referenceSetId = args.reference_set_id mask.append('referenceSetId') request = genomics_messages.GenomicsReadgroupsetsPatchRequest( readGroupSet=updated, readGroupSetId=str(args.id), updateMask=','.join(mask)) return apitools_client.readgroupsets.Patch(request)
def Run(self, args): """This is what gets called when the user runs this command. Args: args: an argparse namespace, All the arguments that were provided to this command invocation. Raises: HttpException: An http error response was received while executing api request. Returns: a VariantSet whose id matches args.id. """ apitools_client = genomics_util.GetGenomicsClient() genomics_messages = genomics_util.GetGenomicsMessages() updated = genomics_messages.VariantSet() mask = [] if args.name: updated.name = args.name mask.append('name') if args.description: updated.description = args.description mask.append('description') request = genomics_messages.GenomicsVariantsetsPatchRequest( variantSetId=args.id, variantSet=updated, updateMask=','.join(mask)) variantset = apitools_client.variantsets.Patch(request) log.status.Print( 'Updated variant set {0}, name: {1}, description: {2}'.format( variantset.id, variantset.name, variantset.description)) return variantset
def Run(self, args): """This is what gets called when the user runs this command. Args: args: argparse.Namespace, All the arguments that were provided to this command invocation. Raises: files.Error: A file argument could not be read. GenomicsError: User input was invalid. HttpException: An http error response was received while executing api request. Returns: Operation representing the running pipeline. """ apitools_client = genomics_util.GetGenomicsClient('v1alpha2') genomics_messages = genomics_util.GetGenomicsMessages('v1alpha2') pipeline = genomics_util.GetFileAsMessage( args.pipeline_file, genomics_messages.Pipeline, self.context[lib.STORAGE_V1_CLIENT_KEY]) pipeline.projectId = genomics_util.GetProjectId() arg_inputs = _ValidateAndMergeArgInputs(args) inputs = genomics_util.ArgDictToAdditionalPropertiesList( arg_inputs, genomics_messages.RunPipelineArgs.InputsValue.AdditionalProperty) outputs = genomics_util.ArgDictToAdditionalPropertiesList( args.outputs, genomics_messages.RunPipelineArgs.OutputsValue.AdditionalProperty) # Set "overrides" on the resources. If the user did not pass anything on # the command line, do not set anything in the resource: preserve the # user-intent "did not set" vs. "set an empty value/list" resources = genomics_messages.PipelineResources( preemptible=args.preemptible) if args.memory: resources.minimumRamGb = args.memory if args.disk_size: resources.disks = [] for disk_encoding in args.disk_size.split(','): disk_args = disk_encoding.split(':', 1) resources.disks.append( genomics_messages.Disk(name=disk_args[0], sizeGb=int(disk_args[1]))) # Progression for picking the right zones... # If specified on the command line, use them. # If specified in the Pipeline definition, use them. # If there is a GCE default zone in the local configuration, use it. # Else let the API select a zone if args.zones: resources.zones = args.zones elif pipeline.resources and pipeline.resources.zones: pass elif properties.VALUES.compute.zone.Get(): resources.zones = [properties.VALUES.compute.zone.Get()] request = genomics_messages.RunPipelineRequest( ephemeralPipeline=pipeline, pipelineArgs=genomics_messages.RunPipelineArgs( inputs=genomics_messages.RunPipelineArgs.InputsValue( additionalProperties=inputs), outputs=genomics_messages.RunPipelineArgs.OutputsValue( additionalProperties=outputs), clientId=args.run_id, logging=genomics_messages.LoggingOptions(gcsPath=args.logging), labels=labels_util.ParseCreateArgs( args, genomics_messages.RunPipelineArgs.LabelsValue), projectId=genomics_util.GetProjectId(), serviceAccount=genomics_messages.ServiceAccount( email=args.service_account_email, scopes=args.service_account_scopes), resources=resources)) result = apitools_client.pipelines.Run(request) log.status.Print('Running [{0}].'.format(result.name)) return result
def Run(self, args): """Run 'operations list'. Args: args: argparse.Namespace, The arguments that this command was invoked with. Returns: The list of operations for this project. """ both = not args.filter and not args.where outputs = [] if both or args.filter: apitools_client = genomics_util.GetGenomicsClient('v2alpha1') genomics_messages = genomics_util.GetGenomicsMessages('v2alpha1') request_filter = None if args.filter: rewriter = filter_rewrite.OperationsBackend() args.filter, request_filter = rewriter.Rewrite(args.filter) log.info('client_filter=%r server_filter=%r', args.filter, request_filter) request = genomics_messages.GenomicsProjectsOperationsListRequest( name='projects/%s/operations' % (genomics_util.GetProjectId(), ), filter=request_filter) outputs.append( list_pager.YieldFromList( apitools_client.projects_operations, request, limit=args.limit, batch_size_attribute='pageSize', batch_size=args. limit, # Use limit if any, else server default. field='operations')) if both or args.where: apitools_client = genomics_util.GetGenomicsClient() genomics_messages = genomics_util.GetGenomicsMessages() if args.where: args.where += ' AND ' args.where += 'projectId=%s' % genomics_util.GetProjectId() request = genomics_messages.GenomicsOperationsListRequest( name='operations', filter=args.where) outputs.append( list_pager.YieldFromList( apitools_client.operations, request, limit=args.limit, batch_size_attribute='pageSize', batch_size=args. limit, # Use limit if any, else server default. field='operations')) return chain.from_iterable(outputs)
def __init__(self, version): self._messages = genomics_util.GetGenomicsMessages(version) self._client = genomics_util.GetGenomicsClient(version) self._registry = resources.REGISTRY.Clone() self._registry.RegisterApiByName('genomics', version)
def Run(self, args): """This is what gets called when the user runs this command. Args: args: argparse.Namespace, All the arguments that were provided to this command invocation. Raises: files.Error: A file argument could not be read. GenomicsError: User input was invalid. HttpException: An http error response was received while executing api request. Returns: Operation representing the running pipeline. """ v2 = False pipeline = None apitools_client = genomics_util.GetGenomicsClient('v1alpha2') genomics_messages = genomics_util.GetGenomicsMessages('v1alpha2') if args.pipeline_file: if args.command_line: # TODO(b/79982664): Use a mutex argument group instead. raise exceptions.GenomicsError( '--command-line cannot be used with --pipeline-file.') pipeline = genomics_util.GetFileAsMessage( args.pipeline_file, genomics_messages.Pipeline, self.context[lib.STORAGE_V1_CLIENT_KEY]) pipeline.projectId = genomics_util.GetProjectId() if not pipeline.docker: v2 = True apitools_client = genomics_util.GetGenomicsClient('v2alpha1') genomics_messages = genomics_util.GetGenomicsMessages( 'v2alpha1') pipeline = genomics_util.GetFileAsMessage( args.pipeline_file, genomics_messages.Pipeline, self.context[lib.STORAGE_V1_CLIENT_KEY]) elif args.command_line: v2 = True apitools_client = genomics_util.GetGenomicsClient('v2alpha1') genomics_messages = genomics_util.GetGenomicsMessages('v2alpha1') pipeline = genomics_messages.Pipeline(actions=[ genomics_messages.Action(imageUri=args.docker_image, commands=['-c', args.command_line], entrypoint='bash') ]) else: raise exceptions.GenomicsError( 'Either --pipeline-file or --command-line is required.') arg_inputs, is_local_file = _ValidateAndMergeArgInputs(args) request = None if v2: # Create messages up front to avoid checking for None everywhere. if not pipeline.resources: pipeline.resources = genomics_messages.Resources() resources = pipeline.resources if not resources.virtualMachine: resources.virtualMachine = genomics_messages.VirtualMachine( machineType='n1-standard-1') virtual_machine = resources.virtualMachine if not virtual_machine.serviceAccount: virtual_machine.serviceAccount = genomics_messages.ServiceAccount( ) # Always set the project id. resources.projectId = genomics_util.GetProjectId() # Update the pipeline based on arguments. if args.memory or args.cpus: # Default to n1-standard1 sizes. virtual_machine.machineType = 'custom-%d-%d' % ( args.cpus or 1, (args.memory or 3.84) * 1000) if args.preemptible: virtual_machine.preemptible = args.preemptible if args.zones: resources.zones = args.zones elif not resources.zones and properties.VALUES.compute.zone.Get(): resources.zones = [properties.VALUES.compute.zone.Get()] if args.regions: resources.regions = args.regions elif not resources.regions and properties.VALUES.compute.region.Get( ): resources.regions = [properties.VALUES.compute.region.Get()] if args.service_account_email != 'default': virtual_machine.serviceAccount.email = args.service_account_email if args.service_account_scopes: virtual_machine.serviceAccount.scopes = args.service_account_scopes # Always add a scope for GCS in case any arguments need it. virtual_machine.serviceAccount.scopes.append( 'https://www.googleapis.com/auth/devstorage.read_write') # Generate paths for inputs and outputs in a shared location and put them # into the environment for actions based on their name. env = {} if arg_inputs: input_generator = _SharedPathGenerator('input') for name, value in arg_inputs.items(): if genomics_util.IsGcsPath(value): env[name] = input_generator.Generate() pipeline.actions.insert( 0, genomics_messages.Action( imageUri=CLOUD_SDK_IMAGE, commands=[ '/bin/sh', '-c', 'gsutil -q cp %s ${%s}' % (value, name) ])) elif name in is_local_file: env[name] = input_generator.Generate() pipeline.actions.insert( 0, genomics_messages.Action( imageUri=CLOUD_SDK_IMAGE, commands=[ '/bin/sh', '-c', 'echo "%s" | base64 -d > ${%s}' % (base64.b64encode(value), name) ])) else: env[name] = value if args.outputs: output_generator = _SharedPathGenerator('output') for name, value in args.outputs.items(): env[name] = output_generator.Generate() pipeline.actions.append( genomics_messages.Action(imageUri=CLOUD_SDK_IMAGE, commands=[ '/bin/sh', '-c', 'gsutil -q cp ${%s} %s' % (name, value) ])) # Merge any existing pipeline arguments into the generated environment and # update the pipeline. if pipeline.environment: for val in pipeline.environment.additionalProperties: if val.key not in env: env[val.key] = val.value pipeline.environment = genomics_messages.Pipeline.EnvironmentValue( additionalProperties=genomics_util. ArgDictToAdditionalPropertiesList( env, genomics_messages.Pipeline.EnvironmentValue. AdditionalProperty)) if arg_inputs or args.outputs: virtual_machine.disks.append( genomics_messages.Disk(name=SHARED_DISK)) for action in pipeline.actions: action.mounts.append( genomics_messages.Mount(disk=SHARED_DISK, path='/' + SHARED_DISK)) if args.logging: pipeline.actions.append( genomics_messages.Action( imageUri=CLOUD_SDK_IMAGE, commands=[ '/bin/sh', '-c', 'gsutil -q cp /google/logs/output ' + args.logging ], flags=[(genomics_messages.Action. FlagsValueListEntryValuesEnum.ALWAYS_RUN)])) # Update disk sizes if specified, potentially including the shared disk. if args.disk_size: disk_sizes = {} for disk_encoding in args.disk_size.split(','): parts = disk_encoding.split(':', 1) try: disk_sizes[parts[0]] = int(parts[1]) except: raise exceptions.GenomicsError('Invalid --disk-size.') for disk in virtual_machine.disks: size = disk_sizes[disk.name] if size: disk.sizeGb = size request = genomics_messages.RunPipelineRequest( pipeline=pipeline, labels=labels_util.ParseCreateArgs( args, genomics_messages.RunPipelineRequest.LabelsValue)) else: inputs = genomics_util.ArgDictToAdditionalPropertiesList( arg_inputs, genomics_messages.RunPipelineArgs.InputsValue. AdditionalProperty) outputs = genomics_util.ArgDictToAdditionalPropertiesList( args.outputs, genomics_messages.RunPipelineArgs.OutputsValue. AdditionalProperty) # Set "overrides" on the resources. If the user did not pass anything on # the command line, do not set anything in the resource: preserve the # user-intent "did not set" vs. "set an empty value/list" resources = genomics_messages.PipelineResources( preemptible=args.preemptible) if args.memory: resources.minimumRamGb = args.memory if args.cpus: resources.minimumCpuCores = args.cpus if args.disk_size: resources.disks = [] for disk_encoding in args.disk_size.split(','): disk_args = disk_encoding.split(':', 1) resources.disks.append( genomics_messages.Disk(name=disk_args[0], sizeGb=int(disk_args[1]))) # Progression for picking the right zones... # If specified on the command line, use them. # If specified in the Pipeline definition, use them. # If there is a GCE default zone in the local configuration, use it. # Else let the API select a zone if args.zones: resources.zones = args.zones elif pipeline.resources and pipeline.resources.zones: pass elif properties.VALUES.compute.zone.Get(): resources.zones = [properties.VALUES.compute.zone.Get()] request = genomics_messages.RunPipelineRequest( ephemeralPipeline=pipeline, pipelineArgs=genomics_messages.RunPipelineArgs( inputs=genomics_messages.RunPipelineArgs.InputsValue( additionalProperties=inputs), outputs=genomics_messages.RunPipelineArgs.OutputsValue( additionalProperties=outputs), clientId=args.run_id, logging=genomics_messages.LoggingOptions( gcsPath=args.logging), labels=labels_util.ParseCreateArgs( args, genomics_messages.RunPipelineArgs.LabelsValue), projectId=genomics_util.GetProjectId(), serviceAccount=genomics_messages.ServiceAccount( email=args.service_account_email, scopes=args.service_account_scopes), resources=resources)) result = apitools_client.pipelines.Run(request) log.status.Print('Running [{0}].'.format(result.name)) return result
def Run(self, args): """This is what gets called when the user runs this command. Args: args: argparse.Namespace, All the arguments that were provided to this command invocation. Raises: files.Error: A file argument could not be read. GenomicsError: User input was invalid. HttpException: An http error response was received while executing api request. Returns: Operation representing the running pipeline. """ pipeline = None apitools_client = genomics_util.GetGenomicsClient('v2alpha1') genomics_messages = genomics_util.GetGenomicsMessages('v2alpha1') if args.pipeline_file: if args.command_line: # TODO(b/79982664): Use a mutex argument group instead. raise exceptions.GenomicsError( '--command-line cannot be used with --pipeline-file.') pipeline = genomics_util.GetFileAsMessage( args.pipeline_file, genomics_messages.Pipeline, self.context[lib.STORAGE_V1_CLIENT_KEY]) elif args.command_line: pipeline = genomics_messages.Pipeline(actions=[ genomics_messages.Action(imageUri=args.docker_image, commands=['-c', args.command_line], entrypoint='bash') ]) else: raise exceptions.GenomicsError( 'Either --pipeline-file or --command-line is required.') arg_inputs, is_local_file = _ValidateAndMergeArgInputs(args) request = None # Create messages up front to avoid checking for None everywhere. if not pipeline.resources: pipeline.resources = genomics_messages.Resources() resources = pipeline.resources if not resources.virtualMachine: resources.virtualMachine = genomics_messages.VirtualMachine( machineType='n1-standard-1') virtual_machine = resources.virtualMachine if not virtual_machine.serviceAccount: virtual_machine.serviceAccount = genomics_messages.ServiceAccount() # Always set the project id. resources.projectId = genomics_util.GetProjectId() # Update the pipeline based on arguments. if args.memory or args.cpus: # Default to n1-standard1 sizes. virtual_machine.machineType = 'custom-%d-%d' % ( args.cpus or 1, (args.memory or 3.75) * 1024) if args.preemptible: virtual_machine.preemptible = args.preemptible if args.zones: resources.zones = args.zones elif not resources.zones and properties.VALUES.compute.zone.Get(): resources.zones = [properties.VALUES.compute.zone.Get()] if args.regions: resources.regions = args.regions elif not resources.regions and properties.VALUES.compute.region.Get(): resources.regions = [properties.VALUES.compute.region.Get()] if args.service_account_email != 'default': virtual_machine.serviceAccount.email = args.service_account_email if args.service_account_scopes: virtual_machine.serviceAccount.scopes = args.service_account_scopes # Always add a scope for GCS in case any arguments need it. virtual_machine.serviceAccount.scopes.append( 'https://www.googleapis.com/auth/devstorage.read_write') # Attach custom network/subnetwork (if set). if args.network or args.subnetwork: if not virtual_machine.network: virtual_machine.network = genomics_messages.Network() if args.network: virtual_machine.network.name = args.network if args.subnetwork: virtual_machine.network.subnetwork = args.subnetwork if args.boot_disk_size is not None: if args.boot_disk_size <= 0: raise exceptions.GenomicsError( 'Boot disk size must be greater than zero.') virtual_machine.bootDiskSizeGb = args.boot_disk_size # Generate paths for inputs and outputs in a shared location and put them # into the environment for actions based on their name. env = {} if arg_inputs: input_generator = _SharedPathGenerator('input') for name, value in arg_inputs.items(): if genomics_util.IsGcsPath(value): env[name] = input_generator.Generate() pipeline.actions.insert( 0, genomics_messages.Action( imageUri=CLOUD_SDK_IMAGE, commands=[ '/bin/sh', '-c', 'gsutil -m -q cp %s ${%s}' % (value, name) ])) elif name in is_local_file: # TODO(b/183206325): Get test coverage to 100%. env[name] = input_generator.Generate() pipeline.actions.insert( 0, genomics_messages.Action( imageUri=CLOUD_SDK_IMAGE, commands=[ '/bin/sh', '-c', 'echo "%s" | base64 -d > ${%s}' % (base64.b64encode( value.encode()).decode(), name) ])) else: env[name] = value if args.outputs: output_generator = _SharedPathGenerator('output') for name, value in args.outputs.items(): env[name] = output_generator.Generate() pipeline.actions.append( genomics_messages.Action(imageUri=CLOUD_SDK_IMAGE, commands=[ '/bin/sh', '-c', 'gsutil -m -q cp ${%s} %s' % (name, value) ])) if args.env_vars: for name, value in args.env_vars.items(): env[name] = value # Merge any existing pipeline arguments into the generated environment and # update the pipeline. if pipeline.environment: for val in pipeline.environment.additionalProperties: if val.key not in env: env[val.key] = val.value pipeline.environment = genomics_messages.Pipeline.EnvironmentValue( additionalProperties=genomics_util. ArgDictToAdditionalPropertiesList( env, genomics_messages.Pipeline.EnvironmentValue. AdditionalProperty)) if arg_inputs or args.outputs: virtual_machine.disks.append( genomics_messages.Disk(name=SHARED_DISK)) for action in pipeline.actions: action.mounts.append( genomics_messages.Mount(disk=SHARED_DISK, path='/' + SHARED_DISK)) if args.logging: pipeline.actions.append( genomics_messages.Action( imageUri=CLOUD_SDK_IMAGE, commands=[ '/bin/sh', '-c', 'gsutil -m -q cp /google/logs/output ' + args.logging ], flags=[(genomics_messages.Action. FlagsValueListEntryValuesEnum.ALWAYS_RUN)])) # Update disk sizes if specified, potentially including the shared disk. if args.disk_size: disk_sizes = {} for disk_encoding in args.disk_size.split(','): parts = disk_encoding.split(':', 1) try: disk_sizes[parts[0]] = int(parts[1]) except: raise exceptions.GenomicsError('Invalid --disk-size.') for disk in virtual_machine.disks: if disk.name in disk_sizes: disk.sizeGb = disk_sizes[disk.name] request = genomics_messages.RunPipelineRequest( pipeline=pipeline, labels=labels_util.ParseCreateArgs( args, genomics_messages.RunPipelineRequest.LabelsValue)) result = apitools_client.pipelines.Run(request) log.status.Print('Running [{0}].'.format(result.name)) return result