def Run(self, args):
"""Run 'variantsets list'.
Args:
args: argparse.Namespace, The arguments that this command was invoked
with.
Returns:
The list of variant sets for this dataset.
"""
apitools_client = genomics_util.GetGenomicsClient()
req_class = genomics_util.GetGenomicsMessages().SearchVariantSetsRequest
request = req_class(datasetIds=[args.dataset_id])
return list_pager.YieldFromList(
apitools_client.variantsets,
request,
method='Search',
limit=args.limit,
batch_size_attribute='pageSize',
batch_size=args.limit, # Use limit if any, else server default.
field='variantSets')
def Run(self, args):
apitools_client = genomics_util.GetGenomicsClient()
messages = genomics_util.GetGenomicsMessages()
dataset_resource = resources.REGISTRY.Parse(
args.id, collection='genomics.datasets')
policy_request = messages.GenomicsDatasetsGetIamPolicyRequest(
resource='datasets/{0}'.format(dataset_resource.Name()),
getIamPolicyRequest=messages.GetIamPolicyRequest(),
)
policy = apitools_client.datasets.GetIamPolicy(policy_request)
iam_util.AddBindingToIamPolicy(messages.Binding, policy, args.member,
args.role)
policy_request = messages.GenomicsDatasetsSetIamPolicyRequest(
resource='datasets/{0}'.format(dataset_resource.Name()),
setIamPolicyRequest=messages.SetIamPolicyRequest(policy=policy),
)
return apitools_client.datasets.SetIamPolicy(policy_request)
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: argparse.Namespace, All the arguments that were provided to this
command invocation.
Raises:
HttpException: An http error response was received while executing api
request.
Returns:
None
"""
apitools_client = genomics_util.GetGenomicsClient()
genomics_messages = genomics_util.GetGenomicsMessages()
call_set = genomics_messages.CallSet(
name=args.name,
variantSetIds=[args.variant_set_id],
)
return apitools_client.callsets.Create(call_set)
def Run(self, args):
"""Run 'callsets list'.
Args:
args: argparse.Namespace, The arguments that this command was invoked
with.
Returns:
The list of callsets matching the given variant set ids.
"""
apitools_client = genomics_util.GetGenomicsClient()
req_class = genomics_util.GetGenomicsMessages().SearchCallSetsRequest
request = req_class(name=args.name, variantSetIds=args.variant_set_ids)
return list_pager.YieldFromList(
apitools_client.callsets,
request,
method='Search',
limit=args.limit,
batch_size_attribute='pageSize',
batch_size=args.limit, # Use limit if any, else server default.
field='callSets')
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace, All the arguments that were provided to this
command invocation.
Returns:
an ImportVariantsResponse message
"""
apitools_client = genomics_util.GetGenomicsClient()
genomics_messages = genomics_util.GetGenomicsMessages()
format_enum = genomics_messages.ImportVariantsRequest.FormatValueValuesEnum
file_format = format_enum.FORMAT_VCF
if args.file_format == 'COMPLETE_GENOMICS':
file_format = format_enum.FORMAT_COMPLETE_GENOMICS
imc = genomics_messages.ImportVariantsRequest.InfoMergeConfigValue
ops_enum = imc.AdditionalProperty.ValueValueValuesEnum
info_merge_config = None
if args.info_merge_config is not None:
additional_properties = []
for k, v in sorted(args.info_merge_config.items()):
op = ops_enum.INFO_MERGE_OPERATION_UNSPECIFIED
if v == 'IGNORE_NEW':
op = ops_enum.IGNORE_NEW
elif v == 'MOVE_TO_CALLS':
op = ops_enum.MOVE_TO_CALLS
additional_properties.append(imc.AdditionalProperty(key=k, value=op))
info_merge_config = imc(additionalProperties=additional_properties)
request = genomics_messages.ImportVariantsRequest(
variantSetId=args.variantset_id,
sourceUris=args.source_uris,
format=file_format,
normalizeReferenceNames=args.normalize_reference_names,
infoMergeConfig=info_merge_config)
return apitools_client.variants.Import(request)
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace, All the arguments that were provided to this
command invocation.
Returns:
A list of variants that meet the search criteria.
"""
apitools_client = genomics_util.GetGenomicsClient()
genomics_messages = genomics_util.GetGenomicsMessages()
fields = genomics_util.GetQueryFields(self.GetReferencedKeyNames(args),
'variants')
if fields:
global_params = genomics_messages.StandardQueryParameters(
fields=fields)
else:
global_params = None
variant_set_id = [args.variant_set_id] if args.variant_set_id else []
pager = list_pager.YieldFromList(
apitools_client.variants,
genomics_messages.SearchVariantsRequest(
variantSetIds=variant_set_id,
callSetIds=args.call_set_ids,
referenceName=args.reference_name,
start=args.start,
end=args.end,
maxCalls=args.limit_calls),
global_params=global_params,
limit=args.limit,
method='Search',
batch_size_attribute='pageSize',
batch_size=args.page_size,
field='variants')
return genomics_util.ReraiseHttpExceptionPager(pager,
self.RewriteError)
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace, All the arguments that were provided to this
command invocation.
Raises:
HttpException: An http error response was received while executing api
request.
Returns:
None
"""
apitools_client = genomics_util.GetGenomicsClient()
genomics_messages = genomics_util.GetGenomicsMessages()
request = genomics_messages.GenomicsDatasetsPatchRequest(
dataset=genomics_messages.Dataset(name=args.name, ),
datasetId=args.id,
)
return apitools_client.datasets.Patch(request)
def Run(self, args):
"""Run 'readgroupsets list'.
Args:
args: argparse.Namespace, The arguments that this command was invoked
with.
Returns:
The list of read group sets matching the given dataset ids.
Raises:
HttpException: An http error response was received while executing api
request.
"""
apitools_client = genomics_util.GetGenomicsClient()
messages = genomics_util.GetGenomicsMessages()
req_class = messages.SearchReadGroupSetsRequest
fields = genomics_util.GetQueryFields(self.GetReferencedKeyNames(args),
'readGroupSets')
if fields:
global_params = messages.StandardQueryParameters(fields=fields)
else:
global_params = None
page_size = args.page_size
if args.limit and args.limit < page_size:
page_size = args.limit
pager = list_pager.YieldFromList(apitools_client.readgroupsets,
req_class(
name=args.name,
datasetIds=args.dataset_ids),
method='Search',
global_params=global_params,
limit=args.limit,
batch_size_attribute='pageSize',
batch_size=page_size,
field='readGroupSets')
return genomics_util.ReraiseHttpExceptionPager(pager)
def Run(self, args):
"""Run 'operations list'.
Args:
args: argparse.Namespace, The arguments that this command was invoked
with.
Yields:
The list of operations for this project.
Raises:
HttpException: An http error response was received while executing api
request.
"""
apitools_client = genomics_util.GetGenomicsClient()
genomics_messages = genomics_util.GetGenomicsMessages()
if args.where:
args.where += ' AND '
args.where += 'projectId=%s' % genomics_util.GetProjectId()
request = genomics_messages.GenomicsOperationsListRequest(
name='operations', filter=args.where)
try:
for resource in list_pager.YieldFromList(
apitools_client.operations,
request,
limit=args.limit,
batch_size_attribute='pageSize',
batch_size=args.
limit, # Use limit if any, else server default.
field='operations'):
yield resource
except apitools_exceptions.HttpError as error:
raise exceptions.HttpException(
genomics_util.GetErrorMessage(error))
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace, All the arguments that were provided to this
command invocation.
Raises:
HttpException: An http error response was received while executing api
request.
GenomicsError: if canceled by the user.
Returns:
None
"""
apitools_client = genomics_util.GetGenomicsClient()
genomics_messages = genomics_util.GetGenomicsMessages()
get_request = genomics_messages.GenomicsVariantsetsGetRequest(
variantSetId=str(args.variant_set_id))
variant_set = apitools_client.variantsets.Get(get_request)
prompt_message = (
'Deleting variant set {0}: "{1}" will also delete all its contents '
'(variants, callsets, and calls).').format(args.variant_set_id,
variant_set.name)
if not console_io.PromptContinue(message=prompt_message):
raise GenomicsError('Deletion aborted by user.')
req = genomics_messages.GenomicsVariantsetsDeleteRequest(
variantSetId=args.variant_set_id, )
ret = apitools_client.variantsets.Delete(req)
log.DeletedResource('{0}: "{1}"'.format(args.variant_set_id,
variant_set.name))
return ret
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace, All the arguments that were provided to this
command invocation.
Raises:
HttpException: An http error response was received while executing api
request.
Returns:
None
"""
apitools_client = genomics_util.GetGenomicsClient()
genomics_messages = genomics_util.GetGenomicsMessages()
variantset = genomics_messages.VariantSet(
datasetId=args.dataset_id,
referenceSetId=args.reference_set_id,
name=args.name,
description=args.description)
return apitools_client.variantsets.Create(variantset)
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace, All the arguments that were provided to this
command invocation.
Raises:
HttpException: An http error response was received while executing api
request.
Returns:
None
"""
apitools_client = genomics_util.GetGenomicsClient()
genomics_messages = genomics_util.GetGenomicsMessages()
if not (args.name or args.reference_set_id):
raise GenomicsError(
'Must specify --name and/or --reference-set-id')
updated = genomics_messages.ReadGroupSet()
mask = []
# TODO(b/26871577): Consider using resource parser here
if args.name:
updated.name = args.name
mask.append('name')
if args.reference_set_id:
updated.referenceSetId = args.reference_set_id
mask.append('referenceSetId')
request = genomics_messages.GenomicsReadgroupsetsPatchRequest(
readGroupSet=updated,
readGroupSetId=str(args.id),
updateMask=','.join(mask))
return apitools_client.readgroupsets.Patch(request)
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace, All the arguments that were provided to this
command invocation.
Raises:
HttpException: An http error response was received while executing api
request.
Returns:
a VariantSet whose id matches args.id.
"""
apitools_client = genomics_util.GetGenomicsClient()
genomics_messages = genomics_util.GetGenomicsMessages()
updated = genomics_messages.VariantSet()
mask = []
if args.name:
updated.name = args.name
mask.append('name')
if args.description:
updated.description = args.description
mask.append('description')
request = genomics_messages.GenomicsVariantsetsPatchRequest(
variantSetId=args.id,
variantSet=updated,
updateMask=','.join(mask))
variantset = apitools_client.variantsets.Patch(request)
log.status.Print(
'Updated variant set {0}, name: {1}, description: {2}'.format(
variantset.id, variantset.name, variantset.description))
return variantset
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: argparse.Namespace, All the arguments that were provided to this
command invocation.
Raises:
files.Error: A file argument could not be read.
GenomicsError: User input was invalid.
HttpException: An http error response was received while executing api
request.
Returns:
Operation representing the running pipeline.
"""
apitools_client = genomics_util.GetGenomicsClient('v1alpha2')
genomics_messages = genomics_util.GetGenomicsMessages('v1alpha2')
pipeline = genomics_util.GetFileAsMessage(
args.pipeline_file, genomics_messages.Pipeline,
self.context[lib.STORAGE_V1_CLIENT_KEY])
pipeline.projectId = genomics_util.GetProjectId()
arg_inputs = _ValidateAndMergeArgInputs(args)
inputs = genomics_util.ArgDictToAdditionalPropertiesList(
arg_inputs,
genomics_messages.RunPipelineArgs.InputsValue.AdditionalProperty)
outputs = genomics_util.ArgDictToAdditionalPropertiesList(
args.outputs,
genomics_messages.RunPipelineArgs.OutputsValue.AdditionalProperty)
# Set "overrides" on the resources. If the user did not pass anything on
# the command line, do not set anything in the resource: preserve the
# user-intent "did not set" vs. "set an empty value/list"
resources = genomics_messages.PipelineResources(
preemptible=args.preemptible)
if args.memory:
resources.minimumRamGb = args.memory
if args.disk_size:
resources.disks = []
for disk_encoding in args.disk_size.split(','):
disk_args = disk_encoding.split(':', 1)
resources.disks.append(
genomics_messages.Disk(name=disk_args[0],
sizeGb=int(disk_args[1])))
# Progression for picking the right zones...
# If specified on the command line, use them.
# If specified in the Pipeline definition, use them.
# If there is a GCE default zone in the local configuration, use it.
# Else let the API select a zone
if args.zones:
resources.zones = args.zones
elif pipeline.resources and pipeline.resources.zones:
pass
elif properties.VALUES.compute.zone.Get():
resources.zones = [properties.VALUES.compute.zone.Get()]
request = genomics_messages.RunPipelineRequest(
ephemeralPipeline=pipeline,
pipelineArgs=genomics_messages.RunPipelineArgs(
inputs=genomics_messages.RunPipelineArgs.InputsValue(
additionalProperties=inputs),
outputs=genomics_messages.RunPipelineArgs.OutputsValue(
additionalProperties=outputs),
clientId=args.run_id,
logging=genomics_messages.LoggingOptions(gcsPath=args.logging),
labels=labels_util.ParseCreateArgs(
args, genomics_messages.RunPipelineArgs.LabelsValue),
projectId=genomics_util.GetProjectId(),
serviceAccount=genomics_messages.ServiceAccount(
email=args.service_account_email,
scopes=args.service_account_scopes),
resources=resources))
result = apitools_client.pipelines.Run(request)
log.status.Print('Running [{0}].'.format(result.name))
return result
def Run(self, args):
"""Run 'operations list'.
Args:
args: argparse.Namespace, The arguments that this command was invoked
with.
Returns:
The list of operations for this project.
"""
both = not args.filter and not args.where
outputs = []
if both or args.filter:
apitools_client = genomics_util.GetGenomicsClient('v2alpha1')
genomics_messages = genomics_util.GetGenomicsMessages('v2alpha1')
request_filter = None
if args.filter:
rewriter = filter_rewrite.OperationsBackend()
args.filter, request_filter = rewriter.Rewrite(args.filter)
log.info('client_filter=%r server_filter=%r', args.filter,
request_filter)
request = genomics_messages.GenomicsProjectsOperationsListRequest(
name='projects/%s/operations' %
(genomics_util.GetProjectId(), ),
filter=request_filter)
outputs.append(
list_pager.YieldFromList(
apitools_client.projects_operations,
request,
limit=args.limit,
batch_size_attribute='pageSize',
batch_size=args.
limit, # Use limit if any, else server default.
field='operations'))
if both or args.where:
apitools_client = genomics_util.GetGenomicsClient()
genomics_messages = genomics_util.GetGenomicsMessages()
if args.where:
args.where += ' AND '
args.where += 'projectId=%s' % genomics_util.GetProjectId()
request = genomics_messages.GenomicsOperationsListRequest(
name='operations', filter=args.where)
outputs.append(
list_pager.YieldFromList(
apitools_client.operations,
request,
limit=args.limit,
batch_size_attribute='pageSize',
batch_size=args.
limit, # Use limit if any, else server default.
field='operations'))
return chain.from_iterable(outputs)
def __init__(self, version):
self._messages = genomics_util.GetGenomicsMessages(version)
self._client = genomics_util.GetGenomicsClient(version)
self._registry = resources.REGISTRY.Clone()
self._registry.RegisterApiByName('genomics', version)
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: argparse.Namespace, All the arguments that were provided to this
command invocation.
Raises:
files.Error: A file argument could not be read.
GenomicsError: User input was invalid.
HttpException: An http error response was received while executing api
request.
Returns:
Operation representing the running pipeline.
"""
v2 = False
pipeline = None
apitools_client = genomics_util.GetGenomicsClient('v1alpha2')
genomics_messages = genomics_util.GetGenomicsMessages('v1alpha2')
if args.pipeline_file:
if args.command_line:
# TODO(b/79982664): Use a mutex argument group instead.
raise exceptions.GenomicsError(
'--command-line cannot be used with --pipeline-file.')
pipeline = genomics_util.GetFileAsMessage(
args.pipeline_file, genomics_messages.Pipeline,
self.context[lib.STORAGE_V1_CLIENT_KEY])
pipeline.projectId = genomics_util.GetProjectId()
if not pipeline.docker:
v2 = True
apitools_client = genomics_util.GetGenomicsClient('v2alpha1')
genomics_messages = genomics_util.GetGenomicsMessages(
'v2alpha1')
pipeline = genomics_util.GetFileAsMessage(
args.pipeline_file, genomics_messages.Pipeline,
self.context[lib.STORAGE_V1_CLIENT_KEY])
elif args.command_line:
v2 = True
apitools_client = genomics_util.GetGenomicsClient('v2alpha1')
genomics_messages = genomics_util.GetGenomicsMessages('v2alpha1')
pipeline = genomics_messages.Pipeline(actions=[
genomics_messages.Action(imageUri=args.docker_image,
commands=['-c', args.command_line],
entrypoint='bash')
])
else:
raise exceptions.GenomicsError(
'Either --pipeline-file or --command-line is required.')
arg_inputs, is_local_file = _ValidateAndMergeArgInputs(args)
request = None
if v2:
# Create messages up front to avoid checking for None everywhere.
if not pipeline.resources:
pipeline.resources = genomics_messages.Resources()
resources = pipeline.resources
if not resources.virtualMachine:
resources.virtualMachine = genomics_messages.VirtualMachine(
machineType='n1-standard-1')
virtual_machine = resources.virtualMachine
if not virtual_machine.serviceAccount:
virtual_machine.serviceAccount = genomics_messages.ServiceAccount(
)
# Always set the project id.
resources.projectId = genomics_util.GetProjectId()
# Update the pipeline based on arguments.
if args.memory or args.cpus:
# Default to n1-standard1 sizes.
virtual_machine.machineType = 'custom-%d-%d' % (
args.cpus or 1, (args.memory or 3.84) * 1000)
if args.preemptible:
virtual_machine.preemptible = args.preemptible
if args.zones:
resources.zones = args.zones
elif not resources.zones and properties.VALUES.compute.zone.Get():
resources.zones = [properties.VALUES.compute.zone.Get()]
if args.regions:
resources.regions = args.regions
elif not resources.regions and properties.VALUES.compute.region.Get(
):
resources.regions = [properties.VALUES.compute.region.Get()]
if args.service_account_email != 'default':
virtual_machine.serviceAccount.email = args.service_account_email
if args.service_account_scopes:
virtual_machine.serviceAccount.scopes = args.service_account_scopes
# Always add a scope for GCS in case any arguments need it.
virtual_machine.serviceAccount.scopes.append(
'https://www.googleapis.com/auth/devstorage.read_write')
# Generate paths for inputs and outputs in a shared location and put them
# into the environment for actions based on their name.
env = {}
if arg_inputs:
input_generator = _SharedPathGenerator('input')
for name, value in arg_inputs.items():
if genomics_util.IsGcsPath(value):
env[name] = input_generator.Generate()
pipeline.actions.insert(
0,
genomics_messages.Action(
imageUri=CLOUD_SDK_IMAGE,
commands=[
'/bin/sh', '-c',
'gsutil -q cp %s ${%s}' % (value, name)
]))
elif name in is_local_file:
env[name] = input_generator.Generate()
pipeline.actions.insert(
0,
genomics_messages.Action(
imageUri=CLOUD_SDK_IMAGE,
commands=[
'/bin/sh', '-c',
'echo "%s" | base64 -d > ${%s}' %
(base64.b64encode(value), name)
]))
else:
env[name] = value
if args.outputs:
output_generator = _SharedPathGenerator('output')
for name, value in args.outputs.items():
env[name] = output_generator.Generate()
pipeline.actions.append(
genomics_messages.Action(imageUri=CLOUD_SDK_IMAGE,
commands=[
'/bin/sh', '-c',
'gsutil -q cp ${%s} %s' %
(name, value)
]))
# Merge any existing pipeline arguments into the generated environment and
# update the pipeline.
if pipeline.environment:
for val in pipeline.environment.additionalProperties:
if val.key not in env:
env[val.key] = val.value
pipeline.environment = genomics_messages.Pipeline.EnvironmentValue(
additionalProperties=genomics_util.
ArgDictToAdditionalPropertiesList(
env, genomics_messages.Pipeline.EnvironmentValue.
AdditionalProperty))
if arg_inputs or args.outputs:
virtual_machine.disks.append(
genomics_messages.Disk(name=SHARED_DISK))
for action in pipeline.actions:
action.mounts.append(
genomics_messages.Mount(disk=SHARED_DISK,
path='/' + SHARED_DISK))
if args.logging:
pipeline.actions.append(
genomics_messages.Action(
imageUri=CLOUD_SDK_IMAGE,
commands=[
'/bin/sh', '-c',
'gsutil -q cp /google/logs/output ' + args.logging
],
flags=[(genomics_messages.Action.
FlagsValueListEntryValuesEnum.ALWAYS_RUN)]))
# Update disk sizes if specified, potentially including the shared disk.
if args.disk_size:
disk_sizes = {}
for disk_encoding in args.disk_size.split(','):
parts = disk_encoding.split(':', 1)
try:
disk_sizes[parts[0]] = int(parts[1])
except:
raise exceptions.GenomicsError('Invalid --disk-size.')
for disk in virtual_machine.disks:
size = disk_sizes[disk.name]
if size:
disk.sizeGb = size
request = genomics_messages.RunPipelineRequest(
pipeline=pipeline,
labels=labels_util.ParseCreateArgs(
args, genomics_messages.RunPipelineRequest.LabelsValue))
else:
inputs = genomics_util.ArgDictToAdditionalPropertiesList(
arg_inputs, genomics_messages.RunPipelineArgs.InputsValue.
AdditionalProperty)
outputs = genomics_util.ArgDictToAdditionalPropertiesList(
args.outputs, genomics_messages.RunPipelineArgs.OutputsValue.
AdditionalProperty)
# Set "overrides" on the resources. If the user did not pass anything on
# the command line, do not set anything in the resource: preserve the
# user-intent "did not set" vs. "set an empty value/list"
resources = genomics_messages.PipelineResources(
preemptible=args.preemptible)
if args.memory:
resources.minimumRamGb = args.memory
if args.cpus:
resources.minimumCpuCores = args.cpus
if args.disk_size:
resources.disks = []
for disk_encoding in args.disk_size.split(','):
disk_args = disk_encoding.split(':', 1)
resources.disks.append(
genomics_messages.Disk(name=disk_args[0],
sizeGb=int(disk_args[1])))
# Progression for picking the right zones...
# If specified on the command line, use them.
# If specified in the Pipeline definition, use them.
# If there is a GCE default zone in the local configuration, use it.
# Else let the API select a zone
if args.zones:
resources.zones = args.zones
elif pipeline.resources and pipeline.resources.zones:
pass
elif properties.VALUES.compute.zone.Get():
resources.zones = [properties.VALUES.compute.zone.Get()]
request = genomics_messages.RunPipelineRequest(
ephemeralPipeline=pipeline,
pipelineArgs=genomics_messages.RunPipelineArgs(
inputs=genomics_messages.RunPipelineArgs.InputsValue(
additionalProperties=inputs),
outputs=genomics_messages.RunPipelineArgs.OutputsValue(
additionalProperties=outputs),
clientId=args.run_id,
logging=genomics_messages.LoggingOptions(
gcsPath=args.logging),
labels=labels_util.ParseCreateArgs(
args, genomics_messages.RunPipelineArgs.LabelsValue),
projectId=genomics_util.GetProjectId(),
serviceAccount=genomics_messages.ServiceAccount(
email=args.service_account_email,
scopes=args.service_account_scopes),
resources=resources))
result = apitools_client.pipelines.Run(request)
log.status.Print('Running [{0}].'.format(result.name))
return result
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: argparse.Namespace, All the arguments that were provided to this
command invocation.
Raises:
files.Error: A file argument could not be read.
GenomicsError: User input was invalid.
HttpException: An http error response was received while executing api
request.
Returns:
Operation representing the running pipeline.
"""
pipeline = None
apitools_client = genomics_util.GetGenomicsClient('v2alpha1')
genomics_messages = genomics_util.GetGenomicsMessages('v2alpha1')
if args.pipeline_file:
if args.command_line:
# TODO(b/79982664): Use a mutex argument group instead.
raise exceptions.GenomicsError(
'--command-line cannot be used with --pipeline-file.')
pipeline = genomics_util.GetFileAsMessage(
args.pipeline_file, genomics_messages.Pipeline,
self.context[lib.STORAGE_V1_CLIENT_KEY])
elif args.command_line:
pipeline = genomics_messages.Pipeline(actions=[
genomics_messages.Action(imageUri=args.docker_image,
commands=['-c', args.command_line],
entrypoint='bash')
])
else:
raise exceptions.GenomicsError(
'Either --pipeline-file or --command-line is required.')
arg_inputs, is_local_file = _ValidateAndMergeArgInputs(args)
request = None
# Create messages up front to avoid checking for None everywhere.
if not pipeline.resources:
pipeline.resources = genomics_messages.Resources()
resources = pipeline.resources
if not resources.virtualMachine:
resources.virtualMachine = genomics_messages.VirtualMachine(
machineType='n1-standard-1')
virtual_machine = resources.virtualMachine
if not virtual_machine.serviceAccount:
virtual_machine.serviceAccount = genomics_messages.ServiceAccount()
# Always set the project id.
resources.projectId = genomics_util.GetProjectId()
# Update the pipeline based on arguments.
if args.memory or args.cpus:
# Default to n1-standard1 sizes.
virtual_machine.machineType = 'custom-%d-%d' % (
args.cpus or 1, (args.memory or 3.75) * 1024)
if args.preemptible:
virtual_machine.preemptible = args.preemptible
if args.zones:
resources.zones = args.zones
elif not resources.zones and properties.VALUES.compute.zone.Get():
resources.zones = [properties.VALUES.compute.zone.Get()]
if args.regions:
resources.regions = args.regions
elif not resources.regions and properties.VALUES.compute.region.Get():
resources.regions = [properties.VALUES.compute.region.Get()]
if args.service_account_email != 'default':
virtual_machine.serviceAccount.email = args.service_account_email
if args.service_account_scopes:
virtual_machine.serviceAccount.scopes = args.service_account_scopes
# Always add a scope for GCS in case any arguments need it.
virtual_machine.serviceAccount.scopes.append(
'https://www.googleapis.com/auth/devstorage.read_write')
# Attach custom network/subnetwork (if set).
if args.network or args.subnetwork:
if not virtual_machine.network:
virtual_machine.network = genomics_messages.Network()
if args.network:
virtual_machine.network.name = args.network
if args.subnetwork:
virtual_machine.network.subnetwork = args.subnetwork
if args.boot_disk_size is not None:
if args.boot_disk_size <= 0:
raise exceptions.GenomicsError(
'Boot disk size must be greater than zero.')
virtual_machine.bootDiskSizeGb = args.boot_disk_size
# Generate paths for inputs and outputs in a shared location and put them
# into the environment for actions based on their name.
env = {}
if arg_inputs:
input_generator = _SharedPathGenerator('input')
for name, value in arg_inputs.items():
if genomics_util.IsGcsPath(value):
env[name] = input_generator.Generate()
pipeline.actions.insert(
0,
genomics_messages.Action(
imageUri=CLOUD_SDK_IMAGE,
commands=[
'/bin/sh', '-c',
'gsutil -m -q cp %s ${%s}' % (value, name)
]))
elif name in is_local_file:
# TODO(b/183206325): Get test coverage to 100%.
env[name] = input_generator.Generate()
pipeline.actions.insert(
0,
genomics_messages.Action(
imageUri=CLOUD_SDK_IMAGE,
commands=[
'/bin/sh', '-c',
'echo "%s" | base64 -d > ${%s}' %
(base64.b64encode(
value.encode()).decode(), name)
]))
else:
env[name] = value
if args.outputs:
output_generator = _SharedPathGenerator('output')
for name, value in args.outputs.items():
env[name] = output_generator.Generate()
pipeline.actions.append(
genomics_messages.Action(imageUri=CLOUD_SDK_IMAGE,
commands=[
'/bin/sh', '-c',
'gsutil -m -q cp ${%s} %s' %
(name, value)
]))
if args.env_vars:
for name, value in args.env_vars.items():
env[name] = value
# Merge any existing pipeline arguments into the generated environment and
# update the pipeline.
if pipeline.environment:
for val in pipeline.environment.additionalProperties:
if val.key not in env:
env[val.key] = val.value
pipeline.environment = genomics_messages.Pipeline.EnvironmentValue(
additionalProperties=genomics_util.
ArgDictToAdditionalPropertiesList(
env, genomics_messages.Pipeline.EnvironmentValue.
AdditionalProperty))
if arg_inputs or args.outputs:
virtual_machine.disks.append(
genomics_messages.Disk(name=SHARED_DISK))
for action in pipeline.actions:
action.mounts.append(
genomics_messages.Mount(disk=SHARED_DISK,
path='/' + SHARED_DISK))
if args.logging:
pipeline.actions.append(
genomics_messages.Action(
imageUri=CLOUD_SDK_IMAGE,
commands=[
'/bin/sh', '-c',
'gsutil -m -q cp /google/logs/output ' + args.logging
],
flags=[(genomics_messages.Action.
FlagsValueListEntryValuesEnum.ALWAYS_RUN)]))
# Update disk sizes if specified, potentially including the shared disk.
if args.disk_size:
disk_sizes = {}
for disk_encoding in args.disk_size.split(','):
parts = disk_encoding.split(':', 1)
try:
disk_sizes[parts[0]] = int(parts[1])
except:
raise exceptions.GenomicsError('Invalid --disk-size.')
for disk in virtual_machine.disks:
if disk.name in disk_sizes:
disk.sizeGb = disk_sizes[disk.name]
request = genomics_messages.RunPipelineRequest(
pipeline=pipeline,
labels=labels_util.ParseCreateArgs(
args, genomics_messages.RunPipelineRequest.LabelsValue))
result = apitools_client.pipelines.Run(request)
log.status.Print('Running [{0}].'.format(result.name))
return result
