def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: argparse.Namespace, All the arguments that were provided to this
command invocation.
Raises:
GenomicsError: User input was invalid.
HttpException: An http error response was received while executing api
request.
Returns:
Operation representing the running pipeline.
"""
apitools_client = self.context[lib.GENOMICS_APITOOLS_V1A2_CLIENT_KEY]
genomics_messages = self.context[lib.GENOMICS_MESSAGES_V1A2_MODULE_KEY]
pipeline = genomics_util.GetFileAsMessage(
args.pipeline_file,
genomics_messages.Pipeline,
self.context[lib.STORAGE_V1_CLIENT_KEY])
pipeline.projectId = genomics_util.GetProjectId()
inputs = genomics_util.ArgDictToAdditionalPropertiesList(
args.inputs,
genomics_messages.RunPipelineArgs.InputsValue.AdditionalProperty)
outputs = genomics_util.ArgDictToAdditionalPropertiesList(
args.outputs,
genomics_messages.RunPipelineArgs.OutputsValue.AdditionalProperty)
resources = genomics_messages.PipelineResources(
preemptible=args.preemptible)
if args.memory:
resources.minimumRamGb = args.memory
if args.disk_size:
resources.disks = []
for disk_encoding in args.disk_size.split(','):
disk_args = disk_encoding.split(':')
resources.disks.append(genomics_messages.Disk(
name=disk_args[0],
sizeGb=int(disk_args[1])
))
request = genomics_messages.RunPipelineRequest(
ephemeralPipeline=pipeline,
pipelineArgs=genomics_messages.RunPipelineArgs(
inputs=genomics_messages.RunPipelineArgs.InputsValue(
additionalProperties=inputs),
outputs=genomics_messages.RunPipelineArgs.OutputsValue(
additionalProperties=outputs),
clientId=args.run_id,
logging=genomics_messages.LoggingOptions(gcsPath=args.logging),
projectId=genomics_util.GetProjectId(),
serviceAccount=genomics_messages.ServiceAccount(
email=args.service_account_email,
scopes=args.service_account_scopes),
resources=resources))
return apitools_client.pipelines.Run(request)
def Run(self, args):
"""Run 'operations list'.
Args:
args: argparse.Namespace, The arguments that this command was invoked
with.
Returns:
The list of operations for this project.
"""
both = not args.filter and not args.where
outputs = []
if both or args.filter:
apitools_client = genomics_util.GetGenomicsClient('v2alpha1')
genomics_messages = genomics_util.GetGenomicsMessages('v2alpha1')
request = genomics_messages.GenomicsProjectsOperationsListRequest(
name='projects/%s/operations' %
(genomics_util.GetProjectId(), ),
filter=args.filter)
outputs.append(
list_pager.YieldFromList(
apitools_client.projects_operations,
request,
limit=args.limit,
batch_size_attribute='pageSize',
batch_size=args.
limit, # Use limit if any, else server default.
field='operations'))
if both or args.where:
apitools_client = genomics_util.GetGenomicsClient()
genomics_messages = genomics_util.GetGenomicsMessages()
if args.where:
args.where += ' AND '
args.where += 'projectId=%s' % genomics_util.GetProjectId()
request = genomics_messages.GenomicsOperationsListRequest(
name='operations', filter=args.where)
outputs.append(
list_pager.YieldFromList(
apitools_client.operations,
request,
limit=args.limit,
batch_size_attribute='pageSize',
batch_size=args.
limit, # Use limit if any, else server default.
field='operations'))
return chain.from_iterable(outputs)
def Run(self, args):
"""Run 'operations list'.
Args:
args: argparse.Namespace, The arguments that this command was invoked
with.
Returns:
The list of operations for this project.
"""
apitools_client = genomics_util.GetGenomicsClient('v2alpha1')
genomics_messages = genomics_util.GetGenomicsMessages('v2alpha1')
request_filter = None
if args.filter:
rewriter = filter_rewrite.OperationsBackend()
args.filter, request_filter = rewriter.Rewrite(args.filter)
log.info('client_filter=%r server_filter=%r', args.filter, request_filter)
request = genomics_messages.GenomicsProjectsOperationsListRequest(
name='projects/%s/operations' % (genomics_util.GetProjectId(),),
filter=request_filter)
return list_pager.YieldFromList(
apitools_client.projects_operations,
request,
limit=args.limit,
batch_size_attribute='pageSize',
batch_size=args.limit, # Use limit if any, else server default.
field='operations')
def Run(self, args):
"""Run 'datasets list'.
Args:
args: argparse.Namespace, The arguments that this command was invoked
with.
Yields:
The list of datasets for this project.
Raises:
HttpException: An http error response was received while executing api
request.
"""
apitools_client = genomics_util.GetGenomicsClient()
request = genomics_util.GetGenomicsMessages(
).GenomicsDatasetsListRequest(projectId=genomics_util.GetProjectId())
try:
for resource in list_pager.YieldFromList(
apitools_client.datasets,
request,
limit=args.limit,
batch_size_attribute='pageSize',
batch_size=args.
limit, # Use limit if any, else server default.
field='datasets'):
yield resource
except apitools_exceptions.HttpError as error:
raise exceptions.HttpException(
genomics_util.GetErrorMessage(error))
def Rewrite(expr):
"""Returns the server side rewrite of a --filter expression.
Args:
expr: A client side --filter expression.
Raises:
resource_exceptions.ExpressionSyntaxError: Expression syntax error.
ValueError: Invalid expression operands.
Returns:
The server side rewrite of a --filter expression, None if the expression is
completely client side.
"""
# Rewrite the expression.
rewrite = resource_filter.Compile(expr, backend=_Backend()).Rewrite()
# Add a project id restriction.
if rewrite:
rewrite += ' AND '
else:
rewrite = ''
rewrite += 'projectId={0}'.format(genomics_util.GetProjectId())
return rewrite
def Run(self, args):
"""Run 'datasets list'.
Args:
args: argparse.Namespace, The arguments that this command was invoked
with.
Returns:
The list of datasets for this project.
Raises:
HttpException: An http error response was received while executing api
request.
"""
genomics_util.ValidateLimitFlag(args.limit)
apitools_client = self.context[lib.GENOMICS_APITOOLS_CLIENT_KEY]
req_class = (self.context[
lib.GENOMICS_MESSAGES_MODULE_KEY].GenomicsDatasetsListRequest)
request = req_class(projectId=genomics_util.GetProjectId())
return list_pager.YieldFromList(
apitools_client.datasets,
request,
limit=args.limit,
batch_size_attribute='pageSize',
batch_size=args.limit, # Use limit if any, else server default.
field='datasets')
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace, All the arguments that were provided to this
command invocation.
Raises:
HttpException: An http error response was received while executing api
request.
Returns:
None
"""
apitools_client = genomics_util.GetGenomicsClient()
genomics_messages = genomics_util.GetGenomicsMessages()
dataset = genomics_messages.Dataset(
name=args.name,
projectId=genomics_util.GetProjectId(),
)
result = apitools_client.datasets.Create(dataset)
log.CreatedResource('{0}, id: {1}'.format(result.name, result.id),
kind='dataset')
return result
def Run(self, args):
"""Run 'operations list'.
Args:
args: argparse.Namespace, The arguments that this command was invoked
with.
Returns:
The list of operations for this project.
"""
apitools_client = genomics_util.GetGenomicsClient()
genomics_messages = genomics_util.GetGenomicsMessages()
if args.where:
args.where += ' AND '
args.where += 'projectId=%s' % genomics_util.GetProjectId()
request = genomics_messages.GenomicsOperationsListRequest(
name='operations', filter=args.where)
return list_pager.YieldFromList(
apitools_client.operations,
request,
limit=args.limit,
batch_size_attribute='pageSize',
batch_size=args.limit, # Use limit if any, else server default.
field='operations')
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace, All the arguments that were provided to this
command invocation.
Raises:
HttpException: An http error response was received while executing api
request.
Returns:
an ExportVariantSetResponse
"""
apitools_client = self.context[lib.GENOMICS_APITOOLS_CLIENT_KEY]
genomics_messages = self.context[lib.GENOMICS_MESSAGES_MODULE_KEY]
enum = genomics_messages.ExportVariantSetRequest.FormatValueValuesEnum
call_set_ids = args.call_set_ids if args.call_set_ids else []
project_id = args.bigquery_project
if not project_id:
project_id = genomics_util.GetProjectId()
request = genomics_messages.GenomicsVariantsetsExportRequest(
variantSetId=args.id,
exportVariantSetRequest=genomics_messages.ExportVariantSetRequest(
callSetIds=call_set_ids,
projectId=project_id,
format=enum.FORMAT_BIGQUERY,
bigqueryDataset=args.bigquery_dataset,
bigqueryTable=args.table))
return apitools_client.variantsets.Export(request)
def Run(self, args):
"""Run 'operations list'.
Args:
args: argparse.Namespace, The arguments that this command was invoked
with.
Returns:
The list of operations for this project.
Raises:
HttpException: An http error response was received while executing api
request.
"""
genomics_util.ValidateLimitFlag(args.limit)
apitools_client = self.context[lib.GENOMICS_APITOOLS_CLIENT_KEY]
genomics_messages = self.context[lib.GENOMICS_MESSAGES_MODULE_KEY]
if args.where:
args.where += ' AND '
args.where += 'projectId=%s' % genomics_util.GetProjectId()
request = genomics_messages.GenomicsOperationsListRequest(
name='operations', filter=args.where)
return list_pager.YieldFromList(
apitools_client.operations,
request,
limit=args.limit,
batch_size_attribute='pageSize',
batch_size=args.limit, # Use limit if any, else server default.
field='operations')
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace, All the arguments that were provided to this
command invocation.
Returns:
an Operation message which tracks the asynchronous export
"""
apitools_client = self.context[lib.GENOMICS_APITOOLS_CLIENT_KEY]
genomics_messages = self.context[lib.GENOMICS_MESSAGES_MODULE_KEY]
try:
return apitools_client.readgroupsets.Export(
genomics_messages.GenomicsReadgroupsetsExportRequest(
readGroupSetId=args.read_group_set_id,
exportReadGroupSetRequest=genomics_messages.
ExportReadGroupSetRequest(
projectId=genomics_util.GetProjectId(),
exportUri=args.export_uri,
referenceNames=args.reference_names)))
except apitools_base.HttpError as error:
# Map our error messages (JSON API camelCased) back into flag names.
msg = (genomics_util.GetErrorMessage(error).replace(
'exportUri', '--export-uri').replace('referenceNames',
'--reference-names'))
unused_type, unused_value, traceback = sys.exc_info()
raise exceptions.HttpException, msg, traceback
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace, All the arguments that were provided to this
command invocation.
Returns:
an Operation message which tracks the asynchronous export
"""
apitools_client = genomics_util.GetGenomicsClient()
genomics_messages = genomics_util.GetGenomicsMessages()
try:
return apitools_client.readgroupsets.Export(
genomics_messages.GenomicsReadgroupsetsExportRequest(
readGroupSetId=args.read_group_set_id,
exportReadGroupSetRequest=genomics_messages.
ExportReadGroupSetRequest(
projectId=genomics_util.GetProjectId(),
exportUri=args.export_uri,
referenceNames=args.reference_names)))
except apitools_exceptions.HttpError as error:
# Map our error messages (JSON API camelCased) back into flag names.
msg = (
exceptions.HttpException(error).payload.status_message.replace(
'exportUri',
'--export-uri').replace('referenceNames',
'--reference-names'))
core_exceptions.reraise(exceptions.HttpException(msg))
def Rewrite(self, expr, defaults=None):
"""Add a project id restriction to the backend rewrite."""
rewrite = super(Backend, self).Rewrite(expr, defaults=defaults)
# Add a project id restriction.
if rewrite:
rewrite += ' AND '
else:
rewrite = ''
rewrite += 'projectId={0}'.format(genomics_util.GetProjectId())
return rewrite
def Run(self, args):
"""Run 'datasets list'.
Args:
args: argparse.Namespace, The arguments that this command was invoked
with.
Returns:
The list of datasets for this project.
"""
apitools_client = genomics_util.GetGenomicsClient()
request = genomics_util.GetGenomicsMessages(
).GenomicsDatasetsListRequest(projectId=genomics_util.GetProjectId())
return list_pager.YieldFromList(
apitools_client.datasets,
request,
limit=args.limit,
batch_size_attribute='pageSize',
batch_size=args.limit, # Use limit if any, else server default.
field='datasets')
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: an argparse namespace, All the arguments that were provided to this
command invocation.
Raises:
HttpException: An http error response was received while executing api
request.
Returns:
None
"""
apitools_client = self.context[lib.GENOMICS_APITOOLS_CLIENT_KEY]
genomics_messages = self.context[lib.GENOMICS_MESSAGES_MODULE_KEY]
dataset = genomics_messages.Dataset(
name=args.name,
projectId=genomics_util.GetProjectId(),
)
return apitools_client.datasets.Create(dataset)
def ResourceFromName(self, name):
return self._registry.Parse(
name,
collection='genomics.projects.operations',
params={'projectsId': genomics_util.GetProjectId()})
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: argparse.Namespace, All the arguments that were provided to this
command invocation.
Raises:
files.Error: A file argument could not be read.
GenomicsError: User input was invalid.
HttpException: An http error response was received while executing api
request.
Returns:
Operation representing the running pipeline.
"""
v2 = False
pipeline = None
apitools_client = genomics_util.GetGenomicsClient('v1alpha2')
genomics_messages = genomics_util.GetGenomicsMessages('v1alpha2')
if args.pipeline_file:
if args.command_line:
# TODO(b/79982664): Use a mutex argument group instead.
raise exceptions.GenomicsError(
'--command-line cannot be used with --pipeline-file.')
pipeline = genomics_util.GetFileAsMessage(
args.pipeline_file, genomics_messages.Pipeline,
self.context[lib.STORAGE_V1_CLIENT_KEY])
pipeline.projectId = genomics_util.GetProjectId()
if not pipeline.docker:
v2 = True
apitools_client = genomics_util.GetGenomicsClient('v2alpha1')
genomics_messages = genomics_util.GetGenomicsMessages(
'v2alpha1')
pipeline = genomics_util.GetFileAsMessage(
args.pipeline_file, genomics_messages.Pipeline,
self.context[lib.STORAGE_V1_CLIENT_KEY])
elif args.command_line:
v2 = True
apitools_client = genomics_util.GetGenomicsClient('v2alpha1')
genomics_messages = genomics_util.GetGenomicsMessages('v2alpha1')
pipeline = genomics_messages.Pipeline(actions=[
genomics_messages.Action(imageUri=args.docker_image,
commands=['-c', args.command_line],
entrypoint='bash')
])
else:
raise exceptions.GenomicsError(
'Either --pipeline-file or --command-line is required.')
arg_inputs, is_local_file = _ValidateAndMergeArgInputs(args)
request = None
if v2:
# Create messages up front to avoid checking for None everywhere.
if not pipeline.resources:
pipeline.resources = genomics_messages.Resources()
resources = pipeline.resources
if not resources.virtualMachine:
resources.virtualMachine = genomics_messages.VirtualMachine(
machineType='n1-standard-1')
virtual_machine = resources.virtualMachine
if not virtual_machine.serviceAccount:
virtual_machine.serviceAccount = genomics_messages.ServiceAccount(
)
# Always set the project id.
resources.projectId = genomics_util.GetProjectId()
# Update the pipeline based on arguments.
if args.memory or args.cpus:
# Default to n1-standard1 sizes.
virtual_machine.machineType = 'custom-%d-%d' % (
args.cpus or 1, (args.memory or 3.84) * 1000)
if args.preemptible:
virtual_machine.preemptible = args.preemptible
if args.zones:
resources.zones = args.zones
elif not resources.zones and properties.VALUES.compute.zone.Get():
resources.zones = [properties.VALUES.compute.zone.Get()]
if args.regions:
resources.regions = args.regions
elif not resources.regions and properties.VALUES.compute.region.Get(
):
resources.regions = [properties.VALUES.compute.region.Get()]
if args.service_account_email != 'default':
virtual_machine.serviceAccount.email = args.service_account_email
if args.service_account_scopes:
virtual_machine.serviceAccount.scopes = args.service_account_scopes
# Always add a scope for GCS in case any arguments need it.
virtual_machine.serviceAccount.scopes.append(
'https://www.googleapis.com/auth/devstorage.read_write')
# Generate paths for inputs and outputs in a shared location and put them
# into the environment for actions based on their name.
env = {}
if arg_inputs:
input_generator = _SharedPathGenerator('input')
for name, value in arg_inputs.items():
if genomics_util.IsGcsPath(value):
env[name] = input_generator.Generate()
pipeline.actions.insert(
0,
genomics_messages.Action(
imageUri=CLOUD_SDK_IMAGE,
commands=[
'/bin/sh', '-c',
'gsutil -q cp %s ${%s}' % (value, name)
]))
elif name in is_local_file:
env[name] = input_generator.Generate()
pipeline.actions.insert(
0,
genomics_messages.Action(
imageUri=CLOUD_SDK_IMAGE,
commands=[
'/bin/sh', '-c',
'echo "%s" | base64 -d > ${%s}' %
(base64.b64encode(value), name)
]))
else:
env[name] = value
if args.outputs:
output_generator = _SharedPathGenerator('output')
for name, value in args.outputs.items():
env[name] = output_generator.Generate()
pipeline.actions.append(
genomics_messages.Action(imageUri=CLOUD_SDK_IMAGE,
commands=[
'/bin/sh', '-c',
'gsutil -q cp ${%s} %s' %
(name, value)
]))
# Merge any existing pipeline arguments into the generated environment and
# update the pipeline.
if pipeline.environment:
for val in pipeline.environment.additionalProperties:
if val.key not in env:
env[val.key] = val.value
pipeline.environment = genomics_messages.Pipeline.EnvironmentValue(
additionalProperties=genomics_util.
ArgDictToAdditionalPropertiesList(
env, genomics_messages.Pipeline.EnvironmentValue.
AdditionalProperty))
if arg_inputs or args.outputs:
virtual_machine.disks.append(
genomics_messages.Disk(name=SHARED_DISK))
for action in pipeline.actions:
action.mounts.append(
genomics_messages.Mount(disk=SHARED_DISK,
path='/' + SHARED_DISK))
if args.logging:
pipeline.actions.append(
genomics_messages.Action(
imageUri=CLOUD_SDK_IMAGE,
commands=[
'/bin/sh', '-c',
'gsutil -q cp /google/logs/output ' + args.logging
],
flags=[(genomics_messages.Action.
FlagsValueListEntryValuesEnum.ALWAYS_RUN)]))
# Update disk sizes if specified, potentially including the shared disk.
if args.disk_size:
disk_sizes = {}
for disk_encoding in args.disk_size.split(','):
parts = disk_encoding.split(':', 1)
try:
disk_sizes[parts[0]] = int(parts[1])
except:
raise exceptions.GenomicsError('Invalid --disk-size.')
for disk in virtual_machine.disks:
size = disk_sizes[disk.name]
if size:
disk.sizeGb = size
request = genomics_messages.RunPipelineRequest(
pipeline=pipeline,
labels=labels_util.ParseCreateArgs(
args, genomics_messages.RunPipelineRequest.LabelsValue))
else:
inputs = genomics_util.ArgDictToAdditionalPropertiesList(
arg_inputs, genomics_messages.RunPipelineArgs.InputsValue.
AdditionalProperty)
outputs = genomics_util.ArgDictToAdditionalPropertiesList(
args.outputs, genomics_messages.RunPipelineArgs.OutputsValue.
AdditionalProperty)
# Set "overrides" on the resources. If the user did not pass anything on
# the command line, do not set anything in the resource: preserve the
# user-intent "did not set" vs. "set an empty value/list"
resources = genomics_messages.PipelineResources(
preemptible=args.preemptible)
if args.memory:
resources.minimumRamGb = args.memory
if args.cpus:
resources.minimumCpuCores = args.cpus
if args.disk_size:
resources.disks = []
for disk_encoding in args.disk_size.split(','):
disk_args = disk_encoding.split(':', 1)
resources.disks.append(
genomics_messages.Disk(name=disk_args[0],
sizeGb=int(disk_args[1])))
# Progression for picking the right zones...
# If specified on the command line, use them.
# If specified in the Pipeline definition, use them.
# If there is a GCE default zone in the local configuration, use it.
# Else let the API select a zone
if args.zones:
resources.zones = args.zones
elif pipeline.resources and pipeline.resources.zones:
pass
elif properties.VALUES.compute.zone.Get():
resources.zones = [properties.VALUES.compute.zone.Get()]
request = genomics_messages.RunPipelineRequest(
ephemeralPipeline=pipeline,
pipelineArgs=genomics_messages.RunPipelineArgs(
inputs=genomics_messages.RunPipelineArgs.InputsValue(
additionalProperties=inputs),
outputs=genomics_messages.RunPipelineArgs.OutputsValue(
additionalProperties=outputs),
clientId=args.run_id,
logging=genomics_messages.LoggingOptions(
gcsPath=args.logging),
labels=labels_util.ParseCreateArgs(
args, genomics_messages.RunPipelineArgs.LabelsValue),
projectId=genomics_util.GetProjectId(),
serviceAccount=genomics_messages.ServiceAccount(
email=args.service_account_email,
scopes=args.service_account_scopes),
resources=resources))
result = apitools_client.pipelines.Run(request)
log.status.Print('Running [{0}].'.format(result.name))
return result
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: argparse.Namespace, All the arguments that were provided to this
command invocation.
Raises:
files.Error: A file argument could not be read.
GenomicsError: User input was invalid.
HttpException: An http error response was received while executing api
request.
Returns:
Operation representing the running pipeline.
"""
apitools_client = genomics_util.GetGenomicsClient('v1alpha2')
genomics_messages = genomics_util.GetGenomicsMessages('v1alpha2')
pipeline = genomics_util.GetFileAsMessage(
args.pipeline_file, genomics_messages.Pipeline,
self.context[lib.STORAGE_V1_CLIENT_KEY])
pipeline.projectId = genomics_util.GetProjectId()
arg_inputs = _ValidateAndMergeArgInputs(args)
inputs = genomics_util.ArgDictToAdditionalPropertiesList(
arg_inputs,
genomics_messages.RunPipelineArgs.InputsValue.AdditionalProperty)
outputs = genomics_util.ArgDictToAdditionalPropertiesList(
args.outputs,
genomics_messages.RunPipelineArgs.OutputsValue.AdditionalProperty)
# Set "overrides" on the resources. If the user did not pass anything on
# the command line, do not set anything in the resource: preserve the
# user-intent "did not set" vs. "set an empty value/list"
resources = genomics_messages.PipelineResources(
preemptible=args.preemptible)
if args.memory:
resources.minimumRamGb = args.memory
if args.disk_size:
resources.disks = []
for disk_encoding in args.disk_size.split(','):
disk_args = disk_encoding.split(':', 1)
resources.disks.append(
genomics_messages.Disk(name=disk_args[0],
sizeGb=int(disk_args[1])))
# Progression for picking the right zones...
# If specified on the command line, use them.
# If specified in the Pipeline definition, use them.
# If there is a GCE default zone in the local configuration, use it.
# Else let the API select a zone
if args.zones:
resources.zones = args.zones
elif pipeline.resources and pipeline.resources.zones:
pass
elif properties.VALUES.compute.zone.Get():
resources.zones = [properties.VALUES.compute.zone.Get()]
request = genomics_messages.RunPipelineRequest(
ephemeralPipeline=pipeline,
pipelineArgs=genomics_messages.RunPipelineArgs(
inputs=genomics_messages.RunPipelineArgs.InputsValue(
additionalProperties=inputs),
outputs=genomics_messages.RunPipelineArgs.OutputsValue(
additionalProperties=outputs),
clientId=args.run_id,
logging=genomics_messages.LoggingOptions(gcsPath=args.logging),
labels=labels_util.ParseCreateArgs(
args, genomics_messages.RunPipelineArgs.LabelsValue),
projectId=genomics_util.GetProjectId(),
serviceAccount=genomics_messages.ServiceAccount(
email=args.service_account_email,
scopes=args.service_account_scopes),
resources=resources))
result = apitools_client.pipelines.Run(request)
log.status.Print('Running [{0}].'.format(result.name))
return result
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: argparse.Namespace, All the arguments that were provided to this
command invocation.
Raises:
files.Error: A file argument could not be read.
GenomicsError: User input was invalid.
HttpException: An http error response was received while executing api
request.
Returns:
Operation representing the running pipeline.
"""
pipeline = None
apitools_client = genomics_util.GetGenomicsClient('v2alpha1')
genomics_messages = genomics_util.GetGenomicsMessages('v2alpha1')
if args.pipeline_file:
if args.command_line:
# TODO(b/79982664): Use a mutex argument group instead.
raise exceptions.GenomicsError(
'--command-line cannot be used with --pipeline-file.')
pipeline = genomics_util.GetFileAsMessage(
args.pipeline_file, genomics_messages.Pipeline,
self.context[lib.STORAGE_V1_CLIENT_KEY])
elif args.command_line:
pipeline = genomics_messages.Pipeline(actions=[
genomics_messages.Action(imageUri=args.docker_image,
commands=['-c', args.command_line],
entrypoint='bash')
])
else:
raise exceptions.GenomicsError(
'Either --pipeline-file or --command-line is required.')
arg_inputs, is_local_file = _ValidateAndMergeArgInputs(args)
request = None
# Create messages up front to avoid checking for None everywhere.
if not pipeline.resources:
pipeline.resources = genomics_messages.Resources()
resources = pipeline.resources
if not resources.virtualMachine:
resources.virtualMachine = genomics_messages.VirtualMachine(
machineType='n1-standard-1')
virtual_machine = resources.virtualMachine
if not virtual_machine.serviceAccount:
virtual_machine.serviceAccount = genomics_messages.ServiceAccount()
# Always set the project id.
resources.projectId = genomics_util.GetProjectId()
# Update the pipeline based on arguments.
if args.memory or args.cpus:
# Default to n1-standard1 sizes.
virtual_machine.machineType = 'custom-%d-%d' % (
args.cpus or 1, (args.memory or 3.75) * 1024)
if args.preemptible:
virtual_machine.preemptible = args.preemptible
if args.zones:
resources.zones = args.zones
elif not resources.zones and properties.VALUES.compute.zone.Get():
resources.zones = [properties.VALUES.compute.zone.Get()]
if args.regions:
resources.regions = args.regions
elif not resources.regions and properties.VALUES.compute.region.Get():
resources.regions = [properties.VALUES.compute.region.Get()]
if args.service_account_email != 'default':
virtual_machine.serviceAccount.email = args.service_account_email
if args.service_account_scopes:
virtual_machine.serviceAccount.scopes = args.service_account_scopes
# Always add a scope for GCS in case any arguments need it.
virtual_machine.serviceAccount.scopes.append(
'https://www.googleapis.com/auth/devstorage.read_write')
# Attach custom network/subnetwork (if set).
if args.network or args.subnetwork:
if not virtual_machine.network:
virtual_machine.network = genomics_messages.Network()
if args.network:
virtual_machine.network.name = args.network
if args.subnetwork:
virtual_machine.network.subnetwork = args.subnetwork
if args.boot_disk_size is not None:
if args.boot_disk_size <= 0:
raise exceptions.GenomicsError(
'Boot disk size must be greater than zero.')
virtual_machine.bootDiskSizeGb = args.boot_disk_size
# Generate paths for inputs and outputs in a shared location and put them
# into the environment for actions based on their name.
env = {}
if arg_inputs:
input_generator = _SharedPathGenerator('input')
for name, value in arg_inputs.items():
if genomics_util.IsGcsPath(value):
env[name] = input_generator.Generate()
pipeline.actions.insert(
0,
genomics_messages.Action(
imageUri=CLOUD_SDK_IMAGE,
commands=[
'/bin/sh', '-c',
'gsutil -m -q cp %s ${%s}' % (value, name)
]))
elif name in is_local_file:
# TODO(b/183206325): Get test coverage to 100%.
env[name] = input_generator.Generate()
pipeline.actions.insert(
0,
genomics_messages.Action(
imageUri=CLOUD_SDK_IMAGE,
commands=[
'/bin/sh', '-c',
'echo "%s" | base64 -d > ${%s}' %
(base64.b64encode(
value.encode()).decode(), name)
]))
else:
env[name] = value
if args.outputs:
output_generator = _SharedPathGenerator('output')
for name, value in args.outputs.items():
env[name] = output_generator.Generate()
pipeline.actions.append(
genomics_messages.Action(imageUri=CLOUD_SDK_IMAGE,
commands=[
'/bin/sh', '-c',
'gsutil -m -q cp ${%s} %s' %
(name, value)
]))
if args.env_vars:
for name, value in args.env_vars.items():
env[name] = value
# Merge any existing pipeline arguments into the generated environment and
# update the pipeline.
if pipeline.environment:
for val in pipeline.environment.additionalProperties:
if val.key not in env:
env[val.key] = val.value
pipeline.environment = genomics_messages.Pipeline.EnvironmentValue(
additionalProperties=genomics_util.
ArgDictToAdditionalPropertiesList(
env, genomics_messages.Pipeline.EnvironmentValue.
AdditionalProperty))
if arg_inputs or args.outputs:
virtual_machine.disks.append(
genomics_messages.Disk(name=SHARED_DISK))
for action in pipeline.actions:
action.mounts.append(
genomics_messages.Mount(disk=SHARED_DISK,
path='/' + SHARED_DISK))
if args.logging:
pipeline.actions.append(
genomics_messages.Action(
imageUri=CLOUD_SDK_IMAGE,
commands=[
'/bin/sh', '-c',
'gsutil -m -q cp /google/logs/output ' + args.logging
],
flags=[(genomics_messages.Action.
FlagsValueListEntryValuesEnum.ALWAYS_RUN)]))
# Update disk sizes if specified, potentially including the shared disk.
if args.disk_size:
disk_sizes = {}
for disk_encoding in args.disk_size.split(','):
parts = disk_encoding.split(':', 1)
try:
disk_sizes[parts[0]] = int(parts[1])
except:
raise exceptions.GenomicsError('Invalid --disk-size.')
for disk in virtual_machine.disks:
if disk.name in disk_sizes:
disk.sizeGb = disk_sizes[disk.name]
request = genomics_messages.RunPipelineRequest(
pipeline=pipeline,
labels=labels_util.ParseCreateArgs(
args, genomics_messages.RunPipelineRequest.LabelsValue))
result = apitools_client.pipelines.Run(request)
log.status.Print('Running [{0}].'.format(result.name))
return result
