def _ValidateAndMergeArgInputs(args):
"""Turn args.inputs and args.inputs_from_file dicts into a single dict.
Args:
args: The parsed command-line arguments
Returns:
A dict that is the merge of args.inputs and args.inputs_from_file
Raises:
files.Error
"""
is_local_file = {}
# If no inputs from file, then no validation or merge needed
if not args.inputs_from_file:
return args.inputs, is_local_file
# Initialize the merged dictionary
arg_inputs = {}
if args.inputs:
# Validate args.inputs and args.inputs-from-file do not overlap
overlap = set(args.inputs.keys()).intersection(
set(args.inputs_from_file.keys()))
if overlap:
raise exceptions.LifeSciencesError(
'--{0} and --{1} may not specify overlapping values: {2}'.
format('inputs', 'inputs-from-file', ', '.join(overlap)))
# Add the args.inputs
arg_inputs.update(args.inputs)
# Read up the inputs-from-file and add the values from the file
for key, value in six.iteritems(args.inputs_from_file):
arg_inputs[key] = files.ReadFileContents(value)
is_local_file[key] = True
return arg_inputs, is_local_file
def Run(self, args):
"""This is what gets called when the user runs this command.
Args:
args: argparse.Namespace, All the arguments that were provided to this
command invocation.
Raises:
files.Error: A file argument could not be read.
LifeSciencesError: User input was invalid.
HttpException: An http error response was received while executing api
request.
Returns:
Operation representing the running pipeline.
"""
pipeline = None
apitools_client = lifesciences_util.GetLifeSciencesClient('v2beta')
lifesciences_messages = lifesciences_util.GetLifeSciencesMessages(
'v2beta')
if args.pipeline_file:
pipeline = lifesciences_util.GetFileAsMessage(
args.pipeline_file, lifesciences_messages.Pipeline,
self.context[lib.STORAGE_V1_CLIENT_KEY])
elif args.command_line:
pipeline = lifesciences_messages.Pipeline(actions=[
lifesciences_messages.Action(
imageUri=args.docker_image,
commands=['-c', args.command_line],
entrypoint='bash')
])
arg_inputs, is_local_file = _ValidateAndMergeArgInputs(args)
request = None
# Create messages up front to avoid checking for None everywhere.
if not pipeline.resources:
pipeline.resources = lifesciences_messages.Resources()
resources = pipeline.resources
if not resources.virtualMachine:
resources.virtualMachine = lifesciences_messages.VirtualMachine(
machineType=args.machine_type)
virtual_machine = resources.virtualMachine
if not virtual_machine.serviceAccount:
virtual_machine.serviceAccount = lifesciences_messages.ServiceAccount(
)
if args.preemptible:
virtual_machine.preemptible = args.preemptible
if args.zones:
resources.zones = args.zones
elif not resources.zones and properties.VALUES.compute.zone.Get():
resources.zones = [properties.VALUES.compute.zone.Get()]
if args.regions:
resources.regions = args.regions
elif not resources.regions and properties.VALUES.compute.region.Get():
resources.regions = [properties.VALUES.compute.region.Get()]
if args.service_account_email != 'default':
virtual_machine.serviceAccount.email = args.service_account_email
if args.service_account_scopes:
virtual_machine.serviceAccount.scopes = args.service_account_scopes
# Always add the cloud-platform scope for user convenience.
virtual_machine.serviceAccount.scopes.append(
'https://www.googleapis.com/auth/cloud-platform')
# Attach custom network/subnetwork (if set).
if args.network or args.subnetwork:
if not virtual_machine.network:
virtual_machine.network = lifesciences_messages.Network()
if args.network:
virtual_machine.network.network = args.network
if args.subnetwork:
virtual_machine.network.subnetwork = args.subnetwork
if args.boot_disk_size is not None:
if args.boot_disk_size <= 0:
raise exceptions.LifeSciencesError(
'Boot disk size must be greater than zero.')
virtual_machine.bootDiskSizeGb = args.boot_disk_size
# Generate paths for inputs and outputs in a shared location and put them
# into the environment for actions based on their name.
env = {}
if arg_inputs:
input_generator = _SharedPathGenerator('input')
for name, value in arg_inputs.items():
if lifesciences_util.IsGcsPath(value):
env[name] = input_generator.Generate()
pipeline.actions.insert(
0,
lifesciences_messages.Action(
imageUri=CLOUD_SDK_IMAGE,
commands=[
'/bin/sh', '-c',
'gsutil -m -q cp %s ${%s}' % (value, name)
]))
elif name in is_local_file:
env[name] = input_generator.Generate()
pipeline.actions.insert(
0,
lifesciences_messages.Action(
imageUri=CLOUD_SDK_IMAGE,
commands=[
'/bin/sh', '-c',
'echo "%s" | base64 -d > ${%s}' %
(base64.b64encode(
value.encode()).decode(), name)
]))
else:
env[name] = value
if args.outputs:
output_generator = _SharedPathGenerator('output')
for name, value in args.outputs.items():
env[name] = output_generator.Generate()
pipeline.actions.append(
lifesciences_messages.Action(
imageUri=CLOUD_SDK_IMAGE,
commands=[
'/bin/sh', '-c',
'gsutil -m -q cp ${%s} %s' % (name, value)
]))
if args.env_vars:
for name, value in args.env_vars.items():
env[name] = value
# Merge any existing pipeline arguments into the generated environment and
# update the pipeline.
if pipeline.environment:
for val in pipeline.environment.additionalProperties:
if val.key not in env:
env[val.key] = val.value
pipeline.environment = lifesciences_messages.Pipeline.EnvironmentValue(
additionalProperties=lifesciences_util.
ArgDictToAdditionalPropertiesList(
env, lifesciences_messages.Pipeline.EnvironmentValue.
AdditionalProperty))
if arg_inputs or args.outputs:
virtual_machine.disks.append(
lifesciences_messages.Disk(name=SHARED_DISK))
for action in pipeline.actions:
action.mounts.append(
lifesciences_messages.Mount(disk=SHARED_DISK,
path='/' + SHARED_DISK))
if args.logging:
pipeline.actions.append(
lifesciences_messages.Action(
imageUri=CLOUD_SDK_IMAGE,
commands=[
'/bin/sh', '-c',
'gsutil -m -q cp /google/logs/output ' + args.logging
],
alwaysRun=True))
# Update disk sizes if specified, potentially including the shared disk.
if args.disk_size:
disk_sizes = {}
for disk_encoding in args.disk_size.split(','):
parts = disk_encoding.split(':', 1)
try:
disk_sizes[parts[0]] = int(parts[1])
except:
raise exceptions.LifeSciencesError('Invalid --disk-size.')
for disk in virtual_machine.disks:
if disk.name in disk_sizes:
disk.sizeGb = disk_sizes[disk.name]
request = lifesciences_messages.RunPipelineRequest(
pipeline=pipeline,
labels=labels_util.ParseCreateArgs(
args, lifesciences_messages.RunPipelineRequest.LabelsValue))
projectId = lifesciences_util.GetProjectId()
location_ref = args.CONCEPTS.location.Parse()
request_wrapper = lifesciences_messages.LifesciencesProjectsLocationsPipelinesRunRequest(
parent=location_ref.RelativeName(), runPipelineRequest=request)
result = apitools_client.projects_locations_pipelines.Run(
request_wrapper)
log.status.Print('Running [{0}].'.format(result.name))
return result