def _get_gcloud_storage_args(self, sub_opts, gsutil_args,
gcloud_storage_map):
if gcloud_storage_map is None:
raise exception.GcloudStorageTranslationError(
'Command "{}" cannot be translated to gcloud storage because the'
' translation mapping is missing.'.format(self.command_name))
args = []
if isinstance(gcloud_storage_map.gcloud_command, str):
args = gcloud_storage_map.gcloud_command.split()
elif isinstance(gcloud_storage_map.gcloud_command, dict):
# If a command has sub-commands, e.g gsutil pap set, gsutil pap get.
# All the flags mapping must be present in the subcommand's map
# because gsutil does not have command specific flags
# if sub-commands are present.
if gcloud_storage_map.flag_map:
raise ValueError(
'Flags mapping should not be present at the top-level command if '
'a sub-command is used. Command: {}.'.format(
self.command_name))
sub_command = gsutil_args[0]
sub_opts, parsed_args = self.ParseSubOpts(
args=gsutil_args[1:], should_update_sub_opts_and_args=False)
return self._get_gcloud_storage_args(
sub_opts, parsed_args,
gcloud_storage_map.gcloud_command.get(sub_command))
else:
raise ValueError('Incorrect mapping found for "{}" command'.format(
self.command_name))
if sub_opts:
for option, value in sub_opts:
if option not in gcloud_storage_map.flag_map:
raise exception.GcloudStorageTranslationError(
'Command option "{}" cannot be translated to'
' gcloud storage'.format(option))
args.append(gcloud_storage_map.flag_map[option].gcloud_flag)
if value != '':
# Empty string represents that the user did not passed in a value
# for the flag.
args.append(value)
return args + gsutil_args
def _get_gcloud_binary_path():
# GCLOUD_BINARY_PATH is used for testing purpose only.
# It helps to run the parity_check.py script directly without having
# to build gcloud.
gcloud_binary_path = os.environ.get('GCLOUD_BINARY_PATH')
if gcloud_binary_path:
return gcloud_binary_path
cloudsdk_root = os.environ.get('CLOUDSDK_ROOT_DIR')
if cloudsdk_root is None:
raise exception.GcloudStorageTranslationError(
'Requested to use "gcloud storage" but the gcloud binary path cannot'
' be found. This might happen if you attempt to use gsutil that was'
' not installed via Cloud SDK. You can manually set the'
' `CLOUDSDK_ROOT_DIR` environment variable to point to the'
' google-cloud-sdk installation directory to resolve the issue.'
' Alternatively, you can set `use_gcloud_storage=False` to disable'
' running the command using gcloud storage.')
return os.path.join(cloudsdk_root, 'bin', 'gcloud')
def _translate_headers(self):
"""Translates gsutil headers to equivalent gcloud storage flags."""
flags = []
for header_key_raw, header_value in self.headers.items():
header_key = header_key_raw.lower()
if header_key == 'x-goog-api-version':
# Gsutil adds this header. We don't have to translate it for gcloud.
continue
flag = get_flag_from_header(header_key, header_value)
if self.command_name in DATA_TRANSFER_COMMANDS:
if flag is None:
raise exception.GcloudStorageTranslationError(
'Header cannot be translated to a gcloud storage equivalent'
' flag. Invalid header: {}:{}'.format(
header_key, header_value))
else:
flags.append(flag)
elif (self.command_name in PRECONDITONS_ONLY_SUPPORTED_COMMANDS
and header_key in PRECONDITIONS_HEADERS):
flags.append(flag)
# We ignore the headers for all other cases, so does gsutil.
return flags
def translate_to_gcloud_storage_if_requested(self):
"""Translates the gsutil command to gcloud storage equivalent.
The translated commands get stored at
self._translated_gcloud_storage_command.
This command also translate the boto config, which gets stored as a dict
at self._translated_env_variables
Returns:
True if the command was successfully translated, else False.
"""
if self.command_name == 'version' or self.command_name == 'test':
# Running any command in debug mode will lead to calling gsutil version
# command. We don't want to translate the version command as this
# should always reflect the version that gsutil is using.
# We don't want to run any translation for the "test" command.
return False
use_gcloud_storage = config.getbool('GSUtil', 'use_gcloud_storage',
False)
try:
hidden_shim_mode = HIDDEN_SHIM_MODE(
config.get('GSUtil', 'hidden_shim_mode', 'none'))
except ValueError:
raise exception.CommandException(
'Invalid option specified for'
' GSUtil:hidden_shim_mode config setting. Should be one of: {}'
.format(' | '.join([x.value for x in HIDDEN_SHIM_MODE])))
if use_gcloud_storage:
try:
top_level_flags, env_variables = self._translate_top_level_flags(
)
header_flags = self._translate_headers()
flags_from_boto, env_vars_from_boto = self._translate_boto_config(
)
env_variables.update(env_vars_from_boto)
gcloud_binary_path = _get_gcloud_binary_path()
gcloud_storage_command = ([gcloud_binary_path] +
self.get_gcloud_storage_args() +
top_level_flags + header_flags +
flags_from_boto)
if hidden_shim_mode == HIDDEN_SHIM_MODE.DRY_RUN:
self._print_gcloud_storage_command_info(
gcloud_storage_command, env_variables, dry_run=True)
elif not os.environ.get(
'CLOUDSDK_CORE_PASS_CREDENTIALS_TO_GSUTIL'):
raise exception.GcloudStorageTranslationError(
'Requested to use "gcloud storage" but gsutil is not using the'
' same credentials as gcloud.'
' You can make gsutil use the same credentials by running:\n'
'{} config set pass_credentials_to_gsutil True'.format(
gcloud_binary_path))
else:
self._print_gcloud_storage_command_info(
gcloud_storage_command, env_variables)
self._translated_gcloud_storage_command = gcloud_storage_command
self._translated_env_variables = env_variables
return True
except exception.GcloudStorageTranslationError as e:
# Raise error if no_fallback mode has been requested. This mode
# should only be used for debuggling and testing purposes.
if hidden_shim_mode == HIDDEN_SHIM_MODE.NO_FALLBACK:
raise exception.CommandException(e)
# For all other cases, we want to run gsutil.
self.logger.error(
'Cannot translate gsutil command to gcloud storage.'
' Going to run gsutil command. Error: %s', e)
return False