def _LoadApi(self, provider, api_selector):
"""Loads a CloudApi into the loaded_apis map for this class.
Args:
provider: Provider to load the API for.
api_selector: cs_api_map.ApiSelector defining the API type.
"""
if provider not in self.api_map[ApiMapConstants.API_MAP]:
raise ArgumentException(
'gsutil Cloud API map contains no entry for provider %s.' %
provider)
if api_selector not in self.api_map[ApiMapConstants.API_MAP][provider]:
raise ArgumentException(
'gsutil Cloud API map does not support API %s for provider %s.'
% (api_selector, provider))
self.loaded_apis[provider][api_selector] = (
self.api_map[ApiMapConstants.API_MAP][provider][api_selector](
self.bucket_storage_uri_class,
self.logger,
self.status_queue,
provider=provider,
debug=self.debug,
trace_token=self.trace_token,
perf_trace_token=self.perf_trace_token,
user_project=self.user_project))
def GetApiSelector(self, provider=None):
"""Returns a cs_api_map.ApiSelector based on input and configuration.
Args:
provider: Provider to return the ApiSelector for. If None, class-wide
default is used.
Returns:
cs_api_map.ApiSelector that will be used for calls to the delegator
for this provider.
"""
selected_provider = provider or self.provider
if not selected_provider:
raise ArgumentException('No provider selected for CloudApi')
if (selected_provider not in self.api_map[ApiMapConstants.DEFAULT_MAP]
or self.api_map[ApiMapConstants.DEFAULT_MAP][selected_provider]
not in self.api_map[ApiMapConstants.API_MAP][selected_provider]
):
raise ArgumentException(
'No default api available for provider %s' % selected_provider)
if selected_provider not in self.api_map[ApiMapConstants.SUPPORT_MAP]:
raise ArgumentException(
'No supported apis available for provider %s' %
selected_provider)
api = self.api_map[ApiMapConstants.DEFAULT_MAP][selected_provider]
# If we have only HMAC credentials for Google Cloud Storage, we must use
# the XML API as the JSON API does not support HMAC.
#
# Technically if we have only HMAC credentials, we should still be able to
# access public read resources via the JSON API, but the XML API can do
# that just as well. It is better to use it than inspect the credentials on
# every HTTP call.
if (provider == 'gs' and
not config.has_option('Credentials', 'gs_oauth2_refresh_token')
and
not (config.has_option('Credentials', 'gs_service_client_id')
and config.has_option('Credentials',
'gs_service_key_file')) and
(config.has_option('Credentials', 'gs_access_key_id')
and config.has_option('Credentials', 'gs_secret_access_key'))):
api = ApiSelector.XML
# Try to force the user's preference to a supported API.
elif self.prefer_api in (
self.api_map[ApiMapConstants.SUPPORT_MAP][selected_provider]):
api = self.prefer_api
return api
def CheckForXmlConfigurationAndRaise(config_type_string, json_txt):
"""Checks a JSON parse exception for provided XML configuration."""
try:
xml.etree.ElementTree.fromstring(str(json_txt))
raise ArgumentException('\n'.join(textwrap.wrap(
'XML {0} data provided; Google Cloud Storage {0} configuration '
'now uses JSON format. To convert your {0}, set the desired XML '
'ACL using \'gsutil {1} set ...\' with gsutil version 3.x. Then '
'use \'gsutil {1} get ...\' with gsutil version 4 or greater to '
'get the corresponding JSON {0}.'.format(config_type_string,
config_type_string.lower()))))
except XmlParseError:
pass
raise ArgumentException('JSON %s data could not be loaded '
'from: %s' % (config_type_string, json_txt))
def BotoEntryToJson(cls, entry):
"""Converts a Boto ACL entry to a valid JSON dictionary."""
acl_entry_json = {}
# JSON API documentation uses camel case.
scope_type_lower = entry.scope.type.lower()
if scope_type_lower == ALL_USERS.lower():
acl_entry_json['entity'] = 'allUsers'
elif scope_type_lower == ALL_AUTHENTICATED_USERS.lower():
acl_entry_json['entity'] = 'allAuthenticatedUsers'
elif scope_type_lower == USER_BY_EMAIL.lower():
acl_entry_json['entity'] = 'user-%s' % entry.scope.email_address
acl_entry_json['email'] = entry.scope.email_address
elif scope_type_lower == USER_BY_ID.lower():
acl_entry_json['entity'] = 'user-%s' % entry.scope.id
acl_entry_json['entityId'] = entry.scope.id
elif scope_type_lower == GROUP_BY_EMAIL.lower():
acl_entry_json['entity'] = 'group-%s' % entry.scope.email_address
acl_entry_json['email'] = entry.scope.email_address
elif scope_type_lower == GROUP_BY_ID.lower():
acl_entry_json['entity'] = 'group-%s' % entry.scope.id
acl_entry_json['entityId'] = entry.scope.id
elif scope_type_lower == GROUP_BY_DOMAIN.lower():
acl_entry_json['entity'] = 'domain-%s' % entry.scope.domain
acl_entry_json['domain'] = entry.scope.domain
else:
raise ArgumentException('ACL contains invalid scope type: %s' %
scope_type_lower)
acl_entry_json['role'] = cls.XML_TO_JSON_ROLES[entry.permission]
return acl_entry_json
def _GetApi(self, provider):
"""Returns a valid CloudApi for use by the caller.
This function lazy-loads connection and credentials using the API map
and credential store provided during class initialization.
Args:
provider: Provider to load API for. If None, class-wide default is used.
Raises:
ArgumentException if there is no matching API available in the API map.
Returns:
Valid API instance that can be used to communicate with the Cloud
Storage provider.
"""
provider = provider or self.provider
if not provider:
raise ArgumentException('No provider selected for _GetApi')
provider = str(provider)
if provider not in self.loaded_apis:
self.loaded_apis[provider] = {}
api_selector = self.GetApiSelector(provider)
if api_selector not in self.loaded_apis[provider]:
# Need to load the API.
self._LoadApi(provider, api_selector)
return self.loaded_apis[provider][api_selector]
def __init__(self,
bucket_storage_uri_class,
gsutil_api_map,
logger,
provider=None,
debug=0):
"""Performs necessary setup for delegating cloud storage requests.
This function has different arguments than the gsutil Cloud API __init__
function because of the delegation responsibilties of this class.
Args:
bucket_storage_uri_class: boto storage_uri class, used by APIs that
provide boto translation or mocking.
gsutil_api_map: Map of providers and API selector tuples to api classes
which can be used to communicate with those providers.
logger: logging.logger for outputting log messages.
provider: Default provider prefix describing cloud storage provider to
connect to.
debug: Debug level for the API implementation (0..3).
"""
super(CloudApiDelegator, self).__init__(bucket_storage_uri_class,
logger,
provider=provider,
debug=debug)
self.api_map = gsutil_api_map
self.prefer_api = boto.config.get('GSUtil', 'prefer_api', '').upper()
self.loaded_apis = {}
if not self.api_map[ApiMapConstants.API_MAP]:
raise ArgumentException(
'No apiclass supplied for gsutil Cloud API map.')
def JsonCorsToMessageEntries(cls, json_cors):
"""Translates CORS JSON to an apitools message.
Args:
json_cors: JSON string representing CORS configuration.
Raises:
ArgumentException on invalid CORS JSON data.
Returns:
List of apitools Bucket.CorsValueListEntry. An empty list represents
no CORS configuration.
"""
deserialized_cors = None
try:
deserialized_cors = json.loads(json_cors)
except ValueError:
CheckForXmlConfigurationAndRaise('CORS', json_cors)
if not isinstance(deserialized_cors, list):
raise ArgumentException(
'CORS JSON should be formatted as a list containing one or more JSON '
'objects.\nSee "gsutil help cors".')
cors = []
for cors_entry in deserialized_cors:
cors.append(
encoding.DictToMessage(
cors_entry, apitools_messages.Bucket.CorsValueListEntry))
return cors
def __init__(self, bucket_storage_uri_class, gsutil_api_map, logger,
status_queue, provider=None, debug=0, trace_token=None,
perf_trace_token=None, user_project=None):
"""Performs necessary setup for delegating cloud storage requests.
This function has different arguments than the gsutil Cloud API __init__
function because of the delegation responsibilties of this class.
Args:
bucket_storage_uri_class: boto storage_uri class, used by APIs that
provide boto translation or mocking.
gsutil_api_map: Map of providers and API selector tuples to api classes
which can be used to communicate with those providers.
logger: logging.logger for outputting log messages.
status_queue: Queue for relaying status to UI.
provider: Default provider prefix describing cloud storage provider to
connect to.
debug: Debug level for the API implementation (0..3).
trace_token: Apiary trace token to pass to API.
perf_trace_token: Performance trace token to use when making API calls.
user_project: Project to be billed for this project.
"""
super(CloudApiDelegator, self).__init__(bucket_storage_uri_class, logger,
status_queue,
provider=provider, debug=debug,
trace_token=trace_token,
perf_trace_token=perf_trace_token,
user_project=user_project)
self.api_map = gsutil_api_map
self.prefer_api = boto.config.get('GSUtil', 'prefer_api', '').upper()
self.loaded_apis = {}
if not self.api_map[ApiMapConstants.API_MAP]:
raise ArgumentException('No apiclass supplied for gsutil Cloud API map.')
def ValidateDstObjectMetadata(dst_obj_metadata):
"""Ensures dst_obj_metadata supplies the needed fields for copy and insert.
Args:
dst_obj_metadata: Metadata to validate.
Raises:
ArgumentException if metadata is invalid.
"""
if not dst_obj_metadata:
raise ArgumentException(
'No object metadata supplied for destination object.')
if not dst_obj_metadata.name:
raise ArgumentException(
'Object metadata supplied for destination object had no object name.')
if not dst_obj_metadata.bucket:
raise ArgumentException(
'Object metadata supplied for destination object had no bucket name.')
def PreconditionsFromHeaders(headers):
"""Creates bucket or object preconditions acccording to the provided headers.
Args:
headers: Dict of headers passed via gsutil -h
Returns:
gsutil Cloud API Preconditions object fields populated from headers, or None
if no precondition headers are present.
"""
return_preconditions = Preconditions()
try:
for header, value in headers.items():
if GOOG_GENERATION_MATCH_REGEX.match(header):
return_preconditions.gen_match = long(value)
if GOOG_METAGENERATION_MATCH_REGEX.match(header):
return_preconditions.meta_gen_match = long(value)
except ValueError, _:
raise ArgumentException('Invalid precondition header specified. '
'x-goog-if-generation-match and '
'x-goog-if-metageneration match must be specified '
'with a positive integer value.')
def ObjectMetadataFromHeaders(headers):
"""Creates object metadata according to the provided headers.
gsutil -h allows specifiying various headers (originally intended
to be passed to boto in gsutil v3). For the JSON API to be compatible with
this option, we need to parse these headers into gsutil_api Object fields.
Args:
headers: Dict of headers passed via gsutil -h
Raises:
ArgumentException if an invalid header is encountered.
Returns:
apitools Object with relevant fields populated from headers.
"""
obj_metadata = apitools_messages.Object()
for header, value in headers.items():
if CACHE_CONTROL_REGEX.match(header):
obj_metadata.cacheControl = value.strip()
elif CONTENT_DISPOSITION_REGEX.match(header):
obj_metadata.contentDisposition = value.strip()
elif CONTENT_ENCODING_REGEX.match(header):
obj_metadata.contentEncoding = value.strip()
elif CONTENT_MD5_REGEX.match(header):
obj_metadata.md5Hash = value.strip()
elif CONTENT_LANGUAGE_REGEX.match(header):
obj_metadata.contentLanguage = value.strip()
elif CONTENT_TYPE_REGEX.match(header):
if not value:
obj_metadata.contentType = DEFAULT_CONTENT_TYPE
else:
obj_metadata.contentType = value.strip()
elif GOOG_API_VERSION_REGEX.match(header):
# API version is only relevant for XML, ignore and rely on the XML API
# to add the appropriate version.
continue
elif GOOG_GENERATION_MATCH_REGEX.match(header):
# Preconditions are handled elsewhere, but allow these headers through.
continue
elif GOOG_METAGENERATION_MATCH_REGEX.match(header):
# Preconditions are handled elsewhere, but allow these headers through.
continue
else:
custom_goog_metadata_match = CUSTOM_GOOG_METADATA_REGEX.match(
header)
custom_amz_metadata_match = CUSTOM_AMZ_METADATA_REGEX.match(header)
custom_amz_header_match = CUSTOM_AMZ_HEADER_REGEX.match(header)
header_key = None
if custom_goog_metadata_match:
header_key = custom_goog_metadata_match.group('header_key')
elif custom_amz_metadata_match:
header_key = custom_amz_metadata_match.group('header_key')
elif custom_amz_header_match:
# If we got here we are guaranteed by the prior statement that this is
# not an x-amz-meta- header.
header_key = (S3_HEADER_PREFIX +
custom_amz_header_match.group('header_key'))
if header_key:
if header_key.lower() == 'x-goog-content-language':
# Work around content-language being inserted into custom metadata.
continue
if not obj_metadata.metadata:
obj_metadata.metadata = apitools_messages.Object.MetadataValue(
)
if not obj_metadata.metadata.additionalProperties:
obj_metadata.metadata.additionalProperties = []
obj_metadata.metadata.additionalProperties.append(
apitools_messages.Object.MetadataValue.AdditionalProperty(
key=header_key, value=value))
else:
raise ArgumentException('Invalid header specifed: %s:%s' %
(header, value))
return obj_metadata
def HeadersFromObjectMetadata(dst_obj_metadata, provider):
"""Creates a header dictionary based on existing object metadata.
Args:
dst_obj_metadata: Object metadata to create the headers from.
provider: Provider string ('gs' or 's3')
Returns:
Headers dictionary.
"""
headers = {}
if not dst_obj_metadata:
return
# Metadata values of '' mean suppress/remove this header.
if dst_obj_metadata.cacheControl is not None:
if not dst_obj_metadata.cacheControl:
headers['cache-control'] = None
else:
headers['cache-control'] = dst_obj_metadata.cacheControl.strip()
if dst_obj_metadata.contentDisposition:
if not dst_obj_metadata.contentDisposition:
headers['content-disposition'] = None
else:
headers['content-disposition'] = (
dst_obj_metadata.contentDisposition.strip())
if dst_obj_metadata.contentEncoding:
if not dst_obj_metadata.contentEncoding:
headers['content-encoding'] = None
else:
headers[
'content-encoding'] = dst_obj_metadata.contentEncoding.strip()
if dst_obj_metadata.contentLanguage:
if not dst_obj_metadata.contentLanguage:
headers['content-language'] = None
else:
headers[
'content-language'] = dst_obj_metadata.contentLanguage.strip()
if dst_obj_metadata.md5Hash:
if not dst_obj_metadata.md5Hash:
headers['Content-MD5'] = None
else:
headers['Content-MD5'] = dst_obj_metadata.md5Hash.strip()
if dst_obj_metadata.contentType is not None:
if not dst_obj_metadata.contentType:
headers['content-type'] = None
else:
headers['content-type'] = dst_obj_metadata.contentType.strip()
if (dst_obj_metadata.metadata
and dst_obj_metadata.metadata.additionalProperties):
for additional_property in dst_obj_metadata.metadata.additionalProperties:
# Work around content-language being inserted into custom metadata by
# the XML API.
if additional_property.key == 'content-language':
continue
# Don't translate special metadata markers.
if additional_property.key in S3_MARKER_GUIDS:
continue
if provider == 'gs':
header_name = 'x-goog-meta-' + additional_property.key
elif provider == 's3':
if additional_property.key.startswith(S3_HEADER_PREFIX):
header_name = (
'x-amz-' +
additional_property.key[len(S3_HEADER_PREFIX):])
else:
header_name = 'x-amz-meta-' + additional_property.key
else:
raise ArgumentException('Invalid provider specified: %s' %
provider)
if (additional_property.value is not None
and not additional_property.value):
headers[header_name] = None
else:
headers[header_name] = additional_property.value
return headers
def GetApiSelector(self, provider=None):
"""Returns a cs_api_map.ApiSelector based on input and configuration.
Args:
provider: Provider to return the ApiSelector for. If None, class-wide
default is used.
Returns:
cs_api_map.ApiSelector that will be used for calls to the delegator
for this provider.
"""
selected_provider = provider or self.provider
if not selected_provider:
raise ArgumentException('No provider selected for CloudApi')
if (selected_provider not in self.api_map[ApiMapConstants.DEFAULT_MAP]
or self.api_map[ApiMapConstants.DEFAULT_MAP][selected_provider]
not in self.api_map[ApiMapConstants.API_MAP][selected_provider]
):
raise ArgumentException(
'No default api available for provider %s' % selected_provider)
if selected_provider not in self.api_map[ApiMapConstants.SUPPORT_MAP]:
raise ArgumentException(
'No supported apis available for provider %s' %
selected_provider)
api = self.api_map[ApiMapConstants.DEFAULT_MAP][selected_provider]
using_gs_hmac = (
provider == 'gs'
and not config.has_option('Credentials', 'gs_oauth2_refresh_token')
and
not (config.has_option('Credentials', 'gs_service_client_id')
and config.has_option('Credentials', 'gs_service_key_file'))
and (config.has_option('Credentials', 'gs_access_key_id')
and config.has_option('Credentials', 'gs_secret_access_key')))
configured_encryption = (
provider == 'gs'
and (config.has_option('GSUtil', 'encryption_key')
or config.has_option('GSUtil', 'decryption_key1')))
if using_gs_hmac and configured_encryption:
raise CommandException(
'gsutil does not support HMAC credentials with customer-supplied '
'encryption keys (CSEK) or customer-managed KMS encryption keys '
'(CMEK). Please generate and include non-HMAC credentials '
'in your .boto configuration file, or to access public encrypted '
'objects, remove your HMAC credentials.')
# If we have only HMAC credentials for Google Cloud Storage, we must use
# the XML API as the JSON API does not support HMAC.
#
# Technically if we have only HMAC credentials, we should still be able to
# access public read resources via the JSON API, but the XML API can do
# that just as well. It is better to use it than inspect the credentials on
# every HTTP call.
elif using_gs_hmac:
api = ApiSelector.XML
# CSEK and CMEK encryption keys are currently only supported in the
# JSON API implementation (GcsJsonApi). We can't stop XML API users from
# interacting with encrypted objects, since we don't know the object is
# encrypted until after the API call is made, but if they specify
# configuration values we will use JSON.
elif configured_encryption:
api = ApiSelector.JSON
# Try to force the user's preference to a supported API.
elif self.prefer_api in (
self.api_map[ApiMapConstants.SUPPORT_MAP][selected_provider]):
api = self.prefer_api
if (api == ApiSelector.XML and context_config.get_context_config() and
context_config.get_context_config().use_client_certificate):
raise ArgumentException(
'User enabled mTLS by setting "use_client_certificate", but mTLS'
' is not supported for the selected XML API. Try configuring for '
' the GCS JSON API or setting "use_client_certificate" to "False" in'
' the Boto config.')
return api
def _SetIam(self):
"""Set IAM policy for given wildcards on the command line."""
self.continue_on_error = False
self.recursion_requested = False
self.all_versions = False
force_etag = False
etag = ''
if self.sub_opts:
for o, arg in self.sub_opts:
if o in ['-r', '-R']:
self.recursion_requested = True
elif o == '-f':
self.continue_on_error = True
elif o == '-a':
self.all_versions = True
elif o == '-e':
etag = str(arg)
force_etag = True
else:
self.RaiseInvalidArgumentException()
file_url = self.args[0]
patterns = self.args[1:]
# Load the IAM policy file and raise error if the file is invalid JSON or
# does not exist.
try:
with open(file_url, 'r') as fp:
policy = json.loads(fp.read())
except IOError:
raise ArgumentException(
'Specified IAM policy file "%s" does not exist.' % file_url)
except ValueError as e:
self.logger.debug('Invalid IAM policy file, ValueError:\n', e)
raise ArgumentException('Invalid IAM policy file "%s".' % file_url)
bindings = policy.get('bindings', [])
if not force_etag:
etag = policy.get('etag', '')
policy_json = json.dumps({'bindings': bindings, 'etag': etag})
try:
policy = protojson.decode_message(apitools_messages.Policy,
policy_json)
except DecodeError:
raise ArgumentException(
'Invalid IAM policy file "%s" or etag "%s".' %
(file_url, etag))
self.everything_set_okay = True
# This list of wildcard strings will be handled by NameExpansionIterator.
threaded_wildcards = []
for pattern in patterns:
surl = StorageUrlFromString(pattern)
if surl.IsBucket():
if self.recursion_requested:
surl.object_name = '*'
threaded_wildcards.append(surl.url_string)
else:
self.SetIamHelper(surl, policy)
else:
threaded_wildcards.append(surl.url_string)
# N.B.: If threaded_wildcards contains a non-existent bucket
# (e.g. ["gs://non-existent", "gs://existent"]), NameExpansionIterator
# will raise an exception in iter.next. This halts all iteration, even
# when -f is set. This behavior is also evident in acl set. This behavior
# also appears for any exception that will be raised when iterating over
# wildcard expansions (access denied if bucket cannot be listed, etc.).
if threaded_wildcards:
name_expansion_iterator = NameExpansionIterator(
self.command_name,
self.debug,
self.logger,
self.gsutil_api,
threaded_wildcards,
self.recursion_requested,
all_versions=self.all_versions,
continue_on_error=self.continue_on_error
or self.parallel_operations,
bucket_listing_fields=['name'])
seek_ahead_iterator = SeekAheadNameExpansionIterator(
self.command_name,
self.debug,
self.GetSeekAheadGsutilApi(),
threaded_wildcards,
self.recursion_requested,
all_versions=self.all_versions)
policy_it = itertools.repeat(protojson.encode_message(policy))
self.Apply(_SetIamWrapper,
itertools.izip(policy_it, name_expansion_iterator),
_SetIamExceptionHandler,
fail_on_error=not self.continue_on_error,
seek_ahead_iterator=seek_ahead_iterator)
self.everything_set_okay &= not GetFailureCount() > 0
# TODO: Add an error counter for files and objects.
if not self.everything_set_okay:
raise CommandException('Some IAM policies could not be set.')
