class FakeCommandWithCompleters(Command):
"""Command with various completer types."""
command_spec = Command.CreateCommandSpec(
'fake2',
argparse_arguments=[
CommandArgument.MakeZeroOrMoreCloudURLsArgument(),
CommandArgument.MakeZeroOrMoreFileURLsArgument(),
CommandArgument.MakeZeroOrMoreCloudOrFileURLsArgument(),
CommandArgument.MakeFreeTextArgument(),
CommandArgument.MakeZeroOrMoreCloudBucketURLsArgument(),
CommandArgument.MakeFileURLOrCannedACLArgument(),
]
)
help_spec = Command.HelpSpec(
help_name='fake2',
help_name_aliases=[],
help_type='command_help',
help_one_line_summary='fake command for tests',
help_text='fake command for tests',
subcommand_help_text={}
)
def __init__(self):
pass
class UrlSignCommand(Command):
"""Implementation of gsutil url_sign command."""
# Command specification. See base class for documentation.
command_spec = Command.CreateCommandSpec(
'signurl',
command_name_aliases=['signedurl', 'queryauth'],
usage_synopsis=_SYNOPSIS,
min_args=1,
max_args=constants.NO_MAX,
supported_sub_args='m:d:c:p:r:u',
supported_private_args=['use-service-account'],
file_url_ok=False,
provider_url_ok=False,
urls_start_arg=1,
gs_api_support=[ApiSelector.XML, ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments=[
CommandArgument.MakeZeroOrMoreFileURLsArgument(),
CommandArgument.MakeZeroOrMoreCloudURLsArgument(),
],
)
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='signurl',
help_name_aliases=[
'signedurl',
'queryauth',
],
help_type='command_help',
help_one_line_summary='Create a signed url',
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={},
)
def _ParseAndCheckSubOpts(self):
# Default argument values
delta = None
method = 'GET'
content_type = ''
passwd = None
region = _AUTO_DETECT_REGION
use_service_account = False
for o, v in self.sub_opts:
# TODO(PY3-ONLY): Delete this if block.
if six.PY2:
v = v.decode(sys.stdin.encoding or constants.UTF8)
if o == '-d':
if delta is not None:
delta += _DurationToTimeDelta(v)
else:
delta = _DurationToTimeDelta(v)
elif o == '-m':
method = v
elif o == '-c':
content_type = v
elif o == '-p':
passwd = v
elif o == '-r':
region = v
elif o == '-u' or o == '--use-service-account':
use_service_account = True
else:
self.RaiseInvalidArgumentException()
if delta is None:
delta = timedelta(hours=1)
else:
if use_service_account and delta > _MAX_EXPIRATION_TIME_WITH_MINUS_U:
# This restriction comes from the IAM SignBlob API. The SignBlob
# API uses a system-managed key which can guarantee validation only
# up to 12 hours. b/156160482#comment4
raise CommandException(
'Max valid duration allowed is %s when -u flag is used. For longer'
' duration, consider using the private-key-file instead of the -u'
' option.' % _MAX_EXPIRATION_TIME_WITH_MINUS_U)
elif delta > _MAX_EXPIRATION_TIME:
raise CommandException('Max valid duration allowed is '
'%s' % _MAX_EXPIRATION_TIME)
if method not in ['GET', 'PUT', 'DELETE', 'HEAD', 'RESUMABLE']:
raise CommandException('HTTP method must be one of'
'[GET|HEAD|PUT|DELETE|RESUMABLE]')
if not use_service_account and len(self.args) < 2:
raise CommandException(
'The command requires a key file argument and one or more '
'url arguments if the --use-service-account flag is missing. '
'Run `gsutil help signurl` for more info')
return method, delta, content_type, passwd, region, use_service_account
def _ProbeObjectAccessWithClient(self, key, use_service_account, provider,
client_email, gcs_path, logger, region):
"""Performs a head request against a signed url to check for read access."""
# Choose a reasonable time in the future; if the user's system clock is
# 60 or more seconds behind the server's this will generate an error.
signed_url = _GenSignedUrl(key=key,
api=self.gsutil_api,
use_service_account=use_service_account,
provider=provider,
client_id=client_email,
method='HEAD',
duration=timedelta(seconds=60),
gcs_path=gcs_path,
logger=logger,
region=region,
string_to_sign_debug=True)
try:
h = GetNewHttp()
req = Request(signed_url, 'HEAD')
response = MakeRequest(h, req)
if response.status_code not in [200, 403, 404]:
raise HttpError.FromResponse(response)
return response.status_code
except HttpError as http_error:
if http_error.has_attr('response'):
error_response = http_error.response
error_string = ('Unexpected HTTP response code %s while querying '
'object readability. Is your system clock accurate?' %
error_response.status_code)
if error_response.content:
error_string += ' Content: %s' % error_response.content
else:
error_string = ('Expected an HTTP response code of '
'200 while querying object readability, but received '
'an error: %s' % http_error)
raise CommandException(error_string)
def _EnumerateStorageUrls(self, in_urls):
ret = []
for url_str in in_urls:
if ContainsWildcard(url_str):
ret.extend([blr.storage_url for blr in self.WildcardIterator(url_str)])
else:
ret.append(StorageUrlFromString(url_str))
return ret
def RunCommand(self):
"""Command entry point for signurl command."""
if not HAVE_OPENSSL:
raise CommandException(
'The signurl command requires the pyopenssl library (try pip '
'install pyopenssl or easy_install pyopenssl)')
method, delta, content_type, passwd, region, use_service_account = (
self._ParseAndCheckSubOpts())
arg_start_index = 0 if use_service_account else 1
storage_urls = self._EnumerateStorageUrls(self.args[arg_start_index:])
region_cache = {}
key = None
if not use_service_account:
try:
key, client_email = _ReadJSONKeystore(
open(self.args[0], 'rb').read(), passwd)
except ValueError:
# Ignore and try parsing as a pkcs12.
if not passwd:
passwd = getpass.getpass('Keystore password:'******'rb').read(), passwd)
except ValueError:
raise CommandException('Unable to parse private key from {0}'.format(
self.args[0]))
else:
client_email = self.gsutil_api.GetServiceAccountId(provider='gs')
print('URL\tHTTP Method\tExpiration\tSigned URL')
for url in storage_urls:
if url.scheme != 'gs':
raise CommandException('Can only create signed urls from gs:// urls')
if url.IsBucket():
if region == _AUTO_DETECT_REGION:
raise CommandException('Generating signed URLs for creating buckets'
' requires a region be specified via the -r '
'option. Run `gsutil help signurl` for more '
'information about the \'-r\' option.')
gcs_path = url.bucket_name
if method == 'RESUMABLE':
raise CommandException('Resumable signed URLs require an object '
'name.')
else:
# Need to url encode the object name as Google Cloud Storage does when
# computing the string to sign when checking the signature.
gcs_path = '{0}/{1}'.format(
url.bucket_name,
urllib.parse.quote(url.object_name.encode(constants.UTF8),
safe=b'/~'))
if region == _AUTO_DETECT_REGION:
if url.bucket_name in region_cache:
bucket_region = region_cache[url.bucket_name]
else:
try:
_, bucket = self.GetSingleBucketUrlFromArg(
'gs://{}'.format(url.bucket_name), bucket_fields=['location'])
except Exception as e:
raise CommandException(
'{}: Failed to auto-detect location for bucket \'{}\'. Please '
'ensure you have storage.buckets.get permission on the bucket '
'or specify the bucket\'s location using the \'-r\' option.'.
format(e.__class__.__name__, url.bucket_name))
bucket_region = bucket.location.lower()
region_cache[url.bucket_name] = bucket_region
else:
bucket_region = region
final_url = _GenSignedUrl(key=key,
api=self.gsutil_api,
use_service_account=use_service_account,
provider=url.scheme,
client_id=client_email,
method=method,
duration=delta,
gcs_path=gcs_path,
logger=self.logger,
region=bucket_region,
content_type=content_type,
string_to_sign_debug=True)
expiration = calendar.timegm((datetime.utcnow() + delta).utctimetuple())
expiration_dt = datetime.fromtimestamp(expiration)
time_str = expiration_dt.strftime('%Y-%m-%d %H:%M:%S')
# TODO(PY3-ONLY): Delete this if block.
if six.PY2:
time_str = time_str.decode(constants.UTF8)
url_info_str = '{0}\t{1}\t{2}\t{3}'.format(url.url_string, method,
time_str, final_url)
# TODO(PY3-ONLY): Delete this if block.
if six.PY2:
url_info_str = url_info_str.encode(constants.UTF8)
print(url_info_str)
response_code = self._ProbeObjectAccessWithClient(
key, use_service_account, url.scheme, client_email, gcs_path,
self.logger, bucket_region)
if response_code == 404:
if url.IsBucket() and method != 'PUT':
raise CommandException(
'Bucket {0} does not exist. Please create a bucket with '
'that name before a creating signed URL to access it.'.format(
url))
else:
if method != 'PUT' and method != 'RESUMABLE':
raise CommandException(
'Object {0} does not exist. Please create/upload an object '
'with that name before a creating signed URL to access it.'.
format(url))
elif response_code == 403:
self.logger.warn(
'%s does not have permissions on %s, using this link will likely '
'result in a 403 error until at least READ permissions are granted',
client_email or 'The account', url)
return 0
class HashCommand(Command):
"""Implementation of gsutil hash command."""
# Command specification. See base class for documentation.
command_spec = Command.CreateCommandSpec(
'hash',
command_name_aliases=[],
usage_synopsis=_SYNOPSIS,
min_args=1,
max_args=constants.NO_MAX,
supported_sub_args='chm',
file_url_ok=True,
provider_url_ok=False,
urls_start_arg=0,
gs_api_support=[ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments=[CommandArgument.MakeZeroOrMoreFileURLsArgument()])
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='hash',
help_name_aliases=['checksum'],
help_type='command_help',
help_one_line_summary='Calculate file hashes',
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={},
)
@classmethod
def _ParseOpts(cls, sub_opts, logger):
"""Returns behavior variables based on input options.
Args:
sub_opts: getopt sub-arguments for the command.
logger: logging.Logger for the command.
Returns:
Tuple of
calc_crc32c: Boolean, if True, command should calculate a CRC32c checksum.
calc_md5: Boolean, if True, command should calculate an MD5 hash.
format_func: Function used for formatting the hash in the desired format.
cloud_format_func: Function used for formatting the hash in the desired
format.
output_format: String describing the hash output format.
"""
calc_crc32c = False
calc_md5 = False
format_func = (
lambda digest: hashing_helper.Base64EncodeHash(digest.hexdigest()))
cloud_format_func = lambda digest: digest
found_hash_option = False
output_format = 'base64'
if sub_opts:
for o, unused_a in sub_opts:
if o == '-c':
calc_crc32c = True
found_hash_option = True
elif o == '-h':
output_format = 'hex'
format_func = lambda digest: digest.hexdigest()
cloud_format_func = lambda digest: (
hashing_helper.Base64ToHexHash(digest).decode('ascii')
) # yapf: disable
elif o == '-m':
calc_md5 = True
found_hash_option = True
if not found_hash_option:
calc_crc32c = True
calc_md5 = True
if calc_crc32c and not boto_util.UsingCrcmodExtension():
logger.warn(hashing_helper.SLOW_CRCMOD_WARNING)
return calc_crc32c, calc_md5, format_func, cloud_format_func, output_format
def _GetHashClassesFromArgs(self, calc_crc32c, calc_md5):
"""Constructs the dictionary of hashes to compute based on the arguments.
Args:
calc_crc32c: If True, CRC32c should be included.
calc_md5: If True, MD5 should be included.
Returns:
Dictionary of {string: hash digester}, where string the name of the
digester algorithm.
"""
hash_dict = {}
if calc_crc32c:
hash_dict['crc32c'] = crcmod.predefined.Crc('crc-32c')
if calc_md5:
hash_dict['md5'] = hashing_helper.GetMd5()
return hash_dict
def RunCommand(self):
"""Command entry point for the hash command."""
(calc_crc32c, calc_md5, format_func, cloud_format_func,
output_format) = (self._ParseOpts(self.sub_opts, self.logger))
matched_one = False
for url_str in self.args:
for file_ref in self.WildcardIterator(url_str).IterObjects(
bucket_listing_fields=[
'crc32c',
'customerEncryption',
'md5Hash',
'size',
]):
matched_one = True
url = StorageUrlFromString(url_str)
file_name = file_ref.storage_url.object_name
if StorageUrlFromString(url_str).IsFileUrl():
file_size = os.path.getsize(file_name)
self.gsutil_api.status_queue.put(
FileMessage(url,
None,
time.time(),
size=file_size,
finished=False,
message_type=FileMessage.FILE_HASH))
callback_processor = ProgressCallbackWithTimeout(
file_size,
FileProgressCallbackHandler(self.gsutil_api.status_queue,
src_url=StorageUrlFromString(url_str),
operation_name='Hashing').call)
hash_dict = self._GetHashClassesFromArgs(calc_crc32c, calc_md5)
with open(file_name, 'rb') as fp:
hashing_helper.CalculateHashesFromContents(
fp, hash_dict, callback_processor=callback_processor)
self.gsutil_api.status_queue.put(
FileMessage(url,
None,
time.time(),
size=file_size,
finished=True,
message_type=FileMessage.FILE_HASH))
else:
hash_dict = {}
obj_metadata = file_ref.root_object
file_size = obj_metadata.size
md5_present = obj_metadata.md5Hash is not None
crc32c_present = obj_metadata.crc32c is not None
if not md5_present and not crc32c_present:
logging.getLogger().warn('No hashes present for %s', url_str)
continue
if md5_present:
hash_dict['md5'] = obj_metadata.md5Hash
if crc32c_present:
hash_dict['crc32c'] = obj_metadata.crc32c
print('Hashes [%s] for %s:' % (output_format, file_name))
for name, digest in six.iteritems(hash_dict):
print('\tHash (%s):\t\t%s' % (name,
(format_func(digest) if url.IsFileUrl()
else cloud_format_func(digest))))
if not matched_one:
raise CommandException('No files matched')
_PutToQueueWithTimeout(self.gsutil_api.status_queue,
FinalMessage(time.time()))
return 0
class HashCommand(Command):
"""Implementation of gsutil hash command."""
# Command specification. See base class for documentation.
command_spec = Command.CreateCommandSpec(
'hash',
command_name_aliases=[],
usage_synopsis=_SYNOPSIS,
min_args=1,
max_args=NO_MAX,
supported_sub_args='chm',
file_url_ok=True,
provider_url_ok=False,
urls_start_arg=0,
gs_api_support=[ApiSelector.JSON],
gs_default_api=ApiSelector.JSON,
argparse_arguments=[CommandArgument.MakeZeroOrMoreFileURLsArgument()])
# Help specification. See help_provider.py for documentation.
help_spec = Command.HelpSpec(
help_name='hash',
help_name_aliases=['checksum'],
help_type='command_help',
help_one_line_summary='Calculate file hashes',
help_text=_DETAILED_HELP_TEXT,
subcommand_help_text={},
)
@classmethod
def _ParseOpts(cls, sub_opts, logger):
"""Returns behavior variables based on input options.
Args:
sub_opts: getopt sub-arguments for the command.
logger: logging.Logger for the command.
Returns:
Tuple of
calc_crc32c: Boolean, if True, command should calculate a CRC32c checksum.
calc_md5: Boolean, if True, command should calculate an MD5 hash.
format_func: Function used for formatting the hash in the desired format.
output_format: String describing the hash output format.
"""
calc_crc32c = False
calc_md5 = False
format_func = lambda digest: Base64EncodeHash(digest.hexdigest())
found_hash_option = False
output_format = 'base64'
if sub_opts:
for o, unused_a in sub_opts:
if o == '-c':
calc_crc32c = True
found_hash_option = True
elif o == '-h':
output_format = 'hex'
format_func = lambda digest: digest.hexdigest()
elif o == '-m':
calc_md5 = True
found_hash_option = True
if not found_hash_option:
calc_crc32c = True
calc_md5 = True
if calc_crc32c and not UsingCrcmodExtension(crcmod):
logger.warn(SLOW_CRCMOD_WARNING)
return calc_crc32c, calc_md5, format_func, output_format
def _GetHashClassesFromArgs(self, calc_crc32c, calc_md5):
"""Constructs the dictionary of hashes to compute based on the arguments.
Args:
calc_crc32c: If True, CRC32c should be included.
calc_md5: If True, MD5 should be included.
Returns:
Dictionary of {string: hash digester}, where string the name of the
digester algorithm.
"""
hash_dict = {}
if calc_crc32c:
hash_dict['crc32c'] = crcmod.predefined.Crc('crc-32c')
if calc_md5:
hash_dict['md5'] = md5()
return hash_dict
def RunCommand(self):
"""Command entry point for the hash command."""
(calc_crc32c, calc_md5, format_func,
output_format) = (self._ParseOpts(self.sub_opts, self.logger))
matched_one = False
for url_str in self.args:
if not StorageUrlFromString(url_str).IsFileUrl():
raise CommandException('"hash" command requires a file URL')
for file_ref in self.WildcardIterator(url_str).IterObjects():
matched_one = True
file_name = file_ref.storage_url.object_name
file_size = os.path.getsize(file_name)
callback_processor = ProgressCallbackWithBackoff(
file_size,
FileProgressCallbackHandler(
ConstructAnnounceText('Hashing', file_name),
self.logger).call)
hash_dict = self._GetHashClassesFromArgs(calc_crc32c, calc_md5)
with open(file_name, 'rb') as fp:
CalculateHashesFromContents(
fp, hash_dict, callback_processor=callback_processor)
print 'Hashes [%s] for %s:' % (output_format, file_name)
for name, digest in hash_dict.iteritems():
print '\tHash (%s):\t\t%s' % (name, format_func(digest))
if not matched_one:
raise CommandException('No files matched')
return 0
