class _NameExpansionIterator(object):
"""
Iterates over all src_uris, expanding wildcards, object-less bucket names,
subdir bucket names, and directory names, generating a flat listing of all
the matching objects/files.
You should instantiate this object using the static factory function
NameExpansionIterator, because consumers of this iterator need the
PluralityCheckableIterator wrapper built by that function.
Yields:
gslib.name_expansion.NameExpansionResult.
Raises:
CommandException: if errors encountered.
"""
def __init__(self,
command_name,
proj_id_handler,
headers,
debug,
bucket_storage_uri_class,
uri_strs,
recursion_requested,
have_existing_dst_container=None,
flat=True,
all_versions=False,
for_all_version_delete=False):
"""
Args:
command_name: name of command being run.
proj_id_handler: ProjectIdHandler to use for current command.
headers: Dictionary containing optional HTTP headers to pass to boto.
debug: Debug level to pass in to boto connection (range 0..3).
bucket_storage_uri_class: Class to instantiate for cloud StorageUris.
Settable for testing/mocking.
uri_strs: PluralityCheckableIterator of URI strings needing expansion.
recursion_requested: True if -R specified on command-line.
have_existing_dst_container: Bool indicator whether this is a copy
request to an existing bucket, bucket subdir, or directory. Default
None value should be used in cases where this is not needed (commands
other than cp).
flat: Bool indicating whether bucket listings should be flattened, i.e.,
so the mapped-to results contain objects spanning subdirectories.
all_versions: Bool indicating whether to iterate over all object versions.
for_all_version_delete: Bool indicating whether this is for an all-version
delete.
Examples of _NameExpansionIterator with flat=True:
- Calling with one of the uri_strs being 'gs://bucket' will enumerate all
top-level objects, as will 'gs://bucket/' and 'gs://bucket/*'.
- 'gs://bucket/**' will enumerate all objects in the bucket.
- 'gs://bucket/abc' will enumerate all next-level objects under directory
abc (i.e., not including subdirectories of abc) if gs://bucket/abc/*
matches any objects; otherwise it will enumerate the single name
gs://bucket/abc
- 'gs://bucket/abc/**' will enumerate all objects under abc or any of its
subdirectories.
- 'file:///tmp' will enumerate all files under /tmp, as will
'file:///tmp/*'
- 'file:///tmp/**' will enumerate all files under /tmp or any of its
subdirectories.
Example if flat=False: calling with gs://bucket/abc/* lists matching objects
or subdirs, but not sub-subdirs or objects beneath subdirs.
Note: In step-by-step comments below we give examples assuming there's a
gs://bucket with object paths:
abcd/o1.txt
abcd/o2.txt
xyz/o1.txt
xyz/o2.txt
and a directory file://dir with file paths:
dir/a.txt
dir/b.txt
dir/c/
"""
self.command_name = command_name
self.proj_id_handler = proj_id_handler
self.headers = headers
self.debug = debug
self.bucket_storage_uri_class = bucket_storage_uri_class
self.suri_builder = StorageUriBuilder(debug, bucket_storage_uri_class)
self.uri_strs = uri_strs
self.recursion_requested = recursion_requested
self.have_existing_dst_container = have_existing_dst_container
self.flat = flat
self.all_versions = all_versions
# Map holding wildcard strings to use for flat vs subdir-by-subdir listings.
# (A flat listing means show all objects expanded all the way down.)
self._flatness_wildcard = {True: '**', False: '*'}
def __iter__(self):
for uri_str in self.uri_strs:
# Step 1: Expand any explicitly specified wildcards. The output from this
# step is an iterator of BucketListingRef.
# Starting with gs://buck*/abc* this step would expand to gs://bucket/abcd
if ContainsWildcard(uri_str):
post_step1_iter = self._WildcardIterator(uri_str)
else:
suri = self.suri_builder.StorageUri(uri_str)
post_step1_iter = iter([BucketListingRef(suri)])
post_step1_iter = PluralityCheckableIterator(post_step1_iter)
# Step 2: Expand bucket subdirs and versions. The output from this
# step is an iterator of (names_container, BucketListingRef).
# Starting with gs://bucket/abcd this step would expand to:
# iter([(True, abcd/o1.txt), (True, abcd/o2.txt)]).
if self.flat and self.recursion_requested:
post_step2_iter = _ImplicitBucketSubdirIterator(
self, post_step1_iter, self.flat)
elif self.all_versions:
post_step2_iter = _AllVersionIterator(self,
post_step1_iter,
headers=self.headers)
else:
post_step2_iter = _NonContainerTuplifyIterator(post_step1_iter)
post_step2_iter = PluralityCheckableIterator(post_step2_iter)
# Step 3. Expand directories and buckets. This step yields the iterated
# values. Starting with gs://bucket this step would expand to:
# [abcd/o1.txt, abcd/o2.txt, xyz/o1.txt, xyz/o2.txt]
# Starting with file://dir this step would expand to:
# [dir/a.txt, dir/b.txt, dir/c/]
exp_src_bucket_listing_refs = []
wc = self._flatness_wildcard[self.flat]
src_uri_expands_to_multi = (post_step1_iter.has_plurality()
or post_step2_iter.has_plurality())
is_multi_src_request = (self.uri_strs.has_plurality()
or src_uri_expands_to_multi)
if post_step2_iter.is_empty():
raise CommandException('No URIs matched: %s' % uri_str)
for (names_container, blr) in post_step2_iter:
if (not blr.GetUri().names_container()
and (self.flat or not blr.HasPrefix())):
yield NameExpansionResult(uri_str,
is_multi_src_request,
src_uri_expands_to_multi,
names_container,
blr.GetUriString(),
self.have_existing_dst_container,
is_latest=blr.IsLatest())
continue
if not self.recursion_requested:
if blr.GetUri().is_file_uri():
desc = 'directory'
else:
desc = 'bucket'
print 'Omitting %s "%s". (Did you mean to do %s -R?)' % (
desc, blr.GetUri(), self.command_name)
continue
if blr.GetUri().is_file_uri():
# Convert dir to implicit recursive wildcard.
uri_to_iterate = '%s/%s' % (blr.GetUriString(), wc)
else:
# Convert bucket to implicit recursive wildcard.
uri_to_iterate = blr.GetUri().clone_replace_name(wc)
wc_iter = PluralityCheckableIterator(
self._WildcardIterator(uri_to_iterate))
src_uri_expands_to_multi = (src_uri_expands_to_multi
or wc_iter.has_plurality())
is_multi_src_request = (self.uri_strs.has_plurality()
or src_uri_expands_to_multi)
for blr in wc_iter:
yield NameExpansionResult(uri_str,
is_multi_src_request,
src_uri_expands_to_multi,
True,
blr.GetUriString(),
self.have_existing_dst_container,
is_latest=blr.IsLatest())
def _WildcardIterator(self, uri_or_str):
"""
Helper to instantiate gslib.WildcardIterator. Args are same as
gslib.WildcardIterator interface, but this method fills in most of the
values from instance state.
Args:
uri_or_str: StorageUri or URI string naming wildcard objects to iterate.
"""
return wildcard_iterator.wildcard_iterator(
uri_or_str,
self.proj_id_handler,
bucket_storage_uri_class=self.bucket_storage_uri_class,
headers=self.headers,
debug=self.debug,
all_versions=self.all_versions)
def _MaybeCheckForAndOfferSoftwareUpdate(self, command_name, debug):
"""Checks the last time we checked for an update, and if it's been longer
than the configured threshold offers the user to update gsutil.
Args:
command_name: The name of the command being run.
debug: Debug level to pass in to boto connection (range 0..3).
Returns:
True if the user decides to update.
"""
# Don't try to interact with user if:
# - gsutil is not connected to a tty (e.g., if being run from cron);
# - user is running gsutil -q
# - user is running the update command (which could otherwise cause an
# additional note that an update is available when user is already trying
# to perform an update);
# - user doesn't have credentials configured; or,
# - user specified gs_host (which could be a non-production different
# service instance, in which case credentials won't work for checking
# gsutil tarball).
gs_host = boto.config.get('Credentials', 'gs_host', None)
if (not sys.stdout.isatty() or not sys.stderr.isatty()
or not sys.stdin.isatty() or command_name == 'update'
or not logging.getLogger().isEnabledFor(logging.INFO)
or not HasConfiguredCredentials() or gs_host):
return False
software_update_check_period = boto.config.get(
'GSUtil', 'software_update_check_period', 30)
# Setting software_update_check_period to 0 means periodic software
# update checking is disabled.
if software_update_check_period == 0:
return False
cur_ts = int(time.time())
if not os.path.isfile(LAST_CHECKED_FOR_GSUTIL_UPDATE_TIMESTAMP_FILE):
# Set last_checked_ts from date of VERSION file, so if the user installed
# an old copy of gsutil it will get noticed (and an update offered) the
# first time they try to run it.
last_checked_ts = int(os.path.getmtime(gslib.VERSION_FILE))
with open(LAST_CHECKED_FOR_GSUTIL_UPDATE_TIMESTAMP_FILE, 'w') as f:
f.write(str(last_checked_ts))
else:
with open(LAST_CHECKED_FOR_GSUTIL_UPDATE_TIMESTAMP_FILE, 'r') as f:
last_checked_ts = int(f.readline())
if (cur_ts - last_checked_ts >
software_update_check_period * SECONDS_PER_DAY):
suri_builder = StorageUriBuilder(debug,
self.bucket_storage_uri_class)
cur_ver = LookUpGsutilVersion(
suri_builder.StorageUri(GSUTIL_PUB_TARBALL))
with open(LAST_CHECKED_FOR_GSUTIL_UPDATE_TIMESTAMP_FILE, 'w') as f:
f.write(str(cur_ts))
if gslib.VERSION != cur_ver:
print '\n'.join(
textwrap.wrap(
'A newer version of gsutil (%s) is available than the version you '
'are running (%s). A detailed log of gsutil release changes is '
'available at gs://pub/gsutil_ReleaseNotes.txt if you would like '
'to read them before updating.' %
(cur_ver, gslib.VERSION),
width=78))
if gslib.IS_PACKAGE_INSTALL:
return False
print
answer = raw_input('Would you like to update [Y/n]? ')
return not answer or answer.lower()[0] != 'n'
return False
class NameExpansionHandler(object):
def __init__(self, command_name, proj_id_handler, headers, debug,
bucket_storage_uri_class):
"""
Args:
command_name: name of command being run.
proj_id_handler: ProjectIdHandler to use for current command.
headers: Dictionary containing optional HTTP headers to pass to boto.
debug: Debug level to pass in to boto connection (range 0..3).
bucket_storage_uri_class: Class to instantiate for cloud StorageUris.
Settable for testing/mocking.
"""
self.command_name = command_name
self.proj_id_handler = proj_id_handler
self.headers = headers
self.debug = debug
self.bucket_storage_uri_class = bucket_storage_uri_class
self.suri_builder = StorageUriBuilder(debug, bucket_storage_uri_class)
# Map holding wildcard strings to use for flat vs subdir-by-subdir listings.
# (A flat listing means show all objects expanded all the way down.)
self._flatness_wildcard = {True: '**', False: '*'}
def WildcardIterator(self, uri_or_str):
"""
Helper to instantiate gslib.WildcardIterator. Args are same as
gslib.WildcardIterator interface, but this method fills in most of the
values from class state.
Args:
uri_or_str: StorageUri or URI string naming wildcard objects to iterate.
"""
return wildcard_iterator.wildcard_iterator(
uri_or_str,
self.proj_id_handler,
bucket_storage_uri_class=self.bucket_storage_uri_class,
headers=self.headers,
debug=self.debug)
def ExpandWildcardsAndContainers(self,
uri_strs,
recursion_requested,
flat=True):
"""
Expands wildcards, object-less bucket names, subdir bucket names, and
directory names, producing a flat listing of all the matching objects/files.
Args:
uri_strs: List of URI strings needing expansion.
recursion_requested: True if -R specified on command-line.
flat: Bool indicating whether bucket listings should be flattened, i.e.,
so the mapped-to results contain objects spanning subdirectories.
Returns:
gslib.name_expansion.NameExpansionResult.
Raises:
CommandException: if errors encountered.
Examples with flat=True:
- Calling with one of the uri_strs being 'gs://bucket' will enumerate all
top-level objects, as will 'gs://bucket/' and 'gs://bucket/*'.
- 'gs://bucket/**' will enumerate all objects in the bucket.
- 'gs://bucket/abc' will enumerate all next-level objects under directory
abc (i.e., not including subdirectories of abc) if gs://bucket/abc/*
matches any objects; otherwise it will enumerate the single name
gs://bucket/abc
- 'gs://bucket/abc/**' will enumerate all objects under abc or any of its
subdirectories.
- 'file:///tmp' will enumerate all files under /tmp, as will
'file:///tmp/*'
- 'file:///tmp/**' will enumerate all files under /tmp or any of its
subdirectories.
Example if flat=False: calling with gs://bucket/abc/* lists matching objects
or subdirs, but not sub-subdirs or objects beneath subdirs.
Note: In step-by-step comments below we give examples assuming there's a
gs://bucket with object paths:
abcd/o1.txt
abcd/o2.txt
xyz/o1.txt
xyz/o2.txt
and a directory file://dir with file paths:
dir/a.txt
dir/b.txt
dir/c/
"""
result = NameExpansionResult()
for uri_str in uri_strs:
# Step 1: Expand any explicitly specified wildcards.
# Starting with gs://buck*/abc* this step would expand to gs://bucket/abcd
if ContainsWildcard(uri_str):
post_step1_bucket_listing_refs = list(
self.WildcardIterator(uri_str))
else:
post_step1_bucket_listing_refs = [
BucketListingRef(self.suri_builder.StorageUri(uri_str))
]
# Step 2: Expand subdirs.
# Starting with gs://bucket/abcd this step would expand to:
# [abcd/o1.txt, abcd/o2.txt].
uri_names_container = False
if flat:
if recursion_requested:
post_step2_bucket_listing_refs = []
for bucket_listing_ref in post_step1_bucket_listing_refs:
(uri_names_container, bucket_listing_refs) = (
self._DoImplicitBucketSubdirExpansionIfApplicable(
bucket_listing_ref.GetUri(), flat))
post_step2_bucket_listing_refs.extend(
bucket_listing_refs)
else:
uri_names_container = False
post_step2_bucket_listing_refs = post_step1_bucket_listing_refs
else:
uri_names_container = False
post_step2_bucket_listing_refs = post_step1_bucket_listing_refs
# Step 3. Expand directories and buckets.
# Starting with gs://bucket this step would expand to:
# [abcd/o1.txt, abcd/o2.txt, xyz/o1.txt, xyz/o2.txt]
# Starting with file://dir this step would expand to:
# [dir/a.txt, dir/b.txt, dir/c/]
exp_src_bucket_listing_refs = []
wc = self._flatness_wildcard[flat]
for bucket_listing_ref in post_step2_bucket_listing_refs:
if (not bucket_listing_ref.GetUri().names_container()
and (flat or not bucket_listing_ref.HasPrefix())):
exp_src_bucket_listing_refs.append(bucket_listing_ref)
continue
if not recursion_requested:
if bucket_listing_ref.GetUri().is_file_uri():
desc = 'directory'
else:
desc = 'bucket'
print 'Omitting %s "%s". (Did you mean to do %s -R?)' % (
desc, bucket_listing_ref.GetUri(), self.command_name)
continue
uri_names_container = True
if bucket_listing_ref.GetUri().is_file_uri():
# Convert dir to implicit recursive wildcard.
uri_to_iter = '%s/%s' % (bucket_listing_ref.GetUriString(),
wc)
else:
# Convert bucket to implicit recursive wildcard.
uri_to_iter = bucket_listing_ref.GetUri(
).clone_replace_name(wc)
wildcard_result = list(self.WildcardIterator(uri_to_iter))
if len(wildcard_result) > 0:
exp_src_bucket_listing_refs.extend(wildcard_result)
result._AddExpansion(self.suri_builder.StorageUri(uri_str),
uri_names_container,
exp_src_bucket_listing_refs)
return result
def _DoImplicitBucketSubdirExpansionIfApplicable(self, uri, flat):
"""
Checks whether uri could be an implicit bucket subdir, and expands if so;
else returns list containing uri. For example gs://abc would be an implicit
bucket subdir if the -R option was specified and gs://abc/* matches
anything.
Can only be called for -R (recursion requested).
Args:
uri: StorageUri.
flat: bool indicating whether bucket listings should be flattened, i.e.,
so the mapped-to results contain objects spanning subdirectories.
Returns:
tuple (names_container, [BucketListingRefs to which uri expanded])
where names_container is true if URI names a directory, bucket,
or bucket subdir (vs how StorageUri.names_container() doesn't
handle latter case).
"""
names_container = False
result_list = []
if uri.names_object():
# URI could be a bucket subdir.
implicit_subdir_matches = list(
self.WildcardIterator(
self.suri_builder.StorageUri(
'%s/%s' %
(uri.uri.rstrip('/'), self._flatness_wildcard[flat]))))
if len(implicit_subdir_matches) > 0:
names_container = True
result_list.extend(implicit_subdir_matches)
else:
result_list.append(BucketListingRef(uri))
else:
result_list.append(BucketListingRef(uri))
return (names_container, result_list)
def StorageUri(self, uri_str):
"""
Helper to instantiate boto.StorageUri with gsutil default flag values.
Uses self.bucket_storage_uri_class to support mocking/testing.
(Identical to the same-named function in command.py; that and this
copy make it convenient to call StorageUri() with a single argument,
from the respective classes.)
Args:
uri_str: StorageUri naming bucket + optional object.
Returns:
boto.StorageUri for given uri_str.
Raises:
InvalidUriError: if uri_str not valid.
"""
return gslib.util.StorageUri(uri_str, self.bucket_storage_uri_class,
self.debug)
def _MaybeCheckForAndOfferSoftwareUpdate(self, command_name, debug):
"""Checks the last time we checked for an update, and if it's been longer
than the configured threshold offers the user to update gsutil.
Args:
command_name: The name of the command being run.
debug: Debug level to pass in to boto connection (range 0..3).
Returns:
True if the user decides to update.
"""
# Don't try to interact with user if:
# - gsutil is not connected to a tty (e.g., if being run from cron);
# - user is running gsutil -q
# - user is running the config command (which could otherwise attempt to
# check for an update for a user running behind a proxy, who has not yet
# configured gsutil to go through the proxy; for such users we need the
# first connection attempt to be made by the gsutil config command).
# - user is running the version command (which gets run when using
# gsutil -D, which would prevent users with proxy config problems from
# sending us gsutil -D output).
# - user is running the update command (which could otherwise cause an
# additional note that an update is available when user is already trying
# to perform an update);
# - user specified gs_host (which could be a non-production different
# service instance, in which case credentials won't work for checking
# gsutil tarball).
gs_host = boto.config.get('Credentials', 'gs_host', None)
if (not IsRunningInteractively()
or command_name in ('config', 'update', 'ver', 'version')
or not logging.getLogger().isEnabledFor(logging.INFO)
or gs_host):
return False
software_update_check_period = boto.config.getint(
'GSUtil', 'software_update_check_period', 30)
# Setting software_update_check_period to 0 means periodic software
# update checking is disabled.
if software_update_check_period == 0:
return False
cur_ts = int(time.time())
if not os.path.isfile(LAST_CHECKED_FOR_GSUTIL_UPDATE_TIMESTAMP_FILE):
# Set last_checked_ts from date of VERSION file, so if the user installed
# an old copy of gsutil it will get noticed (and an update offered) the
# first time they try to run it.
last_checked_ts = GetGsutilVersionModifiedTime()
with open(LAST_CHECKED_FOR_GSUTIL_UPDATE_TIMESTAMP_FILE, 'w') as f:
f.write(str(last_checked_ts))
else:
try:
with open(LAST_CHECKED_FOR_GSUTIL_UPDATE_TIMESTAMP_FILE,
'r') as f:
last_checked_ts = int(f.readline())
except (TypeError, ValueError):
return False
if (cur_ts - last_checked_ts >
software_update_check_period * SECONDS_PER_DAY):
suri_builder = StorageUriBuilder(debug,
self.bucket_storage_uri_class)
cur_ver = LookUpGsutilVersion(
suri_builder.StorageUri(GSUTIL_PUB_TARBALL))
with open(LAST_CHECKED_FOR_GSUTIL_UPDATE_TIMESTAMP_FILE, 'w') as f:
f.write(str(cur_ts))
(g, m) = CompareVersions(cur_ver, gslib.VERSION)
if m:
print '\n'.join(
textwrap.wrap(
'A newer version of gsutil (%s) is available than the version you '
'are running (%s). NOTE: This is a major new version, so it is '
'strongly recommended that you review the release note details at %s '
'before updating to this version, especially if you use gsutil in '
'scripts.' %
(cur_ver, gslib.VERSION, RELEASE_NOTES_URL)))
if gslib.IS_PACKAGE_INSTALL:
return False
print
answer = raw_input('Would you like to update [y/N]? ')
return answer and answer.lower()[0] == 'y'
elif g:
print '\n'.join(
textwrap.wrap(
'A newer version of gsutil (%s) is available than the version you '
'are running (%s). A detailed log of gsutil release changes is '
'available at %s if you would like to read them before updating.'
% (cur_ver, gslib.VERSION, RELEASE_NOTES_URL)))
if gslib.IS_PACKAGE_INSTALL:
return False
print
answer = raw_input('Would you like to update [Y/n]? ')
return not answer or answer.lower()[0] != 'n'
return False