def setUp(self):
super(GsUtilIntegrationTestCase, self).setUp()
self.bucket_uris = []
# Set up API version and project ID handler.
self.api_version = boto.config.get_value(
'GSUtil', 'default_api_version', '1')
self.proj_id_handler = ProjectIdHandler()
def setUpClass(cls):
base.GsUtilTestCase.setUpClass()
cls.mock_bucket_storage_uri = GSMockBucketStorageUri
cls.proj_id_handler = ProjectIdHandler()
config_file_list = boto.pyami.config.BotoConfigLocations
# Use "gsutil_test_commands" as a fake UserAgent. This value will never be
# sent via HTTP because we're using MockStorageService here.
cls.command_runner = CommandRunner(config_file_list,
cls.mock_bucket_storage_uri)
def testCallingGetKeyOnProviderOnlyWildcardIteration(self):
"""Tests that attempting iterating provider-only wildcard raises"""
try:
from gslib.bucket_listing_ref import BucketListingRefException
for iter_result in wildcard_iterator.wildcard_iterator(
'gs://',
ProjectIdHandler(),
bucket_storage_uri_class=self.mock_bucket_storage_uri):
iter_result.GetKey()
self.fail('Expected BucketListingRefException not raised.')
except BucketListingRefException, e:
self.assertTrue(
str(e).find(
'Attempt to call GetKey() on Key-less BucketListingRef') !=
-1)
class Command(object):
REQUIRED_SPEC_KEYS = [COMMAND_NAME]
# Each subclass must define the following map, minimally including the
# keys in REQUIRED_SPEC_KEYS; other values below will be used as defaults,
# although for readbility subclasses should specify the complete map.
command_spec = {
# Name of command.
COMMAND_NAME: None,
# List of command name aliases.
COMMAND_NAME_ALIASES: [],
# Min number of args required by this command.
MIN_ARGS: 0,
# Max number of args required by this command, or NO_MAX.
MAX_ARGS: NO_MAX,
# Getopt-style string specifying acceptable sub args.
SUPPORTED_SUB_ARGS: '',
# True if file URIs are acceptable for this command.
FILE_URIS_OK: False,
# True if provider-only URIs are acceptable for this command.
PROVIDER_URIS_OK: False,
# Index in args of first URI arg.
URIS_START_ARG: 0,
# True if must configure gsutil before running command.
CONFIG_REQUIRED: True,
}
_default_command_spec = command_spec
help_spec = HelpProvider.help_spec
"""Define an empty test specification, which derived classes must populate.
This is a list of tuples containing the following values:
step_name - mnemonic name for test, displayed when test is run
cmd_line - shell command line to run test
expect_ret or None - expected return code from test (None means ignore)
(result_file, expect_file) or None - tuple of result file and expected
file to diff for additional test
verification beyond the return code
(None means no diff requested)
Notes:
- Setting expected_ret to None means there is no expectation and,
hence, any returned value will pass.
- Any occurrences of the string 'gsutil' in the cmd_line parameter
are expanded to the full path to the gsutil command under test.
- The cmd_line, result_file and expect_file parameters may
contain the following special substrings:
$Bn - converted to one of 10 unique-for-testing bucket names (n=0..9)
$On - converted to one of 10 unique-for-testing object names (n=0..9)
$Fn - converted to one of 10 unique-for-testing file names (n=0..9)
$G - converted to the directory where gsutil is installed. Useful for
referencing test data.
- The generated file names are full pathnames, whereas the generated
bucket and object names are simple relative names.
- Tests with a non-None result_file and expect_file automatically
trigger an implicit diff of the two files.
- These test specifications, in combination with the conversion strings
allow tests to be constructed parametrically. For example, here's an
annotated subset of a test_steps for the cp command:
# Copy local file to object, verify 0 return code.
('simple cp', 'gsutil cp $F1 gs://$B1/$O1', 0, None, None),
# Copy uploaded object back to local file and diff vs. orig file.
('verify cp', 'gsutil cp gs://$B1/$O1 $F2', 0, '$F2', '$F1'),
- After pattern substitution, the specs are run sequentially, in the
order in which they appear in the test_steps list.
"""
test_steps = []
# Define a convenience property for command name, since it's used many places.
def _GetDefaultCommandName(self):
return self.command_spec[COMMAND_NAME]
command_name = property(_GetDefaultCommandName)
def __init__(self,
command_runner,
args,
headers,
debug,
parallel_operations,
config_file_list,
bucket_storage_uri_class,
test_method=None,
logging_filters=None):
"""
Args:
command_runner: CommandRunner (for commands built atop other commands).
args: Command-line args (arg0 = actual arg, not command name ala bash).
headers: Dictionary containing optional HTTP headers to pass to boto.
debug: Debug level to pass in to boto connection (range 0..3).
parallel_operations: Should command operations be executed in parallel?
config_file_list: Config file list returned by GetBotoConfigFileList().
bucket_storage_uri_class: Class to instantiate for cloud StorageUris.
Settable for testing/mocking.
test_method: Optional general purpose method for testing purposes.
Application and semantics of this method will vary by
command and test type.
logging_filters: Optional list of logging.Filters to apply to this
command's logger.
Implementation note: subclasses shouldn't need to define an __init__
method, and instead depend on the shared initialization that happens
here. If you do define an __init__ method in a subclass you'll need to
explicitly call super().__init__(). But you're encouraged not to do this,
because it will make changing the __init__ interface more painful.
"""
# Save class values from constructor params.
self.command_runner = command_runner
self.args = args
self.unparsed_args = args
self.headers = headers
self.debug = debug
self.parallel_operations = parallel_operations
self.config_file_list = config_file_list
self.bucket_storage_uri_class = bucket_storage_uri_class
self.test_method = test_method
self.exclude_symlinks = False
self.recursion_requested = False
self.all_versions = False
# Global instance of a threaded logger object.
self.logger = _ThreadedLogger(self.command_name)
if logging_filters:
for filter in logging_filters:
self.logger.addFilter(filter)
# Process sub-command instance specifications.
# First, ensure subclass implementation sets all required keys.
for k in self.REQUIRED_SPEC_KEYS:
if k not in self.command_spec or self.command_spec[k] is None:
raise CommandException(
'"%s" command implementation is missing %s '
'specification' % (self.command_name, k))
# Now override default command_spec with subclass-specified values.
tmp = self._default_command_spec
tmp.update(self.command_spec)
self.command_spec = tmp
del tmp
# Make sure command provides a test specification.
if not self.test_steps:
# TODO: Uncomment following lines when test feature is ready.
#raise CommandException('"%s" command implementation is missing test '
#'specification' % self.command_name)
pass
# Parse and validate args.
try:
(self.sub_opts,
self.args) = getopt.getopt(args,
self.command_spec[SUPPORTED_SUB_ARGS])
except GetoptError, e:
raise CommandException('%s for "%s" command.' %
(e.msg, self.command_name))
if (len(self.args) < self.command_spec[MIN_ARGS]
or len(self.args) > self.command_spec[MAX_ARGS]):
raise CommandException(
'Wrong number of arguments for "%s" command.' %
self.command_name)
if (not self.command_spec[FILE_URIS_OK] and self.HaveFileUris(
self.args[self.command_spec[URIS_START_ARG]:])):
raise CommandException(
'"%s" command does not support "file://" URIs. '
'Did you mean to use a gs:// URI?' % self.command_name)
if (not self.command_spec[PROVIDER_URIS_OK] and self._HaveProviderUris(
self.args[self.command_spec[URIS_START_ARG]:])):
raise CommandException(
'"%s" command does not support provider-only '
'URIs.' % self.command_name)
if self.command_spec[CONFIG_REQUIRED]:
self._ConfigureNoOpAuthIfNeeded()
self.proj_id_handler = ProjectIdHandler()
self.suri_builder = StorageUriBuilder(debug, bucket_storage_uri_class)
# Cross-platform path to run gsutil binary.
self.gsutil_cmd = ''
# Cross-platform list containing gsutil path for use with subprocess.
self.gsutil_exec_list = []
# If running on Windows, invoke python interpreter explicitly.
if gslib.util.IS_WINDOWS:
self.gsutil_cmd += 'python '
self.gsutil_exec_list += ['python']
# Add full path to gsutil to make sure we test the correct version.
self.gsutil_path = gslib.GSUTIL_PATH
self.gsutil_cmd += self.gsutil_path
self.gsutil_exec_list += [self.gsutil_path]
# We're treating recursion_requested like it's used by all commands, but
# only some of the commands accept the -R option.
if self.sub_opts:
for o, unused_a in self.sub_opts:
if o == '-r' or o == '-R':
self.recursion_requested = True
break
class GsUtilIntegrationTestCase(base.GsUtilTestCase):
"""Base class for gsutil integration tests."""
GROUP_TEST_ADDRESS = '*****@*****.**'
GROUP_TEST_ID = '00b4903a97d097895ab58ef505d535916a712215b79c3e54932c2eb502ad97f5'
USER_TEST_ADDRESS = '*****@*****.**'
USER_TEST_ID = '00b4903a9703325c6bfc98992d72e75600387a64b3b6bee9ef74613ef8842080'
DOMAIN_TEST = 'google.com'
def setUp(self):
super(GsUtilIntegrationTestCase, self).setUp()
self.bucket_uris = []
# Set up API version and project ID handler.
self.api_version = boto.config.get_value(
'GSUtil', 'default_api_version', '1')
self.proj_id_handler = ProjectIdHandler()
# Retry with an exponential backoff if a server error is received. This
# ensures that we try *really* hard to clean up after ourselves.
@Retry(GSResponseError, logger=LOGGER)
def tearDown(self):
super(GsUtilIntegrationTestCase, self).tearDown()
while self.bucket_uris:
bucket_uri = self.bucket_uris[-1]
bucket_list = list(bucket_uri.list_bucket(all_versions=True))
while bucket_list:
for k in bucket_list:
k.delete()
bucket_list = list(bucket_uri.list_bucket(all_versions=True))
bucket_uri.delete_bucket()
self.bucket_uris.pop()
def CreateBucket(self, bucket_name=None, test_objects=0, storage_class=None):
"""Creates a test bucket.
The bucket and all of its contents will be deleted after the test.
Args:
bucket_name: Create the bucket with this name. If not provided, a
temporary test bucket name is constructed.
test_objects: The number of objects that should be placed in the bucket.
Defaults to 0.
storage_class: storage class to use. If not provided we us standard.
Returns:
StorageUri for the created bucket.
"""
bucket_name = bucket_name or self.MakeTempName('bucket')
bucket_uri = boto.storage_uri('gs://%s' % bucket_name.lower(),
suppress_consec_slashes=False)
# Apply API version and project ID headers if necessary.
headers = {'x-goog-api-version': self.api_version}
self.proj_id_handler.FillInProjectHeaderIfNeeded(
'test', bucket_uri, headers)
bucket_uri.create_bucket(storage_class=storage_class, headers=headers)
self.bucket_uris.append(bucket_uri)
for i in range(test_objects):
self.CreateObject(bucket_uri=bucket_uri,
object_name=self.MakeTempName('obj'),
contents='test %d' % i)
return bucket_uri
def CreateVersionedBucket(self, bucket_name=None, test_objects=0):
"""Creates a versioned test bucket.
The bucket and all of its contents will be deleted after the test.
Args:
bucket_name: Create the bucket with this name. If not provided, a
temporary test bucket name is constructed.
test_objects: The number of objects that should be placed in the bucket.
Defaults to 0.
Returns:
StorageUri for the created bucket with versioning enabled.
"""
bucket_uri = self.CreateBucket(bucket_name=bucket_name,
test_objects=test_objects)
bucket_uri.configure_versioning(True)
return bucket_uri
def CreateObject(self, bucket_uri=None, object_name=None, contents=None):
"""Creates a test object.
Args:
bucket: The URI of the bucket to place the object in. If not specified, a
new temporary bucket is created.
object_name: The name to use for the object. If not specified, a temporary
test object name is constructed.
contents: The contents to write to the object. If not specified, the key
is not written to, which means that it isn't actually created
yet on the server.
Returns:
A StorageUri for the created object.
"""
bucket_uri = bucket_uri or self.CreateBucket()
object_name = object_name or self.MakeTempName('obj')
key_uri = bucket_uri.clone_replace_name(object_name)
if contents is not None:
key_uri.set_contents_from_string(contents)
return key_uri
def RunGsUtil(self, cmd, return_status=False, return_stdout=False,
return_stderr=False, expected_status=0, stdin=None):
"""Runs the gsutil command.
Args:
cmd: The command to run, as a list, e.g. ['cp', 'foo', 'bar']
return_status: If True, the exit status code is returned.
return_stdout: If True, the standard output of the command is returned.
return_stderr: If True, the standard error of the command is returned.
expected_status: The expected return code. If not specified, defaults to
0. If the return code is a different value, an exception
is raised.
stdin: A string of data to pipe to the process as standard input.
Returns:
A tuple containing the desired return values specified by the return_*
arguments.
"""
cmd = [GSUTIL_PATH] + cmd
if IS_WINDOWS:
cmd = [sys.executable] + cmd
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
stdin=subprocess.PIPE)
(stdout, stderr) = p.communicate(stdin)
status = p.returncode
if expected_status is not None:
self.assertEqual(
status, expected_status,
msg='Expected status %d, got %d.\nCommand:\n%s\n\nstderr:\n%s' % (
expected_status, status, ' '.join(cmd), stderr))
toreturn = []
if return_status:
toreturn.append(status)
if return_stdout:
if IS_WINDOWS:
stdout = stdout.replace('\r\n', '\n')
toreturn.append(stdout)
if return_stderr:
if IS_WINDOWS:
stderr = stderr.replace('\r\n', '\n')
toreturn.append(stderr)
if len(toreturn) == 1:
return toreturn[0]
elif toreturn:
return tuple(toreturn)
class Command(object):
# Global instance of a threaded logger object.
THREADED_LOGGER = _ThreadedLogger()
REQUIRED_SPEC_KEYS = [COMMAND_NAME]
# Each subclass must define the following map, minimally including the
# keys in REQUIRED_SPEC_KEYS; other values below will be used as defaults,
# although for readbility subclasses should specify the complete map.
command_spec = {
# Name of command.
COMMAND_NAME: None,
# List of command name aliases.
COMMAND_NAME_ALIASES: [],
# Min number of args required by this command.
MIN_ARGS: 0,
# Max number of args required by this command, or NO_MAX.
MAX_ARGS: NO_MAX,
# Getopt-style string specifying acceptable sub args.
SUPPORTED_SUB_ARGS: '',
# True if file URIs are acceptable for this command.
FILE_URIS_OK: False,
# True if provider-only URIs are acceptable for this command.
PROVIDER_URIS_OK: False,
# Index in args of first URI arg.
URIS_START_ARG: 0,
# True if must configure gsutil before running command.
CONFIG_REQUIRED: True,
}
_default_command_spec = command_spec
# Define a convenience property for command name, since it's used many places.
def _get_command_name(self):
return self.command_spec[COMMAND_NAME]
command_name = property(_get_command_name)
def __init__(self,
command_runner,
args,
headers,
debug,
parallel_operations,
gsutil_bin_dir,
boto_lib_dir,
config_file_list,
bucket_storage_uri_class,
test_method=None):
"""
Args:
command_runner: CommandRunner (for commands built atop other commands).
args: command-line args (arg0 = actual arg, not command name ala bash).
headers: dictionary containing optional HTTP headers to pass to boto.
debug: debug level to pass in to boto connection (range 0..3).
parallel_operations: Should command operations be executed in parallel?
gsutil_bin_dir: bin dir from which gsutil is running.
boto_lib_dir: lib dir where boto runs.
config_file_list: config file list returned by _GetBotoConfigFileList().
bucket_storage_uri_class: Class to instantiate for cloud StorageUris.
Settable for testing/mocking.
test_method: Optional general purpose method for testing purposes.
Application and semantics of this method will vary by
command and test type.
"""
# Save class values from constructor params.
self.command_runner = command_runner
self.args = args
self.headers = headers
self.debug = debug
self.parallel_operations = parallel_operations
self.gsutil_bin_dir = gsutil_bin_dir
self.boto_lib_dir = boto_lib_dir
self.config_file_list = config_file_list
self.bucket_storage_uri_class = bucket_storage_uri_class
self.test_method = test_method
self.ignore_symlinks = False
# Process sub-command instance specifications.
# First, ensure subclass implementation sets all required keys.
for k in self.REQUIRED_SPEC_KEYS:
if k not in self.command_spec or self.command_spec[k] is None:
raise CommandException(
'"%s" command implementation is missing %s '
'specification' % (self.command_name, k))
# Now override default command_spec with subclass-specified values.
tmp = self._default_command_spec
tmp.update(self.command_spec)
self.command_spec = tmp
del tmp
# Parse and validate args.
try:
(self.sub_opts,
self.args) = getopt.getopt(args,
self.command_spec[SUPPORTED_SUB_ARGS])
except GetoptError, e:
raise CommandException('%s for "%s" command.' %
(e.msg, self.command_name))
if (len(self.args) < self.command_spec[MIN_ARGS]
or len(self.args) > self.command_spec[MAX_ARGS]):
raise CommandException(
'Wrong number of arguments for "%s" command.' %
self.command_name)
if (not self.command_spec[FILE_URIS_OK] and self._HaveFileUris(
self.args[self.command_spec[URIS_START_ARG]:])):
raise CommandException(
'"%s" command does not support "file://" URIs. '
'Did you mean to use a gs:// URI?' % self.command_name)
if (not self.command_spec[PROVIDER_URIS_OK] and self._HaveProviderUris(
self.args[self.command_spec[URIS_START_ARG]:])):
raise CommandException(
'"%s" command does not support provider-only '
'URIs.' % self.command_name)
if self.command_spec[CONFIG_REQUIRED]:
self._ConfigureNoOpAuthIfNeeded()
self.proj_id_handler = ProjectIdHandler()
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
"""Utility class for gslib unit tests"""
import sys
import wildcard_iterator
# Put local libs at front of path so tests will run latest lib code rather
# than whatever code is found on user's PYTHONPATH.
sys.path.insert(0, '.')
sys.path.insert(0, 'boto')
import boto
from gslib.project_id import ProjectIdHandler
from tests.s3 import mock_storage_service
proj_id_handler = ProjectIdHandler()
def test_wildcard_iterator(uri_or_str,
result_type=wildcard_iterator.ResultType.URIS,
debug=0):
"""
Convenience method for instantiating a testing
instance of WildCardIterator, without having to specify
all the params of that class (like
bucket_storage_uri_class=mock_storage_service.MockBucketStorageUri).
Also naming the factory method this way makes it clearer in the test
code that WildcardIterator needs to be set up for testing.
Args, Returns, and Raises are same as for
wildcard_iterator.wildcard_iterator(), except there's no