def _ConfigureLogging(level=logging.INFO): """Similar to logging.basicConfig() except it always adds a handler.""" log_format = '%(levelname)s %(asctime)s %(filename)s] %(message)s' date_format = '%m%d %H:%M:%S.%f' formatter = GsutilFormatter(fmt=log_format, datefmt=date_format) handler = GsutilStreamHandler() handler.setFormatter(formatter) root_logger = logging.getLogger() root_logger.addHandler(handler) root_logger.setLevel(level)
def _ConfigureLogging(level=logging.INFO): """Similar to logging.basicConfig() except it always adds a handler.""" log_format = '%(levelname)s %(asctime)s %(filename)s] %(message)s' date_format = '%m%d %H:%M:%S.%f' formatter = GsutilFormatter(fmt=log_format, datefmt=date_format) handler = GsutilStreamHandler() handler.setFormatter(formatter) root_logger = logging.getLogger() root_logger.addHandler(handler) root_logger.setLevel(level)
def setUp(self):
super(GsUtilUnitTestCase, self).setUp()
self.bucket_uris = []
self.stdout_save = sys.stdout
self.stderr_save = sys.stderr
fd, self.stdout_file = tempfile.mkstemp()
sys.stdout = os.fdopen(fd, 'w+')
fd, self.stderr_file = tempfile.mkstemp()
sys.stderr = os.fdopen(fd, 'w+')
self.accumulated_stdout = []
self.accumulated_stderr = []
self.root_logger = logging.getLogger()
self.is_debugging = self.root_logger.isEnabledFor(logging.DEBUG)
self.log_handlers_save = self.root_logger.handlers
fd, self.log_handler_file = tempfile.mkstemp()
self.log_handler_stream = os.fdopen(fd, 'w+')
self.temp_log_handler = GsutilStreamHandler(self.log_handler_stream)
self.root_logger.handlers = [self.temp_log_handler]
def setUp(self):
super(GsUtilUnitTestCase, self).setUp()
self.bucket_uris = []
self.stdout_save = sys.stdout
self.stderr_save = sys.stderr
fd, self.stdout_file = tempfile.mkstemp()
sys.stdout = os.fdopen(fd, 'w+')
fd, self.stderr_file = tempfile.mkstemp()
sys.stderr = os.fdopen(fd, 'w+')
self.accumulated_stdout = []
self.accumulated_stderr = []
self.root_logger = logging.getLogger()
self.is_debugging = self.root_logger.isEnabledFor(logging.DEBUG)
self.log_handlers_save = self.root_logger.handlers
fd, self.log_handler_file = tempfile.mkstemp()
self.log_handler_stream = os.fdopen(fd, 'w+')
self.temp_log_handler = GsutilStreamHandler(self.log_handler_stream)
self.root_logger.handlers = [self.temp_log_handler]
class GsUtilUnitTestCase(base.GsUtilTestCase):
"""Base class for gsutil unit tests."""
@classmethod
def setUpClass(cls):
base.GsUtilTestCase.setUpClass()
cls.mock_bucket_storage_uri = util.GSMockBucketStorageUri
cls.mock_gsutil_api_class_map_factory = GsutilApiUnitTestClassMapFactory
cls.logger = logging.getLogger()
cls.command_runner = CommandRunner(
bucket_storage_uri_class=cls.mock_bucket_storage_uri,
gsutil_api_class_map_factory=cls.mock_gsutil_api_class_map_factory)
def setUp(self):
super(GsUtilUnitTestCase, self).setUp()
self.bucket_uris = []
self.stdout_save = sys.stdout
self.stderr_save = sys.stderr
fd, self.stdout_file = tempfile.mkstemp()
sys.stdout = os.fdopen(fd, 'w+')
fd, self.stderr_file = tempfile.mkstemp()
sys.stderr = os.fdopen(fd, 'w+')
self.accumulated_stdout = []
self.accumulated_stderr = []
self.root_logger = logging.getLogger()
self.is_debugging = self.root_logger.isEnabledFor(logging.DEBUG)
self.log_handlers_save = self.root_logger.handlers
fd, self.log_handler_file = tempfile.mkstemp()
self.log_handler_stream = os.fdopen(fd, 'w+')
self.temp_log_handler = GsutilStreamHandler(self.log_handler_stream)
self.root_logger.handlers = [self.temp_log_handler]
def tearDown(self):
super(GsUtilUnitTestCase, self).tearDown()
self.root_logger.handlers = self.log_handlers_save
self.temp_log_handler.flush()
self.temp_log_handler.close()
self.log_handler_stream.seek(0)
log_output = self.log_handler_stream.read()
self.log_handler_stream.close()
os.unlink(self.log_handler_file)
sys.stdout.seek(0)
sys.stderr.seek(0)
stdout = sys.stdout.read()
stderr = sys.stderr.read()
stdout += ''.join(self.accumulated_stdout)
stderr += ''.join(self.accumulated_stderr)
sys.stdout.close()
sys.stderr.close()
sys.stdout = self.stdout_save
sys.stderr = self.stderr_save
os.unlink(self.stdout_file)
os.unlink(self.stderr_file)
if self.is_debugging and stdout:
sys.stderr.write('==== stdout %s ====\n' % self.id())
sys.stderr.write(stdout)
sys.stderr.write('==== end stdout ====\n')
if self.is_debugging and stderr:
sys.stderr.write('==== stderr %s ====\n' % self.id())
sys.stderr.write(stderr)
sys.stderr.write('==== end stderr ====\n')
if self.is_debugging and log_output:
sys.stderr.write('==== log output %s ====\n' % self.id())
sys.stderr.write(log_output)
sys.stderr.write('==== end log output ====\n')
def RunCommand(self, command_name, args=None, headers=None, debug=0,
return_stdout=False, return_stderr=False,
return_log_handler=False, cwd=None):
"""Method for calling gslib.command_runner.CommandRunner.
Passes parallel_operations=False for all tests, optionally saving/returning
stdout output. We run all tests multi-threaded, to exercise those more
complicated code paths.
TODO: Change to run with parallel_operations=True for all tests. At
present when you do this it causes many test failures.
Args:
command_name: The name of the command being run.
args: Command-line args (arg0 = actual arg, not command name ala bash).
headers: Dictionary containing optional HTTP headers to pass to boto.
debug: Debug level to pass in to boto connection (range 0..3).
return_stdout: If True, will save and return stdout produced by command.
return_stderr: If True, will save and return stderr produced by command.
return_log_handler: If True, will return a MockLoggingHandler instance
that was attached to the command's logger while running.
cwd: The working directory that should be switched to before running the
command. The working directory will be reset back to its original
value after running the command. If not specified, the working
directory is left unchanged.
Returns:
One or a tuple of requested return values, depending on whether
return_stdout, return_stderr, and/or return_log_handler were specified.
"""
args = args or []
command_line = ' '.join([command_name] + args)
if self.is_debugging:
self.stderr_save.write('\nRunCommand of %s\n' % command_line)
# Save and truncate stdout and stderr for the lifetime of RunCommand. This
# way, we can return just the stdout and stderr that was output during the
# RunNamedCommand call below.
sys.stdout.seek(0)
sys.stderr.seek(0)
stdout = sys.stdout.read()
stderr = sys.stderr.read()
if stdout:
self.accumulated_stdout.append(stdout)
if stderr:
self.accumulated_stderr.append(stderr)
sys.stdout.seek(0)
sys.stderr.seek(0)
sys.stdout.truncate()
sys.stderr.truncate()
mock_log_handler = MockLoggingHandler()
logging.getLogger(command_name).addHandler(mock_log_handler)
try:
with WorkingDirectory(cwd):
self.command_runner.RunNamedCommand(
command_name, args=args, headers=headers, debug=debug,
parallel_operations=False, do_shutdown=False)
finally:
sys.stdout.seek(0)
stdout = sys.stdout.read()
sys.stderr.seek(0)
stderr = sys.stderr.read()
logging.getLogger(command_name).removeHandler(mock_log_handler)
mock_log_handler.close()
log_output = '\n'.join(
'%s:\n ' % level + '\n '.join(records)
for level, records in mock_log_handler.messages.iteritems()
if records)
if self.is_debugging and log_output:
self.stderr_save.write(
'==== logging RunCommand %s %s ====\n' % (self.id(), command_line))
self.stderr_save.write(log_output)
self.stderr_save.write('\n==== end logging ====\n')
if self.is_debugging and stdout:
self.stderr_save.write(
'==== stdout RunCommand %s %s ====\n' % (self.id(), command_line))
self.stderr_save.write(stdout)
self.stderr_save.write('==== end stdout ====\n')
if self.is_debugging and stderr:
self.stderr_save.write(
'==== stderr RunCommand %s %s ====\n' % (self.id(), command_line))
self.stderr_save.write(stderr)
self.stderr_save.write('==== end stderr ====\n')
# Reset stdout and stderr files, so that we won't print them out again
# in tearDown if debugging is enabled.
sys.stdout.seek(0)
sys.stderr.seek(0)
sys.stdout.truncate()
sys.stderr.truncate()
to_return = []
if return_stdout:
to_return.append(stdout)
if return_stderr:
to_return.append(stderr)
if return_log_handler:
to_return.append(mock_log_handler)
if len(to_return) == 1:
return to_return[0]
return tuple(to_return)
@classmethod
def MakeGsUtilApi(cls, debug=0):
gsutil_api_map = {
ApiMapConstants.API_MAP: (
cls.mock_gsutil_api_class_map_factory.GetClassMap()),
ApiMapConstants.SUPPORT_MAP: {
'gs': [ApiSelector.XML, ApiSelector.JSON],
's3': [ApiSelector.XML]
},
ApiMapConstants.DEFAULT_MAP: {
'gs': ApiSelector.JSON,
's3': ApiSelector.XML
}
}
return CloudApiDelegator(
cls.mock_bucket_storage_uri, gsutil_api_map, cls.logger,
DiscardMessagesQueue(), debug=debug)
@classmethod
def _test_wildcard_iterator(cls, uri_or_str, debug=0):
"""Convenience method for instantiating a test instance of WildcardIterator.
This makes it unnecessary to specify all the params of that class
(like bucket_storage_uri_class=mock_storage_service.MockBucketStorageUri).
Also, naming the factory method this way makes it clearer in the test code
that WildcardIterator needs to be set up for testing.
Args are same as for wildcard_iterator.wildcard_iterator(), except
there are no class args for bucket_storage_uri_class or gsutil_api_class.
Args:
uri_or_str: StorageUri or string representing the wildcard string.
debug: debug level to pass to the underlying connection (0..3)
Returns:
WildcardIterator, over which caller can iterate.
"""
# TODO: Remove when tests no longer pass StorageUri arguments.
uri_string = uri_or_str
if hasattr(uri_or_str, 'uri'):
uri_string = uri_or_str.uri
return wildcard_iterator.CreateWildcardIterator(
uri_string, cls.MakeGsUtilApi())
@staticmethod
def _test_storage_uri(uri_str, default_scheme='file', debug=0,
validate=True):
"""Convenience method for instantiating a testing instance of StorageUri.
This makes it unnecessary to specify
bucket_storage_uri_class=mock_storage_service.MockBucketStorageUri.
Also naming the factory method this way makes it clearer in the test
code that StorageUri needs to be set up for testing.
Args, Returns, and Raises are same as for boto.storage_uri(), except there's
no bucket_storage_uri_class arg.
Args:
uri_str: Uri string to create StorageUri for.
default_scheme: Default scheme for the StorageUri
debug: debug level to pass to the underlying connection (0..3)
validate: If True, validate the resource that the StorageUri refers to.
Returns:
StorageUri based on the arguments.
"""
return boto.storage_uri(uri_str, default_scheme, debug, validate,
util.GSMockBucketStorageUri)
def CreateBucket(self, bucket_name=None, test_objects=0, storage_class=None,
provider='gs'):
"""Creates a test bucket.
The bucket and all of its contents will be deleted after the test.
Args:
bucket_name: Create the bucket with this name. If not provided, a
temporary test bucket name is constructed.
test_objects: The number of objects that should be placed in the bucket or
a list of object names to place in the bucket. Defaults to
0.
storage_class: storage class to use. If not provided we us standard.
provider: string provider to use, default gs.
Returns:
StorageUri for the created bucket.
"""
bucket_name = bucket_name or self.MakeTempName('bucket')
bucket_uri = boto.storage_uri(
'%s://%s' % (provider, bucket_name.lower()),
suppress_consec_slashes=False,
bucket_storage_uri_class=util.GSMockBucketStorageUri)
bucket_uri.create_bucket(storage_class=storage_class)
self.bucket_uris.append(bucket_uri)
try:
iter(test_objects)
except TypeError:
test_objects = [self.MakeTempName('obj') for _ in range(test_objects)]
for i, name in enumerate(test_objects):
self.CreateObject(bucket_uri=bucket_uri, object_name=name,
contents='test %d' % i)
return bucket_uri
def CreateObject(self, bucket_uri=None, object_name=None, contents=None):
"""Creates a test object.
Args:
bucket_uri: The URI of the bucket to place the object in. If not
specified, a new temporary bucket is created.
object_name: The name to use for the object. If not specified, a temporary
test object name is constructed.
contents: The contents to write to the object. If not specified, the key
is not written to, which means that it isn't actually created
yet on the server.
Returns:
A StorageUri for the created object.
"""
bucket_uri = bucket_uri or self.CreateBucket(provider=self.default_provider)
object_name = object_name or self.MakeTempName('obj')
key_uri = bucket_uri.clone_replace_name(object_name)
if contents is not None:
key_uri.set_contents_from_string(contents)
return key_uri
class GsUtilUnitTestCase(base.GsUtilTestCase):
"""Base class for gsutil unit tests."""
@classmethod
def setUpClass(cls):
base.GsUtilTestCase.setUpClass()
cls.mock_bucket_storage_uri = util.GSMockBucketStorageUri
cls.mock_gsutil_api_class_map_factory = GsutilApiUnitTestClassMapFactory
cls.logger = logging.getLogger()
cls.command_runner = CommandRunner(
bucket_storage_uri_class=cls.mock_bucket_storage_uri,
gsutil_api_class_map_factory=cls.mock_gsutil_api_class_map_factory)
def setUp(self):
super(GsUtilUnitTestCase, self).setUp()
self.bucket_uris = []
self.stdout_save = sys.stdout
self.stderr_save = sys.stderr
fd, self.stdout_file = tempfile.mkstemp()
sys.stdout = os.fdopen(fd, 'w+')
fd, self.stderr_file = tempfile.mkstemp()
sys.stderr = os.fdopen(fd, 'w+')
self.accumulated_stdout = []
self.accumulated_stderr = []
self.root_logger = logging.getLogger()
self.is_debugging = self.root_logger.isEnabledFor(logging.DEBUG)
self.log_handlers_save = self.root_logger.handlers
fd, self.log_handler_file = tempfile.mkstemp()
self.log_handler_stream = os.fdopen(fd, 'w+')
self.temp_log_handler = GsutilStreamHandler(self.log_handler_stream)
self.root_logger.handlers = [self.temp_log_handler]
def tearDown(self):
super(GsUtilUnitTestCase, self).tearDown()
self.root_logger.handlers = self.log_handlers_save
self.temp_log_handler.flush()
self.temp_log_handler.close()
self.log_handler_stream.seek(0)
log_output = self.log_handler_stream.read()
self.log_handler_stream.close()
os.unlink(self.log_handler_file)
sys.stdout.seek(0)
sys.stderr.seek(0)
stdout = sys.stdout.read()
stderr = sys.stderr.read()
stdout += ''.join(self.accumulated_stdout)
stderr += ''.join(self.accumulated_stderr)
sys.stdout.close()
sys.stderr.close()
sys.stdout = self.stdout_save
sys.stderr = self.stderr_save
os.unlink(self.stdout_file)
os.unlink(self.stderr_file)
if self.is_debugging and stdout:
sys.stderr.write('==== stdout %s ====\n' % self.id())
sys.stderr.write(stdout)
sys.stderr.write('==== end stdout ====\n')
if self.is_debugging and stderr:
sys.stderr.write('==== stderr %s ====\n' % self.id())
sys.stderr.write(stderr)
sys.stderr.write('==== end stderr ====\n')
if self.is_debugging and log_output:
sys.stderr.write('==== log output %s ====\n' % self.id())
sys.stderr.write(log_output)
sys.stderr.write('==== end log output ====\n')
def RunCommand(self, command_name, args=None, headers=None, debug=0,
return_stdout=False, return_stderr=False,
return_log_handler=False, cwd=None):
"""Method for calling gslib.command_runner.CommandRunner.
Passes parallel_operations=False for all tests, optionally saving/returning
stdout output. We run all tests multi-threaded, to exercise those more
complicated code paths.
TODO: Change to run with parallel_operations=True for all tests. At
present when you do this it causes many test failures.
Args:
command_name: The name of the command being run.
args: Command-line args (arg0 = actual arg, not command name ala bash).
headers: Dictionary containing optional HTTP headers to pass to boto.
debug: Debug level to pass in to boto connection (range 0..3).
return_stdout: If True, will save and return stdout produced by command.
return_stderr: If True, will save and return stderr produced by command.
return_log_handler: If True, will return a MockLoggingHandler instance
that was attached to the command's logger while running.
cwd: The working directory that should be switched to before running the
command. The working directory will be reset back to its original
value after running the command. If not specified, the working
directory is left unchanged.
Returns:
One or a tuple of requested return values, depending on whether
return_stdout, return_stderr, and/or return_log_handler were specified.
"""
args = args or []
command_line = ' '.join([command_name] + args)
if self.is_debugging:
self.stderr_save.write('\nRunCommand of %s\n' % command_line)
# Save and truncate stdout and stderr for the lifetime of RunCommand. This
# way, we can return just the stdout and stderr that was output during the
# RunNamedCommand call below.
sys.stdout.seek(0)
sys.stderr.seek(0)
stdout = sys.stdout.read()
stderr = sys.stderr.read()
if stdout:
self.accumulated_stdout.append(stdout)
if stderr:
self.accumulated_stderr.append(stderr)
sys.stdout.seek(0)
sys.stderr.seek(0)
sys.stdout.truncate()
sys.stderr.truncate()
mock_log_handler = MockLoggingHandler()
logging.getLogger(command_name).addHandler(mock_log_handler)
try:
with WorkingDirectory(cwd):
self.command_runner.RunNamedCommand(
command_name, args=args, headers=headers, debug=debug,
parallel_operations=False, do_shutdown=False)
finally:
sys.stdout.seek(0)
stdout = sys.stdout.read()
sys.stderr.seek(0)
stderr = sys.stderr.read()
logging.getLogger(command_name).removeHandler(mock_log_handler)
mock_log_handler.close()
log_output = '\n'.join(
'%s:\n ' % level + '\n '.join(records)
for level, records in mock_log_handler.messages.iteritems()
if records)
if self.is_debugging and log_output:
self.stderr_save.write(
'==== logging RunCommand %s %s ====\n' % (self.id(), command_line))
self.stderr_save.write(log_output)
self.stderr_save.write('\n==== end logging ====\n')
if self.is_debugging and stdout:
self.stderr_save.write(
'==== stdout RunCommand %s %s ====\n' % (self.id(), command_line))
self.stderr_save.write(stdout)
self.stderr_save.write('==== end stdout ====\n')
if self.is_debugging and stderr:
self.stderr_save.write(
'==== stderr RunCommand %s %s ====\n' % (self.id(), command_line))
self.stderr_save.write(stderr)
self.stderr_save.write('==== end stderr ====\n')
# Reset stdout and stderr files, so that we won't print them out again
# in tearDown if debugging is enabled.
sys.stdout.seek(0)
sys.stderr.seek(0)
sys.stdout.truncate()
sys.stderr.truncate()
to_return = []
if return_stdout:
to_return.append(stdout)
if return_stderr:
to_return.append(stderr)
if return_log_handler:
to_return.append(mock_log_handler)
if len(to_return) == 1:
return to_return[0]
return tuple(to_return)
@classmethod
def MakeGsUtilApi(cls, debug=0):
gsutil_api_map = {
ApiMapConstants.API_MAP: (
cls.mock_gsutil_api_class_map_factory.GetClassMap()),
ApiMapConstants.SUPPORT_MAP: {
'gs': [ApiSelector.XML, ApiSelector.JSON],
's3': [ApiSelector.XML]
},
ApiMapConstants.DEFAULT_MAP: {
'gs': ApiSelector.JSON,
's3': ApiSelector.XML
}
}
return CloudApiDelegator(
cls.mock_bucket_storage_uri, gsutil_api_map, cls.logger, debug=debug)
@classmethod
def _test_wildcard_iterator(cls, uri_or_str, debug=0):
"""Convenience method for instantiating a test instance of WildcardIterator.
This makes it unnecessary to specify all the params of that class
(like bucket_storage_uri_class=mock_storage_service.MockBucketStorageUri).
Also, naming the factory method this way makes it clearer in the test code
that WildcardIterator needs to be set up for testing.
Args are same as for wildcard_iterator.wildcard_iterator(), except
there are no class args for bucket_storage_uri_class or gsutil_api_class.
Args:
uri_or_str: StorageUri or string representing the wildcard string.
debug: debug level to pass to the underlying connection (0..3)
Returns:
WildcardIterator, over which caller can iterate.
"""
# TODO: Remove when tests no longer pass StorageUri arguments.
uri_string = uri_or_str
if hasattr(uri_or_str, 'uri'):
uri_string = uri_or_str.uri
return wildcard_iterator.CreateWildcardIterator(
uri_string, cls.MakeGsUtilApi())
@staticmethod
def _test_storage_uri(uri_str, default_scheme='file', debug=0,
validate=True):
"""Convenience method for instantiating a testing instance of StorageUri.
This makes it unnecessary to specify
bucket_storage_uri_class=mock_storage_service.MockBucketStorageUri.
Also naming the factory method this way makes it clearer in the test
code that StorageUri needs to be set up for testing.
Args, Returns, and Raises are same as for boto.storage_uri(), except there's
no bucket_storage_uri_class arg.
Args:
uri_str: Uri string to create StorageUri for.
default_scheme: Default scheme for the StorageUri
debug: debug level to pass to the underlying connection (0..3)
validate: If True, validate the resource that the StorageUri refers to.
Returns:
StorageUri based on the arguments.
"""
return boto.storage_uri(uri_str, default_scheme, debug, validate,
util.GSMockBucketStorageUri)
def CreateBucket(self, bucket_name=None, test_objects=0, storage_class=None,
provider='gs'):
"""Creates a test bucket.
The bucket and all of its contents will be deleted after the test.
Args:
bucket_name: Create the bucket with this name. If not provided, a
temporary test bucket name is constructed.
test_objects: The number of objects that should be placed in the bucket or
a list of object names to place in the bucket. Defaults to
0.
storage_class: storage class to use. If not provided we us standard.
provider: string provider to use, default gs.
Returns:
StorageUri for the created bucket.
"""
bucket_name = bucket_name or self.MakeTempName('bucket')
bucket_uri = boto.storage_uri(
'%s://%s' % (provider, bucket_name.lower()),
suppress_consec_slashes=False,
bucket_storage_uri_class=util.GSMockBucketStorageUri)
bucket_uri.create_bucket(storage_class=storage_class)
self.bucket_uris.append(bucket_uri)
try:
iter(test_objects)
except TypeError:
test_objects = [self.MakeTempName('obj') for _ in range(test_objects)]
for i, name in enumerate(test_objects):
self.CreateObject(bucket_uri=bucket_uri, object_name=name,
contents='test %d' % i)
return bucket_uri
def CreateObject(self, bucket_uri=None, object_name=None, contents=None):
"""Creates a test object.
Args:
bucket_uri: The URI of the bucket to place the object in. If not
specified, a new temporary bucket is created.
object_name: The name to use for the object. If not specified, a temporary
test object name is constructed.
contents: The contents to write to the object. If not specified, the key
is not written to, which means that it isn't actually created
yet on the server.
Returns:
A StorageUri for the created object.
"""
bucket_uri = bucket_uri or self.CreateBucket()
object_name = object_name or self.MakeTempName('obj')
key_uri = bucket_uri.clone_replace_name(object_name)
if contents is not None:
key_uri.set_contents_from_string(contents)
return key_uri
