def test_warn_bad_symlink(self):
"""
This tests to make sure it fails when following bad symlinks.
"""
abs_root = six.text_type(os.path.abspath(self.root) + os.sep)
input_local_dir = {
'src': {
'path': abs_root,
'type': 'local'
},
'dest': {
'path': self.bucket,
'type': 's3'
},
'dir_op': True,
'use_src_name': True
}
file_stats = FileGenerator(self.client, '', True).call(input_local_dir)
file_gen = FileGenerator(self.client, '', True)
file_stats = file_gen.call(input_local_dir)
all_filenames = self.filenames + self.symlink_files
all_filenames.sort()
result_list = []
for file_stat in file_stats:
result_list.append(getattr(file_stat, 'src'))
self.assertEqual(len(result_list), len(all_filenames))
# Just check to make sure the right local files are generated.
for i in range(len(result_list)):
filename = six.text_type(os.path.abspath(all_filenames[i]))
self.assertEqual(result_list[i], filename)
self.assertFalse(file_gen.result_queue.empty())
def test_warn_bad_symlink(self):
"""
This tests to make sure it fails when following bad symlinks.
"""
abs_root = six.text_type(os.path.abspath(self.root) + os.sep)
input_local_dir = {'src': {'path': abs_root,
'type': 'local'},
'dest': {'path': self.bucket,
'type': 's3'},
'dir_op': True, 'use_src_name': True}
file_stats = FileGenerator(self.service, self.endpoint,
'', True).call(input_local_dir)
file_gen = FileGenerator(self.service, self.endpoint, '', True)
file_stats = file_gen.call(input_local_dir)
all_filenames = self.filenames + self.symlink_files
all_filenames.sort()
result_list = []
for file_stat in file_stats:
result_list.append(getattr(file_stat, 'src'))
self.assertEqual(len(result_list), len(all_filenames))
# Just check to make sure the right local files are generated.
for i in range(len(result_list)):
filename = six.text_type(os.path.abspath(all_filenames[i]))
self.assertEqual(result_list[i], filename)
self.assertFalse(file_gen.result_queue.empty())
def test_follow_symlink(self):
# First remove the bad symlink.
os.remove(os.path.join(self.root, 'symlink_2'))
abs_root = six.text_type(os.path.abspath(self.root) + os.sep)
input_local_dir = {
'src': {
'path': abs_root,
'type': 'local'
},
'dest': {
'path': self.bucket,
'type': 's3'
},
'dir_op': True,
'use_src_name': True
}
file_stats = FileGenerator(self.client, '', True).call(input_local_dir)
all_filenames = self.filenames + self.symlink_files
all_filenames.sort()
result_list = []
for file_stat in file_stats:
result_list.append(getattr(file_stat, 'src'))
self.assertEqual(len(result_list), len(all_filenames))
# Just check to make sure the right local files are generated.
for i in range(len(result_list)):
filename = six.text_type(os.path.abspath(all_filenames[i]))
self.assertEqual(result_list[i], filename)
def setUp(self):
self.local_file = six.text_type(
os.path.abspath('.') + os.sep + 'some_directory' + os.sep +
'text1.txt')
self.local_dir = six.text_type(
os.path.abspath('.') + os.sep + 'some_directory' + os.sep)
self.session = FakeSession()
self.service = self.session.get_service('s3')
self.endpoint = self.service.get_endpoint('us-east-1')
self.files = make_loc_files()
def setUp(self):
self.local_file = six.text_type(os.path.abspath('.') +
os.sep + 'some_directory' +
os.sep + 'text1.txt')
self.local_dir = six.text_type(os.path.abspath('.') +
os.sep + 'some_directory' +
os.sep)
self.session = FakeSession()
self.service = self.session.get_service('s3')
self.endpoint = self.service.get_endpoint('us-east-1')
self.files = make_loc_files()
def _unpack_cli_arg(argument_model, value, cli_name):
if argument_model.type_name in SCALAR_TYPES:
return unpack_scalar_cli_arg(argument_model, value, cli_name)
elif argument_model.type_name in COMPLEX_TYPES:
return _unpack_complex_cli_arg(argument_model, value, cli_name)
else:
return six.text_type(value)
def _check_for_uri_param(param, value):
if isinstance(value, list) and len(value) == 1:
value = value[0]
try:
return get_paramfile(value)
except ResourceLoadingError as e:
raise ParamError(param.cli_name, six.text_type(e))
def make_loc_files():
"""
This sets up the test by making a directory named some_directory. It
has the file text1.txt and the directory another_directory inside. Inside
of another_directory it creates the file text2.txt.
"""
directory1 = six.text_type(
os.path.abspath('.') + os.sep + 'some_directory' + os.sep)
if not os.path.exists(directory1):
os.mkdir(directory1)
string1 = b"This is a test."
filename1 = directory1 + u"text1.txt"
with open(filename1, 'wb') as file1:
file1.write(string1)
directory2 = directory1 + u'another_directory' + os.sep
if not os.path.exists(directory2):
os.mkdir(directory2)
string2 = b"This is another test."
filename2 = directory2 + u"text2.txt"
with open(filename2, 'wb') as file2:
file2.write(string2)
return [filename1, filename2, directory2, directory1]
def _check_for_uri_param(param, value):
if isinstance(value, list) and len(value) == 1:
value = value[0]
try:
return get_paramfile(value)
except ResourceLoadingError as e:
raise ParamError(param.cli_name, six.text_type(e))
def _unpack_cli_arg(argument_model, value, cli_name):
if argument_model.type_name in SCALAR_TYPES:
return unpack_scalar_cli_arg(argument_model, value, cli_name)
elif argument_model.type_name in COMPLEX_TYPES:
return _unpack_complex_cli_arg(argument_model, value, cli_name)
else:
return six.text_type(value)
def test_no_follow_symlink(self):
abs_root = six.text_type(os.path.abspath(self.root) + os.sep)
input_local_dir = {'src': {'path': abs_root,
'type': 'local'},
'dest': {'path': self.bucket,
'type': 's3'},
'dir_op': True, 'use_src_name': True}
file_stats = FileGenerator(self.client, '', False).call(input_local_dir)
self.filenames.sort()
result_list = []
for file_stat in file_stats:
result_list.append(getattr(file_stat, 'src'))
self.assertEqual(len(result_list), len(self.filenames))
# Just check to make sure the right local files are generated.
for i in range(len(result_list)):
filename = six.text_type(os.path.abspath(self.filenames[i]))
self.assertEqual(result_list[i], filename)
def __call__(self, event_name, param, value, **kwargs):
"""Handler that supports param values from local files."""
if isinstance(value, list) and len(value) == 1:
value = value[0]
try:
return get_paramfile(value, self._prefixes)
except ResourceLoadingError as e:
raise ParamError(param.cli_name, six.text_type(e))
def make_loc_files(file_creator):
"""
This sets up the test by making a directory named some_directory. It
has the file text1.txt and the directory another_directory inside. Inside
of another_directory it creates the file text2.txt.
"""
body = 'This is a test.'
filename1 = file_creator.create_file(
os.path.join('some_directory', 'text1.txt'), body)
filename2 = file_creator.create_file(
os.path.join('some_directory', 'another_directory', 'text2.txt'), body)
filename1 = six.text_type(filename1)
filename2 = six.text_type(filename2)
return [filename1, filename2, os.path.dirname(filename2),
os.path.dirname(filename1)]
def _format_scalar_list(elements, identifier, stream):
if identifier is not None:
for item in elements:
stream.write('%s\t%s\n' % (identifier.upper(),
item))
else:
# For a bare list, just print the contents.
stream.write('\t'.join([six.text_type(item) for item in elements]))
stream.write('\n')
def get_text_length(text):
# `len(unichar)` measures the number of characters, so we use
# `unicodedata.east_asian_width` to measure the length of characters.
# Following responses are considered to be full-width length.
# * A(Ambiguous)
# * F(Fullwidth)
# * W(Wide)
text = six.text_type(text)
return sum(2 if unicodedata.east_asian_width(char) in 'WFA' else 1
for char in text)
def _format_text(item, stream, identifier=None, scalar_keys=None):
if isinstance(item, dict):
_format_dict(scalar_keys, item, identifier, stream)
elif isinstance(item, list):
_format_list(item, identifier, stream)
else:
# If it's not a list or a dict, we just write the scalar
# value out directly.
stream.write(six.text_type(item))
stream.write('\n')
def main(self, args=None):
"""
:param args: List of arguments, with the 'aws' removed. For example,
the command "aws s3 list-objects --bucket foo" will have an
args list of ``['s3', 'list-objects', '--bucket', 'foo']``.
"""
if args is None:
args = sys.argv[1:]
command_table = self._get_command_table()
parser = self._create_parser(command_table)
self._add_aliases(command_table, parser)
parsed_args, remaining = parser.parse_known_args(args)
try:
# Because _handle_top_level_args emits events, it's possible
# that exceptions can be raised, which should have the same
# general exception handling logic as calling into the
# command table. This is why it's in the try/except clause.
self._handle_top_level_args(parsed_args)
self._emit_session_event(parsed_args)
HISTORY_RECORDER.record('CLI_VERSION', self.session.user_agent(),
'CLI')
HISTORY_RECORDER.record('CLI_ARGUMENTS', args, 'CLI')
return command_table[parsed_args.command](remaining, parsed_args)
except UnknownArgumentError as e:
sys.stderr.write("usage: %s\n" % USAGE)
sys.stderr.write(str(e))
sys.stderr.write("\n")
return 255
except NoRegionError as e:
msg = ('%s You can also configure your region by running '
'"aws configure".' % e)
self._show_error(msg)
return 255
except NoCredentialsError as e:
msg = ('%s. You can configure credentials by running '
'"aws configure".' % e)
self._show_error(msg)
return 255
except KeyboardInterrupt:
# Shell standard for signals that terminate
# the process is to return 128 + signum, in this case
# SIGINT=2, so we'll have an RC of 130.
sys.stdout.write("\n")
return 128 + signal.SIGINT
except Exception as e:
LOG.debug("Exception caught in main()", exc_info=True)
LOG.debug("Exiting with rc 255")
err = get_stderr_text_writer()
err.write("\n")
err.write(six.text_type(e))
err.write("\n")
return 255
def make_loc_files(file_creator, size=None):
"""
This sets up the test by making a directory named some_directory. It
has the file text1.txt and the directory another_directory inside. Inside
of another_directory it creates the file text2.txt.
"""
if size:
body = "*" * size
else:
body = 'This is a test.'
filename1 = file_creator.create_file(
os.path.join('some_directory', 'text1.txt'), body)
filename2 = file_creator.create_file(
os.path.join('some_directory', 'another_directory', 'text2.txt'), body)
filename1 = six.text_type(filename1)
filename2 = six.text_type(filename2)
return [filename1, filename2, os.path.dirname(filename2),
os.path.dirname(filename1)]
def main(self, args=None):
"""
:param args: List of arguments, with the 'aws' removed. For example,
the command "aws s3 list-objects --bucket foo" will have an
args list of ``['s3', 'list-objects', '--bucket', 'foo']``.
"""
if args is None:
args = sys.argv[1:]
command_table = self._get_command_table()
parser = self._create_parser(command_table)
self._add_aliases(command_table, parser)
parsed_args, remaining = parser.parse_known_args(args)
try:
# Because _handle_top_level_args emits events, it's possible
# that exceptions can be raised, which should have the same
# general exception handling logic as calling into the
# command table. This is why it's in the try/except clause.
self._handle_top_level_args(parsed_args)
self._emit_session_event(parsed_args)
HISTORY_RECORDER.record(
'CLI_VERSION', self.session.user_agent(), 'CLI')
HISTORY_RECORDER.record('CLI_ARGUMENTS', args, 'CLI')
return command_table[parsed_args.command](remaining, parsed_args)
except UnknownArgumentError as e:
sys.stderr.write("usage: %s\n" % USAGE)
sys.stderr.write(str(e))
sys.stderr.write("\n")
return 255
except NoRegionError as e:
msg = ('%s You can also configure your region by running '
'"aws configure".' % e)
self._show_error(msg)
return 255
except NoCredentialsError as e:
msg = ('%s. You can configure credentials by running '
'"aws configure".' % e)
self._show_error(msg)
return 255
except KeyboardInterrupt:
# Shell standard for signals that terminate
# the process is to return 128 + signum, in this case
# SIGINT=2, so we'll have an RC of 130.
sys.stdout.write("\n")
return 128 + signal.SIGINT
except Exception as e:
LOG.debug("Exception caught in main()", exc_info=True)
LOG.debug("Exiting with rc 255")
err = get_stderr_text_writer()
err.write("\n")
err.write(six.text_type(e))
err.write("\n")
return 255
def test_follow_symlink(self):
# First remove the bad symlink.
os.remove(os.path.join(self.root, 'symlink_2'))
abs_root = six.text_type(os.path.abspath(self.root) + os.sep)
input_local_dir = {'src': {'path': abs_root,
'type': 'local'},
'dest': {'path': self.bucket,
'type': 's3'},
'dir_op': True, 'use_src_name': True}
file_stats = FileGenerator(self.service, self.endpoint,
'', True).call(input_local_dir)
all_filenames = self.filenames + self.symlink_files
all_filenames.sort()
result_list = []
for file_stat in file_stats:
result_list.append(getattr(file_stat, 'src'))
self.assertEqual(len(result_list), len(all_filenames))
# Just check to make sure the right local files are generated.
for i in range(len(result_list)):
filename = six.text_type(os.path.abspath(all_filenames[i]))
self.assertEqual(result_list[i], filename)
def _partition_dict(item_dict, scalar_keys):
# Given a dictionary, partition it into two list based on the
# values associated with the keys.
# {'foo': 'scalar', 'bar': 'scalar', 'baz': ['not, 'scalar']}
# scalar = [('foo', 'scalar'), ('bar', 'scalar')]
# non_scalar = [('baz', ['not', 'scalar'])]
scalar = []
non_scalar = []
if scalar_keys is None:
# scalar_keys can have more than just the keys in the item_dict,
# but if user does not provide scalar_keys, we'll grab the keys
# from the current item_dict
for key, value in sorted(item_dict.items()):
if isinstance(value, (dict, list)):
non_scalar.append((key, value))
else:
scalar.append(six.text_type(value))
else:
for key in scalar_keys:
scalar.append(six.text_type(item_dict.get(key, '')))
remaining_keys = sorted(set(item_dict.keys()) - set(scalar_keys))
for remaining_key in remaining_keys:
non_scalar.append((remaining_key, item_dict[remaining_key]))
return scalar, non_scalar
def write_exception(ex, outfile):
outfile.write("\n")
outfile.write(six.text_type(ex))
outfile.write("\n")
def setUp(self):
self.directory = six.text_type(tempfile.mkdtemp())
def _format_row(self, row):
return [six.text_type(r) for r in row]
def write_exception(ex, outfile):
outfile.write("\n")
outfile.write(six.text_type(ex))
outfile.write("\n")
def _split_on_commas(self, value):
try:
return utils.split_on_commas(value)
except ValueError as e:
raise ParamSyntaxError(six.text_type(e))
def setUp(self):
self.directory = six.text_type(tempfile.mkdtemp())
