def _ValidateAndGetConfigValues(self):
"""Parses the user's config file to aggregate non-PII config values.
Returns:
A comma-delimited string of config values explicitly set by the user in
key:value pairs, sorted alphabetically by key.
"""
config_values = []
# If a user has an invalid config value set, we will mark it as such with
# this value. If a user did not enter a value, we will not report it.
invalid_value_string = 'INVALID'
def GetAndValidateConfigValue(section, category, validation_fn):
try:
config_value = boto.config.get_value(section, category)
if config_value and validation_fn(config_value):
config_values.append((category, config_value))
# If the user entered a non-valid config value, store it as invalid.
elif config_value:
config_values.append((category, invalid_value_string))
# This function gets called during initialization of the MetricsCollector.
# If any of the logic fails, we do not want to hinder the gsutil command
# being run, and thus ignore any exceptions.
except: # pylint: disable=bare-except
config_values.append((category, invalid_value_string))
# Validate boolean values.
for section, bool_category in (('Boto', 'https_validate_certificates'),
('GSUtil', 'disable_analytics_prompt'),
('GSUtil', 'use_magicfile'),
('GSUtil', 'tab_completion_time_logs')):
GetAndValidateConfigValue(
section=section,
category=bool_category,
validation_fn=lambda val: str(val).lower() in
('true', 'false'))
# Define a threshold for some config values which should be reasonably low.
small_int_threshold = 2000
# Validate small integers.
for section, small_int_category in (
('Boto', 'debug'), ('Boto', 'http_socket_timeout'),
('Boto', 'num_retries'), ('Boto', 'max_retry_delay'),
('GSUtil',
'default_api_version'), ('GSUtil',
'sliced_object_download_max_components'),
('GSUtil', 'parallel_process_count'), ('GSUtil',
'parallel_thread_count'),
('GSUtil', 'software_update_check_period'),
('GSUtil', 'tab_completion_timeout'), ('OAuth2',
'oauth2_refresh_retries')):
GetAndValidateConfigValue(
section=section,
category=small_int_category,
validation_fn=lambda val: str(val).isdigit() and int(
val) < small_int_threshold)
# Validate large integers.
for section, large_int_category in (('GSUtil', 'resumable_threshold'),
('GSUtil', 'rsync_buffer_lines'),
('GSUtil',
'task_estimation_threshold')):
GetAndValidateConfigValue(
section=section,
category=large_int_category,
validation_fn=lambda val: str(val).isdigit())
# Validate data sizes.
for section, data_size_category in (
('GSUtil', 'parallel_composite_upload_component_size'),
('GSUtil', 'parallel_composite_upload_threshold'),
('GSUtil', 'sliced_object_download_component_size'),
('GSUtil', 'sliced_object_download_threshold')):
config_value = boto.config.get_value(section, data_size_category)
if config_value:
try:
size_in_bytes = HumanReadableToBytes(config_value)
config_values.append((data_size_category, size_in_bytes))
except ValueError:
config_values.append(
(data_size_category, invalid_value_string))
# Validate specific options.
# pylint: disable=g-long-lambda
GetAndValidateConfigValue(
section='GSUtil',
category='check_hashes',
validation_fn=lambda val: val in
('if_fast_else_fail', 'if_fast_else_skip', 'always', 'never'))
# pylint: enable=g-long-lambda
GetAndValidateConfigValue(
section='GSUtil',
category='content_language',
validation_fn=lambda val: val.isalpha() and len(val) <= 3)
GetAndValidateConfigValue(section='GSUtil',
category='json_api_version',
validation_fn=lambda val: val[0].lower() ==
'v' and val[1:].isdigit())
GetAndValidateConfigValue(section='GSUtil',
category='prefer_api',
validation_fn=lambda val: val in
('json', 'xml'))
GetAndValidateConfigValue(section='OAuth2',
category='token_cache',
validation_fn=lambda val: val in
('file_system', 'in_memory'))
return ','.join(
sorted([
'{0}:{1}'.format(config[0], config[1])
for config in config_values
]))
def _ParseArgs(self):
"""Parses arguments for perfdiag command."""
# From -n.
self.num_iterations = 5
# From -c.
self.processes = 1
# From -k.
self.threads = 1
# From -s.
self.thru_filesize = 1048576
# From -t.
self.diag_tests = self.ALL_DIAG_TESTS
# From -o.
self.output_file = None
# From -i.
self.input_file = None
# From -m.
self.metadata_keys = {}
if self.sub_opts:
for o, a in self.sub_opts:
if o == '-n':
self.num_iterations = self._ParsePositiveInteger(
a, 'The -n parameter must be a positive integer.')
if o == '-c':
self.processes = self._ParsePositiveInteger(
a, 'The -c parameter must be a positive integer.')
if o == '-k':
self.threads = self._ParsePositiveInteger(
a, 'The -k parameter must be a positive integer.')
if o == '-s':
try:
self.thru_filesize = HumanReadableToBytes(a)
except ValueError:
raise CommandException('Invalid -s parameter.')
if self.thru_filesize > (20 * 1024 ** 3): # Max 20 GB.
raise CommandException(
'Maximum throughput file size parameter (-s) is 20GB.')
if o == '-t':
self.diag_tests = []
for test_name in a.strip().split(','):
if test_name.lower() not in self.ALL_DIAG_TESTS:
raise CommandException("List of test names (-t) contains invalid "
"test name '%s'." % test_name)
self.diag_tests.append(test_name)
if o == '-m':
pieces = a.split(':')
if len(pieces) != 2:
raise CommandException(
"Invalid metadata key-value combination '%s'." % a)
key, value = pieces
self.metadata_keys[key] = value
if o == '-o':
self.output_file = os.path.abspath(a)
if o == '-i':
self.input_file = os.path.abspath(a)
if not os.path.isfile(self.input_file):
raise CommandException("Invalid input file (-i): '%s'." % a)
try:
with open(self.input_file, 'r') as f:
self.results = json.load(f)
self.logger.info("Read input file: '%s'.", self.input_file)
except ValueError:
raise CommandException("Could not decode input file (-i): '%s'." %
a)
return
if not self.args:
raise CommandException('Wrong number of arguments for "perfdiag" '
'command.')
self.bucket_uri = self.suri_builder.StorageUri(self.args[0])
if not self.bucket_uri.names_bucket():
raise CommandException('The perfdiag command requires a URI that '
'specifies a bucket.\n"%s" is not '
'valid.' % self.bucket_uri)
self.bucket = self.bucket_uri.get_bucket()
# TODO: Add MD5 argument support to get_contents_to_file()
# and pass the file md5 as a parameter to avoid any unnecessary
# computation.
self.get_contents_to_file_args = {}
if self.bucket_uri.scheme == 'gs':
self.get_contents_to_file_args = {'hash_algs': {}}