def test_command_argument_parser_setup_completers(self): command_map = { FakeCommandWithCompleters.command_spec.command_name: FakeCommandWithCompleters() } runner = CommandRunner( bucket_storage_uri_class=self.mock_bucket_storage_uri, gsutil_api_class_map_factory=self.mock_gsutil_api_class_map_factory, command_map=command_map) subparsers = FakeArgparseSubparsers() runner.ConfigureCommandArgumentParsers(subparsers) self.assertEqual(1, len(subparsers.parsers)) parser = subparsers.parsers[0] self.assertEqual(6, len(parser.arguments)) self.assertEqual(CloudObjectCompleter, type(parser.arguments[0].completer)) self.assertEqual(LocalObjectCompleter, type(parser.arguments[1].completer)) self.assertEqual( CloudOrLocalObjectCompleter, type(parser.arguments[2].completer)) self.assertEqual( NoOpCompleter, type(parser.arguments[3].completer)) self.assertEqual(CloudObjectCompleter, type(parser.arguments[4].completer)) self.assertEqual( LocalObjectOrCannedACLCompleter, type(parser.arguments[5].completer))
def test_command_argument_parser_setup_nested_argparse_arguments(self): command_map = { FakeCommandWithNestedArguments.command_spec.command_name: FakeCommandWithNestedArguments(), } runner = CommandRunner( bucket_storage_uri_class=self.mock_bucket_storage_uri, gsutil_api_class_map_factory=self. mock_gsutil_api_class_map_factory, command_map=command_map) parser = FakeArgparseParser() runner.ConfigureCommandArgumentParsers(parser) # Start at first level of subparsers. cur_subparser = parser.subparsers.parsers[0] self.assertEqual( cur_subparser.name, FakeCommandWithNestedArguments.command_spec.command_name) # Go one level further, checking the current subparser's first subparser. cur_subparser = cur_subparser.subparsers.parsers[0] self.assertEqual(cur_subparser.name, FakeCommandWithNestedArguments.subcommand_name) cur_subparser = cur_subparser.subparsers.parsers[0] self.assertEqual(cur_subparser.name, FakeCommandWithNestedArguments.subcommand_subname) # FakeArgparseParser adds a FakeArgparseArgument in place of any arguments # passed in via add_argument(), so we can't check for anything beyond the # presence of the correct number of arguments here. self.assertEqual(len(cur_subparser.arguments), FakeCommandWithNestedArguments.num_args)
def setUpClass(cls): base.GsUtilTestCase.setUpClass() cls.mock_bucket_storage_uri = util.GSMockBucketStorageUri cls.mock_gsutil_api_class_map_factory = GsutilApiUnitTestClassMapFactory cls.logger = logging.getLogger() cls.command_runner = CommandRunner( bucket_storage_uri_class=cls.mock_bucket_storage_uri, gsutil_api_class_map_factory=cls.mock_gsutil_api_class_map_factory)
def setUpClass(cls): base.GsUtilTestCase.setUpClass() cls.mock_bucket_storage_uri = GSMockBucketStorageUri cls.proj_id_handler = ProjectIdHandler() config_file_list = boto.pyami.config.BotoConfigLocations # Use "gsutil_test_commands" as a fake UserAgent. This value will never be # sent via HTTP because we're using MockStorageService here. cls.command_runner = CommandRunner(config_file_list, cls.mock_bucket_storage_uri)
def setUpClass(cls): base.GsUtilTestCase.setUpClass() cls.mock_bucket_storage_uri = util.GSMockBucketStorageUri cls.mock_gsutil_api_class_map_factory = GsutilApiUnitTestClassMapFactory cls.logger = logging.getLogger() cls.command_runner = CommandRunner( bucket_storage_uri_class=cls.mock_bucket_storage_uri, gsutil_api_class_map_factory=cls.mock_gsutil_api_class_map_factory) # Ensure unit tests don't fail if no default_project_id is defined in the # boto config file. project_id.UNIT_TEST_PROJECT_ID = 'mock-project-id-for-unit-tests'
def test_command_argument_parser_setup_invalid_completer(self): command_map = { FakeCommandWithInvalidCompleter.command_spec.command_name: FakeCommandWithInvalidCompleter() } runner = CommandRunner( bucket_storage_uri_class=self.mock_bucket_storage_uri, gsutil_api_class_map_factory=self.mock_gsutil_api_class_map_factory, command_map=command_map) subparsers = FakeArgparseSubparsers() try: runner.ConfigureCommandArgumentParsers(subparsers) except RuntimeError as e: self.assertIn('Unknown completer', e.message)
def main(): InitializeSignalHandling() # Any modules used in initializing multiprocessing variables must be # imported after importing gslib.__main__. # pylint: disable=redefined-outer-name,g-import-not-at-top import gslib.boto_translation import gslib.command import gslib.utils.parallelism_framework_util # pylint: disable=unused-variable from gcs_oauth2_boto_plugin import oauth2_client from apitools.base.py import credentials_lib # pylint: enable=unused-variable if (gslib.utils.parallelism_framework_util. CheckMultiprocessingAvailableAndInit().is_available): # These setup methods must be called, and, on Windows, they can only be # called from within an "if __name__ == '__main__':" block. gslib.command.InitializeMultiprocessingVariables() gslib.boto_translation.InitializeMultiprocessingVariables() else: gslib.command.InitializeThreadingVariables() # This needs to be done after InitializeMultiprocessingVariables(), since # otherwise we can't call CreateLock. try: # pylint: disable=unused-import,g-import-not-at-top import gcs_oauth2_boto_plugin gsutil_client_id, gsutil_client_secret = ( system_util.GetGsutilClientIdAndSecret()) gcs_oauth2_boto_plugin.oauth2_helper.SetFallbackClientIdAndSecret( gsutil_client_id, gsutil_client_secret) gcs_oauth2_boto_plugin.oauth2_helper.SetLock( gslib.utils.parallelism_framework_util.CreateLock()) credentials_lib.SetCredentialsCacheFileLock( gslib.utils.parallelism_framework_util.CreateLock()) except ImportError: pass global debug_level global test_exception_traces supported, err = check_python_version_support() if not supported: raise CommandException(err) sys.exit(1) boto_util.MonkeyPatchBoto() system_util.MonkeyPatchHttp() # In gsutil 4.0 and beyond, we don't use the boto library for the JSON # API. However, we still store gsutil configuration data in the .boto # config file for compatibility with previous versions and user convenience. # Many users have a .boto configuration file from previous versions, and it # is useful to have all of the configuration for gsutil stored in one place. command_runner = CommandRunner() if not boto_util.BOTO_IS_SECURE: raise CommandException('\n'.join( textwrap.wrap( 'Your boto configuration has is_secure = False. Gsutil cannot be ' 'run this way, for security reasons.'))) headers = {} parallel_operations = False quiet = False version = False debug_level = 0 trace_token = None perf_trace_token = None test_exception_traces = False user_project = None # If user enters no commands just print the usage info. if len(sys.argv) == 1: sys.argv.append('help') # Change the default of the 'https_validate_certificates' boto option to # True (it is currently False in boto). if not boto.config.has_option('Boto', 'https_validate_certificates'): if not boto.config.has_section('Boto'): boto.config.add_section('Boto') boto.config.setbool('Boto', 'https_validate_certificates', True) for signal_num in GetCaughtSignals(): RegisterSignalHandler(signal_num, _CleanupSignalHandler) try: for o, a in opts: if o in ('-d', '--debug'): # Also causes boto to include httplib header output. debug_level = constants.DEBUGLEVEL_DUMP_REQUESTS elif o in ('-D', '--detailedDebug'): # We use debug level 3 to ask gsutil code to output more detailed # debug output. This is a bit of a hack since it overloads the same # flag that was originally implemented for boto use. And we use -DD # to ask for really detailed debugging (i.e., including HTTP payload). if debug_level == constants.DEBUGLEVEL_DUMP_REQUESTS: debug_level = constants.DEBUGLEVEL_DUMP_REQUESTS_AND_PAYLOADS else: debug_level = constants.DEBUGLEVEL_DUMP_REQUESTS elif o in ('-?', '--help'): _OutputUsageAndExit(command_runner) elif o in ('-h', '--header'): (hdr_name, _, hdr_val) = a.partition(':') if not hdr_name: _OutputUsageAndExit(command_runner) headers[hdr_name.lower()] = hdr_val elif o in ('-m', '--multithreaded'): parallel_operations = True elif o in ('-q', '--quiet'): quiet = True elif o == '-u': user_project = a elif o in ('-v', '--version'): version = True elif o in ('-i', '--impersonate-service-account'): constants.IMPERSONATE_SERVICE_ACCOUNT = a elif o == '--perf-trace-token': perf_trace_token = a elif o == '--trace-token': trace_token = a elif o == '--testexceptiontraces': # Hidden flag for integration tests. test_exception_traces = True # Avoid printing extra warnings to stderr regarding long retries by # setting the threshold very high. constants.LONG_RETRY_WARN_SEC = 3600 elif o in ('-o', '--option'): (opt_section_name, _, opt_value) = a.partition('=') if not opt_section_name: _OutputUsageAndExit(command_runner) (opt_section, _, opt_name) = opt_section_name.partition(':') if not opt_section or not opt_name: _OutputUsageAndExit(command_runner) if not boto.config.has_section(opt_section): boto.config.add_section(opt_section) boto.config.set(opt_section, opt_name, opt_value) # Now that any Boto option overrides (via `-o` args) have been parsed, # perform initialization that depends on those options. boto_util.configured_certs_file = (boto_util.ConfigureCertsFile()) metrics.LogCommandParams(global_opts=opts) httplib2.debuglevel = debug_level if trace_token: sys.stderr.write(TRACE_WARNING) if debug_level >= constants.DEBUGLEVEL_DUMP_REQUESTS: sys.stderr.write(DEBUG_WARNING) _ConfigureRootLogger(level=logging.DEBUG) command_runner.RunNamedCommand('ver', ['-l']) config_items = [] for config_section in ('Boto', 'GSUtil'): try: config_items.extend(boto.config.items(config_section)) except configparser.NoSectionError: pass for i in range(len(config_items)): config_item_key = config_items[i][0] if config_item_key in CONFIG_KEYS_TO_REDACT: config_items[i] = (config_item_key, 'REDACTED') sys.stderr.write('Command being run: %s\n' % ' '.join(sys.argv)) sys.stderr.write('config_file_list: %s\n' % boto_util.GetFriendlyConfigFilePaths()) sys.stderr.write('config: %s\n' % str(config_items)) else: # Non-debug log level. root_logger_level = logging.WARNING if quiet else logging.INFO # oauth2client uses INFO and WARNING logging in places that would better # correspond to gsutil's debug logging (e.g., when refreshing # access tokens), so we bump the threshold one level higher where # appropriate. These log levels work for regular- and quiet-level logging. oa2c_logger_level = logging.WARNING oa2c_multiprocess_file_storage_logger_level = logging.ERROR _ConfigureRootLogger(level=root_logger_level) oauth2client.client.logger.setLevel(oa2c_logger_level) oauth2client.contrib.multiprocess_file_storage.logger.setLevel( oa2c_multiprocess_file_storage_logger_level) # pylint: disable=protected-access oauth2client.transport._LOGGER.setLevel(oa2c_logger_level) reauth_creds._LOGGER.setLevel(oa2c_logger_level) # pylint: enable=protected-access # Initialize context configuration for device mTLS. context_config.create_context_config(logging.getLogger()) # TODO(reauth): Fix once reauth pins to pyu2f version newer than 0.1.3. # Fixes pyu2f v0.1.3 bug. import six # pylint: disable=g-import-not-at-top six.input = six.moves.input if not boto_util.CERTIFICATE_VALIDATION_ENABLED: sys.stderr.write(HTTP_WARNING) if version: command_name = 'version' elif not args: command_name = 'help' else: command_name = args[0] _CheckAndWarnForProxyDifferences() # Both 1 and 2 are valid _ARGCOMPLETE values; this var tells argcomplete at # what argv[] index the command to match starts. We want it to start at the # value for the path to gsutil, so: # $ gsutil <command> # Should be the 1st argument, so '1' # $ python gsutil <command> # Should be the 2nd argument, so '2' # Both are valid; most users invoke gsutil in the first style, but our # integration and prerelease tests invoke it in the second style, as we need # to specify the Python interpreter used to run gsutil. if os.environ.get('_ARGCOMPLETE', '0') in ('1', '2'): return _PerformTabCompletion(command_runner) return _RunNamedCommandAndHandleExceptions( command_runner, command_name, args=args[1:], headers=headers, debug_level=debug_level, trace_token=trace_token, parallel_operations=parallel_operations, perf_trace_token=perf_trace_token, user_project=user_project) finally: _Cleanup()
def main(): InitializeSignalHandling() # Any modules used in initializing multiprocessing variables must be # imported after importing gslib.__main__. # pylint: disable=redefined-outer-name,g-import-not-at-top import gslib.boto_translation import gslib.command import gslib.util from gslib.util import BOTO_IS_SECURE from gslib.util import CERTIFICATE_VALIDATION_ENABLED # pylint: disable=unused-variable from gcs_oauth2_boto_plugin import oauth2_client from apitools.base.py import credentials_lib # pylint: enable=unused-variable from gslib.util import CheckMultiprocessingAvailableAndInit if CheckMultiprocessingAvailableAndInit().is_available: # These setup methods must be called, and, on Windows, they can only be # called from within an "if __name__ == '__main__':" block. gslib.command.InitializeMultiprocessingVariables() gslib.boto_translation.InitializeMultiprocessingVariables() else: gslib.command.InitializeThreadingVariables() # This needs to be done after gslib.util.InitializeMultiprocessingVariables(), # since otherwise we can't call gslib.util.CreateLock. try: # pylint: disable=unused-import,g-import-not-at-top import gcs_oauth2_boto_plugin gsutil_client_id, gsutil_client_secret = GetGsutilClientIdAndSecret() gcs_oauth2_boto_plugin.oauth2_helper.SetFallbackClientIdAndSecret( gsutil_client_id, gsutil_client_secret) gcs_oauth2_boto_plugin.oauth2_helper.SetLock(CreateLock()) credentials_lib.SetCredentialsCacheFileLock(CreateLock()) except ImportError: pass global debug global test_exception_traces if not (2, 7) <= sys.version_info[:3] < (3,): raise CommandException('gsutil requires python 2.7.') # In gsutil 4.0 and beyond, we don't use the boto library for the JSON # API. However, we still store gsutil configuration data in the .boto # config file for compatibility with previous versions and user convenience. # Many users have a .boto configuration file from previous versions, and it # is useful to have all of the configuration for gsutil stored in one place. command_runner = CommandRunner() if not BOTO_IS_SECURE: raise CommandException('\n'.join(textwrap.wrap( 'Your boto configuration has is_secure = False. Gsutil cannot be ' 'run this way, for security reasons.'))) headers = {} parallel_operations = False quiet = False version = False debug = 0 trace_token = None perf_trace_token = None test_exception_traces = False # If user enters no commands just print the usage info. if len(sys.argv) == 1: sys.argv.append('help') # Change the default of the 'https_validate_certificates' boto option to # True (it is currently False in boto). if not boto.config.has_option('Boto', 'https_validate_certificates'): if not boto.config.has_section('Boto'): boto.config.add_section('Boto') boto.config.setbool('Boto', 'https_validate_certificates', True) gslib.util.configured_certs_file = gslib.util.ConfigureCertsFile() for signal_num in GetCaughtSignals(): RegisterSignalHandler(signal_num, _CleanupSignalHandler) GetCertsFile() try: try: opts, args = getopt.getopt(sys.argv[1:], 'dDvo:h:mq', ['debug', 'detailedDebug', 'version', 'option', 'help', 'header', 'multithreaded', 'quiet', 'testexceptiontraces', 'trace-token=', 'perf-trace-token=']) except getopt.GetoptError as e: _HandleCommandException(CommandException(e.msg)) for o, a in opts: if o in ('-d', '--debug'): # Also causes boto to include httplib header output. debug = DEBUGLEVEL_DUMP_REQUESTS elif o in ('-D', '--detailedDebug'): # We use debug level 3 to ask gsutil code to output more detailed # debug output. This is a bit of a hack since it overloads the same # flag that was originally implemented for boto use. And we use -DD # to ask for really detailed debugging (i.e., including HTTP payload). if debug == DEBUGLEVEL_DUMP_REQUESTS: debug = DEBUGLEVEL_DUMP_REQUESTS_AND_PAYLOADS else: debug = DEBUGLEVEL_DUMP_REQUESTS elif o in ('-?', '--help'): _OutputUsageAndExit(command_runner) elif o in ('-h', '--header'): (hdr_name, _, hdr_val) = a.partition(':') if not hdr_name: _OutputUsageAndExit(command_runner) headers[hdr_name.lower()] = hdr_val elif o in ('-m', '--multithreaded'): parallel_operations = True elif o in ('-q', '--quiet'): quiet = True elif o in ('-v', '--version'): version = True elif o == '--perf-trace-token': perf_trace_token = a elif o == '--trace-token': trace_token = a elif o == '--testexceptiontraces': # Hidden flag for integration tests. test_exception_traces = True # Avoid printing extra warnings to stderr regarding long retries by # setting the threshold very high. gslib.util.LONG_RETRY_WARN_SEC = 3600 elif o in ('-o', '--option'): (opt_section_name, _, opt_value) = a.partition('=') if not opt_section_name: _OutputUsageAndExit(command_runner) (opt_section, _, opt_name) = opt_section_name.partition(':') if not opt_section or not opt_name: _OutputUsageAndExit(command_runner) if not boto.config.has_section(opt_section): boto.config.add_section(opt_section) boto.config.set(opt_section, opt_name, opt_value) metrics.LogCommandParams(global_opts=opts) httplib2.debuglevel = debug if trace_token: sys.stderr.write(TRACE_WARNING) if debug >= DEBUGLEVEL_DUMP_REQUESTS: sys.stderr.write(DEBUG_WARNING) _ConfigureLogging(level=logging.DEBUG) command_runner.RunNamedCommand('ver', ['-l']) config_items = [] try: config_items.extend(boto.config.items('Boto')) config_items.extend(boto.config.items('GSUtil')) except ConfigParser.NoSectionError: pass for i in xrange(len(config_items)): config_item_key = config_items[i][0] if config_item_key in CONFIG_KEYS_TO_REDACT: config_items[i] = (config_item_key, 'REDACTED') sys.stderr.write('Command being run: %s\n' % ' '.join(sys.argv)) sys.stderr.write('config_file_list: %s\n' % GetBotoConfigFileList()) sys.stderr.write('config: %s\n' % str(config_items)) elif quiet: _ConfigureLogging(level=logging.WARNING) else: _ConfigureLogging(level=logging.INFO) # oauth2client uses info logging in places that would better # correspond to gsutil's debug logging (e.g., when refreshing # access tokens). oauth2client.client.logger.setLevel(logging.WARNING) if not CERTIFICATE_VALIDATION_ENABLED: sys.stderr.write(HTTP_WARNING) if version: command_name = 'version' elif not args: command_name = 'help' else: command_name = args[0] _CheckAndWarnForProxyDifferences() if os.environ.get('_ARGCOMPLETE', '0') == '1': return _PerformTabCompletion(command_runner) return _RunNamedCommandAndHandleExceptions( command_runner, command_name, args=args[1:], headers=headers, debug_level=debug, trace_token=trace_token, parallel_operations=parallel_operations, perf_trace_token=perf_trace_token) finally: _Cleanup()
def main(): # These modules must be imported after importing gslib.__main__. import gslib.command import gslib.util from gslib.third_party.oauth2_plugin import oauth2_client from gslib.util import MultiprocessingIsAvailable if MultiprocessingIsAvailable()[0]: # These setup methods must be called, and, on Windows, they can only be # called from within an "if __name__ == '__main__':" block. gslib.util.InitializeMultiprocessingVariables() gslib.command.InitializeMultiprocessingVariables() oauth2_client.InitializeMultiprocessingVariables() global debug if not (2, 6) <= sys.version_info[:3] < (3, ): raise gslib.exception.CommandException( 'gsutil requires python 2.6 or 2.7.') config_file_list = GetBotoConfigFileList() command_runner = CommandRunner(config_file_list) headers = {} parallel_operations = False quiet = False version = False debug = 0 # If user enters no commands just print the usage info. if len(sys.argv) == 1: sys.argv.append('help') # Change the default of the 'https_validate_certificates' boto option to # True (it is currently False in boto). if not boto.config.has_option('Boto', 'https_validate_certificates'): if not boto.config.has_section('Boto'): boto.config.add_section('Boto') boto.config.setbool('Boto', 'https_validate_certificates', True) # If ca_certificates_file is configured use it; otherwise configure boto to # use the cert roots distributed with gsutil. if not boto.config.get_value('Boto', 'ca_certificates_file', None): disk_certs_file = os.path.abspath( os.path.join(gslib.GSLIB_DIR, 'data', 'cacerts.txt')) if not os.path.exists(disk_certs_file): # If the file is not present on disk, this means the gslib module doesn't # actually exist on disk anywhere. This can happen if it's being imported # from a zip file. Unfortunately, we have to copy the certs file to a # local temp file on disk because the underlying SSL socket requires it # to be a filesystem path. certs_data = pkgutil.get_data('gslib', 'data/cacerts.txt') if not certs_data: raise gslib.exception.CommandException( 'Certificates file not found. Please reinstall gsutil from scratch' ) fd, fname = tempfile.mkstemp(suffix='.txt', prefix='gsutil-cacerts') f = os.fdopen(fd, 'w') f.write(certs_data) f.close() disk_certs_file = fname cleanup_files.append(disk_certs_file) boto.config.set('Boto', 'ca_certificates_file', disk_certs_file) try: try: opts, args = getopt.getopt(sys.argv[1:], 'dDvo:h:mq', [ 'debug', 'detailedDebug', 'version', 'option', 'help', 'header', 'multithreaded', 'quiet' ]) except getopt.GetoptError as e: _HandleCommandException(gslib.exception.CommandException(e.msg)) for o, a in opts: if o in ('-d', '--debug'): # Passing debug=2 causes boto to include httplib header output. debug = 2 elif o in ('-D', '--detailedDebug'): # We use debug level 3 to ask gsutil code to output more detailed # debug output. This is a bit of a hack since it overloads the same # flag that was originally implemented for boto use. And we use -DD # to ask for really detailed debugging (i.e., including HTTP payload). if debug == 3: debug = 4 else: debug = 3 elif o in ('-?', '--help'): _OutputUsageAndExit(command_runner) elif o in ('-h', '--header'): (hdr_name, _, hdr_val) = a.partition(':') if not hdr_name: _OutputUsageAndExit(command_runner) headers[hdr_name.lower()] = hdr_val elif o in ('-m', '--multithreaded'): parallel_operations = True elif o in ('-q', '--quiet'): quiet = True elif o in ('-v', '--version'): version = True elif o in ('-o', '--option'): (opt_section_name, _, opt_value) = a.partition('=') if not opt_section_name: _OutputUsageAndExit(command_runner) (opt_section, _, opt_name) = opt_section_name.partition(':') if not opt_section or not opt_name: _OutputUsageAndExit(command_runner) if not boto.config.has_section(opt_section): boto.config.add_section(opt_section) boto.config.set(opt_section, opt_name, opt_value) httplib2.debuglevel = debug if debug > 1: sys.stderr.write(DEBUG_WARNING) if debug == 2: _ConfigureLogging(level=logging.DEBUG) elif debug > 2: _ConfigureLogging(level=logging.DEBUG) command_runner.RunNamedCommand('ver', ['-l']) config_items = [] try: config_items.extend(boto.config.items('Boto')) config_items.extend(boto.config.items('GSUtil')) except ConfigParser.NoSectionError: pass sys.stderr.write('config_file_list: %s\n' % config_file_list) sys.stderr.write('config: %s\n' % str(config_items)) elif quiet: _ConfigureLogging(level=logging.WARNING) else: _ConfigureLogging(level=logging.INFO) # apiclient and oauth2client use info logging in places that would better # correspond to gsutil's debug logging (e.g., when refreshing access # tokens). oauth2client.client.logger.setLevel(logging.WARNING) apiclient.discovery.logger.setLevel(logging.WARNING) if version: command_name = 'version' elif not args: command_name = 'help' else: command_name = args[0] # Unset http_proxy environment variable if it's set, because it confuses # boto. (Proxies should instead be configured via the boto config file.) if 'http_proxy' in os.environ: if debug > 1: sys.stderr.write( 'Unsetting http_proxy environment variable within gsutil run.\n' ) del os.environ['http_proxy'] return _RunNamedCommandAndHandleExceptions(command_runner, command_name, args[1:], headers, debug, parallel_operations) finally: _Cleanup()
def _RunCp(self, args): command_runner = CommandRunner(GetBotoConfigFileList) command_runner.RunNamedCommand("cp", args=args, headers=self.headers, parallel_operations=False, skip_update_check=True, logging_filters=[self._CpFilter()])
def main(): # Any modules used in initializing multiprocessing variables must be # imported after importing gslib.__main__. # pylint: disable=redefined-outer-name,g-import-not-at-top import gslib.boto_translation import gslib.command import gslib.util from gslib.util import BOTO_IS_SECURE from gslib.util import CERTIFICATE_VALIDATION_ENABLED from gcs_oauth2_boto_plugin import oauth2_client from gslib.util import MultiprocessingIsAvailable if MultiprocessingIsAvailable()[0]: # These setup methods must be called, and, on Windows, they can only be # called from within an "if __name__ == '__main__':" block. gslib.util.InitializeMultiprocessingVariables() gslib.command.InitializeMultiprocessingVariables() gslib.boto_translation.InitializeMultiprocessingVariables() # This needs to be done after gslib.util.InitializeMultiprocessingVariables(), # since otherwise we can't call gslib.util.CreateLock. try: # pylint: disable=unused-import,g-import-not-at-top import gcs_oauth2_boto_plugin gcs_oauth2_boto_plugin.oauth2_helper.SetFallbackClientIdAndSecret( GSUTIL_CLIENT_ID, GSUTIL_CLIENT_NOTSOSECRET) gcs_oauth2_boto_plugin.oauth2_helper.SetLock(CreateLock()) except ImportError: pass global debug global test_exception_traces if not (2, 6) <= sys.version_info[:3] < (3,): raise gslib.exception.CommandException( 'gsutil requires python 2.6 or 2.7.') # In gsutil 4.0 and beyond, we don't use the boto library for the JSON # API. However, we still store gsutil configuration data in the .boto # config file for compatibility with previous versions and user convenience. # Many users have a .boto configuration file from previous versions, and it # is useful to have all of the configuration for gsutil stored in one place. command_runner = CommandRunner() if not BOTO_IS_SECURE: raise CommandException('\n'.join(textwrap.wrap( 'Your boto configuration has is_secure = False. Gsutil cannot be ' 'run this way, for security reasons.'))) headers = {} parallel_operations = False quiet = False version = False debug = 0 test_exception_traces = False # If user enters no commands just print the usage info. if len(sys.argv) == 1: sys.argv.append('help') # Change the default of the 'https_validate_certificates' boto option to # True (it is currently False in boto). if not boto.config.has_option('Boto', 'https_validate_certificates'): if not boto.config.has_section('Boto'): boto.config.add_section('Boto') boto.config.setbool('Boto', 'https_validate_certificates', True) GetCertsFile() try: try: opts, args = getopt.getopt(sys.argv[1:], 'dDvo:h:mq', ['debug', 'detailedDebug', 'version', 'option', 'help', 'header', 'multithreaded', 'quiet', 'testexceptiontraces']) except getopt.GetoptError as e: _HandleCommandException(gslib.exception.CommandException(e.msg)) for o, a in opts: if o in ('-d', '--debug'): # Passing debug=2 causes boto to include httplib header output. debug = 3 elif o in ('-D', '--detailedDebug'): # We use debug level 3 to ask gsutil code to output more detailed # debug output. This is a bit of a hack since it overloads the same # flag that was originally implemented for boto use. And we use -DD # to ask for really detailed debugging (i.e., including HTTP payload). if debug == 3: debug = 4 else: debug = 3 elif o in ('-?', '--help'): _OutputUsageAndExit(command_runner) elif o in ('-h', '--header'): (hdr_name, _, hdr_val) = a.partition(':') if not hdr_name: _OutputUsageAndExit(command_runner) headers[hdr_name.lower()] = hdr_val elif o in ('-m', '--multithreaded'): parallel_operations = True elif o in ('-q', '--quiet'): quiet = True elif o in ('-v', '--version'): version = True elif o == '--testexceptiontraces': # Hidden flag for integration tests. test_exception_traces = True elif o in ('-o', '--option'): (opt_section_name, _, opt_value) = a.partition('=') if not opt_section_name: _OutputUsageAndExit(command_runner) (opt_section, _, opt_name) = opt_section_name.partition(':') if not opt_section or not opt_name: _OutputUsageAndExit(command_runner) if not boto.config.has_section(opt_section): boto.config.add_section(opt_section) boto.config.set(opt_section, opt_name, opt_value) httplib2.debuglevel = debug if debug > 1: sys.stderr.write(DEBUG_WARNING) if debug >= 2: _ConfigureLogging(level=logging.DEBUG) command_runner.RunNamedCommand('ver', ['-l']) config_items = [] try: config_items.extend(boto.config.items('Boto')) config_items.extend(boto.config.items('GSUtil')) except ConfigParser.NoSectionError: pass for i in xrange(len(config_items)): config_item_key = config_items[i][0] if config_item_key in CONFIG_KEYS_TO_REDACT: config_items[i] = (config_item_key, 'REDACTED') sys.stderr.write('Command being run: %s\n' % ' '.join(sys.argv)) sys.stderr.write('config_file_list: %s\n' % GetBotoConfigFileList()) sys.stderr.write('config: %s\n' % str(config_items)) elif quiet: _ConfigureLogging(level=logging.WARNING) else: _ConfigureLogging(level=logging.INFO) # oauth2client uses info logging in places that would better # correspond to gsutil's debug logging (e.g., when refreshing # access tokens). oauth2client.client.logger.setLevel(logging.WARNING) if not CERTIFICATE_VALIDATION_ENABLED: sys.stderr.write(HTTP_WARNING) if version: command_name = 'version' elif not args: command_name = 'help' else: command_name = args[0] # Unset http_proxy environment variable if it's set, because it confuses # boto. (Proxies should instead be configured via the boto config file.) if 'http_proxy' in os.environ: if debug > 1: sys.stderr.write( 'Unsetting http_proxy environment variable within gsutil run.\n') del os.environ['http_proxy'] if os.environ.get('_ARGCOMPLETE', '0') == '1': return _PerformTabCompletion(command_runner) return _RunNamedCommandAndHandleExceptions( command_runner, command_name, args=args[1:], headers=headers, debug_level=debug, parallel_operations=parallel_operations) finally: _Cleanup()
def main(): global debug if not (2, 6) <= sys.version_info[:3] < (3,): raise gslib.exception.CommandException( 'gsutil requires python 2.6 or 2.7.') # Load the gsutil version number and append it to boto.UserAgent so the value # is set before anything instantiates boto. (If parts of boto were # instantiated first those parts would have the old value of boto.UserAgent, # so we wouldn't be guaranteed that all code paths send the correct user # agent.) boto.UserAgent += ' gsutil/%s (%s)' % (gslib.VERSION, sys.platform) config_file_list = GetBotoConfigFileList() command_runner = CommandRunner(config_file_list) headers = {} parallel_operations = False quiet = False version = False debug = 0 # If user enters no commands just print the usage info. if len(sys.argv) == 1: sys.argv.append('help') # Change the default of the 'https_validate_certificates' boto option to # True (it is currently False in boto). if not boto.config.has_option('Boto', 'https_validate_certificates'): if not boto.config.has_section('Boto'): boto.config.add_section('Boto') boto.config.setbool('Boto', 'https_validate_certificates', True) # If ca_certificates_file is configured use it; otherwise configure boto to # use the cert roots distributed with gsutil. if not boto.config.get_value('Boto', 'ca_certificates_file', None): boto.config.set('Boto', 'ca_certificates_file', DEFAULT_CA_CERTS_FILE) try: opts, args = getopt.getopt(sys.argv[1:], 'dDvh:mq', ['debug', 'detailedDebug', 'version', 'help', 'header', 'multithreaded', 'quiet']) except getopt.GetoptError as e: _HandleCommandException(gslib.exception.CommandException(e.msg)) for o, a in opts: if o in ('-d', '--debug'): # Passing debug=2 causes boto to include httplib header output. debug = 2 elif o in ('-D', '--detailedDebug'): # We use debug level 3 to ask gsutil code to output more detailed # debug output. This is a bit of a hack since it overloads the same # flag that was originally implemented for boto use. And we use -DD # to ask for really detailed debugging (i.e., including HTTP payload). if debug == 3: debug = 4 else: debug = 3 elif o in ('-?', '--help'): _OutputUsageAndExit(command_runner) elif o in ('-h', '--header'): (hdr_name, unused_ptn, hdr_val) = a.partition(':') if not hdr_name: _OutputUsageAndExit(command_runner) headers[hdr_name.lower()] = hdr_val elif o in ('-m', '--multithreaded'): parallel_operations = True elif o in ('-q', '--quiet'): quiet = True elif o in ('-v', '--version'): version = True httplib2.debuglevel = debug if debug > 1: sys.stderr.write( '***************************** WARNING *****************************\n' '*** You are running gsutil with debug output enabled.\n' '*** Be aware that debug output includes authentication ' 'credentials.\n' '*** Do not share (e.g., post to support forums) debug output\n' '*** unless you have sanitized authentication tokens in the\n' '*** output, or have revoked your credentials.\n' '***************************** WARNING *****************************\n') if debug == 2: logging.basicConfig(level=logging.DEBUG) elif debug > 2: logging.basicConfig(level=logging.DEBUG) command_runner.RunNamedCommand('ver', ['-l']) config_items = [] try: config_items.extend(boto.config.items('Boto')) config_items.extend(boto.config.items('GSUtil')) except ConfigParser.NoSectionError: pass sys.stderr.write('config_file_list: %s\n' % config_file_list) sys.stderr.write('config: %s\n' % str(config_items)) elif quiet: logging.basicConfig(level=logging.WARNING) else: logging.basicConfig(level=logging.INFO) # apiclient and oauth2client use info logging in places that would better # correspond to gsutil's debug logging (e.g., when refreshing access # tokens). oauth2client.client.logger.setLevel(logging.WARNING) apiclient.discovery.logger.setLevel(logging.WARNING) if version: command_name = 'version' elif not args: command_name = 'help' else: command_name = args[0] # Unset http_proxy environment variable if it's set, because it confuses # boto. (Proxies should instead be configured via the boto config file.) if 'http_proxy' in os.environ: if debug > 1: sys.stderr.write( 'Unsetting http_proxy environment variable within gsutil run.\n') del os.environ['http_proxy'] return _RunNamedCommandAndHandleExceptions(command_runner, command_name, args[1:], headers, debug, parallel_operations)