def test_searching_through_Search_class(): """ Verify if ``pip.vcs.Search`` uses tests xmlrpclib.Transport class """ original_xmlrpclib_transport = pip.download.xmlrpclib_transport pip.download.xmlrpclib_transport = fake_transport = Mock() query = 'mylittlequerythatdoesnotexists' dumped_xmlrpc_request = b( xmlrpclib.dumps(({ 'name': query, 'summary': query }, 'or'), 'search')) expected = [{ '_pypi_ordering': 100, 'name': 'foo', 'summary': 'foo summary', 'version': '1.0' }] fake_transport.request.return_value = (expected, ) pypi_searcher = SearchCommand(create_main_parser()) result = pypi_searcher.search(query, 'http://pypi.python.org/pypi') try: assert expected == result, result fake_transport.request.assert_called_with('pypi.python.org', '/pypi', dumped_xmlrpc_request, verbose=VERBOSE_FALSE) finally: pip.download.xmlrpclib_transport = original_xmlrpclib_transport
def package_finder(argv): """Return a PackageFinder respecting command-line options. :arg argv: Everything after the subcommand """ # We instantiate an InstallCommand and then use some of its private # machinery--its arg parser--for our own purposes, like a virus. This # approach is portable across many pip versions, where more fine-grained # ones are not. Ignoring options that don't exist on the parser (for # instance, --use-wheel) gives us a straightforward method of backward # compatibility. try: command = InstallCommand() except TypeError: # This is likely pip 1.3.0's "__init__() takes exactly 2 arguments (1 # given)" error. In that version, InstallCommand takes a top=level # parser passed in from outside. from pip.baseparser import create_main_parser command = InstallCommand(create_main_parser()) # The downside is that it essentially ruins the InstallCommand class for # further use. Calling out to pip.main() within the same interpreter, for # example, would result in arguments parsed this time turning up there. # Thus, we deepcopy the arg parser so we don't trash its singletons. Of # course, deepcopy doesn't work on these objects, because they contain # uncopyable regex patterns, so we pickle and unpickle instead. Fun! options, _ = loads(dumps(command.parser)).parse_args(argv) # Carry over PackageFinder kwargs that have [about] the same names as # options attr names: possible_options = [ "find_links", "use_wheel", "allow_external", "allow_unverified", "allow_all_external", ("allow_all_prereleases", "pre"), "process_dependency_links", ] kwargs = {} for option in possible_options: kw, attr = option if isinstance(option, tuple) else (option, option) value = getattr(options, attr, MARKER) if value is not MARKER: kwargs[kw] = value # Figure out index_urls: index_urls = [options.index_url] + options.extra_index_urls if options.no_index: index_urls = [] index_urls += getattr(options, "mirrors", []) # If pip is new enough to have a PipSession, initialize one, since # PackageFinder requires it: if hasattr(command, "_build_session"): kwargs["session"] = command._build_session(options) return PackageFinder(index_urls=index_urls, **kwargs)
def test_run_method_should_raise_command_error_when_command_does_not_exist(): """ Test HelpCommand.run for non-existing command """ options_mock = Mock() args = ('mycommand', ) help_cmd = HelpCommand(create_main_parser()) assert_raises(CommandError, help_cmd.run, options_mock, args)
def test_run_method_should_raise_command_error_when_command_does_not_exist(): """ Test HelpCommand.run for non-existing command """ options_mock = Mock() args = ('mycommand',) help_cmd = HelpCommand(create_main_parser()) assert_raises(CommandError, help_cmd.run, options_mock, args)
def test_run_method_should_return_no_matches_found_when_does_not_find_packages(): """ Test SearchCommand.run for no matches """ options_mock = Mock() options_mock.index = 'http://pypi.python.org/pypi' search_cmd = SearchCommand(create_main_parser()) status = search_cmd.run(options_mock, ('non-existant-package',)) assert status == NO_MATCHES_FOUND, status
def test_run_method_should_return_no_matches_found_when_does_not_find_packages(): """ Test SearchCommand.run for no matches """ options_mock = Mock() options_mock.index = 'https://pypi.python.org/pypi' search_cmd = SearchCommand(create_main_parser()) status = search_cmd.run(options_mock, ('non-existant-package',)) assert status == NO_MATCHES_FOUND, status
def test_run_method_should_return_sucess_when_command_name_not_specified(): """ Test HelpCommand.run when there are no args """ options_mock = Mock() args = () help_cmd = HelpCommand(create_main_parser()) status = help_cmd.run(options_mock, args) assert status == SUCCESS
def test_run_method_should_return_sucess_when_finds_command_name(): """ Test HelpCommand.run for existing command """ options_mock = Mock() args = ('freeze', ) help_cmd = HelpCommand(create_main_parser()) status = help_cmd.run(options_mock, args) assert status == SUCCESS
def test_run_method_should_return_sucess_when_finds_command_name(): """ Test HelpCommand.run for existing command """ options_mock = Mock() args = ('freeze',) help_cmd = HelpCommand(create_main_parser()) status = help_cmd.run(options_mock, args) assert status == SUCCESS
def test_run_method_should_return_sucess_when_find_packages(): """ Test SearchCommand.run for found package """ options_mock = Mock() options_mock.index = 'http://pypi.python.org/pypi' search_cmd = SearchCommand(create_main_parser()) status = search_cmd.run(options_mock, ('pip', )) assert status == SUCCESS
def test_run_method_should_return_sucess_when_find_packages(): """ Test SearchCommand.run for found package """ options_mock = Mock() options_mock.index = 'http://pypi.python.org/pypi' search_cmd = SearchCommand(create_main_parser()) status = search_cmd.run(options_mock, ('pip',)) assert status == SUCCESS
def package_finder(argv): """Return a PackageFinder respecting command-line options. :arg argv: Everything after the subcommand """ # We instantiate an InstallCommand and then use some of its private # machinery--its arg parser--for our own purposes, like a virus. This # approach is portable across many pip versions, where more fine-grained # ones are not. Ignoring options that don't exist on the parser (for # instance, --use-wheel) gives us a straightforward method of backward # compatibility. try: command = InstallCommand() except TypeError: # This is likely pip 1.3.0's "__init__() takes exactly 2 arguments (1 # given)" error. In that version, InstallCommand takes a top=level # parser passed in from outside. from pip.baseparser import create_main_parser command = InstallCommand(create_main_parser()) # The downside is that it essentially ruins the InstallCommand class for # further use. Calling out to pip.main() within the same interpreter, for # example, would result in arguments parsed this time turning up there. # Thus, we deepcopy the arg parser so we don't trash its singletons. Of # course, deepcopy doesn't work on these objects, because they contain # uncopyable regex patterns, so we pickle and unpickle instead. Fun! options, _ = loads(dumps(command.parser)).parse_args(argv) # Carry over PackageFinder kwargs that have [about] the same names as # options attr names: possible_options = [ 'find_links', 'use_wheel', 'allow_external', 'allow_unverified', 'allow_all_external', ('allow_all_prereleases', 'pre'), 'process_dependency_links' ] kwargs = {} for option in possible_options: kw, attr = option if isinstance(option, tuple) else (option, option) value = getattr(options, attr, MARKER) if value is not MARKER: kwargs[kw] = value # Figure out index_urls: index_urls = [options.index_url] + options.extra_index_urls if options.no_index: index_urls = [] index_urls += getattr(options, 'mirrors', []) # If pip is new enough to have a PipSession, initialize one, since # PackageFinder requires it: if hasattr(command, '_build_session'): kwargs['session'] = command._build_session(options) return PackageFinder(index_urls=index_urls, **kwargs)
def test_searching_through_Search_class(): """ Verify if ``pip.vcs.Search`` uses tests xmlrpclib.Transport class """ original_xmlrpclib_transport = pip.download.xmlrpclib_transport pip.download.xmlrpclib_transport = fake_transport = Mock() query = 'mylittlequerythatdoesnotexists' dumped_xmlrpc_request = b(xmlrpclib.dumps(({'name': query, 'summary': query}, 'or'), 'search')) expected = [{'_pypi_ordering': 100, 'name': 'foo', 'summary': 'foo summary', 'version': '1.0'}] fake_transport.request.return_value = (expected,) pypi_searcher = SearchCommand(create_main_parser()) result = pypi_searcher.search(query, 'http://pypi.python.org/pypi') try: assert expected == result, result fake_transport.request.assert_called_with('pypi.python.org', '/pypi', dumped_xmlrpc_request, verbose=VERBOSE_FALSE) finally: pip.download.xmlrpclib_transport = original_xmlrpclib_transport
def parseopts(args): parser = create_main_parser() parser.main = True # so the help formatter knows # create command listing command_summaries = get_summaries() description = [''] + ['%-27s %s' % (i, j) for i, j in command_summaries] parser.description = '\n'.join(description) options, args = parser.parse_args(args) if options.version: sys.stdout.write(parser.version) sys.stdout.write(os.linesep) sys.exit() # pip || pip help || pip --help -> print_help() if not args or (args[0] == 'help' and len(args) == 1): parser.print_help() sys.exit() if not args: msg = ('You must give a command ' '(use "pip --help" to see a list of commands)') raise CommandError(msg) command = args[0].lower() if command not in commands: guess = get_similar_commands(command) msg = ['unknown command "%s"' % command] if guess: msg.append('maybe you meant "%s"' % guess) raise CommandError(' - '.join(msg)) return command, options, args, parser
def run_pip(arguments, use_remote_index): """ Execute a modified ``pip install`` command. This function assumes that the arguments concern a ``pip install`` command (:py:func:`main()` makes sure of this). :param arguments: A list of strings containing the arguments that will be passed to ``pip``. :param use_remote_index: A boolean indicating whether ``pip`` is allowed to contact http://pypi.python.org. :returns: A ``RequirementSet`` object created by ``pip``, unless an exception is raised by ``pip`` (in which case the exception will bubble up). """ command_line = [] for i, arg in enumerate(arguments): if arg == 'install': command_line += ['pip'] + arguments[:i+1] + [ '--download-cache=%s' % download_cache, '--find-links=file://%s' % source_index] if not use_remote_index: command_line += ['--no-index'] command_line += arguments[i+1:] break else: command_line = ['pip'] + arguments logger.info("Executing command: %s", ' '.join(command_line)) parser = create_main_parser() pip = CustomInstallCommand(parser) initial_options, args = parser.parse_args(command_line[1:]) exit_status = pip.main(command_line[2:], initial_options) # Make sure the output of pip and pip-accel are not intermingled. sys.stdout.flush() update_source_dists_index() if exit_status == SUCCESS: return pip.requirement_set else: raise pip.intercepted_exception
def run_pip(arguments, use_remote_index): """ Execute a modified `pip install` command. This function assumes that the arguments concern a "pip install" command. Expects two arguments: A list of strings containing the arguments that will be passed to `pip` followed by a boolean indicating whether `pip` may contact http://pypi.python.org. Returns a pip RequirementSet object. If pip reports an error, the exception will be re-raised by the run_pip() function. """ command_line = [] for i, arg in enumerate(arguments): if arg == 'install': command_line += ['pip'] + arguments[:i+1] + [ '--download-cache=%s' % download_cache, '--find-links=file://%s' % source_index] if not use_remote_index: command_line += ['--no-index'] command_line += arguments[i+1:] break else: command_line = ['pip'] + arguments logger.info("Executing command: %s.", ' '.join(command_line)) parser = create_main_parser() pip = CustomInstallCommand(parser) initial_options, args = parser.parse_args(command_line[1:]) exit_status = pip.main(command_line[2:], initial_options) # Make sure the output of pip and pip-accel are not intermingled. sys.stdout.flush() update_source_dists_index() if exit_status == SUCCESS: return pip.requirement_set else: raise pip.intercepted_exception
def bundle_package(self, cache=True): """Makes the pybundle archive (that :program:`pip` can take to install) with completely resolved dependencies. It yields triple of package name, filename of the pybundle archive, and its full path. :: with build.bundle_package() as (package, filename, path): sftp.put(path, filename) :param cache: whether to cache the package file or not. ``True`` by default :type cache: :class:`bool` """ asuka_logger = self.get_logger('bundle_package') # Makes pip.log.logger to forward records to the standard logging if not getattr(type(self), 'initialized', False): type(self).initialized = True logger.consumers.extend([ (Logger.FATAL, asuka_logger.critical), (Logger.ERROR, asuka_logger.error), (Logger.WARN, asuka_logger.warn), (Logger.NOTIFY, asuka_logger.info), (Logger.INFO, asuka_logger.info), (Logger.DEBUG, asuka_logger.debug), (Logger.VERBOSE_DEBUG, asuka_logger.debug) ]) vcs.register(Git) main_parser = create_main_parser() bundle = commands['bundle'](main_parser) with self.archive_package() as (package_name, filename, filepath): bundle_filename = package_name + '.pybundle' if cache: cache_dir_path = os.path.join( tempfile.gettempdir(), 'asuka-pybundle-cache' ) if not os.path.isdir(cache_dir_path): os.makedirs(cache_dir_path) cache_path = os.path.join(cache_dir_path, bundle_filename) if os.path.isfile(cache_path): asuka_logger.info('cache exists: %s, skipping pybundle...', cache_path) yield package_name, bundle_filename, cache_path return tempdir = tempfile.gettempdir() bundle_path = os.path.join( os.path.dirname(filepath), bundle_filename ) asuka_logger.info('pybundle_path = %r', bundle_path) options = optparse.Values() options.editables = [] options.requirements = [] options.find_links = [] options.index_url = PYPI_INDEX_URLS[0] options.extra_index_urls = PYPI_INDEX_URLS[1:] options.no_index = False options.use_mirrors = False options.mirrors = True options.build_dir = os.path.join( tempdir, 'asuka-dist-build-bundle' ) options.target_dir = None options.download_dir = None options.download_cache = os.path.join( tempdir, 'asuka-dist-download-cache' ) options.src_dir = backup_dir(src_prefix, '-bundle') options.upgrade = False options.force_reinstall = False options.ignore_dependencies = False options.no_install = True options.no_download = False options.install_options = [] options.global_options = [] options.use_user_site = False options.as_egg = False asuka_logger.debug('start: pip bundle %s %s', bundle_path, filepath) retrial = 0 while 1: try: shutil.rmtree(options.build_dir) except (OSError, IOError): pass try: bundle.run(options, [bundle_path, filepath]) except PipError as e: asuka_logger.exception(e) retrial += 1 if retrial < 3: asuka_logger.error( 'retry pip bundle after %d second(s)... (%d)', retrial, retrial ** 2 ) options.index_url = PYPI_INDEX_URLS[retrial] options.extra_index_urls = PYPI_INDEX_URLS[retrial+1:] time.sleep(retrial ** 2) continue raise finally: if os.path.isdir(options.build_dir): shutil.rmtree(options.build_dir) break asuka_logger.debug('end: pip bundle %s %s', bundle_path, filepath) if cache: asuka_logger.info('save pybundle cache %s...', cache_path) shutil.copyfile(bundle_path, cache_path) yield package_name, os.path.basename(bundle_path), bundle_path
def autocomplete(): """Command and option completion for the main option parser (and options) and its subcommands (and options). Enable by sourcing one of the completion shell scripts (bash or zsh). """ # Don't complete if user hasn't sourced bash_completion file. if 'PIP_AUTO_COMPLETE' not in os.environ: return cwords = os.environ['COMP_WORDS'].split()[1:] cword = int(os.environ['COMP_CWORD']) try: current = cwords[cword - 1] except IndexError: current = '' subcommands = [cmd for cmd, summary in get_summaries()] options = [] # subcommand try: subcommand_name = [w for w in cwords if w in subcommands][0] except IndexError: subcommand_name = None parser = create_main_parser() # subcommand options if subcommand_name: # special case: 'help' subcommand has no options if subcommand_name == 'help': sys.exit(1) # special case: list locally installed dists for uninstall command if subcommand_name == 'uninstall' and not current.startswith('-'): installed = [] lc = current.lower() for dist in get_installed_distributions(local_only=True): if dist.key.startswith(lc) and dist.key not in cwords[1:]: installed.append(dist.key) # if there are no dists installed, fall back to option completion if installed: for dist in installed: print(dist) sys.exit(1) subcommand = commands[subcommand_name](parser) options += [(opt.get_opt_string(), opt.nargs) for opt in subcommand.parser.option_list_all if opt.help != optparse.SUPPRESS_HELP] # filter out previously specified options from available options prev_opts = [x.split('=')[0] for x in cwords[1:cword - 1]] options = [(x, v) for (x, v) in options if x not in prev_opts] # filter options by current input options = [(k, v) for k, v in options if k.startswith(current)] for option in options: opt_label = option[0] # append '=' to options which require args if option[1]: opt_label += '=' print(opt_label) else: # show main parser options only when necessary if current.startswith('-') or current.startswith('--'): opts = [i.option_list for i in parser.option_groups] opts.append(parser.option_list) opts = (o for it in opts for o in it) subcommands += [ i.get_opt_string() for i in opts if i.help != optparse.SUPPRESS_HELP ] print(' '.join([x for x in subcommands if x.startswith(current)])) sys.exit(1)
def process_options(self): cmd = commands[self.arguments[0]](create_main_parser()) self._format_options(cmd.parser.option_groups[0].option_list, cmd_name=cmd.name)
def autocomplete(): """Command and option completion for the main option parser (and options) and its subcommands (and options). Enable by sourcing one of the completion shell scripts (bash or zsh). """ # Don't complete if user hasn't sourced bash_completion file. if 'PIP_AUTO_COMPLETE' not in os.environ: return cwords = os.environ['COMP_WORDS'].split()[1:] cword = int(os.environ['COMP_CWORD']) try: current = cwords[cword - 1] except IndexError: current = '' subcommands = [cmd for cmd, summary in get_summaries()] options = [] # subcommand try: subcommand_name = [w for w in cwords if w in subcommands][0] except IndexError: subcommand_name = None parser = create_main_parser() # subcommand options if subcommand_name: # special case: 'help' subcommand has no options if subcommand_name == 'help': sys.exit(1) # special case: list locally installed dists for uninstall command if subcommand_name == 'uninstall' and not current.startswith('-'): installed = [] lc = current.lower() for dist in get_installed_distributions(local_only=True): if dist.key.startswith(lc) and dist.key not in cwords[1:]: installed.append(dist.key) # if there are no dists installed, fall back to option completion if installed: for dist in installed: print(dist) sys.exit(1) subcommand = commands[subcommand_name](parser) options += [(opt.get_opt_string(), opt.nargs) for opt in subcommand.parser.option_list_all if opt.help != optparse.SUPPRESS_HELP] # filter out previously specified options from available options prev_opts = [x.split('=')[0] for x in cwords[1:cword - 1]] options = [(x, v) for (x, v) in options if x not in prev_opts] # filter options by current input options = [(k, v) for k, v in options if k.startswith(current)] for option in options: opt_label = option[0] # append '=' to options which require args if option[1]: opt_label += '=' print(opt_label) else: # show main parser options only when necessary if current.startswith('-') or current.startswith('--'): opts = [i.option_list for i in parser.option_groups] opts.append(parser.option_list) opts = (o for it in opts for o in it) subcommands += [i.get_opt_string() for i in opts if i.help != optparse.SUPPRESS_HELP] print(' '.join([x for x in subcommands if x.startswith(current)])) sys.exit(1)
def bundle_package(self, cache=True): """Makes the pybundle archive (that :program:`pip` can take to install) with completely resolved dependencies. It yields triple of package name, filename of the pybundle archive, and its full path. :: with build.bundle_package() as (package, filename, path): sftp.put(path, filename) :param cache: whether to cache the package file or not. ``True`` by default :type cache: :class:`bool` """ asuka_logger = self.get_logger('bundle_package') # Makes pip.log.logger to forward records to the standard logging if not getattr(type(self), 'initialized', False): type(self).initialized = True logger.consumers.extend([(Logger.FATAL, asuka_logger.critical), (Logger.ERROR, asuka_logger.error), (Logger.WARN, asuka_logger.warn), (Logger.NOTIFY, asuka_logger.info), (Logger.INFO, asuka_logger.info), (Logger.DEBUG, asuka_logger.debug), (Logger.VERBOSE_DEBUG, asuka_logger.debug) ]) vcs.register(Git) main_parser = create_main_parser() bundle = commands['bundle'](main_parser) with self.archive_package() as (package_name, filename, filepath): bundle_filename = package_name + '.pybundle' if cache: cache_dir_path = os.path.join(tempfile.gettempdir(), 'asuka-pybundle-cache') if not os.path.isdir(cache_dir_path): os.makedirs(cache_dir_path) cache_path = os.path.join(cache_dir_path, bundle_filename) if os.path.isfile(cache_path): asuka_logger.info('cache exists: %s, skipping pybundle...', cache_path) yield package_name, bundle_filename, cache_path return tempdir = tempfile.gettempdir() bundle_path = os.path.join(os.path.dirname(filepath), bundle_filename) asuka_logger.info('pybundle_path = %r', bundle_path) options = optparse.Values() options.editables = [] options.requirements = [] options.find_links = [] options.index_url = PYPI_INDEX_URLS[0] options.extra_index_urls = PYPI_INDEX_URLS[1:] options.no_index = False options.use_mirrors = False options.mirrors = True options.build_dir = os.path.join(tempdir, 'asuka-dist-build-bundle') options.target_dir = None options.download_dir = None options.download_cache = os.path.join(tempdir, 'asuka-dist-download-cache') options.src_dir = backup_dir(src_prefix, '-bundle') options.upgrade = False options.force_reinstall = False options.ignore_dependencies = False options.no_install = True options.no_download = False options.install_options = [] options.global_options = [] options.use_user_site = False options.as_egg = False asuka_logger.debug('start: pip bundle %s %s', bundle_path, filepath) retrial = 0 while 1: try: shutil.rmtree(options.build_dir) except (OSError, IOError): pass try: bundle.run(options, [bundle_path, filepath]) except PipError as e: asuka_logger.exception(e) retrial += 1 if retrial < 3: asuka_logger.error( 'retry pip bundle after %d second(s)... (%d)', retrial, retrial**2) options.index_url = PYPI_INDEX_URLS[retrial] options.extra_index_urls = PYPI_INDEX_URLS[retrial + 1:] time.sleep(retrial**2) continue raise finally: if os.path.isdir(options.build_dir): shutil.rmtree(options.build_dir) break asuka_logger.debug('end: pip bundle %s %s', bundle_path, filepath) if cache: asuka_logger.info('save pybundle cache %s...', cache_path) shutil.copyfile(bundle_path, cache_path) yield package_name, os.path.basename(bundle_path), bundle_path