def test_returns_on_failures(self): test_server = TestServer([("*/*@*/*", "*")], [("*/*@*/*", "*")]) servers = {"default": test_server} class Response(object): ok = None status_code = None charset = None text = "" headers = {} def __init__(self, ok, status_code): self.ok = ok self.status_code = status_code class BuggyRequester(object): def __init__(self, *args, **kwargs): pass def get(self, *args, **kwargs): return Response(False, 404) client2 = TestClient(servers=servers, requester_class=BuggyRequester) conan_ref = ConanFileReference.loads("Hello/1.2.1@frodo/stable") package_ref = PackageReference(conan_ref, "123123123") installer = ConanProxy(client2.paths, client2.user_io, client2.remote_manager, "default", recorder=ActionRecorder()) with self.assertRaises(NotFoundException): installer.get_recipe(conan_ref) self.assertFalse(installer.package_available(package_ref, False, True)) class BuggyRequester2(BuggyRequester): def get(self, *args, **kwargs): return Response(False, 500) client2 = TestClient(servers=servers, requester_class=BuggyRequester2) installer = ConanProxy(client2.paths, client2.user_io, client2.remote_manager, "default", recorder=ActionRecorder()) try: installer.get_recipe(conan_ref) except NotFoundException: self.assertFalse(True) # Shouldn't capture here except ConanException: pass try: installer.package_available(package_ref, False, True) except NotFoundException: self.assertFalse(True) # Shouldn't capture here except ConanException: pass
def __init__(self, client_cache, user_io, remote_manager, remote, recorder): self._client_cache = client_cache self._user_io = user_io self._remote_proxy = ConanProxy(self._client_cache, self._user_io, remote_manager, remote, recorder) self._cache_search = DiskSearchManager(self._client_cache)
def user(self, remote=None, name=None, password=None): remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, remote) if password == "": if not remote: remote = remote_proxy.registry.default_remote.name name, password = self._user_io.request_login(remote_name=remote, username=name) return remote_proxy.authenticate(name, password)
def package_files(self, reference, path, profile): """ Bundle pre-existing binaries @param reference: ConanFileReference """ conan_file_path = self._client_cache.conanfile(reference) if not os.path.exists(conan_file_path): raise ConanException("Package recipe '%s' does not exist" % str(reference)) current_path = path remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, remote_name=None, update=False, check_updates=False, manifest_manager=None) loader = ConanFileLoader(self._runner, self._client_cache.settings, profile) conanfile = loader.load_virtual([reference], current_path) graph_builder = self._get_graph_builder(loader, False, remote_proxy) deps_graph = graph_builder.load(conanfile) # this is a bit tricky, but works. The loading of a cache package makes the referenced # one, the first of the first level, always existing nodes = deps_graph.direct_requires() _, conanfile = nodes[0] packages_folder = self._client_cache.packages(reference) package_folder = os.path.join(packages_folder, conanfile.info.package_id()) shutil.copytree(path, package_folder) save(os.path.join(package_folder, CONANINFO), conanfile.info.dumps()) # Create the digest for the package digest = FileTreeManifest.create(package_folder) save(os.path.join(package_folder, CONAN_MANIFEST), str(digest))
def package_files(self, reference, source_folder, build_folder, package_folder, profile, force): """ Bundle pre-existing binaries @param reference: ConanFileReference """ conan_file_path = self._client_cache.conanfile(reference) if not os.path.exists(conan_file_path): raise ConanException("Package recipe '%s' does not exist" % str(reference)) current_path = package_folder remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, remote_name=None, update=False, check_updates=False, manifest_manager=None) loader = self.get_loader(profile) conanfile = loader.load_virtual([reference], current_path) graph_builder = self._get_graph_builder(loader, False, remote_proxy) deps_graph = graph_builder.load(conanfile) # this is a bit tricky, but works. The loading of a cache package makes the referenced # one, the first of the first level, always existing nodes = deps_graph.direct_requires() _, conanfile = nodes[0] pkg_id = conanfile.info.package_id() self._user_io.out.info("Packaging to %s" % pkg_id) pkg_reference = PackageReference(reference, pkg_id) dest_package_folder = self._client_cache.package( pkg_reference, short_paths=conanfile.short_paths) if os.path.exists(dest_package_folder): if force: rmdir(dest_package_folder) else: raise ConanException( "Package already exists. Please use --force, -f to overwrite it" ) recipe_hash = self._client_cache.load_manifest(reference).summary_hash conanfile.info.recipe_hash = recipe_hash if source_folder or build_folder: package_output = ScopedOutput(str(reference), self._user_io.out) packager.create_package(conanfile, source_folder, build_folder, dest_package_folder, package_output, local=True) else: # we are specifying a final package shutil.copytree(package_folder, dest_package_folder, symlinks=True) save(os.path.join(dest_package_folder, CONANINFO), conanfile.info.dumps()) # Create the digest for the package digest = FileTreeManifest.create(dest_package_folder) save(os.path.join(dest_package_folder, CONAN_MANIFEST), str(digest))
def download(self, reference, package_ids, remote=None): """ Download conanfile and specified packages to local repository @param reference: ConanFileReference @param package_ids: Package ids or empty for download all @param remote: install only from that remote """ assert (isinstance(reference, ConanFileReference)) remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, remote) package = remote_proxy.search(reference, None) if not package: # Search the reference first, and raise if it doesn't exist raise ConanException("'%s' not found in remote" % str(reference)) if package_ids: remote_proxy.download_packages(reference, package_ids) else: self._user_io.out.info("Getting the complete package list " "from '%s'..." % str(reference)) packages_props = remote_proxy.search_packages(reference, None) if not packages_props: output = ScopedOutput(str(reference), self._user_io.out) output.warn("No remote binary packages found in remote") else: remote_proxy.download_packages(reference, list(packages_props.keys()))
def remove(self, pattern, src=False, build_ids=None, package_ids_filter=None, force=False, remote=None, packages_query=None, outdated=False): """ Remove conans and/or packages @param pattern: string to match packages @param src: Remove src folder @param package_ids_filter: list of ids or [] for all list @param build_ids: list of ids or [] for all list @param remote: search on another origin to get packages info @param force: if True, it will be deleted without requesting anything @param packages_query: Only if src is a reference. Query settings and options """ remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, remote) remover = ConanRemover(self._client_cache, self._search_manager, self._user_io, remote_proxy) remover.remove(pattern, src, build_ids, package_ids_filter, force=force, packages_query=packages_query, outdated=outdated)
def __init__(self, client_cache, user_io, remote_manager, search_manager, remote): self._client_cache = client_cache self._user_io = user_io self._remote_proxy = ConanProxy(self._client_cache, self._user_io, remote_manager, remote) self._search_manager = search_manager
def info_get_graph(self, reference, profile, remote=None, check_updates=False): """ Fetch and build all dependencies for the given reference @param reference: ConanFileReference or path to user space conanfile @param remote: install only from that remote """ remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, remote, update=False, check_updates=check_updates, recorder=self._recorder) deps_graph, graph_builder, conanfile = self._get_deps_graph( reference, profile, remote_proxy) if check_updates: graph_updates_info = graph_builder.get_graph_updates_info( deps_graph) else: graph_updates_info = {} return deps_graph, graph_updates_info, self._get_project_reference( reference, conanfile)
def get_path(self, reference, package_id=None, path=None, remote=None): remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, remote) if not path and not package_id: path = "conanfile.py" elif not path and package_id: path = "conaninfo.txt" return remote_proxy.get_path(reference, package_id, path), path
def info_nodes_to_build(self, reference, profile, build_modes, remote, check_updates): remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, remote, update=False, check_updates=check_updates, recorder=self._recorder) deps_graph, _, conanfile = self._get_deps_graph( reference, profile, remote_proxy) build_mode = BuildMode(build_modes, self._user_io.out) installer = ConanInstaller(self._client_cache, self._user_io.out, remote_proxy, build_mode, None, recorder=self._recorder) nodes = installer.nodes_to_build(deps_graph) counter = Counter(ref.conan.name for ref, _ in nodes) ret = [ ref if counter[ref.conan.name] > 1 else str(ref.conan) for ref, _ in nodes ] return ret, self._get_project_reference(reference, conanfile)
def info_get_graph(self, reference, current_path, profile, remote=None, filename=None, check_updates=False): """ Fetch and build all dependencies for the given reference @param reference: ConanFileReference or path to user space conanfile @param current_path: where the output files will be saved @param remote: install only from that remote @param profile: Profile object with both the -s introduced options and profile readed values @param build_modes: List of build_modes specified @param filename: Optional filename of the conanfile """ remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, remote, update=False, check_updates=check_updates) deps_graph, graph_builder, conanfile = self._get_deps_graph( reference, profile, filename, current_path, remote_proxy) if check_updates: graph_updates_info = graph_builder.get_graph_updates_info( deps_graph) else: graph_updates_info = {} return deps_graph, graph_updates_info, self._get_project_reference( reference, conanfile)
def upload(self, conan_reference, package_id=None, remote=None, all_packages=None, force=False): t1 = time.time() remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, remote) uploader = ConanUploader(self._client_cache, self._user_io, remote_proxy) # Load conanfile to check if the build policy is set to always try: conanfile_path = self._client_cache.conanfile(conan_reference) conan_file = self._loader().load_class(conanfile_path) except NotFoundException: raise NotFoundException("There is no local conanfile exported as %s" % str(conan_reference)) # Can't use build_policy_always here because it's not loaded (only load_class) if conan_file.build_policy == "always" and (all_packages or package_id): raise ConanException("Conanfile has build_policy='always', " "no packages can be uploaded") if package_id: # Upload package uploader.upload_package(PackageReference(conan_reference, package_id)) else: # Upload conans uploader.upload_conan(conan_reference, all_packages=all_packages, force=force) logger.debug("====> Time manager upload: %f" % (time.time() - t1))
def search(self, pattern_or_reference=None, remote=None, ignorecase=True, packages_query=None): """ Print the single information saved in conan.vars about all the packages or the packages which match with a pattern Attributes: pattern = string to match packages remote = search on another origin to get packages info packages_pattern = String query with binary packages properties: "arch=x86 AND os=Windows" """ printer = Printer(self._user_io.out) if remote: remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, remote) adapter = remote_proxy else: adapter = self._search_manager if isinstance(pattern_or_reference, ConanFileReference): packages_props = adapter.search_packages(pattern_or_reference, packages_query) ordered_packages = OrderedDict(sorted(packages_props.items())) try: recipe_hash = self._client_cache.load_manifest(pattern_or_reference).summary_hash except IOError: # It could not exist in local recipe_hash = None printer.print_search_packages(ordered_packages, pattern_or_reference, recipe_hash, packages_query) else: references = adapter.search(pattern_or_reference, ignorecase) printer.print_search_recipes(references, pattern_or_reference)
def upload(self, conan_reference, package_id=None, remote=None, all_packages=None, force=False): remote_proxy = ConanProxy(self._paths, self._user_io, self._remote_manager, remote) uploader = ConanUploader(self._paths, self._user_io, remote_proxy) # Load conanfile to check if the build policy is set to always try: conanfile_path = self._paths.conanfile(conan_reference) output = ScopedOutput(str(conan_reference), self._user_io.out) conan_file = self._loader().load_conan(conanfile_path, output, consumer=False) except NotFoundException: raise NotFoundException( "There is no local conanfile exported as %s" % str(conan_reference)) if conan_file.build_policy_always and (all_packages or package_id): raise ConanException("Conanfile has build_policy='always', " "no packages can be uploaded") if package_id: # Upload package uploader.upload_package( PackageReference(conan_reference, package_id)) else: # Upload conans uploader.upload_conan(conan_reference, all_packages=all_packages, force=force)
def upload(self, conan_reference_or_pattern, package_id=None, remote=None, all_packages=None, force=False, confirm=False, retry=0, retry_wait=0, skip_upload=False): """If package_id is provided, conan_reference_or_pattern is a ConanFileReference""" t1 = time.time() remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, remote) uploader = ConanUploader(self._client_cache, self._user_io, remote_proxy, self._search_manager) if package_id: # Upload package ref = ConanFileReference.loads(conan_reference_or_pattern) uploader.check_reference(ref) uploader.upload_package(PackageReference(ref, package_id), retry=retry, retry_wait=retry_wait, skip_upload=skip_upload) else: # Upload conans uploader.upload(conan_reference_or_pattern, all_packages=all_packages, force=force, confirm=confirm, retry=retry, retry_wait=retry_wait, skip_upload=skip_upload) logger.debug("====> Time manager upload: %f" % (time.time() - t1))
def search_packages(self, reference=None, remote=None, packages_query=None, outdated=False): """ Return the single information saved in conan.vars about all the packages or the packages which match with a pattern Attributes: pattern = string to match packages remote = search on another origin to get packages info packages_pattern = String query with binary packages properties: "arch=x86 AND os=Windows" """ packages_props = self._get_search_adapter(remote).search_packages( reference, packages_query) ordered_packages = OrderedDict(sorted(packages_props.items())) if remote: remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, remote) remote = remote_proxy.registry.remote(remote) manifest = self._remote_manager.get_conan_digest(reference, remote) recipe_hash = manifest.summary_hash else: try: recipe_hash = self._client_cache.load_manifest( reference).summary_hash except IOError: # It could not exist in local recipe_hash = None if outdated and recipe_hash: ordered_packages = filter_outdated(ordered_packages, recipe_hash) return ordered_packages, reference, recipe_hash, packages_query
def copy(self, reference, package_ids, username, channel, force=False): """ Copy or move conanfile (exported) and packages to another user and or channel @param reference: ConanFileReference containing the packages to be moved @param package_ids: list of ids or [] for all list @param username: Destination username @param channel: Destination channel @param remote: install only from that remote """ # It is necessary to make sure the sources are complete before proceeding remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, None) # Now we can actually copy conan_file_path = self._client_cache.conanfile(reference) conanfile = load_conanfile_class(conan_file_path) remote_proxy.complete_recipe_sources(reference, short_paths=conanfile.short_paths) copier = PackageCopier(self._client_cache, self._user_io, conanfile.short_paths) if not package_ids: packages = self._client_cache.packages(reference) if os.path.exists(packages): package_ids = os.listdir(packages) else: package_ids = [] copier.copy(reference, package_ids, username, channel, force)
def _get_graph(self, reference, current_path, remote, options, settings, filename, update, check_updates, manifest_manager, scopes, package_settings, env, package_env): loader = self._loader(current_path, settings, package_settings, options, scopes, env, package_env) # Not check for updates for info command, it'll be checked when dep graph is built remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, remote, update=update, check_updates=check_updates, manifest_manager=manifest_manager) if isinstance(reference, ConanFileReference): project_reference = None conanfile = loader.load_virtual(reference, current_path) is_txt = True else: conanfile_path = reference project_reference = "PROJECT" output = ScopedOutput(project_reference, self._user_io.out) try: if filename and filename.endswith(".txt"): raise NotFoundException("") conan_file_path = os.path.join(conanfile_path, filename or CONANFILE) conanfile = loader.load_conan(conan_file_path, output, consumer=True) is_txt = False if conanfile.name is not None and conanfile.version is not None: project_reference = "%s/%s@" % (conanfile.name, conanfile.version) project_reference += "PROJECT" except NotFoundException: # Load requirements.txt conan_path = os.path.join(conanfile_path, filename or CONANFILE_TXT) conanfile = loader.load_conan_txt(conan_path, output) is_txt = True # build deps graph and install it local_search = None if update else self._search_manager resolver = RequireResolver(self._user_io.out, local_search, remote_proxy) builder = DepsGraphBuilder(remote_proxy, self._user_io.out, loader, resolver) deps_graph = builder.load(None, conanfile) # These lines are so the conaninfo stores the correct complete info if is_txt: conanfile.info.settings = loader._settings.values conanfile.info.full_settings = loader._settings.values conanfile.info.scope = self._current_scopes conanfile.cpp_info = CppInfo(current_path) conanfile.env_info = EnvInfo(current_path) registry = RemoteRegistry(self._client_cache.registry, self._user_io.out) return (builder, deps_graph, project_reference, registry, conanfile, remote_proxy, loader)
def _get_search_adapter(self, remote): if remote: remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, remote) adapter = remote_proxy else: adapter = self._search_manager return adapter
def _prepare_sources(client_cache, user_io, remote_manager, reference): remote_proxy = ConanProxy(client_cache, user_io, remote_manager, None) conan_file_path = client_cache.conanfile(reference) conanfile = load_conanfile_class(conan_file_path) remote_proxy.complete_recipe_sources(conanfile, reference, short_paths=conanfile.short_paths) return conanfile.short_paths
def get_proxy(self, remote_name=None): remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, remote_name=remote_name, recorder=self._recorder, registry=self._registry) return remote_proxy
def export_pkg(self, reference, source_folder, build_folder, package_folder, install_folder, profile, force): conan_file_path = self._client_cache.conanfile(reference) if not os.path.exists(conan_file_path): raise ConanException("Package recipe '%s' does not exist" % str(reference)) remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, remote_name=None, update=False, check_updates=False, manifest_manager=None, recorder=self._recorder) loader = self.get_loader(profile) conanfile = loader.load_virtual([reference], scope_options=True) if install_folder and existing_info_files(install_folder): _load_deps_info(install_folder, conanfile, required=True) graph_builder = self._get_graph_builder(loader, False, remote_proxy) deps_graph = graph_builder.load(conanfile) # this is a bit tricky, but works. The loading of a cache package makes the referenced # one, the first of the first level, always existing nodes = deps_graph.direct_requires() _, conanfile = nodes[0] pkg_id = conanfile.info.package_id() self._user_io.out.info("Packaging to %s" % pkg_id) pkg_reference = PackageReference(reference, pkg_id) dest_package_folder = self._client_cache.package( pkg_reference, short_paths=conanfile.short_paths) if os.path.exists(dest_package_folder): if force: rmdir(dest_package_folder) else: raise ConanException( "Package already exists. Please use --force, -f to " "overwrite it") recipe_hash = self._client_cache.load_manifest(reference).summary_hash conanfile.info.recipe_hash = recipe_hash conanfile.develop = True package_output = ScopedOutput(str(reference), self._user_io.out) if package_folder: packager.export_pkg(conanfile, package_folder, dest_package_folder, package_output) else: packager.create_package(conanfile, source_folder, build_folder, dest_package_folder, install_folder, package_output, local=True)
def remove(self, pattern, src=False, build_ids=None, package_ids_filter=None, force=False, remote=None): """ Remove conans and/or packages @param pattern: string to match packages @param package_ids: list of ids or [] for all list @param remote: search on another origin to get packages info @param force: if True, it will be deleted without requesting anything """ remote_proxy = ConanProxy(self._paths, self._user_io, self._remote_manager, remote) remover = ConanRemover(self.file_manager, self._user_io, remote_proxy) remover.remove(pattern, src, build_ids, package_ids_filter, force=force)
def info_build_order(self, reference, profile, filename, build_order, remote, check_updates, cwd): remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, remote, update=False, check_updates=check_updates) deps_graph, _, _ = self._get_deps_graph(reference, profile, filename, cwd, remote_proxy) result = deps_graph.build_order(build_order) return result
def download(self, reference, package_ids, remote=None): """ Download conanfile and specified packages to local repository @param reference: ConanFileReference @param package_ids: Package ids or empty for download all @param remote: install only from that remote """ assert(isinstance(reference, ConanFileReference)) remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, remote) if package_ids: remote_proxy.download_packages(reference, package_ids) else: # Not specified packages, download all packages_props = remote_proxy.search_packages(reference, None) if not packages_props: # No filter by properties raise ConanException("'%s' not found in remote" % str(reference)) remote_proxy.download_packages(reference, list(packages_props.keys()))
def search(self, pattern=None, remote=None, ignorecase=True, verbose=False, extra_verbose=False, package_pattern=None): """ Print the single information saved in conan.vars about all the packages or the packages which match with a pattern Attributes: pattern = string to match packages remote = search on another origin to get packages info """ if remote: remote_proxy = ConanProxy(self._paths, self._user_io, self._remote_manager, remote) info = remote_proxy.search(pattern, ignorecase) else: info = self.file_manager.search(pattern, ignorecase) filtered_info = info # Filter packages if package_pattern if package_pattern: try: # Prepare ER to be more user natural if ".*" not in package_pattern: package_pattern = package_pattern.replace("*", ".*") # Compile expression package_pattern = re.compile(package_pattern, re.IGNORECASE) filtered_info = SearchInfo() for conan_ref, packages in sorted(info.items()): filtered_packages = { pid: data for pid, data in packages.items() if package_pattern.match(pid) } if filtered_packages: filtered_info[conan_ref] = filtered_packages except Exception: # Invalid pattern raise ConanException("Invalid package pattern") printer = Printer(self._user_io.out) printer.print_search(filtered_info, pattern, verbose, extra_verbose)
def download(self, reference, package_ids, remote=None): """ Download conanfile and specified packages to local repository @param reference: ConanFileReference @param package_ids: Package ids or empty for download all @param remote: install only from that remote """ assert(isinstance(reference, ConanFileReference)) remote_proxy = ConanProxy(self._paths, self._user_io, self._remote_manager, remote) if package_ids: remote_proxy.download_packages(reference, package_ids) else: # Not specified packages, download all info = remote_proxy.search(str(reference), ignorecase=False) if reference not in info: remote = remote or self._remote_manager.default_remote raise ConanException("'%s' not found in remote '%s'" % (str(reference), remote)) remote_proxy.download_packages(reference, list(info[reference].keys()))
def upload(self, conan_reference, package_id=None, remote=None, all_packages=None, force=False): remote_proxy = ConanProxy(self._paths, self._user_io, self._remote_manager, remote) uploader = ConanUploader(self._paths, self._user_io, remote_proxy) if package_id: # Upload package uploader.upload_package( PackageReference(conan_reference, package_id)) else: # Upload conans uploader.upload_conan(conan_reference, all_packages=all_packages, force=force)
def download(self, reference, package_ids, remote, recipe): """ Download conanfile and specified packages to local repository @param reference: ConanFileReference @param package_ids: Package ids or empty for download all @param remote: install only from that remote @param only_recipe: download only the recipe """ assert (isinstance(reference, ConanFileReference)) remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, remote, recorder=self._recorder) remote, _ = remote_proxy._get_remote() package = self._remote_manager.search_recipes(remote, reference, None) if not package: # Search the reference first, and raise if it doesn't exist raise ConanException("'%s' not found in remote" % str(reference)) # First of all download package recipe remote_proxy.get_recipe(reference) if recipe: return # Download the sources too, don't be lazy conan_file_path = self._client_cache.conanfile(reference) conanfile = load_conanfile_class(conan_file_path) remote_proxy.complete_recipe_sources(conanfile, reference, short_paths=conanfile.short_paths) if package_ids: remote_proxy.download_packages(reference, package_ids) else: self._user_io.out.info("Getting the complete package list " "from '%s'..." % str(reference)) packages_props = self._remote_manager.search_packages( remote, reference, None) if not packages_props: output = ScopedOutput(str(reference), self._user_io.out) output.warn("No remote binary packages found in remote") else: remote_proxy.download_packages(reference, list(packages_props.keys()))