Exemple #1
0
    def child_build_test(self):
        client = TestClient()
        repo_folder = client.current_folder
        build_folder = os.path.join(client.current_folder, "build")
        mkdir(build_folder)
        package_folder = os.path.join(build_folder, "package")
        mkdir(package_folder)
        client.save({"conanfile.py": conanfile_out})

        client.current_folder = build_folder
        client.run("install ..")
        client.run("source ..")
        client.run("build .. --source-folder=.")
        client.current_folder = package_folder
        client.run("package ../.. --build-folder=../")
        self._assert_pkg(package_folder)
        rmdir(package_folder)
        client.current_folder = repo_folder

        client.run("export-pkg . Pkg/0.1@lasote/testing -bf=./build")

        ref = ConanFileReference.loads("Pkg/0.1@lasote/testing")
        cache_package_folder = client.client_cache.packages(ref)
        cache_package_folder = os.path.join(cache_package_folder,
                                            os.listdir(cache_package_folder)[0])
        self._assert_pkg(cache_package_folder)
Exemple #2
0
    def install_outdated_and_dep_test(self):
        # regression test for https://github.com/conan-io/conan/issues/1053
        # A new recipe that depends on Hello0/0.1
        new_client = TestClient(servers=self.servers,
                                users={"default": [("lasote", "mypass")]})
        files = cpp_hello_conan_files("Hello1", "0.1", ["Hello0/0.1@lasote/stable"], build=False)
        new_client.save(files)
        new_client.run("export . lasote/stable")
        self.assertIn("A new conanfile.py version was exported", new_client.user_io.out)
        # It will retrieve from the remote Hello0 and build Hello1
        new_client.run("install Hello1/0.1@lasote/stable --build missing")

        # Then modify REMOTE Hello0 recipe files (WITH THE OTHER CLIENT)
        files = cpp_hello_conan_files("Hello0", "0.1", build=False)
        files["conanfile.py"] = files["conanfile.py"] + "\n#MODIFIED RECIPE"
        self.client.save(files)
        self.client.run("export . lasote/stable")
        self.assertIn("A new conanfile.py version was exported", self.client.user_io.out)
        self.client.run("install Hello0/0.1@lasote/stable --build missing")
        # Upload only the recipe, so the package is outdated in the server
        self.client.run("upload Hello0/0.1@lasote/stable")

        # Now, with the new_client, remove only the binary package from Hello0
        rmdir(new_client.paths.packages(self.ref))
        # And try to install Hello1 again, should not complain because the remote
        # binary is in the "same version" than local cached Hello0
        new_client.run("install Hello1/0.1@lasote/stable --build outdated --build Hello1")
        self.assertIn("Downloading conan_package.tgz", new_client.user_io.out)
        self.assertIn("Hello1/0.1@lasote/stable: WARN: Forced build from source",
                      new_client.user_io.out)
Exemple #3
0
    def get_recipe(self, conan_reference, remote):
        """
        Read the conans from remotes
        Will iterate the remotes to find the conans unless remote was specified

        returns (dict relative_filepath:abs_path , remote_name)"""
        dest_folder = self._client_cache.export(conan_reference)
        rmdir(dest_folder)

        def filter_function(urls):
            if CONANFILE not in list(urls.keys()):
                raise NotFoundException("Conan '%s' doesn't have a %s!"
                                        % (conan_reference, CONANFILE))
            urls.pop(EXPORT_SOURCES_TGZ_NAME, None)
            return urls

        t1 = time.time()
        urls = self._call_remote(remote, "get_recipe_urls", conan_reference)
        urls = filter_function(urls)
        if not urls:
            return conan_reference

        zipped_files = self._call_remote(remote, "download_files_to_folder", urls, dest_folder)

        duration = time.time() - t1
        log_recipe_download(conan_reference, duration, remote, zipped_files)

        unzip_and_get_files(zipped_files, dest_folder, EXPORT_TGZ_NAME)
        # Make sure that the source dir is deleted
        rm_conandir(self._client_cache.source(conan_reference))
        touch_folder(dest_folder)
Exemple #4
0
def create_package(conanfile, build_folder, package_folder, output):
    """ copies built artifacts, libs, headers, data, etc from build_folder to
    package folder
    """
    mkdir(package_folder)

    # Make the copy of all the patterns
    output.info("Generating the package")
    output.info("Package folder %s" % (package_folder))
    conanfile.copy = FileCopier(build_folder, package_folder)

    def wrap(dst_folder):
        def new_method(pattern, src=""):
            conanfile.copy(pattern, dst_folder, src)
        return new_method

    conanfile.copy_headers = wrap(DEFAULT_INCLUDE)
    conanfile.copy_libs = wrap(DEFAULT_LIB)
    conanfile.copy_bins = wrap(DEFAULT_BIN)
    conanfile.copy_res = wrap(DEFAULT_RES)
    try:
        conanfile.package_folder = package_folder
        conanfile.package()
    except Exception as e:
        os.chdir(build_folder)
        try:
            rmdir(package_folder)
        except Exception as e_rm:
            output.error("Unable to remove package folder %s\n%s"
                                    % (package_folder, str(e_rm)))
            output.warn("**** Please delete it manually ****")
        raise ConanException("%s: %s" % (conanfile.name, str(e)))

    _create_aux_files(build_folder, package_folder)
    output.success("Package '%s' created" % os.path.basename(package_folder))
Exemple #5
0
    def get_package(self, package_ref, short_paths):
        """ obtain a package, either from disk or retrieve from remotes if necessary
        and not necessary to build
        """
        output = ScopedOutput(str(package_ref.conan), self._out)
        package_folder = self._client_cache.package(package_ref, short_paths=short_paths)

        # Check current package status
        if os.path.exists(package_folder):
            if self._check_updates:
                read_manifest = self._client_cache.load_package_manifest(package_ref)
                try:  # get_conan_digest can fail, not in server
                    upstream_manifest = self.get_package_digest(package_ref)
                    if upstream_manifest != read_manifest:
                        if upstream_manifest.time > read_manifest.time:
                            output.warn("Current package is older than remote upstream one")
                            if self._update:
                                output.warn("Removing it to retrieve or build an updated one")
                                rmdir(package_folder)
                        else:
                            output.warn("Current package is newer than remote upstream one")
                except ConanException:
                    pass

        installed = False
        local_package = os.path.exists(package_folder)
        if local_package:
            output.info('Already installed!')
            installed = True
            log_package_got_from_local_cache(package_ref)
        else:
            installed = self._retrieve_remote_package(package_ref, package_folder,
                                                      output)
        self.handle_package_manifest(package_ref, installed)
        return installed
Exemple #6
0
    def get_recipe(self, conan_reference, dest_folder, remote):
        """
        Read the conans from remotes
        Will iterate the remotes to find the conans unless remote was specified

        returns (dict relative_filepath:abs_path , remote_name)"""
        rmdir(dest_folder)  # Remove first the destination folder
        t1 = time.time()

        def filter_function(urls):
            if CONANFILE not in list(urls.keys()):
                raise NotFoundException("Conan '%s' doesn't have a %s!"
                                        % (conan_reference, CONANFILE))
            urls.pop(EXPORT_SOURCES_TGZ_NAME, None)
            return urls

        zipped_files = self._call_remote(remote, "get_recipe", conan_reference, dest_folder,
                                         filter_function)
        duration = time.time() - t1
        log_recipe_download(conan_reference, duration, remote, zipped_files)

        unzip_and_get_files(zipped_files, dest_folder, EXPORT_TGZ_NAME)
        # Make sure that the source dir is deleted
        rm_conandir(self._client_cache.source(conan_reference))
        for dirname, _, filenames in os.walk(dest_folder):
            for fname in filenames:
                touch(os.path.join(dirname, fname))
Exemple #7
0
def configuration_install(item, client_cache, output, verify_ssl, requester,
                          config_type=None, args=None):
    tmp_folder = os.path.join(client_cache.conan_folder, "tmp_config_install")
    # necessary for Mac OSX, where the temp folders in /var/ are symlinks to /private/var/
    tmp_folder = os.path.realpath(tmp_folder)
    mkdir(tmp_folder)
    try:
        if item is None:
            try:
                item = client_cache.conan_config.get_item("general.config_install")
            except ConanException:
                raise ConanException("Called config install without arguments and "
                                     "'general.config_install' not defined in conan.conf")

        if item.endswith(".git") or config_type == "git":
            _process_git_repo(item, client_cache, output, tmp_folder, verify_ssl, args)
        elif os.path.isdir(item):
            _process_folder(item, client_cache, output)
        elif os.path.isfile(item):
            _process_zip_file(item, client_cache, output, tmp_folder)
        elif item.startswith("http"):
            _process_download(item, client_cache, output, tmp_folder, verify_ssl,
                              requester=requester)
        else:
            raise ConanException("I don't know how to process %s" % item)
    finally:
        if item:
            client_cache.conan_config.set_item("general.config_install", item)
        rmdir(tmp_folder)
Exemple #8
0
    def package(self, reference, package_id):
        # Package paths
        conan_file_path = self._client_cache.conanfile(reference)
        if not os.path.exists(conan_file_path):
            raise ConanException("Package recipe '%s' does not exist" % str(reference))

        if not package_id:
            packages = [PackageReference(reference, packid)
                        for packid in self._client_cache.conan_builds(reference)]
            if not packages:
                raise NotFoundException("%s: Package recipe has not been built locally\n"
                                        "Please read the 'conan package' command help\n"
                                        "Use 'conan install' or 'conan test_package' to build and "
                                        "create binaries" % str(reference))
        else:
            packages = [PackageReference(reference, package_id)]

        for package_reference in packages:
            build_folder = self._client_cache.build(package_reference, short_paths=None)
            if not os.path.exists(build_folder):
                raise NotFoundException("%s: Package binary '%s' folder doesn't exist\n"
                                        "Please read the 'conan package' command help\n"
                                        "Use 'conan install' or 'conan test_package' to build and "
                                        "create binaries"
                                        % (str(reference), package_reference.package_id))
            # The package already exist, we can use short_paths if they were defined
            package_folder = self._client_cache.package(package_reference, short_paths=None)
            # Will read current conaninfo with specified options and load conanfile with them
            output = ScopedOutput(str(reference), self._user_io.out)
            output.info("Re-packaging %s" % package_reference.package_id)
            loader = self._loader(build_folder)
            conanfile = loader.load_conan(conan_file_path, self._user_io.out)
            self._load_deps_info(build_folder, conanfile, output)
            rmdir(package_folder)
            packager.create_package(conanfile, build_folder, package_folder, output)
Exemple #9
0
    def package(self, reference, package_id, only_manifest, package_all):
        assert(isinstance(reference, ConanFileReference))

        # Package paths
        conan_file_path = self._paths.conanfile(reference)

        if package_all:
            if only_manifest:
                packages_dir = self._paths.packages(reference)
            else:
                packages_dir = self._paths.builds(reference)
            if not os.path.exists(packages_dir):
                raise NotFoundException('%s does not exist' % str(reference))
            packages = [PackageReference(reference, packid)
                        for packid in os.listdir(packages_dir)]
        else:
            packages = [PackageReference(reference, package_id)]

        for package_reference in packages:
            package_folder = self._paths.package(package_reference)
            # Will read current conaninfo with specified options and load conanfile with them
            if not only_manifest:
                self._user_io.out.info("Packaging %s" % package_reference.package_id)
                build_folder = self._paths.build(package_reference)
                loader = self._loader(build_folder)
                conanfile = loader.load_conan(conan_file_path, self._user_io.out)
                rmdir(package_folder)
                output = ScopedOutput(str(reference), self._user_io.out)
                packager.create_package(conanfile, build_folder, package_folder, output)
            else:
                self._user_io.out.info("Creating manifest for %s" % package_reference.package_id)
                if not os.path.exists(package_folder):
                    raise NotFoundException('Package %s does not exist' % str(package_reference))
                packager.generate_manifest(package_folder)
Exemple #10
0
def _recreate_folders(destination_folder):
    try:
        if os.path.exists(destination_folder):
            rmdir(destination_folder)
        os.makedirs(destination_folder)
    except Exception as e:
        raise ConanException("Unable to create folder %s\n%s" % (destination_folder, str(e)))
Exemple #11
0
    def get_recipe(self, ref, remote):
        """
        Read the conans from remotes
        Will iterate the remotes to find the conans unless remote was specified

        returns (dict relative_filepath:abs_path , remote_name)"""

        self._hook_manager.execute("pre_download_recipe", reference=ref, remote=remote)
        dest_folder = self._cache.package_layout(ref).export()
        rmdir(dest_folder)

        ref = self._resolve_latest_ref(ref, remote)

        t1 = time.time()
        zipped_files = self._call_remote(remote, "get_recipe", ref, dest_folder)
        duration = time.time() - t1
        log_recipe_download(ref, duration, remote.name, zipped_files)

        recipe_checksums = calc_files_checksum(zipped_files)

        unzip_and_get_files(zipped_files, dest_folder, EXPORT_TGZ_NAME, output=self._output)
        # Make sure that the source dir is deleted
        package_layout = self._cache.package_layout(ref)
        rm_conandir(package_layout.source())
        touch_folder(dest_folder)
        conanfile_path = package_layout.conanfile()

        with package_layout.update_metadata() as metadata:
            metadata.recipe.revision = ref.revision
            metadata.recipe.checksums = recipe_checksums

        self._hook_manager.execute("post_download_recipe", conanfile_path=conanfile_path,
                                   reference=ref, remote=remote)

        return ref
Exemple #12
0
 def install_build_and_test(self, conanfile_abs_path, reference, profile,
                            remote, update, build_modes=None, manifest_folder=None,
                            manifest_verify=False, manifest_interactive=False, keep_build=False,
                            test_build_folder=None):
     """
     Installs the reference (specified by the parameters or extracted from the test conanfile)
     and builds the test_package/conanfile.py running the test() method.
     """
     base_folder = os.path.dirname(conanfile_abs_path)
     test_build_folder, delete_after_build = self._build_folder(test_build_folder, profile, base_folder)
     rmdir(test_build_folder)
     if build_modes is None:
         build_modes = ["never"]
     try:
         self._manager.install(inject_require=reference,
                               reference=conanfile_abs_path,
                               install_folder=test_build_folder,
                               remote_name=remote,
                               profile=profile,
                               update=update,
                               build_modes=build_modes,
                               manifest_folder=manifest_folder,
                               manifest_verify=manifest_verify,
                               manifest_interactive=manifest_interactive,
                               keep_build=keep_build)
         self._manager.build(conanfile_abs_path, base_folder, test_build_folder, package_folder=None,
                             install_folder=test_build_folder, test=str(reference))
     finally:
         if delete_after_build:
             os.chdir(base_folder)    # Required for windows where deleting the cwd is not possible.
             rmdir(test_build_folder)
Exemple #13
0
    def install_outdated_dep_test(self):
        # A new recipe that depends on Hello0/0.1
        new_client = TestClient(servers=self.servers,
                                users={"default": [("lasote", "mypass")]})
        files = cpp_hello_conan_files("Hello1", "0.1", ["Hello0/0.1@lasote/stable"], build=False)
        new_client.save(files)
        new_client.run("export lasote/stable")
        self.assertIn("A new conanfile.py version was exported", new_client.user_io.out)
        # It will retrieve from the remote Hello0 and build Hello1
        new_client.run("install Hello1/0.1@lasote/stable --build missing")

        # Then modify REMOTE Hello0 recipe files (WITH THE OTHER CLIENT)
        files = cpp_hello_conan_files("Hello0", "0.1", build=False)
        files["conanfile.py"] = files["conanfile.py"] + "\n#MODIFIED RECIPE"
        self.client.save(files)
        self.client.run("export lasote/stable")
        self.assertIn("A new conanfile.py version was exported", self.client.user_io.out)
        self.client.run("install Hello0/0.1@lasote/stable --build missing")
        # Upload only the recipe, so the package is outdated in the server
        self.client.run("upload Hello0/0.1@lasote/stable")

        # Now, with the new_client, remove only the binary package from Hello0
        rmdir(new_client.paths.packages(self.ref))
        # And try to install Hello1 again, should not complain because the remote
        # binary is in the "same version" than local cached Hello0
        new_client.run("install Hello1/0.1@lasote/stable --build outdated")
        self.assertIn("Downloading conan_package.tgz", new_client.user_io.out)
        self.assertIn("Hello0/0.1@lasote/stable: Package is up to date", new_client.user_io.out)

        # But if we remove the full Hello0 local package, will retrieve the updated
        # recipe and the outdated package
        new_client.run("remove Hello0* -f")
        new_client.run("install Hello1/0.1@lasote/stable --build outdated")
        self.assertIn("Hello0/0.1@lasote/stable: Outdated package!", new_client.user_io.out)
        self.assertIn("Hello0/0.1@lasote/stable: Building your package", new_client.user_io.out)
Exemple #14
0
    def test_deleted_source_folder(self):
        path, _ = create_local_git_repo({"myfile": "contents"},
                                        branch="my_release")
        conanfile = base_git.format(url=_quoted("auto"), revision="auto")
        self.client.save({
            "conanfile.py": conanfile,
            "myfile.txt": "My file is copied"
        })
        create_local_git_repo(folder=self.client.current_folder)
        self.client.run_command('git remote add origin "%s"' %
                                path.replace("\\", "/"))
        self.client.run_command('git push origin master')
        self.client.run("export . user/channel")

        # delete old source, but it doesn't matter because the sources are in the cache
        rmdir(self.client.current_folder)
        new_curdir = temp_folder()
        self.client.current_folder = new_curdir

        self.client.run("install lib/0.1@user/channel --build")
        self.assertNotIn(
            "Getting sources from url: '%s'" % path.replace("\\", "/"),
            self.client.out)

        # If the remove the source folder, then it is fetched from the "remote" doing an install
        self.client.run("remove lib/0.1@user/channel -f -s")
        self.client.run("install lib/0.1@user/channel --build")
        self.assertIn(
            "SCM: Getting sources from url: '%s'" % path.replace("\\", "/"),
            self.client.out)
Exemple #15
0
    def get_recipe_sources(self, ref, export_folder, export_sources_folder,
                           remote):
        assert ref.revision, "get_recipe_sources requires RREV"
        t1 = time.time()

        zipped_files = self._call_remote(remote, "get_recipe_sources", ref,
                                         export_folder)
        if not zipped_files:
            mkdir(export_sources_folder
                  )  # create the folder even if no source files
            return

        duration = time.time() - t1
        log_recipe_sources_download(ref, duration, remote.name, zipped_files)

        unzip_and_get_files(zipped_files,
                            export_sources_folder,
                            EXPORT_SOURCES_TGZ_NAME,
                            output=self._output)
        # REMOVE in Conan 2.0
        c_src_path = os.path.join(export_sources_folder,
                                  EXPORT_SOURCES_DIR_OLD)
        if os.path.exists(c_src_path):
            merge_directories(c_src_path, export_sources_folder)
            rmdir(c_src_path)
        touch_folder(export_sources_folder)
Exemple #16
0
    def copy(self, reference, package_ids, username, channel, force=False):
        assert(isinstance(reference, ConanFileReference))
        dest_ref = "%s/%s@%s/%s" % (reference.name, reference.version, username, channel)
        dest_ref = ConanFileReference.loads(dest_ref)
        # Copy export
        export_origin = self._paths.export(reference)
        if not os.path.exists(export_origin):
            raise ConanException("'%s' doesn't exist" % str(reference))
        export_dest = self._paths.export(dest_ref)
        if os.path.exists(export_dest):
            if not force and not self._user_io.request_boolean("'%s' already exist. Override?"
                                                               % str(dest_ref)):
                return
            rmdir(export_dest)
        shutil.copytree(export_origin, export_dest)
        self._user_io.out.info("Copied %s to %s" % (str(reference), str(dest_ref)))

        # Copy packages
        for package_id in package_ids:
            package_origin = PackageReference(reference, package_id)
            package_dest = PackageReference(dest_ref, package_id)
            package_path_origin = self._paths.package(package_origin)
            package_path_dest = self._paths.package(package_dest)
            if os.path.exists(package_path_dest):
                if not force and not self._user_io.request_boolean("Package '%s' already exist."
                                                                   " Override?"
                                                                   % str(package_id)):
                    continue
                rmdir(package_path_dest)
            shutil.copytree(package_path_origin, package_path_dest)
            self._user_io.out.info("Copied %s to %s" % (str(package_id), str(dest_ref)))
Exemple #17
0
def configuration_install(item, client_cache, output, verify_ssl):
    tmp_folder = os.path.join(client_cache.conan_folder, "tmp_config_install")
    # necessary for Mac OSX, where the temp folders in /var/ are symlinks to /private/var/
    tmp_folder = os.path.realpath(tmp_folder)
    mkdir(tmp_folder)
    try:
        if item is None:
            try:
                item = client_cache.conan_config.get_item("general.config_install")
            except ConanException:
                raise ConanException("Called config install without arguments and "
                                     "'general.config_install' not defined in conan.conf")

        if item.endswith(".git"):
            _process_git_repo(item, client_cache, output, tmp_folder, verify_ssl)
        elif os.path.exists(item):
            # is a local file
            _process_zip_file(item, client_cache, output, tmp_folder)
        elif item.startswith("http"):
            _process_download(item, client_cache, output, tmp_folder, verify_ssl)
        else:
            raise ConanException("I don't know how to process %s" % item)
    finally:
        if item:
            client_cache.conan_config.set_item("general.config_install", item)
        rmdir(tmp_folder)
Exemple #18
0
def rm_conandir(path):
    """removal of a directory that might contain a link to a short path"""
    link = os.path.join(path, CONAN_LINK)
    if os.path.exists(link):
        short_path = load(link)
        rmdir(os.path.dirname(short_path))
    rmdir(path)
Exemple #19
0
    def child_build_test(self):
        client = TestClient()
        build_folder = os.path.join(client.current_folder, "build")
        mkdir(build_folder)
        package_folder = os.path.join(build_folder, "package")
        mkdir(package_folder)
        client.save({"conanfile.py": conanfile,
                     "file.h": "file_h_contents!"})

        client.current_folder = build_folder
        client.run("install ..")
        client.run("build ..")
        client.current_folder = package_folder
        client.run("package ../.. --build-folder=../")
        self._assert_pkg(package_folder)
        rmdir(package_folder)  # IMPORTANT: Symptom that package + package_folder is not fitting
        # well now. (To discuss)
        # But I think now you choose you way to develop, local or cache, if you use conan export-pkg
        # you are done, if you use package() you need the "conan project" feature
        client.current_folder = build_folder
        client.run("export-pkg .. Pkg/0.1@lasote/testing --source-folder=.. ")

        ref = ConanFileReference.loads("Pkg/0.1@lasote/testing")
        cache_package_folder = client.client_cache.packages(ref)
        cache_package_folder = os.path.join(cache_package_folder,
                                            os.listdir(cache_package_folder)[0])
        self._assert_pkg(cache_package_folder)
Exemple #20
0
    def install_outdated_and_dep_test(self):
        # regression test for https://github.com/conan-io/conan/issues/1053
        # A new recipe that depends on Hello0/0.1
        new_client = TestClient(servers=self.servers,
                                users={"default": [("lasote", "mypass")]})
        files = cpp_hello_conan_files("Hello1", "0.1", ["Hello0/0.1@lasote/stable"], build=False)
        new_client.save(files)
        new_client.run("export lasote/stable")
        self.assertIn("A new conanfile.py version was exported", new_client.user_io.out)
        # It will retrieve from the remote Hello0 and build Hello1
        new_client.run("install Hello1/0.1@lasote/stable --build missing")

        # Then modify REMOTE Hello0 recipe files (WITH THE OTHER CLIENT)
        files = cpp_hello_conan_files("Hello0", "0.1", build=False)
        files["conanfile.py"] = files["conanfile.py"] + "\n#MODIFIED RECIPE"
        self.client.save(files)
        self.client.run("export lasote/stable")
        self.assertIn("A new conanfile.py version was exported", self.client.user_io.out)
        self.client.run("install Hello0/0.1@lasote/stable --build missing")
        # Upload only the recipe, so the package is outdated in the server
        self.client.run("upload Hello0/0.1@lasote/stable")

        # Now, with the new_client, remove only the binary package from Hello0
        rmdir(new_client.paths.packages(self.ref))
        # And try to install Hello1 again, should not complain because the remote
        # binary is in the "same version" than local cached Hello0
        new_client.run("install Hello1/0.1@lasote/stable --build outdated --build Hello1")
        self.assertIn("Downloading conan_package.tgz", new_client.user_io.out)
        self.assertIn("Hello1/0.1@lasote/stable: WARN: Forced build from source",
                      new_client.user_io.out)
Exemple #21
0
def _remove_folder_raising(folder):
    try:
        rmdir(folder)
    except OSError as e:
        raise ConanException(
            "%s\n\nCouldn't remove folder, might be busy or open\n"
            "Close any app using it, and retry" % str(e))
    def install_outdated_dep_test(self):
        # A new recipe that depends on Hello0/0.1
        new_client = TestClient(servers=self.servers,
                                users={"default": [("lasote", "mypass")]})
        files = cpp_hello_conan_files("Hello1", "0.1", ["Hello0/0.1@lasote/stable"], build=False)
        new_client.save(files)
        new_client.run("export . lasote/stable")
        self.assertIn("A new conanfile.py version was exported", new_client.user_io.out)
        # It will retrieve from the remote Hello0 and build Hello1
        new_client.run("install Hello1/0.1@lasote/stable --build missing")

        # Then modify REMOTE Hello0 recipe files (WITH THE OTHER CLIENT)
        files = cpp_hello_conan_files("Hello0", "0.1", build=False)
        files["conanfile.py"] = files["conanfile.py"] + "\n#MODIFIED RECIPE"
        self.client.save(files)
        self.client.run("export . lasote/stable")
        self.assertIn("A new conanfile.py version was exported", self.client.user_io.out)
        self.client.run("install Hello0/0.1@lasote/stable --build missing")
        # Upload only the recipe, so the package is outdated in the server
        self.client.run("upload Hello0/0.1@lasote/stable")

        # Now, with the new_client, remove only the binary package from Hello0
        rmdir(new_client.paths.packages(self.ref))
        # And try to install Hello1 again, should not complain because the remote
        # binary is in the "same version" than local cached Hello0
        new_client.run("install Hello1/0.1@lasote/stable --build outdated")
        self.assertIn("Downloading conan_package.tgz", new_client.user_io.out)
        self.assertIn("Hello0/0.1@lasote/stable: Package is up to date", new_client.user_io.out)

        # But if we remove the full Hello0 local package, will retrieve the updated
        # recipe and the outdated package
        new_client.run("remove Hello0* -f")
        new_client.run("install Hello1/0.1@lasote/stable --build outdated")
        self.assertIn("Hello0/0.1@lasote/stable: Outdated package!", new_client.user_io.out)
        self.assertIn("Hello0/0.1@lasote/stable: Building your package", new_client.user_io.out)
Exemple #23
0
    def _get_build_folder(self, conanfile, package_layout, pref, keep_build,
                          recorder):
        # Build folder can use a different package_ID if build_id() is defined.
        # This function decides if the build folder should be re-used (not build again)
        # and returns the build folder
        new_id = build_id(conanfile)
        build_pref = PackageReference(pref.ref, new_id) if new_id else pref
        build_folder = package_layout.build(build_pref)

        if is_dirty(build_folder):
            self._output.warn("Build folder is dirty, removing it: %s" %
                              build_folder)
            rmdir(build_folder)

        # Decide if the build folder should be kept
        skip_build = conanfile.develop and keep_build
        if skip_build:
            self._output.info("Won't be built as specified by --keep-build")
            if not os.path.exists(build_folder):
                msg = "--keep-build specified, but build folder not found"
                recorder.package_install_error(
                    pref,
                    INSTALL_ERROR_MISSING_BUILD_FOLDER,
                    msg,
                    remote_name=None)
                raise ConanException(msg)
        elif build_pref != pref and os.path.exists(build_folder) and hasattr(
                conanfile, "build_id"):
            self._output.info(
                "Won't be built, using previous build folder as defined in build_id()"
            )
            skip_build = True

        return build_folder, skip_build
Exemple #24
0
    def get_recipe_sources(self, conan_reference, export_folder, export_sources_folder, remote):
        t1 = time.time()

        def filter_function(urls):
            file_url = urls.get(EXPORT_SOURCES_TGZ_NAME)
            if file_url:
                urls = {EXPORT_SOURCES_TGZ_NAME: file_url}
            else:
                return None
            return urls

        urls = self._call_remote(remote, "get_recipe_urls", conan_reference)
        urls = filter_function(urls)
        if not urls:
            return conan_reference

        zipped_files = self._call_remote(remote, "download_files_to_folder", urls, export_folder)

        duration = time.time() - t1
        log_recipe_sources_download(conan_reference, duration, remote, zipped_files)

        if not zipped_files:
            mkdir(export_sources_folder)  # create the folder even if no source files
            return

        unzip_and_get_files(zipped_files, export_sources_folder, EXPORT_SOURCES_TGZ_NAME)
        c_src_path = os.path.join(export_sources_folder, EXPORT_SOURCES_DIR_OLD)
        if os.path.exists(c_src_path):
            merge_directories(c_src_path, export_sources_folder)
            rmdir(c_src_path)
        touch_folder(export_sources_folder)
Exemple #25
0
def _clean_package(package_folder, output):
    try:
        output.warn("Trying to remove package folder: %s" % package_folder)
        rmdir(package_folder)
    except OSError as e:
        raise ConanException("%s\n\nCouldn't remove folder '%s', might be busy or open. Close any app "
                             "using it, and retry" % (str(e), package_folder))
Exemple #26
0
    def test_package(self, *args):
        """ build and run your package test. Must have conanfile.py with "test"
        method and "test_package" subfolder with package consumer test project
        """
        parser = argparse.ArgumentParser(description=self.test_package.__doc__, prog="conan test",
                                         formatter_class=RawTextHelpFormatter)
        parser.add_argument("path", nargs='?', default="", help='path to conanfile file, '
                            'e.g. /my_project/')
        parser.add_argument("-f", "--folder", help='alternative test folder name')
        self._parse_args(parser)

        args = parser.parse_args(*args)

        root_folder = os.path.normpath(os.path.join(os.getcwd(), args.path))
        if args.folder:
            test_folder_name = args.folder
            test_folder = os.path.join(root_folder, test_folder_name)
            test_conanfile = os.path.join(test_folder, "conanfile.py")
            if not os.path.exists(test_conanfile):
                raise ConanException("test folder '%s' not available, "
                                     "or it doesn't have a conanfile.py" % args.folder)
        else:
            for name in ["test_package", "test"]:
                test_folder_name = name
                test_folder = os.path.join(root_folder, test_folder_name)
                test_conanfile = os.path.join(test_folder, "conanfile.py")
                if os.path.exists(test_conanfile):
                    break
            else:
                raise ConanException("test folder 'test_package' not available, "
                                     "or it doesn't have a conanfile.py")

        lib_to_test = self._detect_tested_library_name()

        # Get False or a list of patterns to check
        if args.build is None and lib_to_test:  # Not specified, force build the tested library
            args.build = [lib_to_test]
        else:
            args.build = self._get_build_sources_parameter(args.build)

        options = args.options or []
        settings = args.settings or []

        sha = hashlib.sha1("".join(options + settings).encode()).hexdigest()
        build_folder = os.path.join(test_folder, "build", sha)
        rmdir(build_folder)
        # shutil.copytree(test_folder, build_folder)

        options = self._get_tuples_list_from_extender_arg(args.options)
        settings = self._get_tuples_list_from_extender_arg(args.settings)

        self._manager.install(reference=test_folder,
                              current_path=build_folder,
                              remote=args.remote,
                              options=options,
                              settings=settings,
                              build_mode=args.build)
        self._test_check(test_folder, test_folder_name)
        self._manager.build(test_folder, build_folder, test=True)
Exemple #27
0
    def _build_node(self, conan_ref, conan_file, build_mode):
        # Compute conan_file package from local (already compiled) or from remote
        output = ScopedOutput(str(conan_ref), self._out)
        package_id = conan_file.info.package_id()
        package_reference = PackageReference(conan_ref, package_id)

        conan_ref = package_reference.conan
        package_folder = self._paths.package(package_reference)
        build_folder = self._paths.build(package_reference,
                                         conan_file.short_paths)
        src_folder = self._paths.source(conan_ref, conan_file.short_paths)
        export_folder = self._paths.export(conan_ref)

        # If already exists do not dirt the output, the common situation
        # is that package is already installed and OK. If don't, the proxy
        # will print some other message about it
        if not os.path.exists(package_folder):
            output.info("Installing package %s" % package_id)

        self._handle_system_requirements(conan_ref, package_reference,
                                         conan_file, output)

        force_build = self._build_forced(conan_ref, build_mode, conan_file)
        if self._remote_proxy.get_package(package_reference, force_build):
            return

        # we need and can build? Only if we are forced or build_mode missing and package not exists
        build = force_build or build_mode is True or conan_file.build_policy_missing

        if build:
            if not force_build and not build_mode:
                output.info(
                    "Building package from source as defined by build_policy='missing'"
                )
            try:
                rmdir(build_folder, conan_file.short_paths)
                rmdir(package_folder)
            except Exception as e:
                raise ConanException(
                    "%s\n\nCouldn't remove folder, might be busy or open\n"
                    "Close any app using it, and retry" % str(e))
            if force_build:
                output.warn('Forced build from source')

            self._build_package(export_folder, src_folder, build_folder,
                                package_folder, conan_file, output)

            # Creating ***info.txt files
            save(os.path.join(build_folder, CONANINFO),
                 conan_file.info.dumps())
            output.info("Generated %s" % CONANINFO)
            save(os.path.join(build_folder, BUILD_INFO),
                 TXTGenerator(conan_file).content)
            output.info("Generated %s" % BUILD_INFO)

            os.chdir(build_folder)
            create_package(conan_file, build_folder, package_folder, output)
        else:
            self._raise_package_not_found_error(conan_ref, conan_file)
Exemple #28
0
    def get_conanfile(self, conan_reference):
        output = ScopedOutput(str(conan_reference), self._out)

        def _refresh():
            conan_dir_path = self._paths.export(conan_reference)
            rmdir(conan_dir_path)
            rmdir(self._paths.source(conan_reference), True)  # It might need to remove shortpath
            current_remote, _ = self._get_remote(conan_reference)
            output.info("Retrieving from remote '%s'..." % current_remote.name)
            self._remote_manager.get_conanfile(conan_reference, current_remote)
            if self._update:
                output.info("Updated!")
            else:
                output.info("Installed!")

        # check if it is in disk
        conanfile_path = self._paths.conanfile(conan_reference)

        path_exist = path_exists(conanfile_path, self._paths.store)

        if path_exist:
            if self._check_integrity:  # Check if package is corrupted
                read_manifest, expected_manifest = self._paths.conan_manifests(conan_reference)
                if read_manifest.file_sums != expected_manifest.file_sums:
                    output.warn("Bad conanfile detected! Removing export directory... ")
                    _refresh()
            else:  # Check for updates
                if self._check_updates:
                    ret = self.update_available(conan_reference)
                    if ret != 0:  # Found and not equal
                        remote, ref_remote = self._get_remote(conan_reference)
                        if ret == 1:
                            if not self._update:
                                if remote != ref_remote:  # Forced new remote
                                    output.warn("There is a new conanfile in '%s' remote. "
                                                "Execute 'install -u -r %s' to update it."
                                                % (remote.name, remote.name))
                                else:
                                    output.warn("There is a new conanfile in '%s' remote. "
                                                "Execute 'install -u' to update it."
                                                % remote.name)
                                output.warn("Refused to install!")
                            else:
                                if remote != ref_remote:
                                    # Delete packages, could be non coherent with new remote
                                    rmdir(self._paths.packages(conan_reference))
                                _refresh()
                        elif ret == -1:
                            if not self._update:
                                output.info("Current conanfile is newer "
                                            "than %s's one" % remote.name)
                            else:
                                output.error("Current conanfile is newer than %s's one. "
                                             "Run 'conan remove %s' and run install again "
                                             "to replace it." % (remote.name, conan_reference))

        else:
            self._retrieve_conanfile(conan_reference, output)
        return conanfile_path
Exemple #29
0
    def get_package(self, pref, dest_folder, remote, output, recorder):

        conanfile_path = self._cache.package_layout(pref.ref).conanfile()
        self._hook_manager.execute("pre_download_package",
                                   conanfile_path=conanfile_path,
                                   reference=pref.ref,
                                   package_id=pref.id,
                                   remote=remote)
        output.info("Retrieving package %s from remote '%s' " %
                    (pref.id, remote.name))
        rm_conandir(dest_folder)  # Remove first the destination folder
        t1 = time.time()
        try:
            pref = self._resolve_latest_pref(pref, remote)
            snapshot = self._call_remote(remote, "get_package_snapshot", pref)
            if not is_package_snapshot_complete(snapshot):
                raise PackageNotFoundException(pref)
            zipped_files = self._call_remote(remote, "get_package", pref,
                                             dest_folder)

            with self._cache.package_layout(
                    pref.ref).update_metadata() as metadata:
                metadata.packages[pref.id].revision = pref.revision
                metadata.packages[pref.id].recipe_revision = pref.ref.revision

            duration = time.time() - t1
            log_package_download(pref, duration, remote, zipped_files)
            unzip_and_get_files(zipped_files,
                                dest_folder,
                                PACKAGE_TGZ_NAME,
                                output=self._output)
            # Issue #214 https://github.com/conan-io/conan/issues/214
            touch_folder(dest_folder)
            if get_env("CONAN_READ_ONLY_CACHE", False):
                make_read_only(dest_folder)
            recorder.package_downloaded(pref, remote.url)
            output.success('Package installed %s' % pref.id)
        except NotFoundException:
            raise PackageNotFoundException(pref)
        except BaseException as e:
            output.error("Exception while getting package: %s" % str(pref.id))
            output.error("Exception: %s %s" % (type(e), str(e)))
            try:
                output.warn("Trying to remove package folder: %s" %
                            dest_folder)
                rmdir(dest_folder)
            except OSError as e:
                raise ConanException(
                    "%s\n\nCouldn't remove folder '%s', might be busy or open. "
                    "Close any app using it, and retry" %
                    (str(e), dest_folder))
            raise
        self._hook_manager.execute("post_download_package",
                                   conanfile_path=conanfile_path,
                                   reference=pref.ref,
                                   package_id=pref.id,
                                   remote=remote)

        return pref
Exemple #30
0
def create_package(conanfile,
                   source_folder,
                   build_folder,
                   package_folder,
                   install_folder,
                   output,
                   local=False,
                   copy_info=False):
    """ copies built artifacts, libs, headers, data, etc from build_folder to
    package folder
    """
    mkdir(package_folder)

    # Make the copy of all the patterns
    output.info("Generating the package")
    output.info("Package folder %s" % (package_folder))

    try:
        package_output = ScopedOutput("%s package()" % output.scope, output)
        output.highlight("Calling package()")
        conanfile.package_folder = package_folder
        conanfile.source_folder = source_folder
        conanfile.build_folder = build_folder

        def recipe_has(conanfile, attribute):
            return attribute in conanfile.__class__.__dict__

        if source_folder != build_folder:
            conanfile.copy = FileCopier(source_folder, package_folder,
                                        build_folder)
            with conanfile_exception_formatter(str(conanfile), "package"):
                with tools.chdir(source_folder):
                    conanfile.package()
            warn = recipe_has(conanfile, "package")
            conanfile.copy.report(package_output, warn=warn)

        conanfile.copy = FileCopier(build_folder, package_folder)
        with tools.chdir(build_folder):
            with conanfile_exception_formatter(str(conanfile), "package"):
                conanfile.package()
        warn = recipe_has(conanfile, "build") and recipe_has(
            conanfile, "package")
        conanfile.copy.report(package_output, warn=warn)
    except Exception as e:
        if not local:
            os.chdir(build_folder)
            try:
                rmdir(package_folder)
            except Exception as e_rm:
                output.error("Unable to remove package folder %s\n%s" %
                             (package_folder, str(e_rm)))
                output.warn("**** Please delete it manually ****")

        if isinstance(e, ConanExceptionInUserConanfileMethod):
            raise
        raise ConanException(e)

    _create_aux_files(install_folder, package_folder, conanfile, copy_info)
    output.success("Package '%s' created" % os.path.basename(package_folder))
Exemple #31
0
def configuration_install(path_or_url,
                          cache,
                          output,
                          verify_ssl,
                          requester,
                          config_type=None,
                          args=None):
    if path_or_url is None:
        try:
            item = cache.conan_config.get_item("general.config_install")
            _config_type, path_or_url, _verify_ssl, _args = _process_config_install_item(
                item)
        except ConanException:
            raise ConanException(
                "Called config install without arguments and "
                "'general.config_install' not defined in conan.conf")
    else:
        _config_type, path_or_url, _verify_ssl, _args = _process_config_install_item(
            path_or_url)

    config_type = config_type or _config_type
    verify_ssl = verify_ssl or _verify_ssl
    args = args or _args

    if os.path.exists(path_or_url):
        path_or_url = os.path.abspath(path_or_url)

    tmp_folder = os.path.join(cache.conan_folder, "tmp_config_install")
    # necessary for Mac OSX, where the temp folders in /var/ are symlinks to /private/var/
    tmp_folder = os.path.realpath(tmp_folder)
    mkdir(tmp_folder)
    try:

        if config_type == "git":
            _process_git_repo(path_or_url, cache, output, tmp_folder,
                              verify_ssl, args)
        elif config_type == "dir":
            args = None
            _process_folder(path_or_url, cache, output)
        elif config_type == "file":
            args = None
            _process_zip_file(path_or_url, cache, output, tmp_folder)
        elif config_type == "url":
            args = None
            _process_download(path_or_url,
                              cache,
                              output,
                              tmp_folder,
                              verify_ssl,
                              requester=requester)
        else:
            raise ConanException("Unable to process config install: %s" %
                                 path_or_url)
    finally:
        if config_type is not None and path_or_url is not None:
            value = "%s, %s, %s, %s" % (config_type, path_or_url, verify_ssl,
                                        args)
            cache.conan_config.set_item("general.config_install", value)
        rmdir(tmp_folder)
Exemple #32
0
    def package(self, reference, package_id):
        # Package paths
        conan_file_path = self._client_cache.conanfile(reference)
        if not os.path.exists(conan_file_path):
            raise ConanException("Package recipe '%s' does not exist" %
                                 str(reference))

        conanfile = load_conanfile_class(conan_file_path)
        if hasattr(conanfile, "build_id"):
            raise ConanException(
                "package command does not support recipes with 'build_id'\n"
                "To repackage them use 'conan install'")

        if not package_id:
            packages = [
                PackageReference(reference, packid)
                for packid in self._client_cache.conan_builds(reference)
            ]
            if not packages:
                raise NotFoundException(
                    "%s: Package recipe has not been built locally\n"
                    "Please read the 'conan package' command help\n"
                    "Use 'conan install' or 'conan test_package' to build and "
                    "create binaries" % str(reference))
        else:
            packages = [PackageReference(reference, package_id)]

        package_source_folder = self._client_cache.source(
            reference, conanfile.short_paths)
        for package_reference in packages:
            build_folder = self._client_cache.build(package_reference,
                                                    short_paths=None)
            if not os.path.exists(build_folder):
                raise NotFoundException(
                    "%s: Package binary '%s' folder doesn't exist\n"
                    "Please read the 'conan package' command help\n"
                    "Use 'conan install' or 'conan test_package' to build and "
                    "create binaries" %
                    (str(reference), package_reference.package_id))
            # The package already exist, we can use short_paths if they were defined
            package_folder = self._client_cache.package(package_reference,
                                                        short_paths=None)
            # Will read current conaninfo with specified options and load conanfile with them
            output = ScopedOutput(str(reference), self._user_io.out)
            output.info("Re-packaging %s" % package_reference.package_id)
            conanfile = load_consumer_conanfile(conan_file_path,
                                                build_folder,
                                                self._client_cache.settings,
                                                self._runner,
                                                output,
                                                reference=reference)
            rmdir(package_folder)
            if getattr(conanfile, 'no_copy_source', False):
                source_folder = package_source_folder
            else:
                source_folder = build_folder
            with environment_append(conanfile.env):
                packager.create_package(conanfile, source_folder, build_folder,
                                        package_folder, output)
Exemple #33
0
 def _rm_conandir(path):
     ''' removal of a directory that might contain a link to a short path
     '''
     link = os.path.join(path, CONAN_LINK)
     if os.path.exists(link):
         short_path = load(link)
         rmdir(os.path.dirname(short_path))
     rmdir(path)
Exemple #34
0
 def source(self):
     z_name = "zlib-%s.tar.gz" % self.version
     tools.download("https://zlib.net/zlib-%s.tar.gz" % self.version, z_name)
     tools.unzip(z_name)
     os.unlink(z_name)
     files.rmdir("%s/contrib" % self.ZIP_FOLDER_NAME)
     if not tools.os_info.is_windows:
         self.run("chmod +x ./%s/configure" % self.ZIP_FOLDER_NAME)
Exemple #35
0
    def export_pkg(self, reference, source_folder, build_folder,
                   package_folder, install_folder, profile, force):

        conan_file_path = self._client_cache.conanfile(reference)
        if not os.path.exists(conan_file_path):
            raise ConanException("Package recipe '%s' does not exist" %
                                 str(reference))

        remote_proxy = ConanProxy(self._client_cache,
                                  self._user_io,
                                  self._remote_manager,
                                  remote_name=None,
                                  update=False,
                                  check_updates=False,
                                  manifest_manager=None,
                                  recorder=self._recorder)

        loader = self.get_loader(profile)
        conanfile = loader.load_virtual([reference], scope_options=True)
        if install_folder and existing_info_files(install_folder):
            _load_deps_info(install_folder, conanfile, required=True)

        graph_builder = self._get_graph_builder(loader, False, remote_proxy)
        deps_graph = graph_builder.load(conanfile)

        # this is a bit tricky, but works. The loading of a cache package makes the referenced
        # one, the first of the first level, always existing
        nodes = deps_graph.direct_requires()
        _, conanfile = nodes[0]
        pkg_id = conanfile.info.package_id()
        self._user_io.out.info("Packaging to %s" % pkg_id)
        pkg_reference = PackageReference(reference, pkg_id)
        dest_package_folder = self._client_cache.package(
            pkg_reference, short_paths=conanfile.short_paths)

        if os.path.exists(dest_package_folder):
            if force:
                rmdir(dest_package_folder)
            else:
                raise ConanException(
                    "Package already exists. Please use --force, -f to "
                    "overwrite it")

        recipe_hash = self._client_cache.load_manifest(reference).summary_hash
        conanfile.info.recipe_hash = recipe_hash
        conanfile.develop = True
        package_output = ScopedOutput(str(reference), self._user_io.out)
        if package_folder:
            packager.export_pkg(conanfile, package_folder, dest_package_folder,
                                package_output)
        else:
            packager.create_package(conanfile,
                                    source_folder,
                                    build_folder,
                                    dest_package_folder,
                                    install_folder,
                                    package_output,
                                    local=True)
Exemple #36
0
    def get_recipe(self, conan_reference):
        output = ScopedOutput(str(conan_reference), self._out)

        def _refresh():
            export_path = self._client_cache.export(conan_reference)
            rmdir(export_path)
            # It might need to remove shortpath
            rmdir(self._client_cache.source(conan_reference), True)
            current_remote, _ = self._get_remote(conan_reference)
            output.info("Retrieving from remote '%s'..." % current_remote.name)
            self._remote_manager.get_recipe(conan_reference, export_path, current_remote)
            if self._update:
                output.info("Updated!")
            else:
                output.info("Installed!")

        # check if it is in disk
        conanfile_path = self._client_cache.conanfile(conan_reference)
        path_exist = path_exists(conanfile_path, self._client_cache.store)

        if path_exist:
            if self._check_updates:
                ret = self.update_available(conan_reference)
                if ret != 0:  # Found and not equal
                    remote, ref_remote = self._get_remote(conan_reference)
                    if ret == 1:
                        if not self._update:
                            if remote != ref_remote:  # Forced new remote
                                output.warn("There is a new conanfile in '%s' remote. "
                                            "Execute 'install -u -r %s' to update it."
                                            % (remote.name, remote.name))
                            else:
                                output.warn("There is a new conanfile in '%s' remote. "
                                            "Execute 'install -u' to update it."
                                            % remote.name)
                            output.warn("Refused to install!")
                        else:
                            if remote != ref_remote:
                                # Delete packages, could be non coherent with new remote
                                rmdir(self._client_cache.packages(conan_reference))
                            _refresh()
                    elif ret == -1:
                        if not self._update:
                            output.info("Current conanfile is newer "
                                        "than %s's one" % remote.name)
                        else:
                            output.error("Current conanfile is newer than %s's one. "
                                         "Run 'conan remove %s' and run install again "
                                         "to replace it." % (remote.name, conan_reference))

        else:
            self._retrieve_recipe(conan_reference, output)

        if self._manifest_manager:
            remote = self._registry.get_ref(conan_reference)
            self._manifest_manager.check_recipe(conan_reference, remote)

        return conanfile_path
Exemple #37
0
    def get_recipe(self, conan_reference):
        output = ScopedOutput(str(conan_reference), self._out)

        def _refresh():
            export_path = self._client_cache.export(conan_reference)
            rmdir(export_path)
            # It might need to remove shortpath
            rmdir(self._client_cache.source(conan_reference), True)
            current_remote, _ = self._get_remote(conan_reference)
            output.info("Retrieving from remote '%s'..." % current_remote.name)
            self._remote_manager.get_recipe(conan_reference, export_path, current_remote)
            if self._update:
                output.info("Updated!")
            else:
                output.info("Installed!")

        # check if it is in disk
        conanfile_path = self._client_cache.conanfile(conan_reference)
        path_exist = path_exists(conanfile_path, self._client_cache.store)

        if path_exist:
            if self._check_updates:
                ret = self.update_available(conan_reference)
                if ret != 0:  # Found and not equal
                    remote, ref_remote = self._get_remote(conan_reference)
                    if ret == 1:
                        if not self._update:
                            if remote != ref_remote:  # Forced new remote
                                output.warn("There is a new conanfile in '%s' remote. "
                                            "Execute 'install -u -r %s' to update it."
                                            % (remote.name, remote.name))
                            else:
                                output.warn("There is a new conanfile in '%s' remote. "
                                            "Execute 'install -u' to update it."
                                            % remote.name)
                            output.warn("Refused to install!")
                        else:
                            if remote != ref_remote:
                                # Delete packages, could be non coherent with new remote
                                rmdir(self._client_cache.packages(conan_reference))
                            _refresh()
                    elif ret == -1:
                        if not self._update:
                            output.info("Current conanfile is newer "
                                        "than %s's one" % remote.name)
                        else:
                            output.error("Current conanfile is newer than %s's one. "
                                         "Run 'conan remove %s' and run install again "
                                         "to replace it." % (remote.name, conan_reference))

        else:
            self._retrieve_recipe(conan_reference, output)

        if self._manifest_manager:
            remote = self._registry.get_ref(conan_reference)
            self._manifest_manager.check_recipe(conan_reference, remote)

        return conanfile_path
Exemple #38
0
def _export_conanfile(conanfile_path, output, client_cache, conanfile,
                      conan_ref, keep_source):

    exports_folder = client_cache.export(conan_ref)
    exports_source_folder = client_cache.export_sources(
        conan_ref, conanfile.short_paths)

    previous_digest = _init_export_folder(exports_folder,
                                          exports_source_folder)
    origin_folder = os.path.dirname(conanfile_path)
    export_recipe(conanfile, origin_folder, exports_folder, output)
    export_source(conanfile, origin_folder, exports_source_folder, output)
    shutil.copy2(conanfile_path, os.path.join(exports_folder, CONANFILE))

    scm_data, captured_revision = _capture_export_scm_data(
        conanfile, os.path.dirname(conanfile_path), exports_folder, output,
        client_cache, conan_ref)

    digest = FileTreeManifest.create(exports_folder, exports_source_folder)

    if previous_digest and previous_digest == digest:
        output.info("The stored package has not changed")
        modified_recipe = False
        digest = previous_digest  # Use the old one, keep old timestamp
    else:
        output.success('A new %s version was exported' % CONANFILE)
        output.info('Folder: %s' % exports_folder)
        modified_recipe = True

    digest.save(exports_folder)

    revision = scm_data.revision if scm_data and captured_revision else digest.summary_hash
    with client_cache.update_metadata(conan_ref) as metadata:
        # Note that there is no time set, the time will come from the remote
        metadata.recipe.revision = revision

    # FIXME: Conan 2.0 Clear the registry entry if the recipe has changed
    source = client_cache.source(conan_ref, conanfile.short_paths)
    remove = False
    if is_dirty(source):
        output.info("Source folder is corrupted, forcing removal")
        remove = True
    elif modified_recipe and not keep_source and os.path.exists(source):
        output.info(
            "Package recipe modified in export, forcing source folder removal")
        output.info("Use the --keep-source, -k option to skip it")
        remove = True
    if remove:
        output.info(
            "Removing 'source' folder, this can take a while for big packages")
        try:
            # remove only the internal
            rmdir(source)
        except BaseException as e:
            output.error("Unable to delete source folder. "
                         "Will be marked as corrupted for deletion")
            output.warn(str(e))
            set_dirty(source)
    def source(self):
        zip_name = "zlib-%s.tar.gz" % self.version
        tools.download("https://zlib.net/fossils/%s" % (zip_name), zip_name)

        tools.unzip(zip_name)
        os.unlink(zip_name)
        files.rmdir("%s/contrib" % self.ZIP_FOLDER_NAME)
        if self.settings.os != "Windows":
            self.run("chmod +x ./%s/configure" % self.ZIP_FOLDER_NAME)
Exemple #40
0
def path_shortener(path, short_paths):
    """ short_paths is 4-state:
    False: Never shorten the path
    True: Always shorten the path, create link if not existing
    None: Use shorten path only if already exists, not create
    """
    use_always_short_paths = get_env("CONAN_USE_ALWAYS_SHORT_PATHS", False)
    short_paths = use_always_short_paths or short_paths

    if short_paths is False or os.getenv("CONAN_USER_HOME_SHORT") == "None":
        return path
    link = os.path.join(path, CONAN_LINK)
    if os.path.exists(link):
        return load(link)
    elif short_paths is None:
        return path

    if os.path.exists(path):
        rmdir(path)

    short_home = os.getenv("CONAN_USER_HOME_SHORT")
    if not short_home:
        drive = os.path.splitdrive(path)[0]
        short_home = os.path.join(drive, os.sep, ".conan")
    mkdir(short_home)

    # Workaround for short_home living in NTFS file systems. Give full control permission
    # to current user to avoid
    # access problems in cygwin/msys2 windows subsystems when using short_home folder
    try:
        userdomain, username = os.getenv("USERDOMAIN"), os.environ["USERNAME"]
        domainname = "%s\%s" % (userdomain,
                                username) if userdomain else username
        cmd = r'cacls %s /E /G "%s":F' % (short_home, domainname)
        subprocess.check_output(
            cmd,
            stderr=subprocess.STDOUT)  # Ignoring any returned output, quiet
    except subprocess.CalledProcessError:
        # cmd can fail if trying to set ACL in non NTFS drives, ignoring it.
        pass

    redirect = hashed_redirect(short_home, path)
    if not redirect:
        logger.warning("Failed to create a deterministic short path in %s",
                       short_home)
        redirect = tempfile.mkdtemp(dir=short_home, prefix="")

    # Save the full path of the local cache directory where the redirect is from.
    # This file is for debugging purposes and not used by Conan.
    save(os.path.join(redirect, CONAN_REAL_PATH), path)

    # This "1" is the way to have a non-existing directory, so commands like
    # shutil.copytree() to it, works. It can be removed without compromising the
    # temp folder generator and conan-links consistency
    redirect = os.path.join(redirect, "1")
    save(link, redirect)
    return redirect
Exemple #41
0
def tmp_config_install_folder(cache):
    tmp_folder = os.path.join(cache.conan_folder, "tmp_config_install")
    # necessary for Mac OSX, where the temp folders in /var/ are symlinks to /private/var/
    tmp_folder = os.path.realpath(tmp_folder)
    mkdir(tmp_folder)
    try:
        yield tmp_folder
    finally:
        rmdir(tmp_folder)
Exemple #42
0
 def export_remove(self):
     export_folder = self.export()
     rmdir(export_folder)
     export_src_folder = os.path.join(self._base_folder, EXPORT_SRC_FOLDER)
     rm_conandir(export_src_folder)
     download_export = self.download_export()
     rmdir(download_export)
     scm_folder = os.path.join(self._base_folder, SCM_SRC_FOLDER)
     rm_conandir(scm_folder)
Exemple #43
0
def create_package(conanfile,
                   source_folder,
                   build_folder,
                   package_folder,
                   output,
                   local=False,
                   copy_info=False):
    """ copies built artifacts, libs, headers, data, etc from build_folder to
    package folder
    """
    mkdir(package_folder)

    # Make the copy of all the patterns
    output.info("Generating the package")
    output.info("Package folder %s" % (package_folder))

    def wrap(dst_folder):
        def new_method(pattern, src=""):
            conanfile.copy(pattern, dst_folder, src)

        return new_method

    # FIXME: Deprecate these methods. Not documented. Confusing. Rely on LINTER
    conanfile.copy_headers = wrap(DEFAULT_INCLUDE)
    conanfile.copy_libs = wrap(DEFAULT_LIB)
    conanfile.copy_bins = wrap(DEFAULT_BIN)
    conanfile.copy_res = wrap(DEFAULT_RES)
    try:
        package_output = ScopedOutput("%s package()" % output.scope, output)
        output.highlight("Calling package()")
        if source_folder != build_folder:
            conanfile.copy = FileCopier(source_folder, package_folder,
                                        build_folder)
            with conanfile_exception_formatter(str(conanfile), "package"):
                conanfile.package()
            conanfile.copy.report(package_output, warn=True)
        conanfile.copy = FileCopier(build_folder, package_folder)
        with conanfile_exception_formatter(str(conanfile), "package"):
            conanfile.package()
        conanfile.copy.report(package_output, warn=True)
    except Exception as e:
        if not local:
            os.chdir(build_folder)
            try:
                rmdir(package_folder)
            except Exception as e_rm:
                output.error("Unable to remove package folder %s\n%s" %
                             (package_folder, str(e_rm)))
                output.warn("**** Please delete it manually ****")

        if isinstance(e, ConanExceptionInUserConanfileMethod):
            raise
        raise ConanException(e)

    _create_aux_files(build_folder, package_folder, conanfile, copy_info)
    output.success("Package '%s' created" % os.path.basename(package_folder))
Exemple #44
0
def package_copy(src_ref, user_channel, package_ids, cache, user_io, short_paths=False,
                 force=False):

    ref = "%s/%s@%s" % (src_ref.name, src_ref.version, user_channel)
    if ref.count('@') > 1:
        raise ConanException("Destination must contain user/channel only.")

    dest_ref = ConanFileReference.loads(ref)
    # Generate metadata
    src_layout = cache.package_layout(src_ref, short_paths)
    src_metadata = src_layout.load_metadata()
    dst_layout = cache.package_layout(dest_ref, short_paths)

    # Copy export
    export_origin = src_layout.export()
    if not os.path.exists(export_origin):
        raise ConanException("'%s' doesn't exist" % str(src_ref))
    export_dest = dst_layout.export()
    if os.path.exists(export_dest):
        if not force and not user_io.request_boolean("'%s' already exist. Override?"
                                                     % str(dest_ref)):
            return
        rmdir(export_dest)
    shutil.copytree(export_origin, export_dest, symlinks=True)
    user_io.out.info("Copied %s to %s" % (str(src_ref), str(dest_ref)))

    export_sources_origin = src_layout.export_sources()
    export_sources_dest = dst_layout.export_sources()
    if os.path.exists(export_sources_dest):
        rmdir(export_sources_dest)
    shutil.copytree(export_sources_origin, export_sources_dest, symlinks=True)
    user_io.out.info("Copied sources %s to %s" % (str(src_ref), str(dest_ref)))

    # Copy packages
    package_revisions = {}  # To be stored in the metadata
    for package_id in package_ids:
        pref_origin = PackageReference(src_ref, package_id)
        pref_dest = PackageReference(dest_ref, package_id)
        package_path_origin = src_layout.package(pref_origin)
        if dst_layout.package_id_exists(package_id):
            if not force and not user_io.request_boolean("Package '%s' already exist."
                                                         " Override?" % str(package_id)):
                continue
            dst_layout.package_remove(pref_dest)
        package_path_dest = dst_layout.package(pref_dest)
        package_revisions[package_id] = (src_metadata.packages[package_id].revision,
                                         src_metadata.recipe.revision)
        shutil.copytree(package_path_origin, package_path_dest, symlinks=True)
        user_io.out.info("Copied %s to %s" % (str(package_id), str(dest_ref)))

    # Generate the metadata
    with dst_layout.update_metadata() as metadata:
        metadata.recipe.revision = src_metadata.recipe.revision
        for package_id, (revision, recipe_revision) in package_revisions.items():
            metadata.packages[package_id].revision = revision
            metadata.packages[package_id].recipe_revision = recipe_revision
Exemple #45
0
 def source(self):
     zip_name = "zlib-%s.tar.gz" % self.version
     tools.download(
         "http://downloads.sourceforge.net/project/libpng/zlib/%s/%s" %
         (self.version, zip_name), zip_name)
     tools.unzip(zip_name)
     os.unlink(zip_name)
     files.rmdir("%s/contrib" % self.ZIP_FOLDER_NAME)
     if self.settings.os != "Windows":
         self.run("chmod +x ./%s/configure" % self.ZIP_FOLDER_NAME)
Exemple #46
0
 def source(self):
     z_name = "zlib-%s.tar.gz" % self.version
     tools.download(
         "https://codeload.github.com/madler/zlib/tar.gz/v%s" %
         self.version, z_name)
     tools.unzip(z_name)
     os.unlink(z_name)
     files.rmdir("%s/contrib" % self.ZIP_FOLDER_NAME)
     if self.settings.os != "Windows":
         self.run("chmod +x ./%s/configure" % self.ZIP_FOLDER_NAME)
 def _export_upload(self, name, version=None, deps=None):
     conan = TestClient(servers=self.servers, users=[("lasote", "mypass")])
     files = cpp_hello_conan_files(name, version, deps, static=self.static)
     conan_ref = ConanFileReference(name, version, "lasote", "stable")
     conan.save(files, clean_first=True)
     conan.run("export lasote/stable")
     conan.run("install '%s' -o static=False -o language=0 --build missing" % str(conan_ref))
     conan.run("upload %s --all" % str(conan_ref))
     rmdir(conan.current_folder)
     shutil.rmtree(conan.paths.store, ignore_errors=True)
Exemple #48
0
    def _export_upload(self, name, version=None, deps=None):
        conan = TestClient(servers=self.servers, users={"default": [("lasote", "mypass")]})
        files = cpp_hello_conan_files(name, version, deps, static=False)
        conan.save(files)

        conan.run("create . lasote/stable")
        conan.run("upload * --all --confirm")
        conan.run("remove * -f")
        rmdir(conan.current_folder)
        shutil.rmtree(conan.cache.store, ignore_errors=True)
Exemple #49
0
    def _make_migrations(self, old_version):
        # ############### FILL THIS METHOD WITH THE REQUIRED ACTIONS ##############

        # VERSION 0.1
        if old_version == Version("0.1"):
            # Remove config, conans, all!
            self.out.warn("Reseting configuration and storage files...")
            if self.conf_path:
                rmdir(self.conf_path)
            if self.store_path:
                rmdir(self.store_path)
Exemple #50
0
 def _make_migrations(self, old_version):
     # ############### FILL THIS METHOD WITH THE REQUIRED ACTIONS ##############
     # VERSION 0.1
     if old_version is None:
         return
     if old_version < Version("0.3"):
         self.out.warn("Migration: Reseting configuration and storage files...")
         if os.path.exists(self.conf_path):
             rmdir(self.conf_path)
         if os.path.exists(self.store_path):
             rmdir(self.store_path)
Exemple #51
0
    def _build_node(self, conan_ref, conan_file, build_mode):
        # Compute conan_file package from local (already compiled) or from remote
        output = ScopedOutput(str(conan_ref), self._out)
        package_id = conan_file.info.package_id()
        package_reference = PackageReference(conan_ref, package_id)

        conan_ref = package_reference.conan
        package_folder = self._paths.package(package_reference, conan_file.short_paths)
        build_folder = self._paths.build(package_reference, conan_file.short_paths)
        src_folder = self._paths.source(conan_ref, conan_file.short_paths)
        export_folder = self._paths.export(conan_ref)

        # If already exists do not dirt the output, the common situation
        # is that package is already installed and OK. If don't, the proxy
        # will print some other message about it
        if not package_exists(package_folder):
            output.info("Installing package %s" % package_id)

        self._handle_system_requirements(conan_ref, package_reference, conan_file, output)

        force_build = self._build_forced(conan_ref, build_mode, conan_file)
        if self._remote_proxy.get_package(package_reference, force_build,
                                          short_paths=conan_file.short_paths):
            return

        # we need and can build? Only if we are forced or build_mode missing and package not exists
        build = force_build or build_mode is True or conan_file.build_policy_missing

        if build:
            if not force_build and not build_mode:
                output.info("Building package from source as defined by build_policy='missing'")
            try:
                rmdir(build_folder, conan_file.short_paths)
                rmdir(package_folder)
            except Exception as e:
                raise ConanException("%s\n\nCouldn't remove folder, might be busy or open\n"
                                     "Close any app using it, and retry" % str(e))
            if force_build:
                output.warn('Forced build from source')

            self._build_package(export_folder, src_folder, build_folder, package_folder,
                                conan_file, output)

            # Creating ***info.txt files
            save(os.path.join(build_folder, CONANINFO), conan_file.info.dumps())
            output.info("Generated %s" % CONANINFO)
            save(os.path.join(build_folder, BUILD_INFO), TXTGenerator(conan_file).content)
            output.info("Generated %s" % BUILD_INFO)

            os.chdir(build_folder)
            create_package(conan_file, build_folder, package_folder, output)
            self._remote_proxy.handle_package_manifest(package_reference, installed=True)
        else:
            self._raise_package_not_found_error(conan_ref, conan_file)
Exemple #52
0
 def _refresh():
     conan_dir_path = self._paths.export(conan_reference)
     rmdir(conan_dir_path)
     rmdir(self._paths.source(conan_reference), True)  # It might need to remove shortpath
     current_remote, _ = self._get_remote(conan_reference)
     output.info("Retrieving from remote '%s'..." % current_remote.name)
     self._remote_manager.get_conanfile(conan_reference, current_remote)
     if self._update:
         output.info("Updated!")
     else:
         output.info("Installed!")
Exemple #53
0
    def get_conanfile(self, conan_reference, remote):
        """
        Read the conans from remotes
        Will iterate the remotes to find the conans unless remote was specified

        returns (dict relative_filepath:content , remote_name)"""
        export_files = self._call_remote(remote, "get_conanfile", conan_reference)
        export_folder = self._paths.export(conan_reference)
        uncompress_files(export_files, export_folder, EXPORT_TGZ_NAME)
        # Make sure that the source dir is deleted
        rmdir(self._paths.source(conan_reference))
Exemple #54
0
    def _export_upload(self, name, version=None, deps=None):
        conan = TestClient(servers=self.servers, users={"default": [("lasote", "mypass")]})
        dll_export = conan.default_compiler_visual_studio
        files = cpp_hello_conan_files(name, version, deps, static=False, dll_export=dll_export)
        conan_ref = ConanFileReference(name, version, "lasote", "stable")
        conan.save(files, clean_first=True)

        conan.run("export lasote/stable")
        conan.run("install '%s' --build missing" % str(conan_ref))
        conan.run("upload %s --all" % str(conan_ref))
        rmdir(conan.current_folder)
        shutil.rmtree(conan.paths.store, ignore_errors=True)
Exemple #55
0
 def _refresh():
     export_path = self._client_cache.export(conan_reference)
     rmdir(export_path)
     # It might need to remove shortpath
     rm_conandir(self._client_cache.source(conan_reference))
     current_remote, _ = self._get_remote(conan_reference)
     output.info("Retrieving from remote '%s'..." % current_remote.name)
     self._remote_manager.get_recipe(conan_reference, export_path, current_remote)
     if self._update:
         output.info("Updated!")
     else:
         output.info("Installed!")
Exemple #56
0
def create_package(conanfile, source_folder, build_folder, package_folder, install_folder,
                   output, local=False, copy_info=False):
    """ copies built artifacts, libs, headers, data, etc. from build_folder to
    package folder
    """
    mkdir(package_folder)

    # Make the copy of all the patterns
    output.info("Generating the package")
    output.info("Package folder %s" % package_folder)

    try:
        package_output = ScopedOutput("%s package()" % output.scope, output)
        output.highlight("Calling package()")
        conanfile.package_folder = package_folder
        conanfile.source_folder = source_folder
        conanfile.install_folder = install_folder
        conanfile.build_folder = build_folder

        def recipe_has(attribute):
            return attribute in conanfile.__class__.__dict__

        if source_folder != build_folder:
            conanfile.copy = FileCopier(source_folder, package_folder, build_folder)
            with conanfile_exception_formatter(str(conanfile), "package"):
                with tools.chdir(source_folder):
                    conanfile.package()
            copy_done = conanfile.copy.report(package_output)
            if not copy_done and recipe_has("package"):
                output.warn("No files copied from source folder!")

        conanfile.copy = FileCopier(build_folder, package_folder)
        with tools.chdir(build_folder):
            with conanfile_exception_formatter(str(conanfile), "package"):
                conanfile.package()
        copy_done = conanfile.copy.report(package_output)
        if not copy_done and recipe_has("build") and recipe_has("package"):
            output.warn("No files copied from build folder!")
    except Exception as e:
        if not local:
            os.chdir(build_folder)
            try:
                rmdir(package_folder)
            except Exception as e_rm:
                output.error("Unable to remove package folder %s\n%s" % (package_folder, str(e_rm)))
                output.warn("**** Please delete it manually ****")

        if isinstance(e, ConanExceptionInUserConanfileMethod):
            raise
        raise ConanException(e)

    _create_aux_files(install_folder, package_folder, conanfile, copy_info)
    output.success("Package '%s' created" % os.path.basename(package_folder))
Exemple #57
0
 def remove_source(raise_error=True):
     output.warn("This can take a while for big packages")
     try:
         rmdir(src_folder)
     except BaseException as e_rm:
         save(dirty, "")  # Creation of DIRTY flag
         msg = str(e_rm)
         if six.PY2:
             msg = str(e_rm).decode("latin1")  # Windows prints some chars in latin1
         output.error("Unable to remove source folder %s\n%s" % (src_folder, msg))
         output.warn("**** Please delete it manually ****")
         if raise_error or isinstance(e_rm, KeyboardInterrupt):
             raise ConanException("Unable to remove source folder")
Exemple #58
0
    def get_recipe(self, conan_reference, dest_folder, remote):
        """
        Read the conans from remotes
        Will iterate the remotes to find the conans unless remote was specified

        returns (dict relative_filepath:abs_path , remote_name)"""
        zipped_files = self._call_remote(remote, "get_recipe", conan_reference, dest_folder)
        files = unzip_and_get_files(zipped_files, dest_folder, EXPORT_TGZ_NAME)
        # Make sure that the source dir is deleted
        rmdir(self._client_cache.source(conan_reference), True)
#       TODO: Download only the CONANFILE file and only download the rest of files
#       in install if needed (not found remote package)
        return files
Exemple #59
0
def _init_export_folder(destination_folder):
    previous_digest = None
    try:
        if os.path.exists(destination_folder):
            if os.path.exists(os.path.join(destination_folder, CONAN_MANIFEST)):
                manifest_content = load(os.path.join(destination_folder, CONAN_MANIFEST))
                previous_digest = FileTreeManifest.loads(manifest_content)
            # Maybe here we want to invalidate cache
            rmdir(destination_folder)
        os.makedirs(destination_folder)
    except Exception as e:
        raise ConanException("Unable to create folder %s\n%s" % (destination_folder, str(e)))
    return previous_digest