Example #1
0
def _get_local_infos_min(paths, reference, v2_compatibility_mode=False):

    result = {}

    if not reference.revision and v2_compatibility_mode:
        recipe_revisions = paths.get_recipe_revisions(reference)
    else:
        recipe_revisions = [reference]

    for recipe_revision in recipe_revisions:
        packages_path = paths.packages(recipe_revision)
        subdirs = list_folder_subdirs(packages_path, level=1)
        for package_id in subdirs:
            if package_id in result:
                continue
            # Read conaninfo
            try:
                package_reference = PackageReference(reference, package_id)
                info_path = os.path.join(paths.package(package_reference,
                                                       short_paths=None), CONANINFO)
                if not os.path.exists(info_path):
                    raise NotFoundException("")
                conan_info_content = load(info_path)
                info = ConanInfo.loads(conan_info_content)
                conan_vars_info = info.serialize_min()
                result[package_id] = conan_vars_info

            except Exception as exc:
                logger.error("Package %s has no ConanInfo file" % str(package_reference))
                if str(exc):
                    logger.error(str(exc))
    return result
Example #2
0
def _get_local_infos_min(client_cache, reference):
    result = {}
    packages_path = client_cache.packages(reference)
    subdirs = list_folder_subdirs(packages_path, level=1)
    for package_id in subdirs:
        # Read conaninfo
        try:
            package_reference = PackageReference(reference, package_id)
            info_path = os.path.join(
                client_cache.package(package_reference, short_paths=None),
                CONANINFO)
            if not os.path.exists(info_path):
                raise NotFoundException("")
            conan_info_content = load(info_path)

            metadata = client_cache.load_metadata(package_reference.conan)
            recipe_revision = metadata.packages[package_id].recipe_revision
            info = ConanInfo.loads(conan_info_content)
            if reference.revision and recipe_revision and recipe_revision != reference.revision:
                continue
            conan_vars_info = info.serialize_min()
            result[package_id] = conan_vars_info

        except Exception as exc:
            logger.error("Package %s has no ConanInfo file" %
                         str(package_reference))
            if str(exc):
                logger.error(str(exc))
    return result
Example #3
0
def _get_local_infos_min(server_store, ref, look_in_all_rrevs):

    result = {}
    rrevs = server_store.get_recipe_revisions(ref) if look_in_all_rrevs else [
        None
    ]

    for rrev in rrevs:
        new_ref = ref.copy_with_rev(rrev.revision) if rrev else ref
        subdirs = list_folder_subdirs(server_store.packages(new_ref), level=1)
        for package_id in subdirs:
            if package_id in result:
                continue
            # Read conaninfo
            try:
                pref = PackageReference(new_ref, package_id)
                revision_entry = server_store.get_last_package_revision(pref)
                if not revision_entry:
                    raise NotFoundException("")
                pref = PackageReference(new_ref, package_id,
                                        revision_entry.revision)
                info_path = os.path.join(server_store.package(pref), CONANINFO)
                if not os.path.exists(info_path):
                    raise NotFoundException("")
                conan_info_content = load(info_path)
                info = ConanInfo.loads(conan_info_content)
                conan_vars_info = info.serialize_min()
                result[package_id] = conan_vars_info
            except Exception as exc:  # FIXME: Too wide
                logger.error("Package %s has no ConanInfo file" % str(pref))
                if str(exc):
                    logger.error(str(exc))
    return result
Example #4
0
    def _search_recipes(self, pattern=None, ignorecase=True):
        def get_ref(_pattern):
            if not isinstance(_pattern, ConanFileReference):
                try:
                    ref_ = ConanFileReference.loads(_pattern)
                except (ConanException, TypeError):
                    ref_ = None
            else:
                ref_ = _pattern
            return ref_

        # Conan references in main storage
        if pattern:
            pattern = str(pattern)
            b_pattern = translate(pattern)
            b_pattern = re.compile(b_pattern, re.IGNORECASE) \
                if ignorecase else re.compile(b_pattern)

        subdirs = list_folder_subdirs(basedir=self._server_store.store,
                                      level=5)
        if not pattern:
            return sorted([
                ConanFileReference(*folder.split("/")).copy_clear_rev()
                for folder in subdirs
            ])
        else:
            ret = set()
            for subdir in subdirs:
                new_ref = ConanFileReference(*subdir.split("/"))
                if _partial_match(b_pattern, repr(new_ref)):
                    ret.add(new_ref.copy_clear_rev())

            return sorted(ret)
Example #5
0
 def remove_locks(self):
     folders = list_folder_subdirs(self._store_folder, 4)
     for folder in folders:
         conan_folder = os.path.join(self._store_folder, folder)
         Lock.clean(conan_folder)
         shutil.rmtree(os.path.join(conan_folder, "locks"),
                       ignore_errors=True)
Example #6
0
    def _get_local_infos_min(self, reference):
        result = {}
        packages_path = self._paths.packages(reference)
        subdirs = list_folder_subdirs(packages_path, level=1)
        for package_id in subdirs:
            # Read conaninfo
            try:
                package_reference = PackageReference(reference, package_id)
                info_path = os.path.join(
                    self._paths.package(package_reference, short_paths=None),
                    CONANINFO)
                if not os.path.exists(info_path):
                    raise NotFoundException("")
                conan_info_content = load(info_path)
                conan_vars_info = ConanInfo.loads(
                    conan_info_content).serialize_min()
                result[package_id] = conan_vars_info

            except Exception as exc:
                logger.error("Package %s has no ConanInfo file" %
                             str(package_reference))
                if str(exc):
                    logger.error(str(exc))

        return result
Example #7
0
    def migrate_to_revisions_layout(self):
        # .conan/data/lib/1.0/user/channel/export/*
        # .conan/data/lib/1.0/user/channel/0/export/*

        # .conan/data/lib/1.0/user/channel/package/*
        # .conan/data/lib/1.0/user/channel/0/package/*

        # .conan/data/lib/1.0/user/channel/package/XXX/*
        # .conan/data/lib/1.0/user/channel/0/package/XXX/0/*
        from six.moves import input as raw_input
        print("**********************************************")
        print("*                                            *")
        print("*       STORAGE MIGRATION NEEDED!            *")
        print("*                                            *")
        print("**********************************************")
        a = raw_input("A migration of your storage is needed, please backup first the storage "
                      "directory before continue: \n\n'%s' \n\nContinue? Y/N " % self.store_path)
        print("**********************************************")

        if a.lower() != "y":
            print("Exiting...")
            exit(1)

        print("**********************************************")
        print("*                                            *")
        print("*       MIGRATION IN PROGRESS                *")
        print("*                                            *")
        print("**********************************************")
        subdirs = list_folder_subdirs(basedir=self.store_path, level=4)
        for subdir in subdirs:
            base_dir = os.path.join(self.store_path, subdir)
            for export_or_package in os.listdir(base_dir):
                the_dir = os.path.join(base_dir, export_or_package)
                dest_dir = os.path.join(base_dir, DEFAULT_REVISION_V1)
                mkdir(dest_dir)
                shutil.move(the_dir, dest_dir)
                print("Moving '%s': %s" % (subdir, export_or_package))

            packages_dir = os.path.join(self.store_path, subdir, DEFAULT_REVISION_V1, PACKAGES_FOLDER)

            if not os.path.exists(packages_dir):
                continue
            for pid in os.listdir(packages_dir):
                package_dir = os.path.join(packages_dir, pid)
                mkdir(os.path.join(package_dir, DEFAULT_REVISION_V1))
                print(" - Package '%s'" % pid)
                for item in os.listdir(package_dir):
                    if item == DEFAULT_REVISION_V1:
                        continue
                    origin_path = os.path.join(package_dir, item)
                    dest_path = os.path.join(package_dir, DEFAULT_REVISION_V1, item)
                    mkdir(dest_dir)
                    shutil.move(origin_path, dest_path)
        print("**********************************************")
        print("*                                            *")
        print("*       MIGRATION COMPLETED!                 *")
        print("*                                            *")
        print("**********************************************")
Example #8
0
def migrate_c_src_export_source(cache, out):
    package_folders = list_folder_subdirs(cache.store, 4)
    for package in package_folders:
        package_folder = os.path.join(cache.store, package)
        c_src = os.path.join(package_folder, "export/%s" % EXPORT_SOURCES_DIR_OLD)
        if os.path.exists(c_src):
            out.warn("Migration: Removing package with old export_sources layout: %s" % package)
            try:
                shutil.rmtree(package_folder)
            except Exception:
                out.warn("Migration: Can't remove the '%s' directory, "
                         "remove it manually" % package_folder)
Example #9
0
def migrate_c_src_export_source(client_cache, out):
    from conans.util.files import list_folder_subdirs
    package_folders = list_folder_subdirs(client_cache.store, 4)
    for package in package_folders:
        package_folder = os.path.join(client_cache.store, package)
        c_src = os.path.join(package_folder, "export/.c_src")
        if os.path.exists(c_src):
            out.warn("Migration: Removing package with old export_sources layout: %s" % package)
            try:
                shutil.rmtree(package_folder)
            except Exception:
                out.warn("Migration: Can't remove the '%s' directory, "
                         "remove it manually" % package_folder)
Example #10
0
def migrate_c_src_export_source(client_cache, out):
    from conans.util.files import list_folder_subdirs
    package_folders = list_folder_subdirs(client_cache.store, 4)
    for package in package_folders:
        package_folder = os.path.join(client_cache.store, package)
        c_src = os.path.join(package_folder, "export/%s" % EXPORT_SOURCES_DIR_OLD)
        if os.path.exists(c_src):
            out.warn("Migration: Removing package with old export_sources layout: %s" % package)
            try:
                shutil.rmtree(package_folder)
            except Exception:
                out.warn("Migration: Can't remove the '%s' directory, "
                         "remove it manually" % package_folder)
Example #11
0
def search_recipes(cache, pattern=None, ignorecase=True):
    # Conan references in main storage
    if pattern:
        if isinstance(pattern, ConanFileReference):
            pattern = repr(pattern)
        pattern = translate(pattern)
        pattern = re.compile(pattern, re.IGNORECASE) if ignorecase else re.compile(pattern)

    subdirs = list_folder_subdirs(basedir=cache.store, level=4)
    refs = [ConanFileReference.load_dir_repr(folder) for folder in subdirs]
    refs.extend(cache.editable_packages.edited_refs.keys())
    if pattern:
        refs = [r for r in refs if _partial_match(pattern, repr(r))]
    refs = sorted(refs)
    return refs
Example #12
0
    def search(self, pattern):
        from fnmatch import translate
        pattern = translate(pattern)
        pattern = re.compile(pattern)

        subdirs = list_folder_subdirs(basedir=self.folder, level=4)

        if not pattern:
            result = [ConanFileReference(*folder.split("/")) for folder in subdirs]
        else:
            result = []
            for subdir in subdirs:
                conan_ref = ConanFileReference(*subdir.split("/"))
                if pattern:
                    if pattern.match(str(conan_ref)):
                        result.append(conan_ref)
        return sorted(result)
Example #13
0
    def search(self, pattern):
        from fnmatch import translate
        pattern = translate(pattern)
        pattern = re.compile(pattern)

        subdirs = list_folder_subdirs(basedir=self.folder, level=4)

        if not pattern:
            result = [ConanFileReference(*folder.split("/")) for folder in subdirs]
        else:
            result = []
            for subdir in subdirs:
                conan_ref = ConanFileReference(*subdir.split("/"))
                if pattern:
                    if pattern.match(str(conan_ref)):
                        result.append(conan_ref)
        return sorted(result)
Example #14
0
 def _copy_to_server(self, client_store_path, server_store):
     subdirs = list_folder_subdirs(basedir=client_store_path.store, level=4)
     refs = [ConanFileReference(*folder.split("/"), revision=DEFAULT_REVISION_V1)
             for folder in subdirs]
     for ref in refs:
         origin_path = client_store_path.export(ref)
         dest_path = server_store.export(ref)
         shutil.copytree(origin_path, dest_path)
         server_store.update_last_revision(ref)
         packages = client_store_path.packages(ref)
         if not os.path.exists(packages):
             continue
         for package in os.listdir(packages):
             pid = PackageReference(ref, package, DEFAULT_REVISION_V1)
             origin_path = client_store_path.package(pid)
             dest_path = server_store.package(pid)
             shutil.copytree(origin_path, dest_path)
             server_store.update_last_package_revision(pid)
Example #15
0
    def search_recipes(self, pattern=None, ignorecase=True):

        def get_ref(_pattern):
            if not isinstance(_pattern, ConanFileReference):
                try:
                    r = ConanFileReference.loads(_pattern)
                except (ConanException, TypeError):
                    r = None
            else:
                r = _pattern
            return r

        def get_folders_levels(_pattern):
            """If a reference with revisions is detected compare with 5 levels of subdirs"""
            r = get_ref(_pattern)
            return 5 if r and r.revision else 4

        # Check directly if it is a reference
        ref = get_ref(pattern)
        if ref:
            # Avoid resolve latest revision if a version range is passed or we are performing a
            # package remove (all revisions)
            path = self._server_store.conan(ref, resolve_latest=False)
            if self._server_store.path_exists(path):
                return [ref]

        # Conan references in main storage
        if pattern:
            pattern = str(pattern)
            b_pattern = translate(pattern)
            b_pattern = re.compile(b_pattern, re.IGNORECASE) if ignorecase else re.compile(b_pattern)

        subdirs = list_folder_subdirs(basedir=self._server_store.store, level=get_folders_levels(pattern))
        if not pattern:
            return sorted([ConanFileReference(*folder.split("/")) for folder in subdirs])
        else:
            ret = []
            for subdir in subdirs:
                conan_ref = ConanFileReference(*subdir.split("/"))
                if _partial_match(b_pattern, conan_ref):
                    ret.append(conan_ref)

            return sorted(ret)
Example #16
0
    def search_recipes(self, pattern=None, ignorecase=True):
        # Conan references in main storage
        if pattern:
            if isinstance(pattern, ConanFileReference):
                pattern = str(pattern)
            pattern = translate(pattern)
            pattern = re.compile(pattern, re.IGNORECASE) if ignorecase else re.compile(pattern)

        subdirs = list_folder_subdirs(basedir=self._paths.store, level=4)
        if not pattern:
            return sorted([ConanFileReference(*folder.split("/")) for folder in subdirs])
        else:
            ret = []
            for subdir in subdirs:
                conan_ref = ConanFileReference(*subdir.split("/"))
                if pattern:
                    if pattern.match(str(conan_ref)):
                        ret.append(conan_ref)
            return sorted(ret)
Example #17
0
def search_recipes(paths, pattern=None, ignorecase=True):
    # Conan references in main storage
    if pattern:
        if isinstance(pattern, ConanFileReference):
            pattern = str(pattern)
        pattern = translate(pattern)
        pattern = re.compile(
            pattern, re.IGNORECASE) if ignorecase else re.compile(pattern)

    subdirs = list_folder_subdirs(basedir=paths.store, level=4)
    if not pattern:
        return sorted(
            [ConanFileReference(*folder.split("/")) for folder in subdirs])
    else:
        ret = []
        for subdir in subdirs:
            conan_ref = ConanFileReference(*subdir.split("/"))
            if pattern:
                if pattern.match(str(conan_ref)):
                    ret.append(conan_ref)
        return sorted(ret)
Example #18
0
def _migrate_lock_files(cache, out):
    out.warn("Migration: Removing old lock files")
    base_dir = cache.store
    pkgs = list_folder_subdirs(base_dir, 4)
    for pkg in pkgs:
        out.info("Removing locks for %s" % pkg)
        try:
            count = os.path.join(base_dir, pkg, "rw.count")
            if os.path.exists(count):
                os.remove(count)
            count = os.path.join(base_dir, pkg, "rw.count.lock")
            if os.path.exists(count):
                os.remove(count)
            locks = os.path.join(base_dir, pkg, "locks")
            if os.path.exists(locks):
                shutil.rmtree(locks)
        except Exception as e:
            raise ConanException("Something went wrong while removing %s locks\n"
                                 "Error: %s\n"
                                 "Please clean your local conan cache manually"
                                 % (pkg, str(e)))
    out.warn("Migration: Removing old lock files finished\n")
Example #19
0
    def _get_local_infos_min(self, reference):
        result = {}
        packages_path = self._paths.packages(reference)
        subdirs = list_folder_subdirs(packages_path, level=1)
        for package_id in subdirs:
            # Read conaninfo
            try:
                package_reference = PackageReference(reference, package_id)
                info_path = os.path.join(self._paths.package(package_reference,
                                                             short_paths=None), CONANINFO)
                if not os.path.exists(info_path):
                    raise NotFoundException("")
                conan_info_content = load(info_path)
                conan_vars_info = ConanInfo.loads(conan_info_content).serialize_min()
                result[package_id] = conan_vars_info

            except Exception as exc:
                logger.error("Package %s has no ConanInfo file" % str(package_reference))
                if str(exc):
                    logger.error(str(exc))

        return result
Example #20
0
    def _search_recipes(self, pattern=None, ignorecase=True):
        subdirs = list_folder_subdirs(basedir=self._server_store.store,
                                      level=5)
        if not pattern:
            return sorted([
                ConanFileReference(*folder.split("/")).copy_clear_rev()
                for folder in subdirs
            ])
        else:
            # Conan references in main storage
            pattern = str(pattern)
            b_pattern = translate(pattern)
            b_pattern = re.compile(
                b_pattern,
                re.IGNORECASE) if ignorecase else re.compile(b_pattern)
            ret = set()
            for subdir in subdirs:
                new_ref = ConanFileReference(*subdir.split("/"))
                if _partial_match(b_pattern, repr(new_ref)):
                    ret.add(new_ref.copy_clear_rev())

            return sorted(ret)
Example #21
0
def _migrate_lock_files(client_cache, out):
    out.warn("Migration: Removing old lock files")
    base_dir = client_cache.store
    pkgs = list_folder_subdirs(base_dir, 4)
    for pkg in pkgs:
        out.info("Removing locks for %s" % pkg)
        try:
            count = os.path.join(base_dir, pkg, "rw.count")
            if os.path.exists(count):
                os.remove(count)
            count = os.path.join(base_dir, pkg, "rw.count.lock")
            if os.path.exists(count):
                os.remove(count)
            locks = os.path.join(base_dir, pkg, "locks")
            if os.path.exists(locks):
                shutil.rmtree(locks)
        except Exception as e:
            raise ConanException("Something went wrong while removing %s locks\n"
                                 "Error: %s\n"
                                 "Please clean your local conan cache manually"
                                 % (pkg, str(e)))
    out.warn("Migration: Removing old lock files finished\n")
Example #22
0
def _get_local_infos_min(server_store, ref, look_in_all_rrevs):

    result = {}
    rrevs = server_store.get_recipe_revisions(ref) if look_in_all_rrevs else [
        None
    ]

    for rrev in rrevs:
        new_ref = ref.copy_with_rev(rrev.revision) if rrev else ref
        subdirs = list_folder_subdirs(server_store.packages(new_ref), level=1)
        for package_id in subdirs:
            if package_id in result:
                continue
            # Read conaninfo
            try:
                pref = PackageReference(new_ref, package_id)
                revision_entry = server_store.get_last_package_revision(pref)
                if not revision_entry:
                    raise NotFoundException("")
                pref = PackageReference(new_ref, package_id,
                                        revision_entry.revision)
                info_path = os.path.join(server_store.package(pref), CONANINFO)
                if not os.path.exists(info_path):
                    raise NotFoundException("")
                content = load(info_path)
                info = ConanInfo.loads(content)
                # From Conan 1.48 the conaninfo.txt is sent raw.
                result[package_id] = {"content": content}
                # FIXME: This could be removed in the conan_server, Artifactory should keep it
                #        to guarantee compatibility with old conan clients.
                conan_vars_info = info.serialize_min()
                result[package_id].update(conan_vars_info)

            except Exception as exc:  # FIXME: Too wide
                logger.error("Package %s has no ConanInfo file" % str(pref))
                if str(exc):
                    logger.error(str(exc))
    return result
Example #23
0
def _get_local_infos_min(package_layout):
    result = OrderedDict()

    packages_path = package_layout.packages()
    subdirs = list_folder_subdirs(packages_path, level=1)
    for package_id in subdirs:
        # Read conaninfo
        pref = PackageReference(package_layout.ref, package_id)
        info_path = os.path.join(package_layout.package(pref), CONANINFO)
        if not os.path.exists(info_path):
            logger.error("There is no ConanInfo: %s" % str(info_path))
            continue
        conan_info_content = load(info_path)

        info = ConanInfo.loads(conan_info_content)
        if package_layout.ref.revision:
            metadata = package_layout.load_metadata()
            recipe_revision = metadata.packages[package_id].recipe_revision
            if recipe_revision and recipe_revision != package_layout.ref.revision:
                continue
        conan_vars_info = info.serialize_min()
        result[package_id] = conan_vars_info

    return result
Example #24
0
def _get_prefs(layout):
    packages_folder = layout.packages()
    folders = list_folder_subdirs(packages_folder, 1)
    return [PackageReference(layout.ref, s) for s in folders]
Example #25
0
def _get_refs(cache):
    folders = list_folder_subdirs(cache.store, 4)
    return [ConanFileReference(*s.split("/")) for s in folders]
Example #26
0
 def all_refs(self):
     subdirs = list_folder_subdirs(basedir=self._store_folder, level=4)
     return [ConanFileReference(*folder.split("/")) for folder in subdirs]
Example #27
0
    def migrate_to_revisions_layout(self):
        # .conan/data/lib/1.0/user/channel/export/*
        # .conan/data/lib/1.0/user/channel/0/export/*

        # .conan/data/lib/1.0/user/channel/package/*
        # .conan/data/lib/1.0/user/channel/0/package/*

        # .conan/data/lib/1.0/user/channel/package/XXX/*
        # .conan/data/lib/1.0/user/channel/0/package/XXX/0/*
        if not self.force_migrations:
            print("**********************************************")
            print("*                                            *")
            print("*      ERROR: STORAGE MIGRATION NEEDED!      *")
            print("*                                            *")
            print("**********************************************")
            msg = "A migration of your storage is needed, please backup first the storage " \
                  "directory and run:\n\n$ conan_server --migrate\n\n"
            logger.error(msg)
            print(msg)
            exit(
                3
            )  # Gunicorn expects error code 3 to stop retrying booting the worker

        print("**********************************************")
        print("*                                            *")
        print("*       MIGRATION IN PROGRESS                *")
        print("*                                            *")
        print("**********************************************")
        subdirs = list_folder_subdirs(basedir=self.store_path, level=4)
        for subdir in subdirs:
            base_dir = os.path.join(self.store_path, subdir)
            for export_or_package in os.listdir(base_dir):
                the_dir = os.path.join(base_dir, export_or_package)
                dest_dir = os.path.join(base_dir, DEFAULT_REVISION_V1)
                mkdir(dest_dir)
                print("Moving '%s': %s" % (subdir, export_or_package))
                shutil.move(the_dir, dest_dir)

            rev_list = RevisionList()
            rev_list.add_revision(DEFAULT_REVISION_V1)
            save(os.path.join(base_dir, REVISIONS_FILE), rev_list.dumps())

            packages_dir = os.path.join(self.store_path, subdir,
                                        DEFAULT_REVISION_V1, PACKAGES_FOLDER)

            if not os.path.exists(packages_dir):
                print("NO PACKAGES")
                continue
            for pid in os.listdir(packages_dir):
                package_dir = os.path.join(packages_dir, pid)
                mkdir(os.path.join(package_dir, DEFAULT_REVISION_V1))
                print(" - Package '%s'" % pid)
                for item in os.listdir(package_dir):
                    if item == DEFAULT_REVISION_V1:
                        continue
                    origin_path = os.path.join(package_dir, item)
                    dest_path = os.path.join(package_dir, DEFAULT_REVISION_V1,
                                             item)
                    mkdir(dest_dir)
                    shutil.move(origin_path, dest_path)
                rev_list = RevisionList()
                rev_list.add_revision(DEFAULT_REVISION_V1)
                save(os.path.join(package_dir, REVISIONS_FILE),
                     rev_list.dumps())
        print("**********************************************")
        print("*                                            *")
        print("*       MIGRATION COMPLETED!                 *")
        print("*                                            *")
        print("**********************************************")
Example #28
0
 def list_folder_subdirs(self, basedir, level):
     from conans.util.files import list_folder_subdirs
     return list_folder_subdirs(basedir, level)
Example #29
0
 def list_folder_subdirs(self, basedir, level):
     from conans.util.files import list_folder_subdirs
     return list_folder_subdirs(basedir, level)
Example #30
0
 def all_refs(self):
     subdirs = list_folder_subdirs(basedir=self._store_folder, level=4)
     return [ConanFileReference.load_dir_repr(folder) for folder in subdirs]