def _get_local_infos_min(paths, reference, v2_compatibility_mode=False):
result = {}
if not reference.revision and v2_compatibility_mode:
recipe_revisions = paths.get_recipe_revisions(reference)
else:
recipe_revisions = [reference]
for recipe_revision in recipe_revisions:
packages_path = paths.packages(recipe_revision)
subdirs = list_folder_subdirs(packages_path, level=1)
for package_id in subdirs:
if package_id in result:
continue
# Read conaninfo
try:
package_reference = PackageReference(reference, package_id)
info_path = os.path.join(paths.package(package_reference,
short_paths=None), CONANINFO)
if not os.path.exists(info_path):
raise NotFoundException("")
conan_info_content = load(info_path)
info = ConanInfo.loads(conan_info_content)
conan_vars_info = info.serialize_min()
result[package_id] = conan_vars_info
except Exception as exc:
logger.error("Package %s has no ConanInfo file" % str(package_reference))
if str(exc):
logger.error(str(exc))
return result
def _get_local_infos_min(client_cache, reference):
result = {}
packages_path = client_cache.packages(reference)
subdirs = list_folder_subdirs(packages_path, level=1)
for package_id in subdirs:
# Read conaninfo
try:
package_reference = PackageReference(reference, package_id)
info_path = os.path.join(
client_cache.package(package_reference, short_paths=None),
CONANINFO)
if not os.path.exists(info_path):
raise NotFoundException("")
conan_info_content = load(info_path)
metadata = client_cache.load_metadata(package_reference.conan)
recipe_revision = metadata.packages[package_id].recipe_revision
info = ConanInfo.loads(conan_info_content)
if reference.revision and recipe_revision and recipe_revision != reference.revision:
continue
conan_vars_info = info.serialize_min()
result[package_id] = conan_vars_info
except Exception as exc:
logger.error("Package %s has no ConanInfo file" %
str(package_reference))
if str(exc):
logger.error(str(exc))
return result
def _get_local_infos_min(server_store, ref, look_in_all_rrevs):
result = {}
rrevs = server_store.get_recipe_revisions(ref) if look_in_all_rrevs else [
None
]
for rrev in rrevs:
new_ref = ref.copy_with_rev(rrev.revision) if rrev else ref
subdirs = list_folder_subdirs(server_store.packages(new_ref), level=1)
for package_id in subdirs:
if package_id in result:
continue
# Read conaninfo
try:
pref = PackageReference(new_ref, package_id)
revision_entry = server_store.get_last_package_revision(pref)
if not revision_entry:
raise NotFoundException("")
pref = PackageReference(new_ref, package_id,
revision_entry.revision)
info_path = os.path.join(server_store.package(pref), CONANINFO)
if not os.path.exists(info_path):
raise NotFoundException("")
conan_info_content = load(info_path)
info = ConanInfo.loads(conan_info_content)
conan_vars_info = info.serialize_min()
result[package_id] = conan_vars_info
except Exception as exc: # FIXME: Too wide
logger.error("Package %s has no ConanInfo file" % str(pref))
if str(exc):
logger.error(str(exc))
return result
def _search_recipes(self, pattern=None, ignorecase=True):
def get_ref(_pattern):
if not isinstance(_pattern, ConanFileReference):
try:
ref_ = ConanFileReference.loads(_pattern)
except (ConanException, TypeError):
ref_ = None
else:
ref_ = _pattern
return ref_
# Conan references in main storage
if pattern:
pattern = str(pattern)
b_pattern = translate(pattern)
b_pattern = re.compile(b_pattern, re.IGNORECASE) \
if ignorecase else re.compile(b_pattern)
subdirs = list_folder_subdirs(basedir=self._server_store.store,
level=5)
if not pattern:
return sorted([
ConanFileReference(*folder.split("/")).copy_clear_rev()
for folder in subdirs
])
else:
ret = set()
for subdir in subdirs:
new_ref = ConanFileReference(*subdir.split("/"))
if _partial_match(b_pattern, repr(new_ref)):
ret.add(new_ref.copy_clear_rev())
return sorted(ret)
def remove_locks(self):
folders = list_folder_subdirs(self._store_folder, 4)
for folder in folders:
conan_folder = os.path.join(self._store_folder, folder)
Lock.clean(conan_folder)
shutil.rmtree(os.path.join(conan_folder, "locks"),
ignore_errors=True)
def _get_local_infos_min(self, reference):
result = {}
packages_path = self._paths.packages(reference)
subdirs = list_folder_subdirs(packages_path, level=1)
for package_id in subdirs:
# Read conaninfo
try:
package_reference = PackageReference(reference, package_id)
info_path = os.path.join(
self._paths.package(package_reference, short_paths=None),
CONANINFO)
if not os.path.exists(info_path):
raise NotFoundException("")
conan_info_content = load(info_path)
conan_vars_info = ConanInfo.loads(
conan_info_content).serialize_min()
result[package_id] = conan_vars_info
except Exception as exc:
logger.error("Package %s has no ConanInfo file" %
str(package_reference))
if str(exc):
logger.error(str(exc))
return result
def migrate_to_revisions_layout(self):
# .conan/data/lib/1.0/user/channel/export/*
# .conan/data/lib/1.0/user/channel/0/export/*
# .conan/data/lib/1.0/user/channel/package/*
# .conan/data/lib/1.0/user/channel/0/package/*
# .conan/data/lib/1.0/user/channel/package/XXX/*
# .conan/data/lib/1.0/user/channel/0/package/XXX/0/*
from six.moves import input as raw_input
print("**********************************************")
print("* *")
print("* STORAGE MIGRATION NEEDED! *")
print("* *")
print("**********************************************")
a = raw_input("A migration of your storage is needed, please backup first the storage "
"directory before continue: \n\n'%s' \n\nContinue? Y/N " % self.store_path)
print("**********************************************")
if a.lower() != "y":
print("Exiting...")
exit(1)
print("**********************************************")
print("* *")
print("* MIGRATION IN PROGRESS *")
print("* *")
print("**********************************************")
subdirs = list_folder_subdirs(basedir=self.store_path, level=4)
for subdir in subdirs:
base_dir = os.path.join(self.store_path, subdir)
for export_or_package in os.listdir(base_dir):
the_dir = os.path.join(base_dir, export_or_package)
dest_dir = os.path.join(base_dir, DEFAULT_REVISION_V1)
mkdir(dest_dir)
shutil.move(the_dir, dest_dir)
print("Moving '%s': %s" % (subdir, export_or_package))
packages_dir = os.path.join(self.store_path, subdir, DEFAULT_REVISION_V1, PACKAGES_FOLDER)
if not os.path.exists(packages_dir):
continue
for pid in os.listdir(packages_dir):
package_dir = os.path.join(packages_dir, pid)
mkdir(os.path.join(package_dir, DEFAULT_REVISION_V1))
print(" - Package '%s'" % pid)
for item in os.listdir(package_dir):
if item == DEFAULT_REVISION_V1:
continue
origin_path = os.path.join(package_dir, item)
dest_path = os.path.join(package_dir, DEFAULT_REVISION_V1, item)
mkdir(dest_dir)
shutil.move(origin_path, dest_path)
print("**********************************************")
print("* *")
print("* MIGRATION COMPLETED! *")
print("* *")
print("**********************************************")
def migrate_c_src_export_source(cache, out):
package_folders = list_folder_subdirs(cache.store, 4)
for package in package_folders:
package_folder = os.path.join(cache.store, package)
c_src = os.path.join(package_folder, "export/%s" % EXPORT_SOURCES_DIR_OLD)
if os.path.exists(c_src):
out.warn("Migration: Removing package with old export_sources layout: %s" % package)
try:
shutil.rmtree(package_folder)
except Exception:
out.warn("Migration: Can't remove the '%s' directory, "
"remove it manually" % package_folder)
def migrate_c_src_export_source(client_cache, out):
from conans.util.files import list_folder_subdirs
package_folders = list_folder_subdirs(client_cache.store, 4)
for package in package_folders:
package_folder = os.path.join(client_cache.store, package)
c_src = os.path.join(package_folder, "export/.c_src")
if os.path.exists(c_src):
out.warn("Migration: Removing package with old export_sources layout: %s" % package)
try:
shutil.rmtree(package_folder)
except Exception:
out.warn("Migration: Can't remove the '%s' directory, "
"remove it manually" % package_folder)
def migrate_c_src_export_source(client_cache, out):
from conans.util.files import list_folder_subdirs
package_folders = list_folder_subdirs(client_cache.store, 4)
for package in package_folders:
package_folder = os.path.join(client_cache.store, package)
c_src = os.path.join(package_folder, "export/%s" % EXPORT_SOURCES_DIR_OLD)
if os.path.exists(c_src):
out.warn("Migration: Removing package with old export_sources layout: %s" % package)
try:
shutil.rmtree(package_folder)
except Exception:
out.warn("Migration: Can't remove the '%s' directory, "
"remove it manually" % package_folder)
def search_recipes(cache, pattern=None, ignorecase=True):
# Conan references in main storage
if pattern:
if isinstance(pattern, ConanFileReference):
pattern = repr(pattern)
pattern = translate(pattern)
pattern = re.compile(pattern, re.IGNORECASE) if ignorecase else re.compile(pattern)
subdirs = list_folder_subdirs(basedir=cache.store, level=4)
refs = [ConanFileReference.load_dir_repr(folder) for folder in subdirs]
refs.extend(cache.editable_packages.edited_refs.keys())
if pattern:
refs = [r for r in refs if _partial_match(pattern, repr(r))]
refs = sorted(refs)
return refs
def search(self, pattern):
from fnmatch import translate
pattern = translate(pattern)
pattern = re.compile(pattern)
subdirs = list_folder_subdirs(basedir=self.folder, level=4)
if not pattern:
result = [ConanFileReference(*folder.split("/")) for folder in subdirs]
else:
result = []
for subdir in subdirs:
conan_ref = ConanFileReference(*subdir.split("/"))
if pattern:
if pattern.match(str(conan_ref)):
result.append(conan_ref)
return sorted(result)
def search(self, pattern):
from fnmatch import translate
pattern = translate(pattern)
pattern = re.compile(pattern)
subdirs = list_folder_subdirs(basedir=self.folder, level=4)
if not pattern:
result = [ConanFileReference(*folder.split("/")) for folder in subdirs]
else:
result = []
for subdir in subdirs:
conan_ref = ConanFileReference(*subdir.split("/"))
if pattern:
if pattern.match(str(conan_ref)):
result.append(conan_ref)
return sorted(result)
def _copy_to_server(self, client_store_path, server_store):
subdirs = list_folder_subdirs(basedir=client_store_path.store, level=4)
refs = [ConanFileReference(*folder.split("/"), revision=DEFAULT_REVISION_V1)
for folder in subdirs]
for ref in refs:
origin_path = client_store_path.export(ref)
dest_path = server_store.export(ref)
shutil.copytree(origin_path, dest_path)
server_store.update_last_revision(ref)
packages = client_store_path.packages(ref)
if not os.path.exists(packages):
continue
for package in os.listdir(packages):
pid = PackageReference(ref, package, DEFAULT_REVISION_V1)
origin_path = client_store_path.package(pid)
dest_path = server_store.package(pid)
shutil.copytree(origin_path, dest_path)
server_store.update_last_package_revision(pid)
def search_recipes(self, pattern=None, ignorecase=True):
def get_ref(_pattern):
if not isinstance(_pattern, ConanFileReference):
try:
r = ConanFileReference.loads(_pattern)
except (ConanException, TypeError):
r = None
else:
r = _pattern
return r
def get_folders_levels(_pattern):
"""If a reference with revisions is detected compare with 5 levels of subdirs"""
r = get_ref(_pattern)
return 5 if r and r.revision else 4
# Check directly if it is a reference
ref = get_ref(pattern)
if ref:
# Avoid resolve latest revision if a version range is passed or we are performing a
# package remove (all revisions)
path = self._server_store.conan(ref, resolve_latest=False)
if self._server_store.path_exists(path):
return [ref]
# Conan references in main storage
if pattern:
pattern = str(pattern)
b_pattern = translate(pattern)
b_pattern = re.compile(b_pattern, re.IGNORECASE) if ignorecase else re.compile(b_pattern)
subdirs = list_folder_subdirs(basedir=self._server_store.store, level=get_folders_levels(pattern))
if not pattern:
return sorted([ConanFileReference(*folder.split("/")) for folder in subdirs])
else:
ret = []
for subdir in subdirs:
conan_ref = ConanFileReference(*subdir.split("/"))
if _partial_match(b_pattern, conan_ref):
ret.append(conan_ref)
return sorted(ret)
def search_recipes(self, pattern=None, ignorecase=True):
# Conan references in main storage
if pattern:
if isinstance(pattern, ConanFileReference):
pattern = str(pattern)
pattern = translate(pattern)
pattern = re.compile(pattern, re.IGNORECASE) if ignorecase else re.compile(pattern)
subdirs = list_folder_subdirs(basedir=self._paths.store, level=4)
if not pattern:
return sorted([ConanFileReference(*folder.split("/")) for folder in subdirs])
else:
ret = []
for subdir in subdirs:
conan_ref = ConanFileReference(*subdir.split("/"))
if pattern:
if pattern.match(str(conan_ref)):
ret.append(conan_ref)
return sorted(ret)
def search_recipes(paths, pattern=None, ignorecase=True):
# Conan references in main storage
if pattern:
if isinstance(pattern, ConanFileReference):
pattern = str(pattern)
pattern = translate(pattern)
pattern = re.compile(
pattern, re.IGNORECASE) if ignorecase else re.compile(pattern)
subdirs = list_folder_subdirs(basedir=paths.store, level=4)
if not pattern:
return sorted(
[ConanFileReference(*folder.split("/")) for folder in subdirs])
else:
ret = []
for subdir in subdirs:
conan_ref = ConanFileReference(*subdir.split("/"))
if pattern:
if pattern.match(str(conan_ref)):
ret.append(conan_ref)
return sorted(ret)
def _migrate_lock_files(cache, out):
out.warn("Migration: Removing old lock files")
base_dir = cache.store
pkgs = list_folder_subdirs(base_dir, 4)
for pkg in pkgs:
out.info("Removing locks for %s" % pkg)
try:
count = os.path.join(base_dir, pkg, "rw.count")
if os.path.exists(count):
os.remove(count)
count = os.path.join(base_dir, pkg, "rw.count.lock")
if os.path.exists(count):
os.remove(count)
locks = os.path.join(base_dir, pkg, "locks")
if os.path.exists(locks):
shutil.rmtree(locks)
except Exception as e:
raise ConanException("Something went wrong while removing %s locks\n"
"Error: %s\n"
"Please clean your local conan cache manually"
% (pkg, str(e)))
out.warn("Migration: Removing old lock files finished\n")
def _get_local_infos_min(self, reference):
result = {}
packages_path = self._paths.packages(reference)
subdirs = list_folder_subdirs(packages_path, level=1)
for package_id in subdirs:
# Read conaninfo
try:
package_reference = PackageReference(reference, package_id)
info_path = os.path.join(self._paths.package(package_reference,
short_paths=None), CONANINFO)
if not os.path.exists(info_path):
raise NotFoundException("")
conan_info_content = load(info_path)
conan_vars_info = ConanInfo.loads(conan_info_content).serialize_min()
result[package_id] = conan_vars_info
except Exception as exc:
logger.error("Package %s has no ConanInfo file" % str(package_reference))
if str(exc):
logger.error(str(exc))
return result
def _search_recipes(self, pattern=None, ignorecase=True):
subdirs = list_folder_subdirs(basedir=self._server_store.store,
level=5)
if not pattern:
return sorted([
ConanFileReference(*folder.split("/")).copy_clear_rev()
for folder in subdirs
])
else:
# Conan references in main storage
pattern = str(pattern)
b_pattern = translate(pattern)
b_pattern = re.compile(
b_pattern,
re.IGNORECASE) if ignorecase else re.compile(b_pattern)
ret = set()
for subdir in subdirs:
new_ref = ConanFileReference(*subdir.split("/"))
if _partial_match(b_pattern, repr(new_ref)):
ret.add(new_ref.copy_clear_rev())
return sorted(ret)
def _migrate_lock_files(client_cache, out):
out.warn("Migration: Removing old lock files")
base_dir = client_cache.store
pkgs = list_folder_subdirs(base_dir, 4)
for pkg in pkgs:
out.info("Removing locks for %s" % pkg)
try:
count = os.path.join(base_dir, pkg, "rw.count")
if os.path.exists(count):
os.remove(count)
count = os.path.join(base_dir, pkg, "rw.count.lock")
if os.path.exists(count):
os.remove(count)
locks = os.path.join(base_dir, pkg, "locks")
if os.path.exists(locks):
shutil.rmtree(locks)
except Exception as e:
raise ConanException("Something went wrong while removing %s locks\n"
"Error: %s\n"
"Please clean your local conan cache manually"
% (pkg, str(e)))
out.warn("Migration: Removing old lock files finished\n")
def _get_local_infos_min(server_store, ref, look_in_all_rrevs):
result = {}
rrevs = server_store.get_recipe_revisions(ref) if look_in_all_rrevs else [
None
]
for rrev in rrevs:
new_ref = ref.copy_with_rev(rrev.revision) if rrev else ref
subdirs = list_folder_subdirs(server_store.packages(new_ref), level=1)
for package_id in subdirs:
if package_id in result:
continue
# Read conaninfo
try:
pref = PackageReference(new_ref, package_id)
revision_entry = server_store.get_last_package_revision(pref)
if not revision_entry:
raise NotFoundException("")
pref = PackageReference(new_ref, package_id,
revision_entry.revision)
info_path = os.path.join(server_store.package(pref), CONANINFO)
if not os.path.exists(info_path):
raise NotFoundException("")
content = load(info_path)
info = ConanInfo.loads(content)
# From Conan 1.48 the conaninfo.txt is sent raw.
result[package_id] = {"content": content}
# FIXME: This could be removed in the conan_server, Artifactory should keep it
# to guarantee compatibility with old conan clients.
conan_vars_info = info.serialize_min()
result[package_id].update(conan_vars_info)
except Exception as exc: # FIXME: Too wide
logger.error("Package %s has no ConanInfo file" % str(pref))
if str(exc):
logger.error(str(exc))
return result
def _get_local_infos_min(package_layout):
result = OrderedDict()
packages_path = package_layout.packages()
subdirs = list_folder_subdirs(packages_path, level=1)
for package_id in subdirs:
# Read conaninfo
pref = PackageReference(package_layout.ref, package_id)
info_path = os.path.join(package_layout.package(pref), CONANINFO)
if not os.path.exists(info_path):
logger.error("There is no ConanInfo: %s" % str(info_path))
continue
conan_info_content = load(info_path)
info = ConanInfo.loads(conan_info_content)
if package_layout.ref.revision:
metadata = package_layout.load_metadata()
recipe_revision = metadata.packages[package_id].recipe_revision
if recipe_revision and recipe_revision != package_layout.ref.revision:
continue
conan_vars_info = info.serialize_min()
result[package_id] = conan_vars_info
return result
def _get_prefs(layout):
packages_folder = layout.packages()
folders = list_folder_subdirs(packages_folder, 1)
return [PackageReference(layout.ref, s) for s in folders]
def _get_refs(cache):
folders = list_folder_subdirs(cache.store, 4)
return [ConanFileReference(*s.split("/")) for s in folders]
def all_refs(self):
subdirs = list_folder_subdirs(basedir=self._store_folder, level=4)
return [ConanFileReference(*folder.split("/")) for folder in subdirs]
def migrate_to_revisions_layout(self):
# .conan/data/lib/1.0/user/channel/export/*
# .conan/data/lib/1.0/user/channel/0/export/*
# .conan/data/lib/1.0/user/channel/package/*
# .conan/data/lib/1.0/user/channel/0/package/*
# .conan/data/lib/1.0/user/channel/package/XXX/*
# .conan/data/lib/1.0/user/channel/0/package/XXX/0/*
if not self.force_migrations:
print("**********************************************")
print("* *")
print("* ERROR: STORAGE MIGRATION NEEDED! *")
print("* *")
print("**********************************************")
msg = "A migration of your storage is needed, please backup first the storage " \
"directory and run:\n\n$ conan_server --migrate\n\n"
logger.error(msg)
print(msg)
exit(
3
) # Gunicorn expects error code 3 to stop retrying booting the worker
print("**********************************************")
print("* *")
print("* MIGRATION IN PROGRESS *")
print("* *")
print("**********************************************")
subdirs = list_folder_subdirs(basedir=self.store_path, level=4)
for subdir in subdirs:
base_dir = os.path.join(self.store_path, subdir)
for export_or_package in os.listdir(base_dir):
the_dir = os.path.join(base_dir, export_or_package)
dest_dir = os.path.join(base_dir, DEFAULT_REVISION_V1)
mkdir(dest_dir)
print("Moving '%s': %s" % (subdir, export_or_package))
shutil.move(the_dir, dest_dir)
rev_list = RevisionList()
rev_list.add_revision(DEFAULT_REVISION_V1)
save(os.path.join(base_dir, REVISIONS_FILE), rev_list.dumps())
packages_dir = os.path.join(self.store_path, subdir,
DEFAULT_REVISION_V1, PACKAGES_FOLDER)
if not os.path.exists(packages_dir):
print("NO PACKAGES")
continue
for pid in os.listdir(packages_dir):
package_dir = os.path.join(packages_dir, pid)
mkdir(os.path.join(package_dir, DEFAULT_REVISION_V1))
print(" - Package '%s'" % pid)
for item in os.listdir(package_dir):
if item == DEFAULT_REVISION_V1:
continue
origin_path = os.path.join(package_dir, item)
dest_path = os.path.join(package_dir, DEFAULT_REVISION_V1,
item)
mkdir(dest_dir)
shutil.move(origin_path, dest_path)
rev_list = RevisionList()
rev_list.add_revision(DEFAULT_REVISION_V1)
save(os.path.join(package_dir, REVISIONS_FILE),
rev_list.dumps())
print("**********************************************")
print("* *")
print("* MIGRATION COMPLETED! *")
print("* *")
print("**********************************************")
def list_folder_subdirs(self, basedir, level):
from conans.util.files import list_folder_subdirs
return list_folder_subdirs(basedir, level)
def list_folder_subdirs(self, basedir, level):
from conans.util.files import list_folder_subdirs
return list_folder_subdirs(basedir, level)
def all_refs(self):
subdirs = list_folder_subdirs(basedir=self._store_folder, level=4)
return [ConanFileReference.load_dir_repr(folder) for folder in subdirs]