def _search_packages_in_local(self, ref=None, query=None, outdated=False):
package_layout = self._cache.package_layout(ref, short_paths=None)
packages_props = search_packages(package_layout, query)
ordered_packages = OrderedDict(sorted(packages_props.items()))
try:
recipe_hash = package_layout.recipe_manifest().summary_hash
except IOError: # It could not exist in local
recipe_hash = None
if outdated:
ordered_packages = filter_outdated(ordered_packages, recipe_hash)
elif self._cache.config.revisions_enabled:
# With revisions, by default filter the packages not belonging to the recipe
# unless outdated is specified.
metadata = package_layout.load_metadata()
ordered_packages = filter_by_revision(metadata, ordered_packages)
references = OrderedDict()
references[None] = self.remote_ref(ordered_packages, recipe_hash)
return references
def search_packages(self, reference, query):
self._authorizer.check_read_conan(self._auth_user, reference)
info = search_packages(self._paths, reference, query)
return info
def remove(self,
pattern,
remote,
src=None,
build_ids=None,
package_ids_filter=None,
force=False,
packages_query=None,
outdated=False):
""" Remove local/remote conans, package folders, etc.
@param src: Remove src folder
@param pattern: it could be OpenCV* or OpenCV or a ConanFileReference
@param build_ids: Lists with ids or empty for all. (Its a filter)
@param package_ids_filter: Lists with ids or empty for all. (Its a filter)
@param force: if True, it will be deleted without requesting anything
@param packages_query: Only if src is a reference. Query settings and options
"""
if remote and (build_ids is not None or src):
raise ConanException(
"Remotes don't have 'build' or 'src' folder, just packages")
if remote:
remote = self._registry.remote(remote)
references = self._remote_manager.search_recipes(remote, pattern)
else:
references = search_recipes(self._client_cache, pattern)
if not references:
self._user_io.out.warn("No package recipe matches '%s'" %
str(pattern))
return
deleted_refs = []
for reference in references:
assert isinstance(reference, ConanFileReference)
package_ids = package_ids_filter
if packages_query or outdated:
# search packages
if remote:
packages = self._remote_manager.search_packages(
remote, reference, packages_query)
else:
packages = search_packages(self._client_cache, reference,
packages_query)
if outdated:
if remote:
recipe_hash = self._remote_manager.get_conan_manifest(
reference, remote).summary_hash
else:
recipe_hash = self._client_cache.load_manifest(
reference).summary_hash
packages = filter_outdated(packages, recipe_hash)
if package_ids_filter:
package_ids = [
p for p in packages if p in package_ids_filter
]
else:
package_ids = list(packages.keys())
if not package_ids:
self._user_io.out.warn(
"No matching packages to remove for %s" %
str(reference))
continue
if self._ask_permission(reference, src, build_ids, package_ids,
force):
deleted_refs.append(reference)
if remote:
self._remote_remove(reference, package_ids, remote)
else:
deleted_refs.append(reference)
self._local_remove(reference, src, build_ids, package_ids)
if not remote:
self._client_cache.delete_empty_dirs(deleted_refs)
def remove(self,
pattern,
remote_name,
src=None,
build_ids=None,
package_ids_filter=None,
force=False,
packages_query=None,
outdated=False):
""" Remove local/remote conans, package folders, etc.
@param src: Remove src folder
@param pattern: it could be OpenCV* or OpenCV or a ConanFileReference
@param build_ids: Lists with ids or empty for all. (Its a filter)
@param package_ids_filter: Lists with ids or empty for all. (Its a filter)
@param force: if True, it will be deleted without requesting anything
@param packages_query: Only if src is a reference. Query settings and options
"""
if remote_name and (build_ids is not None or src):
raise ConanException(
"Remotes don't have 'build' or 'src' folder, just packages")
is_reference = check_valid_ref(pattern)
input_ref = ConanFileReference.loads(pattern) if is_reference else None
if not input_ref and packages_query is not None:
raise ConanException(
"query parameter only allowed with a valid recipe "
"reference as the search pattern.")
if input_ref and package_ids_filter and not input_ref.revision:
for package_id in package_ids_filter:
if "#" in package_id:
raise ConanException(
"Specify a recipe revision if you specify a package "
"revision")
if remote_name:
remote = self._remotes[remote_name]
if input_ref:
if not self._cache.config.revisions_enabled and input_ref.revision:
raise ConanException(
"Revisions not enabled in the client, cannot remove "
"revisions in the server")
refs = [input_ref]
else:
refs = self._remote_manager.search_recipes(remote, pattern)
else:
if input_ref:
refs = []
if self._cache.installed_as_editable(input_ref):
raise ConanException(
self._message_removing_editable(input_ref))
if not self._cache.package_layout(input_ref).recipe_exists():
raise RecipeNotFoundException(
input_ref,
print_rev=self._cache.config.revisions_enabled)
refs.append(input_ref)
else:
refs = search_recipes(self._cache, pattern)
if not refs:
self._user_io.out.warn("No package recipe matches '%s'" %
str(pattern))
return
if input_ref and not input_ref.revision:
# Ignore revisions for deleting if the input was not with a revision
# (Removing all the recipe revisions from a reference)
refs = [r.copy_clear_rev() for r in refs]
deleted_refs = []
for ref in refs:
assert isinstance(ref, ConanFileReference)
package_layout = self._cache.package_layout(ref)
package_ids = package_ids_filter
if packages_query or outdated:
# search packages
if remote_name:
packages = self._remote_manager.search_packages(
remote, ref, packages_query)
else:
packages = search_packages(package_layout, packages_query)
if outdated:
if remote_name:
manifest, ref = self._remote_manager.get_recipe_manifest(
ref, remote)
recipe_hash = manifest.summary_hash
else:
recipe_hash = package_layout.recipe_manifest(
).summary_hash
packages = filter_outdated(packages, recipe_hash)
if package_ids_filter:
package_ids = [
p for p in packages if p in package_ids_filter
]
else:
package_ids = list(packages.keys())
if not package_ids:
self._user_io.out.warn(
"No matching packages to remove for %s" %
ref.full_str())
continue
if self._ask_permission(ref, src, build_ids, package_ids, force):
try:
if remote_name:
self._remote_remove(ref, package_ids, remote)
else:
self._local_remove(ref, src, build_ids, package_ids)
except NotFoundException:
# If we didn't specify a pattern but a concrete ref, fail if there is no
# ref to remove
if input_ref:
raise
else:
deleted_refs.append(ref)
if not remote_name:
self._cache.delete_empty_dirs(deleted_refs)
def upload(self,
recorder,
reference_or_pattern,
package_id=None,
all_packages=None,
confirm=False,
retry=0,
retry_wait=0,
integrity_check=False,
policy=None,
remote_name=None,
query=None):
"""If package_id is provided, conan_reference_or_pattern is a ConanFileReference"""
if package_id and not check_valid_ref(reference_or_pattern,
allow_pattern=False):
raise ConanException(
"-p parameter only allowed with a valid recipe reference, "
"not with a pattern")
t1 = time.time()
if package_id or check_valid_ref(reference_or_pattern,
allow_pattern=False):
# Upload package
ref = ConanFileReference.loads(reference_or_pattern)
references = [
ref,
]
confirm = True
else:
references = search_recipes(self._client_cache,
reference_or_pattern)
if not references:
raise NotFoundException(
("No packages found matching pattern '%s'" %
reference_or_pattern))
for conan_ref in references:
upload = True
if not confirm:
msg = "Are you sure you want to upload '%s'?" % str(conan_ref)
upload = self._user_io.request_boolean(msg)
if upload:
try:
conanfile_path = self._client_cache.conanfile(conan_ref)
conan_file = self._loader.load_class(conanfile_path)
except NotFoundException:
raise NotFoundException(
("There is no local conanfile exported as %s" %
str(conan_ref)))
if all_packages:
packages_ids = self._client_cache.conan_packages(conan_ref)
elif query:
packages = search_packages(self._client_cache, conan_ref,
query)
packages_ids = list(packages.keys())
elif package_id:
packages_ids = [
package_id,
]
else:
packages_ids = []
self._upload(conan_file, conan_ref, packages_ids, retry,
retry_wait, integrity_check, policy, remote_name,
recorder)
logger.debug("UPLOAD: Time manager upload: %f" % (time.time() - t1))
def cmd_export(app,
conanfile_path,
name,
version,
user,
channel,
keep_source,
export=True,
graph_lock=None):
""" Export the recipe
param conanfile_path: the original source directory of the user containing a
conanfile.py
"""
loader, cache, hook_manager, output = app.loader, app.cache, app.hook_manager, app.out
revisions_enabled = app.config.revisions_enabled
conanfile = loader.load_export(conanfile_path, name, version, user,
channel)
# FIXME: Conan 2.0, deprecate CONAN_USER AND CONAN_CHANNEL and remove this try excepts
# Take the default from the env vars if they exist to not break behavior
try:
user = conanfile.user
except ConanException:
user = None
try:
channel = conanfile.channel
except ConanException:
channel = None
ref = ConanFileReference(conanfile.name, conanfile.version, user, channel)
# If we receive lock information, python_requires could have been locked
if graph_lock:
node_id = graph_lock.get_node(ref)
python_requires = graph_lock.python_requires(node_id)
# TODO: check that the locked python_requires are different from the loaded ones
# FIXME: private access, will be improved when api collaborators are improved
loader._python_requires._range_resolver.output # invalidate previous version range output
conanfile = loader.load_export(conanfile_path, conanfile.name,
conanfile.version, conanfile.user,
conanfile.channel, python_requires)
check_casing_conflict(cache=cache, ref=ref)
package_layout = cache.package_layout(ref,
short_paths=conanfile.short_paths)
if not export:
metadata = package_layout.load_metadata()
recipe_revision = metadata.recipe.revision
ref = ref.copy_with_rev(recipe_revision)
if graph_lock:
graph_lock.update_exported_ref(node_id, ref)
return ref
hook_manager.execute("pre_export",
conanfile=conanfile,
conanfile_path=conanfile_path,
reference=package_layout.ref)
logger.debug("EXPORT: %s" % conanfile_path)
output.highlight("Exporting package recipe")
conan_linter(conanfile_path, output)
output = conanfile.output
# Get previous digest
try:
previous_manifest = FileTreeManifest.load(package_layout.export())
except IOError:
previous_manifest = None
finally:
_recreate_folders(package_layout.export(),
package_layout.export_sources())
# Copy sources to target folders
with package_layout.conanfile_write_lock(output=output):
origin_folder = os.path.dirname(conanfile_path)
export_recipe(conanfile, origin_folder, package_layout.export())
export_source(conanfile, origin_folder,
package_layout.export_sources())
shutil.copy2(conanfile_path, package_layout.conanfile())
_capture_export_scm_data(conanfile,
os.path.dirname(conanfile_path),
package_layout.export(),
output,
scm_src_file=package_layout.scm_folder())
# Execute post-export hook before computing the digest
hook_manager.execute("post_export",
conanfile=conanfile,
reference=package_layout.ref,
conanfile_path=package_layout.conanfile())
# Compute the new digest
manifest = FileTreeManifest.create(package_layout.export(),
package_layout.export_sources())
modified_recipe = not previous_manifest or previous_manifest != manifest
if modified_recipe:
output.success('A new %s version was exported' % CONANFILE)
output.info('Folder: %s' % package_layout.export())
else:
output.info("The stored package has not changed")
manifest = previous_manifest # Use the old one, keep old timestamp
manifest.save(package_layout.export())
# Compute the revision for the recipe
revision = _update_revision_in_metadata(
package_layout=package_layout,
revisions_enabled=revisions_enabled,
output=output,
path=os.path.dirname(conanfile_path),
manifest=manifest,
revision_mode=conanfile.revision_mode)
# FIXME: Conan 2.0 Clear the registry entry if the recipe has changed
source_folder = package_layout.source()
if os.path.exists(source_folder):
try:
if is_dirty(source_folder):
output.info("Source folder is corrupted, forcing removal")
rmdir(source_folder)
elif modified_recipe and not keep_source:
output.info(
"Package recipe modified in export, forcing source folder removal"
)
output.info("Use the --keep-source, -k option to skip it")
rmdir(source_folder)
except BaseException as e:
output.error(
"Unable to delete source folder. Will be marked as corrupted for deletion"
)
output.warn(str(e))
set_dirty(source_folder)
# When revisions enabled, remove the packages not matching the revision
if revisions_enabled:
packages = search_packages(package_layout, query=None)
metadata = package_layout.load_metadata()
recipe_revision = metadata.recipe.revision
to_remove = [
pid for pid in packages if metadata.packages.get(pid)
and metadata.packages.get(pid).recipe_revision != recipe_revision
]
if to_remove:
output.info(
"Removing the local binary packages from different recipe revisions"
)
remover = DiskRemover()
remover.remove_packages(package_layout, ids_filter=to_remove)
ref = ref.copy_with_rev(revision)
output.info("Exported revision: %s" % revision)
if graph_lock:
graph_lock.update_exported_ref(node_id, ref)
return ref
def _collect_packages_to_upload(self, refs, confirm, remote_name, all_packages, query,
package_id):
""" compute the references with revisions and the package_ids to be uploaded
"""
# Group recipes by remote
refs_by_remote = defaultdict(list)
default_remote = (self._registry.remotes.get(remote_name) if remote_name else
self._registry.remotes.default)
for ref in refs:
metadata = self._cache.package_layout(ref).load_metadata()
ref = ref.copy_with_rev(metadata.recipe.revision)
if not remote_name:
remote = self._registry.refs.get(ref) or default_remote
else:
remote = default_remote
upload = True
if not confirm:
msg = "Are you sure you want to upload '%s' to '%s'?" % (str(ref), remote.name)
upload = self._user_io.request_boolean(msg)
if upload:
try:
conanfile_path = self._cache.conanfile(ref)
conanfile = self._loader.load_class(conanfile_path)
except NotFoundException:
raise NotFoundException(("There is no local conanfile exported as %s" %
str(ref)))
# TODO: This search of binary packages has to be improved, more robust
# So only real packages are retrieved
if all_packages or query:
if all_packages:
query = None
# better to do a search, that will retrieve real packages with ConanInfo
# Not only "package_id" folders that could be empty
package_layout = self._cache.package_layout(ref.copy_clear_rev())
packages = search_packages(package_layout, query)
packages_ids = list(packages.keys())
elif package_id:
packages_ids = [package_id, ]
else:
packages_ids = []
if packages_ids:
if conanfile.build_policy == "always":
raise ConanException("Conanfile '%s' has build_policy='always', "
"no packages can be uploaded" % str(ref))
prefs = []
# Gather all the complete PREFS with PREV
for package_id in packages_ids:
if package_id not in metadata.packages:
raise ConanException("Binary package %s:%s not found"
% (str(ref), package_id))
# Filter packages that don't match the recipe revision
if self._cache.config.revisions_enabled and ref.revision:
rec_rev = metadata.packages[package_id].recipe_revision
if ref.revision != rec_rev:
self._user_io.out.warn("Skipping package '%s', it doesn't belong to the "
"current recipe revision" % package_id)
continue
package_revision = metadata.packages[package_id].revision
assert package_revision is not None, "PREV cannot be None to upload"
prefs.append(PackageReference(ref, package_id, package_revision))
refs_by_remote[remote].append((ref, conanfile, prefs))
return refs_by_remote
def cmd_export(conanfile_path,
name,
version,
user,
channel,
keep_source,
revisions_enabled,
output,
hook_manager,
loader,
cache,
export=True):
""" Export the recipe
param conanfile_path: the original source directory of the user containing a
conanfile.py
"""
conanfile = loader.load_export(conanfile_path, name, version, user,
channel)
ref = ConanFileReference(conanfile.name, conanfile.version, conanfile.user,
conanfile.channel)
check_casing_conflict(cache=cache, ref=ref)
package_layout = cache.package_layout(ref,
short_paths=conanfile.short_paths)
if not export:
metadata = package_layout.load_metadata()
recipe_revision = metadata.recipe.revision
ref = ref.copy_with_rev(recipe_revision)
return ref
hook_manager.execute("pre_export",
conanfile=conanfile,
conanfile_path=conanfile_path,
reference=package_layout.ref)
logger.debug("EXPORT: %s" % conanfile_path)
output.highlight("Exporting package recipe")
conan_linter(conanfile_path, output)
output = conanfile.output
# Get previous digest
try:
previous_digest = FileTreeManifest.load(package_layout.export())
except IOError:
previous_digest = None
finally:
_recreate_folders(package_layout.export(),
package_layout.export_sources())
# Copy sources to target folders
with package_layout.conanfile_write_lock(output=output):
origin_folder = os.path.dirname(conanfile_path)
export_recipe(conanfile, origin_folder, package_layout.export())
export_source(conanfile, origin_folder,
package_layout.export_sources())
shutil.copy2(conanfile_path, package_layout.conanfile())
_capture_export_scm_data(conanfile,
os.path.dirname(conanfile_path),
package_layout.export(),
output,
scm_src_file=package_layout.scm_folder())
# Execute post-export hook before computing the digest
hook_manager.execute("post_export",
conanfile=conanfile,
reference=package_layout.ref,
conanfile_path=package_layout.conanfile())
# Compute the new digest
digest = FileTreeManifest.create(package_layout.export(),
package_layout.export_sources())
modified_recipe = not previous_digest or previous_digest != digest
if modified_recipe:
output.success('A new %s version was exported' % CONANFILE)
output.info('Folder: %s' % package_layout.export())
else:
output.info("The stored package has not changed")
digest = previous_digest # Use the old one, keep old timestamp
digest.save(package_layout.export())
# Compute the revision for the recipe
revision = _update_revision_in_metadata(
package_layout=package_layout,
revisions_enabled=revisions_enabled,
output=output,
path=os.path.dirname(conanfile_path),
digest=digest,
revision_mode=conanfile.revision_mode)
# FIXME: Conan 2.0 Clear the registry entry if the recipe has changed
source_folder = package_layout.source()
if os.path.exists(source_folder):
try:
if is_dirty(source_folder):
output.info("Source folder is corrupted, forcing removal")
rmdir(source_folder)
elif modified_recipe and not keep_source:
output.info(
"Package recipe modified in export, forcing source folder removal"
)
output.info("Use the --keep-source, -k option to skip it")
rmdir(source_folder)
except BaseException as e:
output.error(
"Unable to delete source folder. Will be marked as corrupted for deletion"
)
output.warn(str(e))
set_dirty(source_folder)
# When revisions enabled, remove the packages not matching the revision
if revisions_enabled:
packages = search_packages(package_layout, query=None)
metadata = package_layout.load_metadata()
recipe_revision = metadata.recipe.revision
to_remove = [
pid for pid in packages if metadata.packages.get(pid)
and metadata.packages.get(pid).recipe_revision != recipe_revision
]
if to_remove:
output.info(
"Removing the local binary packages from different recipe revisions"
)
remover = DiskRemover()
remover.remove_packages(package_layout, ids_filter=to_remove)
ref = ref.copy_with_rev(revision)
return ref
def cmd_export(app,
conanfile_path,
name,
version,
user,
channel,
keep_source,
export=True,
graph_lock=None,
ignore_dirty=False):
""" Export the recipe
param conanfile_path: the original source directory of the user containing a
conanfile.py
"""
loader, cache, hook_manager, output = app.loader, app.cache, app.hook_manager, app.out
revisions_enabled = app.config.revisions_enabled
scm_to_conandata = app.config.scm_to_conandata
conanfile = loader.load_export(conanfile_path, name, version, user,
channel)
# FIXME: Conan 2.0, deprecate CONAN_USER AND CONAN_CHANNEL and remove this try excepts
# Take the default from the env vars if they exist to not break behavior
try:
user = conanfile.user
except ConanV2Exception:
raise
except ConanException:
user = None
try:
channel = conanfile.channel
except ConanV2Exception:
raise
except ConanException:
channel = None
ref = ConanFileReference(conanfile.name, conanfile.version, user, channel)
# If we receive lock information, python_requires could have been locked
if graph_lock:
node_id = graph_lock.get_consumer(ref)
python_requires = graph_lock.python_requires(node_id)
# TODO: check that the locked python_requires are different from the loaded ones
app.range_resolver.clear_output(
) # invalidate previous version range output
conanfile = loader.load_export(conanfile_path, conanfile.name,
conanfile.version, ref.user,
ref.channel, python_requires)
check_casing_conflict(cache=cache, ref=ref)
package_layout = cache.package_layout(ref,
short_paths=conanfile.short_paths)
if not export:
metadata = package_layout.load_metadata()
recipe_revision = metadata.recipe.revision
ref = ref.copy_with_rev(recipe_revision)
if graph_lock:
graph_lock.update_exported_ref(node_id, ref)
return ref
_check_settings_for_warnings(conanfile, output)
hook_manager.execute("pre_export",
conanfile=conanfile,
conanfile_path=conanfile_path,
reference=package_layout.ref)
logger.debug("EXPORT: %s" % conanfile_path)
output.highlight("Exporting package recipe")
output = conanfile.output
# Copy sources to target folders
with package_layout.conanfile_write_lock(output=output):
# Get previous manifest
try:
previous_manifest = package_layout.recipe_manifest()
except IOError:
previous_manifest = None
package_layout.export_remove()
export_folder = package_layout.export()
export_src_folder = package_layout.export_sources()
mkdir(export_folder)
mkdir(export_src_folder)
origin_folder = os.path.dirname(conanfile_path)
export_recipe(conanfile, origin_folder, export_folder)
export_source(conanfile, origin_folder, export_src_folder)
shutil.copy2(conanfile_path, package_layout.conanfile())
# Calculate the "auto" values and replace in conanfile.py
scm_data, local_src_folder = _capture_scm_auto_fields(
conanfile, os.path.dirname(conanfile_path), package_layout, output,
ignore_dirty, scm_to_conandata)
# Clear previous scm_folder
modified_recipe = False
scm_sources_folder = package_layout.scm_sources()
if local_src_folder and not keep_source:
# Copy the local scm folder to scm_sources in the cache
mkdir(scm_sources_folder)
_export_scm(scm_data, local_src_folder, scm_sources_folder, output)
# https://github.com/conan-io/conan/issues/5195#issuecomment-551840597
# It will cause the source folder to be removed (needed because the recipe still has
# the "auto" with uncommitted changes)
modified_recipe = True
# Execute post-export hook before computing the digest
hook_manager.execute("post_export",
conanfile=conanfile,
reference=package_layout.ref,
conanfile_path=package_layout.conanfile())
# Compute the new digest
manifest = FileTreeManifest.create(export_folder, export_src_folder)
modified_recipe |= not previous_manifest or previous_manifest != manifest
if modified_recipe:
output.success('A new %s version was exported' % CONANFILE)
output.info('Folder: %s' % export_folder)
else:
output.info("The stored package has not changed")
manifest = previous_manifest # Use the old one, keep old timestamp
manifest.save(export_folder)
# Compute the revision for the recipe
revision = _update_revision_in_metadata(
package_layout=package_layout,
revisions_enabled=revisions_enabled,
output=output,
path=os.path.dirname(conanfile_path),
manifest=manifest,
revision_mode=conanfile.revision_mode)
# FIXME: Conan 2.0 Clear the registry entry if the recipe has changed
source_folder = package_layout.source()
if os.path.exists(source_folder):
try:
if is_dirty(source_folder):
output.info("Source folder is corrupted, forcing removal")
rmdir(source_folder)
clean_dirty(source_folder)
elif modified_recipe and not keep_source:
output.info(
"Package recipe modified in export, forcing source folder removal"
)
output.info("Use the --keep-source, -k option to skip it")
rmdir(source_folder)
except BaseException as e:
output.error(
"Unable to delete source folder. Will be marked as corrupted for deletion"
)
output.warn(str(e))
set_dirty(source_folder)
# When revisions enabled, remove the packages not matching the revision
if revisions_enabled:
packages = search_packages(package_layout, query=None)
metadata = package_layout.load_metadata()
recipe_revision = metadata.recipe.revision
to_remove = [
pid for pid in packages if metadata.packages.get(pid)
and metadata.packages.get(pid).recipe_revision != recipe_revision
]
if to_remove:
output.info(
"Removing the local binary packages from different recipe revisions"
)
remover = DiskRemover()
remover.remove_packages(package_layout, ids_filter=to_remove)
ref = ref.copy_with_rev(revision)
output.info("Exported revision: %s" % revision)
if graph_lock:
graph_lock.update_exported_ref(node_id, ref)
return ref