def download(reference, package_ids, remote_name, recipe, remote_manager,
client_cache, out, recorder, loader, hook_manager):
assert(isinstance(reference, ConanFileReference))
output = ScopedOutput(str(reference), out)
registry = client_cache.registry
remote = registry.remotes.get(remote_name) if remote_name else registry.remotes.default
hook_manager.execute("pre_download", reference=reference, remote=remote)
# First of all download package recipe
try:
remote_manager.get_recipe(reference, remote)
except NotFoundException:
raise NotFoundException("'%s' not found in remote '%s'" % (str(reference), remote.name))
registry.refs.set(reference, remote.name)
conan_file_path = client_cache.conanfile(reference)
conanfile = loader.load_class(conan_file_path)
if not recipe: # Not only the recipe
# Download the sources too, don't be lazy
complete_recipe_sources(remote_manager, client_cache, conanfile, reference)
if not package_ids: # User didnt specify a specific package binary
output.info("Getting the complete package list from '%s'..." % str(reference))
packages_props = remote_manager.search_packages(remote, reference, None)
package_ids = list(packages_props.keys())
if not package_ids:
output.warn("No remote binary packages found in remote")
_download_binaries(conanfile, reference, package_ids, client_cache, remote_manager,
remote, output, recorder)
hook_manager.execute("post_download", conanfile_path=conan_file_path, reference=reference,
remote=remote)
def _prepare_sources(client_cache, reference, remote_manager, registry,
loader):
conan_file_path = client_cache.conanfile(reference)
conanfile = loader.load_class(conan_file_path)
complete_recipe_sources(remote_manager, client_cache, registry, conanfile,
reference)
return conanfile.short_paths
def _build_package(self, node, package_ref, output, keep_build):
conan_ref, conan_file = node.conan_ref, node.conanfile
skip_build = conan_file.develop and keep_build
if skip_build:
output.info("Won't be built as specified by --keep-build")
t1 = time.time()
builder = _ConanPackageBuilder(conan_file, package_ref, self._client_cache, output)
if skip_build:
if not os.path.exists(builder.build_folder):
msg = "--keep-build specified, but build folder not found"
self._recorder.package_install_error(package_ref,
INSTALL_ERROR_MISSING_BUILD_FOLDER,
msg, remote=None)
raise ConanException(msg)
else:
with self._client_cache.conanfile_write_lock(conan_ref):
complete_recipe_sources(self._remote_manager, self._client_cache,
self._registry, conan_file, conan_ref)
builder.prepare_build()
with self._client_cache.conanfile_read_lock(conan_ref):
try:
if not skip_build:
builder.build()
builder.package()
except ConanException as exc:
self._recorder.package_install_error(package_ref, INSTALL_ERROR_BUILDING,
str(exc), remote=None)
raise exc
else:
# Log build
self._log_built_package(builder.build_folder, package_ref, time.time() - t1)
def _prepare_sources(self, conanfile, pref, package_layout, conanfile_path,
source_folder, build_folder, remotes):
export_folder = package_layout.export()
export_source_folder = package_layout.export_sources()
scm_sources_folder = package_layout.scm_sources()
complete_recipe_sources(self._remote_manager, self._cache, conanfile,
pref.ref, remotes)
_remove_folder_raising(build_folder)
config_source(export_folder, export_source_folder, scm_sources_folder,
source_folder, conanfile, self._output, conanfile_path,
pref.ref, self._hook_manager, self._cache)
if not getattr(conanfile, 'no_copy_source', False):
self._output.info('Copying sources to build folder')
try:
shutil.copytree(source_folder, build_folder, symlinks=True)
except Exception as e:
msg = str(e)
if "206" in msg: # System error shutil.Error 206: Filename or extension too long
msg += "\nUse short_paths=True if paths too long"
raise ConanException(
"%s\nError copying sources to build folder" % msg)
logger.debug("BUILD: Copied to %s", build_folder)
logger.debug("BUILD: Files copied %s",
",".join(os.listdir(build_folder)))
def download(ref, package_ids, remote, recipe, remote_manager, cache, out,
recorder, loader, hook_manager, remotes):
assert (isinstance(ref, ConanFileReference))
output = ScopedOutput(str(ref), out)
hook_manager.execute("pre_download", reference=ref, remote=remote)
ref = remote_manager.get_recipe(ref, remote)
with cache.package_layout(ref).update_metadata() as metadata:
metadata.recipe.remote = remote.name
conan_file_path = cache.package_layout(ref).conanfile()
conanfile = loader.load_class(conan_file_path)
# Download the sources too, don't be lazy
complete_recipe_sources(remote_manager, cache, conanfile, ref, remotes)
if not recipe: # Not only the recipe
if not package_ids: # User didn't specify a specific package binary
output.info("Getting the complete package list from '%s'..." %
ref.full_repr())
packages_props = remote_manager.search_packages(remote, ref, None)
package_ids = list(packages_props.keys())
if not package_ids:
output.warn("No remote binary packages found in remote")
_download_binaries(conanfile, ref, package_ids, cache, remote_manager,
remote, output, recorder)
hook_manager.execute("post_download",
conanfile_path=conan_file_path,
reference=ref,
remote=remote)
def download(app, ref, package_ids, remote, recipe, recorder, remotes):
out, remote_manager, cache, loader = app.out, app.remote_manager, app.cache, app.loader
hook_manager = app.hook_manager
assert (isinstance(ref, ConanFileReference))
output = ScopedOutput(str(ref), out)
hook_manager.execute("pre_download", reference=ref, remote=remote)
try:
ref = remote_manager.get_recipe(ref, remote)
except NotFoundException:
raise RecipeNotFoundException(ref)
conan_file_path = cache.package_layout(ref).conanfile()
conanfile = loader.load_basic(conan_file_path)
# Download the sources too, don't be lazy
complete_recipe_sources(remote_manager, cache, conanfile, ref, remotes)
if not recipe: # Not only the recipe
if not package_ids: # User didn't specify a specific package binary
output.info("Getting the complete package list from '%s'..." %
ref.full_str())
packages_props = remote_manager.search_packages(remote, ref, None)
package_ids = list(packages_props.keys())
if not package_ids:
output.warn("No remote binary packages found in remote")
parallel = cache.config.parallel_download
_download_binaries(conanfile, ref, package_ids, cache, remote_manager,
remote, output, recorder, parallel)
hook_manager.execute("post_download",
conanfile_path=conan_file_path,
reference=ref,
remote=remote)
def _build_package(self, node, package_ref, output, keep_build):
conan_ref, conan_file = node.conan_ref, node.conanfile
t1 = time.time()
# It is necessary to complete the sources of python requires, which might be used
for python_require in conan_file.python_requires:
complete_recipe_sources(self._remote_manager, self._client_cache,
conan_file, python_require.conan_ref)
builder = _ConanPackageBuilder(conan_file, package_ref,
self._client_cache, output,
self._hook_manager)
if is_dirty(builder.build_folder):
output.warn("Build folder is dirty, removing it: %s" %
builder.build_folder)
rmdir(builder.build_folder)
skip_build = conan_file.develop and keep_build
if skip_build:
output.info("Won't be built as specified by --keep-build")
if skip_build:
if not os.path.exists(builder.build_folder):
msg = "--keep-build specified, but build folder not found"
self._recorder.package_install_error(
package_ref,
INSTALL_ERROR_MISSING_BUILD_FOLDER,
msg,
remote_name=None)
raise ConanException(msg)
else:
with self._client_cache.conanfile_write_lock(conan_ref):
set_dirty(builder.build_folder)
complete_recipe_sources(self._remote_manager,
self._client_cache, conan_file,
conan_ref)
builder.prepare_build()
with self._client_cache.conanfile_read_lock(conan_ref):
try:
if not skip_build:
builder.build()
clean_dirty(builder.build_folder)
builder.package()
except ConanException as exc:
self._recorder.package_install_error(package_ref,
INSTALL_ERROR_BUILDING,
str(exc),
remote_name=None)
raise exc
else:
# Log build
self._log_built_package(builder.build_folder,
package_ref.copy_clear_rev(),
time.time() - t1)
def _upload_recipe(self, ref, conanfile, retry, retry_wait, policy, remote, remotes):
current_remote_name = self._cache.package_layout(ref).load_metadata().recipe.remote
if remote.name != current_remote_name:
complete_recipe_sources(self._remote_manager, self._cache, conanfile, ref, remotes)
conanfile_path = self._cache.package_layout(ref).conanfile()
self._hook_manager.execute("pre_upload_recipe", conanfile_path=conanfile_path,
reference=ref, remote=remote)
t1 = time.time()
the_files = self._compress_recipe_files(ref)
with self._cache.package_layout(ref).update_metadata() as metadata:
metadata.recipe.checksums = calc_files_checksum(the_files)
local_manifest = FileTreeManifest.loads(load(the_files["conanmanifest.txt"]))
remote_manifest = None
if policy != UPLOAD_POLICY_FORCE:
# Check SCM data for auto fields
if hasattr(conanfile, "scm") and (
conanfile.scm.get("url") == "auto" or conanfile.scm.get("revision") == "auto"):
raise ConanException("The recipe has 'scm.url' or 'scm.revision' with 'auto' "
"values. Use '--force' to ignore this error or export again "
"the recipe ('conan export' or 'conan create') in a "
"repository with no-uncommitted changes or by "
"using the '--ignore-dirty' option")
remote_manifest = self._check_recipe_date(ref, remote, local_manifest)
if policy == UPLOAD_POLICY_SKIP:
return ref
files_to_upload, deleted = self._recipe_files_to_upload(ref, policy, the_files,
remote, remote_manifest,
local_manifest)
if files_to_upload or deleted:
self._remote_manager.upload_recipe(ref, files_to_upload, deleted,
remote, retry, retry_wait)
self._upload_recipe_end_msg(ref, remote)
else:
self._output.info("Recipe is up to date, upload skipped")
duration = time.time() - t1
log_recipe_upload(ref, duration, the_files, remote.name)
self._hook_manager.execute("post_upload_recipe", conanfile_path=conanfile_path,
reference=ref, remote=remote)
# The recipe wasn't in the registry or it has changed the revision field only
if not current_remote_name:
with self._cache.package_layout(ref).update_metadata() as metadata:
metadata.recipe.remote = remote.name
return ref
def _build_package(self, node, package_id, package_ref, output, keep_build,
profile_build_requires, flat, deps_graph, update):
conan_ref, conan_file = node.conan_ref, node.conanfile
build_allowed = self._build_mode.allowed(conan_file, conan_ref)
if not build_allowed:
raise_package_not_found_error(conan_file, conan_ref, package_id, output, self._recorder, None)
skip_build = conan_file.develop and keep_build
if skip_build:
output.info("Won't be built as specified by --keep-build")
else:
if conan_file.build_policy_missing:
output.info("Building package from source as defined by build_policy='missing'")
elif self._build_mode.forced(conan_file, conan_ref):
output.warn('Forced build from source')
if not skip_build:
self._build_requires.install(conan_ref, conan_file, self,
profile_build_requires, output, update=update)
# It is important that it is done AFTER build_requires install
self._propagate_info(node, flat, deps_graph)
t1 = time.time()
builder = _ConanPackageBuilder(conan_file, package_ref, self._client_cache, output)
if skip_build:
if not os.path.exists(builder.build_folder):
msg = "--keep-build specified, but build folder not found"
self._recorder.package_install_error(package_ref,
INSTALL_ERROR_MISSING_BUILD_FOLDER,
msg, remote=None)
raise ConanException(msg)
else:
with self._client_cache.conanfile_write_lock(conan_ref):
complete_recipe_sources(self._remote_proxy._remote_manager, self._client_cache,
self._remote_proxy.registry, conan_file, conan_ref)
builder.prepare_build()
with self._client_cache.conanfile_read_lock(conan_ref):
try:
if not skip_build:
builder.build()
builder.package()
except ConanException as exc:
self._recorder.package_install_error(package_ref, INSTALL_ERROR_BUILDING,
str(exc), remote=None)
raise exc
else:
self._remote_proxy.handle_package_manifest(package_ref)
# Log build
self._log_built_package(builder.build_folder, package_ref, time.time() - t1)
self._built_packages.add((conan_ref, package_id))
def _build_package(self, node, package_id, package_ref, output, keep_build,
profile_build_requires, inverse_levels, deps_graph, update):
conan_ref, conan_file = node.conan_ref, node.conanfile
build_allowed = self._build_mode.allowed(conan_file, conan_ref)
if not build_allowed:
raise_package_not_found_error(conan_file, conan_ref, package_id, output, self._recorder, None)
skip_build = conan_file.develop and keep_build
if skip_build:
output.info("Won't be built as specified by --keep-build")
else:
if conan_file.build_policy_missing:
output.info("Building package from source as defined by build_policy='missing'")
elif self._build_mode.forced(conan_file, conan_ref):
output.warn('Forced build from source')
if not skip_build:
self._build_requires.install(conan_ref, conan_file, self,
profile_build_requires, output, update=update)
# It is important that it is done AFTER build_requires install
self._propagate_info(node, inverse_levels, deps_graph)
t1 = time.time()
builder = _ConanPackageBuilder(conan_file, package_ref, self._client_cache, output)
if skip_build:
if not os.path.exists(builder.build_folder):
msg = "--keep-build specified, but build folder not found"
self._recorder.package_install_error(package_ref,
INSTALL_ERROR_MISSING_BUILD_FOLDER,
msg, remote=None)
raise ConanException(msg)
else:
with self._client_cache.conanfile_write_lock(conan_ref):
complete_recipe_sources(self._remote_proxy._remote_manager, self._client_cache,
self._remote_proxy.registry, conan_file, conan_ref)
builder.prepare_build()
with self._client_cache.conanfile_read_lock(conan_ref):
try:
if not skip_build:
builder.build()
builder.package()
except ConanException as exc:
self._recorder.package_install_error(package_ref, INSTALL_ERROR_BUILDING,
str(exc), remote=None)
raise exc
else:
self._remote_proxy.handle_package_manifest(package_ref)
# Log build
self._log_built_package(builder.build_folder, package_ref, time.time() - t1)
self._built_packages.add((conan_ref, package_id))
def _get_recipe(self, conan_reference, check_updates, update):
output = ScopedOutput(str(conan_reference), self._out)
check_updates = check_updates or update
# check if it is in disk
conanfile_path = self._client_cache.conanfile(conan_reference)
if os.path.exists(conanfile_path):
if check_updates:
ret = self.update_available(conan_reference)
if ret != 0: # Found and not equal
remote, ref_remote = self._get_remote(conan_reference)
if ret == 1:
if not update:
if remote != ref_remote: # Forced new remote
output.warn("There is a new conanfile in '%s' remote. "
"Execute 'install -u -r %s' to update it."
% (remote.name, remote.name))
else:
output.warn("There is a new conanfile in '%s' remote. "
"Execute 'install -u' to update it."
% remote.name)
output.warn("Refused to install!")
else:
DiskRemover(self._client_cache).remove(conan_reference)
output.info("Retrieving from remote '%s'..." % remote.name)
self._remote_manager.get_recipe(conan_reference, remote)
output.info("Updated!")
elif ret == -1:
if not update:
output.info("Current conanfile is newer than %s's one" % remote.name)
else:
output.error("Current conanfile is newer than %s's one. "
"Run 'conan remove %s' and run install again "
"to replace it." % (remote.name, conan_reference))
log_recipe_got_from_local_cache(conan_reference)
self._recorder.recipe_fetched_from_cache(conan_reference)
else:
self._retrieve_recipe(conan_reference, output)
if self._manifest_manager:
# Just make sure that the recipe sources are there to check
conanfile = load_conanfile_class(conanfile_path)
complete_recipe_sources(self._remote_manager, self._client_cache, self._registry,
conanfile, conan_reference)
remote = self._registry.get_ref(conan_reference)
self._manifest_manager.check_recipe(conan_reference, remote)
return conanfile_path
def _build_package(self, node, output, keep_build, remotes):
conanfile = node.conanfile
# It is necessary to complete the sources of python requires, which might be used
for python_require in conanfile.python_requires.values():
assert python_require.ref.revision is not None, \
"Installer should receive python_require.ref always"
complete_recipe_sources(self._remote_manager, self._cache,
python_require.conanfile, python_require.ref, remotes)
builder = _PackageBuilder(self._cache, output, self._hook_manager, self._remote_manager)
pref = builder.build_package(node, keep_build, self._recorder, remotes)
if node.graph_lock_node:
node.graph_lock_node.modified = BINARY_BUILD
return pref
def _upload_recipe(self, ref, conanfile, retry, retry_wait, policy, remote,
remotes):
current_remote_name = self._cache.package_layout(
ref).load_metadata().recipe.remote
if remote.name != current_remote_name:
complete_recipe_sources(self._remote_manager, self._cache,
conanfile, ref, remotes)
conanfile_path = self._cache.package_layout(ref).conanfile()
self._hook_manager.execute("pre_upload_recipe",
conanfile_path=conanfile_path,
reference=ref,
remote=remote)
t1 = time.time()
the_files = self._compress_recipe_files(ref)
local_manifest = FileTreeManifest.loads(
load(the_files["conanmanifest.txt"]))
remote_manifest = None
if policy != UPLOAD_POLICY_FORCE:
remote_manifest = self._check_recipe_date(ref, remote,
local_manifest)
if policy == UPLOAD_POLICY_SKIP:
return ref
files_to_upload, deleted = self._recipe_files_to_upload(
ref, policy, the_files, remote, remote_manifest, local_manifest)
if files_to_upload or deleted:
self._remote_manager.upload_recipe(ref, files_to_upload, deleted,
remote, retry, retry_wait)
self._upload_recipe_end_msg(ref, remote)
else:
self._user_io.out.info("Recipe is up to date, upload skipped")
duration = time.time() - t1
log_recipe_upload(ref, duration, the_files, remote.name)
self._hook_manager.execute("post_upload_recipe",
conanfile_path=conanfile_path,
reference=ref,
remote=remote)
# The recipe wasn't in the registry or it has changed the revision field only
if not current_remote_name:
with self._cache.package_layout(ref).update_metadata() as metadata:
metadata.recipe.remote = remote.name
return ref
def _upload_recipe(self, conan_reference, retry, retry_wait, skip_upload, no_overwrite, remote):
conan_file_path = self._client_cache.conanfile(conan_reference)
current_remote = self._registry.get_recipe_remote(conan_reference)
if remote != current_remote:
conanfile = self._loader.load_class(conan_file_path)
complete_recipe_sources(self._remote_manager, self._client_cache, self._registry,
conanfile, conan_reference)
ignore_deleted_file = None
else:
ignore_deleted_file = EXPORT_SOURCES_TGZ_NAME
result = self._remote_manager.upload_recipe(conan_reference, remote, retry, retry_wait,
ignore_deleted_file=ignore_deleted_file,
skip_upload=skip_upload,
no_overwrite=no_overwrite)
return result
def _build_package(self, node, pref, output, keep_build):
conanfile = node.conanfile
assert pref.id, "Package-ID without value"
# It is necessary to complete the sources of python requires, which might be used
for python_require in conanfile.python_requires:
assert python_require.ref.revision is not None, \
"Installer should receive python_require.ref always"
complete_recipe_sources(self._remote_manager, self._cache,
conanfile, python_require.ref)
builder = _PackageBuilder(self._cache, output, self._hook_manager,
self._remote_manager)
pref = builder.build_package(node, keep_build, self._recorder)
return pref
def _upload_recipe(self, conan_reference, retry, retry_wait, skip_upload, no_overwrite, remote):
conan_file_path = self._client_cache.conanfile(conan_reference)
current_remote = self._registry.get_ref(conan_reference)
if remote != current_remote:
conanfile = load_conanfile_class(conan_file_path)
complete_recipe_sources(self._remote_manager, self._client_cache, self._registry,
conanfile, conan_reference)
ignore_deleted_file = None
else:
ignore_deleted_file = EXPORT_SOURCES_TGZ_NAME
result = self._remote_manager.upload_recipe(conan_reference, remote, retry, retry_wait,
ignore_deleted_file=ignore_deleted_file,
skip_upload=skip_upload,
no_overwrite=no_overwrite)
return result
def _upload_recipe(self, conan_reference, retry, retry_wait, policy,
remote, remote_manifest):
conan_file_path = self._client_cache.conanfile(conan_reference)
current_remote = self._registry.get_recipe_remote(conan_reference)
if remote != current_remote:
conanfile = self._loader.load_class(conan_file_path)
complete_recipe_sources(self._remote_manager, self._client_cache,
self._registry, conanfile, conan_reference)
result = self._remote_manager.upload_recipe(
conan_reference,
remote,
retry,
retry_wait,
policy=policy,
remote_manifest=remote_manifest)
return result
def _build_package(self, node, output, keep_build, remotes):
conanfile = node.conanfile
# It is necessary to complete the sources of python requires, which might be used
# Only the legacy python_requires allow this
python_requires = getattr(conanfile, "python_requires", None)
if python_requires and isinstance(python_requires, dict): # Old legacy python_requires
for python_require in python_requires.values():
assert python_require.ref.revision is not None, \
"Installer should receive python_require.ref always"
complete_recipe_sources(self._remote_manager, self._cache,
python_require.conanfile, python_require.ref, remotes)
builder = _PackageBuilder(self._cache, output, self._hook_manager, self._remote_manager)
pref = builder.build_package(node, keep_build, self._recorder, remotes)
if node.graph_lock_node:
node.graph_lock_node.modified = GraphLockNode.MODIFIED_BUILT
return pref
def download(reference, package_ids, remote_name, recipe, registry,
remote_manager, client_cache, out, recorder, loader,
hook_manager):
assert (isinstance(reference, ConanFileReference))
output = ScopedOutput(str(reference), out)
remote = registry.remotes.get(
remote_name) if remote_name else registry.remotes.default
package = remote_manager.search_recipes(remote, reference, None)
if not package: # Search the reference first, and raise if it doesn't exist
raise ConanException("'%s' not found in remote" % str(reference))
hook_manager.execute("pre_download", reference=reference, remote=remote)
# First of all download package recipe
remote_manager.get_recipe(reference, remote)
registry.refs.set(reference, remote.name)
conan_file_path = client_cache.conanfile(reference)
conanfile = loader.load_class(conan_file_path)
if not recipe:
# Download the sources too, don't be lazy
complete_recipe_sources(remote_manager, client_cache, registry,
conanfile, reference)
if package_ids:
_download_binaries(reference, package_ids, client_cache,
remote_manager, remote, output, recorder,
loader)
else:
output.info("Getting the complete package list "
"from '%s'..." % str(reference))
packages_props = remote_manager.search_packages(
remote, reference, None)
if not packages_props:
output = ScopedOutput(str(reference), out)
output.warn("No remote binary packages found in remote")
else:
_download_binaries(reference, list(packages_props.keys()),
client_cache, remote_manager, remote,
output, recorder, loader)
hook_manager.execute("post_download",
conanfile_path=conan_file_path,
reference=reference,
remote=remote)
def download(self, reference, package_ids, remote_name, recipe):
""" Download conanfile and specified packages to local repository
@param reference: ConanFileReference
@param package_ids: Package ids or empty for download all
@param remote: install only from that remote
@param only_recipe: download only the recipe
"""
assert (isinstance(reference, ConanFileReference))
output = ScopedOutput(str(reference), self._user_io.out)
remote = self._registry.remote(
remote_name) if remote_name else self._registry.default_remote
package = self._remote_manager.search_recipes(remote, reference, None)
if not package: # Search the reference first, and raise if it doesn't exist
raise ConanException("'%s' not found in remote" % str(reference))
# First of all download package recipe
self._remote_manager.get_recipe(reference, remote)
self._registry.set_ref(reference, remote)
if recipe:
return
# Download the sources too, don't be lazy
conan_file_path = self._client_cache.conanfile(reference)
conanfile = load_conanfile_class(conan_file_path)
complete_recipe_sources(self._remote_manager, self._client_cache,
self._registry, conanfile, reference)
if package_ids:
download_binaries(reference, package_ids, self._client_cache,
self._remote_manager, remote, output,
self._recorder)
else:
output.info("Getting the complete package list "
"from '%s'..." % str(reference))
packages_props = self._remote_manager.search_packages(
remote, reference, None)
if not packages_props:
output = ScopedOutput(str(reference), self._user_io.out)
output.warn("No remote binary packages found in remote")
else:
download_binaries(reference, list(packages_props.keys()),
self._client_cache, self._remote_manager,
remote, output, self._recorder)
def _upload_recipe(self, ref, conanfile, retry, retry_wait, policy,
remote):
if policy != UPLOAD_POLICY_FORCE:
remote_manifest = self._check_recipe_date(ref, remote)
else:
remote_manifest = None
current_remote = self._registry.refs.get(ref)
if remote != current_remote:
complete_recipe_sources(self._remote_manager, self._cache,
conanfile, ref)
conanfile_path = self._cache.conanfile(ref)
self._hook_manager.execute("pre_upload_recipe",
conanfile_path=conanfile_path,
reference=ref,
remote=remote)
t1 = time.time()
the_files = self._compress_recipe_files(ref)
if policy == UPLOAD_POLICY_SKIP:
return ref
files_to_upload, deleted = self._recipe_files_to_upload(
ref, policy, the_files, remote, remote_manifest)
if files_to_upload or deleted:
self._remote_manager.upload_recipe(ref, files_to_upload, deleted,
remote, retry, retry_wait)
self._upload_recipe_end_msg(ref, remote)
else:
self._user_io.out.info("Recipe is up to date, upload skipped")
duration = time.time() - t1
log_recipe_upload(ref, duration, the_files, remote.name)
self._hook_manager.execute("post_upload_recipe",
conanfile_path=conanfile_path,
reference=ref,
remote=remote)
# The recipe wasn't in the registry or it has changed the revision field only
if not current_remote:
self._registry.refs.set(ref, remote.name)
return ref
def _upload_recipe(self, ref, retry, retry_wait, policy, remote,
remote_manifest):
conan_file_path = self._cache.conanfile(ref)
current_remote = self._registry.refs.get(ref)
if remote != current_remote:
conanfile = self._loader.load_class(conan_file_path)
complete_recipe_sources(self._remote_manager, self._cache,
conanfile, ref)
self._remote_manager.upload_recipe(ref,
remote,
retry,
retry_wait,
policy=policy,
remote_manifest=remote_manifest)
# The recipe wasn't in the registry or it has changed the revision field only
if not current_remote and policy != UPLOAD_POLICY_SKIP:
self._registry.refs.set(ref, remote.name)
return ref
def download(self, reference, package_ids, remote_name, recipe):
""" Download conanfile and specified packages to local repository
@param reference: ConanFileReference
@param package_ids: Package ids or empty for download all
@param remote: install only from that remote
@param only_recipe: download only the recipe
"""
assert(isinstance(reference, ConanFileReference))
remote_proxy = self.get_proxy(remote_name=remote_name)
remote, _ = remote_proxy._get_remote()
package = self._remote_manager.search_recipes(remote, reference, None)
if not package: # Search the reference first, and raise if it doesn't exist
raise ConanException("'%s' not found in remote" % str(reference))
# First of all download package recipe
remote_proxy.get_recipe(reference, check_updates=True, update=True)
if recipe:
return
# Download the sources too, don't be lazy
conan_file_path = self._client_cache.conanfile(reference)
conanfile = load_conanfile_class(conan_file_path)
complete_recipe_sources(self._remote_manager, self._client_cache, self._registry,
conanfile, reference)
if package_ids:
remote_proxy.download_packages(reference, package_ids)
else:
self._user_io.out.info("Getting the complete package list "
"from '%s'..." % str(reference))
packages_props = self._remote_manager.search_packages(remote, reference, None)
if not packages_props:
output = ScopedOutput(str(reference), self._user_io.out)
output.warn("No remote binary packages found in remote")
else:
remote_proxy.download_packages(reference, list(packages_props.keys()))
def install(self, ref_or_path, install_folder, graph_info, remotes=None, build_modes=None,
update=False, manifest_folder=None, manifest_verify=False,
manifest_interactive=False, generators=None, no_imports=False, create_reference=None,
keep_build=False):
""" Fetch and build all dependencies for the given reference
@param ref_or_path: ConanFileReference or path to user space conanfile
@param install_folder: where the output files will be saved
@param remote_name: install only from that remote
@param profile: Profile object with both the -s introduced options and profile read values
@param build_modes: List of build_modes specified
@param update: Check for updated in the upstream remotes (and update)
@param manifest_folder: Folder to install the manifests
@param manifest_verify: Verify dependencies manifests against stored ones
@param manifest_interactive: Install deps manifests in folder for later verify, asking user
for confirmation
@param generators: List of generators from command line. If False, no generator will be
written
@param no_imports: Install specified packages but avoid running imports
"""
if generators is not False:
generators = set(generators) if generators else set()
generators.add("txt") # Add txt generator by default
self._user_io.out.info("Configuration:")
self._user_io.out.writeln(graph_info.profile.dumps())
result = self._graph_manager.load_graph(ref_or_path, create_reference, graph_info,
build_modes, False, update, remotes,
self._recorder)
deps_graph, conanfile = result
if conanfile.display_name == "virtual":
self._user_io.out.highlight("Installing package: %s" % str(ref_or_path))
else:
conanfile.output.highlight("Installing package")
print_graph(deps_graph, self._user_io.out)
try:
if cross_building(graph_info.profile.processed_settings):
settings = get_cross_building_settings(graph_info.profile.processed_settings)
message = "Cross-build from '%s:%s' to '%s:%s'" % settings
self._user_io.out.writeln(message, Color.BRIGHT_MAGENTA)
except ConanException: # Setting os doesn't exist
pass
installer = BinaryInstaller(self._cache, self._user_io.out, self._remote_manager,
recorder=self._recorder,
hook_manager=self._hook_manager)
installer.install(deps_graph, remotes, keep_build=keep_build, graph_info=graph_info)
if manifest_folder:
manifest_manager = ManifestManager(manifest_folder, user_io=self._user_io,
cache=self._cache)
for node in deps_graph.nodes:
if node.recipe in (RECIPE_CONSUMER, RECIPE_VIRTUAL):
continue
complete_recipe_sources(self._remote_manager, self._cache, node.conanfile, node.ref,
remotes)
manifest_manager.check_graph(deps_graph,
verify=manifest_verify,
interactive=manifest_interactive)
manifest_manager.print_log()
if install_folder:
# Write generators
output = conanfile.output if conanfile.display_name != "virtual" else self._user_io.out
if generators is not False:
tmp = list(conanfile.generators) # Add the command line specified generators
tmp.extend([g for g in generators if g not in tmp])
conanfile.generators = tmp
write_generators(conanfile, install_folder, output)
if not isinstance(ref_or_path, ConanFileReference):
# Write conaninfo
content = normalize(conanfile.info.dumps())
save(os.path.join(install_folder, CONANINFO), content)
output.info("Generated %s" % CONANINFO)
graph_info.save(install_folder)
output.info("Generated graphinfo")
if not no_imports:
run_imports(conanfile, install_folder)
call_system_requirements(conanfile, conanfile.output)
if not create_reference and isinstance(ref_or_path, ConanFileReference):
# The conanfile loaded is a virtual one. The one w deploy is the first level one
neighbours = deps_graph.root.neighbors()
deploy_conanfile = neighbours[0].conanfile
if hasattr(deploy_conanfile, "deploy") and callable(deploy_conanfile.deploy):
run_deploy(deploy_conanfile, install_folder)
def install(self,
reference,
install_folder,
profile,
remote_name=None,
build_modes=None,
update=False,
manifest_folder=None,
manifest_verify=False,
manifest_interactive=False,
generators=None,
no_imports=False,
create_reference=None,
keep_build=False):
""" Fetch and build all dependencies for the given reference
@param reference: ConanFileReference or path to user space conanfile
@param install_folder: where the output files will be saved
@param remote: install only from that remote
@param profile: Profile object with both the -s introduced options and profile read values
@param build_modes: List of build_modes specified
@param update: Check for updated in the upstream remotes (and update)
@param manifest_folder: Folder to install the manifests
@param manifest_verify: Verify dependencies manifests against stored ones
@param manifest_interactive: Install deps manifests in folder for later verify, asking user
for confirmation
@param generators: List of generators from command line. If False, no generator will be
written
@param no_imports: Install specified packages but avoid running imports
@param inject_require: Reference to add as a requirement to the conanfile
"""
if generators is not False:
generators = set(generators) if generators else set()
generators.add("txt") # Add txt generator by default
result = self._graph_manager.load_graph(reference, create_reference,
profile, build_modes, False,
update, remote_name,
self._recorder, None)
deps_graph, conanfile, cache_settings = result
if not isinstance(reference, ConanFileReference):
output = ScopedOutput(
("%s (test package)" %
str(create_reference)) if create_reference else "PROJECT",
self._user_io.out)
output.highlight("Installing %s" % reference)
else:
output = ScopedOutput(str(reference), self._user_io.out)
output.highlight("Installing package")
print_graph(deps_graph, self._user_io.out)
try:
if cross_building(cache_settings):
b_os, b_arch, h_os, h_arch = get_cross_building_settings(
cache_settings)
message = "Cross-build from '%s:%s' to '%s:%s'" % (
b_os, b_arch, h_os, h_arch)
self._user_io.out.writeln(message, Color.BRIGHT_MAGENTA)
except ConanException: # Setting os doesn't exist
pass
installer = ConanInstaller(self._client_cache,
output,
self._remote_manager,
self._registry,
recorder=self._recorder,
workspace=None)
installer.install(deps_graph, keep_build)
if manifest_folder:
manifest_manager = ManifestManager(manifest_folder,
user_io=self._user_io,
client_cache=self._client_cache)
for node in deps_graph.nodes:
if not node.conan_ref:
continue
complete_recipe_sources(self._remote_manager,
self._client_cache, self._registry,
node.conanfile, node.conan_ref)
manifest_manager.check_graph(deps_graph,
verify=manifest_verify,
interactive=manifest_interactive)
manifest_manager.print_log()
if install_folder:
# Write generators
if generators is not False:
tmp = list(conanfile.generators
) # Add the command line specified generators
tmp.extend([g for g in generators if g not in tmp])
conanfile.generators = tmp
write_generators(conanfile, install_folder, output)
if not isinstance(reference, ConanFileReference):
# Write conaninfo
content = normalize(conanfile.info.dumps())
save(os.path.join(install_folder, CONANINFO), content)
output.info("Generated %s" % CONANINFO)
if not no_imports:
run_imports(conanfile, install_folder, output)
call_system_requirements(conanfile, output)
if not create_reference and isinstance(reference,
ConanFileReference):
# The conanfile loaded is really a virtual one. The one with the deploy is the first level one
neighbours = deps_graph.root.neighbors()
deploy_conanfile = neighbours[0].conanfile
if hasattr(deploy_conanfile, "deploy") and callable(
deploy_conanfile.deploy):
run_deploy(deploy_conanfile, install_folder, output)
def _upload_recipe(self, ref, conanfile, retry, retry_wait, policy, remote,
remotes):
layout = self._cache.package_layout(ref)
current_remote_name = layout.load_metadata().recipe.remote
if remote.name != current_remote_name:
complete_recipe_sources(self._remote_manager, self._cache,
conanfile, ref, remotes)
conanfile_path = layout.conanfile()
self._hook_manager.execute("pre_upload_recipe",
conanfile_path=conanfile_path,
reference=ref,
remote=remote)
t1 = time.time()
cache_files = self._compress_recipe_files(layout, ref)
with layout.update_metadata() as metadata:
metadata.recipe.checksums = calc_files_checksum(cache_files)
local_manifest = FileTreeManifest.loads(
load(cache_files["conanmanifest.txt"]))
remote_manifest = None
if policy != UPLOAD_POLICY_FORCE:
# Check SCM data for auto fields
if hasattr(conanfile,
"scm") and (conanfile.scm.get("url") == "auto"
or conanfile.scm.get("revision") == "auto"
or conanfile.scm.get("type") is None
or conanfile.scm.get("url") is None
or conanfile.scm.get("revision") is None):
raise ConanException(
"The recipe contains invalid data in the 'scm' attribute"
" (some 'auto' values or missing fields 'type', 'url' or"
" 'revision'). Use '--force' to ignore this error or export"
" again the recipe ('conan export' or 'conan create') to"
" fix these issues.")
remote_manifest = self._check_recipe_date(ref, remote,
local_manifest)
if policy == UPLOAD_POLICY_SKIP:
return ref
files_to_upload, deleted = self._recipe_files_to_upload(
ref, policy, cache_files, remote, remote_manifest, local_manifest)
if files_to_upload or deleted:
self._remote_manager.upload_recipe(ref, files_to_upload, deleted,
remote, retry, retry_wait)
self._upload_recipe_end_msg(ref, remote)
else:
self._output.info("Recipe is up to date, upload skipped")
duration = time.time() - t1
log_recipe_upload(ref, duration, cache_files, remote.name)
self._hook_manager.execute("post_upload_recipe",
conanfile_path=conanfile_path,
reference=ref,
remote=remote)
# The recipe wasn't in the registry or it has changed the revision field only
if not current_remote_name:
with layout.update_metadata() as metadata:
metadata.recipe.remote = remote.name
return ref
def deps_install(app,
ref_or_path,
install_folder,
graph_info,
remotes=None,
build_modes=None,
update=False,
manifest_folder=None,
manifest_verify=False,
manifest_interactive=False,
generators=None,
no_imports=False,
create_reference=None,
keep_build=False,
recorder=None):
""" Fetch and build all dependencies for the given reference
@param app: The ConanApp instance with all collaborators
@param ref_or_path: ConanFileReference or path to user space conanfile
@param install_folder: where the output files will be saved
@param build_modes: List of build_modes specified
@param update: Check for updated in the upstream remotes (and update)
@param manifest_folder: Folder to install the manifests
@param manifest_verify: Verify dependencies manifests against stored ones
@param manifest_interactive: Install deps manifests in folder for later verify, asking user
for confirmation
@param generators: List of generators from command line. If False, no generator will be
written
@param no_imports: Install specified packages but avoid running imports
"""
out, user_io, graph_manager, cache = app.out, app.user_io, app.graph_manager, app.cache
remote_manager, hook_manager = app.remote_manager, app.hook_manager
if generators is not False:
generators = set(generators) if generators else set()
generators.add("txt") # Add txt generator by default
if graph_info.profile_build:
out.info("Configuration (profile_host):")
out.writeln(graph_info.profile_host.dumps())
out.info("Configuration (profile_build):")
out.writeln(graph_info.profile_build.dumps())
else:
out.info("Configuration:")
out.writeln(graph_info.profile_host.dumps())
deps_graph = graph_manager.load_graph(ref_or_path, create_reference,
graph_info, build_modes, False,
update, remotes, recorder)
root_node = deps_graph.root
conanfile = root_node.conanfile
if root_node.recipe == RECIPE_VIRTUAL:
out.highlight("Installing package: %s" % str(ref_or_path))
else:
conanfile.output.highlight("Installing package")
print_graph(deps_graph, out)
try:
if cross_building(conanfile):
settings = get_cross_building_settings(conanfile)
message = "Cross-build from '%s:%s' to '%s:%s'" % settings
out.writeln(message, Color.BRIGHT_MAGENTA)
except ConanException: # Setting os doesn't exist
pass
installer = BinaryInstaller(app, recorder=recorder)
# TODO: Extract this from the GraphManager, reuse same object, check args earlier
build_modes = BuildMode(build_modes, out)
installer.install(deps_graph,
remotes,
build_modes,
update,
keep_build=keep_build,
graph_info=graph_info)
graph_info.graph_lock.complete_matching_prevs()
if manifest_folder:
manifest_manager = ManifestManager(manifest_folder,
user_io=user_io,
cache=cache)
for node in deps_graph.nodes:
if node.recipe in (RECIPE_CONSUMER, RECIPE_VIRTUAL):
continue
complete_recipe_sources(remote_manager, cache, node.conanfile,
node.ref, remotes)
manifest_manager.check_graph(deps_graph,
verify=manifest_verify,
interactive=manifest_interactive)
manifest_manager.print_log()
if install_folder:
conanfile.install_folder = install_folder
# Write generators
output = conanfile.output if root_node.recipe != RECIPE_VIRTUAL else out
if generators is not False:
tmp = list(conanfile.generators
) # Add the command line specified generators
tmp.extend([g for g in generators if g not in tmp])
conanfile.generators = tmp
write_generators(conanfile, install_folder, output)
write_toolchain(conanfile, install_folder, output)
if not isinstance(ref_or_path, ConanFileReference):
# Write conaninfo
content = normalize(conanfile.info.dumps())
save(os.path.join(install_folder, CONANINFO), content)
output.info("Generated %s" % CONANINFO)
graph_info.save(install_folder)
output.info("Generated graphinfo")
graph_lock_file = GraphLockFile(graph_info.profile_host,
graph_info.profile_build,
graph_info.graph_lock)
graph_lock_file.save(os.path.join(install_folder, "conan.lock"))
if not no_imports:
run_imports(conanfile, install_folder)
if type(conanfile
).system_requirements != ConanFile.system_requirements:
call_system_requirements(conanfile, conanfile.output)
if not create_reference and isinstance(ref_or_path,
ConanFileReference):
# The conanfile loaded is a virtual one. The one w deploy is the first level one
neighbours = deps_graph.root.neighbors()
deploy_conanfile = neighbours[0].conanfile
if hasattr(deploy_conanfile, "deploy") and callable(
deploy_conanfile.deploy):
run_deploy(deploy_conanfile, install_folder)
def _prepare_sources(cache, ref, remote_manager, loader):
conan_file_path = cache.conanfile(ref)
conanfile = loader.load_class(conan_file_path)
complete_recipe_sources(remote_manager, cache, conanfile, ref)
return conanfile.short_paths
def _prepare_sources(cache, ref, remote_manager, loader, remotes):
conan_file_path = cache.package_layout(ref).conanfile()
conanfile = loader.load_basic(conan_file_path)
complete_recipe_sources(remote_manager, cache, conanfile, ref, remotes)
return conanfile.short_paths
def _get_recipe(self, conan_reference, check_updates, update):
output = ScopedOutput(str(conan_reference), self._out)
check_updates = check_updates or update
# check if it is in disk
conanfile_path = self._client_cache.conanfile(conan_reference)
if os.path.exists(conanfile_path):
if check_updates:
ret = self.update_available(conan_reference)
if ret != 0: # Found and not equal
remote, ref_remote = self._get_remote(conan_reference)
if ret == 1:
if not update:
if remote != ref_remote: # Forced new remote
output.warn(
"There is a new conanfile in '%s' remote. "
"Execute 'install -u -r %s' to update it."
% (remote.name, remote.name))
else:
output.warn(
"There is a new conanfile in '%s' remote. "
"Execute 'install -u' to update it." %
remote.name)
output.warn("Refused to install!")
else:
DiskRemover(
self._client_cache).remove(conan_reference)
output.info("Retrieving from remote '%s'..." %
remote.name)
self._remote_manager.get_recipe(
conan_reference, remote)
self._registry.set_ref(conan_reference, remote)
output.info("Updated!")
elif ret == -1:
if not update:
output.info(
"Current conanfile is newer than %s's one" %
remote.name)
else:
output.error(
"Current conanfile is newer than %s's one. "
"Run 'conan remove %s' and run install again "
"to replace it." %
(remote.name, conan_reference))
log_recipe_got_from_local_cache(conan_reference)
self._recorder.recipe_fetched_from_cache(conan_reference)
else:
self._retrieve_recipe(conan_reference, output)
if self._manifest_manager:
# Just make sure that the recipe sources are there to check
conanfile = load_conanfile_class(conanfile_path)
complete_recipe_sources(self._remote_manager, self._client_cache,
self._registry, conanfile, conan_reference)
remote = self._registry.get_ref(conan_reference)
self._manifest_manager.check_recipe(conan_reference, remote)
return conanfile_path