def download(app, ref, package_ids, remote, recipe, recorder, remotes):
out, remote_manager, cache, loader = app.out, app.remote_manager, app.cache, app.loader
hook_manager = app.hook_manager
assert (isinstance(ref, ConanFileReference))
output = ScopedOutput(str(ref), out)
hook_manager.execute("pre_download", reference=ref, remote=remote)
try:
ref = remote_manager.get_recipe(ref, remote)
except NotFoundException:
raise RecipeNotFoundException(ref)
conan_file_path = cache.package_layout(ref).conanfile()
conanfile = loader.load_basic(conan_file_path)
# Download the sources too, don't be lazy
retrieve_exports_sources(remote_manager, cache, conanfile, ref, remotes)
if not recipe: # Not only the recipe
if not package_ids: # User didn't specify a specific package binary
output.info("Getting the complete package list from '%s'..." %
ref.full_str())
packages_props = remote_manager.search_packages(remote, ref, None)
package_ids = list(packages_props.keys())
if not package_ids:
output.warn("No remote binary packages found in remote")
parallel = cache.config.parallel_download
_download_binaries(conanfile, ref, package_ids, cache, remote_manager,
remote, output, recorder, parallel)
hook_manager.execute("post_download",
conanfile_path=conan_file_path,
reference=ref,
remote=remote)
def prepare_recipe(self, ref, conanfile, remote, remotes, policy):
""" do a bunch of things that are necessary before actually executing the upload:
- retrieve exports_sources to complete the recipe if necessary
- compress the artifacts in conan_export.tgz and conan_export_sources.tgz
- check if package is ok to be uploaded, if scm info missing, will raise
- check if the remote recipe is newer, raise
- compare and decide which files need to be uploaded (and deleted from server)
"""
layout = self._cache.package_layout(ref)
current_remote_name = layout.load_metadata().recipe.remote
if remote.name != current_remote_name:
retrieve_exports_sources(self._remote_manager, self._cache,
conanfile, ref, remotes)
conanfile_path = layout.conanfile()
self._hook_manager.execute("pre_upload_recipe",
conanfile_path=conanfile_path,
reference=ref,
remote=remote)
t1 = time.time()
cache_files = self._compress_recipe_files(layout, ref)
with layout.update_metadata() as metadata:
metadata.recipe.checksums = calc_files_checksum(cache_files)
local_manifest = FileTreeManifest.loads(
load(cache_files["conanmanifest.txt"]))
remote_manifest = None
if policy != UPLOAD_POLICY_FORCE:
# Check SCM data for auto fields
if hasattr(conanfile,
"scm") and (conanfile.scm.get("url") == "auto"
or conanfile.scm.get("revision") == "auto"
or conanfile.scm.get("type") is None
or conanfile.scm.get("url") is None
or conanfile.scm.get("revision") is None):
raise ConanException(
"The recipe contains invalid data in the 'scm' attribute"
" (some 'auto' values or missing fields 'type', 'url' or"
" 'revision'). Use '--force' to ignore this error or export"
" again the recipe ('conan export' or 'conan create') to"
" fix these issues.")
remote_manifest = self._check_recipe_date(ref, remote,
local_manifest)
if policy == UPLOAD_POLICY_SKIP:
return
files_to_upload, deleted = self._recipe_files_to_upload(
ref, policy, cache_files, remote, remote_manifest, local_manifest)
return files_to_upload, deleted, cache_files, conanfile_path, t1, current_remote_name, layout
def _prepare_sources(self, conanfile, pref, package_layout, remotes):
export_folder = package_layout.export()
export_source_folder = package_layout.export_sources()
scm_sources_folder = package_layout.scm_sources()
conanfile_path = package_layout.conanfile()
source_folder = package_layout.source()
retrieve_exports_sources(self._remote_manager, self._cache, conanfile,
pref.ref, remotes)
config_source(export_folder, export_source_folder, scm_sources_folder,
source_folder, conanfile, self._output, conanfile_path,
pref.ref, self._hook_manager, self._cache)
def _build_package(self, node, output, keep_build, remotes):
conanfile = node.conanfile
# It is necessary to complete the sources of python requires, which might be used
# Only the legacy python_requires allow this
python_requires = getattr(conanfile, "python_requires", None)
if python_requires and isinstance(python_requires, dict): # Old legacy python_requires
for python_require in python_requires.values():
assert python_require.ref.revision is not None, \
"Installer should receive python_require.ref always"
retrieve_exports_sources(self._remote_manager, self._cache,
python_require.conanfile, python_require.ref, remotes)
builder = _PackageBuilder(self._cache, output, self._hook_manager, self._remote_manager,
self._generator_manager)
pref = builder.build_package(node, keep_build, self._recorder, remotes)
if node.graph_lock_node:
node.graph_lock_node.prev = pref.revision
return pref
def deps_install(app,
ref_or_path,
install_folder,
base_folder,
graph_info,
remotes=None,
build_modes=None,
update=False,
manifest_folder=None,
manifest_verify=False,
manifest_interactive=False,
generators=None,
no_imports=False,
create_reference=None,
keep_build=False,
recorder=None,
lockfile_node_id=None,
is_build_require=False,
add_txt_generator=True,
require_overrides=None,
conanfile_path=None,
test=None,
source_folder=None,
output_folder=None):
""" Fetch and build all dependencies for the given reference
@param app: The ConanApp instance with all collaborators
@param ref_or_path: ConanFileReference or path to user space conanfile
@param install_folder: where the output files will be saved
@param build_modes: List of build_modes specified
@param update: Check for updated in the upstream remotes (and update)
@param manifest_folder: Folder to install the manifests
@param manifest_verify: Verify dependencies manifests against stored ones
@param manifest_interactive: Install deps manifests in folder for later verify, asking user
for confirmation
@param generators: List of generators from command line.
@param no_imports: Install specified packages but avoid running imports
@param add_txt_generator: Add the txt to the list of generators
"""
out, user_io, graph_manager, cache = app.out, app.user_io, app.graph_manager, app.cache
remote_manager, hook_manager = app.remote_manager, app.hook_manager
profile_host, profile_build = graph_info.profile_host, graph_info.profile_build
if profile_build:
out.info("Configuration (profile_host):")
out.writeln(profile_host.dumps())
out.info("Configuration (profile_build):")
out.writeln(profile_build.dumps())
else:
out.info("Configuration:")
out.writeln(profile_host.dumps())
deps_graph = graph_manager.load_graph(ref_or_path,
create_reference,
graph_info,
build_modes,
False,
update,
remotes,
recorder,
lockfile_node_id=lockfile_node_id,
is_build_require=is_build_require,
require_overrides=require_overrides)
graph_lock = graph_info.graph_lock # After the graph is loaded it is defined
root_node = deps_graph.root
conanfile = root_node.conanfile
if root_node.recipe == RECIPE_VIRTUAL:
out.highlight("Installing package: %s" % str(ref_or_path))
else:
conanfile.output.highlight("Installing package")
print_graph(deps_graph, out)
try:
if cross_building(conanfile):
settings = get_cross_building_settings(conanfile)
message = "Cross-build from '%s:%s' to '%s:%s'" % settings
out.writeln(message, Color.BRIGHT_MAGENTA)
except ConanException: # Setting os doesn't exist
pass
installer = BinaryInstaller(app, recorder=recorder)
# TODO: Extract this from the GraphManager, reuse same object, check args earlier
build_modes = BuildMode(build_modes, out)
installer.install(deps_graph,
remotes,
build_modes,
update,
profile_host,
profile_build,
graph_lock,
keep_build=keep_build)
graph_lock.complete_matching_prevs()
if manifest_folder:
manifest_manager = ManifestManager(manifest_folder,
user_io=user_io,
cache=cache)
for node in deps_graph.nodes:
if node.recipe in (RECIPE_CONSUMER, RECIPE_VIRTUAL):
continue
retrieve_exports_sources(remote_manager, cache, node.conanfile,
node.ref, remotes)
manifest_manager.check_graph(deps_graph,
verify=manifest_verify,
interactive=manifest_interactive)
manifest_manager.print_log()
if hasattr(conanfile, "layout") and not test:
conanfile.folders.set_base_source(source_folder or conanfile_path)
conanfile.folders.set_base_install(output_folder or conanfile_path)
conanfile.folders.set_base_imports(output_folder or conanfile_path)
conanfile.folders.set_base_generators(output_folder or conanfile_path)
else:
conanfile.folders.set_base_install(install_folder)
conanfile.folders.set_base_imports(install_folder)
conanfile.folders.set_base_generators(base_folder)
output = conanfile.output if root_node.recipe != RECIPE_VIRTUAL else out
if install_folder:
# Write generators
tmp = list(
conanfile.generators) # Add the command line specified generators
generators = set(generators) if generators else set()
tmp.extend([g for g in generators if g not in tmp])
if add_txt_generator:
tmp.append("txt")
conanfile.generators = tmp
app.generator_manager.write_generators(conanfile, install_folder,
conanfile.generators_folder,
output)
write_toolchain(conanfile, conanfile.generators_folder, output)
if not isinstance(ref_or_path, ConanFileReference):
# Write conaninfo
content = normalize(conanfile.info.dumps())
save(os.path.join(install_folder, CONANINFO), content)
output.info("Generated %s" % CONANINFO)
graph_info.save(install_folder)
output.info("Generated graphinfo")
graph_lock_file = GraphLockFile(profile_host, profile_build,
graph_lock)
graph_lock_file.save(os.path.join(install_folder, "conan.lock"))
if not no_imports:
run_imports(conanfile)
if type(conanfile
).system_requirements != ConanFile.system_requirements:
call_system_requirements(conanfile, conanfile.output)
if not create_reference and isinstance(ref_or_path,
ConanFileReference):
# The conanfile loaded is a virtual one. The one w deploy is the first level one
neighbours = deps_graph.root.neighbors()
deploy_conanfile = neighbours[0].conanfile
if hasattr(deploy_conanfile, "deploy") and callable(
deploy_conanfile.deploy):
run_deploy(deploy_conanfile, install_folder)
def _upload_recipe(self, ref, conanfile, retry, retry_wait, policy, remote,
remotes):
layout = self._cache.package_layout(ref)
current_remote_name = layout.load_metadata().recipe.remote
if remote.name != current_remote_name:
retrieve_exports_sources(self._remote_manager, self._cache,
conanfile, ref, remotes)
conanfile_path = layout.conanfile()
self._hook_manager.execute("pre_upload_recipe",
conanfile_path=conanfile_path,
reference=ref,
remote=remote)
t1 = time.time()
cache_files = self._compress_recipe_files(layout, ref)
with layout.update_metadata() as metadata:
metadata.recipe.checksums = calc_files_checksum(cache_files)
local_manifest = FileTreeManifest.loads(
load(cache_files["conanmanifest.txt"]))
remote_manifest = None
if policy != UPLOAD_POLICY_FORCE:
# Check SCM data for auto fields
if hasattr(conanfile,
"scm") and (conanfile.scm.get("url") == "auto"
or conanfile.scm.get("revision") == "auto"
or conanfile.scm.get("type") is None
or conanfile.scm.get("url") is None
or conanfile.scm.get("revision") is None):
raise ConanException(
"The recipe contains invalid data in the 'scm' attribute"
" (some 'auto' values or missing fields 'type', 'url' or"
" 'revision'). Use '--force' to ignore this error or export"
" again the recipe ('conan export' or 'conan create') to"
" fix these issues.")
remote_manifest = self._check_recipe_date(ref, remote,
local_manifest)
if policy == UPLOAD_POLICY_SKIP:
return ref
files_to_upload, deleted = self._recipe_files_to_upload(
ref, policy, cache_files, remote, remote_manifest, local_manifest)
if files_to_upload or deleted:
self._remote_manager.upload_recipe(ref, files_to_upload, deleted,
remote, retry, retry_wait)
self._upload_recipe_end_msg(ref, remote)
else:
self._output.info("Recipe is up to date, upload skipped")
duration = time.time() - t1
log_recipe_upload(ref, duration, cache_files, remote.name)
self._hook_manager.execute("post_upload_recipe",
conanfile_path=conanfile_path,
reference=ref,
remote=remote)
# The recipe wasn't in the registry or it has changed the revision field only
if not current_remote_name:
with layout.update_metadata() as metadata:
metadata.recipe.remote = remote.name
return ref
def _prepare_sources(cache, ref, remote_manager, loader, remotes):
conan_file_path = cache.package_layout(ref).conanfile()
conanfile = loader.load_basic(conan_file_path)
retrieve_exports_sources(remote_manager, cache, conanfile, ref, remotes)
return conanfile.short_paths
