def config_source(export_folder, export_source_folder, scm_sources_folder,
src_folder, conanfile, output, conanfile_path, reference,
hook_manager, cache):
""" Implements the sources configuration when a package is going to be built in the
local cache.
"""
def remove_source(raise_error=True):
output.warn("This can take a while for big packages")
try:
rmdir(src_folder)
except BaseException as e_rm:
set_dirty(src_folder)
msg = str(e_rm)
if six.PY2:
msg = str(e_rm).decode(
"latin1") # Windows prints some chars in latin1
output.error("Unable to remove source folder %s\n%s" %
(src_folder, msg))
output.warn("**** Please delete it manually ****")
if raise_error or isinstance(e_rm, KeyboardInterrupt):
raise ConanException("Unable to remove source folder")
if is_dirty(src_folder):
output.warn("Trying to remove corrupted source folder")
remove_source()
elif conanfile.build_policy_always:
output.warn(
"Detected build_policy 'always', trying to remove source folder")
remove_source()
if not os.path.exists(src_folder): # No source folder, need to get it
with set_dirty_context_manager(src_folder):
mkdir(src_folder)
def get_sources_from_exports():
# First of all get the exported scm sources (if auto) or clone (if fixed)
_run_cache_scm(conanfile, scm_sources_folder, src_folder,
output)
# so self exported files have precedence over python_requires ones
merge_directories(export_folder, src_folder)
# Now move the export-sources to the right location
merge_directories(export_source_folder, src_folder)
_run_source(conanfile,
conanfile_path,
src_folder,
hook_manager,
reference,
cache,
get_sources_from_exports=get_sources_from_exports)
def config_source(export_folder, export_source_folder, scm_sources_folder, conanfile, output,
conanfile_path, reference, hook_manager, cache):
""" Implements the sources configuration when a package is going to be built in the
local cache:
- remove old sources if dirty or build_policy=always
- execute SCM logic
- do a copy of the export and exports_sources folders to the source folder in the cache
- run the source() recipe method
"""
def remove_source():
output.warn("This can take a while for big packages")
try:
rmdir(conanfile.folders.base_source)
except BaseException as e_rm:
msg = str(e_rm)
if six.PY2:
msg = str(e_rm).decode("latin1") # Windows prints some chars in latin1
output.error("Unable to remove source folder %s\n%s" %
(conanfile.folders.base_source, msg))
output.warn("**** Please delete it manually ****")
raise ConanException("Unable to remove source folder")
if is_dirty(conanfile.folders.base_source):
output.warn("Trying to remove corrupted source folder")
remove_source()
clean_dirty(conanfile.folders.base_source)
elif conanfile.build_policy_always:
output.warn("Detected build_policy 'always', trying to remove source folder")
remove_source()
if not os.path.exists(conanfile.folders.base_source): # No source folder, need to get it
with set_dirty_context_manager(conanfile.folders.base_source):
mkdir(conanfile.source_folder)
def get_sources_from_exports():
# First of all get the exported scm sources (if auto) or clone (if fixed)
_run_cache_scm(conanfile, scm_sources_folder, output)
if not hasattr(conanfile, "layout"):
# so self exported files have precedence over python_requires ones
merge_directories(export_folder, conanfile.folders.base_source)
# Now move the export-sources to the right location
merge_directories(export_source_folder, conanfile.folders.base_source)
_run_source(conanfile, conanfile_path, hook_manager, reference, cache,
get_sources_from_exports=get_sources_from_exports)
def _handle_node_cache(self, node, keep_build,
processed_package_references, remotes):
pref = node.pref
assert pref.id, "Package-ID without value"
assert pref.id != PACKAGE_ID_UNKNOWN, "Package-ID error: %s" % str(
pref)
conanfile = node.conanfile
output = conanfile.output
layout = self._cache.package_layout(pref.ref, conanfile.short_paths)
package_folder = layout.package(pref)
with layout.package_lock(pref):
if pref not in processed_package_references:
processed_package_references.add(pref)
if node.binary == BINARY_BUILD:
assert node.prev is None, "PREV for %s to be built should be None" % str(
pref)
with set_dirty_context_manager(package_folder):
pref = self._build_package(node, output, keep_build,
remotes)
assert node.prev, "Node PREV shouldn't be empty"
assert node.pref.revision, "Node PREF revision shouldn't be empty"
assert pref.revision is not None, "PREV for %s to be built is None" % str(
pref)
elif node.binary in (BINARY_UPDATE, BINARY_DOWNLOAD):
# this can happen after a re-evaluation of packageID with Package_ID_unknown
self._download_pkg(layout, node.pref, node)
elif node.binary == BINARY_CACHE:
assert node.prev, "PREV for %s is None" % str(pref)
output.success('Already installed!')
log_package_got_from_local_cache(pref)
self._recorder.package_fetched_from_cache(pref)
# Call the info method
self._call_package_info(conanfile, package_folder, ref=pref.ref)
self._recorder.package_cpp_info(pref, conanfile.cpp_info)
def compress_files(files, symlinks, name, dest_dir, output=None):
t1 = time.time()
# FIXME, better write to disk sequentially and not keep tgz contents in memory
tgz_path = os.path.join(dest_dir, name)
with set_dirty_context_manager(tgz_path), open(tgz_path,
"wb") as tgz_handle:
tgz = gzopen_without_timestamps(name, mode="w", fileobj=tgz_handle)
for filename, dest in sorted(symlinks.items()):
info = tarfile.TarInfo(name=filename)
info.type = tarfile.SYMTYPE
info.linkname = dest
info.size = 0 # A symlink shouldn't have size
tgz.addfile(tarinfo=info)
mask = ~(stat.S_IWOTH | stat.S_IWGRP)
with progress_bar.iterate_list_with_progress(sorted(files.items()),
output, "Compressing %s" %
name) as pg_file_list:
for filename, abs_path in pg_file_list:
info = tarfile.TarInfo(name=filename)
info.size = os.stat(abs_path).st_size
info.mode = os.stat(abs_path).st_mode & mask
if os.path.islink(abs_path):
info.type = tarfile.SYMTYPE
info.size = 0 # A symlink shouldn't have size
info.linkname = os.readlink(abs_path) # @UndefinedVariable
tgz.addfile(tarinfo=info)
else:
with open(abs_path, 'rb') as file_handler:
tgz.addfile(tarinfo=info, fileobj=file_handler)
tgz.close()
duration = time.time() - t1
log_compressed_files(files, duration, tgz_path)
return tgz_path
def compress_files(files, symlinks, name, dest_dir, output=None):
t1 = time.time()
# FIXME, better write to disk sequentially and not keep tgz contents in memory
tgz_path = os.path.join(dest_dir, name)
with set_dirty_context_manager(tgz_path), open(tgz_path,
"wb") as tgz_handle:
tgz = gzopen_without_timestamps(name, mode="w", fileobj=tgz_handle)
for filename, dest in sorted(symlinks.items()):
info = tarfile.TarInfo(name=filename)
info.type = tarfile.SYMTYPE
info.linkname = dest
info.size = 0 # A symlink shouldn't have size
tgz.addfile(tarinfo=info)
mask = ~(stat.S_IWOTH | stat.S_IWGRP)
i_file = 0
n_files = len(files)
last_progress = None
if output and n_files > 1 and not output.is_terminal:
output.write("[")
elif output and n_files > 1 and output.is_terminal:
progress_bar = tqdm(total=len(files),
desc="Compressing %s" % name,
unit="files",
leave=True,
dynamic_ncols=False,
ascii=True,
file=output)
for filename, abs_path in sorted(files.items()):
info = tarfile.TarInfo(name=filename)
info.size = os.stat(abs_path).st_size
info.mode = os.stat(abs_path).st_mode & mask
if os.path.islink(abs_path):
info.type = tarfile.SYMTYPE
info.size = 0 # A symlink shouldn't have size
info.linkname = os.readlink(abs_path) # @UndefinedVariable
tgz.addfile(tarinfo=info)
else:
with open(abs_path, 'rb') as file_handler:
tgz.addfile(tarinfo=info, fileobj=file_handler)
if output and n_files > 1:
i_file = i_file + 1
units = min(50, int(50 * i_file / n_files))
if last_progress != units: # Avoid screen refresh if nothing has change
if not output.is_terminal:
output.write('=' * (units - (last_progress or 0)))
last_progress = units
if output.is_terminal:
progress_bar.update()
if output and n_files > 1:
if output.is_terminal:
progress_bar.close()
else:
output.writeln("]")
tgz.close()
duration = time.time() - t1
log_compressed_files(files, duration, tgz_path)
return tgz_path
def export_pkg(app, recorder, full_ref, source_folder, build_folder,
package_folder, install_folder, graph_info, force, remotes):
ref = full_ref.copy_clear_rev()
cache, output, hook_manager = app.cache, app.out, app.hook_manager
graph_manager = app.graph_manager
conan_file_path = cache.package_layout(ref).conanfile()
if not os.path.exists(conan_file_path):
raise ConanException("Package recipe '%s' does not exist" % str(ref))
# The graph has to be loaded with build_mode=[ref.name], so that node is not tried
# to be downloaded from remotes
# passing here the create_reference=ref argument is useful so the recipe is in "develop",
# because the "package()" method is in develop=True already
deps_graph = graph_manager.load_graph(ref,
ref,
graph_info=graph_info,
build_mode=[ref.name],
check_updates=False,
update=False,
remotes=remotes,
recorder=recorder,
apply_build_requires=False)
# this is a bit tricky, but works. The root (virtual), has only 1 neighbor,
# which is the exported pkg
nodes = deps_graph.root.neighbors()
pkg_node = nodes[0]
conanfile = pkg_node.conanfile
def _init_conanfile_infos():
node_order = [
n for n in pkg_node.public_closure if n.binary != BINARY_SKIP
]
subtree_libnames = [node.ref.name for node in node_order]
add_env_conaninfo(conanfile, subtree_libnames)
_init_conanfile_infos()
from conans.client.conan_api import existing_info_files
if install_folder and existing_info_files(install_folder):
load_deps_info(install_folder, conanfile, required=True)
package_id = pkg_node.package_id
output.info("Packaging to %s" % package_id)
pref = PackageReference(ref, package_id)
layout = cache.package_layout(ref, short_paths=conanfile.short_paths)
dest_package_folder = layout.package(pref)
if os.path.exists(dest_package_folder):
if force:
rmdir(dest_package_folder)
else:
raise ConanException(
"Package already exists. Please use --force, -f to "
"overwrite it")
recipe_hash = layout.recipe_manifest().summary_hash
conanfile.info.recipe_hash = recipe_hash
conanfile.develop = True
with set_dirty_context_manager(dest_package_folder):
if package_folder:
prev = packager.export_pkg(conanfile, package_id, package_folder,
dest_package_folder, hook_manager,
conan_file_path, ref)
else:
prev = run_package_method(conanfile,
package_id,
source_folder,
build_folder,
dest_package_folder,
install_folder,
hook_manager,
conan_file_path,
ref,
local=True)
packager.update_package_metadata(prev, layout, package_id,
full_ref.revision)
pref = PackageReference(pref.ref, pref.id, prev)
if graph_info.graph_lock:
# after the package has been created we need to update the node PREV
pkg_node.prev = pref.revision
graph_info.graph_lock.update_check_graph(deps_graph, output)
recorder.package_exported(pref)
