def broken_package_tgz_test(self):
# https://github.com/conan-io/conan/issues/2854
client = self._client()
client.save({"conanfile.py": conanfile, "source.h": "my source"})
client.run("create . user/testing")
pref = PackageReference.loads("Hello0/1.2.1@user/testing:" +
NO_SETTINGS_PACKAGE_ID)
def gzopen_patched(name,
mode="r",
fileobj=None,
compresslevel=None,
**kwargs):
if name == PACKAGE_TGZ_NAME:
raise ConanException("Error gzopen %s" % name)
return gzopen_without_timestamps(name, mode, fileobj,
compresslevel, **kwargs)
with mock.patch('conans.client.cmd.uploader.gzopen_without_timestamps',
new=gzopen_patched):
client.run("upload * --confirm --all", assert_error=True)
self.assertIn("ERROR: Error gzopen conan_package.tgz", client.out)
export_folder = client.cache.package_layout(pref.ref).package(pref)
tgz = os.path.join(export_folder, PACKAGE_TGZ_NAME)
self.assertTrue(os.path.exists(tgz))
self.assertTrue(is_dirty(tgz))
client.run("upload * --confirm --all")
self.assertIn(
"WARN: Hello0/1.2.1@user/testing:%s: "
"Removing conan_package.tgz, marked as dirty" %
NO_SETTINGS_PACKAGE_ID, client.out)
self.assertTrue(os.path.exists(tgz))
self.assertFalse(is_dirty(tgz))
def broken_sources_tgz_test(self):
# https://github.com/conan-io/conan/issues/2854
client = self._client()
client.save({"conanfile.py": conanfile, "source.h": "my source"})
client.run("create . user/testing")
ref = ConanFileReference.loads("Hello0/1.2.1@user/testing")
def gzopen_patched(name,
mode="r",
fileobj=None,
compresslevel=None,
**kwargs):
raise ConanException("Error gzopen %s" % name)
with mock.patch('conans.client.cmd.uploader.gzopen_without_timestamps',
new=gzopen_patched):
client.run("upload * --confirm", assert_error=True)
self.assertIn("ERROR: Error gzopen conan_sources.tgz", client.out)
export_folder = client.cache.package_layout(ref).export()
tgz = os.path.join(export_folder, EXPORT_SOURCES_TGZ_NAME)
self.assertTrue(os.path.exists(tgz))
self.assertTrue(is_dirty(tgz))
client.run("upload * --confirm")
self.assertIn(
"WARN: Hello0/1.2.1@user/testing: Removing conan_sources.tgz, marked as dirty",
client.out)
self.assertTrue(os.path.exists(tgz))
self.assertFalse(is_dirty(tgz))
def test_broken_package_tgz(self):
# https://github.com/conan-io/conan/issues/2854
client = TestClient(default_server_user=True)
client.save({"conanfile.py": conanfile, "source.h": "my source"})
client.run("create . user/testing")
pref = PackageReference.loads("Hello0/1.2.1@user/testing:" +
NO_SETTINGS_PACKAGE_ID)
def gzopen_patched(name, mode="r", fileobj=None, **kwargs):
if name == PACKAGE_TGZ_NAME:
raise ConanException("Error gzopen %s" % name)
return gzopen_without_timestamps(name, mode, fileobj, **kwargs)
with patch('conans.client.cmd.uploader.gzopen_without_timestamps',
new=gzopen_patched):
client.run("upload * --confirm --all", assert_error=True)
self.assertIn(
"ERROR: Hello0/1.2.1@user/testing:5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9"
": Upload package to 'default' failed: Error gzopen conan_package.tgz",
client.out)
download_folder = client.cache.package_layout(
pref.ref).download_package(pref)
tgz = os.path.join(download_folder, PACKAGE_TGZ_NAME)
self.assertTrue(os.path.exists(tgz))
self.assertTrue(is_dirty(tgz))
client.run("upload * --confirm --all")
self.assertIn(
"WARN: Hello0/1.2.1@user/testing:%s: "
"Removing conan_package.tgz, marked as dirty" %
NO_SETTINGS_PACKAGE_ID, client.out)
self.assertTrue(os.path.exists(tgz))
self.assertFalse(is_dirty(tgz))
def test_export_pkg_clean_dirty(self):
# https://github.com/conan-io/conan/issues/6449
client = TestClient()
conanfile = textwrap.dedent("""
from conans import ConanFile
class Pkg(ConanFile):
def build(self):
if self.in_local_cache:
raise Exception("Can't build while installing")
""")
client.save({"conanfile.py": conanfile})
client.run("create . pkg/0.1@", assert_error=True)
self.assertIn("Can't build while installing", client.out)
ref = ConanFileReference.loads("pkg/0.1")
layout = client.cache.package_layout(ref)
pref = PackageReference(ref,
"5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9")
build_folder = layout.build(pref)
self.assertTrue(is_dirty(build_folder))
package_folder = layout.package(pref)
self.assertTrue(is_dirty(package_folder))
client.run("export-pkg . pkg/0.1@")
self.assertFalse(is_dirty(package_folder))
client.run("install pkg/0.1@")
self.assertIn("pkg/0.1: Already installed!", client.out)
def _compress_package_files(self, pref, integrity_check):
t1 = time.time()
# existing package, will use short paths if defined
package_folder = self._cache.package(pref, short_paths=None)
if is_dirty(package_folder):
raise ConanException("Package %s is corrupted, aborting upload.\n"
"Remove it with 'conan remove %s -p=%s'"
% (pref, pref.ref, pref.id))
tgz_path = os.path.join(package_folder, PACKAGE_TGZ_NAME)
if is_dirty(tgz_path):
self._user_io.out.warn("%s: Removing %s, marked as dirty"
% (str(pref), PACKAGE_TGZ_NAME))
os.remove(tgz_path)
clean_dirty(tgz_path)
# Get all the files in that directory
files, symlinks = gather_files(package_folder)
if CONANINFO not in files or CONAN_MANIFEST not in files:
logger.error("Missing info or manifest in uploading files: %s" % (str(files)))
raise ConanException("Cannot upload corrupted package '%s'" % str(pref))
logger.debug("UPLOAD: Time remote_manager build_files_set : %f" % (time.time() - t1))
if integrity_check:
self._package_integrity_check(pref, files, package_folder)
logger.debug("UPLOAD: Time remote_manager check package integrity : %f"
% (time.time() - t1))
the_files = _compress_package_files(files, symlinks, package_folder, self._user_io.out)
return the_files
def upload_package(self, package_reference, remote, retry, retry_wait, integrity_check=False,
policy=None):
"""Will upload the package to the first remote"""
conanfile_path = self._client_cache.conanfile(package_reference.conan)
self._hook_manager.execute("pre_upload_package", conanfile_path=conanfile_path,
reference=package_reference.conan,
package_id=package_reference.package_id,
remote=remote)
t1 = time.time()
# existing package, will use short paths if defined
package_folder = self._client_cache.package(package_reference, short_paths=None)
if is_dirty(package_folder):
raise ConanException("Package %s is corrupted, aborting upload.\n"
"Remove it with 'conan remove %s -p=%s'" % (package_reference,
package_reference.conan,
package_reference.package_id))
tgz_path = os.path.join(package_folder, PACKAGE_TGZ_NAME)
if is_dirty(tgz_path):
self._output.warn("%s: Removing %s, marked as dirty" % (str(package_reference), PACKAGE_TGZ_NAME))
os.remove(tgz_path)
clean_dirty(tgz_path)
# Get all the files in that directory
files, symlinks = gather_files(package_folder)
if CONANINFO not in files or CONAN_MANIFEST not in files:
logger.error("Missing info or manifest in uploading files: %s" % (str(files)))
raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference))
logger.debug("UPLOAD: Time remote_manager build_files_set : %f" % (time.time() - t1))
if integrity_check:
self._package_integrity_check(package_reference, files, package_folder)
logger.debug("UPLOAD: Time remote_manager check package integrity : %f"
% (time.time() - t1))
the_files = compress_package_files(files, symlinks, package_folder, self._output)
if policy == UPLOAD_POLICY_SKIP:
return None
uploaded, new_pref, rev_time = self._call_remote(remote, "upload_package", package_reference,
the_files, retry, retry_wait, policy)
# Update package revision with the rev_time (Created locally but with rev_time None)
with self._client_cache.update_metadata(new_pref.conan) as metadata:
metadata.packages[new_pref.package_id].time = rev_time
duration = time.time() - t1
log_package_upload(package_reference, duration, the_files, remote)
logger.debug("UPLOAD: Time remote_manager upload_package: %f" % duration)
if not uploaded:
self._output.rewrite_line("Package is up to date, upload skipped")
self._output.writeln("")
self._hook_manager.execute("post_upload_package", conanfile_path=conanfile_path,
reference=package_reference.conan,
package_id=package_reference.package_id, remote=remote)
return new_pref
def upload_package(self,
package_reference,
remote,
retry,
retry_wait,
skip_upload=False,
integrity_check=False,
no_overwrite=None):
"""Will upload the package to the first remote"""
t1 = time.time()
# existing package, will use short paths if defined
package_folder = self._client_cache.package(package_reference,
short_paths=None)
if is_dirty(package_folder):
raise ConanException("Package %s is corrupted, aborting upload.\n"
"Remove it with 'conan remove %s -p=%s'" %
(package_reference, package_reference.conan,
package_reference.package_id))
tgz_path = os.path.join(package_folder, PACKAGE_TGZ_NAME)
if is_dirty(tgz_path):
self._output.warn("%s: Removing %s, marked as dirty" %
(str(package_reference), PACKAGE_TGZ_NAME))
os.remove(tgz_path)
clean_dirty(tgz_path)
# Get all the files in that directory
files, symlinks = gather_files(package_folder)
if CONANINFO not in files or CONAN_MANIFEST not in files:
logger.error("Missing info or manifest in uploading files: %s" %
(str(files)))
raise ConanException("Cannot upload corrupted package '%s'" %
str(package_reference))
logger.debug("====> Time remote_manager build_files_set : %f" %
(time.time() - t1))
if integrity_check:
self._package_integrity_check(package_reference, files,
package_folder)
logger.debug(
"====> Time remote_manager check package integrity : %f" %
(time.time() - t1))
the_files = compress_package_files(files, symlinks, package_folder,
self._output)
if skip_upload:
return None
tmp = self._call_remote(remote, "upload_package", package_reference,
the_files, retry, retry_wait, no_overwrite)
duration = time.time() - t1
log_package_upload(package_reference, duration, the_files, remote)
logger.debug("====> Time remote_manager upload_package: %f" % duration)
if not tmp:
self._output.rewrite_line("Package is up to date, upload skipped")
self._output.writeln("")
return tmp
def upload_recipe(self, conan_reference, remote, retry, retry_wait, ignore_deleted_file,
skip_upload=False, no_overwrite=None):
"""Will upload the conans to the first remote"""
t1 = time.time()
export_folder = self._client_cache.export(conan_reference)
for f in (EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME):
tgz_path = os.path.join(export_folder, f)
if is_dirty(tgz_path):
self._output.warn("%s: Removing %s, marked as dirty" % (str(conan_reference), f))
os.remove(tgz_path)
clean_dirty(tgz_path)
files, symlinks = gather_files(export_folder)
if CONANFILE not in files or CONAN_MANIFEST not in files:
raise ConanException("Cannot upload corrupted recipe '%s'" % str(conan_reference))
export_src_folder = self._client_cache.export_sources(conan_reference, short_paths=None)
src_files, src_symlinks = gather_files(export_src_folder)
the_files = _compress_recipe_files(files, symlinks, src_files, src_symlinks, export_folder,
self._output)
if skip_upload:
return None
ret, new_ref = self._call_remote(remote, "upload_recipe", conan_reference, the_files, retry,
retry_wait, ignore_deleted_file, no_overwrite)
duration = time.time() - t1
log_recipe_upload(new_ref, duration, the_files, remote.name)
if ret:
msg = "Uploaded conan recipe '%s' to '%s'" % (str(new_ref), remote.name)
url = remote.url.replace("https://api.bintray.com/conan", "https://bintray.com")
msg += ": %s" % url
else:
msg = "Recipe is up to date, upload skipped"
self._output.info(msg)
return new_ref
def _get_build_folder(self, conanfile, package_layout, pref, keep_build,
recorder):
# Build folder can use a different package_ID if build_id() is defined.
# This function decides if the build folder should be re-used (not build again)
# and returns the build folder
new_id = build_id(conanfile)
build_pref = PackageReference(pref.ref, new_id) if new_id else pref
build_folder = package_layout.build(build_pref)
if is_dirty(build_folder):
self._output.warn("Build folder is dirty, removing it: %s" %
build_folder)
rmdir(build_folder)
clean_dirty(build_folder)
# Decide if the build folder should be kept
skip_build = conanfile.develop and keep_build
if skip_build:
self._output.info("Won't be built as specified by --keep-build")
if not os.path.exists(build_folder):
msg = "--keep-build specified, but build folder not found"
recorder.package_install_error(
pref,
INSTALL_ERROR_MISSING_BUILD_FOLDER,
msg,
remote_name=None)
raise ConanException(msg)
elif build_pref != pref and os.path.exists(build_folder) and hasattr(
conanfile, "build_id"):
self._output.info(
"Won't be built, using previous build folder as defined in build_id()"
)
skip_build = True
return build_folder, skip_build
def _export_conanfile(conanfile_path, output, client_cache, conanfile,
conan_ref, keep_source):
exports_folder = client_cache.export(conan_ref)
exports_source_folder = client_cache.export_sources(
conan_ref, conanfile.short_paths)
previous_digest = _init_export_folder(exports_folder,
exports_source_folder)
origin_folder = os.path.dirname(conanfile_path)
export_recipe(conanfile, origin_folder, exports_folder, output)
export_source(conanfile, origin_folder, exports_source_folder, output)
shutil.copy2(conanfile_path, os.path.join(exports_folder, CONANFILE))
scm_data, captured_revision = _capture_export_scm_data(
conanfile, os.path.dirname(conanfile_path), exports_folder, output,
client_cache, conan_ref)
digest = FileTreeManifest.create(exports_folder, exports_source_folder)
if previous_digest and previous_digest == digest:
output.info("The stored package has not changed")
modified_recipe = False
digest = previous_digest # Use the old one, keep old timestamp
else:
output.success('A new %s version was exported' % CONANFILE)
output.info('Folder: %s' % exports_folder)
modified_recipe = True
digest.save(exports_folder)
revision = scm_data.revision if scm_data and captured_revision else digest.summary_hash
with client_cache.update_metadata(conan_ref) as metadata:
# Note that there is no time set, the time will come from the remote
metadata.recipe.revision = revision
# FIXME: Conan 2.0 Clear the registry entry if the recipe has changed
source = client_cache.source(conan_ref, conanfile.short_paths)
remove = False
if is_dirty(source):
output.info("Source folder is corrupted, forcing removal")
remove = True
elif modified_recipe and not keep_source and os.path.exists(source):
output.info(
"Package recipe modified in export, forcing source folder removal")
output.info("Use the --keep-source, -k option to skip it")
remove = True
if remove:
output.info(
"Removing 'source' folder, this can take a while for big packages")
try:
# remove only the internal
rmdir(source)
except BaseException as e:
output.error("Unable to delete source folder. "
"Will be marked as corrupted for deletion")
output.warn(str(e))
set_dirty(source)
def upload_recipe(self, conan_reference, remote, retry, retry_wait, policy,
remote_manifest):
conanfile_path = self._client_cache.conanfile(conan_reference)
self._hook_manager.execute("pre_upload_recipe",
conanfile_path=conanfile_path,
reference=conan_reference,
remote=remote)
t1 = time.time()
export_folder = self._client_cache.export(conan_reference)
for f in (EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME):
tgz_path = os.path.join(export_folder, f)
if is_dirty(tgz_path):
self._output.warn("%s: Removing %s, marked as dirty" %
(str(conan_reference), f))
os.remove(tgz_path)
clean_dirty(tgz_path)
files, symlinks = gather_files(export_folder)
if CONANFILE not in files or CONAN_MANIFEST not in files:
raise ConanException("Cannot upload corrupted recipe '%s'" %
str(conan_reference))
export_src_folder = self._client_cache.export_sources(conan_reference,
short_paths=None)
src_files, src_symlinks = gather_files(export_src_folder)
the_files = _compress_recipe_files(files, symlinks, src_files,
src_symlinks, export_folder,
self._output)
if policy == UPLOAD_POLICY_SKIP:
return conan_reference
ret, rev_time = self._call_remote(remote, "upload_recipe",
conan_reference, the_files, retry,
retry_wait, policy, remote_manifest)
# Update package revision with the rev_time (Created locally but with rev_time None)
with self._client_cache.update_metadata(conan_reference) as metadata:
metadata.recipe.time = rev_time
duration = time.time() - t1
log_recipe_upload(conan_reference, duration, the_files, remote.name)
if ret:
msg = "Uploaded conan recipe '%s' to '%s'" % (str(conan_reference),
remote.name)
url = remote.url.replace("https://api.bintray.com/conan",
"https://bintray.com")
msg += ": %s" % url
else:
msg = "Recipe is up to date, upload skipped"
self._output.info(msg)
self._hook_manager.execute("post_upload_recipe",
conanfile_path=conanfile_path,
reference=conan_reference,
remote=remote)
def _build_package(self, node, package_ref, output, keep_build):
conan_ref, conan_file = node.conan_ref, node.conanfile
t1 = time.time()
# It is necessary to complete the sources of python requires, which might be used
for python_require in conan_file.python_requires:
complete_recipe_sources(self._remote_manager, self._client_cache,
conan_file, python_require.conan_ref)
builder = _ConanPackageBuilder(conan_file, package_ref,
self._client_cache, output,
self._hook_manager)
if is_dirty(builder.build_folder):
output.warn("Build folder is dirty, removing it: %s" %
builder.build_folder)
rmdir(builder.build_folder)
skip_build = conan_file.develop and keep_build
if skip_build:
output.info("Won't be built as specified by --keep-build")
if skip_build:
if not os.path.exists(builder.build_folder):
msg = "--keep-build specified, but build folder not found"
self._recorder.package_install_error(
package_ref,
INSTALL_ERROR_MISSING_BUILD_FOLDER,
msg,
remote_name=None)
raise ConanException(msg)
else:
with self._client_cache.conanfile_write_lock(conan_ref):
set_dirty(builder.build_folder)
complete_recipe_sources(self._remote_manager,
self._client_cache, conan_file,
conan_ref)
builder.prepare_build()
with self._client_cache.conanfile_read_lock(conan_ref):
try:
if not skip_build:
builder.build()
clean_dirty(builder.build_folder)
builder.package()
except ConanException as exc:
self._recorder.package_install_error(package_ref,
INSTALL_ERROR_BUILDING,
str(exc),
remote_name=None)
raise exc
else:
# Log build
self._log_built_package(builder.build_folder,
package_ref.copy_clear_rev(),
time.time() - t1)
def config_source(export_folder, export_source_folder, src_folder, conanfile,
output, conanfile_path, reference, hook_manager, cache):
""" Implements the sources configuration when a package is going to be built in the
local cache.
"""
def remove_source(raise_error=True):
output.warn("This can take a while for big packages")
try:
rmdir(src_folder)
except BaseException as e_rm:
set_dirty(src_folder)
msg = str(e_rm)
if six.PY2:
msg = str(e_rm).decode(
"latin1") # Windows prints some chars in latin1
output.error("Unable to remove source folder %s\n%s" %
(src_folder, msg))
output.warn("**** Please delete it manually ****")
if raise_error or isinstance(e_rm, KeyboardInterrupt):
raise ConanException("Unable to remove source folder")
sources_pointer = cache.package_layout(reference).scm_folder()
local_sources_path = load(sources_pointer) if os.path.exists(
sources_pointer) else None
if is_dirty(src_folder):
output.warn("Trying to remove corrupted source folder")
remove_source()
elif conanfile.build_policy_always:
output.warn(
"Detected build_policy 'always', trying to remove source folder")
remove_source()
elif local_sources_path and os.path.exists(local_sources_path):
output.warn(
"Detected 'scm' auto in conanfile, trying to remove source folder")
remove_source()
if not os.path.exists(src_folder): # No source folder, need to get it
set_dirty(src_folder)
mkdir(src_folder)
def get_sources_from_exports():
# so self exported files have precedence over python_requires ones
merge_directories(export_folder, src_folder)
# Now move the export-sources to the right location
merge_directories(export_source_folder, src_folder)
_run_source(conanfile,
conanfile_path,
src_folder,
hook_manager,
reference,
cache,
local_sources_path,
get_sources_from_exports=get_sources_from_exports)
clean_dirty(src_folder) # Everything went well, remove DIRTY flag
def config_source(export_folder, export_source_folder, scm_sources_folder,
conanfile, output, conanfile_path, reference, hook_manager,
cache):
""" Implements the sources configuration when a package is going to be built in the
local cache:
- remove old sources if dirty or build_policy=always
- execute SCM logic
- do a copy of the export and exports_sources folders to the source folder in the cache
- run the source() recipe method
"""
def remove_source():
output.warn("This can take a while for big packages")
try:
rmdir(conanfile.folders.base_source)
except BaseException as e_rm:
msg = str(e_rm)
if six.PY2:
msg = str(e_rm).decode(
"latin1") # Windows prints some chars in latin1
output.error("Unable to remove source folder %s\n%s" %
(conanfile.folders.base_source, msg))
output.warn("**** Please delete it manually ****")
raise ConanException("Unable to remove source folder")
if is_dirty(conanfile.folders.base_source):
output.warn("Trying to remove corrupted source folder")
remove_source()
clean_dirty(conanfile.folders.base_source)
elif conanfile.build_policy_always:
output.warn(
"Detected build_policy 'always', trying to remove source folder")
remove_source()
if not os.path.exists(
conanfile.folders.base_source): # No source folder, need to get it
with set_dirty_context_manager(conanfile.folders.base_source):
mkdir(conanfile.source_folder)
def get_sources_from_exports():
# First of all get the exported scm sources (if auto) or clone (if fixed)
_run_cache_scm(conanfile, scm_sources_folder, output)
if not hasattr(conanfile, "layout"):
# so self exported files have precedence over python_requires ones
merge_directories(export_folder,
conanfile.folders.base_source)
# Now move the export-sources to the right location
merge_directories(export_source_folder,
conanfile.folders.base_source)
_run_source(conanfile,
conanfile_path,
hook_manager,
reference,
cache,
get_sources_from_exports=get_sources_from_exports)
def _get_nodes(self, nodes_by_level, skip_nodes):
"""Compute a list of (conan_ref, package_id, conan_file, build_node)
defining what to do with each node
"""
nodes_to_build = []
# Now build each level, starting from the most independent one
package_references = set()
for level in nodes_by_level:
for node in level:
if node in skip_nodes:
continue
conan_ref, conan_file = node.conan_ref, node.conanfile
build_node = False
logger.debug("Processing node %s", repr(conan_ref))
package_id = conan_file.info.package_id()
package_reference = PackageReference(conan_ref, package_id)
# Avoid processing twice the same package reference
if package_reference not in package_references:
package_references.add(package_reference)
package_folder = self._client_cache.package(
package_reference, short_paths=conan_file.short_paths)
local_project = self._workspace[
conan_ref] if self._workspace else None
if not local_project:
with self._client_cache.package_lock(
package_reference):
if is_dirty(package_folder):
output = ScopedOutput(str(conan_ref),
self._out)
output.warn(
"Package is corrupted, removing folder: %s"
% package_folder)
rmdir(package_folder)
check_outdated = self._build_mode.outdated
if self._build_mode.forced(conan_file, conan_ref):
build_node = True
else:
if local_project:
# Maybe the folder is not there, check it!
build_node = False
else:
available = self._remote_proxy.package_available(
package_reference, package_folder,
check_outdated)
build_node = not available
nodes_to_build.append((node, package_id, build_node))
return nodes_to_build
def _compress_package_files(self, layout, pref, integrity_check):
t1 = time.time()
if layout.package_is_dirty(pref):
raise ConanException("Package %s is corrupted, aborting upload.\n"
"Remove it with 'conan remove %s -p=%s'" %
(pref, pref.ref, pref.id))
download_pkg_folder = layout.download_package(pref)
package_tgz = os.path.join(download_pkg_folder, PACKAGE_TGZ_NAME)
if is_dirty(package_tgz):
self._output.warn("%s: Removing %s, marked as dirty" %
(str(pref), PACKAGE_TGZ_NAME))
os.remove(package_tgz)
clean_dirty(package_tgz)
# Get all the files in that directory
# existing package, will use short paths if defined
package_folder = layout.package(pref)
files, symlinks = gather_files(package_folder)
if CONANINFO not in files or CONAN_MANIFEST not in files:
logger.error("Missing info or manifest in uploading files: %s" %
(str(files)))
raise ConanException("Cannot upload corrupted package '%s'" %
str(pref))
logger.debug("UPLOAD: Time remote_manager build_files_set : %f" %
(time.time() - t1))
if integrity_check:
self._package_integrity_check(pref, files, package_folder)
logger.debug(
"UPLOAD: Time remote_manager check package integrity : %f" %
(time.time() - t1))
if not os.path.isfile(package_tgz):
if self._output and not self._output.is_terminal:
self._output.writeln("Compressing package...")
tgz_files = {
f: path
for f, path in files.items()
if f not in [CONANINFO, CONAN_MANIFEST]
}
tgz_path = compress_files(tgz_files, symlinks, PACKAGE_TGZ_NAME,
download_pkg_folder, self._output)
assert tgz_path == package_tgz
assert os.path.exists(package_tgz)
return {
PACKAGE_TGZ_NAME: package_tgz,
CONANINFO: files[CONANINFO],
CONAN_MANIFEST: files[CONAN_MANIFEST]
}
def config_source(export_folder, export_source_folder, scm_sources_folder,
src_folder, conanfile, output, conanfile_path, reference,
hook_manager, cache):
""" Implements the sources configuration when a package is going to be built in the
local cache.
"""
def remove_source(raise_error=True):
output.warn("This can take a while for big packages")
try:
rmdir(src_folder)
except BaseException as e_rm:
set_dirty(src_folder)
msg = str(e_rm)
if six.PY2:
msg = str(e_rm).decode(
"latin1") # Windows prints some chars in latin1
output.error("Unable to remove source folder %s\n%s" %
(src_folder, msg))
output.warn("**** Please delete it manually ****")
if raise_error or isinstance(e_rm, KeyboardInterrupt):
raise ConanException("Unable to remove source folder")
if is_dirty(src_folder):
output.warn("Trying to remove corrupted source folder")
remove_source()
elif conanfile.build_policy_always:
output.warn(
"Detected build_policy 'always', trying to remove source folder")
remove_source()
if not os.path.exists(src_folder): # No source folder, need to get it
with set_dirty_context_manager(src_folder):
mkdir(src_folder)
def get_sources_from_exports():
# First of all get the exported scm sources (if auto) or clone (if fixed)
_run_cache_scm(conanfile, scm_sources_folder, src_folder,
output)
# so self exported files have precedence over python_requires ones
merge_directories(export_folder, src_folder)
# Now move the export-sources to the right location
merge_directories(export_source_folder, src_folder)
_run_source(conanfile,
conanfile_path,
src_folder,
hook_manager,
reference,
cache,
get_sources_from_exports=get_sources_from_exports)
def _export_conanfile(conanfile_path, output, paths, conanfile, conan_ref,
keep_source):
exports_folder = paths.export(conan_ref)
exports_source_folder = paths.export_sources(conan_ref,
conanfile.short_paths)
previous_digest = _init_export_folder(exports_folder,
exports_source_folder)
_execute_export(conanfile_path, conanfile, exports_folder,
exports_source_folder, output)
shutil.copy2(conanfile_path, os.path.join(exports_folder, CONANFILE))
_capture_export_scm_data(conanfile, os.path.dirname(conanfile_path),
exports_folder, output, paths, conan_ref)
digest = FileTreeManifest.create(exports_folder, exports_source_folder)
if previous_digest and previous_digest == digest:
output.info("The stored package has not changed")
modified_recipe = False
digest = previous_digest # Use the old one, keep old timestamp
else:
output.success('A new %s version was exported' % CONANFILE)
output.info('Folder: %s' % exports_folder)
modified_recipe = True
digest.save(exports_folder)
source = paths.source(conan_ref, conanfile.short_paths)
remove = False
if is_dirty(source):
output.info("Source folder is corrupted, forcing removal")
remove = True
elif modified_recipe and not keep_source and os.path.exists(source):
output.info(
"Package recipe modified in export, forcing source folder removal")
output.info("Use the --keep-source, -k option to skip it")
remove = True
if remove:
output.info(
"Removing 'source' folder, this can take a while for big packages")
try:
# remove only the internal
rmdir(source)
except BaseException as e:
output.error("Unable to delete source folder. "
"Will be marked as corrupted for deletion")
output.warn(str(e))
set_dirty(source)
def download(self,
url,
file_path=None,
md5=None,
sha1=None,
sha256=None,
**kwargs):
""" compatible interface of FileDownloader + checksum
"""
checksum = sha256 or sha1 or md5
# If it is a user download, it must contain a checksum
assert (not self._user_download) or (self._user_download and checksum)
h = self._get_hash(url, checksum)
with self._lock(h):
cached_path = os.path.join(self._cache_folder, h)
if is_dirty(cached_path):
if os.path.exists(cached_path):
os.remove(cached_path)
clean_dirty(cached_path)
if os.path.exists(cached_path):
# If exists but it is corrupted, it is removed. Note that v2 downloads
# do not have checksums, this only works for user downloads
try:
check_checksum(cached_path, md5, sha1, sha256)
except ConanException:
logger.error("Cached file corrupt, redownloading")
remove(cached_path)
if not os.path.exists(cached_path):
set_dirty(cached_path)
self._file_downloader.download(url=url,
file_path=cached_path,
md5=md5,
sha1=sha1,
sha256=sha256,
**kwargs)
clean_dirty(cached_path)
if file_path is not None:
file_path = os.path.abspath(file_path)
mkdir(os.path.dirname(file_path))
shutil.copy2(cached_path, file_path)
else:
with open(cached_path, 'rb') as handle:
tmp = handle.read()
return tmp
def package_remove(self, pref):
# Here we could validate and check we own a write lock over this package
assert isinstance(pref, PackageReference)
assert pref.ref == self._ref, "{!r} != {!r}".format(
pref.ref, self._ref)
# This is NOT the short paths, but the standard cache one
pkg_folder = os.path.join(self._base_folder, PACKAGES_FOLDER, pref.id)
try:
rm_conandir(pkg_folder
) # This will remove the shortened path too if exists
except OSError as e:
raise ConanException(
"%s\n\nFolder: %s\n"
"Couldn't remove folder, might be busy or open\n"
"Close any app using it, and retry" % (pkg_folder, str(e)))
if is_dirty(pkg_folder):
clean_dirty(pkg_folder)
def _compress_recipe_files(self, ref):
export_folder = self._cache.export(ref)
for f in (EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME):
tgz_path = os.path.join(export_folder, f)
if is_dirty(tgz_path):
self._user_io.out.warn("%s: Removing %s, marked as dirty" % (str(ref), f))
os.remove(tgz_path)
clean_dirty(tgz_path)
files, symlinks = gather_files(export_folder)
if CONANFILE not in files or CONAN_MANIFEST not in files:
raise ConanException("Cannot upload corrupted recipe '%s'" % str(ref))
export_src_folder = self._cache.export_sources(ref, short_paths=None)
src_files, src_symlinks = gather_files(export_src_folder)
the_files = _compress_recipe_files(files, symlinks, src_files, src_symlinks, export_folder,
self._user_io.out)
return the_files
def _evaluate_clean_pkg_folder_dirty(self, node, package_layout, package_folder, pref):
# Check if dirty, to remove it
with package_layout.package_lock(pref):
assert node.recipe != RECIPE_EDITABLE, "Editable package shouldn't reach this code"
if is_dirty(package_folder):
node.conanfile.output.warn("Package is corrupted, removing folder: %s"
% package_folder)
rmdir(package_folder) # Do not remove if it is EDITABLE
return
if self._cache.config.revisions_enabled:
metadata = package_layout.load_metadata()
rec_rev = metadata.packages[pref.id].recipe_revision
if rec_rev and rec_rev != node.ref.revision:
node.conanfile.output.warn("The package {} doesn't belong to the installed "
"recipe revision, removing folder".format(pref))
rmdir(package_folder)
return metadata
def _get_nodes(self, nodes_by_level, skip_nodes):
"""Compute a list of (conan_ref, package_id, conan_file, build_node)
defining what to do with each node
"""
nodes_to_build = []
# Now build each level, starting from the most independent one
package_references = set()
for level in nodes_by_level:
for node in level:
if node in skip_nodes:
continue
conan_ref, conan_file = node.conan_ref, node.conanfile
build_node = False
logger.debug("Processing node %s", repr(conan_ref))
package_id = conan_file.info.package_id()
package_reference = PackageReference(conan_ref, package_id)
# Avoid processing twice the same package reference
if package_reference not in package_references:
package_references.add(package_reference)
package_folder = self._client_cache.package(package_reference,
short_paths=conan_file.short_paths)
with self._client_cache.package_lock(package_reference):
if is_dirty(package_folder):
output = ScopedOutput(str(conan_ref), self._out)
output.warn("Package is corrupted, removing folder: %s" % package_folder)
rmdir(package_folder)
check_outdated = self._build_mode.outdated
if self._build_mode.forced(conan_file, conan_ref):
build_node = True
else:
available = self._remote_proxy.package_available(package_reference, package_folder,
check_outdated)
build_node = not available
nodes_to_build.append((node, package_id, build_node))
# A check to be sure that if introduced a pattern, something is going to be built
if self._build_mode.patterns:
to_build = [str(n[0].conan_ref.name) for n in nodes_to_build if n[2]]
self._build_mode.check_matches(to_build)
return nodes_to_build
def _get_nodes(self, nodes_by_level, skip_nodes):
"""Compute a list of (conan_ref, package_id, conan_file, build_node)
defining what to do with each node
"""
nodes_to_build = []
# Now build each level, starting from the most independent one
package_references = set()
for level in nodes_by_level:
for node in level:
if node in skip_nodes:
continue
conan_ref, conan_file = node.conan_ref, node.conanfile
build_node = False
logger.debug("Processing node %s", repr(conan_ref))
package_id = conan_file.info.package_id()
package_reference = PackageReference(conan_ref, package_id)
# Avoid processing twice the same package reference
if package_reference not in package_references:
package_references.add(package_reference)
package_folder = self._client_cache.package(package_reference,
short_paths=conan_file.short_paths)
with self._client_cache.package_lock(package_reference):
if is_dirty(package_folder):
output = ScopedOutput(str(conan_ref), self._out)
output.warn("Package is corrupted, removing folder: %s" % package_folder)
rmdir(package_folder)
check_outdated = self._build_mode.outdated
if self._build_mode.forced(conan_file, conan_ref):
build_node = True
else:
available = self._remote_proxy.package_available(package_reference, package_folder,
check_outdated)
build_node = not available
nodes_to_build.append((node, package_id, build_node))
# A check to be sure that if introduced a pattern, something is going to be built
if self._build_mode.patterns:
to_build = [str(n[0].conan_ref.name) for n in nodes_to_build if n[2]]
self._build_mode.check_matches(to_build)
return nodes_to_build
def _compress_recipe_files(self, ref):
layout = self._cache.package_layout(ref)
download_export_folder = layout.download_export()
for f in (EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME):
tgz_path = os.path.join(download_export_folder, f)
if is_dirty(tgz_path):
self._output.warn("%s: Removing %s, marked as dirty" %
(str(ref), f))
os.remove(tgz_path)
clean_dirty(tgz_path)
export_folder = layout.export()
files, symlinks = gather_files(export_folder)
if CONANFILE not in files or CONAN_MANIFEST not in files:
raise ConanException("Cannot upload corrupted recipe '%s'" %
str(ref))
export_src_folder = layout.export_sources()
src_files, src_symlinks = gather_files(export_src_folder)
result = {
CONANFILE: files.pop(CONANFILE),
CONAN_MANIFEST: files.pop(CONAN_MANIFEST)
}
def add_tgz(tgz_name, tgz_files, tgz_symlinks, msg):
tgz = os.path.join(download_export_folder, tgz_name)
if os.path.isfile(tgz):
result[tgz_name] = tgz
elif tgz_files:
if self._output and not self._output.is_terminal:
self._output.writeln(msg)
tgz = compress_files(tgz_files, tgz_symlinks, tgz_name,
download_export_folder, self._output)
result[tgz_name] = tgz
add_tgz(EXPORT_TGZ_NAME, files, symlinks, "Compressing recipe...")
add_tgz(EXPORT_SOURCES_TGZ_NAME, src_files, src_symlinks,
"Compressing recipe sources...")
return result
def upload_package(self, package_reference, remote, retry, retry_wait, skip_upload=False,
integrity_check=False, no_overwrite=None):
"""Will upload the package to the first remote"""
t1 = time.time()
# existing package, will use short paths if defined
package_folder = self._client_cache.package(package_reference, short_paths=None)
if is_dirty(package_folder):
raise ConanException("Package %s is corrupted, aborting upload.\n"
"Remove it with 'conan remove %s -p=%s'" % (package_reference,
package_reference.conan,
package_reference.package_id))
# Get all the files in that directory
files, symlinks = gather_files(package_folder)
if CONANINFO not in files or CONAN_MANIFEST not in files:
logger.error("Missing info or manifest in uploading files: %s" % (str(files)))
raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference))
logger.debug("====> Time remote_manager build_files_set : %f" % (time.time() - t1))
if integrity_check:
self._package_integrity_check(package_reference, files, package_folder)
logger.debug("====> Time remote_manager check package integrity : %f"
% (time.time() - t1))
the_files = compress_package_files(files, symlinks, package_folder, self._output)
if skip_upload:
return None
tmp = self._call_remote(remote, "upload_package", package_reference, the_files,
retry, retry_wait, no_overwrite)
duration = time.time() - t1
log_package_upload(package_reference, duration, the_files, remote)
logger.debug("====> Time remote_manager upload_package: %f" % duration)
if not tmp:
self._output.rewrite_line("Package is up to date, upload skipped")
self._output.writeln("")
return tmp
def download(self, url, file_path=None, md5=None, sha1=None, sha256=None, **kwargs):
""" compatible interface of FileDownloader + checksum
"""
checksum = sha256 or sha1 or md5
# If it is a user download, it must contain a checksum
assert (not self._user_download) or (self._user_download and checksum)
h = self._get_hash(url, checksum)
with self._lock(h):
cached_path = os.path.join(self._cache_folder, h)
if is_dirty(cached_path):
if os.path.exists(cached_path):
os.remove(cached_path)
clean_dirty(cached_path)
if not os.path.exists(cached_path):
set_dirty(cached_path)
self._file_downloader.download(url=url, file_path=cached_path, md5=md5,
sha1=sha1, sha256=sha256, **kwargs)
clean_dirty(cached_path)
else:
# specific check for corrupted cached files, will raise, but do nothing more
# user can report it or "rm -rf cache_folder/path/to/file"
try:
check_checksum(cached_path, md5, sha1, sha256)
except ConanException as e:
raise ConanException("%s\nCached downloaded file corrupted: %s"
% (str(e), cached_path))
if file_path is not None:
file_path = os.path.abspath(file_path)
mkdir(os.path.dirname(file_path))
shutil.copy2(cached_path, file_path)
else:
with open(cached_path, 'rb') as handle:
tmp = handle.read()
return tmp
def config_source(export_folder, export_source_folder, src_folder,
conan_file, output, force=False):
""" creates src folder and retrieve, calling source() from conanfile
the necessary source code
"""
def remove_source(raise_error=True):
output.warn("This can take a while for big packages")
try:
rmdir(src_folder)
except BaseException as e_rm:
set_dirty(src_folder)
msg = str(e_rm)
if six.PY2:
msg = str(e_rm).decode("latin1") # Windows prints some chars in latin1
output.error("Unable to remove source folder %s\n%s" % (src_folder, msg))
output.warn("**** Please delete it manually ****")
if raise_error or isinstance(e_rm, KeyboardInterrupt):
raise ConanException("Unable to remove source folder")
if force:
output.warn("Forced removal of source folder")
remove_source()
elif is_dirty(src_folder):
output.warn("Trying to remove corrupted source folder")
remove_source()
elif conan_file.build_policy_always:
output.warn("Detected build_policy 'always', trying to remove source folder")
remove_source()
if not os.path.exists(src_folder):
output.info('Configuring sources in %s' % src_folder)
shutil.copytree(export_folder, src_folder, symlinks=True)
# Now move the export-sources to the right location
merge_directories(export_source_folder, src_folder)
for f in (EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME, CONANFILE+"c",
CONANFILE+"o", CONANFILE, CONAN_MANIFEST):
try:
os.remove(os.path.join(src_folder, f))
except OSError:
pass
try:
shutil.rmtree(os.path.join(src_folder, "__pycache__"))
except OSError:
pass
set_dirty(src_folder)
os.chdir(src_folder)
conan_file.source_folder = src_folder
try:
with get_env_context_manager(conan_file):
with conanfile_exception_formatter(str(conan_file), "source"):
conan_file.build_folder = None
conan_file.package_folder = None
conan_file.source()
clean_dirty(src_folder) # Everything went well, remove DIRTY flag
except Exception as e:
os.chdir(export_folder)
# in case source() fails (user error, typically), remove the src_folder
# and raise to interrupt any other processes (build, package)
output.warn("Trying to remove corrupted source folder")
remove_source(raise_error=False)
if isinstance(e, ConanExceptionInUserConanfileMethod):
raise e
raise ConanException(e)
def config_source(export_folder,
export_source_folder,
local_sources_path,
src_folder,
conan_file,
output,
force=False):
""" creates src folder and retrieve, calling source() from conanfile
the necessary source code
"""
def remove_source(raise_error=True):
output.warn("This can take a while for big packages")
try:
rmdir(src_folder)
except BaseException as e_rm:
set_dirty(src_folder)
msg = str(e_rm)
if six.PY2:
msg = str(e_rm).decode(
"latin1") # Windows prints some chars in latin1
output.error("Unable to remove source folder %s\n%s" %
(src_folder, msg))
output.warn("**** Please delete it manually ****")
if raise_error or isinstance(e_rm, KeyboardInterrupt):
raise ConanException("Unable to remove source folder")
if force:
output.warn("Forced removal of source folder")
remove_source()
elif is_dirty(src_folder):
output.warn("Trying to remove corrupted source folder")
remove_source()
elif conan_file.build_policy_always:
output.warn(
"Detected build_policy 'always', trying to remove source folder")
remove_source()
if not os.path.exists(src_folder):
output.info('Configuring sources in %s' % src_folder)
shutil.copytree(export_folder, src_folder, symlinks=True)
# Now move the export-sources to the right location
merge_directories(export_source_folder, src_folder)
_clean_source_folder(src_folder)
try:
shutil.rmtree(os.path.join(src_folder, "__pycache__"))
except OSError:
pass
set_dirty(src_folder)
os.chdir(src_folder)
conan_file.source_folder = src_folder
try:
with get_env_context_manager(conan_file):
with conanfile_exception_formatter(str(conan_file), "source"):
conan_file.build_folder = None
conan_file.package_folder = None
scm = get_scm(conan_file, src_folder)
if scm:
# scm.capture_origin before exporting
if local_sources_path and os.path.exists(
local_sources_path):
output.info("Getting sources from folder: %s" %
local_sources_path)
merge_directories(local_sources_path, src_folder)
_clean_source_folder(src_folder)
else:
output.info("Getting sources from url: '%s'" %
scm.url)
scm.clone()
scm.checkout()
conan_file.source()
clean_dirty(src_folder) # Everything went well, remove DIRTY flag
except Exception as e:
os.chdir(export_folder)
# in case source() fails (user error, typically), remove the src_folder
# and raise to interrupt any other processes (build, package)
output.warn("Trying to remove corrupted source folder")
remove_source(raise_error=False)
if isinstance(e, ConanExceptionInUserConanfileMethod):
raise e
raise ConanException(e)
def package_is_dirty(self, pref):
pkg_folder = os.path.join(self._base_folder, PACKAGES_FOLDER, pref.id)
return is_dirty(pkg_folder)
def config_source(export_folder,
export_source_folder,
src_folder,
conan_file,
output,
force=False):
""" creates src folder and retrieve, calling source() from conanfile
the necessary source code
"""
def remove_source(raise_error=True):
output.warn("This can take a while for big packages")
try:
rmdir(src_folder)
except BaseException as e_rm:
set_dirty(src_folder)
msg = str(e_rm)
if six.PY2:
msg = str(e_rm).decode(
"latin1") # Windows prints some chars in latin1
output.error("Unable to remove source folder %s\n%s" %
(src_folder, msg))
output.warn("**** Please delete it manually ****")
if raise_error or isinstance(e_rm, KeyboardInterrupt):
raise ConanException("Unable to remove source folder")
if force:
output.warn("Forced removal of source folder")
remove_source()
elif is_dirty(src_folder):
output.warn("Trying to remove dirty source folder")
remove_source()
elif conan_file.build_policy_always:
output.warn(
"Detected build_policy 'always', trying to remove source folder")
remove_source()
if not os.path.exists(src_folder):
output.info('Configuring sources in %s' % src_folder)
shutil.copytree(export_folder, src_folder, symlinks=True)
# Now move the export-sources to the right location
merge_directories(export_source_folder, src_folder)
for f in (EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME, CONANFILE + "c",
CONANFILE + "o", CONANFILE, CONAN_MANIFEST):
try:
os.remove(os.path.join(src_folder, f))
except OSError:
pass
try:
shutil.rmtree(os.path.join(src_folder, "__pycache__"))
except OSError:
pass
set_dirty(src_folder)
os.chdir(src_folder)
conan_file.source_folder = src_folder
try:
with tools.environment_append(conan_file.env):
with conanfile_exception_formatter(str(conan_file), "source"):
conan_file.build_folder = None
conan_file.package_folder = None
conan_file.source()
clean_dirty(src_folder) # Everything went well, remove DIRTY flag
except Exception as e:
os.chdir(export_folder)
# in case source() fails (user error, typically), remove the src_folder
# and raise to interrupt any other processes (build, package)
output.warn("Trying to remove dirty source folder")
remove_source(raise_error=False)
if isinstance(e, ConanExceptionInUserConanfileMethod):
raise e
raise ConanException(e)
def _evaluate_node(self, node, build_mode, update, evaluated_references,
remote_name):
assert node.binary is None
conan_ref, conanfile = node.conan_ref, node.conanfile
revisions_enabled = get_env("CONAN_CLIENT_REVISIONS_ENABLED", False)
package_id = conanfile.info.package_id()
package_ref = PackageReference(conan_ref, package_id)
# Check that this same reference hasn't already been checked
previous_node = evaluated_references.get(package_ref)
if previous_node:
node.binary = previous_node.binary
node.binary_remote = previous_node.binary_remote
return
evaluated_references[package_ref] = node
output = ScopedOutput(str(conan_ref), self._out)
if build_mode.forced(conanfile, conan_ref):
output.warn('Forced build from source')
node.binary = BINARY_BUILD
return
package_folder = self._client_cache.package(
package_ref, short_paths=conanfile.short_paths)
# Check if dirty, to remove it
local_project = self._workspace[conan_ref] if self._workspace else None
if local_project:
node.binary = BINARY_WORKSPACE
return
with self._client_cache.package_lock(package_ref):
if is_dirty(package_folder):
output.warn("Package is corrupted, removing folder: %s" %
package_folder)
rmdir(package_folder)
if remote_name:
remote = self._registry.remotes.get(remote_name)
else:
# If the remote_name is not given, follow the binary remote, or
# the recipe remote
# If it is defined it won't iterate (might change in conan2.0)
remote = self._registry.prefs.get(
package_ref) or self._registry.refs.get(conan_ref)
remotes = self._registry.remotes.list
if os.path.exists(package_folder):
if update:
if remote:
if self._check_update(package_folder, package_ref, remote,
output, node):
node.binary = BINARY_UPDATE
if build_mode.outdated:
package_hash = self._get_package_info(
package_ref, remote).recipe_hash
elif remotes:
pass
else:
output.warn("Can't update, no remote defined")
if not node.binary:
node.binary = BINARY_CACHE
package_hash = ConanInfo.load_from_package(
package_folder).recipe_hash
else: # Binary does NOT exist locally
if not revisions_enabled and not node.revision_pinned:
# Do not search for packages for the specific resolved recipe revision but all
package_ref = package_ref.copy_clear_rev()
remote_info = None
if remote:
remote_info = self._get_package_info(package_ref, remote)
# If the "remote" came from the registry but the user didn't specified the -r, with
# revisions iterate all remotes
if not remote or (not remote_info and revisions_enabled
and not remote_name):
for r in remotes:
remote_info = self._get_package_info(package_ref, r)
if remote_info:
remote = r
break
if remote_info:
node.binary = BINARY_DOWNLOAD
package_hash = remote_info.recipe_hash
else:
if build_mode.allowed(conanfile, conan_ref):
node.binary = BINARY_BUILD
else:
node.binary = BINARY_MISSING
if build_mode.outdated:
if node.binary in (BINARY_CACHE, BINARY_DOWNLOAD, BINARY_UPDATE):
local_recipe_hash = self._client_cache.load_manifest(
package_ref.conan).summary_hash
if local_recipe_hash != package_hash:
output.info("Outdated package!")
node.binary = BINARY_BUILD
else:
output.info("Package is up to date")
node.binary_remote = remote
def cmd_export(app,
conanfile_path,
name,
version,
user,
channel,
keep_source,
export=True,
graph_lock=None):
""" Export the recipe
param conanfile_path: the original source directory of the user containing a
conanfile.py
"""
loader, cache, hook_manager, output = app.loader, app.cache, app.hook_manager, app.out
revisions_enabled = app.config.revisions_enabled
conanfile = loader.load_export(conanfile_path, name, version, user,
channel)
# FIXME: Conan 2.0, deprecate CONAN_USER AND CONAN_CHANNEL and remove this try excepts
# Take the default from the env vars if they exist to not break behavior
try:
user = conanfile.user
except ConanException:
user = None
try:
channel = conanfile.channel
except ConanException:
channel = None
ref = ConanFileReference(conanfile.name, conanfile.version, user, channel)
# If we receive lock information, python_requires could have been locked
if graph_lock:
node_id = graph_lock.get_node(ref)
python_requires = graph_lock.python_requires(node_id)
# TODO: check that the locked python_requires are different from the loaded ones
# FIXME: private access, will be improved when api collaborators are improved
loader._python_requires._range_resolver.output # invalidate previous version range output
conanfile = loader.load_export(conanfile_path, conanfile.name,
conanfile.version, conanfile.user,
conanfile.channel, python_requires)
check_casing_conflict(cache=cache, ref=ref)
package_layout = cache.package_layout(ref,
short_paths=conanfile.short_paths)
if not export:
metadata = package_layout.load_metadata()
recipe_revision = metadata.recipe.revision
ref = ref.copy_with_rev(recipe_revision)
if graph_lock:
graph_lock.update_exported_ref(node_id, ref)
return ref
hook_manager.execute("pre_export",
conanfile=conanfile,
conanfile_path=conanfile_path,
reference=package_layout.ref)
logger.debug("EXPORT: %s" % conanfile_path)
output.highlight("Exporting package recipe")
conan_linter(conanfile_path, output)
output = conanfile.output
# Get previous digest
try:
previous_manifest = FileTreeManifest.load(package_layout.export())
except IOError:
previous_manifest = None
finally:
_recreate_folders(package_layout.export(),
package_layout.export_sources())
# Copy sources to target folders
with package_layout.conanfile_write_lock(output=output):
origin_folder = os.path.dirname(conanfile_path)
export_recipe(conanfile, origin_folder, package_layout.export())
export_source(conanfile, origin_folder,
package_layout.export_sources())
shutil.copy2(conanfile_path, package_layout.conanfile())
_capture_export_scm_data(conanfile,
os.path.dirname(conanfile_path),
package_layout.export(),
output,
scm_src_file=package_layout.scm_folder())
# Execute post-export hook before computing the digest
hook_manager.execute("post_export",
conanfile=conanfile,
reference=package_layout.ref,
conanfile_path=package_layout.conanfile())
# Compute the new digest
manifest = FileTreeManifest.create(package_layout.export(),
package_layout.export_sources())
modified_recipe = not previous_manifest or previous_manifest != manifest
if modified_recipe:
output.success('A new %s version was exported' % CONANFILE)
output.info('Folder: %s' % package_layout.export())
else:
output.info("The stored package has not changed")
manifest = previous_manifest # Use the old one, keep old timestamp
manifest.save(package_layout.export())
# Compute the revision for the recipe
revision = _update_revision_in_metadata(
package_layout=package_layout,
revisions_enabled=revisions_enabled,
output=output,
path=os.path.dirname(conanfile_path),
manifest=manifest,
revision_mode=conanfile.revision_mode)
# FIXME: Conan 2.0 Clear the registry entry if the recipe has changed
source_folder = package_layout.source()
if os.path.exists(source_folder):
try:
if is_dirty(source_folder):
output.info("Source folder is corrupted, forcing removal")
rmdir(source_folder)
elif modified_recipe and not keep_source:
output.info(
"Package recipe modified in export, forcing source folder removal"
)
output.info("Use the --keep-source, -k option to skip it")
rmdir(source_folder)
except BaseException as e:
output.error(
"Unable to delete source folder. Will be marked as corrupted for deletion"
)
output.warn(str(e))
set_dirty(source_folder)
# When revisions enabled, remove the packages not matching the revision
if revisions_enabled:
packages = search_packages(package_layout, query=None)
metadata = package_layout.load_metadata()
recipe_revision = metadata.recipe.revision
to_remove = [
pid for pid in packages if metadata.packages.get(pid)
and metadata.packages.get(pid).recipe_revision != recipe_revision
]
if to_remove:
output.info(
"Removing the local binary packages from different recipe revisions"
)
remover = DiskRemover()
remover.remove_packages(package_layout, ids_filter=to_remove)
ref = ref.copy_with_rev(revision)
output.info("Exported revision: %s" % revision)
if graph_lock:
graph_lock.update_exported_ref(node_id, ref)
return ref