def child_build_test(self):
client = TestClient()
repo_folder = client.current_folder
build_folder = os.path.join(client.current_folder, "build")
mkdir(build_folder)
package_folder = os.path.join(build_folder, "package")
mkdir(package_folder)
client.save({"conanfile.py": conanfile_out})
client.current_folder = build_folder
client.run("install ..")
client.run("source ..")
client.run("build .. --source-folder=.")
client.current_folder = package_folder
client.run("package ../.. --build-folder=../")
self._assert_pkg(package_folder)
rmdir(package_folder)
client.current_folder = repo_folder
client.run("export-pkg . Pkg/0.1@lasote/testing -bf=./build")
ref = ConanFileReference.loads("Pkg/0.1@lasote/testing")
cache_package_folder = client.client_cache.packages(ref)
cache_package_folder = os.path.join(cache_package_folder,
os.listdir(cache_package_folder)[0])
self._assert_pkg(cache_package_folder)
def install_outdated_and_dep_test(self):
# regression test for https://github.com/conan-io/conan/issues/1053
# A new recipe that depends on Hello0/0.1
new_client = TestClient(servers=self.servers,
users={"default": [("lasote", "mypass")]})
files = cpp_hello_conan_files("Hello1", "0.1", ["Hello0/0.1@lasote/stable"], build=False)
new_client.save(files)
new_client.run("export . lasote/stable")
self.assertIn("A new conanfile.py version was exported", new_client.user_io.out)
# It will retrieve from the remote Hello0 and build Hello1
new_client.run("install Hello1/0.1@lasote/stable --build missing")
# Then modify REMOTE Hello0 recipe files (WITH THE OTHER CLIENT)
files = cpp_hello_conan_files("Hello0", "0.1", build=False)
files["conanfile.py"] = files["conanfile.py"] + "\n#MODIFIED RECIPE"
self.client.save(files)
self.client.run("export . lasote/stable")
self.assertIn("A new conanfile.py version was exported", self.client.user_io.out)
self.client.run("install Hello0/0.1@lasote/stable --build missing")
# Upload only the recipe, so the package is outdated in the server
self.client.run("upload Hello0/0.1@lasote/stable")
# Now, with the new_client, remove only the binary package from Hello0
rmdir(new_client.paths.packages(self.ref))
# And try to install Hello1 again, should not complain because the remote
# binary is in the "same version" than local cached Hello0
new_client.run("install Hello1/0.1@lasote/stable --build outdated --build Hello1")
self.assertIn("Downloading conan_package.tgz", new_client.user_io.out)
self.assertIn("Hello1/0.1@lasote/stable: WARN: Forced build from source",
new_client.user_io.out)
def get_recipe(self, conan_reference, remote):
"""
Read the conans from remotes
Will iterate the remotes to find the conans unless remote was specified
returns (dict relative_filepath:abs_path , remote_name)"""
dest_folder = self._client_cache.export(conan_reference)
rmdir(dest_folder)
def filter_function(urls):
if CONANFILE not in list(urls.keys()):
raise NotFoundException("Conan '%s' doesn't have a %s!"
% (conan_reference, CONANFILE))
urls.pop(EXPORT_SOURCES_TGZ_NAME, None)
return urls
t1 = time.time()
urls = self._call_remote(remote, "get_recipe_urls", conan_reference)
urls = filter_function(urls)
if not urls:
return conan_reference
zipped_files = self._call_remote(remote, "download_files_to_folder", urls, dest_folder)
duration = time.time() - t1
log_recipe_download(conan_reference, duration, remote, zipped_files)
unzip_and_get_files(zipped_files, dest_folder, EXPORT_TGZ_NAME)
# Make sure that the source dir is deleted
rm_conandir(self._client_cache.source(conan_reference))
touch_folder(dest_folder)
def create_package(conanfile, build_folder, package_folder, output):
""" copies built artifacts, libs, headers, data, etc from build_folder to
package folder
"""
mkdir(package_folder)
# Make the copy of all the patterns
output.info("Generating the package")
output.info("Package folder %s" % (package_folder))
conanfile.copy = FileCopier(build_folder, package_folder)
def wrap(dst_folder):
def new_method(pattern, src=""):
conanfile.copy(pattern, dst_folder, src)
return new_method
conanfile.copy_headers = wrap(DEFAULT_INCLUDE)
conanfile.copy_libs = wrap(DEFAULT_LIB)
conanfile.copy_bins = wrap(DEFAULT_BIN)
conanfile.copy_res = wrap(DEFAULT_RES)
try:
conanfile.package_folder = package_folder
conanfile.package()
except Exception as e:
os.chdir(build_folder)
try:
rmdir(package_folder)
except Exception as e_rm:
output.error("Unable to remove package folder %s\n%s"
% (package_folder, str(e_rm)))
output.warn("**** Please delete it manually ****")
raise ConanException("%s: %s" % (conanfile.name, str(e)))
_create_aux_files(build_folder, package_folder)
output.success("Package '%s' created" % os.path.basename(package_folder))
def get_package(self, package_ref, short_paths):
""" obtain a package, either from disk or retrieve from remotes if necessary
and not necessary to build
"""
output = ScopedOutput(str(package_ref.conan), self._out)
package_folder = self._client_cache.package(package_ref, short_paths=short_paths)
# Check current package status
if os.path.exists(package_folder):
if self._check_updates:
read_manifest = self._client_cache.load_package_manifest(package_ref)
try: # get_conan_digest can fail, not in server
upstream_manifest = self.get_package_digest(package_ref)
if upstream_manifest != read_manifest:
if upstream_manifest.time > read_manifest.time:
output.warn("Current package is older than remote upstream one")
if self._update:
output.warn("Removing it to retrieve or build an updated one")
rmdir(package_folder)
else:
output.warn("Current package is newer than remote upstream one")
except ConanException:
pass
installed = False
local_package = os.path.exists(package_folder)
if local_package:
output.info('Already installed!')
installed = True
log_package_got_from_local_cache(package_ref)
else:
installed = self._retrieve_remote_package(package_ref, package_folder,
output)
self.handle_package_manifest(package_ref, installed)
return installed
def get_recipe(self, conan_reference, dest_folder, remote):
"""
Read the conans from remotes
Will iterate the remotes to find the conans unless remote was specified
returns (dict relative_filepath:abs_path , remote_name)"""
rmdir(dest_folder) # Remove first the destination folder
t1 = time.time()
def filter_function(urls):
if CONANFILE not in list(urls.keys()):
raise NotFoundException("Conan '%s' doesn't have a %s!"
% (conan_reference, CONANFILE))
urls.pop(EXPORT_SOURCES_TGZ_NAME, None)
return urls
zipped_files = self._call_remote(remote, "get_recipe", conan_reference, dest_folder,
filter_function)
duration = time.time() - t1
log_recipe_download(conan_reference, duration, remote, zipped_files)
unzip_and_get_files(zipped_files, dest_folder, EXPORT_TGZ_NAME)
# Make sure that the source dir is deleted
rm_conandir(self._client_cache.source(conan_reference))
for dirname, _, filenames in os.walk(dest_folder):
for fname in filenames:
touch(os.path.join(dirname, fname))
def configuration_install(item, client_cache, output, verify_ssl, requester,
config_type=None, args=None):
tmp_folder = os.path.join(client_cache.conan_folder, "tmp_config_install")
# necessary for Mac OSX, where the temp folders in /var/ are symlinks to /private/var/
tmp_folder = os.path.realpath(tmp_folder)
mkdir(tmp_folder)
try:
if item is None:
try:
item = client_cache.conan_config.get_item("general.config_install")
except ConanException:
raise ConanException("Called config install without arguments and "
"'general.config_install' not defined in conan.conf")
if item.endswith(".git") or config_type == "git":
_process_git_repo(item, client_cache, output, tmp_folder, verify_ssl, args)
elif os.path.isdir(item):
_process_folder(item, client_cache, output)
elif os.path.isfile(item):
_process_zip_file(item, client_cache, output, tmp_folder)
elif item.startswith("http"):
_process_download(item, client_cache, output, tmp_folder, verify_ssl,
requester=requester)
else:
raise ConanException("I don't know how to process %s" % item)
finally:
if item:
client_cache.conan_config.set_item("general.config_install", item)
rmdir(tmp_folder)
def package(self, reference, package_id):
# Package paths
conan_file_path = self._client_cache.conanfile(reference)
if not os.path.exists(conan_file_path):
raise ConanException("Package recipe '%s' does not exist" % str(reference))
if not package_id:
packages = [PackageReference(reference, packid)
for packid in self._client_cache.conan_builds(reference)]
if not packages:
raise NotFoundException("%s: Package recipe has not been built locally\n"
"Please read the 'conan package' command help\n"
"Use 'conan install' or 'conan test_package' to build and "
"create binaries" % str(reference))
else:
packages = [PackageReference(reference, package_id)]
for package_reference in packages:
build_folder = self._client_cache.build(package_reference, short_paths=None)
if not os.path.exists(build_folder):
raise NotFoundException("%s: Package binary '%s' folder doesn't exist\n"
"Please read the 'conan package' command help\n"
"Use 'conan install' or 'conan test_package' to build and "
"create binaries"
% (str(reference), package_reference.package_id))
# The package already exist, we can use short_paths if they were defined
package_folder = self._client_cache.package(package_reference, short_paths=None)
# Will read current conaninfo with specified options and load conanfile with them
output = ScopedOutput(str(reference), self._user_io.out)
output.info("Re-packaging %s" % package_reference.package_id)
loader = self._loader(build_folder)
conanfile = loader.load_conan(conan_file_path, self._user_io.out)
self._load_deps_info(build_folder, conanfile, output)
rmdir(package_folder)
packager.create_package(conanfile, build_folder, package_folder, output)
def package(self, reference, package_id, only_manifest, package_all):
assert(isinstance(reference, ConanFileReference))
# Package paths
conan_file_path = self._paths.conanfile(reference)
if package_all:
if only_manifest:
packages_dir = self._paths.packages(reference)
else:
packages_dir = self._paths.builds(reference)
if not os.path.exists(packages_dir):
raise NotFoundException('%s does not exist' % str(reference))
packages = [PackageReference(reference, packid)
for packid in os.listdir(packages_dir)]
else:
packages = [PackageReference(reference, package_id)]
for package_reference in packages:
package_folder = self._paths.package(package_reference)
# Will read current conaninfo with specified options and load conanfile with them
if not only_manifest:
self._user_io.out.info("Packaging %s" % package_reference.package_id)
build_folder = self._paths.build(package_reference)
loader = self._loader(build_folder)
conanfile = loader.load_conan(conan_file_path, self._user_io.out)
rmdir(package_folder)
output = ScopedOutput(str(reference), self._user_io.out)
packager.create_package(conanfile, build_folder, package_folder, output)
else:
self._user_io.out.info("Creating manifest for %s" % package_reference.package_id)
if not os.path.exists(package_folder):
raise NotFoundException('Package %s does not exist' % str(package_reference))
packager.generate_manifest(package_folder)
def _recreate_folders(destination_folder):
try:
if os.path.exists(destination_folder):
rmdir(destination_folder)
os.makedirs(destination_folder)
except Exception as e:
raise ConanException("Unable to create folder %s\n%s" % (destination_folder, str(e)))
def get_recipe(self, ref, remote):
"""
Read the conans from remotes
Will iterate the remotes to find the conans unless remote was specified
returns (dict relative_filepath:abs_path , remote_name)"""
self._hook_manager.execute("pre_download_recipe", reference=ref, remote=remote)
dest_folder = self._cache.package_layout(ref).export()
rmdir(dest_folder)
ref = self._resolve_latest_ref(ref, remote)
t1 = time.time()
zipped_files = self._call_remote(remote, "get_recipe", ref, dest_folder)
duration = time.time() - t1
log_recipe_download(ref, duration, remote.name, zipped_files)
recipe_checksums = calc_files_checksum(zipped_files)
unzip_and_get_files(zipped_files, dest_folder, EXPORT_TGZ_NAME, output=self._output)
# Make sure that the source dir is deleted
package_layout = self._cache.package_layout(ref)
rm_conandir(package_layout.source())
touch_folder(dest_folder)
conanfile_path = package_layout.conanfile()
with package_layout.update_metadata() as metadata:
metadata.recipe.revision = ref.revision
metadata.recipe.checksums = recipe_checksums
self._hook_manager.execute("post_download_recipe", conanfile_path=conanfile_path,
reference=ref, remote=remote)
return ref
def install_build_and_test(self, conanfile_abs_path, reference, profile,
remote, update, build_modes=None, manifest_folder=None,
manifest_verify=False, manifest_interactive=False, keep_build=False,
test_build_folder=None):
"""
Installs the reference (specified by the parameters or extracted from the test conanfile)
and builds the test_package/conanfile.py running the test() method.
"""
base_folder = os.path.dirname(conanfile_abs_path)
test_build_folder, delete_after_build = self._build_folder(test_build_folder, profile, base_folder)
rmdir(test_build_folder)
if build_modes is None:
build_modes = ["never"]
try:
self._manager.install(inject_require=reference,
reference=conanfile_abs_path,
install_folder=test_build_folder,
remote_name=remote,
profile=profile,
update=update,
build_modes=build_modes,
manifest_folder=manifest_folder,
manifest_verify=manifest_verify,
manifest_interactive=manifest_interactive,
keep_build=keep_build)
self._manager.build(conanfile_abs_path, base_folder, test_build_folder, package_folder=None,
install_folder=test_build_folder, test=str(reference))
finally:
if delete_after_build:
os.chdir(base_folder) # Required for windows where deleting the cwd is not possible.
rmdir(test_build_folder)
def install_outdated_dep_test(self):
# A new recipe that depends on Hello0/0.1
new_client = TestClient(servers=self.servers,
users={"default": [("lasote", "mypass")]})
files = cpp_hello_conan_files("Hello1", "0.1", ["Hello0/0.1@lasote/stable"], build=False)
new_client.save(files)
new_client.run("export lasote/stable")
self.assertIn("A new conanfile.py version was exported", new_client.user_io.out)
# It will retrieve from the remote Hello0 and build Hello1
new_client.run("install Hello1/0.1@lasote/stable --build missing")
# Then modify REMOTE Hello0 recipe files (WITH THE OTHER CLIENT)
files = cpp_hello_conan_files("Hello0", "0.1", build=False)
files["conanfile.py"] = files["conanfile.py"] + "\n#MODIFIED RECIPE"
self.client.save(files)
self.client.run("export lasote/stable")
self.assertIn("A new conanfile.py version was exported", self.client.user_io.out)
self.client.run("install Hello0/0.1@lasote/stable --build missing")
# Upload only the recipe, so the package is outdated in the server
self.client.run("upload Hello0/0.1@lasote/stable")
# Now, with the new_client, remove only the binary package from Hello0
rmdir(new_client.paths.packages(self.ref))
# And try to install Hello1 again, should not complain because the remote
# binary is in the "same version" than local cached Hello0
new_client.run("install Hello1/0.1@lasote/stable --build outdated")
self.assertIn("Downloading conan_package.tgz", new_client.user_io.out)
self.assertIn("Hello0/0.1@lasote/stable: Package is up to date", new_client.user_io.out)
# But if we remove the full Hello0 local package, will retrieve the updated
# recipe and the outdated package
new_client.run("remove Hello0* -f")
new_client.run("install Hello1/0.1@lasote/stable --build outdated")
self.assertIn("Hello0/0.1@lasote/stable: Outdated package!", new_client.user_io.out)
self.assertIn("Hello0/0.1@lasote/stable: Building your package", new_client.user_io.out)
def test_deleted_source_folder(self):
path, _ = create_local_git_repo({"myfile": "contents"},
branch="my_release")
conanfile = base_git.format(url=_quoted("auto"), revision="auto")
self.client.save({
"conanfile.py": conanfile,
"myfile.txt": "My file is copied"
})
create_local_git_repo(folder=self.client.current_folder)
self.client.run_command('git remote add origin "%s"' %
path.replace("\\", "/"))
self.client.run_command('git push origin master')
self.client.run("export . user/channel")
# delete old source, but it doesn't matter because the sources are in the cache
rmdir(self.client.current_folder)
new_curdir = temp_folder()
self.client.current_folder = new_curdir
self.client.run("install lib/0.1@user/channel --build")
self.assertNotIn(
"Getting sources from url: '%s'" % path.replace("\\", "/"),
self.client.out)
# If the remove the source folder, then it is fetched from the "remote" doing an install
self.client.run("remove lib/0.1@user/channel -f -s")
self.client.run("install lib/0.1@user/channel --build")
self.assertIn(
"SCM: Getting sources from url: '%s'" % path.replace("\\", "/"),
self.client.out)
def get_recipe_sources(self, ref, export_folder, export_sources_folder,
remote):
assert ref.revision, "get_recipe_sources requires RREV"
t1 = time.time()
zipped_files = self._call_remote(remote, "get_recipe_sources", ref,
export_folder)
if not zipped_files:
mkdir(export_sources_folder
) # create the folder even if no source files
return
duration = time.time() - t1
log_recipe_sources_download(ref, duration, remote.name, zipped_files)
unzip_and_get_files(zipped_files,
export_sources_folder,
EXPORT_SOURCES_TGZ_NAME,
output=self._output)
# REMOVE in Conan 2.0
c_src_path = os.path.join(export_sources_folder,
EXPORT_SOURCES_DIR_OLD)
if os.path.exists(c_src_path):
merge_directories(c_src_path, export_sources_folder)
rmdir(c_src_path)
touch_folder(export_sources_folder)
def copy(self, reference, package_ids, username, channel, force=False):
assert(isinstance(reference, ConanFileReference))
dest_ref = "%s/%s@%s/%s" % (reference.name, reference.version, username, channel)
dest_ref = ConanFileReference.loads(dest_ref)
# Copy export
export_origin = self._paths.export(reference)
if not os.path.exists(export_origin):
raise ConanException("'%s' doesn't exist" % str(reference))
export_dest = self._paths.export(dest_ref)
if os.path.exists(export_dest):
if not force and not self._user_io.request_boolean("'%s' already exist. Override?"
% str(dest_ref)):
return
rmdir(export_dest)
shutil.copytree(export_origin, export_dest)
self._user_io.out.info("Copied %s to %s" % (str(reference), str(dest_ref)))
# Copy packages
for package_id in package_ids:
package_origin = PackageReference(reference, package_id)
package_dest = PackageReference(dest_ref, package_id)
package_path_origin = self._paths.package(package_origin)
package_path_dest = self._paths.package(package_dest)
if os.path.exists(package_path_dest):
if not force and not self._user_io.request_boolean("Package '%s' already exist."
" Override?"
% str(package_id)):
continue
rmdir(package_path_dest)
shutil.copytree(package_path_origin, package_path_dest)
self._user_io.out.info("Copied %s to %s" % (str(package_id), str(dest_ref)))
def configuration_install(item, client_cache, output, verify_ssl):
tmp_folder = os.path.join(client_cache.conan_folder, "tmp_config_install")
# necessary for Mac OSX, where the temp folders in /var/ are symlinks to /private/var/
tmp_folder = os.path.realpath(tmp_folder)
mkdir(tmp_folder)
try:
if item is None:
try:
item = client_cache.conan_config.get_item("general.config_install")
except ConanException:
raise ConanException("Called config install without arguments and "
"'general.config_install' not defined in conan.conf")
if item.endswith(".git"):
_process_git_repo(item, client_cache, output, tmp_folder, verify_ssl)
elif os.path.exists(item):
# is a local file
_process_zip_file(item, client_cache, output, tmp_folder)
elif item.startswith("http"):
_process_download(item, client_cache, output, tmp_folder, verify_ssl)
else:
raise ConanException("I don't know how to process %s" % item)
finally:
if item:
client_cache.conan_config.set_item("general.config_install", item)
rmdir(tmp_folder)
def rm_conandir(path):
"""removal of a directory that might contain a link to a short path"""
link = os.path.join(path, CONAN_LINK)
if os.path.exists(link):
short_path = load(link)
rmdir(os.path.dirname(short_path))
rmdir(path)
def child_build_test(self):
client = TestClient()
build_folder = os.path.join(client.current_folder, "build")
mkdir(build_folder)
package_folder = os.path.join(build_folder, "package")
mkdir(package_folder)
client.save({"conanfile.py": conanfile,
"file.h": "file_h_contents!"})
client.current_folder = build_folder
client.run("install ..")
client.run("build ..")
client.current_folder = package_folder
client.run("package ../.. --build-folder=../")
self._assert_pkg(package_folder)
rmdir(package_folder) # IMPORTANT: Symptom that package + package_folder is not fitting
# well now. (To discuss)
# But I think now you choose you way to develop, local or cache, if you use conan export-pkg
# you are done, if you use package() you need the "conan project" feature
client.current_folder = build_folder
client.run("export-pkg .. Pkg/0.1@lasote/testing --source-folder=.. ")
ref = ConanFileReference.loads("Pkg/0.1@lasote/testing")
cache_package_folder = client.client_cache.packages(ref)
cache_package_folder = os.path.join(cache_package_folder,
os.listdir(cache_package_folder)[0])
self._assert_pkg(cache_package_folder)
def install_outdated_and_dep_test(self):
# regression test for https://github.com/conan-io/conan/issues/1053
# A new recipe that depends on Hello0/0.1
new_client = TestClient(servers=self.servers,
users={"default": [("lasote", "mypass")]})
files = cpp_hello_conan_files("Hello1", "0.1", ["Hello0/0.1@lasote/stable"], build=False)
new_client.save(files)
new_client.run("export lasote/stable")
self.assertIn("A new conanfile.py version was exported", new_client.user_io.out)
# It will retrieve from the remote Hello0 and build Hello1
new_client.run("install Hello1/0.1@lasote/stable --build missing")
# Then modify REMOTE Hello0 recipe files (WITH THE OTHER CLIENT)
files = cpp_hello_conan_files("Hello0", "0.1", build=False)
files["conanfile.py"] = files["conanfile.py"] + "\n#MODIFIED RECIPE"
self.client.save(files)
self.client.run("export lasote/stable")
self.assertIn("A new conanfile.py version was exported", self.client.user_io.out)
self.client.run("install Hello0/0.1@lasote/stable --build missing")
# Upload only the recipe, so the package is outdated in the server
self.client.run("upload Hello0/0.1@lasote/stable")
# Now, with the new_client, remove only the binary package from Hello0
rmdir(new_client.paths.packages(self.ref))
# And try to install Hello1 again, should not complain because the remote
# binary is in the "same version" than local cached Hello0
new_client.run("install Hello1/0.1@lasote/stable --build outdated --build Hello1")
self.assertIn("Downloading conan_package.tgz", new_client.user_io.out)
self.assertIn("Hello1/0.1@lasote/stable: WARN: Forced build from source",
new_client.user_io.out)
def _remove_folder_raising(folder):
try:
rmdir(folder)
except OSError as e:
raise ConanException(
"%s\n\nCouldn't remove folder, might be busy or open\n"
"Close any app using it, and retry" % str(e))
def install_outdated_dep_test(self):
# A new recipe that depends on Hello0/0.1
new_client = TestClient(servers=self.servers,
users={"default": [("lasote", "mypass")]})
files = cpp_hello_conan_files("Hello1", "0.1", ["Hello0/0.1@lasote/stable"], build=False)
new_client.save(files)
new_client.run("export . lasote/stable")
self.assertIn("A new conanfile.py version was exported", new_client.user_io.out)
# It will retrieve from the remote Hello0 and build Hello1
new_client.run("install Hello1/0.1@lasote/stable --build missing")
# Then modify REMOTE Hello0 recipe files (WITH THE OTHER CLIENT)
files = cpp_hello_conan_files("Hello0", "0.1", build=False)
files["conanfile.py"] = files["conanfile.py"] + "\n#MODIFIED RECIPE"
self.client.save(files)
self.client.run("export . lasote/stable")
self.assertIn("A new conanfile.py version was exported", self.client.user_io.out)
self.client.run("install Hello0/0.1@lasote/stable --build missing")
# Upload only the recipe, so the package is outdated in the server
self.client.run("upload Hello0/0.1@lasote/stable")
# Now, with the new_client, remove only the binary package from Hello0
rmdir(new_client.paths.packages(self.ref))
# And try to install Hello1 again, should not complain because the remote
# binary is in the "same version" than local cached Hello0
new_client.run("install Hello1/0.1@lasote/stable --build outdated")
self.assertIn("Downloading conan_package.tgz", new_client.user_io.out)
self.assertIn("Hello0/0.1@lasote/stable: Package is up to date", new_client.user_io.out)
# But if we remove the full Hello0 local package, will retrieve the updated
# recipe and the outdated package
new_client.run("remove Hello0* -f")
new_client.run("install Hello1/0.1@lasote/stable --build outdated")
self.assertIn("Hello0/0.1@lasote/stable: Outdated package!", new_client.user_io.out)
self.assertIn("Hello0/0.1@lasote/stable: Building your package", new_client.user_io.out)
def _get_build_folder(self, conanfile, package_layout, pref, keep_build,
recorder):
# Build folder can use a different package_ID if build_id() is defined.
# This function decides if the build folder should be re-used (not build again)
# and returns the build folder
new_id = build_id(conanfile)
build_pref = PackageReference(pref.ref, new_id) if new_id else pref
build_folder = package_layout.build(build_pref)
if is_dirty(build_folder):
self._output.warn("Build folder is dirty, removing it: %s" %
build_folder)
rmdir(build_folder)
# Decide if the build folder should be kept
skip_build = conanfile.develop and keep_build
if skip_build:
self._output.info("Won't be built as specified by --keep-build")
if not os.path.exists(build_folder):
msg = "--keep-build specified, but build folder not found"
recorder.package_install_error(
pref,
INSTALL_ERROR_MISSING_BUILD_FOLDER,
msg,
remote_name=None)
raise ConanException(msg)
elif build_pref != pref and os.path.exists(build_folder) and hasattr(
conanfile, "build_id"):
self._output.info(
"Won't be built, using previous build folder as defined in build_id()"
)
skip_build = True
return build_folder, skip_build
def get_recipe_sources(self, conan_reference, export_folder, export_sources_folder, remote):
t1 = time.time()
def filter_function(urls):
file_url = urls.get(EXPORT_SOURCES_TGZ_NAME)
if file_url:
urls = {EXPORT_SOURCES_TGZ_NAME: file_url}
else:
return None
return urls
urls = self._call_remote(remote, "get_recipe_urls", conan_reference)
urls = filter_function(urls)
if not urls:
return conan_reference
zipped_files = self._call_remote(remote, "download_files_to_folder", urls, export_folder)
duration = time.time() - t1
log_recipe_sources_download(conan_reference, duration, remote, zipped_files)
if not zipped_files:
mkdir(export_sources_folder) # create the folder even if no source files
return
unzip_and_get_files(zipped_files, export_sources_folder, EXPORT_SOURCES_TGZ_NAME)
c_src_path = os.path.join(export_sources_folder, EXPORT_SOURCES_DIR_OLD)
if os.path.exists(c_src_path):
merge_directories(c_src_path, export_sources_folder)
rmdir(c_src_path)
touch_folder(export_sources_folder)
def _clean_package(package_folder, output):
try:
output.warn("Trying to remove package folder: %s" % package_folder)
rmdir(package_folder)
except OSError as e:
raise ConanException("%s\n\nCouldn't remove folder '%s', might be busy or open. Close any app "
"using it, and retry" % (str(e), package_folder))
def test_package(self, *args):
""" build and run your package test. Must have conanfile.py with "test"
method and "test_package" subfolder with package consumer test project
"""
parser = argparse.ArgumentParser(description=self.test_package.__doc__, prog="conan test",
formatter_class=RawTextHelpFormatter)
parser.add_argument("path", nargs='?', default="", help='path to conanfile file, '
'e.g. /my_project/')
parser.add_argument("-f", "--folder", help='alternative test folder name')
self._parse_args(parser)
args = parser.parse_args(*args)
root_folder = os.path.normpath(os.path.join(os.getcwd(), args.path))
if args.folder:
test_folder_name = args.folder
test_folder = os.path.join(root_folder, test_folder_name)
test_conanfile = os.path.join(test_folder, "conanfile.py")
if not os.path.exists(test_conanfile):
raise ConanException("test folder '%s' not available, "
"or it doesn't have a conanfile.py" % args.folder)
else:
for name in ["test_package", "test"]:
test_folder_name = name
test_folder = os.path.join(root_folder, test_folder_name)
test_conanfile = os.path.join(test_folder, "conanfile.py")
if os.path.exists(test_conanfile):
break
else:
raise ConanException("test folder 'test_package' not available, "
"or it doesn't have a conanfile.py")
lib_to_test = self._detect_tested_library_name()
# Get False or a list of patterns to check
if args.build is None and lib_to_test: # Not specified, force build the tested library
args.build = [lib_to_test]
else:
args.build = self._get_build_sources_parameter(args.build)
options = args.options or []
settings = args.settings or []
sha = hashlib.sha1("".join(options + settings).encode()).hexdigest()
build_folder = os.path.join(test_folder, "build", sha)
rmdir(build_folder)
# shutil.copytree(test_folder, build_folder)
options = self._get_tuples_list_from_extender_arg(args.options)
settings = self._get_tuples_list_from_extender_arg(args.settings)
self._manager.install(reference=test_folder,
current_path=build_folder,
remote=args.remote,
options=options,
settings=settings,
build_mode=args.build)
self._test_check(test_folder, test_folder_name)
self._manager.build(test_folder, build_folder, test=True)
def _build_node(self, conan_ref, conan_file, build_mode):
# Compute conan_file package from local (already compiled) or from remote
output = ScopedOutput(str(conan_ref), self._out)
package_id = conan_file.info.package_id()
package_reference = PackageReference(conan_ref, package_id)
conan_ref = package_reference.conan
package_folder = self._paths.package(package_reference)
build_folder = self._paths.build(package_reference,
conan_file.short_paths)
src_folder = self._paths.source(conan_ref, conan_file.short_paths)
export_folder = self._paths.export(conan_ref)
# If already exists do not dirt the output, the common situation
# is that package is already installed and OK. If don't, the proxy
# will print some other message about it
if not os.path.exists(package_folder):
output.info("Installing package %s" % package_id)
self._handle_system_requirements(conan_ref, package_reference,
conan_file, output)
force_build = self._build_forced(conan_ref, build_mode, conan_file)
if self._remote_proxy.get_package(package_reference, force_build):
return
# we need and can build? Only if we are forced or build_mode missing and package not exists
build = force_build or build_mode is True or conan_file.build_policy_missing
if build:
if not force_build and not build_mode:
output.info(
"Building package from source as defined by build_policy='missing'"
)
try:
rmdir(build_folder, conan_file.short_paths)
rmdir(package_folder)
except Exception as e:
raise ConanException(
"%s\n\nCouldn't remove folder, might be busy or open\n"
"Close any app using it, and retry" % str(e))
if force_build:
output.warn('Forced build from source')
self._build_package(export_folder, src_folder, build_folder,
package_folder, conan_file, output)
# Creating ***info.txt files
save(os.path.join(build_folder, CONANINFO),
conan_file.info.dumps())
output.info("Generated %s" % CONANINFO)
save(os.path.join(build_folder, BUILD_INFO),
TXTGenerator(conan_file).content)
output.info("Generated %s" % BUILD_INFO)
os.chdir(build_folder)
create_package(conan_file, build_folder, package_folder, output)
else:
self._raise_package_not_found_error(conan_ref, conan_file)
def get_conanfile(self, conan_reference):
output = ScopedOutput(str(conan_reference), self._out)
def _refresh():
conan_dir_path = self._paths.export(conan_reference)
rmdir(conan_dir_path)
rmdir(self._paths.source(conan_reference), True) # It might need to remove shortpath
current_remote, _ = self._get_remote(conan_reference)
output.info("Retrieving from remote '%s'..." % current_remote.name)
self._remote_manager.get_conanfile(conan_reference, current_remote)
if self._update:
output.info("Updated!")
else:
output.info("Installed!")
# check if it is in disk
conanfile_path = self._paths.conanfile(conan_reference)
path_exist = path_exists(conanfile_path, self._paths.store)
if path_exist:
if self._check_integrity: # Check if package is corrupted
read_manifest, expected_manifest = self._paths.conan_manifests(conan_reference)
if read_manifest.file_sums != expected_manifest.file_sums:
output.warn("Bad conanfile detected! Removing export directory... ")
_refresh()
else: # Check for updates
if self._check_updates:
ret = self.update_available(conan_reference)
if ret != 0: # Found and not equal
remote, ref_remote = self._get_remote(conan_reference)
if ret == 1:
if not self._update:
if remote != ref_remote: # Forced new remote
output.warn("There is a new conanfile in '%s' remote. "
"Execute 'install -u -r %s' to update it."
% (remote.name, remote.name))
else:
output.warn("There is a new conanfile in '%s' remote. "
"Execute 'install -u' to update it."
% remote.name)
output.warn("Refused to install!")
else:
if remote != ref_remote:
# Delete packages, could be non coherent with new remote
rmdir(self._paths.packages(conan_reference))
_refresh()
elif ret == -1:
if not self._update:
output.info("Current conanfile is newer "
"than %s's one" % remote.name)
else:
output.error("Current conanfile is newer than %s's one. "
"Run 'conan remove %s' and run install again "
"to replace it." % (remote.name, conan_reference))
else:
self._retrieve_conanfile(conan_reference, output)
return conanfile_path
def get_package(self, pref, dest_folder, remote, output, recorder):
conanfile_path = self._cache.package_layout(pref.ref).conanfile()
self._hook_manager.execute("pre_download_package",
conanfile_path=conanfile_path,
reference=pref.ref,
package_id=pref.id,
remote=remote)
output.info("Retrieving package %s from remote '%s' " %
(pref.id, remote.name))
rm_conandir(dest_folder) # Remove first the destination folder
t1 = time.time()
try:
pref = self._resolve_latest_pref(pref, remote)
snapshot = self._call_remote(remote, "get_package_snapshot", pref)
if not is_package_snapshot_complete(snapshot):
raise PackageNotFoundException(pref)
zipped_files = self._call_remote(remote, "get_package", pref,
dest_folder)
with self._cache.package_layout(
pref.ref).update_metadata() as metadata:
metadata.packages[pref.id].revision = pref.revision
metadata.packages[pref.id].recipe_revision = pref.ref.revision
duration = time.time() - t1
log_package_download(pref, duration, remote, zipped_files)
unzip_and_get_files(zipped_files,
dest_folder,
PACKAGE_TGZ_NAME,
output=self._output)
# Issue #214 https://github.com/conan-io/conan/issues/214
touch_folder(dest_folder)
if get_env("CONAN_READ_ONLY_CACHE", False):
make_read_only(dest_folder)
recorder.package_downloaded(pref, remote.url)
output.success('Package installed %s' % pref.id)
except NotFoundException:
raise PackageNotFoundException(pref)
except BaseException as e:
output.error("Exception while getting package: %s" % str(pref.id))
output.error("Exception: %s %s" % (type(e), str(e)))
try:
output.warn("Trying to remove package folder: %s" %
dest_folder)
rmdir(dest_folder)
except OSError as e:
raise ConanException(
"%s\n\nCouldn't remove folder '%s', might be busy or open. "
"Close any app using it, and retry" %
(str(e), dest_folder))
raise
self._hook_manager.execute("post_download_package",
conanfile_path=conanfile_path,
reference=pref.ref,
package_id=pref.id,
remote=remote)
return pref
def create_package(conanfile,
source_folder,
build_folder,
package_folder,
install_folder,
output,
local=False,
copy_info=False):
""" copies built artifacts, libs, headers, data, etc from build_folder to
package folder
"""
mkdir(package_folder)
# Make the copy of all the patterns
output.info("Generating the package")
output.info("Package folder %s" % (package_folder))
try:
package_output = ScopedOutput("%s package()" % output.scope, output)
output.highlight("Calling package()")
conanfile.package_folder = package_folder
conanfile.source_folder = source_folder
conanfile.build_folder = build_folder
def recipe_has(conanfile, attribute):
return attribute in conanfile.__class__.__dict__
if source_folder != build_folder:
conanfile.copy = FileCopier(source_folder, package_folder,
build_folder)
with conanfile_exception_formatter(str(conanfile), "package"):
with tools.chdir(source_folder):
conanfile.package()
warn = recipe_has(conanfile, "package")
conanfile.copy.report(package_output, warn=warn)
conanfile.copy = FileCopier(build_folder, package_folder)
with tools.chdir(build_folder):
with conanfile_exception_formatter(str(conanfile), "package"):
conanfile.package()
warn = recipe_has(conanfile, "build") and recipe_has(
conanfile, "package")
conanfile.copy.report(package_output, warn=warn)
except Exception as e:
if not local:
os.chdir(build_folder)
try:
rmdir(package_folder)
except Exception as e_rm:
output.error("Unable to remove package folder %s\n%s" %
(package_folder, str(e_rm)))
output.warn("**** Please delete it manually ****")
if isinstance(e, ConanExceptionInUserConanfileMethod):
raise
raise ConanException(e)
_create_aux_files(install_folder, package_folder, conanfile, copy_info)
output.success("Package '%s' created" % os.path.basename(package_folder))
def configuration_install(path_or_url,
cache,
output,
verify_ssl,
requester,
config_type=None,
args=None):
if path_or_url is None:
try:
item = cache.conan_config.get_item("general.config_install")
_config_type, path_or_url, _verify_ssl, _args = _process_config_install_item(
item)
except ConanException:
raise ConanException(
"Called config install without arguments and "
"'general.config_install' not defined in conan.conf")
else:
_config_type, path_or_url, _verify_ssl, _args = _process_config_install_item(
path_or_url)
config_type = config_type or _config_type
verify_ssl = verify_ssl or _verify_ssl
args = args or _args
if os.path.exists(path_or_url):
path_or_url = os.path.abspath(path_or_url)
tmp_folder = os.path.join(cache.conan_folder, "tmp_config_install")
# necessary for Mac OSX, where the temp folders in /var/ are symlinks to /private/var/
tmp_folder = os.path.realpath(tmp_folder)
mkdir(tmp_folder)
try:
if config_type == "git":
_process_git_repo(path_or_url, cache, output, tmp_folder,
verify_ssl, args)
elif config_type == "dir":
args = None
_process_folder(path_or_url, cache, output)
elif config_type == "file":
args = None
_process_zip_file(path_or_url, cache, output, tmp_folder)
elif config_type == "url":
args = None
_process_download(path_or_url,
cache,
output,
tmp_folder,
verify_ssl,
requester=requester)
else:
raise ConanException("Unable to process config install: %s" %
path_or_url)
finally:
if config_type is not None and path_or_url is not None:
value = "%s, %s, %s, %s" % (config_type, path_or_url, verify_ssl,
args)
cache.conan_config.set_item("general.config_install", value)
rmdir(tmp_folder)
def package(self, reference, package_id):
# Package paths
conan_file_path = self._client_cache.conanfile(reference)
if not os.path.exists(conan_file_path):
raise ConanException("Package recipe '%s' does not exist" %
str(reference))
conanfile = load_conanfile_class(conan_file_path)
if hasattr(conanfile, "build_id"):
raise ConanException(
"package command does not support recipes with 'build_id'\n"
"To repackage them use 'conan install'")
if not package_id:
packages = [
PackageReference(reference, packid)
for packid in self._client_cache.conan_builds(reference)
]
if not packages:
raise NotFoundException(
"%s: Package recipe has not been built locally\n"
"Please read the 'conan package' command help\n"
"Use 'conan install' or 'conan test_package' to build and "
"create binaries" % str(reference))
else:
packages = [PackageReference(reference, package_id)]
package_source_folder = self._client_cache.source(
reference, conanfile.short_paths)
for package_reference in packages:
build_folder = self._client_cache.build(package_reference,
short_paths=None)
if not os.path.exists(build_folder):
raise NotFoundException(
"%s: Package binary '%s' folder doesn't exist\n"
"Please read the 'conan package' command help\n"
"Use 'conan install' or 'conan test_package' to build and "
"create binaries" %
(str(reference), package_reference.package_id))
# The package already exist, we can use short_paths if they were defined
package_folder = self._client_cache.package(package_reference,
short_paths=None)
# Will read current conaninfo with specified options and load conanfile with them
output = ScopedOutput(str(reference), self._user_io.out)
output.info("Re-packaging %s" % package_reference.package_id)
conanfile = load_consumer_conanfile(conan_file_path,
build_folder,
self._client_cache.settings,
self._runner,
output,
reference=reference)
rmdir(package_folder)
if getattr(conanfile, 'no_copy_source', False):
source_folder = package_source_folder
else:
source_folder = build_folder
with environment_append(conanfile.env):
packager.create_package(conanfile, source_folder, build_folder,
package_folder, output)
def _rm_conandir(path):
''' removal of a directory that might contain a link to a short path
'''
link = os.path.join(path, CONAN_LINK)
if os.path.exists(link):
short_path = load(link)
rmdir(os.path.dirname(short_path))
rmdir(path)
def source(self):
z_name = "zlib-%s.tar.gz" % self.version
tools.download("https://zlib.net/zlib-%s.tar.gz" % self.version, z_name)
tools.unzip(z_name)
os.unlink(z_name)
files.rmdir("%s/contrib" % self.ZIP_FOLDER_NAME)
if not tools.os_info.is_windows:
self.run("chmod +x ./%s/configure" % self.ZIP_FOLDER_NAME)
def export_pkg(self, reference, source_folder, build_folder,
package_folder, install_folder, profile, force):
conan_file_path = self._client_cache.conanfile(reference)
if not os.path.exists(conan_file_path):
raise ConanException("Package recipe '%s' does not exist" %
str(reference))
remote_proxy = ConanProxy(self._client_cache,
self._user_io,
self._remote_manager,
remote_name=None,
update=False,
check_updates=False,
manifest_manager=None,
recorder=self._recorder)
loader = self.get_loader(profile)
conanfile = loader.load_virtual([reference], scope_options=True)
if install_folder and existing_info_files(install_folder):
_load_deps_info(install_folder, conanfile, required=True)
graph_builder = self._get_graph_builder(loader, False, remote_proxy)
deps_graph = graph_builder.load(conanfile)
# this is a bit tricky, but works. The loading of a cache package makes the referenced
# one, the first of the first level, always existing
nodes = deps_graph.direct_requires()
_, conanfile = nodes[0]
pkg_id = conanfile.info.package_id()
self._user_io.out.info("Packaging to %s" % pkg_id)
pkg_reference = PackageReference(reference, pkg_id)
dest_package_folder = self._client_cache.package(
pkg_reference, short_paths=conanfile.short_paths)
if os.path.exists(dest_package_folder):
if force:
rmdir(dest_package_folder)
else:
raise ConanException(
"Package already exists. Please use --force, -f to "
"overwrite it")
recipe_hash = self._client_cache.load_manifest(reference).summary_hash
conanfile.info.recipe_hash = recipe_hash
conanfile.develop = True
package_output = ScopedOutput(str(reference), self._user_io.out)
if package_folder:
packager.export_pkg(conanfile, package_folder, dest_package_folder,
package_output)
else:
packager.create_package(conanfile,
source_folder,
build_folder,
dest_package_folder,
install_folder,
package_output,
local=True)
def get_recipe(self, conan_reference):
output = ScopedOutput(str(conan_reference), self._out)
def _refresh():
export_path = self._client_cache.export(conan_reference)
rmdir(export_path)
# It might need to remove shortpath
rmdir(self._client_cache.source(conan_reference), True)
current_remote, _ = self._get_remote(conan_reference)
output.info("Retrieving from remote '%s'..." % current_remote.name)
self._remote_manager.get_recipe(conan_reference, export_path, current_remote)
if self._update:
output.info("Updated!")
else:
output.info("Installed!")
# check if it is in disk
conanfile_path = self._client_cache.conanfile(conan_reference)
path_exist = path_exists(conanfile_path, self._client_cache.store)
if path_exist:
if self._check_updates:
ret = self.update_available(conan_reference)
if ret != 0: # Found and not equal
remote, ref_remote = self._get_remote(conan_reference)
if ret == 1:
if not self._update:
if remote != ref_remote: # Forced new remote
output.warn("There is a new conanfile in '%s' remote. "
"Execute 'install -u -r %s' to update it."
% (remote.name, remote.name))
else:
output.warn("There is a new conanfile in '%s' remote. "
"Execute 'install -u' to update it."
% remote.name)
output.warn("Refused to install!")
else:
if remote != ref_remote:
# Delete packages, could be non coherent with new remote
rmdir(self._client_cache.packages(conan_reference))
_refresh()
elif ret == -1:
if not self._update:
output.info("Current conanfile is newer "
"than %s's one" % remote.name)
else:
output.error("Current conanfile is newer than %s's one. "
"Run 'conan remove %s' and run install again "
"to replace it." % (remote.name, conan_reference))
else:
self._retrieve_recipe(conan_reference, output)
if self._manifest_manager:
remote = self._registry.get_ref(conan_reference)
self._manifest_manager.check_recipe(conan_reference, remote)
return conanfile_path
def get_recipe(self, conan_reference):
output = ScopedOutput(str(conan_reference), self._out)
def _refresh():
export_path = self._client_cache.export(conan_reference)
rmdir(export_path)
# It might need to remove shortpath
rmdir(self._client_cache.source(conan_reference), True)
current_remote, _ = self._get_remote(conan_reference)
output.info("Retrieving from remote '%s'..." % current_remote.name)
self._remote_manager.get_recipe(conan_reference, export_path, current_remote)
if self._update:
output.info("Updated!")
else:
output.info("Installed!")
# check if it is in disk
conanfile_path = self._client_cache.conanfile(conan_reference)
path_exist = path_exists(conanfile_path, self._client_cache.store)
if path_exist:
if self._check_updates:
ret = self.update_available(conan_reference)
if ret != 0: # Found and not equal
remote, ref_remote = self._get_remote(conan_reference)
if ret == 1:
if not self._update:
if remote != ref_remote: # Forced new remote
output.warn("There is a new conanfile in '%s' remote. "
"Execute 'install -u -r %s' to update it."
% (remote.name, remote.name))
else:
output.warn("There is a new conanfile in '%s' remote. "
"Execute 'install -u' to update it."
% remote.name)
output.warn("Refused to install!")
else:
if remote != ref_remote:
# Delete packages, could be non coherent with new remote
rmdir(self._client_cache.packages(conan_reference))
_refresh()
elif ret == -1:
if not self._update:
output.info("Current conanfile is newer "
"than %s's one" % remote.name)
else:
output.error("Current conanfile is newer than %s's one. "
"Run 'conan remove %s' and run install again "
"to replace it." % (remote.name, conan_reference))
else:
self._retrieve_recipe(conan_reference, output)
if self._manifest_manager:
remote = self._registry.get_ref(conan_reference)
self._manifest_manager.check_recipe(conan_reference, remote)
return conanfile_path
def _export_conanfile(conanfile_path, output, client_cache, conanfile,
conan_ref, keep_source):
exports_folder = client_cache.export(conan_ref)
exports_source_folder = client_cache.export_sources(
conan_ref, conanfile.short_paths)
previous_digest = _init_export_folder(exports_folder,
exports_source_folder)
origin_folder = os.path.dirname(conanfile_path)
export_recipe(conanfile, origin_folder, exports_folder, output)
export_source(conanfile, origin_folder, exports_source_folder, output)
shutil.copy2(conanfile_path, os.path.join(exports_folder, CONANFILE))
scm_data, captured_revision = _capture_export_scm_data(
conanfile, os.path.dirname(conanfile_path), exports_folder, output,
client_cache, conan_ref)
digest = FileTreeManifest.create(exports_folder, exports_source_folder)
if previous_digest and previous_digest == digest:
output.info("The stored package has not changed")
modified_recipe = False
digest = previous_digest # Use the old one, keep old timestamp
else:
output.success('A new %s version was exported' % CONANFILE)
output.info('Folder: %s' % exports_folder)
modified_recipe = True
digest.save(exports_folder)
revision = scm_data.revision if scm_data and captured_revision else digest.summary_hash
with client_cache.update_metadata(conan_ref) as metadata:
# Note that there is no time set, the time will come from the remote
metadata.recipe.revision = revision
# FIXME: Conan 2.0 Clear the registry entry if the recipe has changed
source = client_cache.source(conan_ref, conanfile.short_paths)
remove = False
if is_dirty(source):
output.info("Source folder is corrupted, forcing removal")
remove = True
elif modified_recipe and not keep_source and os.path.exists(source):
output.info(
"Package recipe modified in export, forcing source folder removal")
output.info("Use the --keep-source, -k option to skip it")
remove = True
if remove:
output.info(
"Removing 'source' folder, this can take a while for big packages")
try:
# remove only the internal
rmdir(source)
except BaseException as e:
output.error("Unable to delete source folder. "
"Will be marked as corrupted for deletion")
output.warn(str(e))
set_dirty(source)
def source(self):
zip_name = "zlib-%s.tar.gz" % self.version
tools.download("https://zlib.net/fossils/%s" % (zip_name), zip_name)
tools.unzip(zip_name)
os.unlink(zip_name)
files.rmdir("%s/contrib" % self.ZIP_FOLDER_NAME)
if self.settings.os != "Windows":
self.run("chmod +x ./%s/configure" % self.ZIP_FOLDER_NAME)
def path_shortener(path, short_paths):
""" short_paths is 4-state:
False: Never shorten the path
True: Always shorten the path, create link if not existing
None: Use shorten path only if already exists, not create
"""
use_always_short_paths = get_env("CONAN_USE_ALWAYS_SHORT_PATHS", False)
short_paths = use_always_short_paths or short_paths
if short_paths is False or os.getenv("CONAN_USER_HOME_SHORT") == "None":
return path
link = os.path.join(path, CONAN_LINK)
if os.path.exists(link):
return load(link)
elif short_paths is None:
return path
if os.path.exists(path):
rmdir(path)
short_home = os.getenv("CONAN_USER_HOME_SHORT")
if not short_home:
drive = os.path.splitdrive(path)[0]
short_home = os.path.join(drive, os.sep, ".conan")
mkdir(short_home)
# Workaround for short_home living in NTFS file systems. Give full control permission
# to current user to avoid
# access problems in cygwin/msys2 windows subsystems when using short_home folder
try:
userdomain, username = os.getenv("USERDOMAIN"), os.environ["USERNAME"]
domainname = "%s\%s" % (userdomain,
username) if userdomain else username
cmd = r'cacls %s /E /G "%s":F' % (short_home, domainname)
subprocess.check_output(
cmd,
stderr=subprocess.STDOUT) # Ignoring any returned output, quiet
except subprocess.CalledProcessError:
# cmd can fail if trying to set ACL in non NTFS drives, ignoring it.
pass
redirect = hashed_redirect(short_home, path)
if not redirect:
logger.warning("Failed to create a deterministic short path in %s",
short_home)
redirect = tempfile.mkdtemp(dir=short_home, prefix="")
# Save the full path of the local cache directory where the redirect is from.
# This file is for debugging purposes and not used by Conan.
save(os.path.join(redirect, CONAN_REAL_PATH), path)
# This "1" is the way to have a non-existing directory, so commands like
# shutil.copytree() to it, works. It can be removed without compromising the
# temp folder generator and conan-links consistency
redirect = os.path.join(redirect, "1")
save(link, redirect)
return redirect
def tmp_config_install_folder(cache):
tmp_folder = os.path.join(cache.conan_folder, "tmp_config_install")
# necessary for Mac OSX, where the temp folders in /var/ are symlinks to /private/var/
tmp_folder = os.path.realpath(tmp_folder)
mkdir(tmp_folder)
try:
yield tmp_folder
finally:
rmdir(tmp_folder)
def export_remove(self):
export_folder = self.export()
rmdir(export_folder)
export_src_folder = os.path.join(self._base_folder, EXPORT_SRC_FOLDER)
rm_conandir(export_src_folder)
download_export = self.download_export()
rmdir(download_export)
scm_folder = os.path.join(self._base_folder, SCM_SRC_FOLDER)
rm_conandir(scm_folder)
def create_package(conanfile,
source_folder,
build_folder,
package_folder,
output,
local=False,
copy_info=False):
""" copies built artifacts, libs, headers, data, etc from build_folder to
package folder
"""
mkdir(package_folder)
# Make the copy of all the patterns
output.info("Generating the package")
output.info("Package folder %s" % (package_folder))
def wrap(dst_folder):
def new_method(pattern, src=""):
conanfile.copy(pattern, dst_folder, src)
return new_method
# FIXME: Deprecate these methods. Not documented. Confusing. Rely on LINTER
conanfile.copy_headers = wrap(DEFAULT_INCLUDE)
conanfile.copy_libs = wrap(DEFAULT_LIB)
conanfile.copy_bins = wrap(DEFAULT_BIN)
conanfile.copy_res = wrap(DEFAULT_RES)
try:
package_output = ScopedOutput("%s package()" % output.scope, output)
output.highlight("Calling package()")
if source_folder != build_folder:
conanfile.copy = FileCopier(source_folder, package_folder,
build_folder)
with conanfile_exception_formatter(str(conanfile), "package"):
conanfile.package()
conanfile.copy.report(package_output, warn=True)
conanfile.copy = FileCopier(build_folder, package_folder)
with conanfile_exception_formatter(str(conanfile), "package"):
conanfile.package()
conanfile.copy.report(package_output, warn=True)
except Exception as e:
if not local:
os.chdir(build_folder)
try:
rmdir(package_folder)
except Exception as e_rm:
output.error("Unable to remove package folder %s\n%s" %
(package_folder, str(e_rm)))
output.warn("**** Please delete it manually ****")
if isinstance(e, ConanExceptionInUserConanfileMethod):
raise
raise ConanException(e)
_create_aux_files(build_folder, package_folder, conanfile, copy_info)
output.success("Package '%s' created" % os.path.basename(package_folder))
def package_copy(src_ref, user_channel, package_ids, cache, user_io, short_paths=False,
force=False):
ref = "%s/%s@%s" % (src_ref.name, src_ref.version, user_channel)
if ref.count('@') > 1:
raise ConanException("Destination must contain user/channel only.")
dest_ref = ConanFileReference.loads(ref)
# Generate metadata
src_layout = cache.package_layout(src_ref, short_paths)
src_metadata = src_layout.load_metadata()
dst_layout = cache.package_layout(dest_ref, short_paths)
# Copy export
export_origin = src_layout.export()
if not os.path.exists(export_origin):
raise ConanException("'%s' doesn't exist" % str(src_ref))
export_dest = dst_layout.export()
if os.path.exists(export_dest):
if not force and not user_io.request_boolean("'%s' already exist. Override?"
% str(dest_ref)):
return
rmdir(export_dest)
shutil.copytree(export_origin, export_dest, symlinks=True)
user_io.out.info("Copied %s to %s" % (str(src_ref), str(dest_ref)))
export_sources_origin = src_layout.export_sources()
export_sources_dest = dst_layout.export_sources()
if os.path.exists(export_sources_dest):
rmdir(export_sources_dest)
shutil.copytree(export_sources_origin, export_sources_dest, symlinks=True)
user_io.out.info("Copied sources %s to %s" % (str(src_ref), str(dest_ref)))
# Copy packages
package_revisions = {} # To be stored in the metadata
for package_id in package_ids:
pref_origin = PackageReference(src_ref, package_id)
pref_dest = PackageReference(dest_ref, package_id)
package_path_origin = src_layout.package(pref_origin)
if dst_layout.package_id_exists(package_id):
if not force and not user_io.request_boolean("Package '%s' already exist."
" Override?" % str(package_id)):
continue
dst_layout.package_remove(pref_dest)
package_path_dest = dst_layout.package(pref_dest)
package_revisions[package_id] = (src_metadata.packages[package_id].revision,
src_metadata.recipe.revision)
shutil.copytree(package_path_origin, package_path_dest, symlinks=True)
user_io.out.info("Copied %s to %s" % (str(package_id), str(dest_ref)))
# Generate the metadata
with dst_layout.update_metadata() as metadata:
metadata.recipe.revision = src_metadata.recipe.revision
for package_id, (revision, recipe_revision) in package_revisions.items():
metadata.packages[package_id].revision = revision
metadata.packages[package_id].recipe_revision = recipe_revision
def source(self):
zip_name = "zlib-%s.tar.gz" % self.version
tools.download(
"http://downloads.sourceforge.net/project/libpng/zlib/%s/%s" %
(self.version, zip_name), zip_name)
tools.unzip(zip_name)
os.unlink(zip_name)
files.rmdir("%s/contrib" % self.ZIP_FOLDER_NAME)
if self.settings.os != "Windows":
self.run("chmod +x ./%s/configure" % self.ZIP_FOLDER_NAME)
def source(self):
z_name = "zlib-%s.tar.gz" % self.version
tools.download(
"https://codeload.github.com/madler/zlib/tar.gz/v%s" %
self.version, z_name)
tools.unzip(z_name)
os.unlink(z_name)
files.rmdir("%s/contrib" % self.ZIP_FOLDER_NAME)
if self.settings.os != "Windows":
self.run("chmod +x ./%s/configure" % self.ZIP_FOLDER_NAME)
def _export_upload(self, name, version=None, deps=None):
conan = TestClient(servers=self.servers, users=[("lasote", "mypass")])
files = cpp_hello_conan_files(name, version, deps, static=self.static)
conan_ref = ConanFileReference(name, version, "lasote", "stable")
conan.save(files, clean_first=True)
conan.run("export lasote/stable")
conan.run("install '%s' -o static=False -o language=0 --build missing" % str(conan_ref))
conan.run("upload %s --all" % str(conan_ref))
rmdir(conan.current_folder)
shutil.rmtree(conan.paths.store, ignore_errors=True)
def _export_upload(self, name, version=None, deps=None):
conan = TestClient(servers=self.servers, users={"default": [("lasote", "mypass")]})
files = cpp_hello_conan_files(name, version, deps, static=False)
conan.save(files)
conan.run("create . lasote/stable")
conan.run("upload * --all --confirm")
conan.run("remove * -f")
rmdir(conan.current_folder)
shutil.rmtree(conan.cache.store, ignore_errors=True)
def _make_migrations(self, old_version):
# ############### FILL THIS METHOD WITH THE REQUIRED ACTIONS ##############
# VERSION 0.1
if old_version == Version("0.1"):
# Remove config, conans, all!
self.out.warn("Reseting configuration and storage files...")
if self.conf_path:
rmdir(self.conf_path)
if self.store_path:
rmdir(self.store_path)
def _make_migrations(self, old_version):
# ############### FILL THIS METHOD WITH THE REQUIRED ACTIONS ##############
# VERSION 0.1
if old_version is None:
return
if old_version < Version("0.3"):
self.out.warn("Migration: Reseting configuration and storage files...")
if os.path.exists(self.conf_path):
rmdir(self.conf_path)
if os.path.exists(self.store_path):
rmdir(self.store_path)
def _build_node(self, conan_ref, conan_file, build_mode):
# Compute conan_file package from local (already compiled) or from remote
output = ScopedOutput(str(conan_ref), self._out)
package_id = conan_file.info.package_id()
package_reference = PackageReference(conan_ref, package_id)
conan_ref = package_reference.conan
package_folder = self._paths.package(package_reference, conan_file.short_paths)
build_folder = self._paths.build(package_reference, conan_file.short_paths)
src_folder = self._paths.source(conan_ref, conan_file.short_paths)
export_folder = self._paths.export(conan_ref)
# If already exists do not dirt the output, the common situation
# is that package is already installed and OK. If don't, the proxy
# will print some other message about it
if not package_exists(package_folder):
output.info("Installing package %s" % package_id)
self._handle_system_requirements(conan_ref, package_reference, conan_file, output)
force_build = self._build_forced(conan_ref, build_mode, conan_file)
if self._remote_proxy.get_package(package_reference, force_build,
short_paths=conan_file.short_paths):
return
# we need and can build? Only if we are forced or build_mode missing and package not exists
build = force_build or build_mode is True or conan_file.build_policy_missing
if build:
if not force_build and not build_mode:
output.info("Building package from source as defined by build_policy='missing'")
try:
rmdir(build_folder, conan_file.short_paths)
rmdir(package_folder)
except Exception as e:
raise ConanException("%s\n\nCouldn't remove folder, might be busy or open\n"
"Close any app using it, and retry" % str(e))
if force_build:
output.warn('Forced build from source')
self._build_package(export_folder, src_folder, build_folder, package_folder,
conan_file, output)
# Creating ***info.txt files
save(os.path.join(build_folder, CONANINFO), conan_file.info.dumps())
output.info("Generated %s" % CONANINFO)
save(os.path.join(build_folder, BUILD_INFO), TXTGenerator(conan_file).content)
output.info("Generated %s" % BUILD_INFO)
os.chdir(build_folder)
create_package(conan_file, build_folder, package_folder, output)
self._remote_proxy.handle_package_manifest(package_reference, installed=True)
else:
self._raise_package_not_found_error(conan_ref, conan_file)
def _refresh():
conan_dir_path = self._paths.export(conan_reference)
rmdir(conan_dir_path)
rmdir(self._paths.source(conan_reference), True) # It might need to remove shortpath
current_remote, _ = self._get_remote(conan_reference)
output.info("Retrieving from remote '%s'..." % current_remote.name)
self._remote_manager.get_conanfile(conan_reference, current_remote)
if self._update:
output.info("Updated!")
else:
output.info("Installed!")
def get_conanfile(self, conan_reference, remote):
"""
Read the conans from remotes
Will iterate the remotes to find the conans unless remote was specified
returns (dict relative_filepath:content , remote_name)"""
export_files = self._call_remote(remote, "get_conanfile", conan_reference)
export_folder = self._paths.export(conan_reference)
uncompress_files(export_files, export_folder, EXPORT_TGZ_NAME)
# Make sure that the source dir is deleted
rmdir(self._paths.source(conan_reference))
def _export_upload(self, name, version=None, deps=None):
conan = TestClient(servers=self.servers, users={"default": [("lasote", "mypass")]})
dll_export = conan.default_compiler_visual_studio
files = cpp_hello_conan_files(name, version, deps, static=False, dll_export=dll_export)
conan_ref = ConanFileReference(name, version, "lasote", "stable")
conan.save(files, clean_first=True)
conan.run("export lasote/stable")
conan.run("install '%s' --build missing" % str(conan_ref))
conan.run("upload %s --all" % str(conan_ref))
rmdir(conan.current_folder)
shutil.rmtree(conan.paths.store, ignore_errors=True)
def _refresh():
export_path = self._client_cache.export(conan_reference)
rmdir(export_path)
# It might need to remove shortpath
rm_conandir(self._client_cache.source(conan_reference))
current_remote, _ = self._get_remote(conan_reference)
output.info("Retrieving from remote '%s'..." % current_remote.name)
self._remote_manager.get_recipe(conan_reference, export_path, current_remote)
if self._update:
output.info("Updated!")
else:
output.info("Installed!")
def create_package(conanfile, source_folder, build_folder, package_folder, install_folder,
output, local=False, copy_info=False):
""" copies built artifacts, libs, headers, data, etc. from build_folder to
package folder
"""
mkdir(package_folder)
# Make the copy of all the patterns
output.info("Generating the package")
output.info("Package folder %s" % package_folder)
try:
package_output = ScopedOutput("%s package()" % output.scope, output)
output.highlight("Calling package()")
conanfile.package_folder = package_folder
conanfile.source_folder = source_folder
conanfile.install_folder = install_folder
conanfile.build_folder = build_folder
def recipe_has(attribute):
return attribute in conanfile.__class__.__dict__
if source_folder != build_folder:
conanfile.copy = FileCopier(source_folder, package_folder, build_folder)
with conanfile_exception_formatter(str(conanfile), "package"):
with tools.chdir(source_folder):
conanfile.package()
copy_done = conanfile.copy.report(package_output)
if not copy_done and recipe_has("package"):
output.warn("No files copied from source folder!")
conanfile.copy = FileCopier(build_folder, package_folder)
with tools.chdir(build_folder):
with conanfile_exception_formatter(str(conanfile), "package"):
conanfile.package()
copy_done = conanfile.copy.report(package_output)
if not copy_done and recipe_has("build") and recipe_has("package"):
output.warn("No files copied from build folder!")
except Exception as e:
if not local:
os.chdir(build_folder)
try:
rmdir(package_folder)
except Exception as e_rm:
output.error("Unable to remove package folder %s\n%s" % (package_folder, str(e_rm)))
output.warn("**** Please delete it manually ****")
if isinstance(e, ConanExceptionInUserConanfileMethod):
raise
raise ConanException(e)
_create_aux_files(install_folder, package_folder, conanfile, copy_info)
output.success("Package '%s' created" % os.path.basename(package_folder))
def remove_source(raise_error=True):
output.warn("This can take a while for big packages")
try:
rmdir(src_folder)
except BaseException as e_rm:
save(dirty, "") # Creation of DIRTY flag
msg = str(e_rm)
if six.PY2:
msg = str(e_rm).decode("latin1") # Windows prints some chars in latin1
output.error("Unable to remove source folder %s\n%s" % (src_folder, msg))
output.warn("**** Please delete it manually ****")
if raise_error or isinstance(e_rm, KeyboardInterrupt):
raise ConanException("Unable to remove source folder")
def get_recipe(self, conan_reference, dest_folder, remote):
"""
Read the conans from remotes
Will iterate the remotes to find the conans unless remote was specified
returns (dict relative_filepath:abs_path , remote_name)"""
zipped_files = self._call_remote(remote, "get_recipe", conan_reference, dest_folder)
files = unzip_and_get_files(zipped_files, dest_folder, EXPORT_TGZ_NAME)
# Make sure that the source dir is deleted
rmdir(self._client_cache.source(conan_reference), True)
# TODO: Download only the CONANFILE file and only download the rest of files
# in install if needed (not found remote package)
return files
def _init_export_folder(destination_folder):
previous_digest = None
try:
if os.path.exists(destination_folder):
if os.path.exists(os.path.join(destination_folder, CONAN_MANIFEST)):
manifest_content = load(os.path.join(destination_folder, CONAN_MANIFEST))
previous_digest = FileTreeManifest.loads(manifest_content)
# Maybe here we want to invalidate cache
rmdir(destination_folder)
os.makedirs(destination_folder)
except Exception as e:
raise ConanException("Unable to create folder %s\n%s" % (destination_folder, str(e)))
return previous_digest
