def test_paths(self):
"""Unit test of path_exists"""
tmp_dir = temp_folder()
new_path = os.path.join(tmp_dir, "CapsDir")
mkdir(new_path)
self.assertTrue(path_exists(new_path, tmp_dir))
self.assertFalse(path_exists(os.path.join(tmp_dir, "capsdir"), tmp_dir))
def test_paths(self):
"""Unit test of path_exists"""
tmp_dir = temp_folder()
new_path = os.path.join(tmp_dir, "CapsDir")
mkdir(new_path)
self.assertTrue(path_exists(new_path, tmp_dir))
self.assertFalse(path_exists(os.path.join(tmp_dir, "capsdir"),
tmp_dir))
def get_conanfile(self, conan_reference):
output = ScopedOutput(str(conan_reference), self._out)
def _refresh():
conan_dir_path = self._paths.export(conan_reference)
rmdir(conan_dir_path)
rmdir(self._paths.source(conan_reference), True) # It might need to remove shortpath
current_remote, _ = self._get_remote(conan_reference)
output.info("Retrieving from remote '%s'..." % current_remote.name)
self._remote_manager.get_conanfile(conan_reference, current_remote)
if self._update:
output.info("Updated!")
else:
output.info("Installed!")
# check if it is in disk
conanfile_path = self._paths.conanfile(conan_reference)
path_exist = path_exists(conanfile_path, self._paths.store)
if path_exist:
if self._check_integrity: # Check if package is corrupted
read_manifest, expected_manifest = self._paths.conan_manifests(conan_reference)
if read_manifest.file_sums != expected_manifest.file_sums:
output.warn("Bad conanfile detected! Removing export directory... ")
_refresh()
else: # Check for updates
if self._check_updates:
ret = self.update_available(conan_reference)
if ret != 0: # Found and not equal
remote, ref_remote = self._get_remote(conan_reference)
if ret == 1:
if not self._update:
if remote != ref_remote: # Forced new remote
output.warn("There is a new conanfile in '%s' remote. "
"Execute 'install -u -r %s' to update it."
% (remote.name, remote.name))
else:
output.warn("There is a new conanfile in '%s' remote. "
"Execute 'install -u' to update it."
% remote.name)
output.warn("Refused to install!")
else:
if remote != ref_remote:
# Delete packages, could be non coherent with new remote
rmdir(self._paths.packages(conan_reference))
_refresh()
elif ret == -1:
if not self._update:
output.info("Current conanfile is newer "
"than %s's one" % remote.name)
else:
output.error("Current conanfile is newer than %s's one. "
"Run 'conan remove %s' and run install again "
"to replace it." % (remote.name, conan_reference))
else:
self._retrieve_conanfile(conan_reference, output)
return conanfile_path
def _get_paths(self, absolute_path, files_subset):
if not path_exists(absolute_path, self._store_folder):
raise NotFoundException("")
paths = relative_dirs(absolute_path)
if files_subset is not None:
paths = set(paths).intersection(set(files_subset))
abs_paths = [os.path.join(absolute_path, relpath) for relpath in paths]
return abs_paths
def get_recipe(self, conan_reference):
output = ScopedOutput(str(conan_reference), self._out)
def _refresh():
export_path = self._client_cache.export(conan_reference)
rmdir(export_path)
# It might need to remove shortpath
rm_conandir(self._client_cache.source(conan_reference))
current_remote, _ = self._get_remote(conan_reference)
output.info("Retrieving from remote '%s'..." % current_remote.name)
self._remote_manager.get_recipe(conan_reference, export_path, current_remote)
if self._update:
output.info("Updated!")
else:
output.info("Installed!")
# check if it is in disk
conanfile_path = self._client_cache.conanfile(conan_reference)
path_exist = path_exists(conanfile_path, self._client_cache.store)
if path_exist:
if self._check_updates:
ret = self.update_available(conan_reference)
if ret != 0: # Found and not equal
remote, ref_remote = self._get_remote(conan_reference)
if ret == 1:
if not self._update:
if remote != ref_remote: # Forced new remote
output.warn("There is a new conanfile in '%s' remote. "
"Execute 'install -u -r %s' to update it."
% (remote.name, remote.name))
else:
output.warn("There is a new conanfile in '%s' remote. "
"Execute 'install -u' to update it."
% remote.name)
output.warn("Refused to install!")
else:
if remote != ref_remote:
# Delete packages, could be non coherent with new remote
DiskRemover(self._client_cache).remove_packages(conan_reference)
_refresh()
elif ret == -1:
if not self._update:
output.info("Current conanfile is newer "
"than %s's one" % remote.name)
else:
output.error("Current conanfile is newer than %s's one. "
"Run 'conan remove %s' and run install again "
"to replace it." % (remote.name, conan_reference))
else:
self._retrieve_recipe(conan_reference, output)
if self._manifest_manager:
remote = self._registry.get_ref(conan_reference)
self._manifest_manager.check_recipe(conan_reference, remote)
return conanfile_path
def get_recipe(self, conan_reference):
output = ScopedOutput(str(conan_reference), self._out)
def _refresh():
export_path = self._client_cache.export(conan_reference)
rmdir(export_path)
# It might need to remove shortpath
rmdir(self._client_cache.source(conan_reference), True)
current_remote, _ = self._get_remote(conan_reference)
output.info("Retrieving from remote '%s'..." % current_remote.name)
self._remote_manager.get_recipe(conan_reference, export_path, current_remote)
if self._update:
output.info("Updated!")
else:
output.info("Installed!")
# check if it is in disk
conanfile_path = self._client_cache.conanfile(conan_reference)
path_exist = path_exists(conanfile_path, self._client_cache.store)
if path_exist:
if self._check_updates:
ret = self.update_available(conan_reference)
if ret != 0: # Found and not equal
remote, ref_remote = self._get_remote(conan_reference)
if ret == 1:
if not self._update:
if remote != ref_remote: # Forced new remote
output.warn("There is a new conanfile in '%s' remote. "
"Execute 'install -u -r %s' to update it."
% (remote.name, remote.name))
else:
output.warn("There is a new conanfile in '%s' remote. "
"Execute 'install -u' to update it."
% remote.name)
output.warn("Refused to install!")
else:
if remote != ref_remote:
# Delete packages, could be non coherent with new remote
rmdir(self._client_cache.packages(conan_reference))
_refresh()
elif ret == -1:
if not self._update:
output.info("Current conanfile is newer "
"than %s's one" % remote.name)
else:
output.error("Current conanfile is newer than %s's one. "
"Run 'conan remove %s' and run install again "
"to replace it." % (remote.name, conan_reference))
else:
self._retrieve_recipe(conan_reference, output)
if self._manifest_manager:
remote = self._registry.get_ref(conan_reference)
self._manifest_manager.check_recipe(conan_reference, remote)
return conanfile_path
def get_snapshot(self, absolute_path="", files_subset=None):
"""returns a dict with the filepaths and md5"""
if not path_exists(absolute_path, self._store_folder):
raise NotFoundException("")
paths = relative_dirs(absolute_path)
if files_subset is not None:
paths = set(paths).intersection(set(files_subset))
abs_paths = [os.path.join(absolute_path, relpath) for relpath in paths]
return {filepath: md5sum(filepath) for filepath in abs_paths}
def get_package(self, package_reference, force_build):
""" obtain a package, either from disk or retrieve from remotes if necessary
and not necessary to build
"""
output = ScopedOutput(str(package_reference.conan), self._out)
package_folder = self._paths.package(package_reference)
# Check current package status
if path_exists(package_folder, self._paths.store):
if self._check_integrity or self._check_updates:
read_manifest, expected_manifest = self._paths.package_manifests(
package_reference)
if self._check_integrity: # Check if package is corrupted
if read_manifest.file_sums != expected_manifest.file_sums:
# If not valid package, ensure empty folder
output.warn("Bad package '%s' detected! Removing "
"package directory... " %
str(package_reference.package_id))
rmdir(package_folder)
if self._check_updates:
try: # get_conan_digest can fail, not in server
upstream_manifest = self.get_package_digest(
package_reference)
if upstream_manifest.file_sums != read_manifest.file_sums:
if upstream_manifest.time > read_manifest.time:
output.warn(
"Current package is older than remote upstream one"
)
if self._update:
output.warn(
"Removing it to retrieve or build an updated one"
)
rmdir(package_folder)
else:
output.warn(
"Current package is newer than remote upstream one"
)
except ConanException:
pass
if not force_build:
local_package = os.path.exists(package_folder)
if local_package:
output = ScopedOutput(str(package_reference.conan), self._out)
output.info('Already installed!')
return True
return self._retrieve_remote_package(package_reference, output)
return False
def get_package(self, package_reference, force_build):
""" obtain a package, either from disk or retrieve from remotes if necessary
and not necessary to build
"""
output = ScopedOutput(str(package_reference.conan), self._out)
package_folder = self._client_cache.package(package_reference)
# Check current package status
if path_exists(package_folder, self._client_cache.store):
if self._check_updates:
read_manifest = self._client_cache.load_package_manifest(
package_reference)
try: # get_conan_digest can fail, not in server
upstream_manifest = self.get_package_digest(
package_reference)
if upstream_manifest.file_sums != read_manifest.file_sums:
if upstream_manifest.time > read_manifest.time:
output.warn(
"Current package is older than remote upstream one"
)
if self._update:
output.warn(
"Removing it to retrieve or build an updated one"
)
rmdir(package_folder)
else:
output.warn(
"Current package is newer than remote upstream one"
)
except ConanException:
pass
installed = False
if not force_build:
local_package = os.path.exists(package_folder)
if local_package:
output = ScopedOutput(str(package_reference.conan), self._out)
output.info('Already installed!')
installed = True
else:
installed = self._retrieve_remote_package(
package_reference, output)
self.handle_package_manifest(package_reference, installed)
return installed
def retrieve_conanfile(self, conan_reference, consumer=False):
""" returns the requested conanfile object, retrieving it from
remotes if necessary. Can raise NotFoundException
"""
output = ScopedOutput(str(conan_reference), self._out)
conanfile_path = self._paths.conanfile(conan_reference)
if not self._paths.valid_conan_digest(conan_reference):
conan_dir_path = self._paths.export(conan_reference)
if path_exists(conan_dir_path, self._paths.store):
# If not valid conanfile, ensure empty folder
output.warn("Bad conanfile detected! Removing export directory... ")
rmdir(conan_dir_path)
output.info("Conanfile not found, retrieving from server")
# If not in localhost, download it. Will raise if not found
self._remote_manager.get_conanfile(conan_reference, self._remote)
conanfile = self._loader.load_conan(conanfile_path, output, consumer)
return conanfile
def retrieve_conanfile(self, conan_reference, consumer=False):
""" returns the requested conanfile object, retrieving it from
remotes if necessary. Can raise NotFoundException
"""
conanfile_path = self._paths.conanfile(conan_reference)
if not self._paths.valid_conan_digest(conan_reference):
conan_dir_path = self._paths.export(conan_reference)
if path_exists(conan_dir_path, self._paths.store):
# If not valid conanfile, ensure empty folder
self._user_io.out.warn("Bad conanfile '%s' detected! Removing "
"export directory... " % str(conan_reference))
rmdir(conan_dir_path)
self._user_io.out.info("Conan %s not found, retrieving from server"
% str(conan_reference))
# If not in localhost, download it. Will raise if not found
self._retrieve_remote_conan_file(conan_reference)
conanfile = self._loader.load_conan(conanfile_path, consumer)
return conanfile
def get_package(self, package_reference, force_build):
""" obtain a package, either from disk or retrieve from remotes if necessary
and not necessary to build
"""
output = ScopedOutput(str(package_reference.conan), self._out)
package_folder = self._paths.package(package_reference)
# Check current package status
if path_exists(package_folder, self._paths.store):
if self._check_integrity or self._check_updates:
read_manifest, expected_manifest = self._paths.package_manifests(package_reference)
if self._check_integrity: # Check if package is corrupted
if read_manifest.file_sums != expected_manifest.file_sums:
# If not valid package, ensure empty folder
output.warn("Bad package '%s' detected! Removing "
"package directory... " % str(package_reference.package_id))
rmdir(package_folder)
if self._check_updates:
try: # get_conan_digest can fail, not in server
upstream_manifest = self.get_package_digest(package_reference)
if upstream_manifest.file_sums != read_manifest.file_sums:
if upstream_manifest.time > read_manifest.time:
output.warn("Current package is older than remote upstream one")
if self._update:
output.warn("Removing it to retrieve or build an updated one")
rmdir(package_folder)
else:
output.warn("Current package is newer than remote upstream one")
except ConanException:
pass
if not force_build:
local_package = os.path.exists(package_folder)
if local_package:
output = ScopedOutput(str(package_reference.conan), self._out)
output.info('Already installed!')
return True
return self._retrieve_remote_package(package_reference, output)
return False
def _compute_private_nodes(self, deps_graph, build_mode):
""" computes a list of nodes that are not required to be built, as they are
private requirements of already available shared libraries as binaries
"""
private_closure = deps_graph.private_nodes()
skippable_nodes = []
for private_node, private_requirers in private_closure:
for private_requirer in private_requirers:
conan_ref, conan_file = private_requirer
package_id = conan_file.info.package_id()
package_reference = PackageReference(conan_ref, package_id)
package_folder = self._paths.package(package_reference)
if not path_exists(package_folder, self._paths.store):
if not self._force_build(conan_ref, build_mode): # Not download package
self._user_io.out.info('Package for %s does not exist' % str(conan_ref))
if not self._retrieve_remote_package(package_reference):
break
else:
skippable_nodes.append(private_node)
return skippable_nodes
def retrieve_conanfile(self, conan_reference, consumer=False):
""" returns the requested conanfile object, retrieving it from
remotes if necessary. Can raise NotFoundException
"""
conanfile_path = self._paths.conanfile(conan_reference)
if not self._paths.valid_conan_digest(conan_reference):
conan_dir_path = self._paths.export(conan_reference)
if path_exists(conan_dir_path, self._paths.store):
# If not valid conanfile, ensure empty folder
self._user_io.out.warn("Bad conanfile '%s' detected! Removing "
"export directory... " %
str(conan_reference))
rmdir(conan_dir_path)
self._user_io.out.info(
"Conan %s not found, retrieving from server" %
str(conan_reference))
# If not in localhost, download it. Will raise if not found
self._retrieve_remote_conan_file(conan_reference)
conanfile = self._loader.load_conan(conanfile_path, consumer)
return conanfile
def _compute_private_nodes(self, deps_graph, build_mode):
""" computes a list of nodes that are not required to be built, as they are
private requirements of already available shared libraries as binaries
"""
private_closure = deps_graph.private_nodes()
skippable_nodes = []
for private_node, private_requirers in private_closure:
for private_requirer in private_requirers:
conan_ref, conan_file = private_requirer
if conan_ref is None:
continue
package_id = conan_file.info.package_id()
package_reference = PackageReference(conan_ref, package_id)
package_folder = self._paths.package(package_reference)
if not path_exists(package_folder, self._paths.store):
if not self._force_build(conan_ref, build_mode): # Not download package
output = ScopedOutput(str(conan_ref), self._out)
output.info("Package not installed")
if not self._retrieve_remote_package(package_reference, output):
break
else:
skippable_nodes.append(private_node)
return skippable_nodes
def _compute_private_nodes(self, deps_graph, build_mode):
""" computes a list of nodes that are not required to be built, as they are
private requirements of already available shared libraries as binaries
"""
private_closure = deps_graph.private_nodes()
skippable_nodes = []
for private_node, private_requirers in private_closure:
for private_requirer in private_requirers:
conan_ref, conan_file = private_requirer
package_id = conan_file.info.package_id()
package_reference = PackageReference(conan_ref, package_id)
package_folder = self._paths.package(package_reference)
if not path_exists(package_folder, self._paths.store):
if not self._force_build(
conan_ref, build_mode): # Not download package
self._user_io.out.info(
'Package for %s does not exist' % str(conan_ref))
if not self._retrieve_remote_package(
package_reference):
break
else:
skippable_nodes.append(private_node)
return skippable_nodes
def get_package(self, package_reference, force_build):
""" obtain a package, either from disk or retrieve from remotes if necessary
and not necessary to build
"""
output = ScopedOutput(str(package_reference.conan), self._out)
package_folder = self._client_cache.package(package_reference)
# Check current package status
if path_exists(package_folder, self._client_cache.store):
if self._check_updates:
read_manifest = self._client_cache.load_package_manifest(package_reference)
try: # get_conan_digest can fail, not in server
upstream_manifest = self.get_package_digest(package_reference)
if upstream_manifest.file_sums != read_manifest.file_sums:
if upstream_manifest.time > read_manifest.time:
output.warn("Current package is older than remote upstream one")
if self._update:
output.warn("Removing it to retrieve or build an updated one")
rmdir(package_folder)
else:
output.warn("Current package is newer than remote upstream one")
except ConanException:
pass
installed = False
if not force_build:
local_package = os.path.exists(package_folder)
if local_package:
output = ScopedOutput(str(package_reference.conan), self._out)
output.info('Already installed!')
installed = True
else:
installed = self._retrieve_remote_package(package_reference, output)
self.handle_package_manifest(package_reference, installed)
return installed
def delete_file(self, path):
'''Delete files from bucket. Path already contains base dir'''
if not path_exists(path, self.base_storage_path):
raise NotFoundException("")
os.remove(path)
def _digests(self, digest_path):
if not path_exists(digest_path, self.store):
return None, None
readed_digest = FileTreeManifest.loads(load(digest_path))
expected_digest = FileTreeManifest.create(os.path.dirname(digest_path))
return readed_digest, expected_digest
def conan_manifests(self, conan_reference):
digest_path = self.digestfile_conanfile(conan_reference)
if not path_exists(digest_path, self.store):
return None, None
return self._digests(digest_path)
def path_exists(self, path, basedir=None):
from conans.util.files import path_exists
return path_exists(path, basedir)
def get_file(self, filepath):
"""path already contains the base_storage_path
(obtained through paths object)"""
if not path_exists(filepath, self.base_storage_path):
raise NotFoundException()
return load(filepath)
def delete_file(self, path):
"""Delete files from bucket. Path already contains base dir"""
if not path_exists(path, self._store_folder):
raise NotFoundException("")
os.remove(path)
def valid_conan_digest(self, conan_reference):
digest_path = self.digestfile_conanfile(conan_reference)
if not path_exists(digest_path, self.store):
return False
return self.valid_digest(digest_path)
def valid_conan_digest(self, conan_reference):
digest_path = self.digestfile_conanfile(conan_reference)
if not path_exists(digest_path, self.store):
return False
return self.valid_digest(digest_path)
def delete_folder(self, path):
"""Delete folder from disk. Path already contains base dir"""
if not path_exists(path, self._store_folder):
raise NotFoundException("")
rmdir(path)
def conan_manifests(self, conan_reference):
digest_path = self.digestfile_conanfile(conan_reference)
if not path_exists(digest_path, self.store):
return None, None
return self._digests(digest_path)
def _digests(self, digest_path):
if not path_exists(digest_path, self.store):
return None, None
readed_digest = FileTreeManifest.loads(load(digest_path))
expected_digest = FileTreeManifest.create(os.path.dirname(digest_path))
return readed_digest, expected_digest
def path_exists(self, path, basedir=None):
from conans.util.files import path_exists
return path_exists(path, basedir)
def delete_file(self, path):
'''Delete files from bucket. Path already contains base dir'''
if not path_exists(path, self._store_folder):
raise NotFoundException("")
os.remove(path)
def delete_folder(self, path):
'''Delete folder from disk. Path already contains base dir'''
if not path_exists(path, self._store_folder):
raise NotFoundException("")
rmdir(path)
def delete_folder(self, path):
'''Delete folder from disk. Path already contains base dir'''
if not path_exists(path, self.base_storage_path):
raise NotFoundException("")
rmdir(path)
def get_file(self, filepath):
"""path already contains the base_storage_path
(obtained through paths object)"""
if not path_exists(filepath, self.base_storage_path):
raise NotFoundException("")
return load(filepath)
