def __init__(self, base_folder=None, current_folder=None,
servers=None, users=None, client_version=CLIENT_VERSION,
min_server_compatible_version=MIN_SERVER_COMPATIBLE_VERSION):
"""
storage_folder: Local storage path
current_folder: Current execution folder
servers: dict of {remote_name: TestServer}
logins is a list of (user, password) for auto input in order
if required==> [("lasote", "mypass"), ("other", "otherpass")]
"""
self.users = users or {"default":
[(TESTING_REMOTE_PRIVATE_USER, TESTING_REMOTE_PRIVATE_PASS)]}
self.servers = servers or {}
self.client_version = Version(str(client_version))
self.min_server_compatible_version = Version(str(min_server_compatible_version))
self.base_folder = base_folder or temp_folder()
# Define storage_folder, if not, it will be read from conf file & pointed to real user home
self.storage_folder = os.path.join(self.base_folder, ".conan", "data")
self.paths = ConanPaths(self.base_folder, self.storage_folder,
TestBufferConanOutput())
self.default_settings(get_env("CONAN_COMPILER", "gcc"),
get_env("CONAN_COMPILER_VERSION", "4.8"),
get_env("CONAN_LIBCXX", "libstdc++"))
self.init_dynamic_vars()
save(self.paths.registry, "")
registry = RemoteRegistry(self.paths.registry, TestBufferConanOutput())
for name, server in self.servers.items():
registry.add(name, server.fake_url)
logger.debug("Client storage = %s" % self.storage_folder)
self.current_folder = current_folder or temp_folder()
def __init__(self, base_folder=None, current_folder=None,
servers=None, users=None, client_version=CLIENT_VERSION,
min_server_compatible_version=MIN_SERVER_COMPATIBLE_VERSION):
"""
storage_folder: Local storage path
current_folder: Current execution folder
servers: dict of {remote_name: TestServer}
logins is a list of (user, password) for auto input in order
if required==> [("lasote", "mypass"), ("other", "otherpass")]
"""
self.users = users or {"default":
[(TESTING_REMOTE_PRIVATE_USER, TESTING_REMOTE_PRIVATE_PASS)]}
self.servers = servers or {}
self.client_version = Version(str(client_version))
self.min_server_compatible_version = Version(str(min_server_compatible_version))
self.base_folder = base_folder or temp_folder()
# Define storage_folder, if not, it will be read from conf file & pointed to real user home
self.storage_folder = os.path.join(self.base_folder, ".conan", "data")
self.paths = ConanPaths(self.base_folder, self.storage_folder, TestBufferConanOutput())
self.default_settings(get_env("CONAN_COMPILER", "gcc"),
get_env("CONAN_COMPILER_VERSION", "4.8"))
self.init_dynamic_vars()
save(self.paths.registry, "")
registry = RemoteRegistry(self.paths.registry, TestBufferConanOutput())
for name, server in self.servers.items():
registry.add(name, server.fake_url)
logger.debug("Client storage = %s" % self.storage_folder)
self.current_folder = current_folder or temp_folder()
def update_servers(self, servers):
self.servers = servers or {}
save(self.client_cache.registry, "")
registry = RemoteRegistry(self.client_cache.registry, TestBufferConanOutput())
for name, server in self.servers.items():
if isinstance(server, TestServer):
registry.add(name, server.fake_url)
else:
registry.add(name, server)
def __init__(self, client_cache, user_io, remote_manager, remote_name,
update=False, check_updates=False, manifest_manager=False):
self._client_cache = client_cache
self._out = user_io.out
self._remote_manager = remote_manager
self._registry = RemoteRegistry(self._client_cache.registry, self._out)
self._remote_name = remote_name
self._update = update
self._check_updates = check_updates or update # Update forces check
self._manifest_manager = manifest_manager
def __init__(self, paths, user_io, remote_manager, remote_name,
update=False, check_updates=False, check_integrity=False):
self._paths = paths
self._out = user_io.out
self._remote_manager = remote_manager
self._registry = RemoteRegistry(self._paths.registry, self._out)
self._remote_name = remote_name
self._update = update
self._check_updates = check_updates or update # Update forces check
self._check_integrity = check_integrity
def authenticate(self, name, password, remote=None):
registry = RemoteRegistry(self._client_cache.registry, self._user_io.out)
if not remote:
remote = registry.default_remote
else:
remote = registry.remote(remote)
if password == "":
name, password = self._user_io.request_login(remote_name=remote.name,
username=name)
self._remote_manager.authenticate(remote, name, password)
def user_set(client_cache, output, user, remote_name=None):
localdb = LocalDB(client_cache.localdb)
registry = RemoteRegistry(client_cache.registry, output)
if not remote_name:
remote = registry.default_remote
else:
remote = registry.remote(remote_name)
if user.lower() == "none":
user = None
update_localdb(localdb, user, None, remote, output)
def test_returns_on_failures(self):
test_server = TestServer([("*/*@*/*", "*")], [("*/*@*/*", "*")])
servers = {"default": test_server}
class Response(object):
ok = None
status_code = None
charset = None
text = ""
headers = {}
def __init__(self, ok, status_code):
self.ok = ok
self.status_code = status_code
class BuggyRequester(object):
def __init__(self, *args, **kwargs):
pass
def get(self, *args, **kwargs):
return Response(False, 404)
client2 = TestClient(servers=servers, requester_class=BuggyRequester)
conan_ref = ConanFileReference.loads("Hello/1.2.1@frodo/stable")
registry = RemoteRegistry(client2.client_cache.registry, client2.out)
installer = ConanProxy(client2.paths,
client2.user_io.out,
client2.remote_manager,
registry=registry)
with self.assertRaises(NotFoundException):
installer.get_recipe(conan_ref, False, False, None,
ActionRecorder())
class BuggyRequester2(BuggyRequester):
def get(self, *args, **kwargs):
return Response(False, 500)
client2 = TestClient(servers=servers, requester_class=BuggyRequester2)
registry = RemoteRegistry(client2.client_cache.registry, client2.out)
installer = ConanProxy(client2.paths,
client2.user_io.out,
client2.remote_manager,
registry=registry)
try:
installer.get_recipe(conan_ref, False, False, None,
ActionRecorder())
except NotFoundException:
self.assertFalse(True) # Shouldn't capture here
except ConanException:
pass
def __init__(self, client_cache, user_io, runner, remote_manager,
_settings_preprocessor, interactive=True):
assert isinstance(user_io, UserIO)
assert isinstance(client_cache, ClientCache)
self._client_cache = client_cache
self._user_io = user_io
self._runner = runner
self._remote_manager = remote_manager
self._settings_preprocessor = _settings_preprocessor
self._registry = RemoteRegistry(self._client_cache.registry, self._user_io.out)
if not interactive:
self._user_io.disable_input()
def __init__(self,
paths,
user_io,
remote_manager,
remote_name,
update=False,
check_updates=True):
self._paths = paths
self._out = user_io.out
self._remote_manager = remote_manager
self._registry = RemoteRegistry(self._paths.registry, self._out)
self._remote_name = remote_name
self._update = update
self._check_updates = check_updates
def add_remove_update_test(self):
f = os.path.join(temp_folder(), "aux_file")
save(f, dump_registry(default_remotes, {}, {}))
registry = RemoteRegistry(f, TestBufferConanOutput())
# Add
registry.remotes.add("local", "http://localhost:9300")
self.assertEqual(registry.remotes.list, [("conan-center", "https://conan.bintray.com", True),
("local", "http://localhost:9300", True)])
# Add
registry.remotes.add("new", "new_url", False)
self.assertEqual(registry.remotes.list, [("conan-center", "https://conan.bintray.com", True),
("local", "http://localhost:9300", True),
("new", "new_url", False)])
with self.assertRaises(ConanException):
registry.remotes.add("new", "new_url")
# Update
registry.remotes.update("new", "other_url")
self.assertEqual(registry.remotes.list, [("conan-center", "https://conan.bintray.com", True),
("local", "http://localhost:9300", True),
("new", "other_url", True)])
with self.assertRaises(ConanException):
registry.remotes.update("new2", "new_url")
registry.remotes.update("new", "other_url", False)
self.assertEqual(registry.remotes.list, [("conan-center", "https://conan.bintray.com", True),
("local", "http://localhost:9300", True),
("new", "other_url", False)])
# Remove
registry.remotes.remove("local")
self.assertEqual(registry.remotes.list, [("conan-center", "https://conan.bintray.com", True),
("new", "other_url", False)])
with self.assertRaises(ConanException):
registry.remotes.remove("new2")
def info(self, deps_graph, graph_updates_info, only, remote, package_filter, show_paths, project_reference):
registry = RemoteRegistry(self.client_cache.registry, self.user_io.out)
Printer(self.user_io.out).print_info(deps_graph, project_reference,
only, registry, graph_updates_info=graph_updates_info,
remote=remote, node_times=self._read_dates(deps_graph),
path_resolver=self.client_cache, package_filter=package_filter,
show_paths=show_paths)
def _get_graph(self, reference, current_path, remote, options, settings,
filename, update, check_updates, manifest_manager, scopes,
package_settings, env, package_env):
loader = self._loader(current_path, settings, package_settings,
options, scopes, env, package_env)
# Not check for updates for info command, it'll be checked when dep graph is built
remote_proxy = ConanProxy(self._client_cache,
self._user_io,
self._remote_manager,
remote,
update=update,
check_updates=check_updates,
manifest_manager=manifest_manager)
if isinstance(reference, ConanFileReference):
project_reference = None
conanfile = loader.load_virtual(reference, current_path)
is_txt = True
else:
conanfile_path = reference
project_reference = "PROJECT"
output = ScopedOutput(project_reference, self._user_io.out)
try:
if filename and filename.endswith(".txt"):
raise NotFoundException("")
conan_file_path = os.path.join(conanfile_path, filename
or CONANFILE)
conanfile = loader.load_conan(conan_file_path,
output,
consumer=True)
is_txt = False
if conanfile.name is not None and conanfile.version is not None:
project_reference = "%s/%s@" % (conanfile.name,
conanfile.version)
project_reference += "PROJECT"
except NotFoundException: # Load requirements.txt
conan_path = os.path.join(conanfile_path, filename
or CONANFILE_TXT)
conanfile = loader.load_conan_txt(conan_path, output)
is_txt = True
# build deps graph and install it
local_search = None if update else self._search_manager
resolver = RequireResolver(self._user_io.out, local_search,
remote_proxy)
builder = DepsGraphBuilder(remote_proxy, self._user_io.out, loader,
resolver)
deps_graph = builder.load(None, conanfile)
# These lines are so the conaninfo stores the correct complete info
if is_txt:
conanfile.info.settings = loader._settings.values
conanfile.info.full_settings = loader._settings.values
conanfile.info.scope = self._current_scopes
conanfile.cpp_info = CppInfo(current_path)
conanfile.env_info = EnvInfo(current_path)
registry = RemoteRegistry(self._client_cache.registry,
self._user_io.out)
return (builder, deps_graph, project_reference, registry, conanfile,
remote_proxy, loader)
def insert_test(self):
f = os.path.join(temp_folder(), "aux_file")
save(f, """
{
"remotes": [
{
"url": "https://server.conan.io",
"verify_ssl": true,
"name": "conan.io"
}
],
"references": {}
}
""")
registry = RemoteRegistry(f, TestBufferConanOutput())
registry.remotes.add("repo1", "url1", True, insert=0)
self.assertEqual(registry.remotes.list, [("repo1", "url1", True),
("conan.io", "https://server.conan.io", True)])
registry.remotes.add("repo2", "url2", True, insert=1)
self.assertEqual(registry.remotes.list, [("repo1", "url1", True),
("repo2", "url2", True),
("conan.io", "https://server.conan.io", True)])
registry.remotes.add("repo3", "url3", True, insert=5)
self.assertEqual(registry.remotes.list, [("repo1", "url1", True),
("repo2", "url2", True),
("conan.io", "https://server.conan.io", True),
("repo3", "url3", True)])
def search_packages(self, reference=None, remote=None, query=None, outdated=False):
""" Return the single information saved in conan.vars about all the packages
or the packages which match with a pattern
Attributes:
pattern = string to match packages
remote = search on another origin to get packages info
packages_pattern = String query with binary
packages properties: "arch=x86 AND os=Windows"
"""
if remote:
remote = RemoteRegistry(self._client_cache.registry, self._user_io).remote(remote)
packages_props = self._remote_manager.search_packages(remote, reference, query)
ordered_packages = OrderedDict(sorted(packages_props.items()))
manifest = self._remote_manager.get_conan_digest(reference, remote)
recipe_hash = manifest.summary_hash
else:
searcher = DiskSearchManager(self._client_cache)
packages_props = searcher.search_packages(reference, query)
ordered_packages = OrderedDict(sorted(packages_props.items()))
try:
recipe_hash = self._client_cache.load_manifest(reference).summary_hash
except IOError: # It could not exist in local
recipe_hash = None
if outdated and recipe_hash:
ordered_packages = filter_outdated(ordered_packages, recipe_hash)
return ordered_packages, reference, recipe_hash, query
def retro_compatibility_test(self):
f = os.path.join(temp_folder(), "aux_file")
save(f, """conan.io https://server.conan.io
""") # Without SSL parameter
registry = RemoteRegistry(f, TestBufferConanOutput())
self.assertEqual(registry.remotes,
[("conan.io", "https://server.conan.io", True)])
def _check(self, install_path):
settings_path = self.client.client_cache.settings_path
self.assertEqual(load(settings_path).splitlines(), settings_yml.splitlines())
registry_path = self.client.client_cache.registry
registry = RemoteRegistry(registry_path, TestBufferConanOutput())
self.assertEqual(registry.remotes,
[Remote("myrepo1", "https://myrepourl.net", False),
Remote("my-repo-2", "https://myrepo2.com", True),
])
self.assertEqual(registry.refs, {"MyPkg/0.1@user/channel": "my-repo-2"})
self.assertEqual(sorted(os.listdir(self.client.client_cache.profiles_path)),
sorted(["default", "linux", "windows"]))
self.assertEqual(load(os.path.join(self.client.client_cache.profiles_path, "linux")).splitlines(),
linux_profile.splitlines())
self.assertEqual(load(os.path.join(self.client.client_cache.profiles_path, "windows")).splitlines(),
win_profile.splitlines())
conan_conf = ConanClientConfigParser(self.client.client_cache.conan_conf_path)
self.assertEqual(conan_conf.get_item("log.run_to_output"), "False")
self.assertEqual(conan_conf.get_item("log.run_to_file"), "False")
self.assertEqual(conan_conf.get_item("log.level"), "10")
self.assertEqual(conan_conf.get_item("general.compression_level"), "6")
self.assertEqual(conan_conf.get_item("general.sysrequires_sudo"), "True")
self.assertEqual(conan_conf.get_item("general.cpu_count"), "1")
self.assertEqual(conan_conf.get_item("general.config_install"), install_path)
self.assertEqual(conan_conf.get_item("proxies.no_proxy"), "mylocalhost")
self.assertEqual(conan_conf.get_item("proxies.https"), "None")
self.assertEqual(conan_conf.get_item("proxies.http"), "http://*****:*****@10.10.1.10:3128/")
self.assertEqual("#Custom pylint",
load(os.path.join(self.client.client_cache.conan_folder, "pylintrc")))
def _install(self, references):
self._output.info("Installing build requires: [%s]" %
", ".join(str(r) for r in references))
conanfile = self._loader.load_virtual(
references, None, scope_options=False) # No need current path
# FIXME: Forced update=True, build_mode, Where to define it?
update = False
local_search = None if update else self._search_manager
resolver = RequireResolver(self._output, local_search,
self._remote_proxy)
graph_builder = DepsGraphBuilder(self._remote_proxy, self._output,
self._loader, resolver)
deps_graph = graph_builder.load(conanfile)
registry = RemoteRegistry(self._client_cache.registry, self._output)
Printer(self._output).print_graph(deps_graph, registry)
# Make sure we recursively do not propagate the "*" pattern
build_requires = copy.copy(self)
build_requires._build_requires = self._build_requires.copy()
build_requires._build_requires.pop("*", None)
build_requires._build_requires.pop("&!", None)
installer = ConanInstaller(self._client_cache, self._output,
self._remote_proxy, build_requires)
installer.install(deps_graph, self._build_modes, self._current_path)
self._output.info("Installed build requires: [%s]" %
", ".join(str(r) for r in references))
return deps_graph
def __init__(self, client_cache, user_io, runner, remote_manager, interactive=True):
assert isinstance(user_io, UserIO)
assert isinstance(client_cache, ClientCache)
self._client_cache = client_cache
self._user_io = user_io
self._runner = runner
self._remote_manager = remote_manager
self._registry = RemoteRegistry(self._client_cache.registry, self._user_io.out)
if not interactive:
self._user_io.disable_input()
self._proxy = ConanProxy(client_cache, self._user_io.out, remote_manager, registry=self._registry)
resolver = RangeResolver(self._user_io.out, client_cache, self._proxy)
python_requires = ConanPythonRequire(self._proxy, resolver)
self._loader = ConanFileLoader(self._runner, self._user_io.out, python_requires)
self._graph_manager = GraphManager(self._user_io.out, self._client_cache, self._registry,
self._remote_manager, self._loader, self._proxy, resolver)
def _get_registry():
f = os.path.join(temp_folder(), "aux_file")
remotes, refs = load_registry_txt(
"conan.io https://server.conan.io True\n"
"conan.io2 https://server2.conan.io True\n")
reg = dump_registry(remotes, refs, {})
save(f, reg)
return RemoteRegistry(f, TestBufferConanOutput())
def users_list(client_cache, output, remote_name=None):
# List all users from required remotes
registry = RemoteRegistry(client_cache.registry, output)
if remote_name:
remotes = [registry.remote(remote_name)]
else:
remotes = registry.remotes
if not remotes:
raise ConanException("No remotes defined")
localdb = LocalDB(client_cache.localdb)
result = []
for remote in remotes:
prev_user = localdb.get_username(remote.url)
username = prev_user or "None (anonymous)"
result.append((remote.name, username))
return result
def __init__(self, paths, user_io, remote_manager, remote_name,
update=False, check_updates=True):
self._paths = paths
self._out = user_io.out
self._remote_manager = remote_manager
self._registry = RemoteRegistry(self._paths.registry, self._out)
self._remote_name = remote_name
self._update = update
self._check_updates = check_updates
def info(self, deps_graph, only, package_filter, show_paths):
registry = RemoteRegistry(self.client_cache.registry, self.user_io.out)
Printer(self.user_io.out).print_info(
deps_graph,
only,
registry,
node_times=self._read_dates(deps_graph),
path_resolver=self.client_cache,
package_filter=package_filter,
show_paths=show_paths)
def __init__(self, client_cache, user_io, remote_manager, remote_name,
update=False, check_updates=False, manifest_manager=False):
self._client_cache = client_cache
self._out = user_io.out
self._remote_manager = remote_manager
self._registry = RemoteRegistry(self._client_cache.registry, self._out)
self._remote_name = remote_name
self._update = update
self._check_updates = check_updates or update # Update forces check (and of course the update)
self._manifest_manager = manifest_manager
def refs_test(self):
f = os.path.join(temp_folder(), "aux_file")
registry = RemoteRegistry(f, TestBufferConanOutput())
ref = ConanFileReference.loads("MyLib/0.1@lasote/stable")
remotes = registry.remotes
registry.set_ref(ref, remotes[0])
remote = registry.get_ref(ref)
self.assertEqual(remote, remotes[0])
registry.set_ref(ref, remotes[0])
remote = registry.get_ref(ref)
self.assertEqual(remote, remotes[0])
def search_recipes(self, pattern, remote=None, case_sensitive=False):
ignorecase = not case_sensitive
if not remote:
return DiskSearchManager(self._client_cache).search_recipes(pattern, ignorecase)
registry = RemoteRegistry(self._client_cache.registry, self._user_io.out)
if remote == 'all':
remotes = registry.remotes
# We have to check if there is a remote called "all"
# Deprecate: 2.0 can remove this check
if 'all' not in (r.name for r in remotes):
references = {}
for remote in remotes:
result = self._remote_manager.search_recipes(remote, pattern, ignorecase)
if result:
references[remote.name] = result
return references
# single remote
remote = registry.remote(remote)
return self._remote_manager.search_recipes(remote, pattern, ignorecase)
def __init__(self,
base_folder=None,
current_folder=None,
servers=None,
users=None,
client_version=CLIENT_VERSION,
min_server_compatible_version=MIN_SERVER_COMPATIBLE_VERSION,
requester_class=None,
runner=None,
path_with_spaces=True):
"""
storage_folder: Local storage path
current_folder: Current execution folder
servers: dict of {remote_name: TestServer}
logins is a list of (user, password) for auto input in order
if required==> [("lasote", "mypass"), ("other", "otherpass")]
"""
self.all_output = "" # For debugging purpose, append all the run outputs
self.users = users or {
"default":
[(TESTING_REMOTE_PRIVATE_USER, TESTING_REMOTE_PRIVATE_PASS)]
}
self.servers = servers or {}
self.client_version = Version(str(client_version))
self.min_server_compatible_version = Version(
str(min_server_compatible_version))
self.base_folder = base_folder or temp_folder(path_with_spaces)
# Define storage_folder, if not, it will be read from conf file & pointed to real user home
self.storage_folder = os.path.join(self.base_folder, ".conan", "data")
self.client_cache = ClientCache(self.base_folder, self.storage_folder,
TestBufferConanOutput())
search_adapter = DiskSearchAdapter()
self.search_manager = DiskSearchManager(self.client_cache,
search_adapter)
self._default_settings(get_env("CONAN_COMPILER", "gcc"),
get_env("CONAN_COMPILER_VERSION", "4.8"),
get_env("CONAN_LIBCXX", "libstdc++"))
self.requester_class = requester_class
self.conan_runner = runner
self.init_dynamic_vars()
save(self.client_cache.registry, "")
registry = RemoteRegistry(self.client_cache.registry,
TestBufferConanOutput())
for name, server in self.servers.items():
if isinstance(server, TestServer):
registry.add(name, server.fake_url)
else:
registry.add(name, server)
logger.debug("Client storage = %s" % self.storage_folder)
self.current_folder = current_folder or temp_folder(path_with_spaces)
def refs_test(self):
f = os.path.join(temp_folder(), "aux_file")
save(f, dump_registry(default_remotes, {}, {}))
registry = RemoteRegistry(f, TestBufferConanOutput())
ref = ConanFileReference.loads("MyLib/0.1@lasote/stable")
remotes = registry.remotes.list
registry.refs.set(ref, remotes[0].name)
remote = registry.refs.get(ref)
self.assertEqual(remote, remotes[0])
registry.refs.set(ref, remotes[0].name)
remote = registry.refs.get(ref)
self.assertEqual(remote, remotes[0])
def update_servers(self, servers):
self.servers = servers or {}
save(self.client_cache.registry, "")
registry = RemoteRegistry(self.client_cache.registry, TestBufferConanOutput())
for name, server in self.servers.items():
if isinstance(server, TestServer):
registry.add(name, server.fake_url)
else:
registry.add(name, server)
def refs_test(self):
f = os.path.join(temp_folder(), "aux_file")
registry = RemoteRegistry(f, TestBufferConanOutput())
ref = ConanFileReference.loads("MyLib/0.1@lasote/stable")
remotes = registry.remotes
registry.set_ref(ref, remotes[0])
remote = registry.get_ref(ref)
self.assertEqual(remote, remotes[0])
registry.set_ref(ref, remotes[0])
remote = registry.get_ref(ref)
self.assertEqual(remote, remotes[0])
def insert_test(self):
f = os.path.join(temp_folder(), "aux_file")
save(f, "conan.io https://server.conan.io True")
registry = RemoteRegistry(f, TestBufferConanOutput())
registry.add("repo1", "url1", True, insert=0)
self.assertEqual(registry.remotes,
[("repo1", "url1", True),
("conan.io", "https://server.conan.io", True)])
registry.add("repo2", "url2", True, insert=1)
self.assertEqual(registry.remotes,
[("repo1", "url1", True), ("repo2", "url2", True),
("conan.io", "https://server.conan.io", True)])
registry.add("repo3", "url3", True, insert=5)
self.assertEqual(registry.remotes,
[("repo1", "url1", True), ("repo2", "url2", True),
("conan.io", "https://server.conan.io", True),
("repo3", "url3", True)])
def to_json_migration_test(self):
tmp = temp_folder()
conf_dir = os.path.join(tmp, ".conan")
f = os.path.join(conf_dir, "registry.txt")
save(f, """conan.io https://server.conan.io True
lib/1.0@conan/stable conan.io
other/1.0@lasote/testing conan.io
""")
client = TestClient(base_folder=tmp, servers=False)
new_path = client.client_cache.registry
registry = RemoteRegistry(new_path, TestBufferConanOutput())
self.assertEqual(registry.remotes.list, [("conan.io", "https://server.conan.io", True)])
self.assertEqual(registry.refs.list, {'lib/1.0@conan/stable': 'conan.io',
'other/1.0@lasote/testing': 'conan.io'})
def insert_test(self):
f = os.path.join(temp_folder(), "aux_file")
save(f, "conan.io https://server.conan.io True")
registry = RemoteRegistry(f, TestBufferConanOutput())
registry.add("repo1", "url1", True, insert=0)
self.assertEqual(registry.remotes, [("repo1", "url1", True),
("conan.io", "https://server.conan.io", True)])
registry.add("repo2", "url2", True, insert=1)
self.assertEqual(registry.remotes, [("repo1", "url1", True),
("repo2", "url2", True),
("conan.io", "https://server.conan.io", True)])
registry.add("repo3", "url3", True, insert=5)
self.assertEqual(registry.remotes, [("repo1", "url1", True),
("repo2", "url2", True),
("conan.io", "https://server.conan.io", True),
("repo3", "url3", True)])
def update_servers(self):
save(self.client_cache.registry, dump_registry({}, {}, {}))
registry = RemoteRegistry(self.client_cache.registry,
TestBufferConanOutput())
def add_server_to_registry(name, server):
if isinstance(server, TestServer):
registry.remotes.add(name, server.fake_url)
else:
registry.remotes.add(name, server)
for name, server in self.servers.items():
if name == "default":
add_server_to_registry(name, server)
for name, server in self.servers.items():
if name != "default":
add_server_to_registry(name, server)
def remove_all_package_test(self):
f = os.path.join(temp_folder(), "aux_file")
save(f, dump_registry(default_remotes, {}, {}))
registry = RemoteRegistry(f, TestBufferConanOutput())
registry.remotes.add("r1", "url1", True, insert=0)
registry.remotes.add("r2", "url2", True, insert=0)
ref = ConanFileReference.loads("MyLib/0.1@lasote/stable")
ref2 = ConanFileReference.loads("MyLib2/0.1@lasote/stable")
registry.prefs.set(PackageReference(ref, "1"), "r1")
registry.prefs.set(PackageReference(ref, "2"), "r1")
registry.prefs.set(PackageReference(ref, "3"), "r1")
registry.prefs.set(PackageReference(ref, "4"), "r2")
registry.prefs.set(PackageReference(ref2, "1"), "r1")
registry.prefs.remove_all(ref)
self.assertIsNone(registry.prefs.get(PackageReference(ref, "1")))
self.assertIsNone(registry.prefs.get(PackageReference(ref, "2")))
self.assertIsNone(registry.prefs.get(PackageReference(ref, "3")))
self.assertIsNone(registry.prefs.get(PackageReference(ref, "4")))
self.assertEquals(registry.prefs.get(PackageReference(ref2, "1")).name, "r1")
registry.prefs.set(PackageReference(ref, "1"), "r1")
registry.prefs.set(PackageReference(ref, "2"), "r1")
registry.prefs.set(PackageReference(ref, "3"), "r1")
registry.prefs.set(PackageReference(ref, "4"), "r2")
registry.prefs.set(PackageReference(ref2, "1"), "r1")
registry.prefs.remove_all(ref, "r1")
self.assertIsNone(registry.prefs.get(PackageReference(ref, "1")))
self.assertIsNone(registry.prefs.get(PackageReference(ref, "2")))
self.assertIsNone(registry.prefs.get(PackageReference(ref, "3")))
self.assertEquals(registry.prefs.get(PackageReference(ref, "4")).name, "r2")
self.assertEquals(registry.prefs.get(PackageReference(ref2, "1")).name, "r1")
def add_remove_update_test(self):
f = os.path.join(temp_folder(), "aux_file")
registry = RemoteRegistry(f, TestBufferConanOutput())
# Add
registry.add("local", "http://localhost:9300")
self.assertEqual(registry.remotes, [("conan-center", "https://conan.bintray.com", True),
("conan-transit", "https://conan-transit.bintray.com", True),
("local", "http://localhost:9300", True)])
# Add
registry.add("new", "new_url", False)
self.assertEqual(registry.remotes, [("conan-center", "https://conan.bintray.com", True),
("conan-transit", "https://conan-transit.bintray.com", True),
("local", "http://localhost:9300", True),
("new", "new_url", False)])
with self.assertRaises(ConanException):
registry.add("new", "new_url")
# Update
registry.update("new", "other_url")
self.assertEqual(registry.remotes, [("conan-center", "https://conan.bintray.com", True),
("conan-transit", "https://conan-transit.bintray.com", True),
("local", "http://localhost:9300", True),
("new", "other_url", True)])
with self.assertRaises(ConanException):
registry.update("new2", "new_url")
registry.update("new", "other_url", False)
self.assertEqual(registry.remotes, [("conan-center", "https://conan.bintray.com", True),
("conan-transit", "https://conan-transit.bintray.com", True),
("local", "http://localhost:9300", True),
("new", "other_url", False)])
# Remove
registry.remove("local")
self.assertEqual(registry.remotes, [("conan-center", "https://conan.bintray.com", True),
("conan-transit", "https://conan-transit.bintray.com", True),
("new", "other_url", False)])
with self.assertRaises(ConanException):
registry.remove("new2")
def remote_update_ref(self, reference, remote):
registry = RemoteRegistry(self._client_cache.registry, self._user_io.out)
return registry.update_ref(reference, remote)
def remote(self, *args):
""" manage remotes
"""
parser = argparse.ArgumentParser(description=self.remote.__doc__, prog="conan remote")
subparsers = parser.add_subparsers(dest='subcommand', help='sub-command help')
# create the parser for the "a" command
subparsers.add_parser('list', help='list current remotes')
parser_add = subparsers.add_parser('add', help='add a remote')
parser_add.add_argument('remote', help='name of the remote')
parser_add.add_argument('url', help='url of the remote')
parser_rm = subparsers.add_parser('remove', help='remove a remote')
parser_rm.add_argument('remote', help='name of the remote')
parser_upd = subparsers.add_parser('update', help='update the remote url')
parser_upd.add_argument('remote', help='name of the remote')
parser_upd.add_argument('url', help='url')
subparsers.add_parser('list_ref', help='list the package recipes and its associated remotes')
parser_padd = subparsers.add_parser('add_ref', help="associate a recipe's reference to a remote")
parser_padd.add_argument('reference', help='package recipe reference')
parser_padd.add_argument('remote', help='name of the remote')
parser_prm = subparsers.add_parser('remove_ref', help="dissociate a recipe's reference and its remote")
parser_prm.add_argument('reference', help='package recipe reference')
parser_pupd = subparsers.add_parser('update_ref', help="update the remote associated with a package recipe")
parser_pupd.add_argument('reference', help='package recipe reference')
parser_pupd.add_argument('remote', help='name of the remote')
args = parser.parse_args(*args)
registry = RemoteRegistry(self._conan_paths.registry, self._user_io.out)
if args.subcommand == "list":
for r in registry.remotes:
self._user_io.out.info("%s: %s" % (r.name, r.url))
elif args.subcommand == "add":
registry.add(args.remote, args.url)
elif args.subcommand == "remove":
registry.remove(args.remote)
elif args.subcommand == "update":
registry.update(args.remote, args.url)
elif args.subcommand == "list_ref":
for ref, remote in registry.refs.items():
self._user_io.out.info("%s: %s" % (ref, remote))
elif args.subcommand == "add_ref":
registry.add_ref(args.reference, args.remote)
elif args.subcommand == "remove_ref":
registry.remove_ref(args.reference)
elif args.subcommand == "update_ref":
registry.update_ref(args.reference, args.remote)
def remote(self, *args):
""" manage remotes
"""
parser = argparse.ArgumentParser(description=self.remote.__doc__, prog="conan remote")
subparsers = parser.add_subparsers(dest='subcommand', help='sub-command help')
# create the parser for the "a" command
subparsers.add_parser('list', help='list current remotes')
parser_add = subparsers.add_parser('add', help='add a remote')
parser_add.add_argument('remote', help='name of the remote')
parser_add.add_argument('url', help='url of the remote')
parser_add.add_argument('verify_ssl', help='Verify SSL certificated. Default True',
default="True", nargs="?")
parser_rm = subparsers.add_parser('remove', help='remove a remote')
parser_rm.add_argument('remote', help='name of the remote')
parser_upd = subparsers.add_parser('update', help='update the remote url')
parser_upd.add_argument('remote', help='name of the remote')
parser_upd.add_argument('url', help='url')
parser_upd.add_argument('verify_ssl', help='Verify SSL certificated. Default True',
default="True", nargs="?")
subparsers.add_parser('list_ref',
help='list the package recipes and its associated remotes')
parser_padd = subparsers.add_parser('add_ref',
help="associate a recipe's reference to a remote")
parser_padd.add_argument('reference', help='package recipe reference')
parser_padd.add_argument('remote', help='name of the remote')
parser_prm = subparsers.add_parser('remove_ref',
help="dissociate a recipe's reference and its remote")
parser_prm.add_argument('reference', help='package recipe reference')
parser_pupd = subparsers.add_parser('update_ref', help="update the remote associated "
"with a package recipe")
parser_pupd.add_argument('reference', help='package recipe reference')
parser_pupd.add_argument('remote', help='name of the remote')
args = parser.parse_args(*args)
registry = RemoteRegistry(self._client_cache.registry, self._user_io.out)
if args.subcommand == "list":
for r in registry.remotes:
self._user_io.out.info("%s: %s [Verify SSL: %s]" % (r.name, r.url, r.verify_ssl))
elif args.subcommand == "add":
verify = get_bool_from_text_value(args.verify_ssl)
registry.add(args.remote, args.url, args.verify_ssl)
elif args.subcommand == "remove":
registry.remove(args.remote)
elif args.subcommand == "update":
verify = get_bool_from_text_value(args.verify_ssl)
registry.update(args.remote, args.url, verify)
elif args.subcommand == "list_ref":
for ref, remote in registry.refs.items():
self._user_io.out.info("%s: %s" % (ref, remote))
elif args.subcommand == "add_ref":
registry.add_ref(args.reference, args.remote)
elif args.subcommand == "remove_ref":
registry.remove_ref(args.reference)
elif args.subcommand == "update_ref":
registry.update_ref(args.reference, args.remote)
class ConanProxy(object):
""" Class to access the conan storage, to perform typical tasks as to get packages,
getting conanfiles, uploading, removing from remote, etc.
It uses the RemoteRegistry to control where the packages come from.
"""
def __init__(self, client_cache, user_io, remote_manager, remote_name,
update=False, check_updates=False, manifest_manager=False):
self._client_cache = client_cache
self._out = user_io.out
self._remote_manager = remote_manager
self._registry = RemoteRegistry(self._client_cache.registry, self._out)
self._remote_name = remote_name
self._update = update
self._check_updates = check_updates or update # Update forces check (and of course the update)
self._manifest_manager = manifest_manager
@property
def registry(self):
return self._registry
def package_available(self, package_ref, short_paths, check_outdated):
"""
Returns True if there is a local or remote package available (and up to date if check_outdated).
It wont download the package, just check its hash
"""
output = ScopedOutput(str(package_ref.conan), self._out)
package_folder = self._client_cache.package(package_ref, short_paths=short_paths)
remote_info = None
# No package in local cache
if not os.path.exists(package_folder):
try:
remote_info = self.get_package_info(package_ref)
except ConanException:
return False # Not local nor remote
# Maybe we have the package (locally or in remote) but it's outdated
if check_outdated:
if remote_info:
package_hash = remote_info.recipe_hash
else:
package_hash = self._client_cache.read_package_recipe_hash(package_folder)
local_recipe_hash = self._client_cache.load_manifest(package_ref.conan).summary_hash
up_to_date = local_recipe_hash == package_hash
if not up_to_date:
output.info("Outdated package!")
else:
output.info("Package is up to date")
return up_to_date
return True
def get_package(self, package_ref, short_paths):
""" obtain a package, either from disk or retrieve from remotes if necessary
and not necessary to build
"""
output = ScopedOutput(str(package_ref.conan), self._out)
package_folder = self._client_cache.package(package_ref, short_paths=short_paths)
# Check current package status
if os.path.exists(package_folder):
if self._check_updates:
read_manifest = self._client_cache.load_package_manifest(package_ref)
try: # get_conan_digest can fail, not in server
upstream_manifest = self.get_package_digest(package_ref)
if upstream_manifest != read_manifest:
if upstream_manifest.time > read_manifest.time:
output.warn("Current package is older than remote upstream one")
if self._update:
output.warn("Removing it to retrieve or build an updated one")
rmdir(package_folder)
else:
output.warn("Current package is newer than remote upstream one")
except ConanException:
pass
installed = False
local_package = os.path.exists(package_folder)
if local_package:
output.info('Already installed!')
installed = True
log_package_got_from_local_cache(package_ref)
else:
installed = self._retrieve_remote_package(package_ref, package_folder,
output)
self.handle_package_manifest(package_ref, installed)
return installed
def handle_package_manifest(self, package_ref, installed):
if installed and self._manifest_manager:
remote = self._registry.get_ref(package_ref.conan)
self._manifest_manager.check_package(package_ref, remote)
def get_recipe_sources(self, conan_reference):
export_path = self._client_cache.export(conan_reference)
sources_folder = os.path.join(export_path, EXPORT_SOURCES_DIR)
if os.path.exists(sources_folder):
return
current_remote = self._registry.get_ref(conan_reference)
if not current_remote:
raise ConanException("Error while trying to get recipe sources for %s. "
"No remote defined" % str(conan_reference))
else:
self._remote_manager.get_recipe_sources(conan_reference, export_path, current_remote)
def get_recipe(self, conan_reference):
output = ScopedOutput(str(conan_reference), self._out)
def _refresh():
export_path = self._client_cache.export(conan_reference)
rmdir(export_path)
# It might need to remove shortpath
rm_conandir(self._client_cache.source(conan_reference))
current_remote, _ = self._get_remote(conan_reference)
output.info("Retrieving from remote '%s'..." % current_remote.name)
self._remote_manager.get_recipe(conan_reference, export_path, current_remote)
if self._update:
output.info("Updated!")
else:
output.info("Installed!")
# check if it is in disk
conanfile_path = self._client_cache.conanfile(conan_reference)
if os.path.exists(conanfile_path):
log_recipe_got_from_local_cache(conan_reference)
if self._check_updates:
ret = self.update_available(conan_reference)
if ret != 0: # Found and not equal
remote, ref_remote = self._get_remote(conan_reference)
if ret == 1:
if not self._update:
if remote != ref_remote: # Forced new remote
output.warn("There is a new conanfile in '%s' remote. "
"Execute 'install -u -r %s' to update it."
% (remote.name, remote.name))
else:
output.warn("There is a new conanfile in '%s' remote. "
"Execute 'install -u' to update it."
% remote.name)
output.warn("Refused to install!")
else:
if remote != ref_remote:
# Delete packages, could be non coherent with new remote
DiskRemover(self._client_cache).remove_packages(conan_reference)
_refresh()
elif ret == -1:
if not self._update:
output.info("Current conanfile is newer "
"than %s's one" % remote.name)
else:
output.error("Current conanfile is newer than %s's one. "
"Run 'conan remove %s' and run install again "
"to replace it." % (remote.name, conan_reference))
else:
self._retrieve_recipe(conan_reference, output)
if self._manifest_manager:
# Just make sure that the recipe sources are there to check
self.get_recipe_sources(conan_reference)
remote = self._registry.get_ref(conan_reference)
self._manifest_manager.check_recipe(conan_reference, remote)
return conanfile_path
def update_available(self, conan_reference):
"""Returns 0 if the conanfiles are equal, 1 if there is an update and -1 if
the local is newer than the remote"""
if not conan_reference:
return 0
read_manifest, _ = self._client_cache.conan_manifests(conan_reference)
if read_manifest:
try: # get_conan_digest can fail, not in server
upstream_manifest = self.get_conan_digest(conan_reference)
if upstream_manifest != read_manifest:
return 1 if upstream_manifest.time > read_manifest.time else -1
except ConanException:
pass
return 0
def _retrieve_recipe(self, conan_reference, output):
""" returns the requested conanfile object, retrieving it from
remotes if necessary. Can raise NotFoundException
"""
def _retrieve_from_remote(remote):
output.info("Trying with '%s'..." % remote.name)
export_path = self._client_cache.export(conan_reference)
result = self._remote_manager.get_recipe(conan_reference, export_path, remote)
self._registry.set_ref(conan_reference, remote)
return result
if self._remote_name:
output.info("Not found, retrieving from server '%s' " % self._remote_name)
remote = self._registry.remote(self._remote_name)
return _retrieve_from_remote(remote)
else:
output.info("Not found, looking in remotes...")
remotes = self._registry.remotes
for remote in remotes:
logger.debug("Trying with remote %s" % remote.name)
try:
return _retrieve_from_remote(remote)
# If exception continue with the next
except (ConanOutdatedClient, ConanConnectionError) as exc:
output.warn(str(exc))
if remote == remotes[-1]: # Last element not found
raise ConanConnectionError("All remotes failed")
except NotFoundException as exc:
if remote == remotes[-1]: # Last element not found
logger.debug("Not found in any remote, raising...%s" % exc)
raise NotFoundException("Unable to find '%s' in remotes"
% str(conan_reference))
raise ConanException("No remote defined")
def complete_recipe_sources(self, conan_reference, force_complete=True):
export_path = self._client_cache.export(conan_reference)
sources_folder = os.path.join(export_path, EXPORT_SOURCES_DIR)
ignore_deleted_file = None
if not os.path.exists(sources_folder):
# If not path to sources exists, we have a problem, at least an empty folder
# should be there
upload_remote, current_remote = self._get_remote(conan_reference)
if not current_remote:
raise ConanException("Trying to upload a package recipe without sources, "
"and the remote for the sources no longer exists")
if force_complete or current_remote != upload_remote:
# If uploading to a different remote than the one from which the recipe
# was retrieved, we definitely need to get the sources, so the recipe is complete
self.get_recipe_sources(conan_reference)
else:
# But if same remote, no need to upload again the TGZ, it is already in the server
# But the upload API needs to know it to not remove the server file.
ignore_deleted_file = EXPORT_SOURCES_TGZ_NAME
return ignore_deleted_file
def upload_recipe(self, conan_reference, retry, retry_wait, skip_upload):
""" upload to defined remote in (-r=remote), to current remote
or to default remote, in that order.
If the remote is not set, set it
"""
ignore_deleted_file = self.complete_recipe_sources(conan_reference, force_complete=False)
remote, ref_remote = self._get_remote(conan_reference)
result = self._remote_manager.upload_recipe(conan_reference, remote, retry, retry_wait,
ignore_deleted_file=ignore_deleted_file,
skip_upload=skip_upload)
if not ref_remote and not skip_upload:
self._registry.set_ref(conan_reference, remote)
return result
def _get_remote(self, conan_ref=None):
# Prioritize -r , then reference registry and then the default remote
ref_remote = self._registry.get_ref(conan_ref) if conan_ref else None
if self._remote_name:
remote = self._registry.remote(self._remote_name)
else:
if ref_remote:
remote = ref_remote
else:
remote = self._registry.default_remote
return remote, ref_remote
def upload_package(self, package_ref, retry, retry_wait, skip_upload):
remote, current_remote = self._get_remote(package_ref.conan)
if not current_remote:
self._out.warn("Remote for '%s' not defined, uploading to %s"
% (str(package_ref.conan), remote.name))
result = self._remote_manager.upload_package(package_ref, remote, retry, retry_wait, skip_upload)
if not current_remote and not skip_upload:
self._registry.set_ref(package_ref.conan, remote)
return result
def get_conan_digest(self, conan_ref):
""" used by update to check the date of packages, require force if older
"""
remote, current_remote = self._get_remote(conan_ref)
result = self._remote_manager.get_conan_digest(conan_ref, remote)
if not current_remote:
self._registry.set_ref(conan_ref, remote)
return result
def get_package_digest(self, package_ref):
""" used by update to check the date of packages, require force if older
"""
remote, ref_remote = self._get_remote(package_ref.conan)
result = self._remote_manager.get_package_digest(package_ref, remote)
if not ref_remote:
self._registry.set_ref(package_ref.conan, remote)
return result
def get_package_info(self, package_ref):
""" Gets the package info to check if outdated
"""
remote, ref_remote = self._get_remote(package_ref.conan)
result = self._remote_manager.get_package_info(package_ref, remote)
if not ref_remote:
self._registry.set_ref(package_ref.conan, remote)
return result
def search(self, pattern=None, ignorecase=True):
remote, _ = self._get_remote()
return self._remote_manager.search(remote, pattern, ignorecase)
def search_remotes(self, pattern=None, ignorecase=True):
if self._remote_name:
remote = self._registry.remote(self._remote_name)
search_result = self._remote_manager.search(remote, pattern, ignorecase)
return search_result
for remote in self._registry.remotes:
search_result = self._remote_manager.search(remote, pattern, ignorecase)
if search_result:
return search_result
def search_packages(self, reference, query):
remote, _ = self._get_remote()
return self._remote_manager.search_packages(remote, reference, query)
def remove(self, conan_ref):
if not self._remote_name:
raise ConanException("Cannot remove, remote not defined")
remote = self._registry.remote(self._remote_name)
result = self._remote_manager.remove(conan_ref, remote)
current_remote = self._registry.get_ref(conan_ref)
if current_remote == remote:
self._registry.remove_ref(conan_ref)
return result
def remove_packages(self, conan_ref, remove_ids):
if not self._remote_name:
raise ConanException("Cannot remove, remote not defined")
remote = self._registry.remote(self._remote_name)
return self._remote_manager.remove_packages(conan_ref, remove_ids, remote)
def download_packages(self, reference, package_ids):
assert(isinstance(package_ids, list))
remote, _ = self._get_remote(reference)
export_path = self._client_cache.export(reference)
self._remote_manager.get_recipe(reference, export_path, remote)
conanfile_path = self._client_cache.conanfile(reference)
conanfile = load_conanfile_class(conanfile_path)
short_paths = conanfile.short_paths
self._registry.set_ref(reference, remote)
output = ScopedOutput(str(reference), self._out)
for package_id in package_ids:
package_ref = PackageReference(reference, package_id)
package_folder = self._client_cache.package(package_ref, short_paths=short_paths)
self._retrieve_remote_package(package_ref, package_folder, output, remote)
def _retrieve_remote_package(self, package_ref, package_folder, output, remote=None):
if remote is None:
remote = self._registry.get_ref(package_ref.conan)
if not remote:
output.warn("Package doesn't have a remote defined. "
"Probably created locally and not uploaded")
return False
package_id = str(package_ref.package_id)
try:
output.info("Looking for package %s in remote '%s' " % (package_id, remote.name))
# Will raise if not found NotFoundException
self._remote_manager.get_package(package_ref, package_folder, remote)
output.success('Package installed %s' % package_id)
return True
except ConanConnectionError:
raise # This shouldn't be skipped
except ConanException as e:
output.warn('Binary for %s not in remote: %s' % (package_id, str(e)))
return False
def authenticate(self, name, password):
if not name: # List all users, from all remotes
remotes = self._registry.remotes
if not remotes:
self._out.error("No remotes defined")
for remote in remotes:
self._remote_manager.authenticate(remote, None, None)
return
remote, _ = self._get_remote()
return self._remote_manager.authenticate(remote, name, password)
def remote_add(self, remote, url, verify_ssl=True, insert=None):
registry = RemoteRegistry(self._client_cache.registry, self._user_io.out)
return registry.add(remote, url, verify_ssl, insert)
def remote_update(self, remote, url, verify_ssl=True):
registry = RemoteRegistry(self._client_cache.registry, self._user_io.out)
return registry.update(remote, url, verify_ssl)
def remote_remove(self, remote):
registry = RemoteRegistry(self._client_cache.registry, self._user_io.out)
return registry.remove(remote)
def _handle_remotes(registry_path, remote_file, output):
registry = RemoteRegistry(registry_path, output)
new_registry = RemoteRegistry(remote_file, output)
registry.define_remotes(new_registry.remotes)
def install(self,
reference,
current_path,
profile,
remote=None,
build_modes=None,
filename=None,
update=False,
manifest_folder=None,
manifest_verify=False,
manifest_interactive=False,
generators=None,
no_imports=False,
inject_require=None):
""" Fetch and build all dependencies for the given reference
@param reference: ConanFileReference or path to user space conanfile
@param current_path: where the output files will be saved
@param remote: install only from that remote
@param profile: Profile object with both the -s introduced options and profile readed values
@param build_modes: List of build_modes specified
@param filename: Optional filename of the conanfile
@param update: Check for updated in the upstream remotes (and update)
@param manifest_folder: Folder to install the manifests
@param manifest_verify: Verify dependencies manifests against stored ones
@param manifest_interactive: Install deps manifests in folder for later verify, asking user for confirmation
@param generators: List of generators from command line
@param no_imports: Install specified packages but avoid running imports
"""
generators = generators or []
manifest_manager = ManifestManager(
manifest_folder,
user_io=self._user_io,
client_cache=self._client_cache,
verify=manifest_verify,
interactive=manifest_interactive) if manifest_folder else None
remote_proxy = ConanProxy(self._client_cache,
self._user_io,
self._remote_manager,
remote,
update=update,
check_updates=False,
manifest_manager=manifest_manager)
loader = ConanFileLoader(self._runner, self._client_cache.settings,
self._profile_with_defaults(profile))
conanfile = self._get_conanfile_object(loader, reference, filename,
current_path)
if inject_require:
self._inject_require(conanfile, inject_require)
graph_builder = self._get_graph_builder(loader, update, remote_proxy)
deps_graph = graph_builder.load(conanfile)
# This line is so the conaninfo stores the correct complete info
conanfile.info.scope = profile.scopes
registry = RemoteRegistry(self._client_cache.registry,
self._user_io.out)
if inject_require:
output = ScopedOutput("%s test package" % str(inject_require),
self._user_io.out)
output.info("Installing dependencies")
elif not isinstance(reference, ConanFileReference):
output = ScopedOutput("PROJECT", self._user_io.out)
Printer(self._user_io.out).print_graph(deps_graph, registry)
else:
output = ScopedOutput(str(reference), self._user_io.out)
output.highlight("Installing package")
try:
if loader._settings.os and detected_os() != loader._settings.os:
message = "Cross-platform from '%s' to '%s'" % (
detected_os(), loader._settings.os)
self._user_io.out.writeln(message, Color.BRIGHT_MAGENTA)
except ConanException: # Setting os doesn't exist
pass
build_mode = BuildMode(build_modes, self._user_io.out)
build_requires = BuildRequires(loader, graph_builder, registry, output,
profile.build_requires)
installer = ConanInstaller(self._client_cache, output, remote_proxy,
build_mode, build_requires)
# Apply build_requires to consumer conanfile
if not isinstance(reference, ConanFileReference):
build_requires.install("", conanfile, installer)
installer.install(deps_graph, current_path)
build_mode.report_matches()
# Write generators
tmp = list(
conanfile.generators) # Add the command line specified generators
tmp.extend([g for g in generators if g not in tmp])
conanfile.generators = tmp
write_generators(conanfile, current_path, output)
if not isinstance(reference, ConanFileReference):
content = normalize(conanfile.info.dumps())
save(os.path.join(current_path, CONANINFO), content)
output.info("Generated %s" % CONANINFO)
if not no_imports:
run_imports(conanfile, current_path, output)
installer.call_system_requirements(conanfile, output)
if manifest_manager:
manifest_manager.print_log()
class ConanProxy(object):
""" Class to access the conan storage, to perform typical tasks as to get packages,
getting conanfiles, uploading, removing from remote, etc.
It uses the RemoteRegistry to control where the packages come from.
"""
def __init__(self, paths, user_io, remote_manager, remote_name,
update=False, check_updates=False, check_integrity=False):
self._paths = paths
self._out = user_io.out
self._remote_manager = remote_manager
self._registry = RemoteRegistry(self._paths.registry, self._out)
self._remote_name = remote_name
self._update = update
self._check_updates = check_updates or update # Update forces check
self._check_integrity = check_integrity
@property
def registry(self):
return self._registry
def get_package(self, package_reference, force_build):
""" obtain a package, either from disk or retrieve from remotes if necessary
and not necessary to build
"""
output = ScopedOutput(str(package_reference.conan), self._out)
package_folder = self._paths.package(package_reference)
# Check current package status
if path_exists(package_folder, self._paths.store):
if self._check_integrity or self._check_updates:
read_manifest, expected_manifest = self._paths.package_manifests(package_reference)
if self._check_integrity: # Check if package is corrupted
if read_manifest.file_sums != expected_manifest.file_sums:
# If not valid package, ensure empty folder
output.warn("Bad package '%s' detected! Removing "
"package directory... " % str(package_reference.package_id))
rmdir(package_folder)
if self._check_updates:
try: # get_conan_digest can fail, not in server
upstream_manifest = self.get_package_digest(package_reference)
if upstream_manifest.file_sums != read_manifest.file_sums:
if upstream_manifest.time > read_manifest.time:
output.warn("Current package is older than remote upstream one")
if self._update:
output.warn("Removing it to retrieve or build an updated one")
rmdir(package_folder)
else:
output.warn("Current package is newer than remote upstream one")
except ConanException:
pass
if not force_build:
local_package = os.path.exists(package_folder)
if local_package:
output = ScopedOutput(str(package_reference.conan), self._out)
output.info('Already installed!')
return True
return self._retrieve_remote_package(package_reference, output)
return False
def get_conanfile(self, conan_reference):
output = ScopedOutput(str(conan_reference), self._out)
def _refresh():
conan_dir_path = self._paths.export(conan_reference)
rmdir(conan_dir_path)
rmdir(self._paths.source(conan_reference), True) # It might need to remove shortpath
current_remote, _ = self._get_remote(conan_reference)
output.info("Retrieving from remote '%s'..." % current_remote.name)
self._remote_manager.get_conanfile(conan_reference, current_remote)
if self._update:
output.info("Updated!")
else:
output.info("Installed!")
# check if it is in disk
conanfile_path = self._paths.conanfile(conan_reference)
path_exist = path_exists(conanfile_path, self._paths.store)
if path_exist:
if self._check_integrity: # Check if package is corrupted
read_manifest, expected_manifest = self._paths.conan_manifests(conan_reference)
if read_manifest.file_sums != expected_manifest.file_sums:
output.warn("Bad conanfile detected! Removing export directory... ")
_refresh()
else: # Check for updates
if self._check_updates:
ret = self.update_available(conan_reference)
if ret != 0: # Found and not equal
remote, ref_remote = self._get_remote(conan_reference)
if ret == 1:
if not self._update:
if remote != ref_remote: # Forced new remote
output.warn("There is a new conanfile in '%s' remote. "
"Execute 'install -u -r %s' to update it."
% (remote.name, remote.name))
else:
output.warn("There is a new conanfile in '%s' remote. "
"Execute 'install -u' to update it."
% remote.name)
output.warn("Refused to install!")
else:
if remote != ref_remote:
# Delete packages, could be non coherent with new remote
rmdir(self._paths.packages(conan_reference))
_refresh()
elif ret == -1:
if not self._update:
output.info("Current conanfile is newer "
"than %s's one" % remote.name)
else:
output.error("Current conanfile is newer than %s's one. "
"Run 'conan remove %s' and run install again "
"to replace it." % (remote.name, conan_reference))
else:
self._retrieve_conanfile(conan_reference, output)
return conanfile_path
def update_available(self, conan_reference):
"""Returns 0 if the conanfiles are equal, 1 if there is an update and -1 if
the local is newer than the remote"""
if not conan_reference:
return 0
read_manifest, _ = self._paths.conan_manifests(conan_reference)
if read_manifest:
try: # get_conan_digest can fail, not in server
upstream_manifest = self.get_conan_digest(conan_reference)
if upstream_manifest.file_sums != read_manifest.file_sums:
return 1 if upstream_manifest.time > read_manifest.time else -1
except ConanException:
pass
return 0
def _retrieve_conanfile(self, conan_reference, output):
""" returns the requested conanfile object, retrieving it from
remotes if necessary. Can raise NotFoundException
"""
def _retrieve_from_remote(remote):
output.info("Trying with '%s'..." % remote.name)
result = self._remote_manager.get_conanfile(conan_reference, remote)
self._registry.set_ref(conan_reference, remote)
return result
if self._remote_name:
output.info("Not found, retrieving from server '%s' " % self._remote_name)
remote = self._registry.remote(self._remote_name)
return _retrieve_from_remote(remote)
else:
output.info("Not found, looking in remotes...")
remotes = self._registry.remotes
for remote in remotes:
logger.debug("Trying with remote %s" % remote.name)
try:
return _retrieve_from_remote(remote)
# If exception continue with the next
except (ConanOutdatedClient, ConanConnectionError) as exc:
output.warn(str(exc))
if remote == remotes[-1]: # Last element not found
raise ConanConnectionError("All remotes failed")
except NotFoundException as exc:
if remote == remotes[-1]: # Last element not found
logger.debug("Not found in any remote, raising...%s" % exc)
raise NotFoundException("Unable to find '%s' in remotes"
% str(conan_reference))
raise ConanException("No remote defined")
def upload_conan(self, conan_reference):
""" upload to defined remote in (-r=remote), to current remote
or to default remote, in that order.
If the remote is not set, set it
"""
remote, ref_remote = self._get_remote(conan_reference)
result = self._remote_manager.upload_conan(conan_reference, remote)
if not ref_remote:
self._registry.set_ref(conan_reference, remote)
return result
def _get_remote(self, conan_ref=None):
# Prioritize -r , then reference registry and then the default remote
ref_remote = self._registry.get_ref(conan_ref) if conan_ref else None
if self._remote_name:
remote = self._registry.remote(self._remote_name)
else:
if ref_remote:
remote = ref_remote
else:
remote = self._registry.default_remote
return remote, ref_remote
def upload_package(self, package_reference):
remote, current_remote = self._get_remote(package_reference.conan)
if not current_remote:
self._out.warn("Remote for '%s' not defined, uploading to %s"
% (str(package_reference.conan), remote.name))
result = self._remote_manager.upload_package(package_reference, remote)
if not current_remote:
self._registry.set_ref(package_reference.conan, remote)
return result
def get_conan_digest(self, conan_ref):
""" used by update to check the date of packages, require force if older
"""
remote, current_remote = self._get_remote(conan_ref)
result = self._remote_manager.get_conan_digest(conan_ref, remote)
if not current_remote:
self._registry.set_ref(conan_ref, remote)
return result
def get_package_digest(self, package_reference):
""" used by update to check the date of packages, require force if older
"""
remote, ref_remote = self._get_remote(package_reference.conan)
result = self._remote_manager.get_package_digest(package_reference, remote)
if not ref_remote:
self._registry.set_ref(package_reference.conan, remote)
return result
def search(self, pattern=None, ignorecase=True):
remote, _ = self._get_remote()
return self._remote_manager.search(remote, pattern, ignorecase)
def remove(self, conan_ref):
if not self._remote_name:
raise ConanException("Cannot remove, remote not defined")
remote = self._registry.remote(self._remote_name)
result = self._remote_manager.remove(conan_ref, remote)
current_remote = self._registry.get_ref(conan_ref)
if current_remote == remote:
self._registry.remove_ref(conan_ref)
return result
def remove_packages(self, conan_ref, remove_ids):
if not self._remote_name:
raise ConanException("Cannot remove, remote not defined")
remote = self._registry.remote(self._remote_name)
return self._remote_manager.remove_packages(conan_ref, remove_ids, remote)
def download_packages(self, reference, package_ids):
assert(isinstance(package_ids, list))
remote, _ = self._get_remote(reference)
self._remote_manager.get_conanfile(reference, remote)
self._registry.set_ref(reference, remote)
output = ScopedOutput(str(reference), self._out)
for package_id in package_ids:
package_reference = PackageReference(reference, package_id)
self._retrieve_remote_package(package_reference, output, remote)
def _retrieve_remote_package(self, package_reference, output, remote=None):
if remote is None:
remote = self._registry.get_ref(package_reference.conan)
if not remote:
output.warn("Package doesn't have a remote defined. "
"Probably created locally and not uploaded")
return False
package_id = str(package_reference.package_id)
try:
output.info("Looking for package %s in remote '%s' " % (package_id, remote.name))
# Will raise if not found NotFoundException
self._remote_manager.get_package(package_reference, remote)
output.success('Package installed %s' % package_id)
return True
except ConanConnectionError:
raise # This shouldn't be skipped
except ConanException as e:
output.warn('Binary for %s not in remote: %s' % (package_id, str(e)))
return False
def authenticate(self, name, password):
if not name: # List all users, from all remotes
remotes = self._registry.remotes
if not remotes:
self._out.error("No remotes defined")
for remote in remotes:
self._remote_manager.authenticate(remote, None, None)
return
remote, _ = self._get_remote()
return self._remote_manager.authenticate(remote, name, password)
