def setUp(self):
folder = temp_folder()
paths = SimplePaths(folder)
search_adapter = DiskSearchAdapter()
self.search_manager = DiskSearchManager(paths, search_adapter)
os.chdir(paths.store)
self.paths = paths
def __init__(self, client_cache, user_io, remote_manager, remote,
recorder):
self._client_cache = client_cache
self._user_io = user_io
self._remote_proxy = ConanProxy(self._client_cache, self._user_io,
remote_manager, remote, recorder)
self._cache_search = DiskSearchManager(self._client_cache)
def search_packages(self, reference=None, remote=None, query=None, outdated=False):
""" Return the single information saved in conan.vars about all the packages
or the packages which match with a pattern
Attributes:
pattern = string to match packages
remote = search on another origin to get packages info
packages_pattern = String query with binary
packages properties: "arch=x86 AND os=Windows"
"""
if remote:
remote = RemoteRegistry(self._client_cache.registry, self._user_io).remote(remote)
packages_props = self._remote_manager.search_packages(remote, reference, query)
ordered_packages = OrderedDict(sorted(packages_props.items()))
manifest = self._remote_manager.get_conan_digest(reference, remote)
recipe_hash = manifest.summary_hash
else:
searcher = DiskSearchManager(self._client_cache)
packages_props = searcher.search_packages(reference, query)
ordered_packages = OrderedDict(sorted(packages_props.items()))
try:
recipe_hash = self._client_cache.load_manifest(reference).summary_hash
except IOError: # It could not exist in local
recipe_hash = None
if outdated and recipe_hash:
ordered_packages = filter_outdated(ordered_packages, recipe_hash)
return ordered_packages, reference, recipe_hash, query
def search_packages(self, reference=None, remote_name=None, query=None, outdated=False):
""" Return the single information saved in conan.vars about all the packages
or the packages which match with a pattern
Attributes:
pattern = string to match packages
remote = search on another origin to get packages info
packages_pattern = String query with binary
packages properties: "arch=x86 AND os=Windows"
"""
if remote_name:
remote = self._registry.remote(remote_name)
packages_props = self._remote_manager.search_packages(remote, reference, query)
ordered_packages = OrderedDict(sorted(packages_props.items()))
manifest = self._remote_manager.get_conan_manifest(reference, remote)
recipe_hash = manifest.summary_hash
else:
searcher = DiskSearchManager(self._client_cache)
packages_props = searcher.search_packages(reference, query)
ordered_packages = OrderedDict(sorted(packages_props.items()))
try:
recipe_hash = self._client_cache.load_manifest(reference).summary_hash
except IOError: # It could not exist in local
recipe_hash = None
if outdated and recipe_hash:
ordered_packages = filter_outdated(ordered_packages, recipe_hash)
return ordered_packages, reference, recipe_hash, query
def case_insensitive_test(self):
root_folder2 = "sdl/1.5/lasote/stable"
conan_ref2 = ConanFileReference.loads("sdl/1.5@lasote/stable")
os.makedirs("%s/%s" % (root_folder2, EXPORT_FOLDER))
root_folder3 = "assimp/0.14/phil/testing"
conan_ref3 = ConanFileReference.loads("assimp/0.14@phil/testing")
os.makedirs("%s/%s" % (root_folder3, EXPORT_FOLDER))
root_folder4 = "sdl/2.10/lasote/stable"
conan_ref4 = ConanFileReference.loads("sdl/2.10@lasote/stable")
os.makedirs("%s/%s" % (root_folder4, EXPORT_FOLDER))
root_folder5 = "SDL_fake/1.10/lasote/testing"
conan_ref5 = ConanFileReference.loads("SDL_fake/1.10@lasote/testing")
os.makedirs("%s/%s" % (root_folder5, EXPORT_FOLDER))
# Case insensitive searches
search_adapter = DiskSearchAdapter()
search_manager = DiskSearchManager(self.paths, search_adapter)
reg_conans = sorted([str(_reg) for _reg in search_manager.search("*")])
self.assertEqual(reg_conans, [str(conan_ref5),
str(conan_ref3),
str(conan_ref2),
str(conan_ref4)])
reg_conans = sorted([str(_reg) for _reg in search_manager.search(pattern="sdl*")])
self.assertEqual(reg_conans, [str(conan_ref5), str(conan_ref2), str(conan_ref4)])
# Case sensitive search
self.assertEqual(str(search_manager.search(pattern="SDL*", ignorecase=False)[0]),
str(conan_ref5))
def remove(self, pattern, remote, src=None, build_ids=None, package_ids_filter=None, force=False,
packages_query=None, outdated=False):
""" Remove local/remote conans, package folders, etc.
@param src: Remove src folder
@param pattern: it could be OpenCV* or OpenCV or a ConanFileReference
@param build_ids: Lists with ids or empty for all. (Its a filter)
@param package_ids_filter: Lists with ids or empty for all. (Its a filter)
@param force: if True, it will be deleted without requesting anything
@param packages_query: Only if src is a reference. Query settings and options
"""
if remote and (build_ids is not None or src):
raise ConanException("Remotes don't have 'build' or 'src' folder, just packages")
if remote:
remote = self._registry.remote(remote)
references = self._remote_manager.search_recipes(remote, pattern)
else:
disk_search = DiskSearchManager(self._client_cache)
references = disk_search.search_recipes(pattern)
if not references:
self._user_io.out.warn("No package recipe matches '%s'" % str(pattern))
return
deleted_refs = []
for reference in references:
assert isinstance(reference, ConanFileReference)
package_ids = package_ids_filter
if packages_query or outdated:
# search packages
if remote:
packages = self._remote_manager.search_packages(remote, reference, packages_query)
else:
packages = disk_search.search_packages(reference, packages_query)
if outdated:
if remote:
recipe_hash = self._remote_proxy.get_conan_manifest(reference).summary_hash
else:
recipe_hash = self._client_cache.load_manifest(reference).summary_hash
packages = filter_outdated(packages, recipe_hash)
if package_ids_filter:
package_ids = [p for p in packages if p in package_ids_filter]
else:
package_ids = list(packages.keys())
if not package_ids:
self._user_io.out.warn("No matching packages to remove for %s"
% str(reference))
continue
if self._ask_permission(reference, src, build_ids, package_ids, force):
deleted_refs.append(reference)
if remote:
self._remote_remove(reference, package_ids, remote)
else:
deleted_refs.append(reference)
self._local_remove(reference, src, build_ids, package_ids)
if not remote:
self._client_cache.delete_empty_dirs(deleted_refs)
def setUp(self):
self.conan_reference = ConanFileReference.loads("openssl/2.0.3@lasote/testing")
self.package_reference = PackageReference(self.conan_reference, "123123123")
self.tmp_dir = temp_folder()
read_perms = [("*/*@*/*", "*")]
write_perms = []
authorizer = BasicAuthorizer(read_perms, write_perms)
self.fake_url = "http://url"
updown_auth_manager = JWTUpDownAuthManager("secret",
timedelta(seconds=200))
adapter = ServerDiskAdapter(self.fake_url, self.tmp_dir, updown_auth_manager)
self.paths = SimplePaths(self.tmp_dir)
self.file_manager = FileManager(self.paths, adapter)
search_adapter = DiskSearchAdapter()
self.search_manager = DiskSearchManager(self.paths, search_adapter)
self.service = ConanService(authorizer, self.file_manager, "lasote")
self.search_service = SearchService(authorizer, self.search_manager, "lasote")
files = hello_source_files("test")
save_files(self.paths.export(self.conan_reference), files)
self.conan_digest = FileTreeManifest.create(self.paths.export(self.conan_reference))
conan_digest_path = os.path.join(self.paths.export(self.conan_reference), CONAN_MANIFEST)
save(conan_digest_path, str(self.conan_digest))
files = hello_source_files("package")
save_files(self.paths.package(self.package_reference), files)
def __init__(self, base_path=None, read_permissions=None,
write_permissions=None, users=None, base_url=None, plugins=None,
server_version=None,
min_client_compatible_version=None,
server_capabilities=None):
plugins = plugins or []
if not base_path:
base_path = temp_folder()
if server_capabilities is None:
server_capabilities = SERVER_CAPABILITIES # Default enabled
if not os.path.exists(base_path):
raise Exception("Base path not exist! %s")
# Define storage_folder, if not, it will be readed from conf file and pointed to real user home
self.storage_folder = os.path.join(base_path, ".conan_server", "data")
mkdir(self.storage_folder)
server_config = migrate_and_get_server_config(base_path, self.storage_folder)
if TestServerLauncher.port == 0:
TestServerLauncher.port = server_config.port
# Encode and Decode signature for Upload and Download service
updown_auth_manager = JWTUpDownAuthManager(server_config.updown_secret,
server_config.authorize_timeout)
self.file_manager = get_file_manager(server_config, public_url=base_url,
updown_auth_manager=updown_auth_manager)
self.search_manager = DiskSearchManager(SimplePaths(server_config.disk_storage_path))
# Prepare some test users
if not read_permissions:
read_permissions = server_config.read_permissions
read_permissions.append(("private_library/1.0.0@private_user/testing", "*"))
read_permissions.append(("*/*@*/*", "*"))
if not write_permissions:
write_permissions = server_config.write_permissions
if not users:
users = dict(server_config.users)
users[TESTING_REMOTE_PRIVATE_USER] = TESTING_REMOTE_PRIVATE_PASS
authorizer = BasicAuthorizer(read_permissions, write_permissions)
authenticator = BasicAuthenticator(users)
credentials_manager = JWTCredentialsManager(server_config.jwt_secret,
server_config.jwt_expire_time)
logger.debug("Storage path: %s" % self.storage_folder)
self.port = TestServerLauncher.port
self.ra = ConanServer(self.port, credentials_manager, updown_auth_manager,
authorizer, authenticator, self.file_manager, self.search_manager,
server_version, min_client_compatible_version,
server_capabilities)
for plugin in plugins:
self.ra.api_v1.install(plugin)
def __init__(self):
user_folder = conan_expand_user("~")
server_folder = os.path.join(user_folder, '.conan_server')
server_config = migrate_and_get_server_config(user_folder)
custom_auth = server_config.custom_authenticator
if custom_auth:
authenticator = load_authentication_plugin(server_folder,
custom_auth)
else:
authenticator = BasicAuthenticator(dict(server_config.users))
authorizer = BasicAuthorizer(server_config.read_permissions,
server_config.write_permissions)
credentials_manager = JWTCredentialsManager(
server_config.jwt_secret, server_config.jwt_expire_time)
updown_auth_manager = JWTUpDownAuthManager(
server_config.updown_secret, server_config.authorize_timeout)
file_manager = get_file_manager(
server_config, updown_auth_manager=updown_auth_manager)
search_adapter = DiskSearchAdapter()
search_manager = DiskSearchManager(
SimplePaths(server_config.disk_storage_path), search_adapter)
server_capabilities = SERVER_CAPABILITIES
self.ra = ConanServer(server_config.port, credentials_manager,
updown_auth_manager, authorizer, authenticator,
file_manager, search_manager,
Version(SERVER_VERSION),
Version(MIN_CLIENT_COMPATIBLE_VERSION),
server_capabilities)
def cmd_export(conanfile_path, name, version, user, channel, keep_source,
output, client_cache):
""" Export the recipe
param conanfile_path: the original source directory of the user containing a
conanfile.py
param user: user under this package will be exported
param channel: string (stable, testing,...)
"""
logger.debug("Exporting %s" % conanfile_path)
conan_linter(conanfile_path, output)
conanfile = _load_export_conanfile(conanfile_path, output, name, version)
conan_ref = ConanFileReference(conanfile.name, conanfile.version, user,
channel)
conan_ref_str = str(conan_ref)
# Maybe a platform check could be added, but depends on disk partition
refs = DiskSearchManager(client_cache).search_recipes(conan_ref_str,
ignorecase=True)
if refs and conan_ref not in refs:
raise ConanException(
"Cannot export package with same name but different case\n"
"You exported '%s' but already existing '%s'" %
(conan_ref_str, " ".join(str(s) for s in refs)))
output = ScopedOutput(str(conan_ref), output)
with client_cache.conanfile_write_lock(conan_ref):
_export_conanfile(conanfile_path, output, client_cache, conanfile,
conan_ref, keep_source)
def setUp(self):
folder = temp_folder()
paths = SimplePaths(folder)
search_adapter = DiskSearchAdapter()
self.search_manager = DiskSearchManager(paths, search_adapter)
os.chdir(paths.store)
self.paths = paths
def __init__(self,
base_folder=None,
current_folder=None,
servers=None,
users=None,
client_version=CLIENT_VERSION,
min_server_compatible_version=MIN_SERVER_COMPATIBLE_VERSION,
requester_class=None,
runner=None,
path_with_spaces=True):
"""
storage_folder: Local storage path
current_folder: Current execution folder
servers: dict of {remote_name: TestServer}
logins is a list of (user, password) for auto input in order
if required==> [("lasote", "mypass"), ("other", "otherpass")]
"""
self.all_output = "" # For debugging purpose, append all the run outputs
self.users = users or {
"default":
[(TESTING_REMOTE_PRIVATE_USER, TESTING_REMOTE_PRIVATE_PASS)]
}
self.servers = servers or {}
self.client_version = Version(str(client_version))
self.min_server_compatible_version = Version(
str(min_server_compatible_version))
self.base_folder = base_folder or temp_folder(path_with_spaces)
# Define storage_folder, if not, it will be read from conf file & pointed to real user home
self.storage_folder = os.path.join(self.base_folder, ".conan", "data")
self.client_cache = ClientCache(self.base_folder, self.storage_folder,
TestBufferConanOutput())
search_adapter = DiskSearchAdapter()
self.search_manager = DiskSearchManager(self.client_cache,
search_adapter)
self._default_settings(get_env("CONAN_COMPILER", "gcc"),
get_env("CONAN_COMPILER_VERSION", "4.8"),
get_env("CONAN_LIBCXX", "libstdc++"))
self.requester_class = requester_class
self.conan_runner = runner
self.init_dynamic_vars()
save(self.client_cache.registry, "")
registry = RemoteRegistry(self.client_cache.registry,
TestBufferConanOutput())
for name, server in self.servers.items():
if isinstance(server, TestServer):
registry.add(name, server.fake_url)
else:
registry.add(name, server)
logger.debug("Client storage = %s" % self.storage_folder)
self.current_folder = current_folder or temp_folder(path_with_spaces)
def factory():
"""Factory"""
def instance_remote_manager(client_cache):
requester = get_basic_requester(client_cache)
# Verify client version against remotes
version_checker_req = VersionCheckerRequester(
requester, Version(client_version),
Version(MIN_SERVER_COMPATIBLE_VERSION), out)
# To handle remote connections
put_headers = client_cache.read_put_headers()
rest_api_client = RestApiClient(out,
requester=version_checker_req,
put_headers=put_headers)
# To store user and token
localdb = LocalDB(client_cache.localdb)
# Wraps RestApiClient to add authentication support (same interface)
auth_manager = ConanApiAuthManager(rest_api_client, user_io,
localdb)
# Handle remote connections
remote_manager = RemoteManager(client_cache, auth_manager, out)
return remote_manager
use_color = get_env("CONAN_COLOR_DISPLAY", 1)
if use_color and hasattr(sys.stdout, "isatty") and sys.stdout.isatty():
import colorama
colorama.init()
color = True
else:
color = False
out = ConanOutput(sys.stdout, color)
user_io = UserIO(out=out)
try:
client_cache = migrate_and_get_client_cache(
get_conan_user_home(), out)
except Exception as e:
out.error(str(e))
raise
with tools.environment_append(client_cache.conan_config.env_vars):
# Adjust CONAN_LOGGING_LEVEL with the env readed
conans.util.log.logger = configure_logger()
# Get the new command instance after migrations have been done
remote_manager = instance_remote_manager(client_cache)
# Get a search manager
search_adapter = DiskSearchAdapter()
search_manager = DiskSearchManager(client_cache, search_adapter)
# Settings preprocessor
conan = Conan(client_cache, user_io, get_conan_runner(),
remote_manager, search_manager,
settings_preprocessor)
return conan, client_cache, user_io
def case_insensitive_test(self):
root_folder2 = "sdl/1.5/lasote/stable"
conan_ref2 = ConanFileReference.loads("sdl/1.5@lasote/stable")
os.makedirs("%s/%s" % (root_folder2, EXPORT_FOLDER))
root_folder3 = "assimp/0.14/phil/testing"
conan_ref3 = ConanFileReference.loads("assimp/0.14@phil/testing")
os.makedirs("%s/%s" % (root_folder3, EXPORT_FOLDER))
root_folder4 = "sdl/2.10/lasote/stable"
conan_ref4 = ConanFileReference.loads("sdl/2.10@lasote/stable")
os.makedirs("%s/%s" % (root_folder4, EXPORT_FOLDER))
root_folder5 = "SDL_fake/1.10/lasote/testing"
conan_ref5 = ConanFileReference.loads("SDL_fake/1.10@lasote/testing")
os.makedirs("%s/%s" % (root_folder5, EXPORT_FOLDER))
# Case insensitive searches
search_adapter = DiskSearchAdapter()
search_manager = DiskSearchManager(self.paths, search_adapter)
reg_conans = sorted(
[str(_reg) for _reg in search_manager.search_recipes("*")])
self.assertEqual(reg_conans, [
str(conan_ref5),
str(conan_ref3),
str(conan_ref2),
str(conan_ref4)
])
reg_conans = sorted([
str(_reg) for _reg in search_manager.search_recipes(pattern="sdl*")
])
self.assertEqual(reg_conans,
[str(conan_ref5),
str(conan_ref2),
str(conan_ref4)])
# Case sensitive search
self.assertEqual(
str(
search_manager.search_recipes(pattern="SDL*",
ignorecase=False)[0]),
str(conan_ref5))
def __init__(self,
base_folder=None,
current_folder=None,
servers=None,
users=None,
client_version=CLIENT_VERSION,
min_server_compatible_version=MIN_SERVER_COMPATIBLE_VERSION,
requester_class=None,
runner=None,
path_with_spaces=True,
default_profile=True):
"""
storage_folder: Local storage path
current_folder: Current execution folder
servers: dict of {remote_name: TestServer}
logins is a list of (user, password) for auto input in order
if required==> [("lasote", "mypass"), ("other", "otherpass")]
"""
self.all_output = "" # For debugging purpose, append all the run outputs
self.users = users or {
"default":
[(TESTING_REMOTE_PRIVATE_USER, TESTING_REMOTE_PRIVATE_PASS)]
}
self.client_version = Version(str(client_version))
self.min_server_compatible_version = Version(
str(min_server_compatible_version))
self.base_folder = base_folder or temp_folder(path_with_spaces)
# Define storage_folder, if not, it will be read from conf file & pointed to real user home
self.storage_folder = os.path.join(self.base_folder, ".conan", "data")
self.client_cache = ClientCache(self.base_folder, self.storage_folder,
TestBufferConanOutput())
search_adapter = DiskSearchAdapter()
self.search_manager = DiskSearchManager(self.client_cache,
search_adapter)
self.requester_class = requester_class
self.conan_runner = runner
self.update_servers(servers)
self.init_dynamic_vars()
logger.debug("Client storage = %s" % self.storage_folder)
self.current_folder = current_folder or temp_folder(path_with_spaces)
# Enforcing VS 2015, even if VS2017 is auto detected
if default_profile:
profile = self.client_cache.default_profile
if profile.settings.get("compiler.version") == "15":
profile.settings["compiler.version"] = "14"
save(self.client_cache.default_profile_path, profile.dumps())
def factory(interactive=None):
"""Factory"""
# Respect color env setting or check tty if unset
color_set = "CONAN_COLOR_DISPLAY" in os.environ
if ((color_set and get_env("CONAN_COLOR_DISPLAY", 1))
or (not color_set
and hasattr(sys.stdout, "isatty")
and sys.stdout.isatty())):
import colorama
if get_env("PYCHARM_HOSTED"): # in PyCharm disable convert/strip
colorama.init(convert=False, strip=False)
else:
colorama.init()
color = True
else:
color = False
out = ConanOutput(sys.stdout, color)
user_io = UserIO(out=out)
try:
user_home = get_conan_user_home()
client_cache = migrate_and_get_client_cache(user_home, out)
sys.path.append(os.path.join(user_home, "python"))
except Exception as e:
out.error(str(e))
raise
with tools.environment_append(client_cache.conan_config.env_vars):
# Adjust CONAN_LOGGING_LEVEL with the env readed
conans.util.log.logger = configure_logger()
# Get the new command instance after migrations have been done
requester = get_basic_requester(client_cache)
_, _, remote_manager = ConanAPIV1.instance_remote_manager(
requester,
client_cache, user_io,
Version(client_version),
Version(MIN_SERVER_COMPATIBLE_VERSION))
# Adjust global tool variables
set_global_instances(out, requester)
# Get a search manager
search_manager = DiskSearchManager(client_cache)
# Settings preprocessor
if interactive is None:
interactive = not get_env("CONAN_NON_INTERACTIVE", False)
conan = ConanAPIV1(client_cache, user_io, get_conan_runner(), remote_manager, search_manager,
settings_preprocessor, interactive=interactive)
return conan, client_cache, user_io
def search_recipes(self, pattern, remote=None, case_sensitive=False):
ignorecase = not case_sensitive
if not remote:
return DiskSearchManager(self._client_cache).search_recipes(pattern, ignorecase)
registry = RemoteRegistry(self._client_cache.registry, self._user_io.out)
if remote == 'all':
remotes = registry.remotes
# We have to check if there is a remote called "all"
# Deprecate: 2.0 can remove this check
if 'all' not in (r.name for r in remotes):
references = {}
for remote in remotes:
result = self._remote_manager.search_recipes(remote, pattern, ignorecase)
if result:
references[remote.name] = result
return references
# single remote
remote = registry.remote(remote)
return self._remote_manager.search_recipes(remote, pattern, ignorecase)
class CmdUpload(object):
def __init__(self, client_cache, user_io, remote_manager, registry):
self._client_cache = client_cache
self._user_io = user_io
self._remote_manager = remote_manager
self._registry = registry
self._cache_search = DiskSearchManager(self._client_cache)
def upload(self, recorder, reference_or_pattern, package_id=None, all_packages=None,
force=False, confirm=False, retry=0, retry_wait=0, skip_upload=False,
integrity_check=False, no_overwrite=None, remote_name=None):
"""If package_id is provided, conan_reference_or_pattern is a ConanFileReference"""
if package_id and not _is_a_reference(reference_or_pattern):
raise ConanException("-p parameter only allowed with a valid recipe reference, "
"not with a pattern")
t1 = time.time()
if package_id or _is_a_reference(reference_or_pattern): # Upload package
ref = ConanFileReference.loads(reference_or_pattern)
references = [ref, ]
confirm = True
else:
references = self._cache_search.search_recipes(reference_or_pattern)
if not references:
raise NotFoundException(("No packages found matching pattern '%s'" %
reference_or_pattern))
for conan_ref in references:
upload = True
if not confirm:
msg = "Are you sure you want to upload '%s'?" % str(conan_ref)
upload = self._user_io.request_boolean(msg)
if upload:
try:
conanfile_path = self._client_cache.conanfile(conan_ref)
conan_file = load_conanfile_class(conanfile_path)
except NotFoundException:
raise NotFoundException(("There is no local conanfile exported as %s" %
str(conan_ref)))
if all_packages:
packages_ids = self._client_cache.conan_packages(conan_ref)
elif package_id:
packages_ids = [package_id, ]
else:
packages_ids = []
self._upload(conan_file, conan_ref, force, packages_ids, retry, retry_wait,
skip_upload, integrity_check, no_overwrite, remote_name, recorder)
logger.debug("====> Time manager upload: %f" % (time.time() - t1))
def _upload(self, conan_file, conan_ref, force, packages_ids, retry, retry_wait, skip_upload,
integrity_check, no_overwrite, remote_name, recorder):
"""Uploads the recipes and binaries identified by conan_ref"""
defined_remote = self._registry.get_ref(conan_ref)
if remote_name: # If remote_name is given, use it
upload_remote = self._registry.remote(remote_name)
elif defined_remote: # Else, if the package had defined a remote, use it
upload_remote = defined_remote
else: # Or use the default otherwise
upload_remote = self._registry.default_remote
if not force:
self._check_recipe_date(conan_ref, upload_remote)
self._user_io.out.info("Uploading %s to remote '%s'"
% (str(conan_ref), upload_remote.name))
self._upload_recipe(conan_ref, retry, retry_wait, skip_upload, no_overwrite, upload_remote)
recorder.add_recipe(str(conan_ref), upload_remote.name, upload_remote.url)
if packages_ids:
# Can't use build_policy_always here because it's not loaded (only load_class)
if conan_file.build_policy == "always":
raise ConanException("Conanfile has build_policy='always', "
"no packages can be uploaded")
total = len(packages_ids)
for index, package_id in enumerate(packages_ids):
ret_upload_package = self._upload_package(PackageReference(conan_ref, package_id),
index + 1, total, retry, retry_wait,
skip_upload, integrity_check,
no_overwrite, upload_remote)
if ret_upload_package:
recorder.add_package(str(conan_ref), package_id)
if not defined_remote and not skip_upload:
self._registry.set_ref(conan_ref, upload_remote)
def _upload_recipe(self, conan_reference, retry, retry_wait, skip_upload, no_overwrite, remote):
conan_file_path = self._client_cache.conanfile(conan_reference)
current_remote = self._registry.get_ref(conan_reference)
if remote != current_remote:
conanfile = load_conanfile_class(conan_file_path)
complete_recipe_sources(self._remote_manager, self._client_cache, self._registry,
conanfile, conan_reference)
ignore_deleted_file = None
else:
ignore_deleted_file = EXPORT_SOURCES_TGZ_NAME
result = self._remote_manager.upload_recipe(conan_reference, remote, retry, retry_wait,
ignore_deleted_file=ignore_deleted_file,
skip_upload=skip_upload,
no_overwrite=no_overwrite)
return result
def _upload_package(self, package_ref, index=1, total=1, retry=None, retry_wait=None,
skip_upload=False, integrity_check=False, no_overwrite=None, remote=None):
"""Uploads the package identified by package_id"""
msg = ("Uploading package %d/%d: %s" % (index, total, str(package_ref.package_id)))
t1 = time.time()
self._user_io.out.info(msg)
result = self._remote_manager.upload_package(package_ref, remote, retry, retry_wait,
skip_upload, integrity_check, no_overwrite)
logger.debug("====> Time uploader upload_package: %f" % (time.time() - t1))
return result
def _check_recipe_date(self, conan_ref, remote):
try:
remote_recipe_manifest = self._remote_manager.get_conan_manifest(conan_ref, remote)
except NotFoundException:
return # First time uploading this package
local_manifest = self._client_cache.load_manifest(conan_ref)
if (remote_recipe_manifest != local_manifest and
remote_recipe_manifest.time > local_manifest.time):
raise ConanException("Remote recipe is newer than local recipe: "
"\n Remote date: %s\n Local date: %s" %
(remote_recipe_manifest.time, local_manifest.time))
class SearchTest(unittest.TestCase):
def setUp(self):
folder = temp_folder()
paths = SimplePaths(folder)
search_adapter = DiskSearchAdapter()
self.search_manager = DiskSearchManager(paths, search_adapter)
os.chdir(paths.store)
self.paths = paths
def basic_test2(self):
conan_ref1 = ConanFileReference.loads("opencv/2.4.10@lasote/testing")
root_folder = str(conan_ref1).replace("@", "/")
artifacts = ["a", "b", "c"]
reg1 = "%s/%s" % (root_folder, EXPORT_FOLDER)
os.makedirs(reg1)
for artif_id in artifacts:
build1 = "%s/%s/%s" % (root_folder, BUILD_FOLDER, artif_id)
artif1 = "%s/%s/%s" % (root_folder, PACKAGES_FOLDER, artif_id)
os.makedirs(build1)
info = ConanInfo().loads("[settings]\n[options]")
save(os.path.join(artif1, CONANINFO), info.dumps())
packages = self.search_manager.search_packages(conan_ref1, "")
all_artif = [_artif for _artif in sorted(packages)]
self.assertEqual(all_artif, artifacts)
def pattern_test(self):
refs = ["opencv/2.4.%s@lasote/testing" % ref for ref in ("1", "2", "3")]
refs = [ConanFileReference.loads(ref) for ref in refs]
for ref in refs:
root_folder = str(ref).replace("@", "/")
reg1 = "%s/%s" % (root_folder, EXPORT_FOLDER)
os.makedirs(reg1)
recipes = self.search_manager.search("opencv/*@lasote/testing")
self.assertEqual(recipes, refs)
def case_insensitive_test(self):
root_folder2 = "sdl/1.5/lasote/stable"
conan_ref2 = ConanFileReference.loads("sdl/1.5@lasote/stable")
os.makedirs("%s/%s" % (root_folder2, EXPORT_FOLDER))
root_folder3 = "assimp/0.14/phil/testing"
conan_ref3 = ConanFileReference.loads("assimp/0.14@phil/testing")
os.makedirs("%s/%s" % (root_folder3, EXPORT_FOLDER))
root_folder4 = "sdl/2.10/lasote/stable"
conan_ref4 = ConanFileReference.loads("sdl/2.10@lasote/stable")
os.makedirs("%s/%s" % (root_folder4, EXPORT_FOLDER))
root_folder5 = "SDL_fake/1.10/lasote/testing"
conan_ref5 = ConanFileReference.loads("SDL_fake/1.10@lasote/testing")
os.makedirs("%s/%s" % (root_folder5, EXPORT_FOLDER))
# Case insensitive searches
search_adapter = DiskSearchAdapter()
search_manager = DiskSearchManager(self.paths, search_adapter)
reg_conans = sorted([str(_reg) for _reg in search_manager.search("*")])
self.assertEqual(reg_conans, [str(conan_ref5),
str(conan_ref3),
str(conan_ref2),
str(conan_ref4)])
reg_conans = sorted([str(_reg) for _reg in search_manager.search(pattern="sdl*")])
self.assertEqual(reg_conans, [str(conan_ref5), str(conan_ref2), str(conan_ref4)])
# Case sensitive search
self.assertEqual(str(search_manager.search(pattern="SDL*", ignorecase=False)[0]),
str(conan_ref5))
def __init__(self, client_cache, user_io, remote_manager, registry):
self._client_cache = client_cache
self._user_io = user_io
self._remote_manager = remote_manager
self._registry = registry
self._cache_search = DiskSearchManager(self._client_cache)
def remove(self,
pattern,
remote,
src=None,
build_ids=None,
package_ids_filter=None,
force=False,
packages_query=None,
outdated=False):
""" Remove local/remote conans, package folders, etc.
@param src: Remove src folder
@param pattern: it could be OpenCV* or OpenCV or a ConanFileReference
@param build_ids: Lists with ids or empty for all. (Its a filter)
@param package_ids_filter: Lists with ids or empty for all. (Its a filter)
@param force: if True, it will be deleted without requesting anything
@param packages_query: Only if src is a reference. Query settings and options
"""
if remote and (build_ids is not None or src):
raise ConanException(
"Remotes don't have 'build' or 'src' folder, just packages")
if remote:
remote = self._registry.remote(remote)
references = self._remote_manager.search_recipes(remote, pattern)
else:
disk_search = DiskSearchManager(self._client_cache)
references = disk_search.search_recipes(pattern)
if not references:
self._user_io.out.warn("No package recipe matches '%s'" %
str(pattern))
return
deleted_refs = []
for reference in references:
assert isinstance(reference, ConanFileReference)
package_ids = package_ids_filter
if packages_query or outdated:
# search packages
if remote:
packages = self._remote_manager.search_packages(
remote, reference, packages_query)
else:
packages = disk_search.search_packages(
reference, packages_query)
if outdated:
if remote:
recipe_hash = self._remote_proxy.get_conan_manifest(
reference).summary_hash
else:
recipe_hash = self._client_cache.load_manifest(
reference).summary_hash
packages = filter_outdated(packages, recipe_hash)
if package_ids_filter:
package_ids = [
p for p in packages if p in package_ids_filter
]
else:
package_ids = list(packages.keys())
if not package_ids:
self._user_io.out.warn(
"No matching packages to remove for %s" %
str(reference))
continue
if self._ask_permission(reference, src, build_ids, package_ids,
force):
deleted_refs.append(reference)
if remote:
self._remote_remove(reference, package_ids, remote)
else:
deleted_refs.append(reference)
self._local_remove(reference, src, build_ids, package_ids)
if not remote:
self._client_cache.delete_empty_dirs(deleted_refs)
class CmdUpload(object):
def __init__(self, client_cache, user_io, remote_manager, remote,
recorder):
self._client_cache = client_cache
self._user_io = user_io
self._remote_proxy = ConanProxy(self._client_cache, self._user_io,
remote_manager, remote, recorder)
self._cache_search = DiskSearchManager(self._client_cache)
def upload(self,
conan_reference_or_pattern,
package_id=None,
all_packages=None,
force=False,
confirm=False,
retry=0,
retry_wait=0,
skip_upload=False,
integrity_check=False,
no_overwrite=None):
"""If package_id is provided, conan_reference_or_pattern is a ConanFileReference"""
if package_id and not _is_a_reference(conan_reference_or_pattern):
raise ConanException(
"-p parameter only allowed with a valid recipe reference, "
"not with a pattern")
t1 = time.time()
if package_id: # Upload package
ref = ConanFileReference.loads(conan_reference_or_pattern)
self._check_reference(ref)
self.upload_package(PackageReference(ref, package_id),
retry=retry,
retry_wait=retry_wait,
skip_upload=skip_upload,
integrity_check=integrity_check,
no_overwrite=no_overwrite)
else: # Upload conans
self._run_upload(conan_reference_or_pattern,
all_packages=all_packages,
force=force,
confirm=confirm,
retry=retry,
retry_wait=retry_wait,
skip_upload=skip_upload,
integrity_check=integrity_check,
no_overwrite=no_overwrite)
logger.debug("====> Time manager upload: %f" % (time.time() - t1))
def _run_upload(self,
ref_or_pattern,
force=False,
all_packages=False,
confirm=False,
retry=None,
retry_wait=None,
skip_upload=False,
integrity_check=False,
no_overwrite=None):
"""Upload all the recipes matching 'pattern'"""
if _is_a_reference(ref_or_pattern):
ref = ConanFileReference.loads(ref_or_pattern)
export_path = self._client_cache.export(ref)
if not os.path.exists(export_path):
raise NotFoundException(
"There is no local conanfile exported as %s" % str(ref))
references = [
ref,
]
confirm = True
else:
references = self._cache_search.search_recipes(ref_or_pattern)
if not references:
raise NotFoundException("No packages found matching pattern '%s'" %
ref_or_pattern)
for conan_ref in references:
upload = True
if not confirm:
msg = "Are you sure you want to upload '%s'?" % str(conan_ref)
upload = self._user_io.request_boolean(msg)
if upload:
self._upload(conan_ref, force, all_packages, retry, retry_wait,
skip_upload, integrity_check, no_overwrite)
def _upload(self, conan_ref, force, all_packages, retry, retry_wait,
skip_upload, integrity_check, no_overwrite):
"""Uploads the recipes and binaries identified by conan_ref"""
if not force:
self._check_recipe_date(conan_ref)
self._user_io.out.info("Uploading %s" % str(conan_ref))
self._remote_proxy.upload_recipe(conan_ref, retry, retry_wait,
skip_upload, no_overwrite)
if all_packages:
self._check_reference(conan_ref)
for index, package_id in enumerate(
self._client_cache.conan_packages(conan_ref)):
total = len(self._client_cache.conan_packages(conan_ref))
self.upload_package(PackageReference(conan_ref, package_id),
index + 1, total, retry, retry_wait,
skip_upload, integrity_check, no_overwrite)
def _check_reference(self, conan_reference):
try:
conanfile_path = self._client_cache.conanfile(conan_reference)
conan_file = load_conanfile_class(conanfile_path)
except NotFoundException:
raise NotFoundException(
"There is no local conanfile exported as %s" %
str(conan_reference))
# Can't use build_policy_always here because it's not loaded (only load_class)
if conan_file.build_policy == "always":
raise ConanException("Conanfile has build_policy='always', "
"no packages can be uploaded")
def upload_package(self,
package_ref,
index=1,
total=1,
retry=None,
retry_wait=None,
skip_upload=False,
integrity_check=False,
no_overwrite=None):
"""Uploads the package identified by package_id"""
msg = ("Uploading package %d/%d: %s" %
(index, total, str(package_ref.package_id)))
t1 = time.time()
self._user_io.out.info(msg)
self._remote_proxy.upload_package(package_ref, retry, retry_wait,
skip_upload, integrity_check,
no_overwrite)
logger.debug("====> Time uploader upload_package: %f" %
(time.time() - t1))
def _check_recipe_date(self, conan_ref):
try:
remote_recipe_manifest = self._remote_proxy.get_conan_digest(
conan_ref)
except NotFoundException:
return # First time uploading this package
local_manifest = self._client_cache.load_manifest(conan_ref)
if (remote_recipe_manifest != local_manifest
and remote_recipe_manifest.time > local_manifest.time):
raise ConanException(
"Remote recipe is newer than local recipe: "
"\n Remote date: %s\n Local date: %s" %
(remote_recipe_manifest.time, local_manifest.time))
class SearchTest(unittest.TestCase):
def setUp(self):
folder = temp_folder()
paths = SimplePaths(folder)
search_adapter = DiskSearchAdapter()
self.search_manager = DiskSearchManager(paths, search_adapter)
os.chdir(paths.store)
self.paths = paths
def basic_test2(self):
conan_ref1 = ConanFileReference.loads("opencv/2.4.10@lasote/testing")
root_folder = str(conan_ref1).replace("@", "/")
artifacts = ["a", "b", "c"]
reg1 = "%s/%s" % (root_folder, EXPORT_FOLDER)
os.makedirs(reg1)
for artif_id in artifacts:
build1 = "%s/%s/%s" % (root_folder, BUILD_FOLDER, artif_id)
artif1 = "%s/%s/%s" % (root_folder, PACKAGES_FOLDER, artif_id)
os.makedirs(build1)
info = ConanInfo().loads("[settings]\n[options]")
save(os.path.join(artif1, CONANINFO), info.dumps())
packages = self.search_manager.search_packages(conan_ref1, "")
all_artif = [_artif for _artif in sorted(packages)]
self.assertEqual(all_artif, artifacts)
def pattern_test(self):
refs = [
"opencv/2.4.%s@lasote/testing" % ref for ref in ("1", "2", "3")
]
refs = [ConanFileReference.loads(ref) for ref in refs]
for ref in refs:
root_folder = str(ref).replace("@", "/")
reg1 = "%s/%s" % (root_folder, EXPORT_FOLDER)
os.makedirs(reg1)
recipes = self.search_manager.search("opencv/*@lasote/testing")
self.assertEqual(recipes, refs)
def case_insensitive_test(self):
root_folder2 = "sdl/1.5/lasote/stable"
conan_ref2 = ConanFileReference.loads("sdl/1.5@lasote/stable")
os.makedirs("%s/%s" % (root_folder2, EXPORT_FOLDER))
root_folder3 = "assimp/0.14/phil/testing"
conan_ref3 = ConanFileReference.loads("assimp/0.14@phil/testing")
os.makedirs("%s/%s" % (root_folder3, EXPORT_FOLDER))
root_folder4 = "sdl/2.10/lasote/stable"
conan_ref4 = ConanFileReference.loads("sdl/2.10@lasote/stable")
os.makedirs("%s/%s" % (root_folder4, EXPORT_FOLDER))
root_folder5 = "SDL_fake/1.10/lasote/testing"
conan_ref5 = ConanFileReference.loads("SDL_fake/1.10@lasote/testing")
os.makedirs("%s/%s" % (root_folder5, EXPORT_FOLDER))
# Case insensitive searches
search_adapter = DiskSearchAdapter()
search_manager = DiskSearchManager(self.paths, search_adapter)
reg_conans = sorted([str(_reg) for _reg in search_manager.search("*")])
self.assertEqual(reg_conans, [
str(conan_ref5),
str(conan_ref3),
str(conan_ref2),
str(conan_ref4)
])
reg_conans = sorted(
[str(_reg) for _reg in search_manager.search(pattern="sdl*")])
self.assertEqual(reg_conans,
[str(conan_ref5),
str(conan_ref2),
str(conan_ref4)])
# Case sensitive search
self.assertEqual(
str(search_manager.search(pattern="SDL*", ignorecase=False)[0]),
str(conan_ref5))
