def _run_upload(self,
pattern,
force=False,
all_packages=False,
confirm=False,
retry=None,
retry_wait=None,
skip_upload=False,
integrity_check=False):
"""Upload all the recipes matching 'pattern'"""
if _is_a_reference(pattern):
ref = ConanFileReference.loads(pattern)
export_path = self._client_cache.export(ref)
if not os.path.exists(export_path):
raise NotFoundException(
"There is no local conanfile exported as %s" % str(ref))
references = [
ref,
]
confirm = True
else:
references = self._search_manager.search(pattern)
if not references:
raise NotFoundException("No packages found matching pattern '%s'" %
pattern)
for conan_ref in references:
upload = True
if not confirm:
msg = "Are you sure you want to upload '%s'?" % str(conan_ref)
upload = self._user_io.request_boolean(msg)
if upload:
self._upload(conan_ref, force, all_packages, retry, retry_wait,
skip_upload, integrity_check)
def _get_local_infos_min(server_store, ref, look_in_all_rrevs):
result = {}
rrevs = server_store.get_recipe_revisions(ref) if look_in_all_rrevs else [
None
]
for rrev in rrevs:
new_ref = ref.copy_with_rev(rrev.revision) if rrev else ref
subdirs = list_folder_subdirs(server_store.packages(new_ref), level=1)
for package_id in subdirs:
if package_id in result:
continue
# Read conaninfo
try:
pref = PackageReference(new_ref, package_id)
revision_entry = server_store.get_last_package_revision(pref)
if not revision_entry:
raise NotFoundException("")
pref = PackageReference(new_ref, package_id,
revision_entry.revision)
info_path = os.path.join(server_store.package(pref), CONANINFO)
if not os.path.exists(info_path):
raise NotFoundException("")
conan_info_content = load(info_path)
info = ConanInfo.loads(conan_info_content)
conan_vars_info = info.serialize_min()
result[package_id] = conan_vars_info
except Exception as exc: # FIXME: Too wide
logger.error("Package %s has no ConanInfo file" % str(pref))
if str(exc):
logger.error(str(exc))
return result
def put_file(self, file_saver, abs_filepath, token, upload_size):
"""
file_saver is an object with the save() method without parameters
"""
try:
encoded_path, filesize, user = self.updown_auth_manager.get_resource_info(
token)
# Check size
if upload_size != filesize:
logger.debug("Invalid size file!!: %s: %s" %
(user, abs_filepath))
raise RequestErrorException("Bad file size")
abs_encoded_path = os.path.abspath(
os.path.join(self.base_store_folder, encoded_path))
if not self._valid_path(abs_filepath, abs_encoded_path):
raise NotFoundException("File not found")
logger.debug("Put file: %s: %s" % (user, abs_filepath))
mkdir(os.path.dirname(abs_filepath))
if os.path.exists(abs_filepath):
os.remove(abs_filepath)
file_saver.save(os.path.dirname(abs_filepath))
except (jwt.ExpiredSignature, jwt.DecodeError, AttributeError):
raise NotFoundException("File not found")
def package(self, reference, package_id):
# Package paths
conan_file_path = self._client_cache.conanfile(reference)
if not os.path.exists(conan_file_path):
raise ConanException("Package recipe '%s' does not exist" % str(reference))
if not package_id:
packages = [PackageReference(reference, packid)
for packid in self._client_cache.conan_builds(reference)]
if not packages:
raise NotFoundException("%s: Package recipe has not been built locally\n"
"Please read the 'conan package' command help\n"
"Use 'conan install' or 'conan test_package' to build and "
"create binaries" % str(reference))
else:
packages = [PackageReference(reference, package_id)]
for package_reference in packages:
build_folder = self._client_cache.build(package_reference, short_paths=None)
if not os.path.exists(build_folder):
raise NotFoundException("%s: Package binary '%s' folder doesn't exist\n"
"Please read the 'conan package' command help\n"
"Use 'conan install' or 'conan test_package' to build and "
"create binaries"
% (str(reference), package_reference.package_id))
# The package already exist, we can use short_paths if they were defined
package_folder = self._client_cache.package(package_reference, short_paths=None)
# Will read current conaninfo with specified options and load conanfile with them
output = ScopedOutput(str(reference), self._user_io.out)
output.info("Re-packaging %s" % package_reference.package_id)
loader = self._loader(build_folder)
conanfile = loader.load_conan(conan_file_path, self._user_io.out)
self._load_deps_info(build_folder, conanfile, output)
rmdir(package_folder)
packager.create_package(conanfile, build_folder, package_folder, output)
def package(self, reference, package_id):
# Package paths
conan_file_path = self._client_cache.conanfile(reference)
if not os.path.exists(conan_file_path):
raise ConanException("Package recipe '%s' does not exist" %
str(reference))
conanfile = load_conanfile_class(conan_file_path)
if hasattr(conanfile, "build_id"):
raise ConanException(
"package command does not support recipes with 'build_id'\n"
"To repackage them use 'conan install'")
if not package_id:
packages = [
PackageReference(reference, packid)
for packid in self._client_cache.conan_builds(reference)
]
if not packages:
raise NotFoundException(
"%s: Package recipe has not been built locally\n"
"Please read the 'conan package' command help\n"
"Use 'conan install' or 'conan test_package' to build and "
"create binaries" % str(reference))
else:
packages = [PackageReference(reference, package_id)]
package_source_folder = self._client_cache.source(
reference, conanfile.short_paths)
for package_reference in packages:
build_folder = self._client_cache.build(package_reference,
short_paths=None)
if not os.path.exists(build_folder):
raise NotFoundException(
"%s: Package binary '%s' folder doesn't exist\n"
"Please read the 'conan package' command help\n"
"Use 'conan install' or 'conan test_package' to build and "
"create binaries" %
(str(reference), package_reference.package_id))
# The package already exist, we can use short_paths if they were defined
package_folder = self._client_cache.package(package_reference,
short_paths=None)
# Will read current conaninfo with specified options and load conanfile with them
output = ScopedOutput(str(reference), self._user_io.out)
output.info("Re-packaging %s" % package_reference.package_id)
conanfile = load_consumer_conanfile(conan_file_path,
build_folder,
self._client_cache.settings,
self._runner,
output,
reference=reference)
rmdir(package_folder)
if getattr(conanfile, 'no_copy_source', False):
source_folder = package_source_folder
else:
source_folder = build_folder
with environment_append(conanfile.env):
packager.create_package(conanfile, source_folder, build_folder,
package_folder, output)
def _download_recipe(self, ref, output, remotes, remote, recorder):
def _retrieve_from_remote(the_remote):
output.info("Trying with '%s'..." % the_remote.name)
# If incomplete, resolve the latest in server
_ref = self._remote_manager.get_recipe(ref, the_remote)
output.info("Downloaded recipe revision %s" % _ref.revision)
with self._cache.package_layout(ref).update_metadata() as metadata:
metadata.recipe.remote = the_remote.name
recorder.recipe_downloaded(ref, the_remote.url)
return _ref
if remote:
output.info("Retrieving from server '%s' " % remote.name)
else:
try:
remote_name = self._cache.package_layout(
ref).load_metadata().recipe.remote
if remote_name:
remote = remotes[remote_name]
except (IOError, RecipeNotFoundException):
pass
else:
if remote:
output.info("Retrieving from predefined remote '%s'" %
remote.name)
if remote:
try:
new_ref = _retrieve_from_remote(remote)
return remote, new_ref
except NotFoundException:
msg = "%s was not found in remote '%s'" % (str(ref),
remote.name)
recorder.recipe_install_error(ref, INSTALL_ERROR_MISSING, msg,
remote.url)
raise NotFoundException(msg)
except RequestException as exc:
recorder.recipe_install_error(ref, INSTALL_ERROR_NETWORK,
str(exc), remote.url)
raise exc
output.info("Not found in local cache, looking in remotes...")
remotes = remotes.values()
if not remotes:
raise ConanException("No remote defined")
for remote in remotes:
try:
new_ref = _retrieve_from_remote(remote)
return remote, new_ref
# If not found continue with the next, else raise
except NotFoundException:
pass
else:
msg = "Unable to find '%s' in remotes" % ref.full_repr()
recorder.recipe_install_error(ref, INSTALL_ERROR_MISSING, msg,
None)
raise NotFoundException(msg)
def _retrieve_recipe(self, conan_reference, output):
""" returns the requested conanfile object, retrieving it from
remotes if necessary. Can raise NotFoundException
"""
def _retrieve_from_remote(the_remote):
output.info("Trying with '%s'..." % the_remote.name)
self._remote_manager.get_recipe(conan_reference, the_remote)
self._registry.set_ref(conan_reference, the_remote)
self._recorder.recipe_downloaded(conan_reference, the_remote.url)
if self._remote_name:
output.info("Not found, retrieving from server '%s' " %
self._remote_name)
ref_remote = self._registry.remote(self._remote_name)
else:
ref_remote = self._registry.get_ref(conan_reference)
if ref_remote:
output.info("Retrieving from predefined remote '%s'" %
ref_remote.name)
if ref_remote:
try:
_retrieve_from_remote(ref_remote)
return
except NotFoundException:
msg = "%s was not found in remote '%s'" % (
str(conan_reference), ref_remote.name)
self._recorder.recipe_install_error(conan_reference,
INSTALL_ERROR_MISSING, msg,
ref_remote.url)
raise NotFoundException(msg)
except RequestException as exc:
self._recorder.recipe_install_error(conan_reference,
INSTALL_ERROR_NETWORK,
str(exc), ref_remote.url)
raise exc
output.info("Not found in local cache, looking in remotes...")
remotes = self._registry.remotes
for remote in remotes:
logger.debug("Trying with remote %s" % remote.name)
try:
_retrieve_from_remote(remote)
return
# If not found continue with the next, else raise
except NotFoundException as exc:
if remote == remotes[-1]: # Last element not found
msg = "Unable to find '%s' in remotes" % str(
conan_reference)
logger.debug("Not found in any remote, raising...%s" % exc)
self._recorder.recipe_install_error(
conan_reference, INSTALL_ERROR_MISSING, msg, None)
raise NotFoundException(msg)
raise ConanException("No remote defined")
def _download_recipe(self, conan_reference, output, remote_name, recorder):
def _retrieve_from_remote(the_remote):
output.info("Trying with '%s'..." % the_remote.name)
new_reference = self._remote_manager.get_recipe(
conan_reference, the_remote)
self._registry.set_ref(new_reference, the_remote.name)
recorder.recipe_downloaded(conan_reference, the_remote.url)
return new_reference
if remote_name:
output.info("Not found, retrieving from server '%s' " %
remote_name)
remote = self._registry.remote(remote_name)
else:
remote = self._registry.get_recipe_remote(conan_reference)
if remote:
output.info("Retrieving from predefined remote '%s'" %
remote.name)
if remote:
try:
new_ref = _retrieve_from_remote(remote)
return remote, new_ref
except NotFoundException:
msg = "%s was not found in remote '%s'" % (
str(conan_reference), remote.name)
recorder.recipe_install_error(conan_reference,
INSTALL_ERROR_MISSING, msg,
remote.url)
raise NotFoundException(msg)
except RequestException as exc:
recorder.recipe_install_error(conan_reference,
INSTALL_ERROR_NETWORK, str(exc),
remote.url)
raise exc
output.info("Not found in local cache, looking in remotes...")
remotes = self._registry.remotes
if not remotes:
raise ConanException("No remote defined")
for remote in remotes:
logger.debug("Trying with remote %s" % remote.name)
try:
new_ref = _retrieve_from_remote(remote)
return remote, new_ref
# If not found continue with the next, else raise
except NotFoundException:
pass
else:
msg = "Unable to find '%s' in remotes" % str(conan_reference)
logger.debug("Not found in any remote")
recorder.recipe_install_error(conan_reference,
INSTALL_ERROR_MISSING, msg, None)
raise NotFoundException(msg)
def get_file_path(self, filepath, token):
try:
encoded_path, _, user = self.updown_auth_manager.get_resource_info(token)
if not self._valid_path(filepath, encoded_path):
logger.info("Invalid path file!! %s: %s" % (user, filepath))
raise NotFoundException("File not found")
logger.debug("Get file: user=%s path=%s" % (user, filepath))
file_path = os.path.normpath(os.path.join(self.base_store_folder, encoded_path))
return file_path
except (jwt.ExpiredSignature, jwt.DecodeError, AttributeError):
raise NotFoundException("File not found")
def _download_recipe(self, ref, output, remote_name, recorder):
def _retrieve_from_remote(_ref, the_remote):
output.info("Trying with '%s'..." % the_remote.name)
# If incomplete, resolve the latest in server
_ref = self._remote_manager.get_recipe(_ref, the_remote)
self._registry.refs.set(_ref, the_remote.name)
recorder.recipe_downloaded(ref, the_remote.url)
return _ref
if remote_name:
output.info("Retrieving from server '%s' " % remote_name)
remote = self._registry.remotes.get(remote_name)
else:
remote = self._registry.refs.get(ref)
if remote:
output.info("Retrieving from predefined remote '%s'" %
remote.name)
if remote:
try:
new_ref = _retrieve_from_remote(ref, remote)
return remote, new_ref
except NotFoundException:
msg = "%s was not found in remote '%s'" % (str(ref),
remote.name)
recorder.recipe_install_error(ref, INSTALL_ERROR_MISSING, msg,
remote.url)
raise NotFoundException(msg)
except RequestException as exc:
recorder.recipe_install_error(ref, INSTALL_ERROR_NETWORK,
str(exc), remote.url)
raise exc
output.info("Not found in local cache, looking in remotes...")
remotes = self._registry.remotes.list
if not remotes:
raise ConanException("No remote defined")
for remote in remotes:
try:
new_ref = _retrieve_from_remote(ref, remote)
return remote, new_ref
# If not found continue with the next, else raise
except NotFoundException:
pass
else:
msg = "Unable to find '%s' in remotes" % ref.full_repr()
recorder.recipe_install_error(ref, INSTALL_ERROR_MISSING, msg,
None)
raise NotFoundException(msg)
def _get_local_infos_min(self, reference):
result = {}
packages_path = self._paths.packages(reference)
subdirs = self._adapter.list_folder_subdirs(packages_path, level=1)
for package_id in subdirs:
# Read conaninfo
try:
package_reference = PackageReference(reference, package_id)
info_path = self._adapter.join_paths(
self._paths.package(package_reference, short_paths=None),
CONANINFO)
if not self._adapter.path_exists(info_path):
raise NotFoundException("")
conan_info_content = self._adapter.load(info_path)
conan_vars_info = ConanInfo.loads(
conan_info_content).serialize_min()
result[package_id] = conan_vars_info
except Exception as exc:
logger.error("Package %s has no ConanInfo file" %
str(package_reference))
if str(exc):
logger.error(str(exc))
return result
def search_packages(self, reference, query):
""" Return a dict like this:
{package_ID: {name: "OpenCV",
version: "2.14",
settings: {os: Windows}}}
param conan_ref: ConanFileReference object
"""
# GET PROPERTIES FROM QUERY
properties = get_properties_from_query(query)
logger.debug("SEARCH PACKAGE PROPERTIES: %s" % properties)
result = {}
packages_path = self._paths.packages(reference)
subdirs = self._adapter.list_folder_subdirs(packages_path, level=1)
for package_id in subdirs:
try:
package_reference = PackageReference(reference, package_id)
info_path = self._adapter.join_paths(
self._paths.package(package_reference, short_paths=None),
CONANINFO)
if not self._adapter.path_exists(info_path):
raise NotFoundException("")
conan_info_content = self._adapter.load(info_path)
conan_vars_info = ConanInfo.loads(conan_info_content)
if not self._filtered_by_properties(conan_vars_info,
properties):
result[package_id] = conan_vars_info.serialize_min()
except Exception as exc:
logger.error("Package %s has not ConanInfo file" %
str(package_reference))
if str(exc):
logger.error(str(exc))
return result
def get_path(self, path, package_id=None):
""" Return the contents for the given `path` inside current layout, it can
be a single file or the list of files in a directory
:param package_id: will retrieve the contents from the package directory
:param path: path relative to the cache reference or package folder
"""
assert not os.path.isabs(path)
if package_id is None: # Get the file in the exported files
folder = self.export()
else:
pref = PackageReference(self._ref, package_id)
folder = self.package(pref)
abs_path = os.path.join(folder, path)
if not os.path.exists(abs_path):
raise NotFoundException("The specified path doesn't exist")
if os.path.isdir(abs_path):
keep_python = get_env("CONAN_KEEP_PYTHON_FILES", False)
return sorted([
path for path in os.listdir(abs_path)
if not discarded_file(path, keep_python)
])
else:
return load(abs_path)
def get_package(self, package_reference, dest_folder, remote, output,
recorder):
package_id = package_reference.package_id
conanfile_path = self._client_cache.conanfile(package_reference.conan)
self._hook_manager.execute("pre_download_package",
conanfile_path=conanfile_path,
reference=package_reference.conan,
package_id=package_id,
remote=remote)
output.info("Retrieving package %s from remote '%s' " %
(package_id, remote.name))
rm_conandir(dest_folder) # Remove first the destination folder
t1 = time.time()
try:
zipped_files, new_ref, rev_time = self._call_remote(
remote, "get_package", package_reference, dest_folder)
with self._client_cache.update_metadata(new_ref.conan) as metadata:
metadata.packages[
new_ref.package_id].revision = new_ref.revision
metadata.packages[
new_ref.
package_id].recipe_revision = new_ref.conan.revision
metadata.packages[new_ref.package_id].time = rev_time
duration = time.time() - t1
log_package_download(package_reference, duration, remote,
zipped_files)
unzip_and_get_files(zipped_files,
dest_folder,
PACKAGE_TGZ_NAME,
output=self._output)
# Issue #214 https://github.com/conan-io/conan/issues/214
touch_folder(dest_folder)
if get_env("CONAN_READ_ONLY_CACHE", False):
make_read_only(dest_folder)
recorder.package_downloaded(package_reference, remote.url)
output.success('Package installed %s' % package_id)
except NotFoundException:
raise NotFoundException("Package binary '%s' not found in '%s'" %
(package_reference, remote.name))
except BaseException as e:
output.error("Exception while getting package: %s" %
str(package_reference.package_id))
output.error("Exception: %s %s" % (type(e), str(e)))
try:
output.warn("Trying to remove package folder: %s" %
dest_folder)
rmdir(dest_folder)
except OSError as e:
raise ConanException(
"%s\n\nCouldn't remove folder '%s', might be busy or open. Close any app "
"using it, and retry" % (str(e), dest_folder))
raise
self._hook_manager.execute("post_download_package",
conanfile_path=conanfile_path,
reference=package_reference.conan,
package_id=package_id,
remote=remote)
return new_ref
def _collects_refs_to_upload(self, package_id, reference_or_pattern,
confirm):
""" validate inputs and compute the refs (without revisions) to be uploaded
"""
if package_id and not check_valid_ref(reference_or_pattern,
strict_mode=False):
raise ConanException(
"-p parameter only allowed with a valid recipe reference, "
"not with a pattern")
if package_id or check_valid_ref(reference_or_pattern):
# Upload package
ref = ConanFileReference.loads(reference_or_pattern)
if ref.revision and not self._cache.config.revisions_enabled:
raise ConanException(
"Revisions not enabled in the client, specify a "
"reference without revision")
refs = [
ref,
]
confirm = True
else:
refs = search_recipes(self._cache, reference_or_pattern)
if not refs:
raise NotFoundException(
("No packages found matching pattern '%s'" %
reference_or_pattern))
return refs, confirm
def download(reference, package_ids, remote_name, recipe, remote_manager,
client_cache, out, recorder, loader, hook_manager):
assert(isinstance(reference, ConanFileReference))
output = ScopedOutput(str(reference), out)
registry = client_cache.registry
remote = registry.remotes.get(remote_name) if remote_name else registry.remotes.default
hook_manager.execute("pre_download", reference=reference, remote=remote)
# First of all download package recipe
try:
remote_manager.get_recipe(reference, remote)
except NotFoundException:
raise NotFoundException("'%s' not found in remote '%s'" % (str(reference), remote.name))
registry.refs.set(reference, remote.name)
conan_file_path = client_cache.conanfile(reference)
conanfile = loader.load_class(conan_file_path)
if not recipe: # Not only the recipe
# Download the sources too, don't be lazy
complete_recipe_sources(remote_manager, client_cache, conanfile, reference)
if not package_ids: # User didnt specify a specific package binary
output.info("Getting the complete package list from '%s'..." % str(reference))
packages_props = remote_manager.search_packages(remote, reference, None)
package_ids = list(packages_props.keys())
if not package_ids:
output.warn("No remote binary packages found in remote")
_download_binaries(conanfile, reference, package_ids, client_cache, remote_manager,
remote, output, recorder)
hook_manager.execute("post_download", conanfile_path=conan_file_path, reference=reference,
remote=remote)
def _download_file(self, url, auth, headers, file_path):
t1 = time.time()
try:
response = self.requester.get(url,
stream=True,
verify=self.verify,
auth=auth,
headers=headers)
except Exception as exc:
raise ConanException("Error downloading file %s: '%s'" %
(url, exception_message_safe(exc)))
if not response.ok:
if response.status_code == 404:
raise NotFoundException("Not found: %s" % url)
elif response.status_code == 401:
raise AuthenticationException()
raise ConanException("Error %d downloading file %s" %
(response.status_code, url))
try:
data = self._download_data(response, file_path)
duration = time.time() - t1
log_download(url, duration)
return data
except Exception as e:
logger.debug(e.__class__)
logger.debug(traceback.format_exc())
# If this part failed, it means problems with the connection to server
raise ConanConnectionError(
"Download failed, check server, possibly try again\n%s" %
str(e))
def _get_local_infos_min(paths, reference, v2_compatibility_mode=False):
result = {}
if not reference.revision and v2_compatibility_mode:
recipe_revisions = paths.get_recipe_revisions(reference)
else:
recipe_revisions = [reference]
for recipe_revision in recipe_revisions:
packages_path = paths.packages(recipe_revision)
subdirs = list_folder_subdirs(packages_path, level=1)
for package_id in subdirs:
if package_id in result:
continue
# Read conaninfo
try:
package_reference = PackageReference(reference, package_id)
info_path = os.path.join(paths.package(package_reference,
short_paths=None), CONANINFO)
if not os.path.exists(info_path):
raise NotFoundException("")
conan_info_content = load(info_path)
info = ConanInfo.loads(conan_info_content)
conan_vars_info = info.serialize_min()
result[package_id] = conan_vars_info
except Exception as exc:
logger.error("Package %s has no ConanInfo file" % str(package_reference))
if str(exc):
logger.error(str(exc))
return result
def upload(self, conan_reference, package_id=None, remote=None, all_packages=None,
force=False):
t1 = time.time()
remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, remote)
uploader = ConanUploader(self._client_cache, self._user_io, remote_proxy)
# Load conanfile to check if the build policy is set to always
try:
conanfile_path = self._client_cache.conanfile(conan_reference)
conan_file = self._loader().load_class(conanfile_path)
except NotFoundException:
raise NotFoundException("There is no local conanfile exported as %s"
% str(conan_reference))
# Can't use build_policy_always here because it's not loaded (only load_class)
if conan_file.build_policy == "always" and (all_packages or package_id):
raise ConanException("Conanfile has build_policy='always', "
"no packages can be uploaded")
if package_id: # Upload package
uploader.upload_package(PackageReference(conan_reference, package_id))
else: # Upload conans
uploader.upload_conan(conan_reference, all_packages=all_packages, force=force)
logger.debug("====> Time manager upload: %f" % (time.time() - t1))
def upload_recipe_file(name, version, username, channel, the_path, auth_user,
revision=None):
if "X-Checksum-Deploy" in request.headers:
raise NotFoundException("Not a checksum storage")
reference = ConanFileReference(name, version, username, channel, revision)
conan_service.upload_recipe_file(request.body, request.headers, reference, the_path,
auth_user)
def get_package_info(self, package_reference):
"""Gets a ConanInfo file from a package"""
url = "%s/download_urls" % self._package_url(package_reference)
urls = self._get_file_to_url_dict(url)
if not urls:
raise NotFoundException("Package not found!")
if CONANINFO not in urls:
raise NotFoundException("Package %s doesn't have the %s file!" % (package_reference,
CONANINFO))
# Get the info (in memory)
contents = self._download_files({CONANINFO: urls[CONANINFO]}, quiet=True)
# Unroll generator and decode shas (plain text)
contents = {key: decode_text(value) for key, value in dict(contents).items()}
return ConanInfo.loads(contents[CONANINFO])
def upload(self,
conan_reference,
package_id=None,
remote=None,
all_packages=None,
force=False):
remote_proxy = ConanProxy(self._paths, self._user_io,
self._remote_manager, remote)
uploader = ConanUploader(self._paths, self._user_io, remote_proxy)
# Load conanfile to check if the build policy is set to always
try:
conanfile_path = self._paths.conanfile(conan_reference)
output = ScopedOutput(str(conan_reference), self._user_io.out)
conan_file = self._loader().load_conan(conanfile_path,
output,
consumer=False)
except NotFoundException:
raise NotFoundException(
"There is no local conanfile exported as %s" %
str(conan_reference))
if conan_file.build_policy_always and (all_packages or package_id):
raise ConanException("Conanfile has build_policy='always', "
"no packages can be uploaded")
if package_id: # Upload package
uploader.upload_package(
PackageReference(conan_reference, package_id))
else: # Upload conans
uploader.upload_conan(conan_reference,
all_packages=all_packages,
force=force)
def load_conan(self, conan_file_path, consumer=False):
""" loads a ConanFile object from the given file
"""
# Check if precompiled exist, delete it
if os.path.exists(conan_file_path + "c"):
os.unlink(conan_file_path + "c")
if not os.path.exists(conan_file_path):
raise NotFoundException("%s not found!" % conan_file_path)
# We have to generate a new name for each conans
module_id = uuid.uuid1()
try:
loaded = imp.load_source("conan_conan%s" % module_id, conan_file_path)
except Exception:
import traceback
trace = traceback.format_exc().split('\n')
raise ConanException("Unable to load conanfile in %s\n%s" % (conan_file_path,
'\n'.join(trace[3:])))
try:
result = self._create_check_conan(loaded, consumer)
if consumer:
result.options.initialize_upstream(self._options)
return result
except Exception as e: # re-raise with file name
raise ConanException("%s: %s" % (conan_file_path, str(e)))
def load_conan_txt(self, conan_requirements_path, output):
if not os.path.exists(conan_requirements_path):
raise NotFoundException("Conanfile not found!")
conanfile = ConanFile(output, self._runner, self._settings.copy(),
os.path.dirname(conan_requirements_path))
try:
parser = ConanFileTextLoader(load(conan_requirements_path))
except Exception as e:
raise ConanException("%s:\n%s" % (conan_requirements_path, str(e)))
for requirement_text in parser.requirements:
ConanFileReference.loads(requirement_text) # Raise if invalid
conanfile.requires.add(requirement_text)
conanfile.generators = parser.generators
options = OptionsValues.loads(parser.options)
conanfile.options.values = options
conanfile.options.initialize_upstream(self._options)
# imports method
conanfile.imports = ConanFileTextLoader.imports_method(
conanfile, parser.import_parameters)
return conanfile
def get_package(self, package_reference, dest_folder, remote, output, recorder):
package_id = package_reference.package_id
output.info("Retrieving package %s from remote '%s' " % (package_id, remote.name))
rm_conandir(dest_folder) # Remove first the destination folder
t1 = time.time()
try:
urls = self._call_remote(remote, "get_package_urls", package_reference)
zipped_files = self._call_remote(remote, "download_files_to_folder", urls, dest_folder)
duration = time.time() - t1
log_package_download(package_reference, duration, remote, zipped_files)
unzip_and_get_files(zipped_files, dest_folder, PACKAGE_TGZ_NAME)
# Issue #214 https://github.com/conan-io/conan/issues/214
touch_folder(dest_folder)
if get_env("CONAN_READ_ONLY_CACHE", False):
make_read_only(dest_folder)
recorder.package_downloaded(package_reference, remote.url)
output.success('Package installed %s' % package_id)
except NotFoundException:
raise NotFoundException("Package binary '%s' not found in '%s'" % (package_reference, remote.name))
except BaseException as e:
output.error("Exception while getting package: %s" % str(package_reference.package_id))
output.error("Exception: %s %s" % (type(e), str(e)))
try:
output.warn("Trying to remove package folder: %s" % dest_folder)
rmdir(dest_folder)
except OSError as e:
raise ConanException("%s\n\nCouldn't remove folder '%s', might be busy or open. Close any app "
"using it, and retry" % (str(e), dest_folder))
raise
def _get_graph(self, reference, current_path, remote, options, settings,
filename, update, check_updates, manifest_manager, scopes,
package_settings, env, package_env):
loader = self._loader(current_path, settings, package_settings,
options, scopes, env, package_env)
# Not check for updates for info command, it'll be checked when dep graph is built
remote_proxy = ConanProxy(self._client_cache,
self._user_io,
self._remote_manager,
remote,
update=update,
check_updates=check_updates,
manifest_manager=manifest_manager)
if isinstance(reference, ConanFileReference):
project_reference = None
conanfile = loader.load_virtual(reference, current_path)
is_txt = True
else:
conanfile_path = reference
project_reference = "PROJECT"
output = ScopedOutput(project_reference, self._user_io.out)
try:
if filename and filename.endswith(".txt"):
raise NotFoundException("")
conan_file_path = os.path.join(conanfile_path, filename
or CONANFILE)
conanfile = loader.load_conan(conan_file_path,
output,
consumer=True)
is_txt = False
if conanfile.name is not None and conanfile.version is not None:
project_reference = "%s/%s@" % (conanfile.name,
conanfile.version)
project_reference += "PROJECT"
except NotFoundException: # Load requirements.txt
conan_path = os.path.join(conanfile_path, filename
or CONANFILE_TXT)
conanfile = loader.load_conan_txt(conan_path, output)
is_txt = True
# build deps graph and install it
local_search = None if update else self._search_manager
resolver = RequireResolver(self._user_io.out, local_search,
remote_proxy)
builder = DepsGraphBuilder(remote_proxy, self._user_io.out, loader,
resolver)
deps_graph = builder.load(None, conanfile)
# These lines are so the conaninfo stores the correct complete info
if is_txt:
conanfile.info.settings = loader._settings.values
conanfile.info.full_settings = loader._settings.values
conanfile.info.scope = self._current_scopes
conanfile.cpp_info = CppInfo(current_path)
conanfile.env_info = EnvInfo(current_path)
registry = RemoteRegistry(self._client_cache.registry,
self._user_io.out)
return (builder, deps_graph, project_reference, registry, conanfile,
remote_proxy, loader)
def _get_local_infos_min(client_cache, reference):
result = {}
packages_path = client_cache.packages(reference)
subdirs = list_folder_subdirs(packages_path, level=1)
for package_id in subdirs:
# Read conaninfo
try:
package_reference = PackageReference(reference, package_id)
info_path = os.path.join(
client_cache.package(package_reference, short_paths=None),
CONANINFO)
if not os.path.exists(info_path):
raise NotFoundException("")
conan_info_content = load(info_path)
metadata = client_cache.load_metadata(package_reference.conan)
recipe_revision = metadata.packages[package_id].recipe_revision
info = ConanInfo.loads(conan_info_content)
if reference.revision and recipe_revision and recipe_revision != reference.revision:
continue
conan_vars_info = info.serialize_min()
result[package_id] = conan_vars_info
except Exception as exc:
logger.error("Package %s has no ConanInfo file" %
str(package_reference))
if str(exc):
logger.error(str(exc))
return result
def _retrieve_recipe(self, conan_reference, output):
""" returns the requested conanfile object, retrieving it from
remotes if necessary. Can raise NotFoundException
"""
def _retrieve_from_remote(remote):
output.info("Trying with '%s'..." % remote.name)
export_path = self._client_cache.export(conan_reference)
result = self._remote_manager.get_recipe(conan_reference, export_path, remote)
self._registry.set_ref(conan_reference, remote)
return result
if self._remote_name:
output.info("Not found, retrieving from server '%s' " % self._remote_name)
remote = self._registry.remote(self._remote_name)
return _retrieve_from_remote(remote)
else:
output.info("Not found, looking in remotes...")
remotes = self._registry.remotes
for remote in remotes:
logger.debug("Trying with remote %s" % remote.name)
try:
return _retrieve_from_remote(remote)
# If exception continue with the next
except (ConanOutdatedClient, ConanConnectionError) as exc:
output.warn(str(exc))
if remote == remotes[-1]: # Last element not found
raise ConanConnectionError("All remotes failed")
except NotFoundException as exc:
if remote == remotes[-1]: # Last element not found
logger.debug("Not found in any remote, raising...%s" % exc)
raise NotFoundException("Unable to find '%s' in remotes"
% str(conan_reference))
raise ConanException("No remote defined")
def _get_conanfile_object(self,
loader,
reference_or_path,
conanfile_filename,
cwd=None):
"""cwd only used for virtuals, to pass it the current directory and make available the
conanfile.conanfile_directory (smell)"""
if isinstance(reference_or_path, ConanFileReference):
conanfile = loader.load_virtual([reference_or_path], cwd)
else:
output = ScopedOutput("PROJECT", self._user_io.out)
try:
if conanfile_filename and conanfile_filename.endswith(".txt"):
raise NotFoundException("")
conan_file_path = os.path.join(reference_or_path,
conanfile_filename or CONANFILE)
conanfile = loader.load_conan(conan_file_path,
output,
consumer=True)
except NotFoundException: # Load conanfile.txt
conan_path = os.path.join(reference_or_path, conanfile_filename
or CONANFILE_TXT)
conanfile = loader.load_conan_txt(conan_path, output)
return conanfile
def _get_package_urls(self, pref):
"""Gets a dict of filename:contents from package"""
url = self.conans_router.package_download_urls(pref)
urls = self._get_file_to_url_dict(url)
if not urls:
raise NotFoundException("Package not found!")
return urls