def build(self, conanfile_path, source_folder, build_folder, test=False):
""" Call to build() method saved on the conanfile.py
param conanfile_path: the original source directory of the user containing a
conanfile.py
"""
logger.debug("Building in %s" % build_folder)
logger.debug("Conanfile in %s" % conanfile_path)
try:
# Append env_vars to execution environment and clear when block code ends
output = ScopedOutput("Project", self._user_io.out)
conan_file = load_consumer_conanfile(conanfile_path, build_folder,
self._client_cache.settings,
self._runner, output)
except NotFoundException:
# TODO: Auto generate conanfile from requirements file
raise ConanException("'%s' file is needed for build.\n"
"Create a '%s' and move manually the "
"requirements and generators from '%s' file"
% (CONANFILE, CONANFILE, CONANFILE_TXT))
try:
os.chdir(build_folder)
conan_file._conanfile_directory = source_folder
conan_file.build_folder = build_folder
conan_file.source_folder = source_folder
with environment_append(conan_file.env):
conan_file.build()
if test:
conan_file.test()
except ConanException:
raise # Raise but not let to reach the Exception except (not print traceback)
except Exception:
import traceback
trace = traceback.format_exc().split('\n')
raise ConanException("Unable to build it successfully\n%s" % '\n'.join(trace[3:]))
def upload(self, conan_reference, package_id=None, remote=None, all_packages=None,
force=False):
t1 = time.time()
remote_proxy = ConanProxy(self._client_cache, self._user_io, self._remote_manager, remote)
uploader = ConanUploader(self._client_cache, self._user_io, remote_proxy)
# Load conanfile to check if the build policy is set to always
try:
conanfile_path = self._client_cache.conanfile(conan_reference)
conan_file = self._loader().load_class(conanfile_path)
except NotFoundException:
raise NotFoundException("There is no local conanfile exported as %s"
% str(conan_reference))
# Can't use build_policy_always here because it's not loaded (only load_class)
if conan_file.build_policy == "always" and (all_packages or package_id):
raise ConanException("Conanfile has build_policy='always', "
"no packages can be uploaded")
if package_id: # Upload package
uploader.upload_package(PackageReference(conan_reference, package_id))
else: # Upload conans
uploader.upload_conan(conan_reference, all_packages=all_packages, force=force)
logger.debug("====> Time manager upload: %f" % (time.time() - t1))
def export(self, user, conan_file_path, keep_source=False, filename=None):
""" Export the conans
param conanfile_path: the original source directory of the user containing a
conanfile.py
param user: user under this package will be exported
param channel: string (stable, testing,...)
"""
assert conan_file_path
logger.debug("Exporting %s" % conan_file_path)
try:
user_name, channel = user.split("/")
except:
user_name, channel = user, "testing"
src_folder = conan_file_path
conanfile_name = filename or CONANFILE
conan_file_path = os.path.join(conan_file_path, conanfile_name)
if ((os.path.exists(conan_file_path) and conanfile_name not in os.listdir(src_folder)) or
(conanfile_name != "conanfile.py" and conanfile_name.lower() == "conanfile.py")):
raise ConanException("Wrong '%s' case" % conanfile_name)
conan_linter(conan_file_path, self._user_io.out)
conanfile = load_export_conanfile(conan_file_path, self._user_io.out)
conan_ref = ConanFileReference(conanfile.name, conanfile.version, user_name, channel)
conan_ref_str = str(conan_ref)
# Maybe a platform check could be added, but depends on disk partition
refs = self._search_manager.search(conan_ref_str, ignorecase=True)
if refs and conan_ref not in refs:
raise ConanException("Cannot export package with same name but different case\n"
"You exported '%s' but already existing '%s'"
% (conan_ref_str, " ".join(str(s) for s in refs)))
output = ScopedOutput(str(conan_ref), self._user_io.out)
export_conanfile(output, self._client_cache, conanfile, src_folder, conan_ref, keep_source,
filename)
def __init__(
self,
base_folder=None,
current_folder=None,
servers=None,
users=None,
client_version=CLIENT_VERSION,
min_server_compatible_version=MIN_SERVER_COMPATIBLE_VERSION,
):
"""
storage_folder: Local storage path
current_folder: Current execution folder
servers: dict of {remote_name: TestServer}
logins is a list of (user, password) for auto input in order
if required==> [("lasote", "mypass"), ("other", "otherpass")]
"""
if users is None:
users = [(TESTING_REMOTE_PRIVATE_USER, TESTING_REMOTE_PRIVATE_PASS)]
self.users = users
self.servers = servers or {}
self.client_version = Version(client_version)
self.min_server_compatible_version = Version(min_server_compatible_version)
self.base_folder = base_folder or temp_folder()
# Define storage_folder, if not, it will be readed from conf file and pointed to real user home
self.storage_folder = os.path.join(self.base_folder, ".conan", "data")
# First initialization for set paths (some test copies data before first real initialization)
self.init_dynamic_vars()
self.default_settings(get_env("CONAN_COMPILER", "gcc"), get_env("CONAN_COMPILER_VERSION", "4.8"))
logger.debug("Client storage = %s" % self.storage_folder)
self.current_folder = current_folder or temp_folder()
def export(self, user, conan_file_path, keep_source=False):
""" Export the conans
param conanfile_path: the original source directory of the user containing a
conanfile.py
param user: user under this package will be exported
param channel: string (stable, testing,...)
"""
assert conan_file_path
logger.debug("Exporting %s" % conan_file_path)
user_name, channel = get_user_channel(user)
conan_file = self._loader().load_conan(os.path.join(conan_file_path, CONANFILE),
self._user_io.out)
url = getattr(conan_file, "url", None)
license_ = getattr(conan_file, "license", None)
if not url:
self._user_io.out.warn("Conanfile doesn't have 'url'.\n"
"It is recommended to add your repo URL as attribute")
if not license_:
self._user_io.out.warn("Conanfile doesn't have a 'license'.\n"
"It is recommended to add the package license as attribute")
conan_ref = ConanFileReference(conan_file.name, conan_file.version, user_name, channel)
conan_ref_str = str(conan_ref)
# Maybe a platform check could be added, but depends on disk partition
info = self.file_manager.search(conan_ref_str, ignorecase=True)
refs = {s for s in info.keys() if str(s).lower() == conan_ref_str.lower()}
if refs and conan_ref not in refs:
raise ConanException("Cannot export package with same name but different case\n"
"You exported '%s' but already existing '%s'"
% (conan_ref_str, " ".join(str(s) for s in refs)))
output = ScopedOutput(str(conan_ref), self._user_io.out)
export_conanfile(output, self._paths,
conan_file.exports, conan_file_path, conan_ref, keep_source)
def _retrieve_conanfile(self, conan_reference, output):
""" returns the requested conanfile object, retrieving it from
remotes if necessary. Can raise NotFoundException
"""
def _retrieve_from_remote(remote):
output.info("Trying with '%s'..." % remote.name)
result = self._remote_manager.get_conanfile(conan_reference, remote)
self._registry.set_ref(conan_reference, remote)
return result
if self._remote_name:
output.info("Not found, retrieving from server '%s' " % self._remote_name)
remote = self._registry.remote(self._remote_name)
return _retrieve_from_remote(remote)
else:
output.info("Not found, looking in remotes...")
remotes = self._registry.remotes
for remote in remotes:
logger.debug("Trying with remote %s" % remote.name)
try:
return _retrieve_from_remote(remote)
# If exception continue with the next
except (ConanOutdatedClient, ConanConnectionError) as exc:
output.warn(str(exc))
if remote == remotes[-1]: # Last element not found
raise ConanConnectionError("All remotes failed")
except NotFoundException as exc:
if remote == remotes[-1]: # Last element not found
logger.debug("Not found in any remote, raising...%s" % exc)
raise NotFoundException("Unable to find '%s' in remotes"
% str(conan_reference))
raise ConanException("No remote defined")
def _call_with_remote_selection(self, remote, method, *argc, **argv):
"""
Iterates in remotes until a response is found
"""
if remote:
return self._call_without_remote_selection(remote, method, *argc, **argv)
for remote in self.remote_names:
logger.debug("Trying with remote %s" % self.remote_url(remote))
self._remote_client.remote_url = self.remote_url(remote)
try:
result = self._call_without_remote_selection(remote, method, *argc, **argv)
self._output.success("Found in remote '%s'" % remote)
return result
# If exception continue with the next
except (ConanOutdatedClient, ConanConnectionError) as exc:
self._output.warn(str(exc))
if remote == self._remotes[-1][0]: # Last element not found
raise ConanConnectionError("All remotes failed")
except NotFoundException as exc:
if remote == self._remotes[-1][0]: # Last element not found
logger.debug("Not found in any remote, raising...%s" % exc)
raise
raise ConanException("No remote defined")
def export(self, user, conan_file_path, keep_source=False):
""" Export the conans
param conanfile_path: the original source directory of the user containing a
conanfile.py
param user: user under this package will be exported
param channel: string (stable, testing,...)
"""
assert conan_file_path
logger.debug("Exporting %s" % conan_file_path)
user_name, channel = get_user_channel(user)
conan_file = self._loader().load_conan(os.path.join(conan_file_path, CONANFILE),
self._user_io.out)
url = getattr(conan_file, "url", None)
license_ = getattr(conan_file, "license", None)
if not url:
self._user_io.out.warn("Conanfile doesn't have 'url'.\n"
"It is recommended to add your repo URL as attribute")
if not license_:
self._user_io.out.warn("Conanfile doesn't have a 'license'.\n"
"It is recommended to add the package license as attribute")
conan_ref = ConanFileReference(conan_file.name, conan_file.version, user_name, channel)
output = ScopedOutput(str(conan_ref), self._user_io.out)
export_conanfile(output, self._paths,
conan_file.exports, conan_file_path, conan_ref, keep_source)
def retry_with_new_token(self, *args, **kwargs):
"""Try LOGIN_RETRIES to obtain a password from user input for which
we can get a valid token from api_client. If a token is returned,
credentials are stored in localdb and rest method is called"""
for _ in range(LOGIN_RETRIES):
user, password = self._user_io.request_login(self._remote.name, self.user)
token = None
try:
token = self.authenticate(user, password)
except AuthenticationException:
if self.user is None:
self._user_io.out.error('Wrong user or password')
else:
self._user_io.out.error(
'Wrong password for user "%s"' % self.user)
self._user_io.out.info(
'You can change username with "conan user <username>"')
if token:
logger.debug("Got token: %s" % str(token))
self._rest_client.token = token
self.user = user
self._store_login((user, token))
# Set custom headers of mac_digest and username
self.set_custom_headers(user)
return wrapper(self, *args, **kwargs)
raise AuthenticationException("Too many failed login attempts, bye!")
def _get_file_conf(self, section, varname=None):
""" Gets the section or variable from config file.
If the queried element is not found an exception is raised.
"""
try:
if not os.path.exists(self.config_filename):
jwt_random_secret = ''.join(random.choice(string.ascii_letters) for _ in range(24))
updown_random_secret = ''.join(random.choice(string.ascii_letters) for _ in range(24))
server_conf = default_server_conf.format(jwt_secret=jwt_random_secret,
updown_secret=updown_random_secret)
save(self.config_filename, server_conf)
if not self._loaded:
self._loaded = True
# To avoid encoding problems we use our tools.load
if six.PY3:
self.read_string(tools.load(self.config_filename))
else:
self.read(self.config_filename)
if varname:
section = dict(self.items(section))
return section[varname]
else:
return self.items(section)
except NoSectionError:
raise ConanException("No section '%s' found" % section)
except Exception as exc:
logger.debug(exc)
raise ConanException("Invalid configuration, "
"missing %s: %s" % (section, varname))
def _remove_file(self, path, conan_ref, msg=""):
try:
logger.debug("Removing folder %s" % path)
if os.path.exists(path):
os.remove(path)
except OSError as e:
raise ConanException("Unable to remove %s %s\n\t%s" % (repr(conan_ref), msg, str(e)))
def upload_files(self, file_urls, files, output, retry, retry_wait):
t1 = time.time()
failed = []
uploader = Uploader(self.requester, output, self.verify_ssl)
# Take advantage of filenames ordering, so that conan_package.tgz and conan_export.tgz
# can be < conanfile, conaninfo, and sent always the last, so smaller files go first
for filename, resource_url in sorted(file_urls.items(), reverse=True):
output.rewrite_line("Uploading %s" % filename)
auth, dedup = self._file_server_capabilities(resource_url)
try:
response = uploader.upload(resource_url, files[filename], auth=auth, dedup=dedup,
retry=retry, retry_wait=retry_wait, headers=self._put_headers)
output.writeln("")
if not response.ok:
output.error("\nError uploading file: %s, '%s'" % (filename, response.content))
failed.append(filename)
else:
pass
except Exception as exc:
output.error("\nError uploading file: %s, '%s'" % (filename, exc))
failed.append(filename)
if failed:
raise ConanException("Execute upload again to retry upload the failed files: %s"
% ", ".join(failed))
else:
logger.debug("\nAll uploaded! Total time: %s\n" % str(time.time() - t1))
def __init__(self, base_folder=None, current_folder=None,
servers=None, users=None, client_version=CLIENT_VERSION,
min_server_compatible_version=MIN_SERVER_COMPATIBLE_VERSION,
requester_class=None, runner=None, path_with_spaces=True):
"""
storage_folder: Local storage path
current_folder: Current execution folder
servers: dict of {remote_name: TestServer}
logins is a list of (user, password) for auto input in order
if required==> [("lasote", "mypass"), ("other", "otherpass")]
"""
self.all_output = "" # For debugging purpose, append all the run outputs
self.users = users or {"default":
[(TESTING_REMOTE_PRIVATE_USER, TESTING_REMOTE_PRIVATE_PASS)]}
self.client_version = Version(str(client_version))
self.min_server_compatible_version = Version(str(min_server_compatible_version))
self.base_folder = base_folder or temp_folder(path_with_spaces)
# Define storage_folder, if not, it will be read from conf file & pointed to real user home
self.storage_folder = os.path.join(self.base_folder, ".conan", "data")
self.client_cache = ClientCache(self.base_folder, self.storage_folder, TestBufferConanOutput())
self.search_manager = DiskSearchManager(self.client_cache)
self.requester_class = requester_class
self.conan_runner = runner
self.update_servers(servers)
self.init_dynamic_vars()
logger.debug("Client storage = %s" % self.storage_folder)
self.current_folder = current_folder or temp_folder(path_with_spaces)
def _get_file_conf(self, section, varname=None):
"""Gets the section from config file or raises an exception"""
try:
if not os.path.exists(self.config_filename):
jwt_random_secret = ''.join(random.choice(string.ascii_letters) for _ in range(24))
updown_random_secret = ''.join(random.choice(string.ascii_letters) for _ in range(24))
server_conf = default_server_conf.format(jwt_secret=jwt_random_secret,
updown_secret=updown_random_secret)
save(self.config_filename, server_conf)
if not self._loaded:
self._loaded = True
self.read(self.config_filename)
if varname:
section = dict(self.items(section))
return section[varname]
else:
return self.items(section)
except NoSectionError as exc:
raise ConanException("No section '%s' found" % section)
except Exception as exc:
logger.debug(exc)
raise ConanException("Invalid configuration, "
"missing %s: %s" % (section, varname))
def _handle_ret(self, ret):
ret_version_status = ret.headers.get("X-Conan-Client-Version-Check", None)
logger.debug("Remote version check result '%s' " % ret_version_status)
if ret_version_status:
server_version = ret.headers.get("X-Conan-Server-Version", None)
server_version = Version(server_version)
logger.debug("Remote version '%s' " % server_version)
if ret_version_status == "current":
return
elif ret_version_status == "outdated":
msg = "A new conan version (v%s) is available in current remote. " % server_version
self.out.warn(msg + "Please, upgrade conan client to avoid deprecation.")
elif ret_version_status == "deprecated":
msg = "Your conan's client version is deprecated for the " "current remote (v%s). " % server_version
# If we operate with a fixed remote (-r XXX) this exception will stop the execution
# but if not, remote_manager will continue with the next remote ignoring this.
raise ConanOutdatedClient(msg + "Upgrade conan client.")
elif ret_version_status == "server_oudated":
if server_version < self.min_server_compatible_version:
msg = (
"Your conan's client is incompatible with this remote."
" The server is deprecated. (v%s). "
"Please, contact with your system administrator"
" and upgrade the server." % server_version
)
raise ConanOutdatedClient(msg)
else:
msg = (
"The conan remote version is outdated (v%s). "
"Please, contact with your system administrator and upgrade the "
"remote to avoid deprecation." % server_version
)
self.out.warn(msg)
def export(self, user, conan_file_path, keep_source=False):
""" Export the conans
param conanfile_path: the original source directory of the user containing a
conanfile.py
param user: user under this package will be exported
param channel: string (stable, testing,...)
"""
assert conan_file_path
logger.debug("Exporting %s" % conan_file_path)
user_name, channel = get_user_channel(user)
conan_file = self._loader().load_class(os.path.join(conan_file_path, CONANFILE))
for field in ["url", "license", "description"]:
field_value = getattr(conan_file, field, None)
if not field_value:
self._user_io.out.warn("Conanfile doesn't have '%s'.\n"
"It is recommended to add it as attribute" % field)
conan_ref = ConanFileReference(conan_file.name, conan_file.version, user_name, channel)
conan_ref_str = str(conan_ref)
# Maybe a platform check could be added, but depends on disk partition
refs = self._search_manager.search(conan_ref_str, ignorecase=True)
if refs and conan_ref not in refs:
raise ConanException("Cannot export package with same name but different case\n"
"You exported '%s' but already existing '%s'"
% (conan_ref_str, " ".join(str(s) for s in refs)))
output = ScopedOutput(str(conan_ref), self._user_io.out)
export_conanfile(output, self._client_cache, conan_file.exports, conan_file_path,
conan_ref, conan_file.short_paths, keep_source)
def _build(self, nodes_by_level, skip_private_nodes, build_mode):
""" The build assumes an input of conans ordered by degree, first level
should be indpendent from each other, the next-second level should have
dependencies only to first level conans.
param nodes_by_level: list of lists [[nodeA, nodeB], [nodeC], [nodeD, ...], ...]
build_mode => ["*"] if user wrote "--build"
=> ["hello*", "bye*"] if user wrote "--build hello --build bye"
=> False if user wrote "never"
=> True if user wrote "missing"
"""
# Now build each level, starting from the most independent one
for level in nodes_by_level:
for node in level:
if node in skip_private_nodes:
continue
conan_ref, conan_file = node
# it is possible that the root conans
# is not inside the storage but in a user folder, and thus its
# treatment is different
if not conan_ref:
continue
logger.debug("Building node %s" % repr(conan_ref))
self._build_node(conan_ref, conan_file, build_mode)
def __init__(self, base_folder=None, current_folder=None,
servers=None, users=None, client_version=CLIENT_VERSION,
min_server_compatible_version=MIN_SERVER_COMPATIBLE_VERSION):
"""
storage_folder: Local storage path
current_folder: Current execution folder
servers: dict of {remote_name: TestServer}
logins is a list of (user, password) for auto input in order
if required==> [("lasote", "mypass"), ("other", "otherpass")]
"""
self.users = users or {"default":
[(TESTING_REMOTE_PRIVATE_USER, TESTING_REMOTE_PRIVATE_PASS)]}
self.servers = servers or {}
self.client_version = Version(str(client_version))
self.min_server_compatible_version = Version(str(min_server_compatible_version))
self.base_folder = base_folder or temp_folder()
# Define storage_folder, if not, it will be read from conf file & pointed to real user home
self.storage_folder = os.path.join(self.base_folder, ".conan", "data")
self.paths = ConanPaths(self.base_folder, self.storage_folder, TestBufferConanOutput())
self.default_settings(get_env("CONAN_COMPILER", "gcc"),
get_env("CONAN_COMPILER_VERSION", "4.8"))
self.init_dynamic_vars()
save(self.paths.registry, "")
registry = RemoteRegistry(self.paths.registry, TestBufferConanOutput())
for name, server in self.servers.items():
registry.add(name, server.fake_url)
logger.debug("Client storage = %s" % self.storage_folder)
self.current_folder = current_folder or temp_folder()
def upload_package(self, package_ref, index=1, total=1):
"""Uploads the package identified by package_id"""
msg = ("Uploading package %d/%d: %s" % (index, total, str(package_ref.package_id)))
t1 = time.time()
self._user_io.out.info(msg)
self._remote_proxy.upload_package(package_ref)
logger.debug("====> Time uploader upload_package: %f" % (time.time() - t1))
def search_packages(self, reference, query):
""" Return a dict like this:
{package_ID: {name: "OpenCV",
version: "2.14",
settings: {os: Windows}}}
param conan_ref: ConanFileReference object
"""
# GET PROPERTIES FROM QUERY
properties = get_properties_from_query(query)
logger.debug("SEARCH PACKAGE PROPERTIES: %s" % properties)
result = {}
packages_path = self._paths.packages(reference)
subdirs = self._adapter.list_folder_subdirs(packages_path, level=1)
for package_id in subdirs:
try:
package_reference = PackageReference(reference, package_id)
info_path = self._adapter.join_paths(self._paths.package(package_reference,
short_paths=None),
CONANINFO)
if not self._adapter.path_exists(info_path, self._paths.store):
raise NotFoundException("")
conan_info_content = self._adapter.load(info_path)
conan_vars_info = ConanInfo.loads(conan_info_content)
if not self._filtered_by_properties(conan_vars_info, properties):
result[package_id] = conan_vars_info.serialize_min()
except Exception as exc:
logger.error("Package %s has not ConanInfo file" % str(package_reference))
if str(exc):
logger.error(str(exc))
return result
def __init__(self, base_path=None, read_permissions=None,
write_permissions=None, users=None, base_url=None, plugins=None,
server_version=None,
min_client_compatible_version=None,
server_capabilities=None):
plugins = plugins or []
if not base_path:
base_path = temp_folder()
if server_capabilities is None:
server_capabilities = SERVER_CAPABILITIES # Default enabled
if not os.path.exists(base_path):
raise Exception("Base path not exist! %s")
# Define storage_folder, if not, it will be readed from conf file and pointed to real user home
self.storage_folder = os.path.join(base_path, ".conan_server", "data")
mkdir(self.storage_folder)
server_config = migrate_and_get_server_config(base_path, self.storage_folder)
if TestServerLauncher.port == 0:
TestServerLauncher.port = server_config.port
# Encode and Decode signature for Upload and Download service
updown_auth_manager = JWTUpDownAuthManager(server_config.updown_secret,
server_config.authorize_timeout)
self.file_manager = get_file_manager(server_config, public_url=base_url,
updown_auth_manager=updown_auth_manager)
self.search_manager = DiskSearchManager(SimplePaths(server_config.disk_storage_path))
# Prepare some test users
if not read_permissions:
read_permissions = server_config.read_permissions
read_permissions.append(("private_library/1.0.0@private_user/testing", "*"))
read_permissions.append(("*/*@*/*", "*"))
if not write_permissions:
write_permissions = server_config.write_permissions
if not users:
users = dict(server_config.users)
users[TESTING_REMOTE_PRIVATE_USER] = TESTING_REMOTE_PRIVATE_PASS
authorizer = BasicAuthorizer(read_permissions, write_permissions)
authenticator = BasicAuthenticator(users)
credentials_manager = JWTCredentialsManager(server_config.jwt_secret,
server_config.jwt_expire_time)
logger.debug("Storage path: %s" % self.storage_folder)
self.port = TestServerLauncher.port
self.ra = ConanServer(self.port, credentials_manager, updown_auth_manager,
authorizer, authenticator, self.file_manager, self.search_manager,
server_version, min_client_compatible_version,
server_capabilities)
for plugin in plugins:
self.ra.api_v1.install(plugin)
def upload_package(self, package_reference, the_files, retry, retry_wait):
"""
basedir: Base directory with the files to upload (for read the files in disk)
relative_files: relative paths to upload
"""
self.check_credentials()
t1 = time.time()
# Get the remote snapshot
remote_snapshot = self._get_package_snapshot(package_reference)
local_snapshot = {filename: md5sum(abs_path) for filename, abs_path in the_files.items()}
# Get the diff
new, modified, deleted = diff_snapshots(local_snapshot, remote_snapshot)
files_to_upload = {filename: the_files[filename] for filename in new + modified}
if files_to_upload: # Obtain upload urls
url = "%s/conans/%s/packages/%s/upload_urls" % (self._remote_api_url,
"/".join(package_reference.conan),
package_reference.package_id)
filesizes = {filename: os.stat(abs_path).st_size for filename,
abs_path in files_to_upload.items()}
self._output.rewrite_line("Requesting upload permissions...")
urls = self._get_json(url, data=filesizes)
self._output.rewrite_line("Requesting upload permissions...Done!")
self._output.writeln("")
self.upload_files(urls, files_to_upload, self._output, retry, retry_wait)
else:
self._output.rewrite_line("Package is up to date.")
self._output.writeln("")
if deleted:
self._remove_package_files(package_reference, deleted)
logger.debug("====> Time rest client upload_package: %f" % (time.time() - t1))
def _remove(self, path, conan_ref, msg=""):
try:
logger.debug("Removing folder %s" % path)
rm_conandir(path)
except OSError:
error_msg = "Folder busy (open or some file open): %s" % path
raise ConanException("%s: Unable to remove %s\n\t%s"
% (repr(conan_ref), msg, error_msg))
def log_command(name, parameters):
if name == "authenticate" and "password" in parameters:
parameters = copy.copy(
parameters) # Ensure we don't alter any app object like args
parameters["password"] = MASKED_FIELD
_append_action("COMMAND", {"name": name, "parameters": parameters})
logger.debug("CONAN_API: %s(%s)" %
(name, ",".join("%s=%s" % (k, v)
for k, v in parameters.items())))
def upload_package(self, package_ref, index=1, total=1):
"""Uploads the package identified by package_id"""
msg = ("Uploading package %d/%d: %s" %
(index, total, str(package_ref.package_id)))
t1 = time.time()
self._user_io.out.info(msg)
self._remote_proxy.upload_package(package_ref)
logger.debug("====> Time uploader upload_package: %f" %
(time.time() - t1))
def _remove_file(self, path, conan_ref, msg=""):
try:
logger.debug("Removing folder %s" % path)
if os.path.exists(path):
os.remove(path)
except OSError:
error_msg = "File busy (open): %s" % path
raise ConanException("Unable to remove %s %s\n\t%s"
% (repr(conan_ref), msg, error_msg))
def _remove_file(self, path, conan_ref, msg=""):
try:
logger.debug("Removing file %s" % path)
if os.path.exists(path):
os.remove(path)
except OSError:
error_msg = "File busy (open): %s" % path
raise ConanException("Unable to remove %s %s\n\t%s" %
(repr(conan_ref), msg, error_msg))
def _create_new_node(self,
current_node,
dep_graph,
requirement,
name_req,
check_updates,
update,
remote_name,
processed_profile,
alias_ref=None):
""" creates and adds a new node to the dependency graph
"""
try:
result = self._proxy.get_recipe(requirement.ref, check_updates,
update, remote_name,
self._recorder)
except ConanException as e:
if current_node.ref:
self._output.error(
"Failed requirement '%s' from '%s'" %
(requirement.ref, current_node.conanfile.display_name))
raise e
conanfile_path, recipe_status, remote, new_ref = result
dep_conanfile = self._loader.load_conanfile(conanfile_path,
processed_profile,
ref=requirement.ref)
if recipe_status == RECIPE_EDITABLE:
dep_conanfile.in_local_cache = False
dep_conanfile.develop = True
if getattr(dep_conanfile, "alias", None):
alias_ref = alias_ref or new_ref.copy_clear_rev()
requirement.ref = ConanFileReference.loads(dep_conanfile.alias)
dep_graph.aliased[alias_ref] = requirement.ref
return self._create_new_node(current_node,
dep_graph,
requirement,
name_req,
check_updates,
update,
remote_name,
processed_profile,
alias_ref=alias_ref)
logger.debug("GRAPH: new_node: %s" % str(new_ref))
new_node = Node(new_ref, dep_conanfile)
new_node.revision_pinned = requirement.ref.revision is not None
new_node.recipe = recipe_status
new_node.remote = remote
new_node.ancestors = current_node.ancestors.copy()
new_node.ancestors.add(current_node.name)
dep_graph.add_node(new_node)
dep_graph.add_edge(current_node, new_node, requirement.private,
requirement.build_require)
return new_node
def _upload_package(self,
pref,
retry=None,
retry_wait=None,
integrity_check=False,
policy=None,
p_remote=None):
assert (pref.revision
is not None), "Cannot upload a package without PREV"
assert (pref.ref.revision
is not None), "Cannot upload a package without RREV"
pkg_layout = self._cache.package_layout(pref.ref)
conanfile_path = pkg_layout.conanfile()
self._hook_manager.execute("pre_upload_package",
conanfile_path=conanfile_path,
reference=pref.ref,
package_id=pref.id,
remote=p_remote)
t1 = time.time()
the_files = self._compress_package_files(pref, integrity_check)
if policy == UPLOAD_POLICY_SKIP:
return None
files_to_upload, deleted = self._package_files_to_upload(
pref, policy, the_files, p_remote)
if files_to_upload or deleted:
self._remote_manager.upload_package(pref, files_to_upload, deleted,
p_remote, retry, retry_wait)
logger.debug("UPLOAD: Time upload package: %f" %
(time.time() - t1))
else:
self._output.info("Package is up to date, upload skipped")
duration = time.time() - t1
log_package_upload(pref, duration, the_files, p_remote)
self._hook_manager.execute("post_upload_package",
conanfile_path=conanfile_path,
reference=pref.ref,
package_id=pref.id,
remote=p_remote)
logger.debug("UPLOAD: Time uploader upload_package: %f" %
(time.time() - t1))
# Update the package metadata
checksums = calc_files_checksum(the_files)
with pkg_layout.update_metadata() as metadata:
cur_package_remote = metadata.packages[pref.id].remote
if not cur_package_remote:
metadata.packages[pref.id].remote = p_remote.name
metadata.packages[pref.id].checksums = checksums
return pref
def infix_to_postfix(exp):
"""
Translates an infix expression to postfix using an standard algorithm
with little hacks for parse complex expressions like "compiler.version=4"
instead of just numbers and without taking in account the operands priority
except the priority specified by the "("
@param exp: String with an expression with & and | operators,
e.j: "os=Windows & (compiler=gcc | compiler.version=3)"
e.j: "os=Windows AND (compiler=gcc or compiler.version=3)"
@return List with the postfix expression
"""
# To ease the parser, operators only with one character
exp = exp.replace(" AND ",
"&").replace(" OR ",
"|").replace(" and ",
"&").replace(" or ", "|")
output = []
stack = []
i = -1
while (i < len(exp) - 1):
i += 1
char = exp[i]
if char == " ": # Ignore spaces between expressions and operators
continue
if char == ")": # Pop the stack until "(" and send them to output
popped = None
while (popped != "(" and stack):
popped = stack.pop()
if popped != "(":
output.append(popped)
if popped != "(":
raise Exception("Bad expression, not balanced parenthesis")
elif is_operator(char):
# Same operations has the same priority
# replace this lines if the operators need to have
# some priority
if stack and is_operator(stack[:-1]):
popped = stack.pop()
output.append(popped)
stack.append(char)
elif char == "(":
stack.append("(")
else: # Parse an expression, in our case something like "compiler=gcc"
logger.debug("Parsing expression from : %s" % exp[i:])
expr = _parse_expression(exp[i:])
logger.debug("Parsed expression: %s" % expr)
i += len(expr) - 1
output.append(expr)
# Append remaining elements
if "(" in stack:
raise Exception("Bad expression, not balanced parenthesis")
output.extend(stack)
return output
def download(self, url, file_path=None, auth=None, retry=1, retry_wait=0):
if file_path and os.path.exists(file_path):
# Should not happen, better to raise, probably we had to remove the dest folder before
raise ConanException(
"Error, the file to download already exists: '%s'" % file_path)
t1 = time.time()
ret = bytearray()
response = call_with_retry(self.output, retry, retry_wait,
self._download_file, url, auth)
if not response.ok: # Do not retry if not found or whatever controlled error
raise ConanException("Error %d downloading file %s" %
(response.status_code, url))
try:
total_length = response.headers.get('content-length')
if total_length is None: # no content length header
if not file_path:
ret += response.content
else:
save(file_path, response.content, append=True)
else:
dl = 0
total_length = int(total_length)
last_progress = None
chunk_size = 1024 if not file_path else 1024 * 100
for data in response.iter_content(chunk_size=chunk_size):
dl += len(data)
if not file_path:
ret.extend(data)
else:
save(file_path, data, append=True)
units = progress_units(dl, total_length)
progress = human_readable_progress(dl, total_length)
if last_progress != units: # Avoid screen refresh if nothing has change
if self.output:
print_progress(self.output, units, progress)
last_progress = units
duration = time.time() - t1
log_download(url, duration)
if not file_path:
return bytes(ret)
else:
return
except Exception as e:
logger.debug(e.__class__)
logger.debug(traceback.format_exc())
# If this part failed, it means problems with the connection to server
raise ConanConnectionError(
"Download failed, check server, possibly try again\n%s" %
str(e))
def build(self, conanfile_path, current_path, test=False, filename=None, profile_name=None):
""" Call to build() method saved on the conanfile.py
param conanfile_path: the original source directory of the user containing a
conanfile.py
"""
logger.debug("Building in %s" % current_path)
logger.debug("Conanfile in %s" % conanfile_path)
if filename and filename.endswith(".txt"):
raise ConanException("A conanfile.py is needed to call 'conan build'")
conanfile_file = os.path.join(conanfile_path, filename or CONANFILE)
try:
output = ScopedOutput("Project", self._user_io.out)
conan_file = self._loader(current_path).load_conan(conanfile_file, output,
consumer=True)
except NotFoundException:
# TODO: Auto generate conanfile from requirements file
raise ConanException("'%s' file is needed for build.\n"
"Create a '%s' and move manually the "
"requirements and generators from '%s' file"
% (CONANFILE, CONANFILE, CONANFILE_TXT))
try:
build_info_file = os.path.join(current_path, BUILD_INFO)
if os.path.exists(build_info_file):
try:
deps_cpp_info = DepsCppInfo.loads(load(build_info_file))
conan_file.deps_cpp_info = deps_cpp_info
except:
pass
env_file = os.path.join(current_path, "conanenv.txt")
if os.path.exists(env_file):
try:
deps_env_info = DepsEnvInfo.loads(load(env_file))
conan_file.deps_env_info = deps_env_info
except:
pass
os.chdir(current_path)
conan_file._conanfile_directory = conanfile_path
# Append env_vars to execution environment and clear when block code ends
profile = self._read_profile(profile_name)
env_vars = self._read_profile_env_vars(profile)
with environment_append(env_vars):
conan_file.build()
if test:
conan_file.test()
except ConanException:
raise # Raise but not let to reach the Exception except (not print traceback)
except Exception:
import traceback
trace = traceback.format_exc().split('\n')
raise ConanException("Unable to build it successfully\n%s" % '\n'.join(trace[3:]))
def __init__(self, base_path=None, read_permissions=None,
write_permissions=None, users=None, base_url=None, plugins=None,
server_version=None,
min_client_compatible_version=None):
plugins = plugins or []
if not base_path:
base_path = temp_folder()
if not os.path.exists(base_path):
raise Exception("Base path not exist! %s")
# Define storage_folder, if not, it will be readed from conf file and pointed to real user home
storage_folder = os.path.join(base_path, ".conan_server", "data")
mkdir(storage_folder)
server_config = migrate_and_get_server_config(base_path, storage_folder)
if TestServerLauncher.port == 0:
TestServerLauncher.port = server_config.port
# Encode and Decode signature for Upload and Download service
updown_auth_manager = JWTUpDownAuthManager(server_config.updown_secret,
server_config.authorize_timeout)
self.file_manager = get_file_manager(server_config, public_url=base_url,
updown_auth_manager=updown_auth_manager)
search_adapter = DiskSearchAdapter()
self.search_manager = DiskSearchManager(SimplePaths(server_config.disk_storage_path), search_adapter)
# Prepare some test users
if not read_permissions:
read_permissions = server_config.read_permissions
read_permissions.append(("private_library/1.0.0@private_user/testing", "*"))
read_permissions.append(("*/*@*/*", "*"))
if not write_permissions:
write_permissions = server_config.write_permissions
if not users:
users = dict(server_config.users)
users[TESTING_REMOTE_PRIVATE_USER] = TESTING_REMOTE_PRIVATE_PASS
authorizer = BasicAuthorizer(read_permissions, write_permissions)
authenticator = BasicAuthenticator(users)
credentials_manager = JWTCredentialsManager(server_config.jwt_secret,
server_config.jwt_expire_time)
logger.debug("Storage path: %s" % storage_folder)
self.port = TestServerLauncher.port
TestServerLauncher.port += 1
self.ra = ConanServer(self.port, False, credentials_manager, updown_auth_manager,
authorizer, authenticator, self.file_manager, self.search_manager,
server_version, min_client_compatible_version)
for plugin in plugins:
self.ra.api_v1.install(plugin)
def build(graph_manager, hook_manager, conanfile_path,
source_folder, build_folder, package_folder, install_folder,
test=False, should_configure=True, should_build=True, should_install=True,
should_test=True):
""" Call to build() method saved on the conanfile.py
param conanfile_path: path to a conanfile.py
"""
logger.debug("BUILD: folder '%s'" % build_folder)
logger.debug("BUILD: Conanfile at '%s'" % conanfile_path)
try:
conan_file = graph_manager.load_consumer_conanfile(conanfile_path, install_folder,
deps_info_required=True, test=test)
except NotFoundException:
# TODO: Auto generate conanfile from requirements file
raise ConanException("'%s' file is needed for build.\n"
"Create a '%s' and move manually the "
"requirements and generators from '%s' file"
% (CONANFILE, CONANFILE, CONANFILE_TXT))
if test:
try:
conan_file.requires.add_ref(test)
except ConanException:
pass
conan_file.should_configure = should_configure
conan_file.should_build = should_build
conan_file.should_install = should_install
conan_file.should_test = should_test
try:
mkdir(build_folder)
os.chdir(build_folder)
conan_file.build_folder = build_folder
conan_file.source_folder = source_folder
conan_file.package_folder = package_folder
conan_file.install_folder = install_folder
hook_manager.execute("pre_build", conanfile=conan_file,
conanfile_path=conanfile_path)
with get_env_context_manager(conan_file):
conan_file.output.highlight("Running build()")
with conanfile_exception_formatter(str(conan_file), "build"):
conan_file.build()
hook_manager.execute("post_build", conanfile=conan_file,
conanfile_path=conanfile_path)
if test:
conan_file.output.highlight("Running test()")
with conanfile_exception_formatter(str(conan_file), "test"):
conan_file.test()
except ConanException:
raise # Raise but not let to reach the Exception except (not print traceback)
except Exception:
import traceback
trace = traceback.format_exc().split('\n')
raise ConanException("Unable to build it successfully\n%s" % '\n'.join(trace[3:]))
def upload_package(self, package_ref, index=1, total=1, retry=None, retry_wait=None,
skip_upload=False, integrity_check=False):
"""Uploads the package identified by package_id"""
msg = ("Uploading package %d/%d: %s" % (index, total, str(package_ref.package_id)))
t1 = time.time()
self._user_io.out.info(msg)
self._remote_proxy.upload_package(package_ref, retry, retry_wait, skip_upload,
integrity_check)
logger.debug("====> Time uploader upload_package: %f" % (time.time() - t1))
def check_credentials(self):
"""If token is not valid will raise AuthenticationException.
User will be asked for new user/pass"""
url = self.router.common_check_credentials()
logger.debug("REST: Check credentials: %s" % url)
ret = self.requester.get(url,
auth=self.auth,
headers=self.custom_headers,
verify=self.verify_ssl)
return ret
def run_build_method(conanfile, hook_manager, **hook_kwargs):
hook_manager.execute("pre_build", conanfile=conanfile, **hook_kwargs)
logger.debug("Call conanfile.build() with files in build folder: %s", os.listdir(conanfile.build_folder))
with get_env_context_manager(conanfile):
conanfile.output.highlight("Calling build()")
with conanfile_exception_formatter(str(conanfile), "build"):
conanfile.build()
hook_manager.execute("post_build", conanfile=conanfile, **hook_kwargs)
def remove_conanfile(self, ref):
""" Remove a recipe and packages """
self.check_credentials()
url = self.router.remove_recipe(ref)
logger.debug("REST: remove: %s" % url)
response = self.requester.delete(url,
auth=self.auth,
headers=self.custom_headers,
verify=self.verify_ssl)
return response
def _retrieve_recipe(self, conan_reference, output):
""" returns the requested conanfile object, retrieving it from
remotes if necessary. Can raise NotFoundException
"""
def _retrieve_from_remote(the_remote):
output.info("Trying with '%s'..." % the_remote.name)
self._remote_manager.get_recipe(conan_reference, the_remote)
self._registry.set_ref(conan_reference, the_remote)
self._recorder.recipe_downloaded(conan_reference, the_remote.url)
if self._remote_name:
output.info("Not found, retrieving from server '%s' " %
self._remote_name)
ref_remote = self._registry.remote(self._remote_name)
else:
ref_remote = self._registry.get_ref(conan_reference)
if ref_remote:
output.info("Retrieving from predefined remote '%s'" %
ref_remote.name)
if ref_remote:
try:
_retrieve_from_remote(ref_remote)
return
except NotFoundException:
msg = "%s was not found in remote '%s'" % (
str(conan_reference), ref_remote.name)
self._recorder.recipe_install_error(conan_reference,
INSTALL_ERROR_MISSING, msg,
ref_remote.url)
raise NotFoundException(msg)
except RequestException as exc:
self._recorder.recipe_install_error(conan_reference,
INSTALL_ERROR_NETWORK,
str(exc), ref_remote.url)
raise exc
output.info("Not found in local cache, looking in remotes...")
remotes = self._registry.remotes
for remote in remotes:
logger.debug("Trying with remote %s" % remote.name)
try:
_retrieve_from_remote(remote)
return
# If not found continue with the next, else raise
except NotFoundException as exc:
if remote == remotes[-1]: # Last element not found
msg = "Unable to find '%s' in remotes" % str(
conan_reference)
logger.debug("Not found in any remote, raising...%s" % exc)
self._recorder.recipe_install_error(
conan_reference, INSTALL_ERROR_MISSING, msg, None)
raise NotFoundException(msg)
raise ConanException("No remote defined")
def __init__(self, base_folder=None, current_folder=None, servers=None, users=None,
requester_class=None, runner=None, path_with_spaces=True, block_v2=None,
revisions=None, cpu_count=1):
"""
storage_folder: Local storage path
current_folder: Current execution folder
servers: dict of {remote_name: TestServer}
logins is a list of (user, password) for auto input in order
if required==> [("lasote", "mypass"), ("other", "otherpass")]
"""
self.block_v2 = block_v2 or get_env("CONAN_API_V2_BLOCKED", True)
if self.block_v2:
self.revisions = False
else:
self.revisions = revisions or get_env("CONAN_CLIENT_REVISIONS_ENABLED", False)
self.block_v2 = False
self.all_output = "" # For debugging purpose, append all the run outputs
self.users = users or {"default":
[(TESTING_REMOTE_PRIVATE_USER, TESTING_REMOTE_PRIVATE_PASS)]}
self.base_folder = base_folder or temp_folder(path_with_spaces)
# Define storage_folder, if not, it will be read from conf file & pointed to real user home
self.storage_folder = os.path.join(self.base_folder, ".conan", "data")
self.cache = ClientCache(self.base_folder, self.storage_folder,
TestBufferConanOutput())
self.requester_class = requester_class
self.conan_runner = runner
if cpu_count:
self.cache.conan_config
replace_in_file(self.cache.conan_conf_path,
"# cpu_count = 1", "cpu_count = %s" % cpu_count,
output=TestBufferConanOutput(), strict=not bool(base_folder))
# Invalidate the cached config
self.cache.invalidate()
if servers and len(servers) > 1 and not isinstance(servers, OrderedDict):
raise Exception("""Testing framework error: Servers should be an OrderedDict. e.g:
servers = OrderedDict()
servers["r1"] = server
servers["r2"] = TestServer()
""")
self.servers = servers or {}
if servers is not False: # Do not mess with registry remotes
self.update_servers()
self.init_dynamic_vars()
logger.debug("Client storage = %s" % self.storage_folder)
self.current_folder = current_folder or temp_folder(path_with_spaces)
def upload_package(self, package_reference, the_files, retry, retry_wait,
policy):
"""
basedir: Base directory with the files to upload (for read the files in disk)
relative_files: relative paths to upload
"""
self.check_credentials()
revisions_enabled = get_env("CONAN_CLIENT_REVISIONS_ENABLED", False)
if not revisions_enabled and policy == UPLOAD_POLICY_NO_OVERWRITE:
# Check if the latest revision is not the one we are uploading, with the compatibility
# mode this is supposed to fail if someone tries to upload a different recipe
latest_pref = PackageReference(package_reference.conan,
package_reference.package_id)
latest_snapshot, ref_latest_snapshot, _ = self._get_package_snapshot(
latest_pref)
server_with_revisions = ref_latest_snapshot.revision != DEFAULT_REVISION_V1
if latest_snapshot and server_with_revisions and \
ref_latest_snapshot.revision != package_reference.revision:
raise ConanException(
"Local package is different from the remote package. "
"Forbidden overwrite")
t1 = time.time()
# Get the remote snapshot
pref = package_reference
remote_snapshot, pref_snapshot, rev_time = self._get_package_snapshot(
pref)
if remote_snapshot:
remote_manifest = self.get_package_manifest(pref_snapshot)
local_manifest = FileTreeManifest.loads(
load(the_files["conanmanifest.txt"]))
if remote_manifest == local_manifest:
return False, pref_snapshot, rev_time
if policy == UPLOAD_POLICY_NO_OVERWRITE:
raise ConanException(
"Local package is different from the remote package. "
"Forbidden overwrite")
files_to_upload = the_files
deleted = set(remote_snapshot).difference(the_files)
if files_to_upload:
self._upload_package(package_reference, files_to_upload, retry,
retry_wait)
if deleted:
raise Exception(
"This shouldn't be happening, deleted files "
"in local package present in remote: %s.\n Please, report it at "
"https://github.com/conan-io/conan/issues " % str(deleted))
logger.debug("====> Time rest client upload_package: %f" %
(time.time() - t1))
return files_to_upload or deleted, package_reference, rev_time
def __init__(self,
base_folder=None,
current_folder=None,
servers=None,
users=None,
client_version=CLIENT_VERSION,
min_server_compatible_version=MIN_SERVER_COMPATIBLE_VERSION,
requester_class=None):
"""
storage_folder: Local storage path
current_folder: Current execution folder
servers: dict of {remote_name: TestServer}
logins is a list of (user, password) for auto input in order
if required==> [("lasote", "mypass"), ("other", "otherpass")]
"""
self.all_output = "" # For debugging purpose, append all the run outputs
self.users = users or {
"default":
[(TESTING_REMOTE_PRIVATE_USER, TESTING_REMOTE_PRIVATE_PASS)]
}
self.servers = servers or {}
self.client_version = Version(str(client_version))
self.min_server_compatible_version = Version(
str(min_server_compatible_version))
self.base_folder = base_folder or temp_folder()
# Define storage_folder, if not, it will be read from conf file & pointed to real user home
self.storage_folder = os.path.join(self.base_folder, ".conan", "data")
self.client_cache = ClientCache(self.base_folder, self.storage_folder,
TestBufferConanOutput())
search_adapter = DiskSearchAdapter()
self.search_manager = DiskSearchManager(self.client_cache,
search_adapter)
self.default_settings(get_env("CONAN_COMPILER", "gcc"),
get_env("CONAN_COMPILER_VERSION", "4.8"),
get_env("CONAN_LIBCXX", "libstdc++"))
self.requester_class = requester_class
self.init_dynamic_vars()
save(self.client_cache.registry, "")
registry = RemoteRegistry(self.client_cache.registry,
TestBufferConanOutput())
for name, server in self.servers.items():
if isinstance(server, TestServer):
registry.add(name, server.fake_url)
else:
registry.add(name, server)
logger.debug("Client storage = %s" % self.storage_folder)
self.current_folder = current_folder or temp_folder()
def upload_package(self, package_reference, the_files, retry, retry_wait,
no_overwrite):
"""
basedir: Base directory with the files to upload (for read the files in disk)
relative_files: relative paths to upload
"""
self.check_credentials()
t1 = time.time()
# Get the remote snapshot
remote_snapshot = self._get_package_snapshot(package_reference)
local_snapshot = {
filename: md5sum(abs_path)
for filename, abs_path in the_files.items()
}
# Get the diff
new, modified, deleted = diff_snapshots(local_snapshot,
remote_snapshot)
if not new and not deleted and modified in (["conanmanifest.txt"], []):
return False
if no_overwrite and remote_snapshot:
if no_overwrite == "all":
raise ConanException(
"Local package is different from the remote package. "
"Forbidden overwrite")
files_to_upload = {
filename: the_files[filename]
for filename in new + modified
}
if files_to_upload: # Obtain upload urls
url = "%s/conans/%s/packages/%s/upload_urls" % (
self.remote_api_url, "/".join(
package_reference.conan), package_reference.package_id)
filesizes = {
filename: os.stat(abs_path).st_size
for filename, abs_path in files_to_upload.items()
}
self._output.rewrite_line("Requesting upload permissions...")
urls = self._get_file_to_url_dict(url, data=filesizes)
self._output.rewrite_line("Requesting upload permissions...Done!")
self._output.writeln("")
self._upload_files(urls, files_to_upload, self._output, retry,
retry_wait)
if deleted:
raise Exception(
"This shouldn't be happening, deleted files "
"in local package present in remote: %s.\n Please, report it at "
"https://github.com/conan-io/conan/issues " % str(deleted))
logger.debug("====> Time rest client upload_package: %f" %
(time.time() - t1))
return files_to_upload or deleted
def build(self,
conanfile_path,
source_folder,
build_folder,
package_folder,
test=False):
""" Call to build() method saved on the conanfile.py
param conanfile_path: the original source directory of the user containing a
conanfile.py
"""
logger.debug("Building in %s" % build_folder)
logger.debug("Conanfile in %s" % conanfile_path)
try:
# Append env_vars to execution environment and clear when block code ends
output = ScopedOutput(
("%s test package" % test) if test else "Project",
self._user_io.out)
conan_file = self.load_consumer_conanfile(conanfile_path,
build_folder, output)
except NotFoundException:
# TODO: Auto generate conanfile from requirements file
raise ConanException("'%s' file is needed for build.\n"
"Create a '%s' and move manually the "
"requirements and generators from '%s' file" %
(CONANFILE, CONANFILE, CONANFILE_TXT))
if test:
try:
conan_file.requires.add(test)
except ConanException:
pass
try:
os.chdir(build_folder)
conan_file._conanfile_directory = source_folder
conan_file.build_folder = build_folder
conan_file.source_folder = source_folder
conan_file.package_folder = package_folder
with environment_append(conan_file.env):
output.highlight("Running build()")
with conanfile_exception_formatter(str(conan_file), "build"):
conan_file.build()
if test:
output.highlight("Running test()")
with conanfile_exception_formatter(str(conan_file),
"test"):
conan_file.test()
except ConanException:
raise # Raise but not let to reach the Exception except (not print traceback)
except Exception:
import traceback
trace = traceback.format_exc().split('\n')
raise ConanException("Unable to build it successfully\n%s" %
'\n'.join(trace[3:]))
def _download_recipe(self, conan_reference, output, remote_name, recorder):
def _retrieve_from_remote(the_remote):
output.info("Trying with '%s'..." % the_remote.name)
new_reference = self._remote_manager.get_recipe(
conan_reference, the_remote)
self._registry.set_ref(new_reference, the_remote.name)
recorder.recipe_downloaded(conan_reference, the_remote.url)
return new_reference
if remote_name:
output.info("Not found, retrieving from server '%s' " %
remote_name)
remote = self._registry.remote(remote_name)
else:
remote = self._registry.get_recipe_remote(conan_reference)
if remote:
output.info("Retrieving from predefined remote '%s'" %
remote.name)
if remote:
try:
new_ref = _retrieve_from_remote(remote)
return remote, new_ref
except NotFoundException:
msg = "%s was not found in remote '%s'" % (
str(conan_reference), remote.name)
recorder.recipe_install_error(conan_reference,
INSTALL_ERROR_MISSING, msg,
remote.url)
raise NotFoundException(msg)
except RequestException as exc:
recorder.recipe_install_error(conan_reference,
INSTALL_ERROR_NETWORK, str(exc),
remote.url)
raise exc
output.info("Not found in local cache, looking in remotes...")
remotes = self._registry.remotes
if not remotes:
raise ConanException("No remote defined")
for remote in remotes:
logger.debug("Trying with remote %s" % remote.name)
try:
new_ref = _retrieve_from_remote(remote)
return remote, new_ref
# If not found continue with the next, else raise
except NotFoundException:
pass
else:
msg = "Unable to find '%s' in remotes" % str(conan_reference)
logger.debug("Not found in any remote")
recorder.recipe_install_error(conan_reference,
INSTALL_ERROR_MISSING, msg, None)
raise NotFoundException(msg)
def get_file_path(self, filepath, token):
try:
encoded_path, _, user = self.updown_auth_manager.get_resource_info(token)
if not self._valid_path(filepath, encoded_path):
logger.info("Invalid path file!! %s: %s" % (user, filepath))
raise NotFoundException("File not found")
logger.debug("Get file: user=%s path=%s" % (user, filepath))
file_path = os.path.normpath(os.path.join(self.base_store_folder, encoded_path))
return file_path
except (jwt.ExpiredSignature, jwt.DecodeError, AttributeError):
raise NotFoundException("File not found")
def _capable(self, capability):
capabilities = self._cached_capabilities.get(self.remote_url)
if capabilities is None:
tmp = RestV1Methods(self.remote_url, self.token, self.custom_headers, self._output,
self.requester, self.verify_ssl, self._put_headers)
capabilities = tmp.server_capabilities()
self._cached_capabilities[self.remote_url] = capabilities
logger.debug("REST: Cached capabilities for the remote: %s" % capabilities)
if not self._revisions_enabled and ONLY_V2 in capabilities:
raise OnlyV2Available(self.remote_url)
return capability in capabilities
def infix_to_postfix(exp):
"""
Translates an infix expression to postfix using an standard algorithm
with little hacks for parse complex expressions like "compiler.version=4"
instead of just numbers and without taking in account the operands priority
except the priority specified by the "("
@param exp: String with an expression with & and | operators,
e.j: "os=Windows & (compiler=gcc | compiler.version=3)"
e.j: "os=Windows AND (compiler=gcc or compiler.version=3)"
@return List with the postfix expression
"""
# To ease the parser, operators only with one character
exp = exp.replace(" AND ", "&").replace(" OR ", "|").replace(" and ", "&").replace(" or ", "|")
output = []
stack = []
i = -1
while(i < len(exp) - 1):
i += 1
char = exp[i]
if char == " ": # Ignore spaces between expressions and operators
continue
if char == ")": # Pop the stack until "(" and send them to output
popped = None
while(popped != "(" and stack):
popped = stack.pop()
if popped != "(":
output.append(popped)
if popped != "(":
raise Exception("Bad expression, not balanced parenthesis")
elif is_operator(char):
# Same operations has the same priority
# replace this lines if the operators need to have
# some priority
if stack and is_operator(stack[:-1]):
popped = stack.pop()
output.append(popped)
stack.append(char)
elif char == "(":
stack.append("(")
else: # Parse an expression, in our case something like "compiler=gcc"
logger.debug("Parsing expression from : %s" % exp[i:])
expr = _parse_expression(exp[i:])
logger.debug("Parsed expression: %s" % expr)
i += len(expr) - 1
output.append(expr)
# Append remaining elements
if "(" in stack:
raise Exception("Bad expression, not balanced parenthesis")
output.extend(stack)
return output
def get_exception_from_error(error_code):
try:
tmp = {value: key for key, value in EXCEPTION_CODE_MAPPING.items()}
if error_code in tmp:
logger.debug("From server: %s" % str(tmp[error_code]))
return tmp[error_code]
else:
logger.debug("From server: %s" % str(_base_error(error_code)))
return tmp[_base_error(error_code)]
except KeyError:
return None
def get_file_path(self, filepath, token):
try:
encoded_path, _, user = self.updown_auth_manager.get_resource_info(token)
if not self._valid_path(filepath, encoded_path):
logger.info("Invalid path file!! %s: %s" % (user, filepath))
raise NotFoundException("File not found")
logger.debug("Get file: user=%s path=%s" % (user, filepath))
file_path = os.path.normpath(os.path.join(self.base_store_folder, encoded_path))
return file_path
except (jwt.ExpiredSignature, jwt.DecodeError, AttributeError):
raise NotFoundException("File not found")
def __init__(self,
base_folder=None,
current_folder=None,
servers=None,
users=None,
client_version=CLIENT_VERSION,
min_server_compatible_version=MIN_SERVER_COMPATIBLE_VERSION,
requester_class=None,
runner=None,
path_with_spaces=True):
"""
storage_folder: Local storage path
current_folder: Current execution folder
servers: dict of {remote_name: TestServer}
logins is a list of (user, password) for auto input in order
if required==> [("lasote", "mypass"), ("other", "otherpass")]
"""
self.all_output = "" # For debugging purpose, append all the run outputs
self.users = users or {
"default":
[(TESTING_REMOTE_PRIVATE_USER, TESTING_REMOTE_PRIVATE_PASS)]
}
self.client_version = Version(str(client_version))
self.min_server_compatible_version = Version(
str(min_server_compatible_version))
self.base_folder = base_folder or temp_folder(path_with_spaces)
# Define storage_folder, if not, it will be read from conf file & pointed to real user home
self.storage_folder = os.path.join(self.base_folder, ".conan", "data")
self.client_cache = ClientCache(self.base_folder, self.storage_folder,
TestBufferConanOutput())
self.requester_class = requester_class
self.conan_runner = runner
if servers and len(servers) > 1 and not isinstance(
servers, OrderedDict):
raise Exception(
"""Testing framework error: Servers should be an OrderedDict. e.g:
servers = OrderedDict()
servers["r1"] = server
servers["r2"] = TestServer()
""")
self.servers = servers or {}
if servers is not False: # Do not mess with registry remotes
self.update_servers()
self.init_dynamic_vars()
logger.debug("Client storage = %s" % self.storage_folder)
self.current_folder = current_folder or temp_folder(path_with_spaces)
def _upload_package(self, package_ref, index=1, total=1, retry=None, retry_wait=None,
skip_upload=False, integrity_check=False, no_overwrite=None, remote=None):
"""Uploads the package identified by package_id"""
msg = ("Uploading package %d/%d: %s" % (index, total, str(package_ref.package_id)))
t1 = time.time()
self._user_io.out.info(msg)
result = self._remote_manager.upload_package(package_ref, remote, retry, retry_wait,
skip_upload, integrity_check, no_overwrite)
logger.debug("====> Time uploader upload_package: %f" % (time.time() - t1))
return result
def authenticate(self, user, password):
"""Sends user + password to get:
- A plain response with a regular token (not supported refresh in the remote) and None
"""
auth = HTTPBasicAuth(user, password)
url = self.router.common_authenticate()
logger.debug("REST: Authenticate to get access_token: %s" % url)
ret = self.requester.get(url, auth=auth, headers=self.custom_headers,
verify=self.verify_ssl)
self._check_error_response(ret)
return decode_text(ret.content)
def _build_package(self):
""" calls the imports + conanfile.build() method
"""
os.chdir(self.build_folder)
self._conan_file.build_folder = self.build_folder
self._conan_file.package_folder = self.package_folder
# In local cache, install folder always is build_folder
self._conan_file.install_folder = self.build_folder
# Read generators from conanfile and generate the needed files
logger.info("GENERATORS: Writing generators")
write_generators(self._conan_file, self.build_folder, self._out)
# Build step might need DLLs, binaries as protoc to generate source files
# So execute imports() before build, storing the list of copied_files
from conans.client.importer import run_imports
copied_files = run_imports(self._conan_file, self.build_folder,
self._out)
try:
# This is necessary because it is different for user projects
# than for packages
self._hook_manager.execute(
"pre_build",
conanfile=self._conan_file,
reference=self._conan_ref,
package_id=self._package_reference.package_id)
logger.debug(
"Call conanfile.build() with files in build folder: %s",
os.listdir(self.build_folder))
self._out.highlight("Calling build()")
with conanfile_exception_formatter(str(self._conan_file), "build"):
self._conan_file.build()
self._out.success("Package '%s' built" %
self._conan_file.info.package_id())
self._out.info("Build folder %s" % self.build_folder)
self._hook_manager.execute(
"post_build",
conanfile=self._conan_file,
reference=self._conan_ref,
package_id=self._package_reference.package_id)
except Exception as exc:
self._out.writeln("")
self._out.error("Package '%s' build failed" %
self._conan_file.info.package_id())
self._out.warn("Build folder %s" % self.build_folder)
if isinstance(exc, ConanExceptionInUserConanfileMethod):
raise exc
raise ConanException(exc)
finally:
# Now remove all files that were imported with imports()
remove_imports(self._conan_file, copied_files, self._out)
def __init__(self,
base_folder=None,
current_folder=None,
servers=None,
users=None,
client_version=CLIENT_VERSION,
min_server_compatible_version=MIN_SERVER_COMPATIBLE_VERSION,
requester_class=None,
runner=None,
path_with_spaces=True,
default_profile=True):
"""
storage_folder: Local storage path
current_folder: Current execution folder
servers: dict of {remote_name: TestServer}
logins is a list of (user, password) for auto input in order
if required==> [("lasote", "mypass"), ("other", "otherpass")]
"""
self.all_output = "" # For debugging purpose, append all the run outputs
self.users = users or {
"default":
[(TESTING_REMOTE_PRIVATE_USER, TESTING_REMOTE_PRIVATE_PASS)]
}
self.client_version = Version(str(client_version))
self.min_server_compatible_version = Version(
str(min_server_compatible_version))
self.base_folder = base_folder or temp_folder(path_with_spaces)
# Define storage_folder, if not, it will be read from conf file & pointed to real user home
self.storage_folder = os.path.join(self.base_folder, ".conan", "data")
self.client_cache = ClientCache(self.base_folder, self.storage_folder,
TestBufferConanOutput())
search_adapter = DiskSearchAdapter()
self.search_manager = DiskSearchManager(self.client_cache,
search_adapter)
self.requester_class = requester_class
self.conan_runner = runner
self.update_servers(servers)
self.init_dynamic_vars()
logger.debug("Client storage = %s" % self.storage_folder)
self.current_folder = current_folder or temp_folder(path_with_spaces)
# Enforcing VS 2015, even if VS2017 is auto detected
if default_profile:
profile = self.client_cache.default_profile
if profile.settings.get("compiler.version") == "15":
profile.settings["compiler.version"] = "14"
save(self.client_cache.default_profile_path, profile.dumps())
def _compress_package_files(self, layout, pref, integrity_check):
t1 = time.time()
if layout.package_is_dirty(pref):
raise ConanException("Package %s is corrupted, aborting upload.\n"
"Remove it with 'conan remove %s -p=%s'" %
(pref, pref.ref, pref.id))
download_pkg_folder = layout.download_package(pref)
package_tgz = os.path.join(download_pkg_folder, PACKAGE_TGZ_NAME)
if is_dirty(package_tgz):
self._output.warn("%s: Removing %s, marked as dirty" %
(str(pref), PACKAGE_TGZ_NAME))
os.remove(package_tgz)
clean_dirty(package_tgz)
# Get all the files in that directory
# existing package, will use short paths if defined
package_folder = layout.package(pref)
files, symlinks = gather_files(package_folder)
if CONANINFO not in files or CONAN_MANIFEST not in files:
logger.error("Missing info or manifest in uploading files: %s" %
(str(files)))
raise ConanException("Cannot upload corrupted package '%s'" %
str(pref))
logger.debug("UPLOAD: Time remote_manager build_files_set : %f" %
(time.time() - t1))
if integrity_check:
self._package_integrity_check(pref, files, package_folder)
logger.debug(
"UPLOAD: Time remote_manager check package integrity : %f" %
(time.time() - t1))
if not os.path.isfile(package_tgz):
if self._output and not self._output.is_terminal:
self._output.writeln("Compressing package...")
tgz_files = {
f: path
for f, path in files.items()
if f not in [CONANINFO, CONAN_MANIFEST]
}
tgz_path = compress_files(tgz_files, symlinks, PACKAGE_TGZ_NAME,
download_pkg_folder, self._output)
assert tgz_path == package_tgz
assert os.path.exists(package_tgz)
return {
PACKAGE_TGZ_NAME: package_tgz,
CONANINFO: files[CONANINFO],
CONAN_MANIFEST: files[CONAN_MANIFEST]
}
def build(self, conanfile_path, source_folder, build_folder, package_folder, install_folder,
test=False, should_configure=True, should_build=True, should_install=True):
""" Call to build() method saved on the conanfile.py
param conanfile_path: path to a conanfile.py
"""
logger.debug("Building in %s" % build_folder)
logger.debug("Conanfile in %s" % conanfile_path)
try:
# Append env_vars to execution environment and clear when block code ends
output = ScopedOutput(("%s (test package)" % test) if test else "Project",
self._user_io.out)
conan_file = self._load_consumer_conanfile(conanfile_path, install_folder, output,
deps_info_required=True)
except NotFoundException:
# TODO: Auto generate conanfile from requirements file
raise ConanException("'%s' file is needed for build.\n"
"Create a '%s' and move manually the "
"requirements and generators from '%s' file"
% (CONANFILE, CONANFILE, CONANFILE_TXT))
if test:
try:
conan_file.requires.add(test)
except ConanException:
pass
conan_file.should_configure = should_configure
conan_file.should_build = should_build
conan_file.should_install = should_install
try:
mkdir(build_folder)
os.chdir(build_folder)
conan_file.build_folder = build_folder
conan_file.source_folder = source_folder
conan_file.package_folder = package_folder
conan_file.install_folder = install_folder
with get_env_context_manager(conan_file):
output.highlight("Running build()")
with conanfile_exception_formatter(str(conan_file), "build"):
conan_file.build()
if test:
output.highlight("Running test()")
with conanfile_exception_formatter(str(conan_file), "test"):
conan_file.test()
except ConanException:
raise # Raise but not let to reach the Exception except (not print traceback)
except Exception:
import traceback
trace = traceback.format_exc().split('\n')
raise ConanException("Unable to build it successfully\n%s" % '\n'.join(trace[3:]))
def _get_nodes(self, nodes_by_level, skip_nodes):
"""Compute a list of (conan_ref, package_id, conan_file, build_node)
defining what to do with each node
"""
nodes_to_build = []
# Now build each level, starting from the most independent one
package_references = set()
for level in nodes_by_level:
for node in level:
if node in skip_nodes:
continue
conan_ref, conan_file = node.conan_ref, node.conanfile
build_node = False
logger.debug("Processing node %s", repr(conan_ref))
package_id = conan_file.info.package_id()
package_reference = PackageReference(conan_ref, package_id)
# Avoid processing twice the same package reference
if package_reference not in package_references:
package_references.add(package_reference)
package_folder = self._client_cache.package(
package_reference, short_paths=conan_file.short_paths)
local_project = self._workspace[
conan_ref] if self._workspace else None
if not local_project:
with self._client_cache.package_lock(
package_reference):
if is_dirty(package_folder):
output = ScopedOutput(str(conan_ref),
self._out)
output.warn(
"Package is corrupted, removing folder: %s"
% package_folder)
rmdir(package_folder)
check_outdated = self._build_mode.outdated
if self._build_mode.forced(conan_file, conan_ref):
build_node = True
else:
if local_project:
# Maybe the folder is not there, check it!
build_node = False
else:
available = self._remote_proxy.package_available(
package_reference, package_folder,
check_outdated)
build_node = not available
nodes_to_build.append((node, package_id, build_node))
return nodes_to_build
def get_exception_from_error(error_code):
tmp = {v: k for k, v in EXCEPTION_CODE_MAPPING.items() # All except NotFound
if k not in (RecipeNotFoundException, PackageNotFoundException)}
if error_code in tmp:
logger.debug("REST ERROR: %s" % str(tmp[error_code]))
return tmp[error_code]
else:
base_error = int(str(error_code)[0] + "00")
logger.debug("REST ERROR: %s" % str(base_error))
try:
return tmp[base_error]
except KeyError:
return None
