def _remove(self, path, ref, msg=""):
try:
logger.debug("REMOVE: folder %s" % path)
rm_conandir(path)
except OSError:
error_msg = "Folder busy (open or some file open): %s" % path
raise ConanException("%s: Unable to remove %s\n\t%s" % (repr(ref), msg, error_msg))
def get_recipe(self, conan_reference, remote):
"""
Read the conans from remotes
Will iterate the remotes to find the conans unless remote was specified
returns (dict relative_filepath:abs_path , remote_name)"""
dest_folder = self._client_cache.export(conan_reference)
rmdir(dest_folder)
def filter_function(urls):
if CONANFILE not in list(urls.keys()):
raise NotFoundException("Conan '%s' doesn't have a %s!"
% (conan_reference, CONANFILE))
urls.pop(EXPORT_SOURCES_TGZ_NAME, None)
return urls
t1 = time.time()
urls = self._call_remote(remote, "get_recipe_urls", conan_reference)
urls = filter_function(urls)
if not urls:
return conan_reference
zipped_files = self._call_remote(remote, "download_files_to_folder", urls, dest_folder)
duration = time.time() - t1
log_recipe_download(conan_reference, duration, remote, zipped_files)
unzip_and_get_files(zipped_files, dest_folder, EXPORT_TGZ_NAME)
# Make sure that the source dir is deleted
rm_conandir(self._client_cache.source(conan_reference))
touch_folder(dest_folder)
def get_package(self, pref, dest_folder, remote, output, recorder):
conanfile_path = self._cache.package_layout(pref.ref).conanfile()
self._hook_manager.execute("pre_download_package",
conanfile_path=conanfile_path,
reference=pref.ref,
package_id=pref.id,
remote=remote)
output.info("Retrieving package %s from remote '%s' " %
(pref.id, remote.name))
rm_conandir(dest_folder) # Remove first the destination folder
t1 = time.time()
try:
pref = self._resolve_latest_pref(pref, remote)
snapshot = self._call_remote(remote, "get_package_snapshot", pref)
if not is_package_snapshot_complete(snapshot):
raise PackageNotFoundException(pref)
zipped_files = self._call_remote(remote, "get_package", pref,
dest_folder)
package_checksums = calc_files_checksum(zipped_files)
with self._cache.package_layout(
pref.ref).update_metadata() as metadata:
metadata.packages[pref.id].revision = pref.revision
metadata.packages[pref.id].recipe_revision = pref.ref.revision
metadata.packages[pref.id].checksums = package_checksums
duration = time.time() - t1
log_package_download(pref, duration, remote, zipped_files)
unzip_and_get_files(zipped_files,
dest_folder,
PACKAGE_TGZ_NAME,
output=self._output)
# Issue #214 https://github.com/conan-io/conan/issues/214
touch_folder(dest_folder)
if get_env("CONAN_READ_ONLY_CACHE", False):
make_read_only(dest_folder)
recorder.package_downloaded(pref, remote.url)
output.success('Package installed %s' % pref.id)
except NotFoundException:
raise PackageNotFoundException(pref)
except BaseException as e:
output.error("Exception while getting package: %s" % str(pref.id))
output.error("Exception: %s %s" % (type(e), str(e)))
try:
output.warn("Trying to remove package folder: %s" %
dest_folder)
rmdir(dest_folder)
except OSError as e:
raise ConanException(
"%s\n\nCouldn't remove folder '%s', might be busy or open. "
"Close any app using it, and retry" %
(str(e), dest_folder))
raise
self._hook_manager.execute("post_download_package",
conanfile_path=conanfile_path,
reference=pref.ref,
package_id=pref.id,
remote=remote)
def get_recipe(self, conan_reference, dest_folder, remote):
"""
Read the conans from remotes
Will iterate the remotes to find the conans unless remote was specified
returns (dict relative_filepath:abs_path , remote_name)"""
rmdir(dest_folder) # Remove first the destination folder
t1 = time.time()
def filter_function(urls):
if CONANFILE not in list(urls.keys()):
raise NotFoundException("Conan '%s' doesn't have a %s!" %
(conan_reference, CONANFILE))
urls.pop(EXPORT_SOURCES_TGZ_NAME, None)
return urls
zipped_files = self._call_remote(remote, "get_recipe", conan_reference,
dest_folder, filter_function)
duration = time.time() - t1
log_recipe_download(conan_reference, duration, remote, zipped_files)
unzip_and_get_files(zipped_files, dest_folder, EXPORT_TGZ_NAME)
# Make sure that the source dir is deleted
rm_conandir(self._client_cache.source(conan_reference))
for dirname, _, filenames in os.walk(dest_folder):
for fname in filenames:
touch(os.path.join(dirname, fname))
def get_recipe(self, conan_reference, dest_folder, remote):
"""
Read the conans from remotes
Will iterate the remotes to find the conans unless remote was specified
returns (dict relative_filepath:abs_path , remote_name)"""
rmdir(dest_folder) # Remove first the destination folder
t1 = time.time()
def filter_function(urls):
if CONANFILE not in list(urls.keys()):
raise NotFoundException("Conan '%s' doesn't have a %s!"
% (conan_reference, CONANFILE))
urls.pop(EXPORT_SOURCES_TGZ_NAME, None)
return urls
zipped_files = self._call_remote(remote, "get_recipe", conan_reference, dest_folder,
filter_function)
duration = time.time() - t1
log_recipe_download(conan_reference, duration, remote, zipped_files)
unzip_and_get_files(zipped_files, dest_folder, EXPORT_TGZ_NAME)
# Make sure that the source dir is deleted
rm_conandir(self._client_cache.source(conan_reference))
for dirname, _, filenames in os.walk(dest_folder):
for fname in filenames:
touch(os.path.join(dirname, fname))
def get_recipe(self, ref, remote):
"""
Read the conans from remotes
Will iterate the remotes to find the conans unless remote was specified
returns (dict relative_filepath:abs_path , remote_name)"""
self._hook_manager.execute("pre_download_recipe", reference=ref, remote=remote)
dest_folder = self._cache.package_layout(ref).export()
rmdir(dest_folder)
ref = self._resolve_latest_ref(ref, remote)
t1 = time.time()
zipped_files = self._call_remote(remote, "get_recipe", ref, dest_folder)
duration = time.time() - t1
log_recipe_download(ref, duration, remote.name, zipped_files)
unzip_and_get_files(zipped_files, dest_folder, EXPORT_TGZ_NAME, output=self._output)
# Make sure that the source dir is deleted
package_layout = self._cache.package_layout(ref)
rm_conandir(package_layout.source())
touch_folder(dest_folder)
conanfile_path = package_layout.conanfile()
with package_layout.update_metadata() as metadata:
metadata.recipe.revision = ref.revision
self._hook_manager.execute("post_download_recipe", conanfile_path=conanfile_path,
reference=ref, remote=remote)
return ref
def get_package(self, package_reference, dest_folder, remote, output, recorder):
package_id = package_reference.package_id
output.info("Retrieving package %s from remote '%s' " % (package_id, remote.name))
rm_conandir(dest_folder) # Remove first the destination folder
t1 = time.time()
try:
urls = self._call_remote(remote, "get_package_urls", package_reference)
zipped_files = self._call_remote(remote, "download_files_to_folder", urls, dest_folder)
duration = time.time() - t1
log_package_download(package_reference, duration, remote, zipped_files)
unzip_and_get_files(zipped_files, dest_folder, PACKAGE_TGZ_NAME)
# Issue #214 https://github.com/conan-io/conan/issues/214
touch_folder(dest_folder)
if get_env("CONAN_READ_ONLY_CACHE", False):
make_read_only(dest_folder)
recorder.package_downloaded(package_reference, remote.url)
output.success('Package installed %s' % package_id)
except NotFoundException:
raise NotFoundException("Package binary '%s' not found in '%s'" % (package_reference, remote.name))
except BaseException as e:
output.error("Exception while getting package: %s" % str(package_reference.package_id))
output.error("Exception: %s %s" % (type(e), str(e)))
try:
output.warn("Trying to remove package folder: %s" % dest_folder)
rmdir(dest_folder)
except OSError as e:
raise ConanException("%s\n\nCouldn't remove folder '%s', might be busy or open. Close any app "
"using it, and retry" % (str(e), dest_folder))
raise
def get_recipe(self, conan_reference, remote):
"""
Read the conans from remotes
Will iterate the remotes to find the conans unless remote was specified
returns (dict relative_filepath:abs_path , remote_name)"""
dest_folder = self._client_cache.export(conan_reference)
rmdir(dest_folder)
def filter_function(urls):
if CONANFILE not in list(urls.keys()):
raise NotFoundException("Conan '%s' doesn't have a %s!"
% (conan_reference, CONANFILE))
urls.pop(EXPORT_SOURCES_TGZ_NAME, None)
return urls
t1 = time.time()
urls = self._call_remote(remote, "get_recipe_urls", conan_reference)
urls = filter_function(urls)
if not urls:
return conan_reference
zipped_files = self._call_remote(remote, "download_files_to_folder", urls, dest_folder)
duration = time.time() - t1
log_recipe_download(conan_reference, duration, remote, zipped_files)
unzip_and_get_files(zipped_files, dest_folder, EXPORT_TGZ_NAME)
# Make sure that the source dir is deleted
rm_conandir(self._client_cache.source(conan_reference))
touch_folder(dest_folder)
def _get_remote_package(self, conan_file, package_reference, output,
package_folder):
"""Get remote package. It won't check if it's outdated"""
# Compute conan_file package from local (already compiled) or from remote
# If already exists do not dirt the output, the common situation
# is that package is already installed and OK. If don't, the proxy
# will print some other message about it
if not os.path.exists(package_folder):
output.info("Retrieving package %s" % package_reference.package_id)
try:
if self._remote_proxy.get_package(
package_reference, short_paths=conan_file.short_paths):
_handle_system_requirements(conan_file, package_reference,
self._client_cache, output)
if get_env("CONAN_READ_ONLY_CACHE", False):
make_read_only(package_folder)
return True
except ConanManifestException:
raise # If is a manifest verify exception, do NOT remove folder
except BaseException as e:
output.error("Exception while getting package: %s" %
str(package_reference.package_id))
output.error("Exception: %s %s" % (type(e), str(e)))
try:
output.warn("Trying to remove package folder: %s" %
package_folder)
rm_conandir(package_folder)
except OSError as e:
raise ConanException(
"%s\n\nCouldn't remove folder '%s', might be busy or open. Close any app "
"using it, and retry" % (str(e), package_folder))
raise
_raise_package_not_found_error(conan_file, package_reference.conan,
package_reference.package_id, output)
def get_package(self, package_reference, dest_folder, remote, output,
recorder):
package_id = package_reference.package_id
conanfile_path = self._client_cache.conanfile(package_reference.conan)
self._hook_manager.execute("pre_download_package",
conanfile_path=conanfile_path,
reference=package_reference.conan,
package_id=package_id,
remote=remote)
output.info("Retrieving package %s from remote '%s' " %
(package_id, remote.name))
rm_conandir(dest_folder) # Remove first the destination folder
t1 = time.time()
try:
zipped_files, new_ref, rev_time = self._call_remote(
remote, "get_package", package_reference, dest_folder)
with self._client_cache.update_metadata(new_ref.conan) as metadata:
metadata.packages[
new_ref.package_id].revision = new_ref.revision
metadata.packages[
new_ref.
package_id].recipe_revision = new_ref.conan.revision
metadata.packages[new_ref.package_id].time = rev_time
duration = time.time() - t1
log_package_download(package_reference, duration, remote,
zipped_files)
unzip_and_get_files(zipped_files,
dest_folder,
PACKAGE_TGZ_NAME,
output=self._output)
# Issue #214 https://github.com/conan-io/conan/issues/214
touch_folder(dest_folder)
if get_env("CONAN_READ_ONLY_CACHE", False):
make_read_only(dest_folder)
recorder.package_downloaded(package_reference, remote.url)
output.success('Package installed %s' % package_id)
except NotFoundException:
raise NotFoundException("Package binary '%s' not found in '%s'" %
(package_reference, remote.name))
except BaseException as e:
output.error("Exception while getting package: %s" %
str(package_reference.package_id))
output.error("Exception: %s %s" % (type(e), str(e)))
try:
output.warn("Trying to remove package folder: %s" %
dest_folder)
rmdir(dest_folder)
except OSError as e:
raise ConanException(
"%s\n\nCouldn't remove folder '%s', might be busy or open. Close any app "
"using it, and retry" % (str(e), dest_folder))
raise
self._hook_manager.execute("post_download_package",
conanfile_path=conanfile_path,
reference=package_reference.conan,
package_id=package_id,
remote=remote)
return new_ref
def get_recipe(self, conan_reference, remote):
"""
Read the conans from remotes
Will iterate the remotes to find the conans unless remote was specified
returns (dict relative_filepath:abs_path , remote_name)"""
self._hook_manager.execute("pre_download_recipe", reference=conan_reference, remote=remote)
dest_folder = self._client_cache.export(conan_reference)
rmdir(dest_folder)
t1 = time.time()
tmp = self._call_remote(remote, "get_recipe", conan_reference, dest_folder)
zipped_files, conan_reference, rev_time = tmp
duration = time.time() - t1
log_recipe_download(conan_reference, duration, remote.name, zipped_files)
unzip_and_get_files(zipped_files, dest_folder, EXPORT_TGZ_NAME, output=self._output)
# Make sure that the source dir is deleted
rm_conandir(self._client_cache.source(conan_reference))
touch_folder(dest_folder)
conanfile_path = self._client_cache.conanfile(conan_reference)
self._hook_manager.execute("post_download_recipe", conanfile_path=conanfile_path,
reference=conan_reference, remote=remote)
with self._client_cache.update_metadata(conan_reference) as metadata:
metadata.recipe.revision = conan_reference.revision
metadata.recipe.time = rev_time
return conan_reference
def get_package(self, conanfile, package_reference, dest_folder, remote,
output, recorder):
package_id = package_reference.package_id
output.info("Retrieving package %s from remote '%s' " %
(package_id, remote.name))
rm_conandir(dest_folder) # Remove first the destination folder
t1 = time.time()
try:
urls = self._call_remote(remote, "get_package_urls",
package_reference)
zipped_files = self._call_remote(remote,
"download_files_to_folder", urls,
dest_folder)
except NotFoundException as e:
output.warn('Binary for %s not in remote: %s' %
(package_id, str(e)))
raise_package_not_found_error(conanfile, package_reference.conan,
package_id, output, recorder,
remote.url)
else:
duration = time.time() - t1
log_package_download(package_reference, duration, remote,
zipped_files)
unzip_and_get_files(zipped_files, dest_folder, PACKAGE_TGZ_NAME)
# Issue #214 https://github.com/conan-io/conan/issues/214
touch_folder(dest_folder)
output.success('Package installed %s' % package_id)
def _remove(self, path, conan_ref, msg=""):
try:
logger.debug("Removing folder %s" % path)
rm_conandir(path)
except OSError as e:
raise ConanException("%s: Unable to remove %s\n\t%s" %
(repr(conan_ref), msg, str(e)))
def _remove(self, path, conan_ref, msg=""):
try:
logger.debug("Removing folder %s" % path)
rm_conandir(path)
except OSError:
error_msg = "Folder busy (open or some file open): %s" % path
raise ConanException("%s: Unable to remove %s\n\t%s"
% (repr(conan_ref), msg, error_msg))
def export_remove(self):
export_folder = self.export()
rmdir(export_folder)
export_src_folder = self.export_sources()
rm_conandir(export_src_folder)
download_export = self.download_export()
rmdir(download_export)
scm_folder = self.scm_sources()
rmdir(scm_folder)
def export_remove(self):
export_folder = self.export()
rmdir(export_folder)
export_src_folder = os.path.join(self._base_folder, EXPORT_SRC_FOLDER)
rm_conandir(export_src_folder)
download_export = self.download_export()
rmdir(download_export)
scm_folder = os.path.join(self._base_folder, SCM_SRC_FOLDER)
rm_conandir(scm_folder)
def _refresh():
export_path = self._client_cache.export(conan_reference)
rmdir(export_path)
# It might need to remove shortpath
rm_conandir(self._client_cache.source(conan_reference))
current_remote, _ = self._get_remote(conan_reference)
output.info("Retrieving from remote '%s'..." % current_remote.name)
self._remote_manager.get_recipe(conan_reference, export_path, current_remote)
if self._update:
output.info("Updated!")
else:
output.info("Installed!")
def _refresh():
export_path = self._client_cache.export(conan_reference)
rmdir(export_path)
# It might need to remove shortpath
rm_conandir(self._client_cache.source(conan_reference))
current_remote, _ = self._get_remote(conan_reference)
output.info("Retrieving from remote '%s'..." % current_remote.name)
self._remote_manager.get_recipe(conan_reference, export_path, current_remote)
if self._update:
output.info("Updated!")
else:
output.info("Installed!")
def get_recipe(self, ref, remote):
"""
Read the conans from remotes
Will iterate the remotes to find the conans unless remote was specified
returns (dict relative_filepath:abs_path , remote_name)"""
self._hook_manager.execute("pre_download_recipe",
reference=ref,
remote=remote)
package_layout = self._cache.package_layout(ref)
package_layout.export_remove()
ref = self._resolve_latest_ref(ref, remote)
t1 = time.time()
download_export = package_layout.download_export()
zipped_files = self._call_remote(remote, "get_recipe", ref,
download_export)
duration = time.time() - t1
log_recipe_download(ref, duration, remote.name, zipped_files)
recipe_checksums = calc_files_checksum(zipped_files)
export_folder = package_layout.export()
tgz_file = zipped_files.pop(EXPORT_TGZ_NAME, None)
check_compressed_files(EXPORT_TGZ_NAME, zipped_files)
if tgz_file:
uncompress_file(tgz_file, export_folder, output=self._output)
mkdir(export_folder)
for file_name, file_path in zipped_files.items(): # copy CONANFILE
shutil.move(file_path, os.path.join(export_folder, file_name))
# Make sure that the source dir is deleted
rm_conandir(package_layout.source())
touch_folder(export_folder)
conanfile_path = package_layout.conanfile()
with package_layout.update_metadata() as metadata:
metadata.recipe.revision = ref.revision
metadata.recipe.checksums = recipe_checksums
metadata.recipe.remote = remote.name
self._hook_manager.execute("post_download_recipe",
conanfile_path=conanfile_path,
reference=ref,
remote=remote)
return ref
def get_package(self, package_reference, dest_folder, remote):
"""
Read the conans package from remotes
Will iterate the remotes to find the conans unless remote was specified
returns (dict relative_filepath:abs_path , remote_name)"""
rm_conandir(dest_folder) # Remove first the destination folder
zipped_files = self._call_remote(remote, "get_package", package_reference, dest_folder)
files = unzip_and_get_files(zipped_files, dest_folder, PACKAGE_TGZ_NAME)
# Issue #214 https://github.com/conan-io/conan/issues/214
for dirname, _, filenames in os.walk(dest_folder):
for fname in filenames:
touch(os.path.join(dirname, fname))
return files
def sources_remove(self):
src_folder = os.path.join(self._base_folder, SRC_FOLDER)
try:
rm_conandir(src_folder) # This will remove the shortened path too if exists
except OSError as e:
raise ConanException("%s\n\nFolder: %s\n"
"Couldn't remove folder, might be busy or open\n"
"Close any app using it, and retry" % (src_folder, str(e)))
scm_folder = os.path.join(self._base_folder, SCM_SRC_FOLDER)
try:
rm_conandir(scm_folder) # This will remove the shortened path too if exists
except OSError as e:
raise ConanException("%s\n\nFolder: %s\n"
"Couldn't remove folder, might be busy or open\n"
"Close any app using it, and retry" % (scm_folder, str(e)))
def get_package(self, package_reference, dest_folder, remote):
"""
Read the conans package from remotes
Will iterate the remotes to find the conans unless remote was specified
returns (dict relative_filepath:abs_path , remote_name)"""
rm_conandir(dest_folder) # Remove first the destination folder
t1 = time.time()
zipped_files = self._call_remote(remote, "get_package", package_reference, dest_folder)
duration = time.time() - t1
log_package_download(package_reference, duration, remote, zipped_files)
unzip_and_get_files(zipped_files, dest_folder, PACKAGE_TGZ_NAME)
# Issue #214 https://github.com/conan-io/conan/issues/214
for dirname, _, filenames in os.walk(dest_folder):
for fname in filenames:
touch(os.path.join(dirname, fname))
def get_recipe(self, conan_reference, dest_folder, remote):
"""
Read the conans from remotes
Will iterate the remotes to find the conans unless remote was specified
returns (dict relative_filepath:abs_path , remote_name)"""
rmdir(dest_folder) # Remove first the destination folder
zipped_files = self._call_remote(remote, "get_recipe", conan_reference, dest_folder)
files = unzip_and_get_files(zipped_files, dest_folder, EXPORT_TGZ_NAME)
# Make sure that the source dir is deleted
rm_conandir(self._client_cache.source(conan_reference))
for dirname, _, filenames in os.walk(dest_folder):
for fname in filenames:
touch(os.path.join(dirname, fname))
# TODO: Download only the CONANFILE file and only download the rest of files
# in install if needed (not found remote package)
return files
def package_remove(self, pref):
# Here we could validate and check we own a write lock over this package
assert isinstance(pref, PackageReference)
assert pref.ref == self._ref, "{!r} != {!r}".format(
pref.ref, self._ref)
# This is NOT the short paths, but the standard cache one
pkg_folder = os.path.join(self._base_folder, PACKAGES_FOLDER, pref.id)
try:
rm_conandir(pkg_folder
) # This will remove the shortened path too if exists
except OSError as e:
raise ConanException(
"%s\n\nFolder: %s\n"
"Couldn't remove folder, might be busy or open\n"
"Close any app using it, and retry" % (pkg_folder, str(e)))
if is_dirty(pkg_folder):
clean_dirty(pkg_folder)
def get_recipe(self, conan_reference, dest_folder, remote):
"""
Read the conans from remotes
Will iterate the remotes to find the conans unless remote was specified
returns (dict relative_filepath:abs_path , remote_name)"""
zipped_files = self._call_remote(remote, "get_recipe", conan_reference,
dest_folder)
files = unzip_and_get_files(zipped_files, dest_folder, EXPORT_TGZ_NAME)
# Make sure that the source dir is deleted
rm_conandir(self._client_cache.source(conan_reference))
for dirname, _, filenames in os.walk(dest_folder):
for fname in filenames:
touch(os.path.join(dirname, fname))
# TODO: Download only the CONANFILE file and only download the rest of files
# in install if needed (not found remote package)
return files
def get_package(self, conanfile, package_reference, dest_folder, remote, output, recorder):
package_id = package_reference.package_id
output.info("Retrieving package %s from remote '%s' " % (package_id, remote.name))
rm_conandir(dest_folder) # Remove first the destination folder
t1 = time.time()
try:
urls = self._call_remote(remote, "get_package_urls", package_reference)
zipped_files = self._call_remote(remote, "download_files_to_folder", urls, dest_folder)
except NotFoundException as e:
output.warn('Binary for %s not in remote: %s' % (package_id, str(e)))
raise_package_not_found_error(conanfile, package_reference.conan,
package_id, output, recorder, remote.url)
else:
duration = time.time() - t1
log_package_download(package_reference, duration, remote, zipped_files)
unzip_and_get_files(zipped_files, dest_folder, PACKAGE_TGZ_NAME)
# Issue #214 https://github.com/conan-io/conan/issues/214
touch_folder(dest_folder)
output.success('Package installed %s' % package_id)
def get_recipe(self, conan_reference, remote):
"""
Read the conans from remotes
Will iterate the remotes to find the conans unless remote was specified
returns (dict relative_filepath:abs_path , remote_name)"""
dest_folder = self._client_cache.export(conan_reference)
rmdir(dest_folder)
t1 = time.time()
zipped_files, conan_reference = self._call_remote(remote, "get_recipe", conan_reference,
dest_folder)
duration = time.time() - t1
log_recipe_download(conan_reference, duration, remote.name, zipped_files)
unzip_and_get_files(zipped_files, dest_folder, EXPORT_TGZ_NAME)
# Make sure that the source dir is deleted
rm_conandir(self._client_cache.source(conan_reference))
touch_folder(dest_folder)
return conan_reference