def get_recipe_sources(self, conan_reference, export_folder, export_sources_folder, remote):
t1 = time.time()
def filter_function(urls):
file_url = urls.get(EXPORT_SOURCES_TGZ_NAME)
if file_url:
urls = {EXPORT_SOURCES_TGZ_NAME: file_url}
else:
return None
return urls
urls = self._call_remote(remote, "get_recipe_urls", conan_reference)
urls = filter_function(urls)
if not urls:
return conan_reference
zipped_files = self._call_remote(remote, "download_files_to_folder", urls, export_folder)
duration = time.time() - t1
log_recipe_sources_download(conan_reference, duration, remote, zipped_files)
if not zipped_files:
mkdir(export_sources_folder) # create the folder even if no source files
return
unzip_and_get_files(zipped_files, export_sources_folder, EXPORT_SOURCES_TGZ_NAME)
c_src_path = os.path.join(export_sources_folder, EXPORT_SOURCES_DIR_OLD)
if os.path.exists(c_src_path):
merge_directories(c_src_path, export_sources_folder)
rmdir(c_src_path)
touch_folder(export_sources_folder)
def get_recipe_sources(self, conan_reference, export_folder,
export_sources_folder, remote):
t1 = time.time()
zipped_files = self._call_remote(remote, "get_recipe_sources",
conan_reference, export_folder)
if not zipped_files:
mkdir(export_sources_folder
) # create the folder even if no source files
return conan_reference
duration = time.time() - t1
log_recipe_sources_download(conan_reference, duration, remote.name,
zipped_files)
unzip_and_get_files(zipped_files,
export_sources_folder,
EXPORT_SOURCES_TGZ_NAME,
output=self._output)
c_src_path = os.path.join(export_sources_folder,
EXPORT_SOURCES_DIR_OLD)
if os.path.exists(c_src_path):
merge_directories(c_src_path, export_sources_folder)
rmdir(c_src_path)
touch_folder(export_sources_folder)
return conan_reference
def get_recipe(self, conan_reference, remote):
"""
Read the conans from remotes
Will iterate the remotes to find the conans unless remote was specified
returns (dict relative_filepath:abs_path , remote_name)"""
self._hook_manager.execute("pre_download_recipe", reference=conan_reference, remote=remote)
dest_folder = self._client_cache.export(conan_reference)
rmdir(dest_folder)
t1 = time.time()
tmp = self._call_remote(remote, "get_recipe", conan_reference, dest_folder)
zipped_files, conan_reference, rev_time = tmp
duration = time.time() - t1
log_recipe_download(conan_reference, duration, remote.name, zipped_files)
unzip_and_get_files(zipped_files, dest_folder, EXPORT_TGZ_NAME, output=self._output)
# Make sure that the source dir is deleted
rm_conandir(self._client_cache.source(conan_reference))
touch_folder(dest_folder)
conanfile_path = self._client_cache.conanfile(conan_reference)
self._hook_manager.execute("post_download_recipe", conanfile_path=conanfile_path,
reference=conan_reference, remote=remote)
with self._client_cache.update_metadata(conan_reference) as metadata:
metadata.recipe.revision = conan_reference.revision
metadata.recipe.time = rev_time
return conan_reference
def get_recipe(self, ref, remote):
"""
Read the conans from remotes
Will iterate the remotes to find the conans unless remote was specified
returns (dict relative_filepath:abs_path , remote_name)"""
self._hook_manager.execute("pre_download_recipe", reference=ref, remote=remote)
dest_folder = self._cache.package_layout(ref).export()
rmdir(dest_folder)
ref = self._resolve_latest_ref(ref, remote)
t1 = time.time()
zipped_files = self._call_remote(remote, "get_recipe", ref, dest_folder)
duration = time.time() - t1
log_recipe_download(ref, duration, remote.name, zipped_files)
unzip_and_get_files(zipped_files, dest_folder, EXPORT_TGZ_NAME, output=self._output)
# Make sure that the source dir is deleted
package_layout = self._cache.package_layout(ref)
rm_conandir(package_layout.source())
touch_folder(dest_folder)
conanfile_path = package_layout.conanfile()
with package_layout.update_metadata() as metadata:
metadata.recipe.revision = ref.revision
self._hook_manager.execute("post_download_recipe", conanfile_path=conanfile_path,
reference=ref, remote=remote)
return ref
def get_package(self, pref, dest_folder, remote, output, recorder):
conanfile_path = self._cache.package_layout(pref.ref).conanfile()
self._hook_manager.execute("pre_download_package",
conanfile_path=conanfile_path,
reference=pref.ref,
package_id=pref.id,
remote=remote)
output.info("Retrieving package %s from remote '%s' " %
(pref.id, remote.name))
rm_conandir(dest_folder) # Remove first the destination folder
t1 = time.time()
try:
pref = self._resolve_latest_pref(pref, remote)
snapshot = self._call_remote(remote, "get_package_snapshot", pref)
if not is_package_snapshot_complete(snapshot):
raise PackageNotFoundException(pref)
zipped_files = self._call_remote(remote, "get_package", pref,
dest_folder)
package_checksums = calc_files_checksum(zipped_files)
with self._cache.package_layout(
pref.ref).update_metadata() as metadata:
metadata.packages[pref.id].revision = pref.revision
metadata.packages[pref.id].recipe_revision = pref.ref.revision
metadata.packages[pref.id].checksums = package_checksums
duration = time.time() - t1
log_package_download(pref, duration, remote, zipped_files)
unzip_and_get_files(zipped_files,
dest_folder,
PACKAGE_TGZ_NAME,
output=self._output)
# Issue #214 https://github.com/conan-io/conan/issues/214
touch_folder(dest_folder)
if get_env("CONAN_READ_ONLY_CACHE", False):
make_read_only(dest_folder)
recorder.package_downloaded(pref, remote.url)
output.success('Package installed %s' % pref.id)
except NotFoundException:
raise PackageNotFoundException(pref)
except BaseException as e:
output.error("Exception while getting package: %s" % str(pref.id))
output.error("Exception: %s %s" % (type(e), str(e)))
try:
output.warn("Trying to remove package folder: %s" %
dest_folder)
rmdir(dest_folder)
except OSError as e:
raise ConanException(
"%s\n\nCouldn't remove folder '%s', might be busy or open. "
"Close any app using it, and retry" %
(str(e), dest_folder))
raise
self._hook_manager.execute("post_download_package",
conanfile_path=conanfile_path,
reference=pref.ref,
package_id=pref.id,
remote=remote)
def get_recipe_sources(self, ref, export_folder, export_sources_folder,
remote):
assert ref.revision, "get_recipe_sources requires RREV"
t1 = time.time()
zipped_files = self._call_remote(remote, "get_recipe_sources", ref,
export_folder)
if not zipped_files:
mkdir(export_sources_folder
) # create the folder even if no source files
return
duration = time.time() - t1
log_recipe_sources_download(ref, duration, remote.name, zipped_files)
unzip_and_get_files(zipped_files,
export_sources_folder,
EXPORT_SOURCES_TGZ_NAME,
output=self._output)
# REMOVE in Conan 2.0
c_src_path = os.path.join(export_sources_folder,
EXPORT_SOURCES_DIR_OLD)
if os.path.exists(c_src_path):
merge_directories(c_src_path, export_sources_folder)
rmdir(c_src_path)
touch_folder(export_sources_folder)
def get_recipe_sources(self, conan_reference, export_folder, export_sources_folder, remote):
t1 = time.time()
def filter_function(urls):
file_url = urls.get(EXPORT_SOURCES_TGZ_NAME)
if file_url:
urls = {EXPORT_SOURCES_TGZ_NAME: file_url}
else:
return None
return urls
urls = self._call_remote(remote, "get_recipe_urls", conan_reference)
urls = filter_function(urls)
if not urls:
return conan_reference
zipped_files = self._call_remote(remote, "download_files_to_folder", urls, export_folder)
duration = time.time() - t1
log_recipe_sources_download(conan_reference, duration, remote, zipped_files)
if not zipped_files:
mkdir(export_sources_folder) # create the folder even if no source files
return
unzip_and_get_files(zipped_files, export_sources_folder, EXPORT_SOURCES_TGZ_NAME)
c_src_path = os.path.join(export_sources_folder, EXPORT_SOURCES_DIR_OLD)
if os.path.exists(c_src_path):
merge_directories(c_src_path, export_sources_folder)
rmdir(c_src_path)
touch_folder(export_sources_folder)
def get_package(self, package_reference, dest_folder, remote, output, recorder):
package_id = package_reference.package_id
output.info("Retrieving package %s from remote '%s' " % (package_id, remote.name))
rm_conandir(dest_folder) # Remove first the destination folder
t1 = time.time()
try:
urls = self._call_remote(remote, "get_package_urls", package_reference)
zipped_files = self._call_remote(remote, "download_files_to_folder", urls, dest_folder)
duration = time.time() - t1
log_package_download(package_reference, duration, remote, zipped_files)
unzip_and_get_files(zipped_files, dest_folder, PACKAGE_TGZ_NAME)
# Issue #214 https://github.com/conan-io/conan/issues/214
touch_folder(dest_folder)
if get_env("CONAN_READ_ONLY_CACHE", False):
make_read_only(dest_folder)
recorder.package_downloaded(package_reference, remote.url)
output.success('Package installed %s' % package_id)
except NotFoundException:
raise NotFoundException("Package binary '%s' not found in '%s'" % (package_reference, remote.name))
except BaseException as e:
output.error("Exception while getting package: %s" % str(package_reference.package_id))
output.error("Exception: %s %s" % (type(e), str(e)))
try:
output.warn("Trying to remove package folder: %s" % dest_folder)
rmdir(dest_folder)
except OSError as e:
raise ConanException("%s\n\nCouldn't remove folder '%s', might be busy or open. Close any app "
"using it, and retry" % (str(e), dest_folder))
raise
def get_package(self, conanfile, package_reference, dest_folder, remote,
output, recorder):
package_id = package_reference.package_id
output.info("Retrieving package %s from remote '%s' " %
(package_id, remote.name))
rm_conandir(dest_folder) # Remove first the destination folder
t1 = time.time()
try:
urls = self._call_remote(remote, "get_package_urls",
package_reference)
zipped_files = self._call_remote(remote,
"download_files_to_folder", urls,
dest_folder)
except NotFoundException as e:
output.warn('Binary for %s not in remote: %s' %
(package_id, str(e)))
raise_package_not_found_error(conanfile, package_reference.conan,
package_id, output, recorder,
remote.url)
else:
duration = time.time() - t1
log_package_download(package_reference, duration, remote,
zipped_files)
unzip_and_get_files(zipped_files, dest_folder, PACKAGE_TGZ_NAME)
# Issue #214 https://github.com/conan-io/conan/issues/214
touch_folder(dest_folder)
output.success('Package installed %s' % package_id)
def get_recipe(self, conan_reference, remote):
"""
Read the conans from remotes
Will iterate the remotes to find the conans unless remote was specified
returns (dict relative_filepath:abs_path , remote_name)"""
dest_folder = self._client_cache.export(conan_reference)
rmdir(dest_folder)
def filter_function(urls):
if CONANFILE not in list(urls.keys()):
raise NotFoundException("Conan '%s' doesn't have a %s!"
% (conan_reference, CONANFILE))
urls.pop(EXPORT_SOURCES_TGZ_NAME, None)
return urls
t1 = time.time()
urls = self._call_remote(remote, "get_recipe_urls", conan_reference)
urls = filter_function(urls)
if not urls:
return conan_reference
zipped_files = self._call_remote(remote, "download_files_to_folder", urls, dest_folder)
duration = time.time() - t1
log_recipe_download(conan_reference, duration, remote, zipped_files)
unzip_and_get_files(zipped_files, dest_folder, EXPORT_TGZ_NAME)
# Make sure that the source dir is deleted
rm_conandir(self._client_cache.source(conan_reference))
touch_folder(dest_folder)
def get_recipe(self, conan_reference, remote):
"""
Read the conans from remotes
Will iterate the remotes to find the conans unless remote was specified
returns (dict relative_filepath:abs_path , remote_name)"""
dest_folder = self._client_cache.export(conan_reference)
rmdir(dest_folder)
def filter_function(urls):
if CONANFILE not in list(urls.keys()):
raise NotFoundException("Conan '%s' doesn't have a %s!"
% (conan_reference, CONANFILE))
urls.pop(EXPORT_SOURCES_TGZ_NAME, None)
return urls
t1 = time.time()
urls = self._call_remote(remote, "get_recipe_urls", conan_reference)
urls = filter_function(urls)
if not urls:
return conan_reference
zipped_files = self._call_remote(remote, "download_files_to_folder", urls, dest_folder)
duration = time.time() - t1
log_recipe_download(conan_reference, duration, remote, zipped_files)
unzip_and_get_files(zipped_files, dest_folder, EXPORT_TGZ_NAME)
# Make sure that the source dir is deleted
rm_conandir(self._client_cache.source(conan_reference))
touch_folder(dest_folder)
def get_package(self, package_reference, dest_folder, remote, output,
recorder):
package_id = package_reference.package_id
conanfile_path = self._client_cache.conanfile(package_reference.conan)
self._hook_manager.execute("pre_download_package",
conanfile_path=conanfile_path,
reference=package_reference.conan,
package_id=package_id,
remote=remote)
output.info("Retrieving package %s from remote '%s' " %
(package_id, remote.name))
rm_conandir(dest_folder) # Remove first the destination folder
t1 = time.time()
try:
zipped_files, new_ref, rev_time = self._call_remote(
remote, "get_package", package_reference, dest_folder)
with self._client_cache.update_metadata(new_ref.conan) as metadata:
metadata.packages[
new_ref.package_id].revision = new_ref.revision
metadata.packages[
new_ref.
package_id].recipe_revision = new_ref.conan.revision
metadata.packages[new_ref.package_id].time = rev_time
duration = time.time() - t1
log_package_download(package_reference, duration, remote,
zipped_files)
unzip_and_get_files(zipped_files,
dest_folder,
PACKAGE_TGZ_NAME,
output=self._output)
# Issue #214 https://github.com/conan-io/conan/issues/214
touch_folder(dest_folder)
if get_env("CONAN_READ_ONLY_CACHE", False):
make_read_only(dest_folder)
recorder.package_downloaded(package_reference, remote.url)
output.success('Package installed %s' % package_id)
except NotFoundException:
raise NotFoundException("Package binary '%s' not found in '%s'" %
(package_reference, remote.name))
except BaseException as e:
output.error("Exception while getting package: %s" %
str(package_reference.package_id))
output.error("Exception: %s %s" % (type(e), str(e)))
try:
output.warn("Trying to remove package folder: %s" %
dest_folder)
rmdir(dest_folder)
except OSError as e:
raise ConanException(
"%s\n\nCouldn't remove folder '%s', might be busy or open. Close any app "
"using it, and retry" % (str(e), dest_folder))
raise
self._hook_manager.execute("post_download_package",
conanfile_path=conanfile_path,
reference=package_reference.conan,
package_id=package_id,
remote=remote)
return new_ref
def _get_package(self, layout, pref, remote, output, recorder, info):
t1 = time.time()
try:
headers = _headers_for_info(info)
pref = self._resolve_latest_pref(pref, remote, headers=headers)
snapshot = self._call_remote(remote, "get_package_snapshot", pref)
if not is_package_snapshot_complete(snapshot):
raise PackageNotFoundException(pref)
download_pkg_folder = layout.download_package(pref)
# Download files to the pkg_tgz folder, not to the final one
zipped_files = self._call_remote(remote, "get_package", pref,
download_pkg_folder)
# Compute and update the package metadata
package_checksums = calc_files_checksum(zipped_files)
with layout.update_metadata() as metadata:
metadata.packages[pref.id].revision = pref.revision
metadata.packages[pref.id].recipe_revision = pref.ref.revision
metadata.packages[pref.id].checksums = package_checksums
metadata.packages[pref.id].remote = remote.name
duration = time.time() - t1
log_package_download(pref, duration, remote, zipped_files)
tgz_file = zipped_files.pop(PACKAGE_TGZ_NAME, None)
check_compressed_files(PACKAGE_TGZ_NAME, zipped_files)
package_folder = layout.package(pref)
if tgz_file: # This must happen always, but just in case
# TODO: The output could be changed to the package one, but
uncompress_file(tgz_file, package_folder, output=self._output)
mkdir(
package_folder
) # Just in case it doesn't exist, because uncompress did nothing
for file_name, file_path in zipped_files.items(
): # copy CONANINFO and CONANMANIFEST
shutil.move(file_path, os.path.join(package_folder, file_name))
# Issue #214 https://github.com/conan-io/conan/issues/214
touch_folder(package_folder)
if get_env("CONAN_READ_ONLY_CACHE", False):
make_read_only(package_folder)
recorder.package_downloaded(pref, remote.url)
output.success('Package installed %s' % pref.id)
output.info("Downloaded package revision %s" % pref.revision)
except NotFoundException:
raise PackageNotFoundException(pref)
except BaseException as e:
output.error("Exception while getting package: %s" % str(pref.id))
output.error("Exception: %s %s" % (type(e), str(e)))
raise
def get_recipe(self, ref, remote):
"""
Read the conans from remotes
Will iterate the remotes to find the conans unless remote was specified
returns (dict relative_filepath:abs_path , remote_name)"""
self._hook_manager.execute("pre_download_recipe",
reference=ref,
remote=remote)
package_layout = self._cache.package_layout(ref)
package_layout.export_remove()
ref = self._resolve_latest_ref(ref, remote)
t1 = time.time()
download_export = package_layout.download_export()
zipped_files = self._call_remote(remote, "get_recipe", ref,
download_export)
duration = time.time() - t1
log_recipe_download(ref, duration, remote.name, zipped_files)
recipe_checksums = calc_files_checksum(zipped_files)
export_folder = package_layout.export()
tgz_file = zipped_files.pop(EXPORT_TGZ_NAME, None)
check_compressed_files(EXPORT_TGZ_NAME, zipped_files)
if tgz_file:
uncompress_file(tgz_file, export_folder, output=self._output)
mkdir(export_folder)
for file_name, file_path in zipped_files.items(): # copy CONANFILE
shutil.move(file_path, os.path.join(export_folder, file_name))
# Make sure that the source dir is deleted
rm_conandir(package_layout.source())
touch_folder(export_folder)
conanfile_path = package_layout.conanfile()
with package_layout.update_metadata() as metadata:
metadata.recipe.revision = ref.revision
metadata.recipe.checksums = recipe_checksums
metadata.recipe.remote = remote.name
self._hook_manager.execute("post_download_recipe",
conanfile_path=conanfile_path,
reference=ref,
remote=remote)
return ref
def get_recipe_sources(self, ref, layout, remote):
assert ref.revision, "get_recipe_sources requires RREV"
t1 = time.time()
download_folder = layout.download_export()
export_sources_folder = layout.export_sources()
zipped_files = self._call_remote(remote, "get_recipe_sources", ref, download_folder)
if not zipped_files:
mkdir(export_sources_folder) # create the folder even if no source files
return
duration = time.time() - t1
log_recipe_sources_download(ref, duration, remote.name, zipped_files)
tgz_file = zipped_files[EXPORT_SOURCES_TGZ_NAME]
check_compressed_files(EXPORT_SOURCES_TGZ_NAME, zipped_files)
uncompress_file(tgz_file, export_sources_folder, output=self._output)
touch_folder(export_sources_folder)
def get_package(self, conanfile, package_reference, dest_folder, remote, output, recorder):
package_id = package_reference.package_id
output.info("Retrieving package %s from remote '%s' " % (package_id, remote.name))
rm_conandir(dest_folder) # Remove first the destination folder
t1 = time.time()
try:
urls = self._call_remote(remote, "get_package_urls", package_reference)
zipped_files = self._call_remote(remote, "download_files_to_folder", urls, dest_folder)
except NotFoundException as e:
output.warn('Binary for %s not in remote: %s' % (package_id, str(e)))
raise_package_not_found_error(conanfile, package_reference.conan,
package_id, output, recorder, remote.url)
else:
duration = time.time() - t1
log_package_download(package_reference, duration, remote, zipped_files)
unzip_and_get_files(zipped_files, dest_folder, PACKAGE_TGZ_NAME)
# Issue #214 https://github.com/conan-io/conan/issues/214
touch_folder(dest_folder)
output.success('Package installed %s' % package_id)
def get_recipe(self, conan_reference, remote):
"""
Read the conans from remotes
Will iterate the remotes to find the conans unless remote was specified
returns (dict relative_filepath:abs_path , remote_name)"""
dest_folder = self._client_cache.export(conan_reference)
rmdir(dest_folder)
t1 = time.time()
zipped_files, conan_reference = self._call_remote(remote, "get_recipe", conan_reference,
dest_folder)
duration = time.time() - t1
log_recipe_download(conan_reference, duration, remote.name, zipped_files)
unzip_and_get_files(zipped_files, dest_folder, EXPORT_TGZ_NAME)
# Make sure that the source dir is deleted
rm_conandir(self._client_cache.source(conan_reference))
touch_folder(dest_folder)
return conan_reference