def upload_conan(self, conan_reference, remote, retry, retry_wait):
"""Will upload the conans to the first remote"""
export_folder = self._client_cache.export(conan_reference)
rel_files = relative_dirs(export_folder)
the_files = {
filename: os.path.join(export_folder, filename)
for filename in rel_files
}
if CONANFILE not in rel_files or CONAN_MANIFEST not in rel_files:
raise ConanException("Cannot upload corrupted recipe '%s'" %
str(conan_reference))
# FIXME: Check modified exports by hand?
the_files = compress_conan_files(the_files, export_folder,
EXPORT_TGZ_NAME, CONANFILE,
self._output)
ret = self._call_remote(remote, "upload_conan", conan_reference,
the_files, retry, retry_wait)
msg = "Uploaded conan recipe '%s' to '%s'" % (str(conan_reference),
remote.name)
# FIXME: server dependent
if remote.url == "https://server.conan.io":
msg += ": https://www.conan.io/source/%s" % "/".join(
conan_reference)
else:
msg += ": %s" % remote.url
self._output.info(msg)
return ret
def _get_paths(self, absolute_path, files_subset):
if not path_exists(absolute_path, self._store_folder):
raise NotFoundException("")
paths = relative_dirs(absolute_path)
if files_subset is not None:
paths = set(paths).intersection(set(files_subset))
abs_paths = [os.path.join(absolute_path, relpath) for relpath in paths]
return abs_paths
def get_snapshot(self, absolute_path="", files_subset=None):
"""returns a dict with the filepaths and md5"""
if not path_exists(absolute_path, self._store_folder):
raise NotFoundException("")
paths = relative_dirs(absolute_path)
if files_subset is not None:
paths = set(paths).intersection(set(files_subset))
abs_paths = [os.path.join(absolute_path, relpath) for relpath in paths]
return {filepath: md5sum(filepath) for filepath in abs_paths}
def unzip_and_get_files(files, destination_dir, tgz_name):
'''Moves all files from package_files, {relative_name: tmp_abs_path}
to destination_dir, unzipping the "tgz_name" if found'''
tgz_file = files.pop(tgz_name, None)
if tgz_file:
uncompress_file(tgz_file, destination_dir)
return relative_dirs(destination_dir)
def unzip_and_get_files(files, destination_dir, tgz_name):
'''Moves all files from package_files, {relative_name: tmp_abs_path}
to destination_dir, unzipping the "tgz_name" if found'''
tgz_file = files.pop(tgz_name, None)
if tgz_file:
uncompress_file(tgz_file, destination_dir)
return relative_dirs(destination_dir)
def _check_package_folder(self, mode):
""" Package folder must be always the same (might have tgz after upload)
"""
expected_package = [
"conaninfo.txt", "conanmanifest.txt",
os.sep.join(["include", "hello.h"])
]
if mode == "both":
expected_package.append(os.sep.join(["docs", "data.txt"]))
expected_package = sorted(expected_package)
self.assertEqual(sorted(relative_dirs(self.package_folder)),
expected_package)
def upload_conan(self, conan_reference, remote):
"""Will upload the conans to the first remote"""
export_folder = self._client_cache.export(conan_reference)
rel_files = relative_dirs(export_folder)
the_files = {filename: os.path.join(export_folder, filename) for filename in rel_files}
if CONANFILE not in rel_files or CONAN_MANIFEST not in rel_files:
raise ConanException("Cannot upload corrupted recipe '%s'" % str(conan_reference))
# FIXME: Check modified exports by hand?
the_files = compress_conan_files(the_files, export_folder, EXPORT_TGZ_NAME,
CONANFILE, self._output)
return self._call_remote(remote, "upload_conan", conan_reference, the_files)
def upload_package(self, package_reference, remote):
"""Will upload the package to the first remote"""
t1 = time.time()
# existing package, will use short paths if defined
package_folder = self._client_cache.package(package_reference, short_paths=None)
# Get all the files in that directory
rel_files = relative_dirs(package_folder)
self._output.rewrite_line("Checking package integrity...")
if CONANINFO not in rel_files or CONAN_MANIFEST not in rel_files:
raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference))
the_files = {filename: os.path.join(package_folder, filename) for filename in rel_files}
logger.debug("====> Time remote_manager build_files_set : %f" % (time.time() - t1))
# If package has been modified remove tgz to regenerate it
read_manifest, expected_manifest = self._client_cache.package_manifests(package_reference)
# Deals with the problem of generated .pyc, which is an issue for python packages
# TODO: refactor and improve the reading files + Manifests prior to upload for both
# recipes and packages
diff = set(expected_manifest.file_sums.keys()).difference(read_manifest.file_sums.keys())
for d in diff:
if d.endswith(".pyc"):
del expected_manifest.file_sums[d]
# It has to be in files, otherwise couldn't be in expected_manifest
del the_files[d]
if read_manifest is None or read_manifest.file_sums != expected_manifest.file_sums:
if PACKAGE_TGZ_NAME in the_files:
try:
tgz_path = os.path.join(package_folder, PACKAGE_TGZ_NAME)
os.unlink(tgz_path)
except Exception:
pass
raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference))
else:
self._output.rewrite_line("Package integrity OK!")
self._output.writeln("")
logger.debug("====> Time remote_manager check package integrity : %f" % (time.time() - t1))
the_files = compress_conan_files(the_files, package_folder, PACKAGE_TGZ_NAME,
CONANINFO, self._output)
tmp = self._call_remote(remote, "upload_package", package_reference, the_files)
logger.debug("====> Time remote_manager upload_package: %f" % (time.time() - t1))
return tmp
def upload_package(self, package_reference, remote):
"""Will upload the package to the first remote"""
t1 = time.time()
# existing package, will use short paths if defined
package_folder = self._client_cache.package(package_reference,
short_paths=None)
# Get all the files in that directory
rel_files = relative_dirs(package_folder)
self._output.rewrite_line("Checking package integrity...")
if CONANINFO not in rel_files or CONAN_MANIFEST not in rel_files:
raise ConanException("Cannot upload corrupted package '%s'" %
str(package_reference))
the_files = {
filename: os.path.join(package_folder, filename)
for filename in rel_files
}
logger.debug("====> Time remote_manager build_files_set : %f" %
(time.time() - t1))
# If package has been modified remove tgz to regenerate it
read_manifest, expected_manifest = self._client_cache.package_manifests(
package_reference)
if read_manifest is None or read_manifest.file_sums != expected_manifest.file_sums:
if PACKAGE_TGZ_NAME in the_files:
try:
tgz_path = os.path.join(package_folder, PACKAGE_TGZ_NAME)
os.unlink(tgz_path)
except Exception:
pass
raise ConanException("Cannot upload corrupted package '%s'" %
str(package_reference))
else:
self._output.rewrite_line("Package integrity OK!")
self._output.writeln("")
logger.debug("====> Time remote_manager check package integrity : %f" %
(time.time() - t1))
the_files = compress_package_files(the_files, package_folder,
self._output)
tmp = self._call_remote(remote, "upload_package", package_reference,
the_files)
logger.debug("====> Time remote_manager upload_package: %f" %
(time.time() - t1))
return tmp
def upload_conan(self, conan_reference, remote):
"""Will upload the conans to the first remote"""
export_folder = self._client_cache.export(conan_reference)
rel_files = relative_dirs(export_folder)
the_files = {
filename: os.path.join(export_folder, filename)
for filename in rel_files
}
if CONANFILE not in rel_files or CONAN_MANIFEST not in rel_files:
raise ConanException("Cannot upload corrupted recipe '%s'" %
str(conan_reference))
# FIXME: Check modified exports by hand?
the_files = compress_export_files(the_files, export_folder,
self._output)
return self._call_remote(remote, "upload_conan", conan_reference,
the_files)
def package_paths(self, package_reference):
''' Returns all file paths for a package (relative to conans directory)'''
return relative_dirs(self.package(package_reference))
def export_paths(self, conan_reference):
''' Returns all file paths for a conans (relative to conans directory)'''
return relative_dirs(self.export(conan_reference))
def package_paths(self, package_reference):
''' Returns all file paths for a package (relative to conans directory)'''
return relative_dirs(self.package(package_reference))
def export_paths(self, conan_reference):
''' Returns all file paths for a conans (relative to conans directory)'''
return relative_dirs(self.export(conan_reference))
def upload_package(self, package_reference, remote, retry, retry_wait):
"""Will upload the package to the first remote"""
t1 = time.time()
# existing package, will use short paths if defined
package_folder = self._client_cache.package(package_reference,
short_paths=None)
# Get all the files in that directory
rel_files = relative_dirs(package_folder)
self._output.rewrite_line("Checking package integrity...")
if CONANINFO not in rel_files or CONAN_MANIFEST not in rel_files:
logger.error("Missing info or manifest in uploading files: %s" %
(str(rel_files)))
raise ConanException("Cannot upload corrupted package '%s'" %
str(package_reference))
the_files = {
filename: os.path.join(package_folder, filename)
for filename in rel_files if not discarded_file(filename)
}
logger.debug("====> Time remote_manager build_files_set : %f" %
(time.time() - t1))
# If package has been modified remove tgz to regenerate it
read_manifest, expected_manifest = self._client_cache.package_manifests(
package_reference)
if read_manifest is None or read_manifest.file_sums != expected_manifest.file_sums:
self._output.writeln("")
for fname in read_manifest.file_sums.keys():
if read_manifest.file_sums[
fname] != expected_manifest.file_sums[fname]:
self._output.warn(
"Mismatched checksum for file %s (checksum: %s, expected: %s)"
% (fname, read_manifest.file_sums[fname],
expected_manifest.file_sums[fname]))
if PACKAGE_TGZ_NAME in the_files:
try:
tgz_path = os.path.join(package_folder, PACKAGE_TGZ_NAME)
os.unlink(tgz_path)
except Exception:
pass
logger.error("Manifests doesn't match!: %s != %s" % (str(
read_manifest.file_sums), str(expected_manifest.file_sums)))
raise ConanException("Cannot upload corrupted package '%s'" %
str(package_reference))
else:
self._output.rewrite_line("Package integrity OK!")
self._output.writeln("")
logger.debug("====> Time remote_manager check package integrity : %f" %
(time.time() - t1))
the_files = compress_conan_files(the_files, package_folder,
PACKAGE_TGZ_NAME, CONANINFO,
self._output)
tmp = self._call_remote(remote, "upload_package", package_reference,
the_files, retry, retry_wait)
logger.debug("====> Time remote_manager upload_package: %f" %
(time.time() - t1))
return tmp