def upload_package(self, package_reference, remote, retry, retry_wait, skip_upload=False,
integrity_check=False):
"""Will upload the package to the first remote"""
t1 = time.time()
# existing package, will use short paths if defined
package_folder = self._client_cache.package(package_reference, short_paths=None)
# Get all the files in that directory
files, symlinks = gather_files(package_folder)
if CONANINFO not in files or CONAN_MANIFEST not in files:
logger.error("Missing info or manifest in uploading files: %s" % (str(files)))
raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference))
logger.debug("====> Time remote_manager build_files_set : %f" % (time.time() - t1))
if integrity_check:
self._package_integrity_check(package_reference, files, package_folder)
logger.debug("====> Time remote_manager check package integrity : %f"
% (time.time() - t1))
the_files = compress_package_files(files, symlinks, package_folder, self._output)
if skip_upload:
return None
tmp = self._call_remote(remote, "upload_package", package_reference, the_files,
retry, retry_wait)
duration = time.time() - t1
log_package_upload(package_reference, duration, the_files, remote)
logger.debug("====> Time remote_manager upload_package: %f" % duration)
if not tmp:
self._output.rewrite_line("Package is up to date, upload skipped")
self._output.writeln("")
return tmp
def _upload_package(self,
pref,
retry=None,
retry_wait=None,
integrity_check=False,
policy=None,
p_remote=None):
assert (pref.revision
is not None), "Cannot upload a package without PREV"
assert (pref.ref.revision
is not None), "Cannot upload a package without RREV"
conanfile_path = self._cache.package_layout(pref.ref).conanfile()
self._hook_manager.execute("pre_upload_package",
conanfile_path=conanfile_path,
reference=pref.ref,
package_id=pref.id,
remote=p_remote)
t1 = time.time()
the_files = self._compress_package_files(pref, integrity_check)
with self._cache.package_layout(
pref.ref).update_metadata() as metadata:
metadata.packages[pref.id].checksums = calc_files_checksum(
the_files)
if policy == UPLOAD_POLICY_SKIP:
return None
files_to_upload, deleted = self._package_files_to_upload(
pref, policy, the_files, p_remote)
if files_to_upload or deleted:
self._remote_manager.upload_package(pref, files_to_upload, deleted,
p_remote, retry, retry_wait)
logger.debug("UPLOAD: Time upload package: %f" %
(time.time() - t1))
else:
self._output.info("Package is up to date, upload skipped")
duration = time.time() - t1
log_package_upload(pref, duration, the_files, p_remote)
self._hook_manager.execute("post_upload_package",
conanfile_path=conanfile_path,
reference=pref.ref,
package_id=pref.id,
remote=p_remote)
logger.debug("UPLOAD: Time uploader upload_package: %f" %
(time.time() - t1))
metadata = self._cache.package_layout(pref.ref).load_metadata()
cur_package_remote = metadata.packages[pref.id].remote
if not cur_package_remote and policy != UPLOAD_POLICY_SKIP:
with self._cache.package_layout(
pref.ref).update_metadata() as metadata:
metadata.packages[pref.id].remote = p_remote.name
return pref
def upload_package(self, package_reference, remote, retry, retry_wait, integrity_check=False,
policy=None):
"""Will upload the package to the first remote"""
conanfile_path = self._client_cache.conanfile(package_reference.conan)
self._hook_manager.execute("pre_upload_package", conanfile_path=conanfile_path,
reference=package_reference.conan,
package_id=package_reference.package_id,
remote=remote)
t1 = time.time()
# existing package, will use short paths if defined
package_folder = self._client_cache.package(package_reference, short_paths=None)
if is_dirty(package_folder):
raise ConanException("Package %s is corrupted, aborting upload.\n"
"Remove it with 'conan remove %s -p=%s'" % (package_reference,
package_reference.conan,
package_reference.package_id))
tgz_path = os.path.join(package_folder, PACKAGE_TGZ_NAME)
if is_dirty(tgz_path):
self._output.warn("%s: Removing %s, marked as dirty" % (str(package_reference), PACKAGE_TGZ_NAME))
os.remove(tgz_path)
clean_dirty(tgz_path)
# Get all the files in that directory
files, symlinks = gather_files(package_folder)
if CONANINFO not in files or CONAN_MANIFEST not in files:
logger.error("Missing info or manifest in uploading files: %s" % (str(files)))
raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference))
logger.debug("UPLOAD: Time remote_manager build_files_set : %f" % (time.time() - t1))
if integrity_check:
self._package_integrity_check(package_reference, files, package_folder)
logger.debug("UPLOAD: Time remote_manager check package integrity : %f"
% (time.time() - t1))
the_files = compress_package_files(files, symlinks, package_folder, self._output)
if policy == UPLOAD_POLICY_SKIP:
return None
uploaded, new_pref, rev_time = self._call_remote(remote, "upload_package", package_reference,
the_files, retry, retry_wait, policy)
# Update package revision with the rev_time (Created locally but with rev_time None)
with self._client_cache.update_metadata(new_pref.conan) as metadata:
metadata.packages[new_pref.package_id].time = rev_time
duration = time.time() - t1
log_package_upload(package_reference, duration, the_files, remote)
logger.debug("UPLOAD: Time remote_manager upload_package: %f" % duration)
if not uploaded:
self._output.rewrite_line("Package is up to date, upload skipped")
self._output.writeln("")
self._hook_manager.execute("post_upload_package", conanfile_path=conanfile_path,
reference=package_reference.conan,
package_id=package_reference.package_id, remote=remote)
return new_pref
def upload_package(self, package_reference, remote, retry, retry_wait):
"""Will upload the package to the first remote"""
t1 = time.time()
# existing package, will use short paths if defined
package_folder = self._client_cache.package(package_reference,
short_paths=None)
# Get all the files in that directory
files = gather_files(package_folder)
self._output.rewrite_line("Checking package integrity...")
if CONANINFO not in files or CONAN_MANIFEST not in files:
logger.error("Missing info or manifest in uploading files: %s" %
(str(files)))
raise ConanException("Cannot upload corrupted package '%s'" %
str(package_reference))
logger.debug("====> Time remote_manager build_files_set : %f" %
(time.time() - t1))
# If package has been modified remove tgz to regenerate it
read_manifest, expected_manifest = self._client_cache.package_manifests(
package_reference)
if read_manifest is None or read_manifest.file_sums != expected_manifest.file_sums:
self._output.writeln("")
for fname in read_manifest.file_sums.keys():
if read_manifest.file_sums[
fname] != expected_manifest.file_sums[fname]:
self._output.warn(
"Mismatched checksum for file %s (checksum: %s, expected: %s)"
% (fname, read_manifest.file_sums[fname],
expected_manifest.file_sums[fname]))
if PACKAGE_TGZ_NAME in files:
try:
tgz_path = os.path.join(package_folder, PACKAGE_TGZ_NAME)
os.unlink(tgz_path)
except Exception:
pass
logger.error("Manifests doesn't match!: %s != %s" % (str(
read_manifest.file_sums), str(expected_manifest.file_sums)))
raise ConanException("Cannot upload corrupted package '%s'" %
str(package_reference))
else:
self._output.rewrite_line("Package integrity OK!")
self._output.writeln("")
logger.debug("====> Time remote_manager check package integrity : %f" %
(time.time() - t1))
the_files = compress_package_files(files, package_folder, self._output)
tmp = self._call_remote(remote, "upload_package", package_reference,
the_files, retry, retry_wait)
duration = time.time() - t1
log_package_upload(package_reference, duration, the_files)
logger.debug("====> Time remote_manager upload_package: %f" %
(duration))
return tmp
def upload_package(self,
package_reference,
remote,
retry,
retry_wait,
skip_upload=False,
integrity_check=False,
no_overwrite=None):
"""Will upload the package to the first remote"""
t1 = time.time()
# existing package, will use short paths if defined
package_folder = self._client_cache.package(package_reference,
short_paths=None)
if is_dirty(package_folder):
raise ConanException("Package %s is corrupted, aborting upload.\n"
"Remove it with 'conan remove %s -p=%s'" %
(package_reference, package_reference.conan,
package_reference.package_id))
tgz_path = os.path.join(package_folder, PACKAGE_TGZ_NAME)
if is_dirty(tgz_path):
self._output.warn("%s: Removing %s, marked as dirty" %
(str(package_reference), PACKAGE_TGZ_NAME))
os.remove(tgz_path)
clean_dirty(tgz_path)
# Get all the files in that directory
files, symlinks = gather_files(package_folder)
if CONANINFO not in files or CONAN_MANIFEST not in files:
logger.error("Missing info or manifest in uploading files: %s" %
(str(files)))
raise ConanException("Cannot upload corrupted package '%s'" %
str(package_reference))
logger.debug("====> Time remote_manager build_files_set : %f" %
(time.time() - t1))
if integrity_check:
self._package_integrity_check(package_reference, files,
package_folder)
logger.debug(
"====> Time remote_manager check package integrity : %f" %
(time.time() - t1))
the_files = compress_package_files(files, symlinks, package_folder,
self._output)
if skip_upload:
return None
tmp = self._call_remote(remote, "upload_package", package_reference,
the_files, retry, retry_wait, no_overwrite)
duration = time.time() - t1
log_package_upload(package_reference, duration, the_files, remote)
logger.debug("====> Time remote_manager upload_package: %f" % duration)
if not tmp:
self._output.rewrite_line("Package is up to date, upload skipped")
self._output.writeln("")
return tmp
def upload_package(self, package_reference, remote, retry, retry_wait, skip_upload=False):
"""Will upload the package to the first remote"""
t1 = time.time()
# existing package, will use short paths if defined
package_folder = self._client_cache.package(package_reference, short_paths=None)
# Get all the files in that directory
files = gather_files(package_folder)
self._output.rewrite_line("Checking package integrity...")
if CONANINFO not in files or CONAN_MANIFEST not in files:
logger.error("Missing info or manifest in uploading files: %s" % (str(files)))
raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference))
logger.debug("====> Time remote_manager build_files_set : %f" % (time.time() - t1))
# If package has been modified remove tgz to regenerate it
read_manifest, expected_manifest = self._client_cache.package_manifests(package_reference)
if read_manifest != expected_manifest:
self._output.writeln("")
diff = read_manifest.difference(expected_manifest)
for fname, (h1, h2) in diff.items():
self._output.warn("Mismatched checksum '%s' (manifest: %s, file: %s)"
% (fname, h1, h2))
if PACKAGE_TGZ_NAME in files:
try:
tgz_path = os.path.join(package_folder, PACKAGE_TGZ_NAME)
os.unlink(tgz_path)
except Exception:
pass
error_msg = os.linesep.join("Mismatched checksum '%s' (manifest: %s, file: %s)"
% (fname, h1, h2) for fname, (h1, h2) in diff.items())
logger.error("Manifests doesn't match!\n%s" % error_msg)
raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference))
else:
self._output.rewrite_line("Package integrity OK!")
self._output.writeln("")
logger.debug("====> Time remote_manager check package integrity : %f" % (time.time() - t1))
the_files = compress_package_files(files, package_folder, self._output)
if not skip_upload:
tmp = self._call_remote(remote, "upload_package", package_reference, the_files,
retry, retry_wait)
duration = time.time() - t1
log_package_upload(package_reference, duration, the_files, remote)
logger.debug("====> Time remote_manager upload_package: %f" % duration)
return tmp
else:
return None
def upload_package(self, package_reference, remote, retry, retry_wait, skip_upload=False):
"""Will upload the package to the first remote"""
t1 = time.time()
# existing package, will use short paths if defined
package_folder = self._client_cache.package(package_reference, short_paths=None)
# Get all the files in that directory
files, symlinks = gather_files(package_folder)
self._output.rewrite_line("Checking package integrity...")
if CONANINFO not in files or CONAN_MANIFEST not in files:
logger.error("Missing info or manifest in uploading files: %s" % (str(files)))
raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference))
logger.debug("====> Time remote_manager build_files_set : %f" % (time.time() - t1))
# If package has been modified remove tgz to regenerate it
read_manifest, expected_manifest = self._client_cache.package_manifests(package_reference)
if read_manifest != expected_manifest:
self._output.writeln("")
diff = read_manifest.difference(expected_manifest)
for fname, (h1, h2) in diff.items():
self._output.warn("Mismatched checksum '%s' (manifest: %s, file: %s)"
% (fname, h1, h2))
if PACKAGE_TGZ_NAME in files:
try:
tgz_path = os.path.join(package_folder, PACKAGE_TGZ_NAME)
os.unlink(tgz_path)
except Exception:
pass
error_msg = os.linesep.join("Mismatched checksum '%s' (manifest: %s, file: %s)"
% (fname, h1, h2) for fname, (h1, h2) in diff.items())
logger.error("Manifests doesn't match!\n%s" % error_msg)
raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference))
else:
self._output.rewrite_line("Package integrity OK!")
self._output.writeln("")
logger.debug("====> Time remote_manager check package integrity : %f" % (time.time() - t1))
the_files = compress_package_files(files, symlinks, package_folder, self._output)
if not skip_upload:
tmp = self._call_remote(remote, "upload_package", package_reference, the_files,
retry, retry_wait)
duration = time.time() - t1
log_package_upload(package_reference, duration, the_files, remote)
logger.debug("====> Time remote_manager upload_package: %f" % duration)
return tmp
else:
return None
def _upload_package(self,
pref,
retry=None,
retry_wait=None,
integrity_check=False,
policy=None,
p_remote=None):
assert (pref.revision
is not None), "Cannot upload a package without PREV"
assert (pref.ref.revision
is not None), "Cannot upload a package without RREV"
conanfile_path = self._cache.conanfile(pref.ref)
self._hook_manager.execute("pre_upload_package",
conanfile_path=conanfile_path,
reference=pref.ref,
package_id=pref.id,
remote=p_remote)
t1 = time.time()
the_files = self._compress_package_files(pref, integrity_check)
if policy == UPLOAD_POLICY_SKIP:
return None
files_to_upload, deleted = self._package_files_to_upload(
pref, policy, the_files, p_remote)
if files_to_upload or deleted:
self._remote_manager.upload_package(pref, files_to_upload, deleted,
p_remote, retry, retry_wait)
logger.debug("UPLOAD: Time upload package: %f" %
(time.time() - t1))
else:
self._user_io.out.info("Package is up to date, upload skipped")
duration = time.time() - t1
log_package_upload(pref, duration, the_files, p_remote)
self._hook_manager.execute("post_upload_package",
conanfile_path=conanfile_path,
reference=pref.ref,
package_id=pref.id,
remote=p_remote)
logger.debug("UPLOAD: Time uploader upload_package: %f" %
(time.time() - t1))
cur_package_remote = self._registry.prefs.get(pref.copy_clear_rev())
if not cur_package_remote and policy != UPLOAD_POLICY_SKIP:
self._registry.prefs.set(pref, p_remote.name)
return pref
def upload_package(self, package_reference, remote, retry, retry_wait, skip_upload=False,
integrity_check=False, no_overwrite=None):
"""Will upload the package to the first remote"""
t1 = time.time()
# existing package, will use short paths if defined
package_folder = self._client_cache.package(package_reference, short_paths=None)
if is_dirty(package_folder):
raise ConanException("Package %s is corrupted, aborting upload.\n"
"Remove it with 'conan remove %s -p=%s'" % (package_reference,
package_reference.conan,
package_reference.package_id))
# Get all the files in that directory
files, symlinks = gather_files(package_folder)
if CONANINFO not in files or CONAN_MANIFEST not in files:
logger.error("Missing info or manifest in uploading files: %s" % (str(files)))
raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference))
logger.debug("====> Time remote_manager build_files_set : %f" % (time.time() - t1))
if integrity_check:
self._package_integrity_check(package_reference, files, package_folder)
logger.debug("====> Time remote_manager check package integrity : %f"
% (time.time() - t1))
the_files = compress_package_files(files, symlinks, package_folder, self._output)
if skip_upload:
return None
tmp = self._call_remote(remote, "upload_package", package_reference, the_files,
retry, retry_wait, no_overwrite)
duration = time.time() - t1
log_package_upload(package_reference, duration, the_files, remote)
logger.debug("====> Time remote_manager upload_package: %f" % duration)
if not tmp:
self._output.rewrite_line("Package is up to date, upload skipped")
self._output.writeln("")
return tmp
