def upload_package(self, package_reference, remote, retry, retry_wait, skip_upload=False, integrity_check=False): """Will upload the package to the first remote""" t1 = time.time() # existing package, will use short paths if defined package_folder = self._client_cache.package(package_reference, short_paths=None) # Get all the files in that directory files, symlinks = gather_files(package_folder) if CONANINFO not in files or CONAN_MANIFEST not in files: logger.error("Missing info or manifest in uploading files: %s" % (str(files))) raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference)) logger.debug("====> Time remote_manager build_files_set : %f" % (time.time() - t1)) if integrity_check: self._package_integrity_check(package_reference, files, package_folder) logger.debug("====> Time remote_manager check package integrity : %f" % (time.time() - t1)) the_files = compress_package_files(files, symlinks, package_folder, self._output) if skip_upload: return None tmp = self._call_remote(remote, "upload_package", package_reference, the_files, retry, retry_wait) duration = time.time() - t1 log_package_upload(package_reference, duration, the_files, remote) logger.debug("====> Time remote_manager upload_package: %f" % duration) if not tmp: self._output.rewrite_line("Package is up to date, upload skipped") self._output.writeln("") return tmp
def _upload_package(self, pref, retry=None, retry_wait=None, integrity_check=False, policy=None, p_remote=None): assert (pref.revision is not None), "Cannot upload a package without PREV" assert (pref.ref.revision is not None), "Cannot upload a package without RREV" conanfile_path = self._cache.package_layout(pref.ref).conanfile() self._hook_manager.execute("pre_upload_package", conanfile_path=conanfile_path, reference=pref.ref, package_id=pref.id, remote=p_remote) t1 = time.time() the_files = self._compress_package_files(pref, integrity_check) with self._cache.package_layout( pref.ref).update_metadata() as metadata: metadata.packages[pref.id].checksums = calc_files_checksum( the_files) if policy == UPLOAD_POLICY_SKIP: return None files_to_upload, deleted = self._package_files_to_upload( pref, policy, the_files, p_remote) if files_to_upload or deleted: self._remote_manager.upload_package(pref, files_to_upload, deleted, p_remote, retry, retry_wait) logger.debug("UPLOAD: Time upload package: %f" % (time.time() - t1)) else: self._output.info("Package is up to date, upload skipped") duration = time.time() - t1 log_package_upload(pref, duration, the_files, p_remote) self._hook_manager.execute("post_upload_package", conanfile_path=conanfile_path, reference=pref.ref, package_id=pref.id, remote=p_remote) logger.debug("UPLOAD: Time uploader upload_package: %f" % (time.time() - t1)) metadata = self._cache.package_layout(pref.ref).load_metadata() cur_package_remote = metadata.packages[pref.id].remote if not cur_package_remote and policy != UPLOAD_POLICY_SKIP: with self._cache.package_layout( pref.ref).update_metadata() as metadata: metadata.packages[pref.id].remote = p_remote.name return pref
def upload_package(self, package_reference, remote, retry, retry_wait, integrity_check=False, policy=None): """Will upload the package to the first remote""" conanfile_path = self._client_cache.conanfile(package_reference.conan) self._hook_manager.execute("pre_upload_package", conanfile_path=conanfile_path, reference=package_reference.conan, package_id=package_reference.package_id, remote=remote) t1 = time.time() # existing package, will use short paths if defined package_folder = self._client_cache.package(package_reference, short_paths=None) if is_dirty(package_folder): raise ConanException("Package %s is corrupted, aborting upload.\n" "Remove it with 'conan remove %s -p=%s'" % (package_reference, package_reference.conan, package_reference.package_id)) tgz_path = os.path.join(package_folder, PACKAGE_TGZ_NAME) if is_dirty(tgz_path): self._output.warn("%s: Removing %s, marked as dirty" % (str(package_reference), PACKAGE_TGZ_NAME)) os.remove(tgz_path) clean_dirty(tgz_path) # Get all the files in that directory files, symlinks = gather_files(package_folder) if CONANINFO not in files or CONAN_MANIFEST not in files: logger.error("Missing info or manifest in uploading files: %s" % (str(files))) raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference)) logger.debug("UPLOAD: Time remote_manager build_files_set : %f" % (time.time() - t1)) if integrity_check: self._package_integrity_check(package_reference, files, package_folder) logger.debug("UPLOAD: Time remote_manager check package integrity : %f" % (time.time() - t1)) the_files = compress_package_files(files, symlinks, package_folder, self._output) if policy == UPLOAD_POLICY_SKIP: return None uploaded, new_pref, rev_time = self._call_remote(remote, "upload_package", package_reference, the_files, retry, retry_wait, policy) # Update package revision with the rev_time (Created locally but with rev_time None) with self._client_cache.update_metadata(new_pref.conan) as metadata: metadata.packages[new_pref.package_id].time = rev_time duration = time.time() - t1 log_package_upload(package_reference, duration, the_files, remote) logger.debug("UPLOAD: Time remote_manager upload_package: %f" % duration) if not uploaded: self._output.rewrite_line("Package is up to date, upload skipped") self._output.writeln("") self._hook_manager.execute("post_upload_package", conanfile_path=conanfile_path, reference=package_reference.conan, package_id=package_reference.package_id, remote=remote) return new_pref
def upload_package(self, package_reference, remote, retry, retry_wait): """Will upload the package to the first remote""" t1 = time.time() # existing package, will use short paths if defined package_folder = self._client_cache.package(package_reference, short_paths=None) # Get all the files in that directory files = gather_files(package_folder) self._output.rewrite_line("Checking package integrity...") if CONANINFO not in files or CONAN_MANIFEST not in files: logger.error("Missing info or manifest in uploading files: %s" % (str(files))) raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference)) logger.debug("====> Time remote_manager build_files_set : %f" % (time.time() - t1)) # If package has been modified remove tgz to regenerate it read_manifest, expected_manifest = self._client_cache.package_manifests( package_reference) if read_manifest is None or read_manifest.file_sums != expected_manifest.file_sums: self._output.writeln("") for fname in read_manifest.file_sums.keys(): if read_manifest.file_sums[ fname] != expected_manifest.file_sums[fname]: self._output.warn( "Mismatched checksum for file %s (checksum: %s, expected: %s)" % (fname, read_manifest.file_sums[fname], expected_manifest.file_sums[fname])) if PACKAGE_TGZ_NAME in files: try: tgz_path = os.path.join(package_folder, PACKAGE_TGZ_NAME) os.unlink(tgz_path) except Exception: pass logger.error("Manifests doesn't match!: %s != %s" % (str( read_manifest.file_sums), str(expected_manifest.file_sums))) raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference)) else: self._output.rewrite_line("Package integrity OK!") self._output.writeln("") logger.debug("====> Time remote_manager check package integrity : %f" % (time.time() - t1)) the_files = compress_package_files(files, package_folder, self._output) tmp = self._call_remote(remote, "upload_package", package_reference, the_files, retry, retry_wait) duration = time.time() - t1 log_package_upload(package_reference, duration, the_files) logger.debug("====> Time remote_manager upload_package: %f" % (duration)) return tmp
def upload_package(self, package_reference, remote, retry, retry_wait, skip_upload=False, integrity_check=False, no_overwrite=None): """Will upload the package to the first remote""" t1 = time.time() # existing package, will use short paths if defined package_folder = self._client_cache.package(package_reference, short_paths=None) if is_dirty(package_folder): raise ConanException("Package %s is corrupted, aborting upload.\n" "Remove it with 'conan remove %s -p=%s'" % (package_reference, package_reference.conan, package_reference.package_id)) tgz_path = os.path.join(package_folder, PACKAGE_TGZ_NAME) if is_dirty(tgz_path): self._output.warn("%s: Removing %s, marked as dirty" % (str(package_reference), PACKAGE_TGZ_NAME)) os.remove(tgz_path) clean_dirty(tgz_path) # Get all the files in that directory files, symlinks = gather_files(package_folder) if CONANINFO not in files or CONAN_MANIFEST not in files: logger.error("Missing info or manifest in uploading files: %s" % (str(files))) raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference)) logger.debug("====> Time remote_manager build_files_set : %f" % (time.time() - t1)) if integrity_check: self._package_integrity_check(package_reference, files, package_folder) logger.debug( "====> Time remote_manager check package integrity : %f" % (time.time() - t1)) the_files = compress_package_files(files, symlinks, package_folder, self._output) if skip_upload: return None tmp = self._call_remote(remote, "upload_package", package_reference, the_files, retry, retry_wait, no_overwrite) duration = time.time() - t1 log_package_upload(package_reference, duration, the_files, remote) logger.debug("====> Time remote_manager upload_package: %f" % duration) if not tmp: self._output.rewrite_line("Package is up to date, upload skipped") self._output.writeln("") return tmp
def upload_package(self, package_reference, remote, retry, retry_wait, skip_upload=False): """Will upload the package to the first remote""" t1 = time.time() # existing package, will use short paths if defined package_folder = self._client_cache.package(package_reference, short_paths=None) # Get all the files in that directory files = gather_files(package_folder) self._output.rewrite_line("Checking package integrity...") if CONANINFO not in files or CONAN_MANIFEST not in files: logger.error("Missing info or manifest in uploading files: %s" % (str(files))) raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference)) logger.debug("====> Time remote_manager build_files_set : %f" % (time.time() - t1)) # If package has been modified remove tgz to regenerate it read_manifest, expected_manifest = self._client_cache.package_manifests(package_reference) if read_manifest != expected_manifest: self._output.writeln("") diff = read_manifest.difference(expected_manifest) for fname, (h1, h2) in diff.items(): self._output.warn("Mismatched checksum '%s' (manifest: %s, file: %s)" % (fname, h1, h2)) if PACKAGE_TGZ_NAME in files: try: tgz_path = os.path.join(package_folder, PACKAGE_TGZ_NAME) os.unlink(tgz_path) except Exception: pass error_msg = os.linesep.join("Mismatched checksum '%s' (manifest: %s, file: %s)" % (fname, h1, h2) for fname, (h1, h2) in diff.items()) logger.error("Manifests doesn't match!\n%s" % error_msg) raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference)) else: self._output.rewrite_line("Package integrity OK!") self._output.writeln("") logger.debug("====> Time remote_manager check package integrity : %f" % (time.time() - t1)) the_files = compress_package_files(files, package_folder, self._output) if not skip_upload: tmp = self._call_remote(remote, "upload_package", package_reference, the_files, retry, retry_wait) duration = time.time() - t1 log_package_upload(package_reference, duration, the_files, remote) logger.debug("====> Time remote_manager upload_package: %f" % duration) return tmp else: return None
def upload_package(self, package_reference, remote, retry, retry_wait, skip_upload=False): """Will upload the package to the first remote""" t1 = time.time() # existing package, will use short paths if defined package_folder = self._client_cache.package(package_reference, short_paths=None) # Get all the files in that directory files, symlinks = gather_files(package_folder) self._output.rewrite_line("Checking package integrity...") if CONANINFO not in files or CONAN_MANIFEST not in files: logger.error("Missing info or manifest in uploading files: %s" % (str(files))) raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference)) logger.debug("====> Time remote_manager build_files_set : %f" % (time.time() - t1)) # If package has been modified remove tgz to regenerate it read_manifest, expected_manifest = self._client_cache.package_manifests(package_reference) if read_manifest != expected_manifest: self._output.writeln("") diff = read_manifest.difference(expected_manifest) for fname, (h1, h2) in diff.items(): self._output.warn("Mismatched checksum '%s' (manifest: %s, file: %s)" % (fname, h1, h2)) if PACKAGE_TGZ_NAME in files: try: tgz_path = os.path.join(package_folder, PACKAGE_TGZ_NAME) os.unlink(tgz_path) except Exception: pass error_msg = os.linesep.join("Mismatched checksum '%s' (manifest: %s, file: %s)" % (fname, h1, h2) for fname, (h1, h2) in diff.items()) logger.error("Manifests doesn't match!\n%s" % error_msg) raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference)) else: self._output.rewrite_line("Package integrity OK!") self._output.writeln("") logger.debug("====> Time remote_manager check package integrity : %f" % (time.time() - t1)) the_files = compress_package_files(files, symlinks, package_folder, self._output) if not skip_upload: tmp = self._call_remote(remote, "upload_package", package_reference, the_files, retry, retry_wait) duration = time.time() - t1 log_package_upload(package_reference, duration, the_files, remote) logger.debug("====> Time remote_manager upload_package: %f" % duration) return tmp else: return None
def _upload_package(self, pref, retry=None, retry_wait=None, integrity_check=False, policy=None, p_remote=None): assert (pref.revision is not None), "Cannot upload a package without PREV" assert (pref.ref.revision is not None), "Cannot upload a package without RREV" conanfile_path = self._cache.conanfile(pref.ref) self._hook_manager.execute("pre_upload_package", conanfile_path=conanfile_path, reference=pref.ref, package_id=pref.id, remote=p_remote) t1 = time.time() the_files = self._compress_package_files(pref, integrity_check) if policy == UPLOAD_POLICY_SKIP: return None files_to_upload, deleted = self._package_files_to_upload( pref, policy, the_files, p_remote) if files_to_upload or deleted: self._remote_manager.upload_package(pref, files_to_upload, deleted, p_remote, retry, retry_wait) logger.debug("UPLOAD: Time upload package: %f" % (time.time() - t1)) else: self._user_io.out.info("Package is up to date, upload skipped") duration = time.time() - t1 log_package_upload(pref, duration, the_files, p_remote) self._hook_manager.execute("post_upload_package", conanfile_path=conanfile_path, reference=pref.ref, package_id=pref.id, remote=p_remote) logger.debug("UPLOAD: Time uploader upload_package: %f" % (time.time() - t1)) cur_package_remote = self._registry.prefs.get(pref.copy_clear_rev()) if not cur_package_remote and policy != UPLOAD_POLICY_SKIP: self._registry.prefs.set(pref, p_remote.name) return pref
def upload_package(self, package_reference, remote, retry, retry_wait, skip_upload=False, integrity_check=False, no_overwrite=None): """Will upload the package to the first remote""" t1 = time.time() # existing package, will use short paths if defined package_folder = self._client_cache.package(package_reference, short_paths=None) if is_dirty(package_folder): raise ConanException("Package %s is corrupted, aborting upload.\n" "Remove it with 'conan remove %s -p=%s'" % (package_reference, package_reference.conan, package_reference.package_id)) # Get all the files in that directory files, symlinks = gather_files(package_folder) if CONANINFO not in files or CONAN_MANIFEST not in files: logger.error("Missing info or manifest in uploading files: %s" % (str(files))) raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference)) logger.debug("====> Time remote_manager build_files_set : %f" % (time.time() - t1)) if integrity_check: self._package_integrity_check(package_reference, files, package_folder) logger.debug("====> Time remote_manager check package integrity : %f" % (time.time() - t1)) the_files = compress_package_files(files, symlinks, package_folder, self._output) if skip_upload: return None tmp = self._call_remote(remote, "upload_package", package_reference, the_files, retry, retry_wait, no_overwrite) duration = time.time() - t1 log_package_upload(package_reference, duration, the_files, remote) logger.debug("====> Time remote_manager upload_package: %f" % duration) if not tmp: self._output.rewrite_line("Package is up to date, upload skipped") self._output.writeln("") return tmp