def upload_recipe(self, conan_reference, remote, retry, retry_wait, ignore_deleted_file,
skip_upload=False, no_overwrite=None):
"""Will upload the conans to the first remote"""
t1 = time.time()
export_folder = self._client_cache.export(conan_reference)
for f in (EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME):
tgz_path = os.path.join(export_folder, f)
if is_dirty(tgz_path):
self._output.warn("%s: Removing %s, marked as dirty" % (str(conan_reference), f))
os.remove(tgz_path)
clean_dirty(tgz_path)
files, symlinks = gather_files(export_folder)
if CONANFILE not in files or CONAN_MANIFEST not in files:
raise ConanException("Cannot upload corrupted recipe '%s'" % str(conan_reference))
export_src_folder = self._client_cache.export_sources(conan_reference, short_paths=None)
src_files, src_symlinks = gather_files(export_src_folder)
the_files = _compress_recipe_files(files, symlinks, src_files, src_symlinks, export_folder,
self._output)
if skip_upload:
return None
ret, new_ref = self._call_remote(remote, "upload_recipe", conan_reference, the_files, retry,
retry_wait, ignore_deleted_file, no_overwrite)
duration = time.time() - t1
log_recipe_upload(new_ref, duration, the_files, remote.name)
if ret:
msg = "Uploaded conan recipe '%s' to '%s'" % (str(new_ref), remote.name)
url = remote.url.replace("https://api.bintray.com/conan", "https://bintray.com")
msg += ": %s" % url
else:
msg = "Recipe is up to date, upload skipped"
self._output.info(msg)
return new_ref
def upload_recipe(self, conan_reference, remote, retry, retry_wait, ignore_deleted_file,
skip_upload=False, no_overwrite=None):
"""Will upload the conans to the first remote"""
t1 = time.time()
export_folder = self._client_cache.export(conan_reference)
files, symlinks = gather_files(export_folder)
if CONANFILE not in files or CONAN_MANIFEST not in files:
raise ConanException("Cannot upload corrupted recipe '%s'" % str(conan_reference))
export_src_folder = self._client_cache.export_sources(conan_reference, short_paths=None)
src_files, src_symlinks = gather_files(export_src_folder)
the_files = _compress_recipe_files(files, symlinks, src_files, src_symlinks, export_folder,
self._output)
if skip_upload:
return None
ret = self._call_remote(remote, "upload_recipe", conan_reference, the_files,
retry, retry_wait, ignore_deleted_file, no_overwrite)
duration = time.time() - t1
log_recipe_upload(conan_reference, duration, the_files, remote)
if ret:
msg = "Uploaded conan recipe '%s' to '%s'" % (str(conan_reference), remote.name)
url = remote.url.replace("https://api.bintray.com/conan", "https://bintray.com")
msg += ": %s" % url
else:
msg = "Recipe is up to date, upload skipped"
self._output.info(msg)
return ret
def upload_recipe(self, conan_reference, remote, retry, retry_wait, ignore_deleted_file,
skip_upload=False, no_overwrite=None):
"""Will upload the conans to the first remote"""
t1 = time.time()
export_folder = self._client_cache.export(conan_reference)
files, symlinks = gather_files(export_folder)
if CONANFILE not in files or CONAN_MANIFEST not in files:
raise ConanException("Cannot upload corrupted recipe '%s'" % str(conan_reference))
export_src_folder = self._client_cache.export_sources(conan_reference, short_paths=None)
src_files, src_symlinks = gather_files(export_src_folder)
the_files = _compress_recipe_files(files, symlinks, src_files, src_symlinks, export_folder,
self._output)
if skip_upload:
return None
ret = self._call_remote(remote, "upload_recipe", conan_reference, the_files,
retry, retry_wait, ignore_deleted_file, no_overwrite)
duration = time.time() - t1
log_recipe_upload(conan_reference, duration, the_files, remote)
if ret:
msg = "Uploaded conan recipe '%s' to '%s'" % (str(conan_reference), remote.name)
url = remote.url.replace("https://api.bintray.com/conan", "https://bintray.com")
msg += ": %s" % url
else:
msg = "Recipe is up to date, upload skipped"
self._output.info(msg)
return ret
def upload_recipe(self, conan_reference, remote, retry, retry_wait, ignore_deleted_file,
skip_upload=False):
"""Will upload the conans to the first remote"""
t1 = time.time()
export_folder = self._client_cache.export(conan_reference)
files, symlinks = gather_files(export_folder)
if CONANFILE not in files or CONAN_MANIFEST not in files:
raise ConanException("Cannot upload corrupted recipe '%s'" % str(conan_reference))
export_src_folder = self._client_cache.export_sources(conan_reference, short_paths=None)
src_files, src_symlinks = gather_files(export_src_folder)
the_files = _compress_recipe_files(files, symlinks, src_files, src_symlinks, export_folder,
self._output)
if skip_upload:
return None
ret = self._call_remote(remote, "upload_recipe", conan_reference, the_files,
retry, retry_wait, ignore_deleted_file)
duration = time.time() - t1
log_recipe_upload(conan_reference, duration, the_files, remote)
if ret:
msg = "Uploaded conan recipe '%s' to '%s'" % (str(conan_reference), remote.name)
# FIXME: server dependent
if remote.url == "https://server.conan.io":
msg += ": https://www.conan.io/source/%s" % "/".join(conan_reference)
else:
msg += ": %s" % remote.url
else:
msg = "Recipe is up to date, upload skipped"
self._output.info(msg)
return ret
def upload_recipe(self, conan_reference, remote, retry, retry_wait, policy,
remote_manifest):
conanfile_path = self._client_cache.conanfile(conan_reference)
self._hook_manager.execute("pre_upload_recipe",
conanfile_path=conanfile_path,
reference=conan_reference,
remote=remote)
t1 = time.time()
export_folder = self._client_cache.export(conan_reference)
for f in (EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME):
tgz_path = os.path.join(export_folder, f)
if is_dirty(tgz_path):
self._output.warn("%s: Removing %s, marked as dirty" %
(str(conan_reference), f))
os.remove(tgz_path)
clean_dirty(tgz_path)
files, symlinks = gather_files(export_folder)
if CONANFILE not in files or CONAN_MANIFEST not in files:
raise ConanException("Cannot upload corrupted recipe '%s'" %
str(conan_reference))
export_src_folder = self._client_cache.export_sources(conan_reference,
short_paths=None)
src_files, src_symlinks = gather_files(export_src_folder)
the_files = _compress_recipe_files(files, symlinks, src_files,
src_symlinks, export_folder,
self._output)
if policy == UPLOAD_POLICY_SKIP:
return conan_reference
ret, rev_time = self._call_remote(remote, "upload_recipe",
conan_reference, the_files, retry,
retry_wait, policy, remote_manifest)
# Update package revision with the rev_time (Created locally but with rev_time None)
with self._client_cache.update_metadata(conan_reference) as metadata:
metadata.recipe.time = rev_time
duration = time.time() - t1
log_recipe_upload(conan_reference, duration, the_files, remote.name)
if ret:
msg = "Uploaded conan recipe '%s' to '%s'" % (str(conan_reference),
remote.name)
url = remote.url.replace("https://api.bintray.com/conan",
"https://bintray.com")
msg += ": %s" % url
else:
msg = "Recipe is up to date, upload skipped"
self._output.info(msg)
self._hook_manager.execute("post_upload_recipe",
conanfile_path=conanfile_path,
reference=conan_reference,
remote=remote)
def upload_package(self, package_reference, remote, retry, retry_wait, skip_upload=False,
integrity_check=False):
"""Will upload the package to the first remote"""
t1 = time.time()
# existing package, will use short paths if defined
package_folder = self._client_cache.package(package_reference, short_paths=None)
# Get all the files in that directory
files, symlinks = gather_files(package_folder)
if CONANINFO not in files or CONAN_MANIFEST not in files:
logger.error("Missing info or manifest in uploading files: %s" % (str(files)))
raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference))
logger.debug("====> Time remote_manager build_files_set : %f" % (time.time() - t1))
if integrity_check:
self._package_integrity_check(package_reference, files, package_folder)
logger.debug("====> Time remote_manager check package integrity : %f"
% (time.time() - t1))
the_files = compress_package_files(files, symlinks, package_folder, self._output)
if skip_upload:
return None
tmp = self._call_remote(remote, "upload_package", package_reference, the_files,
retry, retry_wait)
duration = time.time() - t1
log_package_upload(package_reference, duration, the_files, remote)
logger.debug("====> Time remote_manager upload_package: %f" % duration)
if not tmp:
self._output.rewrite_line("Package is up to date, upload skipped")
self._output.writeln("")
return tmp
def _compress_package_files(self, pref, integrity_check):
t1 = time.time()
# existing package, will use short paths if defined
package_folder = self._cache.package(pref, short_paths=None)
if is_dirty(package_folder):
raise ConanException("Package %s is corrupted, aborting upload.\n"
"Remove it with 'conan remove %s -p=%s'"
% (pref, pref.ref, pref.id))
tgz_path = os.path.join(package_folder, PACKAGE_TGZ_NAME)
if is_dirty(tgz_path):
self._user_io.out.warn("%s: Removing %s, marked as dirty"
% (str(pref), PACKAGE_TGZ_NAME))
os.remove(tgz_path)
clean_dirty(tgz_path)
# Get all the files in that directory
files, symlinks = gather_files(package_folder)
if CONANINFO not in files or CONAN_MANIFEST not in files:
logger.error("Missing info or manifest in uploading files: %s" % (str(files)))
raise ConanException("Cannot upload corrupted package '%s'" % str(pref))
logger.debug("UPLOAD: Time remote_manager build_files_set : %f" % (time.time() - t1))
if integrity_check:
self._package_integrity_check(pref, files, package_folder)
logger.debug("UPLOAD: Time remote_manager check package integrity : %f"
% (time.time() - t1))
the_files = _compress_package_files(files, symlinks, package_folder, self._user_io.out)
return the_files
def upload_recipe(self, conan_reference, remote, retry, retry_wait, ignore_deleted_file,
skip_upload=False):
"""Will upload the conans to the first remote"""
t1 = time.time()
export_folder = self._client_cache.export(conan_reference)
files, symlinks = gather_files(export_folder)
if CONANFILE not in files or CONAN_MANIFEST not in files:
raise ConanException("Cannot upload corrupted recipe '%s'" % str(conan_reference))
the_files = compress_recipe_files(files, symlinks, export_folder, self._output)
if skip_upload:
return None
ret = self._call_remote(remote, "upload_recipe", conan_reference, the_files,
retry, retry_wait, ignore_deleted_file)
duration = time.time() - t1
log_recipe_upload(conan_reference, duration, the_files, remote)
msg = "Uploaded conan recipe '%s' to '%s'" % (str(conan_reference), remote.name)
# FIXME: server dependent
if remote.url == "https://server.conan.io":
msg += ": https://www.conan.io/source/%s" % "/".join(conan_reference)
else:
msg += ": %s" % remote.url
self._output.info(msg)
return ret
def upload_package(self, package_reference, remote, retry, retry_wait, integrity_check=False,
policy=None):
"""Will upload the package to the first remote"""
conanfile_path = self._client_cache.conanfile(package_reference.conan)
self._hook_manager.execute("pre_upload_package", conanfile_path=conanfile_path,
reference=package_reference.conan,
package_id=package_reference.package_id,
remote=remote)
t1 = time.time()
# existing package, will use short paths if defined
package_folder = self._client_cache.package(package_reference, short_paths=None)
if is_dirty(package_folder):
raise ConanException("Package %s is corrupted, aborting upload.\n"
"Remove it with 'conan remove %s -p=%s'" % (package_reference,
package_reference.conan,
package_reference.package_id))
tgz_path = os.path.join(package_folder, PACKAGE_TGZ_NAME)
if is_dirty(tgz_path):
self._output.warn("%s: Removing %s, marked as dirty" % (str(package_reference), PACKAGE_TGZ_NAME))
os.remove(tgz_path)
clean_dirty(tgz_path)
# Get all the files in that directory
files, symlinks = gather_files(package_folder)
if CONANINFO not in files or CONAN_MANIFEST not in files:
logger.error("Missing info or manifest in uploading files: %s" % (str(files)))
raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference))
logger.debug("UPLOAD: Time remote_manager build_files_set : %f" % (time.time() - t1))
if integrity_check:
self._package_integrity_check(package_reference, files, package_folder)
logger.debug("UPLOAD: Time remote_manager check package integrity : %f"
% (time.time() - t1))
the_files = compress_package_files(files, symlinks, package_folder, self._output)
if policy == UPLOAD_POLICY_SKIP:
return None
uploaded, new_pref, rev_time = self._call_remote(remote, "upload_package", package_reference,
the_files, retry, retry_wait, policy)
# Update package revision with the rev_time (Created locally but with rev_time None)
with self._client_cache.update_metadata(new_pref.conan) as metadata:
metadata.packages[new_pref.package_id].time = rev_time
duration = time.time() - t1
log_package_upload(package_reference, duration, the_files, remote)
logger.debug("UPLOAD: Time remote_manager upload_package: %f" % duration)
if not uploaded:
self._output.rewrite_line("Package is up to date, upload skipped")
self._output.writeln("")
self._hook_manager.execute("post_upload_package", conanfile_path=conanfile_path,
reference=package_reference.conan,
package_id=package_reference.package_id, remote=remote)
return new_pref
def upload_package(self, package_reference, remote, retry, retry_wait):
"""Will upload the package to the first remote"""
t1 = time.time()
# existing package, will use short paths if defined
package_folder = self._client_cache.package(package_reference,
short_paths=None)
# Get all the files in that directory
files = gather_files(package_folder)
self._output.rewrite_line("Checking package integrity...")
if CONANINFO not in files or CONAN_MANIFEST not in files:
logger.error("Missing info or manifest in uploading files: %s" %
(str(files)))
raise ConanException("Cannot upload corrupted package '%s'" %
str(package_reference))
logger.debug("====> Time remote_manager build_files_set : %f" %
(time.time() - t1))
# If package has been modified remove tgz to regenerate it
read_manifest, expected_manifest = self._client_cache.package_manifests(
package_reference)
if read_manifest is None or read_manifest.file_sums != expected_manifest.file_sums:
self._output.writeln("")
for fname in read_manifest.file_sums.keys():
if read_manifest.file_sums[
fname] != expected_manifest.file_sums[fname]:
self._output.warn(
"Mismatched checksum for file %s (checksum: %s, expected: %s)"
% (fname, read_manifest.file_sums[fname],
expected_manifest.file_sums[fname]))
if PACKAGE_TGZ_NAME in files:
try:
tgz_path = os.path.join(package_folder, PACKAGE_TGZ_NAME)
os.unlink(tgz_path)
except Exception:
pass
logger.error("Manifests doesn't match!: %s != %s" % (str(
read_manifest.file_sums), str(expected_manifest.file_sums)))
raise ConanException("Cannot upload corrupted package '%s'" %
str(package_reference))
else:
self._output.rewrite_line("Package integrity OK!")
self._output.writeln("")
logger.debug("====> Time remote_manager check package integrity : %f" %
(time.time() - t1))
the_files = compress_package_files(files, package_folder, self._output)
tmp = self._call_remote(remote, "upload_package", package_reference,
the_files, retry, retry_wait)
duration = time.time() - t1
log_package_upload(package_reference, duration, the_files)
logger.debug("====> Time remote_manager upload_package: %f" %
(duration))
return tmp
def upload_package(self,
package_reference,
remote,
retry,
retry_wait,
skip_upload=False,
integrity_check=False,
no_overwrite=None):
"""Will upload the package to the first remote"""
t1 = time.time()
# existing package, will use short paths if defined
package_folder = self._client_cache.package(package_reference,
short_paths=None)
if is_dirty(package_folder):
raise ConanException("Package %s is corrupted, aborting upload.\n"
"Remove it with 'conan remove %s -p=%s'" %
(package_reference, package_reference.conan,
package_reference.package_id))
tgz_path = os.path.join(package_folder, PACKAGE_TGZ_NAME)
if is_dirty(tgz_path):
self._output.warn("%s: Removing %s, marked as dirty" %
(str(package_reference), PACKAGE_TGZ_NAME))
os.remove(tgz_path)
clean_dirty(tgz_path)
# Get all the files in that directory
files, symlinks = gather_files(package_folder)
if CONANINFO not in files or CONAN_MANIFEST not in files:
logger.error("Missing info or manifest in uploading files: %s" %
(str(files)))
raise ConanException("Cannot upload corrupted package '%s'" %
str(package_reference))
logger.debug("====> Time remote_manager build_files_set : %f" %
(time.time() - t1))
if integrity_check:
self._package_integrity_check(package_reference, files,
package_folder)
logger.debug(
"====> Time remote_manager check package integrity : %f" %
(time.time() - t1))
the_files = compress_package_files(files, symlinks, package_folder,
self._output)
if skip_upload:
return None
tmp = self._call_remote(remote, "upload_package", package_reference,
the_files, retry, retry_wait, no_overwrite)
duration = time.time() - t1
log_package_upload(package_reference, duration, the_files, remote)
logger.debug("====> Time remote_manager upload_package: %f" % duration)
if not tmp:
self._output.rewrite_line("Package is up to date, upload skipped")
self._output.writeln("")
return tmp
def _compress_recipe_files(self, ref):
export_folder = self._cache.export(ref)
for f in (EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME):
tgz_path = os.path.join(export_folder, f)
if is_dirty(tgz_path):
self._user_io.out.warn("%s: Removing %s, marked as dirty" % (str(ref), f))
os.remove(tgz_path)
clean_dirty(tgz_path)
files, symlinks = gather_files(export_folder)
if CONANFILE not in files or CONAN_MANIFEST not in files:
raise ConanException("Cannot upload corrupted recipe '%s'" % str(ref))
export_src_folder = self._cache.export_sources(ref, short_paths=None)
src_files, src_symlinks = gather_files(export_src_folder)
the_files = _compress_recipe_files(files, symlinks, src_files, src_symlinks, export_folder,
self._user_io.out)
return the_files
def _compress_recipe_files(self, ref):
layout = self._cache.package_layout(ref)
download_export_folder = layout.download_export()
for f in (EXPORT_TGZ_NAME, EXPORT_SOURCES_TGZ_NAME):
tgz_path = os.path.join(download_export_folder, f)
if is_dirty(tgz_path):
self._output.warn("%s: Removing %s, marked as dirty" %
(str(ref), f))
os.remove(tgz_path)
clean_dirty(tgz_path)
export_folder = layout.export()
files, symlinks = gather_files(export_folder)
if CONANFILE not in files or CONAN_MANIFEST not in files:
raise ConanException("Cannot upload corrupted recipe '%s'" %
str(ref))
export_src_folder = layout.export_sources()
src_files, src_symlinks = gather_files(export_src_folder)
result = {
CONANFILE: files.pop(CONANFILE),
CONAN_MANIFEST: files.pop(CONAN_MANIFEST)
}
def add_tgz(tgz_name, tgz_files, tgz_symlinks, msg):
tgz = os.path.join(download_export_folder, tgz_name)
if os.path.isfile(tgz):
result[tgz_name] = tgz
elif tgz_files:
if self._output and not self._output.is_terminal:
self._output.writeln(msg)
tgz = compress_files(tgz_files, tgz_symlinks, tgz_name,
download_export_folder, self._output)
result[tgz_name] = tgz
add_tgz(EXPORT_TGZ_NAME, files, symlinks, "Compressing recipe...")
add_tgz(EXPORT_SOURCES_TGZ_NAME, src_files, src_symlinks,
"Compressing recipe sources...")
return result
def _compress_package_files(self, layout, pref, integrity_check):
t1 = time.time()
if layout.package_is_dirty(pref):
raise ConanException("Package %s is corrupted, aborting upload.\n"
"Remove it with 'conan remove %s -p=%s'" %
(pref, pref.ref, pref.id))
download_pkg_folder = layout.download_package(pref)
package_tgz = os.path.join(download_pkg_folder, PACKAGE_TGZ_NAME)
if is_dirty(package_tgz):
self._output.warn("%s: Removing %s, marked as dirty" %
(str(pref), PACKAGE_TGZ_NAME))
os.remove(package_tgz)
clean_dirty(package_tgz)
# Get all the files in that directory
# existing package, will use short paths if defined
package_folder = layout.package(pref)
files, symlinks = gather_files(package_folder)
if CONANINFO not in files or CONAN_MANIFEST not in files:
logger.error("Missing info or manifest in uploading files: %s" %
(str(files)))
raise ConanException("Cannot upload corrupted package '%s'" %
str(pref))
logger.debug("UPLOAD: Time remote_manager build_files_set : %f" %
(time.time() - t1))
if integrity_check:
self._package_integrity_check(pref, files, package_folder)
logger.debug(
"UPLOAD: Time remote_manager check package integrity : %f" %
(time.time() - t1))
if not os.path.isfile(package_tgz):
if self._output and not self._output.is_terminal:
self._output.writeln("Compressing package...")
tgz_files = {
f: path
for f, path in files.items()
if f not in [CONANINFO, CONAN_MANIFEST]
}
tgz_path = compress_files(tgz_files, symlinks, PACKAGE_TGZ_NAME,
download_pkg_folder, self._output)
assert tgz_path == package_tgz
assert os.path.exists(package_tgz)
return {
PACKAGE_TGZ_NAME: package_tgz,
CONANINFO: files[CONANINFO],
CONAN_MANIFEST: files[CONAN_MANIFEST]
}
def upload_package(self, package_reference, remote, retry, retry_wait, skip_upload=False):
"""Will upload the package to the first remote"""
t1 = time.time()
# existing package, will use short paths if defined
package_folder = self._client_cache.package(package_reference, short_paths=None)
# Get all the files in that directory
files, symlinks = gather_files(package_folder)
self._output.rewrite_line("Checking package integrity...")
if CONANINFO not in files or CONAN_MANIFEST not in files:
logger.error("Missing info or manifest in uploading files: %s" % (str(files)))
raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference))
logger.debug("====> Time remote_manager build_files_set : %f" % (time.time() - t1))
# If package has been modified remove tgz to regenerate it
read_manifest, expected_manifest = self._client_cache.package_manifests(package_reference)
if read_manifest != expected_manifest:
self._output.writeln("")
diff = read_manifest.difference(expected_manifest)
for fname, (h1, h2) in diff.items():
self._output.warn("Mismatched checksum '%s' (manifest: %s, file: %s)"
% (fname, h1, h2))
if PACKAGE_TGZ_NAME in files:
try:
tgz_path = os.path.join(package_folder, PACKAGE_TGZ_NAME)
os.unlink(tgz_path)
except Exception:
pass
error_msg = os.linesep.join("Mismatched checksum '%s' (manifest: %s, file: %s)"
% (fname, h1, h2) for fname, (h1, h2) in diff.items())
logger.error("Manifests doesn't match!\n%s" % error_msg)
raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference))
else:
self._output.rewrite_line("Package integrity OK!")
self._output.writeln("")
logger.debug("====> Time remote_manager check package integrity : %f" % (time.time() - t1))
the_files = compress_package_files(files, symlinks, package_folder, self._output)
if not skip_upload:
tmp = self._call_remote(remote, "upload_package", package_reference, the_files,
retry, retry_wait)
duration = time.time() - t1
log_package_upload(package_reference, duration, the_files, remote)
logger.debug("====> Time remote_manager upload_package: %f" % duration)
return tmp
else:
return None
def upload_package(self, package_reference, remote, retry, retry_wait, skip_upload=False):
"""Will upload the package to the first remote"""
t1 = time.time()
# existing package, will use short paths if defined
package_folder = self._client_cache.package(package_reference, short_paths=None)
# Get all the files in that directory
files = gather_files(package_folder)
self._output.rewrite_line("Checking package integrity...")
if CONANINFO not in files or CONAN_MANIFEST not in files:
logger.error("Missing info or manifest in uploading files: %s" % (str(files)))
raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference))
logger.debug("====> Time remote_manager build_files_set : %f" % (time.time() - t1))
# If package has been modified remove tgz to regenerate it
read_manifest, expected_manifest = self._client_cache.package_manifests(package_reference)
if read_manifest != expected_manifest:
self._output.writeln("")
diff = read_manifest.difference(expected_manifest)
for fname, (h1, h2) in diff.items():
self._output.warn("Mismatched checksum '%s' (manifest: %s, file: %s)"
% (fname, h1, h2))
if PACKAGE_TGZ_NAME in files:
try:
tgz_path = os.path.join(package_folder, PACKAGE_TGZ_NAME)
os.unlink(tgz_path)
except Exception:
pass
error_msg = os.linesep.join("Mismatched checksum '%s' (manifest: %s, file: %s)"
% (fname, h1, h2) for fname, (h1, h2) in diff.items())
logger.error("Manifests doesn't match!\n%s" % error_msg)
raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference))
else:
self._output.rewrite_line("Package integrity OK!")
self._output.writeln("")
logger.debug("====> Time remote_manager check package integrity : %f" % (time.time() - t1))
the_files = compress_package_files(files, package_folder, self._output)
if not skip_upload:
tmp = self._call_remote(remote, "upload_package", package_reference, the_files,
retry, retry_wait)
duration = time.time() - t1
log_package_upload(package_reference, duration, the_files, remote)
logger.debug("====> Time remote_manager upload_package: %f" % duration)
return tmp
else:
return None
def _compress_folder(self, folder, tgz_path, folder_entry=None):
# Create a tar.gz file with the files in the folder and an additional TarInfo entry
# for the folder_entry (the gather files doesn't return empty dirs)
with open(tgz_path, "wb") as tgz_handle:
tgz = gzopen_without_timestamps("name", mode="w", fileobj=tgz_handle)
if folder_entry:
# Create an empty folder in the tgz file
t = tarfile.TarInfo(folder_entry)
t.mode = 488
t.type = tarfile.DIRTYPE
tgz.addfile(t)
files, _ = gather_files(folder)
for filename, abs_path in files.items():
info = tarfile.TarInfo(name=filename)
with open(os.path.join(folder, filename), 'rb') as file_handler:
tgz.addfile(tarinfo=info, fileobj=file_handler)
tgz.close()
def setUp(self):
self.tmp_folder = temp_folder()
with chdir(self.tmp_folder):
# Create a couple of files
ori_files_dir = os.path.join(self.tmp_folder, "ori")
file1 = os.path.join(ori_files_dir, "file1")
file2 = os.path.join(ori_files_dir, "folder", "file2")
save(file1, "")
save(file2, "")
# Create a tar.gz file with the above files
self.tgz_file = os.path.join(self.tmp_folder, "file.tar.gz")
with open(self.tgz_file, "wb") as tgz_handle:
tgz = gzopen_without_timestamps("name", mode="w", fileobj=tgz_handle)
files, _ = gather_files(ori_files_dir)
for filename, abs_path in files.items():
info = tarfile.TarInfo(name=filename)
with open(file1, 'rb') as file_handler:
tgz.addfile(tarinfo=info, fileobj=file_handler)
tgz.close()
def upload_package(self, package_reference, remote, retry, retry_wait, skip_upload=False,
integrity_check=False, no_overwrite=None):
"""Will upload the package to the first remote"""
t1 = time.time()
# existing package, will use short paths if defined
package_folder = self._client_cache.package(package_reference, short_paths=None)
if is_dirty(package_folder):
raise ConanException("Package %s is corrupted, aborting upload.\n"
"Remove it with 'conan remove %s -p=%s'" % (package_reference,
package_reference.conan,
package_reference.package_id))
# Get all the files in that directory
files, symlinks = gather_files(package_folder)
if CONANINFO not in files or CONAN_MANIFEST not in files:
logger.error("Missing info or manifest in uploading files: %s" % (str(files)))
raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference))
logger.debug("====> Time remote_manager build_files_set : %f" % (time.time() - t1))
if integrity_check:
self._package_integrity_check(package_reference, files, package_folder)
logger.debug("====> Time remote_manager check package integrity : %f"
% (time.time() - t1))
the_files = compress_package_files(files, symlinks, package_folder, self._output)
if skip_upload:
return None
tmp = self._call_remote(remote, "upload_package", package_reference, the_files,
retry, retry_wait, no_overwrite)
duration = time.time() - t1
log_package_upload(package_reference, duration, the_files, remote)
logger.debug("====> Time remote_manager upload_package: %f" % duration)
if not tmp:
self._output.rewrite_line("Package is up to date, upload skipped")
self._output.writeln("")
return tmp
