def upload_package(self, package_reference, the_files):
"""
basedir: Base directory with the files to upload (for read the files in disk)
relative_files: relative paths to upload
"""
self.check_credentials()
# Get the remote snapshot
remote_snapshot = self._get_package_snapshot(package_reference)
local_snapshot = {filename: md5(content) for filename, content in the_files.items()}
# Get the diff
new, modified, deleted = diff_snapshots(local_snapshot, remote_snapshot)
files_to_upload = {filename: the_files[filename] for filename in new + modified}
if files_to_upload: # Obtain upload urls
url = "%s/conans/%s/packages/%s/upload_urls" % (self._remote_api_url,
"/".join(package_reference.conan),
package_reference.package_id)
filesizes = {filename: len(content) for filename, content in files_to_upload.items()}
self._output.rewrite_line("Requesting upload permissions...")
urls = self._get_json(url, data=filesizes)
self._output.rewrite_line("Requesting upload permissions...Done!")
self._output.writeln("")
self.upload_files(urls, files_to_upload, self._output)
else:
self._output.rewrite_line("Package is up to date.")
self._output.writeln("")
if deleted:
self.remove_package_files(package_reference, deleted)
def _upload_recipe(self, ref, base_files=None, retry=1, retry_wait=0):
files = {"conanfile.py": GenConanfile("3").with_requires("1", "12")}
if base_files:
files.update(base_files)
content = """
from conans import ConanFile
class MyConan(ConanFile):
name = "%s"
version = "%s"
settings = arch, compiler, os
""" % (ref.name, ref.version)
files[CONANFILE] = content
files_md5s = {filename: md5(content) for filename, content in files.items()}
conan_digest = FileTreeManifest(123123123, files_md5s)
tmp_dir = temp_folder()
abs_paths = {}
for filename, content in files.items():
abs_path = os.path.join(tmp_dir, filename)
save(abs_path, content)
abs_paths[filename] = abs_path
abs_paths[CONAN_MANIFEST] = os.path.join(tmp_dir, CONAN_MANIFEST)
conan_digest.save(tmp_dir)
self.api.upload_recipe(ref, abs_paths, None, retry, retry_wait)
def _upload_recipe(self, conan_reference, base_files=None, retry=1, retry_wait=0):
files = hello_source_files(3, [1, 12])
if base_files:
files.update(base_files)
content = """
from conans import ConanFile
class MyConan(ConanFile):
name = "%s"
version = "%s"
settings = arch, compiler, os
""" % (conan_reference.name, conan_reference.version)
files[CONANFILE] = content
files_md5s = {filename: md5(content) for filename, content in files.items()}
conan_digest = FileTreeManifest(123123123, files_md5s)
files[CONAN_MANIFEST] = str(conan_digest)
tmp_dir = temp_folder()
abs_paths = {}
for filename, content in files.items():
abs_path = os.path.join(tmp_dir, filename)
save(abs_path, content)
abs_paths[filename] = abs_path
self.api.upload_recipe(conan_reference, abs_paths, retry, retry_wait, False)
def _upload_recipe(self,
conan_reference,
base_files=None,
retry=1,
retry_wait=0):
files = hello_source_files(3, [1, 12])
if base_files:
files.update(base_files)
content = """
from conans import ConanFile
class MyConan(ConanFile):
name = "%s"
version = "%s"
settings = arch, compiler, os
""" % (conan_reference.name, conan_reference.version)
files[CONANFILE] = content
files_md5s = {
filename: md5(content)
for filename, content in files.items()
}
conan_digest = FileTreeManifest(123123123, files_md5s)
tmp_dir = temp_folder()
abs_paths = {}
for filename, content in files.items():
abs_path = os.path.join(tmp_dir, filename)
save(abs_path, content)
abs_paths[filename] = abs_path
abs_paths[CONAN_MANIFEST] = os.path.join(tmp_dir, CONAN_MANIFEST)
conan_digest.save(tmp_dir)
self.api.upload_recipe(conan_reference, abs_paths, retry, retry_wait,
False, None)
def test_tree_manifest(self):
tmp_dir = temp_folder()
files = {"one.ext": "aalakjshdlkjahsdlkjahsdljkhsadljkhasljkdhlkjashd",
"path/to/two.txt": "asdas13123",
"two.txt": "asdasdasdasdasdasd",
"folder/damn.pyc": "binarythings",
"folder/damn.pyo": "binarythings2",
"pythonfile.pyc": "binarythings3"}
for filename, content in files.items():
save(os.path.join(tmp_dir, filename), content)
manifest = FileTreeManifest.create(tmp_dir)
save(os.path.join(tmp_dir, "THEMANIFEST.txt"), str(manifest))
readed_manifest = FileTreeManifest.loads(load(os.path.join(tmp_dir, "THEMANIFEST.txt")))
self.assertEqual(readed_manifest.time, manifest.time)
self.assertEqual(readed_manifest, manifest)
# Not included the pycs or pyo
self.assertEquals(set(manifest.file_sums.keys()),
set(["one.ext", "path/to/two.txt", "two.txt"]))
for filepath, md5readed in manifest.file_sums.items():
content = files[filepath]
self.assertEquals(md5(content), md5readed)
def test_tree_manifest(self):
tmp_dir = temp_folder()
files = {
"one.ext": "aalakjshdlkjahsdlkjahsdljkhsadljkhasljkdhlkjashd",
"path/to/two.txt": "asdas13123",
"two.txt": "asdasdasdasdasdasd",
"folder/damn.pyc": "binarythings",
"folder/damn.pyo": "binarythings2",
"pythonfile.pyc": "binarythings3"
}
for filename, content in files.items():
save(os.path.join(tmp_dir, filename), content)
manifest = FileTreeManifest.create(tmp_dir)
manifest.save(tmp_dir)
readed_manifest = FileTreeManifest.load(tmp_dir)
self.assertEqual(readed_manifest.time, manifest.time)
self.assertEqual(readed_manifest, manifest)
# Not included the pycs or pyo
self.assertEqual(set(manifest.file_sums.keys()),
set(["one.ext", "path/to/two.txt", "two.txt"]))
for filepath, md5readed in manifest.file_sums.items():
content = files[filepath]
self.assertEqual(md5(content), md5readed)
def upload_conan(self, conan_reference, the_files):
"""
the_files: dict with relative_path: content
"""
self.check_credentials()
# Get the remote snapshot
remote_snapshot = self._get_conan_snapshot(conan_reference)
local_snapshot = {
filename: md5(content)
for filename, content in the_files.iteritems()
}
# Get the diff
new, modified, deleted = diff_snapshots(local_snapshot,
remote_snapshot)
files_to_upload = {
filename.replace("\\", "/"): the_files[filename]
for filename in new + modified
}
if files_to_upload:
# Get the upload urls
url = "%s/conans/%s/upload_urls" % (self._remote_api_url,
"/".join(conan_reference))
filesizes = {
filename.replace("\\", "/"): len(content)
for filename, content in files_to_upload.iteritems()
}
urls = self._get_json(url, data=filesizes)
self.upload_files(urls, files_to_upload, self._output)
if deleted:
self.remove_conanfile_files(conan_reference, deleted)
def _failed_package_verify(self, reference, remote="local cache"):
self.client.run("install %s --build missing --manifests" %
str(reference))
self.assertIn(
"Installed manifest for 'Hello/0.1@lasote/stable' from %s" %
remote, self.client.user_io.out)
self.assertIn(
"Installed manifest for 'Hello/0.1@lasote/stable:"
"5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9' from %s" % remote,
self.client.user_io.out)
output_folder = os.path.join(self.client.current_folder,
".conan_manifests")
paths = SimplePaths(output_folder)
self.assertTrue(
os.path.exists(
os.path.join(paths.export(self.reference), CONAN_MANIFEST)))
package_reference = PackageReference.loads(
"Hello/0.1@lasote/stable:"
"5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9")
self.assertTrue(
os.path.exists(
os.path.join(paths.package(package_reference),
CONAN_MANIFEST)))
client = TestClient(servers=self.servers,
users={"default": [("lasote", "mypass")]})
client.save(self.files)
client.run("export . lasote/stable")
client.run("install Hello/0.1@lasote/stable --build=missing")
info = os.path.join(client.paths.package(package_reference),
"conaninfo.txt")
info_content = load(info)
info_content += "# Dummy string"
save(info, info_content)
package_folder = client.paths.package(package_reference)
manifest = FileTreeManifest.load(package_folder)
manifest.file_sums["conaninfo.txt"] = md5(info_content)
manifest.save(package_folder)
client.run("upload %s --all" % str(self.reference))
# now verify, with update
self.client.run("remove Hello/0.1@lasote/stable -f")
self.client.run("install %s --build missing --verify" %
str(self.reference),
ignore_error=True)
self.assertNotIn("Manifest for 'Hello/0.1@lasote/stable': OK",
self.client.user_io.out)
self.assertNotIn("Manifest for '%s': OK" % str(package_reference),
self.client.user_io.out)
self.assertIn(
"Modified or new manifest '%s' detected" % str(package_reference),
self.client.user_io.out)
def _failed_package_verify(self, reference, remote="local cache"):
self.client.run("install %s --build missing --manifests" %
str(reference))
self.assertIn(
"Installed manifest for 'Hello/0.1@lasote/stable' from %s" %
remote, self.client.user_io.out)
self.assertIn(
"Installed manifest for 'Hello/0.1@lasote/stable:"
"%s' from %s" % (NO_SETTINGS_PACKAGE_ID, remote),
self.client.user_io.out)
output_folder = os.path.join(self.client.current_folder,
".conan_manifests")
self.assertTrue(
os.path.exists(
os.path.join(export_folder(output_folder, self.ref),
CONAN_MANIFEST)))
pref = PackageReference.loads("Hello/0.1@lasote/stable: %s" %
NO_SETTINGS_PACKAGE_ID)
self.assertTrue(
os.path.exists(
os.path.join(package_folder(output_folder, pref),
CONAN_MANIFEST)))
client = TestClient(servers=self.servers,
users={"default": [("lasote", "mypass")]})
client.save(self.files)
client.run("export . lasote/stable")
client.run("install Hello/0.1@lasote/stable --build=missing")
package_folder_path = client.cache.package_layout(
pref.ref).package(pref)
info = os.path.join(package_folder_path, "conaninfo.txt")
info_content = load(info)
info_content += "# Dummy string"
save(info, info_content)
manifest = FileTreeManifest.load(package_folder_path)
manifest.file_sums["conaninfo.txt"] = md5(info_content)
manifest.save(package_folder_path)
client.run("upload %s --all" % str(self.ref))
# now verify, with update
self.client.run("remove Hello/0.1@lasote/stable -f")
self.client.run("install %s --build missing --verify" % str(self.ref),
assert_error=True)
self.assertNotIn("Manifest for 'Hello/0.1@lasote/stable': OK",
self.client.user_io.out)
self.assertNotIn("Manifest for '%s': OK" % str(pref),
self.client.user_io.out)
self.assertIn("Modified or new manifest '%s' detected" % str(pref),
self.client.user_io.out)
def test_tree_manifest(self):
tmp_dir = temp_folder()
files = {"one.ext": "aalakjshdlkjahsdlkjahsdljkhsadljkhasljkdhlkjashd",
"path/to/two.txt": "asdas13123",
"two.txt": "asdasdasdasdasdasd"}
for filename, content in files.iteritems():
save(os.path.join(tmp_dir, filename), content)
manifest = FileTreeManifest.create(tmp_dir)
save(os.path.join(tmp_dir, "THEMANIFEST.txt"), str(manifest))
readed_manifest = FileTreeManifest.loads(load(os.path.join(tmp_dir, "THEMANIFEST.txt")))
self.assertEquals(readed_manifest, manifest)
for filepath, md5readed in manifest.file_sums.iteritems():
content = files[filepath]
self.assertEquals(md5(content), md5readed)
def _upload_conan(self, conan_reference, base_files=None):
files = hello_source_files(3, [1, 12])
if base_files:
files.update(base_files)
content = """
from conans import ConanFile
class MyConan(ConanFile):
name = "%s"
version = "%s"
settings = arch, compiler, os
""" % (conan_reference.name, conan_reference.version)
files[CONANFILE] = content
files_md5s = {filename: md5(content) for filename, content in files.iteritems()}
conan_digest = FileTreeManifest(123123123, files_md5s)
files[CONAN_MANIFEST] = str(conan_digest)
self.api.upload_conan(conan_reference, files)
def _failed_package_verify(self, reference, remote="local cache"):
self.client.run("install %s --build missing --manifests" % str(reference))
self.assertIn("Installed manifest for 'Hello/0.1@lasote/stable' from %s" % remote,
self.client.user_io.out)
self.assertIn("Installed manifest for 'Hello/0.1@lasote/stable:"
"5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9' from %s" % remote,
self.client.user_io.out)
output_folder = os.path.join(self.client.current_folder, ".conan_manifests")
paths = SimplePaths(output_folder)
self.assertTrue(os.path.exists(paths.digestfile_conanfile(self.reference)))
package_reference = PackageReference.loads("Hello/0.1@lasote/stable:"
"5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9")
self.assertTrue(os.path.exists(paths.digestfile_package(package_reference)))
client = TestClient(servers=self.servers, users={"default": [("lasote", "mypass")]})
client.save(self.files)
client.run("export lasote/stable")
client.run("install Hello/0.1@lasote/stable --build=missing")
info = os.path.join(client.paths.package(package_reference), "conaninfo.txt")
info_content = load(info)
info_content += "# Dummy string"
save(info, info_content)
manifest = client.paths.load_package_manifest(package_reference)
manifest.file_sums["conaninfo.txt"] = md5(info_content)
save(client.paths.digestfile_package(package_reference), str(manifest))
manifest = client.paths.load_package_manifest(package_reference)
client.run("upload %s --all" % str(self.reference))
# now verify, with update
self.client.run("remove Hello/0.1@lasote/stable -f")
self.client.run("install %s --build missing --verify"
% str(self.reference),
ignore_error=True)
self.assertNotIn("Manifest for 'Hello/0.1@lasote/stable': OK", self.client.user_io.out)
self.assertNotIn("Manifest for '%s': OK" % str(package_reference), self.client.user_io.out)
self.assertIn("Modified or new manifest '%s' detected" % str(package_reference),
self.client.user_io.out)
def _upload_conan(self, conan_reference, base_files=None):
files = hello_source_files(3, [1, 12])
if base_files:
files.update(base_files)
content = """
from conans import ConanFile
class MyConan(ConanFile):
name = "%s"
version = "%s"
settings = arch, compiler, os
""" % (conan_reference.name, conan_reference.version)
files[CONANFILE] = content
files_md5s = {
filename: md5(content)
for filename, content in files.items()
}
conan_digest = FileTreeManifest(123123123, files_md5s)
files[CONAN_MANIFEST] = str(conan_digest)
self.api.upload_conan(conan_reference, files)
def upload_conan(self, conan_reference, the_files):
"""
the_files: dict with relative_path: content
"""
self.check_credentials()
# Get the remote snapshot
remote_snapshot = self._get_conan_snapshot(conan_reference)
local_snapshot = {filename: md5(content) for filename, content in the_files.items()}
# Get the diff
new, modified, deleted = diff_snapshots(local_snapshot, remote_snapshot)
files_to_upload = {filename.replace("\\", "/"): the_files[filename] for filename in new + modified}
if files_to_upload:
# Get the upload urls
url = "%s/conans/%s/upload_urls" % (self._remote_api_url, "/".join(conan_reference))
filesizes = {filename.replace("\\", "/"): len(content) for filename, content in files_to_upload.items()}
urls = self._get_json(url, data=filesizes)
self.upload_files(urls, files_to_upload, self._output)
if deleted:
self.remove_conanfile_files(conan_reference, deleted)
def summary_hash(self):
s = ["%s: %s" % (f, fmd5) for f, fmd5 in sorted(self.file_sums.items())]
s.append("")
return md5("\n".join(s))
def summary_hash(self):
ret = "" # Do not include the timestamp in the summary hash
for filepath, file_md5 in self.file_sums.items():
ret += "\n%s: %s" % (filepath, file_md5)
return md5(ret)
def summary_hash(self):
ret = "" # Do not include the timestamp in the summary hash
for filepath, file_md5 in sorted(self.file_sums.items()):
ret += "%s: %s\n" % (filepath, file_md5)
return md5(ret)
def summary_hash(self):
s = ["%s: %s" % (f, fmd5) for f, fmd5 in sorted(self.file_sums.items())]
s.append("")
return md5("\n".join(s))
def md5_test(self):
result = md5(u"äüïöñç")
self.assertEqual("dfcc3d74aa447280a7ecfdb98da55174", result)
def md5_test(self):
result = md5(u"äüïöñç")
self.assertEqual("dfcc3d74aa447280a7ecfdb98da55174", result)
