def test_tree_manifest(self):
tmp_dir = temp_folder()
files = {"one.ext": "aalakjshdlkjahsdlkjahsdljkhsadljkhasljkdhlkjashd",
"path/to/two.txt": "asdas13123",
"two.txt": "asdasdasdasdasdasd",
"folder/damn.pyc": "binarythings",
"folder/damn.pyo": "binarythings2",
"pythonfile.pyc": "binarythings3"}
for filename, content in files.items():
save(os.path.join(tmp_dir, filename), content)
manifest = FileTreeManifest.create(tmp_dir)
save(os.path.join(tmp_dir, "THEMANIFEST.txt"), str(manifest))
readed_manifest = FileTreeManifest.loads(load(os.path.join(tmp_dir, "THEMANIFEST.txt")))
self.assertEqual(readed_manifest.time, manifest.time)
self.assertEqual(readed_manifest, manifest)
# Not included the pycs or pyo
self.assertEquals(set(manifest.file_sums.keys()),
set(["one.ext", "path/to/two.txt", "two.txt"]))
for filepath, md5readed in manifest.file_sums.items():
content = files[filepath]
self.assertEquals(md5(content), md5readed)
def _upload_recipe(self, conan_reference, base_files=None, retry=1, retry_wait=0):
files = hello_source_files(3, [1, 12])
if base_files:
files.update(base_files)
content = """
from conans import ConanFile
class MyConan(ConanFile):
name = "%s"
version = "%s"
settings = arch, compiler, os
""" % (conan_reference.name, conan_reference.version)
files[CONANFILE] = content
files_md5s = {filename: md5(content) for filename, content in files.items()}
conan_digest = FileTreeManifest(123123123, files_md5s)
tmp_dir = temp_folder()
abs_paths = {}
for filename, content in files.items():
abs_path = os.path.join(tmp_dir, filename)
save(abs_path, content)
abs_paths[filename] = abs_path
abs_paths[CONAN_MANIFEST] = os.path.join(tmp_dir, CONAN_MANIFEST)
conan_digest.save(tmp_dir)
self.api.upload_recipe(conan_reference, abs_paths, retry, retry_wait, False, None)
def valid_digest(self, digest_path):
if not os.path.exists(digest_path):
return False
expected_digest = FileTreeManifest.create(os.path.dirname(digest_path))
del expected_digest.file_sums[CONAN_MANIFEST] # Exclude the MANIFEST itself
expected_digest.file_sums.pop(CONANFILE + "c", None) # Exclude the CONANFILE.pyc
expected_digest.file_sums.pop(".DS_Store", None) # Exclude tmp in mac
readed_digest = FileTreeManifest.loads(load(digest_path))
return readed_digest.file_sums == expected_digest.file_sums
def _report_save_manifest(copied_files, output, dest_folder, manifest_name):
report_copied_files(copied_files, output)
if copied_files:
date = calendar.timegm(time.gmtime())
file_dict = {}
for f in copied_files:
abs_path = os.path.join(dest_folder, f)
file_dict[f] = md5sum(abs_path)
manifest = FileTreeManifest(date, file_dict)
manifest.save(dest_folder, manifest_name)
def test_export_a_new_version(self):
self._create_packages_and_builds()
# Export an update of the same conans
conan2 = TestClient(self.conan.base_folder)
files2 = cpp_hello_conan_files("Hello0", "0.1")
files2[CONANFILE] = "# insert comment\n %s" % files2[CONANFILE]
conan2.save(files2)
conan2.run("export lasote/stable")
reg_path3 = conan2.paths.export(self.conan_ref)
digest3 = FileTreeManifest.loads(load(conan2.paths.digestfile_conanfile(self.conan_ref)))
self.assertIn('A new conanfile.py version was exported', conan2.user_io.out)
# self.assertIn('All the previous packs were cleaned', conan2.user_io.out)
self.assertIn('conanfile.py exported as %s in %s'
% (self.conan_ref, reg_path3), conan2.user_io.out)
self.assertTrue(os.path.exists(reg_path3))
for name in files2.keys():
self.assertTrue(os.path.exists(os.path.join(reg_path3, name)))
expected_sums = {'hello.cpp': '4f005274b2fdb25e6113b69774dac184',
'main.cpp': '0479f3c223c9a656a718f3148e044124',
'CMakeLists.txt': '310d65b60943b879286cf8839cf9ba08',
'conanfile.py': '5a2c6e2d6b39638b7d8560786d7935a3',
'helloHello0.h': '9448df034392fc8781a47dd03ae71bdd'}
self.assertEqual(expected_sums, digest3.file_sums)
def test_export_the_same_code(self):
file_list = self._create_packages_and_builds()
# Export the same conans
conan2 = TestClient(self.conan.base_folder)
files2 = cpp_hello_conan_files("Hello0", "0.1")
conan2.save(files2)
conan2.run("export lasote/stable")
reg_path2 = conan2.paths.export(self.conan_ref)
digest2 = FileTreeManifest.loads(load(conan2.paths.digestfile_conanfile(self.conan_ref)))
self.assertNotIn('A new Conan version was exported', conan2.user_io.out)
self.assertNotIn('Cleaning the old builds ...', conan2.user_io.out)
self.assertNotIn('Cleaning the old packs ...', conan2.user_io.out)
self.assertNotIn('All the previous packs were cleaned', conan2.user_io.out)
self.assertIn('conanfile.py exported as %s in %s'
% (self.conan_ref, reg_path2), conan2.user_io.out)
self.assertTrue(os.path.exists(reg_path2))
for name in files2.keys():
self.assertTrue(os.path.exists(os.path.join(reg_path2, name)))
expected_sums = {'hello.cpp': '4f005274b2fdb25e6113b69774dac184',
'main.cpp': '0479f3c223c9a656a718f3148e044124',
'CMakeLists.txt': '310d65b60943b879286cf8839cf9ba08',
'conanfile.py': '4df4b5266ad6d5d55cbdd0a8b12c9d1a',
'helloHello0.h': '9448df034392fc8781a47dd03ae71bdd'}
self.assertEqual(expected_sums, digest2.file_sums)
for f in file_list:
self.assertTrue(os.path.exists(f))
def setUp(self):
self.conan_reference = ConanFileReference.loads("openssl/2.0.3@lasote/testing")
self.package_reference = PackageReference(self.conan_reference, "123123123")
self.tmp_dir = temp_folder()
read_perms = [("*/*@*/*", "*")]
write_perms = []
authorizer = BasicAuthorizer(read_perms, write_perms)
self.fake_url = "http://url"
updown_auth_manager = JWTUpDownAuthManager("secret",
timedelta(seconds=200))
adapter = ServerDiskAdapter(self.fake_url, self.tmp_dir, updown_auth_manager)
self.paths = SimplePaths(self.tmp_dir)
self.file_manager = FileManager(self.paths, adapter)
search_adapter = DiskSearchAdapter()
self.search_manager = DiskSearchManager(self.paths, search_adapter)
self.service = ConanService(authorizer, self.file_manager, "lasote")
self.search_service = SearchService(authorizer, self.search_manager, "lasote")
files = hello_source_files("test")
save_files(self.paths.export(self.conan_reference), files)
self.conan_digest = FileTreeManifest.create(self.paths.export(self.conan_reference))
conan_digest_path = os.path.join(self.paths.export(self.conan_reference), CONAN_MANIFEST)
save(conan_digest_path, str(self.conan_digest))
files = hello_source_files("package")
save_files(self.paths.package(self.package_reference), files)
def test_export_the_same_code(self):
file_list = self._create_packages_and_builds()
# Export the same conans
conan2 = TestClient(self.conan.base_folder)
files2 = cpp_hello_conan_files("Hello0", "0.1")
conan2.save(files2)
conan2.run("export lasote/stable")
reg_path2 = conan2.paths.export(self.conan_ref)
digest2 = FileTreeManifest.loads(load(conan2.paths.digestfile_conanfile(self.conan_ref)))
self.assertNotIn('A new Conan version was exported', conan2.user_io.out)
self.assertNotIn('Cleaning the old builds ...', conan2.user_io.out)
self.assertNotIn('Cleaning the old packs ...', conan2.user_io.out)
self.assertNotIn('All the previous packs were cleaned', conan2.user_io.out)
self.assertIn('%s: conanfile.py exported to local storage' % str(self.conan_ref),
self.conan.user_io.out)
self.assertIn('%s: Folder: %s' % (str(self.conan_ref), reg_path2), self.conan.user_io.out)
self.assertTrue(os.path.exists(reg_path2))
for name in files2.keys():
self.assertTrue(os.path.exists(os.path.join(reg_path2, name)))
expected_sums = {'hello.cpp': '4f005274b2fdb25e6113b69774dac184',
'main.cpp': '0479f3c223c9a656a718f3148e044124',
'CMakeLists.txt': 'bc3405da4bb0b51a3b9f05aca71e58c8',
'conanfile.py': 'f21a98d974e9294b0d709070042c6e78',
'helloHello0.h': '9448df034392fc8781a47dd03ae71bdd'}
self.assertEqual(expected_sums, digest2.file_sums)
for f in file_list:
self.assertTrue(os.path.exists(f))
def test_basic(self):
current_folder = temp_folder()
source_folder = os.path.join(current_folder, "source")
client = TestClient(current_folder=current_folder)
files = cpp_hello_conan_files("Hello0", "0.1")
conan_ref = ConanFileReference("Hello0", "0.1", "lasote", "stable")
client.save(files, path=source_folder)
client.run("export lasote/stable --path=source")
reg_path = client.paths.export(conan_ref)
manif = FileTreeManifest.loads(load(client.paths.digestfile_conanfile(conan_ref)))
self.assertIn('%s: A new conanfile.py version was exported' % str(conan_ref),
client.user_io.out)
self.assertIn('%s: Folder: %s' % (str(conan_ref), reg_path), client.user_io.out)
self.assertTrue(os.path.exists(reg_path))
for name in list(files.keys()):
self.assertTrue(os.path.exists(os.path.join(reg_path, name)))
expected_sums = {'hello.cpp': '4f005274b2fdb25e6113b69774dac184',
'main.cpp': '0479f3c223c9a656a718f3148e044124',
'CMakeLists.txt': 'bc3405da4bb0b51a3b9f05aca71e58c8',
'conanfile.py': '5632cf850a7161388ab24f42b9bdb3fd',
'executable': '68b329da9893e34099c7d8ad5cb9c940',
'helloHello0.h': '9448df034392fc8781a47dd03ae71bdd'}
self.assertEqual(expected_sums, manif.file_sums)
def test_path(self):
base_folder = temp_folder()
source_folder = os.path.join(base_folder, "source")
current_folder = os.path.join(base_folder, "current")
client = TestClient(current_folder=current_folder)
files = cpp_hello_conan_files("Hello0", "0.1")
conan_ref = ConanFileReference("Hello0", "0.1", "lasote", "stable")
conanfile = files.pop("conanfile.py")
client.save(files, path=source_folder)
conanfile = conanfile.replace("exports = '*'", 'exports = "../source*"')
client.save({"conanfile.py": conanfile})
client.run("export lasote/stable")
reg_path = client.paths.export(conan_ref)
manif = FileTreeManifest.loads(load(client.paths.digestfile_conanfile(conan_ref)))
self.assertIn('%s: A new conanfile.py version was exported' % str(conan_ref),
client.user_io.out)
self.assertIn('%s: Folder: %s' % (str(conan_ref), reg_path), client.user_io.out)
self.assertTrue(os.path.exists(reg_path))
for name in ['conanfile.py', 'conanmanifest.txt', 'source/main.cpp',
'source/executable', 'source/hello.cpp', 'source/CMakeLists.txt',
'source/helloHello0.h']:
self.assertTrue(os.path.exists(os.path.join(reg_path, name)))
expected_sums = {'source/hello.cpp': '4f005274b2fdb25e6113b69774dac184',
'source/main.cpp': '0479f3c223c9a656a718f3148e044124',
'source/CMakeLists.txt': '52546396c42f16be3daf72ecf7ab7143',
'conanfile.py': '3ac566eb5b2e4df4417003f0e606e237',
'source/executable': '68b329da9893e34099c7d8ad5cb9c940',
'source/helloHello0.h': '9448df034392fc8781a47dd03ae71bdd'}
self.assertEqual(expected_sums, manif.file_sums)
def test_export_a_new_version(self):
self._create_packages_and_builds()
# Export an update of the same conans
conan2 = TestClient(self.conan.base_folder)
files2 = cpp_hello_conan_files("Hello0", "0.1")
files2[CONANFILE] = "# insert comment\n %s" % files2[CONANFILE]
conan2.save(files2)
conan2.run("export lasote/stable")
reg_path3 = conan2.paths.export(self.conan_ref)
digest3 = FileTreeManifest.loads(load(conan2.paths.digestfile_conanfile(self.conan_ref)))
self.assertIn('A new conanfile.py version was exported', conan2.user_io.out)
self.assertIn('%s: conanfile.py exported to local storage' % str(self.conan_ref),
self.conan.user_io.out)
self.assertIn('%s: Folder: %s' % (str(self.conan_ref), reg_path3), self.conan.user_io.out)
self.assertTrue(os.path.exists(reg_path3))
for name in files2.keys():
self.assertTrue(os.path.exists(os.path.join(reg_path3, name)))
expected_sums = {'hello.cpp': '4f005274b2fdb25e6113b69774dac184',
'main.cpp': '0479f3c223c9a656a718f3148e044124',
'CMakeLists.txt': 'bc3405da4bb0b51a3b9f05aca71e58c8',
'conanfile.py': 'c5889ea6485599c7a0cae02b54270b35',
'helloHello0.h': '9448df034392fc8781a47dd03ae71bdd'}
self.assertEqual(expected_sums, digest3.file_sums)
def test_export_a_new_version(self):
self._create_packages_and_builds()
# Export an update of the same conans
conan2 = TestClient(self.conan.base_folder)
files2 = cpp_hello_conan_files("Hello0", "0.1")
files2[CONANFILE] = "# insert comment\n %s" % files2[CONANFILE]
conan2.save(files2)
conan2.run("export lasote/stable")
reg_path3 = conan2.paths.export(self.conan_ref)
digest3 = FileTreeManifest.loads(load(conan2.paths.digestfile_conanfile(self.conan_ref)))
self.assertIn('%s: A new conanfile.py version was exported' % str(self.conan_ref),
self.conan.user_io.out)
self.assertIn('%s: Folder: %s' % (str(self.conan_ref), reg_path3), self.conan.user_io.out)
self.assertTrue(os.path.exists(reg_path3))
for name in list(files2.keys()):
self.assertTrue(os.path.exists(os.path.join(reg_path3, name)))
expected_sums = {'hello.cpp': '4f005274b2fdb25e6113b69774dac184',
'main.cpp': '0479f3c223c9a656a718f3148e044124',
'CMakeLists.txt': '52546396c42f16be3daf72ecf7ab7143',
'conanfile.py': 'ad17cf00b3142728b03ac37782b9acd9',
'executable': '68b329da9893e34099c7d8ad5cb9c940',
'helloHello0.h': '9448df034392fc8781a47dd03ae71bdd'}
self.assertEqual(expected_sums, digest3.file_sums)
def undo_imports(current_path, output):
manifest_path = os.path.join(current_path, IMPORTS_MANIFESTS)
try:
manifest_content = load(manifest_path)
except:
raise ConanException("Cannot load file %s" % manifest_path)
try:
manifest = FileTreeManifest.loads(manifest_content)
except:
raise ConanException("Wrong manifest file format %s" % manifest_path)
not_removed = 0
files = manifest.files()
for filepath in files:
if not os.path.exists(filepath):
output.warn("File doesn't exist: %s" % filepath)
continue
try:
os.remove(filepath)
except:
output.error("Cannot remove file (open or busy): %s" % filepath)
not_removed += 1
if not_removed:
raise ConanException("Cannot remove %s or more imported files" % not_removed)
output.success("Removed %s imported files" % (len(files)))
try:
os.remove(manifest_path)
output.success("Removed imports manifest file: %s" % manifest_path)
except:
raise ConanException("Cannot remove manifest file (open or busy): %s" % manifest_path)
def test_export_the_same_code(self):
file_list = self._create_packages_and_builds()
# Export the same conans
conan2 = TestClient(self.conan.base_folder)
files2 = cpp_hello_conan_files("Hello0", "0.1")
conan2.save(files2)
conan2.run("export lasote/stable")
reg_path2 = conan2.paths.export(self.conan_ref)
digest2 = FileTreeManifest.loads(load(conan2.paths.digestfile_conanfile(self.conan_ref)))
self.assertNotIn('A new Conan version was exported', conan2.user_io.out)
self.assertNotIn('Cleaning the old builds ...', conan2.user_io.out)
self.assertNotIn('Cleaning the old packs ...', conan2.user_io.out)
self.assertNotIn('All the previous packs were cleaned', conan2.user_io.out)
self.assertIn('%s: A new conanfile.py version was exported' % str(self.conan_ref),
self.conan.user_io.out)
self.assertIn('%s: Folder: %s' % (str(self.conan_ref), reg_path2), self.conan.user_io.out)
self.assertTrue(os.path.exists(reg_path2))
for name in list(files2.keys()):
self.assertTrue(os.path.exists(os.path.join(reg_path2, name)))
expected_sums = {'hello.cpp': '4f005274b2fdb25e6113b69774dac184',
'main.cpp': '0479f3c223c9a656a718f3148e044124',
'CMakeLists.txt': '52546396c42f16be3daf72ecf7ab7143',
'conanfile.py': '355949fbf0b4fc32b8f1c5a338dfe1ae',
'executable': '68b329da9893e34099c7d8ad5cb9c940',
'helloHello0.h': '9448df034392fc8781a47dd03ae71bdd'}
self.assertEqual(expected_sums, digest2.file_sums)
for f in file_list:
self.assertTrue(os.path.exists(f))
def test_rel_path(self):
base_folder = temp_folder()
source_folder = os.path.join(base_folder, "source")
current_folder = os.path.join(base_folder, "current")
os.makedirs(current_folder)
client = TestClient(current_folder=current_folder)
files = cpp_hello_conan_files("Hello0", "0.1")
conan_ref = ConanFileReference("Hello0", "0.1", "lasote", "stable")
client.save(files, path=source_folder)
client.run("export lasote/stable --path=../source")
reg_path = client.paths.export(conan_ref)
manif = FileTreeManifest.loads(load(client.paths.digestfile_conanfile(conan_ref)))
self.assertIn('%s: A new conanfile.py version was exported' % str(conan_ref),
client.user_io.out)
self.assertIn('%s: Folder: %s' % (str(conan_ref), reg_path), client.user_io.out)
self.assertTrue(os.path.exists(reg_path))
for name in list(files.keys()):
self.assertTrue(os.path.exists(os.path.join(reg_path, name)))
expected_sums = {'hello.cpp': '4f005274b2fdb25e6113b69774dac184',
'main.cpp': '0479f3c223c9a656a718f3148e044124',
'CMakeLists.txt': '52546396c42f16be3daf72ecf7ab7143',
'conanfile.py': '355949fbf0b4fc32b8f1c5a338dfe1ae',
'executable': '68b329da9893e34099c7d8ad5cb9c940',
'helloHello0.h': '9448df034392fc8781a47dd03ae71bdd'}
self.assertEqual(expected_sums, manif.file_sums)
def _check_manifest(self):
manifest_content = load(os.path.join(self.client.current_folder, IMPORTS_MANIFESTS))
manifest = FileTreeManifest.loads(manifest_content)
self.assertEqual(manifest.file_sums,
{os.path.join(self.client.current_folder, "file1.txt"):
"8b1a9953c4611296a827abf8c47804d7",
os.path.join(self.client.current_folder, "file2.txt"):
"f5a7924e621e84c9280a9a27e1bcb7f6"})
def setUp(self):
hello_files = cpp_hello_conan_files("Hello")
test_conanfile_contents = hello_files[CONANFILE]
self.server_folder = temp_folder()
test_server = TestServer(users={"fenix": "mypass"},
base_path=self.server_folder) # exported users and passwords
self.server = test_server
servers = {"default": test_server}
client = TestClient(servers=servers, users={"default": [("fenix", "mypass")]})
# Conans with and without packages created
self.root_folder = {"H1": 'Hello/1.4.10/fenix/testing',
"H2": 'Hello/2.4.11/fenix/testing',
"B": 'Bye/0.14/fenix/testing',
"O": 'Other/1.2/fenix/testing'}
files = {}
pack_refs = []
for key, folder in self.root_folder.items():
ref = ConanFileReference.loads(folder)
files["%s/%s/conanfile.py" % (folder, EXPORT_FOLDER)] = test_conanfile_contents
files["%s/%s/conanmanifest.txt" % (folder, EXPORT_FOLDER)] = ""
files["%s/%s/conans.txt" % (folder, SRC_FOLDER)] = ""
for pack_id in (1, 2):
i = pack_id
pack_id = "%s_%s" % (pack_id, key)
pack_refs.append(PackageReference(ref, str(pack_id)))
files["%s/%s/%s/conans.txt" % (folder, BUILD_FOLDER, pack_id)] = ""
files["%s/%s/%s/conans.txt" % (folder, PACKAGES_FOLDER, pack_id)] = ""
files["%s/%s/%s/%s" % (folder, PACKAGES_FOLDER, pack_id, CONANINFO)] = conaninfo % str(i)
files["%s/%s/%s/%s" % (folder, PACKAGES_FOLDER, pack_id, CONAN_MANIFEST)] = ""
exports_sources_dir = os.path.join(client.client_cache.export(ref), EXPORT_SOURCES_DIR)
os.makedirs(exports_sources_dir)
client.save(files, client.client_cache.store)
# Create the manifests to be able to upload
for pack_ref in pack_refs:
digest_path = client.client_cache.digestfile_package(pack_ref)
expected_manifest = FileTreeManifest.create(os.path.dirname(digest_path))
files["%s/%s/%s/%s" % ("/".join(pack_ref.conan),
PACKAGES_FOLDER,
pack_ref.package_id,
CONAN_MANIFEST)] = str(expected_manifest)
client.save(files, client.client_cache.store)
self.client = client
for folder in self.root_folder.values():
client.run("upload %s --all" % folder.replace("/fenix", "@fenix"))
self.assert_folders({"H1": [1, 2], "H2": [1, 2], "B": [1, 2], "O": [1, 2]},
{"H1": [1, 2], "H2": [1, 2], "B": [1, 2], "O": [1, 2]},
{"H1": [1, 2], "H2": [1, 2], "B": [1, 2], "O": [1, 2]},
{"H1": True, "H2": True, "B": True, "O": True})
def force_test(self):
'''Tries to upload a conans exported after than remote version.'''
# Upload all conans and packages
self.client.run('upload %s --all' % str(self.conan_ref))
self.assertTrue(os.path.exists(self.server_reg_folder))
self.assertTrue(os.path.exists(self.server_pack_folder))
# Fake datetime from exported date and upload again
old_digest = self.client.paths.load_manifest(self.conan_ref)
old_digest.file_sums["new_file"] = "012345"
fake_digest = FileTreeManifest(2, old_digest.file_sums)
fake_digest.save(self.client.paths.export(self.conan_ref))
self.client.run('upload %s' % str(self.conan_ref), ignore_error=True)
self.assertIn("Remote recipe is newer than local recipe", self.client.user_io.out)
self.client.run('upload %s --force' % str(self.conan_ref))
self.assertIn("Uploading %s" % str(self.conan_ref),
self.client.user_io.out)
def setUp(self):
self.client = self._get_client()
conan_digest = FileTreeManifest('123123123', {})
self.conan_ref = ConanFileReference.loads("Hello/1.2.1@frodo/stable")
reg_folder = self.client.paths.export(self.conan_ref)
self.client.run('upload %s' % str(self.conan_ref), ignore_error=True)
self.assertIn("There is no local conanfile exported as %s" % str(self.conan_ref),
self.client.user_io.out)
files = hello_source_files()
self.client.save(files, path=reg_folder)
self.client.save({CONANFILE: myconan1,
"include/math/lib1.h": "//copy",
"my_lib/debug/libd.a": "//copy",
"my_data/readme.txt": "//copy",
"my_bin/executable": "//copy"}, path=reg_folder)
conan_digest.save(reg_folder)
mkdir(self.client.client_cache.export_sources(self.conan_ref))
self.package_ref = PackageReference(self.conan_ref, "myfakeid")
self.server_pack_folder = self.test_server.paths.package(self.package_ref)
package_folder = self.client.paths.package(self.package_ref)
save(os.path.join(package_folder, "include", "lib1.h"), "//header")
save(os.path.join(package_folder, "lib", "my_lib", "libd.a"), "//lib")
save(os.path.join(package_folder, "res", "shares", "readme.txt"),
"//res")
save(os.path.join(package_folder, "bin", "my_bin", "executable"), "//bin")
save(os.path.join(package_folder, CONANINFO), "info")
save(os.path.join(package_folder, CONAN_MANIFEST), "manifest")
os.chmod(os.path.join(package_folder, "bin", "my_bin", "executable"),
os.stat(os.path.join(package_folder, "bin", "my_bin", "executable")).st_mode |
stat.S_IRWXU)
digest_path = self.client.client_cache.digestfile_package(self.package_ref)
expected_manifest = FileTreeManifest.create(os.path.dirname(digest_path))
expected_manifest.save(package_folder)
self.server_reg_folder = self.test_server.paths.export(self.conan_ref)
self.assertFalse(os.path.exists(self.server_reg_folder))
self.assertFalse(os.path.exists(self.server_pack_folder))
def test_tree_manifest(self):
tmp_dir = temp_folder()
files = {"one.ext": "aalakjshdlkjahsdlkjahsdljkhsadljkhasljkdhlkjashd",
"path/to/two.txt": "asdas13123",
"two.txt": "asdasdasdasdasdasd"}
for filename, content in files.iteritems():
save(os.path.join(tmp_dir, filename), content)
manifest = FileTreeManifest.create(tmp_dir)
save(os.path.join(tmp_dir, "THEMANIFEST.txt"), str(manifest))
readed_manifest = FileTreeManifest.loads(load(os.path.join(tmp_dir, "THEMANIFEST.txt")))
self.assertEquals(readed_manifest, manifest)
for filepath, md5readed in manifest.file_sums.iteritems():
content = files[filepath]
self.assertEquals(md5(content), md5readed)
def already_pyc_in_manifest_test(self):
tmp_dir = temp_folder()
save(os.path.join(tmp_dir, "man.txt"), "1478122267\nconanfile.pyc: "
"2bcac725a0e6843ef351f4d18cf867ec\n"
"conanfile.py: 2bcac725a0e6843ef351f4d18cf867ec",
"conanfile.pyo: 2bcac725a0e6843ef351f4d18cf867ec")
read_manifest = FileTreeManifest.loads(load(os.path.join(tmp_dir, "man.txt")))
# Not included the pycs or pyo
self.assertEquals(set(read_manifest.file_sums.keys()),
set(["conanfile.py"]))
def get_conan_digest(self, conan_reference):
"""Gets a FileTreeManifest from conans"""
# Obtain the URLs
url = "%s/conans/%s/digest" % (self._remote_api_url, "/".join(conan_reference))
urls = self._get_json(url)
# Get the digest
contents = self.download_files(urls)
contents = {key: decode_text(value) for key, value in dict(contents).items()} # Unroll generator and decode shas (plain text)
return FileTreeManifest.loads(contents[CONAN_MANIFEST])
def get_conan_digest(self, conan_reference):
"""Gets a FileTreeManifest from conans"""
# Obtain the URLs
url = "%s/conans/%s/digest" % (self._remote_api_url, "/".join(conan_reference))
urls = self._get_json(url)
# Get the digest
contents = self.download_files(urls)
contents = dict(contents) # Unroll generator
return FileTreeManifest.loads(contents[CONAN_MANIFEST])
def setUp(self):
self.servers = {"local": TestServer(server_capabilities=[]),
"search_able": TestServer(server_capabilities=[COMPLEX_SEARCH_CAPABILITY])}
self.client = TestClient(servers=self.servers)
# No conans created
self.client.run("search")
output = self.client.user_io.out
self.assertIn('There are no packages', output)
# Conans with and without packages created
self.root_folder1 = 'Hello/1.4.10/fenix/testing'
root_folder2 = 'helloTest/1.4.10/fenix/stable'
root_folder3 = 'Bye/0.14/fenix/testing'
root_folder4 = 'NodeInfo/1.0.2/fenix/stable'
root_folder5 = 'MissFile/1.0.2/fenix/stable'
root_folder11 = 'Hello/1.4.11/fenix/testing'
root_folder12 = 'Hello/1.4.12/fenix/testing'
self.client.save({"Empty/1.10/fake/test/reg/fake.txt": "//",
"%s/%s/WindowsPackageSHA/%s" % (self.root_folder1,
PACKAGES_FOLDER,
CONANINFO): conan_vars1,
"%s/%s/WindowsPackageSHA/%s" % (root_folder11,
PACKAGES_FOLDER,
CONANINFO): conan_vars1,
"%s/%s/WindowsPackageSHA/%s" % (root_folder12,
PACKAGES_FOLDER,
CONANINFO): conan_vars1,
"%s/%s/PlatformIndependantSHA/%s" % (self.root_folder1,
PACKAGES_FOLDER,
CONANINFO): conan_vars1b,
"%s/%s/LinuxPackageSHA/%s" % (self.root_folder1,
PACKAGES_FOLDER,
CONANINFO): conan_vars1c,
"%s/%s/a44f541cd44w57/%s" % (root_folder2,
PACKAGES_FOLDER,
CONANINFO): conan_vars2,
"%s/%s/e4f7vdwcv4w55d/%s" % (root_folder3,
PACKAGES_FOLDER,
CONANINFO): conan_vars3,
"%s/%s/e4f7vdwcv4w55d/%s" % (root_folder4,
PACKAGES_FOLDER,
CONANINFO): conan_vars4,
"%s/%s/e4f7vdwcv4w55d/%s" % (root_folder5,
PACKAGES_FOLDER,
"hello.txt"): "Hello"},
self.client.paths.store)
# Fake some manifests to be able to calculate recipe hash
fake_manifest = FileTreeManifest(1212, {})
fake_manifest.save(os.path.join(self.client.paths.store, self.root_folder1, EXPORT_FOLDER))
fake_manifest.save(os.path.join(self.client.paths.store, root_folder2, EXPORT_FOLDER))
fake_manifest.save(os.path.join(self.client.paths.store, root_folder3, EXPORT_FOLDER))
fake_manifest.save(os.path.join(self.client.paths.store, root_folder4, EXPORT_FOLDER))
fake_manifest.save(os.path.join(self.client.paths.store, root_folder5, EXPORT_FOLDER))
def upload_same_package_dont_compress_test(self):
# Create a manifest for the faked package
pack_path = self.client.paths.package(self.package_ref)
digest_path = self.client.client_cache.digestfile_package(self.package_ref)
expected_manifest = FileTreeManifest.create(os.path.dirname(digest_path))
save(os.path.join(pack_path, CONAN_MANIFEST), str(expected_manifest))
self.client.run("upload %s --all" % str(self.conan_ref), ignore_error=False)
self.assertIn("Compressing package", str(self.client.user_io.out))
self.client.run("upload %s --all" % str(self.conan_ref), ignore_error=False)
self.assertNotIn("Compressing package", str(self.client.user_io.out))
self.assertIn("Package is up to date", str(self.client.user_io.out))
def _check(self, reference, manifest, remote, path):
if os.path.exists(path):
existing_manifest = FileTreeManifest.loads(load(path))
if existing_manifest.file_sums == manifest.file_sums:
self._log.append("Manifest for '%s': OK" % str(reference))
return
if self._verify:
raise ConanException("Modified or new manifest '%s' detected.\n"
"Remote: %s\nProject manifest doesn't match installed one"
% (str(reference), remote))
self._handle_add(reference, remote, manifest, path)
def _init_export_folder(destination_folder):
previous_digest = None
try:
if os.path.exists(destination_folder):
if os.path.exists(os.path.join(destination_folder, CONAN_MANIFEST)):
manifest_content = load(os.path.join(destination_folder, CONAN_MANIFEST))
previous_digest = FileTreeManifest.loads(manifest_content)
# Maybe here we want to invalidate cache
rmdir(destination_folder)
os.makedirs(destination_folder)
except Exception as e:
raise ConanException("Unable to create folder %s\n%s" % (destination_folder, str(e)))
return previous_digest
def get_package_manifest(self, package_reference):
"""Gets a FileTreeManifest from a package"""
# Obtain the URLs
url = "%s/conans/%s/packages/%s/digest" % (self._remote_api_url,
"/".join(package_reference.conan),
package_reference.package_id)
urls = self._get_file_to_url_dict(url)
# Get the digest
contents = self.download_files(urls)
# Unroll generator and decode shas (plain text)
contents = {key: decode_text(value) for key, value in dict(contents).items()}
return FileTreeManifest.loads(contents[CONAN_MANIFEST])
def _create_aux_files(build_folder, package_folder):
""" auxiliary method that creates CONANINFO in
the package_folder
"""
try:
logger.debug("Creating config files to %s" % package_folder)
shutil.copy(os.path.join(build_folder, CONANINFO), package_folder)
# Create the digest for the package
digest = FileTreeManifest.create(package_folder)
save(os.path.join(package_folder, CONAN_MANIFEST), str(digest))
except IOError:
raise ConanException("%s does not exist inside of your % folder. Try to re-build it again"
" to solve it." % (CONANINFO, build_folder))
def setUp(self):
servers = {}
# All can write (for avoid authentication until we mock user_io)
self.test_server = TestServer([("*/*@*/*", "*")], [("*/*@*/*", "*")],
users={"lasote": "mypass"})
servers["default"] = self.test_server
conan_digest = FileTreeManifest('123123123', {})
self.client = TestClient(servers=servers, users={"default": [("lasote", "mypass")]})
self.conan_ref = ConanFileReference.loads("Hello/1.2.1@frodo/stable")
reg_folder = self.client.paths.export(self.conan_ref)
self.client.run('upload %s' % str(self.conan_ref), ignore_error=True)
self.assertIn("There is no local conanfile exported as %s" % str(self.conan_ref),
self.client.user_io.out)
files = hello_source_files()
self.client.save(files, path=reg_folder)
self.client.save({CONANFILE: myconan1,
CONAN_MANIFEST: str(conan_digest),
"include/math/lib1.h": "//copy",
"my_lib/debug/libd.a": "//copy",
"my_data/readme.txt": "//copy",
"my_bin/executable": "//copy"}, path=reg_folder)
self.package_ref = PackageReference(self.conan_ref, "myfakeid")
self.server_pack_folder = self.test_server.paths.package(self.package_ref)
package_folder = self.client.paths.package(self.package_ref)
save(os.path.join(package_folder, "include", "lib1.h"), "//header")
save(os.path.join(package_folder, "lib", "my_lib", "libd.a"), "//lib")
save(os.path.join(package_folder, "res", "shares", "readme.txt"),
"//res")
save(os.path.join(package_folder, "bin", "my_bin", "executable"), "//bin")
save(os.path.join(package_folder, CONANINFO), "info")
save(os.path.join(package_folder, CONAN_MANIFEST), "manifest")
os.chmod(os.path.join(package_folder, "bin", "my_bin", "executable"),
os.stat(os.path.join(package_folder, "bin", "my_bin", "executable")).st_mode |
stat.S_IRWXU)
digest_path = self.client.client_cache.digestfile_package(self.package_ref)
expected_manifest = FileTreeManifest.create(os.path.dirname(digest_path))
save(os.path.join(package_folder, CONAN_MANIFEST), str(expected_manifest))
self.server_reg_folder = self.test_server.paths.export(self.conan_ref)
self.assertFalse(os.path.exists(self.server_reg_folder))
self.assertFalse(os.path.exists(self.server_pack_folder))
def complete_test(self):
""" basic installation of a new conans
"""
servers = {}
# All can write (for avoid authentication until we mock user_io)
test_server = TestServer([("*/*@*/*", "*")], [("*/*@*/*", "*")])
servers["default"] = test_server
conan_digest = FileTreeManifest(123123123, {})
client = TestClient(servers=servers)
client.init_dynamic_vars()
conan_ref = ConanFileReference.loads("Hello/1.2.1@frodo/stable")
reg_folder = client.paths.export(conan_ref)
files = hello_source_files()
client.save(files, path=reg_folder)
client.save(
{
CONANFILE: myconan1,
CONAN_MANIFEST: str(conan_digest),
"include/math/lib1.h": "//copy",
"my_lib/debug/libd.a": "//copy",
"my_data/readme.txt": "//copy"
},
path=reg_folder)
package_ref = PackageReference(conan_ref, "fakeid")
package_folder = client.paths.package(package_ref)
save(os.path.join(package_folder, "include", "lib1.h"), "//header")
save(os.path.join(package_folder, "lib", "my_lib", "libd.a"), "//lib")
save(os.path.join(package_folder, "res", "shares", "readme.txt"),
"//res")
client.remote_manager.upload_conan(conan_ref)
client.remote_manager.upload_package(package_ref)
client2 = TestClient(servers=servers)
client2.init_dynamic_vars()
loader = ConanFileLoader(TestBufferConanOutput(), None, Settings(),
OptionsValues.loads(""))
installer = ConanInstaller(client2.paths, client2.user_io, loader,
client2.remote_manager, "default")
installer.retrieve_conanfile(conan_ref)
installer._retrieve_remote_package(package_ref)
reg_path = client2.paths.export(
ConanFileReference.loads("Hello/1.2.1/frodo/stable"))
pack_folder = client2.paths.package(package_ref)
# Test the file in the downloaded conans
files = [
'CMakeLists.txt', 'my_lib/debug/libd.a', 'hello.cpp', 'hello0.h',
CONANFILE, CONAN_MANIFEST, 'main.cpp', 'include/math/lib1.h',
'my_data/readme.txt'
]
for _file in files:
self.assertTrue(os.path.exists(os.path.join(reg_path, _file)))
self.assertTrue(os.path.exists(pack_folder))
# Test the file in the downloaded package
self.assertTrue(os.path.exists(pack_folder))
self.assertTrue(
os.path.exists(os.path.join(pack_folder, "include", "lib1.h")))
self.assertTrue(
os.path.exists(os.path.join(pack_folder, "lib", "my_lib/libd.a")))
self.assertTrue(
os.path.exists(
os.path.join(pack_folder, "res", "shares/readme.txt")))
def recipe_manifest(self):
return FileTreeManifest.load(self.export())
def setUp(self):
hello_files = cpp_hello_conan_files("Hello")
test_conanfile_contents = hello_files[CONANFILE]
self.server_folder = temp_folder()
test_server = TestServer(
users={"fenix": "mypass"},
base_path=self.server_folder) # exported users and passwords
self.server = test_server
servers = {"default": test_server}
client = TestClient(servers=servers,
users={"default": [("fenix", "mypass")]})
# Conans with and without packages created
self.root_folder = {
"H1": 'Hello/1.4.10/fenix/testing',
"H2": 'Hello/2.4.11/fenix/testing',
"B": 'Bye/0.14/fenix/testing',
"O": 'Other/1.2/fenix/testing'
}
files = {}
pack_refs = []
for key, folder in self.root_folder.items():
ref = ConanFileReference.loads(folder)
files["%s/%s/conanfile.py" %
(folder, EXPORT_FOLDER)] = test_conanfile_contents
files["%s/%s/conanmanifest.txt" % (folder, EXPORT_FOLDER)] = ""
files["%s/%s/conans.txt" % (folder, SRC_FOLDER)] = ""
for pack_id in (1, 2):
i = pack_id
pack_id = "%s_%s" % (pack_id, key)
pack_refs.append(PackageReference(ref, str(pack_id)))
files["%s/%s/%s/conans.txt" %
(folder, BUILD_FOLDER, pack_id)] = ""
files["%s/%s/%s/conans.txt" %
(folder, PACKAGES_FOLDER, pack_id)] = ""
files["%s/%s/%s/%s" % (folder, PACKAGES_FOLDER, pack_id,
CONANINFO)] = conaninfo % str(i)
files["%s/%s/%s/%s" %
(folder, PACKAGES_FOLDER, pack_id, CONAN_MANIFEST)] = ""
exports_sources_dir = client.client_cache.export_sources(ref)
os.makedirs(exports_sources_dir)
client.save(files, client.client_cache.store)
# Create the manifests to be able to upload
for pack_ref in pack_refs:
digest_path = client.client_cache.digestfile_package(pack_ref)
expected_manifest = FileTreeManifest.create(
os.path.dirname(digest_path))
files["%s/%s/%s/%s" % ("/".join(
pack_ref.conan), PACKAGES_FOLDER, pack_ref.package_id,
CONAN_MANIFEST)] = str(expected_manifest)
client.save(files, client.client_cache.store)
self.client = client
for folder in self.root_folder.values():
client.run("upload %s --all" % folder.replace("/fenix", "@fenix"))
self.assert_folders(
{
"H1": [1, 2],
"H2": [1, 2],
"B": [1, 2],
"O": [1, 2]
}, {
"H1": [1, 2],
"H2": [1, 2],
"B": [1, 2],
"O": [1, 2]
}, {
"H1": [1, 2],
"H2": [1, 2],
"B": [1, 2],
"O": [1, 2]
}, {
"H1": True,
"H2": True,
"B": True,
"O": True
})
def load_package_manifest(self, package_reference):
"""conan_id = sha(zip file)"""
package_folder = self.package(package_reference, short_paths=None)
return FileTreeManifest.load(package_folder)
def _get_recipe(self, reference, check_updates, update, remote_name,
recorder):
output = ScopedOutput(str(reference), self._out)
# check if it is in disk
conanfile_path = self._client_cache.conanfile(reference)
# NOT in disk, must be retrieved from remotes
if not os.path.exists(conanfile_path):
remote, new_ref = self._download_recipe(reference, output,
remote_name, recorder)
status = RECIPE_DOWNLOADED
return conanfile_path, status, remote, new_ref
metadata = self._client_cache.load_metadata(reference)
cur_revision = metadata.recipe.revision
remote = self._registry.refs.get(reference)
named_remote = self._registry.remotes.get(
remote_name) if remote_name else None
update_remote = named_remote or remote
# Check if we have a revision different from the requested one
revisions_enabled = get_env("CONAN_CLIENT_REVISIONS_ENABLED", False)
if revisions_enabled and reference.revision and cur_revision != reference.revision:
output.info(
"Different revision requested, removing current local recipe..."
)
DiskRemover(self._client_cache).remove_recipe(reference)
output.info("Retrieving from remote '%s'..." % update_remote.name)
new_ref = self._remote_manager.get_recipe(reference, update_remote)
self._registry.refs.set(new_ref, update_remote.name)
status = RECIPE_UPDATED
return conanfile_path, status, update_remote, new_ref
check_updates = check_updates or update
# Recipe exists in disk, but no need to check updates
if not check_updates:
status = RECIPE_INCACHE
ref = reference.copy_with_rev(cur_revision)
return conanfile_path, status, remote, ref
if not update_remote:
status = RECIPE_NO_REMOTE
ref = reference.copy_with_rev(cur_revision)
return conanfile_path, status, None, ref
try: # get_conan_manifest can fail, not in server
upstream_manifest = self._remote_manager.get_conan_manifest(
reference, update_remote)
except NotFoundException:
status = RECIPE_NOT_IN_REMOTE
ref = reference.copy_with_rev(cur_revision)
return conanfile_path, status, update_remote, ref
export = self._client_cache.export(reference)
read_manifest = FileTreeManifest.load(export)
if upstream_manifest != read_manifest:
if upstream_manifest.time > read_manifest.time:
if update:
DiskRemover(self._client_cache).remove_recipe(reference)
output.info("Retrieving from remote '%s'..." %
update_remote.name)
new_ref = self._remote_manager.get_recipe(
reference, update_remote)
self._registry.refs.set(new_ref, update_remote.name)
status = RECIPE_UPDATED
return conanfile_path, status, update_remote, new_ref
else:
status = RECIPE_UPDATEABLE
else:
status = RECIPE_NEWER
else:
status = RECIPE_INCACHE
ref = reference.copy_with_rev(cur_revision)
return conanfile_path, status, update_remote, ref
def cmd_export(app,
conanfile_path,
name,
version,
user,
channel,
keep_source,
export=True,
graph_lock=None):
""" Export the recipe
param conanfile_path: the original source directory of the user containing a
conanfile.py
"""
loader, cache, hook_manager, output = app.loader, app.cache, app.hook_manager, app.out
revisions_enabled = app.config.revisions_enabled
conanfile = loader.load_export(conanfile_path, name, version, user,
channel)
# FIXME: Conan 2.0, deprecate CONAN_USER AND CONAN_CHANNEL and remove this try excepts
# Take the default from the env vars if they exist to not break behavior
try:
user = conanfile.user
except ConanException:
user = None
try:
channel = conanfile.channel
except ConanException:
channel = None
ref = ConanFileReference(conanfile.name, conanfile.version, user, channel)
# If we receive lock information, python_requires could have been locked
if graph_lock:
node_id = graph_lock.get_node(ref)
python_requires = graph_lock.python_requires(node_id)
# TODO: check that the locked python_requires are different from the loaded ones
# FIXME: private access, will be improved when api collaborators are improved
loader._python_requires._range_resolver.output # invalidate previous version range output
conanfile = loader.load_export(conanfile_path, conanfile.name,
conanfile.version, conanfile.user,
conanfile.channel, python_requires)
check_casing_conflict(cache=cache, ref=ref)
package_layout = cache.package_layout(ref,
short_paths=conanfile.short_paths)
if not export:
metadata = package_layout.load_metadata()
recipe_revision = metadata.recipe.revision
ref = ref.copy_with_rev(recipe_revision)
if graph_lock:
graph_lock.update_exported_ref(node_id, ref)
return ref
hook_manager.execute("pre_export",
conanfile=conanfile,
conanfile_path=conanfile_path,
reference=package_layout.ref)
logger.debug("EXPORT: %s" % conanfile_path)
output.highlight("Exporting package recipe")
conan_linter(conanfile_path, output)
output = conanfile.output
# Get previous digest
try:
previous_manifest = FileTreeManifest.load(package_layout.export())
except IOError:
previous_manifest = None
finally:
_recreate_folders(package_layout.export(),
package_layout.export_sources())
# Copy sources to target folders
with package_layout.conanfile_write_lock(output=output):
origin_folder = os.path.dirname(conanfile_path)
export_recipe(conanfile, origin_folder, package_layout.export())
export_source(conanfile, origin_folder,
package_layout.export_sources())
shutil.copy2(conanfile_path, package_layout.conanfile())
_capture_export_scm_data(conanfile,
os.path.dirname(conanfile_path),
package_layout.export(),
output,
scm_src_file=package_layout.scm_folder())
# Execute post-export hook before computing the digest
hook_manager.execute("post_export",
conanfile=conanfile,
reference=package_layout.ref,
conanfile_path=package_layout.conanfile())
# Compute the new digest
manifest = FileTreeManifest.create(package_layout.export(),
package_layout.export_sources())
modified_recipe = not previous_manifest or previous_manifest != manifest
if modified_recipe:
output.success('A new %s version was exported' % CONANFILE)
output.info('Folder: %s' % package_layout.export())
else:
output.info("The stored package has not changed")
manifest = previous_manifest # Use the old one, keep old timestamp
manifest.save(package_layout.export())
# Compute the revision for the recipe
revision = _update_revision_in_metadata(
package_layout=package_layout,
revisions_enabled=revisions_enabled,
output=output,
path=os.path.dirname(conanfile_path),
manifest=manifest,
revision_mode=conanfile.revision_mode)
# FIXME: Conan 2.0 Clear the registry entry if the recipe has changed
source_folder = package_layout.source()
if os.path.exists(source_folder):
try:
if is_dirty(source_folder):
output.info("Source folder is corrupted, forcing removal")
rmdir(source_folder)
elif modified_recipe and not keep_source:
output.info(
"Package recipe modified in export, forcing source folder removal"
)
output.info("Use the --keep-source, -k option to skip it")
rmdir(source_folder)
except BaseException as e:
output.error(
"Unable to delete source folder. Will be marked as corrupted for deletion"
)
output.warn(str(e))
set_dirty(source_folder)
# When revisions enabled, remove the packages not matching the revision
if revisions_enabled:
packages = search_packages(package_layout, query=None)
metadata = package_layout.load_metadata()
recipe_revision = metadata.recipe.revision
to_remove = [
pid for pid in packages if metadata.packages.get(pid)
and metadata.packages.get(pid).recipe_revision != recipe_revision
]
if to_remove:
output.info(
"Removing the local binary packages from different recipe revisions"
)
remover = DiskRemover()
remover.remove_packages(package_layout, ids_filter=to_remove)
ref = ref.copy_with_rev(revision)
output.info("Exported revision: %s" % revision)
if graph_lock:
graph_lock.update_exported_ref(node_id, ref)
return ref
def setUp(self):
self.servers = {
"local":
TestServer(server_capabilities=[]),
"search_able":
TestServer(server_capabilities=[COMPLEX_SEARCH_CAPABILITY])
}
self.client = TestClient(servers=self.servers)
# No conans created
self.client.run("search")
output = self.client.user_io.out
self.assertIn('There are no packages', output)
# Conans with and without packages created
self.root_folder1 = 'Hello/1.4.10/fenix/testing'
root_folder2 = 'helloTest/1.4.10/fenix/stable'
root_folder3 = 'Bye/0.14/fenix/testing'
root_folder4 = 'NodeInfo/1.0.2/fenix/stable'
root_folder5 = 'MissFile/1.0.2/fenix/stable'
root_folder11 = 'Hello/1.4.11/fenix/testing'
root_folder12 = 'Hello/1.4.12/fenix/testing'
self.client.save(
{
"Empty/1.10/fake/test/reg/fake.txt":
"//",
"%s/%s/WindowsPackageSHA/%s" % (self.root_folder1, PACKAGES_FOLDER, CONANINFO):
conan_vars1,
"%s/%s/WindowsPackageSHA/%s" % (root_folder11, PACKAGES_FOLDER, CONANINFO):
conan_vars1,
"%s/%s/WindowsPackageSHA/%s" % (root_folder12, PACKAGES_FOLDER, CONANINFO):
conan_vars1,
"%s/%s/PlatformIndependantSHA/%s" % (self.root_folder1, PACKAGES_FOLDER, CONANINFO):
conan_vars1b,
"%s/%s/LinuxPackageSHA/%s" % (self.root_folder1, PACKAGES_FOLDER, CONANINFO):
conan_vars1c,
"%s/%s/a44f541cd44w57/%s" % (root_folder2, PACKAGES_FOLDER, CONANINFO):
conan_vars2,
"%s/%s/e4f7vdwcv4w55d/%s" % (root_folder3, PACKAGES_FOLDER, CONANINFO):
conan_vars3,
"%s/%s/e4f7vdwcv4w55d/%s" % (root_folder4, PACKAGES_FOLDER, CONANINFO):
conan_vars4,
"%s/%s/e4f7vdwcv4w55d/%s" % (root_folder5, PACKAGES_FOLDER, "hello.txt"):
"Hello"
}, self.client.paths.store)
# Fake some manifests to be able to calculate recipe hash
fake_manifest = FileTreeManifest(1212, {})
fake_manifest.save(
os.path.join(self.client.paths.store, self.root_folder1,
EXPORT_FOLDER))
fake_manifest.save(
os.path.join(self.client.paths.store, root_folder2, EXPORT_FOLDER))
fake_manifest.save(
os.path.join(self.client.paths.store, root_folder3, EXPORT_FOLDER))
fake_manifest.save(
os.path.join(self.client.paths.store, root_folder4, EXPORT_FOLDER))
fake_manifest.save(
os.path.join(self.client.paths.store, root_folder5, EXPORT_FOLDER))
def deploy_test(self, deploy_to_abs):
client = TestClient()
libconanfile = """from conans import ConanFile
from conans.tools import save
class Lib(ConanFile):
exports_sources = "*"
def build(self):
save("mylib.dll", "mydll")
def package(self):
self.copy("*")
def deploy(self):
self.output.info("Lib deploy()")
"""
client.save({
"conanfile.py": libconanfile,
"License.md": "lib license",
"otherfile": ""
})
client.run("create Lib/0.1@user/testing")
self.assertNotIn("Lib deploy()", client.out)
if deploy_to_abs:
dll_folder = temp_folder()
mkdir(dll_folder)
else:
dll_folder = ""
conanfile = """from conans import ConanFile
from conans.tools import save
class Pkg(ConanFile):
requires = "Lib/0.1@user/testing"
def build(self):
save("myapp.exe", "myexe")
def package(self):
self.copy("*")
def deploy(self):
self.output.info("Pkg deploy()")
self.copy("*.exe")
self.copy_deps("*.dll", dst="%s")
""" % dll_folder.replace("\\", "/")
client.save({"conanfile.py": conanfile})
client.run("create Pkg/0.1@user/testing")
self.assertNotIn("deploy()", client.out)
client.current_folder = temp_folder()
client.run("install Pkg/0.1@user/testing")
self.assertIn(
"Pkg/0.1@user/testing deploy(): Copied 1 '.dll' files: mylib.dll",
client.out)
self.assertIn(
"Pkg/0.1@user/testing deploy(): Copied 1 '.exe' files: myapp.exe",
client.out)
deploy_manifest = FileTreeManifest.loads(
load(os.path.join(client.current_folder, "deploy_manifest.txt")))
app = os.path.join(client.current_folder, "myapp.exe")
if deploy_to_abs:
lib = os.path.join(dll_folder, "mylib.dll")
else:
lib = os.path.join(client.current_folder, "mylib.dll")
self.assertEqual(sorted([app, lib]),
sorted(deploy_manifest.file_sums.keys()))
self.assertEqual(load(app), "myexe")
self.assertEqual(load(lib), "mydll")
def cmd_export(conanfile_path,
name,
version,
user,
channel,
keep_source,
revisions_enabled,
output,
hook_manager,
loader,
cache,
export=True):
""" Export the recipe
param conanfile_path: the original source directory of the user containing a
conanfile.py
"""
conanfile = loader.load_export(conanfile_path, name, version, user,
channel)
ref = ConanFileReference(conanfile.name, conanfile.version, conanfile.user,
conanfile.channel)
check_casing_conflict(cache=cache, ref=ref)
package_layout = cache.package_layout(ref,
short_paths=conanfile.short_paths)
if not export:
metadata = package_layout.load_metadata()
recipe_revision = metadata.recipe.revision
ref = ref.copy_with_rev(recipe_revision)
return ref
hook_manager.execute("pre_export",
conanfile=conanfile,
conanfile_path=conanfile_path,
reference=package_layout.ref)
logger.debug("EXPORT: %s" % conanfile_path)
output.highlight("Exporting package recipe")
conan_linter(conanfile_path, output)
output = conanfile.output
# Get previous digest
try:
previous_digest = FileTreeManifest.load(package_layout.export())
except IOError:
previous_digest = None
finally:
_recreate_folders(package_layout.export(),
package_layout.export_sources())
# Copy sources to target folders
with package_layout.conanfile_write_lock(output=output):
origin_folder = os.path.dirname(conanfile_path)
export_recipe(conanfile, origin_folder, package_layout.export())
export_source(conanfile, origin_folder,
package_layout.export_sources())
shutil.copy2(conanfile_path, package_layout.conanfile())
_capture_export_scm_data(conanfile,
os.path.dirname(conanfile_path),
package_layout.export(),
output,
scm_src_file=package_layout.scm_folder())
# Execute post-export hook before computing the digest
hook_manager.execute("post_export",
conanfile=conanfile,
reference=package_layout.ref,
conanfile_path=package_layout.conanfile())
# Compute the new digest
digest = FileTreeManifest.create(package_layout.export(),
package_layout.export_sources())
modified_recipe = not previous_digest or previous_digest != digest
if modified_recipe:
output.success('A new %s version was exported' % CONANFILE)
output.info('Folder: %s' % package_layout.export())
else:
output.info("The stored package has not changed")
digest = previous_digest # Use the old one, keep old timestamp
digest.save(package_layout.export())
# Compute the revision for the recipe
revision = _update_revision_in_metadata(
package_layout=package_layout,
revisions_enabled=revisions_enabled,
output=output,
path=os.path.dirname(conanfile_path),
digest=digest,
revision_mode=conanfile.revision_mode)
# FIXME: Conan 2.0 Clear the registry entry if the recipe has changed
source_folder = package_layout.source()
if os.path.exists(source_folder):
try:
if is_dirty(source_folder):
output.info("Source folder is corrupted, forcing removal")
rmdir(source_folder)
elif modified_recipe and not keep_source:
output.info(
"Package recipe modified in export, forcing source folder removal"
)
output.info("Use the --keep-source, -k option to skip it")
rmdir(source_folder)
except BaseException as e:
output.error(
"Unable to delete source folder. Will be marked as corrupted for deletion"
)
output.warn(str(e))
set_dirty(source_folder)
# When revisions enabled, remove the packages not matching the revision
if revisions_enabled:
packages = search_packages(package_layout, query=None)
metadata = package_layout.load_metadata()
recipe_revision = metadata.recipe.revision
to_remove = [
pid for pid in packages if metadata.packages.get(pid)
and metadata.packages.get(pid).recipe_revision != recipe_revision
]
if to_remove:
output.info(
"Removing the local binary packages from different recipe revisions"
)
remover = DiskRemover()
remover.remove_packages(package_layout, ids_filter=to_remove)
ref = ref.copy_with_rev(revision)
return ref
def _digests(digest_path, exports_sources_folder=None):
readed_digest = FileTreeManifest.loads(load(digest_path))
expected_digest = FileTreeManifest.create(os.path.dirname(digest_path),
exports_sources_folder)
return readed_digest, expected_digest
def _create_manifest(self, package_reference):
# Create the manifest to be able to upload the package
pack_path = self.client.paths.package(package_reference)
digest_path = self.client.client_cache.digestfile_package(package_reference)
expected_manifest = FileTreeManifest.create(os.path.dirname(digest_path))
save(os.path.join(pack_path, CONAN_MANIFEST), str(expected_manifest))
def inc_package_manifest_timestamp(cache, package_reference, inc_time):
pref = PackageReference.loads(package_reference)
path = cache.package_layout(pref.ref).package(pref)
manifest = FileTreeManifest.load(path)
manifest.time += inc_time
manifest.save(path)
def bump_time(inc_time):
path = client.client_cache.export(
ConanFileReference.loads("Pkg/0.1@lasote/testing"))
manifest = FileTreeManifest.load(path)
manifest.time += inc_time
manifest.save(path)
def load_digest(self, conan_reference):
'''conan_id = sha(zip file)'''
filename = os.path.join(self.export(conan_reference), CONAN_MANIFEST)
return FileTreeManifest.loads(load(filename))
def _digests(self, digest_path):
if not path_exists(digest_path, self.store):
return None, None
readed_digest = FileTreeManifest.loads(load(digest_path))
expected_digest = FileTreeManifest.create(os.path.dirname(digest_path))
return readed_digest, expected_digest
def load_manifest(self, conan_reference):
'''conan_id = sha(zip file)'''
filename = self.digestfile_conanfile(conan_reference)
return FileTreeManifest.loads(load(filename))
def setUp(self):
self.client = self._get_client()
self.ref = ConanFileReference.loads("Hello/1.2.1@frodo/stable#%s" %
DEFAULT_REVISION_V1)
self.pref = PackageReference(self.ref, "myfakeid", DEFAULT_REVISION_V1)
reg_folder = self.client.cache.package_layout(self.ref).export()
self.client.run('upload %s' % str(self.ref), assert_error=True)
self.assertIn("ERROR: Recipe not found: '%s'" % str(self.ref),
self.client.out)
files = hello_source_files()
fake_metadata = PackageMetadata()
fake_metadata.recipe.revision = DEFAULT_REVISION_V1
fake_metadata.packages[self.pref.id].revision = DEFAULT_REVISION_V1
self.client.save({"metadata.json": fake_metadata.dumps()},
path=self.client.cache.package_layout(
self.ref).base_folder())
self.client.save(files, path=reg_folder)
self.client.save(
{
CONANFILE: myconan1,
"include/math/lib1.h": "//copy",
"my_lib/debug/libd.a": "//copy",
"my_data/readme.txt": "//copy",
"my_bin/executable": "//copy"
},
path=reg_folder)
mkdir(self.client.cache.package_layout(self.ref).export_sources())
manifest = FileTreeManifest.create(reg_folder)
manifest.time = '123123123'
manifest.save(reg_folder)
self.test_server.server_store.update_last_revision(self.ref)
self.server_pack_folder = self.test_server.server_store.package(
self.pref)
package_folder = self.client.cache.package_layout(self.ref).package(
self.pref)
save(os.path.join(package_folder, "include", "lib1.h"), "//header")
save(os.path.join(package_folder, "lib", "my_lib", "libd.a"), "//lib")
save(os.path.join(package_folder, "res", "shares", "readme.txt"),
"//res")
save(os.path.join(package_folder, "bin", "my_bin", "executable"),
"//bin")
save(os.path.join(package_folder, CONANINFO),
"""[recipe_hash]\n%s""" % manifest.summary_hash)
FileTreeManifest.create(package_folder).save(package_folder)
self.test_server.server_store.update_last_package_revision(self.pref)
os.chmod(
os.path.join(package_folder, "bin", "my_bin", "executable"),
os.stat(os.path.join(package_folder, "bin", "my_bin",
"executable")).st_mode | stat.S_IRWXU)
expected_manifest = FileTreeManifest.create(package_folder)
expected_manifest.save(package_folder)
self.server_reg_folder = self.test_server.server_store.export(self.ref)
self.assertFalse(os.path.exists(self.server_reg_folder))
self.assertFalse(os.path.exists(self.server_pack_folder))
def load_package_manifest(self, package_reference):
'''conan_id = sha(zip file)'''
filename = self.digestfile_package(package_reference, short_paths=None)
return FileTreeManifest.loads(load(filename))
def load_manifest(self, conan_reference):
"""conan_id = sha(zip file)"""
assert isinstance(conan_reference, ConanFileReference)
export_folder = self.export(conan_reference)
check_ref_case(conan_reference, export_folder, self.store)
return FileTreeManifest.load(export_folder)
def _digests(self, digest_path):
readed_digest = FileTreeManifest.loads(load(digest_path))
expected_digest = FileTreeManifest.create(os.path.dirname(digest_path))
return readed_digest, expected_digest
def _digests(folder, exports_sources_folder=None):
readed_digest = FileTreeManifest.load(folder)
expected_digest = FileTreeManifest.create(folder,
exports_sources_folder)
return readed_digest, expected_digest
def setUp(self):
hello_files = cpp_hello_conan_files("Hello")
test_conanfile_contents = hello_files[CONANFILE]
self.server_folder = temp_folder()
test_server = TestServer(
users={"myuser": "******"},
base_path=self.server_folder) # exported users and passwords
self.server = test_server
servers = {"default": test_server}
client = TestClient(servers=servers,
users={"default": [("myuser", "mypass")]})
# Conans with and without packages created
self.root_folder = {
"H1": 'Hello/1.4.10@myuser/testing',
"H2": 'Hello/2.4.11@myuser/testing',
"B": 'Bye/0.14@myuser/testing',
"O": 'Other/1.2@myuser/testing'
}
files = {}
prefs = []
for key, folder in self.root_folder.items():
ref = ConanFileReference.loads(folder)
folder = folder.replace("@", "/")
fake_metadata = PackageMetadata()
fake_metadata.recipe.revision = DEFAULT_REVISION_V1
files["%s/%s/conanfile.py" %
(folder, EXPORT_FOLDER)] = test_conanfile_contents
files["%s/%s/conanmanifest.txt" %
(folder, EXPORT_FOLDER
)] = "%s\nconanfile.py: 234234234" % fake_recipe_hash
files["%s/%s/conans.txt" % (folder, SRC_FOLDER)] = ""
for pack_id in (1, 2):
i = pack_id
pack_id = "%s_%s" % (pack_id, key)
fake_metadata.packages[pack_id].revision = DEFAULT_REVISION_V1
prefs.append(PackageReference(ref, str(pack_id)))
files["%s/%s/%s/conans.txt" %
(folder, BUILD_FOLDER, pack_id)] = ""
files["%s/%s/%s/conans.txt" %
(folder, PACKAGES_FOLDER, pack_id)] = ""
files["%s/%s/%s/%s" % (
folder, PACKAGES_FOLDER, pack_id, CONANINFO
)] = conaninfo % str(i) + "905eefe3570dd09a8453b30b9272bb44"
files["%s/%s/%s/%s" %
(folder, PACKAGES_FOLDER, pack_id, CONAN_MANIFEST)] = ""
files["%s/metadata.json" % folder] = fake_metadata.dumps()
exports_sources_dir = client.cache.package_layout(
ref).export_sources()
os.makedirs(exports_sources_dir)
client.save(files, client.cache.store)
# Create the manifests to be able to upload
for pref in prefs:
pkg_folder = client.cache.package_layout(pref.ref).package(pref)
expected_manifest = FileTreeManifest.create(pkg_folder)
files["%s/%s/%s/%s" %
(pref.ref.dir_repr(), PACKAGES_FOLDER, pref.id,
CONAN_MANIFEST)] = repr(expected_manifest)
client.save(files, client.cache.store)
self.client = client
for folder in self.root_folder.values():
client.run("upload %s --all" % folder)
self.assert_folders(
{
"H1": [1, 2],
"H2": [1, 2],
"B": [1, 2],
"O": [1, 2]
}, {
"H1": [1, 2],
"H2": [1, 2],
"B": [1, 2],
"O": [1, 2]
}, {
"H1": [1, 2],
"H2": [1, 2],
"B": [1, 2],
"O": [1, 2]
}, {
"H1": True,
"H2": True,
"B": True,
"O": True
})
def _create_manifest(self, client, package_reference):
# Create the manifest to be able to upload the package
pack_path = client.cache.package(package_reference)
expected_manifest = FileTreeManifest.create(pack_path)
expected_manifest.save(pack_path)
def _get_recipe(self, ref, check_updates, update, remotes, recorder):
output = ScopedOutput(str(ref), self._out)
# check if it is in disk
conanfile_path = self._cache.package_layout(ref).conanfile()
# NOT in disk, must be retrieved from remotes
if not os.path.exists(conanfile_path):
remote, new_ref = self._download_recipe(ref, output, remotes,
remotes.selected, recorder)
status = RECIPE_DOWNLOADED
return conanfile_path, status, remote, new_ref
metadata = self._cache.package_layout(ref).load_metadata()
cur_revision = metadata.recipe.revision
cur_remote = metadata.recipe.remote
cur_remote = remotes[cur_remote] if cur_remote else None
selected_remote = remotes.selected or cur_remote
check_updates = check_updates or update
# Recipe exists in disk, but no need to check updates
requested_different_revision = (
ref.revision is not None) and cur_revision != ref.revision
if requested_different_revision and not check_updates:
raise NotFoundException(
"The recipe in the local cache doesn't match the specified "
"revision. Use '--update' to check in the remote.")
if not requested_different_revision:
if not check_updates:
status = RECIPE_INCACHE
ref = ref.copy_with_rev(cur_revision)
return conanfile_path, status, cur_remote, ref
if not selected_remote:
status = RECIPE_NO_REMOTE
ref = ref.copy_with_rev(cur_revision)
return conanfile_path, status, None, ref
else: # Requested different revision and with --update
remote, new_ref = self._download_recipe(ref, output, remotes,
selected_remote, recorder)
status = RECIPE_DOWNLOADED
return conanfile_path, status, remote, new_ref
try: # get_recipe_manifest can fail, not in server
upstream_manifest, ref = self._remote_manager.get_recipe_manifest(
ref, selected_remote)
except NotFoundException:
status = RECIPE_NOT_IN_REMOTE
ref = ref.copy_with_rev(cur_revision)
return conanfile_path, status, selected_remote, ref
export = self._cache.package_layout(ref).export()
read_manifest = FileTreeManifest.load(export)
if upstream_manifest != read_manifest:
if upstream_manifest.time > read_manifest.time:
if update:
DiskRemover().remove_recipe(
self._cache.package_layout(ref), output=output)
output.info("Retrieving from remote '%s'..." %
selected_remote.name)
self._download_recipe(ref, output, remotes,
selected_remote, recorder)
with self._cache.package_layout(
ref).update_metadata() as metadata:
metadata.recipe.remote = selected_remote.name
status = RECIPE_UPDATED
return conanfile_path, status, selected_remote, ref
else:
status = RECIPE_UPDATEABLE
else:
status = RECIPE_NEWER
else:
status = RECIPE_INCACHE
ref = ref.copy_with_rev(cur_revision)
return conanfile_path, status, selected_remote, ref
def inc_recipe_manifest_timestamp(cache, reference, inc_time):
ref = ConanFileReference.loads(reference)
path = cache.package_layout(ref).export()
manifest = FileTreeManifest.load(path)
manifest.time += inc_time
manifest.save(path)
def package_manifests(self, pref):
package_folder = self.package(pref)
readed_manifest = FileTreeManifest.load(package_folder)
expected_manifest = FileTreeManifest.create(package_folder)
return readed_manifest, expected_manifest
def _create_manifest(client, pref):
# Create the manifest to be able to upload the package
pack_path = client.cache.package_layout(pref.ref).package(pref)
expected_manifest = FileTreeManifest.create(pack_path)
expected_manifest.save(pack_path)
def inc_package_manifest_timestamp(client_cache, package_ref, inc_time):
pkg_ref = PackageReference.loads(str(package_ref))
path = client_cache.package(pkg_ref)
manifest = FileTreeManifest.load(path)
manifest.time += inc_time
manifest.save(path)
def generate_manifest(package_folder):
# Create the digest for the package
digest = FileTreeManifest.create(package_folder)
save(os.path.join(package_folder, CONAN_MANIFEST), str(digest))
def inc_recipe_manifest_timestamp(client_cache, conan_ref, inc_time):
conan_ref = ConanFileReference.loads(str(conan_ref))
path = client_cache.export(conan_ref)
manifest = FileTreeManifest.load(path)
manifest.time += inc_time
manifest.save(path)
