def upload_package(self, package_reference, the_files): """ basedir: Base directory with the files to upload (for read the files in disk) relative_files: relative paths to upload """ self.check_credentials() # Get the remote snapshot remote_snapshot = self._get_package_snapshot(package_reference) local_snapshot = {filename: md5(content) for filename, content in the_files.items()} # Get the diff new, modified, deleted = diff_snapshots(local_snapshot, remote_snapshot) files_to_upload = {filename: the_files[filename] for filename in new + modified} if files_to_upload: # Obtain upload urls url = "%s/conans/%s/packages/%s/upload_urls" % (self._remote_api_url, "/".join(package_reference.conan), package_reference.package_id) filesizes = {filename: len(content) for filename, content in files_to_upload.items()} self._output.rewrite_line("Requesting upload permissions...") urls = self._get_json(url, data=filesizes) self._output.rewrite_line("Requesting upload permissions...Done!") self._output.writeln("") self.upload_files(urls, files_to_upload, self._output) else: self._output.rewrite_line("Package is up to date.") self._output.writeln("") if deleted: self.remove_package_files(package_reference, deleted)
def _upload_recipe(self, ref, base_files=None, retry=1, retry_wait=0): files = {"conanfile.py": GenConanfile("3").with_requires("1", "12")} if base_files: files.update(base_files) content = """ from conans import ConanFile class MyConan(ConanFile): name = "%s" version = "%s" settings = arch, compiler, os """ % (ref.name, ref.version) files[CONANFILE] = content files_md5s = {filename: md5(content) for filename, content in files.items()} conan_digest = FileTreeManifest(123123123, files_md5s) tmp_dir = temp_folder() abs_paths = {} for filename, content in files.items(): abs_path = os.path.join(tmp_dir, filename) save(abs_path, content) abs_paths[filename] = abs_path abs_paths[CONAN_MANIFEST] = os.path.join(tmp_dir, CONAN_MANIFEST) conan_digest.save(tmp_dir) self.api.upload_recipe(ref, abs_paths, None, retry, retry_wait)
def _upload_recipe(self, conan_reference, base_files=None, retry=1, retry_wait=0): files = hello_source_files(3, [1, 12]) if base_files: files.update(base_files) content = """ from conans import ConanFile class MyConan(ConanFile): name = "%s" version = "%s" settings = arch, compiler, os """ % (conan_reference.name, conan_reference.version) files[CONANFILE] = content files_md5s = {filename: md5(content) for filename, content in files.items()} conan_digest = FileTreeManifest(123123123, files_md5s) files[CONAN_MANIFEST] = str(conan_digest) tmp_dir = temp_folder() abs_paths = {} for filename, content in files.items(): abs_path = os.path.join(tmp_dir, filename) save(abs_path, content) abs_paths[filename] = abs_path self.api.upload_recipe(conan_reference, abs_paths, retry, retry_wait, False)
def _upload_recipe(self, conan_reference, base_files=None, retry=1, retry_wait=0): files = hello_source_files(3, [1, 12]) if base_files: files.update(base_files) content = """ from conans import ConanFile class MyConan(ConanFile): name = "%s" version = "%s" settings = arch, compiler, os """ % (conan_reference.name, conan_reference.version) files[CONANFILE] = content files_md5s = { filename: md5(content) for filename, content in files.items() } conan_digest = FileTreeManifest(123123123, files_md5s) tmp_dir = temp_folder() abs_paths = {} for filename, content in files.items(): abs_path = os.path.join(tmp_dir, filename) save(abs_path, content) abs_paths[filename] = abs_path abs_paths[CONAN_MANIFEST] = os.path.join(tmp_dir, CONAN_MANIFEST) conan_digest.save(tmp_dir) self.api.upload_recipe(conan_reference, abs_paths, retry, retry_wait, False, None)
def test_tree_manifest(self): tmp_dir = temp_folder() files = {"one.ext": "aalakjshdlkjahsdlkjahsdljkhsadljkhasljkdhlkjashd", "path/to/two.txt": "asdas13123", "two.txt": "asdasdasdasdasdasd", "folder/damn.pyc": "binarythings", "folder/damn.pyo": "binarythings2", "pythonfile.pyc": "binarythings3"} for filename, content in files.items(): save(os.path.join(tmp_dir, filename), content) manifest = FileTreeManifest.create(tmp_dir) save(os.path.join(tmp_dir, "THEMANIFEST.txt"), str(manifest)) readed_manifest = FileTreeManifest.loads(load(os.path.join(tmp_dir, "THEMANIFEST.txt"))) self.assertEqual(readed_manifest.time, manifest.time) self.assertEqual(readed_manifest, manifest) # Not included the pycs or pyo self.assertEquals(set(manifest.file_sums.keys()), set(["one.ext", "path/to/two.txt", "two.txt"])) for filepath, md5readed in manifest.file_sums.items(): content = files[filepath] self.assertEquals(md5(content), md5readed)
def test_tree_manifest(self): tmp_dir = temp_folder() files = { "one.ext": "aalakjshdlkjahsdlkjahsdljkhsadljkhasljkdhlkjashd", "path/to/two.txt": "asdas13123", "two.txt": "asdasdasdasdasdasd", "folder/damn.pyc": "binarythings", "folder/damn.pyo": "binarythings2", "pythonfile.pyc": "binarythings3" } for filename, content in files.items(): save(os.path.join(tmp_dir, filename), content) manifest = FileTreeManifest.create(tmp_dir) manifest.save(tmp_dir) readed_manifest = FileTreeManifest.load(tmp_dir) self.assertEqual(readed_manifest.time, manifest.time) self.assertEqual(readed_manifest, manifest) # Not included the pycs or pyo self.assertEqual(set(manifest.file_sums.keys()), set(["one.ext", "path/to/two.txt", "two.txt"])) for filepath, md5readed in manifest.file_sums.items(): content = files[filepath] self.assertEqual(md5(content), md5readed)
def upload_conan(self, conan_reference, the_files): """ the_files: dict with relative_path: content """ self.check_credentials() # Get the remote snapshot remote_snapshot = self._get_conan_snapshot(conan_reference) local_snapshot = { filename: md5(content) for filename, content in the_files.iteritems() } # Get the diff new, modified, deleted = diff_snapshots(local_snapshot, remote_snapshot) files_to_upload = { filename.replace("\\", "/"): the_files[filename] for filename in new + modified } if files_to_upload: # Get the upload urls url = "%s/conans/%s/upload_urls" % (self._remote_api_url, "/".join(conan_reference)) filesizes = { filename.replace("\\", "/"): len(content) for filename, content in files_to_upload.iteritems() } urls = self._get_json(url, data=filesizes) self.upload_files(urls, files_to_upload, self._output) if deleted: self.remove_conanfile_files(conan_reference, deleted)
def _failed_package_verify(self, reference, remote="local cache"): self.client.run("install %s --build missing --manifests" % str(reference)) self.assertIn( "Installed manifest for 'Hello/0.1@lasote/stable' from %s" % remote, self.client.user_io.out) self.assertIn( "Installed manifest for 'Hello/0.1@lasote/stable:" "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9' from %s" % remote, self.client.user_io.out) output_folder = os.path.join(self.client.current_folder, ".conan_manifests") paths = SimplePaths(output_folder) self.assertTrue( os.path.exists( os.path.join(paths.export(self.reference), CONAN_MANIFEST))) package_reference = PackageReference.loads( "Hello/0.1@lasote/stable:" "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9") self.assertTrue( os.path.exists( os.path.join(paths.package(package_reference), CONAN_MANIFEST))) client = TestClient(servers=self.servers, users={"default": [("lasote", "mypass")]}) client.save(self.files) client.run("export . lasote/stable") client.run("install Hello/0.1@lasote/stable --build=missing") info = os.path.join(client.paths.package(package_reference), "conaninfo.txt") info_content = load(info) info_content += "# Dummy string" save(info, info_content) package_folder = client.paths.package(package_reference) manifest = FileTreeManifest.load(package_folder) manifest.file_sums["conaninfo.txt"] = md5(info_content) manifest.save(package_folder) client.run("upload %s --all" % str(self.reference)) # now verify, with update self.client.run("remove Hello/0.1@lasote/stable -f") self.client.run("install %s --build missing --verify" % str(self.reference), ignore_error=True) self.assertNotIn("Manifest for 'Hello/0.1@lasote/stable': OK", self.client.user_io.out) self.assertNotIn("Manifest for '%s': OK" % str(package_reference), self.client.user_io.out) self.assertIn( "Modified or new manifest '%s' detected" % str(package_reference), self.client.user_io.out)
def _failed_package_verify(self, reference, remote="local cache"): self.client.run("install %s --build missing --manifests" % str(reference)) self.assertIn( "Installed manifest for 'Hello/0.1@lasote/stable' from %s" % remote, self.client.user_io.out) self.assertIn( "Installed manifest for 'Hello/0.1@lasote/stable:" "%s' from %s" % (NO_SETTINGS_PACKAGE_ID, remote), self.client.user_io.out) output_folder = os.path.join(self.client.current_folder, ".conan_manifests") self.assertTrue( os.path.exists( os.path.join(export_folder(output_folder, self.ref), CONAN_MANIFEST))) pref = PackageReference.loads("Hello/0.1@lasote/stable: %s" % NO_SETTINGS_PACKAGE_ID) self.assertTrue( os.path.exists( os.path.join(package_folder(output_folder, pref), CONAN_MANIFEST))) client = TestClient(servers=self.servers, users={"default": [("lasote", "mypass")]}) client.save(self.files) client.run("export . lasote/stable") client.run("install Hello/0.1@lasote/stable --build=missing") package_folder_path = client.cache.package_layout( pref.ref).package(pref) info = os.path.join(package_folder_path, "conaninfo.txt") info_content = load(info) info_content += "# Dummy string" save(info, info_content) manifest = FileTreeManifest.load(package_folder_path) manifest.file_sums["conaninfo.txt"] = md5(info_content) manifest.save(package_folder_path) client.run("upload %s --all" % str(self.ref)) # now verify, with update self.client.run("remove Hello/0.1@lasote/stable -f") self.client.run("install %s --build missing --verify" % str(self.ref), assert_error=True) self.assertNotIn("Manifest for 'Hello/0.1@lasote/stable': OK", self.client.user_io.out) self.assertNotIn("Manifest for '%s': OK" % str(pref), self.client.user_io.out) self.assertIn("Modified or new manifest '%s' detected" % str(pref), self.client.user_io.out)
def test_tree_manifest(self): tmp_dir = temp_folder() files = {"one.ext": "aalakjshdlkjahsdlkjahsdljkhsadljkhasljkdhlkjashd", "path/to/two.txt": "asdas13123", "two.txt": "asdasdasdasdasdasd"} for filename, content in files.iteritems(): save(os.path.join(tmp_dir, filename), content) manifest = FileTreeManifest.create(tmp_dir) save(os.path.join(tmp_dir, "THEMANIFEST.txt"), str(manifest)) readed_manifest = FileTreeManifest.loads(load(os.path.join(tmp_dir, "THEMANIFEST.txt"))) self.assertEquals(readed_manifest, manifest) for filepath, md5readed in manifest.file_sums.iteritems(): content = files[filepath] self.assertEquals(md5(content), md5readed)
def _upload_conan(self, conan_reference, base_files=None): files = hello_source_files(3, [1, 12]) if base_files: files.update(base_files) content = """ from conans import ConanFile class MyConan(ConanFile): name = "%s" version = "%s" settings = arch, compiler, os """ % (conan_reference.name, conan_reference.version) files[CONANFILE] = content files_md5s = {filename: md5(content) for filename, content in files.iteritems()} conan_digest = FileTreeManifest(123123123, files_md5s) files[CONAN_MANIFEST] = str(conan_digest) self.api.upload_conan(conan_reference, files)
def _failed_package_verify(self, reference, remote="local cache"): self.client.run("install %s --build missing --manifests" % str(reference)) self.assertIn("Installed manifest for 'Hello/0.1@lasote/stable' from %s" % remote, self.client.user_io.out) self.assertIn("Installed manifest for 'Hello/0.1@lasote/stable:" "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9' from %s" % remote, self.client.user_io.out) output_folder = os.path.join(self.client.current_folder, ".conan_manifests") paths = SimplePaths(output_folder) self.assertTrue(os.path.exists(paths.digestfile_conanfile(self.reference))) package_reference = PackageReference.loads("Hello/0.1@lasote/stable:" "5ab84d6acfe1f23c4fae0ab88f26e3a396351ac9") self.assertTrue(os.path.exists(paths.digestfile_package(package_reference))) client = TestClient(servers=self.servers, users={"default": [("lasote", "mypass")]}) client.save(self.files) client.run("export lasote/stable") client.run("install Hello/0.1@lasote/stable --build=missing") info = os.path.join(client.paths.package(package_reference), "conaninfo.txt") info_content = load(info) info_content += "# Dummy string" save(info, info_content) manifest = client.paths.load_package_manifest(package_reference) manifest.file_sums["conaninfo.txt"] = md5(info_content) save(client.paths.digestfile_package(package_reference), str(manifest)) manifest = client.paths.load_package_manifest(package_reference) client.run("upload %s --all" % str(self.reference)) # now verify, with update self.client.run("remove Hello/0.1@lasote/stable -f") self.client.run("install %s --build missing --verify" % str(self.reference), ignore_error=True) self.assertNotIn("Manifest for 'Hello/0.1@lasote/stable': OK", self.client.user_io.out) self.assertNotIn("Manifest for '%s': OK" % str(package_reference), self.client.user_io.out) self.assertIn("Modified or new manifest '%s' detected" % str(package_reference), self.client.user_io.out)
def _upload_conan(self, conan_reference, base_files=None): files = hello_source_files(3, [1, 12]) if base_files: files.update(base_files) content = """ from conans import ConanFile class MyConan(ConanFile): name = "%s" version = "%s" settings = arch, compiler, os """ % (conan_reference.name, conan_reference.version) files[CONANFILE] = content files_md5s = { filename: md5(content) for filename, content in files.items() } conan_digest = FileTreeManifest(123123123, files_md5s) files[CONAN_MANIFEST] = str(conan_digest) self.api.upload_conan(conan_reference, files)
def upload_conan(self, conan_reference, the_files): """ the_files: dict with relative_path: content """ self.check_credentials() # Get the remote snapshot remote_snapshot = self._get_conan_snapshot(conan_reference) local_snapshot = {filename: md5(content) for filename, content in the_files.items()} # Get the diff new, modified, deleted = diff_snapshots(local_snapshot, remote_snapshot) files_to_upload = {filename.replace("\\", "/"): the_files[filename] for filename in new + modified} if files_to_upload: # Get the upload urls url = "%s/conans/%s/upload_urls" % (self._remote_api_url, "/".join(conan_reference)) filesizes = {filename.replace("\\", "/"): len(content) for filename, content in files_to_upload.items()} urls = self._get_json(url, data=filesizes) self.upload_files(urls, files_to_upload, self._output) if deleted: self.remove_conanfile_files(conan_reference, deleted)
def summary_hash(self): s = ["%s: %s" % (f, fmd5) for f, fmd5 in sorted(self.file_sums.items())] s.append("") return md5("\n".join(s))
def summary_hash(self): ret = "" # Do not include the timestamp in the summary hash for filepath, file_md5 in self.file_sums.items(): ret += "\n%s: %s" % (filepath, file_md5) return md5(ret)
def summary_hash(self): ret = "" # Do not include the timestamp in the summary hash for filepath, file_md5 in sorted(self.file_sums.items()): ret += "%s: %s\n" % (filepath, file_md5) return md5(ret)
def md5_test(self): result = md5(u"äüïöñç") self.assertEqual("dfcc3d74aa447280a7ecfdb98da55174", result)