def test_update_artifact(tmpdir, files): sharedir = os.path.join(str(tmpdir), "share") with create_artifact_share(sharedir, casd=True) as share: # put files object if files == "present": directory = re_pb2.Directory() digest = share.cas.add_object(buffer=directory.SerializeToString()) elif files == "invalid": digest = share.cas.add_object( buffer="abcdefghijklmnop".encode("utf-8")) elif files == "absent": digest = utils._message_digest("abcdefghijklmnop".encode("utf-8")) url = urlparse(share.repo) with grpc.insecure_channel("{}:{}".format(url.hostname, url.port)) as channel: artifact_stub = ArtifactServiceStub(channel) # initialise an artifact artifact = Artifact() artifact.version = 0 artifact.build_success = True artifact.strong_key = "abcdefghijklmnop" artifact.files.hash = "hashymchashash" artifact.files.size_bytes = 10 artifact.files.CopyFrom(digest) # Put it in the artifact share with an UpdateArtifactRequest request = UpdateArtifactRequest() request.artifact.CopyFrom(artifact) request.cache_key = "a-cache-key" # should return the same artifact back if files == "present": response = artifact_stub.UpdateArtifact(request) assert response == artifact else: try: artifact_stub.UpdateArtifact(request) except grpc.RpcError as e: assert e.code() == grpc.StatusCode.FAILED_PRECONDITION if files == "absent": assert e.details( ) == "Artifact files specified but no files found" elif files == "invalid": assert e.details( ) == "Artifact files specified but directory not found" return # If we uploaded the artifact check GetArtifact request = GetArtifactRequest() request.cache_key = "a-cache-key" response = artifact_stub.GetArtifact(request) assert response == artifact
def _reachable_refs_dir(self, reachable, tree): if tree.hash in reachable: return reachable.add(tree.hash) directory = remote_execution_pb2.Directory() with open(self.cas.objpath(tree), "rb") as f: directory.ParseFromString(f.read()) for filenode in directory.files: if not os.path.exists(self.cas.objpath(filenode.digest)): raise FileNotFoundError reachable.add(filenode.digest.hash) for dirnode in directory.directories: self._reachable_refs_dir(reachable, dirnode.digest)
def test_pull_tree(cli, tmpdir, datafiles): project_dir = str(datafiles) # Set up an artifact cache. with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share: # Configure artifact share rootcache_dir = os.path.join(str(tmpdir), "cache") user_config_file = str(tmpdir.join("buildstream.conf")) user_config = { "scheduler": { "pushers": 1 }, "artifacts": { "url": share.repo, "push": True, }, "cachedir": rootcache_dir, } # Write down the user configuration file _yaml.roundtrip_dump(user_config, file=user_config_file) # Ensure CLI calls will use it cli.configure(user_config) # First build the project with the artifact cache configured result = cli.run(project=project_dir, args=["build", "target.bst"]) result.assert_success() # Assert that we are now cached locally assert cli.get_element_state(project_dir, "target.bst") == "cached" # Assert that we shared/pushed the cached artifact assert share.get_artifact( cli.get_artifact_name(project_dir, "test", "target.bst")) with dummy_context(config=user_config_file) as context: # Load the project and CAS cache project = Project(project_dir, context) project.ensure_fully_loaded() cas = context.get_cascache() # Assert that the element's artifact is cached element = project.load_elements(["target.bst"])[0] element_key = cli.get_element_key(project_dir, "target.bst") assert cli.artifact.is_cached(rootcache_dir, element, element_key) # Retrieve the Directory object from the cached artifact artifact_digest = cli.artifact.get_digest(rootcache_dir, element, element_key) artifactcache = context.artifactcache # Manually setup the CAS remote artifactcache.setup_remotes(use_config=True) assert artifactcache.has_push_remotes() directory = remote_execution_pb2.Directory() with open(cas.objpath(artifact_digest), "rb") as f: directory.ParseFromString(f.read()) # Build the Tree object while we are still cached tree = remote_execution_pb2.Tree() tree_maker(cas, tree, directory) # Push the Tree as a regular message tree_digest = artifactcache.push_message(project, tree) tree_hash, tree_size = tree_digest.hash, tree_digest.size_bytes assert tree_hash and tree_size # Now delete the artifact locally cli.remove_artifact_from_cache(project_dir, "target.bst") # Assert that we are not cached locally anymore artifactcache.close_grpc_channels() cas.close_grpc_channels() assert cli.get_element_state(project_dir, "target.bst") != "cached" tree_digest = remote_execution_pb2.Digest(hash=tree_hash, size_bytes=tree_size) # Pull the artifact using the Tree object directory_digest = artifactcache.pull_tree(project, artifact_digest) directory_hash, directory_size = directory_digest.hash, directory_digest.size_bytes # Directory size now zero with AaaP and stack element commit #1cbc5e63dc assert directory_hash and not directory_size directory_digest = remote_execution_pb2.Digest( hash=directory_hash, size_bytes=directory_size) # Ensure the entire Tree stucture has been pulled assert os.path.exists(cas.objpath(directory_digest))