def test_build_shell_fetch(cli, datafiles):
project = str(datafiles)
element_name = "build-shell-fetch.bst"
# Create a file with unique contents such that it cannot be in the cache already
test_filepath = os.path.join(project, "files", "hello.txt")
test_message = "Hello World! {}".format(uuid.uuid4())
with open(test_filepath, "w") as f:
f.write(test_message)
checksum = utils.sha256sum(test_filepath)
# Create an element that has this unique file as a source
element = {
"kind": "manual",
"depends": ["base.bst"],
"sources": [{"kind": "remote", "url": "project_dir:/files/hello.txt", "ref": checksum}],
}
_yaml.roundtrip_dump(element, os.path.join(project, "elements", element_name))
# Ensure our dependencies are cached
result = cli.run(project=project, args=["build", "base.bst"])
result.assert_success()
# Ensure our sources are not cached
assert cli.get_element_state(project, element_name) == "fetch needed"
# Launching a shell should fetch any uncached sources
result = cli.run(project=project, args=["shell", "--build", element_name, "cat", "hello.txt"])
result.assert_success()
assert result.output == test_message
def test_integration_partial_artifact(cli, datafiles, tmpdir,
integration_cache):
project = str(datafiles)
element_name = "autotools/amhello.bst"
# push to an artifact server so we can pull from it later.
with create_artifact_share(os.path.join(str(tmpdir),
"artifactshare")) as share:
cli.configure(
{"artifacts": {
"servers": [{
"url": share.repo,
"push": True
}]
}})
result = cli.run(project=project, args=["build", element_name])
result.assert_success()
# If the build is cached then it might not push to the artifact cache
result = cli.run(project=project,
args=["artifact", "push", element_name])
result.assert_success()
result = cli.run(project=project, args=["shell", element_name])
result.assert_success()
# do a checkout and get the digest of the hello binary.
result = cli.run(
project=project,
args=[
"artifact",
"checkout",
"--deps",
"none",
"--directory",
os.path.join(str(tmpdir), "tmp"),
"autotools/amhello.bst",
],
)
result.assert_success()
digest = utils.sha256sum(
os.path.join(str(tmpdir), "tmp", "usr", "bin", "hello"))
# Remove the binary from the CAS
cachedir = cli.config["cachedir"]
objpath = os.path.join(cachedir, "cas", "objects", digest[:2],
digest[2:])
os.unlink(objpath)
# check shell doesn't work
result = cli.run(project=project,
args=["shell", element_name, "--", "hello"])
result.assert_main_error(ErrorDomain.APP, "shell-missing-deps")
# check the artifact gets completed with '--pull' specified
result = cli.run(project=project,
args=["shell", "--pull", element_name, "--", "hello"])
result.assert_success()
assert "autotools/amhello.bst" in result.get_pulled_elements()
def test_partial_artifact_checkout_fetch(cli, datafiles, tmpdir):
project = str(datafiles)
checkout_dir = os.path.join(str(tmpdir), "checkout")
repo = create_repo("git", str(tmpdir))
repo.create(os.path.join(str(datafiles), "files"))
element_dir = os.path.join(str(tmpdir), "elements")
project = str(tmpdir)
project_config = {
"name": "partial-artifact-checkout-fetch",
"min-version": "2.0",
"element-path": "elements",
}
project_file = os.path.join(str(tmpdir), "project.conf")
_yaml.roundtrip_dump(project_config, project_file)
input_config = {
"kind": "import",
"sources": [repo.source_config()],
}
input_name = "input.bst"
input_file = os.path.join(element_dir, input_name)
_yaml.roundtrip_dump(input_config, input_file)
with create_artifact_share(os.path.join(str(tmpdir),
"artifactshare")) as share:
cli.configure({"artifacts": {"url": share.repo, "push": True}})
result = cli.run(project=project, args=["source", "track", input_name])
result.assert_success()
result = cli.run(project=project, args=["build", input_name])
result.assert_success()
# A push artifact cache means we have to pull to push to them, so
# delete some blobs from that CAS such that we have to fetch
digest = utils.sha256sum(
os.path.join(project, "files", "bin-files", "usr", "bin", "hello"))
objpath = os.path.join(cli.directory, "cas", "objects", digest[:2],
digest[2:])
os.unlink(objpath)
# Verify that the build-only dependency is not (complete) in the local cache
result = cli.run(project=project,
args=[
"artifact", "checkout", input_name, "--directory",
checkout_dir
])
result.assert_main_error(ErrorDomain.STREAM,
"uncached-checkout-attempt")
# Verify that the pull method fetches relevant artifacts in order to stage
result = cli.run(project=project,
args=[
"artifact", "checkout", "--pull", input_name,
"--directory", checkout_dir
])
result.assert_success()
# should have pulled whatever was deleted previous
assert input_name in result.get_pulled_elements()
def test_pull_missing_local_blob(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
repo = create_repo("git", str(tmpdir))
repo.create(os.path.join(str(datafiles), "files"))
element_dir = os.path.join(str(tmpdir), "elements")
project = str(tmpdir)
project_config = {
"name": "pull-missing-local-blob",
"min-version": "2.0",
"element-path": "elements",
}
project_file = os.path.join(str(tmpdir), "project.conf")
_yaml.roundtrip_dump(project_config, project_file)
input_config = {
"kind": "import",
"sources": [repo.source_config()],
}
input_name = "input.bst"
input_file = os.path.join(element_dir, input_name)
_yaml.roundtrip_dump(input_config, input_file)
depends_name = "depends.bst"
depends_config = {"kind": "stack", "depends": [input_name]}
depends_file = os.path.join(element_dir, depends_name)
_yaml.roundtrip_dump(depends_config, depends_file)
with create_artifact_share(os.path.join(str(tmpdir),
"artifactshare")) as share:
# First build the import-bin element and push to the remote.
cli.configure(
{"artifacts": {
"servers": [{
"url": share.repo,
"push": True
}]
}})
result = cli.run(project=project, args=["source", "track", input_name])
result.assert_success()
result = cli.run(project=project, args=["build", input_name])
result.assert_success()
assert cli.get_element_state(project, input_name) == "cached"
# Delete a file blob from the local cache.
# This is a placeholder to test partial CAS handling until we support
# partial artifact pulling (or blob-based CAS expiry).
#
digest = utils.sha256sum(
os.path.join(project, "files", "bin-files", "usr", "bin", "hello"))
objpath = os.path.join(cli.directory, "cas", "objects", digest[:2],
digest[2:])
os.unlink(objpath)
# Now try bst build
result = cli.run(project=project, args=["build", depends_name])
result.assert_success()
# Assert that the import-bin artifact was pulled (completing the partial artifact)
assert result.get_pulled_elements() == [input_name]
def _download(self, url):
try:
with self.cargo.tempdir() as td:
default_name = os.path.basename(url)
request = urllib.request.Request(url)
request.add_header('Accept', '*/*')
# We do not use etag in case what we have in cache is
# not matching ref in order to be able to recover from
# corrupted download.
if self.sha:
etag = self._get_etag(self.sha)
if etag and self.get_consistency() == Consistency.CACHED:
request.add_header('If-None-Match', etag)
with contextlib.closing(
urllib.request.urlopen(request)) as response:
info = response.info()
etag = info['ETag'] if 'ETag' in info else None
filename = info.get_filename(default_name)
filename = os.path.basename(filename)
local_file = os.path.join(td, filename)
with open(local_file, 'wb') as dest:
shutil.copyfileobj(response, dest)
# Make sure url-specific mirror dir exists.
os.makedirs(self._get_mirror_dir(), exist_ok=True)
# Store by sha256sum
sha256 = utils.sha256sum(local_file)
# Even if the file already exists, move the new file over.
# In case the old file was corrupted somehow.
os.rename(local_file, self._get_mirror_file(sha256))
if etag:
self._store_etag(sha256, etag)
return sha256
except urllib.error.HTTPError as e:
if e.code == 304:
# 304 Not Modified.
# Because we use etag only for matching sha, currently specified sha is what
# we would have downloaded.
return self.sha
raise SourceError("{}: Error mirroring {}: {}".format(
self, url, e),
temporary=True) from e
except (urllib.error.URLError, urllib.error.ContentTooShortError,
OSError) as e:
raise SourceError("{}: Error mirroring {}: {}".format(
self, url, e),
temporary=True) from e
def create(self, directory):
tarball = os.path.join(self.repo, "file.tar.gz")
old_dir = os.getcwd()
os.chdir(directory)
with tarfile.open(tarball, "w:gz") as tar:
tar.add(".")
os.chdir(old_dir)
return sha256sum(tarball)
def generate_remote_import_element(input_path, output_path):
return {
"kind":
"import",
"sources": [{
"kind": "remote",
"url": "file://{}".format(input_path),
"filename": output_path,
"ref": utils.sha256sum(input_path),
}],
}
def create(self, directory):
archive = os.path.join(self.repo, "file.zip")
old_dir = os.getcwd()
os.chdir(directory)
with zipfile.ZipFile(archive, "w") as zipfp:
for root, dirs, files in os.walk("."):
names = dirs + files
names = [os.path.join(root, name) for name in names]
for name in names:
zipfp.write(name)
os.chdir(old_dir)
return sha256sum(archive)
def get_unique_key(self):
return [self.path, utils.sha256sum(self.fullpath), self.strip_level]
def _ensure_mirror(self):
# Downloads from the url and caches it according to its sha256sum.
try:
with self.tempdir() as td:
default_name = os.path.basename(self.url)
request = urllib.request.Request(self.url)
request.add_header("Accept", "*/*")
# We do not use etag in case what we have in cache is
# not matching ref in order to be able to recover from
# corrupted download.
if self.ref:
etag = self._get_etag(self.ref)
# Do not re-download the file if the ETag matches.
if etag and self.is_cached():
request.add_header("If-None-Match", etag)
opener = self.__get_urlopener()
with contextlib.closing(opener.open(request)) as response:
info = response.info()
etag = info["ETag"] if "ETag" in info else None
filename = info.get_filename(default_name)
filename = os.path.basename(filename)
local_file = os.path.join(td, filename)
with open(local_file, "wb") as dest:
shutil.copyfileobj(response, dest)
# Make sure url-specific mirror dir exists.
if not os.path.isdir(self._mirror_dir):
os.makedirs(self._mirror_dir)
# Store by sha256sum
sha256 = utils.sha256sum(local_file)
# Even if the file already exists, move the new file over.
# In case the old file was corrupted somehow.
os.rename(local_file, self._get_mirror_file(sha256))
if etag:
self._store_etag(sha256, etag)
return sha256
except urllib.error.HTTPError as e:
if e.code == 304:
# 304 Not Modified.
# Because we use etag only for matching ref, currently specified ref is what
# we would have downloaded.
return self.ref
raise SourceError(
"{}: Error mirroring {}: {}".format(self, self.url, e),
temporary=True,
) from e
except (
urllib.error.URLError,
urllib.error.ContentTooShortError,
OSError,
ValueError,
) as e:
# Note that urllib.request.Request in the try block may throw a
# ValueError for unknown url types, so we handle it here.
raise SourceError(
"{}: Error mirroring {}: {}".format(self, self.url, e),
temporary=True,
) from e
def _verify_blob(path, expected_digest):
blob_digest = "sha256:" + sha256sum(path)
if expected_digest != blob_digest:
raise SourceError(
"Blob {} is corrupt; got content hash of {}.".format(
path, blob_digest))
