def fetch(self): # pylint: disable=arguments-differ
with self.tempdir() as tmpdir:
packages = self.ref.strip().split("\n")
package_dir = os.path.join(tmpdir, "packages")
os.makedirs(package_dir)
self.call(
[
*self.host_pip,
"download",
"--no-binary",
":all:",
"--index-url",
self.index_url,
"--dest",
package_dir,
*packages,
],
fail="Failed to install python packages: {}".format(packages),
)
# If the mirror directory already exists, assume that some other
# process has fetched the sources before us and ensure that we do
# not raise an error in that case.
try:
utils.move_atomic(package_dir, self._mirror)
except utils.DirectoryExistsError:
# Another process has beaten us and has fetched the sources
# before us.
pass
except OSError as e:
raise SourceError(
"{}: Failed to move downloaded pip packages from '{}' to '{}': {}".format(
self, package_dir, self._mirror, e
)
) from e
def test_move_to_existing_file(src, tmp_path):
dst = tmp_path.joinpath("dst")
with dst.open("w") as fp:
fp.write("error")
with pytest.raises(NotADirectoryError):
move_atomic(src, dst)
def test_move_to_existing_non_empty_dir(src, tmp_path):
dst = tmp_path.joinpath("dst")
dst.mkdir()
with dst.joinpath("existing").open("w") as fp:
fp.write("already there")
with pytest.raises(DirectoryExistsError):
move_atomic(src, dst)
def test_move_file_to_existing_file(tmp_path):
dst = tmp_path.joinpath("dst")
src = tmp_path.joinpath("src")
with src.open("w") as fp:
fp.write("src")
with dst.open("w") as fp:
fp.write("dst")
move_atomic(src, dst)
with dst.open() as fp:
assert fp.read() == "src"
def test_move_to_empty_dir_set_mtime(src, tmp_path):
dst = tmp_path.joinpath("dst")
move_atomic(src, dst)
assert dst.joinpath("test").exists()
_dst = str(dst)
# set the mtime via stamp
timestamp1 = "2020-01-08T11:05:50.832123Z"
_set_file_mtime(_dst, _parse_timestamp(timestamp1))
assert timestamp1 == _get_file_mtimestamp(_dst)
# reset the mtime using an offset stamp
timestamp2 = "2010-02-12T12:05:50.832123+01:00"
_set_file_mtime(_dst, _parse_timestamp(timestamp2))
assert _get_file_mtimestamp(_dst) == "2010-02-12T11:05:50.832123Z"
def _ensure_repo(self):
if not os.path.exists(self.mirror):
with self.source.tempdir() as tmpdir:
self.source.call(
[self.source.host_git, "init", "--bare", tmpdir],
fail="Failed to initialise repository",
)
try:
utils.move_atomic(tmpdir, self.mirror)
except DirectoryExistsError:
# Another process was quicker to download this repository.
# Let's discard our own
self.source.status(
"{}: Discarding duplicate repository".format(
self.source))
except OSError as e:
raise SourceError(
"{}: Failed to move created repository from '{}' to '{}': {}"
.format(self.source, tmpdir, self.mirror, e)) from e
def test_move_to_empty_dir_set_mtime(src, tmp_path):
# Skip this test if we do not have support for subsecond precision mtimes
#
if not have_subsecond_mtime(str(tmp_path)):
pytest.skip(
"Filesystem does not support subsecond mtime precision: {}".format(
str(tmp_path)))
dst = tmp_path.joinpath("dst")
move_atomic(src, dst)
assert dst.joinpath("test").exists()
_dst = str(dst)
# set the mtime via stamp
timestamp1 = "2020-01-08T11:05:50.832123Z"
_set_file_mtime(_dst, _parse_timestamp(timestamp1))
assert timestamp1 == _get_file_mtimestamp(_dst)
# reset the mtime using an offset stamp
timestamp2 = "2010-02-12T12:05:50.832123+01:00"
_set_file_mtime(_dst, _parse_timestamp(timestamp2))
assert _get_file_mtimestamp(_dst) == "2010-02-12T11:05:50.832123Z"
def test_move_to_existing_empty_dir(src, tmp_path):
dst = tmp_path.joinpath("dst")
dst.mkdir()
move_atomic(src, dst)
assert dst.joinpath("test").exists()
def test_move_non_existing_dir(tmp_path):
dst = tmp_path.joinpath("dst")
src = tmp_path.joinpath("src")
with pytest.raises(FileNotFoundError):
move_atomic(src, dst)
def test_move_to_empty_dir_no_create_parents(src, tmp_path):
dst = tmp_path.joinpath("nested/dst")
with pytest.raises(FileNotFoundError):
move_atomic(src, dst, ensure_parents=False)
def test_move_to_empty_dir_create_parents(src, tmp_path):
dst = tmp_path.joinpath("nested/dst")
move_atomic(src, dst)
assert dst.joinpath("test").exists()
def fetch(self):
# pylint: disable=arguments-differ
with self.timed_activity(
"Fetching image {}:{} with digest {}".format(
self.image, self.tag, self.digest),
silent_nested=True,
):
with self.tempdir() as tmpdir:
# move all files to a tmpdir
try:
manifest = self._load_manifest()
except FileNotFoundError as e:
try:
manifest_text, digest = self.client.manifest(
self.image, self.digest)
except requests.RequestException as ee:
raise SourceError(ee) from ee
if digest != self.digest:
raise SourceError(
"Requested image {}, got manifest with digest {}".
format(self.digest, digest)) from e
self._save_manifest(manifest_text, tmpdir)
manifest = json.loads(manifest_text)
except DockerManifestError as e:
self.log("Unexpected manifest", detail=e.manifest)
raise
except (OSError, requests.RequestException) as e:
raise SourceError(e) from e
for layer in manifest["layers"]:
if (layer["mediaType"] !=
"application/vnd.docker.image.rootfs.diff.tar.gzip"
):
raise SourceError("Unsupported layer type: {}".format(
layer["mediaType"]))
layer_digest = layer["digest"]
blob_path = os.path.join(tmpdir, layer_digest + ".tar.gz")
if not os.path.exists(blob_path):
try:
self.client.blob(self.image,
layer_digest,
download_to=blob_path)
except (OSError, requests.RequestException) as e:
if os.path.exists(blob_path):
shutil.rmtree(blob_path)
raise SourceError(e) from e
self._verify_blob(blob_path, expected_digest=layer_digest)
# Only if all sources are successfully fetched, move files to staging directory
# As both the manifest and blobs are content addressable, we can optimize space by having
# a flat mirror directory. We check one-by-one if there is any need to copy a file out of the tmpdir.
for fetched_file in os.listdir(tmpdir):
move_atomic(
os.path.join(tmpdir, fetched_file),
os.path.join(self.get_mirror_directory(),
fetched_file),
)
