def fetch(self): # pylint: disable=arguments-differ with self.tempdir() as tmpdir: packages = self.ref.strip().split("\n") package_dir = os.path.join(tmpdir, "packages") os.makedirs(package_dir) self.call( [ *self.host_pip, "download", "--no-binary", ":all:", "--index-url", self.index_url, "--dest", package_dir, *packages, ], fail="Failed to install python packages: {}".format(packages), ) # If the mirror directory already exists, assume that some other # process has fetched the sources before us and ensure that we do # not raise an error in that case. try: utils.move_atomic(package_dir, self._mirror) except utils.DirectoryExistsError: # Another process has beaten us and has fetched the sources # before us. pass except OSError as e: raise SourceError( "{}: Failed to move downloaded pip packages from '{}' to '{}': {}".format( self, package_dir, self._mirror, e ) ) from e
def test_move_to_existing_file(src, tmp_path): dst = tmp_path.joinpath("dst") with dst.open("w") as fp: fp.write("error") with pytest.raises(NotADirectoryError): move_atomic(src, dst)
def test_move_to_existing_non_empty_dir(src, tmp_path): dst = tmp_path.joinpath("dst") dst.mkdir() with dst.joinpath("existing").open("w") as fp: fp.write("already there") with pytest.raises(DirectoryExistsError): move_atomic(src, dst)
def test_move_file_to_existing_file(tmp_path): dst = tmp_path.joinpath("dst") src = tmp_path.joinpath("src") with src.open("w") as fp: fp.write("src") with dst.open("w") as fp: fp.write("dst") move_atomic(src, dst) with dst.open() as fp: assert fp.read() == "src"
def test_move_to_empty_dir_set_mtime(src, tmp_path): dst = tmp_path.joinpath("dst") move_atomic(src, dst) assert dst.joinpath("test").exists() _dst = str(dst) # set the mtime via stamp timestamp1 = "2020-01-08T11:05:50.832123Z" _set_file_mtime(_dst, _parse_timestamp(timestamp1)) assert timestamp1 == _get_file_mtimestamp(_dst) # reset the mtime using an offset stamp timestamp2 = "2010-02-12T12:05:50.832123+01:00" _set_file_mtime(_dst, _parse_timestamp(timestamp2)) assert _get_file_mtimestamp(_dst) == "2010-02-12T11:05:50.832123Z"
def _ensure_repo(self): if not os.path.exists(self.mirror): with self.source.tempdir() as tmpdir: self.source.call( [self.source.host_git, "init", "--bare", tmpdir], fail="Failed to initialise repository", ) try: utils.move_atomic(tmpdir, self.mirror) except DirectoryExistsError: # Another process was quicker to download this repository. # Let's discard our own self.source.status( "{}: Discarding duplicate repository".format( self.source)) except OSError as e: raise SourceError( "{}: Failed to move created repository from '{}' to '{}': {}" .format(self.source, tmpdir, self.mirror, e)) from e
def test_move_to_empty_dir_set_mtime(src, tmp_path): # Skip this test if we do not have support for subsecond precision mtimes # if not have_subsecond_mtime(str(tmp_path)): pytest.skip( "Filesystem does not support subsecond mtime precision: {}".format( str(tmp_path))) dst = tmp_path.joinpath("dst") move_atomic(src, dst) assert dst.joinpath("test").exists() _dst = str(dst) # set the mtime via stamp timestamp1 = "2020-01-08T11:05:50.832123Z" _set_file_mtime(_dst, _parse_timestamp(timestamp1)) assert timestamp1 == _get_file_mtimestamp(_dst) # reset the mtime using an offset stamp timestamp2 = "2010-02-12T12:05:50.832123+01:00" _set_file_mtime(_dst, _parse_timestamp(timestamp2)) assert _get_file_mtimestamp(_dst) == "2010-02-12T11:05:50.832123Z"
def test_move_to_existing_empty_dir(src, tmp_path): dst = tmp_path.joinpath("dst") dst.mkdir() move_atomic(src, dst) assert dst.joinpath("test").exists()
def test_move_non_existing_dir(tmp_path): dst = tmp_path.joinpath("dst") src = tmp_path.joinpath("src") with pytest.raises(FileNotFoundError): move_atomic(src, dst)
def test_move_to_empty_dir_no_create_parents(src, tmp_path): dst = tmp_path.joinpath("nested/dst") with pytest.raises(FileNotFoundError): move_atomic(src, dst, ensure_parents=False)
def test_move_to_empty_dir_create_parents(src, tmp_path): dst = tmp_path.joinpath("nested/dst") move_atomic(src, dst) assert dst.joinpath("test").exists()
def fetch(self): # pylint: disable=arguments-differ with self.timed_activity( "Fetching image {}:{} with digest {}".format( self.image, self.tag, self.digest), silent_nested=True, ): with self.tempdir() as tmpdir: # move all files to a tmpdir try: manifest = self._load_manifest() except FileNotFoundError as e: try: manifest_text, digest = self.client.manifest( self.image, self.digest) except requests.RequestException as ee: raise SourceError(ee) from ee if digest != self.digest: raise SourceError( "Requested image {}, got manifest with digest {}". format(self.digest, digest)) from e self._save_manifest(manifest_text, tmpdir) manifest = json.loads(manifest_text) except DockerManifestError as e: self.log("Unexpected manifest", detail=e.manifest) raise except (OSError, requests.RequestException) as e: raise SourceError(e) from e for layer in manifest["layers"]: if (layer["mediaType"] != "application/vnd.docker.image.rootfs.diff.tar.gzip" ): raise SourceError("Unsupported layer type: {}".format( layer["mediaType"])) layer_digest = layer["digest"] blob_path = os.path.join(tmpdir, layer_digest + ".tar.gz") if not os.path.exists(blob_path): try: self.client.blob(self.image, layer_digest, download_to=blob_path) except (OSError, requests.RequestException) as e: if os.path.exists(blob_path): shutil.rmtree(blob_path) raise SourceError(e) from e self._verify_blob(blob_path, expected_digest=layer_digest) # Only if all sources are successfully fetched, move files to staging directory # As both the manifest and blobs are content addressable, we can optimize space by having # a flat mirror directory. We check one-by-one if there is any need to copy a file out of the tmpdir. for fetched_file in os.listdir(tmpdir): move_atomic( os.path.join(tmpdir, fetched_file), os.path.join(self.get_mirror_directory(), fetched_file), )