def test_fetch_source_by_pull(mock_update_and_archive, mock_getctime, gitsubmodule):
mock_getctime.side_effect = [
datetime(2020, 3, 1, 20, 0, 0).timestamp(),
datetime(2020, 3, 4, 10, 13, 30).timestamp(),
datetime(2020, 3, 6, 10, 13, 30).timestamp(),
]
scm_git = scm.Git(url, ref)
po = mock.patch.object
if gitsubmodule:
with mock.patch("cachito.workers.scm.SourcesDir") as mock_scr:
scm_git_submodule = scm.Git(url, f"{ref}-with-submodules")
mock_scr.return_value = scm_git_submodule.sources_dir
with po(scm_git_submodule.sources_dir.archive_path, "exists", return_value=False):
with po(
scm_git_submodule.sources_dir.package_dir,
"glob",
return_value=[
"29eh2a.tar.gz",
"a8c2d2.tar.gz",
"a8c2d2-with-submodules.tar.gz",
],
):
scm_git_submodule.fetch_source(gitsubmodule)
else:
with po(scm_git.sources_dir.archive_path, "exists", return_value=False):
with po(
scm_git.sources_dir.package_dir,
"glob",
return_value=["29eh2a.tar.gz", "a8c2d2.tar.gz", "a8c2d2-with-submodules.tar.gz"],
):
scm_git.fetch_source(gitsubmodule)
mock_update_and_archive.assert_called_once_with("a8c2d2.tar.gz", gitsubmodule=gitsubmodule)
def test_fetch_source_invalid_archive_exists(mock_clone, mock_verify, caplog,
gitsubmodule):
mock_verify.side_effect = [CachitoError("stub"), None]
scm_git = scm.Git(url, ref)
po = mock.patch.object
if gitsubmodule:
with mock.patch("cachito.workers.scm.SourcesDir") as mock_scr:
scm_git_submodule = scm.Git(url, f"{ref}-with-submodules")
mock_scr.return_value = scm_git_submodule.sources_dir
with po(scm_git_submodule.sources_dir.archive_path,
"exists",
return_value=True):
with po(scm_git_submodule.sources_dir.package_dir,
"glob") as glob:
scm_git_submodule.fetch_source(gitsubmodule)
glob.assert_called_once()
msg = f'The archive at "{scm_git_submodule.sources_dir.archive_path}" is '
"invalid and will be re-created"
assert msg in caplog.text
else:
with po(scm_git.sources_dir.archive_path, "exists", return_value=True):
with po(scm_git.sources_dir.package_dir, "glob") as glob:
scm_git.fetch_source(gitsubmodule)
glob.assert_called_once()
msg = (
f'The archive at "{scm_git.sources_dir.archive_path}" is invalid and will be re-created'
)
assert msg in caplog.text
mock_clone.assert_called_once()
def test_fetch_source_by_pull_corrupt_archive(mock_clone_and_archive,
mock_update_and_archive,
mock_getctime, all_corrupt,
gitsubmodule):
if all_corrupt:
mock_update_and_archive.side_effect = [
git.exc.InvalidGitRepositoryError, OSError
]
else:
mock_update_and_archive.side_effect = [tarfile.ExtractError, None]
mock_getctime.side_effect = [
datetime(2020, 3, 1, 20, 0, 0).timestamp(),
datetime(2020, 3, 4, 10, 13, 30).timestamp(),
datetime(2020, 3, 1, 20, 0, 0).timestamp(),
]
scm_git = scm.Git(url, ref)
po = mock.patch.object
if gitsubmodule:
with mock.patch("cachito.workers.scm.SourcesDir") as mock_scr:
scm_git_submodule = scm.Git(url, f"{ref}-with-submodules")
mock_scr.return_value = scm_git_submodule.sources_dir
with po(scm_git_submodule.sources_dir.archive_path,
"exists",
return_value=False):
with po(
scm_git_submodule.sources_dir.package_dir,
"glob",
return_value=["29eh2a.tar.gz", "a8c2d2.tar.gz"],
):
scm_git_submodule.fetch_source(gitsubmodule)
else:
with po(scm_git.sources_dir.archive_path, "exists",
return_value=False):
with po(
scm_git.sources_dir.package_dir,
"glob",
return_value=["29eh2a.tar.gz", "a8c2d2.tar.gz"],
):
scm_git.fetch_source(gitsubmodule)
assert mock_update_and_archive.call_count == 2
calls = [
mock.call("a8c2d2.tar.gz", gitsubmodule=gitsubmodule),
mock.call("29eh2a.tar.gz", gitsubmodule=gitsubmodule),
]
mock_update_and_archive.assert_has_calls(calls)
if all_corrupt:
mock_clone_and_archive.assert_called_once_with(
gitsubmodule=gitsubmodule)
else:
mock_clone_and_archive.assert_not_called()
def test_update_and_archive(mock_ugs, mock_exists, mock_fsck, mock_repo,
mock_temp_dir, mock_tarfile_open, gitsubmodule):
# Mock the archive being created
mock_exists.return_value = True
mock_tarfile = mock.Mock()
mock_tarfile_open.return_value.__enter__.return_value = mock_tarfile
# Mock the tempfile.TemporaryDirectory context manager
mock_temp_dir.return_value.__enter__.return_value = "/tmp/cachito-temp"
# Test does not really extract this archive file. The filename could be arbitrary.
scm.Git(url, ref).update_and_archive("/tmp/1234567.tar.gz", gitsubmodule)
# Verify the tempfile.TemporaryDirectory context manager was used twice:
# once for _update_and_archive and once for _verify_archive
assert mock_temp_dir.return_value.__enter__.call_count == 2
repo = mock_repo.return_value
# Verify the changes are pulled.
repo.remote.return_value.fetch.assert_called_once_with(refspec=ref)
# Verify the repo is reset to specific ref
repo.commit.assert_called_once_with(ref)
assert repo.commit.return_value == repo.head.reference
repo.head.reset.assert_called_once_with(index=True, working_tree=True)
mock_tarfile.add.assert_called_once_with(
mock_repo.return_value.working_dir, "app")
# Verify the archive was verified
mock_fsck.assert_called_once()
# Verify the update_git_submodules was called correctly(if applicable)
if gitsubmodule:
mock_ugs.assert_called_once_with(repo)
else:
mock_ugs.assert_not_called()
def test_update_and_verify_archive(fake_repo, caplog):
repo_dir, _ = fake_repo
git_obj = scm.Git(f"file://{repo_dir}", "master")
git_obj.clone_and_archive()
caplog.clear()
git_obj.update_and_archive(git_obj.sources_dir.archive_path)
assert f"Verifying the archive at {git_obj.sources_dir.archive_path}" in caplog.text
def test_verify_invalid_archive(mock_istar, fake_repo):
mock_istar.return_value = False
repo_dir, _ = fake_repo
git_obj = scm.Git(f"file://{repo_dir}", "master")
err_msg = f"No valid archive found at {git_obj.sources_dir.archive_path}"
with pytest.raises(CachitoError, match=err_msg):
git_obj._verify_archive()
def test_fetch_source_archive_exists(mock_verify, gitsubmodule):
scm_git = scm.Git(url, ref)
with mock.patch("pathlib.Path.exists", return_value=True):
with mock.patch("pathlib.Path.glob") as glob:
scm_git.fetch_source(gitsubmodule)
glob.assert_not_called()
def test_clone_and_archive(mock_archive_path, mock_clone, mock_temp_dir,
mock_tarfile_open):
# Mock the archive being created
mock_tarfile = mock.Mock()
mock_tarfile_open.return_value.__enter__.return_value = mock_tarfile
# Mock the commit being returned from repo.commit(self.ref)
mock_commit = mock.Mock()
mock_clone.return_value.commit.return_value = mock_commit
# Mock the tempfile.TemporaryDirectory context manager
mock_temp_dir.return_value.__enter__.return_value = '/tmp/cachito-temp'
# Mock the Git.archives_dir property
mock_archive_path.return_value = archive_path
git_obj = scm.Git(url, ref)
git_obj.clone_and_archive()
# Verify the tempfile.TemporaryDirectory context manager was used
mock_temp_dir.return_value.__enter__.assert_called_once()
# Verify the repo was cloned and checked out properly
mock_clone.assert_called_once_with(url,
'/tmp/cachito-temp/repo',
no_checkout=True)
assert mock_clone.return_value.head.reference == mock_commit
mock_clone.return_value.head.reset.assert_called_once_with(
index=True, working_tree=True)
# Verfiy the archive was created
mock_tarfile.add.assert_called_once_with('/tmp/cachito-temp/repo', 'app')
def test_fetch_source_clone_if_no_archive_yet(mock_clone_and_archive, gitsubmodule):
scm_git = scm.Git(url, ref)
po = mock.patch.object
if gitsubmodule:
with mock.patch("cachito.workers.scm.SourcesDir") as mock_scr:
scm_git_submodule = scm.Git(url, f"{ref}-with-submodules")
mock_scr.return_value = scm_git_submodule.sources_dir
with po(scm_git_submodule.sources_dir.archive_path, "exists", return_value=False):
with po(scm_git_submodule.sources_dir.package_dir, "glob", return_value=[]):
scm_git_submodule.fetch_source(gitsubmodule)
else:
with po(scm_git.sources_dir.archive_path, "exists", return_value=False):
with po(scm_git.sources_dir.package_dir, "glob", return_value=[]):
scm_git.fetch_source(gitsubmodule)
mock_clone_and_archive.assert_called_once_with(gitsubmodule=gitsubmodule)
def test_fetch_source_archive_exists(mock_verify, gitsubmodule):
scm_git = scm.Git(url, ref)
po = mock.patch.object
with po(scm_git.sources_dir.archive_path, "exists", return_value=True):
with po(scm_git.sources_dir.package_dir, "glob") as glob:
scm_git.fetch_source(gitsubmodule)
glob.assert_not_called()
def test_archive_path(mock_makedirs, mock_celery_app):
path = '/tmp/cachito-archives'
mock_celery_app.conf.cachito_archives_dir = path
git = scm.Git(url, ref)
assert git.archives_dir == path
assert git.archive_path == archive_path
mock_makedirs.assert_called_once_with(
'/tmp/cachito-archives/release-engineering/retrodep', exist_ok=True)
def test_update_and_archive_pull_error(mock_repo, mock_tarfile_open,
gitsubmodule):
repo = mock_repo.return_value
repo.remote.return_value.fetch.side_effect = OSError
with pytest.raises(CachitoError,
match="Failed to fetch from the remote Git repository"):
scm.Git(url, ref).update_and_archive("/tmp/1234567.tar.gz",
gitsubmodule)
def test_clone_and_archive_clone_failed(mock_git_clone, mock_temp_dir, gitsubmodule):
# Mock the tempfile.TemporaryDirectory context manager
mock_temp_dir.return_value.__enter__.return_value = "/tmp/cachito-temp"
# Mock the git clone call
mock_git_clone.side_effect = git.GitCommandError("some error", 1)
git_obj = scm.Git(url, ref)
with pytest.raises(CachitoError, match="Cloning the Git repository failed"):
git_obj.clone_and_archive(gitsubmodule)
def test_update_git_submodules_failed(mock_repo):
repo = mock_repo.return_value
# Set up the side effect for submodule_update call
repo.submodule_update.side_effect = git.GitCommandError("some error", 1)
expected = re.escape("Updating the Git submodule(s) failed")
git_obj = scm.Git(url, ref)
with pytest.raises(CachitoError, match=expected):
git_obj.update_git_submodules(repo)
def test_clone_and_archive(
mock_fsync,
mock_link,
mock_ugs,
mock_exists,
mock_fsck,
mock_clone,
mock_temp_file,
mock_temp_dir,
mock_tarfile_open,
gitsubmodule,
):
# Mock the archive being created
mock_exists.return_value = True
mock_tarfile = mock.Mock()
mock_tarfile_open.return_value.__enter__.return_value = mock_tarfile
# Mock the commit being returned from repo.commit(self.ref)
mock_commit = mock.Mock()
mock_clone.return_value.commit.return_value = mock_commit
# Mock the tempfile.TemporaryDirectory context manager
mock_temp_dir.return_value.__enter__.return_value = "/tmp/cachito-temp"
# Mock the tempfile.NamedTemporaryFile context manager
mock_temp_file.return_value.__enter__.return_value = mock.Mock(
name="/dev/null", fileno=lambda: -1)
git_obj = scm.Git(url, ref)
with mock.patch.object(git_obj.sources_dir,
"archive_path",
new=archive_path):
git_obj.clone_and_archive(gitsubmodule)
# Verify the tempfile.TemporaryDirectory context manager was used twice:
# once for _clone_and_archive and once for _verify_archive
assert mock_temp_dir.return_value.__enter__.call_count == 2
# Verify the repo was cloned and checked out properly
mock_clone.assert_called_once_with(url,
"/tmp/cachito-temp/repo",
no_checkout=True,
env={"GIT_TERMINAL_PROMPT": "0"})
assert mock_clone.return_value.head.reference == mock_commit
mock_clone.return_value.head.reset.assert_called_once_with(
index=True, working_tree=True)
# Verfiy the archive was created
mock_tarfile.add.assert_called_once_with(
mock_clone.return_value.working_dir, "app")
# Verify the archive was verified
mock_fsck.assert_called_once()
# Verify the update_git_submodules was called correctly(if applicable)
if gitsubmodule:
mock_ugs.assert_called_once_with(mock_clone.return_value)
else:
mock_ugs.assert_not_called()
mock_clone.return_value.git.gc.assert_called_once_with('--prune=now')
def test_download_source_download_failed(mock_requests, mock_temp_dir):
# Mock the tempfile.TemporaryDirectory context manager
mock_temp_dir.return_value.__enter__.return_value = '/tmp/cachito-temp'
# Mock the requests.get call of downloading the archive from GitHub so that it fails
mock_requests.return_value.__enter__.return_value.ok = False
download_url = f'{url[: -len(".git")]}/archive/{ref}.tar.gz'
git = scm.Git(url, ref)
expected_error = 'An unexpected error was encountered when downloading the source'
with pytest.raises(CachitoError, match=expected_error):
git.download_source_archive(download_url)
def test_clone_and_archive_clone_failed(mock_run, mock_temp_dir):
# Mock the tempfile.TemporaryDirectory context manager
mock_temp_dir.return_value.__enter__.return_value = '/tmp/cachito-temp'
# Mock the git clone call
mock_run.return_value.returncode = 1
mock_run.return_value.stderr = 'failure'
git = scm.Git(url, ref)
with pytest.raises(CachitoError,
match='Cloning the git repository failed'):
git.clone_and_archive()
def test_create_archive_verify_fails(fake_repo, caplog):
repo_dir, _ = fake_repo
git_obj = scm.Git(f"file://{repo_dir}", "master")
# substitute the archive with a broken git repository
os.unlink(os.path.join(repo_dir, ".git", "HEAD"))
err_msg = f"Invalid archive at {git_obj.sources_dir.archive_path}"
with pytest.raises(CachitoError, match=err_msg):
git_obj._create_archive(repo_dir)
# verify the archive was not created
assert f"Removing invalid archive at {git_obj.sources_dir.archive_path}" in caplog.text
assert not os.path.exists(git_obj.sources_dir.archive_path)
def test_verify_invalid_repo(fake_repo, tmp_path):
repo_dir, _ = fake_repo
git_obj = scm.Git(f"file://{repo_dir}", "master")
# substitute the archive with a broken git repository
os.unlink(os.path.join(repo_dir, ".git", "HEAD"))
git_obj.sources_dir.archive_path = tmp_path / "archive.tar.gz"
with tarfile.open(git_obj.sources_dir.archive_path, mode="w:gz") as bundle_archive:
bundle_archive.add(repo_dir, "app")
err_msg = f"Invalid archive at {git_obj.sources_dir.archive_path}"
with pytest.raises(CachitoError, match=err_msg):
git_obj._verify_archive()
def test_download_source_archive(
mock_tarfile_open,
mock_copyfile,
mock_copyfileobj,
mock_archive_path,
mock_requests,
mock_temp_dir,
):
# Mock the tempfile.TemporaryDirectory context manager
mock_temp_dir.return_value.__enter__.return_value = '/tmp/cachito-temp'
# Mock the requests.get call of downloading the archive from GitHub
mock_requests.return_value.__enter__.return_value.ok = True
# Mock the Git.archive_path property
mock_archive_path.return_value = archive_path
# Mock the opening of the initial tar file that was downloaded from GitHub
mock_initial_tarfile = mock.Mock()
mock_initial_tarfile.firstmember.name = 'retrodep'
# Mock the opening of the tar file that will contain the content to put in long-term storage
mock_final_tarfile = mock.Mock()
mock_tarfile_open.return_value.__enter__.side_effect = [
mock_initial_tarfile,
mock_final_tarfile,
]
git = scm.Git(url, ref)
download_url = f'{url[: -len(".git")]}/archive/{ref}.tar.gz'
# Mock the opening of the initial tar file to write the content downloaded from GitHub
with mock.patch('builtins.open', mock.mock_open()) as mock_file:
git.download_source_archive(download_url)
# Verify the tarfile was written to in the temporary directory
mock_file.assert_called_once_with(f'/tmp/cachito-temp/{ref}.tar.gz',
'wb')
# Verify the tempfile.TemporaryDirectory context manager was used
mock_temp_dir.return_value.__enter__.assert_called_once()
# Verify the archive was downloaded
mock_requests.assert_called_once_with(download_url,
stream=True,
timeout=120)
# Verify the archive was written to disk
mock_copyfileobj.assert_called_once()
# Verify that the intial archive that was downloaded was extracted to the temporary directory
mock_initial_tarfile.extractall.assert_called_once_with(
'/tmp/cachito-temp')
# Verify that the final archive was created
mock_final_tarfile.add.assert_called_once_with(
'/tmp/cachito-temp/retrodep', 'app')
mock_copyfile.assert_called_once_with(
'/tmp/cachito-temp/corrected-c50b93a32df1c9d700e3e80996845bc2e13be848.tar.gz',
'/tmp/cachito-archives/release-engineering/retrodep/'
'c50b93a32df1c9d700e3e80996845bc2e13be848.tar.gz',
)
def test_clone_and_archive_checkout_failed(mock_git_clone, mock_temp_dir, gitsubmodule):
# Mock the tempfile.TemporaryDirectory context manager
mock_temp_dir.return_value.__enter__.return_value = "/tmp/cachito-temp"
# Mock the git calls
mock_git_clone.return_value.commit.side_effect = git.GitCommandError("commit is invalid", 1)
git_obj = scm.Git(url, ref)
expected = (
"Checking out the Git repository failed. Please verify the supplied reference of "
f'"{ref}" is valid.'
)
with mock.patch.object(git_obj.sources_dir, "archive_path", new=archive_path):
with pytest.raises(CachitoError, match=expected):
git_obj.clone_and_archive(gitsubmodule)
def test_verify_corrupted_git_repo(mock_fsck, fake_repo, tmp_path):
mock_fsck.side_effect = subprocess.CalledProcessError(0, "stub command")
repo_dir, _ = fake_repo
git_obj = scm.Git(f"file://{repo_dir}", "master")
stub_file = tmp_path / "fake_contents"
with open(stub_file, "w") as f:
f.write("stub\n")
with tarfile.open(git_obj.sources_dir.archive_path, "w:gz") as tar:
tar.add(stub_file)
err_msg = f"Invalid archive at {git_obj.sources_dir.archive_path}"
with pytest.raises(CachitoError, match=err_msg):
git_obj._verify_archive()
def test_verify_corrupted_archive(mock_extract, fake_repo, exception_type, tmp_path):
mock_extract.side_effect = exception_type("Something wrong with the tar archive")
repo_dir, _ = fake_repo
git_obj = scm.Git(f"file://{repo_dir}", "master")
stub_file = tmp_path / "fake_contents"
with open(stub_file, "w") as f:
f.write("stub\n")
with tarfile.open(git_obj.sources_dir.archive_path, "w:gz") as tar:
tar.add(stub_file)
err_msg = f"Invalid archive at {git_obj.sources_dir.archive_path}"
with pytest.raises(CachitoError, match=err_msg):
git_obj._verify_archive()
def test_create_and_verify_archive(fake_repo, caplog):
repo_dir, _ = fake_repo
git_obj = scm.Git(f"file://{repo_dir}", "master")
verify_log_msg = f"Verifying the archive at {git_obj.sources_dir.archive_path}"
already_created_log_msg = (
f"{git_obj.sources_dir.archive_path} was created while this task was running. "
"Will proceed with that archive")
git_obj._create_archive(repo_dir)
assert verify_log_msg in caplog.text
assert already_created_log_msg not in caplog.text
caplog.clear()
# create archive again to simulate race condition. This should not generate errors
git_obj._create_archive(repo_dir)
assert verify_log_msg in caplog.text
assert already_created_log_msg in caplog.text
def test_clone_and_archive_checkout_failed(mock_archive_path, mock_git_clone,
mock_temp_dir):
# Mock the tempfile.TemporaryDirectory context manager
mock_temp_dir.return_value.__enter__.return_value = '/tmp/cachito-temp'
# Mock the git calls
mock_git_clone.return_value.commit.side_effect = git.GitCommandError(
'commit is invalid', 1)
# Mock the Git.archives_dir property
mock_archive_path.return_value = archive_path
git_obj = scm.Git(url, ref)
expected = (
'Checking out the Git repository failed. Please verify the supplied reference of '
f'"{ref}" is valid.')
with pytest.raises(CachitoError, match=expected):
git_obj.clone_and_archive()
def test_clone_and_archive_git_archive_failed(mock_archive_path, mock_run,
mock_temp_dir, archive_error,
expected_error):
# Mock the tempfile.TemporaryDirectory context manager
mock_temp_dir.return_value.__enter__.return_value = '/tmp/cachito-temp'
# Mock the git calls
mock_clone = mock.Mock()
mock_clone.returncode = 0
mock_archive = mock.Mock()
mock_archive.returncode = 1
mock_archive.stderr = archive_error
mock_run.side_effect = [mock_clone, mock_archive]
# Mock the Git.archives_dir property
mock_archive_path.return_value = archive_path
git = scm.Git(url, ref)
with pytest.raises(CachitoError, match=expected_error):
git.clone_and_archive()
def test_clone_and_archive(mock_archive_path, mock_run, mock_temp_dir):
# Mock the tempfile.TemporaryDirectory context manager
mock_temp_dir.return_value.__enter__.return_value = '/tmp/cachito-temp'
# Mock the git calls
mock_run.return_value.returncode = 0
# Mock the Git.archives_dir property
mock_archive_path.return_value = archive_path
git = scm.Git(url, ref)
git.clone_and_archive()
# Verify the tempfile.TemporaryDirectory context manager was used
mock_temp_dir.return_value.__enter__.assert_called_once()
# Verify the git calls were correct
mock_run.assert_has_calls([
mock.call([
'git',
'clone',
'-q',
'--no-checkout',
'https://github.com/release-engineering/retrodep.git',
'/tmp/cachito-temp/repo',
],
capture_output=True,
universal_newlines=True,
encoding='utf-8'),
mock.call([
'git',
'-C',
'/tmp/cachito-temp/repo',
'archive',
'-o',
f'/tmp/cachito-archives/release-engineering/retrodep/{ref}.tar.gz',
'--prefix=app/',
'c50b93a32df1c9d700e3e80996845bc2e13be848',
],
capture_output=True,
universal_newlines=True,
encoding='utf-8'),
])
def test_update_git_submodules(mock_repo):
git_obj = scm.Git(url, ref)
git_obj.update_git_submodules(mock_repo)
# Verify the git submodule update was called correctly
mock_repo.submodule_update.assert_called_once_with(recursive=False)
def test_verify_archive_not_available():
git_obj = scm.Git("invalid", "ref")
err_msg = f"No valid archive found at {git_obj.sources_dir.archive_path}"
with pytest.raises(CachitoError, match=err_msg):
git_obj._verify_archive()
def test_repo_name():
git_obj = scm.Git(url, ref)
assert git_obj.repo_name == "release-engineering/retrodep"
