def test_fetch_source_by_pull(mock_update_and_archive, mock_getctime, gitsubmodule): mock_getctime.side_effect = [ datetime(2020, 3, 1, 20, 0, 0).timestamp(), datetime(2020, 3, 4, 10, 13, 30).timestamp(), datetime(2020, 3, 6, 10, 13, 30).timestamp(), ] scm_git = scm.Git(url, ref) po = mock.patch.object if gitsubmodule: with mock.patch("cachito.workers.scm.SourcesDir") as mock_scr: scm_git_submodule = scm.Git(url, f"{ref}-with-submodules") mock_scr.return_value = scm_git_submodule.sources_dir with po(scm_git_submodule.sources_dir.archive_path, "exists", return_value=False): with po( scm_git_submodule.sources_dir.package_dir, "glob", return_value=[ "29eh2a.tar.gz", "a8c2d2.tar.gz", "a8c2d2-with-submodules.tar.gz", ], ): scm_git_submodule.fetch_source(gitsubmodule) else: with po(scm_git.sources_dir.archive_path, "exists", return_value=False): with po( scm_git.sources_dir.package_dir, "glob", return_value=["29eh2a.tar.gz", "a8c2d2.tar.gz", "a8c2d2-with-submodules.tar.gz"], ): scm_git.fetch_source(gitsubmodule) mock_update_and_archive.assert_called_once_with("a8c2d2.tar.gz", gitsubmodule=gitsubmodule)
def test_fetch_source_invalid_archive_exists(mock_clone, mock_verify, caplog, gitsubmodule): mock_verify.side_effect = [CachitoError("stub"), None] scm_git = scm.Git(url, ref) po = mock.patch.object if gitsubmodule: with mock.patch("cachito.workers.scm.SourcesDir") as mock_scr: scm_git_submodule = scm.Git(url, f"{ref}-with-submodules") mock_scr.return_value = scm_git_submodule.sources_dir with po(scm_git_submodule.sources_dir.archive_path, "exists", return_value=True): with po(scm_git_submodule.sources_dir.package_dir, "glob") as glob: scm_git_submodule.fetch_source(gitsubmodule) glob.assert_called_once() msg = f'The archive at "{scm_git_submodule.sources_dir.archive_path}" is ' "invalid and will be re-created" assert msg in caplog.text else: with po(scm_git.sources_dir.archive_path, "exists", return_value=True): with po(scm_git.sources_dir.package_dir, "glob") as glob: scm_git.fetch_source(gitsubmodule) glob.assert_called_once() msg = ( f'The archive at "{scm_git.sources_dir.archive_path}" is invalid and will be re-created' ) assert msg in caplog.text mock_clone.assert_called_once()
def test_fetch_source_by_pull_corrupt_archive(mock_clone_and_archive, mock_update_and_archive, mock_getctime, all_corrupt, gitsubmodule): if all_corrupt: mock_update_and_archive.side_effect = [ git.exc.InvalidGitRepositoryError, OSError ] else: mock_update_and_archive.side_effect = [tarfile.ExtractError, None] mock_getctime.side_effect = [ datetime(2020, 3, 1, 20, 0, 0).timestamp(), datetime(2020, 3, 4, 10, 13, 30).timestamp(), datetime(2020, 3, 1, 20, 0, 0).timestamp(), ] scm_git = scm.Git(url, ref) po = mock.patch.object if gitsubmodule: with mock.patch("cachito.workers.scm.SourcesDir") as mock_scr: scm_git_submodule = scm.Git(url, f"{ref}-with-submodules") mock_scr.return_value = scm_git_submodule.sources_dir with po(scm_git_submodule.sources_dir.archive_path, "exists", return_value=False): with po( scm_git_submodule.sources_dir.package_dir, "glob", return_value=["29eh2a.tar.gz", "a8c2d2.tar.gz"], ): scm_git_submodule.fetch_source(gitsubmodule) else: with po(scm_git.sources_dir.archive_path, "exists", return_value=False): with po( scm_git.sources_dir.package_dir, "glob", return_value=["29eh2a.tar.gz", "a8c2d2.tar.gz"], ): scm_git.fetch_source(gitsubmodule) assert mock_update_and_archive.call_count == 2 calls = [ mock.call("a8c2d2.tar.gz", gitsubmodule=gitsubmodule), mock.call("29eh2a.tar.gz", gitsubmodule=gitsubmodule), ] mock_update_and_archive.assert_has_calls(calls) if all_corrupt: mock_clone_and_archive.assert_called_once_with( gitsubmodule=gitsubmodule) else: mock_clone_and_archive.assert_not_called()
def test_update_and_archive(mock_ugs, mock_exists, mock_fsck, mock_repo, mock_temp_dir, mock_tarfile_open, gitsubmodule): # Mock the archive being created mock_exists.return_value = True mock_tarfile = mock.Mock() mock_tarfile_open.return_value.__enter__.return_value = mock_tarfile # Mock the tempfile.TemporaryDirectory context manager mock_temp_dir.return_value.__enter__.return_value = "/tmp/cachito-temp" # Test does not really extract this archive file. The filename could be arbitrary. scm.Git(url, ref).update_and_archive("/tmp/1234567.tar.gz", gitsubmodule) # Verify the tempfile.TemporaryDirectory context manager was used twice: # once for _update_and_archive and once for _verify_archive assert mock_temp_dir.return_value.__enter__.call_count == 2 repo = mock_repo.return_value # Verify the changes are pulled. repo.remote.return_value.fetch.assert_called_once_with(refspec=ref) # Verify the repo is reset to specific ref repo.commit.assert_called_once_with(ref) assert repo.commit.return_value == repo.head.reference repo.head.reset.assert_called_once_with(index=True, working_tree=True) mock_tarfile.add.assert_called_once_with( mock_repo.return_value.working_dir, "app") # Verify the archive was verified mock_fsck.assert_called_once() # Verify the update_git_submodules was called correctly(if applicable) if gitsubmodule: mock_ugs.assert_called_once_with(repo) else: mock_ugs.assert_not_called()
def test_update_and_verify_archive(fake_repo, caplog): repo_dir, _ = fake_repo git_obj = scm.Git(f"file://{repo_dir}", "master") git_obj.clone_and_archive() caplog.clear() git_obj.update_and_archive(git_obj.sources_dir.archive_path) assert f"Verifying the archive at {git_obj.sources_dir.archive_path}" in caplog.text
def test_verify_invalid_archive(mock_istar, fake_repo): mock_istar.return_value = False repo_dir, _ = fake_repo git_obj = scm.Git(f"file://{repo_dir}", "master") err_msg = f"No valid archive found at {git_obj.sources_dir.archive_path}" with pytest.raises(CachitoError, match=err_msg): git_obj._verify_archive()
def test_fetch_source_archive_exists(mock_verify, gitsubmodule): scm_git = scm.Git(url, ref) with mock.patch("pathlib.Path.exists", return_value=True): with mock.patch("pathlib.Path.glob") as glob: scm_git.fetch_source(gitsubmodule) glob.assert_not_called()
def test_clone_and_archive(mock_archive_path, mock_clone, mock_temp_dir, mock_tarfile_open): # Mock the archive being created mock_tarfile = mock.Mock() mock_tarfile_open.return_value.__enter__.return_value = mock_tarfile # Mock the commit being returned from repo.commit(self.ref) mock_commit = mock.Mock() mock_clone.return_value.commit.return_value = mock_commit # Mock the tempfile.TemporaryDirectory context manager mock_temp_dir.return_value.__enter__.return_value = '/tmp/cachito-temp' # Mock the Git.archives_dir property mock_archive_path.return_value = archive_path git_obj = scm.Git(url, ref) git_obj.clone_and_archive() # Verify the tempfile.TemporaryDirectory context manager was used mock_temp_dir.return_value.__enter__.assert_called_once() # Verify the repo was cloned and checked out properly mock_clone.assert_called_once_with(url, '/tmp/cachito-temp/repo', no_checkout=True) assert mock_clone.return_value.head.reference == mock_commit mock_clone.return_value.head.reset.assert_called_once_with( index=True, working_tree=True) # Verfiy the archive was created mock_tarfile.add.assert_called_once_with('/tmp/cachito-temp/repo', 'app')
def test_fetch_source_clone_if_no_archive_yet(mock_clone_and_archive, gitsubmodule): scm_git = scm.Git(url, ref) po = mock.patch.object if gitsubmodule: with mock.patch("cachito.workers.scm.SourcesDir") as mock_scr: scm_git_submodule = scm.Git(url, f"{ref}-with-submodules") mock_scr.return_value = scm_git_submodule.sources_dir with po(scm_git_submodule.sources_dir.archive_path, "exists", return_value=False): with po(scm_git_submodule.sources_dir.package_dir, "glob", return_value=[]): scm_git_submodule.fetch_source(gitsubmodule) else: with po(scm_git.sources_dir.archive_path, "exists", return_value=False): with po(scm_git.sources_dir.package_dir, "glob", return_value=[]): scm_git.fetch_source(gitsubmodule) mock_clone_and_archive.assert_called_once_with(gitsubmodule=gitsubmodule)
def test_fetch_source_archive_exists(mock_verify, gitsubmodule): scm_git = scm.Git(url, ref) po = mock.patch.object with po(scm_git.sources_dir.archive_path, "exists", return_value=True): with po(scm_git.sources_dir.package_dir, "glob") as glob: scm_git.fetch_source(gitsubmodule) glob.assert_not_called()
def test_archive_path(mock_makedirs, mock_celery_app): path = '/tmp/cachito-archives' mock_celery_app.conf.cachito_archives_dir = path git = scm.Git(url, ref) assert git.archives_dir == path assert git.archive_path == archive_path mock_makedirs.assert_called_once_with( '/tmp/cachito-archives/release-engineering/retrodep', exist_ok=True)
def test_update_and_archive_pull_error(mock_repo, mock_tarfile_open, gitsubmodule): repo = mock_repo.return_value repo.remote.return_value.fetch.side_effect = OSError with pytest.raises(CachitoError, match="Failed to fetch from the remote Git repository"): scm.Git(url, ref).update_and_archive("/tmp/1234567.tar.gz", gitsubmodule)
def test_clone_and_archive_clone_failed(mock_git_clone, mock_temp_dir, gitsubmodule): # Mock the tempfile.TemporaryDirectory context manager mock_temp_dir.return_value.__enter__.return_value = "/tmp/cachito-temp" # Mock the git clone call mock_git_clone.side_effect = git.GitCommandError("some error", 1) git_obj = scm.Git(url, ref) with pytest.raises(CachitoError, match="Cloning the Git repository failed"): git_obj.clone_and_archive(gitsubmodule)
def test_update_git_submodules_failed(mock_repo): repo = mock_repo.return_value # Set up the side effect for submodule_update call repo.submodule_update.side_effect = git.GitCommandError("some error", 1) expected = re.escape("Updating the Git submodule(s) failed") git_obj = scm.Git(url, ref) with pytest.raises(CachitoError, match=expected): git_obj.update_git_submodules(repo)
def test_clone_and_archive( mock_fsync, mock_link, mock_ugs, mock_exists, mock_fsck, mock_clone, mock_temp_file, mock_temp_dir, mock_tarfile_open, gitsubmodule, ): # Mock the archive being created mock_exists.return_value = True mock_tarfile = mock.Mock() mock_tarfile_open.return_value.__enter__.return_value = mock_tarfile # Mock the commit being returned from repo.commit(self.ref) mock_commit = mock.Mock() mock_clone.return_value.commit.return_value = mock_commit # Mock the tempfile.TemporaryDirectory context manager mock_temp_dir.return_value.__enter__.return_value = "/tmp/cachito-temp" # Mock the tempfile.NamedTemporaryFile context manager mock_temp_file.return_value.__enter__.return_value = mock.Mock( name="/dev/null", fileno=lambda: -1) git_obj = scm.Git(url, ref) with mock.patch.object(git_obj.sources_dir, "archive_path", new=archive_path): git_obj.clone_and_archive(gitsubmodule) # Verify the tempfile.TemporaryDirectory context manager was used twice: # once for _clone_and_archive and once for _verify_archive assert mock_temp_dir.return_value.__enter__.call_count == 2 # Verify the repo was cloned and checked out properly mock_clone.assert_called_once_with(url, "/tmp/cachito-temp/repo", no_checkout=True, env={"GIT_TERMINAL_PROMPT": "0"}) assert mock_clone.return_value.head.reference == mock_commit mock_clone.return_value.head.reset.assert_called_once_with( index=True, working_tree=True) # Verfiy the archive was created mock_tarfile.add.assert_called_once_with( mock_clone.return_value.working_dir, "app") # Verify the archive was verified mock_fsck.assert_called_once() # Verify the update_git_submodules was called correctly(if applicable) if gitsubmodule: mock_ugs.assert_called_once_with(mock_clone.return_value) else: mock_ugs.assert_not_called() mock_clone.return_value.git.gc.assert_called_once_with('--prune=now')
def test_download_source_download_failed(mock_requests, mock_temp_dir): # Mock the tempfile.TemporaryDirectory context manager mock_temp_dir.return_value.__enter__.return_value = '/tmp/cachito-temp' # Mock the requests.get call of downloading the archive from GitHub so that it fails mock_requests.return_value.__enter__.return_value.ok = False download_url = f'{url[: -len(".git")]}/archive/{ref}.tar.gz' git = scm.Git(url, ref) expected_error = 'An unexpected error was encountered when downloading the source' with pytest.raises(CachitoError, match=expected_error): git.download_source_archive(download_url)
def test_clone_and_archive_clone_failed(mock_run, mock_temp_dir): # Mock the tempfile.TemporaryDirectory context manager mock_temp_dir.return_value.__enter__.return_value = '/tmp/cachito-temp' # Mock the git clone call mock_run.return_value.returncode = 1 mock_run.return_value.stderr = 'failure' git = scm.Git(url, ref) with pytest.raises(CachitoError, match='Cloning the git repository failed'): git.clone_and_archive()
def test_create_archive_verify_fails(fake_repo, caplog): repo_dir, _ = fake_repo git_obj = scm.Git(f"file://{repo_dir}", "master") # substitute the archive with a broken git repository os.unlink(os.path.join(repo_dir, ".git", "HEAD")) err_msg = f"Invalid archive at {git_obj.sources_dir.archive_path}" with pytest.raises(CachitoError, match=err_msg): git_obj._create_archive(repo_dir) # verify the archive was not created assert f"Removing invalid archive at {git_obj.sources_dir.archive_path}" in caplog.text assert not os.path.exists(git_obj.sources_dir.archive_path)
def test_verify_invalid_repo(fake_repo, tmp_path): repo_dir, _ = fake_repo git_obj = scm.Git(f"file://{repo_dir}", "master") # substitute the archive with a broken git repository os.unlink(os.path.join(repo_dir, ".git", "HEAD")) git_obj.sources_dir.archive_path = tmp_path / "archive.tar.gz" with tarfile.open(git_obj.sources_dir.archive_path, mode="w:gz") as bundle_archive: bundle_archive.add(repo_dir, "app") err_msg = f"Invalid archive at {git_obj.sources_dir.archive_path}" with pytest.raises(CachitoError, match=err_msg): git_obj._verify_archive()
def test_download_source_archive( mock_tarfile_open, mock_copyfile, mock_copyfileobj, mock_archive_path, mock_requests, mock_temp_dir, ): # Mock the tempfile.TemporaryDirectory context manager mock_temp_dir.return_value.__enter__.return_value = '/tmp/cachito-temp' # Mock the requests.get call of downloading the archive from GitHub mock_requests.return_value.__enter__.return_value.ok = True # Mock the Git.archive_path property mock_archive_path.return_value = archive_path # Mock the opening of the initial tar file that was downloaded from GitHub mock_initial_tarfile = mock.Mock() mock_initial_tarfile.firstmember.name = 'retrodep' # Mock the opening of the tar file that will contain the content to put in long-term storage mock_final_tarfile = mock.Mock() mock_tarfile_open.return_value.__enter__.side_effect = [ mock_initial_tarfile, mock_final_tarfile, ] git = scm.Git(url, ref) download_url = f'{url[: -len(".git")]}/archive/{ref}.tar.gz' # Mock the opening of the initial tar file to write the content downloaded from GitHub with mock.patch('builtins.open', mock.mock_open()) as mock_file: git.download_source_archive(download_url) # Verify the tarfile was written to in the temporary directory mock_file.assert_called_once_with(f'/tmp/cachito-temp/{ref}.tar.gz', 'wb') # Verify the tempfile.TemporaryDirectory context manager was used mock_temp_dir.return_value.__enter__.assert_called_once() # Verify the archive was downloaded mock_requests.assert_called_once_with(download_url, stream=True, timeout=120) # Verify the archive was written to disk mock_copyfileobj.assert_called_once() # Verify that the intial archive that was downloaded was extracted to the temporary directory mock_initial_tarfile.extractall.assert_called_once_with( '/tmp/cachito-temp') # Verify that the final archive was created mock_final_tarfile.add.assert_called_once_with( '/tmp/cachito-temp/retrodep', 'app') mock_copyfile.assert_called_once_with( '/tmp/cachito-temp/corrected-c50b93a32df1c9d700e3e80996845bc2e13be848.tar.gz', '/tmp/cachito-archives/release-engineering/retrodep/' 'c50b93a32df1c9d700e3e80996845bc2e13be848.tar.gz', )
def test_clone_and_archive_checkout_failed(mock_git_clone, mock_temp_dir, gitsubmodule): # Mock the tempfile.TemporaryDirectory context manager mock_temp_dir.return_value.__enter__.return_value = "/tmp/cachito-temp" # Mock the git calls mock_git_clone.return_value.commit.side_effect = git.GitCommandError("commit is invalid", 1) git_obj = scm.Git(url, ref) expected = ( "Checking out the Git repository failed. Please verify the supplied reference of " f'"{ref}" is valid.' ) with mock.patch.object(git_obj.sources_dir, "archive_path", new=archive_path): with pytest.raises(CachitoError, match=expected): git_obj.clone_and_archive(gitsubmodule)
def test_verify_corrupted_git_repo(mock_fsck, fake_repo, tmp_path): mock_fsck.side_effect = subprocess.CalledProcessError(0, "stub command") repo_dir, _ = fake_repo git_obj = scm.Git(f"file://{repo_dir}", "master") stub_file = tmp_path / "fake_contents" with open(stub_file, "w") as f: f.write("stub\n") with tarfile.open(git_obj.sources_dir.archive_path, "w:gz") as tar: tar.add(stub_file) err_msg = f"Invalid archive at {git_obj.sources_dir.archive_path}" with pytest.raises(CachitoError, match=err_msg): git_obj._verify_archive()
def test_verify_corrupted_archive(mock_extract, fake_repo, exception_type, tmp_path): mock_extract.side_effect = exception_type("Something wrong with the tar archive") repo_dir, _ = fake_repo git_obj = scm.Git(f"file://{repo_dir}", "master") stub_file = tmp_path / "fake_contents" with open(stub_file, "w") as f: f.write("stub\n") with tarfile.open(git_obj.sources_dir.archive_path, "w:gz") as tar: tar.add(stub_file) err_msg = f"Invalid archive at {git_obj.sources_dir.archive_path}" with pytest.raises(CachitoError, match=err_msg): git_obj._verify_archive()
def test_create_and_verify_archive(fake_repo, caplog): repo_dir, _ = fake_repo git_obj = scm.Git(f"file://{repo_dir}", "master") verify_log_msg = f"Verifying the archive at {git_obj.sources_dir.archive_path}" already_created_log_msg = ( f"{git_obj.sources_dir.archive_path} was created while this task was running. " "Will proceed with that archive") git_obj._create_archive(repo_dir) assert verify_log_msg in caplog.text assert already_created_log_msg not in caplog.text caplog.clear() # create archive again to simulate race condition. This should not generate errors git_obj._create_archive(repo_dir) assert verify_log_msg in caplog.text assert already_created_log_msg in caplog.text
def test_clone_and_archive_checkout_failed(mock_archive_path, mock_git_clone, mock_temp_dir): # Mock the tempfile.TemporaryDirectory context manager mock_temp_dir.return_value.__enter__.return_value = '/tmp/cachito-temp' # Mock the git calls mock_git_clone.return_value.commit.side_effect = git.GitCommandError( 'commit is invalid', 1) # Mock the Git.archives_dir property mock_archive_path.return_value = archive_path git_obj = scm.Git(url, ref) expected = ( 'Checking out the Git repository failed. Please verify the supplied reference of ' f'"{ref}" is valid.') with pytest.raises(CachitoError, match=expected): git_obj.clone_and_archive()
def test_clone_and_archive_git_archive_failed(mock_archive_path, mock_run, mock_temp_dir, archive_error, expected_error): # Mock the tempfile.TemporaryDirectory context manager mock_temp_dir.return_value.__enter__.return_value = '/tmp/cachito-temp' # Mock the git calls mock_clone = mock.Mock() mock_clone.returncode = 0 mock_archive = mock.Mock() mock_archive.returncode = 1 mock_archive.stderr = archive_error mock_run.side_effect = [mock_clone, mock_archive] # Mock the Git.archives_dir property mock_archive_path.return_value = archive_path git = scm.Git(url, ref) with pytest.raises(CachitoError, match=expected_error): git.clone_and_archive()
def test_clone_and_archive(mock_archive_path, mock_run, mock_temp_dir): # Mock the tempfile.TemporaryDirectory context manager mock_temp_dir.return_value.__enter__.return_value = '/tmp/cachito-temp' # Mock the git calls mock_run.return_value.returncode = 0 # Mock the Git.archives_dir property mock_archive_path.return_value = archive_path git = scm.Git(url, ref) git.clone_and_archive() # Verify the tempfile.TemporaryDirectory context manager was used mock_temp_dir.return_value.__enter__.assert_called_once() # Verify the git calls were correct mock_run.assert_has_calls([ mock.call([ 'git', 'clone', '-q', '--no-checkout', 'https://github.com/release-engineering/retrodep.git', '/tmp/cachito-temp/repo', ], capture_output=True, universal_newlines=True, encoding='utf-8'), mock.call([ 'git', '-C', '/tmp/cachito-temp/repo', 'archive', '-o', f'/tmp/cachito-archives/release-engineering/retrodep/{ref}.tar.gz', '--prefix=app/', 'c50b93a32df1c9d700e3e80996845bc2e13be848', ], capture_output=True, universal_newlines=True, encoding='utf-8'), ])
def test_update_git_submodules(mock_repo): git_obj = scm.Git(url, ref) git_obj.update_git_submodules(mock_repo) # Verify the git submodule update was called correctly mock_repo.submodule_update.assert_called_once_with(recursive=False)
def test_verify_archive_not_available(): git_obj = scm.Git("invalid", "ref") err_msg = f"No valid archive found at {git_obj.sources_dir.archive_path}" with pytest.raises(CachitoError, match=err_msg): git_obj._verify_archive()
def test_repo_name(): git_obj = scm.Git(url, ref) assert git_obj.repo_name == "release-engineering/retrodep"