def test_cli(self): with temp_dir() as td: p = b'Hello, world!' # Write ~1.67 chunks of this phrase f_in = BytesIO(p * int(cli._CHUNK_SIZE * 5 / (3 * len(p)))) f_sid = BytesIO() cli.main([ '--storage', Path.json_dumps({ 'kind': 'filesystem', 'key': 'test', 'base_dir': td / 'storage', }), 'put' ], from_file=f_in, to_file=f_sid) self.assertTrue(f_sid.getvalue().endswith(b'\n')) sid = f_sid.getvalue()[:-1] f_out = BytesIO() cli.main([ '--storage', Path.json_dumps({ 'kind': 'filesystem', 'key': 'test', 'base_dir': td / 'storage', }), 'get', sid ], from_file=None, to_file=f_out) self.assertEqual(f_in.getvalue(), f_out.getvalue())
def test_snapshot(self): with temp_repos.temp_repos_steps( gpg_signing_key=temp_repos.get_test_signing_key(), repo_change_steps=[{'dog': temp_repos.SAMPLE_STEPS[0]['dog']}] ) as repos_root, temp_dir() as td: with open(td / 'fake_gpg_key', 'w'): pass whitelist_dir = td / 'gpg_whitelist' os.mkdir(whitelist_dir) shutil.copy(td / 'fake_gpg_key', whitelist_dir) storage_dict = { 'key': 'test', 'kind': 'filesystem', 'base_dir': td / 'storage', } snapshot_repo([ '--repo-universe=fakeverse', '--repo-name=dog', '--repo-url=' + (repos_root / "0/dog").file_url(), f'--gpg-key-whitelist-dir={whitelist_dir}', '--gpg-url=' + (td / 'fake_gpg_key').file_url(), f'--snapshot-dir={td / "snap"}', f'--storage={Path.json_dumps(storage_dict)}', '--db=' + Path.json_dumps({ 'kind': 'sqlite', 'db_path': td / 'db.sqlite3', }), '--threads=4', ]) # This test simply checks the overall integration, so we don't # bother looking inside the DB or Storage, or inspecting the # details of the snapshot -- those should all be covered by # lower-level tests. with sqlite3.connect(RepoSnapshot.fetch_sqlite_from_storage( Storage.make(**storage_dict), td / 'snap', td / 'snapshot.sql3', )) as db: self.assertEqual({ 'dog-pkgs/rpm-test-carrot-2-rc0.x86_64.rpm', 'dog-pkgs/rpm-test-mice-0.1-a.x86_64.rpm', 'dog-pkgs/rpm-test-milk-1.41-42.x86_64.rpm', 'dog-pkgs/rpm-test-mutable-a-f.x86_64.rpm', }, { path for path, in db.execute( 'SELECT "path" FROM "rpm";' ).fetchall() })
def test_snapshot(self): with temp_repos.temp_repos_steps( gpg_signing_key=temp_repos.get_test_signing_key(), repo_change_steps=[ { # All of the `snap0` repos are in the "mammal" universe 'bunny': temp_repos.SAMPLE_STEPS[0]['bunny'], 'cat': temp_repos.SAMPLE_STEPS[0]['cat'], 'dog': temp_repos.SAMPLE_STEPS[0]['dog'], 'kitteh': 'cat', 'gonna_skip_for_0': 'bunny', }, # None of these are in the "mammal" universe, see `ru_json` # below. { # 'bunny' stays unchanged, with the step 0 `repomd.xml` 'cat': temp_repos.SAMPLE_STEPS[1]['cat'], 'dog': temp_repos.SAMPLE_STEPS[1]['dog'], # 'kitteh' stays unchanged, with the step 0 `repomd.xml` }, ] ) as repos_root, temp_dir() as td: storage_dict = { 'key': 'test', 'kind': 'filesystem', 'base_dir': td / 'storage', } repo_db_path = td / 'db.sqlite3' # Mock all repomd fetch timestamps to be identical to test that # multiple universes do not collide. orig_store_repomd = repo_db.RepoDBContext.store_repomd with unittest.mock.patch.object( repo_db.RepoDBContext, 'store_repomd', lambda self, universe_s, repo_s, repomd: orig_store_repomd( self, universe_s, repo_s, repomd._replace(fetch_timestamp=451, ), )), tempfile.NamedTemporaryFile('w') as ru_json: common_args = [ f'--gpg-key-whitelist-dir={td / "gpg_whitelist"}', '--storage=' + Path.json_dumps(storage_dict), '--db=' + Path.json_dumps({ 'kind': 'sqlite', 'db_path': repo_db_path, }), '--threads=4', ] snapshot_repos_from_args(common_args + [ '--one-universe-for-all-repos=mammal', f'--dnf-conf={repos_root / "0/dnf.conf"}', f'--yum-conf={repos_root / "0/yum.conf"}', f'--snapshot-dir={td / "snap0"}', '--exclude=gonna_skip_for_0', ]) # We want to avoid involving the "mammal" universe to # exercise the fact that a universe **not** mentioned in a # snapshot is not used for mutable RPM detection. The fact # that we also have a "zombie" exercises the fact that we do # detect cross-universe mutable RPMs when the universes # occur in the same snapshot. Search below for # `rpm-test-mutable` and `rpm-test-milk`. json.dump( { 'bunny': 'marsupial', # Same content as in snap0 'cat': 'zombie', # Changes content from snap0 'dog': 'marsupial', # Changes content from snap0 'kitteh': 'marsupial', # Same content as in snap0 'gonna_skip_for_0': 'thisone', }, ru_json) ru_json.flush() snapshot_repos_from_args(common_args + [ f'--repo-to-universe-json={ru_json.name}', f'--dnf-conf={repos_root / "1/dnf.conf"}', f'--yum-conf={repos_root / "1/yum.conf"}', f'--snapshot-dir={td / "snap1"}', ]) updated_headers = {} orig_headers = {} for snap, conf_type in zip(['snap0', 'snap1'], ['yum', 'dnf']): updated_path = td / snap / f'{conf_type}.conf' orig_path = Path(updated_path + b'.orig') updated_headers[snap] = _read_conf_headers(updated_path) orig_headers[snap] = _read_conf_headers(orig_path) # For snap0, orig file should differ from conf since we excluded excluded = '[gonna_skip_for_0]' self.assertNotIn(excluded, updated_headers['snap0']) self.assertEqual(updated_headers['snap0'] | {excluded}, orig_headers['snap0']) # For snap1, orig file should equal conf self.assertEqual(updated_headers['snap1'], orig_headers['snap1']) with sqlite3.connect(repo_db_path) as db: # Check that repomd rows are repeated or duplicated as we'd # expect across `snap[01]`, and the universes. repo_mds = sorted( db.execute(''' SELECT "universe", "repo", "fetch_timestamp", "checksum" FROM "repo_metadata" ''').fetchall()) self.assertEqual( [ ('mammal', 'bunny', 451), # snap0 ('mammal', 'cat', 451), # snap0 ('mammal', 'dog', 451), # snap0 ('mammal', 'kitteh', 451), # snap0 -- index -5 ('marsupial', 'bunny', 451), # snap1 ('marsupial', 'dog', 451), # snap1 ('marsupial', 'kitteh', 451), # snap1 -- index -2 ('thisone', 'gonna_skip_for_0', 451), # snap1 ('zombie', 'cat', 451), # snap1 ], [r[:3] for r in repo_mds]) # The kittehs have the same checksums, but exist separately # due to being in different universes. self.assertEqual(repo_mds[-3][1:], repo_mds[-6][1:]) def _fetch_sorted_by_nevra(nevra): return sorted( db.execute( ''' SELECT "universe", "name", "epoch", "version", "release", "arch", "checksum" FROM "rpm" WHERE "name" = ? AND "epoch" = ? AND "version" = ? AND "release" = ? AND "arch" = ? ''', nevra).fetchall()) # We expect this identical "carrot" RPM (same checksums) to # be repeated because it occurs in two different universes. kitteh_carrot_nevra = [ 'rpm-test-carrot', 0, '1', 'lockme', 'x86_64', ] kitteh_carrots = _fetch_sorted_by_nevra(kitteh_carrot_nevra) kitteh_carrot_chksum = kitteh_carrots[0][-1] self.assertEqual( [ # step0 cat & kitteh ('mammal', *kitteh_carrot_nevra, kitteh_carrot_chksum), # step1 kitteh ('marsupial', *kitteh_carrot_nevra, kitteh_carrot_chksum), ], kitteh_carrots) # This RPM has two variants for its contents at step 1. # This creates a mutable RPM error in `snap1`. Note that we # detect it even though the variants are in different # universes. milk2_nevra = ['rpm-test-milk', 0, '2.71', '8', 'x86_64'] milk2s = _fetch_sorted_by_nevra(milk2_nevra) milk2_chksum_step0 = milk2s[0][-1] # mammal sorts first milk2_chksum_step1, = {milk2s[1][-1], milk2s[2][-1] } - {milk2_chksum_step0} self.assertEqual( [ # snap0 cat & kitteh ('mammal', *milk2_nevra, milk2_chksum_step0), # snap1 kitteh -- mutable RPM error vs "snap1 cat" ('marsupial', *milk2_nevra, milk2_chksum_step0), # snap1 cat -- mutable RPM error vs "snap1 kitteh" ('zombie', *milk2_nevra, milk2_chksum_step1), ], milk2s) # This RPM changes contents between step 0 and step 1, but # since the "mammal" universe is not used in step 1, there # is no mutable RPM error. mutable_nevra = ['rpm-test-mutable', 0, 'a', 'f', 'x86_64'] mutables = _fetch_sorted_by_nevra(mutable_nevra) mutable_chksum_dog = mutables[0][-1] # mammal sorts first mutable_chksum_cat = mutables[1][-1] self.assertEqual( sorted([ # snap0 dog ('mammal', *mutable_nevra, mutable_chksum_dog), # snap1 cat ('zombie', *mutable_nevra, mutable_chksum_cat), ]), mutables) # As with `test_snapshot_repo`, this is not a complete check of # the snapshot state. We only check for sanity, and for the # interactions between multiple snapshots & multiple universes. # Lower-level tests check many other lower-level details. mutable_a_f_checksums = set() milk2_checksums = set() expected_errors = 1 for snap_name, expected_rows in [ # These are just straight up "bunny", "cat" (with alias), # and "dog" from SAMPLE_STEPS[0], as indicated in our setup. ('snap0', { ('bunny', 'bunny-pkgs/rpm-test-carrot-2-rc0'), ('cat', 'cat-pkgs/rpm-test-carrot-1-lockme'), ('cat', 'cat-pkgs/rpm-test-mice-0.1-a'), ('cat', 'cat-pkgs/rpm-test-milk-2.71-8'), ('dog', 'dog-pkgs/rpm-test-milk-1.41-42'), ('dog', 'dog-pkgs/rpm-test-carrot-2-rc0'), ('dog', 'dog-pkgs/rpm-test-mice-0.1-a'), ('dog', 'dog-pkgs/rpm-test-mutable-a-f'), ('kitteh', 'cat-pkgs/rpm-test-carrot-1-lockme'), ('kitteh', 'cat-pkgs/rpm-test-mice-0.1-a'), ('kitteh', 'cat-pkgs/rpm-test-milk-2.71-8'), }), # These are "bunny" & "cat" (as "kitteh") from # SAMPLE_STEPS[0], plus "cat" & "dog from SAMPLE_STEPS[1]. # ( 'snap1', { ('bunny', 'bunny-pkgs/rpm-test-carrot-2-rc0'), ('cat', 'cat-pkgs/rpm-test-milk-2.71-8'), # may error ('cat', 'cat-pkgs/rpm-test-mice-0.2-rc0'), # We'd have gotten a "mutable RPM" error if this # were in the same universe as the "mutable" from # "dog" in snap0. ('cat', 'cat-pkgs/rpm-test-mutable-a-f'), ('dog', 'dog-pkgs/rpm-test-carrot-2-rc0'), ('dog', 'dog-pkgs/rpm-test-bone-5i-beef'), ('gonna_skip_for_0', 'bunny-pkgs/rpm-test-carrot-2-rc0'), ('kitteh', 'cat-pkgs/rpm-test-carrot-1-lockme'), ('kitteh', 'cat-pkgs/rpm-test-mice-0.1-a'), ('kitteh', 'cat-pkgs/rpm-test-milk-2.71-8'), # may error }), ]: with sqlite3.connect( RepoSnapshot.fetch_sqlite_from_storage( Storage.make(**storage_dict), td / snap_name, td / snap_name / 'snapshot.sql3', )) as db: rows = db.execute( 'SELECT "repo", "path", "error", "checksum" FROM "rpm"' ).fetchall() self.assertEqual( {(r, p + '.x86_64.rpm') for r, p in expected_rows}, {(r, p) for r, p, _e, _c in rows}) for repo, path, error, chksum in rows: # There is just 1 error among all the rows. The # "milk-2.71" RPM from either "kitteh" or "cat" in # `snap1` gets marked with "mutable_rpm". Which # repo gets picked depends on the (shuffled) order # of the snapshot. If we were to run the `snap1` # snapshot a second time, both would get marked. if error is not None: expected_errors -= 1 self.assertEqual(( 'snap1', 'cat-pkgs/rpm-test-milk-2.71-8.x86_64.rpm', 'mutable_rpm', ), (snap_name, path, error), repo) self.assertIn(repo, {'cat', 'kitteh'}) # Sanity-check checksums self.assertTrue(chksum.startswith('sha384:'), chksum) if path == 'cat-pkgs/rpm-test-milk-2.71-8.x86_64.rpm': milk2_checksums.add(chksum) if path.endswith('rpm-test-mutable-a-f.x86_64.rpm'): mutable_a_f_checksums.add(chksum) self.assertEqual(0, expected_errors) self.assertEqual(2, len(milk2_checksums)) self.assertEqual(2, len(mutable_a_f_checksums))