def _get_location_row(location): if location in LOCATION_MAP: return LOCATION_MAP[location] location_row = ImageStorageLocation.get(name=location) LOCATION_MAP[location] = location_row return location_row
def process(resources): response = [] changed = True for resource in resources: p_state = resource["state"] p_name = resource["name"] try: image_storage_location = ImageStorageLocation.get(name=p_name) except ImageStorageLocation.DoesNotExist: image_storage_location = None if p_state == "absent": if image_storage_location is not None: changed = True image_storage_location.delete_instance() response.append("Image Storage Location '%s' deleted" % p_name) changed = True else: response.append("Image Storage Location '%s' does not exist" % p_name) else: if image_storage_location is None: changed = True image_storage_location = ImageStorageLocation.create( name=p_name) response.append("Image Storage Location '%s' created" % p_name) else: response.append("Image Storage Location '%s' exists" % p_name) return {"failed": False, "changed": changed, "meta": response}, 200
def _populate_blob(repo, content): digest = str(sha256_digest(content)) location = ImageStorageLocation.get(name="local_us") storage.put_content(["local_us"], storage.blob_path(digest), "somecontent") blob = model.blob.store_blob_record_and_temp_link_in_repo( repo, digest, location, len(content), 120) return blob, digest
def get_or_create_shared_blob(digest, byte_data, storage): """ Returns the ImageStorage blob with the given digest or, if not present, adds a row and writes the given byte data to the storage engine. This method is *only* to be used for shared blobs that are globally accessible, such as the special empty gzipped tar layer that Docker no longer pushes to us. """ assert digest assert byte_data is not None and isinstance(byte_data, bytes) assert storage try: return ImageStorage.get(content_checksum=digest) except ImageStorage.DoesNotExist: preferred = storage.preferred_locations[0] location_obj = ImageStorageLocation.get(name=preferred) record = ImageStorage.create(image_size=len(byte_data), content_checksum=digest) try: storage.put_content([preferred], storage_model.get_layer_path(record), byte_data) ImageStoragePlacement.create(storage=record, location=location_obj) except: logger.exception("Exception when trying to write special layer %s", digest) record.delete_instance() raise return record
def _populate_blob(content): digest = str(sha256_digest(content)) location = ImageStorageLocation.get(name="local_us") blob = store_blob_record_and_temp_link("devtable", "newrepo", digest, location, len(content), 120) storage.put_content(["local_us"], get_layer_path(blob), content) return blob, digest
def test_blob_caching(method, endpoint, client, app): digest = "sha256:" + hashlib.sha256(b"a").hexdigest() location = ImageStorageLocation.get(name="local_us") model.blob.store_blob_record_and_temp_link("devtable", "simple", digest, location, 1, 10000000) params = { "repository": "devtable/simple", "digest": digest, } user = model.user.get_user("devtable") access = [{ "type": "repository", "name": "devtable/simple", "actions": ["pull"], }] context, subject = build_context_and_subject( ValidatedAuthContext(user=user)) token = generate_bearer_token(realapp.config["SERVER_HOSTNAME"], subject, context, access, 600, instance_keys) headers = { "Authorization": "Bearer %s" % token.decode("ascii"), } # Run without caching to make sure the request works. This also preloads some of # our global model caches. conduct_call(client, "v2." + endpoint, url_for, method, params, expected_code=200, headers=headers) with patch("endpoints.v2.blob.model_cache", InMemoryDataModelCache(TEST_CACHE_CONFIG)): # First request should make a DB query to retrieve the blob. conduct_call(client, "v2." + endpoint, url_for, method, params, expected_code=200, headers=headers) # Subsequent requests should use the cached blob. with assert_query_count(0): conduct_call( client, "v2." + endpoint, url_for, method, params, expected_code=200, headers=headers, )
def _populate_blob(repo, content): assert isinstance(repo, Repository) assert isinstance(content, bytes) digest = str(sha256_digest(content)) location = ImageStorageLocation.get(name="local_us") blob = model.blob.store_blob_record_and_temp_link_in_repo( repo, digest, location, len(content), TEMP_BLOB_EXPIRATION) return blob, digest
def _populate_blob(repo, content): assert isinstance(content, bytes) digest = sha256_digest(content) location = ImageStorageLocation.get(name="local_us") storage.put_content(["local_us"], storage.blob_path(digest), content) blob = model.blob.store_blob_record_and_temp_link_in_repo( repo, digest, location, len(content), 120) return blob, digest
def test_blob_caching(method, endpoint, client, app): digest = 'sha256:' + hashlib.sha256("a").hexdigest() location = ImageStorageLocation.get(name='local_us') model.blob.store_blob_record_and_temp_link('devtable', 'simple', digest, location, 1, 10000000) params = { 'repository': 'devtable/simple', 'digest': digest, } user = model.user.get_user('devtable') access = [{ 'type': 'repository', 'name': 'devtable/simple', 'actions': ['pull'], }] context, subject = build_context_and_subject( ValidatedAuthContext(user=user)) token = generate_bearer_token(realapp.config['SERVER_HOSTNAME'], subject, context, access, 600, instance_keys) headers = { 'Authorization': 'Bearer %s' % token, } # Run without caching to make sure the request works. This also preloads some of # our global model caches. conduct_call(client, 'v2.' + endpoint, url_for, method, params, expected_code=200, headers=headers) with patch('endpoints.v2.blob.model_cache', InMemoryDataModelCache()): # First request should make a DB query to retrieve the blob. conduct_call(client, 'v2.' + endpoint, url_for, method, params, expected_code=200, headers=headers) # Subsequent requests should use the cached blob. with assert_query_count(0): conduct_call(client, 'v2.' + endpoint, url_for, method, params, expected_code=200, headers=headers)
def get_blob(layer): content = Bytes.for_string_or_unicode(layer).as_encoded_str() digest = str(sha256_digest(content)) blob = store_blob_record_and_temp_link( self.orgname, self.upstream_repository, digest, ImageStorageLocation.get(name="local_us"), len(content), 120, ) storage.put_content(["local_us"], get_layer_path(blob), content) return blob, digest
def populate_database(minimal=False): logger.debug("Populating the DB with test data.") # Check if the data already exists. If so, we skip. This can happen between calls from the # "old style" tests and the new py.test's. try: User.get(username="******") logger.debug("DB already populated") return except User.DoesNotExist: pass # Note: databases set up with "real" schema (via Alembic) will not have these types # type, so we it here it necessary. try: ImageStorageLocation.get(name="local_eu") ImageStorageLocation.get(name="local_us") except ImageStorageLocation.DoesNotExist: ImageStorageLocation.create(name="local_eu") ImageStorageLocation.create(name="local_us") try: NotificationKind.get(name="test_notification") except NotificationKind.DoesNotExist: NotificationKind.create(name="test_notification") new_user_1 = model.user.create_user("devtable", "password", "*****@*****.**") new_user_1.verified = True new_user_1.stripe_id = TEST_STRIPE_ID new_user_1.save() if minimal: logger.debug( "Skipping most db population because user requested mininal db") return UserRegion.create(user=new_user_1, location=ImageStorageLocation.get(name="local_us")) model.release.set_region_release("quay", "us", "v0.1.2") model.user.create_confirm_email_code(new_user_1, new_email="*****@*****.**") disabled_user = model.user.create_user("disabled", "password", "*****@*****.**") disabled_user.verified = True disabled_user.enabled = False disabled_user.save() dtrobot = model.user.create_robot("dtrobot", new_user_1) dtrobot2 = model.user.create_robot("dtrobot2", new_user_1) new_user_2 = model.user.create_user("public", "password", "*****@*****.**") new_user_2.verified = True new_user_2.save() new_user_3 = model.user.create_user("freshuser", "password", "*****@*****.**") new_user_3.verified = True new_user_3.save() another_robot = model.user.create_robot("anotherrobot", new_user_3) new_user_4 = model.user.create_user("randomuser", "password", "*****@*****.**") new_user_4.verified = True new_user_4.save() new_user_5 = model.user.create_user("unverified", "password", "*****@*****.**") new_user_5.save() reader = model.user.create_user("reader", "password", "*****@*****.**") reader.verified = True reader.save() creatoruser = model.user.create_user("creator", "password", "*****@*****.**") creatoruser.verified = True creatoruser.save() outside_org = model.user.create_user("outsideorg", "password", "*****@*****.**") outside_org.verified = True outside_org.save() model.notification.create_notification( "test_notification", new_user_1, metadata={ "some": "value", "arr": [1, 2, 3], "obj": { "a": 1, "b": 2 } }, ) from_date = datetime.utcnow() to_date = from_date + timedelta(hours=1) notification_metadata = { "from_date": formatdate(calendar.timegm(from_date.utctimetuple())), "to_date": formatdate(calendar.timegm(to_date.utctimetuple())), "reason": "database migration", } model.notification.create_notification("maintenance", new_user_1, metadata=notification_metadata) __generate_repository( new_user_4, "randomrepo", "Random repo repository.", False, [], (4, [], ["latest", "prod"]), ) simple_repo = __generate_repository( new_user_1, "simple", "Simple repository.", False, [], (4, [], ["latest", "prod"]), ) # Add some labels to the latest tag's manifest. repo_ref = RepositoryReference.for_repo_obj(simple_repo) tag = registry_model.get_repo_tag(repo_ref, "latest") manifest = registry_model.get_manifest_for_tag(tag) assert manifest first_label = registry_model.create_manifest_label(manifest, "foo", "bar", "manifest") registry_model.create_manifest_label(manifest, "foo", "baz", "api") registry_model.create_manifest_label(manifest, "anotherlabel", "1234", "internal") registry_model.create_manifest_label(manifest, "jsonlabel", '{"hey": "there"}', "internal", "application/json") label_metadata = { "key": "foo", "value": "bar", "id": first_label._db_id, "manifest_digest": manifest.digest, } logs_model.log_action( "manifest_label_add", new_user_1.username, performer=new_user_1, timestamp=datetime.now(), metadata=label_metadata, repository=simple_repo, ) model.blob.initiate_upload(new_user_1.username, simple_repo.name, str(uuid4()), "local_us", {}) model.notification.create_repo_notification(simple_repo, "repo_push", "quay_notification", {}, {}) __generate_repository( new_user_1, "sharedtags", "Shared tags repository", False, [(new_user_2, "read"), (dtrobot[0], "read")], ( 2, [ (3, [], ["v2.0", "v2.1", "v2.2"]), ( 1, [(1, [(1, [], ["prod", "581a284"]) ], ["staging", "8423b58"]), (1, [], None)], None, ), ], None, ), ) __generate_repository( new_user_1, "history", "Historical repository.", False, [], (4, [(2, [], "#latest"), (3, [], "latest")], None), ) __generate_repository( new_user_1, "complex", "Complex repository with many branches and tags.", False, [(new_user_2, "read"), (dtrobot[0], "read")], ( 2, [(3, [], "v2.0"), (1, [(1, [(2, [], ["prod"])], "staging"), (1, [], None)], None)], None, ), ) __generate_repository( new_user_1, "gargantuan", None, False, [], ( 2, [ (3, [], "v2.0"), (1, [(1, [(1, [], ["latest", "prod"])], "staging"), (1, [], None)], None), (20, [], "v3.0"), (5, [], "v4.0"), (1, [(1, [], "v5.0"), (1, [], "v6.0")], None), ], None, ), ) trusted_repo = __generate_repository( new_user_1, "trusted", "Trusted repository.", False, [], (4, [], ["latest", "prod"]), ) trusted_repo.trust_enabled = True trusted_repo.save() publicrepo = __generate_repository( new_user_2, "publicrepo", "Public repository pullable by the world.", True, [], (10, [], "latest"), ) __generate_repository(outside_org, "coolrepo", "Some cool repo.", False, [], (5, [], "latest")) __generate_repository( new_user_1, "shared", "Shared repository, another user can write.", False, [(new_user_2, "write"), (reader, "read")], (5, [], "latest"), ) __generate_repository( new_user_1, "text-full-repo", "This is a repository for testing text search", False, [(new_user_2, "write"), (reader, "read")], (5, [], "latest"), ) building = __generate_repository( new_user_1, "building", "Empty repository which is building.", False, [(new_user_2, "write"), (reader, "read")], (0, [], None), ) new_token = model.token.create_access_token(building, "write", "build-worker") trigger = model.build.create_build_trigger(building, "github", "123authtoken", new_user_1, pull_robot=dtrobot[0]) trigger.config = json.dumps({ "build_source": "jakedt/testconnect", "subdir": "", "dockerfile_path": "Dockerfile", "context": "/", }) trigger.save() repo = "ci.devtable.com:5000/%s/%s" % (building.namespace_user.username, building.name) job_config = { "repository": repo, "docker_tags": ["latest"], "build_subdir": "", "trigger_metadata": { "commit": "3482adc5822c498e8f7db2e361e8d57b3d77ddd9", "ref": "refs/heads/master", "default_branch": "master", }, } model.repository.star_repository(new_user_1, simple_repo) record = model.repository.create_email_authorization_for_repo( new_user_1.username, "simple", "*****@*****.**") record.confirmed = True record.save() model.repository.create_email_authorization_for_repo( new_user_1.username, "simple", "*****@*****.**") build2 = model.build.create_repository_build( building, new_token, job_config, "68daeebd-a5b9-457f-80a0-4363b882f8ea", "build-name", trigger, ) build2.uuid = "deadpork-dead-pork-dead-porkdeadpork" build2.save() build3 = model.build.create_repository_build( building, new_token, job_config, "f49d07f9-93da-474d-ad5f-c852107c3892", "build-name", trigger, ) build3.uuid = "deadduck-dead-duck-dead-duckdeadduck" build3.save() build1 = model.build.create_repository_build( building, new_token, job_config, "701dcc3724fb4f2ea6c31400528343cd", "build-name", trigger) build1.uuid = "deadbeef-dead-beef-dead-beefdeadbeef" build1.save() org = model.organization.create_organization("buynlarge", "*****@*****.**", new_user_1) org.stripe_id = TEST_STRIPE_ID org.save() liborg = model.organization.create_organization( "library", "*****@*****.**", new_user_1) liborg.save() titiorg = model.organization.create_organization("titi", "*****@*****.**", new_user_1) titiorg.save() thirdorg = model.organization.create_organization( "sellnsmall", "*****@*****.**", new_user_1) thirdorg.save() model.user.create_robot("coolrobot", org) oauth_app_1 = model.oauth.create_application( org, "Some Test App", "http://localhost:8000", "http://localhost:8000/o2c.html", client_id="deadbeef", ) model.oauth.create_application( org, "Some Other Test App", "http://quay.io", "http://localhost:8000/o2c.html", client_id="deadpork", description="This is another test application", ) model.oauth.create_user_access_token(new_user_1, "deadbeef", "repo:admin", access_token="%s%s" % ("b" * 40, "c" * 40)) oauth_credential = Credential.from_string("dswfhasdf1") OAuthAuthorizationCode.create( application=oauth_app_1, code="Z932odswfhasdf1", scope="repo:admin", data='{"somejson": "goeshere"}', code_name="Z932odswfhasdf1Z932o", code_credential=oauth_credential, ) model.user.create_robot("neworgrobot", org) ownerbot = model.user.create_robot("ownerbot", org)[0] creatorbot = model.user.create_robot("creatorbot", org)[0] owners = model.team.get_organization_team("buynlarge", "owners") owners.description = "Owners have unfetterd access across the entire org." owners.save() org_repo = __generate_repository( org, "orgrepo", "Repository owned by an org.", False, [(outside_org, "read")], (4, [], ["latest", "prod"]), ) __generate_repository( org, "anotherorgrepo", "Another repository owned by an org.", False, [], (4, [], ["latest", "prod"]), ) creators = model.team.create_team("creators", org, "creator", "Creators of orgrepo.") reader_team = model.team.create_team("readers", org, "member", "Readers of orgrepo.") model.team.add_or_invite_to_team(new_user_1, reader_team, outside_org) model.permission.set_team_repo_permission(reader_team.name, org_repo.namespace_user.username, org_repo.name, "read") model.team.add_user_to_team(new_user_2, reader_team) model.team.add_user_to_team(reader, reader_team) model.team.add_user_to_team(ownerbot, owners) model.team.add_user_to_team(creatorbot, creators) model.team.add_user_to_team(creatoruser, creators) sell_owners = model.team.get_organization_team("sellnsmall", "owners") sell_owners.description = "Owners have unfettered access across the entire org." sell_owners.save() model.team.add_user_to_team(new_user_4, sell_owners) sync_config = { "group_dn": "cn=Test-Group,ou=Users", "group_id": "somegroupid" } synced_team = model.team.create_team("synced", org, "member", "Some synced team.") model.team.set_team_syncing(synced_team, "ldap", sync_config) another_synced_team = model.team.create_team("synced", thirdorg, "member", "Some synced team.") model.team.set_team_syncing(another_synced_team, "ldap", {"group_dn": "cn=Test-Group,ou=Users"}) __generate_repository( new_user_1, "superwide", None, False, [], [ (10, [], "latest2"), (2, [], "latest3"), (2, [(1, [], "latest11"), (2, [], "latest12")], "latest4"), (2, [], "latest5"), (2, [], "latest6"), (2, [], "latest7"), (2, [], "latest8"), (2, [], "latest9"), (2, [], "latest10"), (2, [], "latest13"), (2, [], "latest14"), (2, [], "latest15"), (2, [], "latest16"), (2, [], "latest17"), (2, [], "latest18"), ], ) mirror_repo = __generate_repository( new_user_1, "mirrored", "Mirrored repository.", False, [(dtrobot[0], "write"), (dtrobot2[0], "write")], (4, [], ["latest", "prod"]), ) mirror_rule = model.repo_mirror.create_mirroring_rule( mirror_repo, ["latest", "3.3*"]) mirror_args = (mirror_repo, mirror_rule, dtrobot[0], "quay.io/coreos/etcd", 60 * 60 * 24) mirror_kwargs = { "external_registry_username": "******", "external_registry_password": "******", "external_registry_config": {}, "is_enabled": True, "sync_start_date": datetime.utcnow(), } mirror = model.repo_mirror.enable_mirroring_for_repository( *mirror_args, **mirror_kwargs) read_only_repo = __generate_repository( new_user_1, "readonly", "Read-Only Repo.", False, [], (4, [], ["latest", "prod"]), ) read_only_repo.state = RepositoryState.READ_ONLY read_only_repo.save() model.permission.add_prototype_permission(org, "read", activating_user=new_user_1, delegate_user=new_user_2) model.permission.add_prototype_permission(org, "read", activating_user=new_user_1, delegate_team=reader_team) model.permission.add_prototype_permission(org, "write", activating_user=new_user_2, delegate_user=new_user_1) today = datetime.today() week_ago = today - timedelta(6) six_ago = today - timedelta(5) four_ago = today - timedelta(4) yesterday = datetime.combine(date.today(), datetime.min.time()) - timedelta(hours=6) __generate_service_key("kid1", "somesamplekey", new_user_1, today, ServiceKeyApprovalType.SUPERUSER) __generate_service_key( "kid2", "someexpiringkey", new_user_1, week_ago, ServiceKeyApprovalType.SUPERUSER, today + timedelta(days=14), ) __generate_service_key("kid3", "unapprovedkey", new_user_1, today, None) __generate_service_key( "kid4", "autorotatingkey", new_user_1, six_ago, ServiceKeyApprovalType.KEY_ROTATION, today + timedelta(days=1), rotation_duration=timedelta(hours=12).total_seconds(), ) __generate_service_key( "kid5", "key for another service", new_user_1, today, ServiceKeyApprovalType.SUPERUSER, today + timedelta(days=14), service="different_sample_service", ) __generate_service_key( "kid6", "someexpiredkey", new_user_1, week_ago, ServiceKeyApprovalType.SUPERUSER, today - timedelta(days=1), ) __generate_service_key( "kid7", "somewayexpiredkey", new_user_1, week_ago, ServiceKeyApprovalType.SUPERUSER, today - timedelta(days=30), ) # Add the test pull key as pre-approved for local and unittest registry testing. # Note: this must match the private key found in the local/test config. _TEST_JWK = { "e": "AQAB", "kty": "RSA", "n": "yqdQgnelhAPMSeyH0kr3UGePK9oFOmNfwD0Ymnh7YYXr21VHWwyM2eVW3cnLd9KXywDFtGSe9oFDbnOuMCdUowdkBcaHju-isbv5KEbNSoy_T2Rip-6L0cY63YzcMJzv1nEYztYXS8wz76pSK81BKBCLapqOCmcPeCvV9yaoFZYvZEsXCl5jjXN3iujSzSF5Z6PpNFlJWTErMT2Z4QfbDKX2Nw6vJN6JnGpTNHZvgvcyNX8vkSgVpQ8DFnFkBEx54PvRV5KpHAq6AsJxKONMo11idQS2PfCNpa2hvz9O6UZe-eIX8jPo5NW8TuGZJumbdPT_nxTDLfCqfiZboeI0Pw", } key = model.service_keys.create_service_key("test_service_key", "test_service_key", "quay", _TEST_JWK, {}, None) model.service_keys.approve_service_key( key.kid, ServiceKeyApprovalType.SUPERUSER, notes="Test service key for local/test registry testing", ) # Add an app specific token. token = model.appspecifictoken.create_token(new_user_1, "some app") token.token_name = "a" * 60 token.token_secret = "b" * 60 token.save() logs_model.log_action( "org_create_team", org.username, performer=new_user_1, timestamp=week_ago, metadata={"team": "readers"}, ) logs_model.log_action( "org_set_team_role", org.username, performer=new_user_1, timestamp=week_ago, metadata={ "team": "readers", "role": "read" }, ) logs_model.log_action( "create_repo", org.username, performer=new_user_1, repository=org_repo, timestamp=week_ago, metadata={ "namespace": org.username, "repo": "orgrepo" }, ) logs_model.log_action( "change_repo_permission", org.username, performer=new_user_2, repository=org_repo, timestamp=six_ago, metadata={ "username": new_user_1.username, "repo": "orgrepo", "role": "admin" }, ) logs_model.log_action( "change_repo_permission", org.username, performer=new_user_1, repository=org_repo, timestamp=six_ago, metadata={ "username": new_user_2.username, "repo": "orgrepo", "role": "read" }, ) logs_model.log_action( "add_repo_accesstoken", org.username, performer=new_user_1, repository=org_repo, timestamp=four_ago, metadata={ "repo": "orgrepo", "token": "deploytoken" }, ) logs_model.log_action( "push_repo", org.username, performer=new_user_2, repository=org_repo, timestamp=today, metadata={ "username": new_user_2.username, "repo": "orgrepo" }, ) logs_model.log_action( "pull_repo", org.username, performer=new_user_2, repository=org_repo, timestamp=today, metadata={ "username": new_user_2.username, "repo": "orgrepo" }, ) logs_model.log_action( "pull_repo", org.username, repository=org_repo, timestamp=today, metadata={ "token": "sometoken", "token_code": "somecode", "repo": "orgrepo" }, ) logs_model.log_action( "delete_tag", org.username, performer=new_user_2, repository=org_repo, timestamp=today, metadata={ "username": new_user_2.username, "repo": "orgrepo", "tag": "sometag" }, ) logs_model.log_action( "pull_repo", org.username, repository=org_repo, timestamp=today, metadata={ "token_code": "somecode", "repo": "orgrepo" }, ) logs_model.log_action( "pull_repo", new_user_2.username, repository=publicrepo, timestamp=yesterday, metadata={ "token_code": "somecode", "repo": "publicrepo" }, ) logs_model.log_action( "build_dockerfile", new_user_1.username, repository=building, timestamp=today, metadata={ "repo": "building", "namespace": new_user_1.username, "trigger_id": trigger.uuid, "config": json.loads(trigger.config), "service": trigger.service.name, }, ) model.message.create([{ "content": "We love you, Quay customers!", "severity": "info", "media_type": "text/plain", }]) model.message.create([{ "content": "This is a **development** install of Quay", "severity": "warning", "media_type": "text/markdown", }]) fake_queue = WorkQueue("fakequeue", tf) fake_queue.put(["canonical", "job", "name"], "{}") model.user.create_user_prompt(new_user_4, "confirm_username") for to_count in Repository.select(): model.repositoryactioncount.count_repository_actions( to_count, datetime.utcnow()) model.repositoryactioncount.update_repository_score(to_count)
def _populate_blob(digest): location = ImageStorageLocation.get(name='local_us') store_blob_record_and_temp_link('devtable', 'simple', digest, location, 1, 120)
def _populate_blob(digest): location = ImageStorageLocation.get(name="local_us") store_blob_record_and_temp_link("devtable", "simple", digest, location, 1, 120)
def test_blob_mounting( mount_digest, source_repo, username, include_from_param, expected_code, client, app ): location = ImageStorageLocation.get(name="local_us") # Store and link some blobs. digest = "sha256:" + hashlib.sha256(b"a").hexdigest() model.blob.store_blob_record_and_temp_link("devtable", "simple", digest, location, 1, 10000000) digest = "sha256:" + hashlib.sha256(b"b").hexdigest() model.blob.store_blob_record_and_temp_link("devtable", "complex", digest, location, 1, 10000000) digest = "sha256:" + hashlib.sha256(b"c").hexdigest() model.blob.store_blob_record_and_temp_link( "public", "publicrepo", digest, location, 1, 10000000 ) params = { "repository": "devtable/building", "mount": mount_digest, } if include_from_param: params["from"] = source_repo user = model.user.get_user(username) access = [ { "type": "repository", "name": "devtable/building", "actions": ["pull", "push"], } ] if source_repo.find(username) == 0: access.append( { "type": "repository", "name": source_repo, "actions": ["pull"], } ) context, subject = build_context_and_subject(ValidatedAuthContext(user=user)) token = generate_bearer_token( realapp.config["SERVER_HOSTNAME"], subject, context, access, 600, instance_keys ) headers = { "Authorization": "Bearer %s" % token.decode("ascii"), } conduct_call( client, "v2.start_blob_upload", url_for, "POST", params, expected_code=expected_code, headers=headers, ) repository = model.repository.get_repository("devtable", "building") if expected_code == 201: # Ensure the blob now exists under the repo. assert model.oci.blob.get_repository_blob_by_digest(repository, mount_digest) else: assert model.oci.blob.get_repository_blob_by_digest(repository, mount_digest) is None
def test_blob_mounting(mount_digest, source_repo, username, expect_success, client, app): location = ImageStorageLocation.get(name='local_us') # Store and link some blobs. digest = 'sha256:' + hashlib.sha256("a").hexdigest() model.blob.store_blob_record_and_temp_link('devtable', 'simple', digest, location, 1, 10000000) digest = 'sha256:' + hashlib.sha256("b").hexdigest() model.blob.store_blob_record_and_temp_link('devtable', 'complex', digest, location, 1, 10000000) digest = 'sha256:' + hashlib.sha256("c").hexdigest() model.blob.store_blob_record_and_temp_link('public', 'publicrepo', digest, location, 1, 10000000) params = { 'repository': 'devtable/building', 'mount': mount_digest, 'from': source_repo, } user = model.user.get_user(username) access = [{ 'type': 'repository', 'name': 'devtable/building', 'actions': ['pull', 'push'], }] if source_repo.find(username) == 0: access.append({ 'type': 'repository', 'name': source_repo, 'actions': ['pull'], }) context, subject = build_context_and_subject( ValidatedAuthContext(user=user)) token = generate_bearer_token(realapp.config['SERVER_HOSTNAME'], subject, context, access, 600, instance_keys) headers = { 'Authorization': 'Bearer %s' % token, } expected_code = 201 if expect_success else 202 conduct_call(client, 'v2.start_blob_upload', url_for, 'POST', params, expected_code=expected_code, headers=headers) if expect_success: # Ensure the blob now exists under the repo. model.blob.get_repo_blob_by_digest('devtable', 'building', mount_digest) else: with pytest.raises(model.blob.BlobDoesNotExist): model.blob.get_repo_blob_by_digest('devtable', 'building', mount_digest)
def populate_database(minimal=False, with_storage=False): logger.debug('Populating the DB with test data.') # Check if the data already exists. If so, we skip. This can happen between calls from the # "old style" tests and the new py.test's. try: User.get(username='******') logger.debug('DB already populated') return except User.DoesNotExist: pass # Note: databases set up with "real" schema (via Alembic) will not have these types # type, so we it here it necessary. try: ImageStorageLocation.get(name='local_eu') ImageStorageLocation.get(name='local_us') except ImageStorageLocation.DoesNotExist: ImageStorageLocation.create(name='local_eu') ImageStorageLocation.create(name='local_us') try: NotificationKind.get(name='test_notification') except NotificationKind.DoesNotExist: NotificationKind.create(name='test_notification') new_user_1 = model.user.create_user('devtable', 'password', '*****@*****.**') new_user_1.verified = True new_user_1.stripe_id = TEST_STRIPE_ID new_user_1.save() if minimal: logger.debug( 'Skipping most db population because user requested mininal db') return UserRegion.create(user=new_user_1, location=ImageStorageLocation.get(name='local_us')) model.release.set_region_release('quay', 'us', 'v0.1.2') model.user.create_confirm_email_code(new_user_1, new_email='*****@*****.**') disabled_user = model.user.create_user('disabled', 'password', '*****@*****.**') disabled_user.verified = True disabled_user.enabled = False disabled_user.save() dtrobot = model.user.create_robot('dtrobot', new_user_1) dtrobot2 = model.user.create_robot('dtrobot2', new_user_1) new_user_2 = model.user.create_user('public', 'password', '*****@*****.**') new_user_2.verified = True new_user_2.save() new_user_3 = model.user.create_user('freshuser', 'password', '*****@*****.**') new_user_3.verified = True new_user_3.save() another_robot = model.user.create_robot('anotherrobot', new_user_3) new_user_4 = model.user.create_user('randomuser', 'password', '*****@*****.**') new_user_4.verified = True new_user_4.save() new_user_5 = model.user.create_user('unverified', 'password', '*****@*****.**') new_user_5.save() reader = model.user.create_user('reader', 'password', '*****@*****.**') reader.verified = True reader.save() creatoruser = model.user.create_user('creator', 'password', '*****@*****.**') creatoruser.verified = True creatoruser.save() outside_org = model.user.create_user('outsideorg', 'password', '*****@*****.**') outside_org.verified = True outside_org.save() model.notification.create_notification('test_notification', new_user_1, metadata={ 'some': 'value', 'arr': [1, 2, 3], 'obj': { 'a': 1, 'b': 2 } }) from_date = datetime.utcnow() to_date = from_date + timedelta(hours=1) notification_metadata = { 'from_date': formatdate(calendar.timegm(from_date.utctimetuple())), 'to_date': formatdate(calendar.timegm(to_date.utctimetuple())), 'reason': 'database migration' } model.notification.create_notification('maintenance', new_user_1, metadata=notification_metadata) __generate_repository(with_storage, new_user_4, 'randomrepo', 'Random repo repository.', False, [], (4, [], ['latest', 'prod'])) simple_repo = __generate_repository(with_storage, new_user_1, 'simple', 'Simple repository.', False, [], (4, [], ['latest', 'prod'])) # Add some labels to the latest tag's manifest. tag_manifest = model.tag.load_tag_manifest(new_user_1.username, 'simple', 'latest') first_label = model.label.create_manifest_label(tag_manifest, 'foo', 'bar', 'manifest') model.label.create_manifest_label(tag_manifest, 'foo', 'baz', 'api') model.label.create_manifest_label(tag_manifest, 'anotherlabel', '1234', 'internal') model.label.create_manifest_label(tag_manifest, 'jsonlabel', '{"hey": "there"}', 'internal') label_metadata = { 'key': 'foo', 'value': 'bar', 'id': first_label.id, 'manifest_digest': tag_manifest.digest } logs_model.log_action('manifest_label_add', new_user_1.username, performer=new_user_1, timestamp=datetime.now(), metadata=label_metadata, repository=tag_manifest.tag.repository) model.blob.initiate_upload(new_user_1.username, simple_repo.name, str(uuid4()), 'local_us', {}) model.notification.create_repo_notification(simple_repo, 'repo_push', 'quay_notification', {}, {}) __generate_repository( with_storage, new_user_1, 'sharedtags', 'Shared tags repository', False, [(new_user_2, 'read'), (dtrobot[0], 'read')], (2, [(3, [], ['v2.0', 'v2.1', 'v2.2']), (1, [(1, [(1, [], ['prod', '581a284'])], ['staging', '8423b58']), (1, [], None)], None)], None)) __generate_repository(with_storage, new_user_1, 'history', 'Historical repository.', False, [], (4, [(2, [], '#latest'), (3, [], 'latest')], None)) __generate_repository(with_storage, new_user_1, 'complex', 'Complex repository with many branches and tags.', False, [(new_user_2, 'read'), (dtrobot[0], 'read')], (2, [(3, [], 'v2.0'), (1, [(1, [(2, [], ['prod'])], 'staging'), (1, [], None)], None)], None)) __generate_repository( with_storage, new_user_1, 'gargantuan', None, False, [], (2, [(3, [], 'v2.0'), (1, [(1, [(1, [], ['latest', 'prod'])], 'staging'), (1, [], None)], None), (20, [], 'v3.0'), (5, [], 'v4.0'), (1, [(1, [], 'v5.0'), (1, [], 'v6.0')], None)], None)) trusted_repo = __generate_repository(with_storage, new_user_1, 'trusted', 'Trusted repository.', False, [], (4, [], ['latest', 'prod'])) trusted_repo.trust_enabled = True trusted_repo.save() publicrepo = __generate_repository( with_storage, new_user_2, 'publicrepo', 'Public repository pullable by the world.', True, [], (10, [], 'latest')) __generate_repository(with_storage, outside_org, 'coolrepo', 'Some cool repo.', False, [], (5, [], 'latest')) __generate_repository(with_storage, new_user_1, 'shared', 'Shared repository, another user can write.', False, [(new_user_2, 'write'), (reader, 'read')], (5, [], 'latest')) __generate_repository(with_storage, new_user_1, 'text-full-repo', 'This is a repository for testing text search', False, [(new_user_2, 'write'), (reader, 'read')], (5, [], 'latest')) building = __generate_repository(with_storage, new_user_1, 'building', 'Empty repository which is building.', False, [(new_user_2, 'write'), (reader, 'read')], (0, [], None)) new_token = model.token.create_access_token(building, 'write', 'build-worker') trigger = model.build.create_build_trigger(building, 'github', '123authtoken', new_user_1, pull_robot=dtrobot[0]) trigger.config = json.dumps({ 'build_source': 'jakedt/testconnect', 'subdir': '', 'dockerfile_path': 'Dockerfile', 'context': '/', }) trigger.save() repo = 'ci.devtable.com:5000/%s/%s' % (building.namespace_user.username, building.name) job_config = { 'repository': repo, 'docker_tags': ['latest'], 'build_subdir': '', 'trigger_metadata': { 'commit': '3482adc5822c498e8f7db2e361e8d57b3d77ddd9', 'ref': 'refs/heads/master', 'default_branch': 'master' } } model.repository.star_repository(new_user_1, simple_repo) record = model.repository.create_email_authorization_for_repo( new_user_1.username, 'simple', '*****@*****.**') record.confirmed = True record.save() model.repository.create_email_authorization_for_repo( new_user_1.username, 'simple', '*****@*****.**') build2 = model.build.create_repository_build( building, new_token, job_config, '68daeebd-a5b9-457f-80a0-4363b882f8ea', 'build-name', trigger) build2.uuid = 'deadpork-dead-pork-dead-porkdeadpork' build2.save() build3 = model.build.create_repository_build( building, new_token, job_config, 'f49d07f9-93da-474d-ad5f-c852107c3892', 'build-name', trigger) build3.uuid = 'deadduck-dead-duck-dead-duckdeadduck' build3.save() build1 = model.build.create_repository_build( building, new_token, job_config, '701dcc3724fb4f2ea6c31400528343cd', 'build-name', trigger) build1.uuid = 'deadbeef-dead-beef-dead-beefdeadbeef' build1.save() org = model.organization.create_organization('buynlarge', '*****@*****.**', new_user_1) org.stripe_id = TEST_STRIPE_ID org.save() liborg = model.organization.create_organization( 'library', '*****@*****.**', new_user_1) liborg.save() titiorg = model.organization.create_organization('titi', '*****@*****.**', new_user_1) titiorg.save() thirdorg = model.organization.create_organization( 'sellnsmall', '*****@*****.**', new_user_1) thirdorg.save() model.user.create_robot('coolrobot', org) oauth_app_1 = model.oauth.create_application( org, 'Some Test App', 'http://localhost:8000', 'http://localhost:8000/o2c.html', client_id='deadbeef') model.oauth.create_application( org, 'Some Other Test App', 'http://quay.io', 'http://localhost:8000/o2c.html', client_id='deadpork', description='This is another test application') model.oauth.create_access_token_for_testing(new_user_1, 'deadbeef', 'repo:admin', access_token='%s%s' % ('b' * 40, 'c' * 40)) oauth_credential = Credential.from_string('dswfhasdf1') OAuthAuthorizationCode.create(application=oauth_app_1, code='Z932odswfhasdf1', scope='repo:admin', data='{"somejson": "goeshere"}', code_name='Z932odswfhasdf1Z932o', code_credential=oauth_credential) model.user.create_robot('neworgrobot', org) ownerbot = model.user.create_robot('ownerbot', org)[0] creatorbot = model.user.create_robot('creatorbot', org)[0] owners = model.team.get_organization_team('buynlarge', 'owners') owners.description = 'Owners have unfetterd access across the entire org.' owners.save() org_repo = __generate_repository(with_storage, org, 'orgrepo', 'Repository owned by an org.', False, [(outside_org, 'read')], (4, [], ['latest', 'prod'])) __generate_repository(with_storage, org, 'anotherorgrepo', 'Another repository owned by an org.', False, [], (4, [], ['latest', 'prod'])) creators = model.team.create_team('creators', org, 'creator', 'Creators of orgrepo.') reader_team = model.team.create_team('readers', org, 'member', 'Readers of orgrepo.') model.team.add_or_invite_to_team(new_user_1, reader_team, outside_org) model.permission.set_team_repo_permission(reader_team.name, org_repo.namespace_user.username, org_repo.name, 'read') model.team.add_user_to_team(new_user_2, reader_team) model.team.add_user_to_team(reader, reader_team) model.team.add_user_to_team(ownerbot, owners) model.team.add_user_to_team(creatorbot, creators) model.team.add_user_to_team(creatoruser, creators) sell_owners = model.team.get_organization_team('sellnsmall', 'owners') sell_owners.description = 'Owners have unfettered access across the entire org.' sell_owners.save() model.team.add_user_to_team(new_user_4, sell_owners) sync_config = { 'group_dn': 'cn=Test-Group,ou=Users', 'group_id': 'somegroupid' } synced_team = model.team.create_team('synced', org, 'member', 'Some synced team.') model.team.set_team_syncing(synced_team, 'ldap', sync_config) another_synced_team = model.team.create_team('synced', thirdorg, 'member', 'Some synced team.') model.team.set_team_syncing(another_synced_team, 'ldap', {'group_dn': 'cn=Test-Group,ou=Users'}) __generate_repository(with_storage, new_user_1, 'superwide', None, False, [], [(10, [], 'latest2'), (2, [], 'latest3'), (2, [(1, [], 'latest11'), (2, [], 'latest12')], 'latest4'), (2, [], 'latest5'), (2, [], 'latest6'), (2, [], 'latest7'), (2, [], 'latest8'), (2, [], 'latest9'), (2, [], 'latest10'), (2, [], 'latest13'), (2, [], 'latest14'), (2, [], 'latest15'), (2, [], 'latest16'), (2, [], 'latest17'), (2, [], 'latest18')]) mirror_repo = __generate_repository(with_storage, new_user_1, 'mirrored', 'Mirrored repository.', False, [(dtrobot[0], 'write'), (dtrobot2[0], 'write')], (4, [], ['latest', 'prod'])) mirror_rule = model.repo_mirror.create_mirroring_rule( mirror_repo, ['latest', '3.3*']) mirror_args = (mirror_repo, mirror_rule, dtrobot[0], 'quay.io/coreos/etcd', 60 * 60 * 24) mirror_kwargs = { 'external_registry_username': '******', 'external_registry_password': '******', 'external_registry_config': {}, 'is_enabled': True, 'sync_start_date': datetime.utcnow() } mirror = model.repo_mirror.enable_mirroring_for_repository( *mirror_args, **mirror_kwargs) read_only_repo = __generate_repository(with_storage, new_user_1, 'readonly', 'Read-Only Repo.', False, [], (4, [], ['latest', 'prod'])) read_only_repo.state = RepositoryState.READ_ONLY read_only_repo.save() model.permission.add_prototype_permission(org, 'read', activating_user=new_user_1, delegate_user=new_user_2) model.permission.add_prototype_permission(org, 'read', activating_user=new_user_1, delegate_team=reader_team) model.permission.add_prototype_permission(org, 'write', activating_user=new_user_2, delegate_user=new_user_1) today = datetime.today() week_ago = today - timedelta(6) six_ago = today - timedelta(5) four_ago = today - timedelta(4) yesterday = datetime.combine(date.today(), datetime.min.time()) - timedelta(hours=6) __generate_service_key('kid1', 'somesamplekey', new_user_1, today, ServiceKeyApprovalType.SUPERUSER) __generate_service_key('kid2', 'someexpiringkey', new_user_1, week_ago, ServiceKeyApprovalType.SUPERUSER, today + timedelta(days=14)) __generate_service_key('kid3', 'unapprovedkey', new_user_1, today, None) __generate_service_key( 'kid4', 'autorotatingkey', new_user_1, six_ago, ServiceKeyApprovalType.KEY_ROTATION, today + timedelta(days=1), rotation_duration=timedelta(hours=12).total_seconds()) __generate_service_key('kid5', 'key for another service', new_user_1, today, ServiceKeyApprovalType.SUPERUSER, today + timedelta(days=14), service='different_sample_service') __generate_service_key('kid6', 'someexpiredkey', new_user_1, week_ago, ServiceKeyApprovalType.SUPERUSER, today - timedelta(days=1)) __generate_service_key('kid7', 'somewayexpiredkey', new_user_1, week_ago, ServiceKeyApprovalType.SUPERUSER, today - timedelta(days=30)) # Add the test pull key as pre-approved for local and unittest registry testing. # Note: this must match the private key found in the local/test config. _TEST_JWK = { 'e': 'AQAB', 'kty': 'RSA', 'n': 'yqdQgnelhAPMSeyH0kr3UGePK9oFOmNfwD0Ymnh7YYXr21VHWwyM2eVW3cnLd9KXywDFtGSe9oFDbnOuMCdUowdkBcaHju-isbv5KEbNSoy_T2Rip-6L0cY63YzcMJzv1nEYztYXS8wz76pSK81BKBCLapqOCmcPeCvV9yaoFZYvZEsXCl5jjXN3iujSzSF5Z6PpNFlJWTErMT2Z4QfbDKX2Nw6vJN6JnGpTNHZvgvcyNX8vkSgVpQ8DFnFkBEx54PvRV5KpHAq6AsJxKONMo11idQS2PfCNpa2hvz9O6UZe-eIX8jPo5NW8TuGZJumbdPT_nxTDLfCqfiZboeI0Pw' } key = model.service_keys.create_service_key('test_service_key', 'test_service_key', 'quay', _TEST_JWK, {}, None) model.service_keys.approve_service_key( key.kid, ServiceKeyApprovalType.SUPERUSER, notes='Test service key for local/test registry testing') # Add an app specific token. token = model.appspecifictoken.create_token(new_user_1, 'some app') token.token_name = 'a' * 60 token.token_secret = 'b' * 60 token.save() logs_model.log_action('org_create_team', org.username, performer=new_user_1, timestamp=week_ago, metadata={'team': 'readers'}) logs_model.log_action('org_set_team_role', org.username, performer=new_user_1, timestamp=week_ago, metadata={ 'team': 'readers', 'role': 'read' }) logs_model.log_action('create_repo', org.username, performer=new_user_1, repository=org_repo, timestamp=week_ago, metadata={ 'namespace': org.username, 'repo': 'orgrepo' }) logs_model.log_action('change_repo_permission', org.username, performer=new_user_2, repository=org_repo, timestamp=six_ago, metadata={ 'username': new_user_1.username, 'repo': 'orgrepo', 'role': 'admin' }) logs_model.log_action('change_repo_permission', org.username, performer=new_user_1, repository=org_repo, timestamp=six_ago, metadata={ 'username': new_user_2.username, 'repo': 'orgrepo', 'role': 'read' }) logs_model.log_action('add_repo_accesstoken', org.username, performer=new_user_1, repository=org_repo, timestamp=four_ago, metadata={ 'repo': 'orgrepo', 'token': 'deploytoken' }) logs_model.log_action('push_repo', org.username, performer=new_user_2, repository=org_repo, timestamp=today, metadata={ 'username': new_user_2.username, 'repo': 'orgrepo' }) logs_model.log_action('pull_repo', org.username, performer=new_user_2, repository=org_repo, timestamp=today, metadata={ 'username': new_user_2.username, 'repo': 'orgrepo' }) logs_model.log_action('pull_repo', org.username, repository=org_repo, timestamp=today, metadata={ 'token': 'sometoken', 'token_code': 'somecode', 'repo': 'orgrepo' }) logs_model.log_action('delete_tag', org.username, performer=new_user_2, repository=org_repo, timestamp=today, metadata={ 'username': new_user_2.username, 'repo': 'orgrepo', 'tag': 'sometag' }) logs_model.log_action('pull_repo', org.username, repository=org_repo, timestamp=today, metadata={ 'token_code': 'somecode', 'repo': 'orgrepo' }) logs_model.log_action('pull_repo', new_user_2.username, repository=publicrepo, timestamp=yesterday, metadata={ 'token_code': 'somecode', 'repo': 'publicrepo' }) logs_model.log_action('build_dockerfile', new_user_1.username, repository=building, timestamp=today, metadata={ 'repo': 'building', 'namespace': new_user_1.username, 'trigger_id': trigger.uuid, 'config': json.loads(trigger.config), 'service': trigger.service.name }) model.message.create([{ 'content': 'We love you, Quay customers!', 'severity': 'info', 'media_type': 'text/plain' }]) model.message.create([{ 'content': 'This is a **development** install of Quay', 'severity': 'warning', 'media_type': 'text/markdown' }]) fake_queue = WorkQueue('fakequeue', tf) fake_queue.put(['canonical', 'job', 'name'], '{}') model.user.create_user_prompt(new_user_4, 'confirm_username') while True: to_count = model.repositoryactioncount.find_uncounted_repository() if not to_count: break model.repositoryactioncount.count_repository_actions(to_count) model.repositoryactioncount.update_repository_score(to_count)