def post(self, namespace_name, repository_name, manifestref): """ Adds a new label into the tag manifest. """ label_data = request.get_json() # Check for any reserved prefixes. if label_validator.has_reserved_prefix(label_data['key']): abort(400, message='Label has a reserved prefix') repo_ref = registry_model.lookup_repository(namespace_name, repository_name) if repo_ref is None: raise NotFound() manifest = registry_model.lookup_manifest_by_digest( repo_ref, manifestref) if manifest is None: raise NotFound() label = None try: label = registry_model.create_manifest_label( manifest, label_data['key'], label_data['value'], 'api', label_data['media_type']) except InvalidLabelKeyException: message = ('Label is of an invalid format or missing please ' + 'use %s format for labels' % VALID_LABEL_KEY_REGEX) abort(400, message=message) except InvalidMediaTypeException: message = 'Media type is invalid please use a valid media type: text/plain, application/json' abort(400, message=message) if label is None: raise NotFound() metadata = { 'id': label.uuid, 'key': label.key, 'value': label.value, 'manifest_digest': manifestref, 'media_type': label.media_type_name, 'namespace': namespace_name, 'repo': repository_name, } log_action('manifest_label_add', namespace_name, metadata, repo_name=repository_name) resp = {'label': _label_dict(label)} repo_string = '%s/%s' % (namespace_name, repository_name) headers = { 'Location': api.url_for(ManageRepositoryManifestLabel, repository=repo_string, manifestref=manifestref, labelid=label.uuid), } return resp, 201, headers
def populate_database(minimal=False): logger.debug("Populating the DB with test data.") # Check if the data already exists. If so, we skip. This can happen between calls from the # "old style" tests and the new py.test's. try: User.get(username="******") logger.debug("DB already populated") return except User.DoesNotExist: pass # Note: databases set up with "real" schema (via Alembic) will not have these types # type, so we it here it necessary. try: ImageStorageLocation.get(name="local_eu") ImageStorageLocation.get(name="local_us") except ImageStorageLocation.DoesNotExist: ImageStorageLocation.create(name="local_eu") ImageStorageLocation.create(name="local_us") try: NotificationKind.get(name="test_notification") except NotificationKind.DoesNotExist: NotificationKind.create(name="test_notification") new_user_1 = model.user.create_user("devtable", "password", "*****@*****.**") new_user_1.verified = True new_user_1.stripe_id = TEST_STRIPE_ID new_user_1.save() if minimal: logger.debug( "Skipping most db population because user requested mininal db") return UserRegion.create(user=new_user_1, location=ImageStorageLocation.get(name="local_us")) model.release.set_region_release("quay", "us", "v0.1.2") model.user.create_confirm_email_code(new_user_1, new_email="*****@*****.**") disabled_user = model.user.create_user("disabled", "password", "*****@*****.**") disabled_user.verified = True disabled_user.enabled = False disabled_user.save() dtrobot = model.user.create_robot("dtrobot", new_user_1) dtrobot2 = model.user.create_robot("dtrobot2", new_user_1) new_user_2 = model.user.create_user("public", "password", "*****@*****.**") new_user_2.verified = True new_user_2.save() new_user_3 = model.user.create_user("freshuser", "password", "*****@*****.**") new_user_3.verified = True new_user_3.save() another_robot = model.user.create_robot("anotherrobot", new_user_3) new_user_4 = model.user.create_user("randomuser", "password", "*****@*****.**") new_user_4.verified = True new_user_4.save() new_user_5 = model.user.create_user("unverified", "password", "*****@*****.**") new_user_5.save() reader = model.user.create_user("reader", "password", "*****@*****.**") reader.verified = True reader.save() creatoruser = model.user.create_user("creator", "password", "*****@*****.**") creatoruser.verified = True creatoruser.save() outside_org = model.user.create_user("outsideorg", "password", "*****@*****.**") outside_org.verified = True outside_org.save() model.notification.create_notification( "test_notification", new_user_1, metadata={ "some": "value", "arr": [1, 2, 3], "obj": { "a": 1, "b": 2 } }, ) from_date = datetime.utcnow() to_date = from_date + timedelta(hours=1) notification_metadata = { "from_date": formatdate(calendar.timegm(from_date.utctimetuple())), "to_date": formatdate(calendar.timegm(to_date.utctimetuple())), "reason": "database migration", } model.notification.create_notification("maintenance", new_user_1, metadata=notification_metadata) __generate_repository( new_user_4, "randomrepo", "Random repo repository.", False, [], (4, [], ["latest", "prod"]), ) simple_repo = __generate_repository( new_user_1, "simple", "Simple repository.", False, [], (4, [], ["latest", "prod"]), ) # Add some labels to the latest tag's manifest. repo_ref = RepositoryReference.for_repo_obj(simple_repo) tag = registry_model.get_repo_tag(repo_ref, "latest") manifest = registry_model.get_manifest_for_tag(tag) assert manifest first_label = registry_model.create_manifest_label(manifest, "foo", "bar", "manifest") registry_model.create_manifest_label(manifest, "foo", "baz", "api") registry_model.create_manifest_label(manifest, "anotherlabel", "1234", "internal") registry_model.create_manifest_label(manifest, "jsonlabel", '{"hey": "there"}', "internal", "application/json") label_metadata = { "key": "foo", "value": "bar", "id": first_label._db_id, "manifest_digest": manifest.digest, } logs_model.log_action( "manifest_label_add", new_user_1.username, performer=new_user_1, timestamp=datetime.now(), metadata=label_metadata, repository=simple_repo, ) model.blob.initiate_upload(new_user_1.username, simple_repo.name, str(uuid4()), "local_us", {}) model.notification.create_repo_notification(simple_repo, "repo_push", "quay_notification", {}, {}) __generate_repository( new_user_1, "sharedtags", "Shared tags repository", False, [(new_user_2, "read"), (dtrobot[0], "read")], ( 2, [ (3, [], ["v2.0", "v2.1", "v2.2"]), ( 1, [(1, [(1, [], ["prod", "581a284"]) ], ["staging", "8423b58"]), (1, [], None)], None, ), ], None, ), ) __generate_repository( new_user_1, "history", "Historical repository.", False, [], (4, [(2, [], "#latest"), (3, [], "latest")], None), ) __generate_repository( new_user_1, "complex", "Complex repository with many branches and tags.", False, [(new_user_2, "read"), (dtrobot[0], "read")], ( 2, [(3, [], "v2.0"), (1, [(1, [(2, [], ["prod"])], "staging"), (1, [], None)], None)], None, ), ) __generate_repository( new_user_1, "gargantuan", None, False, [], ( 2, [ (3, [], "v2.0"), (1, [(1, [(1, [], ["latest", "prod"])], "staging"), (1, [], None)], None), (20, [], "v3.0"), (5, [], "v4.0"), (1, [(1, [], "v5.0"), (1, [], "v6.0")], None), ], None, ), ) trusted_repo = __generate_repository( new_user_1, "trusted", "Trusted repository.", False, [], (4, [], ["latest", "prod"]), ) trusted_repo.trust_enabled = True trusted_repo.save() publicrepo = __generate_repository( new_user_2, "publicrepo", "Public repository pullable by the world.", True, [], (10, [], "latest"), ) __generate_repository(outside_org, "coolrepo", "Some cool repo.", False, [], (5, [], "latest")) __generate_repository( new_user_1, "shared", "Shared repository, another user can write.", False, [(new_user_2, "write"), (reader, "read")], (5, [], "latest"), ) __generate_repository( new_user_1, "text-full-repo", "This is a repository for testing text search", False, [(new_user_2, "write"), (reader, "read")], (5, [], "latest"), ) building = __generate_repository( new_user_1, "building", "Empty repository which is building.", False, [(new_user_2, "write"), (reader, "read")], (0, [], None), ) new_token = model.token.create_access_token(building, "write", "build-worker") trigger = model.build.create_build_trigger(building, "github", "123authtoken", new_user_1, pull_robot=dtrobot[0]) trigger.config = json.dumps({ "build_source": "jakedt/testconnect", "subdir": "", "dockerfile_path": "Dockerfile", "context": "/", }) trigger.save() repo = "ci.devtable.com:5000/%s/%s" % (building.namespace_user.username, building.name) job_config = { "repository": repo, "docker_tags": ["latest"], "build_subdir": "", "trigger_metadata": { "commit": "3482adc5822c498e8f7db2e361e8d57b3d77ddd9", "ref": "refs/heads/master", "default_branch": "master", }, } model.repository.star_repository(new_user_1, simple_repo) record = model.repository.create_email_authorization_for_repo( new_user_1.username, "simple", "*****@*****.**") record.confirmed = True record.save() model.repository.create_email_authorization_for_repo( new_user_1.username, "simple", "*****@*****.**") build2 = model.build.create_repository_build( building, new_token, job_config, "68daeebd-a5b9-457f-80a0-4363b882f8ea", "build-name", trigger, ) build2.uuid = "deadpork-dead-pork-dead-porkdeadpork" build2.save() build3 = model.build.create_repository_build( building, new_token, job_config, "f49d07f9-93da-474d-ad5f-c852107c3892", "build-name", trigger, ) build3.uuid = "deadduck-dead-duck-dead-duckdeadduck" build3.save() build1 = model.build.create_repository_build( building, new_token, job_config, "701dcc3724fb4f2ea6c31400528343cd", "build-name", trigger) build1.uuid = "deadbeef-dead-beef-dead-beefdeadbeef" build1.save() org = model.organization.create_organization("buynlarge", "*****@*****.**", new_user_1) org.stripe_id = TEST_STRIPE_ID org.save() liborg = model.organization.create_organization( "library", "*****@*****.**", new_user_1) liborg.save() titiorg = model.organization.create_organization("titi", "*****@*****.**", new_user_1) titiorg.save() thirdorg = model.organization.create_organization( "sellnsmall", "*****@*****.**", new_user_1) thirdorg.save() model.user.create_robot("coolrobot", org) oauth_app_1 = model.oauth.create_application( org, "Some Test App", "http://localhost:8000", "http://localhost:8000/o2c.html", client_id="deadbeef", ) model.oauth.create_application( org, "Some Other Test App", "http://quay.io", "http://localhost:8000/o2c.html", client_id="deadpork", description="This is another test application", ) model.oauth.create_user_access_token(new_user_1, "deadbeef", "repo:admin", access_token="%s%s" % ("b" * 40, "c" * 40)) oauth_credential = Credential.from_string("dswfhasdf1") OAuthAuthorizationCode.create( application=oauth_app_1, code="Z932odswfhasdf1", scope="repo:admin", data='{"somejson": "goeshere"}', code_name="Z932odswfhasdf1Z932o", code_credential=oauth_credential, ) model.user.create_robot("neworgrobot", org) ownerbot = model.user.create_robot("ownerbot", org)[0] creatorbot = model.user.create_robot("creatorbot", org)[0] owners = model.team.get_organization_team("buynlarge", "owners") owners.description = "Owners have unfetterd access across the entire org." owners.save() org_repo = __generate_repository( org, "orgrepo", "Repository owned by an org.", False, [(outside_org, "read")], (4, [], ["latest", "prod"]), ) __generate_repository( org, "anotherorgrepo", "Another repository owned by an org.", False, [], (4, [], ["latest", "prod"]), ) creators = model.team.create_team("creators", org, "creator", "Creators of orgrepo.") reader_team = model.team.create_team("readers", org, "member", "Readers of orgrepo.") model.team.add_or_invite_to_team(new_user_1, reader_team, outside_org) model.permission.set_team_repo_permission(reader_team.name, org_repo.namespace_user.username, org_repo.name, "read") model.team.add_user_to_team(new_user_2, reader_team) model.team.add_user_to_team(reader, reader_team) model.team.add_user_to_team(ownerbot, owners) model.team.add_user_to_team(creatorbot, creators) model.team.add_user_to_team(creatoruser, creators) sell_owners = model.team.get_organization_team("sellnsmall", "owners") sell_owners.description = "Owners have unfettered access across the entire org." sell_owners.save() model.team.add_user_to_team(new_user_4, sell_owners) sync_config = { "group_dn": "cn=Test-Group,ou=Users", "group_id": "somegroupid" } synced_team = model.team.create_team("synced", org, "member", "Some synced team.") model.team.set_team_syncing(synced_team, "ldap", sync_config) another_synced_team = model.team.create_team("synced", thirdorg, "member", "Some synced team.") model.team.set_team_syncing(another_synced_team, "ldap", {"group_dn": "cn=Test-Group,ou=Users"}) __generate_repository( new_user_1, "superwide", None, False, [], [ (10, [], "latest2"), (2, [], "latest3"), (2, [(1, [], "latest11"), (2, [], "latest12")], "latest4"), (2, [], "latest5"), (2, [], "latest6"), (2, [], "latest7"), (2, [], "latest8"), (2, [], "latest9"), (2, [], "latest10"), (2, [], "latest13"), (2, [], "latest14"), (2, [], "latest15"), (2, [], "latest16"), (2, [], "latest17"), (2, [], "latest18"), ], ) mirror_repo = __generate_repository( new_user_1, "mirrored", "Mirrored repository.", False, [(dtrobot[0], "write"), (dtrobot2[0], "write")], (4, [], ["latest", "prod"]), ) mirror_rule = model.repo_mirror.create_mirroring_rule( mirror_repo, ["latest", "3.3*"]) mirror_args = (mirror_repo, mirror_rule, dtrobot[0], "quay.io/coreos/etcd", 60 * 60 * 24) mirror_kwargs = { "external_registry_username": "******", "external_registry_password": "******", "external_registry_config": {}, "is_enabled": True, "sync_start_date": datetime.utcnow(), } mirror = model.repo_mirror.enable_mirroring_for_repository( *mirror_args, **mirror_kwargs) read_only_repo = __generate_repository( new_user_1, "readonly", "Read-Only Repo.", False, [], (4, [], ["latest", "prod"]), ) read_only_repo.state = RepositoryState.READ_ONLY read_only_repo.save() model.permission.add_prototype_permission(org, "read", activating_user=new_user_1, delegate_user=new_user_2) model.permission.add_prototype_permission(org, "read", activating_user=new_user_1, delegate_team=reader_team) model.permission.add_prototype_permission(org, "write", activating_user=new_user_2, delegate_user=new_user_1) today = datetime.today() week_ago = today - timedelta(6) six_ago = today - timedelta(5) four_ago = today - timedelta(4) yesterday = datetime.combine(date.today(), datetime.min.time()) - timedelta(hours=6) __generate_service_key("kid1", "somesamplekey", new_user_1, today, ServiceKeyApprovalType.SUPERUSER) __generate_service_key( "kid2", "someexpiringkey", new_user_1, week_ago, ServiceKeyApprovalType.SUPERUSER, today + timedelta(days=14), ) __generate_service_key("kid3", "unapprovedkey", new_user_1, today, None) __generate_service_key( "kid4", "autorotatingkey", new_user_1, six_ago, ServiceKeyApprovalType.KEY_ROTATION, today + timedelta(days=1), rotation_duration=timedelta(hours=12).total_seconds(), ) __generate_service_key( "kid5", "key for another service", new_user_1, today, ServiceKeyApprovalType.SUPERUSER, today + timedelta(days=14), service="different_sample_service", ) __generate_service_key( "kid6", "someexpiredkey", new_user_1, week_ago, ServiceKeyApprovalType.SUPERUSER, today - timedelta(days=1), ) __generate_service_key( "kid7", "somewayexpiredkey", new_user_1, week_ago, ServiceKeyApprovalType.SUPERUSER, today - timedelta(days=30), ) # Add the test pull key as pre-approved for local and unittest registry testing. # Note: this must match the private key found in the local/test config. _TEST_JWK = { "e": "AQAB", "kty": "RSA", "n": "yqdQgnelhAPMSeyH0kr3UGePK9oFOmNfwD0Ymnh7YYXr21VHWwyM2eVW3cnLd9KXywDFtGSe9oFDbnOuMCdUowdkBcaHju-isbv5KEbNSoy_T2Rip-6L0cY63YzcMJzv1nEYztYXS8wz76pSK81BKBCLapqOCmcPeCvV9yaoFZYvZEsXCl5jjXN3iujSzSF5Z6PpNFlJWTErMT2Z4QfbDKX2Nw6vJN6JnGpTNHZvgvcyNX8vkSgVpQ8DFnFkBEx54PvRV5KpHAq6AsJxKONMo11idQS2PfCNpa2hvz9O6UZe-eIX8jPo5NW8TuGZJumbdPT_nxTDLfCqfiZboeI0Pw", } key = model.service_keys.create_service_key("test_service_key", "test_service_key", "quay", _TEST_JWK, {}, None) model.service_keys.approve_service_key( key.kid, ServiceKeyApprovalType.SUPERUSER, notes="Test service key for local/test registry testing", ) # Add an app specific token. token = model.appspecifictoken.create_token(new_user_1, "some app") token.token_name = "a" * 60 token.token_secret = "b" * 60 token.save() logs_model.log_action( "org_create_team", org.username, performer=new_user_1, timestamp=week_ago, metadata={"team": "readers"}, ) logs_model.log_action( "org_set_team_role", org.username, performer=new_user_1, timestamp=week_ago, metadata={ "team": "readers", "role": "read" }, ) logs_model.log_action( "create_repo", org.username, performer=new_user_1, repository=org_repo, timestamp=week_ago, metadata={ "namespace": org.username, "repo": "orgrepo" }, ) logs_model.log_action( "change_repo_permission", org.username, performer=new_user_2, repository=org_repo, timestamp=six_ago, metadata={ "username": new_user_1.username, "repo": "orgrepo", "role": "admin" }, ) logs_model.log_action( "change_repo_permission", org.username, performer=new_user_1, repository=org_repo, timestamp=six_ago, metadata={ "username": new_user_2.username, "repo": "orgrepo", "role": "read" }, ) logs_model.log_action( "add_repo_accesstoken", org.username, performer=new_user_1, repository=org_repo, timestamp=four_ago, metadata={ "repo": "orgrepo", "token": "deploytoken" }, ) logs_model.log_action( "push_repo", org.username, performer=new_user_2, repository=org_repo, timestamp=today, metadata={ "username": new_user_2.username, "repo": "orgrepo" }, ) logs_model.log_action( "pull_repo", org.username, performer=new_user_2, repository=org_repo, timestamp=today, metadata={ "username": new_user_2.username, "repo": "orgrepo" }, ) logs_model.log_action( "pull_repo", org.username, repository=org_repo, timestamp=today, metadata={ "token": "sometoken", "token_code": "somecode", "repo": "orgrepo" }, ) logs_model.log_action( "delete_tag", org.username, performer=new_user_2, repository=org_repo, timestamp=today, metadata={ "username": new_user_2.username, "repo": "orgrepo", "tag": "sometag" }, ) logs_model.log_action( "pull_repo", org.username, repository=org_repo, timestamp=today, metadata={ "token_code": "somecode", "repo": "orgrepo" }, ) logs_model.log_action( "pull_repo", new_user_2.username, repository=publicrepo, timestamp=yesterday, metadata={ "token_code": "somecode", "repo": "publicrepo" }, ) logs_model.log_action( "build_dockerfile", new_user_1.username, repository=building, timestamp=today, metadata={ "repo": "building", "namespace": new_user_1.username, "trigger_id": trigger.uuid, "config": json.loads(trigger.config), "service": trigger.service.name, }, ) model.message.create([{ "content": "We love you, Quay customers!", "severity": "info", "media_type": "text/plain", }]) model.message.create([{ "content": "This is a **development** install of Quay", "severity": "warning", "media_type": "text/markdown", }]) fake_queue = WorkQueue("fakequeue", tf) fake_queue.put(["canonical", "job", "name"], "{}") model.user.create_user_prompt(new_user_4, "confirm_username") for to_count in Repository.select(): model.repositoryactioncount.count_repository_actions( to_count, datetime.utcnow()) model.repositoryactioncount.update_repository_score(to_count)
def _build_complete(self, result): """ Wraps up a completed build. Handles any errors and calls self._build_finished. """ build_id = self._current_job.repo_build.uuid try: # Retrieve the result. This will raise an ApplicationError on any error that occurred. result_value = result.result() kwargs = {} # Note: If we are hitting an older builder that didn't return ANY map data, then the result # value will be a bool instead of a proper CallResult object. # Therefore: we have a try-except guard here to ensure we don't hit this pitfall. try: kwargs = result_value.kwresults except: pass try: yield From(self._build_status.set_phase(BUILD_PHASE.COMPLETE)) except InvalidRepositoryBuildException: logger.warning( 'Build %s was not found; repo was probably deleted', build_id) raise Return() yield From(self._build_finished(BuildJobResult.COMPLETE)) # Label the pushed manifests with the build metadata. manifest_digests = kwargs.get('digests') or [] repository = registry_model.lookup_repository( self._current_job.namespace, self._current_job.repo_name) if repository is not None: for digest in manifest_digests: with UseThenDisconnect(app.config): manifest = registry_model.lookup_manifest_by_digest( repository, digest, require_available=True) if manifest is None: continue registry_model.create_manifest_label( manifest, INTERNAL_LABEL_BUILD_UUID, build_id, 'internal', 'text/plain') # Send the notification that the build has completed successfully. self._current_job.send_notification( 'build_success', image_id=kwargs.get('image_id'), manifest_digests=manifest_digests) except ApplicationError as aex: worker_error = WorkerError(aex.error, aex.kwargs.get('base_error')) # Write the error to the log. yield From( self._build_status.set_error( worker_error.public_message(), worker_error.extra_data(), internal_error=worker_error.is_internal_error(), requeued=self._current_job.has_retries_remaining())) # Send the notification that the build has failed. self._current_job.send_notification( 'build_failure', error_message=worker_error.public_message()) # Mark the build as completed. if worker_error.is_internal_error(): logger.exception( '[BUILD INTERNAL ERROR: Remote] Build ID: %s: %s', build_id, worker_error.public_message()) yield From(self._build_finished(BuildJobResult.INCOMPLETE)) else: logger.debug('Got remote failure exception for build %s: %s', build_id, aex) yield From(self._build_finished(BuildJobResult.ERROR)) # Remove the current job. self._current_job = None
def post(self, namespace_name, repository_name, manifestref): """ Adds a new label into the tag manifest. """ label_data = request.get_json() # Check for any reserved prefixes. if label_validator.has_reserved_prefix(label_data["key"]): abort(400, message="Label has a reserved prefix") repo_ref = registry_model.lookup_repository(namespace_name, repository_name) if repo_ref is None: raise NotFound() manifest = registry_model.lookup_manifest_by_digest(repo_ref, manifestref) if manifest is None: raise NotFound() label = None try: label = registry_model.create_manifest_label( manifest, label_data["key"], label_data["value"], "api", label_data["media_type"] ) except InvalidLabelKeyException: message = ( "Label is of an invalid format or missing please " + "use %s format for labels" % VALID_LABEL_KEY_REGEX ) abort(400, message=message) except InvalidMediaTypeException: message = ( "Media type is invalid please use a valid media type: text/plain, application/json" ) abort(400, message=message) if label is None: raise NotFound() metadata = { "id": label.uuid, "key": label.key, "value": label.value, "manifest_digest": manifestref, "media_type": label.media_type_name, "namespace": namespace_name, "repo": repository_name, } log_action("manifest_label_add", namespace_name, metadata, repo_name=repository_name) resp = {"label": _label_dict(label)} repo_string = "%s/%s" % (namespace_name, repository_name) headers = { "Location": api.url_for( ManageRepositoryManifestLabel, repository=repo_string, manifestref=manifestref, labelid=label.uuid, ), } return resp, 201, headers