def post(self): """Subscribe a machine to a Cloud Pub/Sub topic. Params: backend_project: If specified, project that the machine subscription topic is contained in for the backend. backend_attributes: If specified, JSON-encoded dict of attributes to include in the machine subscription message for the backend. backend_topic: If specified, topic that the machine subscription should be published to for the backend. hostname: Hostname being reclaimed. machine_id: ID of the CatalogMachineEntry being reclaimed. machine_service_account: Service account to authorize to consume the subscription. machine_subscription: Cloud Pub/Sub subscription to create for the machine. machine_subscription_project: Project the Cloud Pub/Sub subscription should live in. machine_topic: Cloud Pub/Sub topic to create for the machine. machine_topic_project: Project the Cloud Pub/Sub topic should live in. """ backend_attributes = json.loads( self.request.get('backend_attributes', {})) backend_project = self.request.get('backend_project') backend_topic = self.request.get('backend_topic') hostname = self.request.get('hostname') machine_id = self.request.get('machine_id') machine_service_account = self.request.get('machine_service_account') machine_subscription = self.request.get('machine_subscription') machine_subscription_project = self.request.get( 'machine_subscription_project') machine_topic = self.request.get('machine_topic') machine_topic_project = self.request.get('machine_topic_project') topic = pubsub.full_topic_name(machine_topic_project, machine_topic) subscription = pubsub.full_subscription_name( machine_subscription_project, machine_subscription) pubsub.ensure_subscription_deleted(subscription) pubsub.ensure_subscription_exists(subscription, topic) with pubsub.iam_policy(subscription) as policy: policy.add_member( 'roles/pubsub.subscriber', 'serviceAccount:%s' % machine_service_account, ) if backend_topic: attributes = backend_attributes.copy() attributes['hostname'] = hostname attributes['subscription'] = machine_subscription attributes['subscription_project'] = machine_subscription_project attributes['topic'] = machine_topic attributes['topic_project'] = machine_topic_project pubsub.publish( pubsub.full_topic_name(backend_project, backend_topic), 'SUBSCRIBED', attributes, ) set_available(ndb.Key(models.CatalogMachineEntry, machine_id))
def post(self): """Fulfill a lease request. Params: lease_id: ID of the LeaseRequest being fulfilled. machine_id: ID of the CatalogMachineEntry fulfilling the LeaseRequest. pubsub_topic: If specified, topic that the lease fulfillment should be published to. pubsub_project: If specified, project that the lease fulfillment topic is contained in. """ lease_id = self.request.get('lease_id') machine_id = self.request.get('machine_id') pubsub_project = self.request.get('pubsub_project') pubsub_topic = self.request.get('pubsub_topic') if pubsub_topic: pubsub.publish( pubsub.full_topic_name(pubsub_project, pubsub_topic), 'FULFILLED', { 'machine_id': machine_id, 'request_hash': lease_id, }, )
def post(self): """Reclaim a machine. Params: backend_topic: If specified, topic that the machine reclamation should be published to for the backend. backend_project: If specified, project that the machine reclamation topic is contained in for the backend. lease_id: ID of the LeaseRequest the machine was leased for. lessee_topic: If specified, topic that the machine reclamation and lease expiration should be published to for the lessee. lessee_project: If specified, project that the machine reclamation and lease expiration topic is contained in. machine_id: ID of the CatalogMachineEntry being reclaimed. """ backend_project = self.request.get("backend_project") backend_topic = self.request.get("backend_topic") lease_id = self.request.get("lease_id") lessee_project = self.request.get("lessee_project") lessee_topic = self.request.get("lessee_topic") machine_id = self.request.get("machine_id") if lessee_topic: pubsub.publish(lessee_topic, lessee_project, "RECLAIMED", machine_id=machine_id, request_hash=lease_id) if backend_topic: pubsub.publish(backend_topic, backend_project, "RECLAIMED", machine_id=machine_id)
def post(self, build_id): # pylint: disable=unused-argument body = json.loads(self.request.body) assert body.get('mode') in ('global', 'callback') bundle = model.BuildBundle.get( body['id'], infra=True, input_properties=True, output_properties=True, ) if not bundle: # pragma: no cover return build = bundle.build message = { 'build': api_common.build_to_dict(bundle), 'hostname': app_identity.get_default_version_hostname(), } attrs = {'build_id': str(build.key.id())} if body['mode'] == 'callback': topic = build.pubsub_callback.topic message['user_data'] = build.pubsub_callback.user_data attrs['auth_token'] = build.pubsub_callback.auth_token else: topic = 'projects/%s/topics/builds' % app_identity.get_application_id() pubsub.publish(topic, json.dumps(message, sort_keys=True), attrs)
def post(self): """Subscribe a machine to a Cloud Pub/Sub topic. Params: backend_project: If specified, project that the machine subscription topic is contained in for the backend. backend_attributes: If specified, JSON-encoded dict of attributes to include in the machine subscription message for the backend. backend_topic: If specified, topic that the machine subscription should be published to for the backend. hostname: Hostname being reclaimed. machine_id: ID of the CatalogMachineEntry being reclaimed. machine_service_account: Service account to authorize to consume the subscription. machine_subscription: Cloud Pub/Sub subscription to create for the machine. machine_subscription_project: Project the Cloud Pub/Sub subscription should live in. machine_topic: Cloud Pub/Sub topic to create for the machine. machine_topic_project: Project the Cloud Pub/Sub topic should live in. """ backend_attributes = json.loads(self.request.get('backend_attributes', {})) backend_project = self.request.get('backend_project') backend_topic = self.request.get('backend_topic') hostname = self.request.get('hostname') machine_id = self.request.get('machine_id') machine_service_account = self.request.get('machine_service_account') machine_subscription = self.request.get('machine_subscription') machine_subscription_project = self.request.get( 'machine_subscription_project') machine_topic = self.request.get('machine_topic') machine_topic_project = self.request.get('machine_topic_project') topic = pubsub.full_topic_name(machine_topic_project, machine_topic) subscription = pubsub.full_subscription_name( machine_subscription_project, machine_subscription) pubsub.ensure_subscription_deleted(subscription) pubsub.ensure_subscription_exists(subscription, topic) with pubsub.iam_policy(subscription) as policy: policy.add_member( 'roles/pubsub.subscriber', 'serviceAccount:%s' % machine_service_account, ) if backend_topic: attributes = backend_attributes.copy() attributes['hostname'] = hostname attributes['subscription'] = machine_subscription attributes['subscription_project'] = machine_subscription_project attributes['topic'] = machine_topic attributes['topic_project'] = machine_topic_project pubsub.publish( pubsub.full_topic_name(backend_project, backend_topic), 'SUBSCRIBED', attributes, ) set_available(ndb.Key(models.CatalogMachineEntry, machine_id))
def _publish_pubsub_message( build_id, topic, user_data, auth_token): # pragma: no cover message = json.dumps({ 'build_id': build_id, 'user_data': user_data, }, sort_keys=True) attrs = { 'build_id': str(build_id), 'auth_token': auth_token, } pubsub.publish(topic, message, attrs)
def test_publish_fatal_error(self): self.mock_requests( [ { "url": "https://pubsub.googleapis.com/v1/projects/a/topics/def:publish", "method": "POST", "payload": {"messages": [{"attributes": {"a": 1, "b": 2}, "data": "bXNn"}]}, "response": net.Error("fatal error", 403, ""), } ] ) with self.assertRaises(pubsub.Error): pubsub.publish("projects/a/topics/def", "msg", {"a": 1, "b": 2})
def maybe_notify_lessee(request, response): """Informs the lessee of the status of a request if there's a Pub/Sub topic. Args: request: A dict representation of an rpc_messages.LeaseRequest instance. response: A dict representation of an rpc_messages.LeaseResponse instance. """ if request.get('pubsub_topic'): pubsub.publish( pubsub.full_topic_name(request['pubsub_project'], request['pubsub_topic']), json.dumps(response), )
def maybe_notify_lessee(request, response): """Informs the lessee of the status of a request if there's a Pub/Sub topic. Args: request: A dict representation of an rpc_messages.LeaseRequest instance. response: A dict representation of an rpc_messages.LeaseResponse instance. """ if request.get('pubsub_topic'): pubsub.publish( pubsub.full_topic_name( request['pubsub_project'], request['pubsub_topic']), json.dumps(response), )
def test_publish_ok(self): self.mock_requests([ # First attempt. Encounters 404 due to non-existing topic. { 'url': 'https://pubsub.googleapis.com/v1/projects/a/topics/def:publish', 'method': 'POST', 'payload': { 'messages': [ { 'attributes': { 'a': 1, 'b': 2 }, 'data': 'bXNn', }, ], }, 'response': net.NotFoundError('topic not found', 404, ''), }, # Creates the topic. { 'url': 'https://pubsub.googleapis.com/v1/projects/a/topics/def', 'method': 'PUT', 'payload': None, }, # Second attempt, succeeds. { 'url': 'https://pubsub.googleapis.com/v1/projects/a/topics/def:publish', 'method': 'POST', 'payload': { 'messages': [ { 'attributes': { 'a': 1, 'b': 2 }, 'data': 'bXNn', }, ], }, }, ]) pubsub.publish('projects/a/topics/def', 'msg', {'a': 1, 'b': 2})
def post(self): """Reclaim a machine. Params: backend_project: If specified, project that the machine reclamation topic is contained in for the backend. backend_attributes: If specified, JSON-encoded dict of attributes to include in the machine reclamation message for the backend. backend_topic: If specified, topic that the machine reclamation should be published to for the backend. hostname: Hostname being reclaimed. lease_id: ID of the LeaseRequest the machine was leased for. lessee_project: If specified, project that the machine reclamation and lease expiration topic is contained in. lessee_topic: If specified, topic that the machine reclamation and lease expiration should be published to for the lessee. machine_id: ID of the CatalogMachineEntry being reclaimed. """ backend_attributes = json.loads( self.request.get('backend_attributes', {})) backend_project = self.request.get('backend_project') backend_topic = self.request.get('backend_topic') hostname = self.request.get('hostname') lease_id = self.request.get('lease_id') lessee_project = self.request.get('lessee_project') lessee_topic = self.request.get('lessee_topic') machine_id = self.request.get('machine_id') if lessee_topic: pubsub.publish( pubsub.full_topic_name(lessee_project, lessee_topic), 'RECLAIMED', { 'machine_id': machine_id, 'request_hash': lease_id, }, ) if backend_topic: attributes = backend_attributes.copy() attributes['hostname'] = hostname pubsub.publish( pubsub.full_topic_name(backend_project, backend_topic), 'RECLAIMED', attributes, )
def post(self): """Reclaim a machine. Params: backend_project: If specified, project that the machine reclamation topic is contained in for the backend. backend_attributes: If specified, JSON-encoded dict of attributes to include in the machine reclamation message for the backend. backend_topic: If specified, topic that the machine reclamation should be published to for the backend. hostname: Hostname being reclaimed. lease_id: ID of the LeaseRequest the machine was leased for. lessee_project: If specified, project that the machine reclamation and lease expiration topic is contained in. lessee_topic: If specified, topic that the machine reclamation and lease expiration should be published to for the lessee. machine_id: ID of the CatalogMachineEntry being reclaimed. """ backend_attributes = json.loads(self.request.get('backend_attributes', {})) backend_project = self.request.get('backend_project') backend_topic = self.request.get('backend_topic') hostname = self.request.get('hostname') lease_id = self.request.get('lease_id') lessee_project = self.request.get('lessee_project') lessee_topic = self.request.get('lessee_topic') machine_id = self.request.get('machine_id') if lessee_topic: pubsub.publish( pubsub.full_topic_name(lessee_project, lessee_topic), 'RECLAIMED', { 'machine_id': machine_id, 'request_hash': lease_id, }, ) if backend_topic: attributes = backend_attributes.copy() attributes['hostname'] = hostname pubsub.publish( pubsub.full_topic_name(backend_project, backend_topic), 'RECLAIMED', attributes )
def post(self): """Fulfill a lease request. Params: lease_id: ID of the LeaseRequest being fulfilled. machine_id: ID of the CatalogMachineEntry fulfilling the LeaseRequest. pubsub_topic: If specified, topic that the lease fulfillment should be published to. pubsub_project: If specified, project that the lease fulfillment topic is contained in. """ lease_id = self.request.get("lease_id") machine_id = self.request.get("machine_id") pubsub_project = self.request.get("pubsub_project") pubsub_topic = self.request.get("pubsub_topic") if pubsub_topic: pubsub.publish(pubsub_topic, pubsub_project, "FULFILLED", machine_id=machine_id, request_hash=lease_id)
def test_publish_fatal_error(self): self.mock_requests([ { 'url': 'https://pubsub.googleapis.com/v1/projects/a/topics/def:publish', 'method': 'POST', 'payload': { 'messages': [ { 'attributes': {'a': 1, 'b': 2}, 'data': 'bXNn', }, ], }, 'response': net.Error('fatal error', 403, ''), } ]) with self.assertRaises(pubsub.Error): pubsub.publish('projects/a/topics/def', 'msg', {'a': 1, 'b': 2})
def _pubsub_notify(task_id, topic, auth_token, userdata): """Sends PubSub notification about task completion. Raises pubsub.TransientError on transient errors. Fatal errors are logged, but not retried. """ logging.debug( 'Sending PubSub notify to "%s" (with userdata "%s") about ' 'completion of "%s"', topic, userdata, task_id) msg = {'task_id': task_id} if userdata: msg['userdata'] = userdata try: pubsub.publish( topic=topic, message=utils.encode_to_json(msg), attributes={'auth_token': auth_token} if auth_token else None) except pubsub.Error: logging.exception('Fatal error when sending PubSub notification')
def maybe_notify_backend(message, hostname, policies): """Informs the backend of the status of a request if there's a Pub/Sub topic. Args: message: The message string to send. hostname: The hostname of the machine this message concerns. policies: A dict representation of an rpc_messages.Policies instance. """ if policies.get('backend_topic'): attributes = { attribute['key']: attribute['value'] for attribute in policies['backend_attributes'] } attributes['hostname'] = hostname pubsub.publish( pubsub.full_topic_name(policies['backend_project'], policies['backend_topic']), message, attributes, )
def maybe_notify_backend(message, hostname, policies): """Informs the backend of the status of a request if there's a Pub/Sub topic. Args: message: The message string to send. hostname: The hostname of the machine this message concerns. policies: A dict representation of an rpc_messages.Policies instance. """ if policies.get('backend_topic'): attributes = { attribute['key']: attribute['value'] for attribute in policies['backend_attributes'] } attributes['hostname'] = hostname pubsub.publish( pubsub.full_topic_name( policies['backend_project'], policies['backend_topic']), message, attributes, )
def maybe_notify_backend(message, hostname, policies): """Informs the backend of the status of a request if there's a Pub/Sub topic. Args: message: The message string to send. hostname: The hostname of the machine this message concerns. policies: A dict representation of an rpc_messages.Policies instance. """ if policies.get('backend_topic'): topic = pubsub.full_topic_name(policies['backend_project'], policies['backend_topic']) attributes = { attribute['key']: attribute['value'] for attribute in policies['backend_attributes'] } attributes['hostname'] = hostname pubsub.publish(topic, message, attributes) # There are relatively few backends, so it's safe to include the # backend topic/project as the value for the target field. metrics.pubsub_messages_sent.increment(fields={'target': topic})
def _pubsub_notify(task_id, topic, auth_token, userdata): """Sends PubSub notification about task completion. Raises pubsub.TransientError on transient errors. Fatal errors are logged, but not retried. """ logging.debug( 'Sending PubSub notify to "%s" (with userdata "%s") about ' 'completion of "%s"', topic, userdata, task_id ) msg = {"task_id": task_id} if userdata: msg["userdata"] = userdata try: pubsub.publish( topic=topic, message=utils.encode_to_json(msg), attributes={"auth_token": auth_token} if auth_token else None, ) except pubsub.Error: logging.exception("Fatal error when sending PubSub notification")
def publish_authdb_change(state): """Publishes AuthDB change notification to the topic. Args: state: AuthReplicationState with version info. """ if utils.is_local_dev_server(): return msg = replication_pb2.ReplicationPushRequest() msg.revision.primary_id = app_identity.get_application_id() msg.revision.auth_db_rev = state.auth_db_rev msg.revision.modified_ts = utils.datetime_to_timestamp(state.modified_ts) blob = msg.SerializeToString() key_name, sig = signature.sign_blob(blob) pubsub.publish(topic_name(), blob, { 'X-AuthDB-SigKey-v1': key_name, 'X-AuthDB-SigVal-v1': base64.b64encode(sig), })
def test_publish_ok(self): self.mock_requests( [ # First attempt. Encounters 404 due to non-existing topic. { "url": "https://pubsub.googleapis.com/v1/projects/a/topics/def:publish", "method": "POST", "payload": {"messages": [{"attributes": {"a": 1, "b": 2}, "data": "bXNn"}]}, "response": net.NotFoundError("topic not found", 404, ""), }, # Creates the topic. {"url": "https://pubsub.googleapis.com/v1/projects/a/topics/def", "method": "PUT", "payload": None}, # Second attempt, succeeds. { "url": "https://pubsub.googleapis.com/v1/projects/a/topics/def:publish", "method": "POST", "payload": {"messages": [{"attributes": {"a": 1, "b": 2}, "data": "bXNn"}]}, }, ] ) pubsub.publish("projects/a/topics/def", "msg", {"a": 1, "b": 2})
def post(self): """Reclaim a machine. Params: backend_topic: If specified, topic that the machine reclamation should be published to for the backend. backend_project: If specified, project that the machine reclamation topic is contained in for the backend. lease_id: ID of the LeaseRequest the machine was leased for. lessee_topic: If specified, topic that the machine reclamation and lease expiration should be published to for the lessee. lessee_project: If specified, project that the machine reclamation and lease expiration topic is contained in. machine_id: ID of the CatalogMachineEntry being reclaimed. """ backend_project = self.request.get('backend_project') backend_topic = self.request.get('backend_topic') lease_id = self.request.get('lease_id') lessee_project = self.request.get('lessee_project') lessee_topic = self.request.get('lessee_topic') machine_id = self.request.get('machine_id') if lessee_topic: pubsub.publish( lessee_topic, lessee_project, 'RECLAIMED', machine_id=machine_id, request_hash=lease_id, ) if backend_topic: pubsub.publish( backend_topic, backend_project, 'RECLAIMED', machine_id=machine_id, )
def test_publish_ok(self): self.mock_requests([ # First attempt. Encounters 404 due to non-existing topic. { 'url': 'https://pubsub.googleapis.com/v1/projects/a/topics/def:publish', 'method': 'POST', 'payload': { 'messages': [ { 'attributes': {'a': 1, 'b': 2}, 'data': 'bXNn', }, ], }, 'response': net.NotFoundError('topic not found', 404, ''), }, # Creates the topic. { 'url': 'https://pubsub.googleapis.com/v1/projects/a/topics/def', 'method': 'PUT', 'payload': None, }, # Second attempt, succeeds. { 'url': 'https://pubsub.googleapis.com/v1/projects/a/topics/def:publish', 'method': 'POST', 'payload': { 'messages': [ { 'attributes': {'a': 1, 'b': 2}, 'data': 'bXNn', }, ], }, }, ]) pubsub.publish('projects/a/topics/def', 'msg', {'a': 1, 'b': 2})
def post(self, build_id): # pylint: disable=unused-argument body = json.loads(self.request.body) pubsub.publish( body['topic'], json.dumps(body['message'], sort_keys=True), body['attrs'])