def post(self): """Subscribe a machine to a Cloud Pub/Sub topic. Params: backend_project: If specified, project that the machine subscription topic is contained in for the backend. backend_attributes: If specified, JSON-encoded dict of attributes to include in the machine subscription message for the backend. backend_topic: If specified, topic that the machine subscription should be published to for the backend. hostname: Hostname being reclaimed. machine_id: ID of the CatalogMachineEntry being reclaimed. machine_service_account: Service account to authorize to consume the subscription. machine_subscription: Cloud Pub/Sub subscription to create for the machine. machine_subscription_project: Project the Cloud Pub/Sub subscription should live in. machine_topic: Cloud Pub/Sub topic to create for the machine. machine_topic_project: Project the Cloud Pub/Sub topic should live in. """ backend_attributes = json.loads( self.request.get('backend_attributes', {})) backend_project = self.request.get('backend_project') backend_topic = self.request.get('backend_topic') hostname = self.request.get('hostname') machine_id = self.request.get('machine_id') machine_service_account = self.request.get('machine_service_account') machine_subscription = self.request.get('machine_subscription') machine_subscription_project = self.request.get( 'machine_subscription_project') machine_topic = self.request.get('machine_topic') machine_topic_project = self.request.get('machine_topic_project') topic = pubsub.full_topic_name(machine_topic_project, machine_topic) subscription = pubsub.full_subscription_name( machine_subscription_project, machine_subscription) pubsub.ensure_subscription_deleted(subscription) pubsub.ensure_subscription_exists(subscription, topic) with pubsub.iam_policy(subscription) as policy: policy.add_member( 'roles/pubsub.subscriber', 'serviceAccount:%s' % machine_service_account, ) if backend_topic: attributes = backend_attributes.copy() attributes['hostname'] = hostname attributes['subscription'] = machine_subscription attributes['subscription_project'] = machine_subscription_project attributes['topic'] = machine_topic attributes['topic_project'] = machine_topic_project pubsub.publish( pubsub.full_topic_name(backend_project, backend_topic), 'SUBSCRIBED', attributes, ) set_available(ndb.Key(models.CatalogMachineEntry, machine_id))
def post(self): """Subscribe a machine to a Cloud Pub/Sub topic. Params: backend_project: If specified, project that the machine subscription topic is contained in for the backend. backend_attributes: If specified, JSON-encoded dict of attributes to include in the machine subscription message for the backend. backend_topic: If specified, topic that the machine subscription should be published to for the backend. hostname: Hostname being reclaimed. machine_id: ID of the CatalogMachineEntry being reclaimed. machine_service_account: Service account to authorize to consume the subscription. machine_subscription: Cloud Pub/Sub subscription to create for the machine. machine_subscription_project: Project the Cloud Pub/Sub subscription should live in. machine_topic: Cloud Pub/Sub topic to create for the machine. machine_topic_project: Project the Cloud Pub/Sub topic should live in. """ backend_attributes = json.loads(self.request.get('backend_attributes', {})) backend_project = self.request.get('backend_project') backend_topic = self.request.get('backend_topic') hostname = self.request.get('hostname') machine_id = self.request.get('machine_id') machine_service_account = self.request.get('machine_service_account') machine_subscription = self.request.get('machine_subscription') machine_subscription_project = self.request.get( 'machine_subscription_project') machine_topic = self.request.get('machine_topic') machine_topic_project = self.request.get('machine_topic_project') topic = pubsub.full_topic_name(machine_topic_project, machine_topic) subscription = pubsub.full_subscription_name( machine_subscription_project, machine_subscription) pubsub.ensure_subscription_deleted(subscription) pubsub.ensure_subscription_exists(subscription, topic) with pubsub.iam_policy(subscription) as policy: policy.add_member( 'roles/pubsub.subscriber', 'serviceAccount:%s' % machine_service_account, ) if backend_topic: attributes = backend_attributes.copy() attributes['hostname'] = hostname attributes['subscription'] = machine_subscription attributes['subscription_project'] = machine_subscription_project attributes['topic'] = machine_topic attributes['topic_project'] = machine_topic_project pubsub.publish( pubsub.full_topic_name(backend_project, backend_topic), 'SUBSCRIBED', attributes, ) set_available(ndb.Key(models.CatalogMachineEntry, machine_id))
def post(self): """Fulfill a lease request. Params: lease_id: ID of the LeaseRequest being fulfilled. machine_id: ID of the CatalogMachineEntry fulfilling the LeaseRequest. pubsub_topic: If specified, topic that the lease fulfillment should be published to. pubsub_project: If specified, project that the lease fulfillment topic is contained in. """ lease_id = self.request.get('lease_id') machine_id = self.request.get('machine_id') pubsub_project = self.request.get('pubsub_project') pubsub_topic = self.request.get('pubsub_topic') if pubsub_topic: pubsub.publish( pubsub.full_topic_name(pubsub_project, pubsub_topic), 'FULFILLED', { 'machine_id': machine_id, 'request_hash': lease_id, }, )
def post(self): """Fulfill a lease request. Params: machine_project: Project that the machine communication topic is contained in. machine_topic: Topic that the machine communication should occur on. policies: JSON-encoded string representation of the rpc_messages.Policies governing this machine. request_json: JSON-encoded string representation of the rpc_messages.LeaseRequest being fulfilled. response_json: JSON-encoded string representation of the rpc_messages.LeaseResponse being delivered. """ machine_project = self.request.get('machine_project') machine_topic = self.request.get('machine_topic') policies = json.loads(self.request.get('policies')) request = json.loads(self.request.get('request_json')) response = json.loads(self.request.get('response_json')) maybe_notify_backend('LEASED', response['hostname'], policies) maybe_notify_lessee(request, response) # Inform the machine. messages = { 'LEASED': { 'lease_expiration_ts': str(response['lease_expiration_ts']) } } if request.get('on_lease', {}).get('swarming_server'): messages['CONNECT'] = { 'swarming_server': request['on_lease']['swarming_server'] } pubsub.publish_multi( pubsub.full_topic_name(machine_project, machine_topic), messages)
def post(self): """Fulfill a lease request. Params: machine_project: Project that the machine communication topic is contained in. machine_topic: Topic that the machine communication should occur on. policies: JSON-encoded string representation of the rpc_messages.Policies governing this machine. request_json: JSON-encoded string representation of the rpc_messages.LeaseRequest being fulfilled. response_json: JSON-encoded string representation of the rpc_messages.LeaseResponse being delivered. """ machine_project = self.request.get('machine_project') machine_topic = self.request.get('machine_topic') policies = json.loads(self.request.get('policies')) request = json.loads(self.request.get('request_json')) response = json.loads(self.request.get('response_json')) maybe_notify_backend('LEASED', response['hostname'], policies) maybe_notify_lessee(request, response) # Inform the machine. messages = { 'LEASED': {'lease_expiration_ts': str(response['lease_expiration_ts'])} } if request.get('on_lease', {}).get('swarming_server'): messages['CONNECT'] = { 'swarming_server': request['on_lease']['swarming_server'] } pubsub.publish_multi( pubsub.full_topic_name(machine_project, machine_topic), messages)
def post(self): """Reclaim a machine. Params: backend_project: If specified, project that the machine reclamation topic is contained in for the backend. backend_attributes: If specified, JSON-encoded dict of attributes to include in the machine reclamation message for the backend. backend_topic: If specified, topic that the machine reclamation should be published to for the backend. hostname: Hostname being reclaimed. lease_id: ID of the LeaseRequest the machine was leased for. lessee_project: If specified, project that the machine reclamation and lease expiration topic is contained in. lessee_topic: If specified, topic that the machine reclamation and lease expiration should be published to for the lessee. machine_id: ID of the CatalogMachineEntry being reclaimed. """ backend_attributes = json.loads( self.request.get('backend_attributes', {})) backend_project = self.request.get('backend_project') backend_topic = self.request.get('backend_topic') hostname = self.request.get('hostname') lease_id = self.request.get('lease_id') lessee_project = self.request.get('lessee_project') lessee_topic = self.request.get('lessee_topic') machine_id = self.request.get('machine_id') if lessee_topic: pubsub.publish( pubsub.full_topic_name(lessee_project, lessee_topic), 'RECLAIMED', { 'machine_id': machine_id, 'request_hash': lease_id, }, ) if backend_topic: attributes = backend_attributes.copy() attributes['hostname'] = hostname pubsub.publish( pubsub.full_topic_name(backend_project, backend_topic), 'RECLAIMED', attributes, )
def post(self): """Reclaim a machine. Params: backend_project: If specified, project that the machine reclamation topic is contained in for the backend. backend_attributes: If specified, JSON-encoded dict of attributes to include in the machine reclamation message for the backend. backend_topic: If specified, topic that the machine reclamation should be published to for the backend. hostname: Hostname being reclaimed. lease_id: ID of the LeaseRequest the machine was leased for. lessee_project: If specified, project that the machine reclamation and lease expiration topic is contained in. lessee_topic: If specified, topic that the machine reclamation and lease expiration should be published to for the lessee. machine_id: ID of the CatalogMachineEntry being reclaimed. """ backend_attributes = json.loads(self.request.get('backend_attributes', {})) backend_project = self.request.get('backend_project') backend_topic = self.request.get('backend_topic') hostname = self.request.get('hostname') lease_id = self.request.get('lease_id') lessee_project = self.request.get('lessee_project') lessee_topic = self.request.get('lessee_topic') machine_id = self.request.get('machine_id') if lessee_topic: pubsub.publish( pubsub.full_topic_name(lessee_project, lessee_topic), 'RECLAIMED', { 'machine_id': machine_id, 'request_hash': lease_id, }, ) if backend_topic: attributes = backend_attributes.copy() attributes['hostname'] = hostname pubsub.publish( pubsub.full_topic_name(backend_project, backend_topic), 'RECLAIMED', attributes )
def maybe_notify_lessee(request, response): """Informs the lessee of the status of a request if there's a Pub/Sub topic. Args: request: A dict representation of an rpc_messages.LeaseRequest instance. response: A dict representation of an rpc_messages.LeaseResponse instance. """ if request.get('pubsub_topic'): pubsub.publish( pubsub.full_topic_name(request['pubsub_project'], request['pubsub_topic']), json.dumps(response), )
def maybe_notify_lessee(request, response): """Informs the lessee of the status of a request if there's a Pub/Sub topic. Args: request: A dict representation of an rpc_messages.LeaseRequest instance. response: A dict representation of an rpc_messages.LeaseResponse instance. """ if request.get('pubsub_topic'): pubsub.publish( pubsub.full_topic_name( request['pubsub_project'], request['pubsub_topic']), json.dumps(response), )
def post(self): """Reclaim a machine. Params: hostname: Hostname of the machine being reclaimed. machine_key: URL-safe ndb.Key for a models.CatalogMachineEntry. machine_subscription: Subscription created for the machine to listen for instructions on. machine_subscription_project: Project that the machine subscription is contained in. machine_topic: Topic that the machine communication should occur on. machine_topic_project: Project that the machine communication topic is contained in. policies: JSON-encoded string representation of the rpc_messages.Policies governing this machine. request_json: JSON-encoded string representation of the rpc_messages.LeaseRequest being fulfilled. response_json: JSON-encoded string representation of the rpc_messages.LeaseResponse being delivered. """ hostname = self.request.get('hostname') machine_key = ndb.Key(urlsafe=self.request.get('machine_key')) machine_subscription = self.request.get('machine_subscription') machine_subscription_project = self.request.get( 'machine_subscription_project') machine_topic = self.request.get('machine_topic') machine_topic_project = self.request.get('machine_topic_project') policies = json.loads(self.request.get('policies')) request = json.loads(self.request.get('request_json')) response = json.loads(self.request.get('response_json')) assert machine_key.kind() == 'CatalogMachineEntry', machine_key maybe_notify_backend('RECLAIMED', hostname, policies) maybe_notify_lessee(request, response) # Delete machine Pub/Sub channel. pubsub.ensure_subscription_deleted( pubsub.full_subscription_name(machine_subscription_project, machine_subscription)) pubsub.ensure_topic_deleted( pubsub.full_topic_name(machine_topic_project, machine_topic)) reclaim(machine_key) metrics.lease_requests_expired.increment()
def maybe_notify_backend(message, hostname, policies): """Informs the backend of the status of a request if there's a Pub/Sub topic. Args: message: The message string to send. hostname: The hostname of the machine this message concerns. policies: A dict representation of an rpc_messages.Policies instance. """ if policies.get('backend_topic'): topic = pubsub.full_topic_name(policies['backend_project'], policies['backend_topic']) attributes = { attribute['key']: attribute['value'] for attribute in policies['backend_attributes'] } attributes['hostname'] = hostname pubsub.publish(topic, message, attributes) # There are relatively few backends, so it's safe to include the # backend topic/project as the value for the target field. metrics.pubsub_messages_sent.increment(fields={'target': topic})
def maybe_notify_backend(message, hostname, policies): """Informs the backend of the status of a request if there's a Pub/Sub topic. Args: message: The message string to send. hostname: The hostname of the machine this message concerns. policies: A dict representation of an rpc_messages.Policies instance. """ if policies.get('backend_topic'): attributes = { attribute['key']: attribute['value'] for attribute in policies['backend_attributes'] } attributes['hostname'] = hostname pubsub.publish( pubsub.full_topic_name(policies['backend_project'], policies['backend_topic']), message, attributes, )
def maybe_notify_backend(message, hostname, policies): """Informs the backend of the status of a request if there's a Pub/Sub topic. Args: message: The message string to send. hostname: The hostname of the machine this message concerns. policies: A dict representation of an rpc_messages.Policies instance. """ if policies.get('backend_topic'): attributes = { attribute['key']: attribute['value'] for attribute in policies['backend_attributes'] } attributes['hostname'] = hostname pubsub.publish( pubsub.full_topic_name( policies['backend_project'], policies['backend_topic']), message, attributes, )
def topic_name(): """Full name of PubSub topic that receives AuthDB change notifications.""" return pubsub.full_topic_name( app_identity.get_application_id(), 'auth-db-changed')
def test_full_topic_name(self): self.assertEqual( 'projects/abc/topics/def', pubsub.full_topic_name('abc', 'def'))
def test_full_topic_name(self): self.assertEqual("projects/abc/topics/def", pubsub.full_topic_name("abc", "def"))