def post(self):
"""Subscribe a machine to a Cloud Pub/Sub topic.
Params:
backend_project: If specified, project that the machine subscription
topic is contained in for the backend.
backend_attributes: If specified, JSON-encoded dict of attributes to
include in the machine subscription message for the backend.
backend_topic: If specified, topic that the machine subscription should
be published to for the backend.
hostname: Hostname being reclaimed.
machine_id: ID of the CatalogMachineEntry being reclaimed.
machine_service_account: Service account to authorize to consume the
subscription.
machine_subscription: Cloud Pub/Sub subscription to create for the
machine.
machine_subscription_project: Project the Cloud Pub/Sub subscription
should live in.
machine_topic: Cloud Pub/Sub topic to create for the machine.
machine_topic_project: Project the Cloud Pub/Sub topic should live in.
"""
backend_attributes = json.loads(
self.request.get('backend_attributes', {}))
backend_project = self.request.get('backend_project')
backend_topic = self.request.get('backend_topic')
hostname = self.request.get('hostname')
machine_id = self.request.get('machine_id')
machine_service_account = self.request.get('machine_service_account')
machine_subscription = self.request.get('machine_subscription')
machine_subscription_project = self.request.get(
'machine_subscription_project')
machine_topic = self.request.get('machine_topic')
machine_topic_project = self.request.get('machine_topic_project')
topic = pubsub.full_topic_name(machine_topic_project, machine_topic)
subscription = pubsub.full_subscription_name(
machine_subscription_project, machine_subscription)
pubsub.ensure_subscription_deleted(subscription)
pubsub.ensure_subscription_exists(subscription, topic)
with pubsub.iam_policy(subscription) as policy:
policy.add_member(
'roles/pubsub.subscriber',
'serviceAccount:%s' % machine_service_account,
)
if backend_topic:
attributes = backend_attributes.copy()
attributes['hostname'] = hostname
attributes['subscription'] = machine_subscription
attributes['subscription_project'] = machine_subscription_project
attributes['topic'] = machine_topic
attributes['topic_project'] = machine_topic_project
pubsub.publish(
pubsub.full_topic_name(backend_project, backend_topic),
'SUBSCRIBED',
attributes,
)
set_available(ndb.Key(models.CatalogMachineEntry, machine_id))
def post(self):
"""Subscribe a machine to a Cloud Pub/Sub topic.
Params:
backend_project: If specified, project that the machine subscription
topic is contained in for the backend.
backend_attributes: If specified, JSON-encoded dict of attributes to
include in the machine subscription message for the backend.
backend_topic: If specified, topic that the machine subscription should
be published to for the backend.
hostname: Hostname being reclaimed.
machine_id: ID of the CatalogMachineEntry being reclaimed.
machine_service_account: Service account to authorize to consume the
subscription.
machine_subscription: Cloud Pub/Sub subscription to create for the
machine.
machine_subscription_project: Project the Cloud Pub/Sub subscription
should live in.
machine_topic: Cloud Pub/Sub topic to create for the machine.
machine_topic_project: Project the Cloud Pub/Sub topic should live in.
"""
backend_attributes = json.loads(self.request.get('backend_attributes', {}))
backend_project = self.request.get('backend_project')
backend_topic = self.request.get('backend_topic')
hostname = self.request.get('hostname')
machine_id = self.request.get('machine_id')
machine_service_account = self.request.get('machine_service_account')
machine_subscription = self.request.get('machine_subscription')
machine_subscription_project = self.request.get(
'machine_subscription_project')
machine_topic = self.request.get('machine_topic')
machine_topic_project = self.request.get('machine_topic_project')
topic = pubsub.full_topic_name(machine_topic_project, machine_topic)
subscription = pubsub.full_subscription_name(
machine_subscription_project, machine_subscription)
pubsub.ensure_subscription_deleted(subscription)
pubsub.ensure_subscription_exists(subscription, topic)
with pubsub.iam_policy(subscription) as policy:
policy.add_member(
'roles/pubsub.subscriber',
'serviceAccount:%s' % machine_service_account,
)
if backend_topic:
attributes = backend_attributes.copy()
attributes['hostname'] = hostname
attributes['subscription'] = machine_subscription
attributes['subscription_project'] = machine_subscription_project
attributes['topic'] = machine_topic
attributes['topic_project'] = machine_topic_project
pubsub.publish(
pubsub.full_topic_name(backend_project, backend_topic),
'SUBSCRIBED',
attributes,
)
set_available(ndb.Key(models.CatalogMachineEntry, machine_id))
def post(self):
"""Fulfill a lease request.
Params:
lease_id: ID of the LeaseRequest being fulfilled.
machine_id: ID of the CatalogMachineEntry fulfilling the LeaseRequest.
pubsub_topic: If specified, topic that the lease fulfillment should be
published to.
pubsub_project: If specified, project that the lease fulfillment topic is
contained in.
"""
lease_id = self.request.get('lease_id')
machine_id = self.request.get('machine_id')
pubsub_project = self.request.get('pubsub_project')
pubsub_topic = self.request.get('pubsub_topic')
if pubsub_topic:
pubsub.publish(
pubsub.full_topic_name(pubsub_project, pubsub_topic),
'FULFILLED',
{
'machine_id': machine_id,
'request_hash': lease_id,
},
)
def post(self):
"""Fulfill a lease request.
Params:
machine_project: Project that the machine communication topic is contained
in.
machine_topic: Topic that the machine communication should occur on.
policies: JSON-encoded string representation of the
rpc_messages.Policies governing this machine.
request_json: JSON-encoded string representation of the
rpc_messages.LeaseRequest being fulfilled.
response_json: JSON-encoded string representation of the
rpc_messages.LeaseResponse being delivered.
"""
machine_project = self.request.get('machine_project')
machine_topic = self.request.get('machine_topic')
policies = json.loads(self.request.get('policies'))
request = json.loads(self.request.get('request_json'))
response = json.loads(self.request.get('response_json'))
maybe_notify_backend('LEASED', response['hostname'], policies)
maybe_notify_lessee(request, response)
# Inform the machine.
messages = {
'LEASED': {
'lease_expiration_ts': str(response['lease_expiration_ts'])
}
}
if request.get('on_lease', {}).get('swarming_server'):
messages['CONNECT'] = {
'swarming_server': request['on_lease']['swarming_server']
}
pubsub.publish_multi(
pubsub.full_topic_name(machine_project, machine_topic), messages)
def post(self):
"""Fulfill a lease request.
Params:
lease_id: ID of the LeaseRequest being fulfilled.
machine_id: ID of the CatalogMachineEntry fulfilling the LeaseRequest.
pubsub_topic: If specified, topic that the lease fulfillment should be
published to.
pubsub_project: If specified, project that the lease fulfillment topic is
contained in.
"""
lease_id = self.request.get('lease_id')
machine_id = self.request.get('machine_id')
pubsub_project = self.request.get('pubsub_project')
pubsub_topic = self.request.get('pubsub_topic')
if pubsub_topic:
pubsub.publish(
pubsub.full_topic_name(pubsub_project, pubsub_topic),
'FULFILLED',
{
'machine_id': machine_id,
'request_hash': lease_id,
},
)
def post(self):
"""Fulfill a lease request.
Params:
machine_project: Project that the machine communication topic is contained
in.
machine_topic: Topic that the machine communication should occur on.
policies: JSON-encoded string representation of the
rpc_messages.Policies governing this machine.
request_json: JSON-encoded string representation of the
rpc_messages.LeaseRequest being fulfilled.
response_json: JSON-encoded string representation of the
rpc_messages.LeaseResponse being delivered.
"""
machine_project = self.request.get('machine_project')
machine_topic = self.request.get('machine_topic')
policies = json.loads(self.request.get('policies'))
request = json.loads(self.request.get('request_json'))
response = json.loads(self.request.get('response_json'))
maybe_notify_backend('LEASED', response['hostname'], policies)
maybe_notify_lessee(request, response)
# Inform the machine.
messages = {
'LEASED': {'lease_expiration_ts': str(response['lease_expiration_ts'])}
}
if request.get('on_lease', {}).get('swarming_server'):
messages['CONNECT'] = {
'swarming_server': request['on_lease']['swarming_server']
}
pubsub.publish_multi(
pubsub.full_topic_name(machine_project, machine_topic), messages)
def post(self):
"""Reclaim a machine.
Params:
backend_project: If specified, project that the machine reclamation
topic is contained in for the backend.
backend_attributes: If specified, JSON-encoded dict of attributes to
include in the machine reclamation message for the backend.
backend_topic: If specified, topic that the machine reclamation should
be published to for the backend.
hostname: Hostname being reclaimed.
lease_id: ID of the LeaseRequest the machine was leased for.
lessee_project: If specified, project that the machine reclamation and
lease expiration topic is contained in.
lessee_topic: If specified, topic that the machine reclamation and lease
expiration should be published to for the lessee.
machine_id: ID of the CatalogMachineEntry being reclaimed.
"""
backend_attributes = json.loads(
self.request.get('backend_attributes', {}))
backend_project = self.request.get('backend_project')
backend_topic = self.request.get('backend_topic')
hostname = self.request.get('hostname')
lease_id = self.request.get('lease_id')
lessee_project = self.request.get('lessee_project')
lessee_topic = self.request.get('lessee_topic')
machine_id = self.request.get('machine_id')
if lessee_topic:
pubsub.publish(
pubsub.full_topic_name(lessee_project, lessee_topic),
'RECLAIMED',
{
'machine_id': machine_id,
'request_hash': lease_id,
},
)
if backend_topic:
attributes = backend_attributes.copy()
attributes['hostname'] = hostname
pubsub.publish(
pubsub.full_topic_name(backend_project, backend_topic),
'RECLAIMED',
attributes,
)
def post(self):
"""Reclaim a machine.
Params:
backend_project: If specified, project that the machine reclamation
topic is contained in for the backend.
backend_attributes: If specified, JSON-encoded dict of attributes to
include in the machine reclamation message for the backend.
backend_topic: If specified, topic that the machine reclamation should
be published to for the backend.
hostname: Hostname being reclaimed.
lease_id: ID of the LeaseRequest the machine was leased for.
lessee_project: If specified, project that the machine reclamation and
lease expiration topic is contained in.
lessee_topic: If specified, topic that the machine reclamation and lease
expiration should be published to for the lessee.
machine_id: ID of the CatalogMachineEntry being reclaimed.
"""
backend_attributes = json.loads(self.request.get('backend_attributes', {}))
backend_project = self.request.get('backend_project')
backend_topic = self.request.get('backend_topic')
hostname = self.request.get('hostname')
lease_id = self.request.get('lease_id')
lessee_project = self.request.get('lessee_project')
lessee_topic = self.request.get('lessee_topic')
machine_id = self.request.get('machine_id')
if lessee_topic:
pubsub.publish(
pubsub.full_topic_name(lessee_project, lessee_topic),
'RECLAIMED',
{
'machine_id': machine_id,
'request_hash': lease_id,
},
)
if backend_topic:
attributes = backend_attributes.copy()
attributes['hostname'] = hostname
pubsub.publish(
pubsub.full_topic_name(backend_project, backend_topic),
'RECLAIMED',
attributes
)
def maybe_notify_lessee(request, response):
"""Informs the lessee of the status of a request if there's a Pub/Sub topic.
Args:
request: A dict representation of an rpc_messages.LeaseRequest instance.
response: A dict representation of an rpc_messages.LeaseResponse instance.
"""
if request.get('pubsub_topic'):
pubsub.publish(
pubsub.full_topic_name(request['pubsub_project'],
request['pubsub_topic']),
json.dumps(response),
)
def maybe_notify_lessee(request, response):
"""Informs the lessee of the status of a request if there's a Pub/Sub topic.
Args:
request: A dict representation of an rpc_messages.LeaseRequest instance.
response: A dict representation of an rpc_messages.LeaseResponse instance.
"""
if request.get('pubsub_topic'):
pubsub.publish(
pubsub.full_topic_name(
request['pubsub_project'], request['pubsub_topic']),
json.dumps(response),
)
def post(self):
"""Reclaim a machine.
Params:
hostname: Hostname of the machine being reclaimed.
machine_key: URL-safe ndb.Key for a models.CatalogMachineEntry.
machine_subscription: Subscription created for the machine to listen
for instructions on.
machine_subscription_project: Project that the machine subscription is
contained in.
machine_topic: Topic that the machine communication should occur on.
machine_topic_project: Project that the machine communication topic is
contained in.
policies: JSON-encoded string representation of the
rpc_messages.Policies governing this machine.
request_json: JSON-encoded string representation of the
rpc_messages.LeaseRequest being fulfilled.
response_json: JSON-encoded string representation of the
rpc_messages.LeaseResponse being delivered.
"""
hostname = self.request.get('hostname')
machine_key = ndb.Key(urlsafe=self.request.get('machine_key'))
machine_subscription = self.request.get('machine_subscription')
machine_subscription_project = self.request.get(
'machine_subscription_project')
machine_topic = self.request.get('machine_topic')
machine_topic_project = self.request.get('machine_topic_project')
policies = json.loads(self.request.get('policies'))
request = json.loads(self.request.get('request_json'))
response = json.loads(self.request.get('response_json'))
assert machine_key.kind() == 'CatalogMachineEntry', machine_key
maybe_notify_backend('RECLAIMED', hostname, policies)
maybe_notify_lessee(request, response)
# Delete machine Pub/Sub channel.
pubsub.ensure_subscription_deleted(
pubsub.full_subscription_name(machine_subscription_project,
machine_subscription))
pubsub.ensure_topic_deleted(
pubsub.full_topic_name(machine_topic_project, machine_topic))
reclaim(machine_key)
metrics.lease_requests_expired.increment()
def maybe_notify_backend(message, hostname, policies):
"""Informs the backend of the status of a request if there's a Pub/Sub topic.
Args:
message: The message string to send.
hostname: The hostname of the machine this message concerns.
policies: A dict representation of an rpc_messages.Policies instance.
"""
if policies.get('backend_topic'):
topic = pubsub.full_topic_name(policies['backend_project'],
policies['backend_topic'])
attributes = {
attribute['key']: attribute['value']
for attribute in policies['backend_attributes']
}
attributes['hostname'] = hostname
pubsub.publish(topic, message, attributes)
# There are relatively few backends, so it's safe to include the
# backend topic/project as the value for the target field.
metrics.pubsub_messages_sent.increment(fields={'target': topic})
def maybe_notify_backend(message, hostname, policies):
"""Informs the backend of the status of a request if there's a Pub/Sub topic.
Args:
message: The message string to send.
hostname: The hostname of the machine this message concerns.
policies: A dict representation of an rpc_messages.Policies instance.
"""
if policies.get('backend_topic'):
attributes = {
attribute['key']: attribute['value']
for attribute in policies['backend_attributes']
}
attributes['hostname'] = hostname
pubsub.publish(
pubsub.full_topic_name(policies['backend_project'],
policies['backend_topic']),
message,
attributes,
)
def maybe_notify_backend(message, hostname, policies):
"""Informs the backend of the status of a request if there's a Pub/Sub topic.
Args:
message: The message string to send.
hostname: The hostname of the machine this message concerns.
policies: A dict representation of an rpc_messages.Policies instance.
"""
if policies.get('backend_topic'):
attributes = {
attribute['key']: attribute['value']
for attribute in policies['backend_attributes']
}
attributes['hostname'] = hostname
pubsub.publish(
pubsub.full_topic_name(
policies['backend_project'], policies['backend_topic']),
message,
attributes,
)
def topic_name():
"""Full name of PubSub topic that receives AuthDB change notifications."""
return pubsub.full_topic_name(
app_identity.get_application_id(), 'auth-db-changed')
def test_full_topic_name(self):
self.assertEqual(
'projects/abc/topics/def', pubsub.full_topic_name('abc', 'def'))
def test_full_topic_name(self):
self.assertEqual("projects/abc/topics/def", pubsub.full_topic_name("abc", "def"))
def topic_name():
"""Full name of PubSub topic that receives AuthDB change notifications."""
return pubsub.full_topic_name(
app_identity.get_application_id(), 'auth-db-changed')