def post(self):
"""Subscribe a machine to a Cloud Pub/Sub topic.
Params:
backend_project: If specified, project that the machine subscription
topic is contained in for the backend.
backend_attributes: If specified, JSON-encoded dict of attributes to
include in the machine subscription message for the backend.
backend_topic: If specified, topic that the machine subscription should
be published to for the backend.
hostname: Hostname being reclaimed.
machine_id: ID of the CatalogMachineEntry being reclaimed.
machine_service_account: Service account to authorize to consume the
subscription.
machine_subscription: Cloud Pub/Sub subscription to create for the
machine.
machine_subscription_project: Project the Cloud Pub/Sub subscription
should live in.
machine_topic: Cloud Pub/Sub topic to create for the machine.
machine_topic_project: Project the Cloud Pub/Sub topic should live in.
"""
backend_attributes = json.loads(
self.request.get('backend_attributes', {}))
backend_project = self.request.get('backend_project')
backend_topic = self.request.get('backend_topic')
hostname = self.request.get('hostname')
machine_id = self.request.get('machine_id')
machine_service_account = self.request.get('machine_service_account')
machine_subscription = self.request.get('machine_subscription')
machine_subscription_project = self.request.get(
'machine_subscription_project')
machine_topic = self.request.get('machine_topic')
machine_topic_project = self.request.get('machine_topic_project')
topic = pubsub.full_topic_name(machine_topic_project, machine_topic)
subscription = pubsub.full_subscription_name(
machine_subscription_project, machine_subscription)
pubsub.ensure_subscription_deleted(subscription)
pubsub.ensure_subscription_exists(subscription, topic)
with pubsub.iam_policy(subscription) as policy:
policy.add_member(
'roles/pubsub.subscriber',
'serviceAccount:%s' % machine_service_account,
)
if backend_topic:
attributes = backend_attributes.copy()
attributes['hostname'] = hostname
attributes['subscription'] = machine_subscription
attributes['subscription_project'] = machine_subscription_project
attributes['topic'] = machine_topic
attributes['topic_project'] = machine_topic_project
pubsub.publish(
pubsub.full_topic_name(backend_project, backend_topic),
'SUBSCRIBED',
attributes,
)
set_available(ndb.Key(models.CatalogMachineEntry, machine_id))
def post(self):
"""Fulfill a lease request.
Params:
lease_id: ID of the LeaseRequest being fulfilled.
machine_id: ID of the CatalogMachineEntry fulfilling the LeaseRequest.
pubsub_topic: If specified, topic that the lease fulfillment should be
published to.
pubsub_project: If specified, project that the lease fulfillment topic is
contained in.
"""
lease_id = self.request.get('lease_id')
machine_id = self.request.get('machine_id')
pubsub_project = self.request.get('pubsub_project')
pubsub_topic = self.request.get('pubsub_topic')
if pubsub_topic:
pubsub.publish(
pubsub.full_topic_name(pubsub_project, pubsub_topic),
'FULFILLED',
{
'machine_id': machine_id,
'request_hash': lease_id,
},
)
def post(self):
"""Fulfill a lease request.
Params:
lease_id: ID of the LeaseRequest being fulfilled.
machine_id: ID of the CatalogMachineEntry fulfilling the LeaseRequest.
pubsub_topic: If specified, topic that the lease fulfillment should be
published to.
pubsub_project: If specified, project that the lease fulfillment topic is
contained in.
"""
lease_id = self.request.get('lease_id')
machine_id = self.request.get('machine_id')
pubsub_project = self.request.get('pubsub_project')
pubsub_topic = self.request.get('pubsub_topic')
if pubsub_topic:
pubsub.publish(
pubsub.full_topic_name(pubsub_project, pubsub_topic),
'FULFILLED',
{
'machine_id': machine_id,
'request_hash': lease_id,
},
)
def post(self):
"""Reclaim a machine.
Params:
backend_topic: If specified, topic that the machine reclamation should
be published to for the backend.
backend_project: If specified, project that the machine reclamation
topic is contained in for the backend.
lease_id: ID of the LeaseRequest the machine was leased for.
lessee_topic: If specified, topic that the machine reclamation and lease
expiration should be published to for the lessee.
lessee_project: If specified, project that the machine reclamation and
lease expiration topic is contained in.
machine_id: ID of the CatalogMachineEntry being reclaimed.
"""
backend_project = self.request.get("backend_project")
backend_topic = self.request.get("backend_topic")
lease_id = self.request.get("lease_id")
lessee_project = self.request.get("lessee_project")
lessee_topic = self.request.get("lessee_topic")
machine_id = self.request.get("machine_id")
if lessee_topic:
pubsub.publish(lessee_topic, lessee_project, "RECLAIMED", machine_id=machine_id, request_hash=lease_id)
if backend_topic:
pubsub.publish(backend_topic, backend_project, "RECLAIMED", machine_id=machine_id)
def post(self, build_id): # pylint: disable=unused-argument
body = json.loads(self.request.body)
assert body.get('mode') in ('global', 'callback')
bundle = model.BuildBundle.get(
body['id'],
infra=True,
input_properties=True,
output_properties=True,
)
if not bundle: # pragma: no cover
return
build = bundle.build
message = {
'build': api_common.build_to_dict(bundle),
'hostname': app_identity.get_default_version_hostname(),
}
attrs = {'build_id': str(build.key.id())}
if body['mode'] == 'callback':
topic = build.pubsub_callback.topic
message['user_data'] = build.pubsub_callback.user_data
attrs['auth_token'] = build.pubsub_callback.auth_token
else:
topic = 'projects/%s/topics/builds' % app_identity.get_application_id()
pubsub.publish(topic, json.dumps(message, sort_keys=True), attrs)
def post(self):
"""Subscribe a machine to a Cloud Pub/Sub topic.
Params:
backend_project: If specified, project that the machine subscription
topic is contained in for the backend.
backend_attributes: If specified, JSON-encoded dict of attributes to
include in the machine subscription message for the backend.
backend_topic: If specified, topic that the machine subscription should
be published to for the backend.
hostname: Hostname being reclaimed.
machine_id: ID of the CatalogMachineEntry being reclaimed.
machine_service_account: Service account to authorize to consume the
subscription.
machine_subscription: Cloud Pub/Sub subscription to create for the
machine.
machine_subscription_project: Project the Cloud Pub/Sub subscription
should live in.
machine_topic: Cloud Pub/Sub topic to create for the machine.
machine_topic_project: Project the Cloud Pub/Sub topic should live in.
"""
backend_attributes = json.loads(self.request.get('backend_attributes', {}))
backend_project = self.request.get('backend_project')
backend_topic = self.request.get('backend_topic')
hostname = self.request.get('hostname')
machine_id = self.request.get('machine_id')
machine_service_account = self.request.get('machine_service_account')
machine_subscription = self.request.get('machine_subscription')
machine_subscription_project = self.request.get(
'machine_subscription_project')
machine_topic = self.request.get('machine_topic')
machine_topic_project = self.request.get('machine_topic_project')
topic = pubsub.full_topic_name(machine_topic_project, machine_topic)
subscription = pubsub.full_subscription_name(
machine_subscription_project, machine_subscription)
pubsub.ensure_subscription_deleted(subscription)
pubsub.ensure_subscription_exists(subscription, topic)
with pubsub.iam_policy(subscription) as policy:
policy.add_member(
'roles/pubsub.subscriber',
'serviceAccount:%s' % machine_service_account,
)
if backend_topic:
attributes = backend_attributes.copy()
attributes['hostname'] = hostname
attributes['subscription'] = machine_subscription
attributes['subscription_project'] = machine_subscription_project
attributes['topic'] = machine_topic
attributes['topic_project'] = machine_topic_project
pubsub.publish(
pubsub.full_topic_name(backend_project, backend_topic),
'SUBSCRIBED',
attributes,
)
set_available(ndb.Key(models.CatalogMachineEntry, machine_id))
def _publish_pubsub_message(
build_id, topic, user_data, auth_token): # pragma: no cover
message = json.dumps({
'build_id': build_id,
'user_data': user_data,
}, sort_keys=True)
attrs = {
'build_id': str(build_id),
'auth_token': auth_token,
}
pubsub.publish(topic, message, attrs)
def test_publish_fatal_error(self):
self.mock_requests(
[
{
"url": "https://pubsub.googleapis.com/v1/projects/a/topics/def:publish",
"method": "POST",
"payload": {"messages": [{"attributes": {"a": 1, "b": 2}, "data": "bXNn"}]},
"response": net.Error("fatal error", 403, ""),
}
]
)
with self.assertRaises(pubsub.Error):
pubsub.publish("projects/a/topics/def", "msg", {"a": 1, "b": 2})
def maybe_notify_lessee(request, response):
"""Informs the lessee of the status of a request if there's a Pub/Sub topic.
Args:
request: A dict representation of an rpc_messages.LeaseRequest instance.
response: A dict representation of an rpc_messages.LeaseResponse instance.
"""
if request.get('pubsub_topic'):
pubsub.publish(
pubsub.full_topic_name(request['pubsub_project'],
request['pubsub_topic']),
json.dumps(response),
)
def maybe_notify_lessee(request, response):
"""Informs the lessee of the status of a request if there's a Pub/Sub topic.
Args:
request: A dict representation of an rpc_messages.LeaseRequest instance.
response: A dict representation of an rpc_messages.LeaseResponse instance.
"""
if request.get('pubsub_topic'):
pubsub.publish(
pubsub.full_topic_name(
request['pubsub_project'], request['pubsub_topic']),
json.dumps(response),
)
def test_publish_ok(self):
self.mock_requests([
# First attempt. Encounters 404 due to non-existing topic.
{
'url':
'https://pubsub.googleapis.com/v1/projects/a/topics/def:publish',
'method': 'POST',
'payload': {
'messages': [
{
'attributes': {
'a': 1,
'b': 2
},
'data': 'bXNn',
},
],
},
'response': net.NotFoundError('topic not found', 404, ''),
},
# Creates the topic.
{
'url':
'https://pubsub.googleapis.com/v1/projects/a/topics/def',
'method': 'PUT',
'payload': None,
},
# Second attempt, succeeds.
{
'url':
'https://pubsub.googleapis.com/v1/projects/a/topics/def:publish',
'method': 'POST',
'payload': {
'messages': [
{
'attributes': {
'a': 1,
'b': 2
},
'data': 'bXNn',
},
],
},
},
])
pubsub.publish('projects/a/topics/def', 'msg', {'a': 1, 'b': 2})
def post(self):
"""Reclaim a machine.
Params:
backend_project: If specified, project that the machine reclamation
topic is contained in for the backend.
backend_attributes: If specified, JSON-encoded dict of attributes to
include in the machine reclamation message for the backend.
backend_topic: If specified, topic that the machine reclamation should
be published to for the backend.
hostname: Hostname being reclaimed.
lease_id: ID of the LeaseRequest the machine was leased for.
lessee_project: If specified, project that the machine reclamation and
lease expiration topic is contained in.
lessee_topic: If specified, topic that the machine reclamation and lease
expiration should be published to for the lessee.
machine_id: ID of the CatalogMachineEntry being reclaimed.
"""
backend_attributes = json.loads(
self.request.get('backend_attributes', {}))
backend_project = self.request.get('backend_project')
backend_topic = self.request.get('backend_topic')
hostname = self.request.get('hostname')
lease_id = self.request.get('lease_id')
lessee_project = self.request.get('lessee_project')
lessee_topic = self.request.get('lessee_topic')
machine_id = self.request.get('machine_id')
if lessee_topic:
pubsub.publish(
pubsub.full_topic_name(lessee_project, lessee_topic),
'RECLAIMED',
{
'machine_id': machine_id,
'request_hash': lease_id,
},
)
if backend_topic:
attributes = backend_attributes.copy()
attributes['hostname'] = hostname
pubsub.publish(
pubsub.full_topic_name(backend_project, backend_topic),
'RECLAIMED',
attributes,
)
def post(self):
"""Reclaim a machine.
Params:
backend_project: If specified, project that the machine reclamation
topic is contained in for the backend.
backend_attributes: If specified, JSON-encoded dict of attributes to
include in the machine reclamation message for the backend.
backend_topic: If specified, topic that the machine reclamation should
be published to for the backend.
hostname: Hostname being reclaimed.
lease_id: ID of the LeaseRequest the machine was leased for.
lessee_project: If specified, project that the machine reclamation and
lease expiration topic is contained in.
lessee_topic: If specified, topic that the machine reclamation and lease
expiration should be published to for the lessee.
machine_id: ID of the CatalogMachineEntry being reclaimed.
"""
backend_attributes = json.loads(self.request.get('backend_attributes', {}))
backend_project = self.request.get('backend_project')
backend_topic = self.request.get('backend_topic')
hostname = self.request.get('hostname')
lease_id = self.request.get('lease_id')
lessee_project = self.request.get('lessee_project')
lessee_topic = self.request.get('lessee_topic')
machine_id = self.request.get('machine_id')
if lessee_topic:
pubsub.publish(
pubsub.full_topic_name(lessee_project, lessee_topic),
'RECLAIMED',
{
'machine_id': machine_id,
'request_hash': lease_id,
},
)
if backend_topic:
attributes = backend_attributes.copy()
attributes['hostname'] = hostname
pubsub.publish(
pubsub.full_topic_name(backend_project, backend_topic),
'RECLAIMED',
attributes
)
def post(self):
"""Fulfill a lease request.
Params:
lease_id: ID of the LeaseRequest being fulfilled.
machine_id: ID of the CatalogMachineEntry fulfilling the LeaseRequest.
pubsub_topic: If specified, topic that the lease fulfillment should be
published to.
pubsub_project: If specified, project that the lease fulfillment topic is
contained in.
"""
lease_id = self.request.get("lease_id")
machine_id = self.request.get("machine_id")
pubsub_project = self.request.get("pubsub_project")
pubsub_topic = self.request.get("pubsub_topic")
if pubsub_topic:
pubsub.publish(pubsub_topic, pubsub_project, "FULFILLED", machine_id=machine_id, request_hash=lease_id)
def test_publish_fatal_error(self):
self.mock_requests([
{
'url': 'https://pubsub.googleapis.com/v1/projects/a/topics/def:publish',
'method': 'POST',
'payload': {
'messages': [
{
'attributes': {'a': 1, 'b': 2},
'data': 'bXNn',
},
],
},
'response': net.Error('fatal error', 403, ''),
}
])
with self.assertRaises(pubsub.Error):
pubsub.publish('projects/a/topics/def', 'msg', {'a': 1, 'b': 2})
def _pubsub_notify(task_id, topic, auth_token, userdata):
"""Sends PubSub notification about task completion.
Raises pubsub.TransientError on transient errors. Fatal errors are logged, but
not retried.
"""
logging.debug(
'Sending PubSub notify to "%s" (with userdata "%s") about '
'completion of "%s"', topic, userdata, task_id)
msg = {'task_id': task_id}
if userdata:
msg['userdata'] = userdata
try:
pubsub.publish(
topic=topic,
message=utils.encode_to_json(msg),
attributes={'auth_token': auth_token} if auth_token else None)
except pubsub.Error:
logging.exception('Fatal error when sending PubSub notification')
def maybe_notify_backend(message, hostname, policies):
"""Informs the backend of the status of a request if there's a Pub/Sub topic.
Args:
message: The message string to send.
hostname: The hostname of the machine this message concerns.
policies: A dict representation of an rpc_messages.Policies instance.
"""
if policies.get('backend_topic'):
attributes = {
attribute['key']: attribute['value']
for attribute in policies['backend_attributes']
}
attributes['hostname'] = hostname
pubsub.publish(
pubsub.full_topic_name(policies['backend_project'],
policies['backend_topic']),
message,
attributes,
)
def maybe_notify_backend(message, hostname, policies):
"""Informs the backend of the status of a request if there's a Pub/Sub topic.
Args:
message: The message string to send.
hostname: The hostname of the machine this message concerns.
policies: A dict representation of an rpc_messages.Policies instance.
"""
if policies.get('backend_topic'):
attributes = {
attribute['key']: attribute['value']
for attribute in policies['backend_attributes']
}
attributes['hostname'] = hostname
pubsub.publish(
pubsub.full_topic_name(
policies['backend_project'], policies['backend_topic']),
message,
attributes,
)
def maybe_notify_backend(message, hostname, policies):
"""Informs the backend of the status of a request if there's a Pub/Sub topic.
Args:
message: The message string to send.
hostname: The hostname of the machine this message concerns.
policies: A dict representation of an rpc_messages.Policies instance.
"""
if policies.get('backend_topic'):
topic = pubsub.full_topic_name(policies['backend_project'],
policies['backend_topic'])
attributes = {
attribute['key']: attribute['value']
for attribute in policies['backend_attributes']
}
attributes['hostname'] = hostname
pubsub.publish(topic, message, attributes)
# There are relatively few backends, so it's safe to include the
# backend topic/project as the value for the target field.
metrics.pubsub_messages_sent.increment(fields={'target': topic})
def _pubsub_notify(task_id, topic, auth_token, userdata):
"""Sends PubSub notification about task completion.
Raises pubsub.TransientError on transient errors. Fatal errors are logged, but
not retried.
"""
logging.debug(
'Sending PubSub notify to "%s" (with userdata "%s") about ' 'completion of "%s"', topic, userdata, task_id
)
msg = {"task_id": task_id}
if userdata:
msg["userdata"] = userdata
try:
pubsub.publish(
topic=topic,
message=utils.encode_to_json(msg),
attributes={"auth_token": auth_token} if auth_token else None,
)
except pubsub.Error:
logging.exception("Fatal error when sending PubSub notification")
def publish_authdb_change(state):
"""Publishes AuthDB change notification to the topic.
Args:
state: AuthReplicationState with version info.
"""
if utils.is_local_dev_server():
return
msg = replication_pb2.ReplicationPushRequest()
msg.revision.primary_id = app_identity.get_application_id()
msg.revision.auth_db_rev = state.auth_db_rev
msg.revision.modified_ts = utils.datetime_to_timestamp(state.modified_ts)
blob = msg.SerializeToString()
key_name, sig = signature.sign_blob(blob)
pubsub.publish(topic_name(), blob, {
'X-AuthDB-SigKey-v1': key_name,
'X-AuthDB-SigVal-v1': base64.b64encode(sig),
})
def test_publish_ok(self):
self.mock_requests(
[
# First attempt. Encounters 404 due to non-existing topic.
{
"url": "https://pubsub.googleapis.com/v1/projects/a/topics/def:publish",
"method": "POST",
"payload": {"messages": [{"attributes": {"a": 1, "b": 2}, "data": "bXNn"}]},
"response": net.NotFoundError("topic not found", 404, ""),
},
# Creates the topic.
{"url": "https://pubsub.googleapis.com/v1/projects/a/topics/def", "method": "PUT", "payload": None},
# Second attempt, succeeds.
{
"url": "https://pubsub.googleapis.com/v1/projects/a/topics/def:publish",
"method": "POST",
"payload": {"messages": [{"attributes": {"a": 1, "b": 2}, "data": "bXNn"}]},
},
]
)
pubsub.publish("projects/a/topics/def", "msg", {"a": 1, "b": 2})
def publish_authdb_change(state):
"""Publishes AuthDB change notification to the topic.
Args:
state: AuthReplicationState with version info.
"""
if utils.is_local_dev_server():
return
msg = replication_pb2.ReplicationPushRequest()
msg.revision.primary_id = app_identity.get_application_id()
msg.revision.auth_db_rev = state.auth_db_rev
msg.revision.modified_ts = utils.datetime_to_timestamp(state.modified_ts)
blob = msg.SerializeToString()
key_name, sig = signature.sign_blob(blob)
pubsub.publish(topic_name(), blob, {
'X-AuthDB-SigKey-v1': key_name,
'X-AuthDB-SigVal-v1': base64.b64encode(sig),
})
def post(self):
"""Reclaim a machine.
Params:
backend_topic: If specified, topic that the machine reclamation should
be published to for the backend.
backend_project: If specified, project that the machine reclamation
topic is contained in for the backend.
lease_id: ID of the LeaseRequest the machine was leased for.
lessee_topic: If specified, topic that the machine reclamation and lease
expiration should be published to for the lessee.
lessee_project: If specified, project that the machine reclamation and
lease expiration topic is contained in.
machine_id: ID of the CatalogMachineEntry being reclaimed.
"""
backend_project = self.request.get('backend_project')
backend_topic = self.request.get('backend_topic')
lease_id = self.request.get('lease_id')
lessee_project = self.request.get('lessee_project')
lessee_topic = self.request.get('lessee_topic')
machine_id = self.request.get('machine_id')
if lessee_topic:
pubsub.publish(
lessee_topic,
lessee_project,
'RECLAIMED',
machine_id=machine_id,
request_hash=lease_id,
)
if backend_topic:
pubsub.publish(
backend_topic,
backend_project,
'RECLAIMED',
machine_id=machine_id,
)
def test_publish_ok(self):
self.mock_requests([
# First attempt. Encounters 404 due to non-existing topic.
{
'url': 'https://pubsub.googleapis.com/v1/projects/a/topics/def:publish',
'method': 'POST',
'payload': {
'messages': [
{
'attributes': {'a': 1, 'b': 2},
'data': 'bXNn',
},
],
},
'response': net.NotFoundError('topic not found', 404, ''),
},
# Creates the topic.
{
'url': 'https://pubsub.googleapis.com/v1/projects/a/topics/def',
'method': 'PUT',
'payload': None,
},
# Second attempt, succeeds.
{
'url': 'https://pubsub.googleapis.com/v1/projects/a/topics/def:publish',
'method': 'POST',
'payload': {
'messages': [
{
'attributes': {'a': 1, 'b': 2},
'data': 'bXNn',
},
],
},
},
])
pubsub.publish('projects/a/topics/def', 'msg', {'a': 1, 'b': 2})
def post(self, build_id): # pylint: disable=unused-argument
body = json.loads(self.request.body)
pubsub.publish(
body['topic'],
json.dumps(body['message'], sort_keys=True),
body['attrs'])