def get_service_public_certificates(service_url):
"""Returns jsonish object with public certificates of a service.
Service at |service_url| must have 'auth' component enabled (to serve
the certificates).
"""
cache_key = 'pub_certs:%s' % service_url
certs = memcache.get(cache_key)
if not certs:
protocol = 'http://' if utils.is_local_dev_server() else 'https://'
assert service_url.startswith(protocol)
result = urlfetch.fetch(
url='%s/auth/api/v1/server/certificates' % service_url,
method='GET',
headers={'X-URLFetch-Service-Id': utils.get_urlfetch_service_id()},
follow_redirects=False,
deadline=10,
validate_certificate=True)
if result.status_code != 200:
raise CertificateError(
'Failed to grab public certs from %s: HTTP %d' %
(service_url, result.status_code))
certs = json.loads(result.content)
memcache.set(cache_key, certs, time=3600)
return certs
def get_service_public_certificates(service_url):
"""Returns jsonish object with public certificates of a service.
Service at |service_url| must have 'auth' component enabled (to serve
the certificates).
"""
cache_key = 'pub_certs:%s' % service_url
certs = memcache.get(cache_key)
if not certs:
protocol = 'http://' if utils.is_local_dev_server() else 'https://'
assert service_url.startswith(protocol)
result = urlfetch.fetch(
url='%s/auth/api/v1/server/certificates' % service_url,
method='GET',
headers={'X-URLFetch-Service-Id': utils.get_urlfetch_service_id()},
follow_redirects=False,
deadline=10,
validate_certificate=True)
if result.status_code != 200:
raise CertificateError(
'Failed to grab public certs from %s: HTTP %d' %
(service_url, result.status_code))
certs = json.loads(result.content)
memcache.set(cache_key, certs, time=3600)
return certs
def get_service_public_certificates(service_url):
"""Returns jsonish object with public certificates of a service.
Service at |service_url| must have 'auth' component enabled (to serve
the certificates).
"""
cache_key = 'pub_certs:%s' % service_url
certs = memcache.get(cache_key)
if certs:
return certs
protocol = 'http://' if utils.is_local_dev_server() else 'https://'
assert service_url.startswith(protocol)
url = '%s/auth/api/v1/server/certificates' % service_url
# Retry code is adapted from components/net.py. net.py can't be used directly
# since it depends on components.auth (and dependency cycles between
# components are bad).
attempt = 0
result = None
while attempt < 4:
if attempt:
logging.info('Retrying...')
attempt += 1
logging.info('GET %s', url)
try:
result = urlfetch.fetch(url=url,
method='GET',
headers={
'X-URLFetch-Service-Id':
utils.get_urlfetch_service_id()
},
follow_redirects=False,
deadline=2,
validate_certificate=True)
except (apiproxy_errors.DeadlineExceededError, urlfetch.Error) as e:
# Transient network error or URL fetch service RPC deadline.
logging.warning('GET %s failed: %s', url, e)
continue
# It MUST return 200 on success, it can't return 403, 404 or >=500.
if result.status_code != 200:
logging.warning('GET %s failed, HTTP %d: %r', url,
result.status_code, result.content)
continue
# Success.
certs = json.loads(result.content)
memcache.set(cache_key, certs, time=3600)
return certs
# All attempts failed, give up.
msg = 'Failed to grab public certs from %s (HTTP code %s)' % (
service_url, result.status_code if result else '???')
raise CertificateError(msg, transient=True)
def get_service_public_certificates(service_url):
"""Returns jsonish object with public certificates of a service.
Service at |service_url| must have 'auth' component enabled (to serve
the certificates).
"""
cache_key = 'pub_certs:%s' % service_url
certs = memcache.get(cache_key)
if certs:
return certs
protocol = 'http://' if utils.is_local_dev_server() else 'https://'
assert service_url.startswith(protocol)
url = '%s/auth/api/v1/server/certificates' % service_url
# Retry code is adapted from components/net.py. net.py can't be used directly
# since it depends on components.auth (and dependency cycles between
# components are bad).
attempt = 0
result = None
while attempt < 4:
if attempt:
logging.info('Retrying...')
attempt += 1
logging.info('GET %s', url)
try:
result = urlfetch.fetch(
url=url,
method='GET',
headers={'X-URLFetch-Service-Id': utils.get_urlfetch_service_id()},
follow_redirects=False,
deadline=5,
validate_certificate=True)
except (apiproxy_errors.DeadlineExceededError, urlfetch.Error) as e:
# Transient network error or URL fetch service RPC deadline.
logging.warning('GET %s failed: %s', url, e)
continue
# It MUST return 200 on success, it can't return 403, 404 or >=500.
if result.status_code != 200:
logging.warning(
'GET %s failed, HTTP %d: %r', url, result.status_code, result.content)
continue
# Success.
certs = json.loads(result.content)
memcache.set(cache_key, certs, time=3600)
return certs
# All attempts failed, give up.
msg = 'Failed to grab public certs from %s (HTTP code %s)' % (
service_url, result.status_code if result else '???')
raise CertificateError(msg, transient=True)
def _fetch_service_certs(service_url):
protocol = 'https://'
if utils.is_local_dev_server():
protocol = ('http://', 'https://')
assert service_url.startswith(protocol), (service_url, protocol)
url = '%s/auth/api/v1/server/certificates' % service_url
# Retry code is adapted from components/net.py. net.py can't be used directly
# since it depends on components.auth (and dependency cycles between
# components are bad).
attempt = 0
result = None
while attempt < 4:
if attempt:
logging.info('Retrying...')
attempt += 1
logging.info('GET %s', url)
try:
result = urlfetch.fetch(url=url,
method='GET',
headers={
'X-URLFetch-Service-Id':
utils.get_urlfetch_service_id()
},
follow_redirects=False,
deadline=5,
validate_certificate=True)
except (apiproxy_errors.DeadlineExceededError, urlfetch.Error) as e:
# Transient network error or URL fetch service RPC deadline.
logging.warning('GET %s failed: %s', url, e)
continue
# It MUST return 200 on success, it can't return 403, 404 or >=500.
if result.status_code != 200:
logging.warning('GET %s failed, HTTP %d: %r', url,
result.status_code, result.content)
continue
return json.loads(result.content)
# All attempts failed, give up.
msg = 'Failed to grab public certs from %s (HTTP code %s)' % (
service_url, result.status_code if result else '???')
raise CertificateError(msg, transient=True)
def become_replica(ticket, initiated_by):
"""Converts current service to a replica of a primary specified in a ticket.
Args:
ticket: replication_pb2.ServiceLinkTicket passed from a primary.
initiated_by: Identity of a user that accepted linking request, for logging.
Raises:
ProtocolError in case the request to primary fails.
"""
assert model.is_standalone()
# On dev appserver emulate X-Appengine-Inbound-Appid header.
headers = {'Content-Type': 'application/octet-stream'}
protocol = 'https'
if utils.is_local_dev_server():
headers['X-Appengine-Inbound-Appid'] = app_identity.get_application_id()
protocol = 'http'
headers['X-URLFetch-Service-Id'] = utils.get_urlfetch_service_id()
# Pass back the ticket for primary to verify it, tell the primary to use
# default version hostname to talk to us.
link_request = replication_pb2.ServiceLinkRequest()
link_request.ticket = ticket.ticket
link_request.replica_url = (
'%s://%s' % (protocol, app_identity.get_default_version_hostname()))
link_request.initiated_by = initiated_by.to_bytes()
# Primary will look at X-Appengine-Inbound-Appid and compare it to what's in
# the ticket.
try:
result = urlfetch.fetch(
url='%s/auth_service/api/v1/internal/link_replica' % ticket.primary_url,
payload=link_request.SerializeToString(),
method='POST',
headers=headers,
follow_redirects=False,
deadline=30,
validate_certificate=True)
except urlfetch.Error as exc:
raise ProtocolError(
replication_pb2.ServiceLinkResponse.TRANSPORT_ERROR,
'URLFetch error (%s): %s' % (exc.__class__.__name__, exc))
# Protobuf based protocol is not using HTTP codes (handler always replies with
# HTTP 200, providing error details if needed in protobuf serialized body).
# So any other status code here means there was a transport level error.
if result.status_code != 200:
raise ProtocolError(
replication_pb2.ServiceLinkResponse.TRANSPORT_ERROR,
'Request to the primary failed with HTTP %d.' % result.status_code)
link_response = replication_pb2.ServiceLinkResponse.FromString(result.content)
if link_response.status != replication_pb2.ServiceLinkResponse.SUCCESS:
message = LINKING_ERRORS.get(
link_response.status,
'Request to the primary failed with status %d.' % link_response.status)
raise ProtocolError(link_response.status, message)
# Become replica. Auth DB will be overwritten on a first push from Primary.
state = model.AuthReplicationState(
key=model.replication_state_key(),
primary_id=ticket.primary_id,
primary_url=ticket.primary_url)
state.put()
def push_to_replica(replica_url, auth_db_blob, key_name, sig):
"""Pushes |auth_db_blob| to a replica via URLFetch POST.
Args:
replica_url: root URL of a replica (i.e. https://<host>).
auth_db_blob: binary blob with serialized Auth DB.
key_name: name of a RSA key used to generate a signature.
sig: base64 encoded signature of |auth_db_blob|.
Returns:
Tuple:
AuthDB revision reporter by a replica (as replication_pb2.AuthDBRevision).
Auth component version used by replica (see components.auth.version).
Raises:
FatalReplicaUpdateError if replica rejected the push.
TransientReplicaUpdateError if push should be retried.
"""
replica_url = replica_url.rstrip('/')
logging.info('Updating replica %s', replica_url)
protocol = 'http://' if utils.is_local_dev_server() else 'https://'
assert replica_url.startswith(protocol)
# Pass signature via the headers.
headers = {
'Content-Type': 'application/octet-stream',
'X-URLFetch-Service-Id': utils.get_urlfetch_service_id(),
'X-AuthDB-SigKey-v1': key_name,
'X-AuthDB-SigVal-v1': sig,
}
# On dev appserver emulate X-Appengine-Inbound-Appid header.
if utils.is_local_dev_server():
headers['X-Appengine-Inbound-Appid'] = app_identity.get_application_id()
# 'follow_redirects' set to False is required for 'X-Appengine-Inbound-Appid'
# to work. 70 sec deadline correspond to 60 sec GAE foreground requests
# deadline plus 10 seconds to account for URL fetch own lags.
ctx = ndb.get_context()
result = yield ctx.urlfetch(
url=replica_url + '/auth/api/v1/internal/replication',
payload=auth_db_blob,
method='POST',
headers=headers,
follow_redirects=False,
deadline=70,
validate_certificate=True)
# Any transport level error is transient.
if result.status_code != 200:
raise TransientReplicaUpdateError(
'Push request failed with HTTP code %d' % result.status_code)
# Deserialize the response.
cls = replication_pb2.ReplicationPushResponse
response = cls.FromString(result.content)
if not response.HasField('status'):
raise FatalReplicaUpdateError('Incomplete response, status is missing')
# Convert errors to exceptions.
if response.status == cls.TRANSIENT_ERROR:
raise TransientReplicaUpdateError(
'Transient error (error code %d).' % response.error_code)
if response.status == cls.FATAL_ERROR:
raise FatalReplicaUpdateError(
'Fatal error (error code %d).' % response.error_code)
if response.status not in (cls.APPLIED, cls.SKIPPED):
raise FatalReplicaUpdateError(
'Unexpected response status: %d' % response.status)
# Replica applied the update, current_revision should be set.
if not response.HasField('current_revision'):
raise FatalReplicaUpdateError(
'Incomplete response, current_revision is missing')
# Extract auth component version used by replica if proto is recent enough.
auth_code_version = None
if response.HasField('auth_code_version'):
auth_code_version = response.auth_code_version
raise ndb.Return((response.current_revision, auth_code_version))
def push_to_replica(replica_url, auth_db_blob, key_name, sig):
"""Pushes |auth_db_blob| to a replica via URLFetch POST.
Args:
replica_url: root URL of a replica (i.e. https://<host>).
auth_db_blob: binary blob with serialized Auth DB.
key_name: name of a RSA key used to generate a signature.
sig: base64 encoded signature of |auth_db_blob|.
Returns:
Tuple:
AuthDB revision reporter by a replica (as replication_pb2.AuthDBRevision).
Auth component version used by replica (see components.auth.version).
Raises:
FatalReplicaUpdateError if replica rejected the push.
TransientReplicaUpdateError if push should be retried.
"""
replica_url = replica_url.rstrip('/')
logging.info('Updating replica %s', replica_url)
protocol = 'http://' if utils.is_local_dev_server() else 'https://'
assert replica_url.startswith(protocol)
# Pass signature via the headers.
headers = {
'Content-Type': 'application/octet-stream',
'X-URLFetch-Service-Id': utils.get_urlfetch_service_id(),
'X-AuthDB-SigKey-v1': key_name,
'X-AuthDB-SigVal-v1': sig,
}
# On dev appserver emulate X-Appengine-Inbound-Appid header.
if utils.is_local_dev_server():
headers['X-Appengine-Inbound-Appid'] = app_identity.get_application_id(
)
# 'follow_redirects' set to False is required for 'X-Appengine-Inbound-Appid'
# to work. 70 sec deadline correspond to 60 sec GAE foreground requests
# deadline plus 10 seconds to account for URL fetch own lags.
ctx = ndb.get_context()
result = yield ctx.urlfetch(url=replica_url +
'/auth/api/v1/internal/replication',
payload=auth_db_blob,
method='POST',
headers=headers,
follow_redirects=False,
deadline=70,
validate_certificate=True)
# Any transport level error is transient.
if result.status_code != 200:
raise TransientReplicaUpdateError(
'Push request failed with HTTP code %d' % result.status_code)
# Deserialize the response.
cls = replication_pb2.ReplicationPushResponse
response = cls.FromString(result.content)
if not response.HasField('status'):
raise FatalReplicaUpdateError('Incomplete response, status is missing')
# Convert errors to exceptions.
if response.status == cls.TRANSIENT_ERROR:
raise TransientReplicaUpdateError('Transient error (error code %d).' %
response.error_code)
if response.status == cls.FATAL_ERROR:
raise FatalReplicaUpdateError('Fatal error (error code %d).' %
response.error_code)
if response.status not in (cls.APPLIED, cls.SKIPPED):
raise FatalReplicaUpdateError('Unexpected response status: %d' %
response.status)
# Replica applied the update, current_revision should be set.
if not response.HasField('current_revision'):
raise FatalReplicaUpdateError(
'Incomplete response, current_revision is missing')
# Extract auth component version used by replica if proto is recent enough.
auth_code_version = None
if response.HasField('auth_code_version'):
auth_code_version = response.auth_code_version
raise ndb.Return((response.current_revision, auth_code_version))
def become_replica(ticket, initiated_by):
"""Converts current service to a replica of a primary specified in a ticket.
Args:
ticket: replication_pb2.ServiceLinkTicket passed from a primary.
initiated_by: Identity of a user that accepted linking request, for logging.
Raises:
ProtocolError in case the request to primary fails.
"""
assert model.is_standalone()
# On dev appserver emulate X-Appengine-Inbound-Appid header.
headers = {'Content-Type': 'application/octet-stream'}
protocol = 'https'
if utils.is_local_dev_server():
headers['X-Appengine-Inbound-Appid'] = app_identity.get_application_id()
protocol = 'http'
headers['X-URLFetch-Service-Id'] = utils.get_urlfetch_service_id()
# Pass back the ticket for primary to verify it, tell the primary to use
# default version hostname to talk to us.
link_request = replication_pb2.ServiceLinkRequest()
link_request.ticket = ticket.ticket
link_request.replica_url = (
'%s://%s' % (protocol, app_identity.get_default_version_hostname()))
link_request.initiated_by = initiated_by.to_bytes()
# Primary will look at X-Appengine-Inbound-Appid and compare it to what's in
# the ticket.
try:
result = urlfetch.fetch(
url='%s/auth_service/api/v1/internal/link_replica' % ticket.primary_url,
payload=link_request.SerializeToString(),
method='POST',
headers=headers,
follow_redirects=False,
deadline=30,
validate_certificate=True)
except urlfetch.Error as exc:
raise ProtocolError(
replication_pb2.ServiceLinkResponse.TRANSPORT_ERROR,
'URLFetch error (%s): %s' % (exc.__class__.__name__, exc))
# Protobuf based protocol is not using HTTP codes (handler always replies with
# HTTP 200, providing error details if needed in protobuf serialized body).
# So any other status code here means there was a transport level error.
if result.status_code != 200:
raise ProtocolError(
replication_pb2.ServiceLinkResponse.TRANSPORT_ERROR,
'Request to the primary failed with HTTP %d.' % result.status_code)
link_response = replication_pb2.ServiceLinkResponse.FromString(result.content)
if link_response.status != replication_pb2.ServiceLinkResponse.SUCCESS:
message = LINKING_ERRORS.get(
link_response.status,
'Request to the primary failed with status %d.' % link_response.status)
raise ProtocolError(link_response.status, message)
# Become replica. Auth DB will be overwritten on a first push from Primary.
state = model.AuthReplicationState(
key=model.replication_state_key(),
primary_id=ticket.primary_id,
primary_url=ticket.primary_url)
state.put()
