def _validate_by_service_async(service, config_set, path, content, ctx):
"""Validates a config with an external service."""
try:
metadata = yield services.get_metadata_async(service.id)
except services.DynamicMetadataError as ex:
logging.error("Could not load dynamic metadata for %s: %s", service.id, ex)
return
assert metadata and metadata.validation
url = metadata.validation.url
if not url:
return
match = False
for p in metadata.validation.patterns:
# TODO(nodir): optimize if necessary.
if validation.compile_pattern(p.config_set)(config_set) and validation.compile_pattern(p.path)(path):
match = True
break
if not match:
return
res = None
def report_error(text):
text = ("Error during external validation: %s\n" "url: %s\n" "config_set: %s\n" "path: %s\n" "response: %r") % (
text,
url,
config_set,
path,
res,
)
logging.error(text)
ctx.critical(text)
try:
req = {"config_set": config_set, "path": path, "content": base64.b64encode(content)}
res = yield net.json_request_async(url, method="POST", payload=req, scopes=net.EMAIL_SCOPE)
except net.Error as ex:
report_error("Net error: %s" % ex)
return
try:
for msg in res.get("messages", []):
if not isinstance(msg, dict):
report_error("invalid response: message is not a dict: %r" % msg)
continue
severity = msg.get("severity") or "INFO"
if severity not in service_config_pb2.ValidationResponseMessage.Severity.keys():
report_error("invalid response: unexpected message severity: %s" % severity)
continue
# It is safe because we've validated |severity|.
func = getattr(ctx, severity.lower())
func(msg.get("text") or "")
except Exception as ex:
report_error(ex)
def test_get_metadata_async(self):
self.mock(storage, 'get_self_config_async', mock.Mock())
storage.get_self_config_async.return_value = future(
service_config_pb2.ServicesCfg(services=[
service_config_pb2.Service(
id='foo', metadata_url='https://foo.com/metadata')
]))
self.mock(net, 'json_request_async', mock.Mock())
net.json_request_async.return_value = future({
'version': '1.0',
'validation': {
'url':
'https://a.com/validate',
'patterns': [
{
'config_set': 'projects/foo',
'path': 'bar.cfg'
},
{
'config_set': 'regex:services/.+',
'path': 'regex:.+'
},
]
}
})
metadata = services.get_metadata_async('foo').get_result()
self.assertEqual(
metadata,
service_config_pb2.ServiceDynamicMetadata(
validation=service_config_pb2.Validator(
url='https://a.com/validate',
patterns=[
service_config_pb2.ConfigPattern(
config_set='projects/foo', path='bar.cfg'),
service_config_pb2.ConfigPattern(
config_set='regex:services/.+', path='regex:.+'),
])))
net.json_request_async.assert_called_once_with(
'https://foo.com/metadata', scopes=net.EMAIL_SCOPE)
storage.get_self_config_async.assert_called_once_with(
common.SERVICES_REGISTRY_FILENAME, service_config_pb2.ServicesCfg)
def test_get_metadata_async(self):
self.mock(storage, 'get_self_config_async', mock.Mock())
storage.get_self_config_async.return_value = future(
service_config_pb2.ServicesCfg(
services=[
service_config_pb2.Service(
id='foo', metadata_url='https://foo.com/metadata')
]
))
self.mock(net, 'json_request_async', mock.Mock())
net.json_request_async.return_value = future({
'version': '1.0',
'validation': {
'url': 'https://a.com/validate',
'patterns': [
{'config_set': 'projects/foo', 'path': 'bar.cfg'},
{'config_set': 'regex:services/.+', 'path': 'regex:.+'},
]
}
})
metadata = services.get_metadata_async('foo').get_result()
self.assertEqual(
metadata,
service_config_pb2.ServiceDynamicMetadata(
validation=service_config_pb2.Validator(
url='https://a.com/validate',
patterns=[
service_config_pb2.ConfigPattern(
config_set='projects/foo', path='bar.cfg'),
service_config_pb2.ConfigPattern(
config_set='regex:services/.+', path='regex:.+'),
]
)
)
)
net.json_request_async.assert_called_once_with(
'https://foo.com/metadata', scopes=net.EMAIL_SCOPE)
storage.get_self_config_async.assert_called_once_with(
common.SERVICES_REGISTRY_FILENAME, service_config_pb2.ServicesCfg)
def test_update_service_metadata_async_different(self):
self.mock_metadata_entity()
self.mock(net, 'json_request_async', mock.Mock())
dct = {
'version': '1.0',
'validation': {
'url': 'https://a.com/different_validate',
'patterns': [
{'config_set': 'projects/bar', 'path': 'foo.cfg'},
{'config_set': 'regex:services/.+', 'path': 'regex:.+'},
]
}
}
net.json_request_async.return_value = future(dct)
self.mock(logging, 'info', mock.Mock())
services._update_service_metadata_async(self.service_proto()).get_result()
self.assertTrue(logging.info.called)
md = services.get_metadata_async('deadbeef').get_result()
self.assertEqual(md.validation.url, 'https://a.com/different_validate')
def test_get_metadata_async_no_metadata(self):
metadata = services.get_metadata_async('metadataless').get_result()
self.assertIsNotNone(metadata)
self.assertFalse(metadata.validation.patterns)
def test_get_metadata_async_not_found(self):
with self.assertRaises(services.ServiceNotFoundError):
services.get_metadata_async('non-existent').get_result()
def test_get_metadata_async_no_metadata(self):
metadata = services.get_metadata_async('metadataless').get_result()
self.assertIsNotNone(metadata)
self.assertFalse(metadata.validation.patterns)
def test_get_metadata_async_not_found(self):
with self.assertRaises(services.ServiceNotFoundError):
services.get_metadata_async('non-existent').get_result()
def test_get_metadata_async_no_metadata(self):
storage.ServiceDynamicMetadata(id='metadataless').put()
metadata = services.get_metadata_async('metadataless').get_result()
self.assertIsNotNone(metadata)
self.assertFalse(metadata.validation.patterns)
def _validate_by_service_async(service, config_set, path, content, ctx):
"""Validates a config with an external service."""
try:
metadata = yield services.get_metadata_async(service.id)
except services.DynamicMetadataError as ex:
logging.error('Could not load dynamic metadata for %s: %s', service.id,
ex)
return
assert metadata and metadata.validation
url = metadata.validation.url
if not url:
return
match = False
for p in metadata.validation.patterns:
# TODO(nodir): optimize if necessary.
if (validation.compile_pattern(p.config_set)(config_set)
and validation.compile_pattern(p.path)(path)):
match = True
break
if not match:
return
res = None
def report_error(text):
text = ('Error during external validation: %s\n'
'url: %s\n'
'config_set: %s\n'
'path: %s\n'
'response: %r') % (text, url, config_set, path, res)
logging.error(text)
ctx.critical(text)
try:
req = {
'config_set': config_set,
'path': path,
'content': base64.b64encode(content),
}
res = yield net.json_request_async(url,
method='POST',
payload=req,
scopes=net.EMAIL_SCOPE)
except net.Error as ex:
report_error('Net error: %s' % ex)
return
try:
for msg in res.get('messages', []):
if not isinstance(msg, dict):
report_error('invalid response: message is not a dict: %r' %
msg)
continue
severity = msg.get('severity') or 'INFO'
if (severity not in service_config_pb2.ValidationResponseMessage.
Severity.keys()):
report_error(
'invalid response: unexpected message severity: %s' %
severity)
continue
# It is safe because we've validated |severity|.
func = getattr(ctx, severity.lower())
func(msg.get('text') or '')
except Exception as ex:
report_error(ex)
def _validate_by_service_async(service, config_set, path, content, ctx):
"""Validates a config with an external service.
Validation results will be stored in the validation context.
Args:
service (service_config_pb2.Service): service to be validated against.
config_set (str): config set being validated.
path (str): path of the config file being validated.
content (str): byte-form of the content of the file being validated.
ctx (validation.Context): context in which validation messages
will be stored.
"""
try:
metadata = yield services.get_metadata_async(service.id)
except services.DynamicMetadataError as ex:
logging.error('Could not load dynamic metadata for %s: %s', service.id, ex)
return
assert metadata and metadata.validation
url = metadata.validation.url
if not url:
return
match = False
for p in metadata.validation.patterns:
# TODO(nodir): optimize if necessary.
if (validation.compile_pattern(p.config_set)(config_set) and
validation.compile_pattern(p.path)(path)):
match = True
break
if not match:
return
res = None
def report_error(text):
text = (
'Error during external validation: %s\n'
'url: %s\n'
'config_set: %s\n'
'path: %s\n'
'response: %r') % (text, url, config_set, path, res)
logging.error(text)
ctx.critical('%s', text)
try:
req = {
'config_set': config_set,
'path': path,
'content': base64.b64encode(content),
}
res = yield net.json_request_async(
url, method='POST', payload=req, scopes=net.EMAIL_SCOPE)
except net.Error as ex:
report_error('Net error: %s' % ex)
return
try:
for msg in res.get('messages', []):
if not isinstance(msg, dict):
report_error('invalid response: message is not a dict: %r' % msg)
continue
severity = msg.get('severity') or 'INFO'
# validation library for Go services sends severity as an integer
# corresponding to Python's logging severity level.
if severity in (logging.DEBUG, logging.INFO, logging.WARNING,
logging.ERROR, logging.CRITICAL):
severity = logging.getLevelName(severity)
if (severity not in
service_config_pb2.ValidationResponseMessage.Severity.keys()):
report_error(
'invalid response: unexpected message severity: %r' % severity)
continue
# It is safe because we've validated |severity|.
func = getattr(ctx, severity.lower())
func('%s', msg.get('text') or '')
except Exception as ex:
report_error(ex)
def _validate_by_service_async(service, config_set, path, content, ctx):
"""Validates a config with an external service."""
try:
metadata = yield services.get_metadata_async(service.id)
except services.DynamicMetadataError as ex:
logging.error('Could not load dynamic metadata for %s: %s', service.id, ex)
return
assert metadata and metadata.validation
url = metadata.validation.url
if not url:
return
match = False
for p in metadata.validation.patterns:
# TODO(nodir): optimize if necessary.
if (validation.compile_pattern(p.config_set)(config_set) and
validation.compile_pattern(p.path)(path)):
match = True
break
if not match:
return
res = None
def report_error(text):
text = (
'Error during external validation: %s\n'
'url: %s\n'
'config_set: %s\n'
'path: %s\n'
'response: %r') % (text, url, config_set, path, res)
logging.error(text)
ctx.critical(text)
try:
req = {
'config_set': config_set,
'path': path,
'content': base64.b64encode(content),
}
res = yield net.json_request_async(
url, method='POST', payload=req, scopes=net.EMAIL_SCOPE)
except net.Error as ex:
report_error('Net error: %s' % ex)
return
try:
for msg in res.get('messages', []):
if not isinstance(msg, dict):
report_error('invalid response: message is not a dict: %r' % msg)
continue
severity = msg.get('severity') or 'INFO'
if (severity not in
service_config_pb2.ValidationResponseMessage.Severity.keys()):
report_error(
'invalid response: unexpected message severity: %s' % severity)
continue
# It is safe because we've validated |severity|.
func = getattr(ctx, severity.lower())
func(msg.get('text') or '')
except Exception as ex:
report_error(ex)
