def test_put_infra_config_ok(
zk, client, infra_info, test_application_name, test_application_token,
test_redis, last_audit_log, mocker,
scope_type, scope_name,
exception_on_recording_downstream):
major_application_name = test_application_name
logger = mocker.patch(
'huskar_api.api.middlewares.logger.logger', autospec=True)
exit_scope_name = scope_name.replace('altb1', 'alta1')
infra_info.set_by_name('r100010', scope_type, exit_scope_name, {
'url': 'redis://localhost:6543'})
infra_info.save()
url = 'sam+redis://%s/overall.alta' % test_redis
r = client.put(
'/api/infra-config/%s/redis/r100010' % test_application_name,
query_string={'scope_type': scope_type, 'scope_name': scope_name},
data=json.dumps({'url': url}),
headers={'Authorization': test_application_token,
'Content-Type': 'application/json'})
assert_response_ok(r)
assert r.json['data'] == {'infra_config': [
{'scope_type': scope_type, 'scope_name': exit_scope_name,
'value': {'url': 'redis://localhost:6543'}},
{'scope_type': scope_type, 'scope_name': scope_name,
'value': {'url': url}},
]}
request_args = json.loads(logger.mock_calls[0][1][-6])
assert 'url' not in request_args
InfraDownstream.flush_cache_by_application(test_redis)
ds = InfraDownstream.get_multi_by_application(test_redis)
if exception_on_recording_downstream:
assert len(ds) == 0
else:
assert len(ds) == 1
assert ds[0].user_application_name == major_application_name
assert ds[0].user_infra_type == 'redis'
assert ds[0].user_infra_name == 'r100010'
audit_log = last_audit_log()
assert audit_log and audit_log.action_name == 'UPDATE_INFRA_CONFIG'
assert audit_log.action_json['application_name'] == major_application_name
assert audit_log.action_json['infra_type'] == 'redis'
assert audit_log.action_json['infra_name'] == 'r100010'
assert audit_log.action_json['scope_type'] == scope_type
assert audit_log.action_json['scope_name'] == scope_name
assert audit_log.action_json['data']['new'] == {'url': url}
assert audit_log.action_json['data']['old'] is None
url = 'sam+redis://%s/overall.alta1' % test_redis
r = client.put(
'/api/infra-config/%s/redis/r100010' % test_application_name,
query_string={'scope_type': scope_type, 'scope_name': scope_name},
data=json.dumps({'url': url}),
headers={'Authorization': test_application_token,
'Content-Type': 'application/json'})
assert_response_ok(r)
def test_bind(db):
InfraDownstream.bind('base.foo', 'redis', 'mycache', 'idcs', 'alta1',
'url', 'redis.foo')
InfraDownstream.bind('base.bar', 'redis', 'mycache', 'idcs', 'alta1',
'url', 'redis.foo')
InfraDownstream.bind('base.bar', 'redis', 'mycache', 'idcs', 'altb1',
'url', 'redis.foo')
InfraDownstream.bind('base.bar', 'redis', u'缓存', 'idcs', 'alta1', 'url',
'redis.foo')
result = list_infra_downstream_by_application_name(db, 'redis.foo')
assert len(result) == 4
assert result[0].user_application_name == 'base.foo'
assert result[0].user_scope_name == 'alta1'
assert result[1].user_application_name == 'base.bar'
assert result[1].user_scope_name == 'alta1'
assert result[2].user_application_name == 'base.bar'
assert result[2].user_scope_name == 'altb1'
assert result[3].user_application_name == 'base.bar'
assert result[3].user_scope_name == 'alta1'
assert result[3].user_infra_name == u'缓存'
result = list_infra_downstream_by_application_name(db, 'redis.bar')
assert len(result) == 0
InfraDownstream.bind('base.bar', 'redis', 'mycache', 'idcs', 'alta1',
'url', 'redis.bar')
result = list_infra_downstream_by_application_name(db, 'redis.foo')
assert len(result) == 3
assert result[0].user_application_name == 'base.foo'
assert result[0].user_scope_name == 'alta1'
assert result[1].user_application_name == 'base.bar'
assert result[1].user_scope_name == 'altb1'
result = list_infra_downstream_by_application_name(db, 'redis.bar')
assert len(result) == 1
assert result[0].user_application_name == 'base.bar'
assert result[0].user_scope_name == 'alta1'
def test_get_multi_by_application(db):
InfraDownstream.bindmany() \
.bind('base.foo', 'redis', 'mycache', 'idcs', 'alta1', 'url',
'redis.foo') \
.bind('base.bar', 'redis', 'mycache', 'idcs', 'alta1', 'url',
'redis.foo') \
.bind('base.bar', 'redis', 'frog+1s', 'idcs', 'alta1', 'url',
'redis.bar') \
.bind('base.bar', 'redis', 'frog+1s', 'idcs', 'altb1', 'url',
'redis.bar') \
.commit()
r = InfraDownstream.get_multi_by_application('redis.foo')
assert len(r) == 2
assert r[0].user_application_name == 'base.foo'
assert r[0].user_infra_type == 'redis'
assert r[0].user_infra_name == 'mycache'
assert r[0].user_scope_name == 'alta1'
assert r[1].user_application_name == 'base.bar'
assert r[1].user_infra_type == 'redis'
assert r[1].user_infra_name == 'mycache'
assert r[1].user_scope_name == 'alta1'
r = InfraDownstream.get_multi_by_application('redis.bar')
assert len(r) == 2
assert r[0].user_application_name == 'base.bar'
assert r[0].user_infra_type == 'redis'
assert r[0].user_infra_name == 'frog+1s'
assert r[0].user_scope_name == 'alta1'
assert r[1].user_application_name == 'base.bar'
assert r[1].user_infra_type == 'redis'
assert r[1].user_infra_name == 'frog+1s'
assert r[1].user_scope_name == 'altb1'
def post(self, infra_application_name):
"""Invalidates cache and shows downstream information of infra.
The response is the same to the ``GET`` method.
:param infra_application_name: The application name of infrastructure.
:<header Authorization: Huskar Token (See :ref:`token`)
:status 200: The result is in the response.
"""
InfraDownstream.flush_cache_by_application(infra_application_name)
return self.get(infra_application_name)
def _update(self, application_name, request, infra_type, infra_name):
check_application_auth(application_name, Authority.WRITE)
check_infra_type(infra_type)
scope_type = request.args['scope_type']
scope_name = request.args['scope_name']
check_scope(scope_type, scope_name)
value = request.get_json()
if not value or not isinstance(value, dict):
abort(400, 'Unacceptable content type or content body')
infra_info = InfraInfo(huskar_client.client, application_name,
infra_type)
infra_info.load()
old_value = infra_info.get_by_name(infra_name, scope_type, scope_name)
yield application_name, infra_info, scope_type, scope_name, value
infra_urls = infra_info.extract_urls(value)
infra_application_names = extract_application_names(infra_urls)
new_value = infra_info.get_by_name(infra_name, scope_type, scope_name)
with audit_log(audit_log.types.UPDATE_INFRA_CONFIG,
application_name=application_name,
infra_type=infra_type,
infra_name=infra_name,
scope_type=scope_type,
scope_name=scope_name,
old_value=old_value,
new_value=new_value):
for infra_application_name in infra_application_names:
check_application(infra_application_name)
infra_info.save()
with suppress_exceptions('infra config updating'):
infra_urls = infra_info.extract_urls(value, as_dict=True)
infra_applications = extract_application_names(infra_urls)
for field_name, infra_application_name in \
infra_applications.iteritems():
InfraDownstream.bind(application_name, infra_type,
infra_name, scope_type, scope_name,
field_name, infra_application_name)
def delete(self, application_name, infra_type, infra_name):
"""Deletes the infrastructure configuration in specified application.
:param application_name: The application which uses infrastructure.
:param infra_type: ``database``, ``redis`` or ``amqp``.
:param infra_name: The unique code-reference name of infrastructure.
:query scope_type: ``idcs`` or ``clusters``.
:query scope_name: The ezone id or clsuter name.
:<header Authorization: Huskar Token (See :ref:`token`)
:status 200: The same as ``GET``.
"""
check_application_auth(application_name, Authority.WRITE)
check_infra_type(infra_type)
scope_type = request.args['scope_type']
scope_name = request.args['scope_name']
check_scope(scope_type, scope_name)
infra_info = InfraInfo(huskar_client.client, application_name,
infra_type)
infra_info.load()
old_value = infra_info.get_by_name(infra_name, scope_type, scope_name)
infra_info.delete_by_name(infra_name, scope_type, scope_name)
with audit_log(audit_log.types.DELETE_INFRA_CONFIG,
application_name=application_name,
infra_type=infra_type,
infra_name=infra_name,
scope_type=scope_type,
scope_name=scope_name,
old_value=old_value,
new_value=None):
infra_info.save()
with suppress_exceptions('infra config deletion'):
old_urls = infra_info.extract_urls(old_value or {}, as_dict=True)
for field_name in frozenset(extract_application_names(old_urls)):
InfraDownstream.unbind(application_name, infra_type,
infra_name, scope_type, scope_name,
field_name)
infra_config = infra_info.list_by_infra_name(infra_name)
return api_response({'infra_config': dump_infra_config(infra_config)})
def test_unbind_stale(db):
m1 = InfraDownstream.bindmany() \
.bind('base.foo', 'redis', 'mycache', 'idcs', 'alta1', 'url',
'redis.foo') \
.bind('base.bar', 'redis', 'mycache', 'idcs', 'alta1', 'url',
'redis.foo') \
.commit()
result = list_infra_downstream_by_application_name(db, 'redis.foo')
assert len(result) == 2
m2 = InfraDownstream.bindmany() \
.bind('base.foo', 'redis', 'mycache', 'idcs', 'alta1', 'url',
'redis.foo') \
.unbind_stale() \
.commit()
result = list_infra_downstream_by_application_name(db, 'redis.foo')
assert len(result) == 1
assert result[0].user_application_name == 'base.foo'
assert result[0].version == m2._timestamp
assert result[0].version > m1._timestamp
def collect_infra_config():
logger.info('Looking up application list')
application_list = Application.get_all()
builder = InfraDownstream.bindmany()
client = huskar_client.client
for application in application_list:
application_name = application.application_name
logger.info('Collecting %s', application_name)
iterator = _collect_infra_upstream(client, application_name)
iterator = _bind_infra_upstream(builder, iterator, application_name)
for upstream in iterator:
logger.info('Recorded %r', upstream)
InfraDownstream.flush_cache_by_application(
upstream.infra_application_name)
builder.commit()
logger.info('Committed %s', application_name)
logger.info('Deleting stale records')
builder.unbind_stale().commit()
logger.info('Done')
def test_bind_various_scopes_and_fields(db):
InfraDownstream.bindmany() \
.bind('base.foo', 'database', 'mydb', 'idcs', 'alta1', 'master',
'dal.test.master') \
.bind('base.foo', 'database', 'mydb', 'idcs', 'alta1', 'slave',
'dal.test.auto') \
.bind('base.foo', 'database', 'mydb', 'clusters', 'sandbox', 'master',
'dal.sandbox.master') \
.bind('base.foo', 'database', 'mydb', 'clusters', 'sandbox', 'slave',
'dal.sandbox.auto') \
.bind('base.foo', 'database', 'mydb', 'idcs', 'altb1', 'master',
'dal.test.master') \
.bind('base.foo', 'database', 'mydb', 'idcs', 'altb1', 'slave',
'dal.test.auto') \
.commit()
r = InfraDownstream.get_multi_by_application('dal.test.master')
assert len(r) == 2
assert r[0].user_application_name == 'base.foo'
assert r[0].user_infra_type == 'database'
assert r[0].user_infra_name == 'mydb'
assert r[0].user_scope_pair == ('idcs', 'alta1')
assert r[0].user_field_name == 'master'
assert r[1].user_application_name == 'base.foo'
assert r[1].user_infra_type == 'database'
assert r[1].user_infra_name == 'mydb'
assert r[1].user_scope_pair == ('idcs', 'altb1')
assert r[1].user_field_name == 'master'
r = InfraDownstream.get_multi_by_application('dal.test.auto')
assert len(r) == 2
assert r[0].user_application_name == 'base.foo'
assert r[0].user_infra_type == 'database'
assert r[0].user_infra_name == 'mydb'
assert r[0].user_scope_pair == ('idcs', 'alta1')
assert r[0].user_field_name == 'slave'
assert r[1].user_application_name == 'base.foo'
assert r[1].user_infra_type == 'database'
assert r[1].user_infra_name == 'mydb'
assert r[1].user_scope_pair == ('idcs', 'altb1')
assert r[1].user_field_name == 'slave'
r = InfraDownstream.get_multi_by_application('dal.sandbox.master')
assert len(r) == 1
assert r[0].user_application_name == 'base.foo'
assert r[0].user_infra_type == 'database'
assert r[0].user_infra_name == 'mydb'
assert r[0].user_scope_pair == ('clusters', 'sandbox')
assert r[0].user_field_name == 'master'
r = InfraDownstream.get_multi_by_application('dal.sandbox.auto')
assert len(r) == 1
assert r[0].user_application_name == 'base.foo'
assert r[0].user_infra_type == 'database'
assert r[0].user_infra_name == 'mydb'
assert r[0].user_scope_pair == ('clusters', 'sandbox')
assert r[0].user_field_name == 'slave'
def test_unbind(db):
InfraDownstream.bind('base.foo', 'redis', 'mycache', 'idcs', 'alta1',
'url', 'redis.foo')
InfraDownstream.bind('base.foo', 'redis', 'mycache', 'idcs', 'altb1',
'url', 'redis.foo')
InfraDownstream.bind('base.bar', 'redis', 'mycache', 'idcs', 'alta1',
'url', 'redis.foo')
result = list_infra_downstream_by_application_name(db, 'redis.foo')
assert len(result) == 3
InfraDownstream.unbind('base.bar', 'redis', 'mycache', 'idcs', 'alta1',
'url')
result = list_infra_downstream_by_application_name(db, 'redis.foo')
assert len(result) == 2
assert result[0].user_application_name == 'base.foo'
assert result[0].user_scope_name == 'alta1'
assert result[1].user_application_name == 'base.foo'
assert result[1].user_scope_name == 'altb1'
def get(self, infra_application_name):
"""Shows downstream information of infrastructure.
The response looks like::
[{
"user_application_name": "base.foo",
"user_infra_type": "database",
"user_infra_name": "db-1",
"version": 1000001,
"updated_at": "... (ISO formatted datetime)",
"created_at": "... (ISO formatted datetime)"
}]
:param infra_application_name: The application name of infrastructure.
:<header Authorization: Huskar Token (See :ref:`token`)
:status 200: The result is in the response.
"""
r = InfraDownstream.get_multi_by_application(infra_application_name)
data = infra_downstream_schema.dump(r, many=True).data
return api_response({'downstream': data})
def test_bindmany(db):
m = InfraDownstream.bindmany() \
.bind('base.foo', 'redis', 'mycache', 'idcs', 'alta1', 'url',
'redis.foo') \
.bind('base.bar', 'redis', 'mycache', 'idcs', 'alta1', 'url',
'redis.foo') \
.commit()
result = list_infra_downstream_by_application_name(db, 'redis.foo')
assert len(result) == 2
assert result[0].user_application_name == 'base.foo'
assert result[1].user_application_name == 'base.bar'
result = list_infra_downstream_by_application_name(db, 'redis.bar')
assert len(result) == 0
m.bindmany() \
.bind('base.bar', 'redis', 'mycache', 'idcs', 'alta1', 'url',
'redis.bar') \
.commit()
result = list_infra_downstream_by_application_name(db, 'redis.foo')
assert len(result) == 1
assert result[0].user_application_name == 'base.foo'
result = list_infra_downstream_by_application_name(db, 'redis.bar')
assert len(result) == 1
assert result[0].user_application_name == 'base.bar'
def test_delete_infra_config_ok(
client, db, infra_info, test_application_name, test_application_token,
last_audit_log):
major_application_name = test_application_name
infra_info.set_by_name('r100010', 'idcs', 'alta1', {
'url': 'sam+redis://redis.100010/overall.alta1'})
infra_info.set_by_name('r100010', 'idcs', 'altb1', {
'url': 'sam+redis://redis.100010/overall.altb1'})
infra_info.save()
InfraDownstream.bindmany() \
.bind(major_application_name, 'redis', 'r100010', 'idcs', 'alta1',
'url', 'redis.100010') \
.bind(major_application_name, 'redis', 'r100010', 'idcs', 'altb1',
'url', 'redis.100010') \
.commit()
InfraDownstream.flush_cache_by_application('redis.100010')
ds = InfraDownstream.get_multi_by_application('redis.100010')
assert len(ds) == 2
r = client.delete(
'/api/infra-config/%s/redis/r100010' % test_application_name,
query_string={'scope_type': 'idcs', 'scope_name': 'altb1'},
headers={'Authorization': test_application_token})
assert_response_ok(r)
assert r.json['data'] == {'infra_config': [
{'scope_type': 'idcs', 'scope_name': 'alta1',
'value': {'url': 'sam+redis://redis.100010/overall.alta1'}},
]}
InfraDownstream.flush_cache_by_application('redis.100010')
ds = InfraDownstream.get_multi_by_application('redis.100010')
assert len(ds) == 1
assert ds[0].user_application_name == major_application_name
assert ds[0].user_infra_type == 'redis'
assert ds[0].user_infra_name == 'r100010'
r = client.delete(
'/api/infra-config/%s/redis/r100010' % test_application_name,
query_string={'scope_type': 'idcs', 'scope_name': 'alta1'},
headers={'Authorization': test_application_token})
assert_response_ok(r)
assert r.json['data'] == {'infra_config': []}
InfraDownstream.flush_cache_by_application('redis.100010')
ds = InfraDownstream.get_multi_by_application('redis.100010')
assert len(ds) == 0
audit_log = last_audit_log()
assert audit_log and audit_log.action_name == 'DELETE_INFRA_CONFIG'
assert audit_log.action_json['application_name'] == major_application_name
assert audit_log.action_json['infra_type'] == 'redis'
assert audit_log.action_json['infra_name'] == 'r100010'
assert audit_log.action_json['scope_type'] == 'idcs'
assert audit_log.action_json['scope_name'] == 'alta1'
assert audit_log.action_json['data']['new'] is None
assert audit_log.action_json['data']['old'] == {
'url': 'sam+redis://redis.100010/overall.alta1'}
def test_get_infra_downstream(client, test_token):
InfraDownstream.bindmany() \
.bind('base.foo', 'redis', 'cache-1', 'idcs', 'alta1', 'url',
'redis.100010') \
.bind('base.foo', 'redis', 'cache-1', 'idcs', 'altb1', 'url',
'redis.100010') \
.bind('base.bar', 'redis', 'cache-1', 'idcs', 'alta1', 'url',
'redis.100010') \
.bind('base.bar', 'redis', 'cache-2', 'idcs', 'alta1', 'url',
'redis.100011') \
.commit()
r = client.get('/api/infra-config-downstream/redis.100010',
headers={'Authorization': test_token})
assert_response_ok(r)
downstream = r.json['data']['downstream']
assert len(downstream) == 3
assert downstream[0]['user_application_name'] == 'base.foo'
assert downstream[0]['user_infra_type'] == 'redis'
assert downstream[0]['user_infra_name'] == 'cache-1'
assert downstream[0]['user_scope_pair'] == {
'type': 'idcs',
'name': 'alta1'
}
assert downstream[0]['user_field_name'] == 'url'
assert downstream[1]['user_application_name'] == 'base.foo'
assert downstream[1]['user_infra_type'] == 'redis'
assert downstream[1]['user_infra_name'] == 'cache-1'
assert downstream[1]['user_scope_pair'] == {
'type': 'idcs',
'name': 'altb1'
}
assert downstream[1]['user_field_name'] == 'url'
assert downstream[2]['user_application_name'] == 'base.bar'
assert downstream[2]['user_infra_type'] == 'redis'
assert downstream[2]['user_infra_name'] == 'cache-1'
assert downstream[2]['user_scope_pair'] == {
'type': 'idcs',
'name': 'alta1'
}
assert downstream[2]['user_field_name'] == 'url'
r = client.get('/api/infra-config-downstream/redis.100011',
headers={'Authorization': test_token})
assert_response_ok(r)
downstream = r.json['data']['downstream']
assert len(downstream) == 1
assert downstream[0]['user_application_name'] == 'base.bar'
assert downstream[0]['user_infra_type'] == 'redis'
assert downstream[0]['user_infra_name'] == 'cache-2'
assert downstream[0]['user_scope_pair'] == {
'type': 'idcs',
'name': 'alta1'
}
assert downstream[0]['user_field_name'] == 'url'
InfraDownstream.bind('base.baz', 'redis', 'cache-1', 'idcs', 'alta1',
'url', 'redis.100011')
InfraDownstream.unbind('base.bar', 'redis', 'cache-2', 'idcs', 'alta1',
'url')
# Stale data
r = client.get('/api/infra-config-downstream/redis.100011',
headers={'Authorization': test_token})
assert_response_ok(r)
downstream = r.json['data']['downstream']
assert len(downstream) == 1
assert downstream[0]['user_application_name'] == 'base.bar'
assert downstream[0]['user_infra_name'] == 'cache-2'
assert downstream[0]['user_scope_pair'] == {
'type': 'idcs',
'name': 'alta1'
}
assert downstream[0]['user_field_name'] == 'url'
# Fresh data
r = client.post('/api/infra-config-downstream/redis.100011',
headers={'Authorization': test_token})
assert_response_ok(r)
downstream = r.json['data']['downstream']
assert len(downstream) == 1
assert downstream[0]['user_application_name'] == 'base.baz'
assert downstream[0]['user_infra_name'] == 'cache-1'
assert downstream[0]['user_scope_pair'] == {
'type': 'idcs',
'name': 'alta1'
}
assert downstream[0]['user_field_name'] == 'url'
def test_collect_infra_config(zk, db, faker, test_team):
prefix = faker.uuid4()[:8]
for x in 'foo', 'bar':
x = '%s_%s' % (prefix, x)
Application.create(x, test_team.id)
InfraDownstream.bindmany()\
.bind(x, 'database', 'stale-db', 'idcs', 'altb1', 'master',
'dal.foo.master') \
.bind(x, 'database', 'stale-db', 'idcs', 'altb1', 'slave',
'dal.foo.auto') \
.bind(x, 'redis', 'stale-cache', 'idcs', 'altb1', 'url',
'redis.foo') \
.commit()
mysql_url = {
'master': 'sam+mysql://root:[email protected]/%s_db' % x,
'slave': 'sam+mysql://root:[email protected]/%s_db' % x}
redis_url = {'url': 'sam+redis://redis.foo'}
infra_info = InfraInfo(zk, x, 'database')
infra_info.load()
infra_info.set_by_name('db', 'idcs', 'altb1', mysql_url)
infra_info.save()
infra_info = InfraInfo(zk, x, 'redis')
infra_info.load()
infra_info.set_by_name('cache', 'idcs', 'altb1', redis_url)
infra_info.save()
InfraDownstream.flush_cache_by_application('dal.foo.auto')
ds = InfraDownstream.get_multi_by_application('dal.foo.auto')
assert all(d.user_infra_name == 'stale-db' for d in ds)
InfraDownstream.flush_cache_by_application('redis.auto')
ds = InfraDownstream.get_multi_by_application('redis.foo')
assert all(d.user_infra_name == 'stale-cache' for d in ds)
list(_collect_infra_upstream(zk, u'error\u00a0'))
collect_infra_config()
InfraDownstream.flush_cache_by_application('dal.foo.auto')
ds = InfraDownstream.get_multi_by_application('dal.foo.auto')
assert len(ds) == 2
assert ds[0].user_application_name == '%s_foo' % prefix
assert ds[0].user_infra_type == 'database'
assert ds[0].user_infra_name == 'db'
assert ds[0].user_scope_pair == ('idcs', 'altb1')
assert ds[0].user_field_name == 'slave'
assert ds[1].user_application_name == '%s_bar' % prefix
assert ds[1].user_infra_type == 'database'
assert ds[1].user_infra_name == 'db'
assert ds[1].user_scope_pair == ('idcs', 'altb1')
assert ds[1].user_field_name == 'slave'
InfraDownstream.flush_cache_by_application('redis.auto')
ds = InfraDownstream.get_multi_by_application('redis.foo')
assert len(ds) == 2
assert ds[0].user_application_name == '%s_foo' % prefix
assert ds[0].user_infra_type == 'redis'
assert ds[0].user_infra_name == 'cache'
assert ds[0].user_scope_pair == ('idcs', 'altb1')
assert ds[0].user_field_name == 'url'
assert ds[1].user_application_name == '%s_bar' % prefix
assert ds[1].user_infra_type == 'redis'
assert ds[1].user_infra_name == 'cache'
assert ds[1].user_scope_pair == ('idcs', 'altb1')
assert ds[1].user_field_name == 'url'