def _update(self, application_name, request, infra_type, infra_name):
check_application_auth(application_name, Authority.WRITE)
check_infra_type(infra_type)
scope_type = request.args['scope_type']
scope_name = request.args['scope_name']
check_scope(scope_type, scope_name)
value = request.get_json()
if not value or not isinstance(value, dict):
abort(400, 'Unacceptable content type or content body')
infra_info = InfraInfo(huskar_client.client, application_name,
infra_type)
infra_info.load()
old_value = infra_info.get_by_name(infra_name, scope_type, scope_name)
yield application_name, infra_info, scope_type, scope_name, value
infra_urls = infra_info.extract_urls(value)
infra_application_names = extract_application_names(infra_urls)
new_value = infra_info.get_by_name(infra_name, scope_type, scope_name)
with audit_log(audit_log.types.UPDATE_INFRA_CONFIG,
application_name=application_name,
infra_type=infra_type,
infra_name=infra_name,
scope_type=scope_type,
scope_name=scope_name,
old_value=old_value,
new_value=new_value):
for infra_application_name in infra_application_names:
check_application(infra_application_name)
infra_info.save()
with suppress_exceptions('infra config updating'):
infra_urls = infra_info.extract_urls(value, as_dict=True)
infra_applications = extract_application_names(infra_urls)
for field_name, infra_application_name in \
infra_applications.iteritems():
InfraDownstream.bind(application_name, infra_type,
infra_name, scope_type, scope_name,
field_name, infra_application_name)
def _collect_infra_upstream(client, application_name):
for infra_type in INFRA_CONFIG_KEYS:
try:
infra_info = InfraInfo(client, application_name, infra_type)
except ValueError:
logger.exception('ValueError with appid: %s type: %s',
application_name, infra_type)
continue
infra_info.load()
infra_data = infra_info.data or {}
for scope_type, scope_data in infra_data.iteritems():
for scope_name, scope_dict in scope_data.iteritems():
for infra_name, value in scope_dict.iteritems():
infra_urls = infra_info.extract_urls(value, as_dict=True)
infra_applications = extract_application_names(infra_urls)
for field_name, infra_application_name in \
infra_applications.iteritems():
yield InfraUpstreamInfo(infra_type, infra_name,
scope_type, scope_name,
field_name,
infra_application_name)
def test_new_oss_infra_config(
client, test_application_name, test_application_token,
test_oss, last_audit_log, mocker):
major_application_name = test_application_name
infra_info = InfraInfo(
huskar_client.client, test_application_name, 'oss')
logger = mocker.patch(
'huskar_api.api.middlewares.logger.logger', autospec=True)
infra_info.set_by_name('r100010', 'idcs', 'alta1', {
'url': 'http://*****:*****@%s:666/overall.alta' % test_oss
r = client.patch(
'/api/infra-config/%s/oss/r100010' % test_application_name,
query_string={'scope_type': 'idcs', 'scope_name': 'alta1'},
data=json.dumps({'connect_timeout_ms': 10, 'url': url,
'idle_timeout_ms': 5000}),
headers={'Authorization': test_application_token,
'Content-Type': 'application/json'})
assert_response_ok(r)
assert r.json['data'] == {'infra_config': [
{'scope_type': 'idcs', 'scope_name': 'alta1',
'value': {
'url': url,
'max_pool_size': 100,
'connect_timeout_ms': 10,
'idle_timeout_ms': 5000,
'max_error_retry': 3}
},
]}
request_args = json.loads(logger.mock_calls[0][1][-6])
assert 'url' not in request_args
audit_log = last_audit_log()
assert audit_log and audit_log.action_name == 'UPDATE_INFRA_CONFIG'
assert audit_log.action_json['application_name'] == major_application_name
assert audit_log.action_json['infra_type'] == 'oss'
assert audit_log.action_json['infra_name'] == 'r100010'
assert audit_log.action_json['scope_type'] == 'idcs'
assert audit_log.action_json['scope_name'] == 'alta1'
assert audit_log.action_json['data']['new'] == {
'url': url,
'max_pool_size': 100,
'connect_timeout_ms': 10,
'idle_timeout_ms': 5000,
'max_error_retry': 3,
}
assert audit_log.action_json['data']['old'] == {
'url': 'http://localhost:233',
'max_pool_size': 100,
'connect_timeout_ms': 5,
'max_error_retry': 3,
}
def get(self, application_name, infra_type, infra_name):
"""Gets the configuration of infrastructure in specified application.
The response looks like::
[{
"scope_type": "idcs", "scope_name": "alta1",
"value": {"url": "sam+redis://redis.foobar/overall"}
}]
:param application_name: The application which uses infrastructure.
:param infra_type: ``database``, ``redis`` or ``amqp``.
:param infra_name: The unique code-reference name of infrastructure.
:<header Authorization: Huskar Token (See :ref:`token`)
:status 200: The result is in the response.
"""
check_application_auth(application_name, Authority.READ)
check_infra_type(infra_type)
infra_info = InfraInfo(huskar_client.client, application_name,
infra_type)
infra_info.load()
infra_config = infra_info.list_by_infra_name(infra_name)
return api_response({'infra_config': dump_infra_config(infra_config)})
def test_rollback_infra_config_change(client, db, faker, test_application_name,
admin_token, last_audit_log,
_action_type, _scope_type, _scope_name,
_old_value, _new_value,
_expected_action_type, _expected_value):
infra_type = 'redis'
infra_name = 'default'
infra_info = InfraInfo(huskar_client.client, test_application_name,
infra_type)
infra_info.load()
action_type = getattr(action_types, _action_type)
action = action_creator.make_action(
action_type,
application_name=test_application_name,
infra_type=infra_type,
infra_name=infra_name,
scope_type=_scope_type,
scope_name=_scope_name,
old_value=_old_value,
new_value=_new_value,
)
audit_log = AuditLog.create(0, faker.ipv4(), action)
db.close()
r = client.put('/api/audit-rollback/%s/%s' %
(test_application_name, audit_log.id),
headers={'Authorization': admin_token})
assert_response_ok(r)
last_audit = last_audit_log()
infra_info.load()
value = infra_info.get_by_name(infra_name, _scope_type, _scope_name)
assert value == _expected_value
assert last_audit.action_type == getattr(action_types,
_expected_action_type)
def rollback_infra_config(action_type, action_data):
application_name = action_data['application_name']
infra_type = action_data['infra_type']
infra_name = action_data['infra_name']
scope_type = action_data['scope_type']
scope_name = action_data['scope_name']
infra_info = InfraInfo(huskar_client.client, application_name, infra_type)
infra_info.load()
current = infra_info.get_by_name(infra_name, scope_type, scope_name)
rollback_to = action_data['data']['old']
if rollback_to is None: # new infra config
infra_info.delete_by_name(infra_name, scope_type, scope_name)
new_action_type = action_types.DELETE_INFRA_CONFIG
else:
infra_info.set_by_name(infra_name, scope_type, scope_name, rollback_to)
new_action_type = action_types.UPDATE_INFRA_CONFIG
infra_info.save()
return new_action_type, {
'application_name': application_name,
'infra_type': infra_type,
'infra_name': infra_name,
'scope_type': scope_type,
'scope_name': scope_name,
'old_value': current,
'new_value': rollback_to,
}
def infra_info(test_application_name):
return InfraInfo(
huskar_client.client, test_application_name, 'redis')
def delete(self, application_name, infra_type, infra_name):
"""Deletes the infrastructure configuration in specified application.
:param application_name: The application which uses infrastructure.
:param infra_type: ``database``, ``redis`` or ``amqp``.
:param infra_name: The unique code-reference name of infrastructure.
:query scope_type: ``idcs`` or ``clusters``.
:query scope_name: The ezone id or clsuter name.
:<header Authorization: Huskar Token (See :ref:`token`)
:status 200: The same as ``GET``.
"""
check_application_auth(application_name, Authority.WRITE)
check_infra_type(infra_type)
scope_type = request.args['scope_type']
scope_name = request.args['scope_name']
check_scope(scope_type, scope_name)
infra_info = InfraInfo(huskar_client.client, application_name,
infra_type)
infra_info.load()
old_value = infra_info.get_by_name(infra_name, scope_type, scope_name)
infra_info.delete_by_name(infra_name, scope_type, scope_name)
with audit_log(audit_log.types.DELETE_INFRA_CONFIG,
application_name=application_name,
infra_type=infra_type,
infra_name=infra_name,
scope_type=scope_type,
scope_name=scope_name,
old_value=old_value,
new_value=None):
infra_info.save()
with suppress_exceptions('infra config deletion'):
old_urls = infra_info.extract_urls(old_value or {}, as_dict=True)
for field_name in frozenset(extract_application_names(old_urls)):
InfraDownstream.unbind(application_name, infra_type,
infra_name, scope_type, scope_name,
field_name)
infra_config = infra_info.list_by_infra_name(infra_name)
return api_response({'infra_config': dump_infra_config(infra_config)})
def test_collect_infra_config(zk, db, faker, test_team):
prefix = faker.uuid4()[:8]
for x in 'foo', 'bar':
x = '%s_%s' % (prefix, x)
Application.create(x, test_team.id)
InfraDownstream.bindmany()\
.bind(x, 'database', 'stale-db', 'idcs', 'altb1', 'master',
'dal.foo.master') \
.bind(x, 'database', 'stale-db', 'idcs', 'altb1', 'slave',
'dal.foo.auto') \
.bind(x, 'redis', 'stale-cache', 'idcs', 'altb1', 'url',
'redis.foo') \
.commit()
mysql_url = {
'master': 'sam+mysql://root:[email protected]/%s_db' % x,
'slave': 'sam+mysql://root:[email protected]/%s_db' % x}
redis_url = {'url': 'sam+redis://redis.foo'}
infra_info = InfraInfo(zk, x, 'database')
infra_info.load()
infra_info.set_by_name('db', 'idcs', 'altb1', mysql_url)
infra_info.save()
infra_info = InfraInfo(zk, x, 'redis')
infra_info.load()
infra_info.set_by_name('cache', 'idcs', 'altb1', redis_url)
infra_info.save()
InfraDownstream.flush_cache_by_application('dal.foo.auto')
ds = InfraDownstream.get_multi_by_application('dal.foo.auto')
assert all(d.user_infra_name == 'stale-db' for d in ds)
InfraDownstream.flush_cache_by_application('redis.auto')
ds = InfraDownstream.get_multi_by_application('redis.foo')
assert all(d.user_infra_name == 'stale-cache' for d in ds)
list(_collect_infra_upstream(zk, u'error\u00a0'))
collect_infra_config()
InfraDownstream.flush_cache_by_application('dal.foo.auto')
ds = InfraDownstream.get_multi_by_application('dal.foo.auto')
assert len(ds) == 2
assert ds[0].user_application_name == '%s_foo' % prefix
assert ds[0].user_infra_type == 'database'
assert ds[0].user_infra_name == 'db'
assert ds[0].user_scope_pair == ('idcs', 'altb1')
assert ds[0].user_field_name == 'slave'
assert ds[1].user_application_name == '%s_bar' % prefix
assert ds[1].user_infra_type == 'database'
assert ds[1].user_infra_name == 'db'
assert ds[1].user_scope_pair == ('idcs', 'altb1')
assert ds[1].user_field_name == 'slave'
InfraDownstream.flush_cache_by_application('redis.auto')
ds = InfraDownstream.get_multi_by_application('redis.foo')
assert len(ds) == 2
assert ds[0].user_application_name == '%s_foo' % prefix
assert ds[0].user_infra_type == 'redis'
assert ds[0].user_infra_name == 'cache'
assert ds[0].user_scope_pair == ('idcs', 'altb1')
assert ds[0].user_field_name == 'url'
assert ds[1].user_application_name == '%s_bar' % prefix
assert ds[1].user_infra_type == 'redis'
assert ds[1].user_infra_name == 'cache'
assert ds[1].user_scope_pair == ('idcs', 'altb1')
assert ds[1].user_field_name == 'url'