def test_modify_variables(self):
prepare_variables(self.variable_dao)
# use test variable in 'new_app' to check
new_app_variable = SAMPLE_VARIABLE.copy()
new_app_variable.app = 'new_app'
new_app_variable.name = 'var1'
# first check
url = "/default/variable_models?app=new_app"
response = self.fetch(url)
res = json.loads(response.body)
self.assertEqual(len(res['values']), 1)
get_variable = VariableModel.from_dict(res['values'][0])
self.assertEqual(new_app_variable, get_variable)
# 1. modify variable and test
# 1.1 modify test variable, so it is doesn't equal now
new_app_variable.remark = 'new_remark'
self.assertNotEquals(new_app_variable, get_variable)
# 1.2 post the modified variable
url = "/default/variable_models"
response = self.fetch(url, method='POST', body='[%s]' % new_app_variable.get_json())
res = json.loads(response.body)
self.assertEqual(res['status'], 0)
self.assertEqual(res['msg'], 'ok')
# 1.3 verify again
url = "/default/variable_models?app=new_app"
response = self.fetch(url)
res = json.loads(response.body)
self.assertEqual(len(res['values']), 1)
get_variable = VariableModel.from_dict(res['values'][0])
# now equal again
self.assertEqual(new_app_variable, get_variable)
# 2. add an variable with post
# 2.1 now there are 4 variables
url = "/default/variable_models"
response = self.fetch(url)
res = json.loads(response.body)
self.assertEqual(len(res['values']), 4)
# 2.2 add one
added_variable = SAMPLE_VARIABLE.copy()
added_variable.name = 'added_variable'
response = self.fetch(url, method='POST', body='[%s]' % added_variable.get_json())
res = json.loads(response.body)
self.assertEqual(res['status'], 0)
self.assertEqual(res['msg'], 'ok')
# 2.3 now there are 5
response = self.fetch(url)
res = json.loads(response.body)
self.assertEqual(len(res['values']), 5)
def tests_modify_variable(self):
prepare_variables(self.cust_dao)
url = "/platform/variable_models/variable/{}/{}".format(SAMPLE_VARIABLE.app, SAMPLE_VARIABLE.name)
response = self.fetch(url)
res = json.loads(response.body)
self.assertEqual(res['status'], 0)
self.assertEqual(res['msg'], 'ok')
self.assertEqual(len(res['values']), 1)
get_variable = VariableModel.from_dict(res['values'][0])
self.assertEqual(get_variable, SAMPLE_VARIABLE)
# 1. modify SAMPLE_VARIABLE
# 1.1 first not equal
new_sample = SAMPLE_VARIABLE.copy()
new_sample.remark = 'modified'
self.assertNotEquals(get_variable, new_sample)
# 1.2 second modify
response = self.fetch(url, method='POST', body=new_sample.get_json())
res = json.loads(response.body)
self.assertEqual(res['status'], 0)
self.assertEqual(res['msg'], 'ok')
# 1.3 third check
response = self.fetch(url)
res = json.loads(response.body)
self.assertEqual(res['status'], 0)
self.assertEqual(res['msg'], 'ok')
self.assertEqual(len(res['values']), 1)
get_variable = VariableModel.from_dict(res['values'][0])
self.assertEqual(get_variable, new_sample)
# 2. add a new one
# 2.1 get 404
url = "/platform/variable_models/variable/{}/{}".format(SAMPLE_VARIABLE.app, 'variable_to_be_add')
response = self.fetch(url)
res = json.loads(response.body)
self.assertEqual(res['status'], 404)
# 2.2 add one
variable_to_be_add = SAMPLE_VARIABLE.copy()
variable_to_be_add.name = 'variable_to_be_add'
response = self.fetch(url, method='POST', body=variable_to_be_add.get_json())
res = json.loads(response.body)
self.assertEqual(res['status'], 0)
self.assertEqual(res['msg'], 'ok')
# 2.3 check
response = self.fetch(url)
res = json.loads(response.body)
self.assertEqual(res['status'], 0)
self.assertEqual(res['msg'], 'ok')
self.assertEqual(len(res['values']), 1)
get_variable = VariableModel.from_dict(res['values'][0])
self.assertEqual(get_variable, variable_to_be_add)
def load_config():
print("loading config")
for _ in get_file_json_content('data/event_model_default.json'):
add_event_to_registry(EventModel.from_dict(_))
for _ in get_file_json_content('data/common_variable_default.json'):
add_variable_to_registry(VariableModel.from_dict(_))
for _ in get_file_json_content('data/realtime_variable_default.json'):
add_variable_to_registry(VariableModel.from_dict(_))
for _ in get_file_json_content('data/profile_variable_default.json'):
add_variable_to_registry(VariableModel.from_dict(_))
def post(self):
"""
美化一组variable
@API
summary: 美化一组variable
notes: 美化一组variable
tags:
- platform
parameters:
-
name: variable
in: body
required: true
type: json
description: 一组variable的json表示
produces:
- application/json
"""
self.set_header('content-type', 'application/json')
body = self.request.body
try:
variables = list()
for _ in json.loads(body):
variable = VariableModel.from_dict(_)
variables.append(variable)
variables = [_.get_simplified_ordered_dict() for _ in variables]
self.finish(json_dumps({"status": 0, "msg": "ok", "values": variables}))
except Exception as err:
print_exc()
return self.process_error(400, str(err))
def test_delete_variable(self):
prepare_variables(self.cust_dao)
url = "/platform/variable_models/variable/{}/{}".format(SAMPLE_VARIABLE.app, SAMPLE_VARIABLE.name)
response = self.fetch(url)
res = json.loads(response.body)
self.assertEqual(res['status'], 0)
self.assertEqual(res['msg'], 'ok')
self.assertEqual(len(res['values']), 1)
self.assertEqual(VariableModel.from_dict(res['values'][0]), SAMPLE_VARIABLE)
# delete
url = "/platform/variable_models/variable/{}/{}".format(SAMPLE_VARIABLE.app, SAMPLE_VARIABLE.name)
response = self.fetch(url, method='DELETE')
res = json.loads(response.body)
self.assertEqual(res['status'], 0)
self.assertEqual(res['msg'], 'ok')
# check
response = self.fetch(url)
res = json.loads(response.body)
self.assertEqual(res['status'], 404)
# can not delete default
clear_variables(self.cust_dao)
prepare_variables(self.default_dao)
url = "/platform/variable_models/variable/{}/{}".format(SAMPLE_VARIABLE.app, SAMPLE_VARIABLE.name)
response = self.fetch(url, method='DELETE')
res = json.loads(response.body)
self.assertEqual(res['status'], 0)
self.assertEqual(res['msg'], 'ok')
response = self.fetch(url)
res = json.loads(response.body)
self.assertEqual(res['status'], 0)
def test_get_variables(self):
prepare_variables(self.cust_dao)
# default 0, cust 3
url = "/default/variable_models"
response = self.fetch(url)
res = json.loads(response.body)
self.assertEqual(self.default_dao.count(), 0)
self.assertEqual(res['status'], 0)
self.assertEqual(res['msg'], 'ok')
self.assertEqual(len(res['values']), 0)
url = '/platform/variable_models'
response = self.fetch(url)
res = json.loads(response.body)
self.assertEqual(self.cust_dao.count(), 4)
self.assertEqual(res['status'], 0)
self.assertEqual(res['msg'], 'ok')
self.assertEqual(len(res['values']), 4)
return_variables = res['values']
return_variable = [_ for _ in return_variables if _['name'] == SAMPLE_VARIABLE.name][0]
self.assertEqual(VariableModel.from_dict(return_variable), SAMPLE_VARIABLE)
# check get by app
response = self.fetch(url + '?app=new_app')
res = json.loads(response.body)
self.assertEqual(len(res['values']), 1)
# check get by type
response = self.fetch(url + '?type=new_type')
res = json.loads(response.body)
self.assertEqual(len(res['values']), 1)
def gen_ip_trigger_variable_from_strategy(strategy,
trigger_variable_name,
is_delay=False):
"""
产生ip维度的trigger变量
new mode
:return:
"""
if not strategy:
return
conditions = []
trigger_event = get_trigger_event(strategy, is_delay)
for t in strategy.terms:
left = t.left
if left.type == 'event':
c = gen_filter_from_event_exp(left, t.op, t.right)
if c:
conditions.append(c)
# location condition has lower priority
for t in strategy.terms:
left = t.left
if left.type == 'func' and left.subtype == 'getlocation':
c = gen_filter_from_location_exp(left, trigger_event)
if c:
conditions.append(c)
total_filter = {}
if is_delay:
total_filter = gen_ordinary_filter(trigger_event[1], 'delay_strategy',
'==', strategy.name).get_dict()
else:
if conditions:
conditions = [_.get_dict() for _ in conditions]
total_filter = Filter('', '', '', '', '', '', 'and', '',
conditions).get_dict()
remark = 'ip trigger for strategy {}'.format(utf8(strategy.name))
variable = VariableModel('realtime', 'nebula', trigger_variable_name,
remark, remark, '', 'enable', 'filter', '', '',
'', [{
'app': trigger_event[0],
'name': trigger_event[1]
}], total_filter, {}, {}, ['c_ip'])
add_variable_to_registry(variable)
return variable
def test_get_variable(self):
prepare_variables(self.variable_dao)
url = "/default/variable_models/variable/{}/{}".format(SAMPLE_VARIABLE.app, SAMPLE_VARIABLE.name)
response = self.fetch(url)
res = json.loads(response.body)
self.assertEqual(res['status'], 0)
self.assertEqual(res['msg'], 'ok')
self.assertEqual(len(res['values']), 1)
self.assertEqual(VariableModel.from_dict(res['values'][0]), SAMPLE_VARIABLE)
# get not exist
url = "/default/variable_models/variable/{}/{}".format(SAMPLE_VARIABLE.app, 'not_exist')
response = self.fetch(url)
res = json.loads(response.body)
self.assertEqual(res['status'], 404)
def gen_dimension_trigger_variable_from_strategy(strategy,
trigger_variable_name,
dimension,
is_delay=False):
"""
产生uid/did维度的trigger变量
new mode
:return:
"""
trigger_event = get_trigger_event(strategy, is_delay)
# mappings, getvariable和count的触发字段必须包含在trigger event，这样在collect variable才能获取
dimension_count = 0
for t in strategy.terms:
left = t.left
# only care realtime vars
if t.scope != 'realtime':
continue
# only dimension related
if left.subtype in {'getvariable', 'count'}:
if get_dimension_from_trigger_keys(
left.trigger_fields) != dimension:
continue
else:
dimension_count += 1
if not dimension_count:
return None
dimension_field = get_field_from_dimension(dimension)
filter_dict = gen_ordinary_filter(trigger_event[1], dimension_field, '!=',
'').get_dict()
remark = '{} trigger for strategy {}'.format(dimension,
utf8(strategy.name))
variable = VariableModel('realtime', 'nebula', trigger_variable_name,
remark, remark, '', 'enable', 'filter', '', '',
'', [{
'app': trigger_event[0],
'name': trigger_event[1]
}], filter_dict, {}, {}, [dimension_field])
add_variable_to_registry(variable)
return variable
def post(self, app, name):
"""
增加/修改一个变量
@API
summary: 增加/修改一个变量
notes: 增加/修改一个变量
tags:
- platform
parameters:
-
name: app
in: path
required: true
type: string
description: 变量的app
-
name: name
in: path
required: true
type: string
description: 变量的名称
-
name: variable
in: body
required: true
type: json
description: 变量的json表示
produces:
- application/json
"""
self.set_header('content-type', 'application/json')
body = self.request.body
try:
new_model = VariableModel.from_json(body)
except Exception as err:
return self.process_error(400, str(err))
try:
VariableModelCustDao().add_model(new_model)
self.finish(json_dumps({"status": 0, "msg": "ok", "values": []}))
except Exception as err:
logger.error(err)
self.process_error(500, str(err))
def to_variablemodel(self):
try:
result = VariableModel(self.module, self.app, self.name,
self.remark, self.visible_name,
self.dimension, self.status, self.type,
self.value_type,
self.value_subtype, self.value_category,
json.loads(self.source),
json.loads(self.filter),
json.loads(self.period),
json.loads(self.function),
json.loads(self.groupbykeys),
json.loads(self.hint))
add_variable_to_registry(result)
return result
except Exception as err:
logger.error('variable model default error...')
return None
def gen_counter_from_interval_count_exp(count_exp, counter_name):
""" 为interval生成新的counter.
新模式
:param count_exp:
:param counter_name:
:return:
"""
# 统计的事件
source_event_id = count_exp.source_event
counter_function = Function('last', '', 'timestamp', '', '', '')
sub_filters = []
for c in count_exp.condition:
left = c['left']
op = c['op']
right = c['right']
if op == '=':
# 等于变量，在counter fix的时候被自动修正
continue
sub_filters.append(
gen_ordinary_filter(source_event_id[1], left, op,
right).get_dict())
counter_filter = Filter('', '', '', '', '', '', 'and', '', sub_filters)
remark = 'first temp variable from interval counter'
counter_variable = VariableModel('realtime', 'nebula', counter_name,
remark, remark, '', 'enable', 'aggregate',
'', '', '', [{
'app': source_event_id[0],
'name': source_event_id[1]
}], counter_filter.get_dict(), {
'type': 'ever',
'value': ''
}, counter_function.get_dict(),
count_exp.groupby)
add_variable_to_registry(counter_variable)
return counter_variable
def post(self):
"""
增加一组变量
@API
summary: 增加一组变量
notes: 增加一组变量，如果变量已存在，则修改
tags:
- platform
parameters:
-
name: variables
in: body
required: true
type: json
description: 变量的json表示，list结构
produces:
- application/json
"""
self.set_header('content-type', 'application/json')
body = self.request.body
try:
variables = list()
for _ in json.loads(body):
v = VariableModel.from_dict(_)
add_variable_to_registry(v)
variables.append(v)
except Exception as err:
print_exc()
return self.process_error(400, str(err))
try:
dao = VariableModelCustDao()
for v in variables:
dao.add_model(v)
self.finish(json_dumps({"status": 0, "msg": "ok", "values": []}))
except Exception as err:
print_exc()
return self.process_error(500, str(err))
def sync_to_base_events(self, body):
"""
第二步：新增基础事件到基础事件记录表
参数：依赖/platform/event_models中post的参数
:param body:
:return:
"""
try:
events = list()
base_event = dict()
for _ in json.loads(body):
# 自动设置一些默认值，无需前端重复传递
base_event["status"] = "enable"
base_event["type"] = "event"
base_event["source"] = [{
"app": _.get("app"),
"name": _.get("name")
}]
# 获取body中我们需要的值
base_event["app"] = _.get("app")
base_event["name"] = _.get("name")
base_event["visible_name"] = _.get("visible_name")
base_event["remark"] = _.get("remark")
base_event["module"] = _.get("type")
event = VariableModel.from_dict(base_event)
events.append(event)
except:
logger.error(traceback.format_exc())
return False
try:
dao = VariableModelCustDao()
for event in events:
dao.add_model(event)
return True
except:
logger.error(traceback.format_exc())
return False
def gen_counters_and_filters_from_count_exp(term, termid, name_pattern,
trigger_variable_name):
"""
从count函数配置中生成新的counter.
返回: (all counters, collect counter， filters)，这里有三块内容
all counters: 所有生成的counter, 包括用来参与策略计算的counter和中间counter
collect counter：最终参与策略计算的counter，这一个list就去掉了一些中间counter
fitlers：collect counters关联的filters
"""
# 统计的事件
count_exp = term.left
source_event_id = count_exp.source_event
sub_filters = []
for c in count_exp.condition:
left = c['left']
op = c['op']
right = c['right']
if op == '=':
# 等于变量，在counter fix的时候被自动修正
continue
sub_filters.append(
gen_ordinary_filter(source_event_id[1], left, op,
right).get_dict())
# distinct string should be non empty
if 'distinct' in count_exp.algorithm:
for field in count_exp.operand:
# noinspection PyBroadException
try:
sub_filters.append(
gen_ordinary_filter(source_event_id[1], field, '!=',
'').get_dict())
except:
pass
counter_filter = Filter('', '', '', '', '', '', 'and', '', sub_filters)
if 'interval' == count_exp.algorithm:
first_counter_name = name_pattern % '_1_'
remark = 'interval 1st counter variable(last counter event) for term %s' % termid
first_counter_function = Function('last', '', 'timestamp', '', '')
first_variable = VariableModel('realtime', 'nebula',
first_counter_name, remark, remark, '',
'enable', 'aggregate', '', '', '',
[{
'app': source_event_id[0],
'name': source_event_id[1]
}], counter_filter.get_dict(), {
'type': 'last_n_seconds',
'value': count_exp.interval
}, first_counter_function.get_dict(),
count_exp.groupby)
add_variable_to_registry(first_variable)
second_counter_name = name_pattern % '_2_'
remark = 'interval 2nd counter variable(last trigger event) for term %s' % termid
second_source = [
{
'app': 'nebula',
'name': trigger_variable_name
},
]
second_counter_function = Function('last', '', 'timestamp', '', '')
second_variable = VariableModel('realtime', 'nebula',
second_counter_name, remark, remark,
'', 'enable', 'aggregate', '', '', '',
second_source, {}, {
'type': 'last_n_seconds',
'value': count_exp.interval
}, second_counter_function.get_dict(),
count_exp.groupby)
add_variable_to_registry(second_variable)
third_counter_name = name_pattern % '_3_'
remark = 'interval 3rd dual variable(-value) for term %s' % termid
third_source = [
{
'app': 'nebula',
'name': second_counter_name
},
{
'app': 'nebula',
'name': first_counter_name
},
]
third_counter_function = Function('-', '', 'value', '', '')
third_variable = VariableModel('realtime', 'nebula',
third_counter_name, remark, remark, '',
'enable', 'dual', '', '', '',
third_source, {}, {
'type': 'last_n_seconds',
'value': count_exp.interval
}, third_counter_function.get_dict(),
count_exp.groupby)
add_variable_to_registry(third_variable)
f = gen_ordinary_filter(third_counter_name, 'value', term.op,
term.right.value)
additional_f = gen_ordinary_filter(third_counter_name, 'value', '>',
'0')
return [first_variable, second_variable,
third_variable], third_variable, [f, additional_f]
else:
counter_function = gen_function(count_exp.algorithm,
count_exp.operand[0])
counter_name = name_pattern % ''
remark = 'temp counter for term %s' % termid
counter_variable = VariableModel('realtime', 'nebula', counter_name,
remark, remark, '', 'enable',
'aggregate', '', '', '',
[{
'app': source_event_id[0],
'name': source_event_id[1]
}], counter_filter.get_dict(), {
'type': 'last_n_seconds',
'value': count_exp.interval
}, counter_function.get_dict(),
count_exp.groupby)
add_variable_to_registry(counter_variable)
f = gen_ordinary_filter(counter_name, 'value', term.op,
term.right.value)
return [counter_variable], counter_variable, [f]
def gen_dimension_collector_variable_from_strategy(s,
collect_variable_name,
trigger_variable_name,
dimension,
before_sleep=True,
is_delay=False):
"""
为IP/UID/DID产生collector变量
:param s: 策略
:param collect_variable_name: collector变量
:param trigger_variable_name: trigger变量
:param dimension: 维度，ip/uid/did
:param before_sleep: 是否找sleep之前的条款
:param is_delay: 是否生成delaycollector
:return: 产生该维度的变量
"""
sources = list()
sources.append({'app': 'nebula', 'name': trigger_variable_name})
variables, variables_conditions = gen_getvariables_filters_from_strategy(
s, dimension, before_sleep)
for v in variables:
sources.append({'app': v[0], 'name': v[1]})
counters, collect_counters, counter_conditions = gen_counters_and_filters_from_strategy(
s, dimension, before_sleep, trigger_variable_name)
for counter in collect_counters:
sources.append({'app': 'nebula', 'name': counter.name})
conditions = (variables_conditions or list()) + (counter_conditions
or list())
remark = 'collector for strategy {}'.format(utf8(s.name))
total_filter = {}
if conditions:
conditions = [_.get_dict() for _ in conditions]
total_filter = Filter('', '', '', '', '', '', 'and', '',
conditions).get_dict()
dimension_field = get_field_from_dimension(dimension)
if is_delay:
variable_type = 'delaycollector'
else:
variable_type = 'collector'
collector_function = {
'method': 'setblacklist',
'param': s.name,
'config': {
'trigger': trigger_variable_name
}
}
if is_delay:
sleep_terms = [_ for _ in s.terms if _.left.subtype == 'sleep']
if not sleep_terms:
raise RuntimeError('sleep配置出错')
duration = int(sleep_terms[0].left.duration)
unit = sleep_terms[0].left.unit
if unit == 'm':
duration *= 60
elif unit == 'h':
duration *= 3600
collector_function['config']['sleep'] = duration
variable = VariableModel('realtime', 'nebula', collect_variable_name,
remark, remark, '', 'enable', variable_type, '',
'', '', sources, total_filter, {},
collector_function, [dimension_field])
add_variable_to_registry(variable)
return counters, variable
SAMPLE_EVENTS = json.load(json_file)
new_events = list()
for _ in SAMPLE_EVENTS:
new_event = EventModel.from_dict(_)
add_event_to_registry(new_event)
new_events.append(new_event)
SAMPLE_EVENTS = new_events
SAMPLE_EVENT = SAMPLE_EVENTS[0]
with open('nebula/tests/data/variable_model.json') as json_file:
SAMPLE_VARIABLES = json.load(json_file)
new_variables = list()
for _ in SAMPLE_VARIABLES:
new_variable = VariableModel.from_dict(_)
add_variable_to_registry(new_variable)
new_variables.append(new_variable)
SAMPLE_VARIABLES = new_variables
SAMPLE_VARIABLE = SAMPLE_VARIABLES[-1] # did变量
def prepare_events(event_dao):
for _ in SAMPLE_EVENTS:
event_dao.add_model(_)
def prepare_variables(variable_dao):
variable_dao.add_model(SAMPLE_VARIABLES[0])
variable_dao.add_model(SAMPLE_VARIABLES[1])