def post(self):
"""
增加一组变量
@API
summary: 增加一组变量
notes: 增加一组变量，如果变量已存在，则修改
tags:
- platform
parameters:
-
name: variables
in: body
required: true
type: json
description: 变量的json表示，list结构
produces:
- application/json
"""
self.set_header('content-type', 'application/json')
body = self.request.body
try:
variables = list()
for _ in json.loads(body):
v = VariableModel.from_dict(_)
add_variable_to_registry(v)
variables.append(v)
except Exception as err:
print_exc()
return self.process_error(400, str(err))
try:
dao = VariableModelCustDao()
for v in variables:
dao.add_model(v)
self.finish(json_dumps({"status": 0, "msg": "ok", "values": []}))
except Exception as err:
print_exc()
return self.process_error(500, str(err))
def load_config():
print("loading config")
for _ in get_file_json_content('data/event_model_default.json'):
add_event_to_registry(EventModel.from_dict(_))
for _ in get_file_json_content('data/common_variable_default.json'):
add_variable_to_registry(VariableModel.from_dict(_))
for _ in get_file_json_content('data/realtime_variable_default.json'):
add_variable_to_registry(VariableModel.from_dict(_))
for _ in get_file_json_content('data/profile_variable_default.json'):
add_variable_to_registry(VariableModel.from_dict(_))
def gen_counters_and_filters_from_count_exp(term, termid, name_pattern,
trigger_variable_name):
"""
从count函数配置中生成新的counter.
返回: (all counters, collect counter， filters)，这里有三块内容
all counters: 所有生成的counter, 包括用来参与策略计算的counter和中间counter
collect counter：最终参与策略计算的counter，这一个list就去掉了一些中间counter
fitlers：collect counters关联的filters
"""
# 统计的事件
count_exp = term.left
source_event_id = count_exp.source_event
sub_filters = []
for c in count_exp.condition:
left = c['left']
op = c['op']
right = c['right']
if op == '=':
# 等于变量，在counter fix的时候被自动修正
continue
sub_filters.append(
gen_ordinary_filter(source_event_id[1], left, op,
right).get_dict())
# distinct string should be non empty
if 'distinct' in count_exp.algorithm:
for field in count_exp.operand:
# noinspection PyBroadException
try:
sub_filters.append(
gen_ordinary_filter(source_event_id[1], field, '!=',
'').get_dict())
except:
pass
counter_filter = Filter('', '', '', '', '', '', 'and', '', sub_filters)
if 'interval' == count_exp.algorithm:
first_counter_name = name_pattern % '_1_'
remark = 'interval 1st counter variable(last counter event) for term %s' % termid
first_counter_function = Function('last', '', 'timestamp', '', '')
first_variable = VariableModel('realtime', 'nebula',
first_counter_name, remark, remark, '',
'enable', 'aggregate', '', '', '',
[{
'app': source_event_id[0],
'name': source_event_id[1]
}], counter_filter.get_dict(), {
'type': 'last_n_seconds',
'value': count_exp.interval
}, first_counter_function.get_dict(),
count_exp.groupby)
add_variable_to_registry(first_variable)
second_counter_name = name_pattern % '_2_'
remark = 'interval 2nd counter variable(last trigger event) for term %s' % termid
second_source = [
{
'app': 'nebula',
'name': trigger_variable_name
},
]
second_counter_function = Function('last', '', 'timestamp', '', '')
second_variable = VariableModel('realtime', 'nebula',
second_counter_name, remark, remark,
'', 'enable', 'aggregate', '', '', '',
second_source, {}, {
'type': 'last_n_seconds',
'value': count_exp.interval
}, second_counter_function.get_dict(),
count_exp.groupby)
add_variable_to_registry(second_variable)
third_counter_name = name_pattern % '_3_'
remark = 'interval 3rd dual variable(-value) for term %s' % termid
third_source = [
{
'app': 'nebula',
'name': second_counter_name
},
{
'app': 'nebula',
'name': first_counter_name
},
]
third_counter_function = Function('-', '', 'value', '', '')
third_variable = VariableModel('realtime', 'nebula',
third_counter_name, remark, remark, '',
'enable', 'dual', '', '', '',
third_source, {}, {
'type': 'last_n_seconds',
'value': count_exp.interval
}, third_counter_function.get_dict(),
count_exp.groupby)
add_variable_to_registry(third_variable)
f = gen_ordinary_filter(third_counter_name, 'value', term.op,
term.right.value)
additional_f = gen_ordinary_filter(third_counter_name, 'value', '>',
'0')
return [first_variable, second_variable,
third_variable], third_variable, [f, additional_f]
else:
counter_function = gen_function(count_exp.algorithm,
count_exp.operand[0])
counter_name = name_pattern % ''
remark = 'temp counter for term %s' % termid
counter_variable = VariableModel('realtime', 'nebula', counter_name,
remark, remark, '', 'enable',
'aggregate', '', '', '',
[{
'app': source_event_id[0],
'name': source_event_id[1]
}], counter_filter.get_dict(), {
'type': 'last_n_seconds',
'value': count_exp.interval
}, counter_function.get_dict(),
count_exp.groupby)
add_variable_to_registry(counter_variable)
f = gen_ordinary_filter(counter_name, 'value', term.op,
term.right.value)
return [counter_variable], counter_variable, [f]
new_events = list()
for _ in SAMPLE_EVENTS:
new_event = EventModel.from_dict(_)
add_event_to_registry(new_event)
new_events.append(new_event)
SAMPLE_EVENTS = new_events
SAMPLE_EVENT = SAMPLE_EVENTS[0]
with open('nebula/tests/data/variable_model.json') as json_file:
SAMPLE_VARIABLES = json.load(json_file)
new_variables = list()
for _ in SAMPLE_VARIABLES:
new_variable = VariableModel.from_dict(_)
add_variable_to_registry(new_variable)
new_variables.append(new_variable)
SAMPLE_VARIABLES = new_variables
SAMPLE_VARIABLE = SAMPLE_VARIABLES[-1] # did变量
def prepare_events(event_dao):
for _ in SAMPLE_EVENTS:
event_dao.add_model(_)
def prepare_variables(variable_dao):
variable_dao.add_model(SAMPLE_VARIABLES[0])
variable_dao.add_model(SAMPLE_VARIABLES[1])
# add with new app
def gen_dimension_collector_variable_from_strategy(s,
collect_variable_name,
trigger_variable_name,
dimension,
before_sleep=True,
is_delay=False):
"""
为IP/UID/DID产生collector变量
:param s: 策略
:param collect_variable_name: collector变量
:param trigger_variable_name: trigger变量
:param dimension: 维度，ip/uid/did
:param before_sleep: 是否找sleep之前的条款
:param is_delay: 是否生成delaycollector
:return: 产生该维度的变量
"""
sources = list()
sources.append({'app': 'nebula', 'name': trigger_variable_name})
variables, variables_conditions = gen_getvariables_filters_from_strategy(
s, dimension, before_sleep)
for v in variables:
sources.append({'app': v[0], 'name': v[1]})
counters, collect_counters, counter_conditions = gen_counters_and_filters_from_strategy(
s, dimension, before_sleep, trigger_variable_name)
for counter in collect_counters:
sources.append({'app': 'nebula', 'name': counter.name})
conditions = (variables_conditions or list()) + (counter_conditions
or list())
remark = 'collector for strategy {}'.format(utf8(s.name))
total_filter = {}
if conditions:
conditions = [_.get_dict() for _ in conditions]
total_filter = Filter('', '', '', '', '', '', 'and', '',
conditions).get_dict()
dimension_field = get_field_from_dimension(dimension)
if is_delay:
variable_type = 'delaycollector'
else:
variable_type = 'collector'
collector_function = {
'method': 'setblacklist',
'param': s.name,
'config': {
'trigger': trigger_variable_name
}
}
if is_delay:
sleep_terms = [_ for _ in s.terms if _.left.subtype == 'sleep']
if not sleep_terms:
raise RuntimeError('sleep配置出错')
duration = int(sleep_terms[0].left.duration)
unit = sleep_terms[0].left.unit
if unit == 'm':
duration *= 60
elif unit == 'h':
duration *= 3600
collector_function['config']['sleep'] = duration
variable = VariableModel('realtime', 'nebula', collect_variable_name,
remark, remark, '', 'enable', variable_type, '',
'', '', sources, total_filter, {},
collector_function, [dimension_field])
add_variable_to_registry(variable)
return counters, variable