def _gen_string_filter(source, string_field, op, op_value, param=''):
"""
get string field function
"""
if not op or op not in _string_condition_ops:
raise RuntimeError('string类型不支持({})操作'.format(utf8(op)))
if op == 'in':
parts = op_value.split(',')
if len(parts) > 10:
raise RuntimeError('属于最多支持10个属性')
condition = [_gen_string_filter(source, string_field, '==', part, param).get_dict() for part in parts]
return Filter('', '', '', '', '', '', 'or', '', condition)
elif op == '!in':
parts = op_value.split(',')
if len(parts) > 10:
raise RuntimeError('属于最多支持10个属性')
condition = [_gen_string_filter(source, string_field, '!=', part, param).get_dict() for part in parts]
return Filter('', '', '', '', '', '', 'and', '', condition)
if op == 'regex':
op = 'match'
elif op == '!regex':
op = '!match'
elif op == 'contain':
op = 'contains'
elif op == '!contain':
op = '!contains'
return Filter(source, string_field, '', '', op, utf8(op_value), 'simple', param, None)
def test_variable_filter():
filter_dict = {
'source': 'HTTP_DYNAMIC',
'object': 'c_ip',
'object_type': 'string',
'object_subtype': '',
'operation': '==',
'value': '1',
'param': '',
'type': 'simple',
}
filter = Filter.from_dict(filter_dict)
filter.sanity_check()
assert not filter.is_empty()
assert filter.get_dict() == filter_dict
# 空的filter
filter_dict = {
}
filter = Filter.from_dict(filter_dict)
filter.sanity_check()
assert filter.is_empty()
assert filter.get_dict() == filter_dict
# complex filter
filter_dict = {
'source': 'HTTP_DYNAMIC',
'object': 'c_ip',
'object_type': 'string',
'object_subtype': '',
'operation': '==',
'value': '1',
'param': '',
'type': 'simple',
}
complex_filter_dict = {
'type': 'and',
'condition': [filter_dict, filter_dict]
}
complex_filter = Filter.from_dict(complex_filter_dict)
complex_filter.sanity_check()
assert not complex_filter.is_empty()
assert complex_filter.get_dict() == complex_filter_dict
def gen_counter_from_interval_count_exp(count_exp, counter_name):
""" 为interval生成新的counter.
新模式
:param count_exp:
:param counter_name:
:return:
"""
# 统计的事件
source_event_id = count_exp.source_event
counter_function = Function('last', '', 'timestamp', '', '', '')
sub_filters = []
for c in count_exp.condition:
left = c['left']
op = c['op']
right = c['right']
if op == '=':
# 等于变量，在counter fix的时候被自动修正
continue
sub_filters.append(
gen_ordinary_filter(source_event_id[1], left, op,
right).get_dict())
counter_filter = Filter('', '', '', '', '', '', 'and', '', sub_filters)
remark = 'first temp variable from interval counter'
counter_variable = VariableModel('realtime', 'nebula', counter_name,
remark, remark, '', 'enable', 'aggregate',
'', '', '', [{
'app': source_event_id[0],
'name': source_event_id[1]
}], counter_filter.get_dict(), {
'type': 'ever',
'value': ''
}, counter_function.get_dict(),
count_exp.groupby)
add_variable_to_registry(counter_variable)
return counter_variable
def gen_ip_trigger_variable_from_strategy(strategy,
trigger_variable_name,
is_delay=False):
"""
产生ip维度的trigger变量
new mode
:return:
"""
if not strategy:
return
conditions = []
trigger_event = get_trigger_event(strategy, is_delay)
for t in strategy.terms:
left = t.left
if left.type == 'event':
c = gen_filter_from_event_exp(left, t.op, t.right)
if c:
conditions.append(c)
# location condition has lower priority
for t in strategy.terms:
left = t.left
if left.type == 'func' and left.subtype == 'getlocation':
c = gen_filter_from_location_exp(left, trigger_event)
if c:
conditions.append(c)
total_filter = {}
if is_delay:
total_filter = gen_ordinary_filter(trigger_event[1], 'delay_strategy',
'==', strategy.name).get_dict()
else:
if conditions:
conditions = [_.get_dict() for _ in conditions]
total_filter = Filter('', '', '', '', '', '', 'and', '',
conditions).get_dict()
remark = 'ip trigger for strategy {}'.format(utf8(strategy.name))
variable = VariableModel('realtime', 'nebula', trigger_variable_name,
remark, remark, '', 'enable', 'filter', '', '',
'', [{
'app': trigger_event[0],
'name': trigger_event[1]
}], total_filter, {}, {}, ['c_ip'])
add_variable_to_registry(variable)
return variable
def test_simplified_filter():
"""
简化的filter
:return:
"""
filter_dict = {
'source': 'HTTP_DYNAMIC',
'object': 'c_ip',
'object_type': 'string',
'object_subtype': '',
'operation': '',
'value': '1',
'type': 'simple',
}
with raises(RuntimeError):
filter = Filter.from_dict(filter_dict)
filter.sanity_check()
def _gen_number_filter(source, number_field, op, op_value, is_long=False):
"""
get number field function
"""
if not op or op not in _number_condition_ops:
raise RuntimeError('数字类型不支持({})操作'.format(utf8(op)))
if op == 'between':
parts = op_value.split(',')
if len(parts) != 2:
raise RuntimeError('介于需要两个参数')
left, right = parts
left_condition = Filter(source, number_field, '', '', '>=', utf8(left), 'simple', '', None).get_dict()
right_condition = Filter(source, number_field, '', '', '<=', utf8(right), 'simple', '', None).get_dict()
return Filter('', '', '', '', '', '', 'and', '', [left_condition, right_condition])
elif op == 'in':
parts = op_value.split(',')
if len(parts) > 10:
raise RuntimeError('属于最多支持10个属性')
condition = [_gen_number_filter(source, number_field, '==', part, is_long).get_dict() for part in parts]
return Filter('', '', '', '', '', '', 'or', '', condition)
elif op == '!in':
parts = op_value.split(',')
if len(parts) > 10:
raise RuntimeError('属于最多支持10个属性')
condition = [_gen_number_filter(source, number_field, '!=', part, is_long).get_dict() for part in parts]
return Filter('', '', '', '', '', '', 'and', '', condition)
# simple condition
try:
if is_long:
op_value = int(op_value)
else:
op_value = float(op_value)
except:
raise RuntimeError('({})不是数字'.format(utf8(op_value)))
return Filter(source, number_field, '', '', op, utf8(op_value), 'simple', '', None)
def test_invalid_filter():
filter_dict = {
'source': 'HTTP_DYNAMIC',
'object': 'c_ip',
'object_type': 'string',
'object_subtype': '',
'operation': '',
'value': '1',
'param': '',
'condition': [],
'type': 'simple',
}
with raises(RuntimeError):
filter = Filter.from_dict(filter_dict)
filter.sanity_check()
# null object
filter_dict = {
'source': 'HTTP_DYNAMIC',
'object': '',
'object_type': 'string',
'object_subtype': '',
'operation': '==',
'value': '1',
'param': '',
'condition': [],
'type': 'simple',
}
with raises(RuntimeError):
filter = Filter.from_dict(filter_dict)
filter.sanity_check()
# invalid type
filter_dict = {
'source': 'HTTP_DYNAMIC',
'object': 'c_ip',
'object_type': 'kv',
'object_subtype': '',
'operation': '==',
'value': '1',
'param': '',
'condition': [],
'type': 'simple',
}
with raises(RuntimeError):
filter = Filter.from_dict(filter_dict)
filter.sanity_check()
# invalid operation
filter_dict = {
'source': 'HTTP_DYNAMIC',
'object': 'c_ip',
'object_type': 'string',
'object_subtype': '',
'operation': '>=',
'value': '1',
'param': '',
'condition': [],
'type': 'simple',
}
with raises(RuntimeError):
filter = Filter.from_dict(filter_dict)
filter.sanity_check()
# invalid type
filter_dict = {
'source': 'HTTP_DYNAMIC',
'object': 'c_ip',
'object_type': 'string',
'object_subtype': '',
'operation': '==',
'value': '1',
'param': '',
'condition': [],
'type': 'invalid type',
}
with raises(RuntimeError):
filter = Filter.from_dict(filter_dict)
filter.sanity_check()
def get(self):
"""
get nebula online variables
@API
summary: get nebula online variables
notes: get nebula online variables for online kv
tags:
- nebula
parameters:
produces:
- application/json
"""
self.set_header('content-type', 'application/json')
# try:
if True:
default_variables = VariableModelCustDao().list_all_models()
default_variables = filter(
lambda model: model.module in {'base', 'realtime'},
default_variables)
# hack for distinct count
for dv in default_variables:
if 'distinct' in dv.function.method and dv.function.object_type == 'string':
# 增加一个不为空的field
function = dv.function
not_null_filter = Filter(function.source, function.object,
function.object_type,
function.object_subtype, '!=', '',
'simple', '', [])
if dv.filter and not dv.filter.is_empty():
dv.filter = Filter(
'', '', '', '', '', '', 'and', '',
[dv.filter.get_dict(),
not_null_filter.get_dict()])
else:
dv.filter = Filter('', '', '', '', '', '', 'and', '',
[not_null_filter.get_dict()])
# end of hack
all_generated_variables = []
all_used_variables = []
strategies = context.nebula_strategies
strategies = filter(
lambda s: (s.status in ["test", "online"] and s.start_effect <=
millis_now() <= s.end_effect), strategies)
for s in strategies:
generated_variables, used_variables = gen_variables_from_strategy(
s, effective_check=True)
all_generated_variables.extend(generated_variables)
all_used_variables.extend(used_variables)
# recursively calcuated all_used_variables:
while True:
new_used_variable_found = False
for used_variable_name in all_used_variables[:]:
for source in get_variable_from_registry(
'nebula', used_variable_name).source:
if source['name'] not in all_used_variables:
all_used_variables.append(source['name'])
new_used_variable_found = True
if not new_used_variable_found:
break
default_variables = [
_ for _ in default_variables if _.name in all_used_variables
or _.type in {'event', 'filter'}
]
result = default_variables + all_generated_variables
result = [_.get_dict() for _ in result]
self.finish(
json_dumps({
"status": 0,
"msg": "ok",
"values": result
}))
def gen_counters_and_filters_from_count_exp(term, termid, name_pattern,
trigger_variable_name):
"""
从count函数配置中生成新的counter.
返回: (all counters, collect counter， filters)，这里有三块内容
all counters: 所有生成的counter, 包括用来参与策略计算的counter和中间counter
collect counter：最终参与策略计算的counter，这一个list就去掉了一些中间counter
fitlers：collect counters关联的filters
"""
# 统计的事件
count_exp = term.left
source_event_id = count_exp.source_event
sub_filters = []
for c in count_exp.condition:
left = c['left']
op = c['op']
right = c['right']
if op == '=':
# 等于变量，在counter fix的时候被自动修正
continue
sub_filters.append(
gen_ordinary_filter(source_event_id[1], left, op,
right).get_dict())
# distinct string should be non empty
if 'distinct' in count_exp.algorithm:
for field in count_exp.operand:
# noinspection PyBroadException
try:
sub_filters.append(
gen_ordinary_filter(source_event_id[1], field, '!=',
'').get_dict())
except:
pass
counter_filter = Filter('', '', '', '', '', '', 'and', '', sub_filters)
if 'interval' == count_exp.algorithm:
first_counter_name = name_pattern % '_1_'
remark = 'interval 1st counter variable(last counter event) for term %s' % termid
first_counter_function = Function('last', '', 'timestamp', '', '')
first_variable = VariableModel('realtime', 'nebula',
first_counter_name, remark, remark, '',
'enable', 'aggregate', '', '', '',
[{
'app': source_event_id[0],
'name': source_event_id[1]
}], counter_filter.get_dict(), {
'type': 'last_n_seconds',
'value': count_exp.interval
}, first_counter_function.get_dict(),
count_exp.groupby)
add_variable_to_registry(first_variable)
second_counter_name = name_pattern % '_2_'
remark = 'interval 2nd counter variable(last trigger event) for term %s' % termid
second_source = [
{
'app': 'nebula',
'name': trigger_variable_name
},
]
second_counter_function = Function('last', '', 'timestamp', '', '')
second_variable = VariableModel('realtime', 'nebula',
second_counter_name, remark, remark,
'', 'enable', 'aggregate', '', '', '',
second_source, {}, {
'type': 'last_n_seconds',
'value': count_exp.interval
}, second_counter_function.get_dict(),
count_exp.groupby)
add_variable_to_registry(second_variable)
third_counter_name = name_pattern % '_3_'
remark = 'interval 3rd dual variable(-value) for term %s' % termid
third_source = [
{
'app': 'nebula',
'name': second_counter_name
},
{
'app': 'nebula',
'name': first_counter_name
},
]
third_counter_function = Function('-', '', 'value', '', '')
third_variable = VariableModel('realtime', 'nebula',
third_counter_name, remark, remark, '',
'enable', 'dual', '', '', '',
third_source, {}, {
'type': 'last_n_seconds',
'value': count_exp.interval
}, third_counter_function.get_dict(),
count_exp.groupby)
add_variable_to_registry(third_variable)
f = gen_ordinary_filter(third_counter_name, 'value', term.op,
term.right.value)
additional_f = gen_ordinary_filter(third_counter_name, 'value', '>',
'0')
return [first_variable, second_variable,
third_variable], third_variable, [f, additional_f]
else:
counter_function = gen_function(count_exp.algorithm,
count_exp.operand[0])
counter_name = name_pattern % ''
remark = 'temp counter for term %s' % termid
counter_variable = VariableModel('realtime', 'nebula', counter_name,
remark, remark, '', 'enable',
'aggregate', '', '', '',
[{
'app': source_event_id[0],
'name': source_event_id[1]
}], counter_filter.get_dict(), {
'type': 'last_n_seconds',
'value': count_exp.interval
}, counter_function.get_dict(),
count_exp.groupby)
add_variable_to_registry(counter_variable)
f = gen_ordinary_filter(counter_name, 'value', term.op,
term.right.value)
return [counter_variable], counter_variable, [f]
def gen_dimension_collector_variable_from_strategy(s,
collect_variable_name,
trigger_variable_name,
dimension,
before_sleep=True,
is_delay=False):
"""
为IP/UID/DID产生collector变量
:param s: 策略
:param collect_variable_name: collector变量
:param trigger_variable_name: trigger变量
:param dimension: 维度，ip/uid/did
:param before_sleep: 是否找sleep之前的条款
:param is_delay: 是否生成delaycollector
:return: 产生该维度的变量
"""
sources = list()
sources.append({'app': 'nebula', 'name': trigger_variable_name})
variables, variables_conditions = gen_getvariables_filters_from_strategy(
s, dimension, before_sleep)
for v in variables:
sources.append({'app': v[0], 'name': v[1]})
counters, collect_counters, counter_conditions = gen_counters_and_filters_from_strategy(
s, dimension, before_sleep, trigger_variable_name)
for counter in collect_counters:
sources.append({'app': 'nebula', 'name': counter.name})
conditions = (variables_conditions or list()) + (counter_conditions
or list())
remark = 'collector for strategy {}'.format(utf8(s.name))
total_filter = {}
if conditions:
conditions = [_.get_dict() for _ in conditions]
total_filter = Filter('', '', '', '', '', '', 'and', '',
conditions).get_dict()
dimension_field = get_field_from_dimension(dimension)
if is_delay:
variable_type = 'delaycollector'
else:
variable_type = 'collector'
collector_function = {
'method': 'setblacklist',
'param': s.name,
'config': {
'trigger': trigger_variable_name
}
}
if is_delay:
sleep_terms = [_ for _ in s.terms if _.left.subtype == 'sleep']
if not sleep_terms:
raise RuntimeError('sleep配置出错')
duration = int(sleep_terms[0].left.duration)
unit = sleep_terms[0].left.unit
if unit == 'm':
duration *= 60
elif unit == 'h':
duration *= 3600
collector_function['config']['sleep'] = duration
variable = VariableModel('realtime', 'nebula', collect_variable_name,
remark, remark, '', 'enable', variable_type, '',
'', '', sources, total_filter, {},
collector_function, [dimension_field])
add_variable_to_registry(variable)
return counters, variable