def _load(self, items):
"""
首先解析mapping
然后解析出各个配置之间的依赖关系，同时尝试检查是否有循环依赖
"""
index = self.esIndex
esType = self.esType
keys = set()
for data in items:
key = data['key']
database = data['database']
table = data['table']
if key in keys:
raise IllegalConfigException(
'Duplicated key[%s] detected in [%s, %s]' %
(key, index, esType))
keys.add(key)
configItem = HandlerConfigItem(self, data)
self._allItems.append(configItem)
if database not in self._inverted:
self._inverted[database] = {}
if table not in self._inverted[database]:
self._inverted[database][table] = []
self._inverted[database][table].append(configItem)
if configItem.isMaster:
if self._masterItem:
raise IllegalConfigException(
'Duplicated master item detected in [%s, %s]' %
(index, esType))
self._masterItem = configItem
else:
self._slaveItems.append(configItem)
# 检查是否存在master item
if not self._masterItem:
raise IllegalConfigException('master item NOT found in [%s, %s]' %
(index, esType))
# 生成依赖关系
self._resolveDependences()
# 检查依赖关系是否合法
self._checkValidDependences()
# 检查是否存在循环依赖
loopDedected = self._checkLoopDependences()
if loopDedected:
raise IllegalConfigException(
'Loop Dependence detected in [%s, %s]' % (index, esType))
def _getParentQuery(self, nestedItem, values):
parentQuery = nestedItem.get('parent_query', None)
if parentQuery:
itemKey = nestedItem.key
context = HandlerContext(nestedItem, {itemKey: values})
parentQuery = context.exp_data(parentQuery, nestedItem)
return parentQuery
statement = nestedItem['statement']
matches = _PARENT_EXP_RE.findall(statement)
if not matches:
raise IllegalConfigException(
'could NOT find parent dependence in nested item: %s' %
nestedItem)
# 构造parent values的部分值
parentValues = {}
for match in matches:
f1, f2, f3, f4 = match
if f1 and f2:
parentValues[f2] = values.get(f1, None)
elif f3 and f4:
parentValues[f3] = values.get(f3, None)
parentItem = nestedItem.getLocatedConfigList().getParentItem()
parentKey = parentItem.key
context = HandlerContext(parentItem, {parentKey: parentValues})
# TODO 这里有个bug：当parentItem是master item时，可能不存在query配置
# 暂时的解决方案是在nestedItem中增加配置parent_query
parentQuery = context.exp_data(parentItem['query'], parentItem)
return parentQuery
def _getDependenceFields(self, value):
if not value:
return
if isinstance(value, list):
for item in value:
self._getDependenceFields(item)
elif isinstance(value, dict):
for item in value.values():
self._getDependenceFields(item)
elif isinstance(value, basestring):
depends = _DEPENDENCE_RE.findall(value)
for dep in depends:
percent = dep[0]
key = dep[1]
field = dep[2]
if percent:
# met '%%'
continue
if key and key != self.key:
# document_id, routing, query, parent_query只允许依赖当前的key
raise IllegalConfigException(
'anchor field[%s, %s] can NOT depend on other config item: %s'
% (key, field, self))
self._anchorFields.add(field)
else:
pass # do nothing, just return
def loadFromFile(self, filepath):
try:
loadData = yaml.load(open(filepath), Loader)
if isinstance(loadData, list):
data = {}
for item in loadData:
data.update(item)
else:
data = loadData
for key in data.keys():
if key.startswith('__'):
del data[key]
self._data = data
except Exception as e:
_logger.error('fail to loads handler config from file[%s]: %s',
filepath, e)
raise IllegalConfigException(
'the config file[%s] MUST be valid: %s' %
(filepath, Failure()))
self._resolve()
global _loadCount
_loadCount += 1
_logger.debug('HandlerConfig loaded count: %s, %s', _initCount,
_loadCount)
def _resolveDependenceByItem(self, item):
key = item['key']
statement = item['statement']
depends = _DEPENDENCE_RE.findall(statement)
dependKeys = set()
for dep in depends:
dKey = dep[1]
dField = dep[2]
if not dKey or dKey == key:
continue
elif dKey == '__master':
dKey = self._masterItem['key']
if dKey not in self._dependents:
self._dependents[dKey] = {}
if dField not in self._dependents[dKey]:
self._dependents[dKey][dField] = set()
self._dependents[dKey][dField].add(key)
if dKey not in ('__last', '__parent'):
dependKeys.add(dKey)
if len(dependKeys) > 1:
raise IllegalConfigException(
'dependence count can NOT be greater than one: esIndex[%s], esType[%s], key[%s]'
% (item.esIndex, item.esType, key))
def resolve(funcString, **kwargs):
if not funcString:
return None
funcInfo = _resolveFunction(funcString)
if funcInfo is None:
values = kwargs['values']
if funcString[0] in ('+', '-'):
sign = funcString[0]
funcString = funcString[1:]
oriValue = values[funcString]
if isinstance(oriValue, Number):
if sign == '-':
return -oriValue
else:
return oriValue
else:
raise IllegalConfigException(
'field with a sign(+ or -) must be a number type: %s' %
funcString)
else:
return values[funcString]
else:
funcName = funcInfo['name']
args = funcInfo['args']
if funcName == 'echo':
return echo(*args)
func = getattr(sys.modules[__name__], funcName, None)
if func is None:
func = utils.functionForName(funcName)
if func is None:
raise IllegalConfigException(
'can NOT find function with name[%s]' % funcName)
argv = []
for arg in args:
argv.append(resolve(arg, **kwargs))
return func(*argv, **kwargs)
def _checkValidDependences(self):
"""
检查依赖关系是否合法
"""
# __last 依赖，只能出现在master item/nested master item
lastDependents = self._getDirectDependentKeys('__last')
for key in lastDependents:
config = self.getConfigItemByKey(key)
if not config.isMaster:
raise IllegalConfigException(
'__last dependence can ONLY appear in master config item, not current item[%s]'
% key)
# __parent 依赖，只能出现在nested master item
parentDependents = self._getDirectDependentKeys('__parent')
for key in parentDependents:
config = self.getConfigItemByKey(key)
if not config.isMaster or not config.isNested():
raise IllegalConfigException(
'__parent dependence can ONLY appear in nested master config item, not current item[%s]'
% key)
def loadFromJson(self, jsonData):
try:
self._data = json.loads(jsonData)
except Exception as e:
_logger.error('fail to loads handler config from json data: %s', e)
raise IllegalConfigException(
'the jsonData string MUST be jsonable: %s' % Failure())
self._resolve()
global _loadCount
_loadCount += 1
_logger.debug('HandlerConfig loaded count: %s, %s', _initCount,
_loadCount)
def filterData(filterDict, values):
"""
是否过滤指定数据
True 数据通过
False 数据被过滤
"""
_logger.debug('filterDict[%s] ; values[%s]', filterDict, values)
if filterDict:
for filterKey in filterDict.keys():
if filterKey not in values:
return False
filterValue = filterDict[filterKey]
dataValue = values[filterKey]
if isinstance(filterValue, list):
if dataValue not in filterValue:
return False
elif isinstance(filterValue, dict):
for op in filterValue.keys():
value = filterValue[op]
if op == '==':
if dataValue != value:
return False
elif op == '!=' or op == '<>':
if dataValue == value:
return False
elif op == '>':
if dataValue <= value:
return False
elif op == '>=':
if dataValue < value:
return False
elif op == '<':
if dataValue >= value:
return False
elif op == '<=':
if dataValue > value:
return False
else:
raise IllegalConfigException(
'filter op NOT supported yet: %s' % op)
else:
if dataValue != filterValue:
return False
# 默认通过
return True
def _resolveMapping(self):
mapping = self._data['mapping']
for index, mapItem in enumerate(mapping):
if isinstance(mapItem, dict):
if 'field' in mapItem:
mapItem['db_field'] = mapItem['field']
mapItem['es_field'] = mapItem['field']
del mapItem['field']
if 'type' not in mapItem:
mapItem['type'] = 'default'
if 'eval_on_deleted' not in mapItem:
mapItem['eval_on_deleted'] = False
else:
mapItem['eval_on_deleted'] = bool(
mapItem['eval_on_deleted'])
if 'null_value' not in mapItem:
mapItem['null_value'] = None
if mapItem['type'] == 'nested':
esField = mapItem['es_field']
nestedConfigList = NestedHandlerConfigList(
self, esField, mapItem['db_field'])
mapItem['db_field'] = nestedConfigList
self._locatedList.addNestedConfigList(
esField, nestedConfigList)
for field in nestedConfigList.getParentDependents():
self._nestedDependents[field] = nestedConfigList
self._nestedLists[esField] = nestedConfigList
elif isinstance(mapItem, basestring):
self._data['mapping'][index] = {
'db_field': mapItem,
'es_field': mapItem,
'type': 'default',
'eval_on_deleted': False,
'null_value': None
}
else:
raise IllegalConfigException(
'mapping values MUST be dict or string: %s' % mapItem)
def _resolveArgs(argsStr):
stack = []
result = []
buff = []
inQuoted = None
lastChar = None
for char in argsStr:
if inQuoted:
if char == inQuoted and lastChar != '\\':
inQuoted = None
buff.append(char)
elif char not in ('(', ')', ','):
if char in ('"', '\'') and lastChar != '\\':
inQuoted = char
buff.append(char)
elif char == ',':
if stack:
buff.append(char)
else:
result.append(''.join(buff))
buff = []
elif char == '(':
stack.append(char)
buff.append(char)
elif char == ')':
stack.pop()
buff.append(char)
lastChar = char
if stack:
raise IllegalConfigException('args string is NOT valid: %s' % argsStr)
if buff:
result.append(''.join(buff))
result = [x.strip() for x in result]
return result
def _checkValidation(self):
# config.key必须存在
if not self.key:
raise IllegalConfigException(
'key of config item can NOT be empty: %s' % self)
# mapping中的es_field必须是一个合法的标识符
mapping = self._data['mapping']
for item in mapping:
esField = item['es_field']
if not _VALID_MAPPING_ES_FIELD.match(esField):
raise IllegalConfigException(
'es_field[%s] must be a valid identifier: %s' %
(esField, self))
# parent_query只能出现在 nested master item
if not (self.isMaster
and self.isNested()) and 'parent_query' in self._data:
raise IllegalConfigException(
'parent_query can ONLY appear in nested master item: %s' %
self)
# 非master的item，以及nested中的所有item必须存在query
# routing不允许出现在nested以及非master的item中
if not self.isMaster or self.isNested():
if 'query' not in self._data:
raise IllegalConfigException(
'query property must EXIST in non-master/nesetd config item: %s'
% self)
if 'routing' in self._data:
raise IllegalConfigException(
'routing property can NOT EXIST in non-master/nesetd config item: %s'
% self)
# statment中不能带有limit子句
statement = self._data['statement']
if _SQL_STATEMENT_LIMIT_RE.search(statement) is not None:
raise IllegalConfigException(
'statement can NOT contain limit clause: [%s]' % statement)
def addNestedConfigList(self, esField, nestedList):
raise IllegalConfigException(
'NestedHandlerConfigList must NOT contain any child NestedHandlerConfigList: %s, %s'
% (self.esIndex, self.esType))
def addNestedConfigList(self, esField, nestedList):
if esField in self._nestedLists:
raise IllegalConfigException('es_field[%s] of nested duplicated' %
esField)
self._nestedLists[esField] = nestedList