def parse_conditions_list(c, d):
if not c:
raise P2ESError('Empty list')
if isinstance(c[0], basestring):
if c[0] == 'AND':
if len(c) > 2:
for sub_c in c[1:]:
if not parse_conditions(sub_c, d):
return False
return True
else:
return False
elif c[0] == 'OR':
if len(c) > 2:
for sub_c in c[1:]:
if parse_conditions(sub_c, d):
return True
return False
else:
return True
else:
raise P2ESError(
'Logical groups must begin with "AND" or "OR" '
'("{}" found)'.format(c[0])
)
else:
# default to "AND" if not specified
for sub_c in c:
if not parse_conditions(sub_c, d):
return False
return True
def send_to_es(CONFIG, index_name, data):
# HTTP bulk insert toward ES
url = '{}/{}/{}/_bulk'.format(CONFIG['ES_URL'], index_name,
CONFIG['ES_Type'])
try:
http_res = http(CONFIG, url, method="POST", data=data)
except Exception as e:
raise P2ESError(
'Error while executing HTTP bulk insert on {} - {}'.format(
index_name, str(e)))
# Interpreting HTTP bulk insert response
if http_res.status_code != 200:
raise P2ESError('Bulk insert on {} failed - '
'HTTP status code = {} - '
'Response {}'.format(index_name, http_res.status_code,
http_res.text))
try:
json_res = http_res.json()
except Exception as e:
raise P2ESError('Error while decoding JSON HTTP response - '
'{} - '
'first 100 characters: {}'.format(
str(e),
http_res.text[:100],
))
if json_res['errors']:
raise P2ESError('Bulk insert on {} failed to process '
'one or more documents'.format(index_name))
def create_index(index_name, CONFIG):
# index already exists?
if does_index_exist(index_name, CONFIG):
return
# index does not exist, creating it
tpl_path = '{}/{}'.format(CONFIG['CONF_DIR'],
CONFIG['ES_IndexTemplateFileName'])
try:
with open(tpl_path, "r") as f:
tpl = f.read()
except Exception as e:
raise P2ESError(
'Error while reading index template from file {}: {}'.format(
tpl_path, str(e)))
url = '{}/{}'.format(CONFIG['ES_URL'], index_name)
last_err = None
try:
# using PUT
http_res = http(CONFIG, url, method="PUT", data=tpl)
except Exception as e1:
last_err = "Error using PUT method: {}".format(str(e1))
# trying the old way
try:
http_res = http(CONFIG, url, method="POST", data=tpl)
except Exception as e2:
# something went wrong: does index exist anyway?
last_err += " - "
last_err += "Error using old way: {}".format(str(e2))
pass
try:
if does_index_exist(index_name, CONFIG):
return
except:
pass
err = "An error occurred while creating index {} from template {}: "
if last_err:
err += last_err
else:
err += "error unknown"
raise P2ESError(err.format(index_name, tpl_path))
def parse_conditions_dict(c, d, opfield):
op = '='
n = None
v = None
for k in c:
if k == opfield:
op = c[k]
if not op in ('=', '>', '>=', '<', '<=', '!=', 'in', 'notin'):
raise P2ESError('Unexpected operator: "{}"'.format(op))
else:
if n is None:
n = k
v = c[k]
else:
raise P2ESError('Only one name/value pair allowed')
if op in ('in', 'notin') and not isinstance(v, list):
raise P2ESError('The "{}" operator requires a list'.format(op))
if n is None:
raise P2ESError('Name/value pair expected')
if n not in d:
return False
if op == '=':
return d[n] == v
elif op == '>':
return d[n] > v
elif op == '>=':
return d[n] >= v
elif op == '<':
return d[n] < v
elif op == '<=':
return d[n] <= v
elif op == '!=':
return d[n] != v
elif op == 'in':
return d[n] in v
elif op == 'notin':
return not d[n] in v
else:
raise P2ESError('Operator not implemented: "{}"'.format(op))
def parse_conditions(c, d, opfield='__op__'):
if isinstance(c, list):
return parse_conditions_list(c, d)
elif isinstance(c, dict):
return parse_conditions_dict(c, d, opfield)
else:
raise P2ESError('Unexpected object type {} from {}'.format(
type(c), str(c)
))
def does_index_exist(index_name, CONFIG):
url = '{}/{}'.format(CONFIG['ES_URL'], index_name)
try:
status_code = http(CONFIG, url, method="HEAD").status_code
if status_code == 200:
return True
if status_code == 404:
return False
raise Exception("Unexpected status code: {}".format(status_code))
except Exception as err:
raise P2ESError('Error while checking if {} index exists: {}'.format(
index_name, str(err)))
def __init__(self, *args, **kwargs):
BaseWriterThread.__init__(self, *args, **kwargs)
# Creating index
self.index_name = datetime.datetime.now().strftime(
self.CONFIG['ES_IndexName']
)
try:
create_index(self.index_name, self.CONFIG)
except P2ESError as e:
raise P2ESError(
"Error while creating index {}: {}".format(
self.index_name, str(e)
)
)
def http(CONFIG, url, method="GET", data=None):
auth = None
if CONFIG['ES_AuthType'] != 'none':
if CONFIG['ES_AuthType'] == 'basic':
auth = HTTPBasicAuth(CONFIG['ES_UserName'], CONFIG['ES_Password'])
elif CONFIG['ES_AuthType'] == 'digest':
auth = HTTPDigestAuth(CONFIG['ES_UserName'], CONFIG['ES_Password'])
else:
raise P2ESError('Unexpected authentication type: {}'.format(
CONFIG['ES_AuthType']))
headers = {'Content-Type': 'application/x-ndjson'}
if method == "GET":
return requests.get(url, auth=auth, headers=headers)
elif method == "POST":
return requests.post(url, auth=auth, data=data, headers=headers)
elif method == "PUT":
return requests.put(url, auth=auth, data=data, headers=headers)
elif method == "HEAD":
return requests.head(url, auth=auth, headers=headers)
else:
raise Exception("Method unknown: {}".format(method))
def test_transformation(tr):
ret = True
try:
tr_det = 'Transformations matrix ({})'.format(transformation)
except:
tr_det = 'Transformations matrix'
if 'Conditions' not in tr:
raise P2ESError('{}, "Conditions" is missing'.format(tr_det))
if 'Actions' not in tr:
raise P2ESError('{}, "Actions" is missing'.format(tr_det))
try:
parse_conditions(tr['Conditions'], {})
except P2ESError as e:
raise P2ESError('{}, invalid "Conditions": {}'.format(tr_det, str(e)))
for action in tr['Actions']:
if 'Type' not in action:
raise P2ESError('{}, "Type" is missing'.format(tr_det))
tr_det += ', action type = {}'.format(action['Type'])
if action['Type'] not in ('AddField', 'AddFieldLookup', 'DelField'):
raise P2ESError('{}, "Type" unknown'.format(tr_det))
if 'Name' not in action:
raise P2ESError('{}, "Name" is missing'.format(tr_det))
if action['Type'] == 'AddField':
if 'Value' not in action:
raise P2ESError(
'{}, "Value" is missing for new field "{}"'.format(
tr_det, action['Name']
)
)
if action['Type'] == 'AddFieldLookup':
if 'LookupFieldName' not in action:
raise P2ESError(
'{}, "LookupFieldName" is missing for '
'new field "{}"'.format(tr_det, action['Name'])
)
if 'LookupTable' in action and 'LookupTableFile' in action:
raise P2ESError(
'{}, only one from "LookupTable" and '
'"LookupTableFile" allowed'.format(tr_det)
)
if 'LookupTable' not in action and 'LookupTableFile' not in action:
raise P2ESError(
'{}, "LookupTable" and "LookupTableFile" missing '
'for new field "{}"'.format(tr_det, action['Name'])
)
if 'LookupTableFile' in action:
try:
with open(action['LookupTableFile'], "r") as f:
action['LookupTable'] = json.load(f)
except Exception as e:
raise P2ESError(
'{}, error loading lookup table from {}: {}'.format(
tr_det, action['LookupTableFile'], str(e)
)
)