def parse_conditions_list(c, d): if not c: raise P2ESError('Empty list') if isinstance(c[0], basestring): if c[0] == 'AND': if len(c) > 2: for sub_c in c[1:]: if not parse_conditions(sub_c, d): return False return True else: return False elif c[0] == 'OR': if len(c) > 2: for sub_c in c[1:]: if parse_conditions(sub_c, d): return True return False else: return True else: raise P2ESError( 'Logical groups must begin with "AND" or "OR" ' '("{}" found)'.format(c[0]) ) else: # default to "AND" if not specified for sub_c in c: if not parse_conditions(sub_c, d): return False return True
def send_to_es(CONFIG, index_name, data): # HTTP bulk insert toward ES url = '{}/{}/{}/_bulk'.format(CONFIG['ES_URL'], index_name, CONFIG['ES_Type']) try: http_res = http(CONFIG, url, method="POST", data=data) except Exception as e: raise P2ESError( 'Error while executing HTTP bulk insert on {} - {}'.format( index_name, str(e))) # Interpreting HTTP bulk insert response if http_res.status_code != 200: raise P2ESError('Bulk insert on {} failed - ' 'HTTP status code = {} - ' 'Response {}'.format(index_name, http_res.status_code, http_res.text)) try: json_res = http_res.json() except Exception as e: raise P2ESError('Error while decoding JSON HTTP response - ' '{} - ' 'first 100 characters: {}'.format( str(e), http_res.text[:100], )) if json_res['errors']: raise P2ESError('Bulk insert on {} failed to process ' 'one or more documents'.format(index_name))
def create_index(index_name, CONFIG): # index already exists? if does_index_exist(index_name, CONFIG): return # index does not exist, creating it tpl_path = '{}/{}'.format(CONFIG['CONF_DIR'], CONFIG['ES_IndexTemplateFileName']) try: with open(tpl_path, "r") as f: tpl = f.read() except Exception as e: raise P2ESError( 'Error while reading index template from file {}: {}'.format( tpl_path, str(e))) url = '{}/{}'.format(CONFIG['ES_URL'], index_name) last_err = None try: # using PUT http_res = http(CONFIG, url, method="PUT", data=tpl) except Exception as e1: last_err = "Error using PUT method: {}".format(str(e1)) # trying the old way try: http_res = http(CONFIG, url, method="POST", data=tpl) except Exception as e2: # something went wrong: does index exist anyway? last_err += " - " last_err += "Error using old way: {}".format(str(e2)) pass try: if does_index_exist(index_name, CONFIG): return except: pass err = "An error occurred while creating index {} from template {}: " if last_err: err += last_err else: err += "error unknown" raise P2ESError(err.format(index_name, tpl_path))
def parse_conditions_dict(c, d, opfield): op = '=' n = None v = None for k in c: if k == opfield: op = c[k] if not op in ('=', '>', '>=', '<', '<=', '!=', 'in', 'notin'): raise P2ESError('Unexpected operator: "{}"'.format(op)) else: if n is None: n = k v = c[k] else: raise P2ESError('Only one name/value pair allowed') if op in ('in', 'notin') and not isinstance(v, list): raise P2ESError('The "{}" operator requires a list'.format(op)) if n is None: raise P2ESError('Name/value pair expected') if n not in d: return False if op == '=': return d[n] == v elif op == '>': return d[n] > v elif op == '>=': return d[n] >= v elif op == '<': return d[n] < v elif op == '<=': return d[n] <= v elif op == '!=': return d[n] != v elif op == 'in': return d[n] in v elif op == 'notin': return not d[n] in v else: raise P2ESError('Operator not implemented: "{}"'.format(op))
def parse_conditions(c, d, opfield='__op__'): if isinstance(c, list): return parse_conditions_list(c, d) elif isinstance(c, dict): return parse_conditions_dict(c, d, opfield) else: raise P2ESError('Unexpected object type {} from {}'.format( type(c), str(c) ))
def does_index_exist(index_name, CONFIG): url = '{}/{}'.format(CONFIG['ES_URL'], index_name) try: status_code = http(CONFIG, url, method="HEAD").status_code if status_code == 200: return True if status_code == 404: return False raise Exception("Unexpected status code: {}".format(status_code)) except Exception as err: raise P2ESError('Error while checking if {} index exists: {}'.format( index_name, str(err)))
def __init__(self, *args, **kwargs): BaseWriterThread.__init__(self, *args, **kwargs) # Creating index self.index_name = datetime.datetime.now().strftime( self.CONFIG['ES_IndexName'] ) try: create_index(self.index_name, self.CONFIG) except P2ESError as e: raise P2ESError( "Error while creating index {}: {}".format( self.index_name, str(e) ) )
def http(CONFIG, url, method="GET", data=None): auth = None if CONFIG['ES_AuthType'] != 'none': if CONFIG['ES_AuthType'] == 'basic': auth = HTTPBasicAuth(CONFIG['ES_UserName'], CONFIG['ES_Password']) elif CONFIG['ES_AuthType'] == 'digest': auth = HTTPDigestAuth(CONFIG['ES_UserName'], CONFIG['ES_Password']) else: raise P2ESError('Unexpected authentication type: {}'.format( CONFIG['ES_AuthType'])) headers = {'Content-Type': 'application/x-ndjson'} if method == "GET": return requests.get(url, auth=auth, headers=headers) elif method == "POST": return requests.post(url, auth=auth, data=data, headers=headers) elif method == "PUT": return requests.put(url, auth=auth, data=data, headers=headers) elif method == "HEAD": return requests.head(url, auth=auth, headers=headers) else: raise Exception("Method unknown: {}".format(method))
def test_transformation(tr): ret = True try: tr_det = 'Transformations matrix ({})'.format(transformation) except: tr_det = 'Transformations matrix' if 'Conditions' not in tr: raise P2ESError('{}, "Conditions" is missing'.format(tr_det)) if 'Actions' not in tr: raise P2ESError('{}, "Actions" is missing'.format(tr_det)) try: parse_conditions(tr['Conditions'], {}) except P2ESError as e: raise P2ESError('{}, invalid "Conditions": {}'.format(tr_det, str(e))) for action in tr['Actions']: if 'Type' not in action: raise P2ESError('{}, "Type" is missing'.format(tr_det)) tr_det += ', action type = {}'.format(action['Type']) if action['Type'] not in ('AddField', 'AddFieldLookup', 'DelField'): raise P2ESError('{}, "Type" unknown'.format(tr_det)) if 'Name' not in action: raise P2ESError('{}, "Name" is missing'.format(tr_det)) if action['Type'] == 'AddField': if 'Value' not in action: raise P2ESError( '{}, "Value" is missing for new field "{}"'.format( tr_det, action['Name'] ) ) if action['Type'] == 'AddFieldLookup': if 'LookupFieldName' not in action: raise P2ESError( '{}, "LookupFieldName" is missing for ' 'new field "{}"'.format(tr_det, action['Name']) ) if 'LookupTable' in action and 'LookupTableFile' in action: raise P2ESError( '{}, only one from "LookupTable" and ' '"LookupTableFile" allowed'.format(tr_det) ) if 'LookupTable' not in action and 'LookupTableFile' not in action: raise P2ESError( '{}, "LookupTable" and "LookupTableFile" missing ' 'for new field "{}"'.format(tr_det, action['Name']) ) if 'LookupTableFile' in action: try: with open(action['LookupTableFile'], "r") as f: action['LookupTable'] = json.load(f) except Exception as e: raise P2ESError( '{}, error loading lookup table from {}: {}'.format( tr_det, action['LookupTableFile'], str(e) ) )