def convert(self):
if self.has_index(self.metadata):
self.metadata['index'] = Config.new_index_pattern
if self.has_references(self.data):
for ref in self.data['references']:
if ref['id'] == Config.old_index_pattern:
ref['id'] = Config.new_index_pattern
for o in self.metadata['filter']:
if self.has_meta(o):
models.Meta(o['meta']).convert()
if self.has_exists(o):
replace_value('field', o['exists'])
if self.has_wildcard(o):
replace_keys(o['wildcard'])
if self.has_bool(o):
models.BoolQuery(o['bool']).convert()
if self.has_query(o):
models.Query(o['query']).convert()
if self.has_query(self.metadata): # Kibana 7
if self.has_query(self.metadata['query']):
models.QueryString(self.metadata['query']).convert()
else:
models.Query(self.metadata['query']).convert()
if self.has_visState:
if self.has_aggs(self.source['visState']):
for agg in self.source['visState']['aggs']:
models.Agg(agg).convert()
return self
def validate(self):
for item in self.metadata['filter']:
if self.has_meta(item):
self.invalid_fields += models.Meta(item['meta'])\
.validate().get_invalid_fields()
if self.has_exists(item):
self.invalid_fields += get_invalid_fields(
item['exists']['field'])
if self.has_wildcard(item):
self.invalid_fields += get_invalid_fields(
item['wildcard'].keys())
if self.has_bool(item):
self.invalid_fields += models.BoolQuery(item['bool'])\
.validate().get_invalid_fields()
if self.has_query(item):
self.invalid_fields += models.Query(item['query'])\
.validate().get_invalid_fields()
if self.has_query(self.metadata
): # Kibana 7 'query' at this level is a QueryString
self.invalid_fields += models.QueryString(self.metadata['query'])\
.validate().get_invalid_fields()
if self.has_panelsjson:
for item in self.source['panelsJSON']:
if self.has_columns(item):
self.invalid_fields += get_invalid_fields(item['columns'])
if self.has_embeddable_config(item):
self.invalid_fields += models.EmbeddableConfig(item['embeddableConfig'])\
.validate().get_invalid_fields()
return self
def convert(self):
if self.has_index(self.metadata):
self.metadata['index'] = Config.new_index_pattern
if self.has_references(self.data):
for ref in self.data['references']:
if ref['id'] == Config.old_index_pattern:
ref['id'] = Config.new_index_pattern
for item in self.metadata['filter']:
if self.has_meta(item):
models.Meta(item['meta']).convert()
if self.has_exists(item):
replace_value('field', item['exists'])
if self.has_wildcard(item):
replace_keys(item['wildcard'])
if self.has_bool(item):
models.BoolQuery(item['bool']).convert()
if self.has_query(item):
models.Query(item['query']).convert()
if self.has_query(self.metadata):
models.QueryString(self.metadata['query']).convert()
if self.has_panelsjson:
for item in self.source['panelsJSON']:
if self.has_columns(item):
replace_values(item['columns'])
if self.has_embeddable_config(item):
models.EmbeddableConfig(item['embeddableConfig']).convert()
return self
def validate(self):
for x in self.data.values():
if isinstance(x, list):
for item in x:
self.invalid_fields += models.Query(
item).validate().get_invalid_fields()
return self
def convert(self):
for filt in self.data:
if filt.get('input') is not None:
if filt['input'].get('query') is not None:
if isinstance(filt['input']['query'], str):
models.QueryString(filt['input']).convert()
else:
models.Query(filt['input']).convert()
if filt.get('bool') is not None:
models.BoolQuery(filt['bool']).convert()
return self
def input(request, user):
logging.info('Got the input from user')
if request.method == "GET":
form = SearchForm(request.GET)
if form.is_valid():
input = form.cleaned_data["i"]
if input.strip().lower() in ('random', 'example', 'random example'):
return redirect('/random')
g = SymPyGamma()
r = g.eval(input)
if not r:
r = [{
"title": "Input",
"input": input,
"output": "Can't handle the input."
}]
if user_exists_and_input_not_present(user, input):
logging.info('User exists and input not present')
with ndb_client.context():
query = models.Query(text=input, user_id=user.user_id())
logging.info('query: %s' % query)
query.put()
elif not input_exists(input):
logging.info('Input does not exists')
with ndb_client.context():
query = models.Query(text=input, user_id=None)
logging.info('query: %s' % query)
query.put()
# For some reason the |random tag always returns the same result
return ("result.html", {
"input": input,
"result": r,
"form": form,
"MEDIA_URL": settings.MEDIA_URL,
"promote_live": random.choice(LIVE_PROMOTION_MESSAGES)
})
def validate(self):
for filt in self.data:
if filt.get('input') is not None:
if filt['input'].get('query') is not None:
if isinstance(filt['input']['query'], str):
self.invalid_fields += models.QueryString(filt['input'])\
.validate().get_invalid_fields()
else:
self.invalid_fields += models.Query(filt['input'])\
.validate().get_invalid_fields()
if filt.get('bool') is not None:
self.invalid_fields += models.BoolQuery(filt['bool'])\
.validate().get_invalid_fields()
return self
def convert(self):
if self.has_index(self.metadata):
self.metadata['index'] = Config.new_index_pattern
if self.has_references(self.data):
for ref in self.data['references']:
if ref['id'] == Config.old_index_pattern:
ref['id'] = Config.new_index_pattern
for o in self.metadata['filter']:
if self.has_meta(o):
models.Meta(o['meta']).convert()
if self.has_exists(o):
replace_value('field', o['exists'])
if self.has_wildcard(o):
replace_keys(o['wildcard'])
if self.has_bool(o):
models.BoolQuery(o['bool']).convert()
if self.has_query(o):
models.Query(o['query']).convert()
if self.has_query(self.metadata):
models.Query(self.metadata['query']).convert()
if self.has_columns:
replace_values(self.source['columns'])
return self
def input(request, user):
if request.method == "GET":
form = SearchForm(request.GET)
if form.is_valid():
input = form.cleaned_data["i"]
if input.strip().lower() in ('random', 'example',
'random example'):
return redirect('/random')
g = SymPyGamma()
r = g.eval(input)
if not r:
r = [{
"title": "Input",
"input": input,
"output": "Can't handle the input."
}]
if (user and not models.Query.query(
models.Query.text == input, models.Query.user_id
== user.user_id()).get()):
query = models.Query(text=input, user_id=user.user_id())
query.put()
elif not models.Query.query(models.Query.text == input).get():
query = models.Query(text=input, user_id=None)
query.put()
# For some reason the |random tag always returns the same result
return ("result.html", {
"input": input,
"result": r,
"form": form,
"MEDIA_URL": settings.MEDIA_URL,
"promote_live": random.choice(LIVE_PROMOTION_MESSAGES)
})
def validate(self):
for o in self.metadata['filter']:
if self.has_meta(o):
self.invalid_fields += models.Meta(
o['meta']).validate().get_invalid_fields()
if self.has_exists(o):
self.invalid_fields += get_invalid_fields(o['exists']['field'])
if self.has_wildcard(o):
self.invalid_fields += get_invalid_fields(o['wildcard'].keys())
if self.has_bool(o):
self.invalid_fields += models.bool_query.BoolQuery(
o['bool']).validate().get_invalid_fields()
if self.has_query(o):
self.invalid_fields += models.query.Query(
o['query']).validate().get_invalid_fields()
if self.has_query(self.metadata):
self.invalid_fields += models.Query(
self.metadata['query']).validate().get_invalid_fields()
if self.has_columns:
self.invalid_fields += get_invalid_fields(self.source['columns'])
return self
def count_words(url, user_id):
query = models.Query(url=url,
queried_by=user_id,
queried_at=datetime.now())
try:
response = requests.get(url)
except requests.exceptions.ConnectionError:
count = None
else:
# Extract text contents
h = html2text.HTML2Text()
h.ignore_links = True
text_content = h.handle(
response.text) # ignore html tags, scripts and styles
text_content = ''.join(
e for e in text_content
if (e.isalnum() or e == ' ')) # remove special characters
count = len(text_content.split())
query.word_count = count
db.session.add(query)
db.session.commit()
def validate(self):
for item in self.metadata['filter']:
if self.has_meta(item):
self.invalid_fields += models.Meta(item['meta'])\
.validate().get_invalid_fields()
if self.has_exists(item):
self.invalid_fields += get_invalid_fields(item['exists']['field'])
if self.has_wildcard(item):
self.invalid_fields += get_invalid_fields(item['wildcard'].keys())
if self.has_bool(item):
self.invalid_fields += models.BoolQuery(item['bool'])\
.validate().get_invalid_fields()
if self.has_query(item):
self.invalid_fields += models.Query(item['query'])\
.validate().get_invalid_fields()
if self.has_query(self.metadata):
models.QueryString(self.metadata['query'])\
.validate().get_invalid_fields()
if self.has_visState:
if self.has_aggs(self.source['visState']):
for agg in self.source['visState']['aggs']:
self.invalid_fields += models.Agg(agg)\
.validate().get_invalid_fields()
return self
def convert(self):
for x in self.data.values():
if isinstance(x, list):
for item in x:
models.Query(item).convert()
return self
def get(self):
q = self.request.get('q').strip()
if not q:
self.redirect("/")
return
self.login()
clean_string = search.clean_query_string(q)
query_hash = common.get_query_hash(clean_string)
query_key = db.Key.from_path('Query', query_hash, parent=self.user_key)
existing_query = self.datastore_fetcher.get(query_key)
keyword_hashes = search.get_keyword_hashes(clean_string)
search_hashes = keyword_hashes[:config.MAX_SEARCH_KEYWORDS]
context = self.datastore_fetcher.get(
db.Key.from_path('UserContext', self.user_key.id_or_name()))
search_query = search.get_search_query(search_hashes)
cur = self.request.get('cursor')
if cur is not None:
search_query.with_cursor(cur)
result_keys = []
existing_chats = {}
users_status = {}
unread_chats = {}
cursor = None
res_keys = search_query.fetch(config.ITEMS_PER_PAGE)
if len(res_keys) >= config.ITEMS_PER_PAGE:
cursor = search_query.cursor()
for k in res_keys:
q_key = search.decode_query_index_key_name(k.name())
user_key = q_key.parent()
if user_key != self.user_key:
result_keys.append(q_key)
chat_id = common.get_chat_key_name(self.user_key, user_key)
existing_chats[user_key] = self.datastore_fetcher.get(
db.Key.from_path('User', self.user_key.id_or_name(),
'UserChat', chat_id))
users_status[user_key] = self.memcache_fetcher.get(
config.MEMCACHE_LAST_BEEN_ONLINE(user_key.id_or_name()))
unread_chats[user_key] = self.datastore_fetcher.get(
db.Key.from_path('User', self.user_key.id_or_name(),
'UnreadChat', chat_id))
results = self.datastore_fetcher.get(result_keys)
result_values = []
online_count = 1
for r in results:
user_key = r.parent_key()
status_class = "online" if users_status[user_key].get_result(
) is not None else "offline"
if status_class == 'online':
online_count += 1
v = {
'query': r.query_string,
'key': r.key(),
'username': models.User.get_username(user_key),
'status_class': status_class,
'context': common.htmlize_string(r.context),
}
if existing_chats[user_key].get_result() is not None:
v['existing_chat'] = existing_chats[user_key].name
if unread_chats[user_key].get_result() is not None:
v['existing_chat_unread'] = True
result_values.append(v)
if not result_values:
self.template_values['top_searches'] = get_top_searches()
if not cursor:
context_text = ""
if context:
context_text = context.context
keyword_hashes = list(
keyword_hashes + search.get_keyword_hashes(
search.clean_query_string(context_text)))[:config.
MAX_KEYWORDS]
query = existing_query.get_result()
if query is not None:
existing_index = db.Query(models.QueryIndex,
keys_only=True).filter(
'query =', query).get()
if existing_index is not None:
db.delete(existing_index)
query.query_string = q
query.context = context_text
query.date_time = self.now
else:
query = models.Query(key=query_key,
query_string=q,
context=context_text,
date_time=self.now)
index = models.QueryIndex(
key_name=search.encode_query_index_key_name(query_key),
query=query_key,
user=self.user_key,
keyword_hashes=keyword_hashes)
top_search = models.TopSearch(
key_name=query_key.name(),
query_string=q,
rating=float(len(search_hashes) * online_count))
db.put([query, index, top_search])
self.template_values["results"] = result_values
self.template_values["key"] = query.key()
self.template_values["query"] = q
self.template_values["cursor"] = cursor
self.render_page('SearchPage.html')