def parse_simple_link(s, suffix):
m = re.match(r"((?P<title>.*),\s*)?" + link_re_str(suffix),
s, re.IGNORECASE)
if m is None:
return
d = m.groupdict()
if not d["title"]:
d["title"] = fetch_title(d["link"])
else:
d["title"] = beautify(d["title"])
return d
def parse_msword_link(s, suffix):
m = re.match(r'HYPERLINK\s*"(?P<title>.*)"\s*' + link_re_str(suffix),
s, re.IGNORECASE)
if m is None:
return
d = m.groupdict()
if d["title"] == d["link"]:
d["title"] = fetch_title(d["link"])
else:
d["title"] = beautify(d["title"])
return d
def crawl(line):
result = {}
items = []
t1 = datetime.now()
# get domain name of the feed source
domain = p.search(line)
# generate identical filename for each source
file_dir = "./output/"
if domain:
file_dir += domain.group()[3:].replace('.', '_')
else:
m.update(line)
file_dir += m.hexdigest()
output_file = open(file_dir + ".json", "w+")
item_count = 0
try:
# request data from local server
response = urllib2.urlopen(service_url + '&url=' + line)
single_page_full_rss = response.read()
output_file.write(single_page_full_rss)
output_file.close()
# get main body of the result and leave the rest behind
json_rss = json.loads(single_page_full_rss)
content = json_rss['rss']['channel']['item']
if type(content) is list:
# extract restricted info fields from results of five-filters
for item in content:
if item['result'] == 'success':
newitem = {}
newitem['title'] = item['title']
newitem['datetime'] = item['pubDate']
newitem['content'] = beautifier.beautify(item['description'])
newitem['link'] = item['link']
items.append(newitem)
item_count += 1
else:
l.log("Url extraction failure: \t\t" + item['link'])
else:
# extraction failure
l.log("Feed extraction failure: \t\t" + line)
except Exception, e:
print e.message
pass
def feedback_confirm(request):
""" Webhook action to confirm feedback received from the user """
global training_feedback
words_to_highlight = []
response_text = "In the phrase '**{}**', the words: \n ".format(
training_feedback['original_intent'])
for entity, values in training_feedback['entities'].items():
response_text += ", and ".join(value for value in values.keys())
response_text += " \nmust be considered {}(s);".format(entity)
words_to_highlight = values.keys()
print('words', words_to_highlight)
response_text += " \nIs that right?"
response = make_card_response("Feedback confirmation", response_text,
"Can you confirm your feedback then?",
beautify(response_text, words_to_highlight))
return response
def get_answer(topic, keyword, options="", request_options=None): # pylint: disable=too-many-locals,too-many-branches,too-many-statements
"""
Find cheat sheet for the topic.
If `keyword` is None or rempty, return the whole answer.
Otherwise cut the paragraphs containing keywords.
Args:
topic (str): the name of the topic of the cheat sheet
keyword (str): the name of the keywords to search in the cheat sheets
Returns:
string: the cheat sheet
"""
def _join_paragraphs(paragraphs):
answer = "\n".join(paragraphs)
return answer
def _split_paragraphs(text):
answer = []
paragraph = ""
for line in text.splitlines():
if line == "":
answer.append(paragraph)
paragraph = ""
else:
paragraph += line+"\n"
answer.append(paragraph)
return answer
def _paragraph_contains(paragraph, keyword, insensitive=False, word_boundaries=True):
"""
Check if `paragraph` contains `keyword`.
Several keywords can be joined together using ~
For example: ~ssh~passphrase
"""
answer = True
if '~' in keyword:
keywords = keyword.split('~')
else:
keywords = [keyword]
for kwrd in keywords:
regex = re.escape(kwrd)
if not word_boundaries:
regex = r"\b%s\b" % kwrd
if insensitive:
answer = answer and bool(re.search(regex, paragraph, re.IGNORECASE))
else:
answer = answer and bool(re.search(regex, paragraph))
return answer
def _rewrite_aliases(word):
if word == ':bash.completion':
return ':bash_completion'
return word
def _rewrite_section_name(query):
"""
"""
if '/' not in query:
return query
section_name, rest = query.split('/', 1)
section_name = LANGUAGE_ALIAS.get(section_name, section_name)
return "%s/%s" % (section_name, rest)
def _rewrite_section_name_for_q(query):
"""
"""
if '/' not in query:
return query
section_name, rest = query.split('/', 1)
section_name = SO_NAME.get(section_name, section_name)
print("%s/%s" % (section_name, rest))
return "%s/%s" % (section_name, rest)
answer = None
needs_beautification = False
topic = _rewrite_aliases(topic)
topic = _rewrite_section_name(topic)
# this is pretty unoptimal
# so this part should be rewritten
# for the most queries we could say immediately
# what type the query has
start_time = time.time()
topic_type = get_topic_type(topic)
print((time.time() - start_time)*1000)
# checking if the answer is in the cache
if topic != "":
# temporary hack for "questions":
# the topic name has to be prefixed with q:
# so we can later delete them from redis
# and we known that they need beautification
#if '/' in topic and '+' in topic:
if topic_type == 'question': #'/' in topic and '+' in topic:
topic = _rewrite_section_name_for_q(topic)
topic = "q:" + topic
needs_beautification = True
answer = REDIS.get(topic)
if answer:
answer = answer.decode('utf-8')
# if answer was not found in the cache
# try to find it in one of the repositories
if not answer:
#topic_type = get_topic_type(topic)
for topic_getter_type, topic_getter in TOPIC_GETTERS:
if topic_type == topic_getter_type:
answer = topic_getter(topic)
break
if not answer:
topic_type = "unknown"
answer = _get_unknown(topic)
# saving answers in the cache
if topic_type not in ["search", "internal", "unknown"]:
REDIS.set(topic, answer)
if needs_beautification:
filetype = 'bash'
if '/' in topic:
filetype = topic.split('/', 1)[0]
if filetype.startswith('q:'):
filetype = filetype[2:]
answer = beautifier.beautify(answer.encode('utf-8'), filetype, request_options)
if not keyword:
return answer
#
# shorten the answer, because keyword is specified
#
insensitive = 'i' in options
word_boundaries = 'b' in options
paragraphs = _split_paragraphs(answer)
paragraphs = [p for p in paragraphs
if _paragraph_contains(p, keyword,
insensitive=insensitive,
word_boundaries=word_boundaries)]
if paragraphs == []:
return ""
answer = _join_paragraphs(paragraphs)
return answer
def get_answer(topic, keyword, options="", request_options=None): # pylint: disable=too-many-locals,too-many-branches,too-many-statements
"""
Find cheat sheet for the topic.
If `keyword` is None or rempty, return the whole answer.
Otherwise cut the paragraphs containing keywords.
Args:
topic (str): the name of the topic of the cheat sheet
keyword (str): the name of the keywords to search in the cheat sheets
Returns:
string: the cheat sheet
"""
def _join_paragraphs(paragraphs):
answer = "\n".join(paragraphs)
return answer
def _split_paragraphs(text):
answer = []
paragraph = ""
for line in text.splitlines():
if line == "":
answer.append(paragraph)
paragraph = ""
else:
paragraph += line+"\n"
answer.append(paragraph)
return answer
def _paragraph_contains(paragraph, keyword, insensitive=False, word_boundaries=True):
"""
Check if `paragraph` contains `keyword`.
Several keywords can be joined together using ~
For example: ~ssh~passphrase
"""
answer = True
if '~' in keyword:
keywords = keyword.split('~')
else:
keywords = [keyword]
for kwrd in keywords:
regex = re.escape(kwrd)
if not word_boundaries:
regex = r"\b%s\b" % kwrd
if insensitive:
answer = answer and bool(re.search(regex, paragraph, re.IGNORECASE))
else:
answer = answer and bool(re.search(regex, paragraph))
return answer
def _rewrite_aliases(word):
if word == ':bash.completion':
return ':bash_completion'
return word
def _rewrite_section_name(query):
"""
"""
if '/' not in query:
return query
section_name, rest = query.split('/', 1)
section_name = LANGUAGE_ALIAS.get(section_name, section_name)
return "%s/%s" % (section_name, rest)
def _rewrite_section_name_for_q(query):
"""
"""
if '/' not in query:
return query
section_name, rest = query.split('/', 1)
section_name = SO_NAME.get(section_name, section_name)
print("%s/%s" % (section_name, rest))
return "%s/%s" % (section_name, rest)
answer = None
needs_beautification = False
topic = _rewrite_aliases(topic)
topic = _rewrite_section_name(topic)
# this is pretty unoptimal
# so this part should be rewritten
# for the most queries we could say immediately
# what type the query has
start_time = time.time()
topic_type = get_topic_type(topic)
print((time.time() - start_time)*1000)
# checking if the answer is in the cache
if topic != "":
# temporary hack for "questions":
# the topic name has to be prefixed with q:
# so we can later delete them from redis
# and we known that they need beautification
#if '/' in topic and '+' in topic:
if topic_type == 'question': #'/' in topic and '+' in topic:
topic = _rewrite_section_name_for_q(topic)
topic = "q:" + topic
needs_beautification = True
answer = REDIS.get(topic)
if answer:
answer = answer.decode('utf-8')
# if answer was not found in the cache
# try to find it in one of the repositories
if not answer:
#topic_type = get_topic_type(topic)
for topic_getter_type, topic_getter in TOPIC_GETTERS:
if topic_type == topic_getter_type:
answer = topic_getter(topic)
break
if not answer:
topic_type = "unknown"
answer = _get_unknown(topic)
# saving answers in the cache
if topic_type not in ["search", "internal", "unknown"]:
REDIS.set(topic, answer)
if needs_beautification:
filetype = 'bash'
if '/' in topic:
filetype = topic.split('/', 1)[0]
if filetype.startswith('q:'):
filetype = filetype[2:]
answer = beautifier.beautify(answer.encode('utf-8'), filetype, request_options)
if not keyword:
return answer
#
# shorten the answer, because keyword is specified
#
insensitive = 'i' in options
word_boundaries = 'b' in options
paragraphs = _split_paragraphs(answer)
paragraphs = [p for p in paragraphs
if _paragraph_contains(p, keyword,
insensitive=insensitive,
word_boundaries=word_boundaries)]
if paragraphs == []:
return ""
answer = _join_paragraphs(paragraphs)
return answer
import beautifier
string = "'test_safd' At nearly 7,000 words, you probably don\u2019t want to try</p>sadfsadf"
f = open("tmp.txt", "w+")
print string.decode('unicode-escape')
r = beautifier.beautify(string)
f.write(r)
print r
def read_title(f):
par = read_paragraph(f)
if par is not None:
return beautify(par.strip())
def extract_tags(s, tags):
l = re.split(r"[()]", s)
if len(l) < 2:
return s
tags += [beautify(p.strip()) for p in l[1].split(',')];
return l[0]