def research_keywords(something_unknown, model, websites_to_scan=10, keyword_count=25, attempts=0, google_api_key=GOOGLE_API_KEY):
# searches for something unknown on Google to find related websites and returns a ranked list of keywords from across all sites
maximum_number_of_google_search_attempts = 3
if attempts < maximum_number_of_google_search_attempts:
all_keywords = Manager().dict()
engine = Google(license=google_api_key, throttle=1.0, language="en")
try:
processes = []
for page in range(int(websites_to_scan/10)):
for website in engine.search(something_unknown, start=page+1, count=10, type=SEARCH, cached=False):
web_mining_process = Process(target=extract_keywords, args=(website.url, model, all_keywords))
processes.append(web_mining_process)
web_mining_process.start()
[process.join() for process in processes]
except HTTP403Forbidden:
print "\nToday's maximum number of free searches from Google shared by this API across all words2map users has expired.\nPlease get your own key at https://code.google.com/apis/console\n\nFrom that site, simply:\n1. In the API Manager Overview, find \"Custom Search API\" and enable it\n2. Copy your new API key from \"Credentials\"\n3. Paste it in words2map.py in the global variable \"GOOGLE_API_KEY\"\n"
sys.exit(1)
except (URLError, URLTimeout, HTTPError, SSLError):
print "\nUnable to reach Google Search for {}, trying one more time".format(something_unknown)
return research_keywords(something_unknown, model, attempts=attempts+1)
all_keywords = sorted(all_keywords.items(), key=itemgetter(1), reverse=True)
print "\nKeywords about {} to combine vectors for:".format(something_unknown)
top_keywords = []
for i in range(keyword_count):
try:
keyword, score = all_keywords[i]
top_keywords.append(all_keywords[i])
print "{} {}".format(round(score, 3), unidecode(keyword).replace("_", " "))
except IndexError:
break
return top_keywords
else:
print "After a few tries, it seems that Google is not returning results for us. If you haven't done so already, please try adding your own API key at https://code.google.com/apis/console\n\nFrom that site, simply:\n1. In the API Manager Overview, find \"Custom Search API\" and enable it\n2. Copy your new API key from \"Credentials\"\n3. Paste it in words2map.py in the global variable \"GOOGLE_API_KEY\"\n"
sys.exit(1)
def research_keywords(something_unknown, model, websites_to_scan=10, keyword_count=25, attempts=0, google_api_key=GOOGLE_API_KEY):
# searches for something unknown on Google to find related websites and returns a ranked list of keywords from across all sites
maximum_number_of_google_search_attempts = 3
if attempts < maximum_number_of_google_search_attempts:
all_keywords = Manager().dict()
engine = Google(license=google_api_key, throttle=1.0, language="en")
try:
processes = []
for page in range(int(websites_to_scan/10)):
for website in engine.search(something_unknown, start=page+1, count=10, type=SEARCH, cached=False):
web_mining_process = Process(target=extract_keywords, args=(website.url, model, all_keywords))
processes.append(web_mining_process)
web_mining_process.start()
[process.join() for process in processes]
except HTTP403Forbidden:
print "\nToday's maximum number of free searches from Google shared by this API across all words2map users has expired.\nPlease get your own key at https://code.google.com/apis/console\n\nFrom that site, simply:\n1. In the API Manager Overview, find \"Custom Search API\" and enable it\n2. Copy your new API key from \"Credentials\"\n3. Paste it in words2map.py in the global variable \"GOOGLE_API_KEY\"\n"
sys.exit(1)
except (URLError, URLTimeout, HTTPError, SSLError):
print "\nUnable to reach Google Search for {}, trying one more time".format(something_unknown)
return research_keywords(something_unknown, model, attempts=attempts+1)
all_keywords = sorted(all_keywords.items(), key=itemgetter(1), reverse=True)
print "\nKeywords about {} to combine vectors for:".format(something_unknown)
top_keywords = []
for i in range(keyword_count):
try:
keyword, score = all_keywords[i]
top_keywords.append(all_keywords[i])
print "{} {}".format(round(score, 3), unidecode(keyword).replace("_", " "))
except IndexError:
break
return top_keywords
else:
print "After a few tries, it seems that Google is not returning results for us. If you haven't done so already, please try adding your own API key at https://code.google.com/apis/console\n\nFrom that site, simply:\n1. In the API Manager Overview, find \"Custom Search API\" and enable it\n2. Copy your new API key from \"Credentials\"\n3. Paste it in words2map.py in the global variable \"GOOGLE_API_KEY\"\n"
sys.exit(1)
def google_search(targetword, itemlist,targetpath):
resultnum=0
engine = Google(license=None)
file = codecs.open(targetpath,'a','utf-8')
outtext= ''
patt = ur'\W+'
for item in itemlist:
for i in range(1,5):
for result in engine.search(item, type=SEARCH, start=i):
url = URL(result.url)
text = url.download(unicode=True)
text = plaintext(text)
text = correctPersianString(text)
text = text.replace('\n',' ')
lines = text.split('.')
for line in lines:
if targetword in line:
match = re.findall(patt,line)
output = ' '.join(match)
for item in punclist:
if item in line:
line = line.replace(item,' ')
print output
file.write(output)
file.write('\n')
print str(resultnum)+" found in web"
file.close()
def google_search(match,targetfile):
engine = Google(license=None)
for i in range(1,10):
for result in engine.search(match, type=SEARCH, start=i):
print plaintext(result.description)
targetfile.write(plaintext(result.description))
targetfile.write('\n')
def google_search(match, targetfile):
engine = Google(license=None)
for i in range(1, 10):
for result in engine.search(match, type=SEARCH, start=i):
print plaintext(result.description)
targetfile.write(plaintext(result.description))
targetfile.write('\n')
def youtubeLinkGoogle(author, album): if author == 'unknown': author = '' name = author + ' ' + album key='AIzaSyC0go1dbuPHJhGYnONXvBc8z9Q8GkBSosw' engine = Google(license=key, throttle=0) results = engine.search(name + " youtube") return results[0].url
def run(self,q):# q is the query
engine = Google(license=None, language="en")
# Google is very fast but you can only get up to 100 (10x10) results per query.
urlsGoogle=[]
for i in range(1,11):
for result in engine.search(q, start=i, count=10, type=SEARCH, cached=True):
urlsGoogle.append(result.url)
return urlsGoogle
def get_urls(self, q = "", n = 1, limit = 1):
url = []
reload(sys)
sys.setdefaultencoding(GOOGLE_API_ENCODING)
engine_google = Google(license=GOOGLE_API_KEY, language=GOOGLE_API_LANG)
for i in range(1, (n + 1)):
for result in engine_google.search(q, start=i, count=10, type=SEARCH, cached=False):
url.append(result.url)
return url[:limit]
def __init__(self, query=''):
engine = Google(license=None, throttle=0.5, language=None)
c = 0
for i in range(1, 5):
results = engine.search(query, start=i)
for result in results:
c += 1
print c
print result.url + ':' + result.title
print repr(plaintext(result.text))
print ""
def __init__(self, query=''):
engine = Google(license=None, throttle=0.5, language=None)
c=0
for i in range(1,5):
results = engine.search(query, start=i)
for result in results:
c+=1
print c
print result.url+':'+result.title
print repr(plaintext(result.text))
print ""
def gnews_polarity(topic):
engine = Google()
results = engine.search(topic, type=NEWS)
score = 0
#only 8 results without using paging/cursor
for result in results:
content = heuristic_scrape(urllib.unquote(result.url))
if content:
polarity = sentiment(content)
score = score + polarity
else:
results.remove(result)
return score / float(len(results)) #avg sentiment
def gnews_polarity(topic):
engine = Google()
results = engine.search(topic, type=NEWS)
score = 0
#only 8 results without using paging/cursor
for result in results:
content = heuristic_scrape(urllib.unquote(result.url))
if content:
polarity = sentiment(content)
score = score + polarity
else:
results.remove(result)
return score / float(len(results)) #avg sentiment
def generar_consulta_google(q):
url = []
reload(sys)
sys.setdefaultencoding('utf8')
# engine_google = Google(license="AIzaSyCvvHb8SYgHvS5gEIQabxuJ0Kl0sYdHl9U", language="en")
engine_google = Google(license="AIzaSyCKlCEJ41mE_6gqTN2AI9J4iSB-2L55zR0", language="en")
for consulta in q:
for i in range(1, 2):
for result in engine_google.search(consulta, start=i, count=10, type=SEARCH, cached=False):
titulo = strip_accents(result.title)
url.append(result.url)
return url
def learn(concept):
""" Returns a list of properties for the given concept,
collected from a "I think X is Y".
"""
q = 'I think %s is *' % concept
p = []
g = Google(language='en', license=None)
for i in range(10):
for result in g.search(q, start=i, cached=True):
m = plaintext(result.description)
m = search(q, m) # Use * as a wildcard.
if m:
p.append(m[0][-1].string)
return [w for w in p if w in PROPERTIES] # only handles known properties...
def get_info(search_query):
if isinstance(search_query, str):
search_query = str(search_query)
else:
return { "Error": "Pass a string, from mine.py [7]", "Result": [None] }
result = []
engineGoogle = Google(license=None, throttle=0.5, language=None)
engineBing = Bing(license=None, throttle=0.5, language=None)
engineTwitter = Twitter(license=None, throttle=0.5, language=None)
engineFacebook = Facebook(license=None, throttle=1.0, language='en')
engineWikipedia = Wikipedia(license=None, throttle=5.0, language=None)
engineFlickr = Flickr(license=None, throttle=5.0, language=None)
engineArray = [engineGoogle, engineBing, engineTwitter, engineFacebook, engineWikipedia, engineFlickr]
engineArray = [engineGoogle, engineTwitter]
'''
for i in range(1,2):
# result = result + ([repr(plaintext(para.text)) for para in engine[0].search(search_query, type=SEARCH, start=i, count=5)])
[result.append([result.append(repr(plaintext(para.text))) for para in engine.search(search_query, type=SEARCH, start=i, count=5)]) for engine in engineArray]
# print repr(plaintext(para.text))
# print repr(plaintext(para.url)) + '\n\n'
# result.append(repr(plaintext(para.text)))
'''
# Google
for i in range(1, 5):
result = result + ([para.text for para in engineGoogle.search(search_query, type=SEARCH, start=i, count=10)])
for i in range(1, 5):
result = result + ([para.text for para in engineTwitter.search(search_query, type=SEARCH, start=i, count=10)])
'''
# for i in range(1,2):
# result = result + ([repr(plaintext(para.text)) for para in engineBing.search(search_query, type=SEARCH, start=i, count=5)])
for i in range(1,2):
result = result + ([repr(plaintext(para.text)) for para in engineTwitter.search(search_query, type=SEARCH, start=i, count=10)])
# for i in range(1,2):
# result = result + ([repr(plaintext(para.text)) for para in engineFacebook.search(search_query, type=SEARCH, start=i, count=5)])
# for i in range(1,2):
# result = result + ([repr(plaintext(para.text)) for para in engineWikipedia.search(search_query, type=SEARCH, start=i, count=5)])
# for i in range(1,2):
# result = result + ([repr(plaintext(para.text)) for para in engineFlickr.search(search_query, type=SEARCH, start=i, count=5)])
'''
return { "Error": None, "Result": result }
# return { "Error": None, "Result": ['Hello World', 'Bye Bye Tommy'] }
def learn(concept):
q = 'I think %s is *' % concept
p = []
g = Google(language='en')
for i in range(10):
for result in g.search(q, start=i, cached=True):
m = plaintext(result.description)
m = search(q, m) # use * as wildcard
if m:
p.append(m[0][-1].string)
return [w for w in p if w in PROPERTIES]
def patternSearch(self, n=12, m=50):
logger.info(u"patternSearch")
proxyList = list()
proxyList.append(u"3128")
proxyList.append(u"206.217.138.154")
logger.info(u"proxyList - %s" % proxyList)
engine = Google(license=None, throttle=0.5, language=None)
# engine = Bing(license=None, throttle=0.5, language=None)
for i in range(n):
logger.info(u"Search %d" % i)
results = engine.search(self.searchTerm, start=i+1, count=m, cached=False, proxy=proxyList)
for r in results:
logger.debug(u"Result=%s" % r.text)
url = r.url
logger.debug(u"URL=%s" % url)
# if url[-4:] == ".com":
# continue
s = r.text.lower()
s = plaintext(s)
s = parsetree(s)
# self.logSentences(s)
# Execute a Regular Expression Search
# p = r'(NN)+ (VB)+'
p = r'(NN)+'
q = search(p, s)
# logPOS(q)
# Iterate over all the words in the POS
logger.debug(u" q.Length=%d" % len(q))
logger.debug(u" q[]=%s" % q)
self.g, self.urlConcepts, self.wordConcepts = \
self.addNodes(self.g, q, url, self.urlConcepts, self.wordConcepts)
return self.urlConcepts, self.wordConcepts
def get_info(search_query):
if isinstance(search_query, str):
search_query = str(search_query)
else:
return {"Error": "Pass a string, from mine.py [7]"}
google = [{'text': '', 'url': '', 'title': ''}]
twitter = [{'text': '', 'url': '', 'title': ''}]
engineGoogle = Google(license=None, throttle=0.5, language=None)
# engineBing = Bing(license=None, throttle=0.5, language=None)
engineTwitter = Twitter(license=None, throttle=0.5, language=None)
# engineFacebook = Facebook(license=None, throttle=1.0, language='en')
# engineWikipedia = Wikipedia(license=None, throttle=5.0, language=None)
# engineFlickr = Flickr(license=None, throttle=5.0, language=None)
# engineArray = [engineGoogle, engineBing, engineTwitter, engineFacebook, engineWikipedia, engineFlickr]
engineArray = [engineGoogle, engineTwitter]
# Google
for i in range(1, 2):
for para in engineGoogle.search(search_query,
type=SEARCH,
start=i,
count=5):
google.append({
'text': para.text,
'url': para.url,
'title': para.title
})
#resultGoogle = resultGoogle + ([para.text for para in engineGoogle.search(search_query, type=SEARCH, start=i, count=10)])
# Twitter
for i in range(1, 2):
for para in engineTwitter.search(search_query,
type=SEARCH,
start=i,
count=5):
twitter.append({
'text': para.text,
'url': para.url,
'title': para.title
})
#resultTwitter = resultTwitter + ([para.text for para in engineTwitter.search(search_query, type=SEARCH, start=i, count=10)])
# print 'From data_mine.py --> google: ', google, ', twitter: ', twitter
return {"Error": None, "Google": google, "Twitter": twitter}
def googlecorpsearch(word,concfilter = '', extraquery='',license=None, start=1, count=8):
"""Searches the web for sentences containing a certain keyword, and possibly a co-occurence word. Generator yielding (leftcontext,word,rightcontext,url) tuples.
First queries Google, and then retrieves the pages of the top search results.
Uses 'pattern' (CLiPS, Antwerpen University)
"""
if not concfilter:
query = 'allintext: ' + word
else:
query = 'allintext: "' + word + ' * ' + concfilter + '" OR "' + concfilter + ' * ' + word + '"'
if extraquery:
query += ' ' + extraquery
engine = Google(license=license)
processed = {}
for result in engine.search(query, start=start,count=count):
if not result.url in processed:
processed[result.url] = True
try:
content = plaintext(result.download())
except:
continue
begin = 0
wordindex = None
wordlength = 0
concindex = None
for i in range(1,len(content)):
if content[i] == '.' or content[i] == '?' or content[i] == '!' or content[i] == '\n':
if wordindex >= begin and ((concfilter and concindex >= begin) or (not concfilter)):
if len(content[begin:wordindex].strip()) > 5 or len(content[wordindex+wordlength:i+1].strip()) > 5:
yield (content[begin:wordindex].strip(), content[wordindex:wordindex+wordlength].strip(), content[wordindex+wordlength:i+1], result.url)
wordindex = concindex = None
begin = i + 1
if len(word)+i <= len(content) and content[i:i+len(word)].lower() == word.lower():
wordindex = i
wordlength = len(word)
for j in range(len(word),len(content)):
if i+j < len(content) and (content[i+j] == ' ' or content[i+j] == '?' or content[i+j] == '!' or content[i+j] == '\n'):
wordlength = j
break
if concfilter and content[i:len(concfilter)].lower() == concfilter.lower():
concindex = i
def generar_consulta_google(q):
url = []
reload(sys)
sys.setdefaultencoding('utf8')
# engine_google = Google(license="AIzaSyCvvHb8SYgHvS5gEIQabxuJ0Kl0sYdHl9U", language="en")
engine_google = Google(license="AIzaSyCKlCEJ41mE_6gqTN2AI9J4iSB-2L55zR0",
language="en")
for consulta in q:
for i in range(1, 2):
for result in engine_google.search(consulta,
start=i,
count=10,
type=SEARCH,
cached=False):
titulo = strip_accents(result.title)
url.append(result.url)
return url
def main():
activites = ["Summary", "Named Entity Recognition", "Search", "Keywords"]
choice = st.sidebar.selectbox("Select Activity", activites)
if choice == "Summary":
html_temp = """
<div style="background-color:#16A085;"><p style="color:white;font-size:60px;">Text Summarizer</p></div>
"""
components.html(html_temp)
text = st.text_area("Input Text For Summary", height=300)
if st.button("summarize"):
st.success(summary(text))
text_range = st.sidebar.slider("Summarize words Range", 25, 500)
# Named Entity Recognition
elif choice == "Named Entity Recognition":
html_temp1 = """
<div style="background-color:#16A085;"><p style="color:white;font-size:60px;">Text Tokenizer</p></div>
"""
components.html(html_temp1)
row_data = st.text_area("write Text For Tokenizer")
docx = nlp(row_data)
if st.button("Tokenizer"):
spacy_streamlit.visualize_tokens(
docx, attrs=['text', 'pos_', 'dep_', 'ent_type_'])
if st.button("NER"):
spacy_streamlit.visualize_ner(docx,
labels=nlp.get_pipe('ner').labels)
if st.button("Text Relationship"):
spacy_streamlit.visualize_parser(docx)
#Search Bar
elif choice == "Search":
html_temp4 = """
<div style="background-color:#16A085;"><p style="color:white;font-size:60px;,text-align:center;">Search Bar</p></div>
"""
components.html(html_temp4)
row_text = st.text_input("Search Anything")
google = Google(license=None)
if st.button("search"):
for search_result in google.search(row_text):
st.write(search_result.text)
st.warning(search_result.url)
# The web module has a SearchEngine class with a search() method
# that yields a list of Result objects.
# Each Result has url, title, description, language, author and date and properties.
# Subclasses of SearchEngine include:
# Google, Yahoo, Bing, Twitter, Wikipedia, Flickr.
# This example retrieves results from Google based on a given query.
# The Google search engine can handle SEARCH, IMAGE, NEWS and BLOG type searches.
# Google's "Custom Search API" is a paid service.
# The web module uses a test account with a 100 free queries per day, shared with all users.
# If the limit is exceeded, SearchEngineLimitError is raised.
# You can obtain your own license key at: https://code.google.com/apis/console/
# Activate "Custom Search API" under "Services" and get the key under "API Access".
# Then use Google(license=[YOUR_KEY]).search().
# This will give you 100 personal free queries, or 5$ per 1000 queries.
engine = Google(license=None, language="en")
# Veale & Hao's method for finding simile using Google's wildcard (*) support.
# http://afflatus.ucd.ie/Papers/LearningFigurative_CogSci07.pdf)
# This will match results such as "as light as a feather", "as cute as a cupcake", etc.
q = "as * as a *"
# Google is very fast but you can only get up to 100 (10x10) results per query.
for i in range(1,2):
for result in engine.search(q, start=i, count=10, type=SEARCH):
print plaintext(result.description) # plaintext() removes HTML formatting.
print result.url
print result.date
print
def gnewshits(topic):
engine = Google()
results = engine.search(topic, type=NEWS)
return results.total
def calc_main():
st.title("Nimbus Words")
st.sidebar.header("Input Options")
expander_bar = st.beta_expander("How To Use This App")
expander_bar.markdown("""
**Use the Dropdown Box located within the sidebar on the left to choose 1 of the 6 AI text editing features offered by Nimbus Words.**
1) **Summarizer:** Paste in text that will be summarized by our AI model. The first text box will do an automated summary of our program's recommended word count, and the second box beneath that will provide a summary of the exact word count you choose using the slider located within the sidebar.
2) **Tokenizer:** Paste in text that will be analyzed by our AI model. The **Tokenizer** button will provide a breakdown on each word within the phrase, for example 'Google' is an organization, or 'Jeff Bezos' is a proper noun. The **NER** button will display all named entities, for example 'Steve Jobs' is a person. The **Text Relationship** button will display a visual graph of the dependency each word has within a sentence or phrase.
3) **Synonyms:** Paste in text that will be analyzed by our AI model. The **Synonyms** button will provide you with synonyms to the inputted attribute. The **Definition** checkbox will provide definitions for the attribute. The **Example** checkbox will provide examples of the given attribute in a sentence.
4) **Translator:** Paste in text that will be translated by our AI model. The **Translate** button will translate the inputted text into one of the many languages that we have provided, and we will automatically detect which language the inputted text is written in.
5) **Search:** Paste in text that will be preprcoessed by our AI model. The **Search** button will do a filtered search for your input.
6) **Spell Correction:** Paste in text that will be spell-checked by our AI model. The **Correct** button will offer a correct spelling for any grammatical error that are detected. The **Pluralize**, **Singularize**, **Comparative** and **Superlative** checkboxes do exactly as they say, and ouput those options for the input you provided.
""")
activites = [
"Summary", "Tokenizer", "Synonyms", "Translator", "Search",
"Spell Correction"
]
choice = st.sidebar.selectbox("Select Activity", activites)
if choice == "Summary":
st.title('AI Text Summarizer')
text = st.text_area("Input Text For Summary", height=300)
if st.button("Summarize"):
st.success(summary(text))
text_range = st.sidebar.slider("Summarize words Range", 25, 500)
text = st.text_area("Input Text For Summary", height=250)
if st.button("Summarize with Custom Word Count"):
st.warning(summarize(text, word_count=text_range))
# Tokenizer
elif choice == "Tokenizer":
st.title('Text Tokenizer')
row_data = st.text_area("write Text For Tokenizer")
docx = nlp(row_data)
if st.button("Tokenizer"):
spacy_streamlit.visualize_tokens(
docx, attrs=['text', 'pos_', 'dep_', 'ent_type_'])
if st.button("NER"):
spacy_streamlit.visualize_ner(docx,
labels=nlp.get_pipe('ner').labels)
if st.button("Text Relationship"):
spacy_streamlit.visualize_parser(docx)
# synonyms
elif choice == "Synonyms":
st.title('Synonym Generator')
text = st.text_area("Enter Text")
if st.button("Synonyms"):
for syn in wordnet.synsets(text):
for i in syn.lemmas():
st.success(i.name())
if st.checkbox("Definition"):
for syn in wordnet.synsets(text):
st.warning(syn.definition())
if st.checkbox("Example"):
for syn in wordnet.synsets(text):
st.success(syn.examples())
# Translator
elif choice == "Translator":
st.title('Speech Tranlation')
row_text = st.text_area("Enter Your Text For Translation", height=300)
translation_text = TextBlob(row_text)
list1 = ["en", "ta", "pa", "gu", "hi", "ur", "kn", "bn", "te"]
a = st.selectbox("select", list1)
if st.button("search"):
#input1 = TextBlob("Simple is better than complex")
st.success(translation_text.translate(to=a))
#Search Bar
elif choice == "Search":
st.title('Web Search')
row_text = st.text_input("Search Anything")
google = Google(license=None)
if st.button("search"):
for search_result in google.search(row_text):
st.write(search_result.text)
st.warning(search_result.url)
elif choice == "Spell Correction":
st.title('AI Spell Correction')
text_data = st.text_area("Enter Text Here")
a = TextBlob(text_data)
if st.button("Correct"):
st.success(a.correct())
st.title('Pluralize & Singularize')
text_data1 = st.text_input("Enter a word For pluralize / singularize")
if st.checkbox("Pluralize"):
st.warning(pluralize(text_data1))
if st.checkbox("Singularize"):
st.warning(singularize(text_data1))
st.title('Compartitive & Superlative')
text2 = st.text_input("Enter Text For comparative & superlative")
if st.checkbox("Comparative"):
st.success(comparative(text2))
if st.checkbox("Superlative"):
st.success(superlative(text2))
cnt = Counter()
for i in resultString.split():
try:
content = i
content = u" ".join(content.replace(u"\xa0", u" ").strip().split())
except UnicodeDecodeError as e:
print e, i
if content in schlagList.elements():
schlagList[content] += 1
cnt[content] += 1
return (resultDate, resultUrl, cnt) # why not just write to file?
engine = Google(license=None, language='de') #'0001401664590345725200:dx8-iwqnvyw')
for result in engine.search('Energiewende'):
articleDict = [buildDict(unicode(result.text), result.url, result.date)]
with codecs.open("google.csv", "a", "utf-8") as f:
for l,m in sorted(schlagList.most_common(20)):
f.write(l)
f.write(",")
f.write(str(m).encode("utf-8"))
f.write("\n")
print schlagList.most_common(20)
'''
with codecs.open("google2.csv", "a", "utf-8") as f:
for x,y,z in articleDict:
from pattern.web import Google, plaintext
import re
words = ["jordan", "germany"]
filename = "../data/corpus-addition"
f = open(filename, "w+")
engine = Google(license=None, throttle=0.5, language=None) # Using Google since I had some issues with Bing.
for word in words:
print "Working on '%s' ..." % word
for result in engine.search(word):
lines = plaintext(result.download()).split("\n")
lines = [re.sub("[^\\w]", " ", line) for line in lines]
lines = [re.sub("\\s+", " ", line).strip() for line in lines]
lines = [line if line.count(" ") >= 4 else "" for line in lines]
text = re.sub("\\n+", "\n", "\n".join(lines))
f.write(text + "\n")
f.close()
class SourceChecker(object):
def __init__(self, text, language, max_queries=5, span=5, threshold=.8):
self.max_queries = max_queries
self.span = span
self.threshold = threshold
self.text = text
self.language = language
self.cat_dict = defaultdict(list)
key = ''
self.engine = Google(license=key, throttle=0.5, language=None)
self.cat_weightage_dict = {
'imposter site': 0.2,
'fake news': 0.0,
'parody site': 0.0,
'some fake stories': 0.0,
'conspiracy': 0.4,
'fake': 0.0,
'rumor': 0.2,
'unreliable': 0.3,
'reliable': 0.9,
'bias': 0.7,
'clickbait': 0.3,
'satire': 0.0,
'junksci': 0.3,
'political': 0.8,
'hate': 0.3,
'blog': 0.5,
'satirical': 0.1,
'unrealiable': 0.3,
'questionable': 0.4,
'least_biased': 1.0,
'pseudoscience': 0.5,
'right_center': 0.8,
'pro_science': 0.8,
'left_center': 0.8,
'right': 0.8,
'left': 0.8,
'biased': 0.8,
'state': 0.5
}
def get_queries(self):
"""Function to extract search queries from the text:
breaks text into ngrams, filters ngrams that consist mostly of stopwords or named entities,
selects an evenly spaced sample of the remaining ngrams"""
text = self.text
beg_quotes = re.findall(r'\"\S', text)
for each in beg_quotes:
text = text.replace(each, 'BEGQ' + each[-1])
end_quotes = re.findall(r'\S\"', text)
for each in end_quotes:
text = text.replace(each, each[0] + 'ENDQ')
text = re.sub('(ENDQ)+', 'ENDQ', text)
text = re.sub('(BEGQ)+', 'BEGQ', text)
text = text.replace('--', 'DOUBLEDASH')
all_ngrams = ngrams(text, n=self.span, punctuation="", continuous=True)
if self.language in stopwords.fileids():
stop_words = stopwords.words(self.language)
else:
stop_words = []
queries = []
queries.append(self.text)
for ngram in all_ngrams:
num_stop = len([w for w in ngram if w in stop_words])
stop_score = float(num_stop) / len(ngram)
if self.language == 'english':
chunked = ne_chunk(pos_tag(ngram))
named_entities = [[w for w, t in elt] for elt in chunked
if isinstance(elt, nltk.Tree)]
num_ent = sum([len(ent_list) for ent_list in named_entities])
ent_score = float(num_ent) / len(ngram)
else:
ent_score = 0
if stop_score < self.threshold and ent_score < self.threshold:
r_string = self.reconstruct_ngram(ngram)
if r_string in self.text:
queries.append(r_string)
reduction = len(queries) / self.max_queries
if reduction == 0:
reduction = 1
return queries[0::reduction]
def reconstruct_ngram(self, ngram):
"""Function to reconstruct original substrings from the ngrams"""
punc_b = ['!', '?', '.', ',', ';', ':', '\'', ')', ']', '}']
punc_a = ['(', '[', '}', '$']
ngram = ' '.join(ngram)
for p in punc_b:
ngram = ngram.replace(' ' + p, p)
for p in punc_a:
ngram = ngram.replace(p + ' ', p)
ngram = re.sub('(^| )BEGQ', ' "', ngram)
ngram = re.sub('ENDQ($| )', '" ', ngram)
ngram = ngram.replace('DOUBLEDASH', '--')
return ngram
def load_domains(self):
"""loads domain information"""
sources_path = pd('data', 'source_data.csv')
domain_file = Datasheet.load(sources_path, headers=True)
for row in domain_file:
url = row[1]
if str(row[-1]).find("\""):
cats = row[2:-1]
else:
cats = row[2:]
self.cat_dict[url] = cats
def pairwise(self, t):
it = iter(t)
return izip(it, it)
def get_urls(self, queries):
"""runs search query through search API and collects returned domain information"""
domains = defaultdict(list)
for q in queries:
q = "\"" + q + "\""
results = self.engine.search(q)
for result in results:
url = result.url
domain = self.get_domain(url)
domains[domain].append(q)
return domains
def get_domain(self, full_url):
"""function to extract the domain name from the URL"""
clean_reg = re.compile(r'^((?:https?:\/\/)?(?:www\.)?).*?(\/.*)?$')
match = re.search(clean_reg, full_url)
beg, end = match.group(1), match.group(2)
domain = string.replace(full_url, beg, '')
domain = string.replace(domain, end, '')
return domain
def render_output(self, domains):
"""renders text output"""
output = defaultdict(list)
for d, v in domains.items():
d_cats = [
c for c in self.cat_dict[d]
if len(c) > 0 and len(c.split(' ')) < 3
]
overlap = float(len(v)) / self.max_queries
if overlap <= 0.2:
output['MINIMAL'].append((d, d_cats))
elif 0.2 < overlap < 0.6:
output['SOME'].append((d, d_cats))
elif overlap >= 0.6:
output['HIGH'].append((d, d_cats))
degrees = ['HIGH', 'SOME', 'MINIMAL']
score1 = {}
for deg in degrees:
score2 = 0
len2 = 0
if output[deg]:
for d, cats in sorted(output[deg]):
if cats:
score3 = 0.0
len2 += 1
len3 = len(cats)
for cat in cats:
score3 += self.cat_weightage_dict[cat]
score3 /= len3
score2 += score3
else:
continue
if len2 != 0:
score2 /= len2
score1[deg] = score2
print 'score is'
cred = (0.5 * score1['HIGH'] + 0.3 * score1['SOME'] +
0.2 * score1['MINIMAL'])
print 'credibility score is '
print cred
print '\n'
for deg in degrees:
if output[deg]:
print '%s OVERLAP: ' % deg
for d, cats in sorted(output[deg]):
if cats:
print d + ': ' + ','.join(cats)
else:
print d
print '\n'
if cred == None:
cred = 0
if cred <= 0.5:
output['RESULT'].append(('false', [int(cred * 100)]))
else:
output['RESULT'].append(('true', [int(cred * 100)]))
return output
def cleanup_text(self, inputData):
inputString = re.sub(r'[^\w\s]', "", inputData).strip().lower()
if re.match(r'^(hello|hi|hey)$', inputString):
return [False, "Hello. Please enter something useful!"]
elif re.match(
r'^(how\s+are\s+you(\s+doing)?|hows\s+it\s+going|hows\s+everything|how\s+are\s+things|hows\s+life)$',
inputString):
return [False, "Good. Please enter something useful!"]
elif re.match(
r'^(whats\s+up|whats\s+new|whats\s+going\s+on|s+up|whaz+up)$',
inputString):
return [False, "Nothing. Please enter something useful!"]
elif re.match(r'^good\s+(morning|afternoon|evening|night)$',
inputString):
return [
False,
re.findall(r'^(good\s+(morning|afternoon|evening|night))$',
inputString)[0][0].upper() +
"! Please enter something useful!"
]
elif len(inputString.split()) < 8:
return [
False, "Please make sure the text contains at least 8 words"
]
else:
return [True, inputData]
def getSensationalData(self, text):
model_path = '../SensationalismClassifier/trained_model.pkl'
clf = SensationalismClassifier(train_data=None,
model=model_path,
dump=False,
debug=False)
firstFive = ""
token = text.split()[:5]
for words in token:
firstFive = firstFive + words + " "
myList = [[firstFive, text]]
res = clf.classify(myList)
list = res[0]
# returns 1 if the input is categorized as sensattokenionalist, 0 if not.
return list[2]
def googlecorpsearch(word,
concfilter='',
extraquery='',
license=None,
start=1,
count=8):
"""Searches the web for sentences containing a certain keyword, and possibly a co-occurence word. Generator yielding (leftcontext,word,rightcontext,url) tuples.
First queries Google, and then retrieves the pages of the top search results.
Uses 'pattern' (CLiPS, Antwerpen University)
"""
if not concfilter:
query = 'allintext: ' + word
else:
query = 'allintext: "' + word + ' * ' + concfilter + '" OR "' + concfilter + ' * ' + word + '"'
if extraquery:
query += ' ' + extraquery
engine = Google(license=license)
processed = {}
for result in engine.search(query, start=start, count=count):
if not result.url in processed:
processed[result.url] = True
try:
content = plaintext(result.download())
except:
continue
begin = 0
wordindex = None
wordlength = 0
concindex = None
for i in range(1, len(content)):
if content[i] == '.' or content[i] == '?' or content[
i] == '!' or content[i] == '\n':
if wordindex >= begin and (
(concfilter and concindex >= begin) or
(not concfilter)):
if len(content[begin:wordindex].strip()) > 5 or len(
content[wordindex + wordlength:i +
1].strip()) > 5:
yield (content[begin:wordindex].strip(),
content[wordindex:wordindex +
wordlength].strip(),
content[wordindex + wordlength:i + 1],
result.url)
wordindex = concindex = None
begin = i + 1
if len(word) + i <= len(content) and content[i:i +
len(word)].lower(
) == word.lower():
wordindex = i
wordlength = len(word)
for j in range(len(word), len(content)):
if i + j < len(content) and (
content[i + j] == ' ' or content[i + j] == '?'
or content[i + j] == '!'
or content[i + j] == '\n'):
wordlength = j
break
if concfilter and content[i:len(concfilter)].lower(
) == concfilter.lower():
concindex = i
def google_search():
g = Google()
for r in g.search(query="Barcamp RD 2019", star=1, count=10):
print(r.title, r.text, r.url)
# The web module has a SearchEngine class with a search() method
# that yields a list of Result objects.
# Each Result has url, title, date, description, date and author properties.
# Subclasses of SearchEngine include:
# Google, Yahoo, Bing, Twitter, Wikipedia, Flickr.
# This example retrieves results from Google based on a given query.
# The Google search engine can handle SEARCH, IMAGE, NEWS and BLOG type searches.
# You should obtain your own license key at:
# http://code.google.com/apis/ajaxsearch/signup.html
# Otherwise you will be sharing the default key with all users of this module.
engine = Google(license=None)
q = "as * as a *"
# Veale & Hao's method for finding simile using Google's wildcard (*) support.
# http://afflatus.ucd.ie/Papers/LearningFigurative_CogSci07.pdf)
# This will match results such as
# - "as light as a feather",
# - "as cute as a cupcake",
# - "as mad as a hatter",
# etc.
# Google is very fast but you can only get up to 64 (8x8) results per query.
for i in range(1, 8):
for result in engine.search(q, start=i, type=SEARCH):
print plaintext(
result.description) # plaintext() removes HTML formatting.
print result.url
print
'Scale', 'Compass'
])
a
print(a)
#%%% Working with Numbers
#The Pattern library contains functions that can be used to convert numbers in the form of text strings into their numeric counterparts and vice versa. To convert from text to numeric representation the number function is used. Similarly to convert back from numbers to their corresponding text representation the numerals function is used. Look at the following script:
from pattern.en import number, numerals
print(number("one hundred and twenty two"))
print(numerals(256.390, round=2))
#%%%Data Mining
from pattern.web import Google
google = Google()
for results in google.search('Analytics India Magazine'):
print(results.url)
print(results.text)
for results in google.search('Gamification'):
print(results.url)
#twitter
from pattern.web import Twitter
twitter = Twitter()
for results in twitter.search('Analytics India Magazine'):
print(results.url)
print(results.text)
class FoF(object):
def __init__(self, text, language, max_queries=10, span=20, threshold=.8):
self.max_queries = max_queries
self.span = span
self.threshold = threshold
self.text = text
self.language = language
self.cat_dict = defaultdict(list)
key = 'AIzaSyDZzslNRAsgyiiBKx36S8rblRKungcypEA'
self.engine = Google(license=key, throttle=0.5, language=None)
self.urls = []
def reconstruct_ngram(self, ngram):
punc_b = ['!', '?', '.', ',', ';', ':', '\'', ')', ']', '}']
punc_a = ['(', '[', '}', '$']
ngram = ' '.join(ngram)
for p in punc_b:
ngram = ngram.replace(' ' + p, p)
for p in punc_a:
ngram = ngram.replace(p + ' ', p)
ngram = re.sub('(^| )BEGQ', ' "', ngram)
ngram = re.sub('ENDQ($| )', '" ', ngram)
ngram = ngram.replace('DOUBLEDASH', '--')
return ngram
def get_queries(self):
text = self.text
beg_quotes = re.findall(r'\"\S', text)
for each in beg_quotes:
text = text.replace(each, 'BEGQ' + each[-1])
end_quotes = re.findall(r'\S\"', text)
for each in end_quotes:
text = text.replace(each, each[0] + 'ENDQ')
text = re.sub('(ENDQ)+', 'ENDQ', text)
text = re.sub('(BEGQ)+', 'BEGQ', text)
text = text.replace('--', 'DOUBLEDASH')
all_ngrams = ngrams(text, n=self.span, punctuation="", continuous=True)
if self.language in stopwords.fileids():
stop_words = stopwords.words(self.language)
else:
stop_words = []
queries = []
for ngram in all_ngrams:
num_stop = len([w for w in ngram if w in stop_words])
stop_score = float(num_stop) / len(ngram)
if self.language == 'english':
chunked = ne_chunk(pos_tag(ngram))
named_entities = [[w for w, t in elt] for elt in chunked
if isinstance(elt, nltk.Tree)]
num_ent = sum([len(ent_list) for ent_list in named_entities])
ent_score = float(num_ent) / len(ngram)
else:
ent_score = 0
if stop_score < self.threshold and ent_score < self.threshold:
r_string = self.reconstruct_ngram(ngram)
if r_string in self.text:
queries.append(r_string)
reduction = len(queries) / self.max_queries
return queries[0::reduction]
def get_domain(self, full_url):
clean_reg = re.compile(r'^((?:https?:\/\/)?(?:www\.)?).*?(\/.*)?$')
match = re.search(clean_reg, full_url)
self.urls.append(full_url)
beg, end = match.group(1), match.group(2)
domain = string.replace(full_url, beg, '')
domain = string.replace(domain, end, '')
return domain
def get_urls(self, queries):
domains = defaultdict(list)
for q in queries:
q = "\"" + q + "\""
results = self.engine.search(q)
for result in results:
url = result.url
domain = self.get_domain(url)
domains[domain].append(q)
return domains
def load_domains(self):
sources_path = pd('data', 'data.csv')
domain_file = Datasheet.load(sources_path, headers=True)
for row in domain_file:
url = row[1]
cats = row[2:]
self.cat_dict[url] = cats
def render_output(self, domains):
data = {'SOME': [], 'HIGH': []}
for d, v in domains.items():
d_cats = [
c for c in self.cat_dict[d]
if len(c) > 0 and len(c.split(' ')) < 3
]
overlap = float(len(v)) / self.max_queries
if 0.4 < overlap < 0.6:
data['SOME'].append(d)
elif overlap >= 0.6:
data['HIGH'].append(d)
data['href'] = self.urls
json_data = json.dumps(data)
print json_data
sys.stdout.flush()
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Apr 2 02:17:22 2019
@author: abhijithneilabraham
"""
from pattern.web import Bing, SEARCH, plaintext, Google
from ulmfit import ULMFiT
engine = Google(license=key)
searched = []
search_key = 'സഞ്ജു സാംസൺ'
for result in engine.search(search_key, type=SEARCH, start=1):
print(repr(plaintext(result.text)))
searched.append(repr(plaintext(result.text)))
print(len(searched))
model = ULMFiT("news/")
for i in searched:
x = model.predict(i)
print(x['intent'])
rawurl_wsj=[]
rawurl_foxnews=[]
journal_names=['nytimes', 'cbsnews', 'wsj', 'foxnews']
rawurls=[rawurl_nytimes, rawurl_cbsnews, rawurl_wsj, rawurl_foxnews]
g=Google()
#get the New York Times url
for journal, raw_url_title in zip(journal_names, rawurls):
#in order to get 30 urls with the keyword, used for-loop
for i in range(1,4):
# search google results correspoding to the following keyword
for result in g.search('Donald Trump opinion site:'+journal+'.com', start=i):
print result.url # Better not to print all of these -- they're not informative to the person using your program
# append the urls to the rawurl_ array
# ^ Not a useful comment - you're just stating what the next line does.
# Better to say _why_ the next line does what it does
raw_url_title.append(result.url)
print raw_url_title # Again, better not to print all of these -- I'm not going to keep writing this, but it applies throughout your code
print len(raw_url_title)
# saves the keyword to the local file in order to reduce query
# we will use this file for analyzing later on
f=open('url_'+journal+'.txt', "w")
print >>f, raw_url_title
class Engine(object):
def __init__(self, provider, key=None):
if provider.lower() == "bing":
key = key or 'd6Mz4slIdgIxcKR4609FO+QKOFTEFFRB3i7j8VioPiE'
self._engine = Bing(license=key)
elif provider.lower() == "google":
key = key or 'AIzaSyCAADAKnnkmDwIlLk_Q1p6foqI_ZMrgzcg'
self._engine = Google(license=key)
else:
raise ValueError('Not a recognized provider.')
def fuzzySearch(self, result, query):
best, best_i = 0, None
for i in range(len(result) - len(query) + 1):
score = similarity(result[i:i+len(query)], query)
if best < score:
best = score
best_i = i
return result[best_i+len(query):] if best_i != None else ''
def getPatterns(self, query):
cleaned = query.strip('?')
p = [(cleaned, 3)]
t = parsetree(query)[0]
for chunk in t.chunks:
if chunk.pos == 'NP':
p.append((chunk.string, 2))
for w in cleaned.split():
p.append((w, 1))
return p
def getGrams(self, results):
grams = {}
for text, weight in results:
uni = set(ngrams(text, n=1))
bi = set(ngrams(text, n=2))
tri = set(ngrams(text, n=3))
for gram in uni:
grams[gram] = grams.get(gram, 0) + weight
for gram in bi:
grams[gram] = grams.get(gram, 0) + weight
for gram in tri:
grams[gram] = grams.get(gram, 0) + weight
return grams
def removeStopWords(self, grams, queries):
for gram in grams.keys():
inter = set([g.lower() for g in gram]) & set(wordlist.STOPWORDS)
if len(inter) > 1 or inter and len(gram) == 1:
grams.pop(gram)
return grams
def searchQueries(self, queries):
results = []
for q, w1, d in queries:
for r in self._engine.search(q, count=100):
results.append((re.sub(r'[!,.?]', '', plaintext(r.txt)), w1))
return results
def searchQueriesWithPatterns(self, queries):
# Faster, but still need to refine extraction patterns
results = []
for q, w1, d in queries:
print "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
print q
print "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
for r in self._engine.search(q, count=50):
# Each result is given a preliminary score based on the weight of the query
# that retrieved it and the pattern that was matched to it
for p, w2 in self.getPatterns(q):
if d == 'L':
m = re.search('(.*?)' + p + '.*\.', plaintext(r.txt), re.IGNORECASE)
else:
m = re.search(p + '(.*)', plaintext(r.txt), re.IGNORECASE)
if m:
print plaintext(r.txt)
print "-------------------------------------------------"
print p, "generated", m.group(1)
print "================================================="
results.append((m.group(1), w1 + w2))
break
return results
def searchAndGram(self, queries):
results = self.searchQueries(queries)
grams = self.getGrams(results)
grams = self.removeStopWords(grams, queries)
return grams
def calc_main():
st.write("Nimbus Words")
st.sidebar.header("Input Options")
activites = ["Summary", "Tokenizer","Synonyms","Translator","Search","Spell Correction"]
choice = st.sidebar.selectbox("Select Activity",activites)
if choice == "Summary":
st.title('AI Text Summarizer')
text = st.text_area("Input Text For Summary",height=300)
if st.button("summarize"):
st.success(summary(text))
text_range= st.sidebar.slider("Summarize words Range",25,500)
text = st.text_area("Input Text For Summary",height=250)
if st.button("custom summarization"):
st.warning(summarize(text,word_count=text_range))
# Tokenizer
elif choice == "Tokenizer":
st.title('Text Tokenizer')
row_data = st.text_area("write Text For Tokenizer")
docx= nlp(row_data)
if st.button("Tokenizer"):
spacy_streamlit.visualize_tokens(docx,attrs=['text','pos_','dep_','ent_type_'])
if st.button("NER"):
spacy_streamlit.visualize_ner(docx,labels=nlp.get_pipe('ner').labels)
if st.button("Text Relationship"):
spacy_streamlit.visualize_parser(docx)
# synonyms
elif choice == "Synonyms":
st.title('Synonym Generator')
text = st.text_area("Enter Text")
if st.button("Find"):
for syn in wordnet.synsets(text):
for i in syn.lemmas():
st.success(i.name())
if st.checkbox("Defination"):
for syn in wordnet.synsets(text):
st.warning(syn.definition())
if st.checkbox("Example"):
for syn in wordnet.synsets(text):
st.success(syn.examples())
# Translator
elif choice == "Translator":
st.title('Speech Tranlation')
row_text = st.text_area("Enter Your Text For Translation",height=300)
translation_text = TextBlob(row_text)
list1 = ["en","ta","pa","gu","hi","ur","kn","bn","te"]
a= st.selectbox("select",list1)
if st.button("search"):
#input1 = TextBlob("Simple is better than complex")
st.success(translation_text.translate(to=a))
#Search Bar
elif choice == "Search":
st.title('Web Search')
row_text= st.text_input("Search Anything")
google = Google(license=None)
if st.button("search"):
for search_result in google.search(row_text):
st.write(search_result.text)
st.warning(search_result.url)
elif choice == "Spell Correction":
st.title('AI Spell Correction')
text_data = st.text_area("Enter Text Here")
a = TextBlob(text_data)
if st.button("Correct"):
st.success(a.correct())
st.title('Pluralize & Singularize')
text_data1 = st.text_input("Enter a word For pluralize / singularize")
if st.checkbox("pluralize"):
st.warning(pluralize(text_data1))
if st.checkbox("singularize"):
st.warning(singularize(text_data1))
st.title('Compartitive & Superlative')
text2 = st.text_input("Enter Text For comparative & superlative")
if st.checkbox("comparative"):
st.success(comparative(text2))
if st.checkbox("superlative"):
st.success(superlative(text2))
def search():
inputs = request.form
age = inputs['age']
sex = inputs['sex']
race = inputs['race']
if age == '1-4':
Age = "Age='2a'"
if sex == 'Female' and race == 'Asian/Pacific Islander':
sql_query = "SELECT name,p_f_asian FROM probabilities WHERE "+Age+" and p_f_asian !=1 ORDER BY p_f_asian DESC;"
if sex == 'Female' and race == 'Black/African American':
sql_query = "SELECT name,p_f_black FROM probabilities WHERE "+Age+" and p_f_black !=1 ORDER BY p_f_black DESC;"
if sex == 'Female' and race == 'Hispanic':
sql_query = "SELECT name,p_f_latino FROM probabilities WHERE "+Age+" and p_f_latino !=1 ORDER BY p_f_latino DESC;"
if sex == 'Female' and race == 'Native American/Alaskan':
sql_query = "SELECT name,p_f_native FROM probabilities WHERE "+Age+" and p_f_native !=1 ORDER BY p_f_native DESC;"
if sex == 'Female' and race == 'White':
sql_query = "SELECT name,p_f_white FROM probabilities WHERE "+Age+" and p_f_white !=1 ORDER BY p_f_white DESC;"
if sex == 'Male' and race == 'Asian/Pacific Islander':
sql_query = "SELECT name,p_f_asian FROM probabilities WHERE "+Age+" and p_m_asian !=1 ORDER BY p_m_asian DESC;"
if sex == 'Male' and race == 'Black/African American':
sql_query = "SELECT name,p_m_black FROM probabilities WHERE "+Age+" and p_m_black !=1 ORDER BY p_m_black DESC;"
if sex == 'Male' and race == 'Hispanic':
sql_query = "SELECT name,p_m_latino FROM probabilities WHERE "+Age+" and p_m_latino !=1 ORDER BY p_m_latino DESC;"
if sex == 'Male' and race == 'Native American/Alaskan':
sql_query = "SELECT name,p_m_native FROM probabilities WHERE "+Age+" and p_m_native !=1 ORDER BY p_m_native DESC;"
if sex == 'Male' and race == 'White':
sql_query = "SELECT name,p_m_white FROM probabilities WHERE "+Age+" and p_m_white !=1 ORDER BY p_m_white DESC;"
if age == '5-14':
Age = "Age='3a'"
if sex == 'Female' and race == 'Asian/Pacific Islander':
sql_query = "SELECT name,p_f_asian FROM probabilities WHERE "+Age+" and p_f_asian !=1 ORDER BY p_f_asian DESC;"
if sex == 'Female' and race == 'Black/African American':
sql_query = "SELECT name,p_f_black FROM probabilities WHERE "+Age+" and p_f_black !=1 ORDER BY p_f_black DESC;"
if sex == 'Female' and race == 'Hispanic':
sql_query = "SELECT name,p_f_latino FROM probabilities WHERE "+Age+" and p_f_latino !=1 ORDER BY p_f_latino DESC;"
if sex == 'Female' and race == 'Native American/Alaskan':
sql_query = "SELECT name,p_f_native FROM probabilities WHERE "+Age+" and p_f_native !=1 ORDER BY p_f_native DESC;"
if sex == 'Female' and race == 'White':
sql_query = "SELECT name,p_f_white FROM probabilities WHERE "+Age+" and p_f_white !=1 ORDER BY p_f_white DESC;"
if sex == 'Male' and race == 'Asian/Pacific Islander':
sql_query = "SELECT name,p_f_asian FROM probabilities WHERE "+Age+" and p_m_asian !=1 ORDER BY p_m_asian DESC;"
if sex == 'Male' and race == 'Black/African American':
sql_query = "SELECT name,p_m_black FROM probabilities WHERE "+Age+" and p_m_black !=1 ORDER BY p_m_black DESC;"
if sex == 'Male' and race == 'Hispanic':
sql_query = "SELECT name,p_m_latino FROM probabilities WHERE "+Age+" and p_m_latino !=1 ORDER BY p_m_latino DESC;"
if sex == 'Male' and race == 'Native American/Alaskan':
sql_query = "SELECT name,p_m_native FROM probabilities WHERE "+Age+" and p_m_native !=1 ORDER BY p_m_native DESC;"
if sex == 'Male' and race == 'White':
sql_query = "SELECT name,p_m_white FROM probabilities WHERE "+Age+" and p_m_white !=1 ORDER BY p_m_white DESC;"
if age == '15-24':
Age = "Age='4a'"
if sex == 'Female' and race == 'Asian/Pacific Islander':
sql_query = "SELECT name,p_f_asian FROM probabilities WHERE "+Age+" and p_f_asian !=1 ORDER BY p_f_asian DESC;"
if sex == 'Female' and race == 'Black/African American':
sql_query = "SELECT name,p_f_black FROM probabilities WHERE "+Age+" and p_f_black !=1 ORDER BY p_f_black DESC;"
if sex == 'Female' and race == 'Hispanic':
sql_query = "SELECT name,p_f_latino FROM probabilities WHERE "+Age+" and p_f_latino !=1 ORDER BY p_f_latino DESC;"
if sex == 'Female' and race == 'Native American/Alaskan':
sql_query = "SELECT name,p_f_native FROM probabilities WHERE "+Age+" and p_f_native !=1 ORDER BY p_f_native DESC;"
if sex == 'Female' and race == 'White':
sql_query = "SELECT name,p_f_white FROM probabilities WHERE "+Age+" and p_f_white !=1 ORDER BY p_f_white DESC;"
if sex == 'Male' and race == 'Asian/Pacific Islander':
sql_query = "SELECT name,p_f_asian FROM probabilities WHERE "+Age+" and p_m_asian !=1 ORDER BY p_m_asian DESC;"
if sex == 'Male' and race == 'Black/African American':
sql_query = "SELECT name,p_m_black FROM probabilities WHERE "+Age+" and p_m_black !=1 ORDER BY p_m_black DESC;"
if sex == 'Male' and race == 'Hispanic':
sql_query = "SELECT name,p_m_latino FROM probabilities WHERE "+Age+" and p_m_latino !=1 ORDER BY p_m_latino DESC;"
if sex == 'Male' and race == 'Native American/Alaskan':
sql_query = "SELECT name,p_m_native FROM probabilities WHERE "+Age+" and p_m_native !=1 ORDER BY p_m_native DESC;"
if sex == 'Male' and race == 'White':
sql_query = "SELECT name,p_m_white FROM probabilities WHERE "+Age+" and p_m_white !=1 ORDER BY p_m_white DESC;"
if age == '25-34':
Age = "Age='5a'"
if sex == 'Female' and race == 'Asian/Pacific Islander':
sql_query = "SELECT name,p_f_asian FROM probabilities WHERE "+Age+" and p_f_asian !=1 ORDER BY p_f_asian DESC;"
if sex == 'Female' and race == 'Black/African American':
sql_query = "SELECT name,p_f_black FROM probabilities WHERE "+Age+" and p_f_black !=1 ORDER BY p_f_black DESC;"
if sex == 'Female' and race == 'Hispanic':
sql_query = "SELECT name,p_f_latino FROM probabilities WHERE "+Age+" and p_f_latino !=1 ORDER BY p_f_latino DESC;"
if sex == 'Female' and race == 'Native American/Alaskan':
sql_query = "SELECT name,p_f_native FROM probabilities WHERE "+Age+" and p_f_native !=1 ORDER BY p_f_native DESC;"
if sex == 'Female' and race == 'White':
sql_query = "SELECT name,p_f_white FROM probabilities WHERE "+Age+" and p_f_white !=1 ORDER BY p_f_white DESC;"
if sex == 'Male' and race == 'Asian/Pacific Islander':
sql_query = "SELECT name,p_f_asian FROM probabilities WHERE "+Age+" and p_m_asian !=1 ORDER BY p_m_asian DESC;"
if sex == 'Male' and race == 'Black/African American':
sql_query = "SELECT name,p_m_black FROM probabilities WHERE "+Age+" and p_m_black !=1 ORDER BY p_m_black DESC;"
if sex == 'Male' and race == 'Hispanic':
sql_query = "SELECT name,p_m_latino FROM probabilities WHERE "+Age+" and p_m_latino !=1 ORDER BY p_m_latino DESC;"
if sex == 'Male' and race == 'Native American/Alaskan':
sql_query = "SELECT name,p_m_native FROM probabilities WHERE "+Age+" and p_m_native !=1 ORDER BY p_m_native DESC;"
if sex == 'Male' and race == 'White':
sql_query = "SELECT name,p_m_white FROM probabilities WHERE "+Age+" and p_m_white !=1 ORDER BY p_m_white DESC;"
if age == '35-44':
Age = "Age='6a'"
if sex == 'Female' and race == 'Asian/Pacific Islander':
sql_query = "SELECT name,p_f_asian FROM probabilities WHERE "+Age+" and p_f_asian !=1 ORDER BY p_f_asian DESC;"
if sex == 'Female' and race == 'Black/African American':
sql_query = "SELECT name,p_f_black FROM probabilities WHERE "+Age+" and p_f_black !=1 ORDER BY p_f_black DESC;"
if sex == 'Female' and race == 'Hispanic':
sql_query = "SELECT name,p_f_latino FROM probabilities WHERE "+Age+" and p_f_latino !=1 ORDER BY p_f_latino DESC;"
if sex == 'Female' and race == 'Native American/Alaskan':
sql_query = "SELECT name,p_f_native FROM probabilities WHERE "+Age+" and p_f_native !=1 ORDER BY p_f_native DESC;"
if sex == 'Female' and race == 'White':
sql_query = "SELECT name,p_f_white FROM probabilities WHERE "+Age+" and p_f_white !=1 ORDER BY p_f_white DESC;"
if sex == 'Male' and race == 'Asian/Pacific Islander':
sql_query = "SELECT name,p_f_asian FROM probabilities WHERE "+Age+" and p_m_asian !=1 ORDER BY p_m_asian DESC;"
if sex == 'Male' and race == 'Black/African American':
sql_query = "SELECT name,p_m_black FROM probabilities WHERE "+Age+" and p_m_black !=1 ORDER BY p_m_black DESC;"
if sex == 'Male' and race == 'Hispanic':
sql_query = "SELECT name,p_m_latino FROM probabilities WHERE "+Age+" and p_m_latino !=1 ORDER BY p_m_latino DESC;"
if sex == 'Male' and race == 'Native American/Alaskan':
sql_query = "SELECT name,p_m_native FROM probabilities WHERE "+Age+" and p_m_native !=1 ORDER BY p_m_native DESC;"
if sex == 'Male' and race == 'White':
sql_query = "SELECT name,p_m_white FROM probabilities WHERE "+Age+" and p_m_white !=1 ORDER BY p_m_white DESC;"
if age == '45-54':
Age = "Age='7a'"
if sex == 'Female' and race == 'Asian/Pacific Islander':
sql_query = "SELECT name,p_f_asian FROM probabilities WHERE "+Age+" and p_f_asian !=1 ORDER BY p_f_asian DESC;"
if sex == 'Female' and race == 'Black/African American':
sql_query = "SELECT name,p_f_black FROM probabilities WHERE "+Age+" and p_f_black !=1 ORDER BY p_f_black DESC;"
if sex == 'Female' and race == 'Hispanic':
sql_query = "SELECT name,p_f_latino FROM probabilities WHERE "+Age+" and p_f_latino !=1 ORDER BY p_f_latino DESC;"
if sex == 'Female' and race == 'Native American/Alaskan':
sql_query = "SELECT name,p_f_native FROM probabilities WHERE "+Age+" and p_f_native !=1 ORDER BY p_f_native DESC;"
if sex == 'Female' and race == 'White':
sql_query = "SELECT name,p_f_white FROM probabilities WHERE "+Age+" and p_f_white !=1 ORDER BY p_f_white DESC;"
if sex == 'Male' and race == 'Asian/Pacific Islander':
sql_query = "SELECT name,p_f_asian FROM probabilities WHERE "+Age+" and p_m_asian !=1 ORDER BY p_m_asian DESC;"
if sex == 'Male' and race == 'Black/African American':
sql_query = "SELECT name,p_m_black FROM probabilities WHERE "+Age+" and p_m_black !=1 ORDER BY p_m_black DESC;"
if sex == 'Male' and race == 'Hispanic':
sql_query = "SELECT name,p_m_latino FROM probabilities WHERE "+Age+" and p_m_latino !=1 ORDER BY p_m_latino DESC;"
if sex == 'Male' and race == 'Native American/Alaskan':
sql_query = "SELECT name,p_m_native FROM probabilities WHERE "+Age+" and p_m_native !=1 ORDER BY p_m_native DESC;"
if sex == 'Male' and race == 'White':
sql_query = "SELECT name,p_m_white FROM probabilities WHERE "+Age+" and p_m_white !=1 ORDER BY p_m_white DESC;"
if age == '55-64':
Age = "Age='8a'"
if sex == 'Female' and race == 'Asian/Pacific Islander':
sql_query = "SELECT name,p_f_asian FROM probabilities WHERE "+Age+" and p_f_asian !=1 ORDER BY p_f_asian DESC;"
if sex == 'Female' and race == 'Black/African American':
sql_query = "SELECT name,p_f_black FROM probabilities WHERE "+Age+" and p_f_black !=1 ORDER BY p_f_black DESC;"
if sex == 'Female' and race == 'Hispanic':
sql_query = "SELECT name,p_f_latino FROM probabilities WHERE "+Age+" and p_f_latino !=1 ORDER BY p_f_latino DESC;"
if sex == 'Female' and race == 'Native American/Alaskan':
sql_query = "SELECT name,p_f_native FROM probabilities WHERE "+Age+" and p_f_native !=1 ORDER BY p_f_native DESC;"
if sex == 'Female' and race == 'White':
sql_query = "SELECT name,p_f_white FROM probabilities WHERE "+Age+" and p_f_white !=1 ORDER BY p_f_white DESC;"
if sex == 'Male' and race == 'Asian/Pacific Islander':
sql_query = "SELECT name,p_f_asian FROM probabilities WHERE "+Age+" and p_m_asian !=1 ORDER BY p_m_asian DESC;"
if sex == 'Male' and race == 'Black/African American':
sql_query = "SELECT name,p_m_black FROM probabilities WHERE "+Age+" and p_m_black !=1 ORDER BY p_m_black DESC;"
if sex == 'Male' and race == 'Hispanic':
sql_query = "SELECT name,p_m_latino FROM probabilities WHERE "+Age+" and p_m_latino !=1 ORDER BY p_m_latino DESC;"
if sex == 'Male' and race == 'Native American/Alaskan':
sql_query = "SELECT name,p_m_native FROM probabilities WHERE "+Age+" and p_m_native !=1 ORDER BY p_m_native DESC;"
if sex == 'Male' and race == 'White':
sql_query = "SELECT name,p_m_white FROM probabilities WHERE "+Age+" and p_m_white !=1 ORDER BY p_m_white DESC;"
if age == '65-74':
Age = "Age='9a'"
if sex == 'Female' and race == 'Asian/Pacific Islander':
sql_query = "SELECT name,p_f_asian FROM probabilities WHERE "+Age+" and p_f_asian !=1 ORDER BY p_f_asian DESC;"
if sex == 'Female' and race == 'Black/African American':
sql_query = "SELECT name,p_f_black FROM probabilities WHERE "+Age+" and p_f_black !=1 ORDER BY p_f_black DESC;"
if sex == 'Female' and race == 'Hispanic':
sql_query = "SELECT name,p_f_latino FROM probabilities WHERE "+Age+" and p_f_latino !=1 ORDER BY p_f_latino DESC;"
if sex == 'Female' and race == 'Native American/Alaskan':
sql_query = "SELECT name,p_f_native FROM probabilities WHERE "+Age+" and p_f_native !=1 ORDER BY p_f_native DESC;"
if sex == 'Female' and race == 'White':
sql_query = "SELECT name,p_f_white FROM probabilities WHERE "+Age+" and p_f_white !=1 ORDER BY p_f_white DESC;"
if sex == 'Male' and race == 'Asian/Pacific Islander':
sql_query = "SELECT name,p_f_asian FROM probabilities WHERE "+Age+" and p_m_asian !=1 ORDER BY p_m_asian DESC;"
if sex == 'Male' and race == 'Black/African American':
sql_query = "SELECT name,p_m_black FROM probabilities WHERE "+Age+" and p_m_black !=1 ORDER BY p_m_black DESC;"
if sex == 'Male' and race == 'Hispanic':
sql_query = "SELECT name,p_m_latino FROM probabilities WHERE "+Age+" and p_m_latino !=1 ORDER BY p_m_latino DESC;"
if sex == 'Male' and race == 'Native American/Alaskan':
sql_query = "SELECT name,p_m_native FROM probabilities WHERE "+Age+" and p_m_native !=1 ORDER BY p_m_native DESC;"
if sex == 'Male' and race == 'White':
sql_query = "SELECT name,p_m_white FROM probabilities WHERE "+Age+" and p_m_white !=1 ORDER BY p_m_white DESC;"
if age == '75-84':
Age = "Age='10a'"
if sex == 'Female' and race == 'Asian/Pacific Islander':
sql_query = "SELECT name,p_f_asian FROM probabilities WHERE "+Age+" and p_f_asian !=1 ORDER BY p_f_asian DESC;"
if sex == 'Female' and race == 'Black/African American':
sql_query = "SELECT name,p_f_black FROM probabilities WHERE "+Age+" and p_f_black !=1 ORDER BY p_f_black DESC;"
if sex == 'Female' and race == 'Hispanic':
sql_query = "SELECT name,p_f_latino FROM probabilities WHERE "+Age+" and p_f_latino !=1 ORDER BY p_f_latino DESC;"
if sex == 'Female' and race == 'Native American/Alaskan':
sql_query = "SELECT name,p_f_native FROM probabilities WHERE "+Age+" and p_f_native !=1 ORDER BY p_f_native DESC;"
if sex == 'Female' and race == 'White':
sql_query = "SELECT name,p_f_white FROM probabilities WHERE "+Age+" and p_f_white !=1 ORDER BY p_f_white DESC;"
if sex == 'Male' and race == 'Asian/Pacific Islander':
sql_query = "SELECT name,p_f_asian FROM probabilities WHERE "+Age+" and p_m_asian !=1 ORDER BY p_m_asian DESC;"
if sex == 'Male' and race == 'Black/African American':
sql_query = "SELECT name,p_m_black FROM probabilities WHERE "+Age+" and p_m_black !=1 ORDER BY p_m_black DESC;"
if sex == 'Male' and race == 'Hispanic':
sql_query = "SELECT name,p_m_latino FROM probabilities WHERE "+Age+" and p_m_latino !=1 ORDER BY p_m_latino DESC;"
if sex == 'Male' and race == 'Native American/Alaskan':
sql_query = "SELECT name,p_m_native FROM probabilities WHERE "+Age+" and p_m_native !=1 ORDER BY p_m_native DESC;"
if sex == 'Male' and race == 'White':
sql_query = "SELECT name,p_m_white FROM probabilities WHERE "+Age+" and p_m_white !=1 ORDER BY p_m_white DESC;"
if age == '85+':
Age = "Age='11a'"
if sex == 'Female' and race == 'Asian/Pacific Islander':
sql_query = "SELECT name,p_f_asian FROM probabilities WHERE "+Age+" and p_f_asian !=1 ORDER BY p_f_asian DESC;"
if sex == 'Female' and race == 'Black/African American':
sql_query = "SELECT name,p_f_black FROM probabilities WHERE "+Age+" and p_f_black !=1 ORDER BY p_f_black DESC;"
if sex == 'Female' and race == 'Hispanic':
sql_query = "SELECT name,p_f_latino FROM probabilities WHERE "+Age+" and p_f_latino !=1 ORDER BY p_f_latino DESC;"
if sex == 'Female' and race == 'Native American/Alaskan':
sql_query = "SELECT name,p_f_native FROM probabilities WHERE "+Age+" and p_f_native !=1 ORDER BY p_f_native DESC;"
if sex == 'Female' and race == 'White':
sql_query = "SELECT name,p_f_white FROM probabilities WHERE "+Age+" and p_f_white !=1 ORDER BY p_f_white DESC;"
if sex == 'Male' and race == 'Asian/Pacific Islander':
sql_query = "SELECT name,p_f_asian FROM probabilities WHERE "+Age+" and p_m_asian !=1 ORDER BY p_m_asian DESC;"
if sex == 'Male' and race == 'Black/African American':
sql_query = "SELECT name,p_m_black FROM probabilities WHERE "+Age+" and p_m_black !=1 ORDER BY p_m_black DESC;"
if sex == 'Male' and race == 'Hispanic':
sql_query = "SELECT name,p_m_latino FROM probabilities WHERE "+Age+" and p_m_latino !=1 ORDER BY p_m_latino DESC;"
if sex == 'Male' and race == 'Native American/Alaskan':
sql_query = "SELECT name,p_m_native FROM probabilities WHERE "+Age+" and p_m_native !=1 ORDER BY p_m_native DESC;"
if sex == 'Male' and race == 'White':
sql_query = "SELECT name,p_m_white FROM probabilities WHERE "+Age+" and p_m_white !=1 ORDER BY p_m_white DESC;"
cursor.execute(sql_query)
rows = cursor.fetchall()
new_list=[''.join(list(rows[0][0])),''.join(list(rows[1][0])),''.join(list(rows[2][0])),''.join(list(rows[3][0])),''.join(list(rows[4][0]))]
pvals=[rows[0][1],rows[1][1],rows[2][1],rows[3][1],rows[4][1]]
padded_list=[]
#need to pad the strings so that they are equal length
max_len=len(max(new_list,key=len))
for i in range(len(new_list)):
cur_len=len(new_list[i])
empty_string=" "*cur_len
padded_string=new_list[i]+empty_string
padded_list.append(padded_string)
#using pattern look up google for top diseases
engine=Google(license=key,throttle=0)
empty_link_disease=[]
empty_text_disease=[]
empty_link_treatment=[]
empty_text_treatment=[]
for index in new_list:
#search wikipedia first for snippet about the disease, if it doesn't exist change to google
about=Wikipedia().search(index)
if not about:
about=engine.search(index)
empty_link_disease.append(about[0].url)
empty_text_disease.append(plaintext(about[0].text))
elif about:
about_url=engine.search(index+' Wikipedia')
empty_link_disease.append(about_url[0].url)
empty_text_disease.append(plaintext(about.sections[0].content[0:1000])+'...')
treatment=engine.search(index+' treatment site:www.healthtap.com')
empty_link_treatment.append(treatment[0].url)
empty_text_treatment.append(plaintext(treatment[0].text))
disease_links=[''.join(list(empty_link_disease[0])),''.join(list(empty_link_disease[1])),''.join(list(empty_link_disease[2])),''.join(list(empty_link_disease[3])),''.join(list(empty_link_disease[4]))]
treatment_links=[''.join(list(empty_link_treatment[0])),''.join(list(empty_link_treatment[1])),''.join(list(empty_link_treatment[2])),''.join(list(empty_link_treatment[3])),''.join(list(empty_link_treatment[4]))]
#string_links=[str(list(empty_link[0])),str(list(empty_link[1])),str(list(empty_link[2])),str(list(empty_link[3])),str(list(empty_link[4]))]
#print string_links
return render_template('results_feb1.html',disease_name=padded_list,pvalues=pvals, d_links=disease_links, disease_snippets=empty_text_disease, t_links=treatment_links, treatment_snippets=empty_text_treatment)
#!/usr/bin/python2
# -*- coding: utf-8 -*-
import codecs, os, sys
from collections import Counter
from optparse import OptionParser
from pattern.web import Google
engine = Google(license=None, language='de') #'001401664590345725200:dx8-iwqnvyw', language='de')
for result in engine.search('Energiewende', cached=False):
print repr(result.text)
class SourceChecker(object):
def __init__(self, text, max_queries=10, span=20, threshold=.8):
self.max_queries = max_queries
self.span = span
self.threshold = threshold
self.text = text
self.cat_dict = defaultdict(list)
self.engine = Google(license='AIzaSyCFgnXgb9rcwJspcSeXHo7QHvucgM2nLrI',
throttle=0.5,
language=None)
def get_queries(self):
text = self.text
beg_quotes = re.findall(r'\"\S', text)
for each in beg_quotes:
text = text.replace(each, 'BEGQ' + each[-1])
end_quotes = re.findall(r'\S\"', text)
for each in end_quotes:
text = text.replace(each, each[0] + 'ENDQ')
text = re.sub('(ENDQ)+', 'ENDQ', text)
text = re.sub('(BEGQ)+', 'BEGQ', text)
text = text.replace('--', 'DOUBLEDASH')
all_ngrams = ngrams(text, n=self.span, punctuation="", continuous=True)
stop_words = stopwords.words('english')
queries = []
for ngram in all_ngrams:
num_stop = len([w for w in ngram if w in stop_words])
stop_score = float(num_stop) / len(ngram)
chunked = ne_chunk(pos_tag(ngram))
named_entities = [[w for w, t in elt] for elt in chunked
if isinstance(elt, nltk.Tree)]
num_ent = sum([len(ent_list) for ent_list in named_entities])
ent_score = float(num_ent) / len(ngram)
if stop_score < self.threshold and ent_score < self.threshold:
r_string = self.reconstruct_ngram(ngram)
if r_string in self.text:
queries.append(r_string)
reduction = len(queries) / self.max_queries
return queries[0::reduction]
def reconstruct_ngram(self, ngram):
punc_b = ['!', '?', '.', ',', ';', ':', '\'', ')', ']', '}']
punc_a = ['(', '[', '}', '$']
ngram = ' '.join(ngram)
for p in punc_b:
ngram = ngram.replace(' ' + p, p)
for p in punc_a:
ngram = ngram.replace(p + ' ', p)
ngram = re.sub('(^| )BEGQ', ' "', ngram)
ngram = re.sub('ENDQ($| )', '" ', ngram)
ngram = ngram.replace('DOUBLEDASH', '--')
return ngram
def load_domains(self):
sources_path = pd('data', 'source_data.csv')
domain_file = Datasheet.load(sources_path, headers=True)
for row in domain_file:
url = row[1]
cats = row[2:]
self.cat_dict[url] = cats
def pairwise(self, t):
it = iter(t)
return izip(it, it)
def get_urls(self, queries):
domains = defaultdict(list)
for q in queries:
q = "\"" + q + "\""
results = self.engine.search(q)
for result in results:
url = result.url
domain = self.get_domain(url)
domains[domain].append(q)
return domains
def get_domain(self, full_url):
clean_reg = re.compile(r'^((?:https?:\/\/)?(?:www\.)?).*?(\/.*)?$')
match = re.search(clean_reg, full_url)
beg, end = match.group(1), match.group(2)
domain = string.replace(full_url, beg, '')
domain = string.replace(domain, end, '')
return domain
def render_output(self, domains):
output = defaultdict(list)
for d, v in domains.items():
d_cats = [
c for c in self.cat_dict[d]
if len(c) > 0 and len(c.split(' ')) < 3
]
overlap = float(len(v)) / self.max_queries
if overlap <= 0.2:
output['MINIMAL'].append((d, d_cats))
elif 0.2 < overlap < 0.6:
output['SOME'].append((d, d_cats))
elif overlap >= 0.6:
output['HIGH'].append((d, d_cats))
degrees = ['HIGH', 'SOME', 'MINIMAL']
print '\n'
for deg in degrees:
if output[deg]:
print '%s OVERLAP: ' % deg
for d, cats in sorted(output[deg]):
if cats:
print d + ': ' + ','.join(cats)
else:
print d
print '\n'
def render_graph(self, domains):
g = Graph()
for domain in domains.keys():
if domain in self.cat_dict:
categories = self.cat_dict[domain]
stroke = (0, 0, 0, 0.5)
if 'right' in categories:
stroke = (255, 0, 0, 1)
elif 'right_center' in categories:
stroke = (255, 0, 0, .5)
if 'left' in categories:
stroke = (0, 0, 255, 1)
elif 'left_center' in categories:
stroke = (0, 0, 255, .5)
if 'least_biased' in categories:
stroke = (0, 255, 0, 1)
fill = (128, 128, 0, 0.1)
dub_cats = [
'fake', 'questionable', 'clickbait', 'unreliable', 'conspiracy'
]
score = len([c for c in categories if c in dub_cats])
if score:
fill = (0, 0, 0, float(score) / 5)
g.add_node(domain,
radius=len(domains[domain]) * 6,
stroke=stroke,
strokewidth=6,
fill=fill,
font_size=30)
pairs = self.pairwise(domains.keys())
for x, y in pairs:
x_queries = set(domains[x])
y_queries = set(domains[y])
intersection = len(x_queries.intersection(y_queries))
if intersection > 0:
max_rad = max(len(domains[x]), len(domains[y])) + 1000
g.add_edge(x, y, length=max_rad, strokewidth=intersection)
path = 'graph'
g.export(path,
encoding='utf-8',
distance=6,
directed=False,
width=1400,
height=900)
# Other search engines may also handle IMAGE, NEWS, ...
# Google's "Custom Search API" is a paid service.
# The pattern.web module uses a test account by default,
# with a 100 free queries per day shared by all Pattern users.
# If this limit is exceeded, SearchEngineLimitError is raised.
# You should obtain your own license key at:
# https://code.google.com/apis/console/
# Activate "Custom Search API" under "Services" and get the key under "API Access".
# Then use Google(license=[YOUR_KEY]).search().
# This will give you 100 personal free queries, or 5$ per 1000 queries.
engine = Google(license=None, language="en")
# Veale & Hao's method for finding similes using wildcards (*):
# http://afflatus.ucd.ie/Papers/LearningFigurative_CogSci07.pdf
# This will match results such as:
# - "as light as a feather",
# - "as cute as a cupcake",
# - "as drunk as a lord",
# - "as snug as a bug", etc.
q = "as * as a *"
# Google is very fast but you can only get up to 100 (10x10) results per query.
for i in range(1, 2):
for result in engine.search(q, start=i, count=10, type=SEARCH, cached=True):
# plaintext() removes all HTML formatting.
print(plaintext(result.text).encode("utf-8"))
print(result.url)
print(result.date)
print()
def gnewshits(topic):
engine = Google()
results = engine.search(topic, type=NEWS)
return results.total
while not asyn_req.done:
time.sleep(0.1)
print('searching...')
print(asyn_req.value)
print(find_urls(asyn_req.value, unique=True))
# ### Getting Search Engine Results with APIs
# #### Google
from pattern.web import Google
google = Google(license=None)
for search_result in google.search('artificial intelligence'):
print(search_result.url)
print(search_result.text)
# #### Twitter
from pattern.web import Twitter
twitter = Twitter()
index = None
for j in range(3):
for tweet in twitter.search('artificial intelligence',
start=index,
count=3):
print(tweet.text)
index = tweet.id
class Engine(object):
def __init__(self, provider, key=None):
if provider.lower() == "bing":
key = key or 'd6Mz4slIdgIxcKR4609FO+QKOFTEFFRB3i7j8VioPiE'
self._engine = Bing(license=key)
elif provider.lower() == "google":
key = key or 'AIzaSyCAADAKnnkmDwIlLk_Q1p6foqI_ZMrgzcg'
self._engine = Google(license=key)
else:
raise ValueError('Not a recognized provider.')
def fuzzySearch(self, result, query):
best, best_i = 0, None
for i in range(len(result) - len(query) + 1):
score = similarity(result[i:i + len(query)], query)
if best < score:
best = score
best_i = i
return result[best_i + len(query):] if best_i != None else ''
def getPatterns(self, query):
cleaned = query.strip('?')
p = [(cleaned, 3)]
t = parsetree(query)[0]
for chunk in t.chunks:
if chunk.pos == 'NP':
p.append((chunk.string, 2))
for w in cleaned.split():
p.append((w, 1))
return p
def getGrams(self, results):
grams = {}
for text, weight in results:
uni = set(ngrams(text, n=1))
bi = set(ngrams(text, n=2))
tri = set(ngrams(text, n=3))
for gram in uni:
grams[gram] = grams.get(gram, 0) + weight
for gram in bi:
grams[gram] = grams.get(gram, 0) + weight
for gram in tri:
grams[gram] = grams.get(gram, 0) + weight
return grams
def removeStopWords(self, grams, queries):
for gram in grams.keys():
inter = set([g.lower() for g in gram]) & set(wordlist.STOPWORDS)
if len(inter) > 1 or inter and len(gram) == 1:
grams.pop(gram)
return grams
def searchQueries(self, queries):
results = []
for q, w1, d in queries:
for r in self._engine.search(q, count=100):
results.append((re.sub(r'[!,.?]', '', plaintext(r.txt)), w1))
return results
def searchQueriesWithPatterns(self, queries):
# Faster, but still need to refine extraction patterns
results = []
for q, w1, d in queries:
print "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
print q
print "++++++++++++++++++++++++++++++++++++++++++++++++++++++++++"
for r in self._engine.search(q, count=50):
# Each result is given a preliminary score based on the weight of the query
# that retrieved it and the pattern that was matched to it
for p, w2 in self.getPatterns(q):
if d == 'L':
m = re.search('(.*?)' + p + '.*\.', plaintext(r.txt),
re.IGNORECASE)
else:
m = re.search(p + '(.*)', plaintext(r.txt),
re.IGNORECASE)
if m:
print plaintext(r.txt)
print "-------------------------------------------------"
print p, "generated", m.group(1)
print "================================================="
results.append((m.group(1), w1 + w2))
break
return results
def searchAndGram(self, queries):
results = self.searchQueries(queries)
grams = self.getGrams(results)
grams = self.removeStopWords(grams, queries)
return grams
# This example retrieves results from Google based on a given query.
# The Google search engine can handle SEARCH type searches.
# Other search engines may also handle IMAGE, NEWS, ...
# Google's "Custom Search API" is a paid service.
# The pattern.web module uses a test account by default,
# with a 100 free queries per day shared by all Pattern users.
# If this limit is exceeded, SearchEngineLimitError is raised.
# You should obtain your own license key at:
# https://code.google.com/apis/console/
# Activate "Custom Search API" under "Services" and get the key under "API Access".
# Then use Google(license=[YOUR_KEY]).search().
# This will give you 100 personal free queries, or 5$ per 1000 queries.
engine = Google(license=None, language="en")
# Veale & Hao's method for finding similes using wildcards (*):
# http://afflatus.ucd.ie/Papers/LearningFigurative_CogSci07.pdf
# This will match results such as:
# - "as light as a feather",
# - "as cute as a cupcake",
# - "as drunk as a lord",
# - "as snug as a bug", etc.
q = "as * as a *"
# Google is very fast but you can only get up to 100 (10x10) results per query.
for i in range(1, 2):
for result in engine.search(q, start=i, count=10, type=SEARCH, cached=True):
print plaintext(result.text) # plaintext() removes all HTML formatting.
print result.url
print result.date
print
#!/usr/bin/python2
# -*- coding: utf-8 -*-
import codecs, os, sys
from collections import Counter
from optparse import OptionParser
from pattern.web import Google
engine = Google(
license=None,
language='de') #'001401664590345725200:dx8-iwqnvyw', language='de')
for result in engine.search('Energiewende', cached=False):
print repr(result.text)
