def test_can_silent_fail_web_search(self):
web_bing = PyMsCognitiveWebSearch(SECRET_KEY,
"Python Software Foundation",
silent_fail=True)
result_one = web_bing.search(limit=50)
self.assertTrue(len(result_one) == 50)
self.assertTrue("python" in result_one[0].name.lower())
def run_query(search_terms):
bing_api_key = read_bing_key()
if not bing_api_key:
raise KeyError("Bing Key Not Found")
# Create our results list which we'll populate.
results = []
# Custom params
params = 'mkt=ru-RU'
try:
search_service = PyMsCognitiveWebSearch(bing_api_key, search_terms, custom_params=params)
json_response = search_service.search(limit=11, format='json') # 1 - 10
except:
print("Error when querying the Bing API")
# Loop through each page returned, populating out results list.
# dict_keys(['json', 'deep_links', 'snippet', 'url', 'title', 'name', 'description', 'id', 'display_url'])
for idx in range(11):
results.append({'title': json_response[idx].title,
'link': json_response[idx].display_url,
'summary': json_response[idx].description})
return results
def test_empty_response(self):
'''
This test checks that searching for a non-existent keyword will not error out.
'''
non_existing_result = u'youwillmostdeffinitlynotfindthisveryweirdandlongstringopnanysitewhatsoever123'
web_bing = PyMsCognitiveWebSearch(SECRET_KEY, non_existing_result)
self.assertTrue([] == web_bing.search())
def website_crawl(self, query):
"""
This function issues the given query to ping, then crawls the websites
that were given in the ResultSet for links to a file. To be used with
queries such as "jpg example file" or "inurl:(avi) intitle:index of"
:return: A generator - StopIteration is called when no more links can/should be found.
"""
self.search_service = PyMsCognitiveWebSearch(self.ms_key, query)
self.search_service.SEARCH_WEB_BASE = "https://api.cognitive.microsoft.com/bing/v7.0/search"
results = self.search_service.search_all(format="json",
quota=LIMIT_RESULTS)
print(len(results))
for item in results:
try:
r = requests.get(item.url, timeout=MAX_TIMEOUT)
except Exception as e:
print("Skipping ", item.url, "because of Exception", str(e))
continue
parsed_uri = urlparse(r.url)
subdomain = '{uri.scheme}://{uri.netloc}/'.format(uri=parsed_uri)
# extract the top level domain:
rootdomain = '{uri.scheme}://{ext.domain}.{ext.suffix}'.format(
uri=parsed_uri, ext=tldextract.extract(r.url))
try:
if requests.head(
subdomain +
"/robots.txt").status_code == 404 and requests.head(
rootdomain + "/robots.txt").status_code == 404:
# No Robots TXT - Skip
print("Skipping", subdomain,
"because it does not contain a robots.txt")
continue
except Exception as e:
print("Skipping", subdomain, "because of exception", str(e))
continue
print("Now scanning through", r.url)
html_text = r.text
if "index of" in query and not "index of" in html_text:
# TODO: Really really hacky. This if statement shoud only be
# TODO: in place if we are issuing the index of query
# We probably did not reach a file repository
continue
soup = BeautifulSoup(html_text, "html.parser")
link_anchors = soup.find_all("a")
links = list(map(lambda x: x.get("href"),
link_anchors)) # type: [str]
links = list(
filter(
lambda x: x is not None and x.lower().endswith(
self.filetype), links))
for link in links:
path = link
filelink = urljoin(
r.url, path
) # Join the two urls. Urljoin handles every case: path is relative and path is absolute
if self.is_valid_file(self.filetype, filelink):
print("Yielding", filelink)
yield filelink
def find_omim_link(row):
geneName = safely_convert_val_to_str(
row['Gene Symbol']) #search based on the ingenuity assigned gene name
searchTerm = 'omim ' + geneName
search_service = PyMsCognitiveWebSearch(bingWebSearchKey, searchTerm)
firstFiftyResults = search_service.search(limit=50, format='json')
url = find_first_correct_result_url(firstFiftyResults, 'omim')
return url
def azure_search(claim):
search_term = claim
search_service = PyMsCognitiveWebSearch('75d1a40af4bf4ba4bdf561ae25b5db5c',
claim)
first_three_result = search_service.search(limit=3, format='json') #1-50
urls = []
# To get individual result json:
for i in first_three_result:
urls.append(i.url.encode('utf-8'))
return urls
def do_search(self, search_term):
""" Perform paged searches while we find new FQDN """
print(search_term)
custom_params = {'safeSearch': 'Off'}
if self.args.offset:
custom_params['offset'] = self.args.offset
search_service = MsWeb(self.args.bing_api_key,
query=search_term,
custom_params=custom_params)
bing_results = search_service.search(limit=self.args.limit,
format='json')
self.query_count += 1
results = BingSearch._parse_results(bing_results)
# Find all the things until we can't find new URLs anymore
# We have the first result set, do a new search and compare it to the previous one.
if self.args.find_all and self.query_count < self.args.max_queries:
result_set = set((x["url"] for x in results))
# Internal py_ms_cognitive object deals with the offset
bing_results = search_service.search(limit=self.args.limit,
format='json')
self.query_count += 1
new_results = BingSearch._parse_results(bing_results)
new_result_set = set((x["url"] for x in new_results))
# New result contains everything already inside result_set
while not result_set.issuperset(new_result_set):
results += new_results
if self.query_count < self.args.max_queries:
# Internal py_ms_cognitive object deals with the offset
bing_results = search_service.search(limit=self.args.limit,
format='json')
self.query_count += 1
new_results = BingSearch._parse_results(bing_results)
new_result_set = set((tuple(x) for x in results))
else:
# Do not perform a search, but previous results were added
# Stop looping, exit
break
return results
def get_seed(self, search_term):
search_service = PyMsCognitiveWebSearch(config.key, search_term)
first_twenty_result = search_service.search(limit=10,
format='json') #1-10
for i in range(10):
#Giving Each of the 10 links a pagerank of 0.1 (i.e. 1/10)
link = first_twenty_result[i].json['displayUrl']
link = self.normalize(link)
self.nodesToVisit.put(
(-0.1,
link)) #Negate for priority queue to favor higher pagerank
self.bfs_queue.put(link)
self.ranks[link] = 0.1
def run_query(search_terms):
bing_api_key = read_bing_key()
if not bing_api_key:
raise KeyError('Bing Key Not Found')
results_per_page = 10
results = []
try:
search_service = PyMsCognitiveWebSearch(bing_api_key, search_terms)
response = search_service.search(limit=results_per_page, format='json')
for result in response:
results.append({
'title': result.json['name'],
'link': result.json['displayUrl'],
'summary': result.json['snippet']})
except:
print("Error when querying the Bing API")
return results
def test_search_all(self):
web_bing = PyMsCognitiveWebSearch(SECRET_KEY,
"Python Software Foundation")
result_one = web_bing.search_all(quota=60)
self.assertTrue(len(result_one) == 60)
self.assertTrue("python" in result_one[0].name.lower())
print "Status: 200 OK"
form = cgi.FieldStorage()
query = form.getvalue('query')
logfile.write(query)
logfile.write("\n")
#query = "moon snails"
API_KEY = "54bcf3e9f555466fa24f529b88311f65"
search_term = query
limit = 10
search_array = search_term.split()
search_service = PyMsCognitiveWebSearch(API_KEY, search_term)
results = search_service.search(limit=limit, format='json')
x = 0
h = 550
w = 550
freq_array = []
url_array = []
name_array = []
snippet_array = []
for x in range(0, limit):
try:
dirty_url = results[x].url[20:]
clean_url = dirty_url[dirty_url.find("http"):]
clean_url = clean_url[:clean_url.find("&p=DevEx")]
def query(query):
search_service = PyMsCognitiveWebSearch(keys.get('bing_api_key'), query)
first_fifty_result = search_service.search(limit=10, format='json') #1-50
return [(x.name, x.display_url) for x in first_fifty_result]
class FileCrawler:
"""
This class should be used to seeds a specific file and save it to a specific url
"""
def __init__(self, filetype: str, ms_key: str, out_dir: str):
"""
:param filetype: The filetype to seeds.
:param ms_key: The api key for the bing search api.
"""
self.filetype = filetype.lower()
self.ms_key = ms_key
print("KEY", self.ms_key)
self.out_dir = out_dir
self.search_service = None
@staticmethod
def is_valid_file(filetype, url: str) -> bool:
"""
Given an url, it checks if the content is a file and of the right fileformat or not.
:param url: The url to check
:return: True if url is not an html webpage, false if it is
"""
utils.temp_print("Trying", url)
try:
response = requests.head(url, timeout=MAX_TIMEOUT, headers=headers)
except Exception as e:
return False
if response.headers.get("content-type") is not None:
# return False
if "text/html" in response.headers["content-type"]:
return False
if filetype in response.headers["content-type"]:
return True
part = url.rpartition(
"."
) # Returns a three tuple, last tuple containing the part after the "."
if part[2].lower() == filetype:
return True
return False
def website_crawl(self, query):
"""
This function issues the given query to ping, then crawls the websites
that were given in the ResultSet for links to a file. To be used with
queries such as "jpg example file" or "inurl:(avi) intitle:index of"
:return: A generator - StopIteration is called when no more links can/should be found.
"""
self.search_service = PyMsCognitiveWebSearch(self.ms_key, query)
self.search_service.SEARCH_WEB_BASE = "https://api.cognitive.microsoft.com/bing/v7.0/search"
results = self.search_service.search_all(format="json",
quota=LIMIT_RESULTS)
print(len(results))
for item in results:
try:
r = requests.get(item.url, timeout=MAX_TIMEOUT)
except Exception as e:
print("Skipping ", item.url, "because of Exception", str(e))
continue
parsed_uri = urlparse(r.url)
subdomain = '{uri.scheme}://{uri.netloc}/'.format(uri=parsed_uri)
# extract the top level domain:
rootdomain = '{uri.scheme}://{ext.domain}.{ext.suffix}'.format(
uri=parsed_uri, ext=tldextract.extract(r.url))
try:
if requests.head(
subdomain +
"/robots.txt").status_code == 404 and requests.head(
rootdomain + "/robots.txt").status_code == 404:
# No Robots TXT - Skip
print("Skipping", subdomain,
"because it does not contain a robots.txt")
continue
except Exception as e:
print("Skipping", subdomain, "because of exception", str(e))
continue
print("Now scanning through", r.url)
html_text = r.text
if "index of" in query and not "index of" in html_text:
# TODO: Really really hacky. This if statement shoud only be
# TODO: in place if we are issuing the index of query
# We probably did not reach a file repository
continue
soup = BeautifulSoup(html_text, "html.parser")
link_anchors = soup.find_all("a")
links = list(map(lambda x: x.get("href"),
link_anchors)) # type: [str]
links = list(
filter(
lambda x: x is not None and x.lower().endswith(
self.filetype), links))
for link in links:
path = link
filelink = urljoin(
r.url, path
) # Join the two urls. Urljoin handles every case: path is relative and path is absolute
if self.is_valid_file(self.filetype, filelink):
print("Yielding", filelink)
yield filelink
def try_filetype_crawl(self):
"""
Try to find download links to files of the given file format.
:return: A generator - StopIteration is called when no more links can/should be found.
"""
# First: Try a simple "filetype:" query - works for some, but not all filetypes
query = "filetype:" + self.filetype
PyMsCognitiveWebSearch.SEARCH_WEB_BASE = "https://api.cognitive.microsoft.com/bing/v7.0/search"
self.search_service = PyMsCognitiveWebSearch(self.ms_key, query)
results = self.search_service.search_all(format="json",
quota=LIMIT_RESULTS + 20)
for item in results:
try:
r = requests.get(
item.url, timeout=MAX_TIMEOUT,
headers=headers) # Request the url to resolve the redirect
except Exception as e: # requests.exceptions.ConnectTimeout:
print("Skipping ", item.url, "because of Exception", str(e))
# Then just skip
continue
if self.is_valid_file(self.filetype, r.url):
print("Yielding ", r.url)
yield r.url
# If this fails, maybe the requested filetype is an image? Then perform an image search
if self.filetype in image_list: # Perform an image Search
query = self.filetype + " sample"
PyMsCognitiveImageSearch.SEARCH_IMAGE_BASE = "https://api.cognitive.microsoft.com/bing/v7.0/images/search"
self.search_service = PyMsCognitiveImageSearch(self.ms_key, query)
results = self.search_service._search(
limit=LIMIT_RESULTS,
format="json") # TODO: Class does not implement pagination? :(
for item in results:
utils.temp_print("Checking item", item.content_url)
try:
r = requests.get(item.content_url,
timeout=MAX_TIMEOUT,
headers=headers)
except Exception as e:
print("Skipping ", item.url, "because of Exception",
str(e))
# print("Timeout, checking next item")
continue
print("Url is", r.url)
if self.is_valid_file(self.filetype, r.url):
print("Yielding ", r.url)
yield r.url
for result in self.website_crawl("." + self.filetype +
" example file"):
print("Yielding", result)
yield result
for result in self.website_crawl("." + self.filetype + " sample file"):
print("Yielding", result)
yield result
# Last Resort: The index of trick. Note thatfi this can yield some undesired file samples, use with caution!
query = "inurl:(" + self.filetype + ") intitle:\"index of:\""
self.search_service = PyMsCognitiveWebSearch(self.ms_key, query)
results = self.search_service.search_all(format="json",
quota=LIMIT_RESULTS)
print(len(results))
for item in results:
try:
r = requests.get(item.url, timeout=MAX_TIMEOUT)
except Exception as e:
print("Skipping ", item.url, "because of Exception", str(e))
continue
parsed_uri = urlparse(r.url)
domain = '{uri.scheme}://{uri.netloc}/'.format(uri=parsed_uri)
try:
if requests.head(domain + "/robots.txt").status_code == 404:
# No Robots TXT - Skip
print("Skipping", domain,
"because it does not contain a robots.txt")
continue
except Exception as e:
print("Skipping", domain, "because of exception", str(e))
continue
print("Now scanning through", r.url)
html_text = r.text
if not "index of" in html_text:
# We probably did not reach a file repository
continue
soup = BeautifulSoup(html_text, "html.parser")
link_anchors = soup.find_all("a")
links = list(map(lambda x: x.get("href"),
link_anchors)) # type: [str]
links = list(
filter(
lambda x: x is not None and x.lower().endswith(
self.filetype), links))
for link in links:
path = link
filelink = urljoin(
r.url, path
) # Join the two urls. Urljoin handles every case: path is relative and path is absolute
if self.is_valid_file(self.filetype, filelink):
print("Yielding", filelink)
yield filelink
def download(self, max_download=1) -> int:
"""
Tries to download max number of samples files of the given file format to the self.out_dir folder
:return: The amount of downloaded files.
"""
print("MAX", max_download)
i = 0
for rurl in self.try_filetype_crawl():
print("Yielded", rurl)
filename = self.filetype + "_" + str(uuid.uuid4())
if (not os.path.exists(self.out_dir)):
os.makedirs(self.out_dir)
utils.download_seed_to_folder(download_link=rurl,
to_directory=self.out_dir,
filename=filename)
# with open(self.out_dir + "/" + filename + "." + self.filetype, "wb") as file:
# for chunk in r.iter_content(chunk_size=1024):
# if chunk: # filter out keep-alive new chunks
# file.write(chunk)
i += 1
if i >= max_download:
return max_download
# print("Downloaded",rurl)
return i
def try_filetype_crawl(self):
"""
Try to find download links to files of the given file format.
:return: A generator - StopIteration is called when no more links can/should be found.
"""
# First: Try a simple "filetype:" query - works for some, but not all filetypes
query = "filetype:" + self.filetype
PyMsCognitiveWebSearch.SEARCH_WEB_BASE = "https://api.cognitive.microsoft.com/bing/v7.0/search"
self.search_service = PyMsCognitiveWebSearch(self.ms_key, query)
results = self.search_service.search_all(format="json",
quota=LIMIT_RESULTS + 20)
for item in results:
try:
r = requests.get(
item.url, timeout=MAX_TIMEOUT,
headers=headers) # Request the url to resolve the redirect
except Exception as e: # requests.exceptions.ConnectTimeout:
print("Skipping ", item.url, "because of Exception", str(e))
# Then just skip
continue
if self.is_valid_file(self.filetype, r.url):
print("Yielding ", r.url)
yield r.url
# If this fails, maybe the requested filetype is an image? Then perform an image search
if self.filetype in image_list: # Perform an image Search
query = self.filetype + " sample"
PyMsCognitiveImageSearch.SEARCH_IMAGE_BASE = "https://api.cognitive.microsoft.com/bing/v7.0/images/search"
self.search_service = PyMsCognitiveImageSearch(self.ms_key, query)
results = self.search_service._search(
limit=LIMIT_RESULTS,
format="json") # TODO: Class does not implement pagination? :(
for item in results:
utils.temp_print("Checking item", item.content_url)
try:
r = requests.get(item.content_url,
timeout=MAX_TIMEOUT,
headers=headers)
except Exception as e:
print("Skipping ", item.url, "because of Exception",
str(e))
# print("Timeout, checking next item")
continue
print("Url is", r.url)
if self.is_valid_file(self.filetype, r.url):
print("Yielding ", r.url)
yield r.url
for result in self.website_crawl("." + self.filetype +
" example file"):
print("Yielding", result)
yield result
for result in self.website_crawl("." + self.filetype + " sample file"):
print("Yielding", result)
yield result
# Last Resort: The index of trick. Note thatfi this can yield some undesired file samples, use with caution!
query = "inurl:(" + self.filetype + ") intitle:\"index of:\""
self.search_service = PyMsCognitiveWebSearch(self.ms_key, query)
results = self.search_service.search_all(format="json",
quota=LIMIT_RESULTS)
print(len(results))
for item in results:
try:
r = requests.get(item.url, timeout=MAX_TIMEOUT)
except Exception as e:
print("Skipping ", item.url, "because of Exception", str(e))
continue
parsed_uri = urlparse(r.url)
domain = '{uri.scheme}://{uri.netloc}/'.format(uri=parsed_uri)
try:
if requests.head(domain + "/robots.txt").status_code == 404:
# No Robots TXT - Skip
print("Skipping", domain,
"because it does not contain a robots.txt")
continue
except Exception as e:
print("Skipping", domain, "because of exception", str(e))
continue
print("Now scanning through", r.url)
html_text = r.text
if not "index of" in html_text:
# We probably did not reach a file repository
continue
soup = BeautifulSoup(html_text, "html.parser")
link_anchors = soup.find_all("a")
links = list(map(lambda x: x.get("href"),
link_anchors)) # type: [str]
links = list(
filter(
lambda x: x is not None and x.lower().endswith(
self.filetype), links))
for link in links:
path = link
filelink = urljoin(
r.url, path
) # Join the two urls. Urljoin handles every case: path is relative and path is absolute
if self.is_valid_file(self.filetype, filelink):
print("Yielding", filelink)
yield filelink
from py_ms_cognitive import PyMsCognitiveWebSearch, PyMsCognitiveImageSearch key = '82af3a845a3640318879cf8d6db7320a' query = "New York City" bing_web = PyMsCognitiveWebSearch(key, query) bing_Image = PyMsCognitiveImageSearch(key, query) first_ten_result = bing_web.search(limit=10, format='json') #1-10 first_ten_image = bing_Image.search(limit=10, format='json') #1-10 print(first_ten_image[0].name)
def query(query):
search_service = PyMsCognitiveWebSearch(keys.get('bing_api_key'), query)
first_fifty_result = search_service.search(limit=10, format='json') #1-50
return [(x.name, x.display_url) for x in first_fifty_result]
import json
import os
import csv
import requests
import re
from urllib.parse import urlparse
from py_ms_cognitive import PyMsCognitiveWebSearch
search_service = PyMsCognitiveWebSearch('', search_term)
import clearbit
clearbit.key = 'sk_7177eb671d9fe0c5cbbc4d6113e2157a'
with open(os.path.abspath('out_1_sample.json'), 'r') as f:
items_json = json.loads(f.read())
csv_f = open('out_1_enriched_sample.csv', 'w')
csv_writer = csv.writer(csv_f, dialect='excel')
# "name": "VelocityEHS", "logo": "https://ga0.imgix.net/logo/o/104315-1467798979-3301337?ixlib=rb-1.0.0&ch=Width%2CDPR&auto=format", "desc": "Environment, Health, Safety (EHS) Management Software", "rating": "4.67", "num_reviews": "3", "website_click_out": "/x/velocityehs-application?route=listing_detail&from_listing=104315", "category": "https://www.getapp.com/business-intelligence-analytics-software/data-visualization/?page=2"},
cols = [
"name", "domain", "getapp_logo", "clearbit_logo", "desc", "rating",
"num_reviews", "cat", "subcat", "getapp_cat_link"
]
csv_writer.writerow(cols)
for json_in in items_json:
csv_out = [None] * len(cols)
# csv_out[cols.index("")] = json_in[""]
csv_out[cols.index("name")] = json_in["name"]
csv_out[cols.index("getapp_logo")] = json_in["logo"]
csv_out[cols.index("desc")] = json_in["desc"]
from py_ms_cognitive import PyMsCognitiveWebSearch
search_term = "Amy Burkhardt"
search_service = PyMsCognitiveWebSearch(
"a185959d275247529ba4bb965f9f56ce",
'"Five-star technology solutions" AND "New York City Department of Education" AND assessment'
)
first_result = search_service.search(limit=1, format='json') # 1-50
print(first_result[0].title)
print(first_result[0].url)
print('hello world')
def search(query):
bing_web = PyMsCognitiveWebSearch(key, query)
bing_Image = PyMsCognitiveImageSearch(key, query)
first_ten_result = bing_web.search(limit=10, format='json') #1-10
first_ten_image = bing_Image.search(limit=10, format='json') #1-10
return (first_ten_image[0].content_url)