def export_domains(site):
db_export = CrawlerDb(site)
db_export.connect()
logger.info("=" * 40)
logger.info("Processing...")
domains = db_export.get_all_domains()
logger.info("There are %d domains" % len(domains))
file = open(DOMAINS_FILENAME, "w+")
file.writelines("\n".join(domains))
file.close()
logger.info("All domains saved to ./data/domains.csv")
logger.info("=" * 40)
def export_emails(site):
# Set up the database
db_export = CrawlerDb(site)
db_export.connect()
logger.info("=" * 40)
logger.info("Processing...")
emails = db_export.get_all_emails()
logger.info("There are %d emails" % len(emails))
file = open(EMAILS_FILENAME, "w+")
file.writelines("\n".join(emails))
file.close()
logger.info("All emails saved to ./data/emails.csv")
logger.info("=" * 40)
def crawl(site, keywords, output_ui: OutputUIInterface = None):
# Set up the database
global db
db = CrawlerDb(site)
db.connect()
"""
This method will
1) Google the keywords, and extract MAX_SEARCH_RESULTS
2) For every result (aka website), crawl the website 2 levels deep.
That is the homepage (level 1) and all it's links (level 2).
But if level 1 has the email, then skip going to level 2.
3) Store the html in /data/html/ and update the database of the crawled emails
crawl(keywords):
Extract Google search results and put all in database
Process each search result, the webpage:
Crawl webpage level 1, the homepage
Crawl webpage level 2, a link away from the homepage
Update all crawled page in database, with has_crawled = True immediately
Store the HTML
"""
logger.info("-" * 40)
# logger.info("Keywords to Google for: %s" % keywords.decode('utf-8'))
logger.info("Keywords to Google for: %s" % keywords)
logger.info("-" * 40)
# Step 1: Crawl Google Page
# eg http://www.google.com/search?q=singapore+web+development&start=0
# Next page: https://www.google.com/search?q=singapore+web+development&start=10
# Google search results are paged with 10 urls each. There are also adurls
for page_index in range(0, MAX_SEARCH_RESULTS, 10):
query = {'q': keywords}
url = 'http://%s/search?' % site
url = url + urllib.parse.urlencode(query) + '&start=' + str(page_index)
# query = {'wd': keywords}
# url = 'http://www.baidu.com/s?' + urllib.parse.urlencode(query) + '&pn=' + str(page_index)
try:
data = retrieve_html(url)
# print("data: \n%s" % data)
for url in google_url_regex.findall(data):
db.enqueue(str(url))
for url in google_adurl_regex.findall(data):
db.enqueue(str(url))
except Exception as e:
logger.error(e)
# for url in baidu_url_regex.findall(data):
# db.enqueue(str(url))
# for url in baidu_adurl_regex.findall(data):
# db.enqueue(str(url))
# Step 2: Crawl each of the search result
# We search till level 2 deep
while (True):
# Dequeue an uncrawled webpage from db
uncrawled = db.dequeue()
if (uncrawled == False):
break
email_set = find_emails_2_level_deep(uncrawled.url, output_ui)
if (len(email_set) > 0):
db.crawled(uncrawled, ",".join(list(email_set)))
if output_ui:
output_ui.append(list(email_set))
else:
db.crawled(uncrawled, None)
google_adurl_regex = re.compile('adurl=(.*?)"')
google_url_regex = re.compile('url\?q=(.*?)&sa=')
email_regex = re.compile('([A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,4})', re.IGNORECASE)
url_regex = re.compile('<a\s.*?href=[\'"](.*?)[\'"].*?>')
# Below url_regex will run into 'Castrophic Backtracking'!
# http://stackoverflow.com/questions/8010005/python-re-infinite-execution
# url_regex = re.compile('<a\s(?:.*?\s)*?href=[\'"](.*?)[\'"].*?>')
# Maximum number of search results to start the crawl
MAX_SEARCH_RESULTS = 150
EMAILS_FILENAME = 'data/emails.csv'
DOMAINS_FILENAME = 'data/domains.csv'
# Set up the database
db = CrawlerDb()
db.connect()
def crawl(keywords):
"""
This method will
1) Google the keywords, and extract MAX_SEARCH_RESULTS
2) For every result (aka website), crawl the website 2 levels deep.
That is the homepage (level 1) and all it's links (level 2).
But if level 1 has the email, then skip going to level 2.
3) Store the html in /data/html/ and update the database of the crawled emails
crawl(keywords):
Extract Google search results and put all in database
google_adurl_regex = re.compile('adurl=(.*?)"')
google_url_regex = re.compile('url\?q=(.*?)&sa=')
email_regex = re.compile('([A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,4})', re.IGNORECASE)
url_regex = re.compile('<a\s.*?href=[\'"](.*?)[\'"].*?>')
# Below url_regex will run into 'Castrophic Backtracking'!
# http://stackoverflow.com/questions/8010005/python-re-infinite-execution
# url_regex = re.compile('<a\s(?:.*?\s)*?href=[\'"](.*?)[\'"].*?>')
# Maximum number of search results to start the crawl
MAX_SEARCH_RESULTS = 500
EMAILS_FILENAME = 'data/emails.csv'
DOMAINS_FILENAME = 'data/domains.csv'
# Set up the database
db = CrawlerDb()
db.connect()
def crawl(keywords):
"""
This method will
1) Google the keywords, and extract MAX_SEARCH_RESULTS
2) For every result (aka website), crawl the website 2 levels deep.
That is the homepage (level 1) and all it's links (level 2).
But if level 1 has the email, then skip going to level 2.
3) Store the html in /data/html/ and update the database of the crawled emails
crawl(keywords):
Extract Google search results and put all in database
google_adurl_regex = re.compile('adurl=(.*?)"')
google_url_regex = re.compile('url\?q=(.*?)&sa=')
email_regex = re.compile('([A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,4})', re.IGNORECASE)
url_regex = re.compile('<a\s.*?href=[\'"](.*?)[\'"].*?>')
# Below url_regex will run into 'Castrophic Backtracking'!
# http://stackoverflow.com/questions/8010005/python-re-infinite-execution
# url_regex = re.compile('<a\s(?:.*?\s)*?href=[\'"](.*?)[\'"].*?>')
# Maximum number of search results to start the crawl
MAX_SEARCH_RESULTS = 150
EMAILS_FILENAME = 'data/emails.csv'
DOMAINS_FILENAME = 'data/domains.csv'
# Set up the database
db = CrawlerDb()
db.connect()
def crawl(keywords):
"""
This method will
1) Google the keywords, and extract MAX_SEARCH_RESULTS
2) For every result (aka website), crawl the website 2 levels deep.
That is the homepage (level 1) and all it's links (level 2).
But if level 1 has the email, then skip going to level 2.
3) Store the html in /data/html/ and update the database of the crawled emails
crawl(keywords):
Extract Google search results and put all in database
def crawl(keywords,name):
"""
This method will
1) Google the keywords, and extract MAX_SEARCH_RESULTS
2) For every result (aka website), crawl the website 2 levels deep.
That is the homepage (level 1) and all it's links (level 2) and all it's link's links (level 3).
3) Store the html in /data/html/ and update the database of the crawled emails
crawl(keywords):
Extract Google search results and put all in database
Process each search result, the webpage:
Crawl webpage level 1, the homepage
Crawl webpage level 2, a link away from the homepage
Crawl webpage level 3, a link away from the homepage
Update all crawled page in database, with has_crawled = True immediately
Store the HTML
"""
if internet_on() == False:
print "*******************NO INTERNET CONNECTION*******************"
sys.exit(0)
db = CrawlerDb()
db.connect()
logger.info("-"*40)
logger.info("Keywords to Google for: %s" % keywords)
logger.info("-"*40)
# Step 1: Crawl Google Page
# eg http://www.google.com/search?q=singapore+web+development&start=0
# Next page: https://www.google.com/search?q=singapore+web+development&start=10
# Google search results are paged with 10 urls each. There are also adurls
for page_index in range(0, MAX_SEARCH_RESULTS, 10):
query = {'q': keywords}
url = 'http://www.google.com/search?' + urllib.urlencode(query) + '&start=' + str(page_index)
data = retrieve_html(url)
# print("data: \n%s" % data)
for url in google_url_regex.findall(data):
db.enqueue(url)
break
# Step 2: Crawl each of the search result
# We search till level 2 deep
while (True):
# Dequeue an uncrawled webpage from db
uncrawled = db.dequeue()
if (uncrawled == False):
break
email_set = find_emails_3_level_deep(uncrawled.url,db)
if (len(email_set) > 0):
db.crawled(uncrawled, ",".join(list(email_set)))
else:
db.crawled(uncrawled, None)
write(name,db)