def genRunner(url):
'''This module runs the process'''
page = makePage(url)
if page:
soup = makeSoup(page, url, 'html5lib')
if not soup.body or not soup.head:
print 'no body or no head found...'
else:
CONTENT = getAllContent(soup)
CONTENT = postprocContent(CONTENT)
return CONTENT
def genRunner(url):
'''This module runs the algorithm'''
# Why am I not using soup stuff??
page = makePage(url)
if page:
soup = makeSoup(page, url, 'html5lib')
if not soup.body or not soup.head:
print 'no body or no head found...'
else:
meta_info, CONTENT = getAllContent(soup)
if meta_info and CONTENT:
CONTENT = postprocContent(CONTENT)
return meta_info, CONTENT
else:
return None, None
import sys, os
import prepwork
import alltheMeta
def mergeOthDict(mod, o):
'''merges the new "other" dict, 'o', with the running "other" dict, mod'''
for key in o.keys():
if mod.has_key(key):
mod[key] = mod[key].union(o[key])
else:
mod[key] = o[key]
return mod
if __name__=="__main__":
url_file = sys.argv[1]
mod = dict()
if os.path.isfile(url_file):
with open(url_file, 'r') as f1:
urls = f1.read()
urls = urls.split('\n')
urls = [url.strip() for url in urls if url.strip()]
for url in urls:
page = prepwork.makePage(url)
soup = prepwork.makeSoup(page, url)
_, o = alltheMeta.getAllTheMeta(soup)
mod = mergeOthDict(mod, o)
for key in mod.keys():
print key
print mod[key]
print '\n'
