class SpiderMain(object):
def __init__(self):
self.manager = URLManager()
self.down = Download()
self.parser = HtmlParser()
self.output = DataOutput()
def crawl(self, root_url):
content = self.down.download(root_url)
movie_ids = self.parser.parse_urls(content)
count = 0
for mid in movie_ids:
if count > 10:
break
movie_link = '''http://service.library.mtime.com/Movie.api?\
Ajax_CallBack=true\
&Ajax_CallBackType=Mtime.Library.Services\
&Ajax_CallBackMethod=GetMovieOverviewRating\
&Ajax_CrossDomain=1\
&Ajax_RequestUrl=http%3A%2F%2Fmovie.mtime.com%2F{0}%2F\
&t={1}\
&Ajax_CallBackArgument0={2}\
'''.format(mid, datetime.datetime.now().strftime("%Y%m%d%H%M%S%f"), mid)
res = self.down.download(movie_link.replace(' ', ''))
self.parser.parser_json(res)
count += 1
self.output.store_data(self.parser.items)
self.output.close_connect()
from common import Download
from parsers import HtmlParser
dl = Download()
parse = HtmlParser()
content = dl.download('http://theater.mtime.com/China_Beijing/')
res = parse._parse_movies(content)
print(res)