Python Spider.run Examples

Example #1

File: Main.py Project: caicaicai/pyvoter

def main():
    try:
        opts, args = getopt.getopt(sys.argv[1:], "svh", ["help", "output="])
    except getopt.GetoptError as err:
        # print help information and exit:
        print(str(err)) # will print something like "option -a not recognized"
        usage()
        sys.exit(2)
    for o, a in opts:
        if o == "-s":
            s = Spider()
            s.run()
        elif o == "-v":
            voter = Voter()
            voter.run()
        else:
            assert False, "unhandled option"

Example #2

File: main.py Project: Elnokrashy131/chinese-independent-blogs-2-opml

from Spider import Spider
from save_as_opml import save_to_opml

if __name__ == "__main__":
    spider = Spider()
    content_list = spider.run()
    save_to_opml(content_list, spider.tag_set, spider.name_list)

Example #3

import asyncio
import atexit

from Spider import Spider

if __name__ == '__main__':
    loop = asyncio.get_event_loop()
    root_spider = Spider(loop, "https://exponea.com")
    loop.run_until_complete(root_spider.run())
    atexit.register(loop.close)

Example #4

File: main.py Project: handongke/leetcode-spider-1

#!/usr/bin/python
# coding:utf-8

from Spider import Spider

spider = Spider('/Users/tongchao/Documents/leetcode/LeetCode/concurrency/')
spider.run()