Python url_list_fetch示例

示例#1

文件： crawler_master.py 项目： TaikorRoy/china_court

# -*- coding: utf-8 -*-
"""
Created on Tue Aug 25 11:04:51 2015

@author: Administrator
"""

from url_list_fetch import url_list_fetch
import time
from mysql_updater_baseClass import MySQLUpdater
from check_remote_mysql import check_remote

mysql = MySQLUpdater()

starting_page_num = 1
ending_page_num = 90

existed = []
pages = range(starting_page_num, ending_page_num+1)
pages.reverse()
for page in pages:
    url_list_fetch(page, existed, db = None)
    time.sleep(2)

mysql.clean_up()

示例#2

# -*- coding: utf-8 -*-

from url_list_fetch import url_list_fetch
import time
from mysql_updater_baseClass import MySQLUpdater
from check_remote_mysql import check_remote

time_span = 1*3600

pages = [1, 2, 3, 4, 5]
while True:
    # mysql = MySQLUpdater()
    # existed = mysql.pull_records()
    try:
        existed = check_remote(100)
    except:
        print("Remote check-court-record API unavailable")
        time.sleep(300)
        continue

    for page in pages:
        url_list_fetch(page, existed)
    # mysql.clean_up()
    time.sleep(time_span)

示例#3

# -*- coding: utf-8 -*-
"""
Created on Tue Aug 25 11:04:51 2015

@author: Administrator
"""

from url_list_fetch import url_list_fetch
import time
from mysql_updater_baseClass import MySQLUpdater
from check_remote_mysql import check_remote

mysql = MySQLUpdater()

starting_page_num = 1
ending_page_num = 90

existed = []
pages = range(starting_page_num, ending_page_num + 1)
pages.reverse()
for page in pages:
    url_list_fetch(page, existed, db=None)
    time.sleep(2)

mysql.clean_up()