__author__ = 'Master Wang' ''' JSON API definition ''' import json, inspect, functools import pdb #为自定义的模块添加搜索路径 import sys sysFc = 'D:\\python_learn\\sysFc' sys.path.append(sysFc) from logSf10 import crLog logger = crLog(fname='D:\桌面\exCodOut.log') logger.info('Succeed') class Page(object): ''' Page object for display pages. ''' def __init__(self, item_count, page_index=1, page_size=2): ''' Init Pagination by item_count, page_index and page_size. >>> p1 = Page(100, 1) >>> p1.page_count 10 >>> p1.offset 0
# -*- coding:utf-8 -*- from urllib.request import urlopen from bs4 import BeautifulSoup import re, pdb, os, random #为自定义模块添加搜索路径 import sys sysSf = 'D:\\python_learn\\sysFc' sys.path.append(sysSf) from logSf10 import crLog logger = crLog() logger.info('Succeed!!') from fRcd import fRecord os.chdir('d:\\python_learn\\spider') from fc.urlfc import getHtml,getLinks,Rule fgx = '\n======================================分割线======================================\n' urlst = [ "https://www.yandex.com", "http://www.baidu.com", "http://www.1688.com", "https://www.zhihu.com/question/20271508", "http://www.cnblogs.com/wupeiqi/articles/4731930.html", "http://www.baidu.com/s?", 'http://news.dbanotes.net', 'http://opac.nl.cn', 'http://www.zhcw.com', 'http://www.zhihu.com' ]
from urllib.request import urlopen from bs4 import BeautifulSoup import collections import re import time import random # 为自定义的模块添加搜索路径 import sys sysFc = 'D:\\python_learn\\sysFc' sys.path.append(sysFc) from logSf10 import crLog logger = crLog(fname='D:\桌面\isbn.log') # logger = crLog(fname = 'D:\桌面\handlers.log') logger.info('Succeed') # 引入数据库连接 from connSql import mkcon cursor, mkconn = mkcon('mic', database='catalog') cursorE, mkconnE = mkcon('mic', database='easybook') insertCount = 0 def sqlData(num=10, cursor=cursor, mkconn=mkconn): # cursor.execute("SELECT * FROM booklist WHERE publishdate = '1999-12-13'") global cursorE, mkconnE