#!/usr/bin/pythom
from dytt8.dytt8 import dytt8
t = dytt8(4)
print t.list_url()
for url in t.http_url():
print url
#!/usr/bin/python
#coding=utf-8
import threading
from Queue import Queue
from dytt8.dytt8 import dytt8
from xunbo.xunbo import xunbo
#下面的dy 就是dytt8的这个class的一个引用，
dy = dytt8(10)
xunbo = xunbo(3)
ftp_urls = []
class ThreadUrl(threading.Thread):
'''
封装多线程库，用来多线程跑啊
'''
def __init__(self,queue,site):
threading.Thread.__init__(self)
self.queue = queue
self.site = site #传递的是一个class的实例或者引用
def run(self):
while True:
try:
url = self.queue.get()
t = self.site.ftp_url(url)
if len(t) > 1:
ftp_urls.append(t)
except:
#codeing=utf-8
from dytt8.dytt8 import dytt8
import sqlite3
db = sqlite3.connect('./spider.db')
link = db.cursor()
link.execute(
'slect * from sqlite_master where type ="table" and name="ftp_url";')
if not link.fetchone():
link.execute(
"""CREAT TABLE 'ftp_url'('id' INTEGER PRIMARY KEY NOT NULL,'url' varchar(120) DEFAULT NULL)"""
)
db.commit()
print("开始爬取")
dytt = dytt8(5)
# for url in urls:
# sql="""INSERT INTO 'ftp_url' value(NULL,'%s');"""%(url)
# link.execute(sql)
db.commit()
db.close()
#!/usr/bin/python
#coding=utf-8
import threading
from Queue import Queue
from dytt8.dytt8 import dytt8
from xunbo.xunbo import xunbo
#下面的dy 就是dytt8的这个class的一个引用，
dy = dytt8(10)
xunbo = xunbo(3)
ftp_urls = []
class ThreadUrl(threading.Thread):
'''
封装多线程库，用来多线程跑啊
'''
def __init__(self, queue, site):
threading.Thread.__init__(self)
self.queue = queue
self.site = site #传递的是一个class的实例或者引用
def run(self):
while True:
try:
url = self.queue.get()
t = self.site.ftp_url(url)
if len(t) > 1:
ftp_urls.append(t)
except:
#!/usr/bin/pythom from dytt8.dytt8 import dytt8 t = dytt8(4) print t.list_url() for url in t.http_url(): print url
#!/usr/bin/python
#coding=utf-8
from dytt8.dytt8 import dytt8
from xunbo.xunbo import xunbo
from mythread import mythread
dytt8 = dytt8(5)
#print "开始抓取迅播前三页的电影链接。。。"
#xunbo = xunbo(3)
print "begin print"
ftp_urls = mythread(dytt8)
file = open('test.txt','w+')
for ftp_url in ftp_urls:
file.write((ftp_url + '\n').encode('utf-8'))
file.flush()
file.close()