def get_center_pane(self):
if self.list is None:
self.db = Database()
self.list = FactivaListCtrl(self.parent,
self.db,
style=wx.LC_REPORT | wx.BORDER_NONE)
return self.list
def get_data(self, url):
db = Database()
r = requests.get(url, headers=self.headers)
soup = BeautifulSoup(r.text, 'html.parser')
items = soup.find_all(self.find_tag)
LOG.debug('found %s' % len(items))
recs = []
for p in items:
cts = p.contents
ip = cts[3].string if cts[3].string else ''
port = cts[5].string if cts[5].string else ''
city = cts[7].a.string if cts[7].a else ''
conn_type = cts[11].string if cts[11].string else ''
speed = cts[13].div['title']
speed = speed[:len(speed) - 1]
o = {
'ip': ip,
'port': port,
'country': 'cn',
'city': city,
'speed': speed,
'conn_type': conn_type
}
recs.append(o)
num_insert, num_update = db.insert_update(recs, 'ippool_exist',
'ippool_insert',
'ippool_update',
['ip', 'port'])
def OnInit(self):
t = Timeit()
LOG.debug("OnInit")
self.cfg_path = 'cfg.json'
# init config file
cfg = Config(
self.cfg_path, {
'version': '1.0.0',
'database': {
'dbtype': 'sqlite',
'dbname': 'data.db',
'dbuser': '',
'dbpass': '',
'dbaddr': 'localhost',
},
'ippool': {
'urls': ['http://www.xicidaili.com/nn/'],
'interval': 60,
}
}).data
# init db
Database(cfg['database']).init_db()
frm = RibbonFrame(None,
-1,
'爬虫工具集 v0.1 [email protected]',
size=(1024, 768))
frm.Show()
LOG.debug('Elapsed time: %f ms' % t.end())
return True
def update(self, o):
idx = o['idx']
rec = o['rec']
db = Database()
db.query('ippool_update_valid', id=rec['id'], valid=rec['valid'])
db.close()
self.data[idx] = rec
self.RowChanged(idx)
def load_data(self):
db = Database()
r = db.query('ippool_count')[0]
c = r.c if r.c is not None else 0
s = r.s if r.s is not None else 0
f = r.f if r.f is not None else 0
self.led_total.set_value(c)
self.led_succ.set_value(s)
self.led_fail.set_value(f)
def records(self):
db = Database()
rows = db.query('ippool_all')
rows = rows.as_dict()
if len(rows) == 0:
return []
delt = rows[len(rows) - 1]['speed'] - rows[0]['speed']
rows = [[
d['country'], d['ip'], d['port'], d['city'],
100 - int(100 * (d['speed'] / delt)), d['conn_type']
] for d in rows]
return rows
def OnInit(self):
t = Timeit()
LOG.info("OnInit")
# load locale from cur dir in locale
# TODO mac, place locale in app
locale = wx.Locale(wx.LANGUAGE_CHINESE_SIMPLIFIED)
if locale.IsOk():
locale.AddCatalogLookupPathPrefix('locale')
locale.AddCatalog('app')
self.cfg_path = 'cfg.json'
# init config file
self.cfg = Config(self.cfg_path, {
'version': '1.0.0',
'database': {
'dbtype': 'sqlite',
'dbname': 'data.db',
'dbuser': '',
'dbpass': '',
'dbaddr': 'localhost',
},
'ippool': {
'urls': ['http://www.xicidaili.com/nn/'],
'interval': 60,
}
}).data
# init db
Database(self.cfg['database']).init_db()
# from ui.ribbonframe import RibbonFrame
# frm = RibbonFrame(None, -1, '爬虫工具集 v0.1 [email protected]', size=(1024, 768))
# frm.Show()
from ui.auiframe import AuiFrame
frm = AuiFrame(None, -1, _('Personal Python Tools'), size=(1024, 768))
frm.Show()
LOG.info('Elapsed time: %f ms' % t.end())
return True
self.thread_stop = False
LOG.debug('Start proxy spider')
step = 1
urls = []
for idx, url in enumerate(self.urls):
if self.thread_stop:
break
urls.append(url)
if idx % step == 0:
LOG.debug(urls)
pool = ThreadPool(self.pool_size)
results = pool.map(self.proc_func, urls)
# close the pool and wait for the work to finish
pool.close()
pool.join()
urls[:] = []
SIG_REFRESH.send(self)
time.sleep(self.interval)
LOG.debug('Finish proxy spider')
def stop(self):
self.thread_stop = True
if __name__ == '__main__':
cfg_path = 'cfg.json'
cfg = Config(cfg_path).data
Database(cfg['database']).init_db()
spider = ProxySpider(cfg_path)
spider.start()
def update_record(self, rec):
ip = rec[1]
port = rec[2]
valid = rec[4]
db = Database()
rows = db.query('ippool_update_valid', ip=ip, port=port, valid=valid)
def get_all_record(self):
db = Database()
rows = db.query('ippool_all')
rows = rows.as_dict()
rows = [ [i, d['ip'], d['port'], d['conn_type'], d['valid'] ] for i, d in enumerate(rows)]
return rows
def instance(cls):
db = Database()
rows = db.query('ippool_all')
rows = rows.as_dict()
db.close()
return TestGridModel(rows)