class CompanyPrice: def __init__(self): self.sinaApi = 'http://hq.sinajs.cn/list=%s' self.db = DbConn() def getPriceInfo(self, code): r = random.uniform(0, 3) time.sleep(r) url = self.sinaApi % (code) page = Crawler.getPage(url, "gbk") tmp = page.split('"') return tmp[1].split(',') def getCompanyList(self): sql = "select code,type from company;" sqlIn = """insert into price(companyid,price,createtime) values('%s','%s','%s')""" self.db.execute(sql) res = self.db.getRes() for i in res: if 'sh' == i[1]: post = 'sh' + i[0] else: post = 'sz' + i[0] priceInfo = self.getPriceInfo(post) if len(priceInfo) < 31: print i[0] + " failed!" continue currentPrice = priceInfo[3] currentTime = priceInfo[30] cmd = sqlIn % (i[0],currentPrice,currentTime) self.db.execute(cmd) print i[0], currentPrice, currentTime
class StockList: def __init__(self, url): self.url = url self.db = DbConn() def close(self): self.db.close() def getIndexPage(self): return Crawler.getPage(self.url, "gbk") def getResult(self, res): arr = [] for item in res: tmp = item.text l = len(tmp) name = tmp[:l - 8].replace(' ', '') code = tmp[l - 7:l - 1].replace(' ', '') arr.append((name, code)) return arr def getCompany(self): page = self.getIndexPage() sql = """insert into company(code,name,type) values('%s','%s','%s')""" soup = BeautifulSoup(page) liangshi = soup.findAll('div', attrs={'class': 'result'}) hushi = liangshi[0].find('ul').findAll('li') shenshi = liangshi[1].find('ul').findAll('li') chuangye = liangshi[2].find('ul').findAll('li') arr = self.getResult(hushi) for i in arr: cmd = sql % (i[1], i[0], 'sh') self.db.execute(cmd) arr = self.getResult(shenshi) for i in arr: cmd = sql % (i[1], i[0], 'sz') self.db.execute(cmd) arr = self.getResult(chuangye) for i in arr: cmd = sql % (i[1], i[0], 'cz') self.db.execute(cmd)