def getBuildList(self, itemName, limit=5, source=''):
#获取build基本信息
db = WarframeDB()
itemInfo = db.getBuildItemlikeName(itemName)
if len(itemInfo) == 0:
return '', '', None
itemType = itemInfo[0]['item_type']
itemBuildId = itemInfo[0]['build_id']
nameEn = itemInfo[0]['name_en']
nameZh = itemInfo[0]['name_zh']
#解析url中的mod装配并格式化
#先尝试取DB的记录，如果DB没有或者太老，则取网站上的
#多抓取2个，存到数据库里。但是最终只返回limit个
records = self.getBuildListFromDb(itemBuildId, itemType,
self.BUILD_RECORD_NUM, itemName)
if records == []:
records = self.getBuildListFromUrl(itemBuildId, itemType, nameEn,
nameZh, self.BUILD_RECORD_NUM)
#这里需要经过推荐算法，从N个中，选取M个，格式化，然后返回
finalRecords = records[0:limit]
for rec in finalRecords:
build = self.getBuildFromUrl(rec['url'])
rec['build'] = self.buildDictToStr(build)
rec['build_dict'] = build
return nameEn, nameZh, finalRecords
def getPriceFromDb(self, nameEn, itemType):
wfdb = WarframeDB()
startTime = time.time()
dbprice = wfdb.getPriceByNameEnAndType(nameEn, itemType)
if dbprice is not None:
dbprice['top_rec'] = dbprice['top_rec'].replace('|', '个\n')
dbprice['top_rec'] = dbprice['top_rec'].replace(':', ' : ')
pTime = str(time.time() - startTime)
logTime = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time.time()))
print "[ProcessLog][%s][Price][Item:%s][ItemType:%s][Source:DB][Ptime:%s]" % (
logTime, nameEn, itemType, pTime)
return dbprice
def getItemImgByZh(self, nameZh):
sql = """SELECT item_img from item where name_zh = '%s' limit 1 """ % (
nameZh)
res = WarframeDB().queryBySql(sql)
if len(res) == 0:
return 'None'
return res[0][0]
def getBuildListFromDb(self, itemBuildId, itemType, limit, nameEn):
startTime = time.time()
after = time.time() - self.BUILD_DB_RECORD_EXPIRE_TIME
res = WarframeDB().getBuildByTime(itemBuildId, itemType, after, limit)
pTime = str(time.time() - startTime)
logTime = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time.time()))
print "[ProcessLog][%s][Build][Item:%s][ItemType:%s][Source:DB][Ptime:%s]" % (
logTime, nameEn, itemType, pTime)
return res
def getPriceSimpleStatistic(self, nameEn, gapTime):
gapTimeStamp = int(time.time() - gapTime)
getRecSql = "SELECT cheapest_price,record_time FROM item_price_record WHERE name_en = '%s' AND UNIX_TIMESTAMP(record_time)>'%s' order by record_time asc " % (
nameEn, gapTimeStamp)
records = WarframeDB().queryBySql(getRecSql)
#PIRCE_STATISTIC_COUNT_MIN = 20 #只有当价格记录大于这个值，才会计算最高最低价格
if len(records) < self.PIRCE_STATISTIC_COUNT_MIN:
return None
highest = 0
lowest = 9999
sumP = 0
for r in records:
highest = max(r[0], highest)
lowest = min(r[0], lowest)
sumP += r[0]
res = {}
res['highest'] = highest
res['lowest'] = lowest
res['avg'] = sumP / len(records)
return res
def getPriceFromWm(self, nameEn, itemType):
startTime = time.time()
nameEnQuote = urllib.quote(nameEn)
url = 'http://warframe.market/api/get_orders/' + itemType + '/' + nameEnQuote
try:
req = urllib2.Request(url)
resp = urllib2.urlopen(req)
html = resp.read()
#conn = httplib.HTTPConnection('warframe.market')
#conn.request('GET','/api/get_orders/'+itemType+'/'+nameEn)
#html = conn.getresponse()
data = json.loads(html)
except:
print url
return None
if data['code'] != 200:
return None
sellInfo = data['response']['sell']
if len(sellInfo) == 0:
return None
#get online player records
onlineSellRec = []
onlineSellRecSum = 0
onlineSellRecCount = 0
for info in sellInfo:
if info['online_ingame'] == False:
continue
#ignore xbox and ps4 record
nameStr = info['ingame_name'].encode("utf-8")
nameStr = urllib.unquote(nameStr)
if nameStr.startswith('(PS4)') or nameStr.startswith('(XB1)'):
continue
#else
onlineSellRec.append(info)
onlineSellRecSum += info['price'] * info['count']
onlineSellRecCount += info['count']
#sort and analysis
if len(onlineSellRec) == 0:
return None
onlineSellRec.sort(key=lambda obj: obj.get('price'), reverse=False)
res = {}
res['top_rec'] = ""
topSellerSum = min(self.TOP_SELLER_NUM, len(onlineSellRec))
for i in range(0, topSellerSum):
strT = self.sellerRecFormat(onlineSellRec[i])
res['top_rec'] += strT
res['cheapest_price'] = onlineSellRec[0]['price']
res['all_count'] = onlineSellRecCount
res['all_avg'] = onlineSellRecSum / onlineSellRecCount
topSum = 0
topCount = 0
for rec in onlineSellRec[0:20]:
topSum += rec['price'] * rec['count']
topCount += rec['count']
res['top_count'] = topCount
res['top_sum'] = topSum
res['top_avg'] = topSum / topCount
res['record_time'] = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time.time()))
res['source'] = 'url'
#查询结果计入数据库
wfdb = WarframeDB()
itemIdName = wfdb.getItemLikeName(nameEn)
res['itemId'] = itemIdName[0]['id']
res['item'] = nameEn
res['category'] = itemIdName[0]['type']
wfdb.insertPrice(res)
pTime = str(time.time() - startTime)
logTime = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time.time()))
print "[ProcessLog][%s][Price][Item:%s][ItemType:%s][Source:WM][Ptime:%s]" % (
logTime, nameEn, itemType, pTime)
return res
def getBuildListFromUrl(self, itemBuildId, itemType, nameEn, nameZh,
limit):
startTime = time.time()
#print time.time()
bs = BuildStatic()
typeTxt = bs.typeTxtMap[itemType]
dataInfoPrefix = 't_30_3400020000'
dataInfoStr = '%s/en/%s-0-%s/' % (dataInfoPrefix, itemType,
itemBuildId)
data = {"infos": dataInfoStr}
headerStr = "http://warframe-builder.com/%s/Builder/" % (typeTxt)
requrl = "http://warframe-builder.com/List_builds"
headers = {
"Referer": headerStr,
'content-type': 'application/json;charset=UTF-8',
'Accept': 'text/html, */*; q=0.01',
'Accept-Encoding': 'gzip, deflate',
'Accept-Language':
'en-GB,en;q=0.8,en-US;q=0.6,zh-CN;q=0.4,zh;q=0.2',
'Connection': 'keep-alive',
'Content-Length': '78',
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'Host': 'warframe-builder.com',
'Origin': 'http://warframe-builder.com',
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36',
'X-Requested-With': 'XMLHttpRequest'
}
dataStr = 'infos=' + dataInfoStr
dataJson = urllib.urlencode(data)
r = requests.post(requrl, data=dataJson, headers=headers)
#print r.text
#print time.time()
tree = etree.HTML(r.text)
resTable = tree.xpath("//tr")
resDict = []
resCount = 0
for tr in resTable:
if resCount > limit:
break
resCount += 1
rec = {}
rec['url'] = self.xpathExtractFirst(tr, "td[1]/a/@href")
if rec['url'] == '':
continue
rec['build_des'] = self.xpathExtractFirst(tr, "td[1]/a/text()")
rec['formas'] = self.xpathExtractFirst(tr, "td[5]/text()")
rec['pop'] = self.xpathExtractFirst(tr, "td[7]/text()")
rec['build_time'] = self.xpathExtractFirst(tr, "td[8]/text()")
#build = self.getBuildFromUrl(rec['url'])
#rec['build'] = self.buildDictToStr(build)
resDict.append(rec)
#print rec['build']
#将url获得的结果保存到db做缓存
rec['item_type'] = itemType
rec['build_item_id'] = itemBuildId
rec['name_en'] = nameEn
rec['name_zh'] = nameZh
rec['build_des'] = rec['build_des'].replace('\'', '\\\'')
WarframeDB().insertBuildRecord(rec)
pTime = str(time.time() - startTime)
logTime = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time.time()))
print "[ProcessLog][%s][Build][Item:%s][ItemType:%s][Source:DB][Ptime:%s]" % (
logTime, nameEn, itemType, pTime)
return resDict
#!/usr/bin/python
# -*- coding:utf-8 -*-
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
from lxml import etree
import requests
import json
import urllib
import MySQLdb
import types
import urllib2
import time
import httplib
from db import WarframeDB
from translator import WmTranslator
from BuildStatic import BuildStatic
tr = WmTranslator()
wdb = WarframeDB()
items = wdb.getBuildItemlikeName('')
for it in items:
zh = tr.en2zh(it['name_en'])
sql = """UPDATE build_item set name_zh='%s' where id=%s""" % (zh, it['id'])
print sql
print wdb.queryBySql(sql)