def transfer(code):
try:
#把dv的原始数据变换一下格式放到dv2，原始格式不好看
client = MongoClient()
db = client['stock2']
collection = db['mv']
cursor = collection.find({"_id": code})
date = None
mv = None
for c in cursor:
date = c["date"]
mv = c["market_value"]
#去掉起始的sh
out = {"_id": code[2:], "data": []}
if len(date) == len(mv):
index = 0
for index in range(len(date)):
tmp = {}
tmp["date"] = date[index]
tmp["timestamp"] = util.utils.TimeString2TimeStamp(date[index])
tmp["mv"] = float(mv[index])
out["data"].append(tmp)
util.saveMongoDB2(out, "stock2", "mv2")
else:
print("wrong date {}".format(code))
except Exception as e:
print(e)
def transfer(code):
#把dv的原始数据变换一下格式放到dv2，原始格式不好看
client = MongoClient()
db = client['stock2']
collection = db['dv']
cursor = collection.find({"_id": code})
date = None
allPrice = None
dv = None
for c in cursor:
date = c["date"]
allPrice = c["price"]
dv = c["dv"]
#去掉起始的sh
out = {"_id":code[2:], "data": []}
if len(date) == len(allPrice) and len(date) == len(dv):
index = 0
for index in range(len(date)):
tmp = {}
tmp["date"] = date[index]
tmp["timestamp"] = util.utils.TimeString2TimeStamp(date[index])
tmp["dv"] = dv[index]
tmp["price"] = float(allPrice[index])
out["data"].append(tmp)
util.saveMongoDB2(out, "stock2", "dv2")
else:
print("wrong date {}".format(code))
def Save(out):
for k, one in out.items():
for k2, two in one.items():
data = {}
data["_id"] = str(k) + ":" + str(k2)
data["counter"] = two["counter"]
data["list"] = two["list"]
util.saveMongoDB2(data, "snowball", "zh_digest")
def Copy():
client = MongoClient()
db = client["ald"]
data = toplist_dict.GetList()
for one in data:
collection = db[one['key']]
cursor = collection.find()
for c in cursor:
# 把其他信息写到db中
one = {}
one["_id"] = c["id"]
one["appkey"] = c["appkey"]
one["name"] = c["name"]
one["category"] = c["category"]
# one["desc"] = c["desc"]
print("save data {} ".format(one["name"]))
util.saveMongoDB2(one, "ald", "detail")
def ProcessRank2():
# 找到大于98分为的组合
client = MongoClient()
db = client['snowball']
collection = db['zh_one']
out = {}
index = 0
cursor = collection.find()
for one in cursor:
try:
barrel = int(one["_id"][2:3])
rank = one["rank"]
if rank >= 98:
out["_id"] = one["_id"][2:]
out["rank"] = rank
util.saveMongoDB2(out, "snowball", "zh_digest98")
except Exception as e:
print(e)
def processFirstPage(self, response):
if response.ok == False:
self.notOK += 1
if self.notOK > 20:
print(response.url)
return
try:
data1 = response.content[1:-1]
data2 = data1.decode("ascii")
data3 = data2.replace("\\", "")
jsonData = json.loads(data3) # , encoding='GB2312')
urls = response.url.split('/')
jsonData["_id"] = urls[-1]
util.saveMongoDB2(jsonData, "stock2", "mv")
except UnicodeDecodeError as e:
print(e)
except Exception as e:
print(e)