def fetch_cnstock_hist_to_mongo():
'''get qoute data '''
#set log
LOGGER_NAME = "TONGLIAN_DATA"
mylog = logger.getLogger(LOGGER_NAME)
#get the stock list
today = datetime.strftime(datetime.today(),"%Y%m%d")
mongo = Mongo()
db = mongo.getDB()
cursor = db.stock_list.find({"exchangeCD":{"$in":["XSHE","XSHG"]},"listStatusCD":"L"})
for row in cursor:
ticker = str(row['ticker'])
mylog.info("update history data of %s"%(ticker))
exchangeCD = str(row['exchangeCD'])
listDate = str(row['listDate']).replace("-", "").replace("NaN", "")
if exchangeCD == 'XSHG' and not ticker.startswith("6"):
continue
#get hist data
cursor2 = db.cn_stock_hist.find({"ticker":ticker})
if cursor2.count() > 0:
continue
mylog.info("insert data of %s"%(ticker))
st = ts.Market()
df = st.MktEqud(ticker=ticker,beginDate=listDate, endDate=today, field="")
df['ticker'] = df['ticker'].map(lambda x: str(x).zfill(6))
db.cn_stock_hist.insert(json.loads(df.to_json(orient='records')))
time.sleep(1)
def __init__(self):
self.__logger = logger.getLogger("MainReportToMysql")
self.__db = Mysql()
self.__data = {}
self.__zhongwen_2_pinyin = {}
self.__pinyin_2_zhongwen = {}
self.__db.add_table_not_exists("mainreport")
self.__exists_colnums = self.__db.get_columns("mainreport")
def __init__(self):
self.__logger = logger.getLogger("MainReportToMysql")
self.__db = Mysql()
self.__data = {}
self.__zhongwen_2_pinyin = {}
self.__pinyin_2_zhongwen = {}
self.__db.add_table_not_exists("mainreport")
self.__exists_colnums = self.__db.get_columns("mainreport")
def batchUpdateMainReport(stockCodeList):
LOGGER_NAME = "UPDATE_DATA"
mylog = logger.getLogger(LOGGER_NAME)
mongodb = Mongo()
for code in stockCodeList:
time.sleep(1)
mylog.info("update mainreport data of %s" % (code))
mongodb.updateMainReport(code)
def batchUpdateMainReport(stockCodeList):
LOGGER_NAME = "UPDATE_DATA"
mylog = logger.getLogger(LOGGER_NAME)
mongodb = Mongo()
for code in stockCodeList:
time.sleep(1)
mylog.info("update mainreport data of %s"%(code))
mongodb.updateMainReport(code)
def run():
'''get qoute data '''
#set log
LOGGER_NAME = "HISTORY_DATA"
mylog = logger.getLogger(LOGGER_NAME)
#get the stock list
today = datetime.strftime(datetime.today(),"%Y%m%d")
mongo = Mongo()
db = mongo.getDB()
cursor = db.stock_list.find({"listStatusCD":"L"})
for row in cursor:
ticker = str(row['ticker'])
mylog.info("update history data of %s"%(ticker))
exchangeCD = str(row['exchangeCD'])
listDate = str(row['listDate']).replace("-", "").replace("NaN", "")
if exchangeCD == 'XSHG' and not ticker.startswith("6"):
continue
#get hist data
cursor2 = db.cn_stock_hist.find({"ticker":ticker}).sort("tradeDate",pymongo.DESCENDING).limit(1)
st = ts.Market()
if cursor2.count() > 0:
start_date = str(cursor2[0]['tradeDate']).replace("-", "")
df = st.MktEqud(ticker=ticker,beginDate=start_date, endDate=today, field="")
if df is not None and not df.empty:
if df['accumAdjFactor'][0] == 1:
mylog.info("update new data of %s from %s"%(ticker,start_date))
df = df[1:]
else:
mylog.info("factor change remove data of %s"%(ticker))
db.cn_stock_hist.remove({"ticker":ticker})
df = st.MktEqud(ticker=ticker,beginDate=listDate, endDate=today, field="")
else:
mylog.info("fisrt insert data of %s"%(ticker))
df = st.MktEqud(ticker=ticker,beginDate=listDate, endDate=today, field="")
if df is not None and not df.empty:
mylog.info("processing insert of %s"%(ticker))
df['ticker'] = df['ticker'].map(lambda x: str(x).zfill(6))
db.cn_stock_hist.insert(json.loads(df.to_json(orient='records')))
time.sleep(1)
def fetch_cnstock_hist_to_mongo():
'''get qoute data '''
#set log
LOGGER_NAME = "TONGLIAN_DATA"
mylog = logger.getLogger(LOGGER_NAME)
#get the stock list
today = datetime.strftime(datetime.today(), "%Y%m%d")
mongo = Mongo()
db = mongo.getDB()
cursor = db.stock_list.find({
"exchangeCD": {
"$in": ["XSHE", "XSHG"]
},
"listStatusCD": "L"
})
for row in cursor:
ticker = str(row['ticker'])
mylog.info("update history data of %s" % (ticker))
exchangeCD = str(row['exchangeCD'])
listDate = str(row['listDate']).replace("-", "").replace("NaN", "")
if exchangeCD == 'XSHG' and not ticker.startswith("6"):
continue
#get hist data
cursor2 = db.cn_stock_hist.find({"ticker": ticker})
if cursor2.count() > 0:
continue
mylog.info("insert data of %s" % (ticker))
st = ts.Market()
df = st.MktEqud(ticker=ticker,
beginDate=listDate,
endDate=today,
field="")
df['ticker'] = df['ticker'].map(lambda x: str(x).zfill(6))
db.cn_stock_hist.insert(json.loads(df.to_json(orient='records')))
time.sleep(1)
def __init__(self):
self.__logger = logger.getLogger("GuaidianJson")
self.__data_path = './stockdata/mainreport'
self.__choose_stock = {}
def __init__(self):
self.__logger = logger.getLogger("Guaidian")
def __init__(self):
self.__logger = logger.getLogger("GuaidianJson")
self.__data_path = './stockdata/mainreport'
# -*- coding:utf-8 -*-
from quantdata.db.mongo import Mongo
from datetime import datetime
from quantdata import logger
import pandas as pd
import json
mongo = Mongo()
db = mongo.getDB()
#set log
LOGGER_NAME = "get the PB"
mylog = logger.getLogger(LOGGER_NAME)
#get the stocklist
stock_list = db.stock_list.find()
for row in stock_list:
ticker = str(row['ticker'])
mylog.info("processing %s" % (ticker))
stock_hist_list = db.cn_stock_hist.find({"ticker": ticker})
hist_list = []
for stock_row in stock_hist_list:
tradDate = datetime.strptime(stock_row['tradeDate'], "%Y-%m-%d")
if stock_row['PE'] <= 0 or stock_row['PB'] <= 0:
continue
hist_list.append({
'ticker': ticker,
'date': stock_row['tradeDate'],
'pb': stock_row['PB'],
'pe': stock_row['PE'],
'gr_year': tradDate.year,
def __init__(self):
self.__logger = logger.getLogger("MainReportJson")
self.__data_path = './stockdata/mainreport'
if not os.path.exists(self.__data_path):
os.makedirs(self.__data_path)
from quantdata import logger
from quantdata.stock import quotes
from quantdata.db.mysql import Mysql
import threading
import time,json
from urllib.request import urlopen, Request
from quantdata import cons as ct
from pypinyin import lazy_pinyin
LOGGER_NAME = "SPIDER"
log = logger.getLogger(LOGGER_NAME)
def _getMainReportJson(code):
try:
request = Request(ct.THS_MAIN_DATA_URL % (code))
request.add_header("User-Agent", ct.USER_AGENT)
text = urlopen(request, timeout=ct.API_TIMEOUT).read()
text = text.decode('gbk') if ct.PY3 else text
reportObj = json.loads(text.strip())
if isinstance(reportObj,dict) and "title" in reportObj and "report" in reportObj:
return reportObj['title'],reportObj['report']
except Exception as e:
print(e)
def _fetchData(stockCodeList):
db = Mysql()
db.add_table_not_exists("mainreport")
#拼音中文互相转换
pinyin_2_zhongwen = {}
zhongwen_2_pinyin = {}
for code in stockCodeList:
#todo:判断是否要更新当前代码是否要更新