def Day(facNameList, colList, sql, endDate=None): if endDate is None: dtNow = dt.datetime.now() - dt.timedelta(days=1) endDate = dt.datetime(dtNow.year, dtNow.month, dtNow.day) else: endDate = dt.datetime(endDate.year, endDate.month, endDate.day) if endDate.date() >= dt.datetime.now().date(): endDate = dt.datetime(dt.datetime.now().year, dt.datetime.now().month, dt.datetime.now().day) - \ dt.timedelta(days=1) connStr = Public.GetPara('connStr') conn = co.connect(connStr) cursor = conn.cursor() mongoConn = Public.GetPara('mongoConn') mc = pm.MongoClient(mongoConn) db = mc['factor'] # for each factor for f in range(len(facNameList)): facName = facNameList[f] col = colList[f] lastUpdateDate = Public.GetLastUpdateDate(facName, mc) tradingDateSet = Public.GetCalendar(lastUpdateDate, endDate) mongoDoc = db[facName].find_one({'_id': lastUpdateDate}) mongoDoc = {} if mongoDoc is None else mongoDoc # save data day by day. currentDate = lastUpdateDate + dt.timedelta(days=1) while currentDate <= endDate: cursor.execute( sql.replace('{TRADE_DATE}', currentDate.strftime('%Y-%m-%d'))) dataList = cursor.fetchall() isTrade = (currentDate in tradingDateSet) mongoDoc = { } # if this statement is executed, invalid data will not be filled by previous value mongoDoc['_id'] = currentDate mongoDoc['_isTrade'] = isTrade mongoDoc['_updateTime'] = dt.datetime.now() for record in dataList: symbol = record[1] data = record[col] if data is not None: # and np.isfinite(data): mongoDoc[symbol] = data db[facName].save(mongoDoc) print(facName + ' ' + str(currentDate)) currentDate += dt.timedelta(days=1) db.cfgUpdate.save({'_id': facName, 'lastUpdateDate': endDate}) ddd = 0
def Mom(priceColName, momPrefix, momDaysList, endDate=None): if endDate is None: dtNow = dt.datetime.now() - dt.timedelta(days=1) endDate = dt.datetime(dtNow.year, dtNow.month, dtNow.day) else: endDate = dt.datetime(endDate.year, endDate.month, endDate.day) if endDate.date() >= dt.datetime.now().date(): endDate = dt.datetime(dt.datetime.now().year, dt.datetime.now().month, dt.datetime.now().day) - \ dt.timedelta(days=1) mongoConn = Public.GetPara('mongoConn') mc = pm.MongoClient(mongoConn) db = mc['factor'] for days in momDaysList: facName = momPrefix + '_' + str(days) lastUpdateDate = Public.GetLastUpdateDate(facName, mc) tradingDateSet = Public.GetCalendar(lastUpdateDate, endDate) currentDate = lastUpdateDate - dt.timedelta(days=days) savedDate = lastUpdateDate priceDict = {} while currentDate + dt.timedelta(days=1) <= endDate: currentDate += dt.timedelta(days=1) # if currentDate not in tradingDateSet: # continue # get data record = db[priceColName].find_one({'_id': currentDate}) if record is None: continue for symbol, priceList in priceDict.items(): priceList.append(np.nan) for symbol, price in record.items(): if symbol[0] == '_': continue if symbol not in priceDict.keys(): priceDict[symbol] = [np.nan] priceDict[symbol][-1] = price if currentDate <= lastUpdateDate: continue # evaluate momentum and save to db mongoDoc = { '_id': currentDate, '_updateTime': dt.datetime.now(), '_isTrade': (currentDate in tradingDateSet) } for symbol, priceList in priceDict.items(): arr = np.array(priceList[max(0, len(priceList) - days):]) if np.sum( np.isfinite(arr)) < days / 3: # data is not sufficient continue mom = arr[-1] / np.nanmean(arr[0:len(arr) - 1]) mongoDoc[symbol] = mom db[facName].save(mongoDoc) savedDate = currentDate print(facName + ' ' + str(currentDate)) db.cfgUpdate.save({'_id': facName, 'lastUpdateDate': savedDate})
def CashFlow(): connStr = Public.GetPara('connStr') conn = co.connect(connStr) mongoConn = Public.GetPara('mongoConn') mc = pm.MongoClient(mongoConn) db = mc['factor'] # factor configDictList = [{'facName': 'FIN_OCF_Q', 'column': 5, 'qfData': 2}, # 季度经营活动现金流 {'facName': 'FIN_OCF_QG', 'column': 5, 'qfData': 5}, # 季度经营活动现金流增长率 {'facName': 'FIN_OCF_12M', 'column': 5, 'qfData': 3}, # 12个月经营活动现金流 {'facName': 'FIN_OCE_12MG', 'column': 5, 'qfData': 6}, # 12个月经营活动现金流增长率 {'facName': 'FIN_ICF_Q', 'column': 8, 'qfData': 2}, # 季度投资活动现金流 {'facName': 'FIN_ICF_QG', 'column': 8, 'qfData': 5}, # 季度投资活动现金流增长率 {'facName': 'FIN_ICF_12M', 'column': 8, 'qfData': 3}, # 12个月投资活动现金流 {'facName': 'FIN_ICE_12MG', 'column': 8, 'qfData': 6}, # 12个月投资活动现金流增长率 {'facName': 'FIN_FCF_Q', 'column': 11, 'qfData': 2}, # 季度筹资活动现金流 {'facName': 'FIN_FCF_QG', 'column': 11, 'qfData': 5}, # 季度筹资活动现金流增长率 {'facName': 'FIN_FCF_12M', 'column': 11, 'qfData': 3}, # 12个月筹资活动现金流 {'facName': 'FIN_FCE_12MG', 'column': 11, 'qfData': 6}, # 12个月筹资活动现金流增长率 ] for configDict in configDictList: facName = configDict['facName'] print('==== ' + facName) lastUpdateDate = Public.GetLastUpdateDate(facName, mc) dataList = Public.GetDataList(Public.sqlCash, lastUpdateDate - dt.timedelta(days=365 * 3), dt.datetime.now(), conn) qfDataTuple = Public.QuarterFormat(dataList) column = configDict['column'] dataDict = qfDataTuple[configDict['qfData']] facDict = {} for symbol, data in dataDict.items(): facDict[symbol] = data[:, [0, 1, 2, column]] Public.ToDB(facDict, facName, endDate=None, updateReportDate=False, mongoClient=mc) ddd = 0
def Balance(): connStr = Public.GetPara('connStr') conn = co.connect(connStr) mongoConn = Public.GetPara('mongoConn') mc = pm.MongoClient(mongoConn) db = mc['factor'] # factor configDictList = [ { 'facName': 'FIN_MC', 'column': 3, 'qfData': 1 }, # 货币资金 { 'facName': 'FIN_LA', 'column': 4, 'qfData': 1 }, # 流动资产 { 'facName': 'FIN_FA', 'column': 5, 'qfData': 1 }, # 固定资产 { 'facName': 'FIN_IA', 'column': 6, 'qfData': 1 }, # 无形资产 { 'facName': 'FIN_TNLA', 'column': 7, 'qfData': 1 }, # 非流动资产合计 { 'facName': 'FIN_TA', 'column': 8, 'qfData': 1 }, # 资产总计 { 'facName': 'FIN_STB', 'column': 9, 'qfData': 1 }, # 短期借款 { 'facName': 'FIN_NLL1Y', 'column': 10, 'qfData': 1 }, # 一年内到期的非流动负债 { 'facName': 'FIN_TLL', 'column': 11, 'qfData': 1 }, # 流动负债合计 { 'facName': 'FIN_LTB', 'column': 12, 'qfData': 1 }, # 长期借款 { 'facName': 'FIN_TNLL', 'column': 13, 'qfData': 1 }, # 非流动负债合计 { 'facName': 'FIN_TL', 'column': 14, 'qfData': 1 }, # 负债合计 { 'facName': 'FIN_PUC', 'column': 15, 'qfData': 1 }, # 实收资本(或股本) { 'facName': 'FIN_UDP', 'column': 16, 'qfData': 1 }, # 未分配利润 { 'facName': 'FIN_TOEAPC', 'column': 17, 'qfData': 1 }, # 归母权益合计 { 'facName': 'FIN_MSE', 'column': 18, 'qfData': 1 }, # 少数股东权益 { 'facName': 'FIN_TOE', 'column': 19, 'qfData': 1 }, # 所有者权益合计 { 'facName': 'FIN_TLOE', 'column': 20, 'qfData': 1 }, # 负债和所有者权益合计 { 'facName': 'FIN_IVT', 'column': 21, 'qfData': 1 }, # 存货 { 'facName': 'FIN_TA_G', 'column': 8, 'qfData': 4 }, # 总资产增长率 { 'facName': 'FIN_TL_G', 'column': 14, 'qfData': 4 }, # 总负债增长率 { 'facName': 'FIN_TOEAPC_G', 'column': 17, 'qfData': 4 }, # 归母权益增长率 { 'facName': 'FIN_TOE_G', 'column': 19, 'qfData': 4 }, # 所有者权益增长率 { 'facName': 'FIN_TA_QA', 'column': 8, 'qfData': 7 }, # 季度总资产平均值 { 'facName': 'FIN_TOEAPC_QA', 'column': 17, 'qfData': 7 }, # 季度归母权益平均值 { 'facName': 'FIN_TOE_QA', 'column': 19, 'qfData': 7 }, # 季度所有者权益平均值 { 'facName': 'FIN_IVT_QA', 'column': 21, 'qfData': 7 }, # 季度存货平均值 { 'facName': 'FIN_TA_12MA', 'column': 8, 'qfData': 8 }, # 12个月总资产平均值 { 'facName': 'FIN_TOEAPC_12MA', 'column': 17, 'qfData': 8 }, # 12个月归母权益平均值 { 'facName': 'FIN_TOE_12MA', 'column': 19, 'qfData': 8 }, # 12个月所有者权益平均值 { 'facName': 'FIN_IVT_12MA', 'column': 21, 'qfData': 8 } # 12个月存货平均值 ] for configDict in configDictList: facName = configDict['facName'] print('==== ' + facName) lastUpdateDate = Public.GetLastUpdateDate(facName, mc) dataList = Public.GetDataList( Public.sqlBlc, lastUpdateDate - dt.timedelta(days=365 * 3), dt.datetime.now(), conn) qfDataTuple = Public.QuarterFormat(dataList) column = configDict['column'] dataDict = qfDataTuple[configDict['qfData']] facDict = {} for symbol, data in dataDict.items(): facDict[symbol] = data[:, [0, 1, 2, column]] updateReportDate = True if facName == 'FIN_TA' else False Public.ToDB(facDict, facName, endDate=None, updateReportDate=updateReportDate, mongoClient=mc) ddd = 0
def FinDer(configDictList, endDate): if endDate is None: dtNow = dt.datetime.now() - dt.timedelta(days=1) endDate = dt.datetime(dtNow.year, dtNow.month, dtNow.day) else: endDate = dt.datetime(endDate.year, endDate.month, endDate.day) if endDate.date() >= dt.datetime.now().date(): endDate = dt.datetime(dt.datetime.now().year, dt.datetime.now().month, dt.datetime.now().day) - \ dt.timedelta(days=1) mongoConn = Public.GetPara('mongoConn') mc = pm.MongoClient(mongoConn) db = mc['factor'] # for each factor for configDict in configDictList: facName = configDict['facName'] depFacNameList = configDict['depFacNameList'] EvalFun = configDict['EvalFun'] lastUpdateDate = Public.GetLastUpdateDate(facName, mc) tradingDateSet = Public.GetCalendar(lastUpdateDate, endDate) currentDate = lastUpdateDate + dt.timedelta(days=1) savedDate = lastUpdateDate while currentDate <= endDate: deficientData = False # get all dependent factor symbolSet = set() depFacListDict = {} for d in range(len(depFacNameList)): depFacName = depFacNameList[d] record = db[depFacName].find_one({'_id': currentDate}) if record is None: deficientData = True break for symbol, value in record.items(): if symbol[0] == '_': continue if symbol not in depFacListDict.keys(): depFacListDict[symbol] = np.nan * np.zeros( [len(depFacNameList)]) depFacListDict[symbol][ d] = value if value is not None else np.nan symbolSet = symbolSet | record.keys() if deficientData: break # evaluate factor value and save to db mongoDoc = { '_id': currentDate, '_isTrade': (currentDate in tradingDateSet), '_updateTime': dt.datetime.now() } for symbol in symbolSet: if symbol[0] == '_': # system field continue facValue = EvalFun(depFacListDict[symbol]) mongoDoc[symbol] = facValue db[facName].save(mongoDoc) savedDate = currentDate print(facName + ' ' + str(currentDate)) currentDate += dt.timedelta(days=1) db.cfgUpdate.save({'_id': facName, 'lastUpdateDate': savedDate})
import redis import pymongo as pm import thulac import os import Public dbConfig = Public.GetPara(os.path.join('.', 'config', 'db.txt')) conn = redis.StrictRedis(host=dbConfig['host'], port=dbConfig['port'], db=int(dbConfig['db']), decode_responses=True) mc = pm.MongoClient('mongodb://*****:*****@localhost:27017/text') db = mc['text'] tl = thulac.thulac(user_dict = os.path.join('.', 'dict', 'dict'), filt = False, seg_only=True) config = conn.hgetall('CONFIG_ES') config['_id'] = 'CONFIG_ES' db['CONFIG'].save(config) keys = conn.keys() count = 0 for key in keys: if len(key) <= 7 or key[0 : 7] != 'http://': continue try: item = conn.hgetall(key) except Exception as e: ddd = 0 item['_id'] = key if key[len(key) - 2] == ',' or key[len(key) - 3] == ',': # parse parse = [] cut = tl.cut(item['content']) for c in cut: parse.append(c[0])
def Profit(): connStr = Public.GetPara('connStr') conn = co.connect(connStr) mongoConn = Public.GetPara('mongoConn') mc = pm.MongoClient(mongoConn) db = mc['factor'] # factor configDictList = [ { 'facName': 'FIN_NP_Q', 'column': 12, 'qfData': 2 }, # 季度净利润 { 'facName': 'FIN_NP_QG', 'column': 12, 'qfData': 5 }, # 季度净利润增长率 { 'facName': 'FIN_NP_12M', 'column': 12, 'qfData': 3 }, # 12个月净利润 { 'facName': 'FIN_NP_12MG', 'column': 12, 'qfData': 6 }, # 12个月净利润增长率 { 'facName': 'FIN_NPAPC_Q', 'column': 13, 'qfData': 2 }, # 季度归母净利润 { 'facName': 'FIN_NPAPC_QG', 'column': 13, 'qfData': 5 }, # 季度归母净利润增长率 { 'facName': 'FIN_NPAPC_12M', 'column': 13, 'qfData': 3 }, # 12个月归母净利润 { 'facName': 'FIN_NPAPC_12MG', 'column': 13, 'qfData': 6 }, # 12个月归母净利润增长率 { 'facName': 'FIN_OTR_Q', 'column': 3, 'qfData': 2 }, # 季度营业总收入 { 'facName': 'FIN_OTR_QG', 'column': 3, 'qfData': 5 }, # 季度营业总收入增长率 { 'facName': 'FIN_OTR_12M', 'column': 3, 'qfData': 3 }, # 12个月营业总收入 { 'facName': 'FIN_OTR_12MG', 'column': 3, 'qfData': 6 }, # 12个月营业总收入增长率 { 'facName': 'FIN_OR_Q', 'column': 4, 'qfData': 2 }, # 季度营业收入 { 'facName': 'FIN_OR_QG', 'column': 4, 'qfData': 5 }, # 季度营业收入增长率 { 'facName': 'FIN_OR_12M', 'column': 4, 'qfData': 3 }, # 12个月营业收入 { 'facName': 'FIN_OR_12MG', 'column': 4, 'qfData': 6 }, # 12个月营业收入增长率 { 'facName': 'FIN_OP_Q', 'column': 9, 'qfData': 2 }, # 季度营业利润 { 'facName': 'FIN_OP_QG', 'column': 9, 'qfData': 5 }, # 季度营业利润增长率 { 'facName': 'FIN_OP_12M', 'column': 9, 'qfData': 3 }, # 12个月营业利润 { 'facName': 'FIN_OP_12MG', 'column': 9, 'qfData': 6 }, # 12个月营业利润增长率 { 'facName': 'FIN_TP_Q', 'column': 10, 'qfData': 2 }, # 季度利润总额 { 'facName': 'FIN_TP_QG', 'column': 10, 'qfData': 5 }, # 季度利润总额增长率 { 'facName': 'FIN_TP_12M', 'column': 10, 'qfData': 3 }, # 12个月利润总额 { 'facName': 'FIN_TP_12MG', 'column': 10, 'qfData': 6 }, # 12个月利润总额增长率 { 'facName': 'FIN_OTC_Q', 'column': 6, 'qfData': 2 }, # 季度营业总成本 { 'facName': 'FIN_OTC_12M', 'column': 6, 'qfData': 3 }, # 12个月营业总成本 { 'facName': 'FIN_OC_Q', 'column': 7, 'qfData': 2 }, # 季度营业成本 { 'facName': 'FIN_OC_12M', 'column': 7, 'qfData': 3 }, # 12个月营业成本 { 'facName': 'FIN_ITE_Q', 'column': 11, 'qfData': 2 }, # 季度所得税费用 { 'facName': 'FIN_ITE_12M', 'column': 11, 'qfData': 3 } # 12个月所得税费用 ] for configDict in configDictList: facName = configDict['facName'] print('==== ' + facName) lastUpdateDate = Public.GetLastUpdateDate(facName, mc) dataList = Public.GetDataList( Public.sqlPrf, lastUpdateDate - dt.timedelta(days=365 * 3), dt.datetime.now(), conn) qfDataTuple = Public.QuarterFormat(dataList) column = configDict['column'] dataDict = qfDataTuple[configDict['qfData']] facDict = {} for symbol, data in dataDict.items(): facDict[symbol] = data[:, [0, 1, 2, column]] Public.ToDB(facDict, facName, endDate=None, updateReportDate=False, mongoClient=mc) ddd = 0
def Beta(betaPrefix, betaDaysList, indexSymbolDict, indexSql, stockSql, endDate=None): if endDate is None: dtNow = dt.datetime.now() - dt.timedelta(days=1) endDate = dt.datetime(dtNow.year, dtNow.month, dtNow.day) else: endDate = dt.datetime(endDate.year, endDate.month, endDate.day) if endDate.date() >= dt.datetime.now().date(): endDate = dt.datetime(dt.datetime.now().year, dt.datetime.now().month, dt.datetime.now().day) - \ dt.timedelta(days=1) connStr = Public.GetPara('connStr') conn = co.connect(connStr) cursor = conn.cursor() mongoConn = Public.GetPara('mongoConn') mc = pm.MongoClient(mongoConn) db = mc['factor'] for days in betaDaysList: for indexCode, indexSymbol in indexSymbolDict.items(): facName = betaPrefix + '_' + indexSymbol + '_' + str(days) lastUpdateDate = Public.GetLastUpdateDate(facName, mc) tradingDateSet = Public.GetCalendar(lastUpdateDate, endDate) currentDate = lastUpdateDate - dt.timedelta(days=days) savedDate = lastUpdateDate indexRtnDict = {} stockRtnDict = {} while currentDate + dt.timedelta(days=1) <= endDate: currentDate += dt.timedelta(days=1) # if currentDate not in tradingDateSet: # continue # get data for stockSymbol, stockRtnList in stockRtnDict.items(): stockRtnList.append(np.nan) for indexSymbol, indexRtnList in indexRtnDict.items(): indexRtnList.append(np.nan) cursor.execute( stockSql.replace('{TRADE_DATE}', currentDate.strftime('%Y-%m-%d'))) stockRtnRecordSet = cursor.fetchall() for stockRtnRecord in stockRtnRecordSet: symbol = stockRtnRecord[1] rtn = stockRtnRecord[4] if symbol not in stockRtnDict.keys(): stockRtnDict[symbol] = [np.nan] stockRtnDict[symbol][ -1] = rtn if rtn is not None else np.nan cursor.execute( indexSql.replace('{TRADE_DATE}', currentDate.strftime('%Y-%m-%d')).replace( '{INDEX_CODE}', str(indexCode))) indexRtnRecordSet = cursor.fetchall() for indexRtnRecord in indexRtnRecordSet: symbol = indexRtnRecord[1] if symbol != indexSymbol: raise Exception( 'index code is not fit to index symbol!') rtn = indexRtnRecord[4] if symbol not in indexRtnDict.keys(): indexRtnDict[symbol] = [np.nan] indexRtnDict[symbol][ -1] = rtn if rtn is not None else np.nan if currentDate <= lastUpdateDate: continue # evaluate beta and save to db if indexSymbol not in indexRtnDict.keys() or len( indexRtnDict[indexSymbol]) < days / 2: continue mongoDoc = { '_id': currentDate, '_updateTime': dt.datetime.now(), '_isTrade': (currentDate in tradingDateSet) } indexArr = np.array(indexRtnDict[indexSymbol] [max(0, len(indexRtnDict[indexSymbol]) - days):], dtype=float) for stockSymbol, stockRtnList in stockRtnDict.items(): stockArr = np.array( stockRtnList[max(0, len(stockRtnList) - days):], dtype=float) if len(stockArr) != len(indexArr): continue vld = np.isfinite(stockArr) * np.isfinite((indexArr)) if np.sum(vld) < days / 3: # data is not sufficient continue const = np.ones(len(stockArr)) X = np.vstack([indexArr[vld], const[vld]]).T y = stockArr[vld] c, res, rank, s = np.linalg.lstsq(X, y) beta = c[0] mongoDoc[stockSymbol] = beta db[facName].save(mongoDoc) savedDate = currentDate print(facName + ' ' + str(currentDate)) db.cfgUpdate.save({'_id': facName, 'lastUpdateDate': savedDate})
def Beta(tovPrefix, tovDaysList, stockSql, endDate=None): if endDate is None: dtNow = dt.datetime.now() - dt.timedelta(days=1) endDate = dt.datetime(dtNow.year, dtNow.month, dtNow.day) else: endDate = dt.datetime(endDate.year, endDate.month, endDate.day) if endDate.date() >= dt.datetime.now().date(): endDate = dt.datetime(dt.datetime.now().year, dt.datetime.now().month, dt.datetime.now().day) - \ dt.timedelta(days=1) connStr = Public.GetPara('connStr') conn = co.connect(connStr) cursor = conn.cursor() mongoConn = Public.GetPara('mongoConn') mc = pm.MongoClient(mongoConn) db = mc['factor'] for days in tovDaysList: facName = tovPrefix + '_' + str(days) lastUpdateDate = Public.GetLastUpdateDate(facName, mc) tradingDateSet = Public.GetCalendar(lastUpdateDate, endDate) currentDate = lastUpdateDate - dt.timedelta(days=days) savedDate = lastUpdateDate stockRtnDict = {} while currentDate + dt.timedelta(days=1) <= endDate: currentDate += dt.timedelta(days=1) # if currentDate not in tradingDateSet: # continue # get data for stockSymbol, stockRtnList in stockRtnDict.items(): stockRtnList.append(np.nan) cursor.execute( stockSql.replace('{TRADE_DATE}', currentDate.strftime('%Y-%m-%d'))) stockRtnRecordSet = cursor.fetchall() for stockRtnRecord in stockRtnRecordSet: symbol = stockRtnRecord[1] tov = stockRtnRecord[4] if symbol not in stockRtnDict.keys(): stockRtnDict[symbol] = [np.nan] stockRtnDict[symbol][-1] = tov if tov is not None else np.nan if currentDate <= lastUpdateDate: continue # evaluate beta and save to db mongoDoc = { '_id': currentDate, '_updateTime': dt.datetime.now(), '_isTrade': (currentDate in tradingDateSet) } for stockSymbol, stockRtnList in stockRtnDict.items(): stockArr = np.array(stockRtnList[max(0, len(stockRtnList) - days):], dtype=float) tov = np.nanmean(stockArr) mongoDoc[stockSymbol] = tov db[facName].save(mongoDoc) savedDate = currentDate print(facName + ' ' + str(currentDate)) db.cfgUpdate.save({'_id': facName, 'lastUpdateDate': savedDate})