def _genNullIDSQLStr(self, factor_names, ids, end_date, args={}):
IDField = args.get("ID字段", self.IDField)
DTField = args.get("时点字段", self.DTField)
FactorField = args.get("因子字段", self.FactorField)
FactorValueField = args.get("因子值字段", self.FactorValueField)
SubSQLStr = "SELECT "+IDField+", "
SubSQLStr += FactorField
SubSQLStr += "MAX("+DTField+") "
SubSQLStr += "FROM "+self._DBTableName+" "
SubSQLStr += "WHERE "+DTField+"<'"+end_date.strftime("%Y-%m-%d:%H:%M:%S.%f")+"' "
SubSQLStr += "AND ("+genSQLInCondition(IDField, ids, is_str=True, max_num=1000)+") "
if len(factor_names)<1000:
SubSQLStr += "AND ("+genSQLInCondition(FactorField, factor_names, is_str=True, max_num=1000)+") "
else:
SubSQLStr += "AND "+FactorField+" IS NOT NULL "
FilterStr = args.get("筛选条件", self.FilterCondition)
if FilterStr: SubSQLStr += "AND "+FilterStr+" "
SubSQLStr += "GROUP BY "+IDField+", "+FactorField
SQLStr = "SELECT "+DTField+", "
SQLStr += IDField+", "
SQLStr += FactorField+", "
if FactorValueField is not None:
SQLStr += FactorValueField+" "
else:
SQLStr += "1 "
SQLStr += "FROM "+self._DBTableName+" "
SQLStr += "WHERE ("+IDField+", "+FactorField+", "+DTField+") IN ("+SubSQLStr+") "
if FilterStr: SQLStr += "AND "+FilterStr+" "
return SQLStr
def __QS_prepareRawData__(self, factor_names, ids, dts, args={}):
if (not dts) or (not ids):
return pd.DataFrame(columns=["DateTime", "ID"] + factor_names)
DBTableName = self._FactorDB.TablePrefix + self._FactorDB._Prefix + self.Name
# 形成 SQL 语句, 时点, ID, 因子数据
SQLStr = "SELECT " + DBTableName + ".DateTime, "
SQLStr += DBTableName + ".ID, "
for iField in factor_names:
SQLStr += DBTableName + "." + iField + ", "
SQLStr = SQLStr[:-2] + " FROM " + DBTableName + " "
if (len(dts) == 1) or (np.nanmax(np.diff(np.array(dts))).days <= 1):
SQLStr += "WHERE " + DBTableName + ".DateTime>='" + dts[
0].strftime("%Y-%m-%d %H:%M:%S.%f") + "' "
SQLStr += "AND " + DBTableName + ".DateTime<='" + dts[-1].strftime(
"%Y-%m-%d %H:%M:%S.%f") + "' "
else:
SQLStr += "WHERE (" + genSQLInCondition(
DBTableName + ".DateTime",
[iDT.strftime("%Y-%m-%d %H:%M:%S.%f") for iDT in dts],
is_str=True,
max_num=1000) + ") "
if len(ids) <= 1000:
SQLStr += "AND (" + genSQLInCondition(
DBTableName + ".ID", ids, is_str=True, max_num=1000) + ") "
SQLStr += "ORDER BY " + DBTableName + ".DateTime, " + DBTableName + ".ID"
RawData = self._FactorDB.fetchall(SQLStr)
if not RawData:
return pd.DataFrame(columns=["DateTime", "ID"] + factor_names)
return pd.DataFrame(np.array(RawData),
columns=["DateTime", "ID"] + factor_names)
def deleteData(self, table_name, ids=None, dts=None):
if table_name not in self._TableFactorDict:
Msg = ("因子库 '%s' 调用方法 deleteData 错误: 不存在因子表 '%s'!" %
(self.Name, table_name))
self._QS_Logger.error(Msg)
raise __QS_Error__(Msg)
if (ids is None) and (dts is None):
return self.truncateDBTable(self.InnerPrefix + table_name)
DBTableName = self.TablePrefix + self.InnerPrefix + table_name
SQLStr = "DELETE FROM " + DBTableName
if dts is not None:
DTs = [iDT.strftime("%Y-%m-%d %H:%M:%S.%f") for iDT in dts]
SQLStr += "WHERE " + genSQLInCondition(
DBTableName + ".datetime", DTs, is_str=True,
max_num=1000) + " "
else:
SQLStr += "WHERE " + DBTableName + ".datetime IS NOT NULL "
if ids is not None:
SQLStr += "AND " + genSQLInCondition(
DBTableName + ".code", ids, is_str=True, max_num=1000)
try:
self.execute(SQLStr)
except Exception as e:
Msg = ("'%s' 调用方法 deleteData 删除表 '%s' 中数据时错误: %s" %
(self.Name, table_name, str(e)))
self._QS_Logger.error(Msg)
raise e
return 0
def _getRawData(self, fields, ids=None, start_date=None, end_date=None, args={}):
IndexEquityID = self.FactorDB.ID2EquityID(fields)
DBTableName = self._FactorDB.TablePrefix + self._FactorDB._TableInfo.loc[self.Name, "DBTableName"]
IDTable = self.FactorDB.TablePrefix+"tb_object_0001"
FieldDict = self._FactorDB._FactorInfo["DBFieldName"].loc[self.Name].loc[['证券ID','指数ID','纳入日期','剔除日期','最新标志']]
# 指数中成份股 ID, 指数证券 ID, 纳入日期, 剔除日期, 最新标志
SQLStr = "SELECT "+DBTableName+'.'+FieldDict['指数ID']+', '# 指数证券 ID
SQLStr += IDTable+".f1_0001, "# ID
SQLStr += DBTableName+'.'+FieldDict['纳入日期']+', '# 纳入日期
SQLStr += DBTableName+'.'+FieldDict['剔除日期']+', '# 剔除日期
SQLStr += DBTableName+'.'+FieldDict['最新标志']+' '# 最新标志
SQLStr += 'FROM '+DBTableName+', '+IDTable+" "
SQLStr += 'WHERE '+DBTableName+'.'+FieldDict['证券ID']+'='+IDTable+'.f16_0001 '
SQLStr += "AND ("+genSQLInCondition(DBTableName+'.'+FieldDict['指数ID'], list(IndexEquityID.values), is_str=True, max_num=1000)+") "
if ids is not None:
SQLStr += 'AND ('+genSQLInCondition(IDTable+'.f1_0001', ids, is_str=True, max_num=1000)+') '
if start_date is not None:
SQLStr += "AND (("+DBTableName+"."+FieldDict["剔除日期"]+">'"+start_date.strftime("%Y%m%d")+"') "
SQLStr += "OR ("+DBTableName+"."+FieldDict["剔除日期"]+" IS NULL))"
if end_date is not None:
SQLStr += "AND "+DBTableName+"."+FieldDict["纳入日期"]+"<='"+end_date.strftime("%Y%m%d")+"' "
else:
SQLStr += "AND "+DBTableName+"."+FieldDict["纳入日期"]+" IS NOT NULL "
SQLStr += 'ORDER BY '+DBTableName+'.'+FieldDict['指数ID']+", "+IDTable+'.f1_0001, '+DBTableName+'.'+FieldDict['纳入日期']
RawData = self.FactorDB.fetchall(SQLStr)
if RawData==[]:
RawData = pd.DataFrame(columns=["指数ID", 'ID', '纳入日期', '剔除日期', '最新标志'])
else:
RawData = pd.DataFrame(np.array(RawData),columns=["指数ID", 'ID', '纳入日期', '剔除日期', '最新标志'])
for iID in fields:
RawData["指数ID"][RawData["指数ID"]==IndexEquityID[iID]] = iID
return RawData
def deleteData(self, table_name, ids=None, dts=None):
DBTableName = self.TablePrefix+self.InnerPrefix+table_name
if (self.DBType!="sqlite3") and (ids is None) and (dts is None):
SQLStr = "TRUNCATE TABLE "+DBTableName
return self.execute(SQLStr)
SQLStr = "DELETE * FROM "+DBTableName
if dts is not None:
DTs = [iDT.strftime("%Y-%m-%d %H:%M:%S.%f") for iDT in dts]
SQLStr += "WHERE "+genSQLInCondition(DBTableName+".datetime", DTs, is_str=True, max_num=1000)+" "
else:
SQLStr += "WHERE "+DBTableName+".datetime IS NOT NULL "
if ids is not None:
SQLStr += "AND "+genSQLInCondition(DBTableName+".code", ids, is_str=True, max_num=1000)
return self.execute(SQLStr)
def __QS_prepareRawData__(self, factor_names, ids, dts, args={}):
if (dts==[]) or (ids==[]): return pd.DataFrame(columns=["QS_DT", "ID"]+factor_names)
IDField = args.get("ID字段", self.IDField)
DTField = args.get("时点字段", self.DTField)
FactorField = args.get("因子字段", self.FactorField)
FactorValueField = args.get("因子值字段", self.FactorValueField)
LookBack = args.get("回溯天数", self.LookBack)
if dts is not None:
dts = sorted(dts)
StartDate, EndDate = dts[0].date(), dts[-1].date()
if not np.isinf(LookBack): StartDate -= dt.timedelta(LookBack)
else:
StartDate = EndDate = None
# 形成 SQL 语句, 时点, ID, 因子数据
SQLStr = "SELECT "+DTField+", "
SQLStr += IDField+", "
SQLStr += FactorField+", "
if FactorValueField is not None:
SQLStr += FactorValueField+", "
else:
SQLStr += "1, "
SQLStr = SQLStr[:-2]+" FROM "+self._DBTableName+" "
if StartDate is not None:
SQLStr += "WHERE "+DTField+">='"+StartDate.strftime("%Y-%m-%d %H:%M:%S.%f")+"' "
SQLStr += "AND "+DTField+"<='"+EndDate.strftime("%Y-%m-%d %H:%M:%S.%f")+"' "
else:
SQLStr += "WHERE "+DTField+" IS NOT NULL "
if ids is not None:
SQLStr += "AND ("+genSQLInCondition(IDField, ids, is_str=True, max_num=1000)+") "
if len(factor_names)<1000:
SQLStr += "AND ("+genSQLInCondition(FactorField, factor_names, is_str=True, max_num=1000)+") "
else:
SQLStr += "AND "+FactorField+" IS NOT NULL "
FilterStr = args.get("筛选条件", self.FilterCondition)
if FilterStr: SQLStr += "AND "+FilterStr+" "
SQLStr += "ORDER BY "+DTField+", "+IDField+", "+FactorField
RawData = self._FactorDB.fetchall(SQLStr)
if not RawData: RawData = pd.DataFrame(columns=["QS_DT", "ID", "QS_Factor", "QS_FactorValue"])
RawData = pd.DataFrame(np.array(RawData), columns=["QS_DT", "ID", "QS_Factor", "QS_FactorValue"])
if (StartDate is not None) and np.isinf(LookBack):
NullIDs = set(ids).difference(set(RawData[RawData["QS_DT"]==dt.datetime.combine(StartDate, dt.time(0))]["ID"]))
if NullIDs:
NullRawData = self._FactorDB.fetchall(self._genNullIDSQLStr(factor_names, list(NullIDs), StartDate, args=args))
if NullRawData:
NullRawData = pd.DataFrame(np.array(NullRawData, dtype="O"), columns=["QS_DT", "ID", "QS_Factor", "QS_FactorValue"])
RawData = pd.concat([NullRawData, RawData], ignore_index=True)
RawData.sort_values(by=["QS_DT", "ID", "QS_Factor"])
if self._FactorDB.DBType=="sqlite3": RawData["QS_DT"] = [dt.datetime.strptime(iDT, "%Y-%m-%d %H:%M:%S.%f") for iDT in RawData.pop("QS_DT")]
return RawData
def __QS_readCov__(self, dts, ids=None):
Data = {}
SQLStr = "SELECT DateTime, FactorCov, FactorData, SpecificRisk "
SQLStr += "FROM " + self._DBTableName + " "
SQLStr += "WHERE FactorCov IS NOT NULL "
SQLStr += "AND (" + genSQLInCondition(
"DateTime", [iDT.strftime("%Y-%m-%d %H:%M:%S.%f") for iDT in dts],
is_str=True,
max_num=1000) + ") "
for iDT, iFactorCov, iFactorData, iSpecificRisk in self._RiskDB.fetchall(
SQLStr):
iFactorCov = pd.read_json(iFactorCov, orient="split")
iSpecificRisk = pd.read_json(iSpecificRisk,
orient="split",
typ="series")
iFactorData = pd.read_json(iFactorData, orient="split")
if ids is None:
iIDs = iSpecificRisk.index
iFactorData = iFactorData.loc[iIDs].values
iSpecificRisk = iSpecificRisk.values
else:
iIDs = ids
iFactorData = iFactorData.loc[iIDs].values
iSpecificRisk = iSpecificRisk.loc[iIDs].values
iCov = np.dot(np.dot(iFactorData, iFactorCov.values),
iFactorData.T) + np.diag(iSpecificRisk**2)
Data[iDT] = pd.DataFrame(iCov, index=iIDs, columns=iIDs)
if Data: return pd.Panel(Data).loc[dts]
return pd.Panel(items=dts)
def _getRawData(self, fields, ids=None, start_date=None, end_date=None, args={}):
DBTableName = self._FactorDB.TablePrefix + self._FactorDB._TableInfo.loc[self.Name, "DBTableName"]
IDTable = self.FactorDB.TablePrefix+"tb_object_0001"
FieldDict = self._FactorDB._FactorInfo["DBFieldName"].loc[self.Name].loc[["日期", "证券ID"]+fields]
# 日期, ID, 因子数据
SQLStr = 'SELECT '+DBTableName+'.'+FieldDict["日期"]+', '# 日期
SQLStr += IDTable+".f1_0001, "# ID
for iField in fields:
SQLStr += DBTableName+'.'+FieldDict[iField]+', '# 因子数据
SQLStr = SQLStr[:-2]+' '
SQLStr += 'FROM '+IDTable+", "+DBTableName+' '
SQLStr += 'WHERE '+IDTable+".F16_0001="+DBTableName+'.'+FieldDict['证券ID']+' '
if ids is not None:
SQLStr += 'AND ('+genSQLInCondition(IDTable+".f1_0001", ids, is_str=True, max_num=1000)+") "
if start_date is not None:
SQLStr += 'AND '+DBTableName+'.'+FieldDict["日期"]+'>=\''+start_date.strftime("%Y%m%d")+'\' '
if end_date is not None:
SQLStr += 'AND '+DBTableName+'.'+FieldDict["日期"]+'<=\''+end_date.strftime("%Y%m%d")+'\' '
SQLStr += 'ORDER BY '+IDTable+'.f1_0001, '+DBTableName+'.'+FieldDict["日期"]
RawData = self.FactorDB.fetchall(SQLStr)
if RawData==[]:
RawData = pd.DataFrame(columns=['日期','ID']+fields)
else:
RawData = pd.DataFrame(np.array(RawData), columns=['日期','ID']+fields)
return RawData
def _genNullIDSQLStr(self, factor_names, ids, end_date, args={}):
IDField = args.get("ID字段", self.IDField)
DTField = args.get("时点字段", self.DTField)
DT2Str = args.get("时间转字符串", self.DT2Str)
SubSQLStr = "SELECT " + IDField + ", "
SubSQLStr += "MAX(" + DTField + ") "
SubSQLStr += "FROM " + self._DBTableName + " "
SubSQLStr += "WHERE " + DTField + "<'" + end_date.strftime(
"%Y-%m-%d:%H:%M:%S.%f") + "' "
SubSQLStr += "AND (" + genSQLInCondition(
IDField, ids, is_str=True, max_num=1000) + ") "
FilterStr = args.get("筛选条件", self.FilterCondition)
if FilterStr:
SubSQLStr += "AND " + FilterStr.format(
Table=self._DBTableName) + " "
SubSQLStr += "GROUP BY " + IDField
SQLStr = "SELECT " + DTField + ", "
SQLStr += IDField + ", "
for iField in factor_names:
if iField == "dt":
iDBDataType = "DateTime"
elif iField == "code":
iDBDataType = "String"
else:
iDBDataType = self._FactorDB._TableFieldDataType[
self._Name][iField]
if DT2Str and (iDBDataType.lower().find("date") != -1):
SQLStr += "DATE_FORMAT(" + iField + ", '%Y-%m-%d %H:%i:%s'), "
else:
SQLStr += iField + ", "
SQLStr = SQLStr[:-2] + " FROM " + self._DBTableName + " "
SQLStr += "WHERE (" + IDField + ", " + DTField + ") IN (" + SubSQLStr + ") "
if FilterStr:
SQLStr += "AND " + FilterStr.format(Table=self._DBTableName) + " "
return SQLStr
def deleteDateTime(self, table_name, dts):
DBTableName = self.TablePrefix+self._Prefix+table_name
if dts is None:
SQLStr = "TRUNCATE TABLE "+DBTableName
return self.execute(SQLStr)
SQLStr = "DELETE * FROM "+DBTableName
DTs = [iDT.strftime("%Y-%m-%d %H:%M:%S.%f") for iDT in dts]
SQLStr += "WHERE "+genSQLInCondition(DBTableName+".DateTime", DTs, is_str=True, max_num=1000)+" "
return self.execute(SQLStr)
def readFactorReturn(self, dts):
SQLStr = "SELECT DateTime, FactorReturn "
SQLStr += "FROM "+self._DBTableName+" "
SQLStr += "WHERE FactorReturn IS NOT NULL "
SQLStr += "AND ("+genSQLInCondition("DateTime", [iDT.strftime("%Y-%m-%d %H:%M:%S.%f") for iDT in dts], is_str=True, max_num=1000)+") "
Data = {}
for iDT, iFactorReturn in self._RiskDB.fetchall(SQLStr):
Data[iDT] = pd.read_json(iFactorReturn, orient="split", typ="series")
if not Data: return pd.DataFrame(index=dts)
return pd.DataFrame(Data).T.loc[dts]
def __QS_readFactorCov__(self, dts):
SQLStr = "SELECT DateTime, FactorCov "
SQLStr += "FROM "+self._DBTableName+" "
SQLStr += "WHERE FactorCov IS NOT NULL "
SQLStr += "AND ("+genSQLInCondition("DateTime", [iDT.strftime("%Y-%m-%d %H:%M:%S.%f") for iDT in dts], is_str=True, max_num=1000)+") "
Data = {}
for iDT, iCov in self._RiskDB.fetchall(SQLStr):
iCov = pd.read_json(iCov, orient="split")
iCov.index = iCov.columns
Data[iDT] = iCov
if Data: return pd.Panel(Data).loc[dts]
return pd.Panel(items=dts)
def __QS_readCov__(self, dts, ids=None):
SQLStr = "SELECT DateTime, Cov "
SQLStr += "FROM "+self._DBTableName+" "
SQLStr += "WHERE ("+genSQLInCondition("DateTime", [iDT.strftime("%Y-%m-%d %H:%M:%S.%f") for iDT in dts], is_str=True, max_num=1000)+") "
Data = {}
for iDT, iCov in self._RiskDB.fetchall(SQLStr):
iCov = pd.read_json(iCov, orient="split")
iCov.index = iCov.columns
if ids is not None:
if iCov.index.intersection(ids).shape[0]>0: iCov = iCov.loc[ids, ids]
else: iCov = pd.DataFrame(index=ids, columns=ids)
Data[iDT] = iCov
if Data: return pd.Panel(Data).loc[dts]
return pd.Panel(items=dts, major_axis=ids, minor_axis=ids)
def readSpecificReturn(self, dts, ids=None):
SQLStr = "SELECT DateTime, SpecificReturn "
SQLStr += "FROM "+self._DBTableName+" "
SQLStr += "WHERE SpecificReturn IS NOT NULL "
SQLStr += "AND ("+genSQLInCondition("DateTime", [iDT.strftime("%Y-%m-%d %H:%M:%S.%f") for iDT in dts], is_str=True, max_num=1000)+") "
Data = {}
for iDT, iSpecificReturn in self._RiskDB.fetchall(SQLStr):
Data[iDT] = pd.read_json(iSpecificReturn, orient="split", typ="series")
if not Data: return pd.DataFrame(index=dts, columns=([] if ids is None else ids))
Data = pd.DataFrame(Data).T.loc[dts]
if ids is not None:
if Data.columns.intersection(ids).shape[0]>0: Data = Data.loc[:, ids]
else: Data = pd.DataFrame(index=dts, columns=ids)
return Data
def __QS_readFactorData__(self, dts, ids=None):
SQLStr = "SELECT DateTime, FactorData "
SQLStr += "FROM "+self._DBTableName+" "
SQLStr += "WHERE FactorData IS NOT NULL "
SQLStr += "AND ("+genSQLInCondition("DateTime", [iDT.strftime("%Y-%m-%d %H:%M:%S.%f") for iDT in dts], is_str=True, max_num=1000)+") "
Data = {}
for iDT, iData in self._RiskDB.fetchall(SQLStr):
Data[iDT] = pd.read_json(iData, orient="split").T
if not Data: return pd.Panel(items=[], major_axis=dts, minor_axis=ids)
Data = pd.Panel(Data).swapaxes(0, 1).loc[:, dts, :]
if ids is not None:
if Data.minor_axis.intersection(ids).shape[0]>0: Data = Data.loc[:, :, ids]
else: Data = pd.Panel(items=Data.items, major_axis=dts, minor_axis=ids)
return Data
def readData(self, data_item, dts):
SQLStr = "SELECT DateTime, "+data_item+" "
SQLStr += "FROM "+self._DBTableName+" "
SQLStr += "WHERE "+data_item+" IS NOT NULL "
SQLStr += "AND ("+genSQLInCondition("DateTime", [iDT.strftime("%Y-%m-%d %H:%M:%S.%f") for iDT in dts], is_str=True, max_num=1000)+") "
Data = {}
Type = None
for iDT, iData in self._RiskDB.fetchall(SQLStr):
if Type is None:
try:
Data[iDT] = pd.read_json(iData, orient="split")
Type = "frame"
except:
Type = "series"
Data[iDT] = pd.read_json(iData, orient="split", typ=Type)
else:
Data[iDT] = pd.read_json(iData, orient="split", typ=Type)
if not Data: return None
if Type=="series": return pd.DataFrame(Data).T.loc[dts]
else: return pd.Panel(Data).loc[dts]
def __QS_prepareRawData__(self, factor_names, ids, dts, args={}):
if (dts == []) or (ids == []):
return pd.DataFrame(columns=["QS_DT", "ID"] + factor_names)
IDField = args.get("ID字段", self.IDField)
DTField = args.get("时点字段", self.DTField)
LookBack = args.get("回溯天数", self.LookBack)
DT2Str = args.get("时间转字符串", self.DT2Str)
if dts is not None:
dts = sorted(dts)
StartDate, EndDate = dts[0].date(), dts[-1].date()
if not np.isinf(LookBack): StartDate -= dt.timedelta(LookBack)
else:
StartDate = EndDate = None
# 形成 SQL 语句, 时点, ID, 因子数据
SQLStr = "SELECT " + self._DBTableName + "." + DTField + ", "
SQLStr += self._DBTableName + "." + IDField + ", "
for iField in factor_names:
if iField == "datetime":
iDBDataType = "datetime"
elif iField == "code":
iDBDataType = "varchar(40)"
else:
iDBDataType = self._FactorDB._TableFieldDataType[
self._Name][iField]
if DT2Str and (iDBDataType.lower().find("date") != -1):
SQLStr += "DATE_FORMAT(" + self._DBTableName + "." + iField + ", '%Y-%m-%d %H:%i:%s'), "
else:
SQLStr += self._DBTableName + "." + iField + ", "
SQLStr = SQLStr[:-2] + " FROM " + self._DBTableName + " "
if StartDate is not None:
SQLStr += "WHERE " + self._DBTableName + "." + DTField + ">='" + StartDate.strftime(
"%Y-%m-%d %H:%M:%S.%f") + "' "
SQLStr += "AND " + self._DBTableName + "." + DTField + "<='" + EndDate.strftime(
"%Y-%m-%d %H:%M:%S.%f") + "' "
else:
SQLStr += "WHERE " + self._DBTableName + "." + DTField + " IS NOT NULL "
if ids is not None:
SQLStr += "AND (" + genSQLInCondition(
self._DBTableName + "." + IDField,
ids,
is_str=True,
max_num=1000) + ") "
FilterStr = args.get("筛选条件", self.FilterCondition)
if FilterStr:
SQLStr += "AND " + FilterStr.format(Table=self._DBTableName) + " "
SQLStr += "ORDER BY " + self._DBTableName + "." + DTField + ", " + self._DBTableName + "." + IDField
if args.get("因子值类型", self.ValueType) != "scalar":
SQLStr += ", " + self._DBTableName + "." + factor_names[0]
RawData = self._FactorDB.fetchall(SQLStr)
if not RawData:
RawData = pd.DataFrame(columns=["QS_DT", "ID"] + factor_names)
RawData = pd.DataFrame(np.array(RawData),
columns=["QS_DT", "ID"] + factor_names)
if (StartDate is not None) and np.isinf(LookBack):
NullIDs = set(ids).difference(
set(RawData[RawData["QS_DT"] == dt.datetime.combine(
StartDate, dt.time(0))]["ID"]))
if NullIDs:
NullRawData = self._FactorDB.fetchall(
self._genNullIDSQLStr(factor_names,
list(NullIDs),
StartDate,
args=args))
if NullRawData:
NullRawData = pd.DataFrame(
np.array(NullRawData, dtype="O"),
columns=["QS_DT", "ID"] + factor_names)
RawData = pd.concat([NullRawData, RawData],
ignore_index=True)
RawData.sort_values(by=["QS_DT", "ID"])
if self._FactorDB.DBType == "sqlite3":
RawData["QS_DT"] = [
dt.datetime.strptime(iDT, "%Y-%m-%d %H:%M:%S.%f")
for iDT in RawData.pop("QS_DT")
]
return RawData
