def _QS_initOperation(self, start_dt, dt_dict, prepare_ids, id_dict):
OldStartDT = dt_dict.get(self.Name, None)
if (OldStartDT is None) or (start_dt < OldStartDT):
dt_dict[self.Name] = start_dt
StartInd, EndInd = self._OperationMode.DTRuler.index(
dt_dict[self.Name]), self._OperationMode.DTRuler.index(
self._OperationMode.DateTimes[-1])
DTs = self._OperationMode.DTRuler[StartInd:EndInd + 1]
DTPartition = partitionList(DTs, len(self._OperationMode._PIDs))
self._PID_DTs = {
iPID: DTPartition[i]
for i, iPID in enumerate(self._OperationMode._PIDs)
}
PrepareIDs = id_dict.setdefault(self.Name, prepare_ids)
if prepare_ids != PrepareIDs:
raise __QS_Error__("因子 %s 指定了不同的截面!" % self.Name)
for i, iDescriptor in enumerate(self._Descriptors):
if self.DescriptorSection[i] is None:
iDescriptor._QS_initOperation(start_dt, dt_dict, prepare_ids,
id_dict)
else:
iDescriptor._QS_initOperation(start_dt, dt_dict,
self.DescriptorSection[i],
id_dict)
if (self._OperationMode.SubProcessNum >
0) and (self.Name not in self._OperationMode._Event):
self._OperationMode._Event[self.Name] = (Queue(), Event())
def __QS_prepareCacheData__(self, ids=None):
PID = self._OperationMode._iPID
StartDT = self._OperationMode._FactorStartDT[self.Name]
EndDT = self._OperationMode.DateTimes[-1]
StartInd, EndInd = self._OperationMode.DTRuler.index(
StartDT), self._OperationMode.DTRuler.index(EndDT)
DTs = list(self._OperationMode.DTRuler[StartInd:EndInd + 1])
IDs = self._OperationMode._FactorPrepareIDs[self.Name]
if IDs is None: IDs = list(self._OperationMode._PID_IDs[PID])
else:
IDs = partitionList(IDs, len(self._OperationMode._PIDs))[
self._OperationMode._PIDs.index(PID)]
if IDs:
StdData = self._calcData(ids=IDs,
dts=DTs,
descriptor_data=[
iDescriptor._QS_getData(
DTs, pids=[PID]).values
for iDescriptor in self._Descriptors
])
StdData = pd.DataFrame(StdData, index=DTs, columns=IDs)
else:
StdData = pd.DataFrame(
index=DTs,
columns=IDs,
dtype=("float" if self.DataType == "double" else "O"))
with self._OperationMode._PID_Lock[PID]:
with shelve.open(self._OperationMode._CacheDataDir + os.sep + PID +
os.sep + self.Name +
str(self._OperationMode._FactorID[self.Name])
) as CacheFile:
CacheFile["StdData"] = StdData
CacheFile["_QS_IDs"] = IDs
self._isCacheDataOK = True
return StdData
def _QS_initOperation(self, start_dt, dt_dict, prepare_ids, id_dict):
if len(self._Descriptors) > len(self.LookBack):
raise __QS_Error__("面板运算因子 : '%s' 的参数'回溯期数'序列长度小于描述子个数!" %
self.Name)
OldStartDT = dt_dict.get(self.Name, None)
DTRuler = self._OperationMode.DTRuler
if (OldStartDT is None) or (start_dt < OldStartDT):
StartDT = dt_dict[self.Name] = start_dt
StartInd, EndInd = DTRuler.index(StartDT), DTRuler.index(
self._OperationMode.DateTimes[-1])
if (self.iLookBackMode
== "扩张窗口") and (self.iInitData is not None) and (
self.iInitData.shape[0] > 0):
if self.iInitData.index[-1] not in self._OperationMode.DTRuler:
self._QS_Logger.warning(
"注意: 因子 '%s' 的初始值不在时点标尺的范围内, 初始值和时点标尺之间的时间间隔将被忽略!" %
(self.Name, ))
else:
StartInd = min(
StartInd,
self._OperationMode.DTRuler.index(
self.iInitData.index[-1]) + 1)
DTs = DTRuler[StartInd:EndInd + 1]
if self.iLookBackMode == "扩张窗口":
DTPartition = [DTs
] + [[]] * (len(self._OperationMode._PIDs) - 1)
else:
DTPartition = partitionList(DTs,
len(self._OperationMode._PIDs))
self._PID_DTs = {
iPID: DTPartition[i]
for i, iPID in enumerate(self._OperationMode._PIDs)
}
else:
StartInd = DTRuler.index(OldStartDT)
PrepareIDs = id_dict.setdefault(self.Name, prepare_ids)
if prepare_ids != PrepareIDs:
raise __QS_Error__("因子 %s 指定了不同的截面!" % self.Name)
for i, iDescriptor in enumerate(self._Descriptors):
iStartInd = StartInd - self.LookBack[i]
if iStartInd < 0:
self._QS_Logger.warning("注意: 对于因子 '%s' 的描述子 '%s', 时点标尺长度不足!" %
(self.Name, iDescriptor.Name))
iStartDT = DTRuler[max(0, iStartInd)]
if self.DescriptorSection[i] is None:
iDescriptor._QS_initOperation(iStartDT, dt_dict, prepare_ids,
id_dict)
else:
iDescriptor._QS_initOperation(iStartDT, dt_dict,
self.DescriptorSection[i],
id_dict)
if (self._OperationMode.SubProcessNum >
0) and (self.Name not in self._OperationMode._Event):
self._OperationMode._Event[self.Name] = (Queue(), Event())
def __QS_prepareCacheData__(self, ids=None):
DTs = list(self._PID_DTs[self._OperationMode._iPID])
IDs = self._OperationMode._FactorPrepareIDs[self.Name]
if IDs is None: IDs = list(self._OperationMode.IDs)
if len(DTs) == 0: # 该进程未分配到计算任务
iDTs = [self._OperationMode.DateTimes[-1]]
for i, iDescriptor in enumerate(self._Descriptors):
iDescriptor._QS_getData(iDTs, pids=None)
StdData = pd.DataFrame(
columns=IDs,
dtype=("float" if self.DataType == "double" else "O"))
elif IDs:
StdData = self._calcData(
ids=IDs,
dts=DTs,
descriptor_data=[
iDescriptor._QS_getData(DTs, pids=None).values
for i, iDescriptor in enumerate(self._Descriptors)
])
StdData = pd.DataFrame(StdData, index=DTs, columns=IDs)
else:
StdData = pd.DataFrame(
index=DTs,
columns=IDs,
dtype=("float" if self.DataType == "double" else "O"))
if self._OperationMode._FactorPrepareIDs[self.Name] is None:
PID_IDs = self._OperationMode._PID_IDs
else:
PID_IDs = {
self._OperationMode._PIDs[i]: iSubIDs
for i, iSubIDs in enumerate(
partitionList(IDs, len(self._OperationMode._PIDs)))
}
for iPID, iIDs in PID_IDs.items():
with self._OperationMode._PID_Lock[iPID]:
with shelve.open(self._OperationMode._CacheDataDir + os.sep +
iPID + os.sep + self.Name +
str(self._OperationMode._FactorID[self.Name])
) as CacheFile:
if "StdData" in CacheFile:
CacheFile["StdData"] = pd.concat(
[CacheFile["StdData"],
StdData.loc[:, iIDs]]).sort_index()
else:
CacheFile["StdData"] = StdData.loc[:, iIDs]
CacheFile["_QS_IDs"] = iIDs
StdData = None # 释放数据
if self._OperationMode.SubProcessNum > 0:
Sub2MainQueue, PIDEvent = self._OperationMode._Event[self.Name]
Sub2MainQueue.put(1)
PIDEvent.wait()
self._isCacheDataOK = True
return StdData
def __QS_prepareCacheData__(self, ids=None):
PID = self._OperationMode._iPID
StartDT = self._OperationMode._FactorStartDT[self.Name]
EndDT = self._OperationMode.DateTimes[-1]
StartInd, EndInd = self._OperationMode.DTRuler.index(
StartDT), self._OperationMode.DTRuler.index(EndDT)
DTs = list(self._OperationMode.DTRuler[StartInd:EndInd + 1])
IDs = self._OperationMode._FactorPrepareIDs[self.Name]
if IDs is None: IDs = list(self._OperationMode._PID_IDs[PID])
else:
IDs = partitionList(IDs, len(self._OperationMode._PIDs))[
self._OperationMode._PIDs.index(PID)]
if IDs:
DescriptorData = []
for i, iDescriptor in enumerate(self._Descriptors):
iStartInd = StartInd - self.LookBack[i]
iDTs = list(self._OperationMode.DTRuler[max(0, iStartInd
):StartInd]) + DTs
iDescriptorData = iDescriptor._QS_getData(iDTs,
pids=[PID]).values
if iStartInd < 0:
iDescriptorData = np.r_[np.full(shape=(
abs(iStartInd), iDescriptorData.shape[1]),
fill_value=np.nan),
iDescriptorData]
DescriptorData.append(iDescriptorData)
StdData = self._calcData(ids=IDs,
dts=DTs,
descriptor_data=DescriptorData,
dt_ruler=self._OperationMode.DTRuler)
StdData = pd.DataFrame(StdData, index=DTs, columns=IDs)
else:
StdData = pd.DataFrame(
index=DTs,
columns=IDs,
dtype=("float" if self.DataType == "double" else "O"))
with self._OperationMode._PID_Lock[PID]:
with shelve.open(self._OperationMode._CacheDataDir + os.sep + PID +
os.sep + self.Name +
str(self._OperationMode._FactorID[self.Name])
) as CacheFile:
CacheFile["StdData"] = StdData
CacheFile["_QS_IDs"] = IDs
self._isCacheDataOK = True
return StdData
def _QS_updateStartDT(self, start_dt, dt_dict):
OldStartDT = dt_dict.get(self.Name, None)
if (OldStartDT is None) or (start_dt < OldStartDT):
dt_dict[self.Name] = start_dt
StartInd, EndInd = self._OperationMode.DTRuler.index(
dt_dict[self.Name]), self._OperationMode.DTRuler.index(
self._OperationMode.DateTimes[-1])
DTs = self._OperationMode.DTRuler[StartInd:EndInd + 1]
DTPartition = partitionList(DTs, len(self._OperationMode._PIDs))
self._PID_DTs = {
iPID: DTPartition[i]
for i, iPID in enumerate(self._OperationMode._PIDs)
}
for i, iDescriptor in enumerate(self._Descriptors):
iDescriptor._QS_updateStartDT(start_dt, dt_dict)
if (self._OperationMode.SubProcessNum >
0) and (self.Name not in self._OperationMode._Event):
self._OperationMode._Event[self.Name] = (Queue(), Event())
def __QS_prepareCacheData__(self, ids=None):
DTs = list(self._PID_DTs[self._OperationMode._iPID])
IDs = self._OperationMode._FactorPrepareIDs[self.Name]
if IDs is None: IDs = list(self._OperationMode.IDs)
if len(DTs) == 0: # 该进程未分配到计算任务
iDTs = [self._OperationMode.DateTimes[-1]]
for i, iDescriptor in enumerate(self._Descriptors):
iDescriptor._QS_getData(iDTs, pids=None)
StdData = pd.DataFrame(
columns=IDs,
dtype=("float" if self.DataType == "double" else "O"))
elif IDs:
DescriptorData, StartInd = [], self._OperationMode.DTRuler.index(
DTs[0])
for i, iDescriptor in enumerate(self._Descriptors):
iStartInd = StartInd - self.LookBack[i]
iDTs = list(self._OperationMode.DTRuler[max(0, iStartInd
):StartInd]) + DTs
iDescriptorData = iDescriptor._QS_getData(iDTs,
pids=None).values
if iStartInd < 0:
iDescriptorData = np.r_[np.full(shape=(
abs(iStartInd), iDescriptorData.shape[1]),
fill_value=np.nan),
iDescriptorData]
DescriptorData.append(iDescriptorData)
StdData = self._calcData(ids=IDs,
dts=DTs,
descriptor_data=DescriptorData,
dt_ruler=self._OperationMode.DTRuler)
DescriptorData, iDescriptorData, StdData = None, None, pd.DataFrame(
StdData, index=DTs, columns=IDs)
else:
StdData = pd.DataFrame(
index=DTs,
columns=IDs,
dtype=("float" if self.DataType == "double" else "O"))
if self._OperationMode._FactorPrepareIDs[self.Name] is None:
PID_IDs = self._OperationMode._PID_IDs
else:
PID_IDs = {
self._OperationMode._PIDs[i]: iSubIDs
for i, iSubIDs in enumerate(
partitionList(IDs, len(self._OperationMode._PIDs)))
}
for iPID, iIDs in PID_IDs.items():
with self._OperationMode._PID_Lock[iPID]:
with shelve.open(self._OperationMode._CacheDataDir + os.sep +
iPID + os.sep + self.Name +
str(self._OperationMode._FactorID[self.Name])
) as CacheFile:
if "StdData" in CacheFile:
CacheFile["StdData"] = pd.concat(
[CacheFile["StdData"],
StdData.loc[:, iIDs]]).sort_index()
else:
CacheFile["StdData"] = StdData.loc[:, iIDs]
CacheFile["_QS_IDs"] = iIDs
StdData = None # 释放数据
if self._OperationMode.SubProcessNum > 0:
Sub2MainQueue, PIDEvent = self._OperationMode._Event[self.Name]
Sub2MainQueue.put(1)
PIDEvent.wait()
self._isCacheDataOK = True
return StdData
