def __QS_prepareCacheData__(self, ids=None):
DTs = list(self._PID_DTs[self._OperationMode._iPID])
IDs = self._OperationMode._FactorPrepareIDs[self.Name]
if IDs is None: IDs = list(self._OperationMode.IDs)
if len(DTs) == 0: # 该进程未分配到计算任务
iDTs = [self._OperationMode.DateTimes[-1]]
for i, iDescriptor in enumerate(self._Descriptors):
iDescriptor._QS_getData(iDTs, pids=None)
StdData = pd.DataFrame(
columns=IDs,
dtype=("float" if self.DataType == "double" else "O"))
elif IDs:
StdData = self._calcData(
ids=IDs,
dts=DTs,
descriptor_data=[
iDescriptor._QS_getData(DTs, pids=None).values
for i, iDescriptor in enumerate(self._Descriptors)
])
StdData = pd.DataFrame(StdData, index=DTs, columns=IDs)
else:
StdData = pd.DataFrame(
index=DTs,
columns=IDs,
dtype=("float" if self.DataType == "double" else "O"))
if self._OperationMode._FactorPrepareIDs[self.Name] is None:
PID_IDs = self._OperationMode._PID_IDs
else:
PID_IDs = {
self._OperationMode._PIDs[i]: iSubIDs
for i, iSubIDs in enumerate(
partitionListMovingSampling(
IDs, len(self._OperationMode._PIDs)))
}
for iPID, iIDs in PID_IDs.items():
with self._OperationMode._PID_Lock[iPID]:
with shelve.open(self._OperationMode._CacheDataDir + os.sep +
iPID + os.sep + self.Name +
str(self._OperationMode._FactorID[self.Name])
) as CacheFile:
if "StdData" in CacheFile:
CacheFile["StdData"] = pd.concat(
[CacheFile["StdData"],
StdData.loc[:, iIDs]]).sort_index()
else:
CacheFile["StdData"] = StdData.loc[:, iIDs]
CacheFile["_QS_IDs"] = iIDs
StdData = None # 释放数据
if self._OperationMode.SubProcessNum > 0:
Sub2MainQueue, PIDEvent = self._OperationMode._Event[self.Name]
Sub2MainQueue.put(1)
PIDEvent.wait()
self._isCacheDataOK = True
return StdData
def __QS_prepareCacheData__(self, ids=None):
PID = self._OperationMode._iPID
StartDT = self._OperationMode._FactorStartDT[self.Name]
EndDT = self._OperationMode.DateTimes[-1]
StartInd, EndInd = self._OperationMode.DTRuler.index(
StartDT), self._OperationMode.DTRuler.index(EndDT)
DTs = list(self._OperationMode.DTRuler[StartInd:EndInd + 1])
IDs = self._OperationMode._FactorPrepareIDs[self.Name]
if IDs is None:
IDs = list(self._OperationMode._PID_IDs[PID])
else:
IDs = partitionListMovingSampling(
IDs, len(self._OperationMode._PIDs))[
self._OperationMode._PIDs.index(PID)]
if IDs:
DescriptorData = []
for i, iDescriptor in enumerate(self._Descriptors):
iStartInd = StartInd - self.LookBack[i]
iDTs = list(self._OperationMode.DTRuler[max(0, iStartInd
):StartInd]) + DTs
iDescriptorData = iDescriptor._QS_getData(iDTs,
pids=[PID]).values
if iStartInd < 0:
iDescriptorData = np.r_[np.full(shape=(
abs(iStartInd), iDescriptorData.shape[1]),
fill_value=np.nan),
iDescriptorData]
DescriptorData.append(iDescriptorData)
StdData = self._calcData(ids=IDs,
dts=DTs,
descriptor_data=DescriptorData,
dt_ruler=self._OperationMode.DTRuler)
StdData = pd.DataFrame(StdData, index=DTs, columns=IDs)
else:
StdData = pd.DataFrame(
index=DTs,
columns=IDs,
dtype=("float" if self.DataType == "double" else "O"))
with self._OperationMode._PID_Lock[PID]:
with shelve.open(self._OperationMode._CacheDataDir + os.sep + PID +
os.sep + self.Name +
str(self._OperationMode._FactorID[self.Name])
) as CacheFile:
CacheFile["StdData"] = StdData
CacheFile["_QS_IDs"] = IDs
self._isCacheDataOK = True
return StdData
def __QS_prepareCacheData__(self, ids=None):
PID = self._OperationMode._iPID
StartDT = self._OperationMode._FactorStartDT[self.Name]
EndDT = self._OperationMode.DateTimes[-1]
StartInd, EndInd = self._OperationMode.DTRuler.index(
StartDT), self._OperationMode.DTRuler.index(EndDT)
DTs = list(self._OperationMode.DTRuler[StartInd:EndInd + 1])
IDs = self._OperationMode._FactorPrepareIDs[self.Name]
if IDs is None:
IDs = list(self._OperationMode._PID_IDs[PID])
else:
IDs = partitionListMovingSampling(
IDs, len(self._OperationMode._PIDs))[
self._OperationMode._PIDs.index(PID)]
if IDs:
StdData = self._calcData(ids=IDs,
dts=DTs,
descriptor_data=[
iDescriptor._QS_getData(
DTs, pids=[PID]).values
for iDescriptor in self._Descriptors
])
StdData = pd.DataFrame(StdData, index=DTs, columns=IDs)
else:
StdData = pd.DataFrame(
index=DTs,
columns=IDs,
dtype=("float" if self.DataType == "double" else "O"))
with self._OperationMode._PID_Lock[PID]:
with shelve.open(self._OperationMode._CacheDataDir + os.sep + PID +
os.sep + self.Name +
str(self._OperationMode._FactorID[self.Name])
) as CacheFile:
CacheFile["StdData"] = StdData
CacheFile["_QS_IDs"] = IDs
self._isCacheDataOK = True
return StdData
def __QS_prepareCacheData__(self, ids=None):
DTs = list(self._PID_DTs[self._OperationMode._iPID])
IDs = self._OperationMode._FactorPrepareIDs[self.Name]
if IDs is None: IDs = list(self._OperationMode.IDs)
if len(DTs) == 0: # 该进程未分配到计算任务
iDTs = [self._OperationMode.DateTimes[-1]]
for i, iDescriptor in enumerate(self._Descriptors):
iDescriptor._QS_getData(iDTs, pids=None)
StdData = pd.DataFrame(
columns=IDs,
dtype=("float" if self.DataType == "double" else "O"))
elif IDs:
DescriptorData, StartInd = [], self._OperationMode.DTRuler.index(
DTs[0])
for i, iDescriptor in enumerate(self._Descriptors):
iStartInd = StartInd - self.LookBack[i]
iDTs = list(self._OperationMode.DTRuler[max(0, iStartInd
):StartInd]) + DTs
iDescriptorData = iDescriptor._QS_getData(iDTs,
pids=None).values
if iStartInd < 0:
iDescriptorData = np.r_[np.full(shape=(
abs(iStartInd), iDescriptorData.shape[1]),
fill_value=np.nan),
iDescriptorData]
DescriptorData.append(iDescriptorData)
StdData = self._calcData(ids=IDs,
dts=DTs,
descriptor_data=DescriptorData,
dt_ruler=self._OperationMode.DTRuler)
DescriptorData, iDescriptorData, StdData = None, None, pd.DataFrame(
StdData, index=DTs, columns=IDs)
else:
StdData = pd.DataFrame(
index=DTs,
columns=IDs,
dtype=("float" if self.DataType == "double" else "O"))
if self._OperationMode._FactorPrepareIDs[self.Name] is None:
PID_IDs = self._OperationMode._PID_IDs
else:
PID_IDs = {
self._OperationMode._PIDs[i]: iSubIDs
for i, iSubIDs in enumerate(
partitionListMovingSampling(
IDs, len(self._OperationMode._PIDs)))
}
for iPID, iIDs in PID_IDs.items():
with self._OperationMode._PID_Lock[iPID]:
with shelve.open(self._OperationMode._CacheDataDir + os.sep +
iPID + os.sep + self.Name +
str(self._OperationMode._FactorID[self.Name])
) as CacheFile:
if "StdData" in CacheFile:
CacheFile["StdData"] = pd.concat(
[CacheFile["StdData"],
StdData.loc[:, iIDs]]).sort_index()
else:
CacheFile["StdData"] = StdData.loc[:, iIDs]
CacheFile["_QS_IDs"] = iIDs
StdData = None # 释放数据
if self._OperationMode.SubProcessNum > 0:
Sub2MainQueue, PIDEvent = self._OperationMode._Event[self.Name]
Sub2MainQueue.put(1)
PIDEvent.wait()
self._isCacheDataOK = True
return StdData