def _QS_initOperation(self, start_dt, dt_dict, prepare_ids, id_dict): OldStartDT = dt_dict.get(self.Name, None) if (OldStartDT is None) or (start_dt < OldStartDT): dt_dict[self.Name] = start_dt StartInd, EndInd = self._OperationMode.DTRuler.index( dt_dict[self.Name]), self._OperationMode.DTRuler.index( self._OperationMode.DateTimes[-1]) DTs = self._OperationMode.DTRuler[StartInd:EndInd + 1] DTPartition = partitionList(DTs, len(self._OperationMode._PIDs)) self._PID_DTs = { iPID: DTPartition[i] for i, iPID in enumerate(self._OperationMode._PIDs) } PrepareIDs = id_dict.setdefault(self.Name, prepare_ids) if prepare_ids != PrepareIDs: raise __QS_Error__("因子 %s 指定了不同的截面!" % self.Name) for i, iDescriptor in enumerate(self._Descriptors): if self.DescriptorSection[i] is None: iDescriptor._QS_initOperation(start_dt, dt_dict, prepare_ids, id_dict) else: iDescriptor._QS_initOperation(start_dt, dt_dict, self.DescriptorSection[i], id_dict) if (self._OperationMode.SubProcessNum > 0) and (self.Name not in self._OperationMode._Event): self._OperationMode._Event[self.Name] = (Queue(), Event())
def __QS_prepareCacheData__(self, ids=None): PID = self._OperationMode._iPID StartDT = self._OperationMode._FactorStartDT[self.Name] EndDT = self._OperationMode.DateTimes[-1] StartInd, EndInd = self._OperationMode.DTRuler.index( StartDT), self._OperationMode.DTRuler.index(EndDT) DTs = list(self._OperationMode.DTRuler[StartInd:EndInd + 1]) IDs = self._OperationMode._FactorPrepareIDs[self.Name] if IDs is None: IDs = list(self._OperationMode._PID_IDs[PID]) else: IDs = partitionList(IDs, len(self._OperationMode._PIDs))[ self._OperationMode._PIDs.index(PID)] if IDs: StdData = self._calcData(ids=IDs, dts=DTs, descriptor_data=[ iDescriptor._QS_getData( DTs, pids=[PID]).values for iDescriptor in self._Descriptors ]) StdData = pd.DataFrame(StdData, index=DTs, columns=IDs) else: StdData = pd.DataFrame( index=DTs, columns=IDs, dtype=("float" if self.DataType == "double" else "O")) with self._OperationMode._PID_Lock[PID]: with shelve.open(self._OperationMode._CacheDataDir + os.sep + PID + os.sep + self.Name + str(self._OperationMode._FactorID[self.Name]) ) as CacheFile: CacheFile["StdData"] = StdData CacheFile["_QS_IDs"] = IDs self._isCacheDataOK = True return StdData
def _QS_initOperation(self, start_dt, dt_dict, prepare_ids, id_dict): if len(self._Descriptors) > len(self.LookBack): raise __QS_Error__("面板运算因子 : '%s' 的参数'回溯期数'序列长度小于描述子个数!" % self.Name) OldStartDT = dt_dict.get(self.Name, None) DTRuler = self._OperationMode.DTRuler if (OldStartDT is None) or (start_dt < OldStartDT): StartDT = dt_dict[self.Name] = start_dt StartInd, EndInd = DTRuler.index(StartDT), DTRuler.index( self._OperationMode.DateTimes[-1]) if (self.iLookBackMode == "扩张窗口") and (self.iInitData is not None) and ( self.iInitData.shape[0] > 0): if self.iInitData.index[-1] not in self._OperationMode.DTRuler: self._QS_Logger.warning( "注意: 因子 '%s' 的初始值不在时点标尺的范围内, 初始值和时点标尺之间的时间间隔将被忽略!" % (self.Name, )) else: StartInd = min( StartInd, self._OperationMode.DTRuler.index( self.iInitData.index[-1]) + 1) DTs = DTRuler[StartInd:EndInd + 1] if self.iLookBackMode == "扩张窗口": DTPartition = [DTs ] + [[]] * (len(self._OperationMode._PIDs) - 1) else: DTPartition = partitionList(DTs, len(self._OperationMode._PIDs)) self._PID_DTs = { iPID: DTPartition[i] for i, iPID in enumerate(self._OperationMode._PIDs) } else: StartInd = DTRuler.index(OldStartDT) PrepareIDs = id_dict.setdefault(self.Name, prepare_ids) if prepare_ids != PrepareIDs: raise __QS_Error__("因子 %s 指定了不同的截面!" % self.Name) for i, iDescriptor in enumerate(self._Descriptors): iStartInd = StartInd - self.LookBack[i] if iStartInd < 0: self._QS_Logger.warning("注意: 对于因子 '%s' 的描述子 '%s', 时点标尺长度不足!" % (self.Name, iDescriptor.Name)) iStartDT = DTRuler[max(0, iStartInd)] if self.DescriptorSection[i] is None: iDescriptor._QS_initOperation(iStartDT, dt_dict, prepare_ids, id_dict) else: iDescriptor._QS_initOperation(iStartDT, dt_dict, self.DescriptorSection[i], id_dict) if (self._OperationMode.SubProcessNum > 0) and (self.Name not in self._OperationMode._Event): self._OperationMode._Event[self.Name] = (Queue(), Event())
def __QS_prepareCacheData__(self, ids=None): DTs = list(self._PID_DTs[self._OperationMode._iPID]) IDs = self._OperationMode._FactorPrepareIDs[self.Name] if IDs is None: IDs = list(self._OperationMode.IDs) if len(DTs) == 0: # 该进程未分配到计算任务 iDTs = [self._OperationMode.DateTimes[-1]] for i, iDescriptor in enumerate(self._Descriptors): iDescriptor._QS_getData(iDTs, pids=None) StdData = pd.DataFrame( columns=IDs, dtype=("float" if self.DataType == "double" else "O")) elif IDs: StdData = self._calcData( ids=IDs, dts=DTs, descriptor_data=[ iDescriptor._QS_getData(DTs, pids=None).values for i, iDescriptor in enumerate(self._Descriptors) ]) StdData = pd.DataFrame(StdData, index=DTs, columns=IDs) else: StdData = pd.DataFrame( index=DTs, columns=IDs, dtype=("float" if self.DataType == "double" else "O")) if self._OperationMode._FactorPrepareIDs[self.Name] is None: PID_IDs = self._OperationMode._PID_IDs else: PID_IDs = { self._OperationMode._PIDs[i]: iSubIDs for i, iSubIDs in enumerate( partitionList(IDs, len(self._OperationMode._PIDs))) } for iPID, iIDs in PID_IDs.items(): with self._OperationMode._PID_Lock[iPID]: with shelve.open(self._OperationMode._CacheDataDir + os.sep + iPID + os.sep + self.Name + str(self._OperationMode._FactorID[self.Name]) ) as CacheFile: if "StdData" in CacheFile: CacheFile["StdData"] = pd.concat( [CacheFile["StdData"], StdData.loc[:, iIDs]]).sort_index() else: CacheFile["StdData"] = StdData.loc[:, iIDs] CacheFile["_QS_IDs"] = iIDs StdData = None # 释放数据 if self._OperationMode.SubProcessNum > 0: Sub2MainQueue, PIDEvent = self._OperationMode._Event[self.Name] Sub2MainQueue.put(1) PIDEvent.wait() self._isCacheDataOK = True return StdData
def __QS_prepareCacheData__(self, ids=None): PID = self._OperationMode._iPID StartDT = self._OperationMode._FactorStartDT[self.Name] EndDT = self._OperationMode.DateTimes[-1] StartInd, EndInd = self._OperationMode.DTRuler.index( StartDT), self._OperationMode.DTRuler.index(EndDT) DTs = list(self._OperationMode.DTRuler[StartInd:EndInd + 1]) IDs = self._OperationMode._FactorPrepareIDs[self.Name] if IDs is None: IDs = list(self._OperationMode._PID_IDs[PID]) else: IDs = partitionList(IDs, len(self._OperationMode._PIDs))[ self._OperationMode._PIDs.index(PID)] if IDs: DescriptorData = [] for i, iDescriptor in enumerate(self._Descriptors): iStartInd = StartInd - self.LookBack[i] iDTs = list(self._OperationMode.DTRuler[max(0, iStartInd ):StartInd]) + DTs iDescriptorData = iDescriptor._QS_getData(iDTs, pids=[PID]).values if iStartInd < 0: iDescriptorData = np.r_[np.full(shape=( abs(iStartInd), iDescriptorData.shape[1]), fill_value=np.nan), iDescriptorData] DescriptorData.append(iDescriptorData) StdData = self._calcData(ids=IDs, dts=DTs, descriptor_data=DescriptorData, dt_ruler=self._OperationMode.DTRuler) StdData = pd.DataFrame(StdData, index=DTs, columns=IDs) else: StdData = pd.DataFrame( index=DTs, columns=IDs, dtype=("float" if self.DataType == "double" else "O")) with self._OperationMode._PID_Lock[PID]: with shelve.open(self._OperationMode._CacheDataDir + os.sep + PID + os.sep + self.Name + str(self._OperationMode._FactorID[self.Name]) ) as CacheFile: CacheFile["StdData"] = StdData CacheFile["_QS_IDs"] = IDs self._isCacheDataOK = True return StdData
def _QS_updateStartDT(self, start_dt, dt_dict): OldStartDT = dt_dict.get(self.Name, None) if (OldStartDT is None) or (start_dt < OldStartDT): dt_dict[self.Name] = start_dt StartInd, EndInd = self._OperationMode.DTRuler.index( dt_dict[self.Name]), self._OperationMode.DTRuler.index( self._OperationMode.DateTimes[-1]) DTs = self._OperationMode.DTRuler[StartInd:EndInd + 1] DTPartition = partitionList(DTs, len(self._OperationMode._PIDs)) self._PID_DTs = { iPID: DTPartition[i] for i, iPID in enumerate(self._OperationMode._PIDs) } for i, iDescriptor in enumerate(self._Descriptors): iDescriptor._QS_updateStartDT(start_dt, dt_dict) if (self._OperationMode.SubProcessNum > 0) and (self.Name not in self._OperationMode._Event): self._OperationMode._Event[self.Name] = (Queue(), Event())
def __QS_prepareCacheData__(self, ids=None): DTs = list(self._PID_DTs[self._OperationMode._iPID]) IDs = self._OperationMode._FactorPrepareIDs[self.Name] if IDs is None: IDs = list(self._OperationMode.IDs) if len(DTs) == 0: # 该进程未分配到计算任务 iDTs = [self._OperationMode.DateTimes[-1]] for i, iDescriptor in enumerate(self._Descriptors): iDescriptor._QS_getData(iDTs, pids=None) StdData = pd.DataFrame( columns=IDs, dtype=("float" if self.DataType == "double" else "O")) elif IDs: DescriptorData, StartInd = [], self._OperationMode.DTRuler.index( DTs[0]) for i, iDescriptor in enumerate(self._Descriptors): iStartInd = StartInd - self.LookBack[i] iDTs = list(self._OperationMode.DTRuler[max(0, iStartInd ):StartInd]) + DTs iDescriptorData = iDescriptor._QS_getData(iDTs, pids=None).values if iStartInd < 0: iDescriptorData = np.r_[np.full(shape=( abs(iStartInd), iDescriptorData.shape[1]), fill_value=np.nan), iDescriptorData] DescriptorData.append(iDescriptorData) StdData = self._calcData(ids=IDs, dts=DTs, descriptor_data=DescriptorData, dt_ruler=self._OperationMode.DTRuler) DescriptorData, iDescriptorData, StdData = None, None, pd.DataFrame( StdData, index=DTs, columns=IDs) else: StdData = pd.DataFrame( index=DTs, columns=IDs, dtype=("float" if self.DataType == "double" else "O")) if self._OperationMode._FactorPrepareIDs[self.Name] is None: PID_IDs = self._OperationMode._PID_IDs else: PID_IDs = { self._OperationMode._PIDs[i]: iSubIDs for i, iSubIDs in enumerate( partitionList(IDs, len(self._OperationMode._PIDs))) } for iPID, iIDs in PID_IDs.items(): with self._OperationMode._PID_Lock[iPID]: with shelve.open(self._OperationMode._CacheDataDir + os.sep + iPID + os.sep + self.Name + str(self._OperationMode._FactorID[self.Name]) ) as CacheFile: if "StdData" in CacheFile: CacheFile["StdData"] = pd.concat( [CacheFile["StdData"], StdData.loc[:, iIDs]]).sort_index() else: CacheFile["StdData"] = StdData.loc[:, iIDs] CacheFile["_QS_IDs"] = iIDs StdData = None # 释放数据 if self._OperationMode.SubProcessNum > 0: Sub2MainQueue, PIDEvent = self._OperationMode._Event[self.Name] Sub2MainQueue.put(1) PIDEvent.wait() self._isCacheDataOK = True return StdData