def onCallFunc(filePath: str, fileNames: Iterable, readFunc: Callable):
# 进程名
pid = os.getpid()
flag = 0
resDict = defaultdict(list) # 每日数据容器
for fileSub in fileNames:
flag += 1
if fileSub is None:
continue
sta = time.time()
date = fileSub.split('.')[0]
# 1.Read
try:
data = readFunc(filePath, fileSub)
except Exception as e:
print(f"Read file error: {fileSub}, {e}")
saveData(
DBName=saveM['TxT']['DBName'],
position=saveM['TxT']['Path'],
data=f"Read file error {dt.datetime.now()}: {fileSub}, {e}",
fileName=f"{funcClass}Error")
continue # 读取出错后跳出该次循环
else:
# 2.Cal
for funcName, func in calFuncs.items():
try:
calRes = func(data=data.copy(), date=date)
except Exception as e:
print(f"{funcName} error: {fileSub}, {e}")
saveData(
DBName=saveM['TxT']['DBName'],
position=saveM['TxT']['Path'],
data=
f"{funcName} error {dt.datetime.now()}: {fileSub}, {e}",
fileName=f"{funcClass}Error")
continue # 函数计算出错后跳出该次循环
# None值不存储
if calRes is not None:
resDict[funcName].append(calRes)
# 3.Save
for resName, resValue in resDict.items(): # 对子函数返回值进行格式转化
resSwitch = switchFuncs[resName](resValue)
if resSwitch is not None: # 子函数无返回值不进行存储操作
saveData(DBName=saveM[resName]['DBName'],
position=saveM[resName]['Path'],
data=resSwitch,
fileName=f"{resName}-{date}")
# 记录已处理数据
with mp.Lock():
saveData(DBName=saveM['Record']['DBName'],
position=saveM['Record']['Path'],
data=fileSub,
fileName=funcClass)
print(
f"{funcClass} {pid:<5}:{date} {flag:>4}/{len(fileNames):>3}, 耗时{round(time.time() - sta, 4)}"
)
def onCallFunc(dataParams: Iterable, readFunc: Callable):
# 进程名
dataParams = list(filter(None, dataParams))
pid = os.getpid()
flag = 0
resDict = defaultdict(list) # 数据容器
sta = time.time()
# 循环处理
for sample in dataParams:
date, code = sample.split('_')
flag += 1
year, month, day = date.split('-')
codeID, exchange = code.split('.')
depthPath = DepthPath.format(year=year,
month=month,
day=day,
exchange=exchange,
codeID=codeID)
# 1.Read
try:
data = readFunc(depthPath)
except Exception as e:
print(f"Read file error: {date}, {code}, {e}")
saveData(DBName=saveM['TxT']['DBName'],
position=saveM['TxT']['Path'],
data=f"Read file error {dt.datetime.now()}: {date}, {code}, {e}",
fileName=f"{funcClass}Error")
continue # 读取出错后跳出该次循环
else:
# 2.Cal
for funcName, func in calFuncs.items():
try:
calRes = func(data=data.copy(), code=code, date=date)
except Exception as e:
print(f"{funcName} error: {date}, {code}, {e}")
saveData(DBName=saveM['TxT']['DBName'],
position=saveM['TxT']['Path'],
data=f"{funcName} error {dt.datetime.now()}: {date}, {code}, {e}",
fileName=f"{funcClass}Error")
continue # 函数计算出错后跳出该次循环
# None值不存储
if calRes is not None:
resDict[funcName].append(calRes)
resDict['ID'].append(sample)
if flag >= 1000 or flag == len(dataParams):
# 3.Save
for resName, resValue in resDict.items(): # 对子函数返回值进行格式转化
if resName == 'ID':
continue
resSwitch = switchFuncs[resName](resValue)
if resSwitch is not None: # 子函数无返回值不进行存储操作
saveData(DBName=saveM[resName]['DBName'],
position=saveM[resName]['Path'],
data=resSwitch,
fileName=f"Process-{pid}-{dt.datetime.now().strftime('%Y%m%d_%H%M%S')}")
# 记录已处理数据
with mp.Lock():
saveData(DBName=saveM['Record']['DBName'],
position=saveM['Record']['Path'],
data=resDict['ID'],
fileName=funcClass)
resDict['ID'].clear()
print(f"{funcClass} {pid:<5}: {flag:>3}/{len(dataParams):>3}, 耗时{round(time.time() - sta, 4)}")
def onCallFunc(filePathList: Iterable, readFunc: Callable):
# 进程名
pid = os.getpid()
flag = 0
for folderPathSub in filePathList:
flag += 1
if folderPathSub is None:
continue
resDict = defaultdict(list) # 每日数据容器
folder_date = os.path.split(folderPathSub)[-1]
date = folder_date[:4] + '-' + folder_date[4:6] + '-' + folder_date[
-2:] # 日期
fileNames = os.listdir(folderPathSub)
sta = time.time()
# 循环处理
for fileSub in fileNames:
code = f"{fileSub[2:-4]}.{fileSub[:2].upper()}"
if code not in effectID: # 剔除非股票数据
continue
# if fileSub not in ["sz000010.csv", "sz000010.csv", "sh600000.csv"]:
# continue
# 1.Read
try:
data = readFunc(folderPathSub, fileSub)
except Exception as e:
print(f"Read file error: {date}, {fileSub}, {e}")
saveData(
DBName=saveM['TxT']['DBName'],
position=saveM['TxT']['Path'],
data=
f"Read file error {dt.datetime.now()}: {date}, {fileSub}, {e}",
fileName=f"{funcClass}Error")
continue # 读取出错后跳出该次循环
else:
# 2.Cal
for funcName, func in calFuncs.items():
try:
calRes = func(data=data.copy(), code=code, date=date)
except Exception as e:
print(f"{funcName} error: {date}, {fileSub}, {e}")
saveData(
DBName=saveM['TxT']['DBName'],
position=saveM['TxT']['Path'],
data=
f"{funcName} error {dt.datetime.now()}: {date}, {fileSub}, {e}",
fileName=f"{funcClass}Error")
continue # 函数计算出错后跳出该次循环
# None值不存储
if calRes is not None:
resDict[funcName].append(calRes)
# 3.Save
for resName, resValue in resDict.items(): # 对子函数返回值进行格式转化
resSwitch = switchFuncs[resName](resValue)
if resSwitch is not None: # 子函数无返回值不进行存储操作
saveData(DBName=saveM[resName]['DBName'],
position=saveM[resName]['Path'],
data=resSwitch,
fileName=date)
# 记录已处理数据
with mp.Lock():
saveData(DBName=saveM['Record']['DBName'],
position=saveM['Record']['Path'],
data=folder_date,
fileName=funcClass)
print(
f"{funcClass} {pid:<5}:{date} {flag:>4}/{len(filePathList):>3}, 耗时{round(time.time() - sta, 4)}"
)
def onCallFunc(filePathList: Iterable, readFunc: Callable):
Time = defaultdict(list)
# 进程名
pid = os.getpid()
flag = 0
for folderPathSub in filePathList:
flag += 1
if folderPathSub is None:
continue
resDict = defaultdict(list) # 每日数据容器
date = os.path.split(folderPathSub)[-1]
fileNames = os.listdir(folderPathSub)
sta = time.time()
# 循环处理
for fileSub in fileNames:
print(f"{pid}-{fileSub}")
code = switchCode(fileSub[:-4])
if code not in effectID: # 剔除非股票数据
continue
# if code != '688321.SH':
# continue
try:
data = readFunc(folderPathSub, fileSub)
except Exception as e:
print(f"Read file error: {date}, {fileSub}, {e}")
saveData(DBName=saveM['TxT']['DBName'],
position=saveM['TxT']['Path'],
data=f"Read file error {dt.datetime.now()}: {date}, {fileSub}, {e}",
fileName=f"{funcClass}Error")
continue # 读取出错后跳出该次循环
else:
for funcName, func in calFuncs.items():
try:
staC = time.time()
calRes = func(data=data.copy(), code=code, date=date)
endC = time.time() - staC
Time[funcName].append(endC)
except Exception as e:
print(f"{funcName} error: {date}, {fileSub}, {e}")
saveData(DBName=saveM['TxT']['DBName'],
position=saveM['TxT']['Path'],
data=f"{funcName} error {dt.datetime.now()}: {date}, {fileSub}, {e}",
fileName=f"{funcClass}Error")
continue # 函数计算出错后跳出该次循环
# None值不存储
if calRes is not None:
resDict[funcName].append(calRes)
for resName, resValue in resDict.items(): # 对子函数返回值进行格式转化
resSwitch = switchFuncs[resName](resValue)
if resSwitch is not None: # 子函数无返回值不进行存储操作
saveData(DBName=saveM[resName]['DBName'],
position=saveM[resName]['Path'],
data=resSwitch,
fileName=date)
# 记录已处理数据
with mp.Lock():
saveData(DBName=saveM['Record']['DBName'],
position=saveM['Record']['Path'],
data=date,
fileName=funcClass)
print(f"{funcClass} {pid:<5}:{date} {flag:>4}/{len(filePathList):>3}, 耗时{round(time.time() - sta, 4)}")
dataRes = pd.DataFrame(Time)
res = dataRes.describe()
dataRes.to_csv(f'C:\\Users\\Administrator\\Desktop\\Test\\{pid}.csv')
res.to_csv(f'C:\\Users\\Administrator\\Desktop\\Test\\{pid}_res.csv')