def _one_work(sym, ta, confer, dirname=""):
if True:
filename = os.path.join(base.dir_eod(), dirname, sym + ".csv")
else:
filename = os.path.join(base.dir_eod(), dirname, sym + ".rel.csv")
try:
if not os.path.exists(filename):
print("Not exsits %s!!!!!!" % filename)
return None
df = pd.read_csv(filename)
#df = df[["date", "open", "high", "low", "close", "volume"]]
df[['volume']] = df[["volume"]].astype(float)
if df is None:
print(sym)
return
df["sym"] = sym
if (len(df) < 300):
print(sym, "too short!")
return None
if (len(df[df.closeo < 10]) / len(df) > 0.5):
print(sym, "price too low!")
return None
if (len(df[df.volume < 100000]) / len(df) > 0.5):
print(sym, "volume too low!")
return None
if len(df[df["high"] < df['close']]) > 0 or len(
df[df["low"] > df["close"]]) > 0:
print(sym, "high < close or low > close ")
return None
df = ta.get_ta(df, confer)
#df.to_pickle(os.path.join(base.dir_ta(ta.get_name()), sym+".pkl"))
return df
except:
traceback.print_exc()
assert False
def _one_work(sym, ta, confer, dirname = ""):
if True:
filename = os.path.join(base.dir_eod(), dirname, sym + ".csv")
else:
filename = os.path.join(base.dir_eod(), dirname, sym + ".rel.csv")
try:
if not os.path.exists(filename):
print("Not exsits %s!!!!!!" % filename)
return None
df = pd.read_csv(filename)
#df = df[["date", "open", "high", "low", "close", "volume"]]
df[['volume']] = df[["volume"]].astype(float)
if df is None:
print(sym)
return
df["sym"] = sym
if (len(df) < 300):
print(sym, "too short!")
return None
if (len(df[df.closeo< 10])/len(df) > 0.5):
print(sym, "price too low!")
return None
if (len(df[df.volume< 100000])/len(df) > 0.5):
print(sym, "volume too low!")
return None
if len(df[df["high"] < df['close']])>0 or len(df[df["low"] > df["close"]])>0:
print(sym, "high < close or low > close ")
return None
df = ta.get_ta(df, confer)
#df.to_pickle(os.path.join(base.dir_ta(ta.get_name()), sym+".pkl"))
return df
except:
traceback.print_exc()
assert False
def _one_work(sym, ta, confer, dirname=""):
if True:
filename = os.path.join(base.dir_eod(), dirname, sym + ".csv")
else:
filename = os.path.join(base.dir_eod(), dirname, sym + ".rel.csv")
try:
if not os.path.exists(filename):
print("Not exsits %s!!!!!!" % filename)
return None
df = pd.read_csv(filename)
#df = df[["date", "open", "high", "low", "close", "volume"]]
df[['volume']] = df[["volume"]].astype(float)
if df is None:
print(sym)
return
df["sym"] = sym
if (len(df) < 300):
print(sym, "too short!")
return None
#if (len(df[df.closeo< 10])/len(df) > 0.5):
# print(sym, "price too low!")
# return None
if (len(df[df.volume < 100000]) / len(df) > 0.5):
print(sym, "volume too low!")
return None
if len(df[df["high"] < df['close']]) > 0 or len(
df[df["low"] > df["close"]]) > 0:
print(sym, "high < close or low > close ")
return None
if confer.is_adj:
df2 = df[['sym', 'date', "open", 'high', 'low', 'close', 'volume']]
else:
df2 = df[[
'sym', 'date', "openo", 'higho', 'lowo', 'closeo', 'volume'
]]
df2.columns = [
'sym', 'date', 'open', 'high', 'low', 'close', 'volume'
]
return ta.get_ta(df2, confer)
except:
traceback.print_exc()
assert False
def _one_work(df_index, sym, confer, dirname = ""):
try:
if not os.path.exists(os.path.join(base.dir_eod(), dirname, sym + ".csv")):
print("Not exsits %s!!!!!!" % os.path.join(base.dir_eod(), dirname, sym + ".csv"))
return None
df = pd.read_csv(os.path.join(base.dir_eod(),dirname, sym + ".csv"))
#df = df[["date", "open", "high", "low", "close", "volume"]]
df[['volume']] = df[["volume"]].astype(float)
if df is None:
print(sym)
return
df["sym"] = sym
origin_len = len(df)
if (len(df) < 300):
print(sym, "too short!")
return None
if (len(df[df.close/df["adjrate"]< 10])/len(df) > 0.5):
print(sym, "price too low!")
return None
if (len(df[df.volume< 100000])/len(df) > 0.5):
print(sym, "volume too low!")
return None
if len(df[df["high"] < df['close']])>0 or len(df[df["low"] > df["close"]])>0:
print(sym, "high < close or low > close ")
return None
df_index.set_index("date", drop=True, inplace=True)
df.set_index("date", drop=True, inplace=True)
df_index = df_index[["close"]].rename(columns={"close":"iclose"})
df = pd.concat([df, df_index], axis=1, join='inner')
#df["close"] = df["close"]/df["iclose"]*2000
#df["open"] = df["open"]/df["iclose"]*2000
#df["high"] = df["high"]/df["iclose"]*2000
#df["low"] = df["low"]/df["iclose"]*2000
df.reset_index(drop=False).to_csv(os.path.join(base.dir_eod(), dirname, sym+".rel.csv"))
if len(df) < 100:
return None
return df
except:
traceback.print_exc()
assert False
def _one_work2(sym, tafunc):
exec "import main.pandas_talib.sig_%s as conf" % tafunc
df = pd.read_csv(os.path.join(base.dir_eod(), sym + ".csv"))
if df is None:
print sym
return
df = judge(df)
if 'ta_index_hdiff_close_1' in df.columns:
df = judge2(df)
df["sym"] = sym
df = conf.main(df)
df.to_pickle(os.path.join(base.dir_ta(conf.__name__), sym + ".pkl"))
def _one_work(sym, taname):
impstr = "import ta_%s as conf" % taname
exec impstr
df = pd.read_csv(os.path.join(base.dir_eod(), sym + ".csv"))
if df is None:
print sym
return
df = judge(df)
if 'ta_index_hdiff_close_1' in df.columns:
df = judge2(df)
df["sym"] = sym
df = conf.main(df)
df.to_pickle(os.path.join(base.dir_ta(taname), sym + ".pkl"))
def _one_work(sym, confer, dirname = ""):
if True:
filename = os.path.join(base.dir_eod(), dirname, sym + ".csv")
else:
filename = os.path.join(base.dir_eod(), dirname, sym + ".rel.csv")
try:
if not os.path.exists(filename):
print("Not exsits %s!!!!!!" % filename)
return None
df = pd.read_csv(filename)
#df = df[["date", "open", "high", "low", "close", "volume"]]
df[['volume']] = df[["volume"]].astype(float)
if df is None:
print(sym)
return
df["sym"] = sym
for score in confer.scores:
df = score.agn_score(df)
return df
except:
traceback.print_exc()
assert False
def work(pool_num, symset, confer, dirname = ""):
to_apends = []
Executor = concurrent.futures.ProcessPoolExecutor
df_index = pd.read_csv(os.path.join(base.dir_eod(), "index", "^GSPC.csv"))
with Executor(max_workers=pool_num) as executor:
futures = {executor.submit(_one_work, df_index, sym, confer, dirname): sym for sym in symset}
for future in concurrent.futures.as_completed(futures):
sym = futures[future]
try:
data = future.result()
if data is None:
continue
except Exception as exc:
traceback.print_exc()
executor.shutdown(wait=False)
sys.exit(1)
def _one_work(sym, ta, confer):
try:
df = pd.read_csv(os.path.join(base.dir_eod(), sym + ".csv"))
df = df[["date", "open", "high", "low", "close", "volume"]]
df[["volume"]] = df[["volume"]].astype(float)
if df is None:
print(sym)
return
df["sym"] = sym
if TEST:
print("TEST")
df = df[df.date >= "2000-01-01"]
df = ta.get_ta(df, confer)
df.to_pickle(os.path.join(base.dir_ta(ta.get_name()), sym + ".pkl"))
return df
except:
traceback.print_exc()
assert False
def _one_work(sym, confer, dirname = ""):
filename = os.path.join(base.dir_eod(), dirname, sym + ".csv")
try:
if not os.path.exists(filename):
print("Not exsits %s!!!!!!" % filename)
return None
df = pd.read_csv(filename)
#df = df[["date", "open", "high", "low", "close", "volume"]]
df[['volume']] = df[["volume"]].astype(float)
if df is None:
print(sym)
return
df["sym"] = sym
for score in confer.scores:
df = score.agn_score(df)
return df
except:
traceback.print_exc()
assert False
def main(args):
symbols = eval("get_%s" % args.setname)()
engine.work(symbols, base.dir_eod(), args.poolnum)
def main2(poolnum=10, target=base.dir_eod(), symbols = sp100(0, 150).get_syms()):
engine.work(list(set(symbols)), target, poolnum)