def get_ta(self, df, confer):
df = df.reset_index("date", drop=True)
if TEST:
print("TEST")
df = TaSetBase1().get_ta(df)
else:
df = TaSetBase1Ext4El().get_ta(df)
df = confer.score1.agn_score(df)
df = df[df.ta_NATR_7 > 1.0]
print("step 1")
from main.model import bitlize
df_bit = bitlize.feat_select(df, 0.8, confer.score1.get_name(), 1, 100)
tobe = [df[["date", "open", "high", "low", "close","volume"]]]
for i, each in df_bit.iterrows():
name = each["name"]
fname = each["fname"]
start = each["start"]
end = each["end"]
new = df.apply(lambda row: 1 if (row[fname]>=start and row[fname]<end) else 0, axis=1)
s = pd.Series(new, name = fname)
assert len(s) == len(tobe[0])
tobe.append(pd.Series(new, name = fname))
result = pd.concat(tobe, axis=1)
assert len(df) == len(result)
return result
def work(pool_num, symset, ta, scores, confer):
print(pool_num)
to_apends = []
Executor = concurrent.futures.ProcessPoolExecutor
with Executor(max_workers=pool_num) as executor:
futures = {executor.submit(_one_work, sym, ta, confer): sym for sym in symset}
for future in concurrent.futures.as_completed(futures):
sym = futures[future]
try:
data = future.result()
if len(data) < 300:
print(sym, "too short!")
continue
for score in scores:
assert isinstance(score, ScoreLabel)
data = score.agn_score(data)
data = data[data.ta_NATR_7 > 1.0]
to_apends.append(data)
print(sym)
except Exception as exc:
executor.shutdown(wait=False)
sys.exit(1)
# longs = []
# shorts = []
# for each in to_apends:
# if is_trend_long(each):
# longs.append(each)
# else:
# shorts.append(each)
# print("longs: ", len(longs))
# print("shorts: ", len(shorts))
# return pd.concat(longs)
df = pd.concat(to_apends)
from main.model import bitlize
with Timer("bitlize.feat_select") as t:
df_bit1 = bitlize.feat_select(df, 0.8, confer.score1.get_name(), 1, 100, confer.n_pool)
df_bit2 = bitlize.feat_select(df, 0.8, confer.score1.get_name(), 2, 100, confer.n_pool)
df_bit = pd.concat([df_bit1, df_bit2], axis=0)
assert len(df_bit1) + len(df_bit2) == len(df_bit)
tobe = [df[["sym", "date", "open", "high", "low", "close", "volume", confer.score1.get_name()]]]
if confer.n_pool == 1:
for i, each in df_bit.iterrows():
name = each["name"]
fname = each["fname"]
start = each["start"]
end = each["end"]
new = df.apply(lambda row: 1 if (row[fname] >= start and row[fname] < end) else 0, axis=1)
s = pd.Series(new, name=fname)
assert len(s) == len(tobe[0])
tobe.append(pd.Series(new, name=name))
else:
Executor = concurrent.futures.ProcessPoolExecutor
with Timer("bit_apply") as t:
with Executor(max_workers=confer.n_pool) as executor:
futures = {
executor.submit(
bit_apply, df[[each["fname"]]].copy(), each["name"], each["fname"], each["start"], each["end"]
): (i, each)
for (i, each) in df_bit.iterrows()
}
for future in concurrent.futures.as_completed(futures):
try:
i, each = futures[future]
tobe.append(future.result())
except:
traceback.print_exc()
executor.shutdown(wait=False)
sys.exit(1)
result = pd.concat(tobe, axis=1)
assert len(df) == len(result)
return result