def get_ta(self, df, confer): df = df.reset_index("date", drop=True) if TEST: print("TEST") df = TaSetBase1().get_ta(df) else: df = TaSetBase1Ext4El().get_ta(df) df = confer.score1.agn_score(df) df = df[df.ta_NATR_7 > 1.0] print("step 1") from main.model import bitlize df_bit = bitlize.feat_select(df, 0.8, confer.score1.get_name(), 1, 100) tobe = [df[["date", "open", "high", "low", "close","volume"]]] for i, each in df_bit.iterrows(): name = each["name"] fname = each["fname"] start = each["start"] end = each["end"] new = df.apply(lambda row: 1 if (row[fname]>=start and row[fname]<end) else 0, axis=1) s = pd.Series(new, name = fname) assert len(s) == len(tobe[0]) tobe.append(pd.Series(new, name = fname)) result = pd.concat(tobe, axis=1) assert len(df) == len(result) return result
def work(pool_num, symset, ta, scores, confer): print(pool_num) to_apends = [] Executor = concurrent.futures.ProcessPoolExecutor with Executor(max_workers=pool_num) as executor: futures = {executor.submit(_one_work, sym, ta, confer): sym for sym in symset} for future in concurrent.futures.as_completed(futures): sym = futures[future] try: data = future.result() if len(data) < 300: print(sym, "too short!") continue for score in scores: assert isinstance(score, ScoreLabel) data = score.agn_score(data) data = data[data.ta_NATR_7 > 1.0] to_apends.append(data) print(sym) except Exception as exc: executor.shutdown(wait=False) sys.exit(1) # longs = [] # shorts = [] # for each in to_apends: # if is_trend_long(each): # longs.append(each) # else: # shorts.append(each) # print("longs: ", len(longs)) # print("shorts: ", len(shorts)) # return pd.concat(longs) df = pd.concat(to_apends) from main.model import bitlize with Timer("bitlize.feat_select") as t: df_bit1 = bitlize.feat_select(df, 0.8, confer.score1.get_name(), 1, 100, confer.n_pool) df_bit2 = bitlize.feat_select(df, 0.8, confer.score1.get_name(), 2, 100, confer.n_pool) df_bit = pd.concat([df_bit1, df_bit2], axis=0) assert len(df_bit1) + len(df_bit2) == len(df_bit) tobe = [df[["sym", "date", "open", "high", "low", "close", "volume", confer.score1.get_name()]]] if confer.n_pool == 1: for i, each in df_bit.iterrows(): name = each["name"] fname = each["fname"] start = each["start"] end = each["end"] new = df.apply(lambda row: 1 if (row[fname] >= start and row[fname] < end) else 0, axis=1) s = pd.Series(new, name=fname) assert len(s) == len(tobe[0]) tobe.append(pd.Series(new, name=name)) else: Executor = concurrent.futures.ProcessPoolExecutor with Timer("bit_apply") as t: with Executor(max_workers=confer.n_pool) as executor: futures = { executor.submit( bit_apply, df[[each["fname"]]].copy(), each["name"], each["fname"], each["start"], each["end"] ): (i, each) for (i, each) in df_bit.iterrows() } for future in concurrent.futures.as_completed(futures): try: i, each = futures[future] tobe.append(future.result()) except: traceback.print_exc() executor.shutdown(wait=False) sys.exit(1) result = pd.concat(tobe, axis=1) assert len(df) == len(result) return result