def get_signal_history(): dthour = request.args.get('datetime') du = DateUtil() specific_hour_dt = du.parse_time_string(dthour) predictions = [] coins = generate_coins(specific_hour_dt) coininfo = CoinInfo() for coin in coins: earliest_date = dbconn.get_earliest_date_in_db(coin) print(coin.name, "earliest date", earliest_date) one_day_before = specific_hour_dt + timedelta(days=-1) coin.loadtime = one_day_before.strftime("%Y-%m-%d") print("checking pred in DB for ", coin, "at", specific_hour_dt) tsfrom = (specific_hour_dt + timedelta(hours=-1)).strftime("%Y-%m-%d %H:00:00") tsto = specific_hour_dt.strftime("%Y-%m-%d %H:00:00") print(tsfrom, tsto) pred = dbconn.check_prediction_in_db(coin, tsfrom, tsto) print("pred:", pred) if pred is None: coininfo.do_prepare(coin, specific_hour_dt) predictions = do_predict(coin, predictions, specific_hour_dt) else: predictions.append(pred) return jsonify({'predictions': predictions})
def update_signal_recalculate(): dthour = request.args.get('datetime') du = DateUtil() specific_hour_dt = du.parse_time_string(dthour) predictions = [] coins = generate_coins(specific_hour_dt) for coin in coins: earliest_date = dbconn.get_earliest_date_in_db(coin) print(coin.name, "earliest date", earliest_date) one_day_before = specific_hour_dt + timedelta(days=-1) coin.loadtime = one_day_before.strftime("%Y-%m-%d") print("checking pred in DB for ", coin, "at", specific_hour_dt) tsfrom = (specific_hour_dt + timedelta(hours=-1)).strftime("%Y-%m-%d %H:00:00") tsto = specific_hour_dt.strftime("%Y-%m-%d %H:00:00") print(tsfrom, tsto) do_prepare(coin, specific_hour_dt) predictions = do_predict(coin, predictions, specific_hour_dt) return jsonify({'predictions': predictions})
def get_nr_of_hour_distance_from_server(self): nowdt = datetime.now() dateutil = DateUtil() nowmillis = dateutil.unix_time_millis(nowdt) server_time = self.get_server_time()['serverTime'] diff = (nowmillis - server_time) / 1000 / 3600 diff = round(diff) return diff
def test_dateutil1(self): dateutil = DateUtil() ba = BinanceApi() print(ba.get_server_time()) nowdt = datetime.now() nowdt = nowdt + timedelta(hours=-2) binancedt_int = dateutil.binance_datetime_int(nowdt) print("binancedt_int ", binancedt_int)
def get_last_n_hour_by_specific_hour_by_coin(self, coin, specific_hour, n_hours): coinname = coin.name coinnameupper = coinname.upper() offset_from_server = self.get_nr_of_hour_distance_from_server() #once you have to substract the offset from server targethour = specific_hour + timedelta(hours=-1 * offset_from_server) #twice you have to substract because the server end time means until the next hour. >:) targethour = targethour + timedelta(hours=-1) date_util = DateUtil() start_time = round(date_util.unix_time_millis(targethour)) apiurl='https://api.binance.com/api/v1/klines?symbol='+coinnameupper+'BTC&endTime='+str(start_time)+\ '&interval=1h&limit='+str(n_hours) print("calling: " + apiurl) response = requests.get(apiurl) return response.json()
def get_signal(): predictions = [] try: du = DateUtil() last_round_hour = du.parse_time_string(du.last_round_hour()) coins = generate_coins(last_round_hour) for coin in coins: pred = dbconn.check_prediction_in_db_last_hour(coin) if pred is None: do_prepare(coin, last_round_hour) predictions = do_predict(coin, predictions, last_round_hour) else: predictions.append(pred) return jsonify({'predictions': predictions}) except Exception as e: print("Exception: ", e) return jsonify({'exception': predictions})
def fill_past_signals(): predictions = [] coins = generate_coins(datetime.now()) coininfo = CoinInfo() du = DateUtil() for coin in coins: earliest_date = dbconn.get_earliest_date_in_db(coin) print(coin.name, "earliest date", earliest_date) start_datetime = du.parse_time_string( du.round_datetime_down(earliest_date)) last_round_hour = du.parse_time_string(du.last_round_hour()) #adding 24 hours to have earlier data in the past for prediction curr_datetime = start_datetime + timedelta(hours=+24) while (curr_datetime < last_round_hour): coin.reset_data_frames() one_day_before = curr_datetime + timedelta(days=-1) coin.loadtime = one_day_before.strftime("%Y-%m-%d") curr_datetime = curr_datetime + timedelta(hours=+1) print("checking pred in DB for ", coin, "at", curr_datetime) tsfrom = (curr_datetime + timedelta(hours=-1)).strftime("%Y-%m-%d %H:00:00") tsto = curr_datetime.strftime("%Y-%m-%d %H:00:00") print(tsfrom, tsto) pred = dbconn.check_prediction_in_db(coin, tsfrom, tsto) print("pred:", pred) if pred is None: coininfo.do_prepare(coin, curr_datetime) predictions = do_predict(coin, predictions, curr_datetime) else: predictions.append(pred) return jsonify({'predictions': predictions})
def do_predict(coin, predictions, specific_hour): ##print(coin.gtdf.head()) data = coin.pricehourly.copy() convert_hour_col(data) times = pd.DatetimeIndex(data['datetime']) cointrain = CoinTrain() X_gtdf = cointrain.increase_by_one_hour(coin.gtdf) X_grtdf = cointrain.increase_by_one_hour(coin.grtdf) X = data # not really summing gX = X.groupby([times.year, times.month, times.day, times.hour]).open.sum() gXdf = pd.DataFrame(gX) # not really max just copy gXdf['high'] = X.groupby([times.year, times.month, times.day, times.hour])['high'].max() gXdf['low'] = X.groupby([times.year, times.month, times.day, times.hour])['low'].max() gXdf['close'] = X.groupby([times.year, times.month, times.day, times.hour])['close'].max() gXdf['volumefrom'] = X.groupby( [times.year, times.month, times.day, times.hour])['volumefrom'].max() gXdf['volumeto'] = X.groupby( [times.year, times.month, times.day, times.hour])['volumeto'].max() gXdf['high_raised'] = X.groupby( [times.year, times.month, times.day, times.hour])['high_raised'].max() gXdf['low_raised'] = X.groupby( [times.year, times.month, times.day, times.hour])['low_raised'].max() gXdf['close_raised'] = X.groupby( [times.year, times.month, times.day, times.hour])['close_raised'].max() print("X_grtdf") #print(X_grtdf) print("X_gtdf") #print(X_gtdf) cols = [ 'retweeter_followers', 'retweet_count', 'sum_posmulrfollower', 'sum_negmulrfollower', 'sum_neumulrfollower', 'sum_compmulrfollower' ] #type(data) # gXdf['retweeter_followers']=coin.grtdf['retweeter_followers'] # gXdf['retweet_count']=coin.grtdf['retweet_count'] # data[]=coin.grtdf[] print("renaming cols") gXdf.index = gXdf.index.rename(['year', 'month', 'day', 'hour']) gXdf = gXdf.merge(X_grtdf, how='left', left_index=True, right_index=True) Xdf = gXdf.merge(X_gtdf, how='left', left_index=True, right_index=True) data = Xdf data.reset_index(inplace=True) cointrain.spreadtweeteffect(data) cointrain.add_change_columns(data) data.fillna(0, inplace=True) data.drop( columns=[ 'year', 'month', 'hour', 'open', 'high', 'low', 'close', 'volumefrom', 'volumeto', # 'vf_change1','vt_change1', # 'vfvt_ratio','vtvf_ratio', # 'c_o_change', 'h_o_change', 'l_o_change', # 'c_o_change1', 'h_o_change1', 'l_o_change1', 'o_change1', 'o_change2', # 'o_change3', 'o_change4', 'o_change5', 'o_change6', 'o_change1_3', # 'o_change1_12', # 'retweeter_followers', # 'retweet_count', # 'sum_posmulrfollower', 'sum_negmulrfollower', # 'sum_neumulrfollower', 'sum_compmulrfollower', 'follower_count', # 'tweet_count', # 'sum_posmulfollower', 'sum_negmulfollower', # 'sum_neumulfollower', 'sum_compmulfollower', 'asia_market', 'eu_market', 'us_market', 'day', 'max_datetime_x', 'max_datetime_y' ], inplace=True) #data = data[COLS] coin.data_to_predict = data print(data.tail()) condition = data.columns != shouldbe_cols if (len(condition[condition == True]) > 0): #data.columns!=shouldbe_cols): print("Columns/Features are not the same as in the model, exiting") print("which col not equal: ", data.columns != shouldbe_cols) print("shouldbe_cols ") print(shouldbe_cols) print('data.columns') print(data.columns) exit(1) print("saving to storeage... ") spec_hour_str = str(specific_hour.strftime("%Y-%m-%d_%H-%M-%S")) coin.save_to_storeage(PHASE, tmpdir='runtime/' + spec_hour_str + '/') print("saving is done.") min_max_scaler = coin.read_scaler() scaled_data = min_max_scaler.transform(data) data = pd.DataFrame(scaled_data) #print(data.tail()) # load json and create model def precision(y_true, y_pred): threshold = 0.3 mult = 0.5 / threshold true_positives = K.sum(K.round(K.clip(y_true * y_pred * mult, 0, 1))) predicted_positives = K.sum(K.round(K.clip(y_pred * mult, 0, 1))) precision = true_positives / (predicted_positives + K.epsilon()) return precision def recall(y_true, y_pred): threshold = 0.3 mult = 0.5 / threshold true_positives = K.sum(K.round(K.clip(y_true * y_pred * mult, 0, 1))) possible_positives = K.sum(K.round(K.clip(y_true, 0, 1))) recall = true_positives / (possible_positives + K.epsilon()) return recall #metrics=[precision,recall,'accuracy'] metrics = {"precision": precision, 'recall': recall} print("loading model for coin:" + coin.name) #model = load_model("./data/altcoin-storage/"+coin.name+"_keras_model.h5",custom_objects=metrics) #loading once> p = Prediction() model = p.load_model(coin, metrics) print("doing predictions.") pred = model.predict(data) print("predictions are ready.") coinbinancename = coin.name.upper() + "BTC" chance = pred[len(pred) - 1][0] signal = 0 treshold = coin.treshold if chance > treshold: signal = 1 #generating prediction du = DateUtil() specific_hour_minus_one = specific_hour + timedelta(hours=-1) pred = p.generate_prediction(specific_hour_minus_one, specific_hour, coinbinancename, chance, treshold, signal) dbconn.save_predictions([pred]) predictions.append(pred) return predictions
def test_dateutil0(self): dateutil = DateUtil() print( datetime.fromtimestamp( int("1523689200")).strftime('%Y-%m-%d %H:%M:%S'))
def __init__(self): print("init TweetIO") self.dateutil=DateUtil()
def test_dateutil0(self): dateutil = DateUtil() self.print_unix_as_date("1523692799000")