Пример #1
0
    def run(self, trade_time, trade_price, trade_size):
        self.price_array.append(trade_price)
        self.size_array.append(trade_size)
        if len(self.price_array) == int(self.tick):
            self.data['low'].append(min(self.price_array))
            self.data['high'].append(max(self.price_array))
            self.data['volume'].append(sum(self.size_array))
            close = self.price_array[-1]
            self.data['close'].append(close)
            if len(self.data['volume']) > 1:
                force = ((close - self.data['close'][-2]) *
                         self.data['volume'][-1])
            else:
                force = 0

            self.data['force'].append(force)

            force_array = np.array(self.data['force']).astype('double')
            ema = stream.EMA(force_array, timeperiod=self.force_ema_period)
            if math.isnan(ema):
                self.data['ema'].append(0)
            else:
                self.data['ema'].append(ema)

            self.price_array.clear()
            self.size_array.clear()
Пример #2
0
def longterm():
    # part where I manage the non-funding long/shorts using 1054, 373 dual ema
    global desired_contracts, recalculate_flag, previous_trade
    old_desired_direction = 'none'
    previous_calc_time = datetime.datetime.now(
        tz=pytz.UTC) - datetime.timedelta(minutes=200)
    ema1 = []
    ema2 = []
    recalculate_flag = False
    old_desired_direction = None

    if os.path.isfile('previous_trade'):
        previous_trade = pickle.load(open('previous_trade', 'rb'))
        if previous_trade > 0:
            old_desired_direction = 'long'
        else:
            old_desired_direction = 'short'
    else:
        previous_trade = None
        recalculate_flag = True
    sleep(10)
    while True:

        now = datetime.datetime.now(tz=pytz.UTC)
        if previous_calc_time and now.minute == previous_calc_time.minute and not recalculate_flag:
            sleep(1)
            continue

        while bm.ws.exited or not bm.ws.wst.is_alive():
            bm.reconnect()
            sleep(1)
        if not testing:
            with lock:
                while not get_history_bitmex.main(bm):
                    pass
        time_since_prev_calc = datetime.datetime.now(
            tz=pytz.UTC) - previous_calc_time
        # num_to_calculate = int(time_since_prev_calc.seconds / 60)
        previous_calc_time = datetime.datetime.now(tz=pytz.UTC)
        with lock:
            globalvar.dbcur.execute(
                "SELECT * FROM bmxswap ORDER BY time DESC limit 1427")
            data = globalvar.dbcur.fetchall()
            data = np.array(data[::-1])
        # while num_to_calculate > 0:

        while len(ema1) < 373:
            offset = 373 - len(ema1)
            truncated_data = data[-offset - 1054:-offset]
            ema1.append(ta.EMA(truncated_data[:, 4], timeperiod=1054))
        while len(ema2) < 3:
            offset = 3 - len(ema2)
            truncated_data_ema1 = ema1[-offset - 373:-offset]
            ema2.append(ta.EMA(np.array(truncated_data_ema1), timeperiod=370))
        if old_desired_direction is None:
            if ema2[-1] > ema2[-2]:
                old_desired_direction = 'long'
            else:
                old_desired_direction = 'short'
        prediction_ema = ema2[-1] > ema2[-2]

        if prediction_ema > 0:  #
            desired_direction = 'long'
            if desired_direction != old_desired_direction:
                logger.info("longterm going long")
                recalculate_flag = True
            logger.debug("continuing going long")
        else:
            desired_direction = 'short'
            if desired_direction != old_desired_direction:
                logger.info("longterm going short")
                recalculate_flag = True
            logger.debug("continuing going short")
        if np.nan in np.array(ema1):
            logger.critical("got NAN in prediction_ema, restarting")
            # trade(0)
            with lock:
                desired_contracts = 0
                sleep(10)
            while bm.position(symbol)['currentQty'] != 0:
                sleep(1)
                pass
            raise ValueError("got NAN in prediction_EMA")

        if recalculate_flag:
            desired_contracts_longterm = desired_contracts_func(
                desired_direction=desired_direction, leverage_desired=5)
            previous_trade = desired_contracts_longterm
            recalculate_flag = False
            pickle.dump(desired_contracts_longterm,
                        open('previous_trade', 'wb'))
        else:
            desired_contracts_longterm = previous_trade
        # num_to_calculate += 1
        while bm.ws.exited or not bm.ws.wst.is_alive():
            bm.reconnect()
            sleep(1)
        # trade(desired_contracts_outer, longterm=True)
        if funding_has_control is False:
            with lock:
                desired_contracts = desired_contracts_longterm
        # finishing up
        del ema1[0]
        del ema2[0]
        old_desired_direction = desired_direction
        recalculate_flag = False
Пример #3
0
def objective(variables, data=None, size=None, train=None, plot=None):
    variables[0], variables[1] = \
        int(variables[0]), int(variables[1])
    usd = 100.0  # investing 5 dollars at 20x
    position = []
    long = False
    in_position = False
    trades = 0
    EMA_length = max(int(variables[0]), int(variables[1])) + 1
    #EMA_length = min(EMA_length, 80000)
    ema1, ema2, adx, plusdi, minusdi, buyselllist= [], [], [100], [], [], []
    # plusdi_ema, minusdi_ema, adx_ema = 0,0,0
    returns = 100
    initialized = False
    max_draw_down = 0
    mdd_relative = 0
    peak_value = 0
    stoploss_triggers = 0
    reversal_flag = False
    local_return_graph_x = []
    local_return_graph_y = []

    #if abs(variables[0]) > abs(variables[2]) + abs(variables[4]): return 0
    #if abs(variables[4]) > abs(variables[2]) + abs(variables[0]): return 0

    #variables[0], variables[2], variables[4] = 0,0,0\
    #ema1_ema, ema2_ema = 0,0

    for iteration in range(size):
        if iteration > .9 * size and train: break
        if iteration < .9 * size and not train: continue  # use 90% train, 10% test, separated in time
        line = data[-1 * size + iteration]  # single line
        if iteration != size - 1:
            max_exec_price = np.amax(data[[[-1 * size + iteration],[-1*size+iteration+2]], [1,2,3,4]])
            min_exec_price = np.amin(data[[[-1 * size + iteration],[-1*size+iteration+2]], [1,2,3,4]])
        else:
            max_exec_price = np.amax(line[1:5])
            min_exec_price = np.amin(line[1:5])
        # for item in [plusdi_ema, minusdi_ema, adx_ema]:
        #     if item == 0: item = line[1]
        try:
            # multiplier = (2 / (variables[1] + 1))
            truncated_data = data[-1 * size + iteration - EMA_length:-1 * size + iteration, :]  # numpy array view
            ema1.append(ta.EMA(truncated_data[:, 4], timeperiod=variables[0]))
            # adx_close = ta.ADX(truncated_data[:, 2], truncated_data[:, 3], truncated_data[:, 4], timeperiod=variables[2])/(100/variables[2])
            # if len(adx) == 0: adx = [adx_close]
            # else:
            #     adx.append((adx_close - adx[-1]) * multiplier + adx[-1])
            # plusdi_close = ta.PLUS_DI(truncated_data[:, 2], truncated_data[:, 3], truncated_data[:, 4], timeperiod=variables[2])/(100/variables[2])
            # if len(plusdi) == 0: plusdi = [plusdi_close]
            # else:
            #     plusdi.append((plusdi_close - plusdi[-1]) * multiplier + plusdi[-1])
            # minusdi_close = ta.MINUS_DI(truncated_data[:, 2], truncated_data[:, 3], truncated_data[:, 4], timeperiod=variables[2])/(100/variables[2])
            # if len(minusdi) == 0: minusdi = [minusdi_close]
            # else:
            #     minusdi.append((minusdi_close - minusdi[-1]) * multiplier + minusdi[-1])
            #print(int((variables[1]/stddev[-1])*.0056), stddev[-1])
            ema2.append(ta.EMA(truncated_data[:, 4], timeperiod=variables[1]))  # todo change
            ema1_last = ema1[-1]
            ema2_last = ema2[-1]
            if math.isnan(adx[-1]) or math.isnan(ema1[-1]) or math.isnan(ema2[-1]):
                raise ValueError
        except:
            print(list(variables), EMA_length)
            print(ema1, ema2, adx)
            e = sys.exc_info()
            print(("<p>Error: %s</p>" % e))
        # multiplier = (2 / (10 + 1))
        # plusdi_ema = (plusdi[-1] - plusdi_ema) * multiplier + plusdi_ema
        # minusdi_ema = (minusdi[-1] - minusdi_ema) * multiplier + minusdi_ema
        # plusdi_ema = (adx[-1] - adx_ema) * multiplier + adx_ema

        if (iteration <= 1 and train) or (iteration <= .9 * size + 1 and not train): continue
        truncated_data = data[-1 * size + iteration - EMA_length:-1 * size + iteration, :]  # numpy array view

        buy = 0
        sell = 0
        # if ema1[-1] > line[1]:
        #     buy += variables[0] #* (ema1[-1] - line[1])
        # else:
        #     sell += variables[0] #* (ema1[-1] - line[1])
        # if ema2[-1] > line[1]:
        #     buy += variables[2] #* (ema2[-1] - line[1])
        # else:
        #     sell += variables[2] #* (ema2[-1] - line[1])

        if ema1[-1] > ema2[-1]:
        #if plusdi[-1] > minusdi[-1]:
            buy += 1
        else:
            sell += 1

        buy_sell = buy - sell
        buyselllist.append(buy_sell)
        #print(buy_sell)  # todo debug
        # end programmable bits, begin simulation
        if not initialized:
            initialized = True
            if buy_sell > 0:  # only executed on first run, from dollars to btc/shorts
                position = [usd * .9997, max_exec_price]  # store entry price
                usd = 0
                long = True
                in_position = True
                last_trade_time = iteration
                if plot:
                   #global long_enter_x
                   #global long_enter_y
                    long_enter_x.append(line[0])
                    long_enter_y.append(line[1]-.1)
                    return_graph_x.append(line[0])
                    return_graph_y.append(position[0])
                    local_return_graph_x.append(line[0])
                    local_return_graph_y.append(position[0])
                continue
            elif buy_sell < 0:
                position = [usd * .9997, min_exec_price]  # store entry price
                usd = 0
                long = False
                in_position = True
                last_trade_time = iteration
                if plot:
                   #global short_enter_x
                   #global short_enter_y
                    short_enter_x.append(line[0])
                    short_enter_y.append(line[1]+.1)
                    return_graph_x.append(line[0])
                    return_graph_y.append(position[0])
                    local_return_graph_x.append(line[0])
                    local_return_graph_y.append(position[0])
                continue
            else:
                if buy_sell > 0:
                    long = True
                    in_position = False
                    last_trade_time = iteration
                    trailing_stop_max = 0
                    continue
                if buy_sell < 0:
                    long = False
                    in_position = False
                    last_trade_time = iteration
                    trailing_stop_max = 0
                    continue
        time_since_last_trade = iteration - last_trade_time
        if in_position and position[0] > peak_value:
            peak_value = position[0]
        if initialized and in_position:
            if long:
                returns = usd + (line[1] / position[1]) * position[0]
                draw_down = (returns - peak_value) / peak_value
                if draw_down < max_draw_down:
                    max_draw_down = draw_down
                    #if plot: print("new mdd long from", position[1], "to", line[1])
                dd_relative = (returns - position[0]) / position[0]
                trailing_stop_max = (line[1] - np.max(data[-size + last_trade_time:-size + iteration, 4])) / np.max(
                    data[-size + last_trade_time:-size + iteration, 4])
                if dd_relative < mdd_relative:
                    mdd_relative = dd_relative
                    # if plot: print("new relative dd long from", position[1], "to", line[1], "at", iteration)

            else:
                returns = usd + (position[1] /line[1]) * position[0]
                draw_down = (returns - peak_value) / peak_value
                if draw_down < max_draw_down:
                    max_draw_down = draw_down
                    #if plot: print("new mdd short from", position[1], "to", line[1])
                dd_relative = (returns - position[0]) / position[0]
                trailing_stop_max = (np.min(data[-size + last_trade_time:-size + iteration, 4]) - line[1]) / line[1]

                if dd_relative < mdd_relative:
                    mdd_relative = dd_relative
                    # if plot: print("new rel dd short from", position[1], "to", line[1], "at", iteration)
        if not in_position:
            returns = usd
            if long:
                trailing_stop_max = (line[1] - np.min(data[-size + last_trade_time:-size + iteration, 4])) / np.min(data[-size + last_trade_time:-size + iteration, 4])
            else:
                trailing_stop_max = (np.max(data[-size + last_trade_time:-size + iteration, 4]) - line[1]) / line[1]
        if in_position:
            if long:
                # if adx[-1] > variables[3]:# and line[1] > ema2[-1] - (ema2[-1] * variables[4]):  # trailing_stop_max > variables[4]:
                    desire_position = True
                # else:
                #     desire_position = False
            else:
                # if adx[-1] > variables[3]:# and line[1] < ema2[-1] + (ema2[-1] * variables[4]):
                    desire_position = True
                # else:
                #     desire_position = False
        if not in_position:
            if long:
                # if adx[-1] > variables[3]:# and line[1] > ema2[-1]: # and trailing_stop_max > -variables[4] * 4:
                    desire_position = True
                # else:
                #     desire_position = False
            else:
                # if adx[-1] > variables[3]:# and line[1] < ema2[-1]:
                    desire_position = True
                # else:
                #     desire_position = False

        if not desire_position and in_position:  # stop loss section 1
            if not long:
                usd = (position[1] / max_exec_price) * position[0]
                position = 0
                # position = [usd * .9997, max(line[1:5])]
                # usd = 0.0
                #long = False
                in_position = False
                stoploss_triggers += 1
                last_trade_time = iteration
                reversal_flag = True
                if plot:
                   #global short_exit_x
                   #global short_exit_y
                    short_exit_x.append(line[0])
                    short_exit_y.append(line[1] + .1)
                    return_graph_x.append(line[0])
                    return_graph_y.append(returns)
                    local_return_graph_x.append(line[0])
                    local_return_graph_y.append(returns)
                continue
            if long:  # stoploss part 1
                usd = (min_exec_price / position[1]) * position[0]  # assume bad execution
                position = 0
                # position = [usd * .9997, min(line[1:5])]
                # usd = 0.0
                #long = True
                in_position = False
                stoploss_triggers += 1
                last_trade_time = iteration
                reversal_flag = True
                if plot:
                    #global long_exit_x
                    #global long_exit_y
                    long_exit_x.append(line[0])
                    long_exit_y.append(line[1] - .1)
                    return_graph_x.append(line[0])
                    return_graph_y.append(returns)
                    local_return_graph_x.append(line[0])
                    local_return_graph_y.append(returns)
                continue
        upper_bound = ema2[-1] * (1+(variables[2]*(variables[3]-(min(time_since_last_trade, variables[4])/(variables[4]/(variables[3]-1))))))
        lower_bound = ema2[-1] * (1-(variables[2]*(variables[3]-(min(time_since_last_trade, variables[4])/(variables[4]/(variables[3]-1))))))


        if desire_position and in_position: # section 2
            if long:  # section where we release control back to buy_sell
                if line[1] < lower_bound:
                    reversal_flag = False

            else:
                if line[1] > upper_bound:
                    reversal_flag = False

            if long:
                if buy_sell > 0.0:
                    reversal_flag = False

            else:
                if buy_sell < 0.0:
                    reversal_flag = False


            if not long and ((buy_sell > 0.0) or (line[1] > upper_bound)) and not reversal_flag: # main buy/sell
                usd = (position[1] / max_exec_price) * position[0]
                position = 0
                position = [usd * .9997, max_exec_price]
                usd = 0.0
                long = True
                in_position = True
                trades += 1
                last_trade_time = iteration
                if line[1] > upper_bound:
                    reversal_flag = True
                    stoploss_triggers += 1
                else:
                    reversal_flag = False
                if plot:
                    #global long_enter_x
                    #global long_enter_y
                    long_enter_x.append(line[0])
                    long_enter_y.append(max_exec_price-.1)
                    return_graph_x.append(line[0])
                    return_graph_y.append(returns)
                    local_return_graph_x.append(line[0])
                    local_return_graph_y.append(returns)
                continue

            if long and ((buy_sell < 0.0) or (line[1] < lower_bound)) and not reversal_flag:  #  main buy/sell 2
                #                      on 20x
                usd = (min_exec_price / position[1]) * position[0]  # assume bad execution
                position = 0
                position = [usd * .9997, min_exec_price]
                usd = 0.0
                long = False
                in_position = True
                trades += 1
                last_trade_time = iteration
                if line[1] < lower_bound:
                    reversal_flag = True
                    stoploss_triggers += 1
                else:
                    reversal_flag = False
                if plot:
                   #global short_enter_x
                   #global short_enter_y
                    short_enter_x.append(line[0])
                    short_enter_y.append(min_exec_price+.1)
                    return_graph_x.append(line[0])
                    return_graph_y.append(returns)
                    local_return_graph_x.append(line[0])
                    local_return_graph_y.append(returns)
                continue

        if not desire_position and not in_position:  # section 3
            if not long and ((buy_sell > 0.0) ^ (line[1] > upper_bound)): # moves understanding of long/short desire despite not being in position
                #usd = (position[1] / max(line[1:5])) * position[0]
                #position = 0
                #position = [usd * .9997, max(line[1:5])]
                #usd = 0.0
                long = True
                #in_position = True
                #trades += 1
                last_trade_time = iteration
                if line[1] > ema2[-1] * (1 + variables[2]):
                    reversal_flag = True
                else:
                    reversal_flag = False
                # if plot:
                #    #global long_enter_x
                #    #global long_enter_y
                #     long_enter_x.append(line[0])
                #     long_enter_y.append(line[1]-.1)
                #     return_graph_x.append(line[0])
                #     return_graph_y.append(returns)
                continue

            if long and ((buy_sell < 0.0) ^ (line[1] < lower_bound)):  # moves understanding of long/short desire despite not being in position
                #usd = (min(line[1:5]) / position[1]) * position[0]
                #position = 0
                # position = [usd * .9997, min(line[1:5])]
                # usd = 0.0
                long = False
                # in_position = True
                # trades += 1
                last_trade_time = iteration
                if line[1] < lower_bound:
                    reversal_flag = True
                else:
                    reversal_flag = False
                # if plot:
                #    #global short_enter_x
                #    #global short_enter_y
                #     short_enter_x.append(line[0])
                #     short_enter_y.append(line[1]+.1)
                #     return_graph_x.append(line[0])
                #     return_graph_y.append(returns)
                continue


        if desire_position and not in_position:  # if adx reverses direction, reenter position section 4
            if not long:  #
                #usd = (position[1] / max(line[1:5])) * position[0]
                #position = 0
                position = [usd * .9997, min_exec_price]
                usd = 0.0
                #long = True
                in_position = True
                trades += 1
                last_trade_time = iteration
                reversal_flag = False
                if plot:
                   #global short_enter_x
                   #global short_enter_y
                    short_enter_x.append(line[0])
                    short_enter_y.append(line[1]+.1)
                    return_graph_x.append(line[0])
                    return_graph_y.append(returns)
                    local_return_graph_x.append(line[0])
                    local_return_graph_y.append(returns)
                continue

            if long:
                #usd = (min(line[1:5]) / position[1]) * position[0]  # assume bad execution
                #position = 0
                position = [usd * .9997, min_exec_price]
                usd = 0.0
                #long = False
                in_position = True
                trades += 1
                last_trade_time = iteration
                reversal_flag = False
                if plot:
                   #global long_enter_x
                   #global long_enter_y
                    long_enter_x.append(line[0])
                    long_enter_y.append(line[1] - .1)
                    return_graph_x.append(line[0])
                    return_graph_y.append(returns)
                    local_return_graph_x.append(line[0])
                    local_return_graph_y.append(returns)
                continue


        if returns < 50 and not plot:  # fail-quick
            returns = 0
            position = [0, line[1]]
            max_draw_down = -.9271
            break
    ema1_switches = 0
    for index, item in enumerate(ema1):
        if (item > ema1[index-1]) is not (ema1[index-1] > ema1[index-2]):
            ema1_switches += 1
    ema2_switches = 0
    for index, item in enumerate(ema2):
        if (item > ema2[index - 1]) is not (ema2[index - 1] > ema2[index - 2]):
            ema2_switches += 1
    macd_switches = 0
    for index, item in enumerate(buyselllist):
        if (item > 0) is not (buyselllist[index-1] > 0):
            macd_switches += 1
    buyselllist = []
    longest_time_since_last_high = 0
    for index, tick in enumerate(local_return_graph_y):  # TODO TEMPORARY
        if index == 0:
            buyselllist.append(0)
            continue
        #print((tick - local_return_graph_y[index - 1]) / tick)
        buyselllist.append((tick - local_return_graph_y[index - 1]) / tick)
    print(sorted(buyselllist))
    if plot:
        for leverage in range(1, 101):
            max_value = 0
            last_value = 0
            time_of_last_high = local_return_graph_x[0]
            time_since_last_high = 0

            for tickindex, tick in enumerate(local_return_graph_y):
                if tickindex == 0:
                    max_value = tick
                    last_value = tick
                    continue
                delta = (tick - local_return_graph_y[tickindex - 1]) / tick
                delta *= leverage
                last_value *= delta + 1
                if last_value > max_value:
                    time_since_last_high = local_return_graph_x[tickindex] - time_of_last_high
                    time_of_last_high = local_return_graph_x[tickindex]
                    if time_since_last_high > longest_time_since_last_high:
                        longest_time_since_last_high = time_since_last_high
                    max_value = last_value
                if last_value < max_value / float(4):
                    break
            if last_value < max_value / float(4):  # when broken out of the inner for by too much leverage
                max_leverage = leverage - 1
                break
            elif leverage == 100:
                print("got max leverage 100 somehow")
                max_leverage = leverage
                break
    # max_leverage = min(math.floor(-1/min(mdd_relative, -.0001)), max_leverage)



    # print(max(adx), "avg:", np.average(adx), min(adx))
    if plot:
        print (ema1_switches, ema2_switches, "trade signals:", macd_switches, "MDD%", max_draw_down * 100, "dd_rel%:", mdd_relative * 100, "max leverage:", max_leverage)
    else:
        print(ema1_switches, ema2_switches, "trade signals:", macd_switches, "MDD%", max_draw_down * 100, "dd_rel%:",
              mdd_relative * 100)
    #print(max(atr), min(atr))
    #print("min, max, stddev, avg of stddev", min(stddev), max(stddev), np.std(stddev), np.average(stddev))


    if plot:
        global plot_minusdi
        global plot_plusdi
        if train:
            plt.plot(data[int(-len(ema1)-(size/10)):int(-(size/10)), 0], ema1)
            plt.plot(data[int(-len(ema2)-(size/10)):int(-(size/10)), 0], ema2)
            ##global plot_adx

            # plot_adx += adx
            plot_minusdi += minusdi
            plot_plusdi += plusdi
        else:
            plt.plot(data[-len(ema1):, 0], ema1)
            plt.plot(data[-len(ema2):, 0], ema2)
            # global plot_adx

            # plot_adx += adx
            plot_minusdi += minusdi
            plot_plusdi += plusdi
    if long and in_position:
        returns = usd + (min_exec_price / position[1]) * position[0]
        if plot:
            return_graph_x.append(line[0])
            return_graph_y.append(returns)
            local_return_graph_x.append(line[0])
            local_return_graph_y.append(returns)
    elif in_position:
        returns = usd + (position[1] / max_exec_price) * position[0]
        if plot:
            return_graph_x.append(line[0])
            return_graph_y.append(returns)
            local_return_graph_x.append(line[0])
            local_return_graph_y.append(returns)
    if not in_position:
        returns = usd
        if plot:
            return_graph_x.append(line[0])
            return_graph_y.append(returns)
            local_return_graph_x.append(line[0])
            local_return_graph_y.append(returns)
    if trades < 20 and not plot and train: returns = 0
    print(list(variables), returns, trades)
    if plot:
        print("Score metric:", returns * (1 + max_draw_down), "stoplosses hit:", stoploss_triggers,
              "longest defecit:", round(longest_time_since_last_high/86400., 2), "days")
    else:
        print("Score metric:", returns * (1 + max_draw_down), "stoplosses hit:", stoploss_triggers)

    return -(returns * (1 + max_draw_down))
Пример #4
0
def objective(variables,
              data=None,
              size=None,
              train=None,
              plot=None,
              try_leverages=None):
    variables[0] = int(variables[0])
    variables[1] = int(variables[1])
    if variables[0] < variables[1]: variables[1] = variables[0]

    usd = 1.0  # investing 5 dollars at 20x
    # position = []
    long = False
    in_position = False
    trades = 0
    EMA_length = int(variables[0]) + 1
    # EMA_length = max(EMA_length, 80000)
    ema1, ema2, adx, plusdi, minusdi, buyselllist = [], [], [100], [], [], []
    # plusdi_ema, minusdi_ema, adx_ema = 0,0,0
    returns = 1
    initialized = False
    max_draw_down = 0
    mdd_relative = 0
    peak_value = usd
    stoploss_triggers = 0
    reversal_flag = False
    local_return_graph_x = []
    local_return_graph_y = []

    #if abs(variables[0]) > abs(variables[2]) + abs(variables[4]): return 0
    #if abs(variables[4]) > abs(variables[2]) + abs(variables[0]): return 0

    #variables[0], variables[2], variables[4] = 0,0,0\
    #ema1_ema, ema2_ema = 0,0

    for iteration in range(size):
        if iteration > .9 * size and train: break
        if iteration < .9 * size and not train:
            continue  # use 90% train, 10% test, separated in time
        line = data[-1 * size + iteration]  # single line
        # if iteration != size - 1:
        #     max_exec_price = np.amax(data[[[-1 * size + iteration],[-1*size+iteration+2]], [1,2,3,4]])
        #     min_exec_price = np.amin(data[[[-1 * size + iteration],[-1*size+iteration+2]], [1,2,3,4]])
        # else:
        #     max_exec_price = np.amax(line[1:5])
        #     min_exec_price = np.amin(line[1:5])
        min_exec_price, max_exec_price = np.mean(line[1:5]), np.mean(line[1:5])
        # for item in [plusdi_ema, minusdi_ema, adx_ema]:
        #     if item == 0: item = line[1]
        try:
            # multiplier = (2 / (variables[1] + 1))
            truncated_data = data[-1 * size + iteration -
                                  EMA_length:-1 * size +
                                  iteration, :]  # numpy array view
            # ema1.append(ta.TRIMA(truncated_data[:, 4], timeperiod=variables[0]))
            ema1.append(ta.EMA(truncated_data[:, 4], timeperiod=variables[0]))
            if len(ema1) < 2: continue
            ema2.append(
                ta.EMA(np.array(ema1[int(-variables[1]):]),
                       timeperiod=int(min(len(ema1), variables[1]))))
            # ema1.append(ta.ULTOSC(truncated_data[:, 2], truncated_data[:, 3], truncated_data[:, 4],
            #                      timeperiod1=int(variables[0]/4.0), timeperiod2=int(variables[0]/2.0), timeperiod3=int(variables[0])))
            # adx_close = ta.ADX(truncated_data[:, 2], truncated_data[:, 3], truncated_data[:, 4], timeperiod=variables[2])/(100/variables[2])
            # if len(adx) == 0: adx = [adx_close]
            # else:
            #     adx.append((adx_close - adx[-1]) * multiplier + adx[-1])
            # plusdi_close = ta.PLUS_DI(truncated_data[:, 2], truncated_data[:, 3], truncated_data[:, 4], timeperiod=variables[2])/(100/variables[2])
            # if len(plusdi) == 0: plusdi = [plusdi_close]
            # else:
            #     plusdi.append((plusdi_close - plusdi[-1]) * multiplier + plusdi[-1])
            # minusdi_close = ta.MINUS_DI(truncated_data[:, 2], truncated_data[:, 3], truncated_data[:, 4], timeperiod=variables[2])/(100/variables[2])
            # if len(minusdi) == 0: minusdi = [minusdi_close]
            # else:
            #     minusdi.append((minusdi_close - minusdi[-1]) * multiplier + minusdi[-1])
            #print(int((variables[1]/stddev[-1])*.0056), stddev[-1])
            # ema2.append(ta.EMA(truncated_data[:, 4], timeperiod=variables[1]))  # todo change
            # ema1_last = ema1[-1]
            # ema2_last = ema2[-1]
            # ATR = ta.ATR(truncated_data[:, 2], truncated_data[:, 3], truncated_data[:, 4], timeperiod=variables[1]) / line[4]
            # uband, middleband, lband = ta.BBANDS(truncated_data[:, 4], timeperiod=variables[1], matype=5)
            # bw = (uband - lband) / middleband
            # adx = ta.ADX(truncated_data[:, 2], truncated_data[:, 3], truncated_data[:, 4], timeperiod=variables[1])/100
            # rsi = ta.RSI(truncated_data[:, 4], timeperiod=variables[1])/100.0
            # mfi = ta.MFI(truncated_data[:, 2], truncated_data[:, 3], truncated_data[:, 4], truncated_data[:, 5], timeperiod=variables[1])
            # if math.isnan(adx[-1]) or math.isnan(ema1[-1]) or math.isnan(ema2[-1]):
            #     raise ValueError
        except:
            print(list(variables), EMA_length)
            #print(ema1, ema2, adx)
            e = sys.exc_info()
            print(("<p>Error: %s</p>" % e))
        # multiplier = (2 / (10 + 1))
        # plusdi_ema = (plusdi[-1] - plusdi_ema) * multiplier + plusdi_ema
        # minusdi_ema = (minusdi[-1] - minusdi_ema) * multiplier + minusdi_ema
        # plusdi_ema = (adx[-1] - adx_ema) * multiplier + adx_ema

        if (iteration <= 1 and train) or (iteration <= .9 * size + 1
                                          and not train):
            continue
        # truncated_data = data[-1 * size + iteration - EMA_length:-1 * size + iteration, :]  # numpy array view

        buy = 0
        sell = 0
        # if ema1[-1] > line[1]:
        #     buy += variables[0] #* (ema1[-1] - line[1])
        # else:
        #     sell += variables[0] #* (ema1[-1] - line[1])
        # if ema2[-1] > line[1]:
        #     buy += variables[2] #* (ema2[-1] - line[1])
        # else:
        #     sell += variables[2] #* (ema2[-1] - line[1])

        #if abs(ema1[-1] - ema2[-1])/ema2[-1] > variables[2]:
        #if plusdi[-1] > minusdi[-1]:
        #slope = ema1[-1] - ema1[-2]
        if ema2[-1] > ema2[-2]:
            buy += 1
        else:
            sell += 1

        buy_sell = buy - sell
        buyselllist.append(buy_sell)
        #print(buy_sell)  #
        # end programmable bits, begin simulation
        if not initialized:
            initialized = True
            # position_live = Position(start_btc=usd/line[4])
            # in_position = False
            # returns = position_live.btc * line[4]
            # peak_value = returns

            if buy_sell > 0:  # only executed on first run, from dollars to btc/shorts
                position = [usd * .9997, max_exec_price]  # store entry price
                usd = 0
                long = True
                # in_position = True
                last_trade_time = iteration
                if plot:
                    #global long_enter_x
                    #global long_enter_y
                    long_enter_x.append(line[0])
                    long_enter_y.append(line[1] - .1)
                    return_graph_x.append(line[0])
                    return_graph_y.append(position[0])
                    local_return_graph_x.append(line[0])
                    local_return_graph_y.append(position[0])
                    init_trade = "long"
                continue
            elif buy_sell < 0:
                position = [usd * .9997, min_exec_price]  # store entry price
                usd = 0
                long = False
                # in_position = True
                last_trade_time = iteration
                if plot:
                    #global short_enter_x
                    #global short_enter_y
                    short_enter_x.append(line[0])
                    short_enter_y.append(line[1] + .1)
                    return_graph_x.append(line[0])
                    return_graph_y.append(position[0])
                    local_return_graph_x.append(line[0])
                    local_return_graph_y.append(position[0])
                    init_trade = "short"
                continue

        # time_since_last_trade = iteration - last_trade_time
        if in_position and position[0] > peak_value:
            peak_value = position[0]
        if initialized:
            if long:
                returns = usd + (line[1] / position[1]) * position[0]
            if not long:
                returns = usd + (position[1] / line[1]) * position[0]
            draw_down = (returns - peak_value) / peak_value
            if draw_down < max_draw_down:
                max_draw_down = draw_down
                #if plot: print("new mdd long from", position[1], "to", line[1])
            dd_relative = draw_down
            # trailing_stop_max = (line[1] - np.max(data[-size + last_trade_time:-size + iteration, 4])) / np.max(
            #     data[-size + last_trade_time:-size + iteration, 4])
            if dd_relative < mdd_relative:
                mdd_relative = dd_relative
                # if plot: print("new relative dd long from", position[1], "to", line[1], "at", iteration)
            #

            #     draw_down = (returns - peak_value) / peak_value
            #     if draw_down < max_draw_down:
            #         max_draw_down = draw_down
            #         #if plot: print("new mdd short from", position[1], "to", line[1])
            #     dd_relative = (returns - position[0]) / position[0]
            #     trailing_stop_max = (np.min(data[-size + last_trade_time:-size + iteration, 4]) - line[1]) / line[1]
            #
            #     if dd_relative < mdd_relative:
            #         mdd_relative = dd_relative
            #         # if plot: print("new rel dd short from", position[1], "to", line[1], "at", iteration)
        # if not in_position:
        #     returns = usd
        #     if long:
        #         trailing_stop_max = (line[1] - np.min(data[-size + last_trade_time:-size + iteration, 4])) / np.min(data[-size + last_trade_time:-size + iteration, 4])
        #     else:
        #         trailing_stop_max = (np.max(data[-size + last_trade_time:-size + iteration, 4]) - line[1]) / line[1]
        # if in_position:
        #     if long:
        #         # if adx[-1] > variables[3]:# and line[1] > ema2[-1] - (ema2[-1] * variables[4]):  # trailing_stop_max > variables[4]:
        #             desire_position = True
        #         # else:
        #         #     desire_position = False
        #     else:
        #         # if adx[-1] > variables[3]:# and line[1] < ema2[-1] + (ema2[-1] * variables[4]):
        #             desire_position = True
        #         # else:
        #         #     desire_position = False
        # if not in_position:
        #     if long:
        #         # if adx[-1] > variables[3]:# and line[1] > ema2[-1]: # and trailing_stop_max > -variables[4] * 4:
        #             desire_position = True
        #         # else:
        #         #     desire_position = False
        #     else:
        #         # if adx[-1] > variables[3]:# and line[1] < ema2[-1]:
        #             desire_position = True
        #         # else:
        #         #     desire_position = False

        # if not desire_position and in_position:  # stop loss section 1
        #     if not long:
        #         usd = (position[1] / max_exec_price) * position[0]
        #         position = 0
        #         # position = [usd * .9997, max(line[1:5])]
        #         # usd = 0.0
        #         #long = False
        #         in_position = False
        #         stoploss_triggers += 1
        #         last_trade_time = iteration
        #         reversal_flag = True
        #         if plot:
        #            #global short_exit_x
        #            #global short_exit_y
        #             short_exit_x.append(line[0])
        #             short_exit_y.append(line[1] + .1)
        #             return_graph_x.append(line[0])
        #             return_graph_y.append(returns)
        #             local_return_graph_x.append(line[0])
        #             local_return_graph_y.append(returns)
        #         continue
        #     if long:  # stoploss part 1
        #         usd = (min_exec_price / position[1]) * position[0]  # assume bad execution
        #         position = 0
        #         # position = [usd * .9997, min(line[1:5])]
        #         # usd = 0.0
        #         #long = True
        #         in_position = False
        #         stoploss_triggers += 1
        #         last_trade_time = iteration
        #         reversal_flag = True
        #         if plot:
        #             #global long_exit_x
        #             #global long_exit_y
        #             long_exit_x.append(line[0])
        #             long_exit_y.append(line[1] - .1)
        #             return_graph_x.append(line[0])
        #             return_graph_y.append(returns)
        #             local_return_graph_x.append(line[0])
        #             local_return_graph_y.append(returns)
        #         continue
        # upper_bound = ema2[-1] * (1+(variables[2]*(variables[3]-(min(time_since_last_trade, variables[4])/(variables[4]/(variables[3]-1))))))
        # lower_bound = ema2[-1] * (1-(variables[2]*(variables[3]-(min(time_since_last_trade, variables[4])/(variables[4]/(variables[3]-1))))))

        # if desire_position and in_position: # section 2
        #     if long:  # section where we release control back to buy_sell
        #         if line[1] < lower_bound:
        #             reversal_flag = False
        #
        #     else:
        #         if line[1] > upper_bound:
        #             reversal_flag = False
        #
        #     if long:
        #         if buy_sell > 0.0:
        #             reversal_flag = False
        #
        #     else:
        #         if buy_sell < 0.0:
        #             reversal_flag = False
        # in_position = position_live.in_position
        # if in_position:
        #     position_live.check_margin_call(line[4])
        #     in_position = position_live.in_position
        #     if not in_position:
        #         desire_position = False
        if buy_sell > 0 and not long and initialized:  # main buy/sell todo modify this
            usd = (position[1] / max_exec_price) * position[0]
            position = 0
            position = [usd * .9997, max_exec_price]
            usd = 0.0
            long = True
            # in_position = True
            trades += 1
            last_trade_time = iteration
            # if line[1] > upper_bound:
            #     reversal_flag = True
            #     stoploss_triggers += 1
            # else:
            #     reversal_flag = False
            if plot:
                #global long_enter_x
                #global long_enter_y
                long_enter_x.append(line[0])
                long_enter_y.append(line[4])
                return_graph_x.append(line[0])
                return_graph_y.append(returns)
                local_return_graph_x.append(line[0])
                local_return_graph_y.append(returns)
            continue

        if buy_sell < 0 and long and initialized:  #  main buy/sell 2
            #                      on 20x
            usd = (min_exec_price /
                   position[1]) * position[0]  # assume bad execution
            position = 0
            position = [usd * .9997, min_exec_price]
            usd = 0.0
            long = False
            # in_position = True
            trades += 1
            last_trade_time = iteration
            # if line[1] < lower_bound:
            #     reversal_flag = True
            #     stoploss_triggers += 1
            # else:
            #     reversal_flag = False
            if plot:
                #global short_enter_x
                #global short_enter_y
                short_enter_x.append(line[0])
                short_enter_y.append(line[4])
                return_graph_x.append(line[0])
                return_graph_y.append(returns)
                local_return_graph_x.append(line[0])
                local_return_graph_y.append(returns)
            continue

        # if not desire_position and not in_position:  # section 3
        #     if not long and ((buy_sell > 0.0) ^ (line[1] > upper_bound)): # moves understanding of long/short desire despite not being in position
        #         #usd = (position[1] / max(line[1:5])) * position[0]
        #         #position = 0
        #         #position = [usd * .9997, max(line[1:5])]
        #         #usd = 0.0
        #         long = True
        #         #in_position = True
        #         #trades += 1
        #         last_trade_time = iteration
        #         if line[1] > ema2[-1] * (1 + variables[2]):
        #             reversal_flag = True
        #         else:
        #             reversal_flag = False
        #         # if plot:
        #         #    #global long_enter_x
        #         #    #global long_enter_y
        #         #     long_enter_x.append(line[0])
        #         #     long_enter_y.append(line[1]-.1)
        #         #     return_graph_x.append(line[0])
        #         #     return_graph_y.append(returns)
        #         continue
        #
        #     if long and ((buy_sell < 0.0) ^ (line[1] < lower_bound)):  # moves understanding of long/short desire despite not being in position
        #         #usd = (min(line[1:5]) / position[1]) * position[0]
        #         #position = 0
        #         # position = [usd * .9997, min(line[1:5])]
        #         # usd = 0.0
        #         long = False
        #         # in_position = True
        #         # trades += 1
        #         last_trade_time = iteration
        #         if line[1] < lower_bound:
        #             reversal_flag = True
        #         else:
        #             reversal_flag = False
        #         # if plot:
        #         #    #global short_enter_x
        #         #    #global short_enter_y
        #         #     short_enter_x.append(line[0])
        #         #     short_enter_y.append(line[1]+.1)
        #         #     return_graph_x.append(line[0])
        #         #     return_graph_y.append(returns)
        #         continue

        # if desire_position and not in_position:  # if adx reverses direction, reenter position section 4
        #     if not long:  #
        #         #usd = (position[1] / max(line[1:5])) * position[0]
        #         #position = 0
        #         position = [usd * .9997, min_exec_price]
        #         usd = 0.0
        #         #long = True
        #         in_position = True
        #         trades += 1
        #         last_trade_time = iteration
        #         reversal_flag = False
        #         if plot:
        #            #global short_enter_x
        #            #global short_enter_y
        #             short_enter_x.append(line[0])
        #             short_enter_y.append(line[1]+.1)
        #             return_graph_x.append(line[0])
        #             return_graph_y.append(returns)
        #             local_return_graph_x.append(line[0])
        #             local_return_graph_y.append(returns)
        #         continue
        #
        #     if long:
        #         #usd = (min(line[1:5]) / position[1]) * position[0]  # assume bad execution
        #         #position = 0
        #         position = [usd * .9997, min_exec_price]
        #         usd = 0.0
        #         #long = False
        #         in_position = True
        #         trades += 1
        #         last_trade_time = iteration
        #         reversal_flag = False
        #         if plot:
        #            #global long_enter_x
        #            #global long_enter_y
        #             long_enter_x.append(line[0])
        #             long_enter_y.append(line[1] - .1)
        #             return_graph_x.append(line[0])
        #             return_graph_y.append(returns)
        #             local_return_graph_x.append(line[0])
        #             local_return_graph_y.append(returns)
        #         continue

        if returns < 50 and not plot:  # fail-quick
            returns = 0
            position = [0, line[1]]
            max_draw_down = -.9271
            break
    # ema1_switches = 0
    # for index, item in enumerate(ema1):
    #     if (item > ema1[index-1]) is not (ema1[index-1] > ema1[index-2]):
    #         ema1_switches += 1
    # ema2_switches = 0
    # for index, item in enumerate(ema2):
    #     if (item > ema2[index - 1]) is not (ema2[index - 1] > ema2[index - 2]):
    #         ema2_switches += 1
    # macd_switches = 0
    # for index, item in enumerate(buyselllist):
    #     if (item > 0) is not (buyselllist[index-1] > 0):
    #         macd_switches += 1
    # buyselllist = []
    # longest_time_since_last_high = 0
    # for index, tick in enumerate(local_return_graph_y):  # TODO TEMPORARY
    #     if index == 0:
    #         buyselllist.append(0)
    #         continue
    #     #print((tick - local_return_graph_y[index - 1]) / tick)
    #     buyselllist.append((tick - local_return_graph_y[index - 1]) / tick)
    # print(sorted(buyselllist))
    if plot:
        for leverage in range(1, 101):
            max_value = 0
            last_value = 0
            time_of_last_high = local_return_graph_x[0]
            time_since_last_high = 0

            for tickindex, tick in enumerate(local_return_graph_y):
                if tickindex == 0:
                    max_value = tick
                    last_value = tick
                    continue
                delta = (tick - local_return_graph_y[tickindex - 1]) / tick
                delta *= leverage
                last_value *= delta + 1
                if last_value > max_value:
                    time_since_last_high = local_return_graph_x[
                        tickindex] - time_of_last_high
                    time_of_last_high = local_return_graph_x[tickindex]
                    # if time_since_last_high > longest_time_since_last_high:
                    #     longest_time_since_last_high = time_since_last_high
                    max_value = last_value
                if last_value < max_value / float(2):
                    break
            if last_value < max_value / float(
                    2
            ):  # when broken out of the inner for by too much leverage
                max_leverage = leverage - 1
                break
            elif leverage == 100:
                print("got max leverage 100 somehow")
                max_leverage = leverage
                break
        best_leverage = [0, 0]
        best_value = 0
        for short_leverage in range(0, 101):
            for long_leverage in range(0, 101):
                max_value = 0
                last_value = 0

                for tickindex, tick in enumerate(local_return_graph_y):
                    if tickindex == 0:
                        max_value = tick
                        last_value = tick
                        continue
                    delta = (tick - local_return_graph_y[tickindex - 1]) / tick
                    if (init_trade == "short") ^ tickindex % 2 == 0:
                        delta *= short_leverage
                        short_fee = 1  #- .0000025 * short_leverage
                        last_value *= delta + short_fee

                    else:
                        delta *= long_leverage
                        long_fee = 1  #- .0000025 * long_leverage
                        last_value *= delta + long_fee

                    if last_value > max_value:
                        # if time_since_last_high > longest_time_since_last_high:
                        #     longest_time_since_last_high = time_since_last_high
                        max_value = last_value
                    if last_value < max_value / float(4):
                        last_value = 0
                        break
                if last_value > best_value:
                    best_leverage = [long_leverage, short_leverage]
                    best_value = last_value
        print("best long/short leverage", best_leverage, best_value)
    if try_leverages:
        long_leverage = try_leverages[0]
        short_leverage = try_leverages[1]
        max_value = 0
        last_value = 0

        for tickindex, tick in enumerate(local_return_graph_y):
            if tickindex == 0:
                max_value = tick
                last_value = tick
                continue
            delta = (tick - local_return_graph_y[tickindex - 1]) / tick
            if (init_trade == "short") ^ tickindex % 2 == 0:
                delta *= short_leverage
                short_fee = 1  # - .0000025 * short_leverage
                last_value *= delta + short_fee

            else:
                delta *= long_leverage
                long_fee = 1  # - .0000025 * long_leverage
                last_value *= delta + long_fee

            if last_value > max_value:
                # if time_since_last_high > longest_time_since_last_high:
                #     longest_time_since_last_high = time_since_last_high
                max_value = last_value
        print("for try_leverage", try_leverages, last_value)

    # max_leverage = min(math.floor(-1/min(mdd_relative, -.0001)), max_leverage)

    # print(max(adx), "avg:", np.average(adx), min(adx))
    if plot:
        print("MDD%", max_draw_down * 100, "dd_rel%:", mdd_relative * 100,
              "max leverage:", max_leverage)
    else:
        print("MDD%", max_draw_down * 100, "dd_rel%:", mdd_relative * 100)
    #print(max(atr), min(atr))
    #print("min, max, stddev, avg of stddev", min(stddev), max(stddev), np.std(stddev), np.average(stddev))

    if plot:
        # global plot_minusdi
        # global plot_plusdi
        if train:
            # plt.plot(data[int(-len(ema1)-(size/10)):int(-(size/10)), 0], ema1)
            plt.plot(data[int(-len(ema2) - (size / 10)):int(-(size / 10)), 0],
                     ema2)
            ##global plot_adx

            # plot_adx += adx
            # plot_minusdi += minusdi
            # plot_plusdi += plusdi
        else:
            # plt.plot(data[-len(ema1):, 0], ema1)
            plt.plot(data[-len(ema2):, 0], ema2)
            # global plot_adx

            # plot_adx += adx
            # plot_minusdi += minusdi
            # plot_plusdi += plusdi
    if long:
        returns = usd + (min_exec_price / position[1]) * position[0]
        if plot:
            return_graph_x.append(line[0])
            return_graph_y.append(returns)
            local_return_graph_x.append(line[0])
            local_return_graph_y.append(returns)
        print(list(variables), returns, trades)
        if plot:
            print("Score metric:", returns * (1 + max_draw_down))
        else:
            print("Score metric:", returns * (1 + max_draw_down))
    else:
        returns = usd + (position[1] / max_exec_price) * position[0]
        if plot:
            return_graph_x.append(line[0])
            return_graph_y.append(returns)
            local_return_graph_x.append(line[0])
            local_return_graph_y.append(returns)
            if trades < 10 and not plot and train: returns = 0
        print(list(variables), returns, trades)
        if plot:
            print("Score metric:", returns * (1 + max_draw_down))
        else:
            print("Score metric:", returns * (1 + max_draw_down))

    return -(returns * (1 + max_draw_down))
Пример #5
0
def dataproc_worker(start, end, live, ns, data=None):
    while True:
        # print(start,end, "gonna call the DB and get:")
        start = start.replace(microsecond=0)
        end = end.replace(microsecond=0)
        flip = False
        # if not live:
        #     if randrange(0, 10) > 4: flip = True
        #     else: flip = False
        if data is None:
            globalvar.dbcur.execute(
                "SELECT * FROM futures WHERE time BETWEEN (?) AND (?)",
                [(start - datetime(1970, 1, 1)) / timedelta(seconds=1),
                 (end - datetime(1970, 1, 1)) / timedelta(seconds=1)])
            data = globalvar.dbcur.fetchall()

        # benchstart = datetime.utcnow()
        data = [list(elem) for elem in data]
        # print("#1:", data[0], data[-1])
        # for index, line in enumerate(data):
        #     # data[index][0] = datetime.strptime(line[0], "%Y-%m-%dT%H:%M:%SZ") #problem 2
        #     date = line[0]  # string of '2016-07-07T10:36:42Z' using explicit slicing:
        #     data[index][0] = datetime(int(date[:4]), int(date[5:7]), int(date[8:10]), int(date[11:13]),
        #                               int(date[14:16]), int(date[17:19]))
        #

        averageNumerator = 0.0
        # print("#2:",data[0],data[-1])
        datalength = len(data)
        if not live:
            finaltime = len(data) - 120
        # print(data[-1][0], "as finaltime", data[-1][1], "as lastlast") # consistently accurate
        for index, line in enumerate(data):
            if live:
                averageNumerator += line[4]
            elif index < finaltime:
                averageNumerator += line[4]
        # if (data[-1][0] == data[-120][0]): break
        # here is where we do live adjustments remove 6k seconds at start pad 6k 0's at end
        # print("#3", data[0], data[-1], len(data))
        averagePrice = averageNumerator / (len(data) - 120)
        if live:
            del data[0:121]

        for index, line in enumerate(
                data):  # normalize to the 120 mins average price
            data[index][1] = line[1] / averagePrice  # o
            data[index][2] = line[2] / averagePrice  # h
            data[index][3] = line[3] / averagePrice  # l
            data[index][4] = line[4] / averagePrice  # c
        timeperiods = [
            10, 100, 1000, 10000, 100000
        ]  # (len(data) - 6000)] is too long, 100k max in C talib
        # initlist = [data[0][0]] * 86400
        # # print (timeperiods)
        if not live:
            open = np.array([row[1] for row in data[0:-120]], dtype=np.double)
            high = np.array([row[2] for row in data[0:-120]], dtype=np.double)
            low = np.array([row[3] for row in data[0:-120]], dtype=np.double)
            close = np.array([row[4] for row in data[0:-120]], dtype=np.double)
            volume = np.array([row[5] for row in data[0:-120]],
                              dtype=np.double)
        else:
            open = np.array([row[1] for row in data[0:-1]], dtype=np.double)
            high = np.array([row[2] for row in data[0:-1]], dtype=np.double)
            low = np.array([row[3] for row in data[0:-1]], dtype=np.double)
            close = np.array([row[4] for row in data[0:-1]], dtype=np.double)
            volume = np.array([row[5] for row in data[0:-1]], dtype=np.double)
            # print("internal last:", close[-1])
        if flip:
            open = (open * -1) + 2
            high = (high * -1) + 2
            low = (low * -1) + 2
            close = (close * -1) + 2
            data = np.array([row[4] for row in data[0:-1]], dtype=np.double)
            data = (data * -1) + 2
        # let's do the non-timeperiod indicators first
        # print("problem four took " + str(datetime.utcnow() - benchstart))
        # print("starting TA")
        trange = ta.TRANGE(high, low, close)
        obv = ta.OBV(close, volume)
        ad = ta.AD(high, low, close, volume)
        ht_trendline = ta.HT_TRENDLINE(close)
        ht_dcperiod = ta.HT_DCPERIOD(close)
        ht_dcphase = ta.HT_DCPHASE(close)
        ht_phasor_inphase, ht_phasor_quadrature = ta.HT_PHASOR(close)
        ht_sine, ht_leadsine = ta.HT_SINE(close)
        ht_trendmode = ta.HT_TRENDMODE(close)
        mama, fama = ta.MAMA(close, fastlimit=.5, slowlimit=.05)
        sar = ta.SAR(high, low)  # could crash, no extended params supplied
        sarext = ta.SAREXT(high, low)  # this one too
        bop = ta.BOP(open, high, low, close)
        # print("basic TA took " + str(datetime.utcnow() - benchstart))
        # print("starting timeperiod TA")

        # setting up timeperiod-requiring indicators
        slowk, slowd, fastk, fastd, rsik, rsid, t3, adx, adxr, apo, aroondown, aroonup, aroonosc, cci, cmo, dx, macd, \
        macdsignal, macdhist, macdext, macdsignalext, macdhistext, macdfix, macdhistfix, macdsignalfix, mfi, minus_dm, \
        minus_di, mom, plus_di, plus_dm, ppo, roc, rocr, rsi, trix, ultosc, willr, beta, correl, linearreg, \
        linearreg_angle, linearreg_intercept, linearreg_slope, stddev, tsf, var, atr, natr, adosc, upperband, \
        middleband, lowerband, dema, ema, kama, ma, midpoint, midprice, sma, tema, trima, wma = ([] for i in range(
            63))  # counted 63 twice
        for timeperiod in timeperiods:
            benchstart = datetime.utcnow()
            fastperiod = int(timeperiod / 2)
            slowperiod = timeperiod
            stochk = int(timeperiod / 3)
            stochd = int(timeperiod / 4)

            # n =  int(len(close) / slicefactor)
            # offset = (len(close)-1) - (n * slicefactor)
            # opens= open[offset::slicefactor]
            # high= high[offset::slicefactor]
            # low= low[offset::slicefactor]
            # close= close[offset::slicefactor]
            # volumes= volume[offset::slicefactor]

            # now for the weird indicators matype=1 for life, ema
            stoch1_temp, stoch2_temp = ta.STOCH(high,
                                                low,
                                                close,
                                                fastk_period=stochk,
                                                slowk_period=stochd,
                                                slowk_matype=1,
                                                slowd_period=stochd,
                                                slowd_matype=1)
            slowk.append(stoch1_temp)
            slowd.append(stoch2_temp)
            stoch1_temp, stoch2_temp = ta.STOCHF(high,
                                                 low,
                                                 close,
                                                 fastk_period=stochk,
                                                 fastd_period=stochd,
                                                 fastd_matype=1)
            fastk.append(stoch1_temp)
            fastd.append(stoch2_temp)
            stoch1_temp, stoch2_temp = ta.STOCHRSI(close,
                                                   timeperiod=timeperiod,
                                                   fastk_period=stochk,
                                                   fastd_period=stochd,
                                                   fastd_matype=1)
            rsik.append(stoch1_temp)
            rsid.append(stoch2_temp)
            t3.append(ta.T3(close, timeperiod=timeperiod, vfactor=.7))
            # now for the normal indicators
            # momentum
            adx.append(ta.ADX(high, low, close, timeperiod=timeperiod))
            adxr.append(ta.ADXR(high, low, close, timeperiod=timeperiod))
            apo.append(
                ta.APO(close, fastperiod=fastperiod, slowperiod=slowperiod))
            aroondown_temp, aroonup_temp = ta.AROON(high,
                                                    low,
                                                    timeperiod=timeperiod)
            aroondown.append(aroondown_temp)
            aroonup.append(aroonup_temp)
            aroonosc.append(ta.AROONOSC(high, low, timeperiod=timeperiod))
            cci.append(ta.CCI(high, low, close, timeperiod=timeperiod))
            cmo.append(ta.CMO(close, timeperiod=timeperiod))
            dx.append(ta.DX(high, low, close, timeperiod=timeperiod))
            macd_temp, macdsignal_temp, macdhist_temp = ta.MACD(
                close,
                fastperiod=fastperiod,
                slowperiod=slowperiod,
                signalperiod=stochk)
            macd.append(macd_temp)
            macdsignal.append(macdsignal_temp)
            macdhist.append(macdhist_temp)
            macd_temp, macdsignal_temp, macdhist_temp = ta.MACDEXT(
                close,
                fastperiod=fastperiod,
                fastmatype=1,
                slowperiod=slowperiod,
                slowmatype=1,
                signalperiod=stochk,
                signalmatype=1)
            macdext.append(macd_temp)
            macdsignalext.append(macdsignal_temp)
            macdhistext.append(macdhist_temp)
            macd_temp, macdsignal_temp, macdhist_temp = ta.MACDFIX(
                close, signalperiod=stochk)
            macdfix.append(macd_temp)
            macdsignalfix.append(macdsignal_temp)
            macdhistfix.append(macdhist_temp)
            mfi.append(ta.MFI(high, low, close, volume, timeperiod=timeperiod))
            minus_di.append(
                ta.MINUS_DI(high, low, close, timeperiod=timeperiod))
            minus_dm.append(ta.MINUS_DM(high, low, timeperiod=timeperiod))
            mom.append(ta.MOM(close, timeperiod=timeperiod))
            plus_di.append(ta.PLUS_DI(high, low, close, timeperiod=timeperiod))
            plus_dm.append(ta.PLUS_DM(high, low, timeperiod=timeperiod))
            ppo.append(
                ta.PPO(close,
                       fastperiod=fastperiod,
                       slowperiod=slowperiod,
                       matype=1))
            roc.append(ta.ROC(close, timeperiod=timeperiod))
            rocr.append(ta.ROCR(close, timeperiod=timeperiod))
            rsi.append(ta.RSI(close, timeperiod=timeperiod))
            trix.append(ta.TRIX(close, timeperiod=timeperiod))
            ultosc.append(
                ta.ULTOSC(high,
                          low,
                          close,
                          timeperiod1=stochk,
                          timeperiod2=fastperiod,
                          timeperiod3=timeperiod))
            willr.append(ta.WILLR(high, low, close, timeperiod=timeperiod))
            # end momentum, begin statistics
            beta.append(ta.BETA(high, low, timeperiod=timeperiod))
            correl.append(ta.CORREL(high, low, timeperiod=timeperiod))
            linearreg.append(ta.LINEARREG(close, timeperiod=timeperiod))
            linearreg_angle.append(
                ta.LINEARREG_ANGLE(close, timeperiod=timeperiod))
            linearreg_intercept.append(
                ta.LINEARREG_INTERCEPT(close, timeperiod=timeperiod))
            linearreg_slope.append(
                ta.LINEARREG_SLOPE(close, timeperiod=timeperiod))
            stddev.append(ta.STDDEV(close, timeperiod=timeperiod, nbdev=1))
            tsf.append(ta.TSF(close, timeperiod=timeperiod))
            var.append(ta.VAR(close, timeperiod=timeperiod, nbdev=1))
            # end stats, begin vol
            atr.append(ta.ATR(high, low, close, timeperiod=timeperiod))
            natr.append(ta.NATR(high, low, close, timeperiod=timeperiod))
            adosc.append(
                ta.ADOSC(high,
                         low,
                         close,
                         volume,
                         fastperiod=fastperiod,
                         slowperiod=slowperiod))
            # end vol, begin overlaps
            upperband_temp, middleband_temp, lowerband_temp = ta.BBANDS(
                close, timeperiod=timeperiod, matype=1)
            upperband.append(upperband_temp)
            middleband.append(middleband_temp)
            lowerband.append(lowerband_temp)
            dema.append(ta.DEMA(close, timeperiod=timeperiod))
            ema.append(ta.EMA(close, timeperiod=timeperiod))
            kama.append(ta.KAMA(close, timeperiod=timeperiod))
            ma.append(ta.MA(close, timeperiod=timeperiod, matype=0))
            midpoint.append(ta.MIDPOINT(close, timeperiod=timeperiod))
            midprice.append(ta.MIDPRICE(high, low, timeperiod=timeperiod))
            sma.append(ta.SMA(close, timeperiod=timeperiod))
            tema.append(ta.TEMA(close, timeperiod=timeperiod))
            trima.append(ta.TRIMA(close, timeperiod=timeperiod))
            wma.append(ta.WMA(close, timeperiod=timeperiod))
            # print("finishing timeperiod" + str(timeperiod))
            # print("TA for " + str(timeperiod) + " took " + str(datetime.utcnow() - benchstart))
        # 1 min, 10min, 100m, currentprice, indicators
        if live:
            finishedline = (close[-1])
        else:
            if not flip:
                finishedline = (  # date, flip, 1, 10, 20, 40, 80, 120, current price
                    (end - datetime(1970, 1, 1)) / timedelta(seconds=1), 0,
                    data[-120][4], data[-111][4], data[-101][4], data[-81][4],
                    data[-41][4], data[-1][4], data[-121][4])
            if flip:
                finishedline = (  # date, flip, 1, 10, 20, 40, 80, 120, current price
                    (end - datetime(1970, 1, 1)) / timedelta(seconds=1), 1,
                    data[-120], data[-111], data[-101], data[-81], data[-41],
                    data[-1], data[-121])

        finishedline += (trange, obv, ad, ht_trendline, ht_dcperiod,
                         ht_dcphase, ht_phasor_inphase, ht_phasor_quadrature,
                         ht_sine, ht_leadsine, ht_trendmode, mama, fama, sar,
                         sarext, bop, slowk, slowd, fastk, fastd, rsik, rsid,
                         t3, adx, adxr, apo, aroondown, aroonup, aroonosc, cci,
                         cmo, dx, macd, macdsignal, macdhist, macdext,
                         macdsignalext, macdhistext, macdfix, macdsignalfix,
                         macdhistfix, mfi, minus_di, minus_dm, mom, plus_di,
                         plus_dm, ppo, roc, rocr, rsi, trix, ultosc, willr,
                         beta, correl, linearreg, linearreg_angle,
                         linearreg_intercept, linearreg_slope, stddev, tsf,
                         var, atr, natr, adosc, upperband, middleband,
                         lowerband, dema, ema, kama, ma, midpoint, midprice,
                         sma, tema, trima, wma)
        finishedline = flatten(finishedline)

        if live:
            finishedline = np.array(finishedline, dtype=np.double)
            finishedline = finishedline[~np.isnan(finishedline)]  # remove nans
        else:
            finishedline = np.array(finishedline, dtype=np.double)
            finishedline = finishedline[~np.isnan(finishedline)]
        # print(pid + " has finished")
        if len(finishedline) != 326: break
        return finishedline
        break