def pong(self, timestamp, pid, watchdog_service, msg):
if msg:
Terminal.inst().action("Thread %s is alive %s" % (self.name, msg), view='content')
if watchdog_service:
watchdog_service.service_pong(pid, timestamp, msg)
def process(self, timeframe, timestamp):
# process only when signals of base timeframe
if timeframe != self.base_timeframe:
return
# update data at tick level
if timeframe == self.base_timeframe:
self.gen_candles_from_ticks(timestamp)
accept, compute = self.filter_market(timestamp)
if not accept:
return
# and compute
entries = []
exits = []
if compute:
# we might receive only LONG signals
entries, exits = self.compute(timeframe, timestamp)
#
# global indicators
#
ref_price = self.timeframes[self.ref_timeframe].price.last
# sma200 = self.timeframes[self.ref_timeframe].sma200.last
ref_sma55 = self.timeframes[self.ref_timeframe].sma55.last
ref_sma = self.timeframes[self.ref_timeframe].sma.last
ref_ema = self.timeframes[self.ref_timeframe].ema.last
#
# filters the entries
#
retained_entries = []
# filters entry signal, according to some correlation, parent timeframe signal or trend or trade regions
for entry in entries:
# only allowed range of signal for entry
if not (self.min_traded_timeframe <= entry.timeframe <=
self.max_traded_timeframe):
continue
# trade region
if not self.check_regions(timestamp, self.instrument.market_bid,
self.instrument.market_ofr, entry,
self.region_allow):
continue
# ref timeframe is bear don't take the risk (always long entry)
if not self.timeframes[self.sltp_timeframe].can_long:
continue
atr_stop = self.timeframes[self.sltp_timeframe].atr.stop_loss(
entry.dir)
if atr_stop < self.instrument.open_exec_price(entry.dir):
entry.sl = atr_stop
# and a target
take_profit = self.timeframes[
self.ref_timeframe].pivotpoint.last_resistances[2]
min_take_profit = self.timeframes[
self.ref_timeframe].pivotpoint.last_resistances[0]
# minimal R:R
gain = (take_profit - entry.p) / entry.p
loss = (entry.p - entry.sl) / entry.p
if loss != 0 and (gain / loss < 0.85): # 0.75 1.0
Terminal.inst().message(
"Risk:reward too weak p=%s sl=%s tp=%s rr=%s" %
(entry.p, entry.sl, take_profit, (gain / loss)),
view="debug")
continue
# not enought potential profit (minimal %)
if gain < 0.01:
continue
entry.tp = take_profit
entry.set('partial-take-profit', 1.0)
# max loss in %
if loss < 0.035:
entry.sl = entry.price * (1 - 0.035)
# or not do the trade, to risky
# continue
retained_entries.append(entry)
# # TP 50% entry
# entry_50pc = StrategySignal(0, 0)
# entry_50pc.dup(entry)
# entry_50pc.tp = self.timeframes[self.sltp_timeframe].pivotpoint.last_resistances[0]
# entry_50pc.set('partial-take-profit', 0.25)
# retained_entries.append(entry_50pc)
#
# process exits signals
#
if self.trades:
self.lock()
for trade in self.trades:
retained_exit = None
# important if we dont want to update user controlled trades if it have some operations
user_mgmt = trade.is_user_trade() and trade.has_operations()
# important, do not update user controlled trades if it have some operations
if trade.is_user_trade() and trade.has_operations():
continue
for signal in exits:
# @todo how to managed exit region ?
# receive an exit signal of the timeframe of the trade
if signal.timeframe == trade.timeframe:
retained_exit = signal
break
# exit signal on reference timeframe
if signal.timeframe == self.ref_timeframe:
retained_exit = signal
break
# exit from any parent timeframe signal
# if signal.timeframe > trade.timeframe:
# retained_exit = signal
# break
# can cancel a non filled trade if exit signal occurs before timeout (timeframe)
# if trade.is_entry_timeout(timestamp, trade.timeframe):
# trader = self.strategy.trader()
# trade.cancel_open(trader)
# Terminal.inst().info("Canceled order (exit signal or entry timeout) %s" % (self.instrument.market_id,), view='default')
# continue
if user_mgmt:
retained_exit = None
if trade.is_opened() and not trade.is_valid(
timestamp, trade.timeframe):
# @todo re-adjust entry
Terminal.inst().info("Update order %s trade %s TODO" % (
trade.id,
self.instrument.market_id,
),
view='default')
continue
# only for active and currently not closing trades
if not trade.is_active() or trade.is_closing(
) or trade.is_closed():
continue
close_exec_price = self.instrument.close_exec_price(trade.dir)
#
# stop-loss update
#
# always need a target, even if user trade and a stop order
update_tp = not trade.tp or not trade.has_limit_order()
update_sl = not trade.sl or not trade.has_stop_order()
# current sl/tp
stop_loss = trade.sl
take_profit = trade.tp
# ATR stop-loss (always long)
atr_stop = self.timeframes[self.sltp_timeframe].atr.stop_loss(
trade.direction)
if atr_stop > stop_loss:
stop_loss = atr_stop
if self.timeframes[
self.ref_timeframe].pivotpoint.last_pivot > 0.0:
if close_exec_price > self.timeframes[
self.ref_timeframe].pivotpoint.last_resistances[2]:
if utils.crossover(
self.timeframes[
self.ref_timeframe].price.prices,
self.timeframes[self.ref_timeframe].pivotpoint.
resistances[2]):
update_tp = True
if stop_loss < self.timeframes[
self.
ref_timeframe].pivotpoint.last_resistances[1]:
update_sl = True
stop_loss = self.timeframes[
self.
ref_timeframe].pivotpoint.last_resistances[1]
elif close_exec_price > self.timeframes[
self.ref_timeframe].pivotpoint.last_resistances[1]:
if utils.crossover(
self.timeframes[
self.ref_timeframe].price.prices,
self.timeframes[self.ref_timeframe].pivotpoint.
resistances[1]):
update_tp = True
if stop_loss < self.timeframes[
self.
ref_timeframe].pivotpoint.last_resistances[0]:
update_sl = True
stop_loss = self.timeframes[
self.
ref_timeframe].pivotpoint.last_resistances[0]
elif close_exec_price > self.timeframes[
self.ref_timeframe].pivotpoint.last_resistances[0]:
if utils.crossover(
self.timeframes[
self.ref_timeframe].price.prices,
self.timeframes[self.ref_timeframe].pivotpoint.
resistances[0]):
update_tp = True
if stop_loss < self.timeframes[
self.ref_timeframe].pivotpoint.last_pivot:
update_sl = True
stop_loss = self.timeframes[
self.ref_timeframe].pivotpoint.last_pivot
elif close_exec_price > self.timeframes[
self.ref_timeframe].pivotpoint.last_pivot:
if utils.crossover(
self.timeframes[
self.ref_timeframe].price.prices,
self.timeframes[
self.ref_timeframe].pivotpoint.pivot):
update_tp = True
if stop_loss < self.timeframes[
self.
ref_timeframe].pivotpoint.last_supports[0]:
update_sl = True
# stop_loss = self.timeframes[self.ref_timeframe].pivotpoint.last_supports[0]
elif close_exec_price > self.timeframes[
self.ref_timeframe].pivotpoint.last_supports[0]:
if utils.crossover(
self.timeframes[
self.ref_timeframe].price.prices,
self.timeframes[self.ref_timeframe].pivotpoint.
supports[0]):
update_tp = True
if trade.sl < self.timeframes[
self.
ref_timeframe].pivotpoint.last_supports[1]:
update_sl = True
# stop_loss = self.timeframes[self.ref_timeframe].pivotpoint.last_supports[1]
elif close_exec_price > self.timeframes[
self.ref_timeframe].pivotpoint.last_supports[1]:
if utils.crossover(
self.timeframes[
self.ref_timeframe].price.prices,
self.timeframes[self.ref_timeframe].pivotpoint.
supports[1]):
update_tp = True
if trade.sl < self.timeframes[
self.
ref_timeframe].pivotpoint.last_supports[2]:
update_sl = True
# stop_loss = self.timeframes[self.ref_timeframe].pivotpoint.last_supports[2]
elif close_exec_price > self.timeframes[
self.ref_timeframe].pivotpoint.last_supports[2]:
if utils.crossover(
self.timeframes[
self.ref_timeframe].price.prices,
self.timeframes[self.ref_timeframe].pivotpoint.
supports[2]):
update_tp = True
if close_exec_price < self.timeframes[
self.
ref_timeframe].pivotpoint.last_supports[2]:
update_sl = True
# stop_loss = self.timeframes[self.ref_timeframe].pivotpoint.last_supports[2]
#
# target update
#
take_profit = self.timeframes[
self.ref_timeframe].pivotpoint.last_resistances[int(
2 * trade.get('partial-take-profit', 0))]
# enought potential profit (0.5% min target) (always long)
# if (take_profit - close_exec_price) / close_exec_price < 0.005 and update_tp:
# update_tp = False
# reevaluate the R:R
# gain = (take_profit - trade.entry_price) / trade.entry_price
# loss = (trade.entry_price - trade.sl) / trade.entry_price
# if loss != 0 and (gain / loss < 0.5): # 0.75
# # Terminal.inst().message("%s %s %s %s" % (trade.entry_price, trade.sl, take_profit, (gain/loss)), view="debug")
# # @todo force exit
# continue
# @todo utiliser OCO, et sinon on peu aussi prioriser un ordre SL si le trade est en perte, et plutot un limit si en profit
if update_sl and stop_loss > 0:
stop_loss = self.instrument.adjust_price(stop_loss)
if trade.sl != stop_loss:
# logger.info("SL %s %s %s" % (update_sl, stop_loss, trade.sl))
delta_time = timestamp - trade.last_stop_loss[0]
num_orders = trade.last_stop_loss[1]
# too many stop-loss modifications in the timeframe
if 0: #not trade.has_stop_order() or delta_time > 60.0: #not ((self.sltp_max_rate > num_orders) and (delta_time < self.sltp_max_timeframe)):
try:
# OCO order or only bot managed stop-loss, only a TP limit is defined
trade.modify_stop_loss(self.strategy.trader(),
self.instrument,
stop_loss)
except Exception as e:
logger.error(repr(e))
Terminal.inst().info("%s modify SL" % timestamp,
view="debug")
else:
trade.sl = stop_loss
Terminal.inst().info("%s modify SL" % timestamp,
view="debug")
if update_tp and take_profit > 0:
take_profit = self.instrument.adjust_price(take_profit)
if trade.tp != take_profit:
# logger.info("TP %s %s %s" % (update_tp, take_profit, trade.tp))
delta_time = timestamp - trade.last_take_profit[0]
num_orders = trade.last_take_profit[1]
# too many stop-loss modifications in the timeframe
if not trade.has_limit_order(
) or delta_time > 60.0: #not ((self.sltp_max_rate > num_orders) and (delta_time < self.sltp_max_timeframe)):
try:
trade.modify_take_profit(
self.strategy.trader(), self.instrument,
take_profit)
except Exception as e:
logger.error(repr(e))
# @todo
Terminal.inst().info("%s modify TP" % timestamp,
view="debug")
else:
trade.tp = take_profit
#
# exit trade if an exit signal retained
#
if retained_exit:
self.process_exit(timestamp, trade, retained_exit.price)
Terminal.inst().info("Exit trade %s %s" %
(self.instrument.symbol, trade.id),
view='debug')
self.unlock()
# update actives trades
self.update_trades(timestamp)
# retained long entry do the order entry signal
for entry in retained_entries:
self.process_entry(timestamp, entry.price, entry.tp, entry.sl,
entry.timeframe,
entry.get('partial-take-profit', 0))
# streaming
self.stream()
def post_run(self):
Terminal.inst().info("Joining watcher %s..." % self._name)
self.disconnect()
def execute(self, args):
if not args:
Terminal.inst().action("Missing parameters", view='status')
return False
# ie: ":assign altbtc BTCUSDT EP@8500 SL@8300 TP@9600 0.521"
appliance = None
market_id = None
# direction base on command name
direction = 1
entry_price = None
stop_loss = 0.0
take_profit = 0.0
quantity = 0.0
timeframe = Instrument.TF_4HOUR
if len(args) < 4:
Terminal.inst().action("Missing parameters", view='status')
return False
try:
appliance, market_id = args[0], args[1]
for value in args[2:]:
if not value:
continue
if value.startswith("EP@"):
entry_price = float(value[3:])
elif value.startswith("SL@"):
stop_loss = float(value[3:])
elif value.startswith("TP@"):
take_profit = float(value[3:])
elif value.startswith("'"):
timeframe = timeframe_from_str(value[1:])
else:
quantity = float(value)
except Exception:
Terminal.inst().action("Invalid parameters", view='status')
return False
if entry_price <= 0.0:
Terminal.inst().action("Entry price must be specified",
view='status')
return False
if stop_loss and stop_loss > entry_price:
Terminal.inst().action("Stop-loss must be lesser than entry price",
view='status')
return False
if take_profit and take_profit < entry_price:
Terminal.inst().action(
"Take-profit must be greater than entry price", view='status')
return False
if quantity <= 0.0:
Terminal.inst().action("Quantity must be specified", view='status')
return False
self._strategy_service.command(
Strategy.COMMAND_TRADE_ASSIGN, {
'appliance': appliance,
'market-id': market_id,
'direction': direction,
'entry-price': entry_price,
'quantity': quantity,
'stop-loss': stop_loss,
'take-profit': take_profit,
'timeframe': timeframe
})
return True
def fetch_and_generate(self,
market_id,
timeframe,
from_date=None,
to_date=None,
n_last=1000,
fetch_option="",
cascaded=None):
if timeframe > 0 and timeframe not in self.GENERATED_TF:
logger.error("Timeframe %i is not allowed !" % (timeframe, ))
return
generators = []
from_tf = timeframe
self._last_ticks = []
self._last_ohlcs = {}
if not from_date and n_last:
# compute a from date
today = datetime.now().astimezone(UTC())
if timeframe >= Instrument.TF_MONTH:
from_date = (
today -
timedelta(months=int(timeframe / Instrument.TF_MONTH) *
n_last)).replace(day=1).replace(hour=0).replace(
minute=0).replace(second=0)
elif timeframe >= Instrument.TF_1D:
from_date = (today - timedelta(
days=int(timeframe / Instrument.TF_1D) * n_last)).replace(
hour=0).replace(minute=0).replace(second=0)
elif timeframe >= Instrument.TF_1H:
from_date = (today - timedelta(
hours=int(timeframe / Instrument.TF_1H) * n_last)).replace(
minute=0).replace(second=0)
elif timeframe >= Instrument.TF_1M:
from_date = (
today -
timedelta(minutes=int(timeframe / Instrument.TF_1M) *
n_last)).replace(second=0)
elif timeframe >= Instrument.TF_1S:
from_date = (today - timedelta(
seconds=int(timeframe / Instrument.TF_1S) * n_last))
from_date = from_date.replace(microsecond=0)
# print(from_date)
if not to_date:
today = datetime.now().astimezone(UTC())
if timeframe == Instrument.TF_MONTH:
to_date = today + timedelta(months=1)
else:
to_date = today + timedelta(seconds=timeframe)
to_date = to_date.replace(microsecond=0)
# cascaded generation of candles
if cascaded:
for tf in Fetcher.GENERATED_TF:
if tf > timeframe:
# from timeframe greater than initial
if tf <= cascaded:
# until max cascaded timeframe
generators.append(CandleGenerator(from_tf, tf))
from_tf = tf
# store for generation
self._last_ohlcs[tf] = []
else:
from_tf = tf
if timeframe > 0:
self._last_ohlcs[timeframe] = []
n = 0
t = 0
if timeframe == 0:
for data in self.fetch_trades(market_id, from_date, to_date, None):
# store (int timestamp in ms, str bid, str ofr, str volume)
Database.inst().store_market_trade(
(self.name, market_id, data[0], data[1], data[2], data[3]))
if generators:
self._last_ticks.append(
(float(data[0]) * 0.001, float(data[1]),
float(data[2]), float(data[3])))
# generate higher candles
for generator in generators:
if generator.from_tf == 0:
candles = generator.generate_from_ticks(
self._last_ticks)
if candles:
for c in candles:
self.store_candle(market_id, generator.to_tf,
c)
self._last_ohlcs[generator.to_tf] += candles
# remove consumed ticks
self._last_ticks = []
else:
candles = generator.generate_from_candles(
self._last_ohlcs[generator.from_tf])
if candles:
for c in candles:
self.store_candle(market_id, generator.to_tf,
c)
self._last_ohlcs[generator.to_tf] += candles
# remove consumed candles
self._last_ohlcs[generator.from_tf] = []
n += 1
t += 1
if n == 1000:
n = 0
Terminal.inst().info("%i..." % t)
Terminal.inst().flush()
# calm down the storage of tick, if parsing is faster
while Database.inst().num_pending_ticks_storage(
) > Fetcher.TICK_STORAGE_DELAY:
time.sleep(Fetcher.TICK_STORAGE_DELAY
) # wait a little before continue
logger.info("Fetched %i trades" % t)
elif timeframe > 0:
for data in self.fetch_candles(market_id, timeframe, from_date,
to_date, None):
# store (int timestamp ms, str open bid, high bid, low bid, close bid, open ofr, high ofr, low ofr, close ofr, volume)
Database.inst().store_market_ohlc(
(self.name, market_id, data[0], int(timeframe), data[1],
data[2], data[3], data[4], data[5], data[6], data[7],
data[8], data[9]))
if generators:
candle = Candle(float(data[0]) * 0.001, timeframe)
candle.set_bid_ohlc(float(data[1]), float(data[2]),
float(data[3]), float(data[4]))
candle.set_ofr_ohlc(float(data[5]), float(data[6]),
float(data[7]), float(data[8]))
candle.set_volume(float(data[9]))
candle.set_consolidated(True)
self._last_ohlcs[timeframe].append(candle)
# generate higher candles
for generator in generators:
candles = generator.generate_from_candles(
self._last_ohlcs[generator.from_tf])
if candles:
for c in candles:
self.store_candle(market_id, generator.to_tf, c)
self._last_ohlcs[generator.to_tf].extend(candles)
# remove consumed candles
self._last_ohlcs[generator.from_tf] = []
n += 1
t += 1
if n == 1000:
n = 0
Terminal.inst().info("%i..." % t)
logger.info("Fetched %i candles" % t)
def do_rebuilder(options):
Terminal.inst().info("Starting SIIS rebuilder using %s identity..." % options['identity'])
Terminal.inst().flush()
# database manager
Database.create(options)
Database.inst().setup(options)
timeframe = -1
cascaded = None
if not options.get('timeframe'):
timeframe = 60 # default to 1min
else:
if options['timeframe'] in TIMEFRAME_FROM_STR_MAP:
timeframe = TIMEFRAME_FROM_STR_MAP[options['timeframe']]
else:
try:
timeframe = int(options['timeframe'])
except:
pass
if not options.get('cascaded'):
cascaded = None
else:
if options['cascaded'] in TIMEFRAME_FROM_STR_MAP:
cascaded = TIMEFRAME_FROM_STR_MAP[options['cascaded']]
else:
try:
cascaded = int(options['cascaded'])
except:
pass
if timeframe < 0:
logger.error("Invalid timeframe")
sys.exit(-1)
from_date = options.get('from')
to_date = options.get('to')
if not to_date:
today = datetime.now().astimezone(UTC())
if timeframe == Instrument.TF_MONTH:
to_date = today + timedelta(months=1)
else:
to_date = today + timedelta(seconds=timeframe)
to_date = to_date.replace(microsecond=0)
if timeframe > 0 and timeframe not in GENERATED_TF:
logger.error("Timeframe %i is not allowed !" % (timeframe,))
return
for market in options['market'].split(','):
if market.startswith('!') or market.startswith('*'):
continue
timestamp = from_date.timestamp()
to_timestamp = to_date.timestamp()
progression = 0.0
prev_update = timestamp
count = 0
total_count = 0
progression_incr = (to_timestamp - timestamp) * 0.01
tts = 0.0
prev_tts = 0.0
generators = []
from_tf = timeframe
last_ticks = []
last_ohlcs = {}
if timeframe == Instrument.TF_TICK:
tick_streamer = Database.inst().create_tick_streamer(options['broker'], market, from_date=from_date, to_date=to_date)
else:
ohlc_streamer = Database.inst().create_ohlc_streamer(options['broker'], market, timeframe, from_date=from_date, to_date=to_date)
# cascaded generation of candles
if cascaded:
for tf in GENERATED_TF:
if tf > timeframe:
# from timeframe greater than initial
if tf <= cascaded:
# until max cascaded timeframe
generators.append(CandleGenerator(from_tf, tf))
from_tf = tf
# store for generation
last_ohlcs[tf] = []
else:
from_tf = tf
if timeframe > 0:
last_ohlcs[timeframe] = []
if timeframe == 0:
while not tick_streamer.finished():
ticks = tick_streamer.next(timestamp + Instrument.TF_1M)
count = len(ticks)
total_count += len(ticks)
for data in ticks:
if data[0] > to_timestamp:
break
if generators:
last_ticks.append(data)
# generate higher candles
for generator in generators:
if generator.from_tf == 0:
candles = generator.generate_from_ticks(last_ticks)
if candles:
for c in candles:
store_ohlc(options['broker'], market, generator.to_tf, c)
last_ohlcs[generator.to_tf] += candles
# remove consumed ticks
last_ticks = []
else:
candles = generator.generate_from_candles(last_ohlcs[generator.from_tf])
if candles:
for c in candles:
store_ohlc(options['broker'], market, generator.to_tf, c)
last_ohlcs[generator.to_tf] += candles
# remove consumed candles
last_ohlcs[generator.from_tf] = []
if timestamp - prev_update >= progression_incr:
progression += 1
Terminal.inst().info("%i%% on %s, %s ticks/trades for 1 minute, current total of %s..." % (progression, format_datetime(timestamp), count, total_count))
prev_update = timestamp
count = 0
if timestamp > to_timestamp:
break
timestamp += Instrument.TF_1M # by step of 1m
# calm down the storage of tick, if parsing is faster
while Database.inst().num_pending_ticks_storage() > TICK_STORAGE_DELAY:
time.sleep(TICK_STORAGE_DELAY) # wait a little before continue
elif timeframe > 0:
while not ohlc_streamer.finished():
ohlcs = ohlc_streamer.next(timestamp + timeframe * 100) # per 100
count = len(ohlcs)
total_count += len(ohlcs)
for data in ohlcs:
if data.timestamp > to_timestamp:
break
if generators:
last_ohlcs[timeframe].append(candle)
# generate higher candles
for generator in generators:
candles = generator.generate_from_candles(last_ohlcs[generator.from_tf])
if candles:
for c in candles:
store_ohlc(options['broker'], market, generator.to_tf, c)
last_ohlcs[generator.to_tf].extend(candles)
# remove consumed candles
last_ohlcs[generator.from_tf] = []
prev_tts = tts
timestamp = tts
if timestamp - prev_update >= progression_incr:
progression += 1
Terminal.inst().info("%i%% on %s, %s ticks/trades for 1 minute, current total of %s..." % (progression, format_datetime(timestamp), count, total_count))
prev_update = timestamp
count = 0
if timestamp > to_timestamp:
break
if total_count == 0:
timestamp += timeframe * 100
if progression < 100:
Terminal.inst().info("100%% on %s, %s ticks/trades for 1 minute, current total of %s..." % (format_datetime(timestamp), count, total_count))
Terminal.inst().info("Flushing database...")
Terminal.inst().flush()
Database.terminate()
Terminal.inst().info("Rebuild done!")
Terminal.inst().flush()
Terminal.terminate()
sys.exit(0)
def execute(self, args):
if not args:
Terminal.inst().action("Missing parameters", view='status')
return False
if args[0] == 'apps':
if len(args) == 1:
self._strategy_service.set_activity(True)
Terminal.inst().action(
"Activated any instruments for all appliances",
view='status')
return True
elif len(args) == 2:
appliance = self._strategy_service.appliance(args[1])
if appliance:
appliance.set_activity(True)
Terminal.inst().action(
"Activated any instrument for appliance %s" % args[1],
view='status')
return True
elif len(args) == 3:
appliance = self._strategy_service.appliance(args[1])
if appliance:
appliance.set_activity(True, args[2])
Terminal.inst().action(
"Activated instrument %s for appliance %s" %
(args[2], args[1]),
view='status')
return True
elif args[0] == 'notifiers':
if len(args) == 1:
self.notifier_service.set_activity(True)
Terminal.inst().action("Activated all notifiers",
view='status')
return True
elif len(args) == 2:
notifier = self._notifier_service.notifier(args[1])
if notifier:
notifier.set_activity(True)
Terminal.inst().action("Activated notifier %s" % args[1],
view='status')
return True
return False
def process_td9(self, timestamp, last_timestamp, candles, prices, volumes):
signal = None
# volume sma, increase signal strength when volume increase over its SMA
# volume_sma = utils.MM_n(self.depth-1, self.volume.volumes)
rsi = 0
rsi_30_70 = 0 # 1 <30, -1 >70
rsi_40_60 = 0 # 1 if RSI in 40-60
# rsi_trend = 0
stochrsi = 0
stochrsi_20_80 = 0 # 1 <20, -1 >80
stochrsi_40_60 = 0 # 1 if stochRSI in 40-60
volume_signal = 0
ema_sma_cross = 0
ema_sma_height = 0
if self.rsi:
self.rsi.compute(last_timestamp, prices)
if self.rsi.last < self.rsi_low:
rsi_30_70 = 1.0
elif rsi > self.rsi_high:
rsi_30_70 = -1.0
if self.rsi.last > 0.4 and self.rsi.last < 0.6:
rsi_40_60 = 1
if self.stochrsi:
self.stochrsi.compute(last_timestamp, prices)
stochrsi = self.stochrsi.last_k
if stochrsi < 0.2:
stochrsi_20_80 = 1.0
elif stochrsi > 0.8:
stochrsi_20_80 = -1.0
if stochrsi > 0.4 and stochrsi < 0.6:
stochrsi_40_60 = 1
# if self.volume.last > volume_sma[-1]:
# volume_signal = 1
# elif self.volume.last < volume_sma[-1]:
# volume_signal = -1
if self.sma and self.ema:
self.sma.compute(last_timestamp, prices)
self.ema.compute(last_timestamp, prices)
# ema over sma crossing
ema_sma_cross = utils.cross((self.ema.prev, self.sma.prev),
(self.ema.last, self.sma.last))
if self.ema.last > self.sma.last:
ema_sma_height = 1
elif self.ema.last < self.sma.last:
ema_sma_height = -1
if self.bollingerbands:
self.bollingerbands.compute(last_timestamp, prices)
bb_break = 0
bb_ma = 0
if prices[-1] > self.bollingerbands.last_top:
bb_break = 1
elif prices[-1] < self.bollingerbands.last_bottom:
bb_break = -1
if prices[-1] > self.bollingerbands.last_ma:
bb_ma = -1
elif prices[-1] > self.bollingerbands.last_ma:
bb_ma = 1
if self.atr:
self.atr.compute(last_timestamp, self.price.high, self.price.low,
self.price.close)
if self.tomdemark:
self.tomdemark.compute(last_timestamp, self.price.timestamp,
self.price.high, self.price.low,
self.price.close)
#
# setup entry
#
# long entry on sell-setup + rsi oversell
if (self.tomdemark.c.c == 2 and self.tomdemark.c.d < 0
and self.price.close[-1] > self.price.close[-2]) or (
self.tomdemark.c.c == 3): # avec td3
# # if (td.c == 1 and td.d < 0 and candles[-1].close > candles[-2].close):
# if (td.c == 2 and td.d < 0 and candles[-1].close > candles[-2].close):
# if ((td.c == 2 or td.c == 3) and td.d < 0 and candles[-1].close > candles[-2].close):
# if ((td.c == 2 or td.c == 3) and td.d < 0) and candles[-1].close > candles[-2].close:
# if bb_break == 0:
# if (bb_break == 0 and bb_ma < 0): # test with aggressive + bb
# if (stochrsi_20_80 > 0 or rsi_30_70 > 0):
# if (ema_sma_height > 0 or rsi_30_70 > 0) and bb_break >= 0:
# if (bb_break == 0 and bb_ma < 0) and (stochrsi_20_80 > 0 or rsi_30_70 > 0): # used with average case but not with aggressive
# if (bb_break == 0 and bb_ma < 0) and (ema_sma_height > 0 or rsi_30_70 > 0): # and volume_signal > 0:
# if (stochrsi_20_80 > 0 or rsi_30_70 > 0): # la classique mais prend des risk, a voir avec un bon SL peut-être
# if (rsi_trend > 0) and (ema_sma_height > 0 or rsi_30_70 > 0): # protege de perte mais rate des gains
# if (ema_sma_height > 0 or rsi_30_70 > 0): # and volume_signal > 0: # C4
# if ema_sma_height > 0: # C5
# if ema_sma_height > 0 and bb_break >= 0 and rsi_trend > 0: # C6
# if ema_sma_height > 0 and bb_break >= 0 and stochrsi_20_80 > 0:
# if 1: # C1
# if rsi_trend > 0 and bb_break > 0: # pas mal evite des entrees foireuses mais pas assez de profits
# if (stochrsi_20_80 > 0 or rsi_30_70 > 0): # la classique mais prend des risk, il faut un bon stop-loss
if (ema_sma_height > 0 or rsi_30_70 > 0) and bb_break <= 0:
# if rsi_trend and (stochrsi_20_80 > 0 or rsi_30_70 > 0):
# if (stochrsi_20_80 > 0 and rsi_30_70 > 0):
signal = StrategySignal(self.tf, timestamp)
signal.signal = StrategySignal.SIGNAL_ENTRY
signal.dir = 1
signal.p = self.price.close[-1]
if self.tomdemark.c.tdst:
signal.sl = self.tomdemark.c.tdst
# Terminal.inst().info("Entry %s %s" % (self.tomdemark.c.tdst, signal.sl), view='default')
# aggressive entry
if ((self.tomdemark.c.c == 9 or
(self.tomdemark.c.c == 8 and self.tomdemark.c.p))
and self.tomdemark.c.d > 0):
# if stochrsi_20_80 > 0 and rsi_30_70 > 0 and candles[-1].close > candles[-2].close: # C1 -
# if 1: # C4 +++
# if stochrsi_20_80 > 0 and candles[-1].close > candles[-2].close: # C3 ++
# if rsi_30_70 > 0 and candles[-1].close > candles[-2].close: # C5 ??
# if ema_sma_height > 0 and bb_break >= 0 and rsi_trend > 0: # C6 ??
# if ema_sma_height > 0 and bb_break >= 0 and stochrsi_20_80 > 0: # C7 ??
# if (ema_sma_height > 0 or rsi_30_70 > 0): # C2 +
# if stochrsi_20_80 > 0 and candles[-1].close > candles[-2].close: # C3 ++
if (stochrsi_20_80 > 0 or rsi_30_70 > 0): # C8 ??
signal = StrategySignal(self.tf, timestamp)
signal.signal = StrategySignal.SIGNAL_ENTRY
signal.dir = 1
signal.p = self.price.close[-1]
# Terminal.inst().info("Aggressive entry %s %s" % (self.tomdemark.c.tdst, signal.sl), view='default')
#
# invalidation 2 of opposite setup
#
# @todo or if a >= 3 <= 7 close below the previous candle close
# can make loss in bull run
# second condition might be optimized
elif (self.tomdemark.c.c == 2 and self.tomdemark.c.d > 0
and self.price.close[-1] < self.price.close[-2]) or (
self.tomdemark.c.c == 3 and self.tomdemark.c.d > 0):
# elif td.c == 3 and td.d > 0: # and candles[-1].close < candles[-2].close:
# long cancelation on buy-setup formation
# if ema_sma_height < 0 or rsi_40_60 > 0: # (excess of volume and ema under sma)
# if stochrsi_20_80 < 0: # and volume_signal > 0:
# if bb_break < 0:
# if (rsi_trend < 0): # C2
# if 1: # C1 +
# if (bb_break == 0 and bb_ma < 0): # C3 ++
if ema_sma_height < 0: # and volume_signal > 0:
# Terminal.inst().info("rsi_30_70 %s / rsi_40_60 %s / ema_sma_height %s / volume_signal %s" % (rsi_30_70, rsi_40_60, ema_sma_height, volume_signal), view='default')
signal = StrategySignal(self.tf, timestamp)
signal.signal = StrategySignal.SIGNAL_EXIT # CANCEL
signal.dir = 1
signal.p = self.price.close[-1]
# Terminal.inst().info("Canceled long entry %s c2-c3, p:%s tf:%s" % (self.strategy_trader.instrument.symbol, signal.p, self.tf), view='default')
#
# setup completed
#
elif (((self.tomdemark.c.c >= 8 and self.tomdemark.c.p) or
(self.tomdemark.c.c == 9)) and self.tomdemark.c.d <
0): # and candles[-1].close > candles[-2].close:
if 1: # stochrsi_20_80 > 1:
signal = StrategySignal(self.tf, timestamp)
signal.signal = StrategySignal.SIGNAL_EXIT
signal.dir = 1
signal.p = self.price.close[-1]
# Terminal.inst().info("Exit long %s %s c8p-c9 (%s%s)" % (self.strategy_trader.instrument.symbol, self.tf, self.tomdemark.c.c, 'p' if signal.p else ''), view='default')
#
# setup aborted
#
elif ((self.tomdemark.c.c >= 2 and self.tomdemark.c.c <= 7)
and self.tomdemark.c.d < 0
) and self.price.close[-1] < self.price.close[-2]: # C1
# if stochrsi_20_80 < 0 or rsi_30_70 < 0: # bearish stoch OR rsi # C1
if stochrsi_20_80 < 0 and rsi_30_70 < 0: # bearish stoch AND rsi (seems better) # C2
signal = StrategySignal(self.tf, timestamp)
signal.signal = StrategySignal.SIGNAL_EXIT
signal.dir = 1
signal.p = self.price.close[-1]
#Terminal.inst().info("Abort long %s %s c3-c7 (%s%s)" % (self.strategy_trader.instrument.symbol, self.tf, td.c, 'p' if signal.p else ''), view='default')
#
# CD entry
#
if self.tomdemark.cd.c > 1:
Terminal.inst().info("CD%s" % self.tomdemark.cd.c)
if self.tomdemark.cd.c == 1:
# count-down buy-setup + rsi oversell
Terminal.inst().info("CD1")
if self.tomdemark.c.d < 0: # and candles[-1].close > candles[-2].high:
# if rsi_30_70 > 0:
signal = StrategySignal(self.tf, timestamp)
signal.signal = StrategySignal.SIGNAL_ENTRY
signal.dir = 1
signal.p = self.price.last
signal.sl = self.tomdemark.c.tdst
Terminal.inst().info("Entry long %s %s cd13, sl:%s" % (
self.strategy_trader.instrument.symbol,
self.tf,
signal.sl,
),
view='default')
#
# CD13 setup
#
elif self.tomdemark.cd.c == 13:
# count-down sell-setup completed
if self.tomdemark.cd.d < 0:
signal = StrategySignal(self.tf, timestamp)
signal.signal = StrategySignal.SIGNAL_EXIT
signal.p = self.price.close[-1]
signal.dir = 1
Terminal.inst().info("Exit long cd13", view='default')
return signal
def check_ticks(broker_id, market_id, from_date, to_date):
last_ticks = []
tick_streamer = Database.inst().create_tick_streamer(broker_id,
market_id,
from_date=from_date,
to_date=to_date)
timestamp = from_date.timestamp()
to_timestamp = to_date.timestamp()
progression = 0.0
prev_update = timestamp
count = 0
total_count = 0
progression_incr = (to_timestamp - timestamp) * 0.01
tts = 0.0
prev_tts = 0.0
while not tick_streamer.finished():
# return any ticks until last time to 1 min more
ticks = tick_streamer.next(timestamp + Instrument.TF_1M)
count = len(ticks)
total_count += len(ticks)
for data in ticks:
tts = data[0]
bid = data[1]
ofr = data[2]
vol = data[3]
if not prev_tts:
prev_tts = tts
gap_duration = tts - prev_tts
if tts != prev_tts and gap_duration < 0.0:
date = format_datetime(timestamp)
Terminal.inst().error(
"Tick timestamp is before previous of %s on %s ! Broken file !"
% (format_delta(gap_duration), date))
if gap_duration > 60.0:
date = format_datetime(timestamp)
Terminal.inst().warning("Tick gap of %s on %s !" %
(format_delta(gap_duration), date))
if bid <= 0.0:
Terminal.inst().warning(
"Bid price is lesser than 0 %s on %s !" % (bid, date))
if ofr <= 0.0:
Terminal.inst().warning(
"Ofr price is lesser than 0 %s on %s !" % (ofr, date))
if vol < 0.0:
Terminal.inst().warning(
"Volume quantity is lesser than 0 %s on %s !" %
(vol, date))
prev_tts = tts
if tts > to_timestamp:
break
if tts - prev_update >= progression_incr:
progression += 1
Terminal.inst().info(
"%i%% on %s, %s ticks/trades for 1 minute, current total of %s..."
%
(progression, format_datetime(timestamp), count, total_count))
prev_update = tts
count = 0
if tts > to_timestamp:
break
timestamp += Instrument.TF_1M # by step of 1m
if progression < 100:
Terminal.inst().info(
"100%% on %s, %s ticks/trades for 1 minute, current total of %s..."
% (format_datetime(timestamp), count, total_count))
Terminal.inst().info("Last tick datetime is %s" % (format_datetime(tts), ))
def process(self, timeframe, timestamp):
# process only at base timeframe
if timeframe != self.base_timeframe:
return
# update data at tick level
if timeframe == self.base_timeframe:
self.gen_candles_from_ticks(timestamp)
accept, compute = self.filter_market(timestamp)
if not accept:
return
# and compute
entries = []
exits = []
if compute:
entries, exits = self.compute(timeframe, timestamp)
#
# global indicators
#
ref_price = self.timeframes[self.ref_timeframe].price.last
# sma200 = self.timeframes[self.ref_timeframe].sma200.last
ref_sma55 = self.timeframes[self.ref_timeframe].sma55.last
ref_sma = self.timeframes[self.ref_timeframe].sma.last
ref_ema = self.timeframes[self.ref_timeframe].ema.last
#
# compute the entry
#
retained_entries = []
for entry in entries:
# only allowed range of signal for entry
if not (self.min_traded_timeframe <= entry.timeframe <= self.max_traded_timeframe):
continue
# trade region
if not self.check_regions(timestamp, self.instrument.market_bid, self.instrument.market_ofr, entry, self.region_allow):
continue
# ref timeframe is contrary
if entry.direction > 0 and not self.timeframes[self.sltp_timeframe].can_long:
continue
if entry.direction < 0 and not self.timeframes[self.sltp_timeframe].can_short:
continue
# initial stop-loss
atr_stop = self.timeframes[self.sltp_timeframe].atr.stop_loss(entry.dir)
if entry.direction > 0:
# and an initial target
take_profit = self.timeframes[self.ref_timeframe].pivotpoint.last_resistances[2]
if atr_stop < self.instrument.open_exec_price(entry.dir):
entry.sl = atr_stop
gain = (take_profit - entry.p) / entry.p
loss = (entry.p - entry.sl) / entry.p
elif entry.direction < 0:
# and an initial target
take_profit = self.timeframes[self.ref_timeframe].pivotpoint.last_supports[2]
if atr_stop > self.instrument.open_exec_price(entry.dir):
entry.sl = atr_stop
gain = (entry.p - take_profit) / entry.p
loss = (entry.sl - entry.p) / entry.p
if loss != 0 and (gain / loss < 1.0):
Terminal.inst().message("%s %s %s %s %s %s" % (entry.p, entry.sl, take_profit, gain, loss, (gain/loss)), view="debug")
continue
# not enought potential profit
if gain < 0.005:
continue
entry.tp = take_profit if gain > 0.005 else entry.p * 1.01
entry.ptp = 1.0
# max loss at x%
if loss > 0.035:
if entry.direction > 0:
entry.sl = entry.price * (1-0.035)
elif entry.direction < 0:
entry.sl = entry.price * (1+0.035)
# or do not do the trade to risky
# continue
retained_entries.append(entry)
# TP 50% entry
# entry_50pc = StrategySignal(0, 0)
# entry_50pc.dup(entry)
# entry_50pc.tp = np.max(self.timeframes[self.sltp_timeframe].pivotpoint.resistances[0])#[-1]
# entry_50pc.ptp = 0.25
# retained_entries.append(entry_50pc)
#
# process eventually exits signals
#
if self.trades:
self.lock()
for trade in self.trades:
retained_exit = None
# important if we dont want to update user controlled trades if it have some operations
user_mgmt = trade.is_user_trade()
for signal in exits:
parent_signal_tf = self.parent_timeframe(signal.timeframe)
# @todo how to managed exit region ?
# receive an exit signal of the timeframe of the trade
if signal.timeframe == trade.timeframe:
retained_exit = signal
break
# exit signal on reference timeframe
if signal.timeframe == self.ref_timeframe:
retained_exit = signal
break
# exit from parent timeframe signal
# if parent_signal_tf == trade.timeframe:
# retained_exit = signal
# break
# exit from any parent timeframe signal
# if signal.timeframe > trade.timeframe:
# retained_exit = signal
# break
# can cancel a non filled trade if exit signal occurs before timeout (timeframe)
# if trade.is_entry_timeout(timestamp, trade.timeframe):
# trader = self.strategy.trader()
# trade.cancel_open(trader)
# Terminal.inst().info("Canceled order (exit signal or entry timeout) %s" % (self.instrument.market_id,), view='default')
# continue
if user_mgmt:
retained_exit = None
# if trade.is_opened() and not trade.is_valid(timestamp, trade.timeframe):
# # @todo re-adjust entry
# Terminal.inst().info("Update order %s trade %s TODO" % (trade.id, self.instrument.market_id,), view='default')
# continue
# only for active and currently not closing trades
if not trade.is_active() or trade.is_closing() or trade.is_closed():
continue
close_exec_price = self.instrument.close_exec_price(trade.dir)
#
# stop-loss update
#
# always need a target, even if user trade and a stop order
update_tp = not trade.tp or not trade.has_limit_order()
update_sl = not trade.sl or not trade.has_stop_order()
# current sl/tp
stop_loss = trade.sl
take_profit = trade.tp
# ATR stop-loss (long/short)
atr_stop = self.timeframes[self.sltp_timeframe].atr.stop_loss(trade.direction)
if trade.direction > 0:
# long, greater or initial
if atr_stop > stop_loss and atr_stop < close_exec_price * 0.995:
stop_loss = atr_stop
elif trade.direction < 0:
# short, lesser or initial
if (atr_stop < stop_loss or stop_loss <= 0) and atr_stop > close_exec_price * 1.005:
stop_loss = atr_stop
if self.timeframes[self.ref_timeframe].pivotpoint.last_pivot > 0.0:
if trade.direction > 0:
# long
if close_exec_price > self.timeframes[self.ref_timeframe].pivotpoint.last_resistances[2]:
if utils.crossover(self.timeframes[self.ref_timeframe].price.prices, self.timeframes[self.ref_timeframe].pivotpoint.resistances[2]):
update_tp = True
if stop_loss < self.timeframes[self.ref_timeframe].pivotpoint.last_resistances[1]:
update_sl = True
# stop_loss = self.timeframes[self.ref_timeframe].pivotpoint.last_resistances[1]
elif close_exec_price > self.timeframes[self.ref_timeframe].pivotpoint.last_resistances[1]:
if utils.crossover(self.timeframes[self.ref_timeframe].price.prices, self.timeframes[self.ref_timeframe].pivotpoint.resistances[1]):
update_tp = True
if stop_loss < self.timeframes[self.ref_timeframe].pivotpoint.last_resistances[0]:
update_sl = True
# stop_loss = self.timeframes[self.ref_timeframe].pivotpoint.last_resistances[0]
elif close_exec_price > self.timeframes[self.ref_timeframe].pivotpoint.last_resistances[0]:
if utils.crossover(self.timeframes[self.ref_timeframe].price.prices, self.timeframes[self.ref_timeframe].pivotpoint.resistances[0]):
update_tp = True
if stop_loss < self.timeframes[self.ref_timeframe].pivotpoint.last_pivot:
update_sl = True
# stop_loss = self.timeframes[self.ref_timeframe].pivotpoint.last_pivot
elif close_exec_price > self.timeframes[self.ref_timeframe].pivotpoint.last_pivot:
if utils.crossover(self.timeframes[self.ref_timeframe].price.prices, self.timeframes[self.ref_timeframe].pivotpoint.pivot):
update_tp = True
if stop_loss < self.timeframes[self.ref_timeframe].pivotpoint.last_supports[0]:
update_sl = True
# stop_loss = self.timeframes[self.ref_timeframe].pivotpoint.last_supports[0]
elif close_exec_price > self.timeframes[self.ref_timeframe].pivotpoint.last_supports[0]:
if utils.crossover(self.timeframes[self.ref_timeframe].price.prices, self.timeframes[self.ref_timeframe].pivotpoint.supports[0]):
update_tp = True
if trade.sl < self.timeframes[self.ref_timeframe].pivotpoint.last_supports[1]:
update_sl = True
# stop_loss = self.timeframes[self.ref_timeframe].pivotpoint.last_supports[1]
elif close_exec_price > self.timeframes[self.ref_timeframe].pivotpoint.last_supports[1]:
if utils.crossover(self.timeframes[self.ref_timeframe].price.prices, self.timeframes[self.ref_timeframe].pivotpoint.supports[1]):
update_tp = True
if trade.sl < self.timeframes[self.ref_timeframe].pivotpoint.last_supports[2]:
update_sl = True
# stop_loss = self.timeframes[self.ref_timeframe].pivotpoint.last_supports[2]
elif close_exec_price > self.timeframes[self.ref_timeframe].pivotpoint.last_supports[2]:
if utils.crossover(self.timeframes[self.ref_timeframe].price.prices, self.timeframes[self.ref_timeframe].pivotpoint.supports[2]):
update_tp = True
if close_exec_price < self.timeframes[self.ref_timeframe].pivotpoint.last_supports[2]:
update_sl = True
# stop_loss = self.timeframes[self.ref_timeframe].pivotpoint.last_supports[2]
elif trade.direction < 0:
# short (could use the sign, but if we want a non symmetrical approch...)
if close_exec_price < self.timeframes[self.ref_timeframe].pivotpoint.last_supports[2]:
if utils.crossover(self.timeframes[self.ref_timeframe].price.prices, self.timeframes[self.ref_timeframe].pivotpoint.supports[2]):
update_tp = True
if close_exec_price > self.timeframes[self.ref_timeframe].pivotpoint.last_supports[2]:
update_sl = True
# stop_loss = self.timeframes[self.ref_timeframe].pivotpoint.last_supports[2]
elif close_exec_price < self.timeframes[self.ref_timeframe].pivotpoint.last_supports[1]:
if utils.crossover(self.timeframes[self.ref_timeframe].price.prices, self.timeframes[self.ref_timeframe].pivotpoint.supports[1]):
update_tp = True
if trade.sl > self.timeframes[self.ref_timeframe].pivotpoint.last_supports[2]:
update_sl = True
# stop_loss = self.timeframes[self.ref_timeframe].pivotpoint.last_supports[2]
elif close_exec_price < self.timeframes[self.ref_timeframe].pivotpoint.last_supports[0]:
if utils.crossover(self.timeframes[self.ref_timeframe].price.prices, self.timeframes[self.ref_timeframe].pivotpoint.supports[0]):
update_tp = True
if trade.sl > self.timeframes[self.ref_timeframe].pivotpoint.last_supports[1]:
update_sl = True
# stop_loss = self.timeframes[self.ref_timeframe].pivotpoint.last_supports[1]
elif close_exec_price < self.timeframes[self.ref_timeframe].pivotpoint.last_pivot:
if utils.crossover(self.timeframes[self.ref_timeframe].price.prices, self.timeframes[self.ref_timeframe].pivotpoint.pivot):
update_tp = True
if stop_loss > self.timeframes[self.ref_timeframe].pivotpoint.last_supports[0]:
update_sl = True
# stop_loss = self.timeframes[self.ref_timeframe].pivotpoint.last_supports[0]
elif close_exec_price < self.timeframes[self.ref_timeframe].pivotpoint.last_resistances[0]:
if utils.crossover(self.timeframes[self.ref_timeframe].price.prices, self.timeframes[self.ref_timeframe].pivotpoint.resistances[0]):
update_tp = True
if stop_loss > self.timeframes[self.ref_timeframe].pivotpoint.last_pivot:
update_sl = True
# stop_loss = self.timeframes[self.ref_timeframe].pivotpoint.last_pivot
elif close_exec_price < self.timeframes[self.ref_timeframe].pivotpoint.last_resistances[1]:
if utils.crossover(self.timeframes[self.ref_timeframe].price.prices, self.timeframes[self.ref_timeframe].pivotpoint.resistances[1]):
update_tp = True
if stop_loss > self.timeframes[self.ref_timeframe].pivotpoint.last_resistances[0]:
update_sl = True
# stop_loss = self.timeframes[self.ref_timeframe].pivotpoint.last_resistances[0]
elif close_exec_price < self.timeframes[self.ref_timeframe].pivotpoint.last_resistances[2]:
if utils.crossunder(self.timeframes[self.ref_timeframe].price.prices, self.timeframes[self.ref_timeframe].pivotpoint.resistances[2]):
update_tp = True
if stop_loss > self.timeframes[self.ref_timeframe].pivotpoint.last_resistances[1]:
update_sl = True
# stop_loss = self.timeframes[self.ref_timeframe].pivotpoint.last_resistances[1]
#
# target update
#
# enought potential profit (0.5% min target)
if trade.direction > 0:
take_profit = self.timeframes[self.ref_timeframe].pivotpoint.last_resistances[int(2*trade.partial_tp)]
# if take_profit <= trade.entry_price:
# take_profit = trade.entry_price * 1.05
gain = (take_profit - trade.entry_price) / trade.entry_price
loss = (trade.entry_price - trade.sl) / trade.entry_price
elif trade.direction < 0:
take_profit = self.timeframes[self.ref_timeframe].pivotpoint.last_supports[int(2*trade.partial_tp)]
# if take_profit >= trade.entry_price:
# take_profit = trade.entry_price * 0.95
gain = (trade.entry_price - take_profit) / trade.entry_price
loss = (trade.sl - trade.entry_price) / trade.entry_price
# reevaluate the R:R
# @todo
# if gain < 0.005 and update_tp:
# ...
if update_sl and stop_loss > 0:
stop_loss = self.instrument.adjust_price(stop_loss)
if trade.sl != stop_loss:
# logger.info("SL %s %s %s" % (update_sl, stop_loss, trade.sl))
delta_time = timestamp - trade.last_stop_loss[0]
num_orders = trade.last_stop_loss[1]
# too many stop-loss modifications in the timeframe
if not trade.has_stop_order() or delta_time > 60.0: #not ((self.sltp_max_rate > num_orders) and (delta_time < self.sltp_max_timeframe)):
try:
trade.modify_stop_loss(self.strategy.trader(), self.instrument.market_id, stop_loss)
except Exception as e:
logger.error(repr(e))
Terminal.inst().info("%s modify SL" % timestamp, view="debug")
else:
trade.sl = stop_loss
if update_tp and take_profit > 0:
take_profit = self.instrument.adjust_price(take_profit)
if trade.tp != take_profit:
logger.info("TP %s %s %s" % (update_tp, take_profit, trade.tp))
delta_time = timestamp - trade.last_take_profit[0]
num_orders = trade.last_take_profit[1]
# too many stop-loss modifications in the timeframe
if not trade.has_limit_order() or delta_time > 60.0: #not ((self.sltp_max_rate > num_orders) and (delta_time < self.sltp_max_timeframe)):
try:
trade.modify_take_profit(self.strategy.trader(), self.instrument.market_id, take_profit)
except Exception as e:
logger.error(repr(e))
Terminal.inst().info("%s modify TP" % timestamp, view="debug")
else:
trade.tp = take_profit
#
# exit trade if an exit signal retained
#
if retained_exit:
self.process_exit(timestamp, trade, retained_exit.price)
Terminal.inst().info("Exit trade %s %s" % (self.instrument.symbol, trade.id), view='debug')
self.unlock()
# update actives trades
self.update_trades(timestamp)
# retained long entry do the order entry signal
for entry in retained_entries:
self.process_entry(timestamp, entry.dir, entry.price, entry.tp, entry.sl, entry.timeframe, entry.partial_tp)
# streaming
self.stream()
def do_rebuilder(options):
Terminal.inst().info("Starting SIIS rebuilder using %s identity..." % options['identity'])
Terminal.inst().flush()
# database manager
Database.create(options)
Database.inst().setup(options)
timeframe = -1
cascaded = None
if not options.get('timeframe'):
timeframe = 60 # default to 1min
else:
if options['timeframe'] in TIMEFRAME_FROM_STR_MAP:
timeframe = TIMEFRAME_FROM_STR_MAP[options['timeframe']]
else:
try:
timeframe = int(options['timeframe'])
except:
pass
if not options.get('cascaded'):
cascaded = None
else:
if options['cascaded'] in TIMEFRAME_FROM_STR_MAP:
cascaded = TIMEFRAME_FROM_STR_MAP[options['cascaded']]
else:
try:
cascaded = int(options['cascaded'])
except:
pass
if timeframe < 0:
logger.error("Invalid timeframe")
sys.exit(-1)
from_date = options.get('from')
to_date = options.get('to')
if not to_date:
today = datetime.now().astimezone(UTC())
if timeframe == Instrument.TF_MONTH:
to_date = today + timedelta(months=1)
else:
to_date = today + timedelta(seconds=timeframe)
to_date = to_date.replace(microsecond=0)
timeframe = options['timeframe']
if timeframe > 0 and timeframe not in GENERATED_TF:
logger.error("Timeframe %i is not allowed !" % (timeframe,))
return
for market in options['market'].split(','):
if market.startswith('!') or market.startswith('*'):
continue
generators = []
from_tf = timeframe
last_ticks = []
last_ohlcs = {}
if timeframe == Instrument.TF_TICK:
tick_streamer = Database.inst().create_tick_streamer(options['broker'], market, from_date=from_date, to_date=to_date)
else:
ohlc_streamer = Database.inst().create_ohlc_streamer(options['broker'], market, timeframe, from_date=from_date, to_date=to_date)
# cascaded generation of candles
if cascaded:
for tf in GENERATED_TF:
if tf > timeframe:
# from timeframe greater than initial
if tf <= cascaded:
# until max cascaded timeframe
generators.append(CandleGenerator(from_tf, tf))
from_tf = tf
# store for generation
last_ohlcs[tf] = []
else:
from_tf = tf
if timeframe > 0:
last_ohlcs[timeframe] = []
n = 0
t = 0
timestamp = from_date.timestamp() + Instrument.TF_1M
if timeframe == 0:
while not tick_streamer.finished():
ticks = tick_streamer.next(timestamp)
timestamp += Instrument.TF_1M # by step of 1M
for data in ticks:
if generators:
last_ticks.append((float(data[0]) * 0.001, float(data[1]), float(data[2]), float(data[3])))
# generate higher candles
for generator in generators:
if generator.from_tf == 0:
candles = generator.generate_from_ticks(last_ticks)
if candles:
for c in candles:
store_ohlc(options['broker'], market, generator.to_tf, c)
last_ohlcs[generator.to_tf] += candles
# remove consumed ticks
last_ticks = []
else:
candles = generator.generate_from_candles(last_ohlcs[generator.from_tf])
if candles:
for c in candles:
store_ohlc(options['broker'], market, generator.to_tf, c)
last_ohlcs[generator.to_tf] += candles
# remove consumed candles
last_ohlcs[generator.from_tf] = []
n += 1
t += 1
if n == 1000:
n = 0
Terminal.inst().info("%i..." % t)
Terminal.inst().flush()
# calm down the storage of tick, if parsing is faster
while Database.inst().num_pending_ticks_storage() > TICK_STORAGE_DELAY:
time.sleep(TICK_STORAGE_DELAY) # wait a little before continue
logger.info("Read %i trades" % t)
elif timeframe > 0:
while not ohlc_streamer.finished():
ohlcs = ohlc_streamer.next(timestamp)
timestamp += Instrument.TF_1M # by step of 1M
for data in ohlcs:
if generators:
candle = Candle(float(data[0]) * 0.001, timeframe)
candle.set_bid_ohlc(float(data[1]), float(data[2]), float(data[3]), float(data[4]))
candle.set_ofr_ohlc(float(data[5]), float(data[6]), float(data[7]), float(data[8]))
candle.set_volume(float(data[9]))
candle.set_consolidated(True)
last_ohlcs[timeframe].append(candle)
# generate higher candles
for generator in generators:
candles = generator.generate_from_candles(last_ohlcs[generator.from_tf])
if candles:
for c in candles:
store_ohlc(options['broker'], market, generator.to_tf, c)
last_ohlcs[generator.to_tf].extend(candles)
# remove consumed candles
last_ohlcs[generator.from_tf] = []
n += 1
t += 1
if n == 1000:
n = 0
Terminal.inst().info("%i..." % t)
logger.info("Read %i candles" % t)
Terminal.inst().info("Flushing database...")
Terminal.inst().flush()
Database.terminate()
Terminal.inst().info("Rebuild done!")
Terminal.inst().flush()
Terminal.terminate()
sys.exit(0)
def display_cli_help():
Terminal.inst().message("")
Terminal.inst().message('%s command line usage:' % APP_LONG_NAME)
Terminal.inst().message("")
Terminal.inst().message("\tcmd <identity> <--options>")
Terminal.inst().message("")
Terminal.inst().message(
"\tProfile name must be defined in the identy.py file from .siis local data. With that way you can manage multiple account, having identity for demo."
)
Terminal.inst().message("\t --help display command line help.")
Terminal.inst().message("\t --version display the version number.")
Terminal.inst().message(
"\t --profile=<profile> Use a specific profile of appliance else default loads any."
)
Terminal.inst().message(
"\t --paper-mode instanciate paper mode trader and simulate as best as possible."
)
Terminal.inst().message(
"\t --backtest process a backtesting, uses paper mode traders and data history avalaible in the database."
)
Terminal.inst().message(
"\t --timestep=<seconds> Timestep in seconds to increment the backesting. More precise is more accurate but need more computing simulation. Adjust to at least fits to the minimal candles size uses in the backtested strategies. Default is 60 seconds."
)
Terminal.inst().message(
"\t --time-factor=<factor> in backtesting mode only allow the user to change the time factor and permit to interact during the backtesting. Default speed factor is as fast as possible."
)
Terminal.inst().message(
"\t --check-data @todo Process a test on candles data. Check if there is inconsitencies into the time of the candles and if there is some gaps. The test is done only on the defined range of time."
)
Terminal.inst().message(
"\t --from=<YYYY-MM-DDThh:mm:ss> define the date time from which start the backtesting, fetcher or binarizer. If ommited use whoole data set (take care)."
)
Terminal.inst().message(
"\t --to=<YYYY-MM-DDThh:mm:ss> define the date time to which stop the backtesting, fetcher or binarizer. If ommited use now."
)
Terminal.inst().message(
"\t --last=<number> Fast last number of candles for every watched market (take care can take all requests credits on the broker). By default it is configured to get 1m, 5m and 1h candles."
)
Terminal.inst().message(
"\t --market=<market-id> Specific market identifier to fetch, binarize only."
)
Terminal.inst().message(
"\t --broker=<broker-name> Specific fetcher or watcher name to fetche or binarize market from."
)
Terminal.inst().message(
"\t --timeframe=<timeframe> Time frame unit or 0 for trade level. For fetcher, higher candles are generated. Defined value is in second or an alias in 1m 5m 15m 1h 2h 4h d m w"
)
Terminal.inst().message(
"\t --cascaded=<max-timeframe> During fetch process generate the candles of highers timeframe from lowers. Default is no. Take care to have entire multiple to fullfill the generated candles."
)
Terminal.inst().message(
"\t --spec=<specific-option> Specific fetcher option (exemple STOCK for alphavantage.co fetcher to fetch a stock market)."
)
Terminal.inst().message(
"\t --watcher-only Only watch and save market/candles data into the database. No trade and neither paper mode trades are performed."
)
Terminal.inst().message(
"\t --read-only Don't write market neither candles data to the database. Default is writing to the database."
)
Terminal.inst().message(
"\t --tool=<tool-name> Execute a specific tool @todo.")
Terminal.inst().message("\t --fetch Process the data fetcher.")
Terminal.inst().message(
"\t --binarize Process to text file to binary conversion for a market."
)
Terminal.inst().message(
"\t --sync Process a synchronization of the watched market from a particular broker."
)
Terminal.inst().message("")
Terminal.inst().message(
"\t During usage press ':h<ENTER>' to get interative commands help. Press ':q<ENTER>' to exit. Knows issues can lock one ore more thread, then you will need to kill the process yourself."
)
Terminal.inst().message("")
def display_help(commands_handler, user_context=False):
if user_context:
# user context help
Terminal.inst().message("User contextuel commands:", view='content')
for entry in commands_handler.get_user_summary():
if entry[1]:
Terminal.inst().message(" - '%s' %s " % (entry[0], entry[1]),
view='content')
else:
# general help
Terminal.inst().message(
"General commands. Direct key actions (single key press), view key are in uppercase:",
view='content')
# @todo accelerator with Command
Terminal.inst().message(" - '?' ping all services", view='content')
Terminal.inst().message(" - <space> print a time mark in status bar",
view='content')
Terminal.inst().message(" - 'n' toggle desktop notifications",
view='content')
Terminal.inst().message(" - 'a' toggle audible notifications",
view='content')
Terminal.inst().message(" - 'e' toggle discord notifications",
view='content')
Terminal.inst().message(
" - 'p' list positions (will be replaced by a dedicated view)",
view='content')
Terminal.inst().message(
" - 'o' list orders (will be replaced by a dedicated view)",
view='content')
Terminal.inst().message(
" - 'g' print trader performance (will be replaced by a dedicated view or removed)",
view='content')
Terminal.inst().message(" - 'A' show account view", view='content')
Terminal.inst().message(" - 'Q' show assets view", view='content')
Terminal.inst().message(" - 'M' show markets view", view='content')
Terminal.inst().message(" - 'T' show tickers view", view='content')
Terminal.inst().message(" - 'F' show strategy view", view='content')
Terminal.inst().message(" - 'S' show statistic view", view='content')
Terminal.inst().message(" - 'P' show performance view", view='content')
Terminal.inst().message(" - 'I' show console view", view='content')
Terminal.inst().message(" - 'N' show notification/signal view",
view='content')
# Terminal.inst().message(" - 'U' list positions", view='content')
# Terminal.inst().message(" - 'O' list orders", view='content')
Terminal.inst().message(" - 'D' show debug view", view='content')
Terminal.inst().message(" - 'C' clear current view", view='content')
for entry in commands_handler.get_summary():
if entry[1]:
Terminal.inst().message(" - %s %s " % (entry[0], entry[1]),
view='content')
Terminal.inst().message("", view='content')
Terminal.inst().message(
"Advanced commands have to be completed by <ENTER> key else <ESC> to cancel. Command typing are avoided after fews seconds.",
view='content')
Terminal.inst().message(" - ':quit' or ':q' exit", view='content')
for entry in commands_handler.get_cli_summary():
if entry[2]:
if entry[1]:
Terminal.inst().message(" - ':%s' or ':%s' %s " %
(entry[0], entry[1], entry[2]),
view='content')
else:
Terminal.inst().message(" - ':%s' %s " %
(entry[0], entry[2]),
view='content')
def table_format(self):
return Terminal.inst().style(
), self._row[0], self.height() - 4, self._col
def execute(self, args):
if not args:
Terminal.inst().action("Missing parameters", view='status')
return False
appliance = None
market_id = None
trade_id = None
timeframe = -1
action = "add-region"
reg = "range"
stage = 0
direction = 0
expiry = 0.0
created = self._strategy_service.timestamp
low = 0.0
high = 0.0
cancelation = 0.0
# ie ":RR _ EURUSD 1.12 1.15"
if len(args) < 4:
Terminal.inst().action("Missing parameters", view='status')
return False
try:
appliance, market_id = args[0], args[1]
low = float(args[2])
high = float(args[3])
for value in args[4:]:
if value.startswith("'"):
timeframe = timeframe_from_str(value[1:])
elif value.startswith('C@'):
cancelation = float(value[2:])
elif value.startswith('@'):
# expiry
if 'T' in value:
# as local datetime
expiry = datetime.strptime(
value[1:], '%Y-%m-%dT%H:%M:%S').timestamp(
) # .replace(tzinfo=UTC())
else:
# relative to now
duration = timeframe_from_str(value[1:])
expiry = created + duration
elif value in ("l", "L", "long", "LONG"):
direction = 1
elif value in ("s", "S", "short", "SHORT"):
direction = -1
elif value in ("e", "E", "entry", "ENTRY"):
stage = 1
elif value in ("x", "X", "exit", "EXIT"):
stage = -1
except Exception:
Terminal.inst().action("Invalid parameters", view='status')
return False
self._strategy_service.command(
Strategy.COMMAND_TRADER_MODIFY, {
'appliance': appliance,
'market-id': market_id,
'trade-id': trade_id,
'action': action,
'region': reg,
'created': created,
'stage': stage,
'direction': direction,
'timeframe': timeframe,
'expiry': expiry,
'low': low,
'high': high,
'cancelation': cancelation
})
return True
def do_optimizer(options):
Terminal.inst().info("Starting SIIS optimizer...")
Terminal.inst().flush()
# database manager
Database.create(options)
Database.inst().setup(options)
broker_id = options['broker']
market_id = options['market']
timeframe = None
from_date = options.get('from')
to_date = options.get('to')
if not to_date:
today = datetime.now().astimezone(UTC())
if timeframe == Instrument.TF_MONTH:
to_date = today + timedelta(months=1)
else:
to_date = today + timedelta(seconds=timeframe)
to_date = to_date.replace(microsecond=0)
if not options.get('timeframe'):
timeframe = None
else:
if options['timeframe'] in TIMEFRAME_FROM_STR_MAP:
timeframe = TIMEFRAME_FROM_STR_MAP[options['timeframe']]
else:
try:
timeframe = int(options['timeframe'])
except:
pass
try:
# checking data integrity, gap...
if timeframe is None:
for market in options['market'].split(','):
if market.startswith('!') or market.startswith('*'):
continue
for tf in GENERATED_TF:
Terminal.inst().info("Verifying %s OHLC %s..." %
(market, timeframe_to_str(tf)))
check_ohlcs(options['broker'], market, tf, from_date,
to_date)
elif timeframe == Instrument.TF_TICK:
for market in options['market'].split(','):
if market.startswith('!') or market.startswith('*'):
continue
Terminal.inst().info("Verifying %s ticks/trades..." %
(market, ))
check_ticks(options['broker'], market, from_date, to_date)
elif timeframe > 0:
# particular ohlc
for market in options['market'].split(','):
if market.startswith('!') or market.startswith('*'):
continue
Terminal.inst().info("Verifying %s OHLC %s..." %
(market, timeframe_to_str(timeframe)))
check_ohlcs(options['broker'], market, timeframe, from_date,
to_date)
except KeyboardInterrupt:
pass
finally:
pass
Terminal.inst().info("Flushing database...")
Terminal.inst().flush()
Database.terminate()
Terminal.inst().info("Optimization done!")
Terminal.inst().flush()
Terminal.terminate()
sys.exit(0)
def do_importer(options):
tool = Importer()
Terminal.inst().info("Starting SIIS importer...")
Terminal.inst().flush()
# database manager
Database.create(options)
Database.inst().setup(options)
# want speedup the database inserts
Database.inst().enable_fetch_mode()
filename = options.get('filename')
detected_format = FORMAT_UNDEFINED
detected_timeframe = None
is_mtx_tick = False
pathname = pathlib.Path(filename)
if not pathname.exists():
error_exit(None, "File %s does not exists" % pathname.name)
timeframe = None
if not options.get('timeframe'):
timeframe = None
else:
if options['timeframe'] in TIMEFRAME_FROM_STR_MAP:
timeframe = TIMEFRAME_FROM_STR_MAP[options['timeframe']]
else:
try:
timeframe = int(options['timeframe'])
except:
pass
src = open(filename, "rt")
if filename.endswith(".siis"):
detected_format = FORMAT_SIIS
elif filename.endswith(".csv"):
# detect the format from the first row
row = src.readline().rstrip('\n')
if row.count('\t') > 0:
if row.count(
'\t'
) == 5 and row == "<DATE>\t<TIME>\t<BID>\t<ASK>\t<LAST>\t<VOLUME>":
detected_format = FORMAT_MT5
detected_timeframe = Instrument.TF_TICK
is_mtx_tick = True
elif row.count(
'\t'
) == 8 and row == "<DATE>\t<TIME>\t<OPEN>\t<HIGH>\t<LOW>\t<CLOSE>\t<TICKVOL>\t<VOL>\t<SPREAD>":
detected_format = FORMAT_MT5
is_mtx_tick = False
# from filename try to detect the timeframe
parts = pathname.name.split('_')
if len(parts) >= 2:
detected_timeframe = MT5_TIMEFRAMES.get(parts[1])
# ignore the header line
elif row.count(',') > 0:
if row.count(',') == 4:
detected_format = FORMAT_MT4
detected_timeframe = Instrument.TF_TICK
is_mtx_tick = True
elif row.count(',') == 6:
detected_format = FORMAT_MT4
is_mtx_tick = False
# from filename try to detect the timeframe
parts = pathname.name.split('.')
if len(parts) > 0:
for mt_tf, tf in MT4_TIMEFRAMES.items():
if parts[0].endswith(mt_tf):
detected_timeframe = tf
break
# reset because first row is data
src.seek(0, 0)
if detected_format == FORMAT_UNDEFINED:
error_exit(src, "Unknown file format")
if detected_format in (FORMAT_MT4, FORMAT_MT5):
if detected_timeframe is not None and timeframe is None:
Terminal.inst().message("Auto-detected timeframe %s" %
timeframe_to_str(detected_timeframe))
if detected_timeframe and timeframe and detected_timeframe != timeframe:
error_exit(
src,
"Auto-detected timeframe %s is different of specified timeframe %s"
% (timeframe_to_str(detected_timeframe),
timeframe_to_str(timeframe)))
market_id = ""
broker_id = ""
# UTC option dates
from_date = options.get('from')
to_date = options.get('to')
if detected_format == FORMAT_SIIS:
# first row gives format details
header = src.readline()
if not header.startswith("format=SIIS\t"):
error_exit(src, "Unsupported file format")
info = header.split('\t')
for nfo in info:
k, v = nfo.split('=')
if k == "version":
if v != "1.0.0":
error_exit(src, "Unsupported format version")
elif k == "created":
pass # informational only
elif k == "broker":
broker_id = v
elif k == "market":
market_id = v
elif k == "from":
pass # informational only
elif k == "to":
pass # informational only
elif k == "timeframe":
if v != "any":
timeframe = timeframe_from_str(v)
else:
timeframe = None
else:
# need broker, market and timeframe
broker_id = options.get('broker')
market_id = options.get('market')
if not broker_id:
error_exit(src, "Missing target broker identifier")
if not market_id or ',' in market_id:
error_exit(src, "Missing or invalid target market identifier")
if timeframe is None:
if is_mtx_tick:
timeframe = Instrument.TF_TICK
elif detected_timeframe:
timeframe = detected_timeframe
else:
error_exit(src, "Missing target timeframe")
# limited sub-range
from_date_str = from_date.strftime(
"%Y-%m-%dT%H:%M:%SZ") if from_date else None
to_date_str = to_date.strftime("%Y-%m-%dT%H:%M:%SZ") if to_date else None
total_count = 0
try:
if detected_format == FORMAT_SIIS:
cur_timeframe = None
cur_from_date = from_date
cur_to_date = to_date
while 1:
row = src.readline()
if not row:
break
row = row.rstrip("\n")
if row.startswith("timeframe="):
# specify the timeframe of the next rows
k, v = row.split('=')
cur_timeframe = timeframe_from_str(v)
continue
if cur_timeframe is None:
# need a specified timeframe
continue
if cur_timeframe == Instrument.TF_TICK:
total_count += import_tick_siis_1_0_0(
broker_id, market_id, cur_from_date, cur_to_date, row)
elif cur_timeframe > 0:
total_count += import_ohlc_siis_1_0_0(
broker_id, market_id, cur_timeframe, cur_from_date,
cur_to_date, row)
elif detected_format == FORMAT_MT4:
cur_timeframe = timeframe if not is_mtx_tick else Instrument.TF_TICK
cur_from_date = from_date
cur_to_date = to_date
if cur_timeframe == Instrument.TF_TICK:
while 1:
row = src.readline()
if not row:
break
row = row.rstrip("\n")
total_count += import_tick_mt4(tool, broker_id, market_id,
cur_from_date, cur_to_date,
row)
elif cur_timeframe > 0:
while 1:
row = src.readline()
if not row:
break
row = row.rstrip("\n")
total_count += import_ohlc_mt4(broker_id, market_id,
cur_timeframe,
cur_from_date, cur_to_date,
row)
elif detected_format == FORMAT_MT5:
cur_timeframe = timeframe if not is_mtx_tick else Instrument.TF_TICK
cur_from_date = from_date
cur_to_date = to_date
if cur_timeframe == Instrument.TF_TICK:
while 1:
row = src.readline()
if not row:
break
row = row.rstrip("\n")
total_count += import_tick_mt5(tool, broker_id, market_id,
cur_from_date, cur_to_date,
row)
elif cur_timeframe > 0:
while 1:
row = src.readline()
if not row:
break
row = row.rstrip("\n")
total_count += import_ohlc_mt5(broker_id, market_id,
cur_timeframe,
cur_from_date, cur_to_date,
row)
except Exception as e:
error_logger.error(str(e))
finally:
src.close()
src = None
Terminal.inst().info("Imported %s samples" % (total_count))
Terminal.inst().info("Flushing database...")
Terminal.inst().flush()
Database.terminate()
Terminal.inst().info("Importation done!")
Terminal.inst().flush()
Terminal.terminate()
sys.exit(0)
def check_ohlcs(broker_id, market_id, timeframe, from_date, to_date):
last_ohlcs = {}
ohlc_streamer = Database.inst().create_ohlc_streamer(broker_id,
market_id,
timeframe,
from_date=from_date,
to_date=to_date,
buffer_size=100)
timestamp = from_date.timestamp()
to_timestamp = to_date.timestamp()
progression = 0.0
prev_update = timestamp
count = 0
total_count = 0
progression_incr = (to_timestamp - timestamp) * 0.01
tts = 0.0
prev_tts = 0.0
while not ohlc_streamer.finished():
ohlcs = ohlc_streamer.next(timestamp + timeframe * 100) # per 100
count = len(ohlcs)
total_count += len(ohlcs)
for ohlc in ohlcs:
tts = ohlc.timestamp
if not prev_tts:
prev_tts = tts
gap_duration = tts - prev_tts - timeframe
if gap_duration != 0:
date = format_datetime(tts)
Terminal.inst().warning("Ohlc gap of %s on %s !" %
(format_delta(gap_duration), date))
if ohlc.bid_open <= 0.0:
Terminal.inst().warning(
"Bid open price is lesser than 0 %s on %s !" %
(ohlc.bid_open, date))
if ohlc.bid_high <= 0.0:
Terminal.inst().warning(
"Bid high price is lesser than 0 %s on %s !" %
(ohlc.bid_high, date))
if ohlc.bid_low <= 0.0:
Terminal.inst().warning(
"Bid close price is lesser than 0 %s on %s !" %
(ohlc.bid_low, date))
if ohlc.bid_close <= 0.0:
Terminal.inst().warning(
"Bid close price is lesser than 0 %s on %s !" %
(ohlc.bid_close, date))
if ohlc.ofr_open <= 0.0:
Terminal.inst().warning(
"Ofr open price is lesser than 0 %s on %s !" %
(ohlc.ofr_open, date))
if ohlc.ofr_high <= 0.0:
Terminal.inst().warning(
"Ofr high price is lesser than 0 %s on %s !" %
(ohlc.ofr_high, date))
if ohlc.ofr_low <= 0.0:
Terminal.inst().warning(
"Ofr low price is lesser than 0 %s on %s !" %
(ohlc.ofr_low, date))
if ohlc.ofr_close <= 0.0:
Terminal.inst().warning(
"Ofr close price is lesser than 0 %s on %s !" %
(ohlc.ofr_close, date))
if ohlc.volume < 0.0:
Terminal.inst().warning(
"Volume quantity is lesser than 0 %s on %s !" %
(ohlc.volume, date))
prev_tts = tts
timestamp = tts
if tts > to_timestamp:
break
if tts - prev_update >= progression_incr:
progression += 1
Terminal.inst().info(
"%i%% on %s, %s for last 100 candles, current total of %s..." %
(progression, format_datetime(timestamp), count, total_count))
prev_update = timestamp
count = 0
if tts > to_timestamp:
break
if len(ohlcs) == 0:
# no results, inc from one step
timestamp += timeframe * 100
if progression < 100:
Terminal.inst().info(
"100%% on %s, %s for last 100 candles, current total of %s..." %
(format_datetime(timestamp), count, total_count))
Terminal.inst().info("Last candle datetime is %s" %
(format_datetime(tts), ))
def process_entry(self, timestamp, direction, price, take_profit,
stop_loss, timeframe):
trader = self.strategy.trader()
market = trader.market(self.instrument.market_id)
quantity = 0.0
price = market.ofr # signal is at ofr price (for now limit entry at current ofr price)
date_time = datetime.fromtimestamp(timestamp)
date_str = date_time.strftime('%Y-%m-%d %H:%M:%S')
# ajust max quantity according to free asset of quote, and convert in asset base quantity
if 0: # not trader.has_margin(self.market.margin_cost(self.instrument.trade_quantity)):
Terminal.inst().notice(
"Not enought free margin %s, has %s but need %s" %
(market.quote,
market.format_quantity(trader.account.margin_balance),
market.format_quantity(self.instrument.trade_quantity)),
view='status')
else:
quantity = market.adjust_quantity(self.instrument.trade_quantity)
#
# create an order
#
do_order = self.activity
order_hedging = False
order_quantity = 0.0
order_price = None
order_type = None
order_leverage = 1.0
# @todo check self.hedging (supported by IG using compensate position)
if self.hedging:
order_hedging = True
# simply set the computed quantity
order_quantity = quantity
order_type = Order.ORDER_MARKET # Order.ORDER_LIMIT @todo limit later
# @todo or trade at order book, compute the limit price from what the order book offer
# limit best price at tiniest ofr price
# adjust price to min / tick size / max
order_price = market.adjust_price(market.ofr)
if take_profit > 0:
take_profit = market.adjust_price(take_profit)
if stop_loss > 0:
stop_loss = market.adjust_price(stop_loss)
#
# cancelation of the signal
#
if order_quantity <= 0 or order_quantity * price < market.min_notional:
# min notional not reached
do_order = False
if self.trades:
self.lock()
if len(self.trades) >= self.max_trades:
# no more than max simultaneous trades
do_order = False
for trade in self.trades:
if trade.timeframe == timeframe:
do_order = False
# if self.trades and (self.trades[-1].dir == direction) and ((timestamp - self.trades[-1].entry_open_time) < self.trade_delay):
if self.trades and (self.trades[-1].dir == direction) and (
(timestamp - self.trades[-1].entry_open_time) < timeframe):
# the same order occurs just after, ignore it
do_order = False
self.unlock()
#
# execution of the order
#
if do_order:
trade = StrategyMarginTrade(timeframe)
logger.info("Order %s %s qty=%s p=%s sl=%s tp=%s ts=%s" %
("long" if direction > 0 else "short",
self.instrument.market_id,
market.format_quantity(order_quantity),
market.format_price(order_price),
market.format_price(stop_loss),
market.format_price(take_profit), date_str))
# the new trade must be in the trades list if the event comes before, and removed after only it failed
self.add_trade(trade)
if trade.open(trader,
self.instrument.market_id,
direction,
order_type,
order_price,
order_quantity,
take_profit,
stop_loss,
order_leverage,
hedging=order_hedging):
# initiate the take-profit limit order
if take_profit > 0:
trade.modify_take_profit(trader, self.instrument.market_id,
take_profit)
# # initiate the stop-loss order
# if stop_loss > 0:
# trade.modify_stop_loss(trader, self.instrument.market_id, stop_loss)
# notify
self.strategy.notify_order(trade.id, trade.dir,
self.instrument.market_id,
market.format_price(price),
timestamp, trade.timeframe, 'entry',
None, market.format_price(trade.sl),
market.format_price(trade.tp))
# want it on the streaming (take care its only the order signal, no the real complete execution)
if trade.direction > 0:
self._global_streamer.member('buy-entry').update(
price, timestamp)
elif trade.direction < 0:
self._global_streamer.member('sell-entry').update(
price, timestamp)
else:
self.remove_trade(trade)
def set_active_view(self, view_id):
Terminal.inst().switch_view(view_id)
def execute(self, args):
if not args:
Terminal.inst().action("Missing parameters", view='status')
return False
# ie: ":long altbtc BTCUSDT L@8500 SL@8300 TP@9600 1.0"
appliance = None
market_id = None
# direction base on command name
direction = -1
method = 'market'
limit_price = None
trigger_price = None
stop_loss = 0.0
take_profit = 0.0
quantity_rate = 1.0
timeframe = Instrument.TF_4HOUR
entry_timeout = None
leverage = None
if len(args) < 2:
Terminal.inst().action("Missing parameters", view='status')
return False
try:
appliance, market_id = args[0], args[1]
for value in args[2:]:
if not value:
continue
if value.startswith("L@"):
method = 'limit'
limit_price = float(value[2:])
elif value.startswith("T@"):
method = 'trigger'
trigger_price = float(value[2:])
elif value.startswith("SL@"):
stop_loss = float(value[3:])
elif value.startswith("TP@"):
take_profit = float(value[3:])
elif value.startswith("'"):
timeframe = timeframe_from_str(value[1:])
elif value.startswith("*"):
quantity_rate = float(value[1:])
elif value.endswith("%"):
quantity_rate = float(value[:-1]) * 0.01
elif value.startswith("/"):
entry_timeout = timeframe_from_str(value[1:])
elif value.startswith("x"):
leverage = float(value[1:])
except Exception:
Terminal.inst().action("Invalid parameters", view='status')
return False
if limit_price and stop_loss and stop_loss < limit_price:
Terminal.inst().action(
"Stop-loss must be greater than limit price", view='status')
return False
if limit_price and take_profit and take_profit > limit_price:
Terminal.inst().action(
"Take-profit must be lesser than limit price", view='status')
return False
if quantity_rate <= 0.0:
Terminal.inst().action("Quantity must be non empty", view='status')
return False
self._strategy_service.command(
Strategy.COMMAND_TRADE_ENTRY, {
'appliance': appliance,
'market-id': market_id,
'direction': direction,
'limit-price': limit_price,
'trigger-price': trigger_price,
'method': method,
'quantity-rate': quantity_rate,
'stop-loss': stop_loss,
'take-profit': take_profit,
'timeframe': timeframe,
'entry-timeout': entry_timeout,
'leverage': leverage
})
return True
def pong(self, msg):
Terminal.inst().action("WokerPool::Worker %s is alive %s" % (self._uid, msg), view='content')
def do_rebuilder(options, siis_logger):
Terminal.inst().info("Starting SIIS rebuilder using %s identity..." % options['identity'])
Terminal.inst().flush()
# database manager
Database.create(options)
Database.inst().setup(options)
timeframe = -1
cascaded = None
if not options.get('timeframe'):
timeframe = 60 # default to 1min
else:
if options['timeframe'] in TIMEFRAME_FROM_STR_MAP:
timeframe = TIMEFRAME_FROM_STR_MAP[options['timeframe']]
else:
try:
timeframe = int(options['timeframe'])
except:
pass
if not options.get('cascaded'):
cascaded = None
else:
if options['cascaded'] in TIMEFRAME_FROM_STR_MAP:
cascaded = TIMEFRAME_FROM_STR_MAP[options['cascaded']]
else:
try:
cascaded = int(options['cascaded'])
except:
pass
if timeframe < 0:
siis_logger.error("Invalid timeframe")
sys.exit(-1)
# @todo tick streamer + inject in a cascaded generator
Terminal.inst().info("Flushing database...")
Terminal.inst().flush()
Database.terminate()
Terminal.inst().info("Rebuild done!")
Terminal.inst().flush()
Terminal.terminate()
sys.exit(0)
def sync(self):
# start backtesting
if self._backtesting and not self._backtest:
go_ready = True
self._mutex.acquire()
self._backtest_progress = 0
for k, appl, in self._appliances.items():
self._mutex.release()
if not appl.running or not appl.ready():
go_ready = False
self._mutex.acquire()
break
else:
self._mutex.acquire()
self._mutex.release()
if go_ready:
# start the time thread once all appliance get theirs data and are ready
class TimeStepThread(threading.Thread):
def __init__(self, service, s, e, ts, tf=0.0):
super().__init__(name="backtest")
self.service = service
self.abort = False
self.s = s
self.e = e
self.c = s
self.ts = ts
self.ppc = 0
self.tf = tf
def run(self):
prev = self.c
min_limit = 0.0001
limit = min_limit # starts with min limit
last_saturation = 0
last_sleep = time.time()
Terminal.inst().info("Backtesting started...", view='status')
appliances = self.service._appliances.values()
traders = []
wait = False
appl = None
# get the list of used traders, to sync them after each pass
for appl in appliances:
if appl.trader() and appl.trader() not in traders:
traders.append(appl.trader())
if len(appliances) == 1:
# a signe appliance, don't need to parellelize, and to sync, python sync suxx a lot, avoid the overload in most of the
# backtesting usage
while self.c < self.e + self.ts:
# now sync the trader base time
for trader in traders:
trader.set_timestamp(self.c)
appl.backtest_update(self.c, self.e)
if self.tf > 0:
# wait factor of time step, so 1 mean realtime simulation, 0 mean as fast as possible
time.sleep((1/self.tf)*self.ts)
self.c += self.ts # add one time step
self.service._timestamp = self.c
# one more step then we can update traders (limits orders, P/L update...)
for trader in traders:
trader.update()
time.sleep(0) # yield
if self.abort:
break
else:
# multiple appliances, parralelise them
while self.c < self.e + self.ts:
if not wait:
# now sync the trader base time
for trader in traders:
# @todo it could be better if we add two step, one update the market and then the strategy computation
# to avoid to update multiple time the same market and potentially with different ut... but not really an issue
trader.set_timestamp(self.c)
# query async update per appliance
for appl in appliances:
appl.query_backtest_update(self.c, self.e)
# @todo could use a semaphore or condition counter
wait = False
for appl in appliances:
# appl.backtest_update(self.c, self.e)
# wait all appliance did theirs jobs
if appl._last_done_ts < self.c:
wait = True
break
if not wait:
if self.tf > 0:
# wait factor of time step, so 1 mean realtime simulation, 0 mean as fast as possible
time.sleep((1/self.tf)*self.ts)
self.c += self.ts # add one time step
self.service._timestamp = self.c
# one more step then we can update traders (limits orders, P/L update...)
for trader in traders:
trader.update()
time.sleep(0) # yield
if self.abort:
break
self._timestep_thread = TimeStepThread(self, self._start_ts, self._end_ts, self._timestep, self._time_factor)
self._timestep_thread.setDaemon(True)
self._timestep_thread.start()
# backtesting started, avoid re-enter
self._backtest = True
if self._backtesting and self._backtest and self._backtest_progress < 100.0:
progress = 0
self._mutex.acquire()
for k, appl, in self._appliances.items():
if appl.running:
progress += appl.progress()
if self._appliances:
progress /= float(len(self._appliances))
self._mutex.release()
total = self._end_ts - self._start_ts
remaining = self._end_ts - progress
pc = 100.0 - (remaining / (total+0.001) * 100)
if pc - self._backtest_progress >= 1.0 and pc < 100.0:
self._backtest_progress = pc
Terminal.inst().info("Backtesting %s%%..." % round(pc), view='status')
if self._end_ts - progress <= 0.0:
# finished !
self._backtest_progress = 100.0
# backtesting done => waiting user
Terminal.inst().info("Backtesting 100% finished !", view='status')
def do_optimizer(options, siis_logger):
Terminal.inst().info("Starting SIIS optimizer...")
Terminal.inst().flush()
# database manager
Database.create(options)
Database.inst().setup(options)
broker_id = options['broker']
market_id = options['market']
timeframe = -1
cascaded = None
if options.get(
'timeframe') and options['timeframe'] in TIMEFRAME_FROM_STR_MAP:
timeframe = TIMEFRAME_FROM_STR_MAP[options['timeframe']]
else:
try:
timeframe = int(options['timeframe'])
except:
pass
if timeframe < 0:
siis_logger.error("Invalid timeframe")
sys.exit(-1)
if timeframe == 0:
# tick
pass
# @todo
else:
# ohlc
pass
# @todo
Terminal.inst().info("Flushing database...")
Terminal.inst().flush()
Database.terminate()
Terminal.inst().info("Optimization done!")
Terminal.inst().flush()
Terminal.terminate()
sys.exit(0)
def run(self):
prev = self.c
min_limit = 0.0001
limit = min_limit # starts with min limit
last_saturation = 0
last_sleep = time.time()
Terminal.inst().info("Backtesting started...", view='status')
appliances = self.service._appliances.values()
traders = []
wait = False
appl = None
# get the list of used traders, to sync them after each pass
for appl in appliances:
if appl.trader() and appl.trader() not in traders:
traders.append(appl.trader())
if len(appliances) == 1:
# a signe appliance, don't need to parellelize, and to sync, python sync suxx a lot, avoid the overload in most of the
# backtesting usage
while self.c < self.e + self.ts:
# now sync the trader base time
for trader in traders:
trader.set_timestamp(self.c)
appl.backtest_update(self.c, self.e)
if self.tf > 0:
# wait factor of time step, so 1 mean realtime simulation, 0 mean as fast as possible
time.sleep((1/self.tf)*self.ts)
self.c += self.ts # add one time step
self.service._timestamp = self.c
# one more step then we can update traders (limits orders, P/L update...)
for trader in traders:
trader.update()
time.sleep(0) # yield
if self.abort:
break
else:
# multiple appliances, parralelise them
while self.c < self.e + self.ts:
if not wait:
# now sync the trader base time
for trader in traders:
# @todo it could be better if we add two step, one update the market and then the strategy computation
# to avoid to update multiple time the same market and potentially with different ut... but not really an issue
trader.set_timestamp(self.c)
# query async update per appliance
for appl in appliances:
appl.query_backtest_update(self.c, self.e)
# @todo could use a semaphore or condition counter
wait = False
for appl in appliances:
# appl.backtest_update(self.c, self.e)
# wait all appliance did theirs jobs
if appl._last_done_ts < self.c:
wait = True
break
if not wait:
if self.tf > 0:
# wait factor of time step, so 1 mean realtime simulation, 0 mean as fast as possible
time.sleep((1/self.tf)*self.ts)
self.c += self.ts # add one time step
self.service._timestamp = self.c
# one more step then we can update traders (limits orders, P/L update...)
for trader in traders:
trader.update()
time.sleep(0) # yield
if self.abort:
break
def process_entry(self, timestamp, price, take_profit, stop_loss,
timeframe, partial_tp):
trader = self.strategy.trader()
quantity = 0.0
direction = Order.LONG # entry is always a long
# large limit price because else miss the pumping markets
price = price + self.instrument.market_spread * (
1 if trader.paper_mode else 5) # signal price + spread
# date_time = datetime.fromtimestamp(timestamp)
# date_str = date_time.strftime('%Y-%m-%d %H:%M:%S')
# ajust max quantity according to free asset of quote, and convert in asset base quantity
if trader.has_asset(self.instrument.quote):
# quantity = min(quantity, trader.asset(self.instrument.quote).free) / self.instrument.market_ofr
if trader.has_quantity(self.instrument.quote,
self.instrument.trade_quantity):
quantity = self.instrument.adjust_quantity(
self.instrument.trade_quantity /
price) # and adjusted to 0/max/step
else:
Terminal.inst().notice(
"Not enought free quote asset %s, has %s but need %s" %
(self.instrument.quote,
self.instrument.format_quantity(
trader.asset(self.instrument.quote).free),
self.instrument.format_quantity(
self.instrument.trade_quantity)),
view='status')
#
# create an order
#
# only if active
do_order = self.activity
order_quantity = 0.0
order_price = None
order_type = None
order_leverage = 1.0
# simply set the computed quantity
order_quantity = quantity
# prefered in limit order at the current best price, and with binance in market INSUFISCENT BALANCE can occurs with market orders...
order_type = Order.ORDER_LIMIT
# limit price
order_price = float(self.instrument.format_price(price))
# order_price = self.instrument.adjust_price(price)
#
# cancelation of the signal
#
if order_quantity <= 0 or order_quantity * price < self.instrument.min_notional:
# min notional not reached
do_order = False
if self.trades:
self.lock()
if len(self.trades) >= self.max_trades:
# no more than max simultaneous trades
do_order = False
# for trade in self.trades:
# if trade.timeframe == timeframe:
# do_order = False
# if self.trades and (self.trades[-1].dir == direction) and ((timestamp - self.trades[-1].entry_open_time) < self.trade_delay):
#if self.trades and (self.trades[-1].dir == direction) and ((timestamp - self.trades[-1].entry_open_time) < timeframe):
# # the same order occurs just after, ignore it
# do_order = False
self.unlock()
#
# execution of the order
#
if do_order:
trade = StrategyAssetTrade(timeframe)
# the new trade must be in the trades list if the event comes before, and removed after only it failed
self.add_trade(trade)
trade.set('partial-take-profit', partial_tp)
if trade.open(trader, self.instrument, direction, order_type,
order_price, order_quantity, take_profit, stop_loss,
order_leverage):
# notify
self.strategy.notify_order(
trade.id, trade.dir, self.instrument.market_id,
self.instrument.format_price(price), timestamp,
trade.timeframe, 'entry', None,
self.instrument.format_price(trade.sl),
self.instrument.format_price(trade.tp))
# want it on the streaming
if self._global_streamer:
# @todo remove me after notify manage that
self._global_streamer.member('buy-entry').update(
price, timestamp)
else:
self.remove_trade(trade)
else:
# notify a signal only
self.strategy.notify_order(
-1, Order.LONG, self.instrument.market_id,
self.instrument.format_price(price), timestamp, timeframe,
'entry', None, self.instrument.format_price(stop_loss),
self.instrument.format_price(take_profit))
def application(argv):
fix_thread_set_name()
# init terminal displayer
Terminal()
options = {'log-path': './user/log', 'log-name': 'client.log'}
# create initial siis data structure if necessary
install(options)
siis_log = SiisLog(options)
logger = logging.getLogger('siis.client')
stream = ""
rpc = ""
fifo = -1
fifo_rpc = -1
if len(sys.argv) > 1:
stream = sys.argv[1]
if len(sys.argv) > 2:
rpc = sys.argv[2]
if not stream:
Terminal.inst().error("- Missing stream url !")
if not rpc:
Terminal.inst().error("- Missing RPC url !")
try:
fifo = os.open(stream, os.O_NONBLOCK | posix.O_RDONLY)
except Exception as e:
Terminal.inst().error(repr(e))
if not fifo:
Terminal.inst().error("- Cannot open the stream !")
try:
fifo_rpc = os.open(rpc, os.O_NONBLOCK | posix.O_WRONLY)
except Exception as e:
Terminal.inst().error(repr(e))
if not fifo_rpc:
Terminal.inst().error("- Cannot open the RPC fifo !")
Terminal.inst().info("Starting SIIS simple chart client...")
Terminal.inst().flush()
try:
Charting.inst().start()
except Exception as e:
has_exception(e)
dispatcher = Dispatcher()
running = True
Terminal.inst().message("Running main loop...")
size = 32768
buf = []
content = ""
cur = bytearray()
if not Charting.inst().visible:
if not Charting.inst().running:
# charting service
try:
Charting.inst().start()
except Exception as e:
has_exception(e)
if Charting.inst().running:
Charting.inst().show()
Terminal.inst().action("Charting is now shown")
while running:
# read from fifo
try:
buf = os.read(fifo, size)
if buf:
for n in buf:
if n == 10: # new line as message termination
try:
msg = json.loads(cur.decode('utf8'))
dispatcher.on_message(msg)
except Exception as e:
logger.error(repr(e))
cur = bytearray()
else:
cur.append(n)
except (BrokenPipeError, IOError):
pass
if not Charting.inst().has_charts():
running = False
time.sleep(0.01)
# close message
messages = dispatcher.close()
if fifo:
os.close(fifo)
fifo = -1
if fifo_rpc:
for msg in messages:
try:
# write to fifo
posix.write(fifo_rpc, (json.dumps(msg) + '\n').encode('utf8'))
except (BrokenPipeError, IOError) as e:
logger.error(repr(e))
except (TypeError, ValueError) as e:
logger.error("Error sending message : %s" % repr(c))
fifo_rpc.flush()
# os.flush(fifo_rpc)
os.close(fifo_rpc)
fifo_rpc = -1
Terminal.inst().info("Terminate...")
Terminal.inst().flush()
# terminate charting singleton
Charting.terminate()
Terminal.inst().info("Bye!")
Terminal.inst().flush()
Terminal.terminate()
def pre_run(self):
Terminal.inst().info("Running watcher %s..." % self._name)
self.connect()
def ping(self, timeout):
if self._mutex.acquire(timeout=timeout):
self._ping = (0, None, True)
self._mutex.release()
else:
Terminal.inst().action("Unable to join thread %s for %s seconds" % (self._thread.name if self._thread else "unknown", timeout), view='content')
