def init(self, options):
# database manager
Database.create(options)
Database.inst().setup(options)
# want speedup the database inserts
Database.inst().enable_fetch_mode()
# default no initial fetch, opt-in
if 'initial-fetch' not in options:
options['initial-fetch'] = False
return True
def do_rebuilder(options, siis_logger):
Terminal.inst().info("Starting SIIS rebuilder using %s identity..." % options['identity'])
Terminal.inst().flush()
# database manager
Database.create(options)
Database.inst().setup(options)
timeframe = -1
cascaded = None
if not options.get('timeframe'):
timeframe = 60 # default to 1min
else:
if options['timeframe'] in TIMEFRAME_FROM_STR_MAP:
timeframe = TIMEFRAME_FROM_STR_MAP[options['timeframe']]
else:
try:
timeframe = int(options['timeframe'])
except:
pass
if not options.get('cascaded'):
cascaded = None
else:
if options['cascaded'] in TIMEFRAME_FROM_STR_MAP:
cascaded = TIMEFRAME_FROM_STR_MAP[options['cascaded']]
else:
try:
cascaded = int(options['cascaded'])
except:
pass
if timeframe < 0:
siis_logger.error("Invalid timeframe")
sys.exit(-1)
# @todo tick streamer + inject in a cascaded generator
Terminal.inst().info("Flushing database...")
Terminal.inst().flush()
Database.terminate()
Terminal.inst().info("Rebuild done!")
Terminal.inst().flush()
Terminal.terminate()
sys.exit(0)
def do_optimizer(options, siis_logger):
Terminal.inst().info("Starting SIIS optimizer...")
Terminal.inst().flush()
# database manager
Database.create(options)
Database.inst().setup(options)
broker_id = options['broker']
market_id = options['market']
timeframe = -1
cascaded = None
if options.get(
'timeframe') and options['timeframe'] in TIMEFRAME_FROM_STR_MAP:
timeframe = TIMEFRAME_FROM_STR_MAP[options['timeframe']]
else:
try:
timeframe = int(options['timeframe'])
except:
pass
if timeframe < 0:
siis_logger.error("Invalid timeframe")
sys.exit(-1)
if timeframe == 0:
# tick
pass
# @todo
else:
# ohlc
pass
# @todo
Terminal.inst().info("Flushing database...")
Terminal.inst().flush()
Database.terminate()
Terminal.inst().info("Optimization done!")
Terminal.inst().flush()
Terminal.terminate()
sys.exit(0)
def do_syncer(options):
"""
Make a connection and synchronize the market data in local DB.
"""
Terminal.inst().info("Starting SIIS syncer using %s identity..." %
options['identity'])
Terminal.inst().flush()
# database manager
Database.create(options)
Database.inst().setup(options)
# default no initial fetch, opt-in
if 'initial-fetch' not in options:
options['initial-fetch'] = False
# watcher service
Terminal.inst().info("Starting watcher's service...")
watcher_service = WatcherService(options)
markets = options['market'].split(',')
watcher = watcher_service.create_watcher(options, options['broker'],
markets)
if watcher:
watcher.initial_fetch = options.get('initial-fetch', False)
watcher.connect()
watcher.update_markets_info()
watcher.disconnect()
watcher_service.terminate()
Database.terminate()
Terminal.inst().info("Sync done!")
Terminal.inst().flush()
Terminal.terminate()
sys.exit(0)
def do_fetcher(options):
Terminal.inst().info("Starting SIIS fetcher using %s identity..." %
options['identity'])
Terminal.inst().flush()
# database manager
Database.create(options)
Database.inst().setup(options)
watcher_service = WatcherService(options)
fetcher = watcher_service.create_fetcher(options, options['broker'])
timeframe = -1
cascaded = None
if not options.get('timeframe'):
timeframe = 60 # default to 1min
else:
if options['timeframe'] in TIMEFRAME_FROM_STR_MAP:
timeframe = TIMEFRAME_FROM_STR_MAP[options['timeframe']]
else:
try:
timeframe = int(options['timeframe'])
except:
pass
if not options.get('cascaded'):
cascaded = None
else:
if options['cascaded'] in TIMEFRAME_FROM_STR_MAP:
cascaded = TIMEFRAME_FROM_STR_MAP[options['cascaded']]
else:
try:
cascaded = int(options['cascaded'])
except:
pass
if timeframe < 0:
logger.error("Invalid timeframe")
sys.exit(-1)
try:
fetcher.connect()
except:
sys.exit(-1)
if fetcher.connected:
logger.info("Fetcher authentified to %s, trying to collect data..." %
fetcher.name)
markets = fetcher.matching_symbols_set(options['market'].split(','),
fetcher.available_instruments())
try:
for market_id in markets:
if not fetcher.has_instrument(market_id, options.get('spec')):
logger.error("Market %s not found !" % (market_id, ))
else:
if options.get('install-market', False):
fetcher.install_market(market_id)
else:
fetcher.fetch_and_generate(market_id, timeframe,
options.get('from'),
options.get('to'),
options.get('last'),
options.get('spec'),
cascaded)
except KeyboardInterrupt:
pass
finally:
fetcher.disconnect()
fetcher = None
Terminal.inst().info("Flushing database...")
Terminal.inst().flush()
Database.terminate()
Terminal.inst().info("Fetch done!")
Terminal.inst().flush()
Terminal.terminate()
sys.exit(0)
def do_optimizer(options):
Terminal.inst().info("Starting SIIS optimizer...")
Terminal.inst().flush()
# database manager
Database.create(options)
Database.inst().setup(options)
broker_id = options['broker']
market_id = options['market']
timeframe = None
from_date = options.get('from')
to_date = options.get('to')
if not to_date:
today = datetime.now().astimezone(UTC())
if timeframe == Instrument.TF_MONTH:
to_date = today + timedelta(months=1)
else:
to_date = today + timedelta(seconds=timeframe)
to_date = to_date.replace(microsecond=0)
if not options.get('timeframe'):
timeframe = None
else:
if options['timeframe'] in TIMEFRAME_FROM_STR_MAP:
timeframe = TIMEFRAME_FROM_STR_MAP[options['timeframe']]
else:
try:
timeframe = int(options['timeframe'])
except:
pass
try:
# checking data integrity, gap...
if timeframe is None:
for market in options['market'].split(','):
if market.startswith('!') or market.startswith('*'):
continue
for tf in GENERATED_TF:
Terminal.inst().info("Verifying %s OHLC %s..." % (market, timeframe_to_str(tf)))
check_ohlcs(options['broker'], market, tf, from_date, to_date)
elif timeframe == Instrument.TF_TICK:
for market in options['market'].split(','):
if market.startswith('!') or market.startswith('*'):
continue
Terminal.inst().info("Verifying %s ticks/trades..." % (market,))
check_ticks(options['broker'], market, from_date, to_date)
elif timeframe > 0:
# particular ohlc
for market in options['market'].split(','):
if market.startswith('!') or market.startswith('*'):
continue
Terminal.inst().info("Verifying %s OHLC %s..." % (market, timeframe_to_str(timeframe)))
check_ohlcs(options['broker'], market, timeframe, from_date, to_date)
except KeyboardInterrupt:
pass
finally:
pass
Terminal.inst().info("Flushing database...")
Terminal.inst().flush()
Database.terminate()
Terminal.inst().info("Optimization done!")
Terminal.inst().flush()
Terminal.terminate()
sys.exit(0)
def application(argv):
fix_thread_set_name()
# init terminal displayer
Terminal()
options = {
'working-path': os.getcwd(),
'identity': 'real',
'config-path': './user/config',
'log-path': './user/log',
'reports-path': './user/reports',
'markets-path': './user/markets',
'log-name': 'siis.log',
}
# create initial siis data structure if necessary
install(options)
siis_log = SiisLog(options, Terminal().inst().style())
siis_logger = logging.getLogger('siis')
# parse process command line
if len(argv) > 1:
options['livemode'] = True
# utc or local datetime ?
for arg in argv:
if arg.startswith('--'):
if arg == '--paper-mode':
# livemode but in paper-mode
options['paper-mode'] = True
elif arg == '--fetch':
# use the fetcher
options['fetch'] = True
elif arg == '--binarize':
# use the binarizer
options['binarize'] = True
elif arg == '--optimize':
# use the optimizer
options['optimize'] = True
elif arg == '--sync':
# use the syncer
options['sync'] = True
elif arg == '--rebuild':
# use the rebuilder
options['rebuild'] = True
elif arg == '--install-market':
options['install-market'] = True
elif arg == '--initial-fetch':
# do the initial OHLC fetch for watchers
options['initial-fetch'] = True
elif arg == '--backtest':
# backtest mean always paper-mode
options['paper-mode'] = True
options['backtesting'] = True
elif arg.startswith('--timestep='):
# backesting timestep, default is 60 second
options['timestep'] = float(arg.split('=')[1])
elif arg.startswith('--time-factor='):
# backtesting time-factor
options['time-factor'] = float(arg.split('=')[1])
elif arg.startswith('--from='):
# if backtest from date (if ommited use whoole data) date format is "yyyy-mm-dd-hh:mm:ss", fetch, binarize, optimize to date
options['from'] = datetime.strptime(
arg.split('=')[1],
'%Y-%m-%dT%H:%M:%S').replace(tzinfo=UTC())
elif arg.startswith('--to='):
# if backtest to date (can be ommited), fetch, binarize, optimize to date
options['to'] = datetime.strptime(
arg.split('=')[1],
'%Y-%m-%dT%H:%M:%S').replace(tzinfo=UTC())
elif arg.startswith('--last='):
# fetch the last n data history
options['last'] = int(arg.split('=')[1])
elif arg.startswith('--market='):
# fetch, binarize, optimize the data history for this market
options['market'] = arg.split('=')[1]
elif arg.startswith('--spec='):
# fetcher data history option
options['option'] = arg.split('=')[1]
elif arg.startswith('--broker='):
# broker name for fetcher, watcher, optimize, binarize
options['broker'] = arg.split('=')[1]
elif arg.startswith('--timeframe='):
# fetch, binarize, optimize base timeframe
options['timeframe'] = arg.split('=')[1]
elif arg.startswith('--cascaded='):
# fetch cascaded ohlc generation
options['cascaded'] = arg.split('=')[1]
elif arg == '--watcher-only':
# feed only with live data (not compatible with --read-only)
options['watcher-only'] = True
elif arg == '--read-only':
# does not write to the database (not compatible with --watcher-only)
options['read-only'] = True
elif arg == '--check-data':
# check DB ohlc data (@todo)
options['check-data'] = True
elif arg.startswith('--profile='):
# appliances profile name
options['profile'] = arg.split('=')[1]
elif arg == '--version':
Terminal.inst().info('%s %s' % (APP_SHORT_NAME, '.'.join(
[str(x) for x in APP_VERSION])))
sys.exit(0)
elif arg == '--help' or '-h':
display_cli_help()
sys.exit(0)
else:
options['identity'] = argv[1]
# watcher-only read-only mutual exclusion
if options.get('watcher-only') and options.get('read-only'):
Terminal.inst().error(
"Options --watcher-only and --read-only are mutually exclusive !"
)
sys.exit(-1)
# backtesting
if options.get('backtesting', False):
if options.get('from') is None or options.get('to') is None:
del options['backtesting']
Terminal.inst().error(
"Backtesting need from= and to= date time")
sys.exit(-1)
if options['identity'].startswith('-'):
Terminal.inst().error("First option must be the identity name")
#
# binarizer mode
#
if options.get('binarize'):
if options.get('market') and options.get('from') and options.get(
'to') and options.get('broker'):
from tools.binarizer import do_binarizer
do_binarizer(options)
else:
display_cli_help()
sys.exit(0)
#
# fetcher mode
#
if options.get('fetch'):
if options.get('market') and options.get('broker'):
from tools.fetcher import do_fetcher
do_fetcher(options)
else:
display_cli_help()
sys.exit(0)
#
# optimizer mode
#
if options.get('optimize'):
if options.get('market') and options.get('from') and options.get(
'to') and options.get('broker'):
from tools.optimizer import do_optimizer
do_optimizer(options)
else:
display_cli_help()
sys.exit(0)
#
# sync mode
#
if options.get('sync'):
if options.get('market') and options.get('broker'):
from tools.syncer import do_syncer
do_syncer(options)
else:
display_cli_help()
sys.exit(0)
#
# rebuilder mode
#
if options.get('rebuild'):
if options.get('market') and options.get('from') and options.get(
'to') and options.get('broker') and options.get('timeframe'):
from tools.rebuilder import do_rebuilder
do_rebuilder(options)
else:
display_cli_help()
sys.exit(0)
#
# normal mode
#
Terminal.inst().info("Starting SIIS using %s identity..." %
options['identity'])
Terminal.inst().action("- (Press 'q' twice to terminate)")
Terminal.inst().action("- (Press 'h' for help)")
Terminal.inst().flush()
if options.get('backtesting'):
Terminal.inst().notice("Process a backtesting.")
if options.get('paper-mode'):
Terminal.inst().notice("- Using paper-mode trader.")
else:
Terminal.inst().notice("- Using live-mode trader.")
signal.signal(signal.SIGINT, signal_handler)
# monitoring service
Terminal.inst().info("Starting monitor service...")
monitor_service = MonitorService(options)
# desktop notifier
desktop_service = DesktopNotifier()
# discord_service = DiscordNotifier()
view_service = ViewService()
# database manager
Database.create(options)
Database.inst().setup(options)
# watcher service
Terminal.inst().info("Starting watcher's service...")
watcher_service = WatcherService(options)
watcher_service.start(options)
# trader service
Terminal.inst().info("Starting trader's service...")
trader_service = TraderService(watcher_service, monitor_service, options)
trader_service.start(options)
# want to display desktop notification and update views
watcher_service.add_listener(desktop_service)
watcher_service.add_listener(view_service)
# want to display desktop notification and update views
trader_service.add_listener(desktop_service)
trader_service.add_listener(view_service)
# trader service listen to watcher service and update views
watcher_service.add_listener(trader_service)
# strategy service
Terminal.inst().info("Starting strategy's service...")
strategy_service = StrategyService(watcher_service, trader_service,
monitor_service, options)
strategy_service.start(options)
# strategy service listen to watcher service
watcher_service.add_listener(strategy_service)
# strategy service listen to trader service
trader_service.add_listener(strategy_service)
# want to display desktop notification, update view and notify on discord
# strategy_service.add_listener(notifier_service)
# @todo add notifier service and replace desktop service as desktop notifier into this service same for discord...
strategy_service.add_listener(desktop_service)
strategy_service.add_listener(view_service)
# for display stats (@todo move to views)
desktop_service.strategy_service = strategy_service
desktop_service.trader_service = trader_service
# register terminal commands
commands_handler = CommandsHandler()
commands_handler.init(options)
# cli commands registration
register_general_commands(commands_handler)
register_trading_commands(commands_handler, trader_service,
strategy_service, monitor_service)
register_region_commands(commands_handler, strategy_service)
setup_views(siis_logger, view_service, watcher_service, trader_service,
strategy_service)
# setup and start the monitor service
monitor_service.setup(watcher_service, trader_service, strategy_service)
monitor_service.start()
Terminal.inst().message("Running main loop...")
Terminal.inst().upgrade()
Terminal.inst().message("Steady...", view='notice')
display_welcome()
LOOP_SLEEP = 0.016 # in second
MAX_CMD_ALIVE = 15 # in second
running = True
value = None
value_changed = False
command_timeout = 0
prev_timestamp = 0
try:
while running:
# keyboard input commands
try:
c = Terminal.inst().read()
key = Terminal.inst().key()
if c:
# split the commande line
args = [
arg
for arg in (value[1:].split(' ')
if value and value.startswith(':') else [])
if arg
]
if value and value[-1] == ' ':
args.append('')
# update the current type command
commands_handler.process_char(c, args)
if key:
if key == 'KEY_ESCAPE':
# cancel command
value = None
value_changed = True
command_timeout = 0
# use command mode
Terminal.inst().set_mode(Terminal.MODE_DEFAULT)
# split the commande line
args = [
arg
for arg in (value[1:].split(' ')
if value and value.startswith(':') else [])
if arg
]
if value and value[-1] == ' ':
args.append('')
# process on the arguments
args = commands_handler.process_key(
key, args,
Terminal.inst().mode == Terminal.MODE_COMMAND)
if args:
# regen the updated commande ligne
value = ":" + ' '.join(args)
value_changed = True
command_timeout = 0
desktop_service.on_key_pressed(key)
# @todo move the rest to command_handler
if c:
if value and value[0] == ':':
if c == '\b':
# backspace, erase last command char
value = value[:-1] if value else None
value_changed = True
command_timeout = time.time()
elif c != '\n':
# append to the advanced command value
value += c
value_changed = True
command_timeout = time.time()
elif c == '\n':
result = commands_handler.process_cli(value)
command_timeout = 0
if not result:
# maybe an application level command
if value == ':q' or value == ':quit':
running = False
elif value.startswith(':x '):
# manually exit position at market @todo move as command
target = value[2:]
if target == "all" or target == "ALL":
Terminal.inst().action(
"Send close to market command for any positions",
view='status')
trader_service.command(
Trader.COMMAND_CLOSE_ALL_MARKET,
{})
else:
Terminal.inst().action(
"Send close to market command for position %s"
% (target, ),
view='status')
trader_service.command(
Trader.COMMAND_CLOSE_MARKET,
{'key': target})
elif value.startswith(':d '):
# @deprecated manually duplicate a position entry or exit must be associated to social strategy
# @todo move as command
target = value[2:]
Terminal.inst().action(
"Send replicate to market command for position %s"
% (target, ),
view='status')
trader_service.command(
Trader.COMMAND_TRIGGER,
{'key': target})
# clear command value
value_changed = True
value = None
# use default mode
Terminal.inst().set_mode(Terminal.MODE_DEFAULT)
elif c != '\n':
# initial command value
value = "" + c
value_changed = True
command_timeout = time.time()
if value and value[0] == ':':
# use command mode
Terminal.inst().set_mode(Terminal.MODE_COMMAND)
if value and value[0] != ':':
# direct key
# use default mode
Terminal.inst().set_mode(Terminal.MODE_DEFAULT)
try:
result = commands_handler.process_accelerator(key)
# @todo convert to Command object accelerator
if not result:
result = True
# @todo might be replaced by views
if value == 'p':
trader_service.command(
Trader.COMMAND_LIST_POSITIONS, {})
elif value == 'o':
trader_service.command(
Trader.COMMAND_LIST_ORDERS, {})
elif value == 'g':
trader_service.command(
Trader.COMMAND_SHOW_PERFORMANCE, {})
# display views
elif value == 'C':
Terminal.inst().clear_content()
elif value == 'D':
Terminal.inst().switch_view('debug')
elif value == 'I':
Terminal.inst().switch_view('content')
elif value == 'F':
Terminal.inst().switch_view('strategy')
elif value == 'S':
Terminal.inst().switch_view('stats')
elif value == 'P':
Terminal.inst().switch_view('perf')
elif value == 'T':
Terminal.inst().switch_view('ticker')
elif value == 'A':
Terminal.inst().switch_view('account')
elif value == 'M':
Terminal.inst().switch_view('market')
elif value == 'Q':
Terminal.inst().switch_view('asset')
elif value == 'N':
Terminal.inst().switch_view('signal')
elif value == '?':
# ping services and workers
watcher_service.ping()
trader_service.ping()
strategy_service.ping()
monitor_service.ping()
elif value == ' ':
# a simple mark on the terminal
Terminal.inst().notice(
"Trading time %s" %
(datetime.fromtimestamp(
strategy_service.timestamp).
strftime('%Y-%m-%d %H:%M:%S')),
view='status')
elif value == 'a':
desktop_service.audible = not desktop_service.audible
Terminal.inst().action(
"Audible notification are now %s" %
("actives" if desktop_service.audible
else "disabled", ),
view='status')
elif value == 'n':
desktop_service.popups = not desktop_service.popups
Terminal.inst().action(
"Desktop notification are now %s" %
("actives" if desktop_service.popups
else "disabled", ),
view='status')
elif value == 'e':
desktop_service.discord = not desktop_service.discord
Terminal.inst().action(
"Discord notification are now %s" %
("actives" if desktop_service.discord
else "disabled", ),
view='status')
else:
result = False
if result:
value = None
value_changed = True
command_timeout = 0
except Exception as e:
has_exception(siis_logger, e)
except IOError:
pass
except Exception as e:
has_exception(siis_logger, e)
# display advanced command only
if value_changed:
if value and value.startswith(':'):
Terminal.inst().action("Command: %s" % value[1:],
view='command')
else:
Terminal.inst().message("", view='command')
# clear input if no char hit during the last MAX_CMD_ALIVE
if value and not value.startswith(':'):
if (command_timeout > 0) and (time.time() - command_timeout >=
MAX_CMD_ALIVE):
value = None
value_changed = True
Terminal.inst().info("Current typing canceled",
view='status')
try:
# display strategy tarding time (update max once per second)
if strategy_service.timestamp - prev_timestamp >= 1.0:
mode = "live"
if trader_service.backtesting:
mode = "backtesting"
elif trader_service.paper_mode:
mode = "paper-mode"
Terminal.inst().message(
"%s - %s" %
(mode,
datetime.fromtimestamp(strategy_service.timestamp).
strftime('%Y-%m-%d %H:%M:%S')),
view='notice')
prev_timestamp = strategy_service.timestamp
# synchronous operations here
watcher_service.sync()
trader_service.sync()
strategy_service.sync()
monitor_service.sync()
desktop_service.sync()
view_service.sync()
Terminal.inst().update()
except BaseException as e:
siis_logger.error(traceback.format_exc())
Terminal.inst().error(repr(e))
# don't waste CPU time on main thread
time.sleep(LOOP_SLEEP)
finally:
Terminal.inst().restore_term()
Terminal.inst().info("Terminate...")
Terminal.inst().flush()
commands_handler.terminate(options)
commands_handler = None
# service terminate
monitor_service.terminate()
strategy_service.terminate()
trader_service.terminate()
watcher_service.terminate()
desktop_service.terminate()
# discord_service.terminate()
view_service.terminate()
Terminal.inst().info("Saving database...")
Terminal.inst().flush()
Database.terminate()
Terminal.inst().info("Bye!")
Terminal.inst().flush()
Terminal.terminate()
default=False,
help='Verbose output')
parser.add_argument('-i',
'--id',
type=str,
required=True,
help='Stock or index id')
parser.add_argument('file',
type=str,
default=None,
help='CSV file to import')
args = parser.parse_args()
database = Database(args.database)
if args.database == ':memory:':
database.create()
data = pd.read_csv(args.file, header=0)
for i in range(len(data)):
row = data.ix[i]
if row['Low'] != 'null' and row['High'] != 'null':
entry = StockEntry(to_date(row['Date']), args.id.lower(),
(float(row['High']) + float(row['Low'])) / 2)
database.add(entry)
database.commit()
def application(argv):
fix_thread_set_name()
# init terminal displayer
Terminal()
options = {
'working-path': os.getcwd(),
'identity': 'real',
'config-path': './user/config',
'log-path': './user/log',
'reports-path': './user/reports',
'markets-path': './user/markets',
'log-name': 'siis.log',
}
# create initial siis data structure if necessary
install(options)
siis_log = SiisLog(options, Terminal().inst().style())
siis_logger = logging.getLogger('siis')
traceback_logger = logging.getLogger('siis.traceback')
# parse process command line
if len(argv) > 1:
options['livemode'] = True
# utc or local datetime ?
for arg in argv:
if arg.startswith('--'):
if arg == '--paper-mode':
# livemode but in paper-mode
options['paper-mode'] = True
elif arg == '--fetch':
# use the fetcher
options['tool'] = "fetcher"
elif arg == '--binarize':
# use the binarizer
options['tool'] = "binarize"
elif arg == '--optimizer':
# use the optimizer
options['tool'] = "optimizer"
elif arg == '--sync':
# use the syncer
options['tool'] = "syncer"
elif arg == '--rebuild':
# use the rebuilder
options['tool'] = "rebuilder"
elif arg == '--export':
# use the exporter
options['tool'] = "exporter"
elif arg == '--import':
# use the importer
options['tool'] = "importer"
elif arg == '--clean':
# use the cleaner
options['tool'] = "cleaner"
elif arg.startswith("--tool="):
# use a named tool
options['tool'] = arg.split('=')[1]
elif arg == '--no-conf':
options['no-conf'] = True
elif arg == '--zip':
options['zip'] = True
elif arg == '--install-market':
options['install-market'] = True
elif arg == '--initial-fetch':
# do the initial OHLC fetch for watchers
options['initial-fetch'] = True
elif arg == '--backtest':
# backtest mean always paper-mode
options['paper-mode'] = True
options['backtesting'] = True
elif arg.startswith('--timestep='):
# backesting timestep, default is 60 second
options['timestep'] = float(arg.split('=')[1])
elif arg.startswith('--time-factor='):
# backtesting time-factor
options['time-factor'] = float(arg.split('=')[1])
elif arg.startswith('--filename='):
# used with import or export
options['filename'] = arg.split('=')[1]
elif arg.startswith('--from='):
# if backtest from date and tools
options['from'] = parse_datetime(arg.split('=')[1])
if not options['from']:
Terminal.inst().error("Invalid 'from' datetime format")
sys.exit(-1)
elif arg.startswith('--to='):
# if backtest to date and tools
options['to'] = parse_datetime(arg.split('=')[1])
if not options['to']:
Terminal.inst().error("Invalid 'to' datetime format")
sys.exit(-1)
elif arg.startswith('--last='):
# fetch the last n data history
options['last'] = int(arg.split('=')[1])
if options['last'] <= 0:
Terminal.inst().error(
"Invalid 'last' value. Must be at least 1")
sys.exit(-1)
elif arg.startswith('--market='):
# fetch, binarize, optimize the data history for this market
options['market'] = arg.split('=')[1]
elif arg.startswith('--spec='):
# fetcher data history option
options['option'] = arg.split('=')[1]
elif arg.startswith('--broker='):
# broker name for fetcher, watcher, optimize, binarize
options['broker'] = arg.split('=')[1]
elif arg.startswith('--timeframe='):
# fetch, binarize, optimize base timeframe
options['timeframe'] = arg.split('=')[1]
elif arg.startswith('--cascaded='):
# fetch cascaded ohlc generation
options['cascaded'] = arg.split('=')[1]
elif arg.startswith('--target='):
# target ohlc generation
options['target'] = arg.split('=')[1]
elif arg == '--watcher-only':
# feed only with live data (not compatible with --read-only)
options['watcher-only'] = True
elif arg == '--read-only':
# does not write to the database (not compatible with --watcher-only)
options['read-only'] = True
elif arg.startswith('--profile='):
# appliances profile name
options['profile'] = arg.split('=')[1]
elif arg == '--version':
Terminal.inst().info('%s %s' % (APP_SHORT_NAME, '.'.join(
[str(x) for x in APP_VERSION])))
sys.exit(0)
elif arg == '--help' or '-h':
display_cli_help()
sys.exit(0)
else:
options['identity'] = argv[1]
# watcher-only read-only mutual exclusion
if options.get('watcher-only') and options.get('read-only'):
Terminal.inst().error(
"Options --watcher-only and --read-only are mutually exclusive !"
)
sys.exit(-1)
# backtesting
if options.get('backtesting', False):
if options.get('from') is None or options.get('to') is None:
del options['backtesting']
Terminal.inst().error(
"Backtesting need from= and to= date time")
sys.exit(-1)
#
# binarizer mode
#
if options.get('tool') == "binarizer":
if options.get('market') and options.get('from') and options.get(
'to') and options.get('broker'):
from tools.binarizer import do_binarizer
do_binarizer(options)
else:
display_cli_help()
sys.exit(0)
#
# fetcher mode
#
if options.get('tool') == "fetcher":
if options.get('market') and options.get('broker'):
from tools.fetcher import do_fetcher
do_fetcher(options)
else:
display_cli_help()
sys.exit(0)
#
# optimizer mode
#
if options.get('tool') == "optimizer":
if options.get('market') and options.get('from') and options.get(
'broker'):
from tools.optimizer import do_optimizer
do_optimizer(options)
else:
display_cli_help()
sys.exit(0)
#
# rebuilder mode
#
if options.get('tool') == "rebuilder":
if options.get('market') and options.get('from') and options.get(
'broker') and options.get('timeframe'):
from tools.rebuilder import do_rebuilder
do_rebuilder(options)
else:
display_cli_help()
sys.exit(0)
#
# exporter mode
#
if options.get('tool') == "exporter":
if options.get('market') and options.get('from') and options.get(
'broker') and options.get('filename'):
from tools.exporter import do_exporter
do_exporter(options)
else:
display_cli_help()
sys.exit(0)
#
# importer mode
#
if options.get('tool') == "importer":
if options.get('filename'):
from tools.importer import do_importer
do_importer(options)
else:
display_cli_help()
sys.exit(0)
#
# tool mode
#
if options.get('tool'):
ToolClazz = Tool.load_tool(options.get('tool'))
if ToolClazz:
if ToolClazz.need_identity():
if options['identity'].startswith('-'):
Terminal.inst().error(
"First option must be the identity name")
Terminal.inst().flush()
sys.exit(-1)
tool = ToolClazz(options)
if not tool.check_options(options):
display_cli_help()
sys.exit(-1)
if ToolClazz.need_identity():
Terminal.inst().info(
"Starting SIIS %s using %s identity..." %
(options.get('tool'), options['identity']))
else:
Terminal.inst().info("Starting SIIS %s..." %
options.get('tool'))
Terminal.inst().flush()
tool.execute(options)
Terminal.inst().info(
"%s done!" %
(ToolClazz.alias() or options.get('tool')).capitalize())
Terminal.inst().flush()
Terminal.terminate()
sys.exit(0)
else:
sys.exit(-1)
if options['identity'].startswith('-'):
Terminal.inst().error("First option must be the identity name")
#
# normal mode
#
Terminal.inst().info("Starting SIIS using %s identity..." %
options['identity'])
Terminal.inst().action("- type ':q<Enter> or :quit<Enter>' to terminate")
Terminal.inst().action(
"- type ':h<Enter> or :help<Enter>' to display help")
Terminal.inst().flush()
if options.get('backtesting'):
Terminal.inst().notice("Process a backtesting.")
if options.get('paper-mode'):
Terminal.inst().notice("- Using paper-mode trader.")
else:
Terminal.inst().notice("- Using live-mode trader.")
signal.signal(signal.SIGINT, signal_handler)
#
# application
#
watchdog_service = WatchdogService(options)
watchdog_service.start(options)
# application services
view_service = None
notifier_service = None
watcher_service = None
trader_service = None
strategy_service = None
# monitoring service
Terminal.inst().info("Starting monitor service...")
monitor_service = MonitorService(options)
# notifier service
try:
notifier_service = NotifierService(options)
notifier_service.start(options)
except Exception as e:
Terminal.inst().error(str(e))
terminate(watchdog_service, watcher_service, trader_service,
strategy_service, monitor_service, view_service,
notifier_service)
sys.exit(-1)
# view service
try:
view_service = ViewService(options)
watchdog_service.add_service(view_service)
except Exception as e:
Terminal.inst().error(str(e))
terminate(watchdog_service, watcher_service, trader_service,
strategy_service, monitor_service, view_service,
notifier_service)
sys.exit(-1)
# database manager
try:
Database.create(options)
Database.inst().setup(options)
except Exception as e:
Terminal.inst().error(str(e))
terminate(watchdog_service, watcher_service, trader_service,
strategy_service, monitor_service, view_service,
notifier_service)
sys.exit(-1)
# watcher service
Terminal.inst().info("Starting watcher's service...")
try:
watcher_service = WatcherService(options)
watcher_service.start(options)
watchdog_service.add_service(watcher_service)
except Exception as e:
Terminal.inst().error(str(e))
terminate(watchdog_service, watcher_service, trader_service,
strategy_service, monitor_service, view_service,
notifier_service)
sys.exit(-1)
# trader service
Terminal.inst().info("Starting trader's service...")
try:
trader_service = TraderService(watcher_service, monitor_service,
options)
trader_service.start(options)
watchdog_service.add_service(trader_service)
except Exception as e:
Terminal.inst().error(str(e))
terminate(watchdog_service, watcher_service, trader_service,
strategy_service, monitor_service, view_service,
notifier_service)
sys.exit(-1)
# want to display desktop notification and update views
watcher_service.add_listener(view_service)
# want to display desktop notification and update views
trader_service.add_listener(view_service)
# trader service listen to watcher service and update views
watcher_service.add_listener(trader_service)
# strategy service
Terminal.inst().info("Starting strategy's service...")
try:
strategy_service = StrategyService(watcher_service, trader_service,
monitor_service, options)
strategy_service.start(options)
watchdog_service.add_service(strategy_service)
except Exception as e:
Terminal.inst().error(str(e))
terminate(watchdog_service, watcher_service, trader_service,
strategy_service, monitor_service, view_service,
notifier_service)
sys.exit(-1)
# wan't to be notifier of system errors
watchdog_service.add_listener(notifier_service)
# strategy service listen to watcher service
watcher_service.add_listener(strategy_service)
# strategy service listen to trader service
trader_service.add_listener(strategy_service)
# want to display desktop notification, update view and notify on discord
strategy_service.add_listener(notifier_service)
strategy_service.add_listener(view_service)
# want signal and important notifications
notifier_service.set_strategy_service(strategy_service)
notifier_service.set_trader_service(trader_service)
# register terminal commands
commands_handler = CommandsHandler()
commands_handler.init(options)
# cli commands registration
register_general_commands(commands_handler)
register_trading_commands(commands_handler, trader_service,
strategy_service, monitor_service,
notifier_service)
register_region_commands(commands_handler, strategy_service)
# setup and start the monitor service
monitor_service.setup(watcher_service, trader_service, strategy_service)
try:
monitor_service.start()
watchdog_service.add_service(monitor_service)
except Exception as e:
Terminal.inst().error(str(e))
terminate(watchdog_service, watcher_service, trader_service,
strategy_service, monitor_service, view_service,
notifier_service)
sys.exit(-1)
Terminal.inst().message("Running main loop...")
Terminal.inst().upgrade()
Terminal.inst().message("Steady...", view='notice')
if view_service:
# setup the default views
try:
setup_default_views(view_service, watcher_service, trader_service,
strategy_service)
except Exception as e:
Terminal.inst().error(str(e))
terminate(watchdog_service, watcher_service, trader_service,
strategy_service, monitor_service, view_service,
notifier_service)
sys.exit(-1)
display_welcome()
LOOP_SLEEP = 0.016 # in second
MAX_CMD_ALIVE = 15 # in second
running = True
value = None
value_changed = False
command_timeout = 0
prev_timestamp = 0
try:
while running:
# keyboard input commands
try:
c = Terminal.inst().read()
key = Terminal.inst().key()
if c:
# split the commande line
args = [
arg
for arg in (value[1:].split(' ')
if value and value.startswith(':') else [])
if arg
]
if value and value[-1] == ' ':
args.append('')
# update the current type command
commands_handler.process_char(c, args)
if key:
if key == 'KEY_ESCAPE':
# cancel command
value = None
value_changed = True
command_timeout = 0
# split the commande line
args = [
arg
for arg in (value[1:].split(' ')
if value and value.startswith(':') else [])
if arg
]
if value and value[-1] == ' ':
args.append('')
# process on the arguments
args = commands_handler.process_key(
key, args,
Terminal.inst().mode == Terminal.MODE_COMMAND)
if args:
# regen the updated commande ligne
value = ":" + ' '.join(args)
value_changed = True
command_timeout = 0
view_service.on_key_pressed(key)
if key == 'KEY_ESCAPE':
# was in command me, now in default mode
Terminal.inst().set_mode(Terminal.MODE_DEFAULT)
# @todo move the rest to command_handler
if c:
if value and value[0] == ':':
if c == '\b':
# backspace, erase last command char
value = value[:-1] if value else None
value_changed = True
command_timeout = time.time()
elif c != '\n':
# append to the advanced command value
value += c
value_changed = True
command_timeout = time.time()
elif c == '\n':
result = commands_handler.process_cli(value)
command_timeout = 0
if not result:
# maybe an application level command
if value == ':q' or value == ':quit':
running = False
elif value.startswith(':x '):
# manually exit position at market
# @todo move as command
target = value[3:]
if target == "all" or target == "ALL":
Terminal.inst().action(
"Send close to market command for any positions",
view='status')
trader_service.command(
Trader.COMMAND_CLOSE_ALL_MARKET,
{})
else:
Terminal.inst().action(
"Send close to market command for position %s"
% (target, ),
view='status')
trader_service.command(
Trader.COMMAND_CLOSE_MARKET,
{'key': target})
# clear command value
value_changed = True
value = None
# use default mode
Terminal.inst().set_mode(Terminal.MODE_DEFAULT)
elif c != '\n':
# initial command value
value = "" + c
value_changed = True
command_timeout = time.time()
if value and value[0] == ':':
# use command mode
Terminal.inst().set_mode(Terminal.MODE_COMMAND)
if value and value[0] != ':':
# direct key
# use default mode
Terminal.inst().set_mode(Terminal.MODE_DEFAULT)
try:
result = commands_handler.process_accelerator(key)
# @todo convert to Command object accelerator
if not result:
result = True
# display views @todo must be managed by view_service
if value == 'C':
Terminal.inst().clear_content()
elif value == 'D':
Terminal.inst().switch_view('debug')
elif value == 'I':
Terminal.inst().switch_view('content')
elif value == 'F':
Terminal.inst().switch_view('strategy')
elif value == 'S':
Terminal.inst().switch_view('stats')
elif value == 'P':
Terminal.inst().switch_view('perf')
elif value == 'X':
Terminal.inst().switch_view('position')
elif value == 'O':
Terminal.inst().switch_view('order')
elif value == 'T':
Terminal.inst().switch_view('ticker')
elif value == 'A':
Terminal.inst().switch_view('account')
elif value == 'M':
Terminal.inst().switch_view('market')
elif value == 'Q':
Terminal.inst().switch_view('asset')
elif value == 'N':
Terminal.inst().switch_view('signal')
elif value == '?':
# ping services and workers
watchdog_service.ping(1.0)
elif value == ' ':
# a simple mark on the terminal
Terminal.inst().notice(
"Trading time %s" %
(datetime.fromtimestamp(
strategy_service.timestamp).
strftime('%Y-%m-%d %H:%M:%S')),
view='status')
elif value == 'a':
if notifier_service:
notifier_service.command(
Notifier.COMMAND_TOGGLE, {
'notifier': "desktop",
'value': "audible"
})
elif value == 'n':
if notifier_service:
notifier_service.command(
Notifier.COMMAND_TOGGLE, {
'notifier': "desktop",
'value': "popup"
})
elif value == '%':
if view_service:
view_service.toggle_percent()
elif value == ',':
if view_service:
view_service.toggle_group()
elif value == '!':
if view_service:
view_service.toggle_datetime_format()
else:
result = False
if result:
value = None
value_changed = True
command_timeout = 0
except Exception as e:
siis_logger.error(repr(e))
traceback_logger.error(traceback.format_exc())
except IOError:
pass
except Exception as e:
siis_logger.error(repr(e))
traceback_logger.error(traceback.format_exc())
# display advanced command only
if value_changed:
if value and value.startswith(':'):
Terminal.inst().message("Command: %s" % value[1:],
view='command')
else:
Terminal.inst().message("", view='command')
# clear input if no char hit during the last MAX_CMD_ALIVE
if value and not value.startswith(':'):
if (command_timeout > 0) and (time.time() - command_timeout >=
MAX_CMD_ALIVE):
value = None
value_changed = True
Terminal.inst().info("Current typing canceled",
view='status')
try:
# display strategy tarding time (update max once per second)
if strategy_service.timestamp - prev_timestamp >= 1.0:
mode = "live"
if trader_service.backtesting:
mode = "backtesting"
elif trader_service.paper_mode:
mode = "paper-mode"
Terminal.inst().message(
"%s - %s" %
(mode,
datetime.fromtimestamp(strategy_service.timestamp).
strftime('%Y-%m-%d %H:%M:%S')),
view='notice')
prev_timestamp = strategy_service.timestamp
# synchronous operations here
watcher_service.sync()
trader_service.sync()
strategy_service.sync()
if monitor_service:
monitor_service.sync()
if view_service:
view_service.sync()
if notifier_service:
notifier_service.sync()
Terminal.inst().update()
except BaseException as e:
siis_logger.error(repr(e))
traceback_logger.error(traceback.format_exc())
# don't waste CPU time on main thread
time.sleep(LOOP_SLEEP)
finally:
Terminal.inst().restore_term()
Terminal.inst().info("Terminate...")
Terminal.inst().flush()
commands_handler.terminate(options) if commands_handler else None
commands_handler = None
# service terminate
monitor_service.terminate() if monitor_service else None
strategy_service.terminate() if strategy_service else None
trader_service.terminate() if trader_service else None
watcher_service.terminate() if watcher_service else None
view_service.terminate() if view_service else None
notifier_service.terminate() if notifier_service else None
Terminal.inst().info("Saving database...")
Terminal.inst().flush()
Database.terminate()
watchdog_service.terminate() if watchdog_service else None
Terminal.inst().info("Bye!")
Terminal.inst().flush()
Terminal.terminate()
def do_rebuilder(options):
Terminal.inst().info("Starting SIIS rebuilder using %s identity..." % options['identity'])
Terminal.inst().flush()
# database manager
Database.create(options)
Database.inst().setup(options)
timeframe = -1
cascaded = None
if not options.get('timeframe'):
timeframe = 60 # default to 1min
else:
if options['timeframe'] in TIMEFRAME_FROM_STR_MAP:
timeframe = TIMEFRAME_FROM_STR_MAP[options['timeframe']]
else:
try:
timeframe = int(options['timeframe'])
except:
pass
if not options.get('cascaded'):
cascaded = None
else:
if options['cascaded'] in TIMEFRAME_FROM_STR_MAP:
cascaded = TIMEFRAME_FROM_STR_MAP[options['cascaded']]
else:
try:
cascaded = int(options['cascaded'])
except:
pass
if timeframe < 0:
logger.error("Invalid timeframe")
sys.exit(-1)
from_date = options.get('from')
to_date = options.get('to')
if not to_date:
today = datetime.now().astimezone(UTC())
if timeframe == Instrument.TF_MONTH:
to_date = today + timedelta(months=1)
else:
to_date = today + timedelta(seconds=timeframe)
to_date = to_date.replace(microsecond=0)
if timeframe > 0 and timeframe not in GENERATED_TF:
logger.error("Timeframe %i is not allowed !" % (timeframe,))
return
for market in options['market'].split(','):
if market.startswith('!') or market.startswith('*'):
continue
timestamp = from_date.timestamp()
to_timestamp = to_date.timestamp()
progression = 0.0
prev_update = timestamp
count = 0
total_count = 0
progression_incr = (to_timestamp - timestamp) * 0.01
tts = 0.0
prev_tts = 0.0
generators = []
from_tf = timeframe
last_ticks = []
last_ohlcs = {}
if timeframe == Instrument.TF_TICK:
tick_streamer = Database.inst().create_tick_streamer(options['broker'], market, from_date=from_date, to_date=to_date)
else:
ohlc_streamer = Database.inst().create_ohlc_streamer(options['broker'], market, timeframe, from_date=from_date, to_date=to_date)
# cascaded generation of candles
if cascaded:
for tf in GENERATED_TF:
if tf > timeframe:
# from timeframe greater than initial
if tf <= cascaded:
# until max cascaded timeframe
generators.append(CandleGenerator(from_tf, tf))
from_tf = tf
# store for generation
last_ohlcs[tf] = []
else:
from_tf = tf
if timeframe > 0:
last_ohlcs[timeframe] = []
if timeframe == 0:
while not tick_streamer.finished():
ticks = tick_streamer.next(timestamp + Instrument.TF_1M)
count = len(ticks)
total_count += len(ticks)
for data in ticks:
if data[0] > to_timestamp:
break
if generators:
last_ticks.append(data)
# generate higher candles
for generator in generators:
if generator.from_tf == 0:
candles = generator.generate_from_ticks(last_ticks)
if candles:
for c in candles:
store_ohlc(options['broker'], market, generator.to_tf, c)
last_ohlcs[generator.to_tf] += candles
# remove consumed ticks
last_ticks = []
else:
candles = generator.generate_from_candles(last_ohlcs[generator.from_tf])
if candles:
for c in candles:
store_ohlc(options['broker'], market, generator.to_tf, c)
last_ohlcs[generator.to_tf] += candles
# remove consumed candles
last_ohlcs[generator.from_tf] = []
if timestamp - prev_update >= progression_incr:
progression += 1
Terminal.inst().info("%i%% on %s, %s ticks/trades for 1 minute, current total of %s..." % (progression, format_datetime(timestamp), count, total_count))
prev_update = timestamp
count = 0
if timestamp > to_timestamp:
break
timestamp += Instrument.TF_1M # by step of 1m
# calm down the storage of tick, if parsing is faster
while Database.inst().num_pending_ticks_storage() > TICK_STORAGE_DELAY:
time.sleep(TICK_STORAGE_DELAY) # wait a little before continue
elif timeframe > 0:
while not ohlc_streamer.finished():
ohlcs = ohlc_streamer.next(timestamp + timeframe * 100) # per 100
count = len(ohlcs)
total_count += len(ohlcs)
for data in ohlcs:
if data.timestamp > to_timestamp:
break
if generators:
last_ohlcs[timeframe].append(candle)
# generate higher candles
for generator in generators:
candles = generator.generate_from_candles(last_ohlcs[generator.from_tf])
if candles:
for c in candles:
store_ohlc(options['broker'], market, generator.to_tf, c)
last_ohlcs[generator.to_tf].extend(candles)
# remove consumed candles
last_ohlcs[generator.from_tf] = []
prev_tts = tts
timestamp = tts
if timestamp - prev_update >= progression_incr:
progression += 1
Terminal.inst().info("%i%% on %s, %s ticks/trades for 1 minute, current total of %s..." % (progression, format_datetime(timestamp), count, total_count))
prev_update = timestamp
count = 0
if timestamp > to_timestamp:
break
if total_count == 0:
timestamp += timeframe * 100
if progression < 100:
Terminal.inst().info("100%% on %s, %s ticks/trades for 1 minute, current total of %s..." % (format_datetime(timestamp), count, total_count))
Terminal.inst().info("Flushing database...")
Terminal.inst().flush()
Database.terminate()
Terminal.inst().info("Rebuild done!")
Terminal.inst().flush()
Terminal.terminate()
sys.exit(0)
def do_importer(options):
tool = Importer()
Terminal.inst().info("Starting SIIS importer...")
Terminal.inst().flush()
# database manager
Database.create(options)
Database.inst().setup(options)
# want speedup the database inserts
Database.inst().enable_fetch_mode()
filename = options.get('filename')
detected_format = FORMAT_UNDEFINED
detected_timeframe = None
is_mtx_tick = False
pathname = pathlib.Path(filename)
if not pathname.exists():
error_exit(None, "File %s does not exists" % pathname.name)
timeframe = None
if not options.get('timeframe'):
timeframe = None
else:
if options['timeframe'] in TIMEFRAME_FROM_STR_MAP:
timeframe = TIMEFRAME_FROM_STR_MAP[options['timeframe']]
else:
try:
timeframe = int(options['timeframe'])
except:
pass
src = open(filename, "rt")
if filename.endswith(".siis"):
detected_format = FORMAT_SIIS
elif filename.endswith(".csv"):
# detect the format from the first row
row = src.readline().rstrip('\n')
if row.count('\t') > 0:
if row.count(
'\t'
) == 5 and row == "<DATE>\t<TIME>\t<BID>\t<ASK>\t<LAST>\t<VOLUME>":
detected_format = FORMAT_MT5
detected_timeframe = Instrument.TF_TICK
is_mtx_tick = True
elif row.count(
'\t'
) == 8 and row == "<DATE>\t<TIME>\t<OPEN>\t<HIGH>\t<LOW>\t<CLOSE>\t<TICKVOL>\t<VOL>\t<SPREAD>":
detected_format = FORMAT_MT5
is_mtx_tick = False
# from filename try to detect the timeframe
parts = pathname.name.split('_')
if len(parts) >= 2:
detected_timeframe = MT5_TIMEFRAMES.get(parts[1])
# ignore the header line
elif row.count(',') > 0:
if row.count(',') == 4:
detected_format = FORMAT_MT4
detected_timeframe = Instrument.TF_TICK
is_mtx_tick = True
elif row.count(',') == 6:
detected_format = FORMAT_MT4
is_mtx_tick = False
# from filename try to detect the timeframe
parts = pathname.name.split('.')
if len(parts) > 0:
for mt_tf, tf in MT4_TIMEFRAMES.items():
if parts[0].endswith(mt_tf):
detected_timeframe = tf
break
# reset because first row is data
src.seek(0, 0)
if detected_format == FORMAT_UNDEFINED:
error_exit(src, "Unknown file format")
if detected_format in (FORMAT_MT4, FORMAT_MT5):
if detected_timeframe is not None and timeframe is None:
Terminal.inst().message("Auto-detected timeframe %s" %
timeframe_to_str(detected_timeframe))
if detected_timeframe and timeframe and detected_timeframe != timeframe:
error_exit(
src,
"Auto-detected timeframe %s is different of specified timeframe %s"
% (timeframe_to_str(detected_timeframe),
timeframe_to_str(timeframe)))
market_id = ""
broker_id = ""
# UTC option dates
from_date = options.get('from')
to_date = options.get('to')
if detected_format == FORMAT_SIIS:
# first row gives format details
header = src.readline()
if not header.startswith("format=SIIS\t"):
error_exit(src, "Unsupported file format")
info = header.split('\t')
for nfo in info:
k, v = nfo.split('=')
if k == "version":
if v != "1.0.0":
error_exit(src, "Unsupported format version")
elif k == "created":
pass # informational only
elif k == "broker":
broker_id = v
elif k == "market":
market_id = v
elif k == "from":
pass # informational only
elif k == "to":
pass # informational only
elif k == "timeframe":
if v != "any":
timeframe = timeframe_from_str(v)
else:
timeframe = None
else:
# need broker, market and timeframe
broker_id = options.get('broker')
market_id = options.get('market')
if not broker_id:
error_exit(src, "Missing target broker identifier")
if not market_id or ',' in market_id:
error_exit(src, "Missing or invalid target market identifier")
if timeframe is None:
if is_mtx_tick:
timeframe = Instrument.TF_TICK
elif detected_timeframe:
timeframe = detected_timeframe
else:
error_exit(src, "Missing target timeframe")
# limited sub-range
from_date_str = from_date.strftime(
"%Y-%m-%dT%H:%M:%SZ") if from_date else None
to_date_str = to_date.strftime("%Y-%m-%dT%H:%M:%SZ") if to_date else None
total_count = 0
try:
if detected_format == FORMAT_SIIS:
cur_timeframe = None
cur_from_date = from_date
cur_to_date = to_date
while 1:
row = src.readline()
if not row:
break
row = row.rstrip("\n")
if row.startswith("timeframe="):
# specify the timeframe of the next rows
k, v = row.split('=')
cur_timeframe = timeframe_from_str(v)
continue
if cur_timeframe is None:
# need a specified timeframe
continue
if cur_timeframe == Instrument.TF_TICK:
total_count += import_tick_siis_1_0_0(
broker_id, market_id, cur_from_date, cur_to_date, row)
elif cur_timeframe > 0:
total_count += import_ohlc_siis_1_0_0(
broker_id, market_id, cur_timeframe, cur_from_date,
cur_to_date, row)
elif detected_format == FORMAT_MT4:
cur_timeframe = timeframe if not is_mtx_tick else Instrument.TF_TICK
cur_from_date = from_date
cur_to_date = to_date
if cur_timeframe == Instrument.TF_TICK:
while 1:
row = src.readline()
if not row:
break
row = row.rstrip("\n")
total_count += import_tick_mt4(tool, broker_id, market_id,
cur_from_date, cur_to_date,
row)
elif cur_timeframe > 0:
while 1:
row = src.readline()
if not row:
break
row = row.rstrip("\n")
total_count += import_ohlc_mt4(broker_id, market_id,
cur_timeframe,
cur_from_date, cur_to_date,
row)
elif detected_format == FORMAT_MT5:
cur_timeframe = timeframe if not is_mtx_tick else Instrument.TF_TICK
cur_from_date = from_date
cur_to_date = to_date
if cur_timeframe == Instrument.TF_TICK:
while 1:
row = src.readline()
if not row:
break
row = row.rstrip("\n")
total_count += import_tick_mt5(tool, broker_id, market_id,
cur_from_date, cur_to_date,
row)
elif cur_timeframe > 0:
while 1:
row = src.readline()
if not row:
break
row = row.rstrip("\n")
total_count += import_ohlc_mt5(broker_id, market_id,
cur_timeframe,
cur_from_date, cur_to_date,
row)
except Exception as e:
error_logger.error(str(e))
finally:
src.close()
src = None
Terminal.inst().info("Imported %s samples" % (total_count))
Terminal.inst().info("Flushing database...")
Terminal.inst().flush()
Database.terminate()
Terminal.inst().info("Importation done!")
Terminal.inst().flush()
Terminal.terminate()
sys.exit(0)
def do_rebuilder(options):
Terminal.inst().info("Starting SIIS rebuilder using %s identity..." % options['identity'])
Terminal.inst().flush()
# database manager
Database.create(options)
Database.inst().setup(options)
timeframe = -1
cascaded = None
if not options.get('timeframe'):
timeframe = 60 # default to 1min
else:
if options['timeframe'] in TIMEFRAME_FROM_STR_MAP:
timeframe = TIMEFRAME_FROM_STR_MAP[options['timeframe']]
else:
try:
timeframe = int(options['timeframe'])
except:
pass
if not options.get('cascaded'):
cascaded = None
else:
if options['cascaded'] in TIMEFRAME_FROM_STR_MAP:
cascaded = TIMEFRAME_FROM_STR_MAP[options['cascaded']]
else:
try:
cascaded = int(options['cascaded'])
except:
pass
if timeframe < 0:
logger.error("Invalid timeframe")
sys.exit(-1)
from_date = options.get('from')
to_date = options.get('to')
if not to_date:
today = datetime.now().astimezone(UTC())
if timeframe == Instrument.TF_MONTH:
to_date = today + timedelta(months=1)
else:
to_date = today + timedelta(seconds=timeframe)
to_date = to_date.replace(microsecond=0)
timeframe = options['timeframe']
if timeframe > 0 and timeframe not in GENERATED_TF:
logger.error("Timeframe %i is not allowed !" % (timeframe,))
return
for market in options['market'].split(','):
if market.startswith('!') or market.startswith('*'):
continue
generators = []
from_tf = timeframe
last_ticks = []
last_ohlcs = {}
if timeframe == Instrument.TF_TICK:
tick_streamer = Database.inst().create_tick_streamer(options['broker'], market, from_date=from_date, to_date=to_date)
else:
ohlc_streamer = Database.inst().create_ohlc_streamer(options['broker'], market, timeframe, from_date=from_date, to_date=to_date)
# cascaded generation of candles
if cascaded:
for tf in GENERATED_TF:
if tf > timeframe:
# from timeframe greater than initial
if tf <= cascaded:
# until max cascaded timeframe
generators.append(CandleGenerator(from_tf, tf))
from_tf = tf
# store for generation
last_ohlcs[tf] = []
else:
from_tf = tf
if timeframe > 0:
last_ohlcs[timeframe] = []
n = 0
t = 0
timestamp = from_date.timestamp() + Instrument.TF_1M
if timeframe == 0:
while not tick_streamer.finished():
ticks = tick_streamer.next(timestamp)
timestamp += Instrument.TF_1M # by step of 1M
for data in ticks:
if generators:
last_ticks.append((float(data[0]) * 0.001, float(data[1]), float(data[2]), float(data[3])))
# generate higher candles
for generator in generators:
if generator.from_tf == 0:
candles = generator.generate_from_ticks(last_ticks)
if candles:
for c in candles:
store_ohlc(options['broker'], market, generator.to_tf, c)
last_ohlcs[generator.to_tf] += candles
# remove consumed ticks
last_ticks = []
else:
candles = generator.generate_from_candles(last_ohlcs[generator.from_tf])
if candles:
for c in candles:
store_ohlc(options['broker'], market, generator.to_tf, c)
last_ohlcs[generator.to_tf] += candles
# remove consumed candles
last_ohlcs[generator.from_tf] = []
n += 1
t += 1
if n == 1000:
n = 0
Terminal.inst().info("%i..." % t)
Terminal.inst().flush()
# calm down the storage of tick, if parsing is faster
while Database.inst().num_pending_ticks_storage() > TICK_STORAGE_DELAY:
time.sleep(TICK_STORAGE_DELAY) # wait a little before continue
logger.info("Read %i trades" % t)
elif timeframe > 0:
while not ohlc_streamer.finished():
ohlcs = ohlc_streamer.next(timestamp)
timestamp += Instrument.TF_1M # by step of 1M
for data in ohlcs:
if generators:
candle = Candle(float(data[0]) * 0.001, timeframe)
candle.set_bid_ohlc(float(data[1]), float(data[2]), float(data[3]), float(data[4]))
candle.set_ofr_ohlc(float(data[5]), float(data[6]), float(data[7]), float(data[8]))
candle.set_volume(float(data[9]))
candle.set_consolidated(True)
last_ohlcs[timeframe].append(candle)
# generate higher candles
for generator in generators:
candles = generator.generate_from_candles(last_ohlcs[generator.from_tf])
if candles:
for c in candles:
store_ohlc(options['broker'], market, generator.to_tf, c)
last_ohlcs[generator.to_tf].extend(candles)
# remove consumed candles
last_ohlcs[generator.from_tf] = []
n += 1
t += 1
if n == 1000:
n = 0
Terminal.inst().info("%i..." % t)
logger.info("Read %i candles" % t)
Terminal.inst().info("Flushing database...")
Terminal.inst().flush()
Database.terminate()
Terminal.inst().info("Rebuild done!")
Terminal.inst().flush()
Terminal.terminate()
sys.exit(0)
def do_exporter(options):
Terminal.inst().info("Starting SIIS exporter...")
Terminal.inst().flush()
# database manager
Database.create(options)
Database.inst().setup(options)
broker_id = options['broker']
market_id = options['market']
timeframe = None
# UTC option dates
from_date = options.get('from')
to_date = options.get('to')
if not to_date:
today = datetime.now().astimezone(UTC())
if timeframe == Instrument.TF_MONTH:
to_date = today + timedelta(months=1)
else:
to_date = today + timedelta(seconds=timeframe or 1.0)
to_date = to_date.replace(microsecond=0)
if not options.get('timeframe'):
timeframe = None
else:
if options['timeframe'] in TIMEFRAME_FROM_STR_MAP:
timeframe = TIMEFRAME_FROM_STR_MAP[options['timeframe']]
else:
try:
timeframe = int(options['timeframe'])
except:
pass
filename = options.get('filename')
cur_datetime = datetime.now().astimezone(
UTC()).strftime("%Y-%m-%dT%H:%M:%SZ")
from_date_str = from_date.strftime("%Y-%m-%dT%H:%M:%SZ")
to_date_str = to_date.strftime("%Y-%m-%dT%H:%M:%SZ")
try:
# exporting data...
if timeframe is None:
for market in options['market'].split(','):
if market.startswith('!') or market.startswith('*'):
continue
dst = open("%s-%s-%s-any.siis" % (filename, broker_id, market),
"wt")
# write file header
dst.write(
"format=SIIS\tversion=%s\tcreated=%s\tbroker=%s\tmarket=%s\tfrom=%s\tto=%s\ttimeframe=any\n"
% (EXPORT_VERSION, cur_datetime, broker_id, market,
from_date_str, to_date_str))
for tf in EXPORT_TF:
Terminal.inst().info("Exporting %s OHLC %s..." %
(market, timeframe_to_str(tf)))
dst.write("timeframe=%s\n" % timeframe_to_str(tf))
export_ohlcs_siis_1_0_0(options['broker'], market, tf,
from_date, to_date, dst)
dst.close()
dst = None
elif timeframe == Instrument.TF_TICK:
for market in options['market'].split(','):
if market.startswith('!') or market.startswith('*'):
continue
dst = open("%s-%s-%s-t.siis" % (filename, broker_id, market),
"wt")
# write file header
dst.write(
"format=SIIS\tversion=%s\tcreated=%s\tbroker=%s\tmarket=%s\tfrom=%s\tto=%s\ttimeframe=t\n"
% (EXPORT_VERSION, cur_datetime, broker_id, market,
from_date_str, to_date_str))
Terminal.inst().info("Exporting %s ticks/trades..." %
(market, ))
dst.write("timeframe=t\n")
export_ticks_siis_1_0_0(options['broker'], market, from_date,
to_date, dst)
dst.close()
dst = None
elif timeframe > 0:
# particular ohlc
for market in options['market'].split(','):
if market.startswith('!') or market.startswith('*'):
continue
dst = open(
"%s-%s-%s-%s.siis" %
(filename, broker_id, market, timeframe_to_str(timeframe)),
"wt")
# write file header
dst.write(
"format=SIIS\tversion=%s\tcreated=%s\tbroker=%s\tmarket=%s\tfrom=%s\tto=%s\ttimeframe=%s\n"
%
(EXPORT_VERSION, cur_datetime, broker_id, market,
from_date_str, to_date_str, timeframe_to_str(timeframe)))
Terminal.inst().info("Exporting %s OHLC %s..." %
(market, timeframe_to_str(timeframe)))
dst.write("timeframe=%s\n" % timeframe_to_str(timeframe))
export_ohlcs_siis_1_0_0(options['broker'], market, timeframe,
from_date, to_date, dst)
dst.close()
dst = None
except KeyboardInterrupt:
pass
except Exception as e:
error_logger.error(str(e))
dst.close()
dst = None
finally:
pass
Database.terminate()
Terminal.inst().info("Exportation done!")
Terminal.inst().flush()
Terminal.terminate()
sys.exit(0)
def application(argv):
fix_thread_set_name()
options = {
'identity': 'real',
'config-path': './user/config',
'connectors-path': './user/config/connectors',
'log-path': './user/log',
'reports-path': './user/reports',
'markets-path': './user/markets',
'log-name': 'connector.log',
'default': "connector.json",
'connector-config': "",
'database': {
'name': "siis",
'type': "pgsql",
'host': "127.0.0.1",
'port': 5432,
'user': "******",
'password': "******"
},
'cache': {
'name': "siis",
'type': "redis",
'host': "127.0.0.1",
'port': 6379,
'user': "******",
'password': "******"
},
'strategy': {
'protocol': "tcp",
'host': "127.0.0.1",
'port': 5600
},
'connector': {
'name': "",
'host': ""
},
'markets': {},
'connectors': {
'binance.com': {
'classpath': 'connectors.binance.connector.BinanceConnector',
},
'bitmex.com': {
'classpath': 'connectors.bitmex.connector.BitMexConnector',
},
'ig.com': {
'classpath': 'connectors.ig.connector.IGConnector',
},
}
}
# create initial siis data structure if necessary
install(options)
siis_log = SiisLog(options, "uterm")
siis_logger = logging.getLogger('siis.connector')
if len(argv) > 1:
# utc or local datetime ?
for n, arg in enumerate(argv):
if arg.startswith('-'):
if (arg == '--fetch' or arg == '-F'):
# use the fetcher
options['fetch'] = True
elif (arg == '--spec' or arg == '-S') and n + 1 < len(argv):
# fetcher data history option
options['option'] = argv[n + 1]
elif arg == '--binarize':
# use the binarizer
options['binarize'] = True
elif arg == '--sync':
# use the syncer
options['sync'] = True
elif (arg == '--connector'
or arg == '-c') and n + 1 < len(argv):
# connector conf filename
options['connector-config'] = argv[n + 1]
elif (arg == '--from' or arg == '-f') and n + 1 < len(argv):
# if backtest from date (if ommited use whoole data) date format is "yyyy-mm-dd-hh:mm:ss", fetch, binarize to date
options['from'] = datetime.datetime.strptime(
argv[n + 1], '%Y-%m-%dT%H:%M:%S').replace(tzinfo=UTC())
elif (arg == '--to' or arg == '-t') and n + 1 < len(argv):
# if backtest to date (can be ommited), fetch, binarize to date
options['to'] = datetime.datetime.strptime(
argv[n + 1], '%Y-%m-%dT%H:%M:%S').replace(tzinfo=UTC())
elif (arg == '--last' or arg == '-l') and n + 1 < len(argv):
# fetch the last n data history
options['last'] = int(argv[n + 1])
elif (arg == '--market' or arg == '-m') and n + 1 < len(argv):
# fetch, binarize the data history for this market
options['market'] = argv[n + 1]
elif (arg == '--timeframe'
or arg == '-s') and n + 1 < len(argv):
# fetch, binarize base timeframe
options['timeframe'] = argv[n + 1]
elif (arg == '--cascaded='
or arg == '-C') and n + 1 < len(argv):
# fetch cascaded ohlc generation
options['cascaded'] = argv[n + 1]
elif arg == '--read-only':
options['read-only'] = True
elif arg == '--check-data':
options['check-data'] = True
elif arg == '--version' or arg == '-v':
print('%s %s' % (APP_SHORT_NAME, '.'.join(
[str(x) for x in APP_VERSION])))
sys.exit(0)
elif arg == '--help' or arg == '-h':
display_cmd_line_help()
sys.exit(0)
# replay
if options.get('replay', False):
if options.get('from') is None or options.get('to') is None:
del options['replay']
print("Replay need from= and to= date time")
sys.exit(-1)
if not options['connector-config']:
print("Connector configuration filename must be specified")
sys.exit(-1)
#
# config
#
parse_config(siis_logger, options)
parse_connector_spec(siis_logger, options)
siis_log.upgrade(options)
#
# binarizer
#
if options.get('binarize'):
if options.get('market') and options.get('from') and options.get(
'to') and options.get('connector'):
do_binarizer(options, siis_logger)
else:
display_cmd_line_help()
sys.exit(0)
#
# fetcher mode
#
if options.get('fetch'):
if options.get('market') and options.get('connector') and options.get(
'timeframe'):
do_fetcher(options, siis_logger)
else:
display_cmd_line_help()
sys.exit(0)
#
# running mode
#
print("Starting SIIS simple runner using %s..." %
options['connector-config'])
print("Hit CTRL-C twice to terminate")
if options.get('replay'):
print("Process a replay.")
# monitoring service
# print("Starting monitor service...")
# monitor_service = MonitorService(options)
# monitor_service.start()
# database manager
Database.create(options)
Database.inst().setup(options)
print("Starting connector handler...")
handler = DefaultHandler(options)
handler.init(options)
handler.start()
run(siis_logger, handler)
print("Terminate...")
handler.stop()
handler.terminate()
handler = None
print("Saving database...")
Database.terminate()
print("Bye!")
