def run_loop(self):
for key in self.get_registered().items():
if key not in self.pools:
pool = serviceclient.get_pool(*key)
if pool:
self.pools[key] = pool
for key in list(self.pools.keys()):
if key[0] not in self.registered:
del self.pools[key]
def extend_prices(_pool: Pool):
with _pool.connection() as client:
l = client.get_prices() or []
for x in l:
x['time'] = now
prices.extend(l)
now = timeutil.jst_now()
prices = []
spawns = {key: gevent.spawn(lambda _pool: extend_prices(_pool), pool)
for key, pool in random.sample(self.pools.items(), len(self.pools))}
gevent.joinall(list(spawns.values()), timeout=1)
is_timeout = False
for key, spawn in spawns.items():
if not spawn.successful():
is_timeout = True
try:
del self.pools[key]
except KeyError:
pass
try:
del self.registered[key[0]]
except KeyError:
pass
for price in prices:
if price['instrument'] not in self.instruments:
self.instruments.add(price['instrument'])
elapsed = timeutil.jst_now() - now
self.request_elapsed_list.append(elapsed)
self.window.update_prices(prices)
elapsed = timeutil.jst_now() - now
self.elapsed_list.append(elapsed)
N = 10
if len(self.elapsed_list) >= N:
total = sum(map(lambda x: x.total_seconds(), self.elapsed_list))
print('# elapsed total:{}, N:{}, one:{}'.format(total, N, total / N))
total = sum(map(lambda x: x.total_seconds(), self.request_elapsed_list))
print('# request total:{}, N:{}, one:{}'.format(total, N, total / N))
self.elapsed_list = []
self.request_elapsed_list = []
if is_timeout:
gevent.sleep(3)
print('#timeout sleep(3)')
else:
gevent.sleep(max((timedelta(milliseconds=500) - elapsed).total_seconds(), 0.1))
def get_prices(self):
now = timeutil.jst_now()
def extend_prices(_pool: Pool):
with _pool.connection() as client:
l = client.get_prices() or []
for x in l:
x['time'] = now
prices.extend(l)
prices = []
spawns = {
key: gevent.spawn(lambda _pool: extend_prices(_pool), pool)
for key, pool in random.sample(list(self.pools.items()),
len(self.pools))
}
gevent.joinall(list(spawns.values()), timeout=1)
is_timeout = False
for key, spawn in spawns.items():
if not spawn.successful():
is_timeout = True
try:
del self.pools[key]
except KeyError:
pass
try:
del self.registered[key[0]]
except KeyError:
pass
elapsed = timeutil.jst_now() - now
self.request_elapsed_list.append(elapsed)
if prices:
GlobalSignal().update_prices.emit(prices)
elapsed = timeutil.jst_now() - now
self.elapsed_list.append(elapsed)
N = 10
if elapsed > timedelta() and len(self.elapsed_list) >= N:
request_total = sum(
map(lambda x: x.total_seconds(), self.request_elapsed_list))
total = sum(map(lambda x: x.total_seconds(), self.elapsed_list))
self.logger.info(
'# elapsed:{:.3f}, request:{:.3f}({:.2f}%), 1loop: {:.3f}, request:{:.3f}'
.format(total, request_total, request_total / total * 100,
total / N, request_total / N))
self.elapsed_list = []
self.request_elapsed_list = []
if is_timeout:
self.logger.warn('#timeout sleep(3)')
gevent.sleep(3)
else:
gevent.sleep(
max((timedelta(seconds=Config()['interval']) -
elapsed).total_seconds(), 0.05))
def __init__(self, instrument: str, bidder: str, asker: str, *, name: str):
super().__init__(name=name)
self.key = (instrument, bidder, asker)
self.instrument = instrument
self.bidder = bidder
self.asker = asker
self.history = []
self.last_sp = 0 # type: Union[int, float]
self.last_received = timeutil.jst_now()
self.last_signaled = timeutil.jst_now()
def on_prices(self):
for conn in self.driver.connections():
gevent.sleep(0)
if conn.url != self.PRICE_URL:
continue
html = conn.get_html(css_selector='#rateListPanel')
if not html:
return
dom = lxml.html.fromstring(html)
prices = []
now = timeutil.jst_now()
for tr in dom.cssselect('tr.rateList-tbody'):
text = tr.cssselect('.product-pulldown-name')[0].text_content()
text += tr.cssselect('.rateList-tbody-bid')[0].text_content()
text += tr.cssselect('.rateList-tbody-ask')[0].text_content()
text = re.sub('\s+', ' ', text)
m = re.search(
'(?P<instrument>\w+/\w+) (?P<bid>[.\d]+) (?P<ask>[.\d]+)',
text)
if m:
d = m.groupdict()
prices += [
dict(service=self.name,
instrument=d['instrument'],
time=now,
bid=float(d['bid']),
ask=float(d['ask']))
]
gevent.sleep(0.0)
return self.update_prices(prices)
def update_prices():
services = [
'click', 'gaitame', 'lion', 'pfx', 'nano', 'sbi', 'try', 'yjfx',
'noaccount'
]
instruments = ('USD/JPY', 'EUR/JPY', 'GBP/JPY', 'AUD/JPY')
prices = []
accounts = []
for service in services:
positions = {}
for i, instrument in enumerate(instruments):
bid = 101 + i + random.random() * 0.8 / 100
ask = 100 + i + random.random() * 0.2 / 100
prices.append(
dict(time=timeutil.jst_now(),
service=service,
instrument=instrument,
bid=bid,
ask=ask))
if services.index(service) < len(services) / 2:
positions[instrument] = random.randint(0, 3) * 1000
else:
positions[instrument] = random.randint(0, 3) * -1000
if service != 'noaccount':
accounts += [new_account(service, positions=positions)]
GlobalSignal().update_prices.emit(prices)
GlobalSignal().update_accounts.emit(accounts)
def on_prices(self, conn: CustomConnection):
if not conn.url.startswith(self.PRICE_URL):
return
html = conn.get_html(
css_selector='#workspaceContainer .rateBoxContainerPanel')
if not html:
return
dom = lxml.html.fromstring(html)
prices = []
now = timeutil.jst_now()
for dom in dom.cssselect('.rateBoxPanel'):
instrument = re.sub(
'\s+', '',
dom.cssselect('div[uifield="currencyPairSelect"] span')
[0].text_content())
bid = re.sub(
'\s+', '',
dom.cssselect('.pricePanel.bid .price')[0].text_content())
ask = re.sub(
'\s+', '',
dom.cssselect('.pricePanel.ask .price')[0].text_content())
prices.append(
dict(service=self.name,
instrument=instrument,
time=now,
bid=float(bid),
ask=float(ask)))
self.update_prices(prices)
def _extract_prices(self, service: str, conn: CustomConnection):
key = ('id', 'rate')
node_id = conn.frame_ids.get(key)
if not node_id:
self.logger.warn('#no key={}'.format(key))
return
html = conn.get_html(css_selector='#PriceList', node_id=node_id)
if not html:
self.logger.warn('#no #PriceList')
return
bid_str = '#bidCurrencyPrice{}'
ask_str = '#askCurrencyPrice{}'
dom = lxml.html.fromstring(html)
prices = []
now = timeutil.jst_now()
for i, instrument in enumerate(self.INSTRUMENTS[service]):
i += 1
bid = float(dom.cssselect(bid_str.format(i))[0].text_content())
ask = float(dom.cssselect(ask_str.format(i))[0].text_content())
prices.append(
dict(service=service,
instrument=instrument,
time=now,
bid=bid,
ask=ask))
return prices
def run(self):
while not self.stopped:
gevent.sleep(0.01)
now = timeutil.jst_now()
with self.lock:
for name in list(self.registered.keys()):
try:
last_checked = self.last_checked[name]
if now - last_checked >= timedelta(seconds=3.0):
try:
del self.registered[name]
except KeyError:
pass
self.logger.warn(
'timeouted {} unregistered'.format(name))
try:
del self.last_checked[name]
except KeyError:
pass
except KeyError:
pass
try:
self.run_loop()
except KeyboardInterrupt:
raise
except Exception as e:
self.logger.exception(str(e))
def run_publish(self):
while not self.stopped:
try:
start_time = time.time()
now = timeutil.jst_now()
accounts = {}
prices = {}
for service, account in self.accounts.items():
account = copy.deepcopy(account)
# account.update(time=str(account['time']))
accounts[service] = account
for service, instrument_price in self.prices.items():
for instrument, price in instrument_price.items():
if now - price['time'] < timedelta(seconds=1):
price = copy.deepcopy(price)
# price.update(time=str(price['time']))
prices.setdefault(service, {})[instrument] = price
data = dict(accounts=accounts, prices=prices)
gevent.sleep(0.0)
for key, pool in list(self.subscriber_pools.items()):
try:
with pool.connection() as client:
client.put_data(data)
except Exception as e:
self.logger.warning(
'disconnected {}. exception:{}'.format(
key, str(e)))
del self.subscriber_pools[key]
gevent.sleep(0.0)
except Exception as e:
self.logger.exception(str(e))
gevent.sleep(1.0)
else:
elapsed = time.time() - start_time
gevent.sleep(max(self.interval - elapsed, self.interval / 10))
def on_prices(self, content: str):
# rate
"""
,
20
USDJPY,112.643,112.646,112.485,112.885,112.405,0.100,112.543,,0
EURJPY,118.840,118.846,118.671,118.890,118.506,0.108,118.732,,0
"""
if not content:
return
prices = []
now = timeutil.jst_now()
for row in csv.reader(StringIO(content)):
if len(row) < 3:
continue
instrument = row[0][:3] + '/' + row[0][-3:]
bid = float(row[1])
ask = float(row[2])
prices.append(
dict(service=self.name,
instrument=instrument,
time=now,
bid=bid,
ask=ask))
self.update_prices(prices)
def save_train_data(self, f_name):
with open(f_name, 'wb') as f:
pickle.dump(self.train_data, f)
self.logger.info('train_data saved to {}'.format(f_name))
f_name += timeutil.jst_now().strftime('.%Y%m%dT%H%M%S')
with open(f_name, 'wb') as f:
pickle.dump(self.train_data, f)
self.logger.info('train_data saved to {}'.format(f_name))
def save_train_data(self, f_name=None):
f_name = os.path.join(self.dir_path, f_name or 'train_data.pickle')
with open(f_name, 'wb') as f:
pickle.dump(self.train_data, f)
self.logger.info('train_data saved to {}'.format(f_name))
f_name += timeutil.jst_now().strftime('.%Y%m%dT%H%M%S')
with open(f_name, 'wb') as f:
pickle.dump(self.train_data, f)
self.logger.info('train_data saved to {}'.format(f_name))
def register(self, name: str, client_address: Tuple[str, int]):
with self.lock:
try:
address = self.registered[name]
if address == client_address:
self.logger.debug('{}({}) health-check'.format(
name, address))
self.last_checked[name] = timeutil.jst_now()
return True
else:
self.logger.info('{}: changed address {} -> {}'.format(
name, address, client_address))
except KeyError:
pass
self.registered[name] = client_address
self.last_checked[name] = timeutil.jst_now()
self.logger.info('{}({}) registered'.format(name, client_address))
return True
def on_prices(self, content: str):
prices = []
now = timeutil.jst_now()
for k, v in json.loads(content)['rateMap'].items():
instrument = '{}/{}'.format(k[:3], k[-3:])
bid, ask = float(v['bid']), float(v['ask'])
d = dict(service=self.name, instrument=instrument, time=now, bid=bid, ask=ask)
prices += [d]
self.update_prices(prices)
def get_prices(self) -> List[dict]:
now = timeutil.jst_now()
prices = []
for price in self.prices:
if (now - price['time']).total_seconds() < self.PRICE_TIMEOUT:
price = copy.deepcopy(price)
price['time'] = str(price['time'])
prices.append(price)
return prices
def save(self, f_name=None, mode='RGB'):
if self.orig_img:
if not f_name:
dir_name = self.dir_name()
os.makedirs(dir_name, exist_ok=True)
f_name = os.path.join(
'.', dir_name,
timeutil.jst_now().strftime('%Y%m%dT%H%M%S.png'))
self.orig_img.convert(mode=mode).save(f_name)
return self
def test_mysqlutil():
conn = connect()
with conn as c:
try:
c.execute('CREATE DATABASE testdb')
c.execute('USE testdb')
c.execute('CREATE TABLE t(time DATETIME(6) PRIMARY KEY)')
dt = timeutil.jst_now()
c.execute('INSERT INTO t SET time=%s', dt)
c.execute('SELECT * FROM t')
print(c.fetchall())
print('#', dt)
finally:
c.execute('DROP DATABASE testdb')
def main():
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s|%(name)s|%(levelname)s| %(message)s')
args = docopt("""
Usage:
{f} [options]
Options:
--log-file FILE [default: ./pyapp.csv]
--disabled SERVICES [default: pfx]
--from DATETIME
--to DATETIME
""".format(f=sys.argv[0]))
log_file = args['--log-file']
disabled = set(args['--disabled'].split(','))
from_dt = timeutil.to_datetime(
args['--from']) if args['--from'] else (timeutil.jst_now() -
timedelta(days=1))
to_dt = timeutil.to_datetime(
args['--to']) if args['--to'] else timeutil.jst_now()
signals = [] # type: List[dict]
# time,bidder,asker,bid,ask,sp
with open(log_file, 'r') as f:
labels = ('time', 'instrument', 'bidder', 'asker', 'bid', 'ask', 'sp')
dt = timeutil.to_datetime('20161022T050000+0900')
for i, row in enumerate(csv.reader(f), 1):
try:
d = dict(zip(labels, row))
d['time'] = timeutil.to_datetime(d['time'])
if d['time'] >= dt:
continue
print(','.join(map(str, [d[k] for k in labels])))
except Exception as e:
logging.exception('#{}\n{}'.format(i, str(e)))
raise
def __init__(self, *, config: Config):
config = config.setdefault('signaler', dict(
timeout=1.0,
duration=5,
sp_threshold_ratio=0.4,
))
self.config = config
self.last_received = timeutil.jst_now()
self.history = []
self.last_sp = 0 # type: Union[int, float]
self.initialized = False
self.key = None
self.key_tuple = None
self.duration = timedelta()
self.last_signaled = datetime.min
def save_model(self, f_name=None):
f_name = os.path.join(self.dir_path, f_name or 'model')
def save(model, f):
try:
chainer.serializers.save_npz(f, model)
self.logger.info('model saved to {}'.format(f))
except FileNotFoundError:
self.logger.warn('!!!model not loaded from {}'.format(f))
save(self.char_recognizer, f_name + '.char.npz')
save(self.width_recognizer, f_name + '.width.npz')
timestamp = timeutil.jst_now().strftime('%Y%m%dT%H%M%S')
save(self.char_recognizer, f_name + '.char.npz' + '.' + timestamp)
save(self.width_recognizer, f_name + '.width.npz' + '.' + timestamp)
def put_prices(self, price_list: List[dict]):
now = timeutil.jst_now()
for price in price_list:
price.update(time=now)
self.prices[price['service']][price['instrument']] = price
gevent.sleep(0.0)
def main():
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s|%(name)s|%(levelname)s| %(message)s')
args = docopt("""
Usage:
{f} [options] NAME
Options:
--master IP_PORT [default: 127.0.0.1:10000]
--bind IP_PORT [default: 127.0.0.1:0]
--make-train-data N [default: 0]
--train EPOCH [default: 0]
--snap N [default: 0]
--snap-interval SEC [default: 1]
--chrome
--title TITLE_RE
""".format(f=sys.argv[0]))
name = args['NAME']
name = 'gaitame'
l = args['--master'].split(':')
master_address = (l[0], int(l[1]))
l = args['--bind'].split(':')
bind_address = (l[0], int(l[1]))
train_data_n = int(args['--make-train-data'])
epoch = int(args['--train'])
snap_n = int(args['--snap'])
snap_interval = float(args['--snap-interval'])
chrome = args['--chrome']
if args['--title']:
title_re = args['--title']
for i in range(snap_n):
for handle in CaptureServer.iter_windows(title_re):
title = CaptureServer.get_window_title(handle)
save_dir = os.path.join(env.get_desktop_dir(), 'market_snap',
name, title)
os.makedirs(save_dir, exist_ok=True)
img = CaptureServer.capture_window(handle)
f_name = os.path.join(
save_dir, '{}.png'.format(
timeutil.jst_now().strftime('%Y%m%dT%H%M%S')))
img.save(f_name)
print('#{}/{} {}'.format(i + 1, snap_n, f_name))
time.sleep(snap_interval)
return
server = GaitameServer(name,
bind_address,
master_address=master_address,
chrome=chrome,
**server_options(name, chrome=chrome))
if snap_n > 0:
for i in range(snap_n):
img = server.capture()
f = os.path.join(server.dir_path,
timeutil.jst_now().strftime('%Y%m%dT%H%M%S.png'))
img.save(f)
print('#SNAP {}/{} file:{}'.format(i + 1, snap_n, f))
time.sleep(snap_interval)
return
if not epoch and train_data_n <= 0:
server.load_model()
server.start()
return
server.load_model()
server.load_train_data()
if train_data_n:
server.make_train_data(server.gen_image(crop=True, n=train_data_n))
server.save_train_data()
if epoch:
server.train(epoch=epoch)
server.save_model()
def update():
try:
from capwin import get_services
nonlocal services
services = get_services()
# services = dict(filter(lambda x: x[0] == 'nano' or x[0] == 'mnp' or x[0] == 'try', services.items()))
for s in services.values():
s.find_window()
for service in services.values():
service.load_model()
if int(train):
for service in services.values():
service.train(epoch=int(train))
for service in services.values():
service.save_model()
nonlocal launched
launched = True
last = timeutil.jst_now()
db_last = timeutil.jst_now()
conn = sqlite3.connect(db)
# conn.execute("""CREATE TABLE IF NOT EXISTS pyapp
# (time DATETIME, name TEXT, instrument TEXT, bid FLOAT, ask FLOAT,
# PRIMARY KEY (time, name, instrument))""")
cur = conn.cursor()
tables = set()
def create_table(name: str, instrument: str):
table_name = '{}_{}'.format(name, instrument.replace('/', ''))
if table_name not in tables:
cur.execute("""CREATE TABLE IF NOT EXISTS {}
(time DATETIME PRIMARY KEY, bid FLOAT, ask FLOAT)""".format(table_name))
tables.add(table_name)
return table_name
def insert(_time: datetime, _all_results: dict):
for name, results in _all_results.items():
for instrument, d in results.items():
table_name = create_table(name, instrument)
cur.execute('INSERT INTO {} VALUES(?,?,?)'.format(table_name),
(_time, d['bid'], d['ask']))
while True:
try:
time.sleep(0.02)
all_results = update_main(services)
now = timeutil.jst_now()
if notifier:
x_results = dict(filter(lambda x: x[0] not in excepted_rank, all_results.items()))
notifier.update(now, x_results)
if os.name == 'posix':
time.sleep(1)
continue
if (now - last).total_seconds() >= 600:
for s in services.values():
s.save()
last = timeutil.jst_now()
insert(now, all_results)
if (now - db_last).total_seconds() >= 5:
conn.commit()
db_last = timeutil.jst_now()
except Exception as e:
logging.exception('{}'.format(str(e)))
time.sleep(1)
except Exception as e:
logging.exception('{}'.format(str(e)))
finally:
nonlocal stop
stop = True
def __init__(self, name, nodes):
super().__init__(name, bind_address=nodes[name], nodes=nodes)
self.config = Config()
self.window = AppWindow(name='appwindow')
GlobalSignal().load_config.emit()
GlobalSignal().call_lazy()
self.window.show()
self.pools = {} # type: Dict[str, Pool]
self.elapsed_list = []
self.request_elapsed_list = []
self.do_reload = False
self.do_refresh = False
def on_reload():
self.do_reload = True
def on_refresh():
def call():
self.info('refresh start')
for name, address in self._nodes.items():
if name == self.name:
continue
try:
self.info('refresh {}{}'.format(name, address))
self.rpc(address).refresh()
except Exception as e:
self.exception('refresh {}{} {}'.format(
name, address, str(e)))
threading.Thread(target=call, daemon=True).start()
self.do_refresh = True
GlobalSignal().reload.connect(on_reload)
GlobalSignal().refresh.connect(on_refresh)
import time
self._last_tick = time.time()
self._accounts = {}
self._setsock = False
self.run_notify_nodes = lambda: None
def write_csv(file_name: str, flush_interval: float):
now = datetime.now().date()
with open(file_name + now.strftime('_%Y%m%d'), 'a') as f:
w = csv.writer(f)
flush_at = time.time()
while True:
d = yield
if d:
keys = [
'time', 'instrument', 'bidder', 'asker', 'bid',
'ask', 'sp'
]
d = dict(**d)
instrument = d['instrument']
d['time'] = d['time'].astimezone(timeutil.TOKYO)
d['bid'] = get_float_format(instrument).format(
d['bid'])
d['ask'] = get_float_format(instrument).format(
d['ask'])
if isinstance(d['sp'], float):
d['sp'] = '{:.2f}'.format(d['sp'])
values = [d[k] for k in keys]
w.writerow(values)
if time.time() - flush_at >= flush_interval:
f.flush()
flush_at = time.time()
self.csv_writer = write_csv('./spread_log.csv', 10.0)
self.csv_writer.send(None)
self.last_spread_at = timeutil.jst_now()
def exit_at():
now = timeutil.jst_now()
if now.hour == exit_hour and (exit_minute <= now.minute):
sys.exit(0)
def main():
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s|%(name)s|%(levelname)s| %(message)s')
args = docopt("""
Usage:
{f} [options]
Options:
--log-file FILE [default: ./signal_log.txt]
--disabled SERVICES [default: pfx]
--from DATETIME
--to DATETIME
--rank RANK [default: 20]
""".format(f=sys.argv[0]))
log_file = args['--log-file']
disabled = set(args['--disabled'].split(','))
from_dt = timeutil.to_datetime(
args['--from']) if args['--from'] else (timeutil.jst_now() -
timedelta(days=1))
to_dt = timeutil.to_datetime(
args['--to']) if args['--to'] else timeutil.jst_now()
rank = int(args['--rank'])
signals = [] # type: List[dict]
# time,bidder,asker,bid,ask,sp
with open(log_file, 'r') as f:
labels = ['time', 'instrument', 'bidder', 'asker', 'bid', 'ask', 'sp']
for i, row in enumerate(csv.reader(f)):
try:
d = dict(zip(labels, row))
d['time'] = timeutil.to_datetime(d['time'])
d['bid'] = float(d['bid'])
d['ask'] = float(d['ask'])
d['sp'] = float(d['sp'])
d['key'] = (d['instrument'], min(d['bidder'], d['asker']),
max(d['bidder'], d['asker']))
d['direction'] = (d['bidder'], d['asker'])
if d['sp'] > 2:
continue
if d['bidder'] in disabled or d['asker'] in disabled:
continue
if not (from_dt <= d['time'] < to_dt):
continue
signals.append(d)
except Exception as e:
logging.exception('#{} {}'.format(i + 1, str(e)))
raise
records = defaultdict(lambda: dict(direction=
(), profit=0, n=0, history=[]))
def pip_scale(instrument: str):
if 'JPY' in instrument.upper():
return 100
return 10000
for signal in signals:
record = records[signal['key']]
record['key'] = signal['key']
if record['direction'] != signal['direction']:
if record['profit'] > 0:
record['profit'] += signal[
'sp'] * 2 # / (signal['bid'] * pip_scale(signal['instrument']) / 10000)
else:
record['profit'] += signal[
'sp'] # / (signal['bid'] * pip_scale(signal['instrument']) / 10000)
record['direction'] = signal['direction']
record['n'] += 1
# record['history'].append(signal)
# records.sort(key=lambda x: str(x['key']))
l = list(records.values())
l.sort(key=lambda x: -x['profit'])
pprint(l[:rank])
l = list(records.values())
l.sort(key=lambda x: -x['n'])
pprint(l[:rank])
def judge(self, instrument: str, bidder: dict, asker: dict, duration_threshold: int = None) -> Optional[dict]:
b, a = bidder['service'], asker['service']
self.init(instrument, b, a)
next_received = max(bidder['time'], asker['time'])
assert bidder['instrument'] == asker['instrument']
if int((bidder['time'] - asker['time']).total_seconds()) >= 1.0:
self.history.clear()
return
try:
middle = sum([bidder['bid'], asker['ask']]) / 2
sp_threshold = middle * self.config['sp_threshold_ratio'] / 10000 * get_pip_scale(instrument)
timeout = timedelta(seconds=self.config['timeout'])
if next_received - self.last_received >= timeout:
self.history.clear()
return
pip_scale = get_pip_scale(instrument)
sp = (bidder['bid'] - asker['ask']) * pip_scale
d = copy.deepcopy(bidder)
d.update(direction=(b, a), bidder=b, asker=a, ask=asker['ask'], sp=sp, time=next_received,
bid_data=bidder, ask_data=asker)
if sp < sp_threshold:
self.history.clear()
if sp > 0:
return d, None
return
self.history.append(d)
duration_threshold = duration_threshold or timedelta(seconds=self.config['duration'])
duration = next_received - self.history[0]['time']
if duration >= duration_threshold:
sp_list = list(map(lambda x: x['sp'], self.history))
sp_min = min(sp_list)
sp_stdev = statistics.stdev(sp_list)
self.history = list(filter(lambda x: next_received - x['time'] <= duration_threshold * 2, self.history))
bid_list = list(map(lambda x: x['bid_data']['bid'], self.history))
ask_list = list(map(lambda x: x['ask_data']['ask'], self.history))
bid_stdev = statistics.stdev(bid_list)
ask_stdev = statistics.stdev(ask_list)
if sp_stdev > 0.5:
logging.warning(
'too volatile. sp_stdev:{}, sp_threshold:{}, instrument:{}, {} -> {}'.format(sp_stdev,
sp_threshold,
instrument, b, a))
return d, None
if bid_stdev * 2 / bid_list[-1] * 100 > 0.03:
logging.warning(
'too volatile. bid_stdev:{}, instrument:{}, {} -> {}'.format(bid_stdev, instrument, b, a))
return d, None
if ask_stdev * 2 / ask_list[-1] * 100 > 0.03:
logging.warning(
'too volatile. ask_stdev:{}, instrument:{}, {} -> {}'.format(ask_stdev, instrument, b, a))
return d, None
if sp_min > self.last_sp + 0.01 or (timeutil.jst_now() - self.last_signaled >= timedelta(minutes=1)):
self.last_sp = sp_min
d2 = copy.deepcopy(d)
d2.update(sp=self.last_sp, not_closed=self.not_closed)
self.last_signaled = timeutil.jst_now()
return d, d2
finally:
self.last_received = next_received
def clear(self):
self.last_received = timeutil.jst_now()
self.history.clear()
self.last_sp = 0
self.duration = timedelta()
def judge(self, bid_data: dict, ask_data: dict) -> Optional[dict]:
instrument = self.instrument
b, a = bid_data['service'], ask_data['service']
next_received = max(bid_data['time'], ask_data['time'])
assert bid_data['instrument'] == ask_data['instrument']
if int((bid_data['time'] - ask_data['time']).total_seconds()) >= 0.5:
self.history.clear()
return
config = Config()
bid_positions = defaultdict(
int,
**config.accounts.get(b, {}).get('positions', {}))
ask_positions = defaultdict(
int,
**config.accounts.get(a, {}).get('positions', {}))
if bid_positions[instrument] <= 0 or ask_positions[instrument] >= 0:
sp_threshold = config.filters.get(instrument,
{}).get('open_spread', 3.0)
duration_threshold = timedelta(seconds=config.filters.get(
instrument, {}).get('open_duration', 4.0))
else:
sp_threshold = config.filters.get(instrument,
{}).get('close_spread', 3.0)
duration_threshold = timedelta(seconds=config.filters.get(
instrument, {}).get('close_duration', 4.0))
try:
timeout = timedelta(seconds=1) # self.config['timeout'])
if next_received - self.last_received >= timeout:
self.history.clear()
return
pip_scale = get_pip_scale(instrument)
sp = (bid_data['bid'] - ask_data['ask']) * pip_scale
d = copy.deepcopy(bid_data)
d.update(direction=(b, a),
bidder=b,
asker=a,
ask=ask_data['ask'],
sp=sp,
time=next_received,
bid_data=bid_data,
ask_data=ask_data,
signaled=False)
if sp < sp_threshold:
self.history.clear()
if sp >= 0.1:
return d
return
self.history.append(d)
duration = next_received - self.history[0]['time']
if duration >= duration_threshold:
self.history = list(
filter(
lambda x: next_received - x['time'] <=
duration_threshold * 2, self.history))
if len(self.history) >= 2:
sp_list = list(map(lambda x: x['sp'], self.history))
sp_min = min(sp_list)
sp_stdev = statistics.stdev(sp_list)
bid_list = list(
map(lambda x: x['bid_data']['bid'], self.history))
ask_list = list(
map(lambda x: x['ask_data']['ask'], self.history))
bid_stdev = statistics.stdev(bid_list)
ask_stdev = statistics.stdev(ask_list)
if sp_stdev > 0.5:
logging.warning(
'too volatile. sp_stdev:{}, sp_threshold:{}, instrument:{}, {} -> {}'
.format(sp_stdev, sp_threshold, instrument, b, a))
return
if bid_stdev * 2 / bid_list[-1] * 100 > 0.03:
logging.warning(
'too volatile. bid_stdev:{}, instrument:{}, {} -> {}'
.format(bid_stdev, instrument, b, a))
return
if ask_stdev * 2 / ask_list[-1] * 100 > 0.03:
logging.warning(
'too volatile. ask_stdev:{}, instrument:{}, {} -> {}'
.format(ask_stdev, instrument, b, a))
return
if sp_min > self.last_sp + 0.01 or (
timeutil.jst_now() - self.last_signaled >=
timedelta(minutes=1)):
self.last_sp = sp_min
d.update(sp=self.last_sp, signaled=True)
self.last_signaled = timeutil.jst_now()
return copy.deepcopy(d)
finally:
self.last_received = next_received
def clear(self):
self.history.clear()
self.last_sp = 0
self.last_received = timeutil.jst_now()
self.last_signaled = timeutil.jst_now()
