def display(series: List[Clazz], score: int) -> List[Clazz]:
results = []
if score:
assert 1 <= score <= 8
begin = series[0]
end = series[-1]
results += [Clazz(begin, value=begin.open)]
for s in series[1:-1]:
# assume order is (open, low, high, close) for (open < close)
if s.open < s.close:
if score <= s.valid_low_score:
results += [Clazz(s, value=s.low)]
if score <= s.valid_high_score:
results += [Clazz(s, value=s.high)]
else:
if score <= s.valid_high_score:
results += [Clazz(s, value=s.high)]
if score <= s.valid_low_score:
results += [Clazz(s, value=s.low)]
results += [Clazz(end, value=end.close)]
else:
for s in series:
# assume order is (open, low, high, close) for (open < close)
value1, value2 = (s.low, s.high) if s.close > s.open else (s.high,
s.low)
results += [
Clazz(timestamp=s.timestamp, value=s.open),
Clazz(timestamp=s.timestamp, value=value1),
Clazz(timestamp=s.timestamp, value=value2),
Clazz(timestamp=s.timestamp, value=s.close)
]
return results
def action(series: List[Clazz]) -> Clazz:
"""
profit - total profit or loss
total - total cash required for all open positions
long - cash required for single long position
open_timestamp - timestamp at the opening
volume - number of all opened longs
"""
profit = total = long = 0.0
open_timestamp = volume = 0
for s in series:
s.test = Clazz()
if (long == 0.0) and (4 <= s.low_score):
open_timestamp = s.timestamp
long = s.test.long = s.close
total += long
volume += 1
elif (long > 0.0) and (2 <= s.low_score):
s.test.open_timestamp = open_timestamp
s.test.open_long = long
s.test.short = s.close
s.test.profit = s.test.short - s.test.open_long
long = 0.0
profit += s.test.profit
return Clazz(profit=profit, total=total, volume=volume)
def plot_candidate_swings(ax: Axes, series: List[Clazz], score: int):
results = []
for s in series:
if score <= s.low_score:
results += [Clazz(timestamp=s.timestamp, value=s.low)]
if score <= s.high_score:
results += [Clazz(timestamp=s.timestamp, value=s.high)]
color = Color.from_score(score)
plot_dots(
ax, [s.timestamp for s in results], [s.value for s in results],
label=f'score-{score} [{int(100 * swings.limit_ratio(score)):03}%]',
color=color,
size=1 + score)
def plot_bars(ax: Axes, series: List[Clazz]):
results = []
for s in series:
value1, value2 = (s.low, s.high) if s.close > s.open else (s.high,
s.low)
results += [
Clazz(timestamp=s.timestamp, value=s.open),
Clazz(timestamp=s.timestamp, value=value1),
Clazz(timestamp=s.timestamp, value=value2),
Clazz(timestamp=s.timestamp, value=s.close)
]
plot_line(ax, [s.timestamp for s in results], [s.value for s in results],
label='bars',
color=Color.grey)
def read_series(symbol: str, begin: int, end: int) -> List[Clazz]:
interval = tool.INTERVAL_1D
series = []
with exante.SecuritySeries(interval) as security_series:
for s in security_series[symbol]:
if begin <= s.timestamp <= end:
x = s.timestamp
y1, y2 = (s.low, s.high) if s.close > s.open else (s.high,
s.low)
series += [
Clazz(x=x, y=s.open),
Clazz(x=x, y=y1),
Clazz(x=x, y=y2),
Clazz(x=x, y=s.close)
]
return series
def security_update_by_interval(engine: Any, interval: timedelta):
LOG.info(f'>> {security_update.__name__} source: {tool.source_name(engine, interval)}')
default_range = Clazz(dt_to=config.datetime_from())
with engine.SecuritySeries(interval) as security_series:
time_range = security_series.time_range()
LOG.debug(f'Time range entries: {len(time_range)}')
for exchange_name in config.EXCHANGES:
with store.ExchangeSeries() as exchange_series:
securities = exchange_series[exchange_name]
with engine.Session() as session:
with flow.Progress(f'security-update: {exchange_name}', securities) as progress:
for security in securities:
progress(security.symbol)
dt_from = time_range.get(security.symbol, default_range).dt_to
dt_to = tool.last_session(exchange_name, interval, DateTime.now())
for slice_from, slice_to in tool.time_slices(dt_from, dt_to, interval, 4096):
time_series = session.series(security.symbol, slice_from, slice_to, interval)
with engine.SecuritySeries(interval, editable=True) as security_series:
security_series += time_series
LOG.info(f'Securities: {len(securities)} updated in the exchange: {exchange_name}')
def init(series: List[Clazz]) -> List[Clazz]:
for s in series:
s.update(tool.SECURITY_SCORE_DEFAULT)
results = []
for s in series:
if s.low != s.high: # avoid duplicates
# assume order is (open, low, high, close) for (open < close)
value1, value2 = (s.low, s.high) if s.close > s.open else (s.high,
s.low)
results += [
Clazz(data=s, value=value1),
Clazz(data=s, value=value2)
]
else:
results += [Clazz(data=s, value=s.low)]
return results
def read_series(begin: int, end: int) -> List[Clazz]:
interval = tool.INTERVAL_1D
with yahoo.SecuritySeries(interval) as security_series:
abc_series = security_series['ABC.NYSE']
return [
Clazz(x=s.timestamp / 1e6, y1=s.low, y2=s.high) for s in abc_series
if begin <= s.timestamp <= end
]
def test_reduce_4():
with config.TESTS_PATH.joinpath('sample.json').open() as sample_io:
sample = json.loads(sample_io.read())
security = [Clazz(s) for s in sample['KGH.WSE']]
score = 4
reduced = swings.init(security)
reduced = swings.reduce(reduced, score)
assert len(reduced) == 68
def __getitem__(self, exchange: str) -> List[Clazz]:
query = '''
FOR datum IN @@collection
FILTER datum.exchange == @exchange
RETURN datum
'''
records = self.tnx_db.aql.execute(query,
bind_vars={
'exchange': exchange,
'@collection': self.name
})
return [Clazz(**r) for r in records]
def datum_from_yahoo(dt: Dict, symbol: str) -> Optional[Clazz]:
try:
return Clazz(symbol=symbol,
timestamp=timestamp_from_yahoo(dt['Date']),
open=float(dt['Open']),
close=float(dt['Close']),
low=float(dt['Low']),
high=float(dt['High']),
volume=int(dt['Volume']),
**tool.SECURITY_SCORE_DEFAULT)
except:
return None
def datum_from_exante(dt: Dict, symbol: str) -> Optional[Clazz]:
try:
return Clazz(symbol=symbol,
timestamp=timestamp_from_exante(dt['timestamp']),
open=float(dt['open']),
close=float(dt['close']),
low=float(dt['low']),
high=float(dt['high']),
volume=int(dt['volume']),
**tool.SECURITY_SCORE_DEFAULT)
except:
return None
def datum_from_stooq(dt: Dict, symbol: str) -> Optional[Clazz]:
try:
return Clazz(symbol=symbol,
timestamp=timestamp_from_stooq(dt['<DATE>']),
open=float(dt['<OPEN>']),
close=float(dt['<CLOSE>']),
low=float(dt['<LOW>']),
high=float(dt['<HIGH>']),
volume=int(dt['<VOL>']),
**tool.SECURITY_SCORE_DEFAULT)
except:
return None
def security_verify(engine: Any):
interval = tool.INTERVAL_1D
source_name = tool.source_name(engine, interval)
health_name = tool.health_name(engine, interval)
LOG.info(f'>> {security_verify.__name__} source: {source_name}')
with engine.SecuritySeries(interval) as security_series:
time_range = security_series.time_range()
with store.File(health_name, editable=True) as health:
for exchange_name in config.EXCHANGES:
health[exchange_name] = {}
last_session = tool.last_session(exchange_name, interval, DateTime.now())
with store.ExchangeSeries() as exchange_series:
securities = exchange_series[exchange_name]
entries = []
with flow.Progress(health_name, securities) as progress:
for security in securities:
progress(security.symbol)
result = Clazz()
symbol_range = time_range.get(security.symbol)
if symbol_range:
overlap, missing = time_series_verify(engine,
security.symbol,
symbol_range.dt_from,
last_session,
interval)
if overlap:
result.overlap = overlap
if missing:
result.missing = missing
if len(missing) > config.HEALTH_MISSING_LIMIT:
result.message = f'The missing limit reached: {len(missing)}'
if last_session in missing:
result.message = f'The last session {symbol_range.dt_to} < {last_session}'
else:
result.message = 'There is no time series for this symbol'
if result:
short_symbol, _ = tool.symbol_split(security.symbol)
health[exchange_name][short_symbol] = result
entry = security.entry(health_name)
entry[health_name] = 'message' not in result
entries += [entry]
with store.ExchangeSeries(editable=True) as exchange_series:
exchange_series |= entries
LOG.info(f'Securities: {len(securities)} verified in the exchange: {exchange_name}')
def time_range(self) -> Dict[str, Clazz]:
query = '''
FOR datum IN @@collection
COLLECT symbol = datum.symbol
AGGREGATE ts_from = MIN(datum.timestamp), ts_to = MAX(datum.timestamp)
RETURN {symbol, ts_from, ts_to}
'''
records = self.tnx_db.aql.execute(query,
bind_vars={'@collection': self.name})
return {
r['symbol']: Clazz(dt_from=DateTime.from_timestamp(r['ts_from']),
dt_to=DateTime.from_timestamp(r['ts_to']))
for r in records
}
def securities(self, exchange: str) -> List[Clazz]:
url = f'{DATA_URL}/exchanges/{exchange}'
response = self.get(url)
assert response.status_code == 200, f'url: {url} reply: {response.text}'
keys = dict(symbol='symbolId',
type='symbolType',
exchange='exchange',
currency='currency',
name='name',
description='description',
short_symbol='ticker')
return [
Clazz({k: item[v]
for k, v in keys.items()}) for item in response.json()
]
def transactions(self) -> List[Clazz]:
url = f'{DATA_URL}/transactions'
response = self.get(url)
convert = {
'asset': 'asset',
'id': 'id',
'type': 'operationType',
'sum': 'sum',
'symbol': 'symbolId',
'timestamp': 'timestamp'
}
return [
Clazz(**{k: d[v]
for k, v in convert.items()}) for d in response.json()
]
def __getitem__(self, symbol: str) -> List[Clazz]:
query = '''
FOR datum IN @@collection
SORT datum.timestamp
FILTER datum.symbol == @symbol
AND datum.timestamp >= @timestamp
RETURN datum
'''
bind_vars = {
'symbol': symbol,
'@collection': self.name,
'timestamp': self.ts_from
}
records = self.tnx_db.aql.execute(query, bind_vars=bind_vars)
return [Clazz(**r) for r in records]
def schedule_endpoint():
if request.method == 'POST':
LOG.info(f'Scheduling function {task_daily.__name__}')
task = Clazz(next_run=DateTime.now().replace(microsecond=0),
running=False,
function=task_daily)
TASKS.append(task)
LOG.info('Listing threads and tasks')
threads = [{
'name': thread.name,
'daemon': thread.daemon,
'alive': thread.is_alive()
} for thread in threading.enumerate()]
content = dict(threads=threads, tasks=TASKS)
return json.dumps(content,
option=json.OPT_INDENT_2,
default=tool.json_default).decode('utf-8')
def show_strategy(symbol: str, interval: timedelta, begin: int, end: int):
ax1, ax2 = init_widget()
series = read_series(symbol, interval, begin, end)
plot_bars(ax1, series)
reduced = swings.init(series)
swings.reduce(reduced, Swing.DROP.value)
state = State.START
lowest_low = None
for s in series:
s.test = Clazz(drop=None, long=None, short=None)
if state in (State.START, State.DROPPED):
if Swing.DROP.value <= s.low_score:
state = State.DROPPED
lowest_low = s.test.drop = s.low
continue
if state == State.DROPPED:
if s.low > lowest_low:
state = State.LONG
lowest_low = s.test.long = s.low
continue
if state == State.LONG:
if s.low < lowest_low:
state = State.START
s.test.short = s.low
lowest_low = None
continue
plot_strategy(ax1, series)
w_size = 14
analyse.volatile(series, w_size)
plot_atr(ax2, series, w_size, Color.red)
show_widget((ax1, ax2), symbol, begin, end)
def flatten_series(series: Iterable[Clazz]):
series = [[Clazz(x=s.x, y=s.y1), Clazz(x=s.x, y=s.y2)] for s in series]
return sum(series, [])
def mid_series(series: Iterable[Clazz]):
return [Clazz(x=s.x, y=(s.y1 + s.y2) / 2) for s in series]
def source_name(engine: Any, interval: Union[timedelta, str]) -> str:
if not isinstance(engine, str):
engine = engine.__name__
engine_name = engine.split('.')[-1]
if isinstance(interval, timedelta):
interval = interval_name(interval)
return f'{engine_name}_{interval}'
ENV_TEST = 'test'
ENV_LIVE = 'live'
ENV_RESULT_DEFAULT = {
name: Clazz(profit=0.0, total=0.0, volume=0)
for name in ('stooq_1d_test', 'yahoo_1d_test', 'exante_1d_test')
}
def result_name(engine: Any, interval: Union[timedelta, str],
env_name: str) -> str:
source = source_name(engine, interval)
return f'{source}_{env_name}'
HEALTH_RESULT_DEFAULT = {
name: False
for name in ('stooq_1d_health', 'yahoo_1d_health', 'exante_1d_health')
}
def test_transpose():
key, = tool.transpose([Clazz(key='v1'), Clazz(key='v2')], ['key'])
assert key == ['v1', 'v2']
import orjson as json
from src import swings, config
from src.clazz import Clazz
SERIES = [
Clazz(symbol='XOM.NYSE',
timestamp=1514851200,
open=1.0,
close=1.0,
low=1.0,
high=1.0,
volume=3),
Clazz(symbol='XOM.NYSE',
timestamp=1514937600,
open=2.0,
close=2.0,
low=2.0,
high=2.0,
volume=8),
Clazz(symbol='XOM.NYSE',
timestamp=1515024000,
open=3.0,
close=3.0,
low=3.0,
high=3.0,
volume=1),
Clazz(symbol='XOM.NYSE',
timestamp=1515110400,
open=4.0,
close=4.0,
def clean(series: List[Clazz]):
required = ['_id', '_rev', '_key'
] + schema.SECURITY_SCHEMA['rule']['required']
for i, s in enumerate(series):
series[i] = Clazz({k: v for k, v in s.items() if k in required})
def test_clean():
extra = [Clazz(s, extra=1) for s in SERIES]
analyse.clean(extra)
assert extra == SERIES
# gunicorn src.schedule:wsgi -b :8882
return app(environ, start_response)
@tool.catch_exception(LOG)
def task_daily():
data.exchange_update()
for engine in (yahoo, stooq, exante):
data.security_daily(engine)
TASKS = [
Clazz(interval=tool.INTERVAL_1D,
hour=1,
minute=7,
next_run=None,
last_run=None,
running=False,
function=task_daily)
]
def run_scheduled_tasks():
for task in TASKS:
utc_now = DateTime.now()
task.next_run = utc_now.replace(hour=task.hour,
minute=task.minute,
second=0,
microsecond=0)
if task.next_run < utc_now:
task.next_run += task.interval
