def display(series: List[Clazz], score: int) -> List[Clazz]: results = [] if score: assert 1 <= score <= 8 begin = series[0] end = series[-1] results += [Clazz(begin, value=begin.open)] for s in series[1:-1]: # assume order is (open, low, high, close) for (open < close) if s.open < s.close: if score <= s.valid_low_score: results += [Clazz(s, value=s.low)] if score <= s.valid_high_score: results += [Clazz(s, value=s.high)] else: if score <= s.valid_high_score: results += [Clazz(s, value=s.high)] if score <= s.valid_low_score: results += [Clazz(s, value=s.low)] results += [Clazz(end, value=end.close)] else: for s in series: # assume order is (open, low, high, close) for (open < close) value1, value2 = (s.low, s.high) if s.close > s.open else (s.high, s.low) results += [ Clazz(timestamp=s.timestamp, value=s.open), Clazz(timestamp=s.timestamp, value=value1), Clazz(timestamp=s.timestamp, value=value2), Clazz(timestamp=s.timestamp, value=s.close) ] return results
def action(series: List[Clazz]) -> Clazz: """ profit - total profit or loss total - total cash required for all open positions long - cash required for single long position open_timestamp - timestamp at the opening volume - number of all opened longs """ profit = total = long = 0.0 open_timestamp = volume = 0 for s in series: s.test = Clazz() if (long == 0.0) and (4 <= s.low_score): open_timestamp = s.timestamp long = s.test.long = s.close total += long volume += 1 elif (long > 0.0) and (2 <= s.low_score): s.test.open_timestamp = open_timestamp s.test.open_long = long s.test.short = s.close s.test.profit = s.test.short - s.test.open_long long = 0.0 profit += s.test.profit return Clazz(profit=profit, total=total, volume=volume)
def plot_candidate_swings(ax: Axes, series: List[Clazz], score: int): results = [] for s in series: if score <= s.low_score: results += [Clazz(timestamp=s.timestamp, value=s.low)] if score <= s.high_score: results += [Clazz(timestamp=s.timestamp, value=s.high)] color = Color.from_score(score) plot_dots( ax, [s.timestamp for s in results], [s.value for s in results], label=f'score-{score} [{int(100 * swings.limit_ratio(score)):03}%]', color=color, size=1 + score)
def plot_bars(ax: Axes, series: List[Clazz]): results = [] for s in series: value1, value2 = (s.low, s.high) if s.close > s.open else (s.high, s.low) results += [ Clazz(timestamp=s.timestamp, value=s.open), Clazz(timestamp=s.timestamp, value=value1), Clazz(timestamp=s.timestamp, value=value2), Clazz(timestamp=s.timestamp, value=s.close) ] plot_line(ax, [s.timestamp for s in results], [s.value for s in results], label='bars', color=Color.grey)
def read_series(symbol: str, begin: int, end: int) -> List[Clazz]: interval = tool.INTERVAL_1D series = [] with exante.SecuritySeries(interval) as security_series: for s in security_series[symbol]: if begin <= s.timestamp <= end: x = s.timestamp y1, y2 = (s.low, s.high) if s.close > s.open else (s.high, s.low) series += [ Clazz(x=x, y=s.open), Clazz(x=x, y=y1), Clazz(x=x, y=y2), Clazz(x=x, y=s.close) ] return series
def security_update_by_interval(engine: Any, interval: timedelta): LOG.info(f'>> {security_update.__name__} source: {tool.source_name(engine, interval)}') default_range = Clazz(dt_to=config.datetime_from()) with engine.SecuritySeries(interval) as security_series: time_range = security_series.time_range() LOG.debug(f'Time range entries: {len(time_range)}') for exchange_name in config.EXCHANGES: with store.ExchangeSeries() as exchange_series: securities = exchange_series[exchange_name] with engine.Session() as session: with flow.Progress(f'security-update: {exchange_name}', securities) as progress: for security in securities: progress(security.symbol) dt_from = time_range.get(security.symbol, default_range).dt_to dt_to = tool.last_session(exchange_name, interval, DateTime.now()) for slice_from, slice_to in tool.time_slices(dt_from, dt_to, interval, 4096): time_series = session.series(security.symbol, slice_from, slice_to, interval) with engine.SecuritySeries(interval, editable=True) as security_series: security_series += time_series LOG.info(f'Securities: {len(securities)} updated in the exchange: {exchange_name}')
def init(series: List[Clazz]) -> List[Clazz]: for s in series: s.update(tool.SECURITY_SCORE_DEFAULT) results = [] for s in series: if s.low != s.high: # avoid duplicates # assume order is (open, low, high, close) for (open < close) value1, value2 = (s.low, s.high) if s.close > s.open else (s.high, s.low) results += [ Clazz(data=s, value=value1), Clazz(data=s, value=value2) ] else: results += [Clazz(data=s, value=s.low)] return results
def read_series(begin: int, end: int) -> List[Clazz]: interval = tool.INTERVAL_1D with yahoo.SecuritySeries(interval) as security_series: abc_series = security_series['ABC.NYSE'] return [ Clazz(x=s.timestamp / 1e6, y1=s.low, y2=s.high) for s in abc_series if begin <= s.timestamp <= end ]
def test_reduce_4(): with config.TESTS_PATH.joinpath('sample.json').open() as sample_io: sample = json.loads(sample_io.read()) security = [Clazz(s) for s in sample['KGH.WSE']] score = 4 reduced = swings.init(security) reduced = swings.reduce(reduced, score) assert len(reduced) == 68
def __getitem__(self, exchange: str) -> List[Clazz]: query = ''' FOR datum IN @@collection FILTER datum.exchange == @exchange RETURN datum ''' records = self.tnx_db.aql.execute(query, bind_vars={ 'exchange': exchange, '@collection': self.name }) return [Clazz(**r) for r in records]
def datum_from_yahoo(dt: Dict, symbol: str) -> Optional[Clazz]: try: return Clazz(symbol=symbol, timestamp=timestamp_from_yahoo(dt['Date']), open=float(dt['Open']), close=float(dt['Close']), low=float(dt['Low']), high=float(dt['High']), volume=int(dt['Volume']), **tool.SECURITY_SCORE_DEFAULT) except: return None
def datum_from_exante(dt: Dict, symbol: str) -> Optional[Clazz]: try: return Clazz(symbol=symbol, timestamp=timestamp_from_exante(dt['timestamp']), open=float(dt['open']), close=float(dt['close']), low=float(dt['low']), high=float(dt['high']), volume=int(dt['volume']), **tool.SECURITY_SCORE_DEFAULT) except: return None
def datum_from_stooq(dt: Dict, symbol: str) -> Optional[Clazz]: try: return Clazz(symbol=symbol, timestamp=timestamp_from_stooq(dt['<DATE>']), open=float(dt['<OPEN>']), close=float(dt['<CLOSE>']), low=float(dt['<LOW>']), high=float(dt['<HIGH>']), volume=int(dt['<VOL>']), **tool.SECURITY_SCORE_DEFAULT) except: return None
def security_verify(engine: Any): interval = tool.INTERVAL_1D source_name = tool.source_name(engine, interval) health_name = tool.health_name(engine, interval) LOG.info(f'>> {security_verify.__name__} source: {source_name}') with engine.SecuritySeries(interval) as security_series: time_range = security_series.time_range() with store.File(health_name, editable=True) as health: for exchange_name in config.EXCHANGES: health[exchange_name] = {} last_session = tool.last_session(exchange_name, interval, DateTime.now()) with store.ExchangeSeries() as exchange_series: securities = exchange_series[exchange_name] entries = [] with flow.Progress(health_name, securities) as progress: for security in securities: progress(security.symbol) result = Clazz() symbol_range = time_range.get(security.symbol) if symbol_range: overlap, missing = time_series_verify(engine, security.symbol, symbol_range.dt_from, last_session, interval) if overlap: result.overlap = overlap if missing: result.missing = missing if len(missing) > config.HEALTH_MISSING_LIMIT: result.message = f'The missing limit reached: {len(missing)}' if last_session in missing: result.message = f'The last session {symbol_range.dt_to} < {last_session}' else: result.message = 'There is no time series for this symbol' if result: short_symbol, _ = tool.symbol_split(security.symbol) health[exchange_name][short_symbol] = result entry = security.entry(health_name) entry[health_name] = 'message' not in result entries += [entry] with store.ExchangeSeries(editable=True) as exchange_series: exchange_series |= entries LOG.info(f'Securities: {len(securities)} verified in the exchange: {exchange_name}')
def time_range(self) -> Dict[str, Clazz]: query = ''' FOR datum IN @@collection COLLECT symbol = datum.symbol AGGREGATE ts_from = MIN(datum.timestamp), ts_to = MAX(datum.timestamp) RETURN {symbol, ts_from, ts_to} ''' records = self.tnx_db.aql.execute(query, bind_vars={'@collection': self.name}) return { r['symbol']: Clazz(dt_from=DateTime.from_timestamp(r['ts_from']), dt_to=DateTime.from_timestamp(r['ts_to'])) for r in records }
def securities(self, exchange: str) -> List[Clazz]: url = f'{DATA_URL}/exchanges/{exchange}' response = self.get(url) assert response.status_code == 200, f'url: {url} reply: {response.text}' keys = dict(symbol='symbolId', type='symbolType', exchange='exchange', currency='currency', name='name', description='description', short_symbol='ticker') return [ Clazz({k: item[v] for k, v in keys.items()}) for item in response.json() ]
def transactions(self) -> List[Clazz]: url = f'{DATA_URL}/transactions' response = self.get(url) convert = { 'asset': 'asset', 'id': 'id', 'type': 'operationType', 'sum': 'sum', 'symbol': 'symbolId', 'timestamp': 'timestamp' } return [ Clazz(**{k: d[v] for k, v in convert.items()}) for d in response.json() ]
def __getitem__(self, symbol: str) -> List[Clazz]: query = ''' FOR datum IN @@collection SORT datum.timestamp FILTER datum.symbol == @symbol AND datum.timestamp >= @timestamp RETURN datum ''' bind_vars = { 'symbol': symbol, '@collection': self.name, 'timestamp': self.ts_from } records = self.tnx_db.aql.execute(query, bind_vars=bind_vars) return [Clazz(**r) for r in records]
def schedule_endpoint(): if request.method == 'POST': LOG.info(f'Scheduling function {task_daily.__name__}') task = Clazz(next_run=DateTime.now().replace(microsecond=0), running=False, function=task_daily) TASKS.append(task) LOG.info('Listing threads and tasks') threads = [{ 'name': thread.name, 'daemon': thread.daemon, 'alive': thread.is_alive() } for thread in threading.enumerate()] content = dict(threads=threads, tasks=TASKS) return json.dumps(content, option=json.OPT_INDENT_2, default=tool.json_default).decode('utf-8')
def show_strategy(symbol: str, interval: timedelta, begin: int, end: int): ax1, ax2 = init_widget() series = read_series(symbol, interval, begin, end) plot_bars(ax1, series) reduced = swings.init(series) swings.reduce(reduced, Swing.DROP.value) state = State.START lowest_low = None for s in series: s.test = Clazz(drop=None, long=None, short=None) if state in (State.START, State.DROPPED): if Swing.DROP.value <= s.low_score: state = State.DROPPED lowest_low = s.test.drop = s.low continue if state == State.DROPPED: if s.low > lowest_low: state = State.LONG lowest_low = s.test.long = s.low continue if state == State.LONG: if s.low < lowest_low: state = State.START s.test.short = s.low lowest_low = None continue plot_strategy(ax1, series) w_size = 14 analyse.volatile(series, w_size) plot_atr(ax2, series, w_size, Color.red) show_widget((ax1, ax2), symbol, begin, end)
def flatten_series(series: Iterable[Clazz]): series = [[Clazz(x=s.x, y=s.y1), Clazz(x=s.x, y=s.y2)] for s in series] return sum(series, [])
def mid_series(series: Iterable[Clazz]): return [Clazz(x=s.x, y=(s.y1 + s.y2) / 2) for s in series]
def source_name(engine: Any, interval: Union[timedelta, str]) -> str: if not isinstance(engine, str): engine = engine.__name__ engine_name = engine.split('.')[-1] if isinstance(interval, timedelta): interval = interval_name(interval) return f'{engine_name}_{interval}' ENV_TEST = 'test' ENV_LIVE = 'live' ENV_RESULT_DEFAULT = { name: Clazz(profit=0.0, total=0.0, volume=0) for name in ('stooq_1d_test', 'yahoo_1d_test', 'exante_1d_test') } def result_name(engine: Any, interval: Union[timedelta, str], env_name: str) -> str: source = source_name(engine, interval) return f'{source}_{env_name}' HEALTH_RESULT_DEFAULT = { name: False for name in ('stooq_1d_health', 'yahoo_1d_health', 'exante_1d_health') }
def test_transpose(): key, = tool.transpose([Clazz(key='v1'), Clazz(key='v2')], ['key']) assert key == ['v1', 'v2']
import orjson as json from src import swings, config from src.clazz import Clazz SERIES = [ Clazz(symbol='XOM.NYSE', timestamp=1514851200, open=1.0, close=1.0, low=1.0, high=1.0, volume=3), Clazz(symbol='XOM.NYSE', timestamp=1514937600, open=2.0, close=2.0, low=2.0, high=2.0, volume=8), Clazz(symbol='XOM.NYSE', timestamp=1515024000, open=3.0, close=3.0, low=3.0, high=3.0, volume=1), Clazz(symbol='XOM.NYSE', timestamp=1515110400, open=4.0, close=4.0,
def clean(series: List[Clazz]): required = ['_id', '_rev', '_key' ] + schema.SECURITY_SCHEMA['rule']['required'] for i, s in enumerate(series): series[i] = Clazz({k: v for k, v in s.items() if k in required})
def test_clean(): extra = [Clazz(s, extra=1) for s in SERIES] analyse.clean(extra) assert extra == SERIES
# gunicorn src.schedule:wsgi -b :8882 return app(environ, start_response) @tool.catch_exception(LOG) def task_daily(): data.exchange_update() for engine in (yahoo, stooq, exante): data.security_daily(engine) TASKS = [ Clazz(interval=tool.INTERVAL_1D, hour=1, minute=7, next_run=None, last_run=None, running=False, function=task_daily) ] def run_scheduled_tasks(): for task in TASKS: utc_now = DateTime.now() task.next_run = utc_now.replace(hour=task.hour, minute=task.minute, second=0, microsecond=0) if task.next_run < utc_now: task.next_run += task.interval