def __init__(self, exchanges=None):
self.exchanges = exchanges if exchanges else {
e: [tool.INTERVAL_1D]
for e in config.EXCHANGES
}
for exchange, intervals in self.exchanges.items():
for interval in intervals:
zip_path = stooq_zip_path(interval, exchange)
zip_path.parent.mkdir(parents=True, exist_ok=True)
if not tool.is_latest(zip_path, interval, exchange):
# start streaming from url
url = stooq_url(interval, exchange)
response = requests.get(url, stream=True)
message = f'Loading {url} to {zip_path.as_posix()}'
LOG.info(message)
size = int(response.headers["Content-Length"]
) // URL_CHUNK_SIZE + 1
LOG.debug(f'Size {zip_path.as_posix()}: {size}M')
# streaming to the zip file
zip_path_pending = zip_path.with_suffix('.pending')
with zip_path_pending.open('wb') as zip_io:
with flow.Progress(message, size) as progress:
for chunk in response.iter_content(URL_CHUNK_SIZE):
progress('+')
zip_io.write(chunk)
zip_path_pending.rename(zip_path)
def security_update_by_interval(engine: Any, interval: timedelta):
LOG.info(f'>> {security_update.__name__} source: {tool.source_name(engine, interval)}')
default_range = Clazz(dt_to=config.datetime_from())
with engine.SecuritySeries(interval) as security_series:
time_range = security_series.time_range()
LOG.debug(f'Time range entries: {len(time_range)}')
for exchange_name in config.EXCHANGES:
with store.ExchangeSeries() as exchange_series:
securities = exchange_series[exchange_name]
with engine.Session() as session:
with flow.Progress(f'security-update: {exchange_name}', securities) as progress:
for security in securities:
progress(security.symbol)
dt_from = time_range.get(security.symbol, default_range).dt_to
dt_to = tool.last_session(exchange_name, interval, DateTime.now())
for slice_from, slice_to in tool.time_slices(dt_from, dt_to, interval, 4096):
time_series = session.series(security.symbol, slice_from, slice_to, interval)
with engine.SecuritySeries(interval, editable=True) as security_series:
security_series += time_series
LOG.info(f'Securities: {len(securities)} updated in the exchange: {exchange_name}')
def security_analyse(engine: Any):
interval = tool.INTERVAL_1D
w_sizes = [50, 100, 200]
source_name = tool.source_name(engine, interval)
result_name = tool.result_name(engine, interval, tool.ENV_TEST)
LOG.info(f'>> {security_analyse.__name__} source: {source_name}')
for exchange_name in config.EXCHANGES:
with store.ExchangeSeries() as exchange_series:
securities = exchange_series[exchange_name]
entries = []
with flow.Progress(f'security-analyse {exchange_name}', securities) as progress:
for security in securities:
progress(security.symbol)
with engine.SecuritySeries(interval, editable=True) as security_series:
time_series = security_series[security.symbol]
analyse.clean(time_series)
swings.calculate(time_series)
for w_size in w_sizes:
analyse.sma(time_series, w_size)
analyse.vma(time_series, w_size)
action = analyse.action(time_series)
security_series *= time_series
entry = security.entry(result_name)
entry[result_name] = action
entries += [entry]
with store.ExchangeSeries(editable=True) as exchange_series:
exchange_series |= entries
LOG.info(f'Securities: {len(securities)} analysed in the exchange: {exchange_name}')
def test_exception(caplog):
lst = ['1st', '2nd']
try:
with flow.Progress(test_progress.__name__, lst) as progress:
progress('1st')
raise Exception('broken iteration of list')
except:
assert '0.0% 1st' in caplog.text
def security_verify(engine: Any):
interval = tool.INTERVAL_1D
source_name = tool.source_name(engine, interval)
health_name = tool.health_name(engine, interval)
LOG.info(f'>> {security_verify.__name__} source: {source_name}')
with engine.SecuritySeries(interval) as security_series:
time_range = security_series.time_range()
with store.File(health_name, editable=True) as health:
for exchange_name in config.EXCHANGES:
health[exchange_name] = {}
last_session = tool.last_session(exchange_name, interval, DateTime.now())
with store.ExchangeSeries() as exchange_series:
securities = exchange_series[exchange_name]
entries = []
with flow.Progress(health_name, securities) as progress:
for security in securities:
progress(security.symbol)
result = Clazz()
symbol_range = time_range.get(security.symbol)
if symbol_range:
overlap, missing = time_series_verify(engine,
security.symbol,
symbol_range.dt_from,
last_session,
interval)
if overlap:
result.overlap = overlap
if missing:
result.missing = missing
if len(missing) > config.HEALTH_MISSING_LIMIT:
result.message = f'The missing limit reached: {len(missing)}'
if last_session in missing:
result.message = f'The last session {symbol_range.dt_to} < {last_session}'
else:
result.message = 'There is no time series for this symbol'
if result:
short_symbol, _ = tool.symbol_split(security.symbol)
health[exchange_name][short_symbol] = result
entry = security.entry(health_name)
entry[health_name] = 'message' not in result
entries += [entry]
with store.ExchangeSeries(editable=True) as exchange_series:
exchange_series |= entries
LOG.info(f'Securities: {len(securities)} verified in the exchange: {exchange_name}')
def security_clean(engine: Any):
interval = tool.INTERVAL_1D
source_name = tool.source_name(engine, interval)
LOG.info(f'>> {security_clean.__name__} source: {source_name}')
for exchange_name in config.EXCHANGES:
with store.ExchangeSeries() as exchange_series:
securities = exchange_series[exchange_name]
with flow.Progress(f'security-clean {exchange_name}', securities) as progress:
for security in securities:
progress(security.symbol)
with engine.SecuritySeries(interval, editable=True) as security_series:
time_series = security_series[security.symbol]
analyse.clean(time_series)
security_series *= time_series
LOG.info(f'Securities: {len(securities)} cleaned in the exchange: {exchange_name}')
def test_progress():
lst = [1, 2]
with flow.Progress(test_progress.__name__, lst) as progress:
progress('1')
progress('2')
def test_empty_list():
with flow.Progress(test_progress.__name__, []) as progress:
pass