def _process_tick(self, ticker_id, tick_type, value):
try:
symbol = self.ticker_id_to_symbol[ticker_id]
except KeyError:
log.error("Tick {} for id={} is not registered".format(tick_type, ticker_id))
return
self.tick_dict["SYMBOL_" + str(ticker_id)] = symbol
self.tick_dict[TickTypeEnum.to_str(tick_type) + "_" + str(ticker_id)] = value
def publish(self, model):
try:
log.info("Percent completed: %3.0f%% (%s - %s): %s" %
(model.percent_complete,
str(model.current_chunk_bounds[0].date()),
str(model.current_chunk_bounds[1].date()),
model.current_work))
except:
log.error("Cannot publish progress state.")
def _check_field(self, field, expected):
sql = "SELECT DISTINCT(value) as r FROM equity_supplementary_mappings WHERE field = '%s' ORDER BY r;" % field
ret = [x[0] for x in self.engine.execute(sql).fetchall()]
ok = np.array_equal(ret, expected)
if not ok:
sane = False
from sharadar.util.logger import log
log.error("Field '%s' changed!\nActual:\n %s\nExpected:\n %s" %
(field, ret, expected))
return False
else:
return True
def ingest(environ, asset_db_writer, minute_bar_writer, daily_bar_writer,
adjustment_writer, calendar, start_session, end_session, cache,
show_progress, output_dir):
# remove the output_dir with the timestamp, it should be empty
try:
os.rmdir(output_dir)
except OSError as e:
log.error("%s : %s" % (output_dir, e.strerror))
try:
_ingest(calendar, start_session, end_session)
except Exception as e:
log.error(traceback.format_exc())
def publish(self, model):
try:
start = str(model.current_chunk_bounds[0].date())
end = str(model.current_chunk_bounds[1].date())
completed = model.percent_complete
work = model.current_work
if start == end:
log.info("Percent completed: %3.0f%% (%s): %s" %
(completed, start, work))
else:
log.info("Percent completed: %3.0f%% (%s - %s): %s" %
(completed, start, end, work))
except:
log.error("Cannot publish progress state.")
def error(self, id_=None, error_code=None, error_msg=None):
if isinstance(id_, Exception):
log.exception(id_)
if isinstance(error_code, int):
if error_code in (502, 503, 326):
# 502: Couldn't connect to TWS.
# 503: The TWS is out of date and must be upgraded.
# 326: Unable connect as the client id is already in use.
self.unrecoverable_error = True
if error_code < 1000:
log.error("[{}] {} ({})".format(error_code, error_msg, id_))
else:
log.info("[{}] {} ({})".format(error_code, error_msg, id_))
else:
log.error("[{}] {} ({})".format(error_code, error_msg, id_))
def write(self, data):
self._validate(data)
df = data[['open', 'high', 'low', 'close', 'volume']]
with closing(sqlite3.connect(self._filename)) as con, con, closing(con.cursor()) as c:
properties = pd.Series({'calendar_name' : self._calendar.name})
properties.to_sql('properties', con, index_label='key', if_exists="replace")
with click.progressbar(length=len(df), label="Inserting price data...") as pbar:
count = 0
for index, row in df.iterrows():
sql = "INSERT OR REPLACE INTO prices (date, sid, open, high, low, close, volume) VALUES ('%s',%f,%f,%f,%f,%f,%f)"
values = index + tuple(row.values)
try:
c.execute(sql % values)
except sqlite3.OperationalError as e:
log.error("SqlError %s: %s" % (e, (sql % values)))
count += 1
pbar.update(count)
def _write(self, tablename, expected_dtypes, frame):
if frame is None or frame.empty:
# keeping the dtypes correct for empty frames is not easy
frame = pd.DataFrame(
np.array([], dtype=list(expected_dtypes.items())),
)
else:
if frozenset(frame.columns) != frozenset(expected_dtypes):
raise ValueError(
"Unexpected frame columns:\n"
"Expected Columns: %s\n"
"Received Columns: %s" % (
set(expected_dtypes),
frame.columns.tolist(),
)
)
actual_dtypes = frame.dtypes
for colname, expected in iteritems(expected_dtypes):
actual = actual_dtypes[colname]
if actual != expected:
raise TypeError(
"Expected data of type {expected} for column"
" '{colname}', but got '{actual}'.".format(
expected=expected,
colname=colname,
actual=actual,
),
)
with closing(sqlite3.connect(self._filename)) as con, con, closing(con.cursor()) as c:
with click.progressbar(length=len(frame), label="Inserting price data...") as pbar:
count = 0
for index, row in frame.iterrows():
sql = "INSERT OR REPLACE INTO %s VALUES ('%s', %s)"
cmd = sql % (tablename, index, ', '.join(map(str, row.values)))
try:
c.execute(cmd)
except sqlite3.OperationalError as e:
log.error(str(e) + ": " + cmd)
count += 1
pbar.update(count)