def __list():
result = sheet.get_values(INDEX_RANGE)
values = result.get('values', [])
if not values:
return []
return [IndexRecord.build(value) for value in values]
def test_update_indexes(self):
indexes = [
IndexRecord.build(row)
for row in self.load_fixture('fixtures/sheet_indexes.json')
]
self.repo.update_indexes(indexes)
results = self.repo.list_indexes()
for result in results:
self.assertTrue(result in indexes)
def update_security_task(data_json, url):
data = json.loads(data_json)
index = IndexRecord.build_from_json(data['data'])
page = requests.get(url, timeout=(connect_timeout, read_timeout)).json()
start_date = data['start']
end_date = data['end']
tree = html.fromstring(page['html'])
rows = tree.xpath('//table//tbody//tr')
for row in rows:
security = SecurityRecord.build(row, index)
security_index = SecurityIndexRecord.build(security, index)
update_price_task.delay(__msg(security.data(), start_date, end_date))
repo.update_security(security)
repo.update_security_index(security_index)
def update_constituents_task(data_json):
"""Update the shares that consitute this index in the databases"""
data = json.loads(data_json)
start_date = data['start']
end_date = data['end']
logger.debug(data)
index = IndexRecord.build_from_json(data['data'])
if index == Repository.NULL_INDEX:
return
logger.debug(index.data())
for page in range(1, index.pages + 1):
data = __msg(index.data(), start_date, end_date)
url = index.url + str(page)
update_security_task.delay(data, url)
def test_update_securities(self):
index = IndexRecord.build(
self.load_fixture('fixtures/sheet_indexes.json')[0])
securities = [
SecurityRecord.build(row, index)
for row in self.__get_securities()
]
security_indexes = [
SecurityIndexRecord.build(security, index)
for security in securities
]
self.repo.update_securities(securities)
self.repo.update_security_indexes(security_indexes)
results = self.repo.list_securities(index=index)
for result in results:
self.assertTrue(result in securities)
def get_index(self, code):
stmt = select([IndexRecord.schema]).where(
IndexRecord.schema.c.code == code)
result = self.__get(stmt)
return IndexRecord.build_from_repo(result)
def list_indexes(self):
results = self.__list(select([IndexRecord.schema]))
return [IndexRecord.build_from_repo(result) for result in results]
class Repository:
NULL_INDEX = IndexRecord()
NULL_SECURITY = SecurityRecord()
def __init__(self, database_url):
self.log = logging.getLogger('repository')
self.engine = create_engine(database_url, pool_size=20, max_overflow=0)
def list_indexes(self):
results = self.__list(select([IndexRecord.schema]))
return [IndexRecord.build_from_repo(result) for result in results]
def get_index(self, code):
stmt = select([IndexRecord.schema]).where(
IndexRecord.schema.c.code == code)
result = self.__get(stmt)
return IndexRecord.build_from_repo(result)
def update_indexes(self, indexes):
self.log.info('save indexes')
self.__update_all(indexes)
def update_index(self, index):
self.__update(index)
def update_security_index(self, security_index):
self.__update(security_index)
def list_securities(self, index=None):
query = select([SecurityRecord.schema])
if index is not None:
query.where(
and_(SecurityRecord.schema.c.code ==
SecurityIndexRecord.schema.c.security_code,
SecurityRecord.schema.c.exchange_code ==
SecurityIndexRecord.schema.c.security_exchange, index.code
== SecurityIndexRecord.schema.c.index_code, index.code ==
SecurityIndexRecord.schema.c.index_exchange))
logger.info(query)
results = self.__list(query)
results = [result for result in results]
return [SecurityRecord.build_from_repo(result) for result in results]
def get_security(self, exchange, code):
stmt = select([SecurityRecord.schema]).where(
and_(SecurityRecord.schema.c.code == code,
SecurityRecord.schema.c.exchange_code == exchange))
result = self.__get(stmt)
return SecurityRecord.build_from_repo(result)
def update_securities(self, securities):
self.log.info('save securities')
self.__update_all(securities)
def update_security(self, security):
self.__update(security)
def update_security_indexes(self, security_indexes):
self.log.info('save security indexes')
self.__update_all(security_indexes)
def update_price_eod(self, price_eod):
self.__update(price_eod)
def __list(self, select):
logger.info(select)
with self.engine.connect() as conn:
return conn.execute(select)
def __get(self, select):
logger.info(select)
with self.engine.connect() as conn:
return conn.execute(select).first()
def __update(self, record):
self.__update_all([record])
def __update_all(self, records):
if not records:
return
with self.engine.connect() as conn:
for record in records:
conn.execute(record.upsert())
def test_update_index(self):
fixture = next(iter(self.load_fixture('fixtures/sheet_indexes.json')))
index = IndexRecord.build(fixture)
self.repo.update_index(index)
result = self.repo.get_index(index.code)
self.assertTrue(result == index)