def import_fund(code, from_date, to_date): """Imports fund data from KOFIA. :param code: e.g., KR5223941018 :param from_date: e.g., 2016-01-01 :param to_date: e.g., 2016-02-28 """ provider = Kofia() app = create_app(__name__) with app.app_context(): asset = get_asset_by_fund_code(code) # FIXME: Target asset should also be determined by asset.data.code base_asset = Asset.query.filter_by(name='KRW').first() data = provider.fetch_data( code, parse_date(from_date), parse_date(to_date)) for date, unit_price, quantity in data: log.info('Import data on {}', date) unit_price /= 1000.0 try: AssetValue.create( asset=asset, base_asset=base_asset, evaluated_at=date, close=unit_price, granularity=Granularity.day, source='kofia') except IntegrityError: log.warn('Identical record has been found for {}. Skipping.', date) db.session.rollback()
def insert_test_data(): """Inserts some sample data for testing.""" app = create_app(__name__) with app.app_context(): user = User.create( family_name='Byeon', given_name='Sumin', email='*****@*****.**', ignore_if_exists=True) account_checking = create_account( AccountType.checking, 'Shinhan', 'checking', user) account_stock = create_account( AccountType.investment, 'Mirae Asset', 'stock', user) asset_krw = create_asset(AssetType.currency, 'KRW', 'Korean Won') create_asset(AssetType.currency, 'USD', 'United States Dollar') for _ in insert_stock_assets(): pass create_asset(AssetType.security, 'KR5223941018', 'KB S&P500') create_asset(AssetType.security, 'KR5229221225', '이스트스프링차이나') portfolio = Portfolio() portfolio.base_asset = asset_krw portfolio.add_accounts(account_checking, account_stock)
def import_sp500_records(): """Import S&P500 fund sample data. Expects a tab seprated value document. """ app = create_app(__name__) app.app_context().push() account_checking = Account.get(id=1001) account_sp500 = Account.get(id=7001) asset_krw = Asset.query.filter_by(name='KRW').first() asset_sp500 = Asset.query.filter_by(name='KB S&P500').first() # Expected number of columns expected_col_count = 6 with open('sample-data/sp500.csv') as fin: # Skip the first row (headers) headers = next(fin) col_count = len(headers.split()) if col_count != expected_col_count: raise Exception( 'Expected number of columns = {}, ' 'actual number of columns = {}'.format( expected_col_count, col_count)) for line in fin: cols = line.split('\t') if len(cols) != expected_col_count: continue date = parse_date(cols[0], '%Y.%m.%d') _type = cols[1] quantity_krw, quantity_sp500 = \ [int(extract_numbers(v)) for v in cols[3:5]] log.info(', '.join([c.strip() for c in cols])) if not (_type == '일반입금' or _type == '일반신규'): log.info('Record type \'{}\' will be ignored', _type) continue with Transaction.create() as t: # NOTE: The actual deposit date and the buying date generally # differ by a few days. Need to figure out how to parse this # properly from the raw data. try: Record.create( created_at=date, account=account_checking, asset=asset_krw, quantity=-quantity_krw, transaction=t) except IntegrityError: log.warn('Identical record exists') db.session.rollback() try: Record.create( created_at=date, account=account_sp500, asset=asset_sp500, quantity=quantity_sp500, transaction=t) except IntegrityError: log.warn('Identical record exists') db.session.rollback()
def import_sp500(): app = create_app(__name__) with app.app_context(): account_checking = Account.get(id=1001) account_sp500 = Account.get(id=7001) asset_krw = Asset.query.filter_by(name='KRW').first() asset_sp500 = Asset.query.filter_by(name='KB S&P500').first() with open('sample-data/sp500.csv') as fin: for line in fin: cols = line.split() if len(cols) != 5: continue date = parse_date(cols[0], '%Y.%m.%d') _type = cols[1] quantity_krw, quantity_sp500 = \ [int(extract_numbers(v)) for v in cols[2:4]] print(cols) withdraw = _type == '일반입금' with Transaction.create() as t: if withdraw: Record.create( created_at=date, account=account_checking, asset=asset_krw, quantity=-quantity_krw, transaction=t) Record.create( created_at=date, account=account_sp500, asset=asset_sp500, quantity=quantity_sp500, transaction=t) print(account_sp500.net_worth(make_date('2016-02-25'), target_asset=asset_krw))
def insert_test_data(): app = create_app(__name__) with app.app_context(): user = User.create(family_name='Byeon', given_name='Sumin', email='*****@*****.**') account_checking, _, _, _, _, account_stock, account_8p, _ = \ insert_accounts(user) for _ in insert_stock_assets(): pass asset_krw = insert_asset('currency, KRW, Korean Won') asset_usd = insert_asset('currency, USD, United States Dollar') asset_gold = insert_asset('commodity, Gold, Gold') asset_sp500 = insert_asset('security, KB S&P500,', data={'code': 'KR5223941018'}) asset_esch = insert_asset('security, 이스트스프링차이나펀드,', data={'code': 'KR5229221225'}) asset_kjp = insert_asset('security, 키움일본인덱스,', data={'code': 'KR5206689717'}) asset_hf1 = insert_asset('bond, 포트폴리오 투자상품 1호,') portfolio = Portfolio() portfolio.base_asset = asset_krw portfolio.add_accounts(account_checking, account_stock, account_8p)
def import_stock_values(code, from_date, to_date): """Import stock price information.""" app = create_app(__name__) with app.app_context(): # NOTE: We assume all Asset records are already in the database, but # this is a temporary workaround. We should implement some mechanism to # automatically insert an Asset record when it is not found. import_stock_values_(code, parse_date(from_date), parse_date(to_date))
def import_sp500_records(): """Import S&P500 fund sample data. Expects a tab seprated value document. """ app = create_app(__name__) app.app_context().push() account_checking = Account.get(id=1001) account_sp500 = Account.get(id=7001) asset_krw = Asset.query.filter_by(name='KRW').first() asset_sp500 = Asset.query.filter_by(name='KB S&P500').first() # Expected number of columns expected_col_count = 6 with open('sample-data/sp500.csv') as fin: # Skip the first row (headers) headers = next(fin) col_count = len(headers.split()) if col_count != expected_col_count: raise Exception( 'Expected number of columns = {}, ' 'actual number of columns = {}'.format( expected_col_count, col_count)) for line in fin: cols = line.split('\t') if len(cols) != expected_col_count: continue date = parse_date(cols[0], '%Y.%m.%d') _type = cols[1] quantity_krw, quantity_sp500 = \ [int(extract_numbers(v)) for v in cols[3:5]] log.info(', '.join([c.strip() for c in cols])) if not (_type == '일반입금' or _type == '일반신규'): log.info('Record type \'{}\' will be ignored', _type) continue with Transaction.create() as t: # NOTE: The actual deposit date and the buying date generally # differ by a few days. Need to figure out how to parse this # properly from the raw data. try: deposit(account_checking, asset_krw, -quantity_krw, date, t) except IntegrityError: log.warn('Identical record exists') db.session.rollback() try: deposit(account_sp500, asset_sp500, quantity_sp500, date, t) except IntegrityError: log.warn('Identical record exists') db.session.rollback()
def import_stock_records(filename): """Parses exported data from the Shinhan HTS.""" app = create_app(__name__) with app.app_context(): account_bank = Account.query \ .filter(Account.name == '신한 입출금').first() account_stock = Account.query \ .filter(Account.name == '신한 주식').first() with open(filename) as fin: for parsed in parse_stock_records(fin): insert_stock_record(parsed, account_stock, account_bank)
def import_stock_values(code): """Import stock price information.""" app = create_app(__name__) with app.app_context(): # NOTE: We assume all Asset records are already in the database, but # this is a temporary workaround. We should implement some mechanism to # automatically insert an Asset record when it is not found. stdin = click.get_text_stream('stdin') for _ in import_stock_values_(stdin, code): pass
def import_miraeasset_foreign_data(filename, account_institution, account_number): """Imports a CSV file exported in 해외거래내역 (9465).""" from finance.importers import import_miraeasset_foreign_records app = create_app(__name__) with app.app_context(): account = Account.get_by_number(account_institution, account_number) with open(filename) as fin: import_miraeasset_foreign_records(fin, account)
def import_rf(): app = create_app(__name__) app.app_context().push() account = Account.get(id=1001) asset = Asset.query.filter_by(name='KRW').first() with open('sample-data/rf.txt') as fin: for line in fin: if line.strip(): insert_record(line, account, asset, None)
def import_miraeasset_foreign_data( filename, account_institution, account_number ): """Imports a CSV file exported in 해외거래내역 (9465).""" from finance.importers import import_miraeasset_foreign_records app = create_app(__name__) with app.app_context(): account = Account.get_by_number(account_institution, account_number) with open(filename) as fin: import_miraeasset_foreign_records(fin, account)
def fetch_asset_values_handler(event, context): config = {'SQLALCHEMY_DATABASE_URI': os.environ['DB_URL']} sqs_region = os.environ['SQS_REGION'] queue_url = os.environ['REQUEST_IMPORT_STOCK_VALUES_QUEUE_URL'] app = create_app(__name__, config=config) with app.app_context(): requests = poll_import_stock_values_requests(sqs_region, queue_url) for request in requests: code = request['code'] start_time = datetime.fromtimestamp(request['start_time']) end_time = datetime.fromtimestamp(request['end_time']) fetch_asset_values(code, start_time, end_time)
def insert_test_data(): app = create_app(__name__) with app.app_context(): user = User.create(family_name='Byeon', given_name='Sumin', email='*****@*****.**') account_checking = Account.create(id=1001, type='checking', name='Shinhan Checking', user=user) account_gold = Account.create(id=9001, type='investment', name='Woori Gold Banking', user=user) account_sp500 = Account.create(id=7001, type='investment', name='S&P500 Fund', user=user) account_esch = Account.create(id=7002, type='investment', name='East Spring China Fund', user=user) account_kjp = Account.create(id=7003, type='investment', name='키움일본인덱스 주식재간접', user=user) account_8p = Account.create(id=8001, type='investment', name='8퍼센트', user=user) account_hf = Account.create(id=8002, type='virtual', name='어니스트펀드', user=user) asset_krw = insert_asset('currency, KRW, Korean Won') asset_usd = insert_asset('currency, USD, United States Dollar') asset_gold = insert_asset('commodity, Gold, Gold') asset_sp500 = insert_asset('security, KB S&P500,', data={'code': 'KR5223941018'}) asset_esch = insert_asset('security, 이스트스프링차이나펀드,', data={'code': 'KR5229221225'}) asset_kjp = insert_asset('security, 키움일본인덱스,', data={'code': 'KR5206689717'}) asset_hf1 = insert_asset('bond, 포트폴리오 투자상품 1호,') portfolio = Portfolio() portfolio.target_asset = asset_krw portfolio.add_accounts(account_checking, account_8p)
def import_8percent(filename): """Imports a single file.""" app = create_app(__name__) with app.app_context(): with open(filename) as fin: raw = fin.read() account_8p = Account.query.get(8001) account_checking = Account.query.filter( Account.name == 'Shinhan Checking').first() asset_krw = Asset.query.filter(Asset.name == 'KRW').first() parsed_data = parse_8percent_data(raw) import_8percent_data( parsed_data, account_checking=account_checking, account_8p=account_8p, asset_krw=asset_krw)
def app(request): """Session-wide test `Flask` application.""" settings_override = { 'TESTING': True, } app = create_app(__name__, config=settings_override) # Establish an application context before running the tests. ctx = app.app_context() ctx.push() def teardown(): ctx.pop() request.addfinalizer(teardown) return app
def import_8percent(filename): """Imports a single file.""" app = create_app(__name__) with app.app_context(): with open(filename) as fin: raw = fin.read() account_8p = Account.query.get(8001) account_checking = Account.query.filter( Account.name == 'Shinhan Checking').first() asset_krw = Asset.query.filter(Asset.name == 'KRW').first() parsed_data = parse_8percent_data(raw) import_8percent_data(parsed_data, account_checking=account_checking, account_8p=account_8p, asset_krw=asset_krw)
def app(request): """Session-wide test `Flask` application.""" settings_override = { 'TESTING': True, } settings_override['SQLALCHEMY_DATABASE_URI'] = os.environ['TEST_DB_URL'] app = create_app(__name__, config=settings_override) # Establish an application context before running the tests. # ctx = app.app_context() # ctx.push() # def teardown(): # ctx.pop() # request.addfinalizer(teardown) return app
def import_8percent(filename): """Imports a single file.""" app = create_app(__name__) provider = _8Percent() with app.app_context(): with open(filename) as fin: raw = fin.read() account_8p = Account.query.filter(Account.name == '8퍼센트').first() account_checking = Account.query.filter( Account.name == '신한은행 입출금').first() asset_krw = Asset.query.filter(Asset.name == 'KRW').first() parsed_data = provider.parse_data(raw) import_8percent_data(parsed_data, account_checking=account_checking, account_8p=account_8p, asset_krw=asset_krw)
def import_dart(fin): """Import DART (전자공시) data.""" try: data = json.loads(fin.read()) except json.decoder.JSONDecodeError as e: log.error('Valid JSON data expected: {}', e) app = create_app(__name__) with app.app_context(): for row in data: try: report = DartReport.create(**row) except IntegrityError: log.info('DartReport-{} already exists', row['id']) db.session.rollback() else: log.info('Fetched report: {}', report)
def import_sp500(): app = create_app(__name__) with app.app_context(): account_checking = Account.get(id=1001) account_sp500 = Account.get(id=7001) asset_krw = Asset.query.filter_by(name='KRW').first() asset_sp500 = Asset.query.filter_by(name='KB S&P500').first() with open('sample-data/sp500.csv') as fin: for line in fin: cols = line.split() if len(cols) != 5: continue date = parse_date(cols[0], '%Y.%m.%d') _type = cols[1] quantity_krw, quantity_sp500 = \ [int(extract_numbers(v)) for v in cols[2:4]] print(cols) withdraw = _type == '일반입금' with Transaction.create() as t: if withdraw: Record.create(created_at=date, account=account_checking, asset=asset_krw, quantity=-quantity_krw, transaction=t) Record.create(created_at=date, account=account_sp500, asset=asset_sp500, quantity=quantity_sp500, transaction=t) print( account_sp500.net_worth(make_date('2016-02-25'), target_asset=asset_krw))
def insert_test_data(): app = create_app(__name__) with app.app_context(): user = User.create( family_name='Byeon', given_name='Sumin', email='*****@*****.**') account_checking = Account.create( id=1001, type='checking', name='Shinhan Checking', user=user) account_gold = Account.create( id=9001, type='investment', name='Woori Gold Banking', user=user) account_sp500 = Account.create( id=7001, type='investment', name='S&P500 Fund', user=user) account_esch = Account.create( id=7002, type='investment', name='East Spring China Fund', user=user) account_kjp = Account.create( id=7003, type='investment', name='키움일본인덱스 주식재간접', user=user) account_8p = Account.create( id=8001, type='investment', name='8퍼센트', user=user) account_hf = Account.create( id=8002, type='virtual', name='어니스트펀드', user=user) asset_krw = insert_asset('currency, KRW, Korean Won') asset_usd = insert_asset('currency, USD, United States Dollar') asset_gold = insert_asset('commodity, Gold, Gold') asset_sp500 = insert_asset('security, KB S&P500,', data={'code': 'KR5223941018'}) asset_esch = insert_asset('security, 이스트스프링차이나펀드,', data={'code': 'KR5229221225'}) asset_kjp = insert_asset('security, 키움일본인덱스,', data={'code': 'KR5206689717'}) asset_hf1 = insert_asset('bond, 포트폴리오 투자상품 1호,') portfolio = Portfolio() portfolio.target_asset = asset_krw portfolio.add_accounts(account_checking, account_8p)
import os from finance import create_app if __name__ == '__main__': application = create_app() host = os.environ.get('HOST', '0.0.0.0') port = int(os.environ.get('PORT', 8002)) debug = bool(os.environ.get('DEBUG', False)) application.run(host=host, port=port, debug=debug)
def drop_all(): """Drops all database tables.""" app = create_app(__name__) with app.app_context(): db.drop_all()
def create_all(): """Creates necessary database tables.""" app = create_app(__name__) with app.app_context(): db.create_all()
def import_fund(code, from_date, to_date): """Imports fund data from KOFIA. :param from_date: e.g., 20160101 :param to_date: e.g., 20160228 """ url = 'http://dis.kofia.or.kr/proframeWeb/XMLSERVICES/' headers = { 'Origin': 'http://dis.kofia.or.kr', 'Accept-Encoding': 'gzip, deflate', 'Accept-Language': 'en-US,en;q=0.8,ko;q=0.6', 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) ' 'AppleWebKit/537.36 (KHTML, like Gecko) ' 'Chrome/48.0.2564.109 Safari/537.36', 'Content-Type': 'text/xml', 'Accept': 'text/xml', 'Referer': 'http://dis.kofia.or.kr/websquare/popup.html?w2xPath=' '/wq/com/popup/DISComFundSmryInfo.xml&companyCd=20090602&' 'standardCd=KR5223941018&standardDt=20160219&grntGb=S&' 'search=&check=1&isMain=undefined&companyGb=A&uFundNm=' '/v8ASwBCwqTQwLv4rW0AUwAmAFAANQAwADDHeLNxwqTJna2Mx5DSLMeQwu' 'DQwQBbyPzC3QAt0wzA%0A3dYVAF0AQwAtAEU%3D&popupID=undefined&' 'w2xHome=/wq/fundann/&w2xDocumentRoot=', } data = """<?xml version="1.0" encoding="utf-8"?> <message> <proframeHeader> <pfmAppName>FS-COM</pfmAppName> <pfmSvcName>COMFundPriceModSO</pfmSvcName> <pfmFnName>priceModSrch</pfmFnName> </proframeHeader> <systemHeader></systemHeader> <COMFundUnityInfoInputDTO> <standardCd>{code}</standardCd> <companyCd>A01031</companyCd> <vSrchTrmFrom>{from_date}</vSrchTrmFrom> <vSrchTrmTo>{to_date}</vSrchTrmTo> <vSrchStd>1</vSrchStd> </COMFundUnityInfoInputDTO> </message> """.format(code=code, from_date=from_date, to_date=to_date) resp = requests.post(url, headers=headers, data=data) app = create_app(__name__) with app.app_context(): # NOTE: I know this looks really stupid, but we'll stick with this # temporary workaround until we figure out how to create an instance of # Asset model from a raw query result # (sqlalchemy.engine.result.RowProxy) query = "SELECT * FROM asset WHERE data->>'code' = :code LIMIT 1" raw_asset = db.session.execute(query, {'code': code}).first() if not raw_asset: raise AssetNotFoundException( 'Fund code {} is not mapped to any asset'.format(code)) asset_id = raw_asset[0] asset = Asset.query.get(asset_id) # FIXME: Target asset should also be determined by asset.data.code target_asset = Asset.query.filter_by(name='KRW').first() schema = AssetValueSchema() schema.load(resp.text) for date, unit_price, original_quantity in schema.get_data(): log.info('Import data on {}', date) unit_price /= 1000.0 try: AssetValue.create( asset=asset, target_asset=target_asset, evaluated_at=date, close=unit_price, granularity='1day') except IntegrityError: log.warn('Identical record has been found for {}. Skipping.', date) db.session.rollback()
def drop_all(): app = create_app(__name__) with app.app_context(): db.drop_all()
def import_fund(code, from_date, to_date): """Imports fund data from KOFIA. :param from_date: e.g., 20160101 :param to_date: e.g., 20160228 """ url = 'http://dis.kofia.or.kr/proframeWeb/XMLSERVICES/' headers = { 'Origin': 'http://dis.kofia.or.kr', 'Accept-Encoding': 'gzip, deflate', 'Accept-Language': 'en-US,en;q=0.8,ko;q=0.6', 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) ' 'AppleWebKit/537.36 (KHTML, like Gecko) ' 'Chrome/48.0.2564.109 Safari/537.36', 'Content-Type': 'text/xml', 'Accept': 'text/xml', 'Referer': 'http://dis.kofia.or.kr/websquare/popup.html?w2xPath=' '/wq/com/popup/DISComFundSmryInfo.xml&companyCd=20090602&' 'standardCd=KR5223941018&standardDt=20160219&grntGb=S&' 'search=&check=1&isMain=undefined&companyGb=A&uFundNm=' '/v8ASwBCwqTQwLv4rW0AUwAmAFAANQAwADDHeLNxwqTJna2Mx5DSLMeQwu' 'DQwQBbyPzC3QAt0wzA%0A3dYVAF0AQwAtAEU%3D&popupID=undefined&' 'w2xHome=/wq/fundann/&w2xDocumentRoot=', } data = """<?xml version="1.0" encoding="utf-8"?> <message> <proframeHeader> <pfmAppName>FS-COM</pfmAppName> <pfmSvcName>COMFundPriceModSO</pfmSvcName> <pfmFnName>priceModSrch</pfmFnName> </proframeHeader> <systemHeader></systemHeader> <COMFundUnityInfoInputDTO> <standardCd>{code}</standardCd> <companyCd>A01031</companyCd> <vSrchTrmFrom>{from_date}</vSrchTrmFrom> <vSrchTrmTo>{to_date}</vSrchTrmTo> <vSrchStd>1</vSrchStd> </COMFundUnityInfoInputDTO> </message> """.format(code=code, from_date=from_date, to_date=to_date) resp = requests.post(url, headers=headers, data=data) app = create_app(__name__) with app.app_context(): # NOTE: I know this looks really stupid, but we'll stick with this # temporary workaround until we figure out how to create an instance of # Asset model from a raw query result # (sqlalchemy.engine.result.RowProxy) query = "SELECT * FROM asset WHERE data->>'code' = :code LIMIT 1" raw_asset = db.session.execute(query, {'code': code}).first() if not raw_asset: raise AssetNotFoundException( 'Fund code {} is not mapped to any asset'.format(code)) asset_id = raw_asset[0] asset = Asset.query.get(asset_id) # FIXME: Target asset should also be determined by asset.data.code target_asset = Asset.query.filter_by(name='KRW').first() schema = AssetValueSchema() schema.load(resp.text) for date, unit_price, original_quantity in schema.get_data(): log.info('Import data on {}', date) unit_price /= 1000.0 try: AssetValue.create(asset=asset, target_asset=target_asset, evaluated_at=date, close=unit_price, granularity='1day') except IntegrityError: log.warn('Identical record has been found for {}. Skipping.', date) db.session.rollback()
from finance import create_app app = create_app() if __name__ == "__main__": app.run(host='0.0.0.0')
import os from finance import create_app application = create_app() if __name__ == '__main__': host = os.environ.get('HOST', '0.0.0.0') port = int(os.environ.get('PORT', 8002)) application.run(host=host, port=port)
from finance import create_app app = create_app()