def test_store_data_frames(self):
# '''
df_cash_flow_before = _sync_cash_flow_statement(4564, 2013, to_year=2019)
db_repository = MongoDBRepository(MongoDBMeta.DATAFRAME_CASH_FLOW)
db_repository.put_data(4564, df_cash_flow_before)
data_frame = db_repository.get_data(4564)
print(data_frame)
print(data_frame.index)
# '''
# '''
df_profit_statement_before = _sync_profit_statement(4564, 2013, to_year=2019)
db_repository = MongoDBRepository(MongoDBMeta.DATAFRAME_PROFIT_STATEMENT)
db_repository.put_data(4564, df_profit_statement_before)
data_frame = db_repository.get_data(4564)
print(data_frame)
print(data_frame.index)
# '''
# '''
df_balance_sheet_before = _sync_balance_sheet(4564, 2013, to_year=2020)
print(df_balance_sheet_before)
db_repository = MongoDBRepository(MongoDBMeta.DATAFRAME_BALANCE_SHEET)
db_repository.put_data(4564, df_balance_sheet_before)
data_frame = db_repository.get_data(4564)
print(data_frame)
print(data_frame.columns)
# '''
# '''
df_dividend_before = _sync_dividend_policy(4564, 2013)
print(df_dividend_before)
print(df_dividend_before.index)
db_repository = MongoDBRepository(MongoDBMeta.DATAFRAME_DIVIDEND_POLICY)
db_repository.put_data(4564, df_dividend_before)
data_frame = db_repository.get_data(4564)
print(data_frame)
print(data_frame.index)
# '''
# '''
df_dividend_before = _sync_dividend_policy(4564, 2013)
print(df_dividend_before)
print(type(df_dividend_before.index))
self.assertIsInstance(df_dividend_before.index, pd.PeriodIndex)
db_repository = MongoDBRepository(MongoDBMeta.DATAFRAME_DIVIDEND_POLICY)
db_repository.put_data(4564, df_dividend_before)
data_frame = db_repository.get_data(4564)
print(data_frame)
print(data_frame.index)
self.assertIsInstance(data_frame.index, pd.PeriodIndex)
def test_fetch_data_utils(self):
'''
stock_code_list = get_stock_codes(stock_type='上市')
fetch_twse_price_measurement_raw_datas(stock_code_list[0: 1])
tpex_stock_code_list = get_stock_codes(stock_type='上櫃')
fetch_tpex_price_measurement_raw_datas(tpex_stock_code_list[0:1])
result = MongoDBRepository(MongoDBMeta.TPEX_PRICE_MEASUREMENT).get_data(stock_code_list[0])
self.assertIsNotNone(result)
'''
'''
fetch_dividend_policy_raw_datas(2884)
result = MongoDBRepository(MongoDBMeta.DIVIDEND_POLICY).get_data(2884)
self.assertIsNotNone(result)
'''
# '''
fetch_shareholder_equity_raw_data(2884, 2020, 3)
result = MongoDBRepository(MongoDBMeta.SHARE_HOLDER).get_data(2809, {'year': 2020, 'season': 3})
self.assertIsNotNone(result)
# '''
'''
fetch_simple_balance_sheet_raw_data(2884, 2020, 3)
result = MongoDBRepository(MongoDBMeta.SIMPLE_BALANCE_SHEET).get_data(2884, {'year': 2020, 'season': 3})
self.assertIsNotNone(result)
'''
'''
fetch_balance_sheet_raw_data(2884, 2020, 3)
result = MongoDBRepository(MongoDBMeta.FULL_BALANCE_SHEET).get_data(2884, {'year': 2020, 'season': 3})
self.assertIsNotNone(result)
'''
'''
def test_sync_performance(self):
_sync_performance(2841)
_repository = MongoDBRepository(MongoDBMeta.DATAFRAME_PERFORMANCE)
content = _repository.get_data(2841)
print(content)
print('index = ', content.index)
print('columns = ', content.columns)
def __init__(self, stock_id):
super().__init__(stock_id)
self.__tag = "ShareholderEquityProcessor"
self.__repository = MongoDBRepository(MongoDBMeta.SHARE_HOLDER)
# self._data_fetcher = _ShareholderEquityFetcher()
self.items_to_get = ('期初餘額', '期末餘額')
self.fields_to_get = ('權益總額', )
def test_sync_statements(self):
# '''
db_repository = MongoDBRepository(MongoDBMeta.DATAFRAME_CASH_FLOW)
data_frame_before = db_repository.get_data(2841)
data_frame_after = _sync_cash_flow_statement(2841, 2013, to_year=2021, df_cash_flow_statement=data_frame_before)
print('before = ', data_frame_before)
print('after = ', data_frame_after)
# '''
# '''
db_repository = MongoDBRepository(MongoDBMeta.DATAFRAME_PROFIT_STATEMENT)
data_frame_before = db_repository.get_data(2841)
data_frame_after = _sync_profit_statement(2841, 2013, df_profit_statement=data_frame_before)
print('before = ', data_frame_before)
print('after = ', data_frame_after)
# '''
# '''
db_repository = MongoDBRepository(MongoDBMeta.DATAFRAME_BALANCE_SHEET)
data_frame_before = db_repository.get_data(2841)
data_frame_after = _sync_balance_sheet(2841, 2013, 2019, df_balance_sheet=data_frame_before)
print('before = ', data_frame_before)
print('after = ', data_frame_after)
def __init__(self):
super().__init__(None)
self.dividend_policy_fetcher = _DividendPolicyFetcher2()
self.__repository = MongoDBRepository(MongoDBMeta.DIVIDEND_POLICY)
def __init__(self):
self.__repository = MongoDBRepository(MongoDBMeta.SIMPLE_BALANCE_SHEET)
def __init__(self):
self.__repository = MongoDBRepository(MongoDBMeta.STOCK_COUNT)
def __init__(self, stock_id):
super().__init__(stock_id)
self.__repository = MongoDBRepository(MongoDBMeta.CASH_FLOW)
self._fetch_fields = ('營業活動之淨現金流入', '取得不動產、廠房及設備', '其他投資活動',
'投資活動之淨現金流入')
import pandas as pd
from evaluation_utils import get_stock_list
from rdss.balance_sheet import SimpleBalanceSheetProcessor
from rdss.cashflow_statment import CashFlowStatementProcessor
from rdss.dividend_policy2 import DividendPolicyProcessor2
from rdss.shareholder_equity import ShareholderEquityProcessor
from rdss.statement_fetchers import SimpleIncomeStatementProcessor
from rdss.stock_count import StockCountProcessor
from repository.mongodb_repository import MongoDBRepository, MongoDBMeta
from stock_data import store_df, read_dfs
from twse_crawler import gen_output_path
from utils import get_time_lines
from value_measurement import PriceMeasurementProcessor2
_cash_flow_repository = MongoDBRepository(MongoDBMeta.DATAFRAME_CASH_FLOW)
_profit_statement_repository = MongoDBRepository(MongoDBMeta.DATAFRAME_PROFIT_STATEMENT)
_balance_sheet_repository = MongoDBRepository(MongoDBMeta.DATAFRAME_BALANCE_SHEET)
_dividend_policy_repository = MongoDBRepository(MongoDBMeta.DATAFRAME_DIVIDEND_POLICY)
_performance_repository = MongoDBRepository(MongoDBMeta.DATAFRAME_PERFORMANCE)
class Option(enum.IntEnum):
BALANCE_SHEET = 1
PROFIT_STATEMENT = 1 << 2
CASH_FLOW_STATEMENT = 1 << 3
DIVIDEND_POLICY = 1 << 4
ALL = BALANCE_SHEET | PROFIT_STATEMENT | CASH_FLOW_STATEMENT | DIVIDEND_POLICY
def sync_statements(stock_codes, times_to_retry=10, break_after_retry=True, option=Option.ALL, isSync=True):
PATH_DIR_RAW_DATA_SHAREHOLDER_EQUITY = "out/raw_datas/shareholder_equity/"
PATH_DIR_RAW_DATA_DIVIDEND_POLICY = "out/raw_datas/dividend_policy"
PATH_DIR_RAW_DATA_STOCK_COUNT = "out/raw_datas/stock_count/"
PATH_DIR_RAW_DATA_CASH_FLOW = "out/raw_datas/cash_flow/"
PATH_DIR_RAW_DATA_PRICE_MEASUREMENT = "out/raw_datas/price_measurement/"
PATH_DIR_RAW_DATA_TPEX_PRICE_MEASUREMENT = "out/raw_datas/tpex_price_measurement"
__balance_sheet_data_fetcher = DataFetcher('https://mops.twse.com.tw/mops/web/ajax_t164sb03')
__simple_balance_sheet_data_fetcher = DataFetcher('https://mops.twse.com.tw/mops/web/ajax_t163sb01')
__shareholder_equity_fetcher = DataFetcher('https://mops.twse.com.tw/mops/web/ajax_t164sb06')
__dividend_policy_fetcher = DataFetcher('https://mops.twse.com.tw/mops/web/ajax_t05st09_2')
__stock_count_fetcher = DataFetcher('https://mops.twse.com.tw/mops/web/ajax_t16sn02')
__cash_flow_fetcher = DataFetcher('https://mops.twse.com.tw/mops/web/ajax_t164sb05')
__stock_count_repository = MongoDBRepository(MongoDBMeta.STOCK_COUNT)
__twse_price_measurement_repository = MongoDBRepository(MongoDBMeta.TWSE_PRICE_MEASUREMENT)
__tpex_price_measurement_repository = MongoDBRepository(MongoDBMeta.TPEX_PRICE_MEASUREMENT)
__dividend_policy_repository = MongoDBRepository(MongoDBMeta.DIVIDEND_POLICY)
__shareholder_repository = MongoDBRepository(MongoDBMeta.SHARE_HOLDER)
__simple_balance_sheet_repository = MongoDBRepository(MongoDBMeta.SIMPLE_BALANCE_SHEET)
__full_balance_sheet_repository = MongoDBRepository(MongoDBMeta.FULL_BALANCE_SHEET)
__cash_flow_repository = MongoDBRepository(MongoDBMeta.CASH_FLOW)
__logger = logging.getLogger("twse.DataFetcher")
# mongo_client = MongoClient('localhost', 27017)
mongo_client = MongoClient('192.168.1.109', 27017)
DB_TWSE = "TWSE"
TABLE_TWSE_PRICE_MEASUREMENT = "twse_price_measurement"
TABLE_TPEX_PRICE_MEASUREMENT = "tpex_price_measurement"
def __init__(self):
self.__in_repository = MongoDBRepository(MongoDBMeta.TPEX_PRICE_MEASUREMENT)
self.__out_repository = _data_frame_repository
print(revamp_list)
return revamp_list
def __data_frame_in_transform(content):
data_frame = pd.read_json(content, orient='split', typ='frame')
print(data_frame.index.values)
index_dict = {item: pd.Period(value=str(item)) for item in data_frame.index.values}
new_data_frame = data_frame.rename(index_dict)
return new_data_frame
__data_frame_repository_transformer = Transformer(in_transform=yearly_period_data_frame_in_transform,
out_transform=default_data_frame_out_transform)
_data_frame_repository = MongoDBRepository(MongoDBMeta.DATAFRAME_PRICE_MEASUREMENT,
transformer=__data_frame_repository_transformer)
class PriceMeasurementProcessor2:
def __init__(self):
self.__twsePriceTransformer = TWSEPriceMeasurementTransformer()
self.__tpexPriceTransformer = TPEXPriceMeasurementTransformer()
from evaluation_utils import get_stock_codes
self.list_twse = get_stock_codes(stock_type='上市')
self.list_tpex = get_stock_codes(stock_type='上櫃')
def get_data_frame(self, stock_id):
data_frame = _data_frame_repository.get_data(stock_id)
current_years = None
if data_frame is not None:
current_years = list(map(lambda year_index: int(year_index.year), data_frame.index.values))
current_years.sort()