def process_business_plan_data(sec_id, symbol):
path = URL_BUSINESS_PLAN_BASE.format(symbol)
headers = {
"Accept":
"application/json, text/javascript, */*; q=0.01",
"Accept-Encoding":
"gzip, deflate",
"Accept-Language":
"en - US, en;q = 0.9, vi;q = 0.8",
"Connection":
"keep-alive",
"Host":
"e.cafef.vn",
"Origin":
"http://s.cafef.vn",
"Referer":
"http://s.cafef.vn",
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36",
}
try:
response = requests.get(path, headers=headers)
data = json.loads(response.text)
if data is None:
return
for item in data:
year_crawl = item.get('KYear')
total_income_crawl = item.get('Totalncome')
profit_crawl = item.get('TotalProfit')
net_income_crawl = item.get('Netlncome')
dividend_stock_crawl = item.get('DivStock')
dividend_money_crawl = item.get('Dividend')
records = get_business_plan(sec_id, year_crawl)
if len(records) == 0:
x = 1000000000
business = BusinessPlanData(sec_id, year_crawl,
total_income_crawl * x,
profit_crawl * x,
net_income_crawl * x,
dividend_stock_crawl,
dividend_money_crawl)
insert_object(business)
except Exception as e:
error('Exception %s' % str(e))
error('process_business_plan_data: %s' % symbol)
error('========================***========================')
def process_quarterly_data(sec_id, symbol, start_year, end_year):
path = URL_BASE_QUARTER.format(symbol, start_year - 1, 1, end_year, 4)
response = requests.get(path, headers=headers)
try:
data = json.loads(response.text)
for year in range(start_year - 1, end_year + 1):
for quarter in range(1, 5):
records = get_quarterly_data(sec_id, year, quarter)
if len(records) > 0:
continue
for item in data:
year_crawl = item.get("Year")
quarter_crawl = item.get("Quarter")
roe_crawl = item.get("ROE_TTM")
roa_crawl = item.get("ROA_TTM")
roic_crawl = item.get("ROIC_TTM")
eps_crawl = item.get("BasicEPS_MRQ")
cash_crawl = item.get("Cash_MRQ")
assets_crawl = item.get("TotalAssets_MRQ")
equity_crawl = item.get("Equity_MRQ")
profit_after_tax_crawl = item.get("ProfitAfterTax_MRQ")
profit_before_tax_crawl = item.get("ProfitBeforeTax_MRQ")
revenue_crawl = item.get("NetSales_MRQ")
liabilities_crawl = item.get("Liabilities_MRQ")
current_liabilities_crawl = item.get(
"CurrentLiabilities_MRQ")
bookvalue_per_share_crawl = item.get(
"BookValuePerShare_MRQ")
sales_per_share_crawl = item.get("SalesPerShare_TTM")
if year == year_crawl and quarter == quarter_crawl:
data_db = QuarterlyData(
sec_id, year_crawl, quarter_crawl, roe_crawl,
roa_crawl, roic_crawl, eps_crawl, cash_crawl,
assets_crawl, equity_crawl, profit_after_tax_crawl,
profit_before_tax_crawl, revenue_crawl,
liabilities_crawl, current_liabilities_crawl,
bookvalue_per_share_crawl, sales_per_share_crawl)
insert_object(data_db)
break
info('done process_quarterly_data: {0}'.format(symbol))
except Exception as e:
error('Exception %s' % str(e))
error('process_quarterly_data: %s' % symbol)
error('========================***========================')
def process_daily_data(sec_id, symbol, group_type, start_date, end_date):
path = URL_DAILY_DATA_BASE.format(symbol, start_date, end_date)
path_inventory = URL_INVENTORY_BASE.format(symbol)
response = requests.get(path, headers=headers)
response_inventory = requests.get(path_inventory, headers=headers)
try:
data = json.loads(response.text)
data_inventory = json.loads(response_inventory.text)
historical_datas = []
daily_datas = []
is_adj = False
for item in data:
date = item.get("Date")
price_open = item.get("PriceOpen")
price_high = item.get("PriceHigh")
price_low = item.get("PriceLow")
price_close = item.get("PriceClose")
adj_close = item.get("AdjClose")
adj_open = item.get("AdjOpen")
adj_high = item.get("AdjHigh")
adj_low = item.get("AdjLow")
if adj_close != price_close:
is_adj = True
deal_volume = item.get("DealVolume")
put_through_volume = item.get("PutthroughVolume")
total_volume = item.get("Volume")
buy_foreign_quantity = item.get("BuyForeignQuantity")
sell_foreign_quantity = item.get("SellForeignQuantity")
buy_quantity = item.get("BuyQuantity")
sell_quantity = item.get("SellQuantity")
market_cap = item.get("MarketCap")
state_ownership = data_inventory.get("StateOwnership")
foreign_ownership = data_inventory.get("ForeignOwnership")
other_ownership = data_inventory.get("OtherOwnership")
if date is None:
continue
fn_date = str(date)[:10]
historical_data = HistoricalData(sec_id, fn_date, price_open,
price_high, price_low,
price_close, adj_open, adj_high,
adj_low, adj_close, deal_volume,
put_through_volume, total_volume,
datetime.datetime.now())
daily_data = DailyData(sec_id, fn_date, buy_quantity,
sell_quantity, buy_foreign_quantity,
sell_foreign_quantity, market_cap,
state_ownership, foreign_ownership,
other_ownership)
if group_type == 1:
trade_price = price_close
else:
trade_price = price_close / 1000
lastest_tick_data = LastestTickData(sec_id, fn_date, trade_price,
deal_volume,
datetime.datetime.now())
records = get_daily_data(sec_id, fn_date)
if len(records) == 0:
daily_datas.append(daily_data)
else:
info(
"Exist record in table daily_data with stock: {0}\t date: {1}"
.format(symbol, fn_date))
records = get_historical_data(sec_id, fn_date)
if len(records) == 0:
historical_datas.append(historical_data)
else:
info(
"Exist record in table historical_data with stock: {0}\t date: {1}"
.format(symbol, fn_date))
records = get_lastest_tick_data(sec_id, fn_date)
if len(records) == 0:
insert_object(lastest_tick_data)
else:
update_lastest_tick_data(sec_id, fn_date, trade_price,
deal_volume, datetime.datetime.now())
insert_list_object(historical_datas)
insert_list_object(daily_datas)
if is_adj:
process_update_daily_data(sec_id, symbol, '2008-01-01', end_date)
except Exception as e:
error('Exception %s' % str(e))
error('process_daily_data: %s' % symbol)
error('========================***========================')