def stop(self): ''' Override to add logic before the stop eg. invalidating running condition ''' error("Service is being stopped at: {0}".format( datetime.datetime.now()))
def process_business_plan_data(sec_id, symbol): path = URL_BUSINESS_PLAN_BASE.format(symbol) headers = { "Accept": "application/json, text/javascript, */*; q=0.01", "Accept-Encoding": "gzip, deflate", "Accept-Language": "en - US, en;q = 0.9, vi;q = 0.8", "Connection": "keep-alive", "Host": "e.cafef.vn", "Origin": "http://s.cafef.vn", "Referer": "http://s.cafef.vn", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36", } try: response = requests.get(path, headers=headers) data = json.loads(response.text) if data is None: return for item in data: year_crawl = item.get('KYear') total_income_crawl = item.get('Totalncome') profit_crawl = item.get('TotalProfit') net_income_crawl = item.get('Netlncome') dividend_stock_crawl = item.get('DivStock') dividend_money_crawl = item.get('Dividend') records = get_business_plan(sec_id, year_crawl) if len(records) == 0: x = 1000000000 business = BusinessPlanData(sec_id, year_crawl, total_income_crawl * x, profit_crawl * x, net_income_crawl * x, dividend_stock_crawl, dividend_money_crawl) insert_object(business) except Exception as e: error('Exception %s' % str(e)) error('process_business_plan_data: %s' % symbol) error('========================***========================')
def process_quarterly_data(sec_id, symbol, start_year, end_year): path = URL_BASE_QUARTER.format(symbol, start_year - 1, 1, end_year, 4) response = requests.get(path, headers=headers) try: data = json.loads(response.text) for year in range(start_year - 1, end_year + 1): for quarter in range(1, 5): records = get_quarterly_data(sec_id, year, quarter) if len(records) > 0: continue for item in data: year_crawl = item.get("Year") quarter_crawl = item.get("Quarter") roe_crawl = item.get("ROE_TTM") roa_crawl = item.get("ROA_TTM") roic_crawl = item.get("ROIC_TTM") eps_crawl = item.get("BasicEPS_MRQ") cash_crawl = item.get("Cash_MRQ") assets_crawl = item.get("TotalAssets_MRQ") equity_crawl = item.get("Equity_MRQ") profit_after_tax_crawl = item.get("ProfitAfterTax_MRQ") profit_before_tax_crawl = item.get("ProfitBeforeTax_MRQ") revenue_crawl = item.get("NetSales_MRQ") liabilities_crawl = item.get("Liabilities_MRQ") current_liabilities_crawl = item.get( "CurrentLiabilities_MRQ") bookvalue_per_share_crawl = item.get( "BookValuePerShare_MRQ") sales_per_share_crawl = item.get("SalesPerShare_TTM") if year == year_crawl and quarter == quarter_crawl: data_db = QuarterlyData( sec_id, year_crawl, quarter_crawl, roe_crawl, roa_crawl, roic_crawl, eps_crawl, cash_crawl, assets_crawl, equity_crawl, profit_after_tax_crawl, profit_before_tax_crawl, revenue_crawl, liabilities_crawl, current_liabilities_crawl, bookvalue_per_share_crawl, sales_per_share_crawl) insert_object(data_db) break info('done process_quarterly_data: {0}'.format(symbol)) except Exception as e: error('Exception %s' % str(e)) error('process_quarterly_data: %s' % symbol) error('========================***========================')
def process_update_daily_data(sec_id, symbol, start_date, end_date): path = URL_DAILY_DATA_ADJ_BASE.format(symbol, start_date, end_date) response = requests.get(path, headers=headers) try: data = json.loads(response.text) for item in data: date = item.get("Date") if date is None: # xu ly lai continue fn_date = str(date)[:10] open = item.get("Open") close = item.get("Close") high = item.get("High") low = item.get("Low") update_adj_price_data(sec_id, fn_date, close, open, high, low) except Exception as e: error('error info: %s' % str(e)) error('error at process_update_daily_data: %s' % symbol) error('========================***========================')
def process_daily_data(sec_id, symbol, group_type, start_date, end_date): path = URL_DAILY_DATA_BASE.format(symbol, start_date, end_date) path_inventory = URL_INVENTORY_BASE.format(symbol) response = requests.get(path, headers=headers) response_inventory = requests.get(path_inventory, headers=headers) try: data = json.loads(response.text) data_inventory = json.loads(response_inventory.text) historical_datas = [] daily_datas = [] is_adj = False for item in data: date = item.get("Date") price_open = item.get("PriceOpen") price_high = item.get("PriceHigh") price_low = item.get("PriceLow") price_close = item.get("PriceClose") adj_close = item.get("AdjClose") adj_open = item.get("AdjOpen") adj_high = item.get("AdjHigh") adj_low = item.get("AdjLow") if adj_close != price_close: is_adj = True deal_volume = item.get("DealVolume") put_through_volume = item.get("PutthroughVolume") total_volume = item.get("Volume") buy_foreign_quantity = item.get("BuyForeignQuantity") sell_foreign_quantity = item.get("SellForeignQuantity") buy_quantity = item.get("BuyQuantity") sell_quantity = item.get("SellQuantity") market_cap = item.get("MarketCap") state_ownership = data_inventory.get("StateOwnership") foreign_ownership = data_inventory.get("ForeignOwnership") other_ownership = data_inventory.get("OtherOwnership") if date is None: continue fn_date = str(date)[:10] historical_data = HistoricalData(sec_id, fn_date, price_open, price_high, price_low, price_close, adj_open, adj_high, adj_low, adj_close, deal_volume, put_through_volume, total_volume, datetime.datetime.now()) daily_data = DailyData(sec_id, fn_date, buy_quantity, sell_quantity, buy_foreign_quantity, sell_foreign_quantity, market_cap, state_ownership, foreign_ownership, other_ownership) if group_type == 1: trade_price = price_close else: trade_price = price_close / 1000 lastest_tick_data = LastestTickData(sec_id, fn_date, trade_price, deal_volume, datetime.datetime.now()) records = get_daily_data(sec_id, fn_date) if len(records) == 0: daily_datas.append(daily_data) else: info( "Exist record in table daily_data with stock: {0}\t date: {1}" .format(symbol, fn_date)) records = get_historical_data(sec_id, fn_date) if len(records) == 0: historical_datas.append(historical_data) else: info( "Exist record in table historical_data with stock: {0}\t date: {1}" .format(symbol, fn_date)) records = get_lastest_tick_data(sec_id, fn_date) if len(records) == 0: insert_object(lastest_tick_data) else: update_lastest_tick_data(sec_id, fn_date, trade_price, deal_volume, datetime.datetime.now()) insert_list_object(historical_datas) insert_list_object(daily_datas) if is_adj: process_update_daily_data(sec_id, symbol, '2008-01-01', end_date) except Exception as e: error('Exception %s' % str(e)) error('process_daily_data: %s' % symbol) error('========================***========================')