def parse_quarterly(code): print('분기 {}'.format(code)) url = NAVER_QUARTERLY % (code) tree = tree_from_url(url) tds = tree.xpath("/html/body/table/tbody/tr[22]/td") ROEs = [first_or_none(td.xpath('span/text()')) for td in tds] while ROEs and ROEs[-1] is None: ROEs.pop() if len(ROEs) == 0: print('*** 분기 ROE 정보가 없음 >>>') return ths = tree.xpath("/html/body/table/thead/tr[2]/th") quarters = [quarter_from(th.text.strip()) for th in ths] tds = tree.xpath("/html/body/table/tbody/tr[28]/td") BPSs = [first_or_none(td.xpath('span/text()')) for td in tds] QROEs = list(zip(quarters, ROEs)) QBPSs = list(zip(quarters, BPSs)) stock = { 'code': code, 'QROEs': QROEs, 'QBPSs': QBPSs, } print(stock) stock = db.save_stock(stock) stock.save_record()
def convert_mos_list_to_df(mo_list: list): """ Разворачивает список по мо в df """ df_contain = [] for mo_dict in mo_list: its_code = first_or_none(list(mo_dict.keys()), lambda x: x != '_id') mo_info = mo_dict[its_code] mo_df = pd.DataFrame(mo_info) mo_df['soc_type'] = mo_df.index mo_df['its_code'] = [its_code for x in range(len(mo_df))] mo_df['geometry'] = [gload(x.geometry_wkt) for x in mo_df.itertuples()] mo_df = mo_df.reset_index() mo_df = mo_df[[ 'its_code', 'oktmo', 'tr', 'sub_name', 'mun_name', 'mun_center_name', 'soc_type', 'geometry', '2018', '2019', '2020', '2021', '2022', '2023', '2024', '2025', '2026', '2027', '2028', '2029', '2030', '2031', '2032', '2033', '2034', '2035' ]] df_contain.append(mo_df) mos_df = pd.concat(df_contain, ignore_index=True) return mos_df
def save_dataset(self, dataset, output_type: str = 'mongo', **kwargs): """saves data according to its output type """ loader = first_or_none(self.loaders, lambda x: x.source_type == output_type) if loader: if output_type == 'mongo': collection_name = kwargs.get('collection_name') if 'version_name' in kwargs and 'transport_type' not in kwargs: version_name = kwargs['version_name'] loader.save_data(dataset, collection_name, version_name=version_name) elif 'transport_type' in kwargs and 'version_name' not in kwargs: transport_type = kwargs['transport_type'] loader.save_data(dataset, collection_name, transport_type=transport_type) elif 'transport_type' in kwargs and 'version_name' in kwargs: transport_type = kwargs['transport_type'] version_name = kwargs['version_name'] loader.save_data(dataset, collection_name, version_name=version_name, transport_type=transport_type) else: loader.save_data(dataset, collection_name) elif output_type == 'Excel' or output_type == 'CSV': flname = kwargs.get('flname') path = kwargs.get('path') geometry_column = kwargs.get('geometry_column') loader.save_data(dataset, flname, path) elif output_type == 'QGSLayers': loader.save_data(dataset, geometry_column, flname)
def get_failed_pr_build_jobs(repo_name, pr_number): statuses_data = get_pr_statuses(repo_name, pr_number) failed_statuses = [ dict(state=status["state"], target_url=status["target_url"]) for status in statuses_data if status["state"] == FAILED_STATUS ] top_failed_status = first_or_none(failed_statuses) if not top_failed_status: return [] build_id = get_build_id_from_url(top_failed_status["target_url"]) return get_failed_build_jobs(build_id)
def get_dataset(self, dataset_name: str, source_type: str = 'mongo', dataset_source: str = 'mongo', **kwargs): """Loads data from loaders Parameters ---------- dataset_name : str source_type : str dataset_source : str Returns ------- data: dict or pd.DataFrame """ loader = first_or_none(self.loaders, lambda x: x.source_type == source_type) if not loader: print('Загрузчики не добавлены при инициации рабочей области') if 'version_name' in kwargs and 'transport_type' not in kwargs: # print(1) version_name = kwargs['version_name'] data = loader.load_data(dataset_source, dataset_name, version_name=version_name) elif 'transport_type' in kwargs and 'version_name' not in kwargs: # print(2) transport_type = kwargs['transport_type'] data = loader.load_data(dataset_source, dataset_name, transport_type=transport_type) elif 'transport_type' in kwargs and 'version_name' in kwargs: # print(3) transport_type = kwargs['transport_type'] version_name = kwargs['version_name'] data = loader.load_data(dataset_source, dataset_name, version_name=version_name, transport_type=transport_type) else: # print(4) data = loader.load_data(dataset_source, dataset_name) return data
def convert_dm_matrix_to_array(contain_list: list): """ Разворачивает список """ # print('convert') _arr_contain = [] for x in contain_list: its_code = first_or_none( list(x.keys()), lambda x: x != '_id' and x != 'version' and x != 'transport_type') _arr_contain.append(np.array(x[its_code])) res = np.array(_arr_contain) return res
def convert_flow_base_matrix_to_array(flows_list: list): """ Разворачивает список """ # ignore_keys = ['_id'] # print('convert') _arr_contain = [] for x in flows_list: try: its_code = first_or_none( list(x.keys()), lambda x: x != '_id' and x != 'version' and x != 'transport_type') _arr_contain.append(np.array(x[its_code])) except: continue res = np.array(_arr_contain) return res
def convert_trs_to_valid_format(trs_list: list): trs_contain = {} for tr_dict in trs_list: its_code = first_or_none(list(tr_dict.keys()), lambda x: x != '_id' and x != 'version') tr_info = tr_dict[its_code] _tr_dict = {} for key, val in tr_info.items(): if key != 'version': if key == 'geometry': val = gload(val) _tr_dict[key] = val trs_contain[int(its_code)] = _tr_dict return trs_contain
def parse_fnguide(code: str): print('종목 {} FnGuide...'.format(code)) url = FNGUIDE + code print('FnGuide {}'.format(url)) tree = tree_from_url(url) title = first_or_none(tree.xpath('//*[@id="giName"]/text()')) if not title: return False groups = first_or_none( tree.xpath('//*[@id="compBody"]/div[1]/div[1]/p/span[1]/text()')) groups = groups.split(' ') group = groups[1] if len(groups) > 1 else None subgroup = first_or_none( tree.xpath('//*[@id="compBody"]/div[1]/div[1]/p/span[4]/text()')) subgroup = subgroup.replace('\xa0', '') closing_month = first_or_none( tree.xpath('//*[@id="compBody"]/div[1]/div[1]/p/span[6]/text()')) closing_month = parse_int(closing_month.split(' ')[0][:-1]) forward_per = parse_float( first_or_none(tree.xpath('//*[@id="corp_group2"]/dl[2]/dd/text()'))) group_per = parse_float( first_or_none(tree.xpath('//*[@id="corp_group2"]/dl[3]/dd/text()'))) dividend_rate = parse_float( first_or_none(tree.xpath('//*[@id="corp_group2"]/dl[5]/dd/text()'))) relative_earning_rate = parse_float( first_or_none(tree.xpath('//*[@id="svdMainChartTxt13"]/text()'))) momentums = tree.xpath( '//*[@id="svdMainGrid1"]/table/tbody/tr[3]/td[1]/span/text()') momentums = [parse_float(m) for m in momentums] month1 = momentums[0] if len(momentums) >= 1 else 0 month3 = momentums[1] if len(momentums) >= 2 else 0 month6 = momentums[2] if len(momentums) >= 3 else 0 month12 = momentums[3] if len(momentums) >= 4 else 0 foreigner_weight = parse_float( first_or_none( tree.xpath( '//*[@id="svdMainGrid1"]/table/tbody/tr[3]/td[2]/text()'))) beta = parse_float( first_or_none( tree.xpath( '//*[@id="svdMainGrid1"]/table/tbody/tr[4]/td[2]/text()'))) stocks = first_or_none( tree.xpath('//*[@id="svdMainGrid1"]/table/tbody/tr[5]/td[1]/text()')) stocks = stocks.split('/ ') has_preferred_stock = False if stocks[1] == '0' else True floating_rate = parse_float( first_or_none( tree.xpath( '//*[@id="svdMainGrid1"]/table/tbody/tr[6]/td[2]/text()'))) YoY = parse_float( first_or_none( tree.xpath( '//*[@id="svdMainGrid2"]/table/tbody/tr/td[4]/span/text()'))) consensus_point = parse_float( first_or_none( tree.xpath('//*[@id="svdMainGrid9"]/table/tbody/tr/td[1]/text()'))) consensus_price = parse_int( first_or_none( tree.xpath('//*[@id="svdMainGrid9"]/table/tbody/tr/td[2]/text()'))) consensus_count = parse_int( first_or_none( tree.xpath('//*[@id="svdMainGrid9"]/table/tbody/tr/td[5]/text()'))) bps = parse_int( first_or_none( tree.xpath( '//*[@id="highlight_D_A"]/table/tbody/tr[19]/td[3]/text()'))) stock = { 'code': code, 'group': group, 'subgroup': subgroup, 'closing_month': closing_month, 'forward_per': forward_per, 'group_per': group_per, 'dividend_rate': dividend_rate, 'relative_earning_rate': relative_earning_rate, 'month1': month1, 'month3': month3, 'month6': month6, 'month12': month12, 'foreigner_weight': foreigner_weight, 'beta': beta, 'has_preferred_stock': has_preferred_stock, 'floating_rate': floating_rate, 'YoY': YoY, 'consensus_point': consensus_point, 'consensus_price': consensus_price, 'consensus_count': consensus_count, 'bps': bps, 'use_fnguide': True, } db.save_stock(stock) return True
def parse_snowball(code): if not parse_basic(code): print('수집 실패') return if not parse_fnguide(code): print('FnGuide 수집실패') if not parse_naver_company(code): return print('종목 {} 스노우볼...'.format(code)) url = NAVER_YEARLY % (code) tree = tree_from_url(url) try: years = list( filter( lambda x: x != '', map(lambda x: x.strip().split('/')[0], tree.xpath('/html/body/table/thead/tr[2]/th/text()')))) last_year_index = years.index(LAST_YEAR) except ValueError: return tds = tree.xpath('/html/body/table/tbody/tr[22]/td') ROEs = [first_or_none(td.xpath('span/text()')) for td in tds] while ROEs and ROEs[-1] is None: ROEs.pop() if len(ROEs) == 0: print('*** ROE 정보가 없음 >>>') return CAPEXs = tree.xpath('/html/body/table/tbody/tr[17]/td/span/text()') CAPEXs = [parse_float(x) for x in CAPEXs] ROEs = [float_or_none(x) for x in ROEs] DEPTs = tree.xpath('/html/body/table/tbody/tr[24]/td/span/text()') DEPTs = [parse_float(x) for x in DEPTs] EPSs = tree.xpath('/html/body/table/tbody/tr[26]/td/span/text()') EPSs = [parse_float(x) for x in EPSs] PERs = tree.xpath('/html/body/table/tbody/tr[27]/td/span/text()') PERs = [parse_float(x) for x in PERs] BPSs = tree.xpath('/html/body/table/tbody/tr[28]/td/span/text()') BPSs = [parse_int(x) for x in BPSs] PBRs = tree.xpath('/html/body/table/tbody/tr[29]/td/span/text()') PBRs = [parse_float(x) for x in PBRs] #자산총계 TAs = tree.xpath('/html/body/table/tbody/tr[8]/td/span/text()') TAs = [parse_int(x) for x in TAs] #당기순이익 NPs = tree.xpath('/html/body/table/tbody/tr[5]/td/span/text()') NPs = [parse_int(x) for x in NPs] #영업활동현금흐름 CFOs = tree.xpath('/html/body/table/tbody/tr[14]/td/span/text()') CFOs = [parse_int(x) for x in CFOs] #발행주식수 TIs = tree.xpath('/html/body/table/tbody/tr[33]/td/span/text()') TIs = [parse_int(x) for x in TIs] stock = { 'code': code, 'ROEs': ROEs, 'last_year_index': last_year_index, 'PBRs': PBRs, 'EPSs': EPSs, 'TAs': TAs, 'NPs': NPs, 'CFOs': CFOs, 'PERs': PERs, 'TIs': TIs, 'DEPTs': DEPTs, 'BPSs': BPSs, 'CAPEXs': CAPEXs, } stock = db.save_stock(stock) stock.save_record() parse_quarterly(code) parse_json(code)
def net_working_capital_by(self, year): return first_or_none([ v[1] for v in self.get('net_working_capital', []) if v[0] == year ])
def total_asset_turnover_by(self, year): return first_or_none([ v[1] for v in self.get('total_asset_turnover', []) if v[0] == year ])
def value_by_year(self, key, year): return first_or_none( [v[1] for v in attr_or_key_getter(key, self, []) if v[0] == year])
def get_env_by_name(env_name): env = first_or_none(env for env in ENV_MAP.values() if env.full_name == env_name) if not env: raise Exception("No env found with name '{}'".format(env_name)) return env
def parse_fnguide(code: str): print('종목 {} FnGuide...'.format(code)) url = FNGUIDE + code print('FnGuide {}'.format(url)) tree = tree_from_url(url) title = first_or_none(tree.xpath('//*[@id="giName"]/text()')) if not title: return False groups = first_or_none(tree.xpath('//*[@id="compBody"]/div[1]/div[1]/p/span[1]/text()')) groups = groups.split(' ') group = groups[1] if len(groups) > 1 else None subgroup = first_or_none(tree.xpath('//*[@id="compBody"]/div[1]/div[1]/p/span[4]/text()')) subgroup = subgroup.replace('\xa0', '') closing_month = first_or_none(tree.xpath('//*[@id="compBody"]/div[1]/div[1]/p/span[6]/text()')) closing_month = parse_int(closing_month.split(' ')[0][:-1]) forward_per = parse_float(first_or_none(tree.xpath('//*[@id="corp_group2"]/dl[2]/dd/text()'))) group_per = parse_float(first_or_none(tree.xpath('//*[@id="corp_group2"]/dl[3]/dd/text()'))) dividend_rate = parse_float(first_or_none(tree.xpath('//*[@id="corp_group2"]/dl[5]/dd/text()'))) relative_earning_rate = parse_float(first_or_none(tree.xpath('//*[@id="svdMainChartTxt13"]/text()'))) momentums = tree.xpath('//*[@id="svdMainGrid1"]/table/tbody/tr[3]/td[1]/span/text()') momentums = [parse_float(m) for m in momentums] month1 = momentums[0] if len(momentums) >= 1 else 0 month3 = momentums[1] if len(momentums) >= 2 else 0 month6 = momentums[2] if len(momentums) >= 3 else 0 month12 = momentums[3] if len(momentums) >= 4 else 0 foreigner_weight = parse_float(first_or_none(tree.xpath('//*[@id="svdMainGrid1"]/table/tbody/tr[3]/td[2]/text()'))) beta = parse_float(first_or_none(tree.xpath('//*[@id="svdMainGrid1"]/table/tbody/tr[4]/td[2]/text()'))) stocks = first_or_none(tree.xpath('//*[@id="svdMainGrid1"]/table/tbody/tr[7]/td[1]/text()')) stocks = stocks.split('/ ') has_preferred_stock = False if stocks[1] == '0' else True floating_rate = parse_float(first_or_none(tree.xpath('//*[@id="svdMainGrid1"]/table/tbody/tr[6]/td[2]/text()'))) YoY = parse_float(first_or_none(tree.xpath('//*[@id="svdMainGrid2"]/table/tbody/tr/td[4]/span/text()'))) consensus_point = parse_float(first_or_none(tree.xpath('//*[@id="svdMainGrid9"]/table/tbody/tr/td[1]/text()'))) consensus_price = parse_int(first_or_none(tree.xpath('//*[@id="svdMainGrid9"]/table/tbody/tr/td[2]/text()'))) consensus_count = parse_int(first_or_none(tree.xpath('//*[@id="svdMainGrid9"]/table/tbody/tr/td[5]/text()'))) bps = parse_int(first_or_none(tree.xpath('//*[@id="highlight_D_A"]/table/tbody/tr[19]/td[3]/text()'))) try: years = tree.xpath('//*[@id="highlight_D_Y"]/table/thead/tr[2]/th/div/text()') years = [x.split('/')[0] for x in years] last_year_index = years.index(LAST_YEAR) except ValueError: print("** 작년 데이터 없음 **") return NPs = tree.xpath('//*[@id="highlight_D_Y"]/table/tbody/tr[3]/td/text()') NPs = [parse_float(x) for x in NPs] TAs = tree.xpath('//*[@id="highlight_D_Y"]/table/tbody/tr[6]/td/text()') TAs = [parse_float(x) for x in TAs] ROEs = tree.xpath('//*[@id="highlight_D_Y"]/table/tbody/tr[17]/td/text()') ROEs = [parse_float(x) for x in ROEs] EPSs = tree.xpath('//*[@id="highlight_D_Y"]/table/tbody/tr[18]/td/text()') EPSs = [parse_float(x) for x in EPSs] BPSs = tree.xpath('//*[@id="highlight_D_Y"]/table/tbody/tr[19]/td/text()') BPSs = [parse_float(x) for x in BPSs] DPSs = tree.xpath('//*[@id="highlight_D_Y"]/table/tbody/tr[20]/td/text()') DPSs = [parse_float(x) for x in DPSs] PERs = tree.xpath('//*[@id="highlight_D_Y"]/table/tbody/tr[21]/td/text()') PERs = [parse_float(x) for x in PERs] PBRs = tree.xpath('//*[@id="highlight_D_Y"]/table/tbody/tr[22]/td/text()') PBRs = [parse_float(x) for x in PBRs] DEPTs = tree.xpath('//*[@id="highlight_D_Y"]/table/tbody/tr[7]/td/text()') DEPTs = [parse_float(x) for x in DEPTs] stock = { 'code': code, 'group': group, 'subgroup': subgroup, 'closing_month': closing_month, 'forward_per': forward_per, 'group_per': group_per, 'dividend_rate': dividend_rate, 'relative_earning_rate': relative_earning_rate, 'month1': month1, 'month3': month3, 'month6': month6, 'month12': month12, 'foreigner_weight': foreigner_weight, 'beta': beta, 'has_preferred_stock': has_preferred_stock, 'floating_rate': floating_rate, 'YoY': YoY, 'consensus_point': consensus_point, 'consensus_price': consensus_price, 'consensus_count': consensus_count, 'bps': bps, 'use_fnguide': True, 'last_year_index': last_year_index, 'NPs': NPs, 'TAs': TAs, 'ROEs': ROEs, 'EPSs': EPSs, 'BPSs': BPSs, 'DPSs': DPSs, 'PERs': PERs, 'PBRs': PBRs, 'DEPTs': DEPTs, } db.save_stock(stock) return True