def parse_quarterly(code):
print('분기 {}'.format(code))
url = NAVER_QUARTERLY % (code)
tree = tree_from_url(url)
tds = tree.xpath("/html/body/table/tbody/tr[22]/td")
ROEs = [first_or_none(td.xpath('span/text()')) for td in tds]
while ROEs and ROEs[-1] is None:
ROEs.pop()
if len(ROEs) == 0:
print('*** 분기 ROE 정보가 없음 >>>')
return
ths = tree.xpath("/html/body/table/thead/tr[2]/th")
quarters = [quarter_from(th.text.strip()) for th in ths]
tds = tree.xpath("/html/body/table/tbody/tr[28]/td")
BPSs = [first_or_none(td.xpath('span/text()')) for td in tds]
QROEs = list(zip(quarters, ROEs))
QBPSs = list(zip(quarters, BPSs))
stock = {
'code': code,
'QROEs': QROEs,
'QBPSs': QBPSs,
}
print(stock)
stock = db.save_stock(stock)
stock.save_record()
def convert_mos_list_to_df(mo_list: list):
"""
Разворачивает список по мо в df
"""
df_contain = []
for mo_dict in mo_list:
its_code = first_or_none(list(mo_dict.keys()), lambda x: x != '_id')
mo_info = mo_dict[its_code]
mo_df = pd.DataFrame(mo_info)
mo_df['soc_type'] = mo_df.index
mo_df['its_code'] = [its_code for x in range(len(mo_df))]
mo_df['geometry'] = [gload(x.geometry_wkt) for x in mo_df.itertuples()]
mo_df = mo_df.reset_index()
mo_df = mo_df[[
'its_code', 'oktmo', 'tr', 'sub_name', 'mun_name',
'mun_center_name', 'soc_type', 'geometry', '2018', '2019', '2020',
'2021', '2022', '2023', '2024', '2025', '2026', '2027', '2028',
'2029', '2030', '2031', '2032', '2033', '2034', '2035'
]]
df_contain.append(mo_df)
mos_df = pd.concat(df_contain, ignore_index=True)
return mos_df
def save_dataset(self, dataset, output_type: str = 'mongo', **kwargs):
"""saves data according to its output type
"""
loader = first_or_none(self.loaders,
lambda x: x.source_type == output_type)
if loader:
if output_type == 'mongo':
collection_name = kwargs.get('collection_name')
if 'version_name' in kwargs and 'transport_type' not in kwargs:
version_name = kwargs['version_name']
loader.save_data(dataset,
collection_name,
version_name=version_name)
elif 'transport_type' in kwargs and 'version_name' not in kwargs:
transport_type = kwargs['transport_type']
loader.save_data(dataset,
collection_name,
transport_type=transport_type)
elif 'transport_type' in kwargs and 'version_name' in kwargs:
transport_type = kwargs['transport_type']
version_name = kwargs['version_name']
loader.save_data(dataset,
collection_name,
version_name=version_name,
transport_type=transport_type)
else:
loader.save_data(dataset, collection_name)
elif output_type == 'Excel' or output_type == 'CSV':
flname = kwargs.get('flname')
path = kwargs.get('path')
geometry_column = kwargs.get('geometry_column')
loader.save_data(dataset, flname, path)
elif output_type == 'QGSLayers':
loader.save_data(dataset, geometry_column, flname)
def get_failed_pr_build_jobs(repo_name, pr_number):
statuses_data = get_pr_statuses(repo_name, pr_number)
failed_statuses = [
dict(state=status["state"], target_url=status["target_url"])
for status in statuses_data if status["state"] == FAILED_STATUS
]
top_failed_status = first_or_none(failed_statuses)
if not top_failed_status:
return []
build_id = get_build_id_from_url(top_failed_status["target_url"])
return get_failed_build_jobs(build_id)
def get_dataset(self,
dataset_name: str,
source_type: str = 'mongo',
dataset_source: str = 'mongo',
**kwargs):
"""Loads data from loaders
Parameters
----------
dataset_name : str
source_type : str
dataset_source : str
Returns
-------
data: dict or pd.DataFrame
"""
loader = first_or_none(self.loaders,
lambda x: x.source_type == source_type)
if not loader:
print('Загрузчики не добавлены при инициации рабочей области')
if 'version_name' in kwargs and 'transport_type' not in kwargs:
# print(1)
version_name = kwargs['version_name']
data = loader.load_data(dataset_source,
dataset_name,
version_name=version_name)
elif 'transport_type' in kwargs and 'version_name' not in kwargs:
# print(2)
transport_type = kwargs['transport_type']
data = loader.load_data(dataset_source,
dataset_name,
transport_type=transport_type)
elif 'transport_type' in kwargs and 'version_name' in kwargs:
# print(3)
transport_type = kwargs['transport_type']
version_name = kwargs['version_name']
data = loader.load_data(dataset_source,
dataset_name,
version_name=version_name,
transport_type=transport_type)
else:
# print(4)
data = loader.load_data(dataset_source, dataset_name)
return data
def convert_dm_matrix_to_array(contain_list: list):
"""
Разворачивает список
"""
# print('convert')
_arr_contain = []
for x in contain_list:
its_code = first_or_none(
list(x.keys()),
lambda x: x != '_id' and x != 'version' and x != 'transport_type')
_arr_contain.append(np.array(x[its_code]))
res = np.array(_arr_contain)
return res
def convert_flow_base_matrix_to_array(flows_list: list):
"""
Разворачивает список
"""
# ignore_keys = ['_id']
# print('convert')
_arr_contain = []
for x in flows_list:
try:
its_code = first_or_none(
list(x.keys()), lambda x: x != '_id' and x != 'version' and x
!= 'transport_type')
_arr_contain.append(np.array(x[its_code]))
except:
continue
res = np.array(_arr_contain)
return res
def convert_trs_to_valid_format(trs_list: list):
trs_contain = {}
for tr_dict in trs_list:
its_code = first_or_none(list(tr_dict.keys()),
lambda x: x != '_id' and x != 'version')
tr_info = tr_dict[its_code]
_tr_dict = {}
for key, val in tr_info.items():
if key != 'version':
if key == 'geometry':
val = gload(val)
_tr_dict[key] = val
trs_contain[int(its_code)] = _tr_dict
return trs_contain
def parse_fnguide(code: str):
print('종목 {} FnGuide...'.format(code))
url = FNGUIDE + code
print('FnGuide {}'.format(url))
tree = tree_from_url(url)
title = first_or_none(tree.xpath('//*[@id="giName"]/text()'))
if not title:
return False
groups = first_or_none(
tree.xpath('//*[@id="compBody"]/div[1]/div[1]/p/span[1]/text()'))
groups = groups.split(' ')
group = groups[1] if len(groups) > 1 else None
subgroup = first_or_none(
tree.xpath('//*[@id="compBody"]/div[1]/div[1]/p/span[4]/text()'))
subgroup = subgroup.replace('\xa0', '')
closing_month = first_or_none(
tree.xpath('//*[@id="compBody"]/div[1]/div[1]/p/span[6]/text()'))
closing_month = parse_int(closing_month.split(' ')[0][:-1])
forward_per = parse_float(
first_or_none(tree.xpath('//*[@id="corp_group2"]/dl[2]/dd/text()')))
group_per = parse_float(
first_or_none(tree.xpath('//*[@id="corp_group2"]/dl[3]/dd/text()')))
dividend_rate = parse_float(
first_or_none(tree.xpath('//*[@id="corp_group2"]/dl[5]/dd/text()')))
relative_earning_rate = parse_float(
first_or_none(tree.xpath('//*[@id="svdMainChartTxt13"]/text()')))
momentums = tree.xpath(
'//*[@id="svdMainGrid1"]/table/tbody/tr[3]/td[1]/span/text()')
momentums = [parse_float(m) for m in momentums]
month1 = momentums[0] if len(momentums) >= 1 else 0
month3 = momentums[1] if len(momentums) >= 2 else 0
month6 = momentums[2] if len(momentums) >= 3 else 0
month12 = momentums[3] if len(momentums) >= 4 else 0
foreigner_weight = parse_float(
first_or_none(
tree.xpath(
'//*[@id="svdMainGrid1"]/table/tbody/tr[3]/td[2]/text()')))
beta = parse_float(
first_or_none(
tree.xpath(
'//*[@id="svdMainGrid1"]/table/tbody/tr[4]/td[2]/text()')))
stocks = first_or_none(
tree.xpath('//*[@id="svdMainGrid1"]/table/tbody/tr[5]/td[1]/text()'))
stocks = stocks.split('/ ')
has_preferred_stock = False if stocks[1] == '0' else True
floating_rate = parse_float(
first_or_none(
tree.xpath(
'//*[@id="svdMainGrid1"]/table/tbody/tr[6]/td[2]/text()')))
YoY = parse_float(
first_or_none(
tree.xpath(
'//*[@id="svdMainGrid2"]/table/tbody/tr/td[4]/span/text()')))
consensus_point = parse_float(
first_or_none(
tree.xpath('//*[@id="svdMainGrid9"]/table/tbody/tr/td[1]/text()')))
consensus_price = parse_int(
first_or_none(
tree.xpath('//*[@id="svdMainGrid9"]/table/tbody/tr/td[2]/text()')))
consensus_count = parse_int(
first_or_none(
tree.xpath('//*[@id="svdMainGrid9"]/table/tbody/tr/td[5]/text()')))
bps = parse_int(
first_or_none(
tree.xpath(
'//*[@id="highlight_D_A"]/table/tbody/tr[19]/td[3]/text()')))
stock = {
'code': code,
'group': group,
'subgroup': subgroup,
'closing_month': closing_month,
'forward_per': forward_per,
'group_per': group_per,
'dividend_rate': dividend_rate,
'relative_earning_rate': relative_earning_rate,
'month1': month1,
'month3': month3,
'month6': month6,
'month12': month12,
'foreigner_weight': foreigner_weight,
'beta': beta,
'has_preferred_stock': has_preferred_stock,
'floating_rate': floating_rate,
'YoY': YoY,
'consensus_point': consensus_point,
'consensus_price': consensus_price,
'consensus_count': consensus_count,
'bps': bps,
'use_fnguide': True,
}
db.save_stock(stock)
return True
def parse_snowball(code):
if not parse_basic(code):
print('수집 실패')
return
if not parse_fnguide(code):
print('FnGuide 수집실패')
if not parse_naver_company(code):
return
print('종목 {} 스노우볼...'.format(code))
url = NAVER_YEARLY % (code)
tree = tree_from_url(url)
try:
years = list(
filter(
lambda x: x != '',
map(lambda x: x.strip().split('/')[0],
tree.xpath('/html/body/table/thead/tr[2]/th/text()'))))
last_year_index = years.index(LAST_YEAR)
except ValueError:
return
tds = tree.xpath('/html/body/table/tbody/tr[22]/td')
ROEs = [first_or_none(td.xpath('span/text()')) for td in tds]
while ROEs and ROEs[-1] is None:
ROEs.pop()
if len(ROEs) == 0:
print('*** ROE 정보가 없음 >>>')
return
CAPEXs = tree.xpath('/html/body/table/tbody/tr[17]/td/span/text()')
CAPEXs = [parse_float(x) for x in CAPEXs]
ROEs = [float_or_none(x) for x in ROEs]
DEPTs = tree.xpath('/html/body/table/tbody/tr[24]/td/span/text()')
DEPTs = [parse_float(x) for x in DEPTs]
EPSs = tree.xpath('/html/body/table/tbody/tr[26]/td/span/text()')
EPSs = [parse_float(x) for x in EPSs]
PERs = tree.xpath('/html/body/table/tbody/tr[27]/td/span/text()')
PERs = [parse_float(x) for x in PERs]
BPSs = tree.xpath('/html/body/table/tbody/tr[28]/td/span/text()')
BPSs = [parse_int(x) for x in BPSs]
PBRs = tree.xpath('/html/body/table/tbody/tr[29]/td/span/text()')
PBRs = [parse_float(x) for x in PBRs]
#자산총계
TAs = tree.xpath('/html/body/table/tbody/tr[8]/td/span/text()')
TAs = [parse_int(x) for x in TAs]
#당기순이익
NPs = tree.xpath('/html/body/table/tbody/tr[5]/td/span/text()')
NPs = [parse_int(x) for x in NPs]
#영업활동현금흐름
CFOs = tree.xpath('/html/body/table/tbody/tr[14]/td/span/text()')
CFOs = [parse_int(x) for x in CFOs]
#발행주식수
TIs = tree.xpath('/html/body/table/tbody/tr[33]/td/span/text()')
TIs = [parse_int(x) for x in TIs]
stock = {
'code': code,
'ROEs': ROEs,
'last_year_index': last_year_index,
'PBRs': PBRs,
'EPSs': EPSs,
'TAs': TAs,
'NPs': NPs,
'CFOs': CFOs,
'PERs': PERs,
'TIs': TIs,
'DEPTs': DEPTs,
'BPSs': BPSs,
'CAPEXs': CAPEXs,
}
stock = db.save_stock(stock)
stock.save_record()
parse_quarterly(code)
parse_json(code)
def net_working_capital_by(self, year):
return first_or_none([
v[1] for v in self.get('net_working_capital', []) if v[0] == year
])
def total_asset_turnover_by(self, year):
return first_or_none([
v[1] for v in self.get('total_asset_turnover', []) if v[0] == year
])
def value_by_year(self, key, year):
return first_or_none(
[v[1] for v in attr_or_key_getter(key, self, []) if v[0] == year])
def get_env_by_name(env_name):
env = first_or_none(env for env in ENV_MAP.values()
if env.full_name == env_name)
if not env:
raise Exception("No env found with name '{}'".format(env_name))
return env
def parse_fnguide(code: str):
print('종목 {} FnGuide...'.format(code))
url = FNGUIDE + code
print('FnGuide {}'.format(url))
tree = tree_from_url(url)
title = first_or_none(tree.xpath('//*[@id="giName"]/text()'))
if not title:
return False
groups = first_or_none(tree.xpath('//*[@id="compBody"]/div[1]/div[1]/p/span[1]/text()'))
groups = groups.split(' ')
group = groups[1] if len(groups) > 1 else None
subgroup = first_or_none(tree.xpath('//*[@id="compBody"]/div[1]/div[1]/p/span[4]/text()'))
subgroup = subgroup.replace('\xa0', '')
closing_month = first_or_none(tree.xpath('//*[@id="compBody"]/div[1]/div[1]/p/span[6]/text()'))
closing_month = parse_int(closing_month.split(' ')[0][:-1])
forward_per = parse_float(first_or_none(tree.xpath('//*[@id="corp_group2"]/dl[2]/dd/text()')))
group_per = parse_float(first_or_none(tree.xpath('//*[@id="corp_group2"]/dl[3]/dd/text()')))
dividend_rate = parse_float(first_or_none(tree.xpath('//*[@id="corp_group2"]/dl[5]/dd/text()')))
relative_earning_rate = parse_float(first_or_none(tree.xpath('//*[@id="svdMainChartTxt13"]/text()')))
momentums = tree.xpath('//*[@id="svdMainGrid1"]/table/tbody/tr[3]/td[1]/span/text()')
momentums = [parse_float(m) for m in momentums]
month1 = momentums[0] if len(momentums) >= 1 else 0
month3 = momentums[1] if len(momentums) >= 2 else 0
month6 = momentums[2] if len(momentums) >= 3 else 0
month12 = momentums[3] if len(momentums) >= 4 else 0
foreigner_weight = parse_float(first_or_none(tree.xpath('//*[@id="svdMainGrid1"]/table/tbody/tr[3]/td[2]/text()')))
beta = parse_float(first_or_none(tree.xpath('//*[@id="svdMainGrid1"]/table/tbody/tr[4]/td[2]/text()')))
stocks = first_or_none(tree.xpath('//*[@id="svdMainGrid1"]/table/tbody/tr[7]/td[1]/text()'))
stocks = stocks.split('/ ')
has_preferred_stock = False if stocks[1] == '0' else True
floating_rate = parse_float(first_or_none(tree.xpath('//*[@id="svdMainGrid1"]/table/tbody/tr[6]/td[2]/text()')))
YoY = parse_float(first_or_none(tree.xpath('//*[@id="svdMainGrid2"]/table/tbody/tr/td[4]/span/text()')))
consensus_point = parse_float(first_or_none(tree.xpath('//*[@id="svdMainGrid9"]/table/tbody/tr/td[1]/text()')))
consensus_price = parse_int(first_or_none(tree.xpath('//*[@id="svdMainGrid9"]/table/tbody/tr/td[2]/text()')))
consensus_count = parse_int(first_or_none(tree.xpath('//*[@id="svdMainGrid9"]/table/tbody/tr/td[5]/text()')))
bps = parse_int(first_or_none(tree.xpath('//*[@id="highlight_D_A"]/table/tbody/tr[19]/td[3]/text()')))
try:
years = tree.xpath('//*[@id="highlight_D_Y"]/table/thead/tr[2]/th/div/text()')
years = [x.split('/')[0] for x in years]
last_year_index = years.index(LAST_YEAR)
except ValueError:
print("** 작년 데이터 없음 **")
return
NPs = tree.xpath('//*[@id="highlight_D_Y"]/table/tbody/tr[3]/td/text()')
NPs = [parse_float(x) for x in NPs]
TAs = tree.xpath('//*[@id="highlight_D_Y"]/table/tbody/tr[6]/td/text()')
TAs = [parse_float(x) for x in TAs]
ROEs = tree.xpath('//*[@id="highlight_D_Y"]/table/tbody/tr[17]/td/text()')
ROEs = [parse_float(x) for x in ROEs]
EPSs = tree.xpath('//*[@id="highlight_D_Y"]/table/tbody/tr[18]/td/text()')
EPSs = [parse_float(x) for x in EPSs]
BPSs = tree.xpath('//*[@id="highlight_D_Y"]/table/tbody/tr[19]/td/text()')
BPSs = [parse_float(x) for x in BPSs]
DPSs = tree.xpath('//*[@id="highlight_D_Y"]/table/tbody/tr[20]/td/text()')
DPSs = [parse_float(x) for x in DPSs]
PERs = tree.xpath('//*[@id="highlight_D_Y"]/table/tbody/tr[21]/td/text()')
PERs = [parse_float(x) for x in PERs]
PBRs = tree.xpath('//*[@id="highlight_D_Y"]/table/tbody/tr[22]/td/text()')
PBRs = [parse_float(x) for x in PBRs]
DEPTs = tree.xpath('//*[@id="highlight_D_Y"]/table/tbody/tr[7]/td/text()')
DEPTs = [parse_float(x) for x in DEPTs]
stock = {
'code': code,
'group': group,
'subgroup': subgroup,
'closing_month': closing_month,
'forward_per': forward_per,
'group_per': group_per,
'dividend_rate': dividend_rate,
'relative_earning_rate': relative_earning_rate,
'month1': month1,
'month3': month3,
'month6': month6,
'month12': month12,
'foreigner_weight': foreigner_weight,
'beta': beta,
'has_preferred_stock': has_preferred_stock,
'floating_rate': floating_rate,
'YoY': YoY,
'consensus_point': consensus_point,
'consensus_price': consensus_price,
'consensus_count': consensus_count,
'bps': bps,
'use_fnguide': True,
'last_year_index': last_year_index,
'NPs': NPs,
'TAs': TAs,
'ROEs': ROEs,
'EPSs': EPSs,
'BPSs': BPSs,
'DPSs': DPSs,
'PERs': PERs,
'PBRs': PBRs,
'DEPTs': DEPTs,
}
db.save_stock(stock)
return True
