def collect_margin_trading(req_url):
html = send_request(req_url)
margin_data = eval(html)
for i in margin_data:
margin_item = i.decode('utf-8').split(',')
stock_number = margin_item[0]
stock_name = margin_item[2]
rz_net_buy_amount = margin_item[3]
rq_repay_volume = margin_item[5]
rq_sell_volume = margin_item[6]
rq_remaining_volume = margin_item[8]
rz_repay_amount = margin_item[9]
rz_buy_amount = margin_item[10]
rz_remaining_amount = margin_item[12]
smt = SMT(stock_number=stock_number,
stock_name=stock_name,
rz_net_buy_amount=rz_net_buy_amount,
rq_repay_volume=rq_repay_volume,
rq_sell_volume=rq_sell_volume,
rq_remaining_volume=rq_remaining_volume,
rz_repay_amount=rz_repay_amount,
rz_buy_amount=rz_buy_amount,
rz_remaining_amount=rz_remaining_amount)
if not is_duplicate(smt):
smt.save()
def collect_stock_data(stock_number, start_date, end_date):
if not isinstance(start_date, datetime.date) or not isinstance(
end_date, datetime.date):
return
end_date += datetime.timedelta(days=7)
url = datayes_week_trading.format(stock_number,
start_date.strftime('%Y%m%d'),
end_date.strftime('%Y%m%d'))
res_data = json.loads(send_request(url, datayes_headers))
if res_data.get('retCode', 0) != 1:
return
trading_data = res_data.get('data', [])
for i in trading_data:
trade_days = int(i.get('tradeDays'))
if trade_days == 0:
continue
stock_number = i.get('ticker')
try:
first_trade_date = datetime.datetime.strptime(
i.get('weekBeginDate'), '%Y-%m-%d')
end_date = datetime.datetime.strptime(i.get('endDate'), '%Y-%m-%d')
last_trade_date = datetime.datetime.strptime(
i.get('endDate'), '%Y-%m-%d')
except Exception as e:
logging.error('Format time failed:' + str(e))
continue
former_swt = SWT.objects(
Q(stock_number=stock_number)
& Q(first_trade_date=first_trade_date))
new_object = True
if former_swt:
swt = former_swt.next()
new_object = False
else:
swt = SWT()
swt.stock_number = stock_number
swt.stock_name = i.get('secShortName')
swt.trade_days = trade_days
swt.first_trade_date = first_trade_date
swt.last_trade_date = last_trade_date
swt.end_date = end_date
swt.pre_close_price = float(i.get('preClosePrice'))
swt.weekly_open_price = float(i.get('openPrice'))
swt.weekly_close_price = float(i.get('closePrice'))
swt.weekly_highest_price = float(i.get('highestPrice'))
swt.weekly_lowest_price = float(i.get('lowestPrice'))
swt.increase_rate = str(round(i.get('chgPct') * 100, 2)) + '%'
swt.turnover_amount = int(i.get('turnoverValue')) / 10000
swt.turnover_volume = int(i.get('turnoverVol')) / 100
if new_object:
if not check_duplicate(swt):
swt.save()
else:
swt.save()
def collect_company_report():
report_list = requests.get(company_report).json().get('data', [])
for r in report_list:
stock_number = r.get('secuFullCode', '').strip().split('.')[0]
stock_name = r.get('secuName', '').strip()
date = datetime.datetime.strptime(r.get('datetime').split('T')[0], '%Y-%m-%d')
title = r.get('title')
author = r.get('author')
rate_change = r.get('change')
rate = r.get('rate')
institution = r.get('insName')
info_code = r.get('infoCode').strip()
content_url = 'http://data.eastmoney.com/report/' + date.strftime('%Y%m%d') + '/' + info_code + '.html'
try:
content = BeautifulSoup(send_request(content_url), 'lxml').find('div', class_='newsContent').text.strip()
except Exception as e:
logging.error('Error when get %s report content:%s' % (stock_number, e))
continue
if not check_duplicate(info_code, date):
stock_report = StockReport(stock_number=stock_number, stock_name=stock_name, date=date, title=title,
author=author, rate_change=rate_change, rate=rate, institution=institution,
info_code=info_code, content=content)
try:
stock_report.save()
except Exception as e:
logging.error('Error when save %s report %s:%s' % (stock_number, info_code, e))
def collect_company_survey(stock_info):
query_id = stock_info.stock_number + estimate_market(
stock_info.stock_number)
company_survey_url = f9_survey.format(query_id)
retry = 5
survey_table = ''
while retry:
try:
survey_html = send_request(company_survey_url)
survey_soup = BeautifulSoup(survey_html, 'lxml')
survey_table = survey_soup.find('table',
id='tablefont').find_all('td')
break
except Exception as e:
retry -= 1
time.sleep(1)
if not survey_soup or not survey_table:
return
stock_info.company_name_cn = survey_table[1].text.strip()
stock_info.company_name_en = survey_table[3].text.strip()
stock_info.used_name = survey_table[5].text.strip()
stock_info.account_firm = survey_table[43].text.strip()
stock_info.law_firm = survey_table[45].text.strip()
stock_info.industry_involved = survey_table[17].text.strip()
stock_info.business_scope = survey_table[47].text.strip()
stock_info.company_introduce = survey_table[7].text.strip()
stock_info.area = survey_table[25].text.strip()
core_concept_url = f10_core_content.format(
estimate_market(stock_info.stock_number, 'market') +
stock_info.stock_number)
concept_data = send_request(core_concept_url)
try:
concept_json = json.loads(concept_data)
if concept_json.get('hxtc'):
market_plate = concept_json.get('hxtc')[0].get('ydnr', '')
stock_info.market_plate = market_plate
except Exception as e:
logging.error('parse concept data error, e = ' + str(e))
pass
stock_info.update_time = datetime.datetime.now()
stock_info.save()
def collect_share_holder(stockinfo):
stock_number = stockinfo.stock_number
stock_name = stockinfo.stock_name
print(stock_number)
if stock_number.startswith('6'):
req_code = stock_number + '01'
else:
req_code = stock_number + '02'
for k, v in share_holder_url.items():
print(k, v)
req_url = k + req_code
share_html = send_request(req_url)
share_soup = BeautifulSoup(share_html, 'lxml')
share_table = share_soup.find('table', id='tablefont')
for i in share_table.find_all('tr'):
items = i.find_all('td')
print(items)
if items[0].text.startswith('20'):
print(i)
def collect_notice(stock_info):
req_url = company_notice.format(stock_info.stock_number)
raw_data = send_request(req_url).replace('var',
'').replace('=', '').replace(
';', '').strip()
notice_data = json.loads(raw_data).get('data', [])
for n in notice_data:
notice_title = n.get('NOTICETITLE')
notice_code = n.get('INFOCODE')
notice_date = datetime.datetime.strptime(
n.get('NOTICEDATE').split('T')[0], '%Y-%m-%d')
notice_url = single_notice.format(stock_info.stock_number, notice_code)
if not is_exists(notice_code):
stock_notice = StockNotice(title=notice_title,
code=notice_code,
date=notice_date,
content_url=notice_url,
stock_number=stock_info.stock_number,
stock_name=stock_info.stock_name)
stock_notice.save()
def collect_stock_data(stock_number, start_date, end_date):
if not isinstance(start_date, datetime.date) or not isinstance(
end_date, datetime.date):
return
end_date += datetime.timedelta(days=7)
url = datayes_week_ad.format(stock_number, start_date.strftime('%Y%m%d'),
end_date.strftime('%Y%m%d'))
res_data = json.loads(send_request(url, datayes_headers))
if res_data.get('retCode', 0) != 1:
return
trading_data = res_data.get('data', [])
for i in trading_data:
trade_days = int(i.get('tradeDays'))
if trade_days == 0:
continue
stock_number = i.get('ticker')
try:
first_trade_date = datetime.datetime.strptime(
i.get('weekBeginDate'), '%Y-%m-%d')
except Exception as e:
logging.error('Format time failed:' + str(e))
continue
former_swt = SWT.objects(
Q(stock_number=stock_number)
& Q(first_trade_date=first_trade_date))
if former_swt:
swt = former_swt.next()
swt.ad_open_price = float(i.get('openPrice'))
swt.ad_close_price = float(i.get('closePrice'))
swt.ad_highest_price = float(i.get('highestPrice'))
swt.ad_lowest_price = float(i.get('lowestPrice'))
swt.save()
def collect_company_survey(stock_info):
query_id = estimate_market(
stock_info.stock_number) + stock_info.stock_number
company_survey_url = company_survey.format(query_id)
retry = 5
survey_table = ''
while retry:
try:
survey_html = send_request(company_survey_url)
survey_soup = BeautifulSoup(survey_html, 'lxml')
survey_table = survey_soup.find('table',
id='Table0').find_all('td')
break
except Exception:
retry -= 1
time.sleep(1)
if not survey_soup or not survey_table:
return
stock_info.stock_name = survey_table[4].text.strip()
stock_info.company_name_cn = survey_table[0].text.strip()
stock_info.company_name_en = survey_table[1].text.strip()
stock_info.used_name = survey_table[2].text.strip()
stock_info.account_firm = survey_table[30].text.strip()
stock_info.law_firm = survey_table[29].text.strip()
stock_info.industry_involved = survey_table[10].text.strip()
stock_info.business_scope = survey_table[32].text.strip()
stock_info.company_introduce = survey_table[31].text.strip()
stock_info.area = survey_table[23].text.strip()
core_concept_url = core_concept.format(query_id)
concept_html = send_request(core_concept_url)
market_plate = BeautifulSoup(concept_html, 'lxml').find('div', class_='summary').find('p').text\
.replace(u'要点一：所属板块 ', '').replace(u'。', '').strip()
stock_info.market_plate = market_plate
if 'sh' in query_id:
q_id = stock_info.stock_number + '1'
elif 'sz' in query_id:
q_id = stock_info.stock_number + '2'
if q_id:
stock_value_url = stock_value.format(q_id)
try:
res = send_request(stock_value_url)
data = json.loads(res.replace('callback(',
'').replace(')', ''))['Value']
if data:
circulated_value = int(data[45])
total_value = int(data[46])
stock_info.circulated_value = circulated_value
stock_info.total_value = total_value
except Exception as e:
logging.error('Error when get %s value:%s' %
(stock_info.stock_number, e))
stock_info.update_time = datetime.datetime.now()
stock_info.save()
def collect_his_trading(stock_number, stock_name):
if stock_number.startswith('6'):
req_url = history_trading.format(stock_number + '01')
else:
req_url = history_trading.format(stock_number + '02')
his_html = send_request(req_url)
his_soup = BeautifulSoup(his_html, 'lxml')
his_table = his_soup.find('table', id='tablefont')
if his_table:
his_data = his_table.find_all('tr')[1:]
for i in his_data:
date = datetime.datetime.strptime(
i.find('p', class_='date').text.strip(), '%Y-%m-%d')
try:
today_opening_price = float(
i.find_all('td')[1].text.replace(' ', '').strip())
today_highest_price = float(
i.find_all('td')[2].text.replace(' ', '').strip())
today_lowest_price = float(
i.find_all('td')[3].text.replace(' ', '').strip())
today_closing_price = float(
i.find_all('td')[4].text.replace(' ', '').strip())
increase_rate = i.find_all('td')[5].text.replace(
' ', '').strip() + '%'
increase_amount = float(
i.find_all('td')[6].text.replace(' ', '').strip())
turnover_rate = i.find_all('td')[7].text.replace(
' ', '').strip() + '%'
total_stock = int(
i.find_all('td')[10].text.replace(' ',
'').replace(',',
'').strip())
circulation_stock = int(
i.find_all('td')[12].text.replace(' ',
'').replace(',',
'').strip())
turnover_amount_msg = i.find_all('td')[9].text.replace(
' ', '').strip()
turnover_amount = 0
if u'万' in turnover_amount_msg:
turnover_amount = int(
float(turnover_amount_msg.replace(u'万', '')))
elif u'亿' in turnover_amount_msg:
turnover_amount = int(
float(turnover_amount_msg.replace(u'亿', '')) * 10000)
if turnover_amount_msg.isdigit() and int(
turnover_amount_msg) == 0:
return
except Exception as e:
if '--' not in str(e):
logging.error('Collect %s %s trading data failed:%s' %
(stock_number, str(date), e))
continue
if float(increase_rate.replace('%', '')) == 0.0 and float(
turnover_rate.replace('%', '')) == 0.0:
# 去掉停牌期间的行情数据
continue
if check_exists(stock_number, date):
sdt = SDT.objects(Q(stock_number=stock_number)
& Q(date=date)).next()
if not sdt.total_stock or not sdt.circulation_stock or not sdt.turnover_amount:
sdt.total_stock = total_stock
sdt.circulation_stock = circulation_stock
sdt.turnover_amount = turnover_amount
sdt.save()
else: # 添加股本相关数据
sdt = SDT(stock_number=stock_number,
stock_name=stock_name,
date=date,
today_opening_price=today_opening_price,
today_highest_price=today_highest_price,
today_lowest_price=today_lowest_price,
today_closing_price=today_closing_price,
increase_rate=increase_rate,
increase_amount=increase_amount,
turnover_rate=turnover_rate,
total_stock=total_stock,
circulation_stock=circulation_stock,
turnover_amount=turnover_amount)
sdt.save()