def collect_stock_daily_trading():
"""
获取并保存每日股票交易数据
"""
url = eastmoney_stock_api
data = request_and_handle_data(url)
stock_data = data["rank"]
for i in stock_data:
stock = i.split(",")
stock_number = stock[1]
stock_name = stock[2]
sdt = SDT(stock_number=stock_number, stock_name=stock_name)
sdt.yesterday_closed_price = float(stock[3])
sdt.today_opening_price = float(stock[4])
sdt.today_closing_price = float(stock[5])
sdt.today_highest_price = float(stock[6])
sdt.today_lowest_price = float(stock[7])
sdt.turnover_amount = int(stock[8])
sdt.turnover_volume = int(stock[9])
sdt.increase_amount = float(stock[10])
sdt.increase_rate = stock[11]
sdt.today_average_price = float(stock[12])
sdt.quantity_relative_ratio = float(stock[22])
sdt.turnover_rate = stock[23]
if not check_duplicate(sdt):
sdt.save()
def collect_stock_daily_trading():
"""
获取并保存每日股票交易数据
"""
url = eastmoney_stock_api
data = request_and_handle_data(url)
stock_data = data['rank']
for i in stock_data:
stock = i.split(',')
stock_number = stock[1]
stock_name = stock[2]
sdt = SDT(stock_number=stock_number, stock_name=stock_name)
sdt.yesterday_closed_price = float(stock[3])
sdt.today_opening_price = float(stock[4])
sdt.today_closing_price = float(stock[5])
sdt.today_highest_price = float(stock[6])
sdt.today_lowest_price = float(stock[7])
sdt.turnover_amount = int(stock[8])
sdt.turnover_volume = int(stock[9])
sdt.increase_amount = float(stock[10])
sdt.increase_rate = stock[11]
sdt.today_average_price = float(stock[12])
sdt.quantity_relative_ratio = float(stock[22])
sdt.turnover_rate = stock[23]
if float(sdt.turnover_rate.replace('%', '')) == 0.0:
# 去掉停牌的交易数据
continue
if not check_duplicate(sdt):
sdt.save()
def collect_his_trading(stock_number, stock_name):
if stock_number.startswith('6'):
req_url = history_trading.format(stock_number+'01')
else:
req_url = history_trading.format(stock_number+'02')
his_html = send_request(req_url)
his_soup = BeautifulSoup(his_html, 'lxml')
his_table = his_soup.find('table', id='tablefont')
if his_table:
his_data = his_table.find_all('tr')[1:]
for i in his_data:
date = datetime.datetime.strptime(i.find('p', class_='date').text, '%Y-%m-%d')
today_opening_price = float(i.find_all('td')[1].text.replace(' ', '').strip())
today_highest_price = float(i.find_all('td')[2].text.replace(' ', '').strip())
today_lowest_price = float(i.find_all('td')[3].text.replace(' ', '').strip())
today_closing_price = float(i.find_all('td')[4].text.replace(' ', '').strip())
increase_rate = i.find_all('td')[5].text.replace(' ', '').strip() + '%'
increase_amount = float(i.find_all('td')[6].text.replace(' ', '').strip())
turnover_rate = i.find_all('td')[7].text.replace(' ', '').strip() + '%'
if float(increase_rate.replace('%', '')) == 0.0 and float(turnover_rate.replace('%', '')) == 0.0:
# 去掉停牌期间的行情数据
continue
if not check_exists(stock_number, date):
sdt = SDT(stock_number=stock_number, stock_name=stock_name, date=date,
today_opening_price=today_opening_price, today_highest_price=today_highest_price,
today_lowest_price=today_lowest_price, today_closing_price=today_closing_price,
increase_rate=increase_rate, increase_amount=increase_amount, turnover_rate=turnover_rate)
sdt.save()
def collect_datayes_data(date):
url = datayes_day_trading.format(date.strftime('%Y%m%d'))
datayes_data = send_requests(url)
if datayes_data.get('retCode') != 1:
return
trading_data = datayes_data.get('data', [])
for i in trading_data:
is_open = i['isOpen']
if is_open != 1:
continue
stock_number = i['ticker']
stock_name = i['secShortName']
yesterday_closed_price = i['actPreClosePrice']
today_opening_price = i['openPrice']
today_closing_price = i['closePrice']
today_highest_price = i['highestPrice']
today_lowest_price = i['lowestPrice']
turnover_amount = int(i['turnoverValue'] / 10000)
turnover_volume = int(i['turnoverVol'] / 100)
increase_amount = i['closePrice'] - i['actPreClosePrice']
increase_rate = str(
round(increase_amount / i['actPreClosePrice'], 4) * 100) + '%'
turnover_rate = str(i['turnoverRate'] * 100) + '%'
total_stock = int(i['marketValue'] / i['closePrice'])
circulation_stock = int(i['negMarketValue'] / i['closePrice'])
date = datetime.datetime.strptime(i['tradeDate'], '%Y-%m-%d')
sdt = SDT(stock_number=stock_number,
stock_name=stock_name,
yesterday_closed_price=yesterday_closed_price,
today_opening_price=today_opening_price,
today_closing_price=today_closing_price,
today_highest_price=today_highest_price,
today_lowest_price=today_lowest_price,
turnover_amount=turnover_amount,
turnover_volume=turnover_volume,
increase_amount=increase_amount,
increase_rate=increase_rate,
turnover_rate=turnover_rate,
total_stock=total_stock,
circulation_stock=circulation_stock,
date=date)
try:
if not check_duplicate(sdt):
sdt.save()
except Exception, e:
logging.error('Error when query or saving %s data:%s' %
(sdt.stock_number, e))
def collect_his_trading(stock_number, stock_name):
if stock_number.startswith('6'):
req_url = history_trading.format(stock_number+'01')
else:
req_url = history_trading.format(stock_number+'02')
his_html = send_request(req_url)
his_soup = BeautifulSoup(his_html, 'lxml')
his_table = his_soup.find('table', id='tablefont')
if his_table:
his_data = his_table.find_all('tr')[1:]
for i in his_data:
date = datetime.datetime.strptime(i.find('p', class_='date').text, '%Y-%m-%d')
try:
today_opening_price = float(i.find_all('td')[1].text.replace(' ', '').strip())
today_highest_price = float(i.find_all('td')[2].text.replace(' ', '').strip())
today_lowest_price = float(i.find_all('td')[3].text.replace(' ', '').strip())
today_closing_price = float(i.find_all('td')[4].text.replace(' ', '').strip())
increase_rate = i.find_all('td')[5].text.replace(' ', '').strip() + '%'
increase_amount = float(i.find_all('td')[6].text.replace(' ', '').strip())
turnover_rate = i.find_all('td')[7].text.replace(' ', '').strip() + '%'
total_stock = int(i.find_all('td')[10].text.replace(' ', '').replace(',', '').strip())
circulation_stock = int(i.find_all('td')[12].text.replace(' ', '').replace(',', '').strip())
except Exception, e:
if '--' not in str(e):
logging.error('Collect %s %s trading data failed:%s' % (stock_number, str(date), e))
continue
if float(increase_rate.replace('%', '')) == 0.0 and float(turnover_rate.replace('%', '')) == 0.0:
# 去掉停牌期间的行情数据
continue
if not check_exists(stock_number, date):
sdt = SDT(stock_number=stock_number, stock_name=stock_name, date=date,
today_opening_price=today_opening_price, today_highest_price=today_highest_price,
today_lowest_price=today_lowest_price, today_closing_price=today_closing_price,
increase_rate=increase_rate, increase_amount=increase_amount, turnover_rate=turnover_rate,
total_stock=total_stock, circulation_stock=circulation_stock)
sdt.save()
else: # 添加股本相关数据
sdt = SDT.objects(Q(stock_number=stock_number) & Q(date=date)).next()
if not sdt.total_stock:
sdt.total_stock = total_stock
if not sdt.circulation_stock:
sdt.circulation_stock = circulation_stock
sdt.save()
def collect_his_trading(stock_number, stock_name):
if stock_number.startswith("6"):
req_url = history_trading.format(stock_number + "01")
else:
req_url = history_trading.format(stock_number + "02")
his_html = send_request(req_url)
his_soup = BeautifulSoup(his_html, "lxml")
his_table = his_soup.find("table", id="tablefont")
if his_table:
his_data = his_table.find_all("tr")[1:]
for i in his_data:
date = datetime.datetime.strptime(i.find("p", class_="date").text, "%Y-%m-%d")
try:
today_opening_price = float(i.find_all("td")[1].text.replace(" ", "").strip())
today_highest_price = float(i.find_all("td")[2].text.replace(" ", "").strip())
today_lowest_price = float(i.find_all("td")[3].text.replace(" ", "").strip())
today_closing_price = float(i.find_all("td")[4].text.replace(" ", "").strip())
increase_rate = i.find_all("td")[5].text.replace(" ", "").strip() + "%"
increase_amount = float(i.find_all("td")[6].text.replace(" ", "").strip())
turnover_rate = i.find_all("td")[7].text.replace(" ", "").strip() + "%"
except Exception, e:
if "--" not in str(e):
logging.error("Collect %s %s trading data failed:%s" % (stock_number, str(date), e))
continue
if float(increase_rate.replace("%", "")) == 0.0 and float(turnover_rate.replace("%", "")) == 0.0:
# 去掉停牌期间的行情数据
continue
if not check_exists(stock_number, date):
sdt = SDT(
stock_number=stock_number,
stock_name=stock_name,
date=date,
today_opening_price=today_opening_price,
today_highest_price=today_highest_price,
today_lowest_price=today_lowest_price,
today_closing_price=today_closing_price,
increase_rate=increase_rate,
increase_amount=increase_amount,
turnover_rate=turnover_rate,
)
sdt.save()
def collect_stock_daily_trading(date):
trading_data = get_pro_client().daily(trade_date=date.strftime('%Y%m%d'))
for i in range(0, len(trading_data)):
stock = trading_data.iloc[i]
stock_number = stock.ts_code.split('.')[0]
existSdt = SDT.objects(Q(stock_number=stock_number) & Q(date=date))
if existSdt and len(existSdt) > 0:
sdt = existSdt[0]
else:
sdt = SDT(stock_number=stock_number)
sdt.date = date
sdt.yesterday_closed_price = stock.pre_close
sdt.today_opening_price = stock.open
sdt.today_closing_price = stock.close
sdt.today_highest_price = stock.high
sdt.today_lowest_price = stock.low
sdt.turnover_amount = stock.amount
sdt.turnover_volume = stock.vol
sdt.increase_amount = stock.change
sdt.increase_rate = str(stock.pct_chg) + '%'
sdt.save()
def collect_his_trading(stock_number, stock_name, start_date, end_date):
ts_code = tushare_util.gen_ts_code(stock_number)
his_data = tushare_util.get_pro_client().query(
'daily',
ts_code=ts_code,
start_date=start_date.strftime('%Y%m%d'),
end_date=end_date.strftime('%Y%m%d'))
for i in range(0, len(his_data)):
trade_data = his_data.iloc[i]
try:
date = datetime.datetime.strptime(trade_data.trade_date, '%Y%m%d')
except Exception as e:
continue
existSdt = SDT.objects(Q(stock_number=stock_number) & Q(date=date))
if existSdt and len(existSdt) > 0:
sdt = existSdt[0]
else:
sdt = SDT(stock_number=stock_number)
sdt.date = date
sdt.stock_name = stock_name
sdt.yesterday_closed_price = trade_data.pre_close
sdt.today_opening_price = trade_data.open
sdt.today_closing_price = trade_data.close
sdt.today_highest_price = trade_data.high
sdt.today_lowest_price = trade_data.low
sdt.turnover_amount = trade_data.amount
sdt.turnover_volume = trade_data.vol
sdt.increase_amount = trade_data.change
sdt.increase_rate = str(trade_data.pct_chg) + '%'
try:
sdt.save()
except Exception as e:
continue
def collect_his_trading(stock_number, stock_name):
if stock_number.startswith('6'):
req_url = history_trading.format(stock_number + '01')
else:
req_url = history_trading.format(stock_number + '02')
his_html = send_request(req_url)
his_soup = BeautifulSoup(his_html, 'lxml')
his_table = his_soup.find('table', id='tablefont')
if his_table:
his_data = his_table.find_all('tr')[1:]
for i in his_data:
date = datetime.datetime.strptime(
i.find('p', class_='date').text.strip(), '%Y-%m-%d')
try:
today_opening_price = float(
i.find_all('td')[1].text.replace(' ', '').strip())
today_highest_price = float(
i.find_all('td')[2].text.replace(' ', '').strip())
today_lowest_price = float(
i.find_all('td')[3].text.replace(' ', '').strip())
today_closing_price = float(
i.find_all('td')[4].text.replace(' ', '').strip())
increase_rate = i.find_all('td')[5].text.replace(
' ', '').strip() + '%'
increase_amount = float(
i.find_all('td')[6].text.replace(' ', '').strip())
turnover_rate = i.find_all('td')[7].text.replace(
' ', '').strip() + '%'
total_stock = int(
i.find_all('td')[10].text.replace(' ',
'').replace(',',
'').strip())
circulation_stock = int(
i.find_all('td')[12].text.replace(' ',
'').replace(',',
'').strip())
turnover_amount_msg = i.find_all('td')[9].text.replace(
' ', '').strip()
turnover_amount = 0
if u'万' in turnover_amount_msg:
turnover_amount = int(
float(turnover_amount_msg.replace(u'万', '')))
elif u'亿' in turnover_amount_msg:
turnover_amount = int(
float(turnover_amount_msg.replace(u'亿', '')) * 10000)
if turnover_amount_msg.isdigit() and int(
turnover_amount_msg) == 0:
return
except Exception as e:
if '--' not in str(e):
logging.error('Collect %s %s trading data failed:%s' %
(stock_number, str(date), e))
continue
if float(increase_rate.replace('%', '')) == 0.0 and float(
turnover_rate.replace('%', '')) == 0.0:
# 去掉停牌期间的行情数据
continue
if check_exists(stock_number, date):
sdt = SDT.objects(Q(stock_number=stock_number)
& Q(date=date)).next()
if not sdt.total_stock or not sdt.circulation_stock or not sdt.turnover_amount:
sdt.total_stock = total_stock
sdt.circulation_stock = circulation_stock
sdt.turnover_amount = turnover_amount
sdt.save()
else: # 添加股本相关数据
sdt = SDT(stock_number=stock_number,
stock_name=stock_name,
date=date,
today_opening_price=today_opening_price,
today_highest_price=today_highest_price,
today_lowest_price=today_lowest_price,
today_closing_price=today_closing_price,
increase_rate=increase_rate,
increase_amount=increase_amount,
turnover_rate=turnover_rate,
total_stock=total_stock,
circulation_stock=circulation_stock,
turnover_amount=turnover_amount)
sdt.save()