def crawl(site, maxpage=None):
proxy = ybk.config.conf.get('proxy')
if proxy:
session.proxies = {'http': proxy}
conf = get_conf(site)
ex = Exchange({
'name': conf['name'],
'url': conf['url'],
'abbr': conf['abbr'],
})
ex.upsert()
for type_ in ['result', 'offer', 'stock']:
tconf = conf.get(type_)
if not tconf:
continue
if maxpage is None:
maxpage = tconf['maxpage']
else:
maxpage = min(maxpage, tconf['maxpage'])
index = tconf['index']
if not isinstance(index, list):
index = [index]
for url in index:
content = session.get(url, timeout=(5, 10)).content
content = fix_javascript(url, content)
parse_index(ex, type_, content, tconf)
for page in range(2, maxpage + 1):
url = tconf['page'].format(page=page)
content = session.get(url, timeout=(5, 10)).content
content = fix_javascript(url, content)
parse_index(ex, type_, content, tconf)
def crawl(site, maxpage=None):
proxy = ybk.config.conf.get('proxy')
if proxy:
session.proxies = {'http': proxy}
conf = get_conf(site)
ex = Exchange({
'name': conf['name'],
'url': conf['url'],
'abbr': conf['abbr'],
})
ex.upsert()
for type_ in ['result', 'offer', 'stock']:
tconf = conf.get(type_)
if not tconf:
continue
if maxpage is None:
maxpage = tconf['maxpage']
else:
maxpage = min(maxpage, tconf['maxpage'])
index = tconf['index']
if not isinstance(index, list):
index = [index]
for url in index:
content = session.get(url, timeout=(5, 10)).content
content = fix_javascript(url, content)
parse_index(ex, type_, content, tconf)
for page in range(2, maxpage + 1):
url = tconf['page'].format(page=page)
content = session.get(url, timeout=(5, 10)).content
content = fix_javascript(url, content)
parse_index(ex, type_, content, tconf)
def offset(self, incr):
c = get_conf(self.exchange)
notrade = [int(x) for x in str(c['notrade'] or '').split(',') if x]
delta = 1 if incr > 0 else -1
odate = self.offers_at
while incr != 0:
incr -= delta
odate += timedelta(days=delta)
while odate.weekday() in notrade:
odate += timedelta(days=delta)
return odate
def analysis():
nav = 'analysis'
exchange = request.args.get('exchange')
exs = sorted(list(Exchange.query()), key=lambda x: x.abbr)
exchanges = [e.abbr for e in exs]
ratings = [ex.rating for ex in exs]
if not exchange:
exchange = exchanges[0]
ex = None
for ex in exs:
if ex.abbr == exchange:
break
# invest history
ih_dates = []
ih_values_self = []
ih_values_all = []
for h in ex.invest_cash_history:
ih_dates.append(h['date'].strftime('%Y-%m-%d'))
ih_values_self.append(h['invest_cash'] / 1e8)
ih_values_all.append(h['total_cash'] / 1e8)
# increase history
inc_days = []
inc_series = []
symbols = []
for symbol, values in ex.increase_history.items():
if len(values) > len(inc_days):
inc_days = list(range(1, len(values)))
inc_series.append({
'name': symbol,
'type': 'line',
'data': [v * 100 for v in values],
})
symbols.append(symbol)
# predict
conf = get_conf(ex.abbr)
today = datetime.utcnow() + timedelta(hours=8)
today = today.replace(hour=0, minute=0, second=0, microsecond=0)
before = today - timedelta(days=conf['cashout'])
cashout_at = today + timedelta(days=conf['cashout'])
colls = list(
Collection.query({
'exchange': ex.abbr,
'offers_at': {
'$gte': before
}
}))
locals()['zip'] = zip
return render_template('frontend/analysis.html', **locals())
def history(site, force=False):
conf = get_conf(site)
exchange = conf['abbr']
url = conf['quote']['history']['url']
type_ = conf['quote']['history']['type']
if not url:
log.warning('{}尚未配置'.format(exchange))
return
if type_ == 'sysframe':
history_sysframe(exchange, url, force)
elif type_ == 'winner':
history_winner(exchange, url, force)
else:
log.warning('{}无法识别'.format(type_))
return
def history(site, force=False):
conf = get_conf(site)
exchange = conf['abbr']
url = conf['quote']['history']['url']
type_ = conf['quote']['history']['type']
if not url:
log.warning('{}尚未配置'.format(exchange))
return
if type_ == 'sysframe':
history_sysframe(exchange, url, force)
elif type_ == 'winner':
history_winner(exchange, url, force)
else:
log.warning('{}无法识别'.format(type_))
return
def analysis():
nav = 'analysis'
exchange = request.args.get('exchange')
exs = sorted(list(Exchange.query()), key=lambda x: x.abbr)
exchanges = [e.abbr for e in exs]
ratings = [ex.rating for ex in exs]
if not exchange:
exchange = exchanges[0]
ex = None
for ex in exs:
if ex.abbr == exchange:
break
# invest history
ih_dates = []
ih_values_self = []
ih_values_all = []
for h in ex.invest_cash_history:
ih_dates.append(h['date'].strftime('%Y-%m-%d'))
ih_values_self.append(h['invest_cash'] / 1e8)
ih_values_all.append(h['total_cash'] / 1e8)
# increase history
inc_days = []
inc_series = []
symbols = []
for symbol, values in ex.increase_history.items():
if len(values) > len(inc_days):
inc_days = list(range(1, len(values)))
inc_series.append({
'name': symbol,
'type': 'line',
'data': [v * 100 for v in values],
})
symbols.append(symbol)
# predict
conf = get_conf(ex.abbr)
today = datetime.utcnow() + timedelta(hours=8)
today = today.replace(hour=0, minute=0, second=0, microsecond=0)
before = today - timedelta(days=conf['cashout'])
cashout_at = today + timedelta(days=conf['cashout'])
colls = list(Collection.query({'exchange': ex.abbr,
'offers_at': {'$gte': before}}))
locals()['zip'] = zip
return render_template('frontend/analysis.html', **locals())
def realtime(site):
conf = get_conf(site)
exchange = conf['abbr']
url = conf['quote']['realtime']['url']
type_ = conf['quote']['realtime']['type']
today = datetime.utcnow().replace(
minute=0, second=0, microsecond=0) + timedelta(hours=8)
if not url:
log.warning('{}尚未配置实时行情url'.format(exchange))
return
if today.hour < 9 or today.hour > 22:
log.warning('不在9点到22点之间, 不做解析')
return
today = today.replace(hour=0)
text = session.get(url, timeout=(3, 7)).text
quotes = parse_quotes(type_, text)
saved = 0
for q in quotes:
Collection.update_one({'exchange': exchange,
'symbol': q['symbol'].strip()},
{'$set': {'name': q['name']}},
upsert=True)
q['exchange'] = exchange
q['quote_type'] = '1d'
q['quote_at'] = today
if q['open_'] in ['—', '-', None, '']:
continue
else:
# 找到上一个交易日的数据, 如果和lclose不符则舍弃
# 需要保证数据每天更新/不足时需要把日线补足才能正常显示
lq = Quote.query_one({'exchange': exchange,
'symbol': q['symbol'].strip(),
'quote_type': '1d',
'quote_at': {'$lt': today}},
sort=[('quote_at', -1)])
if not lq or abs(lq.close - q['lclose']) < 0.01:
Quote(q).upsert()
saved += 1
log.info('{} 导入 {}/{} 条实时交易记录'.format(exchange, saved, len(quotes)))
def cashout_at(self):
""" 出金日期 """
c = get_conf(self.exchange)
return self.offset(c['cashout'])
def calendar():
nav = 'calendar'
starts_at = request.args.get('starts_at')
ends_at = request.args.get('ends_at')
if starts_at:
starts_at = datetime.strptime(starts_at, '%Y%m%d')
today = datetime.utcnow() + timedelta(hours=8)
today = today.replace(hour=0, minute=0, second=0, microsecond=0)
if not starts_at:
starts_at = today - timedelta(days=3)
ends_at = starts_at + timedelta(days=10)
# 表头
heads = []
d = starts_at
while d <= ends_at:
heads.append(
('周' + '一二三四五六日'[d.weekday()], '{}/{}'.format(d.month, d.day)))
d += timedelta(days=1)
# 表身
exs = [] # 交易所所在行
rowdict = defaultdict(list) # 交易所 -> 每天有/没有
seen = set()
ddict = {}
for c in Collection.query(
{'offers_at': {
'$gte': starts_at,
'$lte': ends_at
}},
sort=[('offers_at', 1)]):
if (c.exchange, c.offers_at) in seen:
continue
seen.add((c.exchange, c.offers_at))
if c.exchange not in exs:
exs.append(c.exchange)
d = ddict.get(c.exchange, starts_at)
while d < c.cashout_at:
if d >= c.offers_at and d < c.cashout_at:
cs = list(
Collection.query({
'offers_at': c.offers_at,
'exchange': c.exchange
}))
ndays = (c.cashout_at - c.offers_at).days
if c.offers_at + timedelta(days=ndays) > ends_at:
ndays = (ends_at - c.offers_at).days + 1
rowdict[c.exchange].append({
'colspan':
ndays,
'exchange':
c.exchange,
'count':
len(cs),
'cs':
cs,
'symbols':
','.join([c.symbol for c in cs])
})
ddict[c.exchange] = c.cashout_at
break
else:
rowdict[c.exchange].append({'colspan': 1})
d += timedelta(days=1)
banks = {}
details = {}
for ex in ddict:
d = ddict[ex]
while d <= ends_at:
spans = sum(x['colspan'] for x in rowdict[ex])
if spans < 11:
rowdict[ex].append({'colspan': 1})
d += timedelta(days=1)
c = get_conf(ex)
banks[ex] = c['opening']['bank']
details[ex] = {}
for cell in rowdict[ex]:
if 'cs' in cell:
for c in cell['cs']:
details[ex][c.symbol] = {
'name': c.name,
'price': c.offer_price,
'offer_cash': c.offer_cash or 0,
'expected_ratio': c.expected_result_cash_ratio or 0,
'expected_revenue': c.expected_annual_profit or 0,
}
if not exs:
exs = ['无申购']
prev_starts_at = (starts_at - timedelta(days=10)).strftime('%Y%m%d')
next_starts_at = (starts_at + timedelta(days=10)).strftime('%Y%m%d')
thisdate = (datetime.utcnow() + timedelta(hours=8))
thisdate = '{}/{}'.format(thisdate.month, thisdate.day)
return render_template('frontend/calendar.html', **locals())
def calendar():
nav = "calendar"
starts_at = request.args.get("starts_at")
ends_at = request.args.get("ends_at")
if starts_at:
starts_at = datetime.strptime(starts_at, "%Y%m%d")
today = datetime.utcnow() + timedelta(hours=8)
today = today.replace(hour=0, minute=0, second=0, microsecond=0)
if not starts_at:
starts_at = today - timedelta(days=3)
ends_at = starts_at + timedelta(days=10)
# 表头
heads = []
d = starts_at
while d <= ends_at:
heads.append(("周" + "一二三四五六日"[d.weekday()], "{}/{}".format(d.month, d.day)))
d += timedelta(days=1)
# 表身
exs = [] # 交易所所在行
rowdict = defaultdict(list) # 交易所 -> 每天有/没有
seen = set()
ddict = {}
for c in Collection.query({"offers_at": {"$gte": starts_at, "$lte": ends_at}}, sort=[("offers_at", 1)]):
if (c.exchange, c.offers_at) in seen:
continue
seen.add((c.exchange, c.offers_at))
if c.exchange not in exs:
exs.append(c.exchange)
d = ddict.get(c.exchange, starts_at)
while d < c.cashout_at:
if d >= c.offers_at and d < c.cashout_at:
cs = list(Collection.query({"offers_at": c.offers_at, "exchange": c.exchange}))
ndays = (c.cashout_at - c.offers_at).days
if c.offers_at + timedelta(days=ndays) > ends_at:
ndays = (ends_at - c.offers_at).days + 1
rowdict[c.exchange].append(
{
"colspan": ndays,
"exchange": c.exchange,
"count": len(cs),
"cs": cs,
"symbols": ",".join([c.symbol for c in cs]),
}
)
ddict[c.exchange] = c.cashout_at
break
else:
rowdict[c.exchange].append({"colspan": 1})
d += timedelta(days=1)
banks = {}
details = {}
for ex in ddict:
d = ddict[ex]
while d <= ends_at:
spans = sum(x["colspan"] for x in rowdict[ex])
if spans < 11:
rowdict[ex].append({"colspan": 1})
d += timedelta(days=1)
c = get_conf(ex)
banks[ex] = c["opening"]["bank"]
details[ex] = {}
for cell in rowdict[ex]:
if "cs" in cell:
for c in cell["cs"]:
details[ex][c.symbol] = {
"name": c.name,
"price": c.offer_price,
"offer_cash": c.offer_cash or 0,
"expected_ratio": c.expected_result_cash_ratio or 0,
"expected_revenue": c.expected_annual_profit or 0,
}
if not exs:
exs = ["无申购"]
prev_starts_at = (starts_at - timedelta(days=10)).strftime("%Y%m%d")
next_starts_at = (starts_at + timedelta(days=10)).strftime("%Y%m%d")
thisdate = datetime.utcnow() + timedelta(hours=8)
thisdate = "{}/{}".format(thisdate.month, thisdate.day)
return render_template("frontend/calendar.html", **locals())
