def crawl(site, maxpage=None): proxy = ybk.config.conf.get('proxy') if proxy: session.proxies = {'http': proxy} conf = get_conf(site) ex = Exchange({ 'name': conf['name'], 'url': conf['url'], 'abbr': conf['abbr'], }) ex.upsert() for type_ in ['result', 'offer', 'stock']: tconf = conf.get(type_) if not tconf: continue if maxpage is None: maxpage = tconf['maxpage'] else: maxpage = min(maxpage, tconf['maxpage']) index = tconf['index'] if not isinstance(index, list): index = [index] for url in index: content = session.get(url, timeout=(5, 10)).content content = fix_javascript(url, content) parse_index(ex, type_, content, tconf) for page in range(2, maxpage + 1): url = tconf['page'].format(page=page) content = session.get(url, timeout=(5, 10)).content content = fix_javascript(url, content) parse_index(ex, type_, content, tconf)
def announcement(): locals()['type_to_cn'] = type_to_cn nav = 'announcement' tab = 'raw' type_ = request.args.get('type', '') typecn = type_to_cn(type_) exchange = request.args.get('exchange', '') page = int(request.args.get('page', 1) or 1) limit = 50 skip = limit * (page - 1) cond = {} if type_: cond['type_'] = type_ if exchange: cond['exchange'] = exchange total = Announcement.count(cond) pagination = Pagination(page, limit, total) exchanges = list(sorted(list(e.abbr for e in Exchange.query()))) types = ['offer', 'result', 'stock'] announcements = list( Announcement.query(cond, sort=[('updated_at', -1)], skip=skip, limit=limit)) for a in announcements: a.typecn = type_to_cn(a.type_) ex = Exchange.query_one(sort=[('updated_at', -1)]) updated_at = None if not ex else ex.updated_at return render_template('frontend/announcement.html', **locals())
def analysis(): nav = 'analysis' exchange = request.args.get('exchange') exs = sorted(list(Exchange.query()), key=lambda x: x.abbr) exchanges = [e.abbr for e in exs] ratings = [ex.rating for ex in exs] if not exchange: exchange = exchanges[0] ex = None for ex in exs: if ex.abbr == exchange: break # invest history ih_dates = [] ih_values_self = [] ih_values_all = [] for h in ex.invest_cash_history: ih_dates.append(h['date'].strftime('%Y-%m-%d')) ih_values_self.append(h['invest_cash'] / 1e8) ih_values_all.append(h['total_cash'] / 1e8) # increase history inc_days = [] inc_series = [] symbols = [] for symbol, values in ex.increase_history.items(): if len(values) > len(inc_days): inc_days = list(range(1, len(values))) inc_series.append({ 'name': symbol, 'type': 'line', 'data': [v * 100 for v in values], }) symbols.append(symbol) # predict conf = get_conf(ex.abbr) today = datetime.utcnow() + timedelta(hours=8) today = today.replace(hour=0, minute=0, second=0, microsecond=0) before = today - timedelta(days=conf['cashout']) cashout_at = today + timedelta(days=conf['cashout']) colls = list( Collection.query({ 'exchange': ex.abbr, 'offers_at': { '$gte': before } })) locals()['zip'] = zip return render_template('frontend/analysis.html', **locals())
def analysis(): nav = 'analysis' exchange = request.args.get('exchange') exs = sorted(list(Exchange.query()), key=lambda x: x.abbr) exchanges = [e.abbr for e in exs] ratings = [ex.rating for ex in exs] if not exchange: exchange = exchanges[0] ex = None for ex in exs: if ex.abbr == exchange: break # invest history ih_dates = [] ih_values_self = [] ih_values_all = [] for h in ex.invest_cash_history: ih_dates.append(h['date'].strftime('%Y-%m-%d')) ih_values_self.append(h['invest_cash'] / 1e8) ih_values_all.append(h['total_cash'] / 1e8) # increase history inc_days = [] inc_series = [] symbols = [] for symbol, values in ex.increase_history.items(): if len(values) > len(inc_days): inc_days = list(range(1, len(values))) inc_series.append({ 'name': symbol, 'type': 'line', 'data': [v * 100 for v in values], }) symbols.append(symbol) # predict conf = get_conf(ex.abbr) today = datetime.utcnow() + timedelta(hours=8) today = today.replace(hour=0, minute=0, second=0, microsecond=0) before = today - timedelta(days=conf['cashout']) cashout_at = today + timedelta(days=conf['cashout']) colls = list(Collection.query({'exchange': ex.abbr, 'offers_at': {'$gte': before}})) locals()['zip'] = zip return render_template('frontend/analysis.html', **locals())
def trade_account(): nav = 'accounts' tab = 'trade_account' investor = request.args.get('investor', '') exchange = request.args.get('exchange', '') add_investor = request.args.get('add_investor', '') add_exchange = request.args.get('add_exchange', '') bank = request.args.get('bank', '') investor = Investor.query_one({'_id': investor}) exchange = Exchange.query_one({'abbr': exchange}) user = current_user._id trade_accounts = TradeAccount.user_accounts(user) if investor: trade_accounts = [ta for ta in trade_accounts if ta.investor == investor._id] if exchange: trade_accounts = [ta for ta in trade_accounts if ta.exchange == exchange.abbr] if bank: trade_accounts = [ta for ta in trade_accounts if ta.bank == bank] investors = Investor.user_investors(user) exchanges = sorted(list(Exchange.query()), key=lambda x: x.abbr) exchange_list = [{'text': e.abbr, 'value': e.abbr} for e in exchanges] investor_list = [{'text': i.name, 'value': i.name} for i in investors] investor_banks = {i.name: sorted([ba.bank for ba in i.bank_accounts]) for i in investors} exchange_banks = {conf['abbr']: sorted(conf['opening']['bank']) for conf in CONFS} bank_exchanges = {} for ex, banks in exchange_banks.items(): for b in banks: if b not in bank_exchanges: bank_exchanges[b] = [] bank_exchanges[b].append(ex) investor_exchanges = { invesotr: sorted(set(sum( [bank_exchanges[b] for b in banks], []))) for invesotr, banks in investor_banks.items() } avail_investors = set(ta.investor for ta in trade_accounts) avail_exchanges = set(ta.exchange for ta in trade_accounts) avail_banks = set(ta.bank for ta in trade_accounts) investors = [i for i in investors if i._id in avail_investors] exchanges = [e for e in exchanges if e.abbr in avail_exchanges] thebanks = sorted(list(avail_banks)) account_positions = {ta._id: sorted([p.to_dict() for p in ta.position], key=lambda p: p['symbol']) for ta in trade_accounts} account_moneys = {ta._id: ta.money.to_dict() if ta.money else {} for ta in trade_accounts} account_order_status = {ta._id: sorted([o.to_dict() for o in ta.order_status], key=lambda o: o['order']) for ta in trade_accounts} account_orders = {ta._id: sorted([o.to_dict() for o in ta.orders], key=lambda o: o['symbol']) for ta in trade_accounts} return render_template('user/trade_account.html', **locals())
def trade_account(): nav = 'accounts' tab = 'trade_account' investor = request.args.get('investor', '') exchange = request.args.get('exchange', '') add_investor = request.args.get('add_investor', '') add_exchange = request.args.get('add_exchange', '') bank = request.args.get('bank', '') investor = Investor.query_one({'_id': investor}) exchange = Exchange.query_one({'abbr': exchange}) user = current_user._id trade_accounts = TradeAccount.user_accounts(user) if investor: trade_accounts = [ ta for ta in trade_accounts if ta.investor == investor._id ] if exchange: trade_accounts = [ ta for ta in trade_accounts if ta.exchange == exchange.abbr ] if bank: trade_accounts = [ta for ta in trade_accounts if ta.bank == bank] investors = Investor.user_investors(user) exchanges = sorted(list(Exchange.query()), key=lambda x: x.abbr) exchange_list = [{'text': e.abbr, 'value': e.abbr} for e in exchanges] investor_list = [{'text': i.name, 'value': i.name} for i in investors] investor_banks = { i.name: sorted([ba.bank for ba in i.bank_accounts]) for i in investors } exchange_banks = { conf['abbr']: sorted(conf['opening']['bank']) for conf in CONFS } bank_exchanges = {} for ex, banks in exchange_banks.items(): for b in banks: if b not in bank_exchanges: bank_exchanges[b] = [] bank_exchanges[b].append(ex) investor_exchanges = { invesotr: sorted(set(sum([bank_exchanges[b] for b in banks], []))) for invesotr, banks in investor_banks.items() } avail_investors = set(ta.investor for ta in trade_accounts) avail_exchanges = set(ta.exchange for ta in trade_accounts) avail_banks = set(ta.bank for ta in trade_accounts) investors = [i for i in investors if i._id in avail_investors] exchanges = [e for e in exchanges if e.abbr in avail_exchanges] thebanks = sorted(list(avail_banks)) account_positions = { ta._id: sorted([p.to_dict() for p in ta.position], key=lambda p: p['symbol']) for ta in trade_accounts } account_moneys = { ta._id: ta.money.to_dict() if ta.money else {} for ta in trade_accounts } account_order_status = { ta._id: sorted([o.to_dict() for o in ta.order_status], key=lambda o: o['order']) for ta in trade_accounts } account_orders = { ta._id: sorted([o.to_dict() for o in ta.orders], key=lambda o: o['symbol']) for ta in trade_accounts } return render_template('user/trade_account.html', **locals())