def get_stock_news(symbol, query, collection=None):
    for query_format in querys:
        html = safe_request(query_format.format(symbol,
                                                query.split('GBGB')[0]))
        soup = BeautifulSoup(html, 'html.parser')

        for news in soup.find_all('li', class_='newsContainer'):
            info = news.find('a')
            title = info.string.strip()
            url = url_base + info['href'].strip().split(
                '.html')[0].strip().split("openWin2('")[-1] + '.html'
            date_str = news.find('span', class_='hour').string.strip()
            if len(date_str) <= 5:
                date_str = timezone.datetime.now().strftime(
                    '%d %b %Y') + ' ' + date_str
            date = timezone.datetime.strptime(date_str, '%d %b %Y %H:%M')

            if collection:
                obj, created = collection.objects.get_or_create(Symbol=symbol)
                try:
                    obj_news, created = obj.stocknews_set.get_or_create(
                        pub_date=date, url=url, title=title)
                    if created:
                        obj_news.save()
                except:
                    pass
def getLSEURLSymbol(symbol, collection=None):
    url = 'http://www.londonstockexchange.com/exchange/searchengine/search.html?q={}&page={}'

    new_query = ""
    for p in itertools.count(start=1):
        html = safe_request(url.format(symbol, p))
        soup = BeautifulSoup(html, 'html.parser')
        table = soup.find('table', class_='table_dati')
        if table and not new_query:
            for tr in table.find('tbody').find_all('tr'):
                [sSymbol, sLink, sType] = tr.find_all('td')[:3]
                type = sType.string.strip()
                if type != 'Stocks':
                    continue
                check = ''.join(map(lambda x: x if isinstance(x,str) else x.string, sSymbol.contents)).strip()
                if check == symbol.upper():
                    href = sLink.find('a')['href']
                    new_query = href.split('.html')[0].split('/')[-1]
                    if collection:
                        obj = collection.objects.get(Symbol=symbol)
                        obj.Query = new_query
                        obj.save()
                    break

        else:
            break

    return new_query
    def get(url):
        lists = []
        n_pages = 1
        index = 0

        while index < n_pages:
            index += 1
            html = safe_request(url.format(index))
            soup = BeautifulSoup(html, 'html.parser')

            # get n_pages
            if n_pages == 1:
                n_pages = int(
                    soup.find('div', class_='paging').find(
                        'p'
                    ).string.split('of')[-1].strip()
                )

            print 'Fetching page {} of {}'.format(index, n_pages)

            # parse list
            for tr in soup.find('tbody').find_all('tr'):
                tds = tr.find_all('td')
                #construct data
                info = {}
                info['symbol'] = tds[0].string.strip()
                a = tds[1].find('a')
                info['name'] = a.string.strip()
                info['query'] = a.get('href').strip().split(
                    '/'
                )[-1].split('?')[0][:-5]

                lists.append(info)

                if collection:
                    objs = collection.objects.filter(Symbol=info['symbol'])
                    if objs.count() > 1:
                        for q in objs.order_by("-pub_date")[1:]:
                            q.delete()
                    obj, _ = objs.get_or_create(Symbol=info['symbol'])
                    obj.Query = info['query']
                    obj.Name = info['name']
                    obj.pub_date = timezone.now()
                    obj.save()

        return lists
def get_stock_news(symbol, query, collection=None):
    for query_format in querys:
        html = safe_request(query_format.format(symbol, query.split('GBGB')[0]))
        soup = BeautifulSoup(html, 'html.parser')

        for news in soup.find_all('li', class_='newsContainer'):
            info = news.find('a')
            title = info.string.strip()
            url = url_base + info['href'].strip().split('.html')[0].strip().split("openWin2('")[-1] + '.html'
            date_str = news.find('span', class_='hour').string.strip()
            if len(date_str) <=5:
                date_str = timezone.datetime.now().strftime('%d %b %Y') + ' ' + date_str
            date = timezone.datetime.strptime(date_str, '%d %b %Y %H:%M')

            if collection:
                obj, created = collection.objects.get_or_create(Symbol=symbol)
                try:
                    obj_news, created = obj.stocknews_set.get_or_create(
                        pub_date=date, url=url, title=title
                    )
                    if created:
                        obj_news.save()
                except:
                    pass
def getLSEInfo(query,symbol,collection=None):

    def valid_str(input_str):
        input_str = re.sub('[^0-9a-zA-Z]+', '', input_str)
        return input_str

    def valid_num(input_str):
        try:
            input_str = re.sub('[^0-9.-]+', '', input_str)

            if len(input_str) and input_str != '-':
                return float(input_str)
        except:
            pass
        return 0.0

    header = {'User-Agent': 'Mozilla/5.0'}

    info = {}

    querys = [
        query,
        getLSEURLSymbol(symbol, collection=collection)
    ]

    found = False
    for query in querys:
        try:
            html = safe_request(query_url.format(query))
            soup = BeautifulSoup(html, 'html.parser')
            tIncome,tBalance,tRatio,tCompany,tTrading = soup.find_all('table')
            found = True
            break
        except:
            continue

    if not found:
        print 'Data not Available'
        return info


    # Income Table
    detail = {}
    for tr in tIncome.find('tbody').find_all('tr'):
        tds = tr.find_all('td')
        if len(tds) > 1:
            index = valid_str(tds[0].string)
            detail[index] = [valid_num(td.string) for td in tds[1:]]
    info['Income'] = detail

    # Balance Table
    detail = {}
    for tr in tBalance.find('tbody').find_all('tr'):
        tds = tr.find_all('td')
        if len(tds) > 1:
            index = valid_str(tds[0].string)
            detail[index] = [valid_num(td.string) for td in tds[1:]]
    info['Balance'] = detail

    # Ratio Table
    detail = {}
    for tr in tRatio.find('tbody').find_all('tr'):
        tds = tr.find_all('td')
        if len(tds) > 1:
            index = valid_str(tds[0].string)
            detail[index] = [valid_num(td.string) for td in tds[1:]]
    info['Ratio'] = detail

    # Company
    detail = {}
    for tr in tCompany.find('tbody').find_all('tr'):
        tds = tr.find_all('td')
        if len(tds) > 1:
            index = valid_str(tds[0].string)
            detail[index] = valid_num(tds[-1].string) if 'Marketcap' in index else tds[-1].string
    info['Company'] = detail

    # Trading
    detail = {}
    for tr in tTrading.find('tbody').find_all('tr'):
        tds = tr.find_all('td')
        if len(tds) > 1:
            index = valid_str(tds[0].string)
            detail[index] = valid_num(tds[-1].string) if 'Exchange' in index else tds[-1].string
    info['Trading'] = detail

    # Get Spread
    url = 'http://www.londonstockexchange.com/exchange/prices-and-markets/stocks/summary/company-summary/{}.html'
    html = urllib2.urlopen(
        urllib2.Request(
            url.format(query),headers=header
        )
    )
    soup = BeautifulSoup(html, 'html.parser')
    try:
        tSummary = soup.find_all('table')[0]
    except:
        print 'Data not Available'
        return info

    detail = {}
    for tr in tSummary.find('tbody').find_all('tr'):
        tds = tr.find_all('td')
        if len(tds) > 1:
            for index, value in zip(tds[0::2],tds[1::2]):
                index = valid_str(index.string)
                detail[index] = value.string if 'Var' in index or 'Last' in index or 'status' in index or 'Special' in index or index == '' else valid_num(value.string)
    info['Summary'] = detail

    stats = {}
    # DATA orgainising --------------------------
    stats['MarketCap'] = info['Company']['Marketcapinmillions']
    stats['Profit'] = info['Income']['ProfitBeforeTax'][-1]
    stats['MPRatio'] = stats['MarketCap'] / stats['Profit'] if stats['Profit'] > 0 else 999
    stats['PE'] = info['Ratio']['PERatioAdjusted'][-1]
    stats['EMS'] = info['Trading']['Exchangemarketsize']
    offer = info['Summary']['Offer']
    bid = info['Summary']['Bid']
    stats['Spread'] = 100 * (offer - bid) / bid if bid > 0 else 99
    stats['Dividend'] = info['Ratio']['DividendYield'][-1]
    stats['NetDebt'] = info['Balance']['TotalLiabilities'][-1]
    try:
        stats['Price'] = info['Summary']['PriceGBX']
    except:
        stats['Price'] = 0
        print 'Price is not GBP'
    stats['Bid'] = info['Summary']['Bid']
    stats['Offer'] = info['Summary']['Offer']
    stats['Liquidity'] = stats['EMS'] * stats['Price'] / 100.0
    stats['DPRatio'] = stats['NetDebt'] / stats['Profit'] * -1.0 if stats['Profit'] != 0 else 3
    # -------------------------------------------
    info['stats'] = stats

    if collection:
        obj, created = collection.objects.get_or_create(Symbol=symbol)
        obj.MarketCap = stats['MarketCap']
        obj.Profit = stats['Profit']
        obj.MPRatio = stats['MPRatio']
        obj.PE = stats['PE']
        obj.EMS = stats['EMS']
        obj.Bid = stats['Bid']
        obj.Offer = stats['Offer']
        obj.Spread = stats['Spread']
        obj.Price = stats['Price']
        obj.Dividend = stats['Dividend']
        obj.NetDebt = stats['NetDebt']
        obj.Liquidity = stats['Liquidity']
        obj.DPRatio = stats['DPRatio']
        obj.Sector = info['Trading']['FTSEsector']
        obj.Catagory = info['Trading']['FTSEindex']
        obj.ProfitTrend = get_slope(info['Income']['ProfitBeforeTax'])
        obj.DividendTrend = get_slope(info['Ratio']['DividendYield'])
        obj.DebtTrend = get_slope(info['Balance']['TotalLiabilities'])
        obj.pub_date = timezone.now()
        obj.save()

    return info