def LISTVIDEOS(url):
    baseurl = url
    if '/?s=' in url:
        is_search = 1
    else:
        is_search = 0

    content = utils.grab_url(url)
    video_info = utils.scrape_data(content)

    if is_search and (len(video_info) < 1):
        print 'No search results'
        dialog = xbmcgui.Dialog()
        dialog.notification('No results!', 'No videos found with this search!', xbmcgui.NOTIFICATION_INFO, 1000)
        # TODO: Go Back to Mode 1
        return

    # Get current page and max page number
    matchc = re.compile('class="pages">Page (.+?) of (.+?)</span>').findall(content)
    if matchc == []:
        print 'This show or search result only has one page'  #DEBUG INFO
        for url, name, thumbnail, plot in video_info:
            addLink(name, url, 3, thumbnail, plot)
        return
    else:
        curp = int(matchc[0][0])
        nextp = curp + 1
        prevp = curp - 1
        lastp = int(matchc[0][1])
        position = 'Page ' + str(curp) + ' of ' + str(lastp)
        # Display Page Indicator if activated in settings!
        if get_bool_settings('page_indicator'):
            addInfo(matchc[0][0], matchc[0][1], baseurl)

    for url, name, thumbnail, plot in video_info:
        addLink(name, url, 3, thumbnail, plot, show_url=baseurl, max_page=lastp)

    if is_search == 0:
        if '/page/' in baseurl:
            baseurl = re.sub("/page/.+", "/page/", baseurl)
        else:
            baseurl = baseurl + '/page/'

        if curp == 1:
            addDir('Prev Page (' + str(lastp) + ' of ' + str(lastp) + ')', baseurl + str(lastp), 2, '')
        else:
            addDir('Prev Page (' + str(prevp) + ' of ' + str(lastp) + ')', baseurl + str(prevp), 2, '')

        if curp < lastp:
            addDir('Next Page (' + str(nextp) + ' of ' + str(lastp) + ')', baseurl + str(nextp), 2, '')
    else:
        sstring = re.sub(".+?/?s=", "/?s=", baseurl)
        surl = 'http://channelawesome.com/page/'
        if curp == 1:
            addDir('Prev Page (' + str(lastp) + ' of ' + str(lastp) + ')', surl + str(lastp) + sstring, 2, '')
        else:
            addDir('Prev Page (' + str(prevp) + ' of ' + str(lastp) + ')', surl + str(prevp) + sstring, 2, '')

        if curp < lastp:
            addDir('Next Page (' + str(nextp) + ' of ' + str(lastp) + ')', surl + str(nextp) + sstring, 2, '')
def LISTALL2(url):
    baseurl = url
    content = utils.grab_url(url)
    # Get max page number
    matchc = re.compile('class="pages">Page (.+?) of (.+?)</span>').findall(content)
    lastp = int(matchc[0][1])

    progress = xbmcgui.DialogProgress()
    progress.create('Progress', 'Listing all videos...')
    for i in range(1, lastp + 1):
        percent = int( (100 / lastp) * i)
        message = "Scraping page " + str(i) + " out of " + str(lastp)
        progress.update( percent, "", message, "" )
        xbmc.sleep( 1000 )
        url = baseurl + '/page/' + str(i)
        page_content = utils.grab_url(url)
        video_info = utils.scrape_data(page_content)
        for url, name, thumbnail, plot in video_info:
            addLink(name, url, 4, thumbnail, plot)
        if progress.iscanceled():
            break
    progress.close()
def LISTALL2(url):
    baseurl = url
    content = utils.grab_url(url)
    # Get max page number
    matchc = re.compile('class="pages">Page (.+?) of (.+?)</span>').findall(
        content)
    lastp = int(matchc[0][1])

    progress = xbmcgui.DialogProgress()
    progress.create('Progress', 'Listing all videos...')
    for i in range(1, lastp + 1):
        percent = int((100 / lastp) * i)
        message = "Scraping page " + str(i) + " out of " + str(lastp)
        progress.update(percent, "", message, "")
        xbmc.sleep(1000)
        url = baseurl + '/page/' + str(i)
        page_content = utils.grab_url(url)
        video_info = utils.scrape_data(page_content)
        for url, name, thumbnail, plot in video_info:
            addLink(name, url, 4, thumbnail, plot)
        if progress.iscanceled():
            break
    progress.close()
def LISTVIDEOS(url):
    baseurl = url
    if '/?s=' in url:
        is_search = 1
    else:
        is_search = 0

    content = utils.grab_url(url)
    video_info = utils.scrape_data(content)

    if is_search and (len(video_info) < 1):
        print 'No search results'
        dialog = xbmcgui.Dialog()
        dialog.notification('No results!', 'No videos found with this search!',
                            xbmcgui.NOTIFICATION_INFO, 1000)
        # TODO: Go Back to Mode 1
        return

    # Get current page and max page number
    matchc = re.compile('class="pages">Page (.+?) of (.+?)</span>').findall(
        content)
    if matchc == []:
        print 'This show or search result only has one page'  #DEBUG INFO
        for url, name, thumbnail, plot in video_info:
            addLink(name, url, 3, thumbnail, plot)
        return
    else:
        curp = int(matchc[0][0])
        nextp = curp + 1
        prevp = curp - 1
        lastp = int(matchc[0][1])
        position = 'Page ' + str(curp) + ' of ' + str(lastp)
        # Display Page Indicator if activated in settings!
        if get_bool_settings('page_indicator'):
            addInfo(matchc[0][0], matchc[0][1], baseurl)

    for url, name, thumbnail, plot in video_info:
        addLink(name,
                url,
                3,
                thumbnail,
                plot,
                show_url=baseurl,
                max_page=lastp)

    if is_search == 0:
        if '/page/' in baseurl:
            baseurl = re.sub("/page/.+", "/page/", baseurl)
        else:
            baseurl = baseurl + '/page/'

        if curp == 1:
            addDir('Prev Page (' + str(lastp) + ' of ' + str(lastp) + ')',
                   baseurl + str(lastp), 2, '')
        else:
            addDir('Prev Page (' + str(prevp) + ' of ' + str(lastp) + ')',
                   baseurl + str(prevp), 2, '')

        if curp < lastp:
            addDir('Next Page (' + str(nextp) + ' of ' + str(lastp) + ')',
                   baseurl + str(nextp), 2, '')
    else:
        sstring = re.sub(".+?/?s=", "/?s=", baseurl)
        surl = 'http://channelawesome.com/page/'
        if curp == 1:
            addDir('Prev Page (' + str(lastp) + ' of ' + str(lastp) + ')',
                   surl + str(lastp) + sstring, 2, '')
        else:
            addDir('Prev Page (' + str(prevp) + ' of ' + str(lastp) + ')',
                   surl + str(prevp) + sstring, 2, '')

        if curp < lastp:
            addDir('Next Page (' + str(nextp) + ' of ' + str(lastp) + ')',
                   surl + str(nextp) + sstring, 2, '')
def EDIT_DB(db_arg):
    print 'Scrape for DB Activated'
    #print 'Parameter: ' + str(db_arg)
    aborted = 0
    data = db_arg.split(';')
    db_mode = data[0].replace('MODE:', '')
    db_name = data[1].replace('NAME:', '')
    db_url = data[2].replace('URL:', '')
    show_name = db_name
    db_name = db_name.replace(' ', '').lower()
    print 'DB Mode: ' + db_mode
    print 'DB Url:' + db_url
    print 'DB name:' + db_name
    print 'Show name:' + show_name

    # Connect to DB
    conn = sqlite3.connect(local_db)
    conn.text_factory = str
    c = conn.cursor()
    if db_mode == 'UPDATE':
        print 'UPDATE STARTED!!!'
        print 'Show URL: ' + db_url
        # Progress Dialog
        progress = xbmcgui.DialogProgress()
        progress.create('Updating Show', 'Scraping new videos...')
        progress.update(1, "", 'Initializing scrape...', "")
        # Grab Page Content
        content = utils.grab_url(db_url)
        # Grab Show Icon
        show_icon = utils.grab_icon(content)
        # Get max page number
        matchc = re.compile(
            'class="pages">Page (.+?) of (.+?)</span>').findall(content)
        lastp = int(matchc[0][1])

        counter = 0
        video_list = []
        for i in range(1, lastp + 1):
            allready_in_db = 0
            percent = int((100 / lastp) * i)
            message = "Scraping page " + str(i) + " of " + str(lastp)
            progress.update(percent, "", message, "")
            xbmc.sleep(1000)
            url = db_url + 'page/' + str(i)
            page_content = utils.grab_url(url)

            video_info = utils.scrape_data(page_content, 'DB')

            for url, name, thumbnail, plot, airdate in video_info:
                # Test to see if entry is allready in DB
                c.execute('SELECT airdate FROM {} WHERE name="{}"'.format(
                    db_name, name))
                test = c.fetchone()
                if test is None:
                    print 'Episode: "{}" not in DB, adding to update list now!'.format(
                        name)
                    entry = (name, url, thumbnail, plot, airdate)
                    video_list.append(entry)
                    counter = counter + 1
                else:  # Entry available in local DB
                    print 'Episode: "{}" Allready in DB, breaking loop now!'.format(
                        name)
                    print 'Stopping update process but will commit changes (if any) to DB'
                    allready_in_db = 1
                    break

            if progress.iscanceled():
                aborted = True
                break
            if allready_in_db:
                break

        progress.close()
        if aborted:
            print 'Scrape was aborted!'  # DEBUG INFO
            dialog = xbmcgui.Dialog()
            dialog.notification('Update canceled!',
                                'No updates were stored in the DB',
                                xbmcgui.NOTIFICATION_WARNING, 3000)
        else:
            dialog = xbmcgui.Dialog()
            c.execute(
                'UPDATE show_list SET show_icon="{}" WHERE table_name="{}"'.
                format(show_icon, db_name))
            if len(video_list) > 0:
                video_list.reverse()
                c.executemany(
                    'INSERT INTO {} (name, url, thumb, plot, airdate) VALUES (?,?,?,?,?)'
                    .format(db_name), video_list)
                dialog.notification(
                    'Update Successfull',
                    '{} entries added to the DB'.format(str(counter)),
                    xbmcgui.NOTIFICATION_INFO, 3000)
            else:
                dialog.notification('No Updates',
                                    'No updates were found for this show',
                                    xbmcgui.NOTIFICATION_INFO, 3000)
        conn.commit()
        conn.close()

    elif db_mode == 'ADD':
        c.execute(
            'SELECT name FROM sqlite_master WHERE type="table" AND name="{}"'.
            format(db_name))
        test = c.fetchone()
        if test is None:
            print 'Table: "{}" does not exist in DB, i will continue'.format(
                db_name)
            # Progress Dialog
            progress = xbmcgui.DialogProgress()
            progress.create('Adding Show to local Database',
                            'Scraping all videos...')
            progress.update(1, "", 'Initializing scrape...', "")
            # Grab Page Conent
            content = utils.grab_url(db_url)
            # Grab Show Icon
            show_icon = utils.grab_icon(content)
            # Get max page number
            matchc = re.compile(
                'class="pages">Page (.+?) of (.+?)</span>').findall(content)
            lastp = int(matchc[0][1])
            video_list = []
            for i in range(1, lastp + 1):
                percent = int((100 / lastp) * i)
                message = "Scraping page " + str(i) + " of " + str(lastp)
                progress.update(percent, "", message, "")
                xbmc.sleep(1000)
                url = db_url + 'page/' + str(i)  # Modified "/page/" to "page/"
                page_content = utils.grab_url(url)

                video_info = utils.scrape_data(page_content, 'DB')

                for url, name, thumbnail, plot, airdate in video_info:
                    print 'name: ' + utils.cleanName(
                        name
                    )  # + '\nplot: ' + utils.cleanName(plot) # DEBUG INFO
                    entry = (name, url, thumbnail, plot, airdate)
                    video_list.append(entry)
                if progress.iscanceled():
                    aborted = True
                    break

            progress.close()
            dialog = xbmcgui.Dialog()
            if aborted:
                print 'Scrape was aborted!'  # DEBUG INFO
                dialog.notification('Process canceled!',
                                    'The show was not added to the local DB.',
                                    xbmcgui.NOTIFICATION_WARNING, 3000)
            else:
                if len(video_list) > 0:
                    # Turn List of Videos arround to get accurate list based on release date
                    video_list.reverse()
                    # Create DB Tables
                    c.execute(
                        'CREATE TABLE IF NOT EXISTS {} (id INTEGER PRIMARY KEY, name TEXT, url TEXT, thumb TEXT, plot TEXT, airdate TEXT)'
                        .format(db_name))
                    c.execute(
                        'CREATE TABLE IF NOT EXISTS show_list (table_name  TEXT PRIMARY KEY, show_name TEXT, show_url TEXT, show_icon TEXT)'
                    )
                    # Add Content to DB
                    c.execute(
                        'INSERT OR REPLACE INTO show_list VALUES ("{}", "{}", "{}", "{}")'
                        .format(db_name, show_name, db_url, show_icon))
                    c.executemany(
                        'INSERT INTO {} (name, url, thumb, plot, airdate) VALUES (?,?,?,?,?)'
                        .format(db_name), video_list)
                    dialog.notification(
                        'Process Complete!',
                        'The show "{}" was added to the local DB.'.format(
                            show_name), xbmcgui.NOTIFICATION_INFO, 3000)
                    conn.commit()
        else:
            print 'Table: "{}" does exist in DB, i will not continue'.format(
                db_name)
            dialog = xbmcgui.Dialog()
            dialog.ok(
                'Database',
                'Show "{}" is allready in the local DB. Update from there.'.
                format(show_name))
        conn.close()

    elif db_mode == 'REMOVE':
        dialog = xbmcgui.Dialog()
        if dialog.yesno("Remove from local DB",
                        'Dou you really want to remove the show:',
                        '{}'.format(show_name), "", 'No', 'Yes'):
            c.execute('DROP TABLE IF EXISTS "{}"'.format(db_name))
            c.execute(
                'DELETE FROM show_list WHERE table_name="{}"'.format(db_name))
            conn.commit()
        conn.close()
        xbmc.executebuiltin("Container.Refresh")
Exemplo n.º 6
0
# Import packages
from datetime import datetime, timedelta
import pandas as pd

import streamlit as st
import altair as alt

# Import parameters and utility functions
from params import stocks, threads, margin, long_term, short_term, buy_asap, sell_asap
from utils import style_df, scrape_data, get_history, rearrange_df


# Scrape and parse the data
df = scrape_data(stocks, threads)


# Display df
st.text('Stocks overview')
st.dataframe(style_df(df, bar=[]))

st.text('Stocks with > 4% dip')
df_dip = df[(df['% Change Previous Close'] < margin) | (df['% Change Cost Basis'] < margin)]
st.dataframe(style_df(df_dip, bar=[]))

st.text('Stocks below desired buy price')
df_buy = df[(df['Current Price'] < df['Sell/Buy Price']) & (df['Sell Pct'] < 0)]
st.dataframe(style_df(df_buy, bar=[]))

st.text('Stocks above desired sell price')
df_sell = df[(df['Current Price'] > df['Sell/Buy Price']) & (df['Sell Pct'] > 0)]
st.dataframe(style_df(df_sell, bar=[]))
def EDIT_DB(db_arg):
    print 'Scrape for DB Activated'
    #print 'Parameter: ' + str(db_arg)
    aborted = 0
    data = db_arg.split(';')
    db_mode = data[0].replace('MODE:', '')
    db_name = data[1].replace('NAME:', '')
    db_url = data[2].replace('URL:', '')
    show_name = db_name
    db_name = db_name.replace(' ', '').lower()
    print 'DB Mode: ' + db_mode
    print 'DB Url:' + db_url
    print 'DB name:' + db_name
    print 'Show name:' + show_name

    # Connect to DB
    conn = sqlite3.connect(local_db)
    conn.text_factory = str
    c = conn.cursor()
    if db_mode == 'UPDATE':
        print 'UPDATE STARTED!!!'
        print 'Show URL: '+ db_url
        # Progress Dialog
        progress = xbmcgui.DialogProgress()
        progress.create('Updating Show', 'Scraping new videos...')
        progress.update( 1, "", 'Initializing scrape...', "" )
        # Grab Page Content
        content = utils.grab_url(db_url)
        # Grab Show Icon
        show_icon = utils.grab_icon(content)
        # Get max page number
        matchc = re.compile('class="pages">Page (.+?) of (.+?)</span>').findall(content)
        lastp = int(matchc[0][1])

        counter = 0
        video_list = []
        for i in range(1, lastp + 1):
            allready_in_db = 0
            percent = int( (100 / lastp) * i)
            message = "Scraping page " + str(i) + " of " + str(lastp)
            progress.update( percent, "", message, "" )
            xbmc.sleep( 1000 )
            url = db_url + 'page/' + str(i)
            page_content = utils.grab_url(url)

            video_info = utils.scrape_data(page_content, 'DB')

            for url, name, thumbnail, plot, airdate in video_info:
                # Test to see if entry is allready in DB
                c.execute('SELECT airdate FROM {} WHERE name="{}"'.format(db_name, name))
                test = c.fetchone()
                if test is None:
                    print 'Episode: "{}" not in DB, adding to update list now!'.format(name)
                    entry = (name, url, thumbnail, plot, airdate)
                    video_list.append(entry)
                    counter = counter + 1
                else: # Entry available in local DB
                    print 'Episode: "{}" Allready in DB, breaking loop now!'.format(name)
                    print 'Stopping update process but will commit changes (if any) to DB'
                    allready_in_db = 1
                    break

            if progress.iscanceled():
                aborted = True
                break
            if allready_in_db:
                break

        progress.close()
        if aborted:
            print 'Scrape was aborted!' # DEBUG INFO
            dialog = xbmcgui.Dialog()
            dialog.notification('Update canceled!', 'No updates were stored in the DB', xbmcgui.NOTIFICATION_WARNING, 3000)
        else:
            dialog = xbmcgui.Dialog()
            c.execute('UPDATE show_list SET show_icon="{}" WHERE table_name="{}"'.format(show_icon, db_name))
            if len(video_list) > 0:
                video_list.reverse()
                c.executemany('INSERT INTO {} (name, url, thumb, plot, airdate) VALUES (?,?,?,?,?)'.format(db_name), video_list)
                dialog.notification('Update Successfull', '{} entries added to the DB'.format(str(counter)), xbmcgui.NOTIFICATION_INFO, 3000)
            else:
                dialog.notification('No Updates', 'No updates were found for this show', xbmcgui.NOTIFICATION_INFO, 3000)
        conn.commit()
        conn.close()

    elif db_mode == 'ADD':
        c.execute('SELECT name FROM sqlite_master WHERE type="table" AND name="{}"'.format(db_name))
        test = c.fetchone()
        if test is None:
            print 'Table: "{}" does not exist in DB, i will continue'.format(db_name)
            # Progress Dialog
            progress = xbmcgui.DialogProgress()
            progress.create('Adding Show to local Database', 'Scraping all videos...')
            progress.update( 1, "", 'Initializing scrape...', "" )
            # Grab Page Conent
            content = utils.grab_url(db_url)
            # Grab Show Icon
            show_icon = utils.grab_icon(content)
            # Get max page number
            matchc = re.compile('class="pages">Page (.+?) of (.+?)</span>').findall(content)
            lastp = int(matchc[0][1])
            video_list = []
            for i in range(1, lastp + 1):
                percent = int( (100 / lastp) * i)
                message = "Scraping page " + str(i) + " of " + str(lastp)
                progress.update( percent, "", message, "" )
                xbmc.sleep( 1000 )
                url = db_url + 'page/' + str(i) # Modified "/page/" to "page/"
                page_content = utils.grab_url(url)

                video_info = utils.scrape_data(page_content, 'DB')

                for url, name, thumbnail, plot, airdate in video_info:
                    print 'name: ' + utils.cleanName(name) # + '\nplot: ' + utils.cleanName(plot) # DEBUG INFO
                    entry = (name, url, thumbnail, plot, airdate)
                    video_list.append(entry)
                if progress.iscanceled():
                    aborted = True
                    break

            progress.close()
            dialog = xbmcgui.Dialog()
            if aborted:
                print 'Scrape was aborted!' # DEBUG INFO
                dialog.notification('Process canceled!', 'The show was not added to the local DB.', xbmcgui.NOTIFICATION_WARNING, 3000)
            else:
                if len(video_list) > 0:
                    # Turn List of Videos arround to get accurate list based on release date
                    video_list.reverse()
                    # Create DB Tables
                    c.execute('CREATE TABLE IF NOT EXISTS {} (id INTEGER PRIMARY KEY, name TEXT, url TEXT, thumb TEXT, plot TEXT, airdate TEXT)'.format(db_name))
                    c.execute('CREATE TABLE IF NOT EXISTS show_list (table_name  TEXT PRIMARY KEY, show_name TEXT, show_url TEXT, show_icon TEXT)')
                    # Add Content to DB
                    c.execute('INSERT OR REPLACE INTO show_list VALUES ("{}", "{}", "{}", "{}")'.format(db_name, show_name, db_url, show_icon))
                    c.executemany('INSERT INTO {} (name, url, thumb, plot, airdate) VALUES (?,?,?,?,?)'.format(db_name), video_list)
                    dialog.notification('Process Complete!',
                                        'The show "{}" was added to the local DB.'.format(show_name),
                                        xbmcgui.NOTIFICATION_INFO, 3000)
                    conn.commit()
        else:
            print 'Table: "{}" does exist in DB, i will not continue'.format(db_name)
            dialog = xbmcgui.Dialog()
            dialog.ok('Database', 'Show "{}" is allready in the local DB. Update from there.'.format(show_name))
        conn.close()

    elif db_mode == 'REMOVE':
        dialog = xbmcgui.Dialog()
        if dialog.yesno("Remove from local DB", 'Dou you really want to remove the show:','{}'.format(show_name), "",'No','Yes'):
            c.execute('DROP TABLE IF EXISTS "{}"'.format(db_name))
            c.execute('DELETE FROM show_list WHERE table_name="{}"'.format(db_name))
            conn.commit()
        conn.close()
        xbmc.executebuiltin("Container.Refresh")