def LISTVIDEOS(url): baseurl = url if '/?s=' in url: is_search = 1 else: is_search = 0 content = utils.grab_url(url) video_info = utils.scrape_data(content) if is_search and (len(video_info) < 1): print 'No search results' dialog = xbmcgui.Dialog() dialog.notification('No results!', 'No videos found with this search!', xbmcgui.NOTIFICATION_INFO, 1000) # TODO: Go Back to Mode 1 return # Get current page and max page number matchc = re.compile('class="pages">Page (.+?) of (.+?)</span>').findall(content) if matchc == []: print 'This show or search result only has one page' #DEBUG INFO for url, name, thumbnail, plot in video_info: addLink(name, url, 3, thumbnail, plot) return else: curp = int(matchc[0][0]) nextp = curp + 1 prevp = curp - 1 lastp = int(matchc[0][1]) position = 'Page ' + str(curp) + ' of ' + str(lastp) # Display Page Indicator if activated in settings! if get_bool_settings('page_indicator'): addInfo(matchc[0][0], matchc[0][1], baseurl) for url, name, thumbnail, plot in video_info: addLink(name, url, 3, thumbnail, plot, show_url=baseurl, max_page=lastp) if is_search == 0: if '/page/' in baseurl: baseurl = re.sub("/page/.+", "/page/", baseurl) else: baseurl = baseurl + '/page/' if curp == 1: addDir('Prev Page (' + str(lastp) + ' of ' + str(lastp) + ')', baseurl + str(lastp), 2, '') else: addDir('Prev Page (' + str(prevp) + ' of ' + str(lastp) + ')', baseurl + str(prevp), 2, '') if curp < lastp: addDir('Next Page (' + str(nextp) + ' of ' + str(lastp) + ')', baseurl + str(nextp), 2, '') else: sstring = re.sub(".+?/?s=", "/?s=", baseurl) surl = 'http://channelawesome.com/page/' if curp == 1: addDir('Prev Page (' + str(lastp) + ' of ' + str(lastp) + ')', surl + str(lastp) + sstring, 2, '') else: addDir('Prev Page (' + str(prevp) + ' of ' + str(lastp) + ')', surl + str(prevp) + sstring, 2, '') if curp < lastp: addDir('Next Page (' + str(nextp) + ' of ' + str(lastp) + ')', surl + str(nextp) + sstring, 2, '')
def LISTALL2(url): baseurl = url content = utils.grab_url(url) # Get max page number matchc = re.compile('class="pages">Page (.+?) of (.+?)</span>').findall(content) lastp = int(matchc[0][1]) progress = xbmcgui.DialogProgress() progress.create('Progress', 'Listing all videos...') for i in range(1, lastp + 1): percent = int( (100 / lastp) * i) message = "Scraping page " + str(i) + " out of " + str(lastp) progress.update( percent, "", message, "" ) xbmc.sleep( 1000 ) url = baseurl + '/page/' + str(i) page_content = utils.grab_url(url) video_info = utils.scrape_data(page_content) for url, name, thumbnail, plot in video_info: addLink(name, url, 4, thumbnail, plot) if progress.iscanceled(): break progress.close()
def LISTALL2(url): baseurl = url content = utils.grab_url(url) # Get max page number matchc = re.compile('class="pages">Page (.+?) of (.+?)</span>').findall( content) lastp = int(matchc[0][1]) progress = xbmcgui.DialogProgress() progress.create('Progress', 'Listing all videos...') for i in range(1, lastp + 1): percent = int((100 / lastp) * i) message = "Scraping page " + str(i) + " out of " + str(lastp) progress.update(percent, "", message, "") xbmc.sleep(1000) url = baseurl + '/page/' + str(i) page_content = utils.grab_url(url) video_info = utils.scrape_data(page_content) for url, name, thumbnail, plot in video_info: addLink(name, url, 4, thumbnail, plot) if progress.iscanceled(): break progress.close()
def LISTVIDEOS(url): baseurl = url if '/?s=' in url: is_search = 1 else: is_search = 0 content = utils.grab_url(url) video_info = utils.scrape_data(content) if is_search and (len(video_info) < 1): print 'No search results' dialog = xbmcgui.Dialog() dialog.notification('No results!', 'No videos found with this search!', xbmcgui.NOTIFICATION_INFO, 1000) # TODO: Go Back to Mode 1 return # Get current page and max page number matchc = re.compile('class="pages">Page (.+?) of (.+?)</span>').findall( content) if matchc == []: print 'This show or search result only has one page' #DEBUG INFO for url, name, thumbnail, plot in video_info: addLink(name, url, 3, thumbnail, plot) return else: curp = int(matchc[0][0]) nextp = curp + 1 prevp = curp - 1 lastp = int(matchc[0][1]) position = 'Page ' + str(curp) + ' of ' + str(lastp) # Display Page Indicator if activated in settings! if get_bool_settings('page_indicator'): addInfo(matchc[0][0], matchc[0][1], baseurl) for url, name, thumbnail, plot in video_info: addLink(name, url, 3, thumbnail, plot, show_url=baseurl, max_page=lastp) if is_search == 0: if '/page/' in baseurl: baseurl = re.sub("/page/.+", "/page/", baseurl) else: baseurl = baseurl + '/page/' if curp == 1: addDir('Prev Page (' + str(lastp) + ' of ' + str(lastp) + ')', baseurl + str(lastp), 2, '') else: addDir('Prev Page (' + str(prevp) + ' of ' + str(lastp) + ')', baseurl + str(prevp), 2, '') if curp < lastp: addDir('Next Page (' + str(nextp) + ' of ' + str(lastp) + ')', baseurl + str(nextp), 2, '') else: sstring = re.sub(".+?/?s=", "/?s=", baseurl) surl = 'http://channelawesome.com/page/' if curp == 1: addDir('Prev Page (' + str(lastp) + ' of ' + str(lastp) + ')', surl + str(lastp) + sstring, 2, '') else: addDir('Prev Page (' + str(prevp) + ' of ' + str(lastp) + ')', surl + str(prevp) + sstring, 2, '') if curp < lastp: addDir('Next Page (' + str(nextp) + ' of ' + str(lastp) + ')', surl + str(nextp) + sstring, 2, '')
def EDIT_DB(db_arg): print 'Scrape for DB Activated' #print 'Parameter: ' + str(db_arg) aborted = 0 data = db_arg.split(';') db_mode = data[0].replace('MODE:', '') db_name = data[1].replace('NAME:', '') db_url = data[2].replace('URL:', '') show_name = db_name db_name = db_name.replace(' ', '').lower() print 'DB Mode: ' + db_mode print 'DB Url:' + db_url print 'DB name:' + db_name print 'Show name:' + show_name # Connect to DB conn = sqlite3.connect(local_db) conn.text_factory = str c = conn.cursor() if db_mode == 'UPDATE': print 'UPDATE STARTED!!!' print 'Show URL: ' + db_url # Progress Dialog progress = xbmcgui.DialogProgress() progress.create('Updating Show', 'Scraping new videos...') progress.update(1, "", 'Initializing scrape...', "") # Grab Page Content content = utils.grab_url(db_url) # Grab Show Icon show_icon = utils.grab_icon(content) # Get max page number matchc = re.compile( 'class="pages">Page (.+?) of (.+?)</span>').findall(content) lastp = int(matchc[0][1]) counter = 0 video_list = [] for i in range(1, lastp + 1): allready_in_db = 0 percent = int((100 / lastp) * i) message = "Scraping page " + str(i) + " of " + str(lastp) progress.update(percent, "", message, "") xbmc.sleep(1000) url = db_url + 'page/' + str(i) page_content = utils.grab_url(url) video_info = utils.scrape_data(page_content, 'DB') for url, name, thumbnail, plot, airdate in video_info: # Test to see if entry is allready in DB c.execute('SELECT airdate FROM {} WHERE name="{}"'.format( db_name, name)) test = c.fetchone() if test is None: print 'Episode: "{}" not in DB, adding to update list now!'.format( name) entry = (name, url, thumbnail, plot, airdate) video_list.append(entry) counter = counter + 1 else: # Entry available in local DB print 'Episode: "{}" Allready in DB, breaking loop now!'.format( name) print 'Stopping update process but will commit changes (if any) to DB' allready_in_db = 1 break if progress.iscanceled(): aborted = True break if allready_in_db: break progress.close() if aborted: print 'Scrape was aborted!' # DEBUG INFO dialog = xbmcgui.Dialog() dialog.notification('Update canceled!', 'No updates were stored in the DB', xbmcgui.NOTIFICATION_WARNING, 3000) else: dialog = xbmcgui.Dialog() c.execute( 'UPDATE show_list SET show_icon="{}" WHERE table_name="{}"'. format(show_icon, db_name)) if len(video_list) > 0: video_list.reverse() c.executemany( 'INSERT INTO {} (name, url, thumb, plot, airdate) VALUES (?,?,?,?,?)' .format(db_name), video_list) dialog.notification( 'Update Successfull', '{} entries added to the DB'.format(str(counter)), xbmcgui.NOTIFICATION_INFO, 3000) else: dialog.notification('No Updates', 'No updates were found for this show', xbmcgui.NOTIFICATION_INFO, 3000) conn.commit() conn.close() elif db_mode == 'ADD': c.execute( 'SELECT name FROM sqlite_master WHERE type="table" AND name="{}"'. format(db_name)) test = c.fetchone() if test is None: print 'Table: "{}" does not exist in DB, i will continue'.format( db_name) # Progress Dialog progress = xbmcgui.DialogProgress() progress.create('Adding Show to local Database', 'Scraping all videos...') progress.update(1, "", 'Initializing scrape...', "") # Grab Page Conent content = utils.grab_url(db_url) # Grab Show Icon show_icon = utils.grab_icon(content) # Get max page number matchc = re.compile( 'class="pages">Page (.+?) of (.+?)</span>').findall(content) lastp = int(matchc[0][1]) video_list = [] for i in range(1, lastp + 1): percent = int((100 / lastp) * i) message = "Scraping page " + str(i) + " of " + str(lastp) progress.update(percent, "", message, "") xbmc.sleep(1000) url = db_url + 'page/' + str(i) # Modified "/page/" to "page/" page_content = utils.grab_url(url) video_info = utils.scrape_data(page_content, 'DB') for url, name, thumbnail, plot, airdate in video_info: print 'name: ' + utils.cleanName( name ) # + '\nplot: ' + utils.cleanName(plot) # DEBUG INFO entry = (name, url, thumbnail, plot, airdate) video_list.append(entry) if progress.iscanceled(): aborted = True break progress.close() dialog = xbmcgui.Dialog() if aborted: print 'Scrape was aborted!' # DEBUG INFO dialog.notification('Process canceled!', 'The show was not added to the local DB.', xbmcgui.NOTIFICATION_WARNING, 3000) else: if len(video_list) > 0: # Turn List of Videos arround to get accurate list based on release date video_list.reverse() # Create DB Tables c.execute( 'CREATE TABLE IF NOT EXISTS {} (id INTEGER PRIMARY KEY, name TEXT, url TEXT, thumb TEXT, plot TEXT, airdate TEXT)' .format(db_name)) c.execute( 'CREATE TABLE IF NOT EXISTS show_list (table_name TEXT PRIMARY KEY, show_name TEXT, show_url TEXT, show_icon TEXT)' ) # Add Content to DB c.execute( 'INSERT OR REPLACE INTO show_list VALUES ("{}", "{}", "{}", "{}")' .format(db_name, show_name, db_url, show_icon)) c.executemany( 'INSERT INTO {} (name, url, thumb, plot, airdate) VALUES (?,?,?,?,?)' .format(db_name), video_list) dialog.notification( 'Process Complete!', 'The show "{}" was added to the local DB.'.format( show_name), xbmcgui.NOTIFICATION_INFO, 3000) conn.commit() else: print 'Table: "{}" does exist in DB, i will not continue'.format( db_name) dialog = xbmcgui.Dialog() dialog.ok( 'Database', 'Show "{}" is allready in the local DB. Update from there.'. format(show_name)) conn.close() elif db_mode == 'REMOVE': dialog = xbmcgui.Dialog() if dialog.yesno("Remove from local DB", 'Dou you really want to remove the show:', '{}'.format(show_name), "", 'No', 'Yes'): c.execute('DROP TABLE IF EXISTS "{}"'.format(db_name)) c.execute( 'DELETE FROM show_list WHERE table_name="{}"'.format(db_name)) conn.commit() conn.close() xbmc.executebuiltin("Container.Refresh")
# Import packages from datetime import datetime, timedelta import pandas as pd import streamlit as st import altair as alt # Import parameters and utility functions from params import stocks, threads, margin, long_term, short_term, buy_asap, sell_asap from utils import style_df, scrape_data, get_history, rearrange_df # Scrape and parse the data df = scrape_data(stocks, threads) # Display df st.text('Stocks overview') st.dataframe(style_df(df, bar=[])) st.text('Stocks with > 4% dip') df_dip = df[(df['% Change Previous Close'] < margin) | (df['% Change Cost Basis'] < margin)] st.dataframe(style_df(df_dip, bar=[])) st.text('Stocks below desired buy price') df_buy = df[(df['Current Price'] < df['Sell/Buy Price']) & (df['Sell Pct'] < 0)] st.dataframe(style_df(df_buy, bar=[])) st.text('Stocks above desired sell price') df_sell = df[(df['Current Price'] > df['Sell/Buy Price']) & (df['Sell Pct'] > 0)] st.dataframe(style_df(df_sell, bar=[]))
def EDIT_DB(db_arg): print 'Scrape for DB Activated' #print 'Parameter: ' + str(db_arg) aborted = 0 data = db_arg.split(';') db_mode = data[0].replace('MODE:', '') db_name = data[1].replace('NAME:', '') db_url = data[2].replace('URL:', '') show_name = db_name db_name = db_name.replace(' ', '').lower() print 'DB Mode: ' + db_mode print 'DB Url:' + db_url print 'DB name:' + db_name print 'Show name:' + show_name # Connect to DB conn = sqlite3.connect(local_db) conn.text_factory = str c = conn.cursor() if db_mode == 'UPDATE': print 'UPDATE STARTED!!!' print 'Show URL: '+ db_url # Progress Dialog progress = xbmcgui.DialogProgress() progress.create('Updating Show', 'Scraping new videos...') progress.update( 1, "", 'Initializing scrape...', "" ) # Grab Page Content content = utils.grab_url(db_url) # Grab Show Icon show_icon = utils.grab_icon(content) # Get max page number matchc = re.compile('class="pages">Page (.+?) of (.+?)</span>').findall(content) lastp = int(matchc[0][1]) counter = 0 video_list = [] for i in range(1, lastp + 1): allready_in_db = 0 percent = int( (100 / lastp) * i) message = "Scraping page " + str(i) + " of " + str(lastp) progress.update( percent, "", message, "" ) xbmc.sleep( 1000 ) url = db_url + 'page/' + str(i) page_content = utils.grab_url(url) video_info = utils.scrape_data(page_content, 'DB') for url, name, thumbnail, plot, airdate in video_info: # Test to see if entry is allready in DB c.execute('SELECT airdate FROM {} WHERE name="{}"'.format(db_name, name)) test = c.fetchone() if test is None: print 'Episode: "{}" not in DB, adding to update list now!'.format(name) entry = (name, url, thumbnail, plot, airdate) video_list.append(entry) counter = counter + 1 else: # Entry available in local DB print 'Episode: "{}" Allready in DB, breaking loop now!'.format(name) print 'Stopping update process but will commit changes (if any) to DB' allready_in_db = 1 break if progress.iscanceled(): aborted = True break if allready_in_db: break progress.close() if aborted: print 'Scrape was aborted!' # DEBUG INFO dialog = xbmcgui.Dialog() dialog.notification('Update canceled!', 'No updates were stored in the DB', xbmcgui.NOTIFICATION_WARNING, 3000) else: dialog = xbmcgui.Dialog() c.execute('UPDATE show_list SET show_icon="{}" WHERE table_name="{}"'.format(show_icon, db_name)) if len(video_list) > 0: video_list.reverse() c.executemany('INSERT INTO {} (name, url, thumb, plot, airdate) VALUES (?,?,?,?,?)'.format(db_name), video_list) dialog.notification('Update Successfull', '{} entries added to the DB'.format(str(counter)), xbmcgui.NOTIFICATION_INFO, 3000) else: dialog.notification('No Updates', 'No updates were found for this show', xbmcgui.NOTIFICATION_INFO, 3000) conn.commit() conn.close() elif db_mode == 'ADD': c.execute('SELECT name FROM sqlite_master WHERE type="table" AND name="{}"'.format(db_name)) test = c.fetchone() if test is None: print 'Table: "{}" does not exist in DB, i will continue'.format(db_name) # Progress Dialog progress = xbmcgui.DialogProgress() progress.create('Adding Show to local Database', 'Scraping all videos...') progress.update( 1, "", 'Initializing scrape...', "" ) # Grab Page Conent content = utils.grab_url(db_url) # Grab Show Icon show_icon = utils.grab_icon(content) # Get max page number matchc = re.compile('class="pages">Page (.+?) of (.+?)</span>').findall(content) lastp = int(matchc[0][1]) video_list = [] for i in range(1, lastp + 1): percent = int( (100 / lastp) * i) message = "Scraping page " + str(i) + " of " + str(lastp) progress.update( percent, "", message, "" ) xbmc.sleep( 1000 ) url = db_url + 'page/' + str(i) # Modified "/page/" to "page/" page_content = utils.grab_url(url) video_info = utils.scrape_data(page_content, 'DB') for url, name, thumbnail, plot, airdate in video_info: print 'name: ' + utils.cleanName(name) # + '\nplot: ' + utils.cleanName(plot) # DEBUG INFO entry = (name, url, thumbnail, plot, airdate) video_list.append(entry) if progress.iscanceled(): aborted = True break progress.close() dialog = xbmcgui.Dialog() if aborted: print 'Scrape was aborted!' # DEBUG INFO dialog.notification('Process canceled!', 'The show was not added to the local DB.', xbmcgui.NOTIFICATION_WARNING, 3000) else: if len(video_list) > 0: # Turn List of Videos arround to get accurate list based on release date video_list.reverse() # Create DB Tables c.execute('CREATE TABLE IF NOT EXISTS {} (id INTEGER PRIMARY KEY, name TEXT, url TEXT, thumb TEXT, plot TEXT, airdate TEXT)'.format(db_name)) c.execute('CREATE TABLE IF NOT EXISTS show_list (table_name TEXT PRIMARY KEY, show_name TEXT, show_url TEXT, show_icon TEXT)') # Add Content to DB c.execute('INSERT OR REPLACE INTO show_list VALUES ("{}", "{}", "{}", "{}")'.format(db_name, show_name, db_url, show_icon)) c.executemany('INSERT INTO {} (name, url, thumb, plot, airdate) VALUES (?,?,?,?,?)'.format(db_name), video_list) dialog.notification('Process Complete!', 'The show "{}" was added to the local DB.'.format(show_name), xbmcgui.NOTIFICATION_INFO, 3000) conn.commit() else: print 'Table: "{}" does exist in DB, i will not continue'.format(db_name) dialog = xbmcgui.Dialog() dialog.ok('Database', 'Show "{}" is allready in the local DB. Update from there.'.format(show_name)) conn.close() elif db_mode == 'REMOVE': dialog = xbmcgui.Dialog() if dialog.yesno("Remove from local DB", 'Dou you really want to remove the show:','{}'.format(show_name), "",'No','Yes'): c.execute('DROP TABLE IF EXISTS "{}"'.format(db_name)) c.execute('DELETE FROM show_list WHERE table_name="{}"'.format(db_name)) conn.commit() conn.close() xbmc.executebuiltin("Container.Refresh")