def run(self): while True: start_time = time.time() _logger.info("Preparing to query servers for price snapshots...") server_count = 0 try: for (site_id, site_server, site_name, site_url, site_passphrase, _) in DATABASE.aggregate_list_sites(): server_count += 1 _logger.info("Querying {url} ({server})...".format( url=site_url, server=site_server, )) sub_start_time = time.time() try: data = urllib.urlencode({ 'snapshot_passphrase': site_passphrase, 'include_null': False, 'max_age': int(sub_start_time - CONFIG['aggregation']['snapshots']['frequency_hours'] * TIME_HOUR), #Good enough }) data = urllib2.urlopen(site_url + 'items/price-snapshot', data).read() data_len = len(data) data = json.loads(data) data_count = len(data) DATABASE.aggregate_write_prices(site_id, data) except Exception as e: _logger.error("Unable to retrieve data from {url} ({server}): {error}".format( url=site_url, server=site_server, error=str(e), )) else: _logger.info("Retrieved {count} records ({size:.1f}kB) in {time:.2f}s".format( count=data_count, size=(data_len / 1024.0), time=(time.time() - sub_start_time), )) except Exception as e: _logger.error("Unable to retrieve site-list: {error}".format( error=str(e), )) try: _logger.info("Pruning old snapshot data...") DATABASE.aggregate_prune_prices(start_time - (CONFIG['aggregation']['snapshots']['retention_days'] * TIME_DAY)) except Exception as e: _logger.error("Unable to prune old snapshot data: {error}".format( error=str(e), )) execution_time = time.time() - start_time sleep_time = max(0, (CONFIG['aggregation']['snapshots']['frequency_hours'] * TIME_HOUR) - execution_time) _logger.info("Queried {count} servers and cleaned up database in {time:.2f}s; sleeping for {sleep:.2f}s".format( count=server_count, time=execution_time, sleep=sleep_time, )) time.sleep(sleep_time)
def BrowseDir(cur_dir, page_size=20, offset=0, user=None): """Browse directories, with paging. Show directories and compatible archives.""" t1 = time.time() oc = ObjectContainer(title2=unicode(os.path.basename(cur_dir)), no_cache=True) try: dir_list = utils.filtered_listdir(cur_dir) page = dir_list[offset:offset + page_size] except Exception as e: Log.Error(e) return error_message('bad path', 'bad path') for item, is_dir in page: full_path = os.path.join(cur_dir, item) if is_dir: state = DATABASE.dir_read_state(user, full_path) oc.add(DirectoryObject( key=Callback(BrowseDir, cur_dir=full_path, page_size=page_size, user=user), title=unicode(utils.decorate_directory(user, state, item)), thumb=R('folder.png'))) else: state = DATABASE.comic_read_state(user, full_path) title = os.path.splitext(item)[0] oc.add(DirectoryObject( key=Callback(ComicMenu, archive_path=full_path, title=title, user=user), title=unicode(utils.decorate_title(full_path, user, state, title)), thumb=utils.thumb_transcode(Callback(get_cover, archive_path=full_path)))) if offset + page_size < len(dir_list): oc.add(NextPageObject(key=Callback(BrowseDir, cur_dir=cur_dir, page_size=page_size, offset=offset + page_size, user=user))) t2 = time.time() Log.Info('Directory loaded in {:.0f} ms. {}'.format((t2 - t1) * 1000.0, cur_dir)) return oc
def ComicMenu(archive_path, title, user=None): """The 'main menu' for a comic. this allows for different functions to be added.""" oc = ObjectContainer(title2=unicode(os.path.basename(archive_path)), no_cache=True) state = DATABASE.comic_read_state(user, archive_path) # Full comic oc.add(PhotoAlbumObject( key=Callback(Comic, archive_path=archive_path, user=user), rating_key=hashlib.md5(archive_path).hexdigest(), title=unicode(utils.decorate_title(archive_path, user, state, title)), thumb=utils.thumb_transcode(Callback(get_cover, archive_path=archive_path)))) # Resume if state == utils.State.IN_PROGRESS: cur, total = DATABASE.get_page_state(user, archive_path) if cur > 0: oc.add(PhotoAlbumObject(title=unicode(L('resume')), thumb=R('resume.png'), key=Callback(Comic, archive_path=archive_path, user=user, page=cur), rating_key=hashlib.md5('{}{}'.format(archive_path, cur)).hexdigest())) # Read/Unread toggle if state == utils.State.UNREAD or state == utils.State.IN_PROGRESS: oc.add(DirectoryObject(title=unicode(L('mark_read')), thumb=R('mark-read.png'), key=Callback(MarkRead, user=user, archive_path=archive_path))) else: oc.add(DirectoryObject(title=unicode(L('mark_unread')), thumb=R('mark-unread.png'), key=Callback(MarkUnread, user=user, archive_path=archive_path))) return oc
def render_film_info(id, db_name): db = DATABASE(db_name) base_info = db.read(table_name="films", where=" id = '{}'".format(id))[0] more_info = db.read(table_name="film_info", where=" film_id = '{}'".format(id))[0] MESSAGE = "" MESSAGE+=base_info[1]+"\n" for c,v in loads(more_info[-1]).items(): MESSAGE += "{0} : {1}\n".format(c,v) MESSAGE+=base_info[3] return MESSAGE
def get_image(archive_path, filename, user): """Return the contents of `filename` from within `archive_path`. also do some other stuff.""" archive = archives.get_archive(archive_path) x, total_pages = DATABASE.get_page_state(user, archive_path) m = utils.PAGE_NUM_REGEX.search(filename) cur_page = int(m.group(1)) if m else 0 Log.Info('{}: <{}> ({}/{})'.format(user, os.path.basename(archive_path), cur_page, total_pages)) if cur_page > 0: DATABASE.set_page_state(user, archive_path, cur_page) return utils.data_object(archive, filename)
def MainMenu(): DATABASE.ensure_keys() archives.init_rar(Prefs['unrar']) archives.init_sz(Prefs['seven_zip']) user = DATABASE.get_user(Request.Headers.get('X-Plex-Token', 'default')) Log.Info('USER: {}'.format(user)) oc = ObjectContainer(title2=unicode(user), no_cache=True) if bool(Prefs['update']): Updater(PREFIX + '/updater', oc) for x in BrowseDir(Prefs['cb_path'], page_size=int(Prefs['page_size']), user=user).objects: oc.add(x) return oc
def add_user(db_name, table_name="users", message=None): if not table_name and not message: raise AttributeError( "send too function table_name and bot message instance") db = DATABASE(db_name) read = db.read(table_name=table_name) try: new_id = read[-1][0] + 1 except: new_id = 0 uids = get_all_subscribers_id(db_name, table_name) data = (new_id, message.chat.id, message.from_user.username, message.from_user.first_name, message.from_user.last_name) if not str(data[1]) in uids: db.insert(table_name=table_name, args=data)
def BrowseDir(cur_dir, page_size=20, offset=0, user=None): """Browse directories, with paging. Show directories and compatible archives.""" t1 = time.time() oc = ObjectContainer(title2=unicode(os.path.basename(cur_dir)), no_cache=True) try: dir_list = utils.filtered_listdir(cur_dir) page = dir_list[offset:offset + page_size] except Exception as e: Log.Error('BrowseDir: failed to get directory listing.') Log.Error('BrowseDir: {}, cur_dir={}, page_size={}, offset={}, user={}'.format( e, cur_dir, page_size, offset, user)) return error_message('bad path', 'bad path') # Read/Unread toggle if os.path.abspath(cur_dir) != os.path.abspath(Prefs['cb_path']): oc.add(DirectoryObject(title=unicode(L('mark_all_read')), thumb=R('mark-read.png'), key=Callback(Confirmation, f='MarkReadDir', action=L('mark_all_read'), user=user, path=cur_dir))) oc.add(DirectoryObject(title=unicode(L('mark_all_unread')), thumb=R('mark-unread.png'), key=Callback(Confirmation, f='MarkUnreadDir', action=L('mark_all_unread'), user=user, path=cur_dir))) for item, is_dir in page: full_path = os.path.join(cur_dir, item) if is_dir: state = DATABASE.dir_read_state(user, full_path) oc.add(DirectoryObject( key=Callback(BrowseDir, cur_dir=full_path, page_size=page_size, user=user), title=unicode(utils.decorate_directory(user, state, item)), thumb=R('folder.png'))) else: state = DATABASE.comic_read_state(user, full_path) title = os.path.splitext(item)[0] oc.add(DirectoryObject( key=Callback(ComicMenu, archive_path=full_path, title=title, user=user), title=unicode(utils.decorate_title(full_path, user, state, title)), thumb=utils.thumb_transcode(Callback(get_cover, archive_path=full_path)))) if offset + page_size < len(dir_list): oc.add(NextPageObject(key=Callback(BrowseDir, cur_dir=cur_dir, page_size=page_size, offset=offset + page_size, user=user))) t2 = time.time() Log.Info('Directory loaded in {:.0f} ms. {}'.format((t2 - t1) * 1000.0, cur_dir)) return oc
def Users(): oc = ObjectContainer(no_cache=True) for username in DATABASE.usernames(): oc.add(DirectoryObject(key=Callback(SwitchUser, new_username=username), title='Switch to: {}'.format(username), thumb=R('icon-default.png'))) oc.add(DirectoryObject(key=Callback(ClearUsers), title='Clear username cache.', thumb=R('icon-default.png'))) oc.add(DirectoryObject(key=Callback(RefreshUser), title='Refresh User.', thumb=R('icon-default.png'))) return oc
def delete_subscriber(chat, db_name, table_name="users"): if not chat: raise AttributeError("params id and chat is none") db = DATABASE(db_name) db.delete(table_name=table_name, where=" chatid = {}".format(chat)) db.close() return True
def decorate_title(archive, user, state, title): if state == State.UNREAD: indicator = Prefs['unread_symbol'] elif state == State.IN_PROGRESS: cur, total = DATABASE.get_page_state(user, archive) if cur <= 0 or total <= 0: indicator = Prefs['in_progress_symbol'] else: indicator = '{} [{}/{}]'.format(Prefs['in_progress_symbol'], cur, total) elif state == State.READ: indicator = Prefs['read_symbol'] else: return title return '{} {}'.format('' if indicator is None else indicator.strip(), title)
def MainMenu(): DATABASE.ensure_keys() Log.Debug('test_token: {}'.format(test_token(Request.Headers.get('X-Plex-Token')))) archives.init_rar(Prefs['unrar']) archives.init_sz(Prefs['seven_zip']) user = DATABASE.get_user(Request.Headers.get('X-Plex-Token', 'default')) Log.Info('USER: {}'.format(user)) oc = ObjectContainer(title2=unicode(user), no_cache=True) if bool(Prefs['update']): Updater(PREFIX + '/updater', oc) oc.add(DirectoryObject(key=Callback(Users), title='Hello {}. Switch User?'.format(user), thumb=R('icon-default.png'))) browse_dir = BrowseDir(Prefs['cb_path'], page_size=int(Prefs['page_size']), user=user) if not hasattr(browse_dir, 'objects'): return browse_dir for x in browse_dir.objects: oc.add(x) return oc
def load_films(db_name, table_name="films", only_load=False, count=10, find=None, find_perc=None, random=False, category=None): try: db = DATABASE(db_name) read = db.read(table_name=table_name) try: len_db = read[-1][0] except: len_db = 0 have_films = [(x[1], x[0]) for x in read] cats = split_by_category(read) if random: if category: return sample(cats[category], count) else: return sample(have_films, count) if only_load: if len_db > (only_load + count) and len_db > only_load: return have_films[only_load:only_load + count] elif len_db > only_load: return have_films[only_load:None] if find: return db.read(table_name=table_name, where="name LIKE '%{}%'".format(find)) if find_perc: finded = [] for film in read: if ratio(find_perc[0], film[1]) > int(find_perc[1]): if not category: finded.append(film) continue elif category == film[-1]: finded.append(film) return finded db.close() return have_films[0:count] except Exception as e: log(str(e), type="ERROR")
def run(self): while True: start_time = time.time() try: DATABASE.users_registration_cleanup( start_time - (CONFIG['data']['users']['registration_timeout_days'] * TIME_DAY) ) DATABASE.watchlist_cleanup( start_time - (CONFIG['data']['users']['watchlist_abandoned_days'] * TIME_DAY) ) DATABASE.items_prices_cleanup( start_time - (CONFIG['data']['prices']['retention_days'] * TIME_DAY) ) except Exception as e: _logger.error("Unable to clean up database: {error}".format( error=str(e), )) sleep_time = TIME_DAY + random.randint(TIME_HOUR * -3, TIME_HOUR * 3) _logger.info("Cycle completed in {time:.2f}s; sleeping for {sleep_time:.2f}s".format( time=(time.time() - start_time), sleep_time=sleep_time, )) time.sleep(sleep_time)
import threading from time import gmtime, strftime import os import atexit import webbrowser import pprint import eventlet import sys from db import DATABASE if sys.version_info[0] < 3: raise Exception("Must be using Python 3") pp = pprint.PrettyPrinter(indent=1) DB = DATABASE('DATABASE_FILE') APP = Flask(__name__) SOCKETIO = SocketIO(APP) @APP.route('/') def index(): js_var = '/static/js/index.js' + '?' + strftime("%Y%m%d%H%M%S", gmtime()) return render_template("index.html", js_var=js_var) @APP.route('/save_and_quit') def shut_off_save(): SOCKETIO.emit('disconnect_it', broadcast=True, namespace='/socketio') close_server(1) return "CLOSED"
def run(self): while True: start_time = time.time() _logger.info("Preparing to query servers for price snapshots...") server_count = 0 try: for (site_id, site_server, site_name, site_url, site_passphrase, _) in DATABASE.aggregate_list_sites(): server_count += 1 _logger.info("Querying {url} ({server})...".format( url=site_url, server=site_server, )) sub_start_time = time.time() try: data = urllib.urlencode({ 'snapshot_passphrase': site_passphrase, 'include_null': False, 'max_age': int(sub_start_time - CONFIG['aggregation']['snapshots'] ['frequency_hours'] * TIME_HOUR), #Good enough }) data = urllib2.urlopen( site_url + 'items/price-snapshot', data).read() data_len = len(data) data = json.loads(data) data_count = len(data) DATABASE.aggregate_write_prices(site_id, data) except Exception as e: _logger.error( "Unable to retrieve data from {url} ({server}): {error}" .format( url=site_url, server=site_server, error=str(e), )) else: _logger.info( "Retrieved {count} records ({size:.1f}kB) in {time:.2f}s" .format( count=data_count, size=(data_len / 1024.0), time=(time.time() - sub_start_time), )) except Exception as e: _logger.error("Unable to retrieve site-list: {error}".format( error=str(e), )) try: _logger.info("Pruning old snapshot data...") DATABASE.aggregate_prune_prices(start_time - ( CONFIG['aggregation']['snapshots']['retention_days'] * TIME_DAY)) except Exception as e: _logger.error( "Unable to prune old snapshot data: {error}".format( error=str(e), )) execution_time = time.time() - start_time sleep_time = max( 0, (CONFIG['aggregation']['snapshots']['frequency_hours'] * TIME_HOUR) - execution_time) _logger.info( "Queried {count} servers and cleaned up database in {time:.2f}s; sleeping for {sleep:.2f}s" .format( count=server_count, time=execution_time, sleep=sleep_time, )) time.sleep(sleep_time)
def MarkUnread(user, archive_path): Log.Info('Mark unread. a={}'.format(archive_path)) DATABASE.mark_unread(user, archive_path) return error_message('marked', 'marked')
def MarkReadDir(user, path): Log.Info('Mark read. {} a={}'.format(user, path)) DATABASE.mark_read_dir(user, path) return error_message('marked', 'marked')
def ClearUsers(): DATABASE.clear_usernames() return error_message('cleared cache', 'cleared cache')
def update_DB(db_name, categories, table_name="films", fah=EMPTY_FUNCTION, fhh=EMPTY_FUNCTION): try: db = DATABASE(db_name) tm = d.datetime.now() upd = db.read(table_name="updates") read = db.read(table_name=table_name) try: start_index = read[-1][0]+1 except: start_index = 0 try: film_info_index = db.read(table_name="film_info")[-1][0]+1 except: film_info_index = 0 try: id = upd[-1][0] except: id = 0 try: lmfi = upd[-1][1] except: lmfi = int(str(tm.year)+str(tm.month)+str(tm.day)+str(tm.hour)+str(tm.minute)) if lmfi > int(str(tm.year)+str(tm.month)+str(tm.day+UPDATE_TIME)+str(tm.hour)+str(tm.minute)): return "db is up to date" have_films = [x[1] for x in read] cat_keys = list(categories.keys()) urls = forep("https://armfilm.co/hy/CTG/page/ID/", repl="CTG", count=cat_keys) for i, url in enumerate(urls): films = load(url=url, repl="ID", count=categories[cat_keys[i]]) for film in films: if not film['name'] in have_films: args = [ start_index, film['name'], film['image'], film['link'], film['categori'] ] film_info = ( film_info_index, start_index, dumps(load_film_info(film['link'])) ) db.insert(table_name="film_info", args=film_info) db.insert(table_name=table_name, args=args) start_index+=1 film_info_index+=1 have_films.append(film["name"]) fah(film) else: fhh(film) db.insert("updates", args=(id+1, int(str(tm.year)+str(tm.month)+str(tm.day)+str(tm.hour)+str(tm.minute)))) db.close() except Exception as e: log(str(e), type="ERROR")
def get_all_subscribers_id(db_name, table_name="users"): db = DATABASE(db_name) users = db.read(table_name=table_name) db.close() return tuple([user[1] for user in users])
def MarkRead(user, archive_path): Log.Info('Mark read. {} a={}'.format(user, archive_path)) DATABASE.mark_read(user, archive_path) return error_message('marked', 'marked')
def DbClean(): return utils.JSONResponse(JSON.StringFromObject(DATABASE.clean_states()))
def SwitchUser(new_username): DATABASE.switch_user(Request.Headers.get('X-Plex-Token', 'default'), new_username) return error_message('changed user', 'changed user')
def user_exist(db_name, table_name="users", chat_id=None): db = DATABASE(db_name) user = db.read(table_name=table_name, where=" chat_id = '{}'".format(chat_id))[0] db.close() return (len(user) > 0)
def get_all_subscribers(db_name, table_name="users"): db = DATABASE(db_name) users = db.read(table_name=table_name) db.close() return users
def Db(): return utils.JSONResponse(DATABASE.dumps())
def MarkUnreadDir(user, path): Log.Info('Mark unread. a={}'.format(path)) DATABASE.mark_unread_dir(user, path) return error_message('marked', 'marked')
def _p_load_(_name_, _age_): server_is_running = False new_server_is_running = True # This is where extra information about server port will be stored _additional_ = [] port_info_upd = {} # Asking the user to start either a default server or a server of there own time.sleep(1) server_port = input(Fore.GREEN + '\nPort(default 3000): ' + Fore.GREEN) time.sleep(2.4) # Setting default port to 3000 if server_port == '': server_port = '3000' # Storing server_port into a dictionary port = {'Server_Port': server_port} _con_() _impo_(server_is_running, new_server_is_running) # EXPLINATION OF LIVE OR NOT LIVE SERVER print(Style.RESET_ALL + Fore.WHITE + '\nLive Port: A live port being live means that there is a 80% chance that someone could link there root to it and use it for any need they want. A Not Live server port is a secured port that NO ONE can get to or use but you at that given time \n\n NOTE: Your user info AND port info is stored in a file after the project is done executing, just heads up in case you might want to see what the program knows about you.') # Setting the server to Live or Not Live server_port_is_live_ = input(Fore.GREEN + '\nDo You Want Port ' + port['Server_Port'] + ' To Be Live (yes by default)[Y/n]: ') time.sleep(2) # Storing server_port_is_live_ into a dictionary is_port_live = {'Server_Port_Is_Live': server_port_is_live_} # Checking if the server is live if server_port_is_live_ == 'y' or server_port_is_live_ == 'Y': is_port_live['Server_Port_Is_Live'] = 'Live' print('Port ' + port['Server_Port'] + ' is ' + is_port_live['Server_Port_Is_Live']) print('\nPort Is Marked As Public, No Worries Though!') elif server_port_is_live_ == 'n' or server_port_is_live_ == 'N': is_port_live['Server_Port_Is_Live'] = 'Not Live' print('Port ' + port['Server_Port'] + ' is ' + is_port_live['Server_Port_Is_Live']) time.sleep(3) print('\nPort Marked As Secured! All Set') elif server_port_is_live_ == '': is_port_live['Server_Port_Is_Live'] = 'Live by default' print('Port ' + port['Server_Port'] + ' is ' + is_port_live['Server_Port_Is_Live']) print('\nPort Is Marked As Public(by default), No Worries Though!') else: error() server_port_is_live = input(Fore.GREEN + '\nDo You Want Port ' + port['Server_Port'] + ' To Be Live (yes by default)[Y/n]: ') if server_port_is_live == 'Y' or server_port_is_live == 'n': print(Fore.RED + 'Cannot follow through with change..sorry') else: print(Fore.RED + 'Cannot follow through with change..sorry') time.sleep(2) port_l(port['Server_Port'], is_port_live['Server_Port_Is_Live']) port_data = [] port_info = {'Is_New': False, 'Dict_Name': 'Port_Data_Info', 'Type': 'basic', 'Port_ID': server_port, 'Is_Live': is_port_live['Server_Port_Is_Live'], 'Port_Security': True} # Set true by default # Giving the port more choices print(Fore.GREEN + Style.BRIGHT + '\nWe want to know more about what you want for your server port ' + port['Server_Port'] + '\n') def _ch_(): print('1) Keep Port At Default Of Everything') print('2) Let Server Port Know Your Name') print('3) Let Server Port Know Your Name AND Age') _ch_() port_choice = input(Fore.GREEN + '> ') # We Add To The Dictionary port_info_upd On What The User Had Entered For port_choice if port_choice == '1': print('No Problem, we will keep your port at default!') port_info_upd.update({'Type': 'default_generated', 'Name': None, 'Age': None}) _additional_.append(def_pkg()) elif port_choice == '2': print('We will update your port data shortly!') port_info_upd.update({'Type': 'Extra_Port_Information','Name': _name_}) _additional_.append(port_info_upd) _additional_.append(pkg_1()) elif port_choice == '3': print('We will update your port data shortly!') port_info_upd.update({'Type': 'Extra_Port_Info', 'Name': _name_, 'Age': _age_}) _additional_.append(port_info_upd) _additional_.append(pkg_2()) else: print(Fore.RED + 'THERE WAS A ERROR' + Fore.GREEN) # Checking whether or not Port_Security is true or False if port_info['Is_Live'] == 'Live': port_info['Port_Security'] = True if port_info['Port_Security']: port_info['Port_Security'] = 'Your Server Port might be used by others' elif port_info['Is_Live'] == 'Not Live': port_info['Port_Security'] = False if port_info['Port_Security'] == False: port_info['Port_Security'] = 'Your Server Port is secured' elif port_info['Is_Live'] == 'Live by default': port_info['Port_Security'] = True if port_info['Port_Security']: port_info['Port_Security'] = 'Server Port is being shared with others By Default' # If for some reason the code doesn't execute correctly, this else statement will make the Data_Saved value UNKNOWN else: port_info['Data_Saved'] = 'UNKNOWN' # PRINTING PORT INFORMATION port_data.append(port_info) # Here we replace the value None that _additional_ gives us by default and tell the user why there is no additional information about there port if _additional_ == [None]: _additional_.append(['Information Could Not Be Loaded']) _additional_.remove(None) # If port_info_upd['Name'] has a value of None we change it to a string if port_info_upd['Name'] == [None] or port_info_upd['Name'] == '': port_info_upd['Name'] = 'User did not give port a name' # This will be stored in a file that the user can access after the project is done running _server_(port_data, port_info_upd['Name'], _additional_) _data_info_(port['Server_Port'], port_info['Is_New'], port_info_upd['Name']) DATABASE(port_info['Port_ID'], port_data) for item in port_data: print(Fore.YELLOW + Style.NORMAL + '\n' + '--' * 10) print('Data stored about your server port\n') print(item) print('--' * 10)
def RefreshUser(): DATABASE.get_user(Request.Headers.get('X-Plex-Token', 'default'), force=True) return error_message('refreshed user', 'refreshed user')