def create_tags(): c = s['cursor'] tags = s['tags'] res = c.execute(q.get_tags) tags_in_db = [] for each in res: tags_in_db.append(each[0]) non_existant_tags = [] existant_tags = [] tags = [a[1:] for a in tags] for each in tags: if each not in tags_in_db: print(f'#{each} does not exist.') non_existant_tags.append(each) continue existant_tags.append(each) d(print, f'Non existant tags: {non_existant_tags}') d(print, f'Existant tags: {existant_tags}') create_tags_script = '' for each in non_existant_tags: # FIXME: extract this y/n to a function. print(f'> #{each} does not exist. Create?') if(askPositive()): # FIXME: move this to queries.py for safety :) create_tags_script = f'{create_tags_script}\nINSERT INTO tags(name,questions_count) VALUES (\'{each}\',1);' c.executescript(create_tags_script) s['conn'].commit()
def get_countdown_days(): if 'eta_date' not in s: return None end_date = None try: t = str(s['eta_date']) t = [int(a) for a in t.split('-')] end_date = date(t[2], t[1], t[0]) except: d(print, f'Invalid date for countdown.') return end_date start_date = date.today() days = (end_date - start_date).days return days
def crawl_metadata(): c = s['cursor'] c.execute(q.get_unscraped_question_ids) res = c.fetchall() stmt = "" for each in res: each = int(each[0]) data = get_metadata(f'https://gateoverflow.in/{each}') d(print, f'{data}') d(print, f'questionid: {each}') c.execute(q.update_crawl_attempts, [each]) if data == None: continue c.execute(q.insert_into_metadata, (each, data['title'], data['desc'], data['image_url'])) c.execute(q.update_metadata_scraped_questions, [each]) c.execute(q.delete_invalid_questions, [s['crawl_attempts_limit']])
def get_default_config(): f = None try: f = open( os.path.join(os.path.dirname(os.path.abspath(__file__)), "default_config.toml"), 'r') except: d( print, f"Error: Cannot open default_config.toml, something's wrong with packaging." ) if f == None: a.abort_program() res = '' for line in f.readlines(): res = f'{res}{line}' f.close() return res
def open_link(link): # TODO: Figure out a way to suppress terminal output of browser # google-chrome output is shown on the terminal d(print, f'opening {link} in browser') try: webbrowser.get().open(link) # subprocess.run(shlex.split( # f'xdg-open {link}'), stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL) except: # we're on termux or android and there's no native browser available # there's an alternative to this # ask user to set $BROWSER='termux-open-url' / or do it during install? try: subprocess.run(shlex.split(f'termux-open-url {link}'), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) except: print("No browser found. Sorry.")
def get_metadata(link): print(f'crawling meta information for site: {link}') res = None try: r = requests.get( f'{constants.metadata_api_base}/api/metadata?url={link}') data = json.loads(r.text) data = data['data'] res = {} res['title'] = data['title'] res['desc'] = data['description'] res['image_url'] = data['image'] except Exception as e: print( 'Maybe internet is down, or question_id is invalid, or other Error, Skipping!' ) d(print, f'Error: {e}') res = None return res
def readable_date(date): delta = relativedelta(datetime.now(), datetime.fromtimestamp(int(date))) res = '' d(pprint, delta) order = ['years', 'months', 'days', 'hours', 'minutes', 'seconds'] delta = [ delta.years, delta.months, delta.days, delta.hours, delta.minutes, delta.seconds ] i = 0 while i < len(delta): if delta[i] == 0: i += 1 else: break if (i >= len(delta)): res = "Just Now" else: res = f'{delta[i]} {order[i]} ago' return res
def latest_version_check(): print('Checking for latest update...') res = None try: res = requests.get('https://pypi.org/project/gateoverflow') if not res.ok: d('t', "Request to pypi website failed, result of GET is not OK") print("Request to server failed.") return res = str(res.text) except: print("Couldn't connect to the Internet.") return p = re.compile("gateoverflow \d\.\d\.\d") matches = [str(each) for each in p.findall(res)] if len(matches) < 1: return matches = [matches[0][len('gateoverflow '):]] matches.append(__version__) d(print, f'Matches are: {matches}') if matches[0] != matches[1]: one = [int(each) for each in matches[0].split('.')] two = [int(each) for each in matches[1].split('.')] print(f'Latest release: {matches[0]}') for i in range(len(one)): if (one[i] == two[i]): continue if (two[i] > one[i]): print('You are ahead of the lastest stable release.\nNice!') return print( f'You are not using the latest release. To upgrade to a latest release, run ' ) print( prettify_table( [['python -m pip install gateoverflow --upgrade']], [])) break else: print('You are already at a latest release.') pass
def add_questions_to_tags(): print(f'Adding {s["questions_list"]} to {s["tags"]}...') tags, questions = s['tags'], s['questions_list'] c = s['cursor'] res = c.execute(q.get_tags) tags_in_db = [] for each in res: tags_in_db.append(each[0]) non_existant_tags = [] existant_tags = [] tags = [a[1:] for a in tags] for each in tags: if each not in tags_in_db: print(f'#{each} does not exist.') non_existant_tags.append(each) continue existant_tags.append(each) d(print, f'Non existant tags: {non_existant_tags}') d(print, f'Existant tags: {existant_tags}') create_tags_script = '' for each in non_existant_tags: # FIXME: extract this y/n to a function. ans = input( f'Tag #{each} does not exist, do you want to create it?(y/n)') ans = ans.lower() if(ans == 'y' or ans == 'yes'): # FIXME: move this to queries.py for safety :) create_tags_script = f'{create_tags_script}\nINSERT INTO tags(name,questions_count) VALUES (\'{each}\',1);' c.executescript(create_tags_script) s['conn'].commit() tq_insert_script = '' for q_id in questions: for tag in tags: tq_insert_script = f'{tq_insert_script}\nINSERT OR REPLACE INTO tq_relation(question_id,tag_id) VALUES ({q_id},(SELECT id FROM tags WHERE name=\'{tag}\'));' c.executescript(tq_insert_script) for tag in tags: c.execute(q.update_questions_count, [each]) s['conn'].commit()
def parse_cmd(in_str): error = False nums = [] tags = [] action = None if (in_str == 'tags' or in_str == '#'): action = constants.parser_actions.LIST_TAGS return (error, nums, tags, action) line = [a.strip() for a in in_str.split(',')] d(print, line) for each in line: try: if (len(each.split(' ')) > 1): [nums.append(int(a)) for a in each.split(' ')] else: nums.append(int(each)) except: # fixes trailing commas and inputs like `,,,` `#,#,#` if (len(each) < 1 or each == '#'): continue # if multiple #'s are there, then it should be invalid if (each[0] == '#' and each.count('#') == 1): if (len(each.split(' ')) > 1): [tags.append(a) for a in each.split(' ')] else: tags.append(each) else: error = True break if error: return (error, nums, tags, action) if (len(nums) == 0): action = constants.parser_actions.LIST_QUESTIONS_OF_TAGS if (len(tags) == 0): action = constants.parser_actions.OPEN_QUESTIONS if (len(tags) == 0 and len(nums) == 0): action = constants.parser_actions.DO_NOTHING d(print, f'error: {error}') d(print, f'nums: {nums}') d(print, f'tags: {tags}') if action == None: action = constants.parser_actions.ADD_QUESTIONS_TO_TAGS return (error, nums, tags, action)
def list_questions_of_tags(): print(f'Only tag(s) found, listing questions of specified tags...') d('t', 'listing questions for tags') tags, questions = s['tags'], s['questions_list'] c = s['cursor'] res = c.execute(q.get_tags) tags_in_db = [] for each in res: tags_in_db.append(each[0]) non_existant_tags = [] existant_tags = [] tags = [a[1:] for a in tags] for each in tags: if each not in tags_in_db: print(f'#{each} does not exist.') non_existant_tags.append(each) continue existant_tags.append(each) d(print, f'Existant tags: {existant_tags}') if len(existant_tags) != 0: # TODO: find a way to move this query to queries.py bruh = (",?"*len(existant_tags)).split(',')[1:] bruh = ','.join(bruh) query = f'select ogq as question_id, tagname, title, visited_count desc from (select question_id as ogq, (select tags.name from tags where id=tag_id)as tagname from tq_relation where tag_id in (select id from tags where name in ({bruh}))) left join metadata on ogq = metadata.question_id left join recents on ogq = recents.question_id;' d(print, query) res = c.execute(query, existant_tags) if res == None: print('Nothing to show.') return headers = ['QuestionID', 'Tag', 'Title', 'Description', 'Visited Count'] k = s['column_width'] data = [] for row in res: row = [str(each) for each in row] data.append(row) print(prettify_table(data, headers)) else: print("Nothing to show.")
def startup_routine(): Path = pathlib.Path # check if the appData directory exists, create it if it doesn't exist, and should be writable home_dir = Path.home() project_home = Path.joinpath( home_dir, f'.{constants.project_name}') # requires python >= 3.5 for now # TODO: make it work for lower python versions, and windows # TODO: check if project_home is writable config_file = Path.joinpath(project_home, f'config.toml') db_file = Path.joinpath(project_home, f'{s["database_name"]}') # home folder may not exists, in that case it's a fresh start if not Path.exists(project_home): # it is a fresh start print("It appears you are running this program for the first time, so I need to configure...\n") print("Creating project home directory...", end="") Path.mkdir(project_home, exist_ok=True, parents=True) d('t', 'created path') print(f"{constants.colors.GREEN}DONE.{constants.colors.END}") # config file may not exist if not Path.exists(config_file): print('Creating default config...', end="") config_file.write_text(get_default_config(), encoding="utf-8") d('t', 'wrote sample config.') print( f"{constants.colors.GREEN}DONE.{constants.colors.END} ({str(config_file)}).\n") # load the config into state # parse the config file if exists, fallback to default ones. try: user_config = toml.load(str(config_file)) d("t", "successfully loaded config file into an object") except Exception as e: d(print, f'{e}') print(f"{constants.colors.FAIL}user config is invalid.{constants.colors.END}") a.abort_program() # load the relevant config into state # TODO: hacky solution, improve this parser later for key in user_config: if key not in s.keys(): continue s[key] = user_config[key] d(print, f'Config: key: {key}, value: {user_config[key]}') d('t', 'user config loaded into state') db_file = Path.joinpath(project_home, f'{s["database_name"]}') if not Path.exists(db_file): print("No database found.") print() print("> Should I create a database?") if(askPositive()): # Create a database s['db_path'] = str(db_file) else: print( "Fine, you should copy your *.db file to ${project_home}") a.abort_program() s['project_home'] = project_home if Path.exists(db_file): s["db_path"] = str(db_file)
def act(cmd): # do something based on action action = cmd.lower().split(' ')[0] switcher = s["switcher"] err, questions, tags, parser_action = parse_cmd(cmd) if action == 'create': cmd = cmd[6:] err, questions, tags, parser_action = parse_cmd(cmd) if parser_action == constants.parser_actions.LIST_QUESTIONS_OF_TAGS or parser_action == constants.parser_actions.LIST_TAGS: parser_action = constants.parser_actions.CREATE_TAGS err = False d('t', 'create tags command') d(print, f'cmd: {cmd}') if action == 'ls': if(len(cmd.split(' ')) > 1): s["list_string"] = cmd else: s["list_string"] = f'ls {s["how_many"]}' if not err: s["questions_list"] = questions s["tags"] = tags s["parser_action"] = parser_action action = 'parser' d(print, f'action: {action}') if action == 'parser': d(print, f'err: {err}') d(print, f'tags: {tags}') d(print, f'questions: {questions}') d(print, f'parser_action: {parser_action}') if(action == 'session-id'): print(s['session_id']) if action not in switcher.keys(): action = 'invalid' f = switcher[action] f()
def cleanup(con): d('t', "Performing cleanup operations") con.cursor().execute(q.end_session, [s['session_id']]) con.commit() con.close()
def main(): a.clear_screen() try: startup_routine() except Exception as e: print(f"{constants.colors.FAIL}Startup failed.{constants.colors.END}") d(print, f'{e}') a.abort_program() if s["project_home"] == None: print( f"{constants.colors.FAIL}No place to store my things.{constants.colors.END}") a.abort_program() # start sqlite connection if s["db_path"] == None: print("No database found. Creating a brand new database...") if(askPositive()): s["db_path"] = str(pathlib.Path.joinpath( s["project_home"], f'{s["database_name"]}')) else: print( f'I cannot run without a database, please create on, or copy already existing *.db file to {s["project_home"]}') a.abort_program() # print('deleting for ease of development') # os.system(f'rm {constants.database_name}') connection = conn(s['db_path']) s['conn'] = connection s['cursor'] = connection.cursor() c = connection.cursor() c.executescript(q.create_tables) try: c.executescript(q.alter_tables) except: d(print, "No need to alter tables.") c.executescript(q.create_triggers) c.executescript(q.create_default_tags) c.execute(q.get_user) res = c.fetchone() if res == None: d('t', "res is none for q.get_user") print("You haven't added your details yet.") print("\n> Details: username and name (one time only)") if(askPositive()): username = input('Enter Username: '******'Enter Name: ') if(len(username) > 0 and len(name) > 0): s['user'] = constants.User(username, name) c.execute(q.create_user, [username, name]) d('t', 'added user info after asking the same to the user') else: print( f"{constants.colors.FAIL}Why invalid details?{constants.colors.END}") a.abort_program() else: d('t', 'user refused to give username and name') print("Fine, stay anonymous then") s['user'] = constants.User() else: d('t', 'res is not null') d(print, f'{res}') res = [str(each) for each in res] s['user'] = constants.User(res[1], res[0]) # c.executescript(q.insert_dummy_values) try: c.execute(q.start_session) s['session_id'] = c.lastrowid except: d('t', "Failed to start a session") atexit.register(cleanup, connection) a.clear_screen() s['switcher'] = a.get_switcher() if s['DEBUG'] == True: for row in c.execute(q.get_all): d(pprint, f'[test]: row is: {row}') # Display info, and take input while(not s['stop']): act(poll()) # c.execute(q.delete_invalid_sessions) connection.commit()
def uncrawled_metadata_count(): c = s['cursor'] c.execute(q.uncrawled_metadata_count) res = c.fetchone() d(print, f'Unscraped Records: {res[0]}') return int(res[0])
def list_command(): # list_options = set(['r', 'q', 't', 'recent', 'questions', 'tags']) row_offset = 0 cmd = s['list_string'].split(' ') c = s['cursor'] d(print, 'lists recents') d(print, f'probably, you wanted to list: {cmd}') if len(cmd) > 2: d(print, 'Invalid argument to ls.') return how_many = None try: how_many = cmd[1] except: how_many = s['how_many'] # default show only 10 records? # what_to_show = cmd[1] d(print, f"omg you want this? : recents, {how_many} items") c.execute(q.get_recent, (how_many, row_offset)) res = c.fetchall() # title = 'QuestionID'.ljust(12) + 'Visited'.ljust(12) + 'Time'.ljust(12) # print(title) headers = ['Question ID', 'Title', 'Description', 'Visits', 'Last Visited'] data = [] k = s['column_width'] for row in res: row = [str(each) for each in row] frow = [] for i, each in enumerate(row): temp = '' d(print, f'{each} - {i}') if i == 4: frow.append(each) continue if(each == 'None'): temp = 'NA' if(len(each) > k): d(print, f'length: {len(each)}') each = each.replace("- GATE Overflow", "") temp = each if(len(each) > k): temp = each[:k-3] + "..." else: temp = each frow.append(temp) for each in frow: d(print, f'after len: {len(each)}') data.append([frow[0], frow[1], frow[2], frow[3], readable_date(frow[4])]) print(prettify_table(data, headers))