def create_tags():
c = s['cursor']
tags = s['tags']
res = c.execute(q.get_tags)
tags_in_db = []
for each in res:
tags_in_db.append(each[0])
non_existant_tags = []
existant_tags = []
tags = [a[1:] for a in tags]
for each in tags:
if each not in tags_in_db:
print(f'#{each} does not exist.')
non_existant_tags.append(each)
continue
existant_tags.append(each)
d(print, f'Non existant tags: {non_existant_tags}')
d(print, f'Existant tags: {existant_tags}')
create_tags_script = ''
for each in non_existant_tags:
# FIXME: extract this y/n to a function.
print(f'> #{each} does not exist. Create?')
if(askPositive()):
# FIXME: move this to queries.py for safety :)
create_tags_script = f'{create_tags_script}\nINSERT INTO tags(name,questions_count) VALUES (\'{each}\',1);'
c.executescript(create_tags_script)
s['conn'].commit()
def get_countdown_days():
if 'eta_date' not in s:
return None
end_date = None
try:
t = str(s['eta_date'])
t = [int(a) for a in t.split('-')]
end_date = date(t[2], t[1], t[0])
except:
d(print, f'Invalid date for countdown.')
return end_date
start_date = date.today()
days = (end_date - start_date).days
return days
def crawl_metadata():
c = s['cursor']
c.execute(q.get_unscraped_question_ids)
res = c.fetchall()
stmt = ""
for each in res:
each = int(each[0])
data = get_metadata(f'https://gateoverflow.in/{each}')
d(print, f'{data}')
d(print, f'questionid: {each}')
c.execute(q.update_crawl_attempts, [each])
if data == None:
continue
c.execute(q.insert_into_metadata,
(each, data['title'], data['desc'], data['image_url']))
c.execute(q.update_metadata_scraped_questions, [each])
c.execute(q.delete_invalid_questions, [s['crawl_attempts_limit']])
def get_default_config():
f = None
try:
f = open(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
"default_config.toml"), 'r')
except:
d(
print,
f"Error: Cannot open default_config.toml, something's wrong with packaging."
)
if f == None:
a.abort_program()
res = ''
for line in f.readlines():
res = f'{res}{line}'
f.close()
return res
def open_link(link):
# TODO: Figure out a way to suppress terminal output of browser
# google-chrome output is shown on the terminal
d(print, f'opening {link} in browser')
try:
webbrowser.get().open(link)
# subprocess.run(shlex.split(
# f'xdg-open {link}'), stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL)
except:
# we're on termux or android and there's no native browser available
# there's an alternative to this
# ask user to set $BROWSER='termux-open-url' / or do it during install?
try:
subprocess.run(shlex.split(f'termux-open-url {link}'),
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
except:
print("No browser found. Sorry.")
def get_metadata(link):
print(f'crawling meta information for site: {link}')
res = None
try:
r = requests.get(
f'{constants.metadata_api_base}/api/metadata?url={link}')
data = json.loads(r.text)
data = data['data']
res = {}
res['title'] = data['title']
res['desc'] = data['description']
res['image_url'] = data['image']
except Exception as e:
print(
'Maybe internet is down, or question_id is invalid, or other Error, Skipping!'
)
d(print, f'Error: {e}')
res = None
return res
def readable_date(date):
delta = relativedelta(datetime.now(), datetime.fromtimestamp(int(date)))
res = ''
d(pprint, delta)
order = ['years', 'months', 'days', 'hours', 'minutes', 'seconds']
delta = [
delta.years, delta.months, delta.days, delta.hours, delta.minutes,
delta.seconds
]
i = 0
while i < len(delta):
if delta[i] == 0:
i += 1
else:
break
if (i >= len(delta)):
res = "Just Now"
else:
res = f'{delta[i]} {order[i]} ago'
return res
def latest_version_check():
print('Checking for latest update...')
res = None
try:
res = requests.get('https://pypi.org/project/gateoverflow')
if not res.ok:
d('t', "Request to pypi website failed, result of GET is not OK")
print("Request to server failed.")
return
res = str(res.text)
except:
print("Couldn't connect to the Internet.")
return
p = re.compile("gateoverflow \d\.\d\.\d")
matches = [str(each) for each in p.findall(res)]
if len(matches) < 1:
return
matches = [matches[0][len('gateoverflow '):]]
matches.append(__version__)
d(print, f'Matches are: {matches}')
if matches[0] != matches[1]:
one = [int(each) for each in matches[0].split('.')]
two = [int(each) for each in matches[1].split('.')]
print(f'Latest release: {matches[0]}')
for i in range(len(one)):
if (one[i] == two[i]):
continue
if (two[i] > one[i]):
print('You are ahead of the lastest stable release.\nNice!')
return
print(
f'You are not using the latest release. To upgrade to a latest release, run '
)
print(
prettify_table(
[['python -m pip install gateoverflow --upgrade']], []))
break
else:
print('You are already at a latest release.')
pass
def add_questions_to_tags():
print(f'Adding {s["questions_list"]} to {s["tags"]}...')
tags, questions = s['tags'], s['questions_list']
c = s['cursor']
res = c.execute(q.get_tags)
tags_in_db = []
for each in res:
tags_in_db.append(each[0])
non_existant_tags = []
existant_tags = []
tags = [a[1:] for a in tags]
for each in tags:
if each not in tags_in_db:
print(f'#{each} does not exist.')
non_existant_tags.append(each)
continue
existant_tags.append(each)
d(print, f'Non existant tags: {non_existant_tags}')
d(print, f'Existant tags: {existant_tags}')
create_tags_script = ''
for each in non_existant_tags:
# FIXME: extract this y/n to a function.
ans = input(
f'Tag #{each} does not exist, do you want to create it?(y/n)')
ans = ans.lower()
if(ans == 'y' or ans == 'yes'):
# FIXME: move this to queries.py for safety :)
create_tags_script = f'{create_tags_script}\nINSERT INTO tags(name,questions_count) VALUES (\'{each}\',1);'
c.executescript(create_tags_script)
s['conn'].commit()
tq_insert_script = ''
for q_id in questions:
for tag in tags:
tq_insert_script = f'{tq_insert_script}\nINSERT OR REPLACE INTO tq_relation(question_id,tag_id) VALUES ({q_id},(SELECT id FROM tags WHERE name=\'{tag}\'));'
c.executescript(tq_insert_script)
for tag in tags:
c.execute(q.update_questions_count, [each])
s['conn'].commit()
def parse_cmd(in_str):
error = False
nums = []
tags = []
action = None
if (in_str == 'tags' or in_str == '#'):
action = constants.parser_actions.LIST_TAGS
return (error, nums, tags, action)
line = [a.strip() for a in in_str.split(',')]
d(print, line)
for each in line:
try:
if (len(each.split(' ')) > 1):
[nums.append(int(a)) for a in each.split(' ')]
else:
nums.append(int(each))
except:
# fixes trailing commas and inputs like `,,,` `#,#,#`
if (len(each) < 1 or each == '#'):
continue
# if multiple #'s are there, then it should be invalid
if (each[0] == '#' and each.count('#') == 1):
if (len(each.split(' ')) > 1):
[tags.append(a) for a in each.split(' ')]
else:
tags.append(each)
else:
error = True
break
if error:
return (error, nums, tags, action)
if (len(nums) == 0):
action = constants.parser_actions.LIST_QUESTIONS_OF_TAGS
if (len(tags) == 0):
action = constants.parser_actions.OPEN_QUESTIONS
if (len(tags) == 0 and len(nums) == 0):
action = constants.parser_actions.DO_NOTHING
d(print, f'error: {error}')
d(print, f'nums: {nums}')
d(print, f'tags: {tags}')
if action == None:
action = constants.parser_actions.ADD_QUESTIONS_TO_TAGS
return (error, nums, tags, action)
def list_questions_of_tags():
print(f'Only tag(s) found, listing questions of specified tags...')
d('t', 'listing questions for tags')
tags, questions = s['tags'], s['questions_list']
c = s['cursor']
res = c.execute(q.get_tags)
tags_in_db = []
for each in res:
tags_in_db.append(each[0])
non_existant_tags = []
existant_tags = []
tags = [a[1:] for a in tags]
for each in tags:
if each not in tags_in_db:
print(f'#{each} does not exist.')
non_existant_tags.append(each)
continue
existant_tags.append(each)
d(print, f'Existant tags: {existant_tags}')
if len(existant_tags) != 0:
# TODO: find a way to move this query to queries.py
bruh = (",?"*len(existant_tags)).split(',')[1:]
bruh = ','.join(bruh)
query = f'select ogq as question_id, tagname, title, visited_count desc from (select question_id as ogq, (select tags.name from tags where id=tag_id)as tagname from tq_relation where tag_id in (select id from tags where name in ({bruh}))) left join metadata on ogq = metadata.question_id left join recents on ogq = recents.question_id;'
d(print, query)
res = c.execute(query, existant_tags)
if res == None:
print('Nothing to show.')
return
headers = ['QuestionID', 'Tag', 'Title',
'Description', 'Visited Count']
k = s['column_width']
data = []
for row in res:
row = [str(each) for each in row]
data.append(row)
print(prettify_table(data, headers))
else:
print("Nothing to show.")
def startup_routine():
Path = pathlib.Path
# check if the appData directory exists, create it if it doesn't exist, and should be writable
home_dir = Path.home()
project_home = Path.joinpath(
home_dir, f'.{constants.project_name}')
# requires python >= 3.5 for now
# TODO: make it work for lower python versions, and windows
# TODO: check if project_home is writable
config_file = Path.joinpath(project_home, f'config.toml')
db_file = Path.joinpath(project_home, f'{s["database_name"]}')
# home folder may not exists, in that case it's a fresh start
if not Path.exists(project_home):
# it is a fresh start
print("It appears you are running this program for the first time, so I need to configure...\n")
print("Creating project home directory...", end="")
Path.mkdir(project_home, exist_ok=True, parents=True)
d('t', 'created path')
print(f"{constants.colors.GREEN}DONE.{constants.colors.END}")
# config file may not exist
if not Path.exists(config_file):
print('Creating default config...', end="")
config_file.write_text(get_default_config(), encoding="utf-8")
d('t', 'wrote sample config.')
print(
f"{constants.colors.GREEN}DONE.{constants.colors.END} ({str(config_file)}).\n")
# load the config into state
# parse the config file if exists, fallback to default ones.
try:
user_config = toml.load(str(config_file))
d("t", "successfully loaded config file into an object")
except Exception as e:
d(print, f'{e}')
print(f"{constants.colors.FAIL}user config is invalid.{constants.colors.END}")
a.abort_program()
# load the relevant config into state
# TODO: hacky solution, improve this parser later
for key in user_config:
if key not in s.keys():
continue
s[key] = user_config[key]
d(print, f'Config: key: {key}, value: {user_config[key]}')
d('t', 'user config loaded into state')
db_file = Path.joinpath(project_home, f'{s["database_name"]}')
if not Path.exists(db_file):
print("No database found.")
print()
print("> Should I create a database?")
if(askPositive()):
# Create a database
s['db_path'] = str(db_file)
else:
print(
"Fine, you should copy your *.db file to ${project_home}")
a.abort_program()
s['project_home'] = project_home
if Path.exists(db_file):
s["db_path"] = str(db_file)
def act(cmd):
# do something based on action
action = cmd.lower().split(' ')[0]
switcher = s["switcher"]
err, questions, tags, parser_action = parse_cmd(cmd)
if action == 'create':
cmd = cmd[6:]
err, questions, tags, parser_action = parse_cmd(cmd)
if parser_action == constants.parser_actions.LIST_QUESTIONS_OF_TAGS or parser_action == constants.parser_actions.LIST_TAGS:
parser_action = constants.parser_actions.CREATE_TAGS
err = False
d('t', 'create tags command')
d(print, f'cmd: {cmd}')
if action == 'ls':
if(len(cmd.split(' ')) > 1):
s["list_string"] = cmd
else:
s["list_string"] = f'ls {s["how_many"]}'
if not err:
s["questions_list"] = questions
s["tags"] = tags
s["parser_action"] = parser_action
action = 'parser'
d(print, f'action: {action}')
if action == 'parser':
d(print, f'err: {err}')
d(print, f'tags: {tags}')
d(print, f'questions: {questions}')
d(print, f'parser_action: {parser_action}')
if(action == 'session-id'):
print(s['session_id'])
if action not in switcher.keys():
action = 'invalid'
f = switcher[action]
f()
def cleanup(con):
d('t', "Performing cleanup operations")
con.cursor().execute(q.end_session, [s['session_id']])
con.commit()
con.close()
def main():
a.clear_screen()
try:
startup_routine()
except Exception as e:
print(f"{constants.colors.FAIL}Startup failed.{constants.colors.END}")
d(print, f'{e}')
a.abort_program()
if s["project_home"] == None:
print(
f"{constants.colors.FAIL}No place to store my things.{constants.colors.END}")
a.abort_program()
# start sqlite connection
if s["db_path"] == None:
print("No database found. Creating a brand new database...")
if(askPositive()):
s["db_path"] = str(pathlib.Path.joinpath(
s["project_home"], f'{s["database_name"]}'))
else:
print(
f'I cannot run without a database, please create on, or copy already existing *.db file to {s["project_home"]}')
a.abort_program()
# print('deleting for ease of development')
# os.system(f'rm {constants.database_name}')
connection = conn(s['db_path'])
s['conn'] = connection
s['cursor'] = connection.cursor()
c = connection.cursor()
c.executescript(q.create_tables)
try:
c.executescript(q.alter_tables)
except:
d(print, "No need to alter tables.")
c.executescript(q.create_triggers)
c.executescript(q.create_default_tags)
c.execute(q.get_user)
res = c.fetchone()
if res == None:
d('t', "res is none for q.get_user")
print("You haven't added your details yet.")
print("\n> Details: username and name (one time only)")
if(askPositive()):
username = input('Enter Username: '******'Enter Name: ')
if(len(username) > 0 and len(name) > 0):
s['user'] = constants.User(username, name)
c.execute(q.create_user, [username, name])
d('t', 'added user info after asking the same to the user')
else:
print(
f"{constants.colors.FAIL}Why invalid details?{constants.colors.END}")
a.abort_program()
else:
d('t', 'user refused to give username and name')
print("Fine, stay anonymous then")
s['user'] = constants.User()
else:
d('t', 'res is not null')
d(print, f'{res}')
res = [str(each) for each in res]
s['user'] = constants.User(res[1], res[0])
# c.executescript(q.insert_dummy_values)
try:
c.execute(q.start_session)
s['session_id'] = c.lastrowid
except:
d('t', "Failed to start a session")
atexit.register(cleanup, connection)
a.clear_screen()
s['switcher'] = a.get_switcher()
if s['DEBUG'] == True:
for row in c.execute(q.get_all):
d(pprint, f'[test]: row is: {row}')
# Display info, and take input
while(not s['stop']):
act(poll())
# c.execute(q.delete_invalid_sessions)
connection.commit()
def uncrawled_metadata_count():
c = s['cursor']
c.execute(q.uncrawled_metadata_count)
res = c.fetchone()
d(print, f'Unscraped Records: {res[0]}')
return int(res[0])
def list_command():
# list_options = set(['r', 'q', 't', 'recent', 'questions', 'tags'])
row_offset = 0
cmd = s['list_string'].split(' ')
c = s['cursor']
d(print, 'lists recents')
d(print, f'probably, you wanted to list: {cmd}')
if len(cmd) > 2:
d(print, 'Invalid argument to ls.')
return
how_many = None
try:
how_many = cmd[1]
except:
how_many = s['how_many'] # default show only 10 records?
# what_to_show = cmd[1]
d(print, f"omg you want this? : recents, {how_many} items")
c.execute(q.get_recent, (how_many, row_offset))
res = c.fetchall()
# title = 'QuestionID'.ljust(12) + 'Visited'.ljust(12) + 'Time'.ljust(12)
# print(title)
headers = ['Question ID', 'Title', 'Description', 'Visits', 'Last Visited']
data = []
k = s['column_width']
for row in res:
row = [str(each) for each in row]
frow = []
for i, each in enumerate(row):
temp = ''
d(print, f'{each} - {i}')
if i == 4:
frow.append(each)
continue
if(each == 'None'):
temp = 'NA'
if(len(each) > k):
d(print, f'length: {len(each)}')
each = each.replace("- GATE Overflow", "")
temp = each
if(len(each) > k):
temp = each[:k-3] + "..."
else:
temp = each
frow.append(temp)
for each in frow:
d(print, f'after len: {len(each)}')
data.append([frow[0], frow[1], frow[2],
frow[3], readable_date(frow[4])])
print(prettify_table(data, headers))
