def main():
try:
start_time = time.time()
logger.info("Start running...")
original_URL = get_URL()
if not WikiUtils.is_wiki_page(original_URL):
raise Exception("Incorrect Input.")
wiki_links = WikiUtils.get_wiki_links(original_URL)
page_name = original_URL.split(r'/')[-1]
logger.info(f"Total number of Wikipedia links is the page '{page_name}' - {len(wiki_links)}")
WikiUtils.print_wiki_back_links(original_URL, wiki_links)
logger.info("Done. Execution time %.2f sec" % (time.time()-start_time))
except Exception as e:
logger.exception(e)
def __init__(self, redis_connection=None):
if redis_connection is None:
self.redis_conn = self.connect_redis()
else:
self.redis_conn = redis_connection
self.wu = WikiUtils(self.redis_conn)
self._check_db_version() # will create initial data if missing
class SiteUtils(object):
def __init__(self, redis_connection=None):
if redis_connection is None:
self.redis_conn = self.connect_redis()
else:
self.redis_conn = redis_connection
self.wu = WikiUtils(self.redis_conn)
self._check_db_version() # will create initial data if missing
def connect_redis(self):
# TODO connection pooling
#r = redis.StrictRedis(db=REDIS_DB)
r = redis.Redis(db=REDIS_DB)
assert isinstance(r, redis.Redis)
return r
def _init_wiki(self):
#print("Wiki init **************************************************************")
root_id = self.wu.create_wiki_root_category()
default_text = """***Root article"""
article_slug = 'Index'
article_name = "Makerspace Wiki Index Page"
self.wu.create_wiki_article(root_id, article_slug, article_name, default_text)
default_text = """
Test article about badgers
===
We like __badgers__ because badgers are _awesome_
NB: unit tests look for this text.
"""
article_slug = 'Badgers'
article_name = "Test Page About Badgers"
subcat_id = self.wu.create_wiki_category(root_id, "Wildlife")
self.wu.create_wiki_article(subcat_id, article_slug, article_name, default_text)
def _init_superuser(self):
self.redis_conn.delete('mkrspc_superusers')
dkpw_passwd = u'password1'
dkpw_user = u'dkpw'
self.redis_conn.lpush('mkrspc_superusers', dkpw_user)
self.make_user(dkpw_user, dkpw_passwd)
alice_passwd = u'puppies'
alice_user = u'alice'
self.make_user(alice_user, alice_passwd)
def _check_db_version(self):
key = 'mkrspc_db_version'
r = self.redis_conn
db_version = r.get(key)
if db_version is None:
db_version = "0.1.0"
r.set(key, db_version)
self._init_superuser()
self._init_wiki()
return db_version
def make_user(self, username, password):
hash_object = hashlib.sha256(auth_salt_secret + password)
hex_dig = hash_object.hexdigest()
self.redis_conn.set('User_Pwd_%s' % username, hex_dig)
def user_exists(self, user):
r = self.redis_conn
user_passwd_record = r.get('User_Pwd_%s' % user)
if user_passwd_record is not None:
return True
else:
return False
def change_user_password(self, username, old_passwd, new_password):
if self.compare_password(username, old_passwd):
self.set_user_password(username, new_password)
def set_user_password(self, username, new_password):
hash_object = hashlib.sha256(auth_salt_secret + new_password)
hex_dig = hash_object.hexdigest()
self.redis_conn.set('User_Pwd_%s' % username, hex_dig)
def do_login(self, user, passwd):
r = self.redis_conn
if self.compare_password(user, passwd):
# The password is valid, so create a token.
token = str(uuid.uuid4())
# Store token in Redis. (Valid for 1000 milli-fortnights.)
r.setex(('User_Auth_Cookie_%s' % token), user, 60 * 60 * 24 * 14)
# Return token to be set as a cookie.
return token
else:
return None
def compare_password(self, user, entered_password):
r = self.redis_conn
user_passwd_record = r.get('User_Pwd_%s' % user)
if user_passwd_record is not None:
hash_object = hashlib.sha256(auth_salt_secret + entered_password)
hex_dig = hash_object.hexdigest()
#print(hex_dig)
if hex_dig == user_passwd_record:
return True
else:
return False
def take_backup(self):
outdir = os.path.join(static_files_root, 'backups')
if not os.path.exists(outdir):
os.mkdir(outdir)
dt = datetime.datetime
outfilename = "mkrspc_web_backup_%s.txt" % (dt.now().isoformat())
outpath = os.path.join(outdir, outfilename)
outf = open(outpath, 'wt')
def wr(line):
print(line, file=outf)
r = self.redis_conn
keys = r.keys('*')
wr("====BEGIN====")
wr(len(keys))
for k in keys:
t = r.type(k)
wr('--------------------------------')
wr(len(k))
wr(k)
wr(len(t))
wr(t)
if t == 'list':
list_length = r.llen(k)
wr('%d' % list_length)
list_items = r.lrange(k, 0, list_length)
for li in list_items:
b64_li = base64.b64encode(li)
wr(len(b64_li))
wr(b64_li)
elif t == 'string':
val = r.get(k)
b64_val = base64.b64encode(val)
wr(len(b64_val))
wr(b64_val)
else:
raise ValueError("Unhandled Redis value type: %s" % t)
wr("====END====")
outf.close()
return outfilename
def check_auth_cookie(self, req):
assert isinstance(req, Request)
req_token = req.cookies.ncms_auth
if req_token is None:
return None
else:
r = self.redis_conn
auth_token_name = r.get('User_Auth_Cookie_%s' % req_token)
if auth_token_name is None:
#print(" - auth cookie not found in Redis")
return None
else:
#print(" - auth cookie found OK")
superusers = r.lrange('mkrspc_superusers', 0, 9)
#print(superusers)
if auth_token_name in superusers:
return (auth_token_name, True)
else:
return (auth_token_name, False)
def remove_auth_cookie(self, req):
assert isinstance(req, Request)
req_token = req.cookies.ncms_auth
if req_token is None:
return None
else:
r = self.redis_conn
r.delete('User_Auth_Cookie_%s' % req_token)
response.set_cookie('ncms_auth', '')
def _menu_entry(self, label, url, icon, selection_id, selected_entry, nav_style='default'):
if selection_id == selected_entry:
selection_class = "selected"
else:
selection_class = "sibling"
if nav_style == 'default':
html = """
<li class="%s">
<a href="%s"><i class="fa %s fa-fw"></i> %s</a>
</li>
""" % (selection_class, url, icon, label)
elif nav_style == 'wiki':
html = """
<li class="%s">
<a href="%s"><i class="fa %s fa-fw"></i> %s</a>
</li>
""" % (selection_class, url, icon, label)
else:
raise Exception("Unknown nav_style [%s]" % nav_style)
return html
def menu(self, selected, user_info, nav_style='default'):
_menu_entry = self._menu_entry # local ref to function
menu_template = """
<ul class="%(nav_style)s icons">
%(home)s
%(about)s
%(wiki)s
%(profile)s
%(admin)s
</ul>
"""
replacements = dict()
replacements['nav_style'] = 'menu_%s' % nav_style
if user_info:
assert isinstance(user_info, tuple)
assert len(user_info) == 2
assert isinstance(user_info[0], basestring)
replacements['profile'] = _menu_entry('Profile', '/user_profile', 'fa-user', 'sel_profile', selected, nav_style)
else:
replacements['profile'] = ''
if user_info and user_info[1] is True:
replacements['admin'] = _menu_entry('Admin', '/admin', 'fa-cog', 'sel_admin', selected, nav_style)
else:
replacements['admin'] = ''
replacements['home'] = _menu_entry('Home', '/', 'fa-home', 'sel_home', selected, nav_style)
replacements['about'] = _menu_entry('About', '/about', 'fa-question-circle', 'sel_about', selected, nav_style)
replacements['wiki'] = _menu_entry('Wiki', '/wiki/Index', 'fa-pencil-square-o', 'sel_wiki', selected, nav_style)
return menu_template % replacements
def user_greeting(self, user_info):
if user_info is not None:
assert isinstance(user_info, tuple)
assert len(user_info) == 2
links = '<a href="/user_profile">%s</a> - <a href="/logout">log out</a>' % (user_info[0])
if user_info[1] == True:
return "Hail " + links
else:
return 'Hi ' + links
else:
return None
def wiki_index(self):
root = self.wu.wiki_root_category()
root_cats = self.wu.wiki_categories_in_category(root)
cats = []
for cat_key in root_cats:
cat = self.wu.name_for_wiki_cat_id(cat_key)
article_keys = self.wu.wiki_articles_in_category(cat_key)
articles = []
subcat_keys = self.wu.wiki_categories_in_category(cat_key)
subcats = subcat_keys
for art_key in article_keys:
slug = self.wu.wiki_article_slug(art_key)
title = self.wu.wiki_article_title(art_key)
articles.append((slug, title))
cats.append((cat_key, cat, articles, subcats))
return cats
def wiki_root_categories(self):
r = self.redis_conn
cat_count = r.llen("wiki_cats")
cats = r.lrange("wiki_cats", 0, cat_count)
return cats
#"wiki_cat_%s" % str(uuid.uuid4())
#r.set(cat_key, cat_name)
#r.lpush("wiki_cats", cat_key)
def build_site_message_model(self, site_message):
message_model = object()
message_model.message_intent = "site-message"
message_model.message = site_message
message_model.icon = "fa-asterisk"
if site_message[0] == "!":
message_model.css_class = "site-message-fail"
message_model.message = site_message[1:]
message_model.icon = "fa-exclamation-triangle"
elif site_message[0] == "?":
message_model.css_class = "site-message-validation"
message_model.message = site_message[1:]
message_model.icon = "fa-question-circle"
elif site_message[0] == "*":
message_model.css_class = "site-message-success"
message_model.message = site_message[1:]
message_model.icon = "fa-check-circle"
return message_model
def setUp(self):
self.r = redis.Redis(db=3) # 3 for testing
self.r.flushdb()
self.wu = WikiUtils(self.r)
self.wu.create_wiki_root_category()
class WikiUtilsTest(TestCase):
def setUp(self):
self.r = redis.Redis(db=3) # 3 for testing
self.r.flushdb()
self.wu = WikiUtils(self.r)
self.wu.create_wiki_root_category()
def test_create_category(self):
cat_name = "TestCategory"
wiki_root_cat_id = self.wu.wiki_root_category()
print "Wiki root id is: %s" % wiki_root_cat_id
self.wu.create_wiki_category(wiki_root_cat_id, cat_name)
cats = self.wu.wiki_root_categories()
existing_cat_names = []
for cat_id in cats:
existing_cat_names.append(self.wu.name_for_wiki_cat_id(cat_id))
self.assertIn(cat_name, existing_cat_names)
def test_new_article_in_category(self):
cat_name = "CategoryA"
root_cat_id = self.wu.wiki_root_category()
new_cat_id = self.wu.create_wiki_category(root_cat_id, cat_name)
title = 'Article in Category A'
slug = 'ArticleInCategoryA'
body = '##An Article about Aardvarks (WIP)'
art_id = self.wu.create_wiki_article(new_cat_id, title, slug, body)
print "The created article id is %s" % art_id
art_body = self.wu.wiki_article_body(art_id)
self.assertIsNotNone(art_body, "Wiki article body should exist.")
art_ids = self.wu.wiki_articles_in_category(new_cat_id)
self.assertIn(art_id, art_ids, "New article should be listed in category articles.")
