def admin_pages_new():
"""
Create a new page.
Administrator should have logged in to access this page.
GET: Show the 'create page' page.
POST: Save the new page. Return one of the following messages.
same name exist (err_code = -1)
database error (err_code = -2)
success (err_code = 0)
"""
if not check_auth():
abort(403)
if request.method == 'GET':
return render_template('admin_pages_new.html')
else:
try:
name = request.form['name']
Page.create(name = request.form['name'],
title = request.form['title'],
content = request.form['content'],
layout = "pure_page")
except PageNameExist:
gen_csrf_token()
return jsonify(err_code=-1, msg=u'页面名称（%s）已存在' % name)
except MySQLdb.IntegrityError:
gen_csrf_token()
return jsonify(err_code=-2, msg=u'数据库错误')
return jsonify(err_code=0, msg=u'页面新建成功')
def get(self):
user = User.checkToken(self.request.cookies.get('session'))
if self.request.cookies.get('our_token'): #the cookie that should contain the access token!
user = User.checkToken(self.request.cookies.get('our_token'))
if not user:
self.error(403)
self.response.write('No user - access denied')
return
page = None
page.title = self.request.get('title')
page.name = self.request.get('name')
page.address = self.request.get('address')
page.details = self.request.get('details')
page.emailBuss = self.request.get('emailBuss')
page.admin = user.key
if page:
if Page.checkIfPageExists(page.title,user.email):
self.response.write(json.dumps({'status':'exists'}))
return
else:
page = Page.addPage(page.title,page.name,page.address,page.details,page.emailBuss,page.admin)
time.sleep(0.5);
def test_expect_successful_page_audit(self, webmock):
# Arrange
domain = 'httpbin.org'
test_axe_report_path = helper.fixture_file_path('httpbin-org-page-all-violations.json')
webmock.get(requests_mock.ANY, text='ok')
site = Site.from_domain_or_url(domain)
page = Page(site)
audit_type = None
# Assume
self.assertIsNone(page.audit)
self.assertPathExists(test_axe_report_path)
# Act
# Mock the AxeAudit generate_report method to return our test fixture file
# path when page.axe_audit called.
with patch.object(AxePageAudit, 'generate_report') as mocked_method:
mocked_method.return_value = test_axe_report_path
page.axe_audit(audit_type)
# Assert
self.assertIsNotNone(page.audit)
self.assertEqual(page.site, site)
self.assertIn(domain, page.url)
self.assertEqual(5, len(page.audit.violations))
self.assertEqual(5, len(page.audit.errors))
self.assertEqual(0, len(page.audit.warnings))
def admin_pages_delete(name):
"""
Delete the specific page.
Administrator should have logged in to access this page.
"""
if not check_auth():
abort(403)
Page.delete(name)
return jsonify(err_code=0, msg=u'页面（%s）已删除' % name)
def audit(self):
AxeAudit.validate_type(self.audit_type)
urls = self.extract_site_page_urls_from_sitemap()
for url in urls:
page = Page(self, url)
page.axe_audit(self.audit_type)
self.pages.append(page)
return AxeAudit.from_site(self)
def get(self, id):
'''This method receives an ID from an page and returns the page'''
if Commons.isValidId(id):
page = Page.objects(id=id)
if Commons.checkIfNotExists(page):
return Commons.notFound('page')
if auth.isAuthorized(page[0].userId):
Page.objects(id=id).update(views=page[0].views + 1)
return make_response(jsonify({'data': page}), 201)
return auth.unauthorized()
def put(self, id):
'''This method receives an ID from an page and updates the page'''
params = self.reqparse.parse_args()
if Commons.isValidId(id):
page = Page.objects(id=id)
if Commons.checkIfNotExists(page):
return Commons.notFound('page')
if auth.isAuthorized(page[0].userId):
data = Commons.filterQueryParams(params)
Page.objects(id=id).update_one(upsert=False,
write_concern=None,
**data)
return make_response(jsonify({'data': 'Page updated'}), 201)
return auth.unauthorized()
def admin_pages_edit(name):
"""
Change an existing page.
Administrator should have logged in to access this page.
If argument 'name' is not an existing page name, abort 404
GET: Show the 'edit page' page.
POST: Save the change.
"""
if not check_auth():
abort(403)
try:
page = Page.get(name)
except NoSuchPage:
abort(404)
if request.method == 'GET':
return render_template('admin_pages_edit.html', **page.__dict__)
else:
page.update(name = request.form['name'],
title = request.form['title'],
content = request.form['content'],
layout = 'pure_page')
return jsonify(err_code=0, msg=u'修改保存成功')
def test_expects_to_filter_by_audit_type(self):
# Arrange
domain = 'sub.domain.com'
site = Site.from_domain_or_url(domain)
page = Page(site)
test_axe_report_path = helper.fixture_file_path(
'httpbin-org-page-all-violations.json')
with open(test_axe_report_path, "r") as f:
data = json.loads(f.read())
axe_errors = data["violations"]
test_cases = [
# audit_type expected_violations_length
("design", 2),
("code", 3),
(None, 5)
]
for audit_type, expected_violations_length in test_cases:
audit = AxePageAudit(page, audit_type)
sorted_violations = []
# Act
for error in axe_errors:
sorted_violations += Violation.s_from_audit_axe_error(
audit, error)
# Assert
self.assertEqual(expected_violations_length,
len(sorted_violations))
def test_expects_violations_in_csv(self, webmock):
# Arrange
test_dir = pathjoin(AUDITS_DIR, "sub-domain-com")
violations_csv_path = pathjoin(test_dir, "sub-domain-com.csv")
webmock.get(requests_mock.ANY, text='ok')
domain = 'sub.domain.com'
site = Site(domain)
page = Page(site)
audit = AxeSiteAudit(site)
source = 'test'
identifier = 'test-error'
severity = 'low'
violation = Violation(
page=page,
source=source,
identifier=identifier,
severity=severity
)
violation.kind = "error"
violation.help = "Error must be fixed"
violation.help_url = "https://help.com"
violation.html = "<p>Test</p>"
violation.failure = "This is incorrect"
# Act
audit.write_to_violation_csv(violations_csv_path, [violation])
with open(violations_csv_path, 'r') as file:
csv_rows = list(csv.reader(file))
row_count = len(csv_rows)
# Assert
self.assertEqual(row_count, 2)
self.assertEqual(csv_rows[0][0], "page_url")
self.assertEqual(csv_rows[1][8], violation.failure)
def post(self, name):
self.check_login()
page = Page.by_name(name)
content = self.request.get('content')
time_fmt = '%a %c'
curr_time = datetime.datetime.today().strftime(time_fmt)
entry = Entry(curr_time, content, self.user.name)
if page:
page.content=content
page.put()
else:
page = Page(name=name, content=content, history=[])
page.put()
page.history.append(entry)
self.redirect('/wiki/'+name)
def get(self, page):
if page == "":
page = "/"
page = Page.get_by_name(page)
if page:
self.response.headers.add_header("Content-Type", "text/css")
self.response.out.write(page.theme.css)
def load_page_data(self, page_file):
page_item_hash = {}
page_item_list = []
con = open(page_file, "r")
data = csv.DictReader(con, delimiter="\t")
count = 0
for row in data:
page_item = Page(row)
page_item_hash[page_item.urlid] = page_item
page_item_list.append(page_item.urlid)
if page_item.alchemy_category_score:
self.cat_score_sum+=page_item.alchemy_category_score
if page_item.avglinksize:
self.avglinksize_sum+=page_item.avglinksize
if page_item.commonLinkRatio_1:
self.commonlinkratio_1_sum+=page_item.commonLinkRatio_1
if page_item.commonLinkRatio_2:
self.commonlinkratio_2_sum+=page_item.commonLinkRatio_2
if page_item.commonLinkRatio_3:
self.commonlinkratio_3_sum+=page_item.commonLinkRatio_3
if page_item.commonLinkRatio_4:
self.commonlinkratio_4_sum+=page_item.commonLinkRatio_4
# for key in row.keys():
# if page_item.compression_ratio:
# self.add_val_to_hash(self.field_value_sum, compression_ratio, page_item.compression_ratio)
count+=1
self.page_item_list = page_item_list
self.page_item_hash = page_item_hash
self.count = count
def get(self):
page = Page.get_by_id(int(self.request.get('page_id')))
user = None
if self.request.cookies.get('our_token'): #the cookie that should contain the access token!
user = User.checkToken(self.request.cookies.get('our_token'))
if not user or page.admin != user.key:
self.error(403)
self.response.write('access denied')
return
new_user_email = self.request.get('member_email')
logging.info("Email {}".format(new_user_email))
new_user = User.query(User.email == new_user_email).get()
if not new_user:
self.error(404)
self.response.write('User with email {} not found'.format(new_user_email))
return
page.members.append(new_user.key)
page.put()
members = page.getMembers()
time.sleep(0.5)
self.response.write(json.dumps({"status": "OK", "members": members}))
def admin_pages():
if not check_auth():
abort(403)
items_list = filter(lambda x: x.layout != 'form_page', Page.get_pages_list())
return render_template('admin_pages.html',
items_list = items_list)
def crawl(page, visited, pool):
"""Crawl url, build site's map and list assets"""
logging.info('Crawling {}'.format(page.url))
visited.add(page)
if soft:
time.sleep(random.random())
try:
links = page.extract_internal_links()
except eventlet.Timeout:
page.retries_left -= 1
if page.retries_left > 0:
pool.spawn_n(crawl, page, visited, pool)
else:
logging.warning('Couldn\'t fetch {} after {} retries.'.format(
page.url, Page.MAX_RETRIES))
return
for link in links:
new_page = Page(link)
if new_page not in visited:
pool.spawn_n(crawl, new_page, visited, pool)
else:
# Url already crawled
pass
page.print_assets()
def get(self, id):
if Commons.isValidId(id):
pages = Page.objects(categoryId=id)
if Commons.checkIfNotExists(pages):
return Commons.notFound('page')
if auth.isAuthorized(pages[0].userId):
return make_response(jsonify({'data': pages}), 201)
return auth.unauthorized()
def add_page(self):
last_page = Page.query.filter_by(book_id=self.book.id).order_by(
sqlalchemy.desc(Page.id)).first()
if last_page != None:
self.page = Page(page_num=self.page_num,
book_id=self.book.id,
location_in_book=self.location_in_book,
page_image=self.page_image,
page_order=last_page.page_order + 10)
else:
self.page = Page(page_num=self.page_num,
book_id=self.book.id,
location_in_book=self.location_in_book,
page_image=self.page_image,
page_order=10)
db.session.add(self.page)
db.session.commit()
def get(self, page_id):
template_params = {}
user = None
if self.request.cookies.get('session'):
user = User.checkToken(self.request.cookies.get('session'))
if not user:
self.redirect('/')
page = Page.getPageUser(user,title)
def generate_admin_html(self, page, user):
contents = Content.all().fetch(1000)
roles = Role.all().fetch(1000)
emaildata = {"contents": contents, "roles": roles}
emailcontext = template.Context(emaildata)
email_template = template.Template(open("templates/email.html").read())
email_html = email_template.render(emailcontext)
admindata = {
"page_edit": Page.to_form(self.request.path, "edit", page.key()),
"theme_edit": Theme.to_form(self.request.path, "edit", page.theme.key(), page.key()),
"page_key": page.key(),
"path": self.request.path,
"permission_table": Permission.get_table(page.key()),
"sections": (
{
"name": section.name.replace("section_", "").capitalize(),
"theme_key": section.theme.key(),
"theme_html": section.theme.html,
"theme_css": section.theme.css,
"theme_js": section.theme.js,
"content": (
{
"title": content.title,
"content_edit": Content.to_edit_list("title", self.request.path),
"content_form": Content.to_form(
self.request.path, "edit", content.key(), section.key()
),
"content_deepform": Content.to_form(self.request.path, rel_key=section.key()),
}
for content in section.get_contents()
),
}
for section in page.get_sections()
),
"page_form": Page.to_form(self.request.path, rel_key=self.ws.site.key()),
"user_form": User.to_form(self.request.path),
"user_list": User.to_edit_list("email", self.request.path, True),
"user_edit_form": User.to_form(self.request.path, "edit", user.key()),
"user_import": open("defaults/admin/user_import.html").read(),
"images": self.ws.site.images_for_use(),
"email_blast": email_html,
}
context = template.Context(admindata)
admin_template = template.Template(open("defaults/admin/tabs.html").read())
admin_html = admin_template.render(context)
return admin_html
def show_page(name):
try:
page = Page.get(name)
return render_template('%s.html' % page.layout,
**page.__dict__)
except NoSuchPage:
abort(404)
except TemplateNotFound:
abort(404)
def get(self, name):
self.check_login()
params = {}
page = Page.by_name(name)
params['title'] = name
if page:
params['content'] = page.content
self.render('wiki-form.html', **params)
def get(self, name):
params={}
params['title'] = name
page = Page.by_name(name)
if not page:
self.redirect('/wiki/_edit/'+name)
else:
page.render()
params['page'] = page
self.render('wiki.html', **params)
def get(self, name):
params = {}
params['title'] = name
page = Page.by_name(name)
if not page:
self.redirect('/wiki/_edit/'+name)
return
params['history'] = page.history
self.render('wiki-history.html', **params)
def post(self):
'''This method creates a new page related to an user'''
params = self.reqparse.parse_args()
if Commons.isValidId(params['categoryId']):
Page(title=params['title'],
url=params['url'],
categoryId=params['categoryId'],
userId=auth.user['id']).save()
return make_response(jsonify({'data': 'Page created'}), 201)
return make_response(jsonify({'error': 'Invalid categoryId'}), 500)
def run(self, max_depth):
depth = 0
while len(self.pending) > 0:
page = self.pending.popleft()
# get links for the current page
page_data = self.scraper.run(page)
page['title'] = page_data['title']
page['links'] = page_data['links']
# persist to db
instance = Page(title=page['title'],
url=page['link'],
search_term=self.search_term,
links=page['links'],
created_at=datetime.now(),
updated_at=datetime.now())
if page['parent'] is not None:
instance.parent = page['parent']
instance.save()
if self.curr_parent is None or page[
'parent'] is not self.curr_parent:
parent_id = self.curr_parent
if parent_id is None:
parent_id = instance.id
if page['parent'] is not self.curr_parent:
depth += 1
parent_id = instance.id
self.curr_parent = page['parent']
logger.info('crawl level: %d, max depth: %d, queue size: %d',
depth, max_depth, len(self.pending))
logger.info('crawling page id %s, url: %s', str(instance.id),
page['link'])
# queue up the links for another level of crawling
if depth < max_depth:
self.enqueue(page_data['links'], parent_id)
def delete(self, id):
'''This method receives an ID from an page and deletes the page'''
if Commons.isValidId(id):
page = Page.objects(id=id)
if Commons.checkIfNotExists(page):
return Commons.notFound('page')
if auth.isAuthorized(page[0].userId):
page.delete()
return make_response(jsonify({'data': 'Page was deleted'}),
201)
return auth.unauthorized()
def test_expects_page_instance(self):
# Arrange
domain = 'sub.domain.com'
site = Site.from_domain_or_url(domain)
# Act
page = Page(site)
# Assert
self.assertIsInstance(page, Page)
self.assertEqual(page.site, site)
self.assertIn(domain, page.url)
def get(self):
contents = Content.all().fetch(1000)
theme_packages = ThemePackage.all().fetch(1000)
themes = Theme.all().fetch(1000)
pages = Page.all().fetch(1000)
images = Image.all().fetch(1000)
roles = Role.all().fetch(1000)
sections = Section.all().fetch(1000)
_users = User.all().fetch(1000)
actions = ACTIONS
template_values = {'logout_url':self.ws.users.create_logout_url('/'),'theme_packages': theme_packages,'themes': themes, 'images': images, 'pages': pages, 'contents':contents, 'roles':roles, 'users':_users, 'actions': actions, 'sections': sections, 'site': self.ws.site}
self.response.out.write(template.render('templates/manage.html',template_values))
def test_expects_new_axe_page_audit(self):
# Arrange
url = 'https://sub.domain.com'
site = Site(url)
page = Page(site)
# Act
page_audit = AxePageAudit(page)
# Assert
self.assertIsInstance(page_audit, AxePageAudit)
self.assertEqual(url, page_audit.url)
self.assertListEqual([], page_audit.violations)
def get(self):
user = User.checkToken(self.request.cookies.get('session'))
if self.request.cookies.get(
'our_token'
): #the cookie that should contain the access token!
user = User.checkToken(self.request.cookies.get('our_token'))
if not user:
self.error(403)
self.response.write('No user - access denied')
return
pages = Page.getAllPages(user)
self.response.write(json.dumps({"status": "OK", "pages": pages}))
def get(self):
page = Page.get_by_id(int(self.request.get('page')))
user = None
if self.request.cookies.get(
'our_token'
): #the cookie that should contain the access token!
user = User.checkToken(self.request.cookies.get('our_token'))
if not user or (page.admin != user.key
and user.key not in page.members):
self.error(403)
self.response.write('access denied')
return
members = page.getMembers()
self.response.write(json.dumps({"status": "OK", "members": members}))
def audit(self):
# Command-line options
domain_or_url = self.app.pargs.domain_or_url[0]
audit_type = self.app.pargs.audit_type
use_templates = not self.app.pargs.no_templates
site = Site.from_domain_or_url(domain_or_url,
audit_type=audit_type,
templates=use_templates)
if self.app.pargs.crawl:
audit = site.audit()
else:
audit = Page.audit(site, audit_type=audit_type)
audit.write_violations_to_csv()
print(audit.summary)
def test_expects_templates(self):
# Arrange
url = 'https://sub.domain.com/path/subpath/subsubpath/index.html'
site = Site.from_domain_or_url(url)
page = Page(site)
expected_templates = [
'path/subpath/subsubpath/index.html',
'path/subpath/subsubpath',
'path/subpath',
'path'
]
# Act
templates = page.templates
# Assert
self.assertEqual(expected_templates, templates)
def get_page(url) -> Optional[Page]:
try:
log.info(f"Started fetching..")
req = requests.get(url, headers=get_randomized_headers(), timeout=15)
# Make the received data searchable
soup = BeautifulSoup(req.text, features="html.parser")
log.info(f"Souped {req.url} ({req.status_code})")
title = soup.title.string if soup.title and soup.title.string else "Error: Title not found"
log.info(f"Title: {title}")
page = Page(req.url, req.status_code, title, soup)
return page
# If the submitted page couldn't be fetched, throw an exception
except (ConnectionError, Timeout, AttributeError, Exception):
log.debug(traceback.format_exc())
log.warning(f"Unable to fetch {url}")
return None
def test_expects_subtemplate(self):
# Arrange
test_cases = [
# url, expected_subtemplate
('https://sub.domain.com/path/subpath/subsubpath/index.html', 'path/subpath'),
('https://sub.domain.com/path/subpath/', 'path/subpath'),
('https://sub.domain.com/path/subpath', 'path/subpath'),
('https://sub.domain.com/path/', None),
('https://sub.domain.com/', None)
]
# Act / Assert
for url, expected_subtemplate in test_cases:
# Act
site = Site.from_domain_or_url(url)
page = Page(site)
# Assert
self.assertEqual(expected_subtemplate, page.subtemplate, url)
def test_expects_paths(self):
# Arrange
test_cases = [
# url, expected_path
('https://sub.domain.com', ''),
('https://sub.domain.com/', ''),
('https://sub.domain.com/path', 'path'),
('https://sub.domain.com/path/', 'path'),
('https://sub.domain.com/path/subpath/', 'path/subpath'),
]
# Act / Assert
for url, expected_path in test_cases:
# Act
site = Site.from_domain_or_url(url)
page = Page(site)
# Assert
self.assertEqual(expected_path, page.path, url)
self.assertEqual(url, page.url)
def get_page(url):
try:
# Fetch amp page
log.info(f"Started fetching..")
req = requests.get(url, headers=random_headers())
current_url = req.url
# Make the received data searchable
soup = BeautifulSoup(req.text, features="html.parser")
title = soup.title.string if soup.title else "Error: Title not found"
log.info(f"Made a soup of {current_url} ({req.status_code})")
log.info(f"Page title = {title}")
page = Page(current_url, req.status_code, title, soup)
return page
# If the submitted page couldn't be fetched, throw an exception
except (ConnectionError, Exception, AttributeError):
log.warning(traceback.format_exc())
log.warning("the page could not be fetched")
return None
def test_expects_violation_string_to_have_correct_information(self):
# Arrange
domain = 'sub.domain.com'
site = Site.from_domain_or_url(domain)
page = Page(site)
source = 'test'
identifier = 'test-error'
severity = 'low'
expected_output = 'test reported a low test-error error on http://sub.domain.com'
# Act
violation = Violation(page=page,
source=source,
identifier=identifier,
severity=severity)
# Same as __str__ magic method in Violation
violation_str = str(violation)
# Assert
self.assertEqual(expected_output, violation_str)
def test_expects_violation_instance(self):
# Arrange
domain = 'sub.domain.com'
site = Site.from_domain_or_url(domain)
page = Page(site)
source = 'test'
identifier = 'test-error'
severity = 'low'
# Act
violation = Violation(page=page,
source=source,
identifier=identifier,
severity=severity)
# Assert
self.assertIsInstance(violation, Violation)
self.assertEqual(page, violation.page)
self.assertEqual(source, violation.source)
self.assertEqual(identifier, violation.identifier)
self.assertEqual(severity, violation.severity)
def get_state(self):
url = self.driver.current_url
page = Page.objects(url=url).first()
if page is None:
self.driver.get(url)
default_state = state_builder.get_current_state(self.driver)
default_state.name = self.page
default_state.save()
page = Page(url=url,
default_state=default_state,
states=[default_state])
page.name = self.page
page.save()
for state in page.states:
if state.name == self.page:
print "Found state %s" % state.name
return state
print "State not found, creating new state"
new_state = state_builder.get_current_state(self.driver)
new_state.save()
new_state.name = self.page
page.states.append(new_state)
page.save()
return new_state
def test_page(self):
default_state = State.objects().first()
states = State.objects[:5]
page = Page(url="http://www.google.com/", default_state=default_state, states=states)
page.save()
assert len(page.states) > 0
def get(self):
if self.ws.site is None:
self.redirect("/install")
return False
path = self.request.path
# print path
user_control = ""
user_label = ""
def kv_q(x):
x = x.split("=")
if len(x) < 2:
return x[0]
return {x[0]: x[1]}
page = Page.get_by_name(path)
if not page:
self.error(404)
return False
if not self.permission_check(page):
self.error(403)
self.redirect("/")
return False
admin_html = ""
if page:
if not page.theme:
page.theme = Theme.create(
{
"name": ["default" + page.name],
"html": [open("defaults/template.html").read()],
"css": [open("defaults/template.css").read()],
"js": [""],
}
)
page.put()
if not page.sections:
page.get_or_make_sections()
user = self.ws.users.get_current_user(self)
if user:
auth = [{"user_control": self.ws.users.create_account_url(path), "user_label": "Account"}]
if self.ws.users.is_current_user_admin(self):
admin_html = self.generate_admin_html(page, user)
else:
auth = [
{"user_control": self.ws.users.create_register_url(path), "user_label": "Register"},
{"user_control": self.ws.users.create_login_url(path), "user_label": "Login"},
]
page_theme = page.theme
page_content_template = template.Template(page.build_template())
sections = page.get_sections()
section_dict = {}
site_users = User.all().fetch(1000)
for section in sections:
section_dict[section.name] = section
user_control_link = ""
for control in auth:
user_control_link += "<a href='%s' class='account control'>%s</a>" % (
control["user_control"],
control["user_label"],
)
page_content_template_values = {
"site_users": site_users,
"ws": self.ws,
"page": page,
"sections": section_dict,
"user_control_link": user_control_link,
}
page_content = self.render_string(page_content_template, page_content_template_values)
page_template_html = open("defaults/outer.html").read()
page_template = template.Template(page_template_html)
template_values = {
"title": page.title,
"css": page_theme.css,
"content": page_content,
"js": page_theme.js,
"admin_content": admin_html,
}
self.render_string_out(page_template, template_values)
else:
self.error(404)
def show_pages_list():
return render_template('pages.html', pageList = Page.get_pages_list())
