def admin_pages_new(): """ Create a new page. Administrator should have logged in to access this page. GET: Show the 'create page' page. POST: Save the new page. Return one of the following messages. same name exist (err_code = -1) database error (err_code = -2) success (err_code = 0) """ if not check_auth(): abort(403) if request.method == 'GET': return render_template('admin_pages_new.html') else: try: name = request.form['name'] Page.create(name = request.form['name'], title = request.form['title'], content = request.form['content'], layout = "pure_page") except PageNameExist: gen_csrf_token() return jsonify(err_code=-1, msg=u'页面名称(%s)已存在' % name) except MySQLdb.IntegrityError: gen_csrf_token() return jsonify(err_code=-2, msg=u'数据库错误') return jsonify(err_code=0, msg=u'页面新建成功')
def get(self): user = User.checkToken(self.request.cookies.get('session')) if self.request.cookies.get('our_token'): #the cookie that should contain the access token! user = User.checkToken(self.request.cookies.get('our_token')) if not user: self.error(403) self.response.write('No user - access denied') return page = None page.title = self.request.get('title') page.name = self.request.get('name') page.address = self.request.get('address') page.details = self.request.get('details') page.emailBuss = self.request.get('emailBuss') page.admin = user.key if page: if Page.checkIfPageExists(page.title,user.email): self.response.write(json.dumps({'status':'exists'})) return else: page = Page.addPage(page.title,page.name,page.address,page.details,page.emailBuss,page.admin) time.sleep(0.5);
def test_expect_successful_page_audit(self, webmock): # Arrange domain = 'httpbin.org' test_axe_report_path = helper.fixture_file_path('httpbin-org-page-all-violations.json') webmock.get(requests_mock.ANY, text='ok') site = Site.from_domain_or_url(domain) page = Page(site) audit_type = None # Assume self.assertIsNone(page.audit) self.assertPathExists(test_axe_report_path) # Act # Mock the AxeAudit generate_report method to return our test fixture file # path when page.axe_audit called. with patch.object(AxePageAudit, 'generate_report') as mocked_method: mocked_method.return_value = test_axe_report_path page.axe_audit(audit_type) # Assert self.assertIsNotNone(page.audit) self.assertEqual(page.site, site) self.assertIn(domain, page.url) self.assertEqual(5, len(page.audit.violations)) self.assertEqual(5, len(page.audit.errors)) self.assertEqual(0, len(page.audit.warnings))
def admin_pages_delete(name): """ Delete the specific page. Administrator should have logged in to access this page. """ if not check_auth(): abort(403) Page.delete(name) return jsonify(err_code=0, msg=u'页面(%s)已删除' % name)
def audit(self): AxeAudit.validate_type(self.audit_type) urls = self.extract_site_page_urls_from_sitemap() for url in urls: page = Page(self, url) page.axe_audit(self.audit_type) self.pages.append(page) return AxeAudit.from_site(self)
def get(self, id): '''This method receives an ID from an page and returns the page''' if Commons.isValidId(id): page = Page.objects(id=id) if Commons.checkIfNotExists(page): return Commons.notFound('page') if auth.isAuthorized(page[0].userId): Page.objects(id=id).update(views=page[0].views + 1) return make_response(jsonify({'data': page}), 201) return auth.unauthorized()
def put(self, id): '''This method receives an ID from an page and updates the page''' params = self.reqparse.parse_args() if Commons.isValidId(id): page = Page.objects(id=id) if Commons.checkIfNotExists(page): return Commons.notFound('page') if auth.isAuthorized(page[0].userId): data = Commons.filterQueryParams(params) Page.objects(id=id).update_one(upsert=False, write_concern=None, **data) return make_response(jsonify({'data': 'Page updated'}), 201) return auth.unauthorized()
def admin_pages_edit(name): """ Change an existing page. Administrator should have logged in to access this page. If argument 'name' is not an existing page name, abort 404 GET: Show the 'edit page' page. POST: Save the change. """ if not check_auth(): abort(403) try: page = Page.get(name) except NoSuchPage: abort(404) if request.method == 'GET': return render_template('admin_pages_edit.html', **page.__dict__) else: page.update(name = request.form['name'], title = request.form['title'], content = request.form['content'], layout = 'pure_page') return jsonify(err_code=0, msg=u'修改保存成功')
def test_expects_to_filter_by_audit_type(self): # Arrange domain = 'sub.domain.com' site = Site.from_domain_or_url(domain) page = Page(site) test_axe_report_path = helper.fixture_file_path( 'httpbin-org-page-all-violations.json') with open(test_axe_report_path, "r") as f: data = json.loads(f.read()) axe_errors = data["violations"] test_cases = [ # audit_type expected_violations_length ("design", 2), ("code", 3), (None, 5) ] for audit_type, expected_violations_length in test_cases: audit = AxePageAudit(page, audit_type) sorted_violations = [] # Act for error in axe_errors: sorted_violations += Violation.s_from_audit_axe_error( audit, error) # Assert self.assertEqual(expected_violations_length, len(sorted_violations))
def test_expects_violations_in_csv(self, webmock): # Arrange test_dir = pathjoin(AUDITS_DIR, "sub-domain-com") violations_csv_path = pathjoin(test_dir, "sub-domain-com.csv") webmock.get(requests_mock.ANY, text='ok') domain = 'sub.domain.com' site = Site(domain) page = Page(site) audit = AxeSiteAudit(site) source = 'test' identifier = 'test-error' severity = 'low' violation = Violation( page=page, source=source, identifier=identifier, severity=severity ) violation.kind = "error" violation.help = "Error must be fixed" violation.help_url = "https://help.com" violation.html = "<p>Test</p>" violation.failure = "This is incorrect" # Act audit.write_to_violation_csv(violations_csv_path, [violation]) with open(violations_csv_path, 'r') as file: csv_rows = list(csv.reader(file)) row_count = len(csv_rows) # Assert self.assertEqual(row_count, 2) self.assertEqual(csv_rows[0][0], "page_url") self.assertEqual(csv_rows[1][8], violation.failure)
def post(self, name): self.check_login() page = Page.by_name(name) content = self.request.get('content') time_fmt = '%a %c' curr_time = datetime.datetime.today().strftime(time_fmt) entry = Entry(curr_time, content, self.user.name) if page: page.content=content page.put() else: page = Page(name=name, content=content, history=[]) page.put() page.history.append(entry) self.redirect('/wiki/'+name)
def get(self, page): if page == "": page = "/" page = Page.get_by_name(page) if page: self.response.headers.add_header("Content-Type", "text/css") self.response.out.write(page.theme.css)
def load_page_data(self, page_file): page_item_hash = {} page_item_list = [] con = open(page_file, "r") data = csv.DictReader(con, delimiter="\t") count = 0 for row in data: page_item = Page(row) page_item_hash[page_item.urlid] = page_item page_item_list.append(page_item.urlid) if page_item.alchemy_category_score: self.cat_score_sum+=page_item.alchemy_category_score if page_item.avglinksize: self.avglinksize_sum+=page_item.avglinksize if page_item.commonLinkRatio_1: self.commonlinkratio_1_sum+=page_item.commonLinkRatio_1 if page_item.commonLinkRatio_2: self.commonlinkratio_2_sum+=page_item.commonLinkRatio_2 if page_item.commonLinkRatio_3: self.commonlinkratio_3_sum+=page_item.commonLinkRatio_3 if page_item.commonLinkRatio_4: self.commonlinkratio_4_sum+=page_item.commonLinkRatio_4 # for key in row.keys(): # if page_item.compression_ratio: # self.add_val_to_hash(self.field_value_sum, compression_ratio, page_item.compression_ratio) count+=1 self.page_item_list = page_item_list self.page_item_hash = page_item_hash self.count = count
def get(self): page = Page.get_by_id(int(self.request.get('page_id'))) user = None if self.request.cookies.get('our_token'): #the cookie that should contain the access token! user = User.checkToken(self.request.cookies.get('our_token')) if not user or page.admin != user.key: self.error(403) self.response.write('access denied') return new_user_email = self.request.get('member_email') logging.info("Email {}".format(new_user_email)) new_user = User.query(User.email == new_user_email).get() if not new_user: self.error(404) self.response.write('User with email {} not found'.format(new_user_email)) return page.members.append(new_user.key) page.put() members = page.getMembers() time.sleep(0.5) self.response.write(json.dumps({"status": "OK", "members": members}))
def admin_pages(): if not check_auth(): abort(403) items_list = filter(lambda x: x.layout != 'form_page', Page.get_pages_list()) return render_template('admin_pages.html', items_list = items_list)
def crawl(page, visited, pool): """Crawl url, build site's map and list assets""" logging.info('Crawling {}'.format(page.url)) visited.add(page) if soft: time.sleep(random.random()) try: links = page.extract_internal_links() except eventlet.Timeout: page.retries_left -= 1 if page.retries_left > 0: pool.spawn_n(crawl, page, visited, pool) else: logging.warning('Couldn\'t fetch {} after {} retries.'.format( page.url, Page.MAX_RETRIES)) return for link in links: new_page = Page(link) if new_page not in visited: pool.spawn_n(crawl, new_page, visited, pool) else: # Url already crawled pass page.print_assets()
def get(self, id): if Commons.isValidId(id): pages = Page.objects(categoryId=id) if Commons.checkIfNotExists(pages): return Commons.notFound('page') if auth.isAuthorized(pages[0].userId): return make_response(jsonify({'data': pages}), 201) return auth.unauthorized()
def add_page(self): last_page = Page.query.filter_by(book_id=self.book.id).order_by( sqlalchemy.desc(Page.id)).first() if last_page != None: self.page = Page(page_num=self.page_num, book_id=self.book.id, location_in_book=self.location_in_book, page_image=self.page_image, page_order=last_page.page_order + 10) else: self.page = Page(page_num=self.page_num, book_id=self.book.id, location_in_book=self.location_in_book, page_image=self.page_image, page_order=10) db.session.add(self.page) db.session.commit()
def get(self, page_id): template_params = {} user = None if self.request.cookies.get('session'): user = User.checkToken(self.request.cookies.get('session')) if not user: self.redirect('/') page = Page.getPageUser(user,title)
def generate_admin_html(self, page, user): contents = Content.all().fetch(1000) roles = Role.all().fetch(1000) emaildata = {"contents": contents, "roles": roles} emailcontext = template.Context(emaildata) email_template = template.Template(open("templates/email.html").read()) email_html = email_template.render(emailcontext) admindata = { "page_edit": Page.to_form(self.request.path, "edit", page.key()), "theme_edit": Theme.to_form(self.request.path, "edit", page.theme.key(), page.key()), "page_key": page.key(), "path": self.request.path, "permission_table": Permission.get_table(page.key()), "sections": ( { "name": section.name.replace("section_", "").capitalize(), "theme_key": section.theme.key(), "theme_html": section.theme.html, "theme_css": section.theme.css, "theme_js": section.theme.js, "content": ( { "title": content.title, "content_edit": Content.to_edit_list("title", self.request.path), "content_form": Content.to_form( self.request.path, "edit", content.key(), section.key() ), "content_deepform": Content.to_form(self.request.path, rel_key=section.key()), } for content in section.get_contents() ), } for section in page.get_sections() ), "page_form": Page.to_form(self.request.path, rel_key=self.ws.site.key()), "user_form": User.to_form(self.request.path), "user_list": User.to_edit_list("email", self.request.path, True), "user_edit_form": User.to_form(self.request.path, "edit", user.key()), "user_import": open("defaults/admin/user_import.html").read(), "images": self.ws.site.images_for_use(), "email_blast": email_html, } context = template.Context(admindata) admin_template = template.Template(open("defaults/admin/tabs.html").read()) admin_html = admin_template.render(context) return admin_html
def show_page(name): try: page = Page.get(name) return render_template('%s.html' % page.layout, **page.__dict__) except NoSuchPage: abort(404) except TemplateNotFound: abort(404)
def get(self, name): self.check_login() params = {} page = Page.by_name(name) params['title'] = name if page: params['content'] = page.content self.render('wiki-form.html', **params)
def get(self, name): params={} params['title'] = name page = Page.by_name(name) if not page: self.redirect('/wiki/_edit/'+name) else: page.render() params['page'] = page self.render('wiki.html', **params)
def get(self, name): params = {} params['title'] = name page = Page.by_name(name) if not page: self.redirect('/wiki/_edit/'+name) return params['history'] = page.history self.render('wiki-history.html', **params)
def post(self): '''This method creates a new page related to an user''' params = self.reqparse.parse_args() if Commons.isValidId(params['categoryId']): Page(title=params['title'], url=params['url'], categoryId=params['categoryId'], userId=auth.user['id']).save() return make_response(jsonify({'data': 'Page created'}), 201) return make_response(jsonify({'error': 'Invalid categoryId'}), 500)
def run(self, max_depth): depth = 0 while len(self.pending) > 0: page = self.pending.popleft() # get links for the current page page_data = self.scraper.run(page) page['title'] = page_data['title'] page['links'] = page_data['links'] # persist to db instance = Page(title=page['title'], url=page['link'], search_term=self.search_term, links=page['links'], created_at=datetime.now(), updated_at=datetime.now()) if page['parent'] is not None: instance.parent = page['parent'] instance.save() if self.curr_parent is None or page[ 'parent'] is not self.curr_parent: parent_id = self.curr_parent if parent_id is None: parent_id = instance.id if page['parent'] is not self.curr_parent: depth += 1 parent_id = instance.id self.curr_parent = page['parent'] logger.info('crawl level: %d, max depth: %d, queue size: %d', depth, max_depth, len(self.pending)) logger.info('crawling page id %s, url: %s', str(instance.id), page['link']) # queue up the links for another level of crawling if depth < max_depth: self.enqueue(page_data['links'], parent_id)
def delete(self, id): '''This method receives an ID from an page and deletes the page''' if Commons.isValidId(id): page = Page.objects(id=id) if Commons.checkIfNotExists(page): return Commons.notFound('page') if auth.isAuthorized(page[0].userId): page.delete() return make_response(jsonify({'data': 'Page was deleted'}), 201) return auth.unauthorized()
def test_expects_page_instance(self): # Arrange domain = 'sub.domain.com' site = Site.from_domain_or_url(domain) # Act page = Page(site) # Assert self.assertIsInstance(page, Page) self.assertEqual(page.site, site) self.assertIn(domain, page.url)
def get(self): contents = Content.all().fetch(1000) theme_packages = ThemePackage.all().fetch(1000) themes = Theme.all().fetch(1000) pages = Page.all().fetch(1000) images = Image.all().fetch(1000) roles = Role.all().fetch(1000) sections = Section.all().fetch(1000) _users = User.all().fetch(1000) actions = ACTIONS template_values = {'logout_url':self.ws.users.create_logout_url('/'),'theme_packages': theme_packages,'themes': themes, 'images': images, 'pages': pages, 'contents':contents, 'roles':roles, 'users':_users, 'actions': actions, 'sections': sections, 'site': self.ws.site} self.response.out.write(template.render('templates/manage.html',template_values))
def test_expects_new_axe_page_audit(self): # Arrange url = 'https://sub.domain.com' site = Site(url) page = Page(site) # Act page_audit = AxePageAudit(page) # Assert self.assertIsInstance(page_audit, AxePageAudit) self.assertEqual(url, page_audit.url) self.assertListEqual([], page_audit.violations)
def get(self): user = User.checkToken(self.request.cookies.get('session')) if self.request.cookies.get( 'our_token' ): #the cookie that should contain the access token! user = User.checkToken(self.request.cookies.get('our_token')) if not user: self.error(403) self.response.write('No user - access denied') return pages = Page.getAllPages(user) self.response.write(json.dumps({"status": "OK", "pages": pages}))
def get(self): page = Page.get_by_id(int(self.request.get('page'))) user = None if self.request.cookies.get( 'our_token' ): #the cookie that should contain the access token! user = User.checkToken(self.request.cookies.get('our_token')) if not user or (page.admin != user.key and user.key not in page.members): self.error(403) self.response.write('access denied') return members = page.getMembers() self.response.write(json.dumps({"status": "OK", "members": members}))
def audit(self): # Command-line options domain_or_url = self.app.pargs.domain_or_url[0] audit_type = self.app.pargs.audit_type use_templates = not self.app.pargs.no_templates site = Site.from_domain_or_url(domain_or_url, audit_type=audit_type, templates=use_templates) if self.app.pargs.crawl: audit = site.audit() else: audit = Page.audit(site, audit_type=audit_type) audit.write_violations_to_csv() print(audit.summary)
def test_expects_templates(self): # Arrange url = 'https://sub.domain.com/path/subpath/subsubpath/index.html' site = Site.from_domain_or_url(url) page = Page(site) expected_templates = [ 'path/subpath/subsubpath/index.html', 'path/subpath/subsubpath', 'path/subpath', 'path' ] # Act templates = page.templates # Assert self.assertEqual(expected_templates, templates)
def get_page(url) -> Optional[Page]: try: log.info(f"Started fetching..") req = requests.get(url, headers=get_randomized_headers(), timeout=15) # Make the received data searchable soup = BeautifulSoup(req.text, features="html.parser") log.info(f"Souped {req.url} ({req.status_code})") title = soup.title.string if soup.title and soup.title.string else "Error: Title not found" log.info(f"Title: {title}") page = Page(req.url, req.status_code, title, soup) return page # If the submitted page couldn't be fetched, throw an exception except (ConnectionError, Timeout, AttributeError, Exception): log.debug(traceback.format_exc()) log.warning(f"Unable to fetch {url}") return None
def test_expects_subtemplate(self): # Arrange test_cases = [ # url, expected_subtemplate ('https://sub.domain.com/path/subpath/subsubpath/index.html', 'path/subpath'), ('https://sub.domain.com/path/subpath/', 'path/subpath'), ('https://sub.domain.com/path/subpath', 'path/subpath'), ('https://sub.domain.com/path/', None), ('https://sub.domain.com/', None) ] # Act / Assert for url, expected_subtemplate in test_cases: # Act site = Site.from_domain_or_url(url) page = Page(site) # Assert self.assertEqual(expected_subtemplate, page.subtemplate, url)
def test_expects_paths(self): # Arrange test_cases = [ # url, expected_path ('https://sub.domain.com', ''), ('https://sub.domain.com/', ''), ('https://sub.domain.com/path', 'path'), ('https://sub.domain.com/path/', 'path'), ('https://sub.domain.com/path/subpath/', 'path/subpath'), ] # Act / Assert for url, expected_path in test_cases: # Act site = Site.from_domain_or_url(url) page = Page(site) # Assert self.assertEqual(expected_path, page.path, url) self.assertEqual(url, page.url)
def get_page(url): try: # Fetch amp page log.info(f"Started fetching..") req = requests.get(url, headers=random_headers()) current_url = req.url # Make the received data searchable soup = BeautifulSoup(req.text, features="html.parser") title = soup.title.string if soup.title else "Error: Title not found" log.info(f"Made a soup of {current_url} ({req.status_code})") log.info(f"Page title = {title}") page = Page(current_url, req.status_code, title, soup) return page # If the submitted page couldn't be fetched, throw an exception except (ConnectionError, Exception, AttributeError): log.warning(traceback.format_exc()) log.warning("the page could not be fetched") return None
def test_expects_violation_string_to_have_correct_information(self): # Arrange domain = 'sub.domain.com' site = Site.from_domain_or_url(domain) page = Page(site) source = 'test' identifier = 'test-error' severity = 'low' expected_output = 'test reported a low test-error error on http://sub.domain.com' # Act violation = Violation(page=page, source=source, identifier=identifier, severity=severity) # Same as __str__ magic method in Violation violation_str = str(violation) # Assert self.assertEqual(expected_output, violation_str)
def test_expects_violation_instance(self): # Arrange domain = 'sub.domain.com' site = Site.from_domain_or_url(domain) page = Page(site) source = 'test' identifier = 'test-error' severity = 'low' # Act violation = Violation(page=page, source=source, identifier=identifier, severity=severity) # Assert self.assertIsInstance(violation, Violation) self.assertEqual(page, violation.page) self.assertEqual(source, violation.source) self.assertEqual(identifier, violation.identifier) self.assertEqual(severity, violation.severity)
def get_state(self): url = self.driver.current_url page = Page.objects(url=url).first() if page is None: self.driver.get(url) default_state = state_builder.get_current_state(self.driver) default_state.name = self.page default_state.save() page = Page(url=url, default_state=default_state, states=[default_state]) page.name = self.page page.save() for state in page.states: if state.name == self.page: print "Found state %s" % state.name return state print "State not found, creating new state" new_state = state_builder.get_current_state(self.driver) new_state.save() new_state.name = self.page page.states.append(new_state) page.save() return new_state
def test_page(self): default_state = State.objects().first() states = State.objects[:5] page = Page(url="http://www.google.com/", default_state=default_state, states=states) page.save() assert len(page.states) > 0
def get(self): if self.ws.site is None: self.redirect("/install") return False path = self.request.path # print path user_control = "" user_label = "" def kv_q(x): x = x.split("=") if len(x) < 2: return x[0] return {x[0]: x[1]} page = Page.get_by_name(path) if not page: self.error(404) return False if not self.permission_check(page): self.error(403) self.redirect("/") return False admin_html = "" if page: if not page.theme: page.theme = Theme.create( { "name": ["default" + page.name], "html": [open("defaults/template.html").read()], "css": [open("defaults/template.css").read()], "js": [""], } ) page.put() if not page.sections: page.get_or_make_sections() user = self.ws.users.get_current_user(self) if user: auth = [{"user_control": self.ws.users.create_account_url(path), "user_label": "Account"}] if self.ws.users.is_current_user_admin(self): admin_html = self.generate_admin_html(page, user) else: auth = [ {"user_control": self.ws.users.create_register_url(path), "user_label": "Register"}, {"user_control": self.ws.users.create_login_url(path), "user_label": "Login"}, ] page_theme = page.theme page_content_template = template.Template(page.build_template()) sections = page.get_sections() section_dict = {} site_users = User.all().fetch(1000) for section in sections: section_dict[section.name] = section user_control_link = "" for control in auth: user_control_link += "<a href='%s' class='account control'>%s</a>" % ( control["user_control"], control["user_label"], ) page_content_template_values = { "site_users": site_users, "ws": self.ws, "page": page, "sections": section_dict, "user_control_link": user_control_link, } page_content = self.render_string(page_content_template, page_content_template_values) page_template_html = open("defaults/outer.html").read() page_template = template.Template(page_template_html) template_values = { "title": page.title, "css": page_theme.css, "content": page_content, "js": page_theme.js, "admin_content": admin_html, } self.render_string_out(page_template, template_values) else: self.error(404)
def show_pages_list(): return render_template('pages.html', pageList = Page.get_pages_list())