def test_page_twice(self):
# make sure database is cleared out between tests
url = 'http://www.example.com/foo'
Page.create(url=url, content='hi world', status_code=200)
self.assertEqual(Page.select().count(), 1)
self.assertRaises(IntegrityError, Page.create, url=url, content='hi world', status_code=200)
def test_create_links(self):
# no pages should exist
self.assertEqual(Page.select().count(), 0)
# create links
links = create_links(link1, [link2])
self.assertTrue(len(links), 1)
link = links[0]
self.assertEqual(link.from_page.url, link1)
self.assertEqual(link.to_page.url, link2)
# 2 pages should exist
self.assertEqual(Page.select().count(), 2)
def delete(path):
if not ModuleAPI.can_write('page'):
return abort(403)
page = Page.get_by_path(path)
if not page:
flash(_('The page you tried to delete does not exist.'), 'danger')
return redirect(url_for('page.get_page', path=path))
abort(404)
rev = page.get_latest_revision()
class DeleteForm(Form):
title = StringField(_('Page title'))
form = DeleteForm(request.form)
if form.validate_on_submit():
if rev.title == form.title.data:
db.session.delete(page)
db.session.commit()
flash(_('The page has been deleted'), 'success')
return redirect(url_for('home.home'))
else:
flash(_('The given title does not match the page title.'),
'warning')
else:
flash_form_errors(form)
return render_template('page/delete.htm', rev=rev, form=form)
def test_crawl_existing_page(self):
content = 'foobar'
page = Page.create(url=link1, content=content, status_code=200)
crawled_page, crawled = crawl_page(link1)
self.assertFalse(crawled)
self.assertEqual(page.id, crawled_page.id)
def get_seo_fields(language="nl", module_name=None, request_path=None):
"""Get the seo fields as dict."""
# Check if the module and path are set.
if module_name is None:
module_name = request.blueprint
if request_path is None:
request_path = request.path
# Check which type of seo fields should be retrieved, based on
# the module name.
if module_name == "activity":
# Get activity id
activity_id = re.search(r"\/([0-9]+)\/", request_path)
# Check seo existance
if activity_id is not None:
# Get the seo object of an activity
seo = SEO.get_by_activity(activity_id.group(1))
else:
# No seo was found for this activity
seo = None
elif module_name == "page":
# Retrieve the page for its id
path = request_path[1:]
page = Page.get_by_path(path)
# Retrieve the revision by page id
if page is not None:
seo = SEO.get_by_page(page.id)
else:
seo = None
else:
# Retrieve seo fields based on the module name.
seo = SEO.get_by_url(module_name)
# Retrieve the seo fields based on the seo object
# or global set values.
if seo is not None:
# Retrieve the language specific SEO fields
if language == "nl":
return {"title": seo.nl_title, "description": seo.nl_description, "tags": seo.nl_tags}
elif language == "en":
return {"title": seo.en_title, "description": seo.en_description, "tags": seo.en_tags}
else:
# TODO, good standard tags
return {
"title": "via",
"description": "Studievereniging via - Informatica, "
+ "Informatiekunde, Informatica, "
+ "University of Amsterdam",
"tags": "Studievereniging,via, informatica, " + "informatiekunde, University of Amsterdam",
}
def addPage(request):
if request.method == 'GET':
page_url = request.GET.get('url')
category_id = request.GET.get('category')
webpage = urllib2.urlopen(page_url).read()
soup = BeautifulSoup(''.join(webpage))
user = User.objects.get(pk=request.user.id)
category = Category.objects.get(id=category_id)
page_title = soup('title')[0].string
page_title = slugify.slugify_ru(page_title)
file_name = u''.join(e for e in page_title if e.isalnum())
file_name = slugify.slugify_ru(file_name)
os.system(STATIC_ROOT+"/webkit2png.py -D "+STATIC_ROOT+"'/images/'"+str(request.user.id)+" -T -o "+file_name+" "+page_url)
page = Page(title=page_title, url=page_url, image_file=file_name+'.png',category=category, user=user)
page.save()
return redirect('home')
else:
return Response(status=403)
def test_permanent_redirect(self, requests_get, requests_head):
url = "http://www.example.com/foo"
redirect_url = "http://www.example.com/bar"
headers = {'location': redirect_url}
requests_head.return_value = MagicMock(status_code=301,
headers=headers)
page = Page.create(url=url, content='', status_code=0)
add_page_info_to_page(page)
to_page = Page.select().where(Page.url == redirect_url).first()
self.assertTrue(to_page)
url_redirect_link = Link.select().where(Link.from_page == page,
Link.to_page == to_page)
self.assertTrue(url_redirect_link.exists())
self.assertEqual(requests_head.call_count, 1)
self.assertFalse(requests_get.called)
self.assertEqual(page.content, redirect_url)
def test_permanent_redirect(self, requests_get, requests_head):
url = "http://www.example.com/foo"
redirect_url = "http://www.example.com/bar"
headers = { 'location': redirect_url }
requests_head.return_value = MagicMock(status_code=301, headers=headers)
page = Page.create(url=url, content='', status_code=0)
add_page_info_to_page(page)
to_page = Page.select().where(Page.url == redirect_url).first()
self.assertTrue(to_page)
url_redirect_link = Link.select().where(
Link.from_page == page,
Link.to_page == to_page)
self.assertTrue(url_redirect_link.exists())
self.assertEqual(requests_head.call_count, 1)
self.assertFalse(requests_get.called)
self.assertEqual(page.content, redirect_url)
def delete_page(id):
"""
Returns:
"""
if id > 0:
page = Page()
page.set_id(id)
page.load()
page.delete()
return redirect(url_for("backend.pages"))
def import_rechem_data():
c = sqlite3.connect('rechem_listings.db')
with open('rechem_drug_titles', 'r') as f:
s = f.read()
drug_title_dict = ast.literal_eval(s)
rechem_pages = pd.read_sql_query("SELECT * FROM Listings", c)
rechem_listings = pd.read_sql_query("SELECT * FROM Listing_index", c)
for i, row in rechem_listings.iterrows():
rechem_listings.loc[i, 'drug'] = drug_title_dict[row['title']]
m = Market.query.filter_by(name="rechem_real").first()
if not m:
m = Market(name="rechem_real")
db.session.add(m)
db.session.commit()
print("market added")
listings = []
for i, row in rechem_listings.iterrows():
drug = Drug.query.filter_by(name=row['drug']).first()
if not drug:
db.session.add(Drug(name=row['drug']))
db.session.commit()
#check if listing for this drug exists. if not, add it.
listing = Listing.query.filter_by(drug=drug).first()
if not listing:
listings.append(Listing(url=row['url'], market=m, drug=drug))
if listings:
db.session.add_all(listings)
db.session.commit()
print("Listings added")
pages = []
for i, row in rechem_pages.iterrows():
listing = rechem_listings[rechem_listings['id'] == row['listing_index_id']]
listing_drug = listing['drug'].item()
listing_name = listing['title'].item()
drug = Drug.query.filter_by(name=listing_drug).first()
listing = Listing.query.filter_by(drug=drug, market=m).first()
# check if a page exsits for this time/listing. If not, add it.
timestamp = datetime.strptime(row['scraped_time'], "%Y-%m-%d %H:%M:%S")
page = Page.query.filter_by(listing=listing, timestamp=timestamp).first()
if not page:
pages.append(Page(html=row['page_text'], timestamp=timestamp, name=listing_name, listing=listing))
if pages:
db.session.add_all(pages)
db.session.commit()
print("Pages added")
return ('', 204)
def addPage(pageInfo):
page = Page()
getId = False
# 获取页面ID
if 'id' in pageInfo:
page = getPageById(pageInfo['id'])
getId = True
if len(pageInfo['title']) > 30:
pageInfo['title'] = pageInfo['title'][0:30]
page.page_title = pageInfo['title']
# 默认短地址为标题
if 'slug' in pageInfo:
page.page_slug = pageInfo['slug']
else:
page.page_slug = page.page_title
if len(pageInfo['content']) > 5000:
pageInfo['content'] = pageInfo['content'][0:5000]
page.page_content = pageInfo['content']
page.page_date = pageInfo['date']
if 'pass' in pageInfo:
page.page_password = hashlib.md5(pageInfo['pass']).hexdigest()
page.page_status = pageInfo['status']
page.user_id = pageInfo['userId']
db.session.add(page)
if getId is True:
db.session.flush()
db.session.commit()
return page
def test_turns_file_models_into_database_models(self):
file_page = PageFile.create("filename.md",
name="Test Page",
slug="/test_page/",
hero_image="/path/to/image")
ModelConverter.convert(Page)
database_page = Page.where("slug", "=", "/test_page/").first_or_fail()
expect(database_page.name).to(equal(file_page.metadata['name']))
expect(database_page.slug).to(equal(file_page.metadata['slug']))
expect(database_page.hero_image).to(
equal(file_page.metadata['hero_image']))
def get_page(path=''):
path = Page.strip_path(path)
page = Page.get_by_path(path)
if not page:
# Try if this might be a redirect.
print("not page")
redirection = Redirect.query.filter(Redirect.fro == path).first()
if redirection:
# get GET parameters so they can be applied to the redirected
# URL
if request.args:
redir_url = redirection.to + '?'
for key in request.args:
redir_url += key + '=' + \
request.args[key] + '&'
print(redir_url)
# this is necssary to prevent incorrect escaping
return redirect(iri_to_uri(redir_url))
return redirect(redirection.to)
return abort(404)
if not PageAPI.can_read(page):
return abort(403)
revision = page.get_latest_revision()
if not revision:
return abort(500)
return render_template('%s/view_single.htm' % (page.type), page=page,
revision=revision, title=revision.title,
context=revision.__class__.context)
def seed_pages():
count = 100
result = []
while count > 0:
result.append(Page(title=fake.text(max_nb_chars=50), text=fake.text(max_nb_chars=100), book_id=randint(1, 100)))
count -= 1
for page in result:
db.session.add(page)
db.session.commit()
amount = 0
temp = []
while amount < 10:
temp.append(Page(title=fake.text(max_nb_chars=50), text=fake.text(max_nb_chars=100), book_id=int(f'10{amount}')))
amount += 1
for demo in temp:
db.session.add(demo)
db.session.commit()
def add_page(pageinfo):
page = Page()
getid = False
# 获取页面ID
if 'id' in pageinfo:
page = get_page_by_id(pageinfo['id'])
getid = True
if len(pageinfo['title']) > 30:
pageinfo['title'] = pageinfo['title'][0:30]
page.page_title = pageinfo['title']
# 默认短地址为标题
if 'slug' in pageinfo:
page.page_slug = pageinfo['slug']
else:
page.page_slug = page.page_title
if len(pageinfo['content']) > 5000:
pageinfo['content'] = pageinfo['content'][0:5000]
page.page_content = pageinfo['content']
page.page_date = pageinfo['date']
if 'pass' in pageinfo:
page.page_password = hashlib.md5(pageinfo['pass']).hexdigest()
page.page_status = pageinfo['status']
if 'userId' in pageinfo:
page.user_id = pageinfo['userId']
db.session.add(page)
if getid is True:
db.session.flush()
db.session.commit()
return page
def index(path):
comment_form = AuthenticatedCommentForm(
) if current_user.is_authenticated else CommentForm()
comment_form.subscribe.data = False if current_user.is_authenticated else True
Page.set_nav()
current_app.logger.debug(request.host_url)
current_app.logger.debug(request.host.lower())
if request.host.lower() == "sprig.houstonhare.com":
path = f"/stories/sprig/{path}"
else:
path = f"/{path}"
page = Page.query.filter_by(path=path).first()
print(f"path: {path}")
print(f"page: {page}")
if page:
comment_form.page_id.data = page.id
code = request.args['code'] if 'code' in request.args else None
if page.published or page.check_view_code(code):
return render_template(f'page/{page.template}.html',
page=page,
comment_form=comment_form,
js='comments.js')
page = Page.query.filter_by(slug='404-error').first()
return render_template(f'page/{page.template}.html', page=page), 404
def rendered_page(self, path):
site = {
"games": Game.all(),
"pages": Page.all(),
"calendar": Event.future_events(),
}
# is game?
if path.startswith("games/"):
# remove trailing slash
slug = "/".join(path.split("/")[1:])
game = Game.where("slug", "=", slug).first()
if game:
return self._render(game.layout, page=game, site=site)
# not game
if path != "/":
path = "/{}/".format(path)
page = Page.where("slug", "=", path).first()
if page:
return self._render(page.layout, page=page, site=site)
return None
def add_admin():
from app.models import Admin, LoveMe
from config import Config
# 创建管理员
admin = Admin(site_name=Config.SITE_NAME, site_title=Config.SITE_TITLE, name=Config.ADMIN_NAME,
profile=Config.ADMIN_PROFILE, login_name=Config.ADMIN_LOGIN_NAME,
password=Config.ADMIN_PASSWORD)
# 创建love-me
love = LoveMe(loveMe=666)
# 创建留言板
guest_book = Page(title='留言板', url_name='guest-book', canComment=True,
isNav=False, body='留言板')
db.session.add(admin)
db.session.add(love)
db.session.add(guest_book)
db.session.commit()
def install_data():
for setting in default_settings:
db.session.add(setting)
for i, (name, url) in enumerate(pages, 1):
page = Page(name=name, url=url, template=u'page.html', position=i)
db.session.add(page)
for i, (name, page_id, parent_id) in enumerate(menu, 1):
menuitem = MenuItem(name=name,
page_id=page_id,
parent_id=parent_id,
position=i)
db.session.add(menuitem)
sec1html = u'<p>Content for pages can be written in the administration panel' \
u' under Content -> Pages or Content -> Sections.</p>'
section = Section(name=u'Introduction',
html=sec1html,
page_id=1,
template=u'section.html',
position=1)
db.session.add(section)
section = Section(name=u'Members',
html=u'',
page_id=2,
template=u'members.html',
position=1)
db.session.add(section)
mem1desc = u'<p>Information about team members can be written in the administration panel' \
u' under Content -> Entities.</p><p>Entities with role assigned as Member will be shown here.</p>'
member = Entity(name=u'Team Member 1',
description=mem1desc,
role=u'Member',
position=1)
db.session.add(member)
member = Entity(name=u'Team Member 2',
description=u'',
role=u'Member',
position=2)
db.session.add(member)
db.session.commit()
def test_add_page_info_to_page(self, requests_get, requests_head):
url = "http://www.example.com/foo"
text = '<div>hi world</div>'
content_type = 'text/html'
headers = {
'content-type': '{}; charset=utf-8'.format(content_type),
}
requests_head.return_value = MagicMock(status_code=200, headers=headers)
requests_get.return_value = MagicMock(status_code=200, text=text)
page = Page.create(url=url, content='', status_code=0)
add_page_info_to_page(page)
self.assertEqual(page.content, text)
self.assertEqual(page.content_type, content_type)
self.assertEqual(requests_head.call_count, 1)
self.assertEqual(requests_get.call_count, 1)
def test_add_page_info_to_page(self, requests_get, requests_head):
url = "http://www.example.com/foo"
text = '<div>hi world</div>'
content_type = 'text/html'
headers = {
'content-type': '{}; charset=utf-8'.format(content_type),
}
requests_head.return_value = MagicMock(status_code=200,
headers=headers)
requests_get.return_value = MagicMock(status_code=200, text=text)
page = Page.create(url=url, content='', status_code=0)
add_page_info_to_page(page)
self.assertEqual(page.content, text)
self.assertEqual(page.content_type, content_type)
self.assertEqual(requests_head.call_count, 1)
self.assertEqual(requests_get.call_count, 1)
def get_resources(module_name=None, request_path=None):
# Check if the module and path are set.
if module_name is None:
module_name = request.blueprint
if request_path is None:
request_path = request.path
# Empty resources for all resources that are not retrieved.
page = None
page_id = None
activity = None
activity_id = None
path = None
# Check which type of seo fields should be retrieved, based on
# the module name.
if module_name == "activity":
# Regex search for acitivity id
activity_result = re.search(r"\/([0-9]+)\/", request_path)
# Fetch id from regex
activity_id = activity_result.group(1)
# Find activity
activity = Activity.query.filter(Activity.id == activity_id).first()
print("lalalala")
elif module_name == "page":
# Retrieve the page for its id
path = request_path[1:]
page = Page.get_by_path(path)
# Retrieve the revision by page id
if page is not None:
page_id = page.id
print("lalala")
else:
# Retrieve seo fields based on the module name.
path = module_name
return {"page": page, "page_id": page_id, "activity": activity, "activity_id": activity_id, "url": path}
def test_task(self):
self.update_state(state='PENDING',
meta={'status': "Beginning loop"})
total = 20
for i in range(total):
logging.info("on loop {}".format(i+1))
status = "Working through task {} of {}".format(i + 1, total)
db.session.add(Page(listing_id=1, html="test"))
db.session.commit()
sleeptime=4
for remaining in range(sleeptime, 0, -1):
self.update_state(state='PROGRESS',
meta={'current': i + 1, 'total': total, 'successes': i + 1,
'failures': 0, 'sleeptime': remaining, 'status': status})
sleep(1)
self.update_state(state='SUCCESS')
return {'current': total, 'total': total, 'successes': 5, 'failures': 0,
'status': 'Completed'}
def new_page():
form = NewPageForm()
if form.validate_on_submit():
data = {
"title": form.title.data,
"body": form.bodyhtml.data,
"category": form.category.data,
"divider_below": form.divider_below.data,
"index": form.index.data,
"name": form.name
}
newpage = Page(**data)
db.session.add(newpage)
db.session.commit()
time.sleep(0.5)
return redirect("/page/" + form.name)
return utils.render_with_navbar("page/form.html", form=form)
def add_page():
"""
Returns:
"""
form = PageEditorForm()
page = Page()
page.init_default()
page.save()
return render_template("content/pages/add_page.html",
form=form,
form_object=page)
def create():
"""Page creating"""
pages = Page.query.all()
if request.method == 'POST':
page = Page()
page.title = request.form['title']
page.source = request.form['source']
page.private = 'private' in request.form
page.user_id = current_user.id
page.parent_id = request.form['parent_id'] if request.form[
'parent_id'] else None
db.session.add(page)
db.session.commit()
flash('Page "%s" successfully created' % page.title, 'success')
return redirect(url_for('backend_page.view', page_id=page.id))
return render_template('page/create.j2', pages=pages)
def get_seo(self, module_name=None, request_path=None):
# Check if the module and path are set.
if module_name is None:
module_name = request.blueprint
if request_path is None:
request_path = request.path
# Check which type of seo fields should be retrieved, based on
# the module name.
if module_name == "activity":
# Get activity id
activity_id = re.search(r"\/([0-9]+)\/", request_path)
# Check seo existance
if activity_id is not None:
# Get the seo object of an activity
return SEO.get_by_activity(activity_id.group(1))
else:
# No seo was found for this activity
return None
elif module_name == "page":
# Retrieve the page for its id
path = request_path[1:]
page = Page.get_by_path(path)
# Retrieve the revision by page id
if page is not None:
return SEO.get_by_page(page.id)
else:
return None
else:
# Retrieve seo fields based on the module name.
seo = SEO.get_by_url(module_name)
return seo
return None
def pages(workspace_id):
'''
For GET requests, return all pages for the given workspace.
For POST requests, add a new pages to the given workspace.
'''
if not validate_workspace(workspace_id):
return 'workspace does not exist', 404
# request is a GET
if request.method == 'GET':
all_pages = Page.query.filter_by(workspace_id=workspace_id).order_by(
Page.updated_at.desc()).all()
schema = PageSchema(many=True, strict=True)
page_data = schema.dump(all_pages)
return jsonify(page_data[0])
# request is a POST
elif request.method == 'POST':
name = request.form.get('Name')
html = request.form.get('HTML').encode()
url = request.form.get('URL')
page = Page.query.filter_by(name=name).first()
if page is not None:
return json.dumps({'success': False}), 200, {
'ContentType': 'application/json'
}
page = Page(name=name, html=html, workspace_id=workspace_id, url=url)
db.session.add(page)
update_workspace_ts(Workspace.query.filter_by(id=workspace_id).first())
db.session.commit()
schema = PageSchema(strict=True)
page_data = schema.dump(page)
return jsonify(page_data[0]), 201
def get_page_history(path=''):
form = HistoryPageForm(request.form)
page = Page.get_by_path(path)
if not page:
return abort(404)
if not PageAPI.can_write(page):
return abort(403)
revisions = page.revision_cls.get_query()\
.filter(page.revision_cls.page_id == page.id)\
.all()
form.previous.choices = [(revision.id, '') for revision in revisions]
form.current.choices = [(revision.id, '') for revision in revisions]
if form.validate_on_submit():
previous = request.form['previous']
current = request.form['current']
previous_revision = page.revision_cls.get_query()\
.filter(page.revision_cls.id == previous).first()
current_revision = page.revision_cls.get_query()\
.filter(page.revision_cls.id == current).first()
prev = previous_revision.get_comparable()
cur = current_revision.get_comparable()
diff = htmldiff(prev, cur)
return render_template('page/compare_page_history.htm', diff=diff)
return render_template('page/get_page_history.htm', form=form,
revisions=zip(revisions, form.previous,
form.current))
def home():
Page.set_nav()
page = Page.query.filter_by(path='/home', published=True).first()
if page:
return render_template(f'page/{page.template}.html', page=page)
return render_template('home.html', page='page')
def latest(path):
Page.set_nav()
path = f"/{path}"
page = Page.query.filter_by(path=path).first()
return redirect(url_for('page.index', path=page.latest().path))
def edit_page(path=''):
if not ModuleAPI.can_write('page'):
return abort(403)
page = Page.get_by_path(path)
form = request.form
if page:
revision = page.get_latest_revision()
# Add the `needs_paid` option to the revision, so it will be inside
# the form.
revision.needs_paid = revision.page.needs_paid
form = PageForm(form, revision)
else:
form = PageForm()
groups = Group.query.all()
# on page submit (edit or create)
if form.validate_on_submit():
# if there was no page we want to create an entire new page (and not
# just a revision)
if not page:
page = Page(path)
page.needs_paid = form['needs_paid'].data
db.session.add(page)
db.session.commit()
custom_form_id = int(form.custom_form_id.data)
if custom_form_id == 0:
custom_form_id = None
new_revision = PageRevision(page,
form.nl_title.data.strip(),
form.en_title.data.strip(),
form.comment.data.strip(),
current_user,
form.nl_content.data.strip(),
form.en_content.data.strip(),
'filter_html' in form,
custom_form_id)
db.session.add(new_revision)
db.session.commit()
# Enter permission in db
for form_entry, group in zip(form.permissions, groups):
permission_entry = PagePermission.query\
.filter(PagePermission.group_id == group.id,
PagePermission.page_id == page.id).first()
permission_level = form_entry.select.data
if permission_entry:
permission_entry.permission = permission_level
else:
permission_entry = PagePermission(group.id, page.id,
permission_level)
db.session.add(permission_entry)
db.session.commit()
flash(_('The page has been saved'), 'success')
# redirect newly created page
return redirect(url_for('page.get_page', path=path))
else:
flash_form_errors(form)
for group in groups:
permission = None
if page:
permission = PagePermission.query\
.filter(PagePermission.group_id == group.id,
PagePermission.page_id == page.id)\
.first()
if permission:
form.permissions\
.append_entry({'select': permission.permission})
else:
form.permissions.append_entry({})
return render_template('page/edit_page.htm', page=page, form=form,
path=path, groups=zip(groups, form.permissions))
def create_mock_data(market_id):
form = CreateMockDataForm()
market = Market.query.filter_by(id=market_id).first()
if form.validate_on_submit():
if not market:
flash("market not found")
return redirect(url_for('main.index'))
total_pages = form.number_of_cases.data
min_price = form.min_price.data
max_price = form.max_price.data
start_date = form.start_date.data
end_date = form.end_date.data
# sellers = list(range(1, 10)) if form.seller.data else []
# origins = Country.query.all() if form.origin.data else []
listings = market.listings
pages_per_listing = int(total_pages / listings.count())
spread = int(pages_per_listing * 0.4)
spread = 1 if spread == 0 else spread
base = pages_per_listing - spread
base = 0 if base < 0 else base
listings_and_pages = []
for listing in listings:
pages = randint(base, pages_per_listing + spread)
listings_and_pages.append([listing, pages])
pre_total = sum([lp[1] for lp in listings_and_pages])
difference = total_pages - pre_total
for i in range(abs(difference)):
if difference > 0:
listings_and_pages[randint(0, len(listings_and_pages)) -
1][1] += 1
else:
idx = randint(0, len(listings_and_pages)) - 1
while True:
if listings_and_pages[idx][1] > 0:
listings_and_pages[idx][1] -= 1
break
else:
idx += 1 if idx < len(listings_and_pages) - 1 else 0
pages = []
for lp in listings_and_pages:
for i in range(lp[1]):
delta = end_date - start_date
int_delta = (delta.days * 24 * 60 * 60) + delta.seconds
random_delta = randrange(int_delta)
date = start_date + timedelta(seconds=random_delta)
price = np.random.randint(min_price, max_price)
page = Page(name="mock data sample",
listing=lp[0],
html="mock data sample",
timestamp=date,
price=price)
pages.append(page)
db.session.add_all(pages)
db.session.commit()
# # create one listing for each drug, or a couple if sellers are used
# for drug in drugs:
# rand_sellers = sample(population=sellers, k=randint(1, 3)) if sellers else None
# rand_origins = sample(population=origins, k=len(rand_sellers)) if origins else None
# while True:
# seller = rand_sellers.pop() if rand_sellers else ''
# origin = rand_origins.pop() if rand_origins else None
# if seller:
# url = "https://{}.com/mock_data{}/drug={}".format(
# market.name,
# "/seller={}".format(seller) if seller else '',
# drug)
# listings.append(Listing(market=market, drug=drug, url=url, seller=seller, origin=origin))
# if not rand_sellers:
# break
# db.session.add_all(listings)
# db.session.commit()
return redirect(url_for('main.index'))
return render_template('create_mock_data.html',
form=form,
market_name=market.name)
def sf4fefffdsf():
c = sqlite3.connect('app/rechem_listings.db')
countries = pd.read_sql_query("SELECT * FROM country", c)
new_countries = []
for i, row in countries.iterrows():
country = Country(name=row['name'], c2=row['c2'])
if not Country.query.filter_by(name=country.name).first():
new_countries.append(country)
db.session.add_all(new_countries)
db.session.commit()
drugs = pd.read_sql_query("SELECT * FROM drug", c)
new_drugs = []
for i, row in drugs.iterrows():
drug = Drug(name=row['name'])
if not Drug.query.filter_by(name=drug.name).first():
new_drugs.append(drug)
db.session.add_all(new_drugs)
db.session.commit()
markets = pd.read_sql_query("SELECT * FROM market", c)
new_markets = []
for i, row in markets.iterrows():
market = Market(name=row['name'])
if not Market.query.filter_by(name=market.name).first():
new_markets.append(market)
db.session.add_all(new_markets)
db.session.commit()
listings = pd.read_sql_query("SELECT * FROM listing", c)
rechem_listings = listings[listings['market_id'] == '4']
new_listings = []
for i, row in rechem_listings.iterrows():
market_name = markets[markets['id'] == int(
row['market_id'])]['name'].item()
new_market_id = Market.query.filter_by(name=market_name).first().id
drug_name = drugs[drugs['id'] == int(row['drug_id'])]['name'].item()
new_drug_id = Drug.query.filter_by(name=drug_name).first().id
# TODO: this is required for sqlite, but may or may not work with sqlalchemy
time_format = "%Y-%m-%d %H:%M:%S.%f"
timestamp = datetime.strptime(row['timestamp'], time_format)
listing = Listing(url=row['url'],
seller=None,
timestamp=timestamp,
market_id=new_market_id,
drug_id=new_drug_id,
origin_id=None)
if not Listing.query.filter_by(url=listing.url).first():
new_listings.append(listing)
db.session.add_all(new_listings)
db.session.commit()
# Get all pages with a listing id that is from the rechem market
pages = pd.read_sql_query("SELECT * FROM page", c)
rechem_pages = pages[pages['listing_id'].isin(rechem_listings['id'])]
new_pages = []
for i, row in rechem_pages.iterrows():
listing_url = listings[listings['id'] == int(
row['listing_id'])]['url'].item()
new_listing_id = Listing.query.filter_by(url=listing_url).first().id
# TODO: this is required for sqlite, but may or may not work with sqlalchemy
time_format = "%Y-%m-%d %H:%M:%S.%f"
timestamp = datetime.strptime(row['timestamp'], time_format)
page = Page(name=row['name'],
html=row['html'],
timestamp=timestamp,
listing_id=new_listing_id)
if not Page.query.filter_by(listing_id=page.listing_id,
timestamp=page.timestamp).first():
new_pages.append(page)
else:
print("page already found:")
db.session.add_all(new_pages)
db.session.commit()
return "data successfully added"
def image_upload():
"""
Returns:
"""
try:
if "file" in request.files:
file = request.files["file"]
if file and allowed_file(file.filename):
filename = secure_filename(file.filename)
id = escape(request.form["id"])
type = escape(request.form["type"])
module = escape(request.form["module"])
max_files = escape(request.form["max_files"])
max_size = escape(request.form["max_size"])
image_format = escape(request.form["image_format"])
"""
z.B: ein page objekt erzeugen die id setzen und laden
anhand von max_files prüfen ob eine liste benötigt wird
wenn ja bild hinzufügen. andernfalls mit image_format einfach nur dem wert im
dataitem setzen
"""
if module == "pages":
module_object = Page()
module_object.set_id(id)
module_object.load()
base_path = current_app.config["ROOT_DIR"] + "app/static/"
final_upload_path = base_path + type + "/" + module + "/" + id + "/"
if not os.path.isdir(final_upload_path):
os.makedirs(final_upload_path)
final_path = os.path.join(final_upload_path, filename)
if int(max_files) > 1:
current_data = module_object.get_list_or_dict(image_format)
if len(current_data) < max_files:
module_object.add(image_format, final_path)
else:
return make_response(400)
else:
module_object.set(image_format, final_path)
file.save(final_path)
module_object.save()
return filename
return make_response(500)
except Exception as error:
print(error)
def edit_page(id):
"""
Returns:
"""
form = PageEditorForm()
form.id = id
page = Page()
page_element = PageElement()
form.parent_id.choices = page.get_id_label_list()
page_elements = list()
if id > 0:
page.set_id(id)
page.load()
page_elements = page_element.get_list_for_page(page.get_id())
if request.method == "POST":
if form.validate_on_submit():
page.prepare_form_input(request.form)
page.save()
page.load()
else:
form.get_error_messages()
form.init_values(page)
return render_template("content/pages/edit_page.html",
form=form,
page_elements=page_elements)
def edit_committee(committee=''):
if not ModuleAPI.can_write('committee'):
return abort(403)
path = 'commissie/' + committee
page = Page.get_by_path(path)
form = request.form
if page:
revision = page.get_latest_revision()
form = CommitteeForm(form, revision)
else:
revision = None
form = CommitteeForm()
try:
url_group_id = int(request.args.get('group_id', None))
except:
url_group_id = None
form.group_id.choices = [(group.id, group.name) for group in
Group.query.order_by(Group.name).all()]
if len(request.form) == 0:
if revision:
selected_group_id = revision.group_id
elif url_group_id is not None:
selected_group_id = url_group_id
else:
selected_group_id = form.group_id.choices[0][0]
else:
selected_group_id = int(form.group_id.data)
form.group_id.data = selected_group_id
selected_group = Group.query.get(selected_group_id)
form.coordinator_id.choices = [
(user.id, user.name) for user in
selected_group.users.order_by(User.first_name, User.last_name).all()]
form.nl_title.data = selected_group.name
if form.validate_on_submit():
committee_nl_title = form.nl_title.data.strip()
committee_en_title = form.en_title.data.strip()
if not page:
root_entry_url = url_for('committee.list').rstrip('/')
root_entry = NavigationEntry.query\
.filter(NavigationEntry.url == root_entry_url)\
.first()
# Check whether the root navigation entry exists.
if not root_entry:
last_root_entry = NavigationEntry.query\
.filter(NavigationEntry.parent_id == None)\
.order_by(NavigationEntry.position.desc()).first() # noqa
root_entry_position = 1
if last_root_entry:
root_entry_position = last_root_entry.position + 1
root_entry = NavigationEntry(
None, 'Commissies', 'Committees', root_entry_url, False,
False, root_entry_position)
db.session.add(root_entry)
db.session.commit()
page = Page(path, 'committee')
# Never needs paid.
page.needs_paid = False
# Create a navigation entry for the new committee.
last_navigation_entry = NavigationEntry.query\
.filter(NavigationEntry.parent_id == root_entry.id)\
.first()
entry_position = 1
if last_navigation_entry:
entry_position = last_navigation_entry.position + 1
navigation_entry = NavigationEntry(
root_entry, committee_nl_title, committee_en_title, '/' + path,
False, False, entry_position)
db.session.add(navigation_entry)
db.session.commit()
# Sort these navigation entries.
NavigationAPI.alphabeticalize(root_entry)
# Assign the navigation entry to the new page (committee).
page.navigation_entry_id = navigation_entry.id
db.session.add(page)
db.session.commit()
# Assign read rights to all, and edit rights to BC.
all_group = Group.query.filter(Group.name == 'all').first()
bc_group = Group.query.filter(Group.name == 'BC').first()
all_entry = PagePermission(all_group.id, page.id, 1)
bc_entry = PagePermission(bc_group.id, page.id, 2)
db.session.add(all_entry)
db.session.add(bc_entry)
db.session.commit()
else:
# If the committee's title has changed, the navigation needs to be
# updated. Look for the entry, compare the titles, and change where
# necessary.
entry = NavigationEntry.query\
.filter(NavigationEntry.url == '/' + path).first()
if entry.title != committee_nl_title:
entry.title = committee_nl_title
db.session.add(entry)
db.session.commit()
group_id = int(form.group_id.data)
coordinator_id = int(form.coordinator_id.data)
# Add coordinator to BC
bc_group = Group.query.filter(Group.name == "BC").first()
if bc_group is not None:
new_coordinator = User.query.filter(
User.id == coordinator_id).first()
bc_group.add_user(new_coordinator)
new_revision = CommitteeRevision(
page, committee_nl_title, committee_en_title,
form.comment.data.strip(), current_user.id,
form.nl_description.data.strip(), form.en_description.data.strip(),
group_id, coordinator_id, form.interim.data)
db.session.add(new_revision)
db.session.commit()
flash(_('The committee has been saved.'), 'success')
return redirect(url_for('page.get_page', path=path))
else:
flash_form_errors(form)
return render_template('committee/edit.htm', page=page,
form=form, path=path)
def install():
objects = []
objects += [User.query.filter_by(username='******').first()]
objects += [Page.query.filter_by(slug='home').first()]
objects += [Page.query.filter_by(slug='admin').first()]
objects += [Page.query.filter_by(slug='search').first()]
objects += [Page.query.filter_by(slug='shop').first()]
objects += [Page.query.filter_by(slug='subscriber-welcome').first()]
objects += [Page.query.filter_by(slug='purchase-thank-you').first()]
if any(objects):
return print(
'Flask writer was previously installed. Command not run. Create a new database and run \'flask db upgrade\' to run this command.'
)
user = User(username='******')
user.set_password('password')
db.session.add(user)
db.session.commit()
home_page = Page(title='Home',
slug='home',
template='page',
published=True,
user_id=user.id)
home_page.set_path()
db.session.add(home_page)
admin_page = Page(title='Admin',
slug='admin',
template='page',
published=False,
user_id=user.id)
admin_page.set_path()
db.session.add(admin_page)
search_page = Page(title='Search',
slug='search',
template='page',
user_id=user.id,
published=False)
search_page.set_path()
db.session.add(search_page)
shop_page = Page(title='Shop',
slug='shop',
template='page',
user_id=user.id,
published=False)
shop_page.set_path()
db.session.add(shop_page)
error_page = Page(title='404 Error',
slug='404-error',
template='page',
user_id=user.id,
published=False)
error_page.set_path()
db.session.add(error_page)
sub_email = Page(title='Subscription Confirmation',
slug='subscriber-welcome',
template='page',
user_id=user.id,
published=False)
sub_email.set_path()
db.session.add(sub_email)
product_email = Page(title='eBook Delivery',
slug='purchase-thank-you',
template='page',
user_id=user.id,
published=False)
product_email.set_path()
db.session.add(product_email)
db.session.commit()
return print('Insttalled!')
def get_menu_pages():
pages = Page.objects(in_top_menu=True, published=True).only("url", "label")
return pages
def sizeof_fmt(num, suffix='B'):
""" print formatted file size
http://stackoverflow.com/a/1094933
"""
for unit in ['', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi']:
if abs(num) < 1024.0:
return "%3.1f%s%s" % (num, unit, suffix)
num /= 1024.0
return "%.1f%s%s" % (num, 'Yi', suffix)
if __name__ == "__main__":
initialize('corpus.db')
page_count = Page.select().count()
crawled_count = Page.select().where(
(Page.status_code == 200) &
((Page.content_type == 'text/html') |
(Page.content_type == 'text/plain')))\
.count()
redirect_count = Page.select().where(Page.status_code == 301).count()
to_crawl_count = Page.select().where(Page.status_code == 0).count()
other_count = page_count - crawled_count - redirect_count - to_crawl_count
link_count = Link.select().count()
corpus_size = os.stat('corpus.db').st_size
corpus_size = sizeof_fmt(corpus_size)
print('crawled pages: {}'.format(crawled_count))
def test_page(self):
now = datetime.datetime.utcnow()
page = Page.create(url='http://www.example.com/foo', content='hi world', status_code=200)
self.assertTrue(page.first_visited > now)
def get(self, slug):
return Page.where("slug", "=", slug.replace("--", "/")).first_or_fail()
def get_menu_pages():
pages = Page.objects(in_top_menu=True, published=True).only("url", "label")
return pages
def set_nav():
Page.set_nav()
def delete_children(node):
for child in node.children:
PageAdminView.delete_children(child)
Page.objects(pk=node.value.id).delete()
def initdb():
# fix for CASCADE not work issue
db.session.execute('PRAGMA foreign_keys=ON;')
try:
db.create_all()
except:
db.drop_all()
# add some records
try:
# add some users
admin = createUser(u'admin', u'111111', u'*****@*****.**',
300)
simon = createUser(u'simon', u'111111', u'*****@*****.**', 300)
ray = createUser(u'ray', u'111111', u'*****@*****.**')
yxm0513 = createUser(u'yxm0513', u'111111', u'*****@*****.**', 200)
gonglidi = createUser(u'gonglidi', u'111111', u'*****@*****.**')
infoprint(u"create users table successfully.", 'successfully')
# add some tags
# search function
search_url = 'http://api.douban.com/book/subjects?tag=python&max-results=200'
books = parse(search_url)
link = ''
ID = []
for e in books['entries']:
if hasattr(e, 'link'):
link = e.link
p = re.compile(r'(?P<id>\d+)')
r = p.search(link)
id = r.group("id")
ID.append(id)
# add some book info from douban
#ID = range(11505944, 11505960)
for id in ID:
book = parse(setting.DOUBAN_API_PATH + '/' + str(id))
title = u''
summary = u''
tags = u''
for e in book['entries']:
if hasattr(e, 'title'):
title = e.title
if hasattr(e, 'summary'):
summary = e.summary
if hasattr(e, 'db_tag'):
if 'python' in e.db_tag.get('name').lower():
tags = e.db_tag.get('name')
else:
tags = e.db_tag.get('name') + ',python'
post = createPost(simon, title, summary, tags=tags)
infoprint(u"create posts table successfully.", 'successfully')
# add some page info
rules_source = u"""
免责条款
=================================
:作者: simon.yang.sh <at> gamil <dot> com
:版本: 0.1 of 2012/07
关于提交内容
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
* 如果你觉得有什么破坏条款的请 联系我们
* 如果你愿意监督我们的内容，联系我们吧。
"""
rules = Page(u"免责条款", rules_source)
rules.save()
about_source = u"""
===================================
关于我们
===================================
我们想做的是什么呢，简单的说就是
*用 digg的方式给用户提供一个读书的榜单.*
缘由：
通常我们会在很多论坛看见很多新手问类似于，“我想学python,各位大侠有没有好的书推荐”， 我们为什么不用digg的方式来给我们构建这么的榜单呢，让大家的投票说话。
我们有什么不一样：
* 我们只做过滤工作，给用户需要的信息 -- 做信息的减法
* 豆瓣也有书，但不是一个榜单。
* 我们很社会化，可构建圈子，评论也需要言简意赅，字数被限制在140个字符
* 未来期望构建知识的架构图，树形的展示知识的架构，给学习者直观的知道，学习者可以由浅入深的研究自己希望的课题。
"""
about = Page(u"关于我们", about_source)
about.save()
api_source = u"""
API
===================================
* Coming Soon!!!
"""
api = Page(u"API", api_source)
api.save()
help_source = u"""
帮助
===================================
* Coming Soon!!!
"""
helppage = Page(u"帮助", help_source)
helppage.save()
infoprint(u"create pages table successfully.", 'successfully')
except Exception as e:
infoprint(u"create tables failed: %s" % e, 'error')
cate1 = Category(name=u"计算机")
cate2 = Category(name=u"编程", parent=cate1)
cate1.save()
cate2.save()
infoprint(u"create categories table successfully.", 'successfully')
def initdb():
# fix for CASCADE not work issue
db.session.execute('PRAGMA foreign_keys=ON;')
try:
db.create_all()
except:
db.drop_all()
# add some records
try:
# add some users
admin = createUser(u'admin', u'111111', u'*****@*****.**', 300)
simon = createUser(u'simon', u'111111', u'*****@*****.**', 300)
ray = createUser(u'ray', u'111111', u'*****@*****.**')
yxm0513 = createUser(u'yxm0513', u'111111', u'*****@*****.**', 200)
gonglidi = createUser(u'gonglidi', u'111111', u'*****@*****.**')
infoprint(u"create users table successfully.", 'successfully')
# add some tags
# search function
search_url = 'http://api.douban.com/book/subjects?tag=python&max-results=200'
books = parse(search_url)
link = ''
ID = []
for e in books['entries']:
if hasattr(e, 'link'):
link = e.link
p = re.compile(r'(?P<id>\d+)')
r = p.search(link)
id = r.group("id")
ID.append(id)
# add some book info from douban
#ID = range(11505944, 11505960)
for id in ID:
book = parse(setting.DOUBAN_API_PATH + '/' + str(id))
title = u''
summary = u''
tags = u''
for e in book['entries']:
if hasattr(e, 'title'):
title = e.title
if hasattr(e, 'summary'):
summary = e.summary
if hasattr(e, 'db_tag'):
if 'python' in e.db_tag.get('name').lower():
tags = e.db_tag.get('name')
else:
tags = e.db_tag.get('name') + ',python'
post = createPost(simon, title, summary, tags=tags)
infoprint(u"create posts table successfully.", 'successfully')
# add some page info
rules_source = u"""
免责条款
=================================
:作者: simon.yang.sh <at> gamil <dot> com
:版本: 0.1 of 2012/07
关于提交内容
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
* 如果你觉得有什么破坏条款的请 联系我们
* 如果你愿意监督我们的内容，联系我们吧。
"""
rules = Page(u"免责条款", rules_source)
rules.save()
about_source = u"""
===================================
关于我们
===================================
我们想做的是什么呢，简单的说就是
*用 digg的方式给用户提供一个读书的榜单.*
缘由：
通常我们会在很多论坛看见很多新手问类似于，“我想学python,各位大侠有没有好的书推荐”， 我们为什么不用digg的方式来给我们构建这么的榜单呢，让大家的投票说话。
我们有什么不一样：
* 我们只做过滤工作，给用户需要的信息 -- 做信息的减法
* 豆瓣也有书，但不是一个榜单。
* 我们很社会化，可构建圈子，评论也需要言简意赅，字数被限制在140个字符
* 未来期望构建知识的架构图，树形的展示知识的架构，给学习者直观的知道，学习者可以由浅入深的研究自己希望的课题。
"""
about = Page(u"关于我们", about_source)
about.save()
api_source = u"""
API
===================================
* Coming Soon!!!
"""
api = Page(u"API", api_source)
api.save()
help_source = u"""
帮助
===================================
* Coming Soon!!!
"""
helppage = Page(u"帮助", help_source)
helppage.save()
infoprint(u"create pages table successfully.", 'successfully')
except Exception as e:
infoprint(u"create tables failed: %s" % e, 'error')
cate1 = Category(name=u"计算机")
cate2 = Category(name=u"编程", parent=cate1)
cate1.save()
cate2.save()
infoprint(u"create categories table successfully.", 'successfully')
def make_dummy_seed():
from app.models import Creation, Page, Component, Design
with app.app_context():
design = Design.query.filter_by(name='design1').first() or Design(
name='design1')
creation = Creation.query.filter_by(
domain='creator').first() or Creation()
creation.name = 'Dummy'
creation.domain = 'creator'
creation.active = True
creation.design = design
creation.menu = {
'position':
'top',
'links': [
{
'href': '/page1',
'name': 'Media'
},
{
'href': '/page2',
'name': 'Poll'
},
{
'href': '/page3',
'name': 'My Page'
},
]
}
db.session.add(creation)
pages = (('page1', (
{
'name': 'TOP',
'type': 'media',
'config': {
'length': 5
}
},
{
'name': 'Twitter',
'type': 'twitter',
'config': {
'account_id': 403614288
}
},
)), ('page2', ({
'name': 'POLL Z',
'type': 'poll',
'config': {
'id_poll': 123
}
}, )), ('page3', ({
'name': 'History',
'type': 'html',
'config': {
'file': 'custom_html_27.html'
}
}, )))
for p, components in pages:
page = Page.query.filter_by(
creation=creation, name=p).first() or Page(name=p,
creation=creation)
db.session.add(page)
for component in components:
c = Component.query.filter_by(
name=component['name'], page=page,
type=component['type']).first() or Component()
c.page = page
c.name = component['name']
c.type = component['type']
c.config = component['config']
db.session.add(c)
db.session.commit()
def edit_page(id, ver_id=None):
page = Page.query.filter_by(id=id).first()
was_published = page.published
print(f"ANCESTORS: {page.ancestors()}")
for anc in page.ancestors():
print(f"ANCESTOR: {anc}")
form = AddPageForm()
form.parent_id.choices = [(0, '---')] + [(p.id, f"{p.title} ({p.path})")
for p in Page.query.all()]
form.user_id.choices = [(u.id, u.username) for u in User.query.all()]
form.notify_group.choices = [
('', ''), ('all', 'All')
] + Subscriber.SUBSCRIPTION_CHOICES + [('discord', 'Discord Only')]
for field in form:
print(f"{field.name}: {field.data}")
if form.validate_on_submit():
prev_parentid = page.parent_id if page.parent_id else None
# Create version from current
version = PageVersion(
original_id=id,
title=page.title,
slug=page.slug,
template=page.template,
parent_id=prev_parentid,
banner=page.banner,
body=page.body,
notes=page.notes,
summary=page.summary,
author_note=page.author_note,
author_note_location=page.author_note_location,
sidebar=page.sidebar,
tags=page.tags,
user_id=page.user_id,
notify_group=page.notify_group,
pub_date=page.pub_date,
published=page.published,
path=page.path,
dir_path=page.dir_path,
)
db.session.add(version)
# Update page
log_orig = log_change(page)
parentid = form.parent_id.data if form.parent_id.data else None
page.title = form.title.data
page.slug = form.slug.data
page.template = form.template.data
page.parent_id = parentid
page.banner = form.banner.data
page.body = form.body.data
page.notes = form.notes.data
page.summary = form.summary.data
page.author_note = form.author_note.data
page.author_note_location = form.author_note_location.data
page.sidebar = form.sidebar.data
page.tags = form.tags.data
page.user_id = form.user_id.data
page.notify_group = form.notify_group.data
page.published = form.published.data
page.edit_date = datetime.utcnow()
pdate = form.pub_date.data
ptime = form.pub_time.data
local_tz = form.timezone.data if form.timezone.data else current_user.timezone
if pdate and ptime:
page.set_local_pub_date(f"{pdate} {ptime}", local_tz)
else:
page.pub_date = None
page.set_path()
log_change(log_orig, page, 'edited a page')
db.session.commit()
if form.notify_subs.data:
current_app.logger.debug(form.notify_group.data)
page.notify_subscribers(form.notify_group.data)
flash("Page updated successfully.", "success")
Page.set_nav()
return redirect(url_for('admin.edit_page', id=id))
if form.errors:
flash("<b>Error!</b> Please fix the errors below.", "danger")
versions = PageVersion.query.filter_by(original_id=id).order_by(
desc('edit_date')).all()
version = PageVersion.query.filter_by(
id=ver_id).first() if ver_id else None
if version:
form.title.data = version.title
form.slug.data = version.slug
form.template.data = version.template
form.parent_id.data = version.parent_id
form.banner.data = version.banner
form.body.data = version.body
form.notes.data = version.notes
form.summary.data = version.summary
form.author_note.data = version.author_note
form.author_note_location.data = version.author_note_location
form.sidebar.data = version.sidebar
form.tags.data = version.tags
form.user_id.data = version.user_id
form.notify_group.data = version.notify_group
form.pub_date.data = version.local_pub_date(current_user.timezone)
form.pub_time.data = version.local_pub_date(current_user.timezone)
form.published.data = version.published
else:
form.title.data = page.title
form.slug.data = page.slug
form.template.data = page.template
form.parent_id.data = page.parent_id
form.banner.data = page.banner
form.body.data = page.body
form.notes.data = page.notes
form.summary.data = page.summary
form.author_note.data = page.author_note
form.author_note_location.data = page.author_note_location
form.sidebar.data = page.sidebar
form.tags.data = page.tags
form.user_id.data = page.user_id
form.notify_group.data = page.notify_group
form.pub_date.data = page.local_pub_date(current_user.timezone)
form.pub_time.data = page.local_pub_date(current_user.timezone)
form.published.data = page.published
return render_template('admin/page-edit.html',
form=form,
tab='pages',
action='Edit',
edit_page=page,
versions=versions,
version=version,
page=Page.query.filter_by(slug='admin').first())
def sizeof_fmt(num, suffix='B'):
""" print formatted file size
http://stackoverflow.com/a/1094933
"""
for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
if abs(num) < 1024.0:
return "%3.1f%s%s" % (num, unit, suffix)
num /= 1024.0
return "%.1f%s%s" % (num, 'Yi', suffix)
if __name__ == "__main__":
initialize('corpus.db')
page_count = Page.select().count()
crawled_count = Page.select().where(
(Page.status_code == 200) &
((Page.content_type == 'text/html') |
(Page.content_type == 'text/plain')))\
.count()
redirect_count = Page.select().where(Page.status_code == 301).count()
to_crawl_count = Page.select().where(Page.status_code == 0).count()
other_count = page_count - crawled_count - redirect_count - to_crawl_count
link_count = Link.select().count()
corpus_size = os.stat('corpus.db').st_size
corpus_size = sizeof_fmt(corpus_size)
print('crawled pages: {}'.format(crawled_count))
