def get(self, slug):
slug = slug.rstrip("/")
guest_access_token = self.get_argument("guest-access-token", None)
postdoc = yield self.settings["db"].posts.find_one({"slug": slug}, {"summary": False, "original": False})
if not postdoc:
raise tornado.web.HTTPError(404)
post = Post(**postdoc)
if post.status == "publish":
# Not a draft any more
self.redirect(self.reverse_url("post", slug))
return
# If an access token is provided, it must be valid. Otherwise,
# administrator must be logged in.
if guest_access_token:
if not post.has_guest_access_token(ObjectId(guest_access_token)):
raise tornado.web.HTTPError(401)
elif not self.current_user:
# Redirect to login page. Return here after login.
next_url = self.request.uri
url = self.get_login_url() + "?" + urlencode({"next": next_url})
self.redirect(url)
return
categories = yield self.get_categories()
self.render("draft.jade", post=post, prev=None, next=None, categories=categories)
def get(self, slug):
slug = slug.rstrip('/')
guest_access_token = self.get_argument('guest-access-token', None)
postdoc = yield self.settings['db'].posts.find_one(
{'slug': slug},
{'summary': False, 'original': False})
if not postdoc:
raise tornado.web.HTTPError(404)
post = Post(**postdoc)
# If an access token is provided, it must be valid. Otherwise,
# administrator must be logged in.
if guest_access_token:
if not post.has_guest_access_token(ObjectId(guest_access_token)):
raise tornado.web.HTTPError(401)
elif not self.current_user:
raise tornado.web.HTTPError(401)
if post.status == 'publish':
# Not a draft any more
self.redirect(self.reverse_url('post', slug))
return
category_docs = yield self.get_categories()
categories = [Category(**doc) for doc in category_docs]
self.render(
'draft.jade',
post=post, prev=None, next=None, categories=categories)
def get(self, slug):
slug = slug.rstrip('/')
guest_access_token = self.get_argument('guest-access-token', None)
postdoc = yield self.settings['db'].posts.find_one({'slug': slug}, {
'summary': False,
'original': False
})
if not postdoc:
raise tornado.web.HTTPError(404)
post = Post(**postdoc)
# If an access token is provided, it must be valid. Otherwise,
# administrator must be logged in.
if guest_access_token:
if not post.has_guest_access_token(ObjectId(guest_access_token)):
raise tornado.web.HTTPError(401)
elif not self.current_user:
raise tornado.web.HTTPError(401)
if post.status == 'publish':
# Not a draft any more
self.redirect(self.reverse_url('post', slug))
return
category_docs = yield self.get_categories()
categories = [Category(**doc) for doc in category_docs]
self.render('draft.jade',
post=post,
prev=None,
next=None,
categories=categories)
def _edit_post(self, postid, struct, post_type):
new_post = Post.from_metaweblog(struct, post_type, is_edit=True)
db = self.settings['db']
old_post_doc = yield db.posts.find_one(ObjectId(postid))
if not old_post_doc:
self.result(xmlrpclib.Fault(404, "Not found"))
else:
old_post = Post(**old_post_doc)
if not old_post.pub_date and new_post.status == 'publish':
new_post.pub_date = datetime.datetime.utcnow()
update_result = yield db.posts.update(
{'_id': old_post_doc['_id']},
{'$set': new_post.to_python()}) # set fields to new values
if update_result['n'] != 1:
self.result(xmlrpclib.Fault(404, "Not found"))
else:
# If link changes, add redirect from old
if (old_post.slug != new_post.slug
and old_post['status'] == 'publish'):
redirect_post = Post(
redirect=new_post.slug,
slug=old_post.slug,
status='publish',
type='redirect',
mod=datetime.datetime.utcnow())
yield db.posts.insert(redirect_post.to_python())
# Done
self.result(True)
def _get_post(self, postid):
postdoc = yield self.settings['db'].posts.find_one(ObjectId(postid))
if not postdoc:
self.result(xmlrpclib.Fault(404, "Not found"))
else:
post = Post(**postdoc)
self.result(post.to_metaweblog(self.application))
def got_post(postdoc, error):
if error:
self.result(xmlrpclib.Fault(500, str(error)))
elif not postdoc:
self.result(xmlrpclib.Fault(404, "Not found"))
else:
post = Post(**postdoc)
self.result(post.to_metaweblog(self.application))
def _get_post(self, postid):
postdoc = yield self.settings['db'].posts.find_one(ObjectId(postid))
if not postdoc:
self.result(xmlrpclib.Fault(404, "Not found"))
else:
post = Post(**postdoc)
self.result(post.to_metaweblog(self.application))
def got_post(postdoc, error):
if error:
raise error
if not postdoc:
self.result(xmlrpclib.Fault(404, "Not found"))
else:
post = Post(**postdoc)
self.result(post.to_metaweblog(self.application))
def get(self, slug):
slug = slug.rstrip('/')
posts = self.settings['db'].posts
post_doc = yield posts.find_one(
{'slug': slug, 'status': 'publish'},
{'summary': False, 'original': False})
if not post_doc:
raise tornado.web.HTTPError(404)
if post_doc['type'] == 'redirect':
# This redirect marks where a real post or page used to be.
url = self.reverse_url('post', post_doc['redirect'])
self.redirect(url, permanent=True)
return
post = Post(**post_doc)
# Only posts have prev / next navigation, not pages.
if post.type == 'post':
fields = {'summary': False, 'body': False, 'original': False}
prev_doc_future = posts.find_one({
'status': 'publish', 'type': 'post',
'pub_date': {'$lt': post.pub_date}
}, fields, sort=[('pub_date', -1)])
next_doc_future = posts.find_one({
'status': 'publish', 'type': 'post',
'pub_date': {'$gt': post.pub_date}
}, fields, sort=[('pub_date', 1)])
# Overkill for this case, but in theory we reduce latency by
# querying for previous and next posts at once, and waiting for
# both.
prev_doc, next_doc = yield [prev_doc_future, next_doc_future]
else:
prev_doc, next_doc = None, None
prev_post = Post(**prev_doc) if prev_doc else None
next_post = Post(**next_doc) if next_doc else None
categories = yield self.get_categories()
self.update_last_mod(post)
self.update_last_mod(prev_post)
self.update_last_mod(next_post)
self.update_last_mod_from_list(categories)
yield self.render_async(
'single.jade',
post=post,
prev=prev_post,
next=next_post,
categories=categories)
def got_recent_posts(posts, error):
if error:
raise error
self.result([
Post(**post).to_metaweblog(self.application) for post in posts
])
def _get(self, *args, **kwargs):
categorydocs = yield motor.Op(self.get_categories)
self.categories = categories = [Category(**doc) for doc in categorydocs]
postdocs = yield motor.Op(self.get_posts, *args, **kwargs)
self.posts = posts = [
Post(**doc) if doc else None
for doc in postdocs]
mod = max(
thing.last_modified
for things in (posts, categories)
for thing in things if thing)
if mod:
# If-Modified-Since header is only good to the second. Truncate
# our own mod-date to match its precision.
mod = mod.replace(microsecond=0)
self.set_header('Last-Modified', mod)
# Adapted from StaticFileHandler
ims_value = self.request.headers.get("If-Modified-Since")
if ims_value is not None:
date_tuple = email.utils.parsedate(ims_value)
if_since = models.utc_tz.localize(
datetime.datetime.fromtimestamp(time.mktime(date_tuple)))
if if_since >= mod:
# No change since client's last request. Tornado will take
# care of the rest.
self.set_status(304)
self.finish()
return
gen.engine(get)(self, *args, **kwargs)
def get(self):
categories = yield self.get_categories()
q = self.get_argument('q', None)
if q:
response = yield self.settings['db'].command('text',
'posts',
search=q,
filter={
'status':
'publish',
'type': 'post'
},
projection={
'original': False,
'plain': False
},
limit=50)
posts = [Post(**result['obj']) for result in response['results']]
else:
posts = []
yield self.render_async('search.jade',
q=q,
posts=posts,
categories=categories)
def get(self):
# TODO: refactor with check_last_modified(), this is gross
# we need an async version of RequestHandler.prepare()
category_docs = yield self.get_categories()
self.categories = [Category(**doc) for doc in category_docs]
q = self.get_argument('q', None)
if q:
response = yield self.db.command('text',
'posts',
search=q,
filter={
'status': 'publish',
'type': 'post'
},
projection={
'original': False,
'plain': False
},
limit=50)
posts = [Post(**result['obj']) for result in response['results']]
else:
posts = []
self.render('search.jade', q=q, posts=posts)
def _recent(self, num_posts, type):
cursor = self.settings['db'].posts.find({'type': type})
# _id starts with timestamp.
cursor.sort([('_id', -1)]).limit(num_posts)
posts = yield cursor.to_list(num_posts)
self.result(
[Post(**post).to_metaweblog(self.application) for post in posts])
def _new_post(self, user, password, struct, type):
new_post = Post.from_metaweblog(struct, type)
if new_post.status == 'publish':
new_post.pub_date = datetime.datetime.utcnow()
_id = yield self.settings['db'].posts.insert(new_post.to_python())
self.result(str(_id))
def _edit_post(self, postid, user, password, struct, type):
try:
self.new_post = Post.from_metaweblog(struct, type, is_edit=True)
self.settings['db'].posts.find_one({'_id': ObjectId(postid)},
callback=self._got_old_post)
except Exception as error:
logging.exception("Editing post")
self.result(xmlrpclib.Fault(500, str(error)))
def _new_post(self, user, password, struct, publish, type):
try:
new_post = Post.from_metaweblog(struct, type, publish=publish)
self.settings['db'].posts.insert(
new_post.to_python(),
callback=self._new_post_inserted)
except Exception as error:
logging.exception("Creating post")
self.result(xmlrpclib.Fault(500, str(error)))
def _new_post(self, user, password, struct, type):
new_post = Post.from_metaweblog(struct, type)
if new_post.status == 'publish':
new_post.pub_date = datetime.datetime.utcnow()
_id = yield self.settings['db'].posts.insert(new_post.to_python())
self.result(str(_id))
cache.event('post_created')
def _new_post(self, user, password, struct, publish, type):
def new_post_inserted(_id, error):
if error:
raise error
self.result(str(_id))
new_post = Post.from_metaweblog(struct, type, publish=publish)
self.settings['db'].posts.insert(new_post.to_python(),
callback=new_post_inserted)
def got_post(post, error):
if error:
raise error
if not post:
self.result(xmlrpclib.Fault(404, "Not found"))
else:
self.result([
cat.to_metaweblog(self.application)
for cat in Post(**post).categories
])
def mt_getPostCategories(self, postid, user, password):
post = yield self.settings['db'].posts.find_one(ObjectId(postid))
if not post:
self.result(xmlrpclib.Fault(404, "Not found"))
else:
self.result([
cat.to_metaweblog(self.application)
for cat in Post(**post).categories
])
def get(self):
# TODO: pagination
db = self.settings['db']
draftdocs = yield motor.Op(db.posts.find(
{'status': 'draft', 'type': 'post'},
{'original': False, 'body': False},
).sort([('_id', -1)]).to_list)
drafts = [Post(**draftdoc) for draftdoc in draftdocs]
self.render('admin-templates/drafts.html', drafts=drafts)
def _new_post(self, user, password, struct, publish, type):
def new_post_inserted(_id, error):
if error:
raise error
self.result(str(_id))
new_post = Post.from_metaweblog(struct, type, publish=publish)
self.settings['db'].posts.insert(
new_post.to_python(),
callback=new_post_inserted)
def _new_post(self, user, password, struct, type):
try:
new_post = Post.from_metaweblog(struct, type)
if new_post.status == 'publish':
new_post.pub_date = datetime.datetime.utcnow()
self.settings['db'].posts.insert(
new_post.to_python(),
callback=self._new_post_inserted)
except Exception as error:
logging.exception("Creating post")
self.result(xmlrpclib.Fault(500, str(error)))
def _edit_post(self, postid, user, password, struct, publish, type):
# TODO: if link changes, add redirect from old
def edited_post(result, error):
if result['n'] != 1:
self.result(xmlrpclib.Fault(404, "Not found"))
else:
self.result(True)
new_post = Post.from_metaweblog(struct, type, publish=publish, is_edit=True)
self.settings['db'].posts.update(
{'_id': ObjectId(postid)},
{'$set': new_post.to_python()}, # set fields to new values
callback=edited_post)
def _edited_post(self, result, error):
if error:
self.result(xmlrpclib.Fault(500, str(error)))
elif result['n'] != 1:
self.result(xmlrpclib.Fault(404, "Not found"))
else:
# If link changes, add redirect from old
if (self.old_post.slug != self.new_post.slug
and self.old_post['status'] == 'publish'
):
redirect_post = Post(
redirect=self.new_post.slug,
slug=self.old_post.slug,
status='publish',
type='redirect',
mod=datetime.datetime.utcnow())
self.settings['db'].posts.insert(
redirect_post.to_python(), callback=self._redirected)
else:
# Done
self.result(True)
def _edit_post(self, postid, struct, post_type):
new_post = Post.from_metaweblog(struct, post_type, is_edit=True)
db = self.settings['db']
old_post_doc = yield db.posts.find_one(ObjectId(postid))
if not old_post_doc:
self.result(xmlrpclib.Fault(404, "Not found"))
else:
old_post = Post(**old_post_doc)
if not old_post.pub_date and new_post.status == 'publish':
new_post.pub_date = datetime.datetime.utcnow()
# TODO: more general solution for fields that must be preserved.
new_post.guest_access_tokens = old_post.guest_access_tokens
update_result = yield db.posts.update(
{'_id': old_post_doc['_id']},
{'$set': new_post.to_python()}) # set fields to new values
if update_result['n'] != 1:
self.result(xmlrpclib.Fault(404, "Not found"))
else:
# If link changes, add redirect from old
if (old_post.slug != new_post.slug
and old_post['status'] == 'publish'):
redirect_post = Post(
redirect=new_post.slug,
slug=old_post.slug,
status='publish',
type='redirect',
mod=datetime.datetime.utcnow())
yield db.posts.insert(redirect_post.to_python())
# Done
self.result(True)
cache.event('post_changed')
def get(self, slug):
slug = slug.rstrip('/')
postdoc = yield motor.Op(
self.settings['db'].posts.find_one,
{'slug': slug},
{'summary': False, 'original': False})
if not postdoc:
raise tornado.web.HTTPError(404)
post=Post(**postdoc)
categories = yield motor.Op(get_categories, self.settings['db'])
self.render(
'single.html',
post=post, prev=None, next=None, categories=categories)
def _edit_post(self, postid, user, password, struct, publish, type):
# TODO: if link changes, add redirect from old
def edited_post(result, error):
if result['n'] != 1:
self.result(xmlrpclib.Fault(404, "Not found"))
else:
self.result(True)
new_post = Post.from_metaweblog(struct,
type,
publish=publish,
is_edit=True)
self.settings['db'].posts.update(
{'_id': ObjectId(postid)},
{'$set': new_post.to_python()}, # set fields to new values
callback=edited_post)
def recent_posts(handler, db, n, tag=None):
"""Show summaries of N most recent posts."""
limit = int(n)
query = {'status': 'publish', 'type': 'post'}
if tag:
query['tags'] = tag
cursor = db.posts.find(query, {'original': False})
docs = yield cursor.sort([('pub_date', -1)]).limit(limit).to_list(limit)
posts = [Post(**doc) for doc in docs]
modified = max(p.last_modified for p in posts) if posts else None
rv = cStringIO.StringIO()
rv.write('<ul class="post-list">')
for post in posts:
rv.write(handler.render_string('post-summary.jade', post=post))
rv.write('</ul>')
raise gen.Return((rv.getvalue(), modified))
def _get(self, *args, **kwargs):
category_docs = yield self.get_categories()
self.categories = categories = [
Category(**doc) for doc in category_docs]
post_docs = yield self.get_posts(*args, **kwargs)
if post_docs:
self.posts = posts = [
Post(**doc) if doc else None
for doc in post_docs]
else:
self.posts = posts = []
if posts or categories:
mod = max(
thing.last_modified
for things in (posts, categories)
for thing in things if thing)
# If-Modified-Since header is only good to the second. Truncate
# our own mod-date to match its precision.
mod = mod.replace(microsecond=0)
self.set_header('Last-Modified', mod)
# Adapted from StaticFileHandler
ims_value = self.request.headers.get("If-Modified-Since")
if ims_value is not None:
date_tuple = email.utils.parsedate(ims_value)
if_since = models.utc_tz.localize(
datetime.datetime.fromtimestamp(time.mktime(date_tuple)))
if if_since >= mod:
# No change since client's last request. Tornado will take
# care of the rest.
self.set_status(304)
self.finish()
return
# Yielding, and returning result, are unneeded. We're not waiting for
# a return value, we're waiting for get() to call finish(). But let's
# yield and return anyway for sanity's sake.
result = yield gen.coroutine(get)(self, *args, **kwargs)
raise gen.Return(result)
def on_message(self, json_message):
"""Receive messages from browsers viewing draft pages.
In draft.jade we include the draft's timestamp at the moment it was
rendered. When rendering completes, Javascript sends the timestamp
back to see if the draft was modified while it was rendering; if so
it reloads again.
This is surprisingly common. If a draft's categories change, MarsEdit
makes two API calls: first to edit the draft's body, then to set its
categories. The first call causes the draft to reload, and while it
reloads the second API call completes. To ensure the browser has loaded
the latest version we must check at the end of each page-load whether
we're still up to date.
# TODO: might reduce flickering by waiting a second to see if we'll
change again, before sending post_changed.
"""
try:
# The client tells us this page's last-modified date.
message = json.loads(json_message)
post_id = ObjectId(message['post_id'])
mod = message['mod']
db = self.session.handler.settings['db']
post_doc = yield db.posts.find_one({
'status': 'draft',
'_id': post_id
})
post = Post(**post_doc)
if str(post.last_modified) != mod:
# Post changed since we served the draft page; make it reload.
self.send('post_changed')
except Exception:
logging.exception('Processing message: %r' % json_message)
def main(args):
start = time.time()
opts = options.options()
destination_url = '/' + opts.base_url.lstrip('/')
parts = urlparse(args.source_url)
source_base_url = urljoin('%s://%s' % (parts[0], parts[1]),
parts[2].split('/xmlrpc.php')[0])
print 'Base URL', source_base_url
db = pymongo.Connection(safe=True).motorblog
motordb = motor.MotorClient().open_sync().motorblog
if args.wipe:
print 'Wiping motorblog database'
db.connection.drop_database('motorblog')
print 'Creating capped collection "events"'
create_events_collection(motordb)
print 'Recreating indexes'
ensure_indexes(db)
source = Blog(args.source_url,
args.source_username,
args.source_password,
use_cache=not args.refresh,
verbose=args.verbose)
print 'Getting media library'
media_library = set([m['link'] for m in source.get_media_library()])
print ' %s assets\n' % len(media_library)
print 'Getting posts and pages'
post_structs = source.get_recent_posts(args.nposts)
print ' %s posts' % len(post_structs)
page_structs = source.get_pages()
print ' %s pages' % len(page_structs)
print
for structs, post_type in [
(post_structs, 'post'),
(page_structs, 'page'),
]:
print '%sS' % post_type.upper()
for struct in structs:
categories = struct.pop('categories', [])
struct['description'] = wordpress_to_markdown(
struct, media_library, db, destination_url, source_base_url)
post = Post.from_metaweblog(struct, post_type)
print '%-34s %s' % (post.title, post.status.upper())
for category_name in categories:
doc = db.categories.find_one({'name': category_name})
if doc:
category = Category(**doc)
else:
category = Category(name=category_name,
slug=slugify(category_name))
category.id = db.categories.insert(category.to_python())
print ' %-30s %s' % (category_name,
' NEW' if not doc else '')
post.categories.append(category)
db.posts.insert(post.to_python())
print '\nFinished %s %ss' % (len(structs), post_type)
print 'Posting "categories_changed" event'
db.events.insert(
{
'ts': datetime.datetime.utcnow(),
'name': 'categories_changed'
},
manipulate=False) # No need to add _id
print '\nFinished in %.2f seconds' % (time.time() - start)
def main(args):
start = time.time()
opts = options.options()
destination_url = '/' + opts.base_url.lstrip('/')
parts = urlparse(args.source_url)
source_base_url = urljoin(
'%s://%s' % (parts[0], parts[1]), parts[2].split('/xmlrpc.php')[0])
print 'Base URL', source_base_url
db = pymongo.Connection(safe=True).motorblog
motordb = motor.MotorConnection().open_sync().motorblog
if args.wipe:
print 'Wiping motorblog database'
db.connection.drop_database('motorblog')
print 'Creating capped collection "events"'
create_events_collection(motordb)
print 'Recreating indexes'
ensure_indexes(db)
source = Blog(
args.source_url, args.source_username, args.source_password,
use_cache=not args.refresh, verbose=args.verbose)
print 'Getting media library'
media_library = set([
m['link'] for m in source.get_media_library()])
print ' %s assets\n' % len(media_library)
print 'Getting posts and pages'
post_structs = source.get_recent_posts(args.nposts)
print ' %s posts' % len(post_structs)
page_structs = source.get_pages()
print ' %s pages' % len(page_structs)
print
for structs, type in [
(post_structs, 'post'),
(page_structs, 'page'),
]:
print '%sS' % type.upper()
for struct in structs:
categories = struct.pop('categories', [])
struct['description'] = wordpress_to_markdown(
struct, media_library, db, destination_url, source_base_url)
post = Post.from_metaweblog(struct, type)
print '%-34s %s' % (post.title, post.status.upper())
for category_name in categories:
doc = db.categories.find_one({'name': category_name})
if doc:
category = Category(**doc)
else:
category = Category(
name=category_name, slug=slugify(category_name))
category.id = db.categories.insert(category.to_python())
print ' %-30s %s' % (
category_name, ' NEW' if not doc else ''
)
post.categories.append(category)
db.posts.insert(post.to_python())
print '\nFinished %s %ss' % (len(structs), type)
print 'Posting "categories_changed" event'
db.events.insert(
{'ts': datetime.datetime.utcnow(), 'name': 'categories_changed'},
manipulate=False) # No need to add _id
print '\nFinished in %.2f seconds' % (time.time() - start)
def get_posts(self, query, fields, sort, skip, limit):
collection = self.settings['db'].posts
cursor = collection.find(query, fields).sort(sort).skip(skip)
docs = yield cursor.limit(limit).to_list(limit)
posts = [Post(**doc) for doc in docs]
raise gen.Return(posts)
