def test_getRecentPosts_own_blog(self):
# create 10 posts
blogid = self.test_blog.id
username = self.test_user1.username
password = self.test_user1.author.remote_access_key
for i in range(10):
title = "Test Post %s" % str(i)
body = "This is test post number %s" % i
blog = self.test_blog
author = self.test_user1.author
p = Post(title=title, body=body, author=author, blog=blog)
p.save()
num_posts = 10
posts = self.s.metaWeblog.getRecentPosts(blogid, username, password,
num_posts)
self.assertEqual(10, len(posts))
def metaWeblog_newPost(user, blogid, struct, publish="PUBLISH"):
""" mt's newpost function..."""
logger.debug("metaWeblog.newPost called")
logger.debug("user: %s" % user)
logger.debug("blogid: %s" % blogid)
logger.debug("struct: %s" % struct)
logger.debug("publish: %s" % publish)
body = struct['description']
try:
logger.info("Checking for passed blog parameter")
blog = Blog.objects.get(pk=blogid)
except ValueError:
# probably expecting wp behavior
logger.info("Specified blog not found, using default")
blog = Blog.objects.filter(owner=user)[0]
pub_date = datetime.datetime.now()
post = Post(title=struct['title'],
body=body,
create_date=pub_date,
update_date=pub_date,
pub_date=pub_date,
status=publish and 'publish' or 'draft',
blog=blog,
author=user.author)
post.prepopulate()
logger.debug("Saving")
# need to save beffore setting many-to-many fields, silly django
post.save()
categories = struct.get("categories", [])
# logger.debug("Setting categories: %s" % categories)
logger.warn("Categories no longer supported")
# clist = []
# for category in categories:
# try:
# c = Category.objects.filter(blog=blog, title=category)[0]
# logger.debug("Got %s" % c)
# clist.append(c)
# except Exception, e:
# logger.warn(str(e))
# post.categories=clist
post.save()
logger.info("Post %s saved" % post)
logger.info("Setting Tags")
setTags(post, struct, key="mt_keywords")
logger.debug("Handling Pings")
logger.info("sending pings to host")
send_pings(post)
logger.debug("newPost finished")
return post.id
def metaWeblog_newPost(user, blogid, struct, publish="PUBLISH"):
""" mt's newpost function..."""
logger.debug("metaWeblog.newPost called")
logger.debug("user: %s" % user)
logger.debug("blogid: %s" % blogid)
logger.debug("struct: %s" % struct)
logger.debug("publish: %s" % publish)
body = struct["description"]
try:
logger.info("Checking for passed blog parameter")
blog = Blog.objects.get(pk=blogid)
except ValueError:
# probably expecting wp behavior
logger.info("Specified blog not found, using default")
blog = Blog.objects.filter(owner=user)[0]
pub_date = datetime.datetime.now()
post = Post(
title=struct["title"],
body=body,
create_date=pub_date,
update_date=pub_date,
pub_date=pub_date,
status=publish and "publish" or "draft",
blog=blog,
author=user.author,
)
post.prepopulate()
logger.debug("Saving")
# need to save beffore setting many-to-many fields, silly django
post.save()
categories = struct.get("categories", [])
# logger.debug("Setting categories: %s" % categories)
logger.warn("Categories no longer supported")
# clist = []
# for category in categories:
# try:
# c = Category.objects.filter(blog=blog, title=category)[0]
# logger.debug("Got %s" % c)
# clist.append(c)
# except Exception, e:
# logger.warn(str(e))
# post.categories=clist
post.save()
logger.info("Post %s saved" % post)
logger.info("Setting Tags")
setTags(post, struct, key="mt_keywords")
logger.debug("Handling Pings")
logger.info("sending pings to host")
send_pings(post)
logger.debug("newPost finished")
return post.id
def test_getRecentPosts_own_blog(self):
# create 10 posts
blogid = self.test_blog.id
username = self.test_user1.username
password = self.test_user1.author.remote_access_key
for i in range(10):
title = "Test Post %s" % str(i)
body = "This is test post number %s" % i
blog = self.test_blog
author = self.test_user1.author
p = Post(
title=title,
body=body,
author=author,
blog=blog
)
p.save()
num_posts = 10
posts = self.s.metaWeblog.getRecentPosts(blogid, username, password, num_posts)
self.assertEqual(10, len(posts))
def wp_newPost(user, blog_id, content):
"""
Parameters
int blog_id
string username
string password
struct content
string post_type
string post_status
string post_title
int post_author
string post_excerpt
string post_content
datetime post_date_gmt | post_date
string post_format
string post_name: Encoded URL (slug)
string post_password
string comment_status
string ping_status
int sticky
int post_thumbnail
int post_parent
array custom_fields
struct
string key
string value
struct terms: Taxonomy names as keys, array of term IDs as values.
struct terms_names: Taxonomy names as keys, array of term names as values.
struct enclosure
string url
int length
string type
any other fields supported by wp_insert_post
## EXAMPLE FROM DeskPM
{ 'post_format': 'text',
'post_title': 'Test Post for desktop clients',
'post_status': 'publish',
'post_thumbnail': 0,
'sticky': False,
'post_content': '<p>This is a test post. </p><p>Go forth, and publish my good man...</p>',
'terms_names': {'post_tag': []},
'comment_status': 'open'
}
## Full-Featured Example
{ 'post_format': 'text',
'post_title': 'Full-featured Posts',
'post_status': 'publish',
'post_thumbnail': 0,
'sticky': False,
'post_content': "Fully Featured, With Pics & Stuff.\n\nMy, aren't **we** fancypants.",
'terms_names': {'post_tag': ['tag']},
'comment_status': 'open'}
Return Values
string post_id
Errors
401
- If the user does not have the edit_posts cap for this post type.
- If user does not have permission to create post of the specified post_status.
- If post_author is different than the user's ID and the user does not have the edit_others_posts cap for this post type.
- If sticky is passed and user does not have permission to make the post sticky, regardless if sticky is set to 0, 1, false or true.
- If a taxonomy in terms or terms_names is not supported by this post type.
- If terms or terms_names is set but user does not have assign_terms cap.
- If an ambiguous term name is used in terms_names.
403
- If invalid post_type is specified.
- If an invalid term ID is specified in terms.
404
- If no author with that post_author ID exists.
- If no attachment with that post_thumbnail ID exists.
"""
logger.debug("wp.newPost entered")
logger.debug("user: %s" % str(user))
logger.debug("blog_id: %s" % str(blog_id))
logger.debug("content:\n%s" % str(content))
blog = Blog.objects.get(pk=blog_id)
logger.info("blog: %s" % str(blog))
pub_date = datetime.datetime.now()
logger.info("blog: %s" % str(blog))
logger.info("pub_date: %s" % str(pub_date))
post = Post(title=content['post_title'],
body=content['post_content'],
create_date=pub_date,
update_date=pub_date,
pub_date=pub_date,
status=content['post_status'],
blog=blog,
author=user.author)
post.save()
logger.info("Post %s saved" % post)
# logger.info("Setting Tags")
# setTags(post, struct)
#logger.debug("Handling Pings")
#logger.info("sending pings to host")
# send_pings(post)
struct = {
'tags': content['terms_names']['post_tag'],
}
setTags(post, struct)
logger.debug("newPost finished")
# set categories? Hmm... categories for posts seem to be legacy thinking
# set tags
return str(post.id)
class Command(BaseCommand):
""" Import blog posts from a remote blog """
option_list = BaseCommand.option_list + (
# make_option("-u", "--username", dest="username", default=[], metavar="username", help="User name for remote API endpoint"),
make_option("-o",
"--owner",
dest="owner",
metavar="ownername",
help="Owner's Django user name"),
make_option("-a",
"--all",
dest="all_blogs",
action="store_true",
help="Import all user blogs from API endpoint"),
make_option("-x",
"--api-endpoint",
dest="api_endpoint",
default=None,
metavar="API endpoint",
help="Direct URL for remote API endpoint"),
make_option("-p",
"--password",
dest="password",
default=[],
help="Password for remote API connection"),
make_option(
"-f",
"--filename",
dest="filename",
metavar="json_filename",
help="JSON file which contains dump blog data (created with --dump)"
),
make_option("-n",
"--dry-run",
dest="dry_run",
action="store_true",
help="Do not write anything to the database"),
make_option("-d",
"--dump",
dest="dump",
action="store_true",
help="Write retrieved data to screen in JSON format"),
)
args = "[blog_url] [xblog_id] [-u username] [-p password]"
help = """ Import blog posts from a remote blog """
def handle(self, *args, **options):
"""
imports a remote blog via the WordPress API
"""
if options.get("api_endpoint", None):
self.xmlrpc_import(args, options)
elif options.get('filename'):
self.json_import(args, options)
def json_import(self, args, options):
"""
Takes a json file of another xblog
importantly, this script will try to sanitize it.
Categories are imported as tags.
Blog posts are assigned to an existing user, so make sure that user exists.
New owner is passed using --owner=username
"""
username = options.get('username')
filename = options.get('filename')
try:
local_user = User.objects.get(username=options.get('owner'))
except ObjectDoesNotExist:
local_user = None
print "Parsing %s for %s" % (username, filename)
# owner = None
owner = local_user
while not owner:
try:
owner = User.objects.get(username=username)
print "Found %s" % owner
except ObjectDoesNotExist:
print "User name '%s' not found" % username
print "Choose from one of the following:"
for user in User.objects.all():
print "- %s" % user.username
print "[q] Quit"
username = raw_input('> ').strip()
if username.lower() == 'q':
sys.exit()
print "Proceeding with %s (id=%s)" % (owner, owner.id)
data = json.load(open(options.get("filename")))
new_data = []
bad_cats = [1, 4, 5] # specifically for YB; remove this
for point in data:
if point['model'] == 'xblog.category':
continue
if point['model'] == 'xblog.blog':
pprint(point)
point['fields']['owner'] = owner.id
if point['model'] == 'xblog.author':
pprint(point)
print "Removing author record for %s" % point['fields'][
'fullname']
data.remove(point)
continue
if point['model'] == 'xblog.post':
for category in point['fields']['categories']:
if category in bad_cats:
continue
# print point['fields']['author']
point['fields']['author'] = owner.id
# remove primary_category_name
for old_field in ["primary_category_name", "categories"]:
if old_field in point.get('fields').keys():
point['fields'].pop(old_field, None)
new_data.append(point)
# for point in new_data:
# print point
json.dump(new_data, open('new_data.json', 'w'), indent=4)
def xmlrpc_import(self, args, options):
"""
Sucks a blog over the internet using its XMLRPC protocol.
"""
blog_url = args[0]
xblog_id = args[1]
if options.get("dump"):
dry_run = True
notify("Dumping %s to screen" % args[0])
username = options.get("username") or raw_input("Username: "******"password") or getpass.getpass()
wp_url = options.get('api_endpoint')
if not wp_url:
notify("Couldn't find API link in page %s" % blog_url)
notify("Please re-run with '-x url_to_api_endpoint'")
# create the xmlrpc connection
if wp_url[:4] != 'http':
# this is a relative URL
notify("Normalizing URL")
a = blog_url
b = wp_url
c = os.path.join(str(blog_url), str(wp_url))
notify(c)
notify(os.path.join(blog_url, wp_url))
wp_url = os.path.join(blog_url, wp_url)
notify(os.path.join(blog_url, wp_url))
notify(wp_url)
notify("Logging in at %s" % wp_url)
wp = xmlrpclib.ServerProxy(wp_url, use_datetime=True, verbose=1)
blogs = wp.wp.getUsersBlogs(username, password)
notify("Got %d blogs" % len(blogs))
active_blogs = []
if len(blogs) > 1 and not options.get("all_blogs"):
notify("Please enter the number of the blog you'd like to import:")
counter = 1
for blog in blogs:
notify("[%d]: %s (ID:%s) " %
(counter, blog["blogName"], blog["blogid"]))
counter += 1
notify("[a]: All blogs")
choice = raw_input("Enter %s or 'a' for all blogs" %
str(range(1, len(blogs)))).strip()
if choice.lower() == 'a':
notify("Grabbing all blogs from this user")
active_blogs = blogs
elif int(choice) in range(1, counter):
notify("Grabbing blog %s" % choice)
# print blogs
# print blogs[0]
# print blogs[1]
# print '---'
# print int(choice)-1
# print blogs[0]
active_blogs = blogs[int(choice) - 1:int(choice)]
else:
active_blogs = blogs
notify("Importing:")
notify(",".join(["'%s'" % blog['blogName'] for blog in active_blogs]))
xblog = Blog.objects.get(id=int(xblog_id))
# xblog_author = User.objects.all()[0] # FIXME: should probably try to get fancy and match email address or some'n
owner_username = options.get("owner") or raw_input(
"Owner's username").strip()
# print "|%s|" % owner_username
xblog_author = User.objects.get(username=owner_username)
categories = []
posts = []
for blog in active_blogs:
# create a dictionary of categories
categories = categories + wp.metaWeblog.getCategories(
blog['blogid'], username, password)
# create a list of posts
notify("%d categories" % len(categories))
# walk through the posts
filter = {}
filter['offset'] = 0
all_posts = []
counter = 0
while True:
filter['number'] = 10
posts = wp.wp.getPosts(blog['blogid'], username, password,
filter)
# print "%d posts" % len(posts)
if len(posts) == 0:
notify("Done!")
break
for post in posts:
# convert the post date.
all_posts.append(post)
counter = counter + 1
# print "[%3d] %s (%s) %s" % (counter, post['post_title'], post['post_date'], post['post_status']),
notify("Title: %s" % post['post_title'])
# notify("Date: %s" % post['post_date'])
# notify("Status: %s" % post['post_status'])
# notify("Author: %s" % post['post_author'])
# notify("Type: %s" % post['post_type'])
# notify(10 * "-")
# print post['post_content']
filter['offset'] += len(posts)
notify("Downloaded %d posts" % len(all_posts))
# category_map = {}
# for category in categories:
# category_map[category['categoryId']] = self.get_or_create_category(category, xblog)
for post in all_posts:
xblog_post = self.get_or_create_post(post, xblog)
xblog_post.author = xblog_author
if options.get('dry_run'):
notify("Not saving '%s'" % xblog_post.title)
else:
xblog_post.save()
categories = self.get_post_categories(post, xblog)
for category in categories:
xblog_post.categories.add(category)
xblog_post.save()
def get_or_create_post(self, post, xblog):
# takes a wp.getPost object and set of categories, and creates
# an xblog post object from it
# extract the relevant information from the post
keys = post.keys()
keys.sort()
# pprint(keys)
# try to find if this post has already been imported?
try:
xpost = Post.objects.get(guid=post['guid'])
return xpost
except ObjectDoesNotExist, e:
pass
xblog_post = {}
xblog_post['pub_date'] = post['post_date_gmt']
xblog_post['update_date'] = post['post_modified_gmt']
xblog_post['create_date'] = post['post_date_gmt']
xblog_post['enable_comments'] = False
xblog_post['title'] = post['post_title']
# # xblog_post['slug'] =
xblog_post['body'] = post['post_content']
xblog_post['summary'] = post['post_excerpt']
# xblog_post['categories'] = self.get_post_categories(post, xblog)
# xblog_post['author'] =
xblog_post['status'] = post['post_status']
xblog_post['guid'] = post['guid']
xblog_post['blog'] = xblog
# print xblog_post
xpost = Post(**xblog_post)
return xpost
# let's see if this post already exists...
p = Post.objects.get(title__exact=post['post_title'])
# print "Found similar post", p.title
if p.create_date == post['post_date']:
print "NOT IMPORTING %s -- it exists" % str(p)
except:
status = status + ": %s" % post['post_title']
# mod the dates...
for dtime in ['post_date', 'post_modified']:
if not post[dtime]:
post[dtime] = datetime.datetime.now()
p = Post(title=post['post_title'],
body=post['post_content'],
create_date=post['post_date'],
update_date=post['post_modified'],
pub_date=post['post_date'],
blog=loblog,
author=louser,
status=post['post_status'],
slug=post['post_name'])
p.prepopulate()
try:
p.save()
p.categories = catlist
p.save()
except:
import traceback
traceback.print_exc(sys.stderr)
# get comments for this post...
p = Post.objects.get(title__exact=post['post_title'])
# print "Found similar post", p.title
if p.create_date == post['post_date']:
print "NOT IMPORTING %s -- it exists" % str(p)
except:
status = status + ": %s" % post['post_title']
# mod the dates...
for dtime in ['post_date', 'post_modified']:
if not post[dtime]:
post[dtime] = datetime.datetime.now()
p = Post(
title=post['post_title'],
body = post['post_content'],
create_date = post['post_date'],
update_date = post['post_modified'],
pub_date = post['post_date'],
blog = loblog,
author =louser,
status = post['post_status'],
slug=post['post_name']
)
p.prepopulate()
try:
p.save()
p.categories = catlist
p.save()
except:
import traceback
traceback.print_exc(sys.stderr)
def wp_newPost(user, blog_id, content):
"""
Parameters
int blog_id
string username
string password
struct content
string post_type
string post_status
string post_title
int post_author
string post_excerpt
string post_content
datetime post_date_gmt | post_date
string post_format
string post_name: Encoded URL (slug)
string post_password
string comment_status
string ping_status
int sticky
int post_thumbnail
int post_parent
array custom_fields
struct
string key
string value
struct terms: Taxonomy names as keys, array of term IDs as values.
struct terms_names: Taxonomy names as keys, array of term names as values.
struct enclosure
string url
int length
string type
any other fields supported by wp_insert_post
## EXAMPLE FROM DeskPM
{ 'post_format': 'text',
'post_title': 'Test Post for desktop clients',
'post_status': 'publish',
'post_thumbnail': 0,
'sticky': False,
'post_content': '<p>This is a test post. </p><p>Go forth, and publish my good man...</p>',
'terms_names': {'post_tag': []},
'comment_status': 'open'
}
## Full-Featured Example
{ 'post_format': 'text',
'post_title': 'Full-featured Posts',
'post_status': 'publish',
'post_thumbnail': 0,
'sticky': False,
'post_content': "Fully Featured, With Pics & Stuff.\n\nMy, aren't **we** fancypants.",
'terms_names': {'post_tag': ['tag']},
'comment_status': 'open'}
Return Values
string post_id
Errors
401
- If the user does not have the edit_posts cap for this post type.
- If user does not have permission to create post of the specified post_status.
- If post_author is different than the user's ID and the user does not have the edit_others_posts cap for this post type.
- If sticky is passed and user does not have permission to make the post sticky, regardless if sticky is set to 0, 1, false or true.
- If a taxonomy in terms or terms_names is not supported by this post type.
- If terms or terms_names is set but user does not have assign_terms cap.
- If an ambiguous term name is used in terms_names.
403
- If invalid post_type is specified.
- If an invalid term ID is specified in terms.
404
- If no author with that post_author ID exists.
- If no attachment with that post_thumbnail ID exists.
"""
logger.debug("wp.newPost entered")
logger.debug("user: %s" % str(user))
logger.debug("blog_id: %s" % str(blog_id))
logger.debug("content:\n%s" % str(content))
blog = Blog.objects.get(pk=blog_id)
logger.info("blog: %s" % str(blog))
pub_date = datetime.datetime.now()
logger.info("blog: %s" % str(blog))
logger.info("pub_date: %s" % str(pub_date))
post = Post(
title=content["post_title"],
body=content["post_content"],
create_date=pub_date,
update_date=pub_date,
pub_date=pub_date,
status=content["post_status"],
blog=blog,
author=user.author,
)
post.save()
logger.info("Post %s saved" % post)
# logger.info("Setting Tags")
# setTags(post, struct)
# logger.debug("Handling Pings")
# logger.info("sending pings to host")
# send_pings(post)
struct = {"tags": content["terms_names"]["post_tag"]}
setTags(post, struct)
logger.debug("newPost finished")
# set categories? Hmm... categories for posts seem to be legacy thinking
# set tags
return str(post.id)
def main():
"""docstring for main"""
config['cursorclass'] = DictCursor
cur = MySQLdb.connect(**config).cursor()
# load the posts
cur.execute("select * from ybwp_posts")
posts = cur.fetchall()
print "Cleaning out posts..."
Post.objects.all().delete()
print "Cleaning out categories..."
Category.objects.all().delete()
print "Cleaning out tags..."
Tag.objects.all().delete()
print "Cleaning out Links..."
Link.objects.all().delete()
print "Cleaning out pingbacks..."
Pingback.objects.all().delete()
print "Killing comments"
FreeComment.objects.filter(content_type=\
ContentType.objects.get(\
model__exact='post')).delete()
# load the categories
# cur.execute("select * from ybwp_categories")
# cats = cur.fetchall()
loblog = Blog.objects.get(pk=1)
print loblog
louser = User.objects.filter(username__exact='Rube')
# get links...
cur.execute("""
SELECT * FROM
ybwp_categories, ybwp_links, ybwp_link2cat
WHERE
ybwp_links.link_id = ybwp_link2cat.link_id
AND
ybwp_categories.cat_ID = ybwp_link2cat.category_id
""")
for link in cur.fetchall():
# try to get the category...
try:
cat = LinkCategory.objects.get(title__iexact=link['cat_name'])
except LinkCategory.DoesNotExist:
# create a new category...blah.
print "Creating category", link['cat_name']
cat = LinkCategory(title=link['cat_name'],
description=link['category_description'],
blog=loblog,
visible=True)
cat.save()
# create a new link...
link = Link(
link_name=link['link_name'],
category=cat,
description=link['link_description'],
rss=link['link_rss'],
visible=True,
url=link['link_url'],
blog=loblog,
)
print link.description
try:
link.save()
except Link.DoesNotExist:
pass
# for each wordpress entry
for post in posts:
status = "Processing %d" % (len(posts))
cur.execute("""
select ybwp_categories.cat_name, ybwp_categories.category_description
from ybwp_categories, ybwp_post2cat
where ybwp_post2cat.post_id = %d and ybwp_categories.cat_ID = ybwp_post2cat.category_id
""" % (post['ID']))
# status
# cq = "select * from ybwp_comments where comment_post_ID = %d" % entry['ID']
# cur.execute(q)
# comments = cur.fetchall()
#
catlist = []
for cat in cur.fetchall():
# try to get the category
try:
locat = Category.objects.get(title__iexact=cat['cat_name'])
# print "Found category", locat
except Category.DoesNotExist, error:
print "Error:", error
# create the category...
print "Creating", cat['cat_name']
locat = Category(title=cat['cat_name'],
description=cat['category_description'],
blog=loblog)
locat.save()
catlist.append(locat)
# print chardet.detect(post['post_title'])
# post['post_title'] = post['post_title'].encode('latin-1')
# post['post_content'] = post['post_content'].encode('latin-1')
try:
# let's see if this post already exists...
xpost = Post.objects.get(title__exact=post['post_title'])
# print "Found similar post", p.title
if xpost.create_date == post['post_date']:
print "NOT IMPORTING %s -- it exists" % str(post)
except Post.DoesNotExist:
status = status + ": %s" % post['post_title']
# mod the dates...
for dtime in ['post_date', 'post_modified']:
if not post[dtime]:
post[dtime] = datetime.datetime.now()
xpost = Post(title=post['post_title'],
body=post['post_content'],
create_date=post['post_date'],
update_date=post['post_modified'],
pub_date=post['post_date'],
blog=loblog,
author=louser,
status=post['post_status'],
slug=post['post_name'])
xpost.prepopulate()
try:
xpost.save()
xpost.categories = catlist
xpost.save()
except Post.DoesNotExist:
traceback.print_exc(sys.stderr)
# get comments for this post...
try:
cur.execute("""
SELECT * FROM ybwp_comments
WHERE
comment_post_ID = %d
""" % post['ID'])
comments = cur.fetchall()
contenttype = ContentType.objects.get(model__exact='post')
site = Site.objects.get(id__exact=settings.SITE_ID)
# create a comment for each comment...
except Site.DoesNotExist:
traceback.print_exc(sys.stdout)
comments = []
raw_input("press enter...")
for comment in comments:
if comment['comment_type'] == 'pingback':
# create a pingback for this post...
continue
else:
if comment['comment_approved'] != 'spam':
newcomment = FreeComment(content_type=contenttype,
object_id=xpost.id,
comment=xmlify(comment['comment_content']),
person_name=comment['comment_author'],
person_email=\
xmlify(comment['comment_author_email']),
person_url=xmlify(comment['comment_author_url']),
# submit_date=comment['comment_date'],
is_public=comment['comment_approved'],
ip_address=comment['comment_author_IP'],
# approved=comment['comment_approved'],
site=site,
)
try:
newcomment.save()
newcomment.submit_date = comment['comment_date']
newcomment.save()
except FreeComment.DoesNotExist:
print "ERROR:"
traceback.print_exc(sys.stderr)
print newcomment.person_name
print newcomment.person_url
print newcomment.comment
raw_input('error: hit enter to continue')
status = status + ": %d comments" % len(comments)
print status
