def import_threads(self, user_api_key, forum):
"Import Threads from the Disqus API"
client = DisqusClient()
start = 0
limit = 1000
threads = []
has_threads = True
while has_threads:
new_threads = client.get_thread_list(
user_api_key=user_api_key,
forum_id=forum.id,
limit=limit, start=start)
if not new_threads:
has_threads = False
else:
threads.extend(new_threads)
start += limit
for thread in threads:
date = datetime.strptime(thread['created_at'],
'%Y-%m-%dT%H:%M').strftime("%Y-%m-%d %H:%M")
t = Thread(
id = thread['id'],
forum = forum,
slug = thread['slug'],
title = thread['title'],
created_at = date,
allow_comments = thread['allow_comments'],
url = thread['url'],
identifier = ' '.join(thread['identifier']))
t.save()
def import_forums(self, user_api_key):
"Import forums from the Disqus API"
client = DisqusClient()
try:
forums = client.get_forum_list(user_api_key=user_api_key)
except URLError, e:
raise CommandError('Could not get forums. Check your ' \
'"DISQUS_API_KEY" setting.')
def disqus_popular():
client = DisqusClient()
forum_list = client.get_most_commented_threads(user_api_key=settings.DISQUS_API_KEY)
try:
forum = [f for f in forum_list\
if f['shortname'] == settings.DISQUS_WEBSITE_SHORTNAME][0]
return forum
except IndexError:
raise CommandError("Could not find forum. " +
"Please check your " +
"'DISQUS_WEBSITE_SHORTNAME' setting.")
def test_client_init(self):
"""
First, we test if the DisqusClient class can be initialized
and parameters that were passed are set correctly.
"""
c = DisqusClient(foo='bar', bar='foo')
self.assertEqual('bar', c.foo)
self.assertEqual('foo', c.bar)
with self.assertRaises(AttributeError):
c.baz
def test_init_if_passed_args_with_name_like_in_METHODS(self):
c = DisqusClient(**DisqusClient.METHODS)
with self.assertRaises(TypeError):
# str is not callable
getattr(c, 'get_thread_list')()
with self.assertRaises(TypeError):
getattr(c, 'get_forum_posts')()
with self.assertRaises(TypeError):
getattr(c, 'moderate_post')()
def handle(self, **options):
from django.conf import settings
client = DisqusClient()
indent = options.get('indent')
filter_ = options.get('filter')
exclude = options.get('exclude')
# Get a list of all forums for an API key. Each API key can have
# multiple forums associated. This application only supports the one
# set in the DISQUS_WEBSITE_SHORTNAME variable
forum_list = client.get_forum_list(
user_api_key=settings.DISQUS_API_KEY)
try:
forum = [
f for f in forum_list
if f['shortname'] == settings.DISQUS_WEBSITE_SHORTNAME
][0]
except IndexError:
raise CommandError("Could not find forum. " +
"Please check your " +
"'DISQUS_WEBSITE_SHORTNAME' setting.")
posts = []
has_new_posts = True
start = 0
step = 100
while has_new_posts:
new_posts = client.get_forum_posts(
user_api_key=settings.DISQUS_API_KEY,
forum_id=forum['id'],
start=start,
limit=start + step,
filter=filter_,
exclude=exclude)
if not new_posts:
has_new_posts = False
else:
start += step
posts.append(new_posts)
print(json.dumps(posts, indent=indent))
def handle(self, **options):
from django.conf import settings
client = DisqusClient()
indent = options.get('indent')
filter_ = options.get('filter')
exclude = options.get('exclude')
# Get a list of all forums for an API key. Each API key can have
# multiple forums associated. This application only supports the one
# set in the DISQUS_WEBSITE_SHORTNAME variable
forum_list = client.get_forum_list(user_api_key=settings.DISQUS_API_KEY)
try:
forum = [f for f in forum_list\
if f['shortname'] == settings.DISQUS_WEBSITE_SHORTNAME][0]
except IndexError:
raise CommandError("Could not find forum. " +
"Please check your " +
"'DISQUS_WEBSITE_SHORTNAME' setting.")
posts = []
has_new_posts = True
start = 0
step = 100
while has_new_posts:
new_posts = client.get_forum_posts(
user_api_key=settings.DISQUS_API_KEY,
forum_id=forum['id'],
start=start,
limit=start+step,
filter=filter_,
exclude=exclude)
if not new_posts:
has_new_posts = False
else:
start += step
posts.append(new_posts)
self.stdout.write(json.dumps(posts, indent=indent))
self.stdout.write('\n')
def handle(self, **options):
current_site = Site.objects.get_current()
client = DisqusClient()
verbosity = int(options.get('verbosity'))
dry_run = bool(options.get('dry_run'))
state_file = options.get('state_file')
last_exported_id = None
if state_file is not None and os.path.exists(state_file):
last_exported_id = self._get_last_state(state_file)
comments = self._get_comments_to_export(last_exported_id)
comments_count = comments.count()
if verbosity >= 1:
print("Exporting %d comment(s)" % comments_count)
# if this is a dry run, we output the comments and exit
if dry_run:
print(comments)
return
# if no comments were found we also exit
if not comments_count:
return
# Get a list of all forums for an API key. Each API key can have
# multiple forums associated. This application only supports the one
# set in the DISQUS_WEBSITE_SHORTNAME variable
forum_list = client.get_forum_list(
user_api_key=settings.DISQUS_API_KEY)
try:
forum = [
f for f in forum_list
if f['shortname'] == settings.DISQUS_WEBSITE_SHORTNAME
][0]
except IndexError:
raise CommandError("Could not find forum. " +
"Please check your " +
"'DISQUS_WEBSITE_SHORTNAME' setting.")
# Get the forum API key
forum_api_key = client.get_forum_api_key(
user_api_key=settings.DISQUS_API_KEY, forum_id=forum['id'])
for comment in comments:
if verbosity >= 1:
print("Exporting comment '%s'" % comment)
# Try to find a thread with the comments URL.
url = 'http://%s%s' % (current_site.domain,
comment.content_object.get_absolute_url())
thread = client.get_thread_by_url(url=url,
forum_api_key=forum_api_key)
# if no thread with the URL could be found, we create a new one.
# to do this, we first need to create the thread and then
# update the thread with a URL.
if not thread:
thread = client.thread_by_identifier(
forum_api_key=forum_api_key,
identifier=str(comment.content_object),
title=str(comment.content_object),
)['thread']
client.update_thread(forum_api_key=forum_api_key,
thread_id=thread['id'],
url=url)
# name and email are optional in contrib.comments but required
# in DISQUS. If they are not set, dummy values will be used
client.create_post(
forum_api_key=forum_api_key,
thread_id=thread['id'],
message=comment.comment.encode('utf-8'),
author_name=comment.userinfo.get('name',
'nobody').encode('utf-8'),
author_email=comment.userinfo.get('email',
'*****@*****.**'),
author_url=comment.userinfo.get('url', ''),
created_at=comment.submit_date.strftime('%Y-%m-%dT%H:%M'))
if state_file is not None:
self._save_state(state_file, comment.pk)
def handle(self, **options):
current_site = Site.objects.get_current()
client = DisqusClient()
verbosity = int(options.get('verbosity'))
dry_run = bool(options.get('dry_run'))
state_file = options.get('state_file')
last_exported_id = None
if state_file is not None and os.path.exists(state_file):
last_exported_id = self._get_last_state(state_file)
comments = self._get_comments_to_export(last_exported_id)
comments_count = comments.count()
if verbosity >= 1:
print "Exporting %d comment(s)" % comments_count
# if this is a dry run, we output the comments and exit
if dry_run:
print comments
return
# if no comments were found we also exit
if not comments_count:
return
# Get a list of all forums for an API key. Each API key can have
# multiple forums associated. This application only supports the one
# set in the DISQUS_WEBSITE_SHORTNAME variable
forum_list = client.get_forum_list(user_api_key=settings.DISQUS_API_KEY)
try:
forum = [f for f in forum_list\
if f['shortname'] == settings.DISQUS_WEBSITE_SHORTNAME][0]
except IndexError:
raise CommandError("Could not find forum. " +
"Please check your " +
"'DISQUS_WEBSITE_SHORTNAME' setting.")
# Get the forum API key
forum_api_key = client.get_forum_api_key(
user_api_key=settings.DISQUS_API_KEY,
forum_id=forum['id'])
for comment in comments:
if verbosity >= 1:
print "Exporting comment '%s'" % comment
# Try to find a thread with the comments URL.
url = 'http://%s%s' % (
current_site.domain,
comment.content_object.get_absolute_url())
thread = client.get_thread_by_url(
url=url,
forum_api_key=forum_api_key)
# if no thread with the URL could be found, we create a new one.
# to do this, we first need to create the thread and then
# update the thread with a URL.
if not thread:
thread = client.thread_by_identifier(
forum_api_key=forum_api_key,
identifier=unicode(comment.content_object),
title=unicode(comment.content_object),
)['thread']
client.update_thread(
forum_api_key=forum_api_key,
thread_id=thread['id'],
url=url)
# name and email are optional in contrib.comments but required
# in DISQUS. If they are not set, dummy values will be used
client.create_post(
forum_api_key=forum_api_key,
thread_id=thread['id'],
message=comment.comment.encode('utf-8'),
author_name=comment.userinfo.get('name',
'nobody').encode('utf-8'),
author_email=comment.userinfo.get('email',
'*****@*****.**'),
author_url=comment.userinfo.get('url', ''),
created_at=comment.submit_date.strftime('%Y-%m-%dT%H:%M'))
if state_file is not None:
self._save_state(state_file, comment.pk)
def setUp(self):
self.client = DisqusClient()
self.attr = {'user_api_key': 'spam', 'developer_api_key': 'ham'}
class DisqusClientTest(TestCase):
def setUp(self):
self.client = DisqusClient()
self.attr = {'user_api_key': 'spam', 'developer_api_key': 'ham'}
def test_init_properly(self):
"""
First, we test if the DisqusClient class can be initialized
and parameters that were passed are set correctly.
"""
c = DisqusClient(foo='bar', bar='foo')
self.assertEqual('bar', c.foo)
self.assertEqual('foo', c.bar)
with self.assertRaises(AttributeError):
c.baz
# XXX bug or feature?
def test_init_if_passed_args_with_name_like_in_METHODS(self):
c = DisqusClient(**DisqusClient.METHODS)
with self.assertRaises(TypeError):
# str is not callable
getattr(c, 'get_thread_list')()
with self.assertRaises(TypeError):
getattr(c, 'get_forum_posts')()
with self.assertRaises(TypeError):
getattr(c, 'moderate_post')()
@mock.patch('disqus.api.DisqusClient.call')
def test_call_method_is_triggered_by_api_methods_from_METHODS(
self, call_mock):
call = getattr(self.client, 'get_user_name')
call(**self.attr)
call_mock.assert_called_with('get_user_name', **self.attr)
call = getattr(self.client, 'get_num_posts')
call(**self.attr)
call_mock.assert_called_with('get_num_posts', **self.attr)
call = getattr(self.client, 'get_thread_by_url')
call(**self.attr)
call_mock.assert_called_with('get_thread_by_url', **self.attr)
@mock.patch('disqus.api.urlopen', new_callable=FakeUrlopen)
@mock.patch('disqus.api.DisqusClient._get_request')
def test__get_request_is_triggered_by_call_method(self, _get_request_mock,
urlopen_mock):
for method in DisqusClient.METHODS:
url = self.client.api_url % method
call = getattr(self.client, method)
call(**self.attr)
_get_request_mock.assert_called_with(url,
self.client.METHODS[method],
**self.attr)
@mock.patch('disqus.api.urlopen', new_callable=FakeUrlopen)
def test_call_method_if_requst_is_succeeded(self, urlopen_mock):
rest_response = '''
{
"message": [
{
"created_at": "2007-07-31 17:44:00",
"shortname": "disqus",
"description":
"The official Disqus threads. [...]",
"id": "NN", "name": "DISQUS Blog and Forum"
},
{
"created_at": "2008-09-10 14:37:31.744838",
"shortname": "antonkovalyov",
"description": "",
"id": "NN",
"name": "Anton Kovalyov"
}
],
"code": "ok",
"succeeded": true
}
'''
response_json = json.loads(rest_response)
message = response_json['message']
response = self.client.get_forum_list(user_api_key='spam')
self.assertEqual(response, message)
@mock.patch('disqus.api.urlopen', new_callable=FakeUrlopenNegative)
def test_call_method_if_requst_is_not_succeeded(self, urlopen_mock):
with self.assertRaises(DisqusException):
self.client.get_forum_list()
@mock.patch('disqus.api.DisqusClient._get_request')
def test_call_method_if_during_request_error_occurred(
self, _get_request_mock):
with self.assertRaises(URLError):
self.client.create_post()
def test__get_request_if_http_method_is_get(self):
attr_ = {
'user_api_key': ['spam'],
'developer_api_key': ['ham'],
'api_version': ['1.1']
}
for api_method, http_method in DisqusClient.METHODS.items():
if http_method == "GET":
url = self.client.api_url % api_method
request_params = self.client._get_request(
url, http_method, **self.attr)
request_no_params = self.client._get_request(url, http_method)
self.assertEqual(request_params.get_host(), 'disqus.com')
self.assertEqual(request_no_params.get_host(), 'disqus.com')
# check actual request method
self.assertEqual(request_params.get_method(), http_method)
self.assertEqual(request_no_params.get_method(), http_method)
# getting url's query string
# since parameters passed to api_url from a dict, mean randomly
url_parsed1 = urlparse(request_params.get_full_url())
qs_params = parse_qs(url_parsed1.query)
url_parsed2 = urlparse(request_no_params.get_full_url())
qs_no_params = parse_qs(url_parsed2.query)
self.assertEqual(qs_params, attr_)
# hardcoded in api_url
self.assertEqual(qs_no_params, {'api_version': ['1.1']})
def test__get_request_if_http_method_is_post(self):
attr_ = {'user_api_key': ['spam'], 'developer_api_key': ['ham']}
for api_method, http_method in DisqusClient.METHODS.items():
if http_method == "POST":
url = self.client.api_url % api_method
request_params = self.client._get_request(
url, http_method, **self.attr)
request_no_params = self.client._get_request(url, http_method)
self.assertEqual(request_params.get_host(), 'disqus.com')
self.assertEqual(request_no_params.get_host(), 'disqus.com')
self.assertEqual(request_params.get_method(), http_method)
self.assertEqual(request_no_params.get_method(), http_method)
qs_params = parse_qs(request_params.data)
qs_no_params = parse_qs(request_no_params.data)
self.assertEqual(qs_params, attr_)
self.assertEqual(qs_no_params, {})
# XXX maybe exception must be raised explicitly (DisqusException)
def test__get_request_if_http_method_is_not_post_or_get(self):
url1 = self.client.api_url % 'get_forum_api_key'
url2 = self.client.api_url % 'create_post'
url3 = self.client.api_url % 'foobar'
with self.assertRaises(UnboundLocalError):
self.client._get_request(url1, 'PUSH', **self.attr)
with self.assertRaises(UnboundLocalError):
self.client._get_request(url1, 'PUSH')
with self.assertRaises(UnboundLocalError):
self.client._get_request(url2, 'PUSH', **self.attr)
with self.assertRaises(UnboundLocalError):
self.client._get_request(url2, 'PUSH')
with self.assertRaises(UnboundLocalError):
self.client._get_request(url3, 'PUSH', **self.attr)
with self.assertRaises(UnboundLocalError):
self.client._get_request(url3, 'PUSH')
# XXX Don't know how to implement this and if should.
def test_call_method_if_api_version_passed_as_method_argument(self):
pass
def import_posts(self, user_api_key, forum):
"Import Posts from the Disqus API"
client = DisqusClient()
start = 0
limit = 1000
posts = []
has_posts = True
while has_posts:
new_posts = client.get_forum_posts(
user_api_key=user_api_key,
forum_id=forum.id,
limit=limit, start=start,
exclude='spam,killed')
if not new_posts:
has_posts = False
else:
posts.extend(new_posts)
start += limit
for post in posts:
try:
thread = Thread.objects.get(id=post['thread']['id'])
except Thread.DoesNotExist:
continue
date = datetime.strptime(post['created_at'],
'%Y-%m-%dT%H:%M').strftime("%Y-%m-%d %H:%M")
if post['is_anonymous']:
# anonymouse user
anon_author = AnonymousAuthor(
url=post['anonymous_author']['url'],
email=post['anonymous_author']['email'],
email_hash=post['anonymous_author']['email_hash'],
name=post['anonymous_author']['name'])
anon_author.save()
author = None
else:
# registered user
author = Author(
id=post['author']['id'],
url=post['author']['url'],
username=post['author']['username'],
email=post['author']['email'],
email_hash=post['author']['email_hash'],
has_avatar=post['author']['has_avatar'],
display_name=post['author']['display_name'])
author.save()
anon_author = None
p = Post(id=post['id'],
forum = forum,
thread = thread,
created_at = date,
message = post['message'],
parent_post = post['parent_post'],
ip_address = post['ip_address'],
has_been_moderated = post['has_been_moderated'],
status = post['status'],
points = post['points'],
is_anonymous = post['is_anonymous'],
anonymous_author = anon_author,
author = author)
p.save()
def setUp(self):
self.client = DisqusClient()
self.attr = {'user_api_key': 'spam', 'developer_api_key': 'ham'}
class DisqusClientTest(TestCase):
def setUp(self):
self.client = DisqusClient()
self.attr = {'user_api_key': 'spam', 'developer_api_key': 'ham'}
def test_init_properly(self):
"""
First, we test if the DisqusClient class can be initialized
and parameters that were passed are set correctly.
"""
c = DisqusClient(foo='bar', bar='foo')
self.assertEqual('bar', c.foo)
self.assertEqual('foo', c.bar)
with self.assertRaises(AttributeError):
c.baz
# XXX bug or feature?
def test_init_if_passed_args_with_name_like_in_METHODS(self):
c = DisqusClient(**DisqusClient.METHODS)
with self.assertRaises(TypeError):
# str is not callable
getattr(c, 'get_thread_list')()
with self.assertRaises(TypeError):
getattr(c, 'get_forum_posts')()
with self.assertRaises(TypeError):
getattr(c, 'moderate_post')()
@mock.patch('disqus.api.DisqusClient.call')
def test_call_method_is_triggered_by_api_methods_from_METHODS(self,
call_mock):
call = getattr(self.client, 'get_user_name')
call(**self.attr)
call_mock.assert_called_with('get_user_name', **self.attr)
call = getattr(self.client, 'get_num_posts')
call(**self.attr)
call_mock.assert_called_with('get_num_posts', **self.attr)
call = getattr(self.client, 'get_thread_by_url')
call(**self.attr)
call_mock.assert_called_with('get_thread_by_url', **self.attr)
@mock.patch('disqus.api.urlopen', new_callable=FakeUrlopen)
@mock.patch('disqus.api.DisqusClient._get_request')
def test__get_request_is_triggered_by_call_method(self,
_get_request_mock,
urlopen_mock):
for method in DisqusClient.METHODS:
url = self.client.api_url % method
call = getattr(self.client, method)
call(**self.attr)
_get_request_mock.assert_called_with(
url, self.client.METHODS[method],
**self.attr)
@mock.patch('disqus.api.urlopen', new_callable=FakeUrlopen)
def test_call_method_if_requst_is_succeeded(self, urlopen_mock):
rest_response = '''
{
"message": [
{
"created_at": "2007-07-31 17:44:00",
"shortname": "disqus",
"description":
"The official Disqus forum. [...]",
"id": "NN", "name": "DISQUS Blog and Forum"
},
{
"created_at": "2008-09-10 14:37:31.744838",
"shortname": "antonkovalyov",
"description": "",
"id": "NN",
"name": "Anton Kovalyov"
}
],
"code": "ok",
"succeeded": true
}
'''
response_json = json.loads(rest_response)
message = response_json['message']
response = self.client.get_forum_list(user_api_key='spam')
self.assertEqual(response, message)
@mock.patch('disqus.api.urlopen',
new_callable=FakeUrlopenNegative)
def test_call_method_if_requst_is_not_succeeded(self, urlopen_mock):
with self.assertRaises(DisqusException):
self.client.get_forum_list()
@mock.patch('disqus.api.DisqusClient._get_request')
def test_call_method_if_during_request_error_occurred(self,
_get_request_mock):
with self.assertRaises(URLError):
self.client.create_post()
def test__get_request_if_http_method_is_get(self):
attr_ = {'user_api_key': ['spam'],
'developer_api_key': ['ham'],
'api_version': ['1.1']
}
for api_method, http_method in DisqusClient.METHODS.items():
if http_method == "GET":
url = self.client.api_url % api_method
request_params = self.client._get_request(
url, http_method,
**self.attr)
request_no_params = self.client._get_request(url, http_method)
self.assertEqual(request_params.get_host(), 'disqus.com')
self.assertEqual(request_no_params.get_host(), 'disqus.com')
# check actual request method
self.assertEqual(request_params.get_method(), http_method)
self.assertEqual(request_no_params.get_method(), http_method)
# getting url's query string
# since parameters passed to api_url from a dict, mean randomly
url_parsed1 = urlparse(request_params.get_full_url())
qs_params = parse_qs(url_parsed1.query)
url_parsed2 = urlparse(request_no_params.get_full_url())
qs_no_params = parse_qs(url_parsed2.query)
self.assertEqual(qs_params, attr_)
# hardcoded in api_url
self.assertEqual(qs_no_params, {'api_version': ['1.1']})
def test__get_request_if_http_method_is_post(self):
attr_ = {'user_api_key': ['spam'], 'developer_api_key': ['ham']}
for api_method, http_method in DisqusClient.METHODS.items():
if http_method == "POST":
url = self.client.api_url % api_method
request_params = self.client._get_request(url,
http_method,
**self.attr)
request_no_params = self.client._get_request(url, http_method)
self.assertEqual(request_params.get_host(), 'disqus.com')
self.assertEqual(request_no_params.get_host(), 'disqus.com')
self.assertEqual(request_params.get_method(), http_method)
self.assertEqual(request_no_params.get_method(), http_method)
qs_params = parse_qs(request_params.data)
qs_no_params = parse_qs(request_no_params.data)
self.assertEqual(qs_params, attr_)
self.assertEqual(qs_no_params, {})
# XXX maybe exception must be raised explicitly (DisqusException)
def test__get_request_if_http_method_is_not_post_or_get(self):
url1 = self.client.api_url % 'get_forum_api_key'
url2 = self.client.api_url % 'create_post'
url3 = self.client.api_url % 'foobar'
with self.assertRaises(UnboundLocalError):
self.client._get_request(url1, 'PUSH', **self.attr)
with self.assertRaises(UnboundLocalError):
self.client._get_request(url1, 'PUSH')
with self.assertRaises(UnboundLocalError):
self.client._get_request(url2, 'PUSH', **self.attr)
with self.assertRaises(UnboundLocalError):
self.client._get_request(url2, 'PUSH')
with self.assertRaises(UnboundLocalError):
self.client._get_request(url3, 'PUSH', **self.attr)
with self.assertRaises(UnboundLocalError):
self.client._get_request(url3, 'PUSH')
# XXX Don't know how to implement this and if should.
def test_call_method_if_api_version_passed_as_method_argument(self):
pass