def disqus_static(generator):
disqus = DisqusAPI(generator.settings['DISQUS_SECRET_KEY'],
generator.settings['DISQUS_PUBLIC_KEY'])
# first retrieve the threads
threads = Paginator(disqus.threads.list,
forum=generator.settings['DISQUS_SITENAME'])
# build a {thread_id: title} dict
thread_dict = {}
for thread in threads:
thread_dict[thread['id']] = thread['title']
# now retrieve the posts
posts = Paginator(disqus.posts.list,
forum=generator.settings['DISQUS_SITENAME'])
# build a {post_id: [child_post1, child_post2, ...]} dict
child_dict = {}
for post in posts:
if post['id'] not in child_dict.keys():
child_dict[post['id']] = []
if post['parent'] is not None:
if str(post['parent']) not in child_dict.keys():
child_dict[str(post['parent'])] = []
child_dict[str(post['parent'])].append(post)
# build a {title: [post1, post2, ...]} dict
post_dict = {}
for post in posts:
build_post_dict(post_dict, child_dict, thread_dict, post)
for article in generator.articles:
if article.title in post_dict:
article.disqus_comments = post_dict[article.title]
article.disqus_comment_count = sum(
[postcounter(post) for post in post_dict[article.title]])
def list(cls, offset=0, limit=100):
thread_ids = []
for talk in sorted(schedule.itervalues(),
key=lambda x: (x['start'], x['room'])):
thread_ids.append(talk['disqus:thread']['id'])
if not thread_ids:
return []
thread_ids = thread_ids[:limit]
result = threads.get_many(thread_ids)
missing_thread_ids = [t for t, v in result.iteritems() if not v]
if missing_thread_ids:
thread_list = Paginator(disqusapi.threads.list,
thread=missing_thread_ids,
forum=app.config['DISQUS_FORUM'])
for thread in thread_list:
result[thread['id']] = Thread.save(thread)
for thread in result.itervalues():
if not thread:
continue
thread['session'] = schedule.get(thread.get('link'))
return [result[t] for t in thread_ids if result.get(t)]
def refresh_data(cls):
disqus = get_apicli()
data = []
for t in Paginator(disqus.threads.list, forum=conf.FORUM):
data.append((t['link'], t['id']))
cache.set(conf.THREAD_MAP_CACHE_KEY, data, timeout=conf.THREAD_MAP_CACHE_TIMEOUT)
return data
def list_by_thread(cls, thread_id, offset=0, limit=100):
result = posts.list(thread_id=thread_id, offset=offset, limit=limit)
if result is None:
result = []
paginator = Paginator(disqusapi.threads.listPosts,
thread=thread_id)
for post in paginator:
if post['author']['isAnonymous']:
continue
result.append(cls.save(post, incr_posts=False))
return result
def list(cls, offset=0, limit=100):
result = threads.list(offset=offset, limit=limit)
if result is None:
result = []
for thread in Paginator(disqusapi.threads.list,
forum=app.config['DISQUS_FORUM'],
category=Category.get('General')['id'],
method='GET'):
result.append(Thread.save(thread))
result.reverse()
return result
def get_comments(self, since=None):
# First, sync thread map to ensure nothing is missing.
ThreadMap.refresh_data()
# Now, query comments.
kwargs = {
'forum': conf.FORUM,
}
if since:
kwargs['since'] = since.isoformat()
kwargs['order'] = 'asc'
for c in Paginator(self.disqus.forums.listPosts, **kwargs):
yield c
import json
import os
from disqusapi import DisqusAPI, Paginator
disqus = DisqusAPI(secret_key="KEY_HERE", public_key="KEY_HERE")
p = Paginator(
disqus.api,
"users.listPosts",
user="******",
limit="100",
order="asc",
method="GET",
)
for post in p:
try:
filename = "comments/{0[createdAt]}_{0[id]}.json".format(post)
with open(filename, "x") as f:
json.dump(post, f)
except FileExistsError:
continue
for f in os.listdir("comments"):
# Further processing…
pass
def api(method, *args, **kwargs):
return method(*args, **api_args(**kwargs))
print 'Trying to find %s category in existing categories...' % TALK_CATEGORY_NAME
for existing_category in api(disqusapi.categories.list):
if existing_category['title'] == TALK_CATEGORY_NAME:
talk_category_id = existing_category['id']
if not talk_category_id:
print "Couldn't find it, adding it..."
new_category = api(disqusapi.categories.create, title=TALK_CATEGORY_NAME)
talk_category_id = new_category['id']
print "Downloading list of existing threads..."
paginator = Paginator(disqusapi.threads.list,
**api_args(category=talk_category_id))
existing_urls = dict((t['link'], t) for t in paginator)
print "%s existing threads found!" % len(existing_urls)
print 'Downloading and processing sessionlist...'
schedule = json.loads(
urllib.urlopen('https://us.pycon.org/2012/schedule/json/').read())
schedule = [s for s in schedule if s.get('url')]
for talk in schedule:
print ' - Processing session %s' % talk['title']
for key in ('start', 'end', 'last_updated'):
dt = datetime.datetime(*map(int, talk[key][0:6]))
talk[key] = dt + datetime.timedelta(hours=5)
def backup_disqus(short_name,
key,
secret,
outfile,
min_comments=5,
verbose=False):
"""
backup disqus threads and comments for a given forum shortname
:param short_name: Disqus forum short name / ID
:type short_name: string
:param key: Disqus API public key
:type key: string
:param secret: Disqus API secret key
:type secret: string
:param outfile: path to the file to write JSON output to
:type outfile: string
:param min_comments: minimum number of posts to have, else error and exit
:type min_comments: integer (default 5)
:param verbose: whether to write verbose output
:type verbose: boolean
"""
result = {}
disqus = DisqusAPI(secret, key)
if verbose:
print("Connected to Disqus API")
try:
details = disqus.forums.details(forum=short_name)
except disqusapi.APIError:
sys.stderr.write("ERROR: unable to find forum '%s'\n" % short_name)
sys.exit(1)
result['forum_details'] = details
if verbose:
print("Got forum details for '%s': %s" % (short_name, str(details)))
try:
threads = Paginator(disqus.forums.listThreads, forum=short_name)
except APIError:
sys.stderr.write("ERROR listing threads for forum '%s'\n" % short_name)
sys.exit(1)
thread_count = 0
all_threads = []
for t in threads:
thread_count = thread_count + 1
all_threads.append(t)
if verbose:
print("Found %d threads" % thread_count)
result['threads'] = all_threads
try:
posts = Paginator(disqus.forums.listPosts,
forum=short_name,
include=['unapproved', 'approved'])
except APIError:
sys.stderr.write("ERROR listing posts for forum '%s'\n" % short_name)
sys.exit(1)
post_count = 0
all_posts = []
for p in posts:
post_count = post_count + 1
all_posts.append(p)
if verbose:
print("Found %d posts" % post_count)
result['posts'] = all_posts
j = anyjson.serialize(result)
with open(outfile, 'w') as fh:
fh.write(j)
sys.stderr.write("Output written to %s\n" % outfile)
return True