def test_anonymize(self):
examples = [(u'12.34.56.78', u'12.34.56.0'),
(u'1234:5678:90ab:cdef:fedc:ba09:8765:4321',
u'1234:5678:90ab:0000:0000:0000:0000:0000'),
(u'::ffff:127.0.0.1', u'127.0.0.0')]
for (addr, anonymized) in examples:
self.assertEqual(utils.anonymize(addr), anonymized)
def migrate(self):
tree = ElementTree.parse(self.xmlfile)
res = defaultdict(list)
for post in tree.findall(Disqus.ns + 'post'):
item = {
'dsq:id': post.attrib.get(Disqus.internals + 'id'),
'text': post.find(Disqus.ns + 'message').text,
'author': post.find('{0}author/{0}name'.format(Disqus.ns)).text,
'email': post.find('{0}author/{0}email'.format(Disqus.ns)).text,
'created': mktime(strptime(
post.find(Disqus.ns + 'createdAt').text, '%Y-%m-%dT%H:%M:%SZ')),
'remote_addr': anonymize(post.find(Disqus.ns + 'ipAddress').text),
'mode': 1 if post.find(Disqus.ns + "isDeleted").text == "false" else 4
}
if post.find(Disqus.ns + 'parent') is not None:
item['dsq:parent'] = post.find(Disqus.ns + 'parent').attrib.get(Disqus.internals + 'id')
res[post.find('%sthread' % Disqus.ns).attrib.get(Disqus.internals + 'id')].append(item)
progress = Progress(len(tree.findall(Disqus.ns + 'thread')))
for i, thread in enumerate(tree.findall(Disqus.ns + 'thread')):
progress.update(i, thread.find(Disqus.ns + 'id').text)
# skip (possibly?) duplicate, but empty thread elements
if thread.find(Disqus.ns + 'id').text is None:
continue
id = thread.attrib.get(Disqus.internals + 'id')
if id in res:
self.threads.add(id)
self.insert(thread, res[id])
# in case a comment has been deleted (and no further childs)
self.db.comments._remove_stale()
progress.finish("{0} threads, {1} comments".format(
len(self.threads), len(self.comments)))
orphans = set(map(lambda e: e.attrib.get(Disqus.internals + "id"), tree.findall(Disqus.ns + "post"))) - self.comments
if orphans:
print("Found %i orphans:" % len(orphans))
for post in tree.findall(Disqus.ns + "post"):
if post.attrib.get(Disqus.internals + "id") not in orphans:
continue
print(" * {0} by {1} <{2}>".format(
post.attrib.get(Disqus.internals + "id"),
post.find("{0}author/{0}name".format(Disqus.ns)).text,
post.find("{0}author/{0}email".format(Disqus.ns)).text))
print(textwrap.fill(post.find(Disqus.ns + "message").text,
initial_indent=" ", subsequent_indent=" "))
print("")
def test_str(self):
# Accept a str on both Python 2 and Python 3, for
# convenience.
examples = [
('12.34.56.78', u'12.34.56.0'),
('1234:5678:90ab:cdef:fedc:ba09:8765:4321',
'1234:5678:90ab:0000:0000:0000:0000:0000'),
('::ffff:127.0.0.1', u'127.0.0.0')]
for (addr, anonymized) in examples:
self.assertEqual(utils.anonymize(addr), anonymized)
def _remote_addr(self, request):
"""Return the anonymized IP address of the requester.
Takes into consideration a potential X-Forwarded-For HTTP header
if a necessary server.trusted-proxies configuration entry is set.
Recipe source: https://stackoverflow.com/a/22936947/636849
"""
remote_addr = request.remote_addr
if self.trusted_proxies:
route = request.access_route + [remote_addr]
remote_addr = next((addr for addr in reversed(route)
if addr not in self.trusted_proxies), remote_addr)
return utils.anonymize(str(remote_addr))
def Comment(self, el):
return {
"text": strip(el.find(self.ns + "comment_content").text),
"author": strip(el.find(self.ns + "comment_author").text),
"email": strip(el.find(self.ns + "comment_author_email").text),
"website": strip(el.find(self.ns + "comment_author_url").text),
"remote_addr": anonymize(
strip(el.find(self.ns + "comment_author_IP").text)),
"created": mktime(strptime(
strip(el.find(self.ns + "comment_date_gmt").text),
"%Y-%m-%d %H:%M:%S")),
"mode": 1 if el.find(self.ns + "comment_approved").text == "1" else 2,
"id": int(el.find(self.ns + "comment_id").text),
"parent": int(el.find(self.ns + "comment_parent").text) or None
}
def dislike(self, environ, request, id):
nv = self.comments.vote(
False, id, utils.anonymize(str(request.remote_addr)))
return JSON(nv, 200)
def new(self, environ, request, uri):
data = request.get_json()
for field in set(data.keys()) - API.ACCEPT:
data.pop(field)
for key in ("author", "email", "website", "parent"):
data.setdefault(key, None)
valid, reason = API.verify(data)
if not valid:
return BadRequest(reason)
for field in ("author", "email", "website"):
if data.get(field) is not None:
data[field] = cgi.escape(data[field])
if data.get("website"):
data["website"] = normalize(data["website"])
data['mode'] = 2 if self.moderated else 1
data['remote_addr'] = utils.anonymize(str(request.remote_addr))
with self.isso.lock:
if uri not in self.threads:
if 'title' not in data:
with http.curl('GET', local("origin"), uri) as resp:
if resp and resp.status == 200:
uri, title = parse.thread(resp.read(), id=uri)
else:
return NotFound('URI does not exist %s')
else:
title = data['title']
thread = self.threads.new(uri, title)
self.signal("comments.new:new-thread", thread)
else:
thread = self.threads[uri]
# notify extensions that the new comment is about to save
self.signal("comments.new:before-save", thread, data)
valid, reason = self.guard.validate(uri, data)
if not valid:
self.signal("comments.new:guard", reason)
raise Forbidden(reason)
with self.isso.lock:
rv = self.comments.add(uri, data)
# notify extension, that the new comment has been successfully saved
self.signal("comments.new:after-save", thread, rv)
cookie = functools.partial(dump_cookie,
value=self.isso.sign(
[rv["id"], sha1(rv["text"])]),
max_age=self.conf.getint('max-age'))
rv["text"] = self.isso.render(rv["text"])
rv["hash"] = self.hash(rv['email'] or rv['remote_addr'])
self.cache.set(
'hash', (rv['email'] or rv['remote_addr']).encode('utf-8'), rv['hash'])
rv = self._add_gravatar_image(rv)
for key in set(rv.keys()) - API.FIELDS:
rv.pop(key)
# success!
self.signal("comments.new:finish", thread, rv)
resp = JSON(rv, 202 if rv["mode"] == 2 else 201)
resp.headers.add("Set-Cookie", cookie(str(rv["id"])))
resp.headers.add("X-Set-Cookie", cookie("isso-%i" % rv["id"]))
return resp
def checkip(self, env, req):
return Response(utils.anonymize(str(req.remote_addr)), 200)
def new(app, environ, request, uri):
data = request.get_json()
for field in set(data.keys()) - set(
['text', 'author', 'website', 'email', 'parent']):
data.pop(field)
if not data.get("text"):
raise BadRequest("no text given")
if "id" in data and not isinstance(data["id"], int):
raise BadRequest("parent id must be an integer")
for field in ("author", "email"):
if data.get(field):
data[field] = cgi.escape(data[field])
data['mode'] = (app.conf.getboolean('moderation', 'enabled') and 2) or 1
data['remote_addr'] = utils.anonymize(str(request.remote_addr))
with app.lock:
if uri not in app.db.threads:
for host in app.conf.getiter('general', 'host'):
with http.curl('GET', host, uri) as resp:
if resp and resp.status == 200:
title = parse.title(resp.read())
break
else:
return Response('URI does not exist', 404)
app.db.threads.new(uri, title)
logger.info('new thread: %s -> %s', uri, title)
else:
title = app.db.threads[uri].title
try:
with app.lock:
rv = app.db.comments.add(uri, data)
except db.IssoDBException:
raise Forbidden
host = list(app.conf.getiter('general', 'host'))[0].rstrip("/")
href = host + uri + "#isso-%i" % rv["id"]
deletion = host + environ["SCRIPT_NAME"] + "/delete/" + app.sign(
str(rv["id"]))
activation = None
if app.conf.getboolean('moderation', 'enabled'):
activation = host + environ["SCRIPT_NAME"] + "/activate/" + app.sign(
str(rv["id"]))
app.notify(
title,
notify.format(rv,
href,
utils.anonymize(str(request.remote_addr)),
activation_key=activation,
deletion_key=deletion))
# save checksum of text into cookie, so mallory can't modify/delete a comment, if
# he add a comment, then removed it but not the signed cookie.
checksum = hashlib.md5(rv["text"].encode('utf-8')).hexdigest()
rv["text"] = app.markdown(rv["text"])
rv["hash"] = str(
pbkdf2(rv.get('email') or rv['remote_addr'], app.salt, 1000, 6))
for key in set(rv.keys()) - FIELDS:
rv.pop(key)
# success!
logger.info('comment created: %s', json.dumps(rv))
resp = Response(json.dumps(rv),
202 if rv["mode"] == 2 else 201,
content_type='application/json')
resp.set_cookie(str(rv["id"]),
app.sign([rv["id"], checksum]),
max_age=app.conf.getint('general', 'max-age'))
return resp
def dislike(app, environ, request, id):
nv = app.db.comments.vote(False, id,
utils.anonymize(str(request.remote_addr)))
return Response(json.dumps(nv), 200)