def put(self, username):
"Add a session for a user and return the session cookie"
parser = restful.reqparse.RequestParser()
parser.add_argument("password", type=str, help="password.", required=True)
args = parser.parse_args()
user = User.query.filter(User.username == username).first()
if not user:
return{}, 404
if not user.verify_password(args.password):
return {}, 401
if not user.active:
return {}, 304
s = Session()
s.from_request(request)
user.sessions.append(s)
db.session.add(user)
db.session.add(s)
db.session.commit()
session['session'] = s.session_id
log("%s logged in." % user.username)
response = user.jsonify()
response['session'] = s.jsonify()
return response
def index(self):
user = auth(session, required=True)
current_session = session['session']
[db.session.delete(s) for s in user.sessions if s.session_id == current_session]
session['session'] = ''
log("%s logged out." % user.username)
return redirect('/')
def on_edit(self, update):
"""
Send edit data to channel subscribers (remember that the channel is a
URL), and any known participants.
"""
if not self.channel or not update:
return
log('DocumentStream: %s %s:%i.' % \
(self.user.username, self.channel, len(update)))
body = {"user":self.user.username,"document":update}
self.broadcast(self.channel, "fragment", body)
for addr in self.participants:
network, node_id, remote_uid = addr.split("/")
router = app.routes.get(network)
if router:
message = {"type": "edit",
"url": self.channel,
"body": update}
message['to'] = addr
message['from'] = self.user.get_address(router)
response = router.protocol.rpc_edit(message)
self.emit(self.channel, body)
def on_update_status(self, status):
"""
"""
log("%s changed status to %s." % (self.user.username, status.title()))
self.user.status = status
#db.session.commit()
self.broadcast(self.channel[1], "update_status", self.user.jsonify())
def recv_json(self, data):
self.emit("test", data)
self.broadcast("test", data)
if self.user:
log("Received JSON from %s: %s" % (self.user.username, str(data)))
else:
log("Received JSON: %s" % str(data))
def parse(html, url):
# TODO(ljb): Recognise licensensing attributes to prevent piracy.
domain = urlparse.urlparse(url).netloc
# Redefining XMLHttpRequest in the following file is not limited to the
# iframes' scope.
append_text = '<script src="/static/js/iframe.js"></script>\n'
#appendage = BeautifulSoup(append_text)
soup = BeautifulSoup(html)
request_endpoint = "/request/"
elements = ["a", "link", "img", "audio", "video"]
# Determine whether to remove all <script> tags or process their src attrs
if "DISABLE_JAVASCRIPT" in app.config and app.config["DISABLE_JAVASCRIPT"]:
[_.extract() for _ in soup.findAll("script")]
else:
elements.append("script")
# Known to omit some images such as the main one on uk.reuters.com
# Also far slower than the previous implementation (it's simply doing more).
def correct(soup, element):
for _ in soup.findAll(element):
if _.has_key("href"):
if _.has_key("license") and _['license'].lower() != "cc by":
log("Ignoring licensed object %s" % _['href'])
del _['href']
continue
log("%s -> %s%s%s" % (unicode(_['href']), request_endpoint, domain, unicode(_['href'])), "debug")
if _['href'].startswith('https'): _['href'] = _['href'].replace("https://", request_endpoint)
elif _['href'].startswith('http'): _['href'] = _['href'].replace("http://", request_endpoint)
elif _['href'].startswith('/'): _['href'] = '%s%s%s' % (request_endpoint, domain, _['href'])
else: _['href'] = '%s%s/%s' % (request_endpoint, domain, _['href'].encode("utf-8", "ignore"))
elif _.has_key("src"):
if _.has_key("license") and _['license'].lower() != "cc by":
log("Ignoring licensed object %s" % _['src'])
del _['src']
continue
log("%s -> %s%s%s" % (str(_['src']), request_endpoint, domain, str(_['src'])), "debug")
if _['src'].startswith('https'): _['src'] = _['src'].replace("https://", request_endpoint)
elif _['src'].startswith('http'): _['src'] = _['src'].replace("http://", request_endpoint)
elif _['src'].startswith('/'): _['src'] = '%s%s%s' % (request_endpoint, domain, _['src'])
else: _['src'] = '%s%s/%s' % (request_endpoint, domain, _['src'])
[correct(soup, element) for element in elements]
log('Should have cycled through urls by now.')
# try:
# soup.head.insert(0, appendage)
# except:
# pass
return unicode(soup)
def authenticate(self):
if not self.user:
user = auth(self.request)
if user and not self.user:
log("Received %s stream connection from %s" % \
(self.socket_type, user.username))
self.user = user
return True
return False
def remind():
for router in app.routes.values():
if len(router):
log("DHT: %s: Telling peers of our public revisions." % \
router.network)
for revision in Revision.query.filter(
and_(Revision.public == True, Revision.parent == None)
).all():
router[revision] = revision
def post(self, username):
"""
Account modification
"""
parser = reqparse.RequestParser()
parser.add_argument("email", type=str)
parser.add_argument("password", type=str)
parser.add_argument("verify_password", type=str)
parser.add_argument("public", type=bool, default=None)
parser.add_argument("active", type=bool, default=None)
args = parser.parse_args()
calling_user = auth(session, required=True)
if calling_user.username != username and not calling_user.can("reset_user_pw"):
return {}, 403
user = User.query.filter(User.username == username).first()
if not user:
return {}, 404
if args.verify_password: # It's here to spare from getting in the logs.
return user.verify_password(args.verify_password)
if args.password:
# must be at least six characters though
if len(args.password) < 6:
return "Must be at least six characters.", 304
user.change_password(args.password)
db.session.add(user)
db.session.commit()
return True
if args.email:
user.email = args.email
if args.public:
user.public = args.public != None
if calling_user.can("deactivate") and args.active != None:
user.active = args.active
if args.active == True:
log("%s reactivated %s's user account." % \
(calling_user.username, user.username))
else:
log("%s deactivated %s's user account." % \
(calling_user.username, user.username))
for s in user.sessions:
db.session.delete(s)
db.session.add(user)
db.session.commit()
return user.jsonify()
def recv_connect(self):
user = auth(self.request)
if not user:
self.request_reconnect()
else:
log("Received presence connection from %s" % user.username)
self.user = user
if not user.can("chat"):
body = {"message":"You don't have permission to chat."}
self.emit("disconnect", body)
self.send("disconnect")
log("%s isn't permitted to chat." % user.username)
def initialize(self):
log("Event stream init")
self.user = None
# Access to app.routes for
# admin users to access via the /eval command...
# "eval" is a nonexistent privilege that has to be created
# and associated in the database manually.
self.routes = app.routes
self.channels = {}
self.channel = None
self.modes = []
self.socket_type = "events"
# This lets us cycle through stream connections on
# the httpd and easily determine the session type.
#self.socket.socket_type = "events"
self.socket.appearing_offline = False
def initialize(self):
"""
Individual connections have timestamped versions of documents meaning
they're garbage collected when people disconnect and remaining users
are left with the official history.
"""
log("Document stream init")
self.user = None
self.fragments = []
self.documents = []
self.participants = []
self.channel = "" # Current URL.
self.socket_type = "document"
if 'channels' not in self.session:
self.session['channels'] = set()
def on_join(self, channel_name):
"""
Join a channel by name. If the channel name is the address of a friend
then we initiate an RPC_CHAT session to the remote host.
"""
if self.user and self.user.username:
if self.user.can("chat"):
if self.channel and channel_name == self.channel[1]: return
log("%s joined %s" % (self.user.username, channel_name))
channel = Channel(name=channel_name)
channel.clients.add(self)
self.channels[channel_name] = channel
self.channel = channel
self.join(channel_name)
# Send an "init" message via RPC_CHAT to the remote host
if channel_name.count("/") == 2:
network, node_id, uid = channel_name.split("/")
router = self.routes.get(network, None)
if router == None: return
friend = [f for f in self.user.friends if f.address == channel_name]
if not friend: return
friend = friend[0]
# if not friend.peer: # Return if no known pubkey
# return
data = {}
data['to'] = friend.uid
data['from'] = [self.user.uid, self.user.username]
data['type'] = "init"
data['body'] = ""
peer_node = friend.most_recent_peer_node(router)
#peer_node = Peer.query.filter(Peer.node_id == node_id).first()
print peer_node
if not peer_node:
log("No peer node associated with %s." % friend, "error")
return
resp = router.protocol.rpc_chat((peer_node.ip, peer_node.port), data)
if resp and "state" in resp and resp['state'] == "delivered":
self.emit("rpc_chat_init", resp)
def write_revision_to_disk(options):
# query for revision by hash
rev = Revision.query.filter(Revision.hash == options.write).first()
# query dht for hash
if not rev:
rev = app.routes._default[options.write]
if not rev:
log("Couldn't find %s locally or via the overlay network." % options.write, "error")
raise SystemExit
if not rev.size:
log("Revision contains no data.", "error")
raise SystemExit
output_path = options.out
if rev.resource and not output_path:
output_path = rev.resource.path.split('/')[-1]
if not output_path:
output_path = options.write
# write to resource name, index.html or options.out.
if not rev.content:
fd = open(output_path, 'wb')
else:
fd = open(output_path, 'w')
fd.write(rev.read())
fd.close()
log("Wrote %s to disk." % output_path)
raise SystemExit
def delete(self, username):
"Delete a session for a user"
user = auth(session, required=True)
parser = reqparse.RequestParser()
parser.add_argument("timestamp", type=str, help="session timestamp", required=True)
args = parser.parse_args()
if user.username != username and not user.can("delete_at_will"):
return {}, 304
target = User.query.filter(User.username == username).first()
if not target:
return {}, 404
timestamp = float(args.timestamp + '.0')
for s in user.sessions:
if time.mktime(s.created.timetuple()) == timestamp:
db.session.delete(s)
db.session.commit()
log("%s deleted a session for %s." % (user.username, target.username))
return {}, 204
def on_msg(self, msg):
"""
Handle sending a message to a local user or a friend on a remote instance.
"""
if self.user and self.channels.values():
if self.user.created:
body = {"u":self.user.username,"m":escape(msg)}
else:
body = {"u":self.user.username,"m":escape(msg),"a": True}
channel = self.channel
# Send message via RPC_CHAT to a remote host
if channel.count("/") == 2:
network, node_id, uid = channel.split("/")
router = self.routes.get(network, None)
if router == None: return
friend = [f for f in self.user.friends if f.address == channel]
if not friend: return
friend = friend[0]
# if not friend.peer: # Return if no known pubkey
# return
data = {}
data['to'] = friend.uid
data['from'] = [self.user.uid, self.user.username]
data['type'] = "message"
data['body'] = msg
# We use the peer node we know this friend to connect from but
# it's best to let the user select which specific node to use.
# For the time being we just go with whoever corresponds to the
# node ID section of Friend.address or the Friend.most_recent_*
# implementation.
peer_node = friend.most_recent_peer_node(router)
print peer_node
if not peer_node:
log("No peer node associated with %s." % friend, "error")
return
if peer_node.ip == router.node.ip and peer_node.port == router.node.port:
log("Cannot transmit message to ourselves.", "error")
return
resp = router.protocol.rpc_chat((peer_node.ip, peer_node.port), data)
if resp:
self.emit("privmsg", body)
return
log("Message to %s from %s: %s" % (channel, self.user.username, msg))
self.broadcast(channel, "privmsg", body)
self.emit("privmsg", body)
else:
self.request_reconnect()
def correct(soup, element):
for _ in soup.findAll(element):
if _.has_key("href"):
if _.has_key("license") and _['license'].lower() != "cc by":
log("Ignoring licensed object %s" % _['href'])
del _['href']
continue
log("%s -> %s%s%s" % (unicode(_['href']), request_endpoint, domain, unicode(_['href'])), "debug")
if _['href'].startswith('https'): _['href'] = _['href'].replace("https://", request_endpoint)
elif _['href'].startswith('http'): _['href'] = _['href'].replace("http://", request_endpoint)
elif _['href'].startswith('/'): _['href'] = '%s%s%s' % (request_endpoint, domain, _['href'])
else: _['href'] = '%s%s/%s' % (request_endpoint, domain, _['href'].encode("utf-8", "ignore"))
elif _.has_key("src"):
if _.has_key("license") and _['license'].lower() != "cc by":
log("Ignoring licensed object %s" % _['src'])
del _['src']
continue
log("%s -> %s%s%s" % (str(_['src']), request_endpoint, domain, str(_['src'])), "debug")
if _['src'].startswith('https'): _['src'] = _['src'].replace("https://", request_endpoint)
elif _['src'].startswith('http'): _['src'] = _['src'].replace("http://", request_endpoint)
elif _['src'].startswith('/'): _['src'] = '%s%s%s' % (request_endpoint, domain, _['src'])
else: _['src'] = '%s%s/%s' % (request_endpoint, domain, _['src'])
def get(url, user_agent, user=None):
"""
Return a Revision of a url.
Check for the presence of a URL
At its original location on the web.
In the DHT.
In the database.
"""
revision = Revision()
if user and not user.can("retrieve_resource"):
revision.status = 403
revision.mimetype = "text"
revision.content = "This user account isn't allowed to retrieve resources."
return revision
# Ignore for now if the addr is on our LAN, VLAN or localhost.
url = urlparse.urlparse(url)
domain = url.netloc
path = url.path or '/'
try: host = socket.gethostbyname(domain)
except: host = ''
if host:
log(host)
# Deep assumptions about the future of ipv4 here.
if any([host.startswith(subnet) for subnet in local_subnets]):
revision.status = 403
revision.mimetype = "text"
revision.content = "We're not currently proxying to local subnets."
return revision
# Check the web
response = None
try:
log("Fetching %s from the original domain." % url.geturl())
response = requests.get(url.geturl(), headers={'User-Agent': user_agent},
timeout=app.config['HTTP_TIMEOUT'])
except Exception, e:
log("Error retrieving %s: %s" % (url.geturl(), e.message))
def can(self, priv_name):
"""
Logs whether a user has an access right and returns True, None or False.
None here means the user wasn't a member of any groups associated with
the Priv being queried.
"""
log_message = "Checking whether %s can %s: " % (self.username, priv_name)
permission = []
priv = Priv.query.filter(Priv.name == priv_name).first()
if priv:
for r in self.user_groups:
for p in r.privs:
if p.priv_id == priv.id:
permission.append(p.allowed)
if any(permission):
log(log_message + "Yes.")
return True
else:
log(log_message + "No.")
return None
log(log_message + "No.")
return False
log("Fetching %s from the original domain." % url.geturl())
response = requests.get(url.geturl(), headers={'User-Agent': user_agent},
timeout=app.config['HTTP_TIMEOUT'])
except Exception, e:
log("Error retrieving %s: %s" % (url.geturl(), e.message))
if response:
revision.add(response)
if 'content-type' in response.headers:
revision.mimetype = response.headers['content-type']
revision.save(user, domain, path) # Calls to Revision.save will check
return revision # whether user.can("create_revision")
# Check an overlay network
if user and user.can("retrieve_from_dht"):
log("Fetching %s from network \"%s\"." % \
(url.geturl(), app.routes._default.network))
revision = app.routes._default[url.netloc + url.path]
if revision:
revision.public = True
revision.save(user, domain, path)
return revision
# Last but not least, check the database
domain_from_db = Domain.query.filter_by(name=domain).first()
rev = Revision.query.filter(
and_(Revision.resource.has(domain=domain_from_db),
Revision.resource.has(path=path))
).order_by(desc(Revision.created)).first()
log(domain_from_db)
if rev:
return rev
def on_join(self, channel):
log('%s has subscribed to the document stream for "%s".' % \
(self.user.username, channel))
self.join(channel)
self.broadcast(self.channel, "join", self.user.jsonify())
def post(self):
"""
Unserialise the data and accept the following calls:
PING
CHAT
EDIT
APPEND
LEAVING
FIND_NODE
FIND_VALUE
APPEND signifies the requesting host has data for the given hash.
"""
parser = restful.reqparse.RequestParser()
parser.add_argument("data", type=str, help="The RPC body.", default=None)
# parser.add_argument("signature", type=str, help="Signature.", default=None)
# parser.add_argument("pubkey", type=str, help="Public key.", default=None)
args = parser.parse_args()
if not args.data:
log("Received request from %s with no data." % request.remote_addr, "warning")
return {}, 400
response = []
data = json.loads(args.data)
if not validate_signature(data):
log("Received message from %s with an invalid signature." % request.remote_addr, "warning")
return "Invalid message signature.", 400
if not 'node' in data:
log("%s didn't provide useable information about themselves." % request.remote_addr, "warning")
return {}, 400
# Validate the node field for internet hosts
if not any([request.remote_addr.startswith(subnet) for subnet in local_subnets]):
stated_addr = data['node'][1]
if stated_addr != request.remote_addr:
log("Request made from %s stated it originated from %s" % (request.remote_addr, stated_addr), "warning")
return "sicillian shrug", 418
# Ensure this peer is using the node ID that corresponds to their ip, port and public key
seed = "%s:%i:%s" % (data['node'][1], data['node'][2], data['pubkey'])
expected_id = long(generate_node_id(seed).encode('hex'), 16)
if data['node'][0] != expected_id:
log("%s is using an incorrect node ID." % request.remote_addr, "warning")
log("Expecting %s but received %s" % (str(expected_id), str(data['node'][0])), "warning")
return {}, 400
# Determine which overlay network this request is concerned with
if not 'network' in data:
return {}, 400
router = app.routes.get(data['network'], None)
if router is None:
return {}, 404
# Execute the corresponding RPC handler
for field in data.keys():
if field.startswith('rpc_'):
rpc_name = 'handle_%s' % field.replace('rpc_', '')
# data[rpc_name] = data[field]
# del data[field]
rpc_method = getattr(router.protocol, rpc_name, None)
if not rpc_method:
log("%s tried to call unknown procedure %s." % (request.remote_addr, rpc_name), "warning")
return {}, 400
response = rpc_method(data)
break
return response
def put(self, username):
"""
Add a friend on a remote instance.
"""
user = auth(session, required=True)
parser = reqparse.RequestParser()
parser.add_argument("name", type=str, default="")
parser.add_argument("address", type=str, required=True)
args = parser.parse_args()
# Parse the address and contact the node in question.
# node = NodeSpider(node)
if args.address.count("/") != 2:
return False
network_name, node_id, remote_uid = args.address.split("/")
router = app.routes.get(network_name, None)
if router == None:
return {}, 404
log("%s is adding a remote friend." % user.username)
request = {"add": {"from": user.uid, "to": args.address}}
response, node = router.protocol.rpc_friend(request)
if not response:
return {}, 404
if Friend.query.filter(and_(Friend.address == args.address,
Friend.user == user)).first():
return {}, 304
# Add a peer instance so the remote sides' public key is
# accesible for encrypting data we transmit to this friend.
network = Network.query.filter(Network.name == network_name).first()
if network != None:
network = Network(name = network_name)
peer = Peer.query.filter(
and_(Peer.network == network,
Peer.ip == node.ip,
Peer.port == node.port)
).first()
if peer == None:
peer = Peer()
peer.load_node(node)
friend = Friend(address=args.address)
friend.name = args.name
friend.state = 1
user.friends.append(friend)
peer.pubkey.friends.append(friend)
db.session.add(peer)
db.session.add(network)
db.session.add(user)
db.session.add(friend)
db.session.commit()
return response, 201
def request_reconnect(self):
"""
Shortcut for asking a client to reconnect.
"""
log("Received chat connection before authentication. Requesting client reconnects.")
self.emit("reconnect", {"m":"Reconnecting.."})
def init_user_groups():
"""
Administrators:
see_all
delete_at_will
reset_user_pw
modify_user
modify_usergroup
deactivate
manage_networks
review_downloads
toggle_signups
Users:
chat
initiate_rtc
create_revision
retrieve_from_dht
browse_peer_nodes
retrieve_resource
stream_document
There's also a secret privilege called "eval" that we don't create but is
checked for in streams.chat.on_cmd. Perhaps make a special UserGroup for
yourself.
"""
# Bans happen by setting User.active to False and clearing their existing sessions.
groups = ["Administrators", "Users"]
admin_privs = [ "see_all", "delete_at_will", "reset_user_pw", "modify_user",
"modify_usergroup", "deactivate", "manage_networks",
"review_downloads", "toggle_signups"]
privs = [ "create_revision_group", "delete_revision_group", "chat",
"initiate_rtc", "create_revision", "retrieve_from_dht",
"browse_peer_nodes", "retrieve_resource", "stream_document"]
privs.extend(admin_privs)
if not Priv.query.first():
log("Creating privileges.")
for priv in privs:
p = Priv.query.filter(Priv.name == priv).first()
if not p:
p = Priv(name=priv)
db.session.add(p)
db.session.commit()
for group in groups:
g = UserGroup.query.filter(UserGroup.name == group).first()
if not g:
g = UserGroup(name=group)
for p in Priv.query.all():
if group != "Administrators" and p.name in admin_privs:
continue
a = Acl()
a.group = g
a.priv = p
a.allowed = True
db.session.add(a)
db.session.add(p)
db.session.commit()
db.session.add(g)
db.session.commit()
log("Created user group \"%s\"." % group)
if pid > 0:
sys.exit(0) # parent
except OSError, e:
sys.stderr.write("fork #1 failed: %d (%s)\n" % (e.errno, e.strerror))
sys.exit(-2)
os.setsid()
os.umask(0)
try:
pid = os.fork()
if pid > 0:
try:
f = file(pidfile, 'w')
f.write(str(pid))
f.close()
except IOError, err:
log(err,'error')
sys.exit(0) # parent
except OSError, e:
sys.stderr.write("fork #2 failed: %d (%s)\n" % (e.errno, e.strerror))
sys.exit(-2)
for fd in (0, 1, 2):
try:
os.close(fd)
except OSError: pass
if __name__ == "__main__":
epilog = "Available test suites: %s" % ', '.join([test for test in maps.keys()])
parser = optparse.OptionParser(epilog=epilog)
parser.add_option("-p", "--port", dest="port", action="store", default=8080, help="(defaults to 8080)")
parser.add_option("--key", dest="key", action="store", default="id_rsa", help="(defaults to id_rsa)")
parser.add_option("-c", "--config", dest="config", action="store", default='synchrony.config')