def handshake(self):
data = ""
for attempt in xrange(0, 20):
data += os.read(0, 1024)#.strip()
data = data.replace("\n\n", "\r\n")
with file('/tmp/log', 'w') as f:
f.write(data)
try:
headers = Message(StringIO(data.split('\r\n', 1)[1]))
if headers.get("Upgrade", None) == "websocket":
break
except:
continue
headers = Message(StringIO(data.split('\r\n', 1)[1]))
if headers.get("Upgrade", None) == "websocket":
return
key = headers['Sec-WebSocket-Key']
digest = b64encode(sha1(key + self.magic).hexdigest().decode('hex'))
response = 'HTTP/1.1 101 Switching Protocols\r\n'
response += 'Upgrade: websocket\r\n'
response += 'Connection: Upgrade\r\n'
response += 'Sec-WebSocket-Accept: %s\r\n\r\n' % digest
self.handshake_done = os.write(1, response)
def test_spamc_unix_process(self):
with open(self.filename) as handle:
result = self.spamc_unix.process(handle)
self.assertIn('message', result)
with open(self.filename) as headerhandle:
headers1 = Message(headerhandle)
headers2 = Message(StringIO(result['message']))
self.assertEqual(headers1.get('Subject'), headers2.get('Subject'))
def test_spamc_tcp_process(self):
with open(self.filename) as _handle:
data = _handle.read()
handle = StringIO(data)
result = self.spamc_tcp.process(handle)
self.assertIn('message', result)
with open(self.filename) as headerhandle:
headers1 = Message(headerhandle)
headers2 = Message(StringIO(result['message']))
self.assertEqual(headers1.get('Subject'), headers2.get('Subject'))
def test_spamc_tcp_process(self):
with open(self.filename) as _handle:
data = _handle.read()
handle = StringIO(data)
result = self.spamc_tcp.process(handle)
self.assertIn("message", result)
with open(self.filename) as headerhandle:
headers1 = Message(headerhandle)
headers2 = Message(StringIO(result["message"]))
self.assertEqual(headers1.get("Subject"), headers2.get("Subject"))
def test_spamc_unix_process(self):
with open(self.filename) as handle:
result = self.spamc_unix.process(handle)
self.assertIn('message', result)
with open(self.filename) as headerhandle:
headers1 = Message(headerhandle)
headers2 = Message(StringIO(result['message']))
self.assertEqual(
headers1.get('Subject'),
headers2.get('Subject')
)
def handshake(self):
data = self.request.recv(1024).strip()
new_data = data.split('\r\n', 1)
if not new_data:
return
headers = Message(StringIO(data.split('\r\n', 1)[1]))
if headers.get("Upgrade", None) == "Websocket" or headers.get("Upgrade", None) == "websocket":
key = headers['Sec-WebSocket-Key']
digest = b64encode(sha1(key + self.magic).hexdigest().decode('hex'))
response = 'HTTP/1.1 101 Switching Protocols\r\n'
response += 'Upgrade: websocket\r\n'
response += 'Connection: Upgrade\r\n'
response += 'Sec-WebSocket-Accept: %s\r\n\r\n' % digest
self.handshake_done = self.request.send(response)
self.valid_client = True
def _perform_handshake(self):
"""Perform The WebSocket Handshake"""
try:
Log.add("Got To Handshake")
data = self.recv(1024).strip()
# Log.add("Data: %s" % data)
headers = Message(StringIO(data.split('\r\n', 1)[1]))
Log.add("Parsed Headers:")
# Log.add(headers)
if headers.get('Upgrade', None) == 'websocket':
Log.add("Attempting Handshake")
# create response key
key = b64encode(sha1(headers['Sec-WebSocket-Key'] + self.SALT).digest())
# create response headers
response = (
"HTTP/1.1 101 Web Socket Protocol Handshake\r\n"
"Upgrade: websocket\r\n"
"Connection: Upgrade\r\n"
"Sec-WebSocket-Origin: %s\r\n"
"Sec-WebSocket-Accept: %s\r\n\r\n" % (headers["Origin"], key)
)
if self.send_bytes(response):
Log.add("Handshake successful")
self._assign_room(data)
self._ready_state = "authenticating"
except Exception as e:
Log.add(e.args)
def _perform_handshake(self):
"""Perform The WebSocket Handshake"""
try:
Log.add("Got To Handshake")
data = self.recv(1024).strip()
# Log.add("Data: %s" % data)
headers = Message(StringIO(data.split('\r\n', 1)[1]))
Log.add("Parsed Headers:")
# Log.add(headers)
if headers.get('Upgrade', None) == 'websocket':
Log.add("Attempting Handshake")
# create response key
key = b64encode(
sha1(headers['Sec-WebSocket-Key'] + self.SALT).digest())
# create response headers
response = ("HTTP/1.1 101 Web Socket Protocol Handshake\r\n"
"Upgrade: websocket\r\n"
"Connection: Upgrade\r\n"
"Sec-WebSocket-Origin: %s\r\n"
"Sec-WebSocket-Accept: %s\r\n\r\n" %
(headers["Origin"], key))
if self.send_bytes(response):
Log.add("Handshake successful")
self._assign_room(data)
self._ready_state = "authenticating"
except Exception as e:
Log.add(e.args)
def handshake(self):
data = self.sock.readTo('\r\n\r\n').strip()
headers = Message(StringIO(data.split('\r\n', 1)[1]))
if headers.get('Upgrade', None).lower() != 'websocket':
print self.client_address, 'missing header "Upgrade: websocket"'
return False
self.client_ip = headers.get('X-Forwarded-For', self.client_address[0])
key = headers['Sec-WebSocket-Key']
digest = b64encode(sha1(key + self.magic).hexdigest().decode('hex'))
response = 'HTTP/1.1 101 Switching Protocols\r\n'
response += 'Upgrade: websocket\r\n'
response += 'Connection: Upgrade\r\n'
response += 'Sec-WebSocket-Accept: %s\r\n\r\n' % digest
self.sock.send(response)
self.on_handshake()
return True
def handshake(self):
log.debug("Request")
data = self.request.recv(1024).strip()
headers = Message(StringIO(data.split('\r\n', 1)[1]))
log.debug("Headers")
if headers.get("Upgrade", None) != "websocket":
return
log.debug("Handshake")
key = headers['Sec-WebSocket-Key']
digest = b64encode(sha1(key + self.magic).hexdigest().decode('hex'))
response = 'HTTP/1.1 101 Switching Protocols\r\n'
response += 'Upgrade: websocket\r\n'
response += 'Connection: Upgrade\r\n'
response += 'Sec-WebSocket-Accept: %s\r\n\r\n' % digest
self.handshake_done = self.request.send(response)
global clients
clients += [self];
# Send over a full delta
combo = {
"name": "combo",
"connections": connections,
"links": links
}
c = json.dumps(combo)
self.send_message(c)
def _receive_handshake(self):
while True:
buf = self.sock.recv(2048)
if "\r\n\r\n" in buf: break
headers, buf = buf.split("\r\n\r\n", 1)
status_line, headers = headers.split("\r\n", 1)
headers = Message(StringIO(headers))
if (status_line != 'HTTP/1.1 101 Web Socket Protocol Handshake'
or headers.get('Connection') != 'Upgrade'
or headers.get('Upgrade') != 'WebSocket'):
raise WebSocketError('Invalid handshake')
return buf
def handshake(self):
try:
data = self.client.recv(1024).strip()
headers = Message(StringIO(data.split('\r\n', 1)[1]))
if headers.get("Upgrade", None) != "websocket":
return
print 'Handshaking with web application...'
key = headers['Sec-WebSocket-Key']
digest = b64encode(
sha1(key + self.magic).hexdigest().decode('hex'))
response = 'HTTP/1.1 101 Switching Protocols\r\n'
response += 'Upgrade: websocket\r\n'
response += 'Connection: Upgrade\r\n'
response += 'Sec-WebSocket-Accept: %s\r\n\r\n' % digest
self.client.send(response)
# Have to make sure the handshake is successful before we add
# it as a client thread or it will get lots of data dumps before
# connection is ready. Doing it here so it doesn't hold up the main thread.
self.server.serverLock.acquire()
self.server.webthreads.append(self)
self.server.serverLock.release()
print "added thread"
self.handshake_done = "websocket"
except Exception as e:
print "Error in handshake: %s" % (str(e))
print traceback.format_exc()
self.kill()
def handshake(self):
data = self.request.recv(1024).strip()
print data
dSplit = data.split('\r\n', 1)
if len(dSplit) > 1:
headers = Message(StringIO(data.split('\r\n', 1)[1]))
else:
headers = Message(StringIO(data.split('\r\n', 1)))
if headers.get("Upgrade", None).lower() != "websocket":
print "no upgrade"
return
print 'Handshaking...'
try:
key = headers['Sec-WebSocket-Key']
digest = b64encode(
sha1(key + self.magic).hexdigest().decode('hex'))
except KeyError:
self.hasSecKey = False
print "no Sec-WebSocket-Key"
response = 'HTTP/1.1 101 Switching Protocols\r\n'
response += 'Upgrade: websocket\r\n'
response += 'Connection: Upgrade\r\n'
if (self.hasSecKey):
response += 'Sec-WebSocket-Accept: %s\r\n\r\n' % digest
self.handshake_done = self.request.send(response)
def handshake(self):
data = self.channel.recv(1024).strip()
new_data = data.split('\r\n', 1)
if not new_data:
return
headers = Message(StringIO(data.split('\r\n', 1)[1]))
if headers.get("Upgrade", None) == "Websocket" or headers.get(
"Upgrade", None) == "websocket":
key = headers['Sec-WebSocket-Key']
digest = b64encode(
sha1(key + self.magic).hexdigest().decode('hex'))
response = 'HTTP/1.1 101 Switching Protocols\r\n'
response += 'Upgrade: websocket\r\n'
response += 'Connection: Upgrade\r\n'
response += 'Sec-WebSocket-Accept: %s\r\n\r\n' % digest
self.handshake_done = self.channel.send(response)
self.valid_client = True
def fetch(self, url, redirections=None):
if not self.is_fetch_allowed_by_robots_txt(url):
print('Skipped: %s' % url)
return FetchResult(headers='', content='', status=403, filepath=None)
try:
digest = hashlib.sha256(url.encode('utf-8')).digest()
filepath = os.path.join(self.cache_dir,
"f" + base64.urlsafe_b64encode(digest).decode('utf-8'))
except:
digest = hashlib.sha256(url).digest()
filepath = os.path.join(self.cache_dir,
"f" + base64.urlsafe_b64encode(digest))
try:
with open(filepath, 'r', encoding='utf-8-sig', newline='') as f:
cached = f.read().split('\r\n\r\n\r\n', 1)
if len(cached) == 2:
print('Cache-Hit: %s' % url)
headers, content = cached
try:
content = content.encode('utf-8')
except:
# already encoded as bytes
pass
if py3:
headers = Message(headers)
else:
headers = Message(StringIO(headers))
status = int(headers.get('Status', '200').split()[0])
return FetchResult(headers, content, status, filepath)
except IOError:
pass
print('Downloading: %s' % url)
delay = random.uniform(self.crawldelay, self.crawldelay + 2) # jitter
time.sleep(delay)
request = Request(url, headers={'User-Agent': self.useragent})
context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
try:
response = urlopen(request, context=context)
except HTTPError as err:
response = err
except UnicodeDecodeError as err:
# The Arabic edition of Sputnik News sometimes emits redirects
# which the urllib2 module of Python 2.7.13 tries to follow,
# but then it causes an urllib2-internal crash. We catch the
# error here, and return it as '400 Bad Request' to our caller.
return FetchResult({}, 'Bad Request: UnicodeDecodeError', 400, None)
status = response.getcode()
content = response.read()
response.close()
if status == 200 or status >= 400 and status <= 499:
with open(filepath, 'wb') as f:
f.write(b'Status: %d\r\n' % response.getcode())
f.write(str(response.headers).rstrip().encode('utf-8'))
f.write(b'\r\n\r\n\r\n')
f.write(content)
return FetchResult(response.headers, content, status, filepath)
def test_spamc_unix_headers(self):
with open(self.filename) as handle:
result = self.spamc_unix.headers(handle)
self.assertIn('message', result)
with open(self.filename) as headerhandle:
headers = Message(headerhandle)
org_subject = "Subject: %s" % headers.get('Subject')
new_subject = "Subject: %s" % result['headers'].get('Subject')
self.assertEqual(org_subject, new_subject)
def _receive_handshake(self):
while True:
buf = self.sock.recv(2048)
if "\r\n\r\n" in buf: break
headers, buf = buf.split("\r\n\r\n", 1)
status_line, headers = headers.split("\r\n", 1)
headers = Message(StringIO(headers))
if (
(not status_line.startswith('HTTP/1.1 101'))
or headers.get('Connection') != 'Upgrade'
or headers.get('Upgrade') != 'WebSocket'
):
raise WebSocketError('Invalid handshake')
return buf.split("fQJ,fN/4F4!~K~MH")[-1]
def _receive_handshake(self):
while True:
buf = self.sock.recv(2048)
if "\r\n\r\n" in buf: break
headers, buf = buf.split("\r\n\r\n", 1)
status_line, headers = headers.split("\r\n", 1)
headers = Message(StringIO(headers))
if (
status_line != 'HTTP/1.1 101 Web Socket Protocol Handshake'
or headers.get('Connection') != 'Upgrade'
or headers.get('Upgrade') != 'WebSocket'
):
raise WebSocketError('Invalid handshake')
return buf
def test_spamc_unix_headers(self):
with open(self.filename) as handle:
result = self.spamc_unix.headers(handle)
self.assertIn('message', result)
with open(self.filename) as headerhandle:
headers = Message(headerhandle)
org_subject = "Subject: %s" % headers.get('Subject')
new_subject = "Subject: %s" % result['headers'].get('Subject')
self.assertEqual(org_subject, new_subject)
def test_spamc_tcp_headers(self):
with open(self.filename) as _handle:
data = _handle.read()
handle = StringIO(data)
result = self.spamc_tcp.headers(handle)
self.assertIn('message', result)
with open(self.filename) as headerhandle:
headers = Message(headerhandle)
org_subject = "Subject: %s" % headers.get('Subject')
new_subject = "Subject: %s" % result['headers'].get('Subject')
self.assertEqual(org_subject, new_subject)
def test_spamc_tcp_headers(self):
with open(self.filename) as _handle:
data = _handle.read()
handle = StringIO(data)
result = self.spamc_tcp.headers(handle)
self.assertIn("message", result)
with open(self.filename) as headerhandle:
headers = Message(headerhandle)
org_subject = "Subject: %s" % headers.get("Subject")
new_subject = "Subject: %s" % result["headers"].get("Subject")
self.assertEqual(org_subject, new_subject)
def handshake(self):
data = self.request.recv(1024).strip()
headers = Message(StringIO(data.split('\r\n', 1)[1]))
if headers.get("Upgrade", None) != "websocket":
return
print 'Handshaking...'
key = headers['Sec-WebSocket-Key']
digest = b64encode(sha1(key + self.magic).hexdigest().decode('hex'))
response = 'HTTP/1.1 101 Switching Protocols\r\n'
response += 'Upgrade: websocket\r\n'
response += 'Connection: Upgrade\r\n'
response += 'Sec-WebSocket-Accept: %s\r\n\r\n' % digest
self.handshake_done = self.request.send(response)
def handshake(self):
data = self.request.recv(1024).strip()
headers = Message(StringIO(data.split('\r\n', 1)[1]))
if headers.get("Upgrade", None) != "websocket":
return
print "\r", self.thread.getName(), 'Handshaking'
key = headers['Sec-WebSocket-Key']
digest = b64encode(sha1(key + self.magic).hexdigest().decode('hex'))
response = 'HTTP/1.1 101 Switching Protocols\r\n'
response += 'Upgrade: websocket\r\n'
response += 'Connection: Upgrade\r\n'
response += 'Sec-WebSocket-Accept: %s\r\n\r\n' % digest
self.handshake_done = self.request.send(response)
def handshake(self):
print "handshake"
data = self.request.recv(1024).strip()
dSplit = data.split('\r\n', 1)
if len(dSplit) > 1:
headers = Message(StringIO(data.split('\r\n', 1)[1]))
else:
headers = Message(StringIO(data.split('\r\n', 1)))
if headers.get("Upgrade", None) == None:
return
if headers.get("Upgrade", None).lower() != "websocket":
return
try:
key = headers['Sec-WebSocket-Key']
digest = b64encode(
sha1(key + self.magic).hexdigest().decode('hex'))
print "has key"
except KeyError:
self.hasSecKey = False
print "no Sec-WebSocket-Key"
response = 'HTTP/1.1 101 Switching Protocols\r\n'
response += 'Upgrade: websocket\r\n'
response += 'Connection: Upgrade\r\n'
#this is also where we can distinguish a wifly from a browers
if (self.hasSecKey):
response += 'Sec-WebSocket-Accept: %s\r\n\r\n' % digest
print "sending back handshake"
self.handshake_done = self.request.send(response)
if (self.hasSecKey):
self.server.addBrowser(self)
else:
self.server.addWiFly(self)
def handshake(self):
print "handshake"
data = self.request.recv(1024).strip()
dSplit = data.split('\r\n', 1)
if len(dSplit) > 1 :
headers = Message(StringIO(data.split('\r\n', 1)[1]))
else:
headers = Message(StringIO(data.split('\r\n', 1)))
if headers.get("Upgrade", None) == None:
return
if headers.get("Upgrade", None).lower() != "websocket":
return
try:
key = headers['Sec-WebSocket-Key']
digest = b64encode(sha1(key + self.magic).hexdigest().decode('hex'))
print "has key"
except KeyError:
self.hasSecKey = False
print "no Sec-WebSocket-Key"
response = 'HTTP/1.1 101 Switching Protocols\r\n'
response += 'Upgrade: websocket\r\n'
response += 'Connection: Upgrade\r\n'
#this is also where we can distinguish a wifly from a browers
if(self.hasSecKey):
response += 'Sec-WebSocket-Accept: %s\r\n\r\n' % digest
print "sending back handshake"
self.handshake_done = self.request.send(response)
if(self.hasSecKey):
self.server.addBrowser(self)
else:
self.server.addWiFly(self)
def handshake(self, data):
print(data)
headers = Message(StringIO(data.strip().split('\r\n', 1)[1]))
if headers.get("Upgrade", None) != "websocket":
return False
logging.debug("handshaking...")
key = headers['Sec-WebSocket-Key']
digest = b64encode(sha1(key + self.magic).hexdigest().decode('hex'))
response = 'HTTP/1.1 101 Switching Protocols\r\n'
response += 'Upgrade: websocket\r\n'
response += 'Connection: Upgrade\r\n'
response += 'Sec-WebSocket-Accept: %s\r\n\r\n' % digest
#self.handshake_done = self.request.send(response)
self.out_buffer = response
return True
def handshake(self):
data = self.request.recv(1024).strip()
headers = Message(StringIO(data.split('\r\n', 1)[1]))
self.originally_recieved_headers = headers
if headers.get("Upgrade", None) != "websocket":
return
print 'Handshaking...'
key = headers['Sec-WebSocket-Key']
digest = b64encode(sha1(key + self.magic).hexdigest().decode('hex'))
response = 'HTTP/1.1 101 Switching Protocols\r\n'
response += 'Upgrade: websocket\r\n'
response += 'Connection: Upgrade\r\n'
response += 'Sec-WebSocket-Accept: %s\r\n\r\n' % digest
self.handshake_done = self.request.send(response)
if self.handshake_done:
Thread(target=self.send_notif_to_client).start()
def handshake(self):
data = self.request.recv(1024).strip()
headers = Message(StringIO(data.split('\r\n', 1)[1]))
if headers.get("Upgrade", None) != "websocket":
return
print 'Handshaking...'
key = headers['Sec-WebSocket-Key']
digest = b64encode(sha1(key + self.magic).hexdigest().decode('hex'))
response = 'HTTP/1.1 101 Switching Protocols\r\n'
response += 'Upgrade: websocket\r\n'
response += 'Connection: Upgrade\r\n'
response += 'Sec-WebSocket-Accept: %s\r\n\r\n' % digest
self.handshake_done = self.request.send(response)
if self.handshake_done:
me = {}
me["type"]="m"
me["time"]=time.time()
me["x"]=0
me["y"]=0
me["d"]=0
me["v"]=0
players.append(self)
me["pid"]=players.index(self)
memsg=jsone.encode(me)
for p in players:
if not p == self:
other = {}
other["type"]="m"
other["time"]=p.lastUpdate
other["x"]=p.lastX
other["y"]=p.lastY
other["d"]=p.lastDir
other["v"]=p.lastV
other["pid"]=players.index(p)
self.send_message(jsone.encode(other))
p.send_message(memsg)
tellAboutAsteroids(self)
tellAboutBot(self)
print "Player connected ("+str(len(players))+")"
def handshake(self):
data = self.request.recv(1024)
headers = Message(StringIO(data.split("\r\n", 1)[1]))
body = data.split("\r\n\r\n")[1]
upgrade = headers.get("Upgrade", "")
if upgrade.lower() != "websocket":
print "WebSockets client " + self.client_address[0] + " wrong Upgrade"
return
if headers.getheader("Sec-WebSocket-Key") != None:
self.ws_rfc = True
response = self.handshake_singleKey(headers)
elif (headers.getheader("Sec-WebSocket-Key1") != None) and (headers.getheader("Sec-WebSocket-Key2") != None):
self.ws_rfc = False
response = self.handshake_twoKeys(headers, body)
else:
print "WebSockets client " + self.client_address[0] + " wrong structure"
return
self.handshake_done = self.request.sendall(response)
self.server.clients += [self]
print "WebSockets client " + self.client_address[0] + " handshaken"
def process_archive(self, peer, sender, mail_options, recips, rcptopts,
data):
"""Archives email meta data using a Backend"""
LOG(
E_INFO, '%s: Sender is <%s> - Recipients (Envelope): %s' %
(self.type, sender, ','.join(recips)))
size = len(data)
if size < MINSIZE:
return self.do_exit(550, 'Invalid Mail')
if not data.endswith(NL):
data = data + NL
args = {}
aid = None
mid = None
stream = StringIO(data)
msg = Message(stream)
if sender == '':
LOG(E_INFO,
'%s: Null return path mail, not archived' % (self.type))
return self.sendmail('<>', mail_options, recips, rcptopts,
data, aid)
## Check if I have msgid in my cache
mid = msg.get('message-id', self.new_mid())
hash = hash_headers(msg.get)
if self.hashdb.has_key(hash):
LOG(E_TRACE, '%s: Message-id: %s' % (self.type, mid))
aid = self.hashdb[hash]
LOG(
E_TRACE,
'%s: Message already has year/pid pair, only adding header'
% self.type)
return self.sendmail(sender, mail_options, recips, rcptopts,
self.add_aid(data, msg, aid), aid, hash)
args['m_mid'] = mid
args['hash'] = hash
## Check for duplicate headers
dupe = dupe_check(msg.headers)
if dupe is not None:
LOG(E_ERR, '%s: Duplicate header %s' % (self.type, dupe))
return self.do_exit(552, 'Duplicate header %s' % dupe)
## Extraction of From field
m_from = msg.getaddrlist('From')
if len(m_from) == 1:
m_from = safe_parseaddr(m_from[0][1])
else:
m_from = None
## Empty or invalid 'From' field, try to use sender
if m_from is None:
LOG(E_ERR,
'%s: no From header in mail using sender' % self.type)
m_from = safe_parseaddr(sender)
## No luck
if m_from is None:
return self.do_exit(552, 'Mail has not suitable From/Sender')
args['m_from'] = m_from
## Extract 'To' field
m_to = []
for h in msg.getaddrlist('To'):
rec = safe_parseaddr(h[1])
if rec is None: continue
m_to.append(rec)
## Empty 'To' field use recipients
if len(m_to) == 0:
LOG(E_ERR,
'%s: no To header in mail using recipients' % self.type)
for recipient in recips:
rec = safe_parseaddr(recipient)
if rec is None:
continue
m_to.append(rec)
if len(m_to) == 0:
return self.do_exit(552,
'Mail has not suitable To/Recipient')
## Extract 'Cc' field
for h in msg.getaddrlist('Cc'):
rec = safe_parseaddr(h[1])
if rec is None: continue
m_to.append(rec)
## Cleanup: remove duplicates
recs = []
for rec in m_to:
if rec not in recs:
recs.append(rec)
args['m_rec'] = recs
## Extract 'Subject' field
m_sub = mime_decode_header(msg.get('Subject', 'No Subject'))
if subjpattern is not None and m_sub.find(subjpattern) != -1:
LOG(E_INFO,
'%s: Subject pattern matched, not archived' % self.type)
return self.sendmail(sender, mail_options, recips, rcptopts,
self.remove_aid(data, msg))
args['m_sub'] = m_sub
## Whitelist check: From, To and Sender (envelope)
checklist = [m_from] + m_to
ss = safe_parseaddr(sender)
if ss is not None:
checklist.append(ss)
for check in checklist:
if check.split('@', 1)[0] in whitelist:
LOG(
E_INFO, '%s: Mail to: %s in whitelist, not archived' %
(self.type, check))
return self.sendmail(sender, mail_options, recips,
rcptopts, self.remove_aid(data, msg))
## Sender size limit check - in kb
if dbchecker is not None and dbchecker.quota_check(
m_from, size >> 10):
return self.do_exit(422, 'Sender quota execeded')
args['m_size'] = size
## Extract 'Date' field
m_date = None
if self.datefromemail:
m_date = msg.getdate('Date')
try:
mktime(m_date)
except:
m_date = None
if m_date is None:
m_date = localtime(time())
args['m_date'] = m_date
m_attach = []
if msg.maintype != 'multipart':
m_parse = parse_message(msg)
if m_parse is not None:
m_attach.append(m_parse)
else:
filepart = MultiFile(stream)
filepart.push(msg.getparam('boundary'))
try:
while filepart.next():
submsg = Message(filepart)
subpart = parse_message(submsg)
if subpart is not None:
m_attach.append(subpart)
except:
LOG(E_ERR, '%s: Error in multipart splitting' % self.type)
args['m_attach'] = m_attach
if dbchecker is not None:
## Collect data for mb lookup
addrs = []
for addr in [m_from] + m_to:
addrs.append(addr)
args['m_mboxes'] = dbchecker.mblookup(addrs)
else:
args['m_mboxes'] = []
year, pid, error = self.backend.process(args)
if year == 0:
LOG(E_ERR, '%s: Backend Error: %s' % (self.type, error))
return self.do_exit(pid, error)
## Adding X-Archiver-ID: header
aid = '%d-%d' % (year, pid)
data = self.add_aid(data, msg, aid)
LOG(E_TRACE, '%s: inserting %s msg in hashdb' % (self.type, aid))
self.hashdb[hash] = aid
self.hashdb.sync()
## Next hop
LOG(E_TRACE, '%s: backend worked fine' % self.type)
LOG(
E_TRACE, '%s: passing data to nexthop: %s:%s' %
(self.type, self.output_address, self.output_port))
return self.sendmail(sender, mail_options, recips, rcptopts, data,
aid, hash)
def _websocket_client_handshake(self):
buf = []
while True:
fds = yield
if self.socket in fds:
try:
data = self._recv(BUF_SIZE)
except socket.error as e:
handle_socket_error(e)
else:
buf.append(data)
score = 0
for char in itertools.chain(*buf):
if char in ('\r', '\n'):
score += 1
if score == 4:
break
else:
score = 0
else:
continue
break
header, body = ''.join(buf).split('\r\n\r\n', 1)
headers = Message(StringIO(header.split(b'\r\n', 1)[1]))
if headers.get(b'Upgrade', '').lower() != b'websocket':
raise protocol.ConnectionClosed()
# common headers:
response = b''.join(
(b'HTTP/1.1 101 Switching Protocols\r\n',
b'Upgrade: websocket\r\n', b'Connection: Upgrade\r\n'))
# if override_host:
# headers[b'host'] = override_host
if b'Sec-WebSocket-Key1' in headers and \
b'Sec-WebSocket-Key2' in headers:
self._decode_message = self._decode_hixie
accept = hashlib.md5(b''.join((
struct.pack(
b'>II',
*list(
int(
b''.join(x for x in headers[key]
if x in string.digits), 10) /
headers[key].count(b' ')
for key in self.HIXIE_WEBSOCKET_SEC_KEY_ORDER)),
body,
))).digest()
response = b''.join((
response,
b'Sec-WebSocket-Origin: %s\r\n' % headers[b'origin'],
b'Sec-WebSocket-Location: ws://%s/\r\n' % headers[b'host'],
b'\r\n%s' % str(accept),
))
self._send(response)
self._send = self._send_hixie
else:
self._decode_message = self._decode_rfc
secret_key = \
b''.join((headers[b'Sec-WebSocket-Key'], self.RFC_MAGIC_HASH,))
digest = b64encode(
hashlib.sha1(secret_key).hexdigest().decode(b'hex'))
response = \
b''.join((response, b'Sec-WebSocket-Accept: %s\r\n\r\n' % digest,))
self._send(response)
self._send = self._send_rfc
print("Ready for reading.")
self.state = self.READ_WHEN
def process_archive(self, peer, sender, mail_options, recips, rcptopts, data):
"""Archives email meta data using a Backend"""
LOG(E_INFO, "%s: Sender is <%s> - Recipients (Envelope): %s" % (self.type, sender, ",".join(recips)))
size = len(data)
if size < MINSIZE:
return self.do_exit(550, "Invalid Mail")
if not data.endswith(NL):
data = data + NL
args = {}
aid = None
mid = None
stream = StringIO(data)
msg = Message(stream)
if sender == "":
LOG(E_INFO, "%s: Null return path mail, not archived" % (self.type))
return self.sendmail("<>", mail_options, recips, rcptopts, data, aid)
## Check if I have msgid in my cache
mid = msg.get("message-id", self.new_mid())
hash = hash_headers(msg.get)
if self.hashdb.has_key(hash):
LOG(E_TRACE, "%s: Message-id: %s" % (self.type, mid))
aid = self.hashdb[hash]
LOG(E_TRACE, "%s: Message already has year/pid pair, only adding header" % self.type)
return self.sendmail(sender, mail_options, recips, rcptopts, self.add_aid(data, msg, aid), aid, hash)
args["m_mid"] = mid
args["hash"] = hash
## Check for duplicate headers
dupe = dupe_check(msg.headers)
if dupe is not None:
LOG(E_ERR, "%s: Duplicate header %s" % (self.type, dupe))
return self.do_exit(552, "Duplicate header %s" % dupe)
## Extraction of From field
m_from = msg.getaddrlist("From")
if len(m_from) == 1:
m_from = safe_parseaddr(m_from[0][1])
else:
m_from = None
## Empty or invalid 'From' field, try to use sender
if m_from is None:
LOG(E_ERR, "%s: no From header in mail using sender" % self.type)
m_from = safe_parseaddr(sender)
## No luck
if m_from is None:
return self.do_exit(552, "Mail has not suitable From/Sender")
args["m_from"] = m_from
## Extract 'To' field
m_to = []
for h in msg.getaddrlist("To"):
rec = safe_parseaddr(h[1])
if rec is None:
continue
m_to.append(rec)
## Empty 'To' field use recipients
if len(m_to) == 0:
LOG(E_ERR, "%s: no To header in mail using recipients" % self.type)
for recipient in recips:
rec = safe_parseaddr(recipient)
if rec is None:
continue
m_to.append(rec)
if len(m_to) == 0:
return self.do_exit(552, "Mail has not suitable To/Recipient")
## Extract 'Cc' field
for h in msg.getaddrlist("Cc"):
rec = safe_parseaddr(h[1])
if rec is None:
continue
m_to.append(rec)
## Cleanup: remove duplicates
recs = []
for rec in m_to:
if rec not in recs:
recs.append(rec)
args["m_rec"] = recs
## Extract 'Subject' field
m_sub = mime_decode_header(msg.get("Subject", "No Subject"))
if subjpattern is not None and m_sub.find(subjpattern) != -1:
LOG(E_INFO, "%s: Subject pattern matched, not archived" % self.type)
return self.sendmail(sender, mail_options, recips, rcptopts, self.remove_aid(data, msg))
args["m_sub"] = m_sub
## Whitelist check: From, To and Sender (envelope)
checklist = [m_from] + m_to
ss = safe_parseaddr(sender)
if ss is not None:
checklist.append(ss)
for check in checklist:
if check.split("@", 1)[0] in whitelist:
LOG(E_INFO, "%s: Mail to: %s in whitelist, not archived" % (self.type, check))
return self.sendmail(sender, mail_options, recips, rcptopts, self.remove_aid(data, msg))
## Sender size limit check - in kb
if dbchecker is not None and dbchecker.quota_check(m_from, size >> 10):
return self.do_exit(422, "Sender quota execeded")
args["m_size"] = size
## Extract 'Date' field
m_date = None
if self.datefromemail:
m_date = msg.getdate("Date")
try:
mktime(m_date)
except:
m_date = None
if m_date is None:
m_date = localtime(time())
args["m_date"] = m_date
m_attach = []
if msg.maintype != "multipart":
m_parse = parse_message(msg)
if m_parse is not None:
m_attach.append(m_parse)
else:
filepart = MultiFile(stream)
filepart.push(msg.getparam("boundary"))
try:
while filepart.next():
submsg = Message(filepart)
subpart = parse_message(submsg)
if subpart is not None:
m_attach.append(subpart)
except:
LOG(E_ERR, "%s: Error in multipart splitting" % self.type)
args["m_attach"] = m_attach
if dbchecker is not None:
## Collect data for mb lookup
addrs = []
for addr in [m_from] + m_to:
addrs.append(addr)
args["m_mboxes"] = dbchecker.mblookup(addrs)
else:
args["m_mboxes"] = []
year, pid, error = self.backend.process(args)
if year == 0:
LOG(E_ERR, "%s: Backend Error: %s" % (self.type, error))
return self.do_exit(pid, error)
## Adding X-Archiver-ID: header
aid = "%d-%d" % (year, pid)
data = self.add_aid(data, msg, aid)
LOG(E_TRACE, "%s: inserting %s msg in hashdb" % (self.type, aid))
self.hashdb[hash] = aid
self.hashdb.sync()
## Next hop
LOG(E_TRACE, "%s: backend worked fine" % self.type)
LOG(E_TRACE, "%s: passing data to nexthop: %s:%s" % (self.type, self.output_address, self.output_port))
return self.sendmail(sender, mail_options, recips, rcptopts, data, aid, hash)
class HttpRequest:
http_version = (1, 1)
http_version_string = ("HTTP/%d.%d" % http_version)
max_content_length = 10000
max_headers = 500
request_line = None
request_method = None
request_uri = None
request_path = None
request_query = None
request_version = None
content_length = 0
content = None
etag = None
close_connection = True
response_code = 200
response_status = "OK"
response_sent = False
cached = False
last_modified = None
forceSSL = False
def __init__(self, host, rin, out):
self.host = host
self.rin = rin
self.out = out
self.request_args = {}
self.args = self.request_args
self.request_headers = {}
self.request_cookies = {}
self.response_headers = {}
self.response_cookies = {}
self.output = StringIO()
self.parseRequest()
def isSecure(self):
return self.forceSSL
def getRequestMethod(self):
return self.request_method
def trim(self, str, ends):
for end in ends:
if str.endswith(end):
str = str[ : -len(end) ]
break
return str
def requestError(self, code, msg=None):
self.sendError(code, msg)
raise ValueError(self.response_status)
def sendError(self, code, msg=None):
self.setResponseCode(code, msg=msg)
self.sendResponse()
def parseRequestVersion(self, version):
try:
if not version.startswith('HTTP/'):
raise ValueError
version_string = version.split('/', 1)[1]
version_codes = version_string.split('.')
if len(version_codes) != 2:
raise ValueError
request_version = (int(version_codes[0]), int(version_codes[1]))
except (ValueError, IndexError):
self.requestError(400, "Bad request version (%s)" % `version`)
def parseRequestLine(self):
line = self.trim(self.request_line, ['\r\n', '\n'])
line_fields = line.split()
n = len(line_fields)
if n == 3:
[method, uri, version] = line_fields
elif n == 2:
[method, uri] = line_fields
version = 'HTTP/0.9'
else:
self.requestError(BAD_REQUEST,
"Bad request (%s)" % `line`)
request_version = self.parseRequestVersion(version)
if request_version > (2, 0):
self.requestError(VERSION_NOT_SUPPORTED,
"HTTP version not supported (%s)" % `version`)
#if request_version >= (1, 1) and self.http_version >= (1, 1):
# self.close_connection = False
#else:
# self.close_connection = True
self.request_method = method
self.method = method
self.request_uri = uri
self.request_version = version
uri_query = uri.split('?')
if len(uri_query) == 1:
self.request_path = uri
else:
self.request_path = uri_query[0]
self.request_query = uri_query[1]
self.request_args = parseQueryArgs(self.request_query)
self.args = self.request_args
def parseRequestHeaders(self):
header_bytes = ""
header_count = 0
while True:
if header_count >= self.max_headers:
self.requestError(BAD_REQUEST,
"Bad request (too many headers)")
line = self.rin.readline()
header_bytes += line
header_count += 1
if line == '\r\n' or line == '\n' or line == '':
break
header_input = StringIO(header_bytes)
self.request_headers = Message(header_input)
def parseRequestCookies(self):
cookie_hdr = self.getHeader("cookie")
if not cookie_hdr: return
for cookie in cookie_hdr.split(';'):
try:
cookie = cookie.lstrip()
(k, v) = cookie.split('=', 1)
self.request_cookies[k] = v
except ValueError:
pass
def parseRequestArgs(self):
if ((self.content is None) or
(self.request_method != "POST")):
return
content_type = self.getHeader('content-type')
if not content_type:
return
(encoding, params) = cgi.parse_header(content_type)
if encoding == URLENCODED:
xargs = cgi.parse_qs(self.content.getvalue(),
keep_blank_values=True)
elif encoding == MULTIPART_FORM_DATA:
xargs = cgi.parse_multipart(self.content, params)
else:
xargs = {}
self.request_args.update(xargs)
def getCookie(self, k):
return self.request_cookies[k]
def readContent(self):
try:
self.content_length = int(self.getHeader("Content-Length"))
except:
return
if self.content_length > self.max_content_length:
self.requestError(REQUEST_ENTITY_TOO_LARGE)
self.content = self.rin.read(self.content_length)
self.content = StringIO(self.content)
self.content.seek(0,0)
def parseRequest(self):
self.request_line = self.rin.readline()
self.parseRequestLine()
self.parseRequestHeaders()
self.parseRequestCookies()
connection_mode = self.getHeader('Connection')
self.setCloseConnection(connection_mode)
self.readContent()
self.parseRequestArgs()
def setCloseConnection(self, mode):
if not mode: return
mode = mode.lower()
if mode == 'close':
self.close_connection = True
elif (mode == 'keep-alive') and (self.http_version >= (1, 1)):
self.close_connection = False
def getCloseConnection(self):
return self.close_connection
def getHeader(self, k, v=None):
return self.request_headers.get(k, v)
def getRequestMethod(self):
return self.request_method
def getRequestPath(self):
return self.request_path
def setResponseCode(self, code, status=None, msg=None):
self.response_code = code
if not status:
status = getStatus(code)
self.response_status = status
def setResponseHeader(self, k, v):
k = k.lower()
self.response_headers[k] = v
if k == 'connection':
self.setCloseConnection(v)
setHeader = setResponseHeader
def setLastModified(self, when):
# time.time() may be a float, but the HTTP-date strings are
# only good for whole seconds.
when = long(math.ceil(when))
if (not self.last_modified) or (self.last_modified < when):
self.lastModified = when
modified_since = self.getHeader('if-modified-since')
if modified_since:
modified_since = stringToDatetime(modified_since)
if modified_since >= when:
self.setResponseCode(NOT_MODIFIED)
self.cached = True
def setContentType(self, ty):
self.setResponseHeader("Content-Type", ty)
def setEtag(self, etag):
if etag:
self.etag = etag
tags = self.getHeader("if-none-match")
if tags:
tags = tags.split()
if (etag in tags) or ('*' in tags):
if self.request_method in ("HEAD", "GET"):
code = NOT_MODIFIED
else:
code = PRECONDITION_FAILED
self.setResponseCode(code)
self.cached = True
def addCookie(self, k, v, expires=None, domain=None, path=None,
max_age=None, comment=None, secure=None):
cookie = v
if expires != None:
cookie += "; Expires=%s" % expires
if domain != None:
cookie += "; Domain=%s" % domain
if path != None:
cookie += "; Path=%s" % path
if max_age != None:
cookie += "; Max-Age=%s" % max_age
if comment != None:
cookie += "; Comment=%s" % comment
if secure:
cookie += "; Secure"
self.response_cookies[k] = cookie
def sendResponseHeaders(self):
if self.etag:
self.setResponseHeader("ETag", self.etag)
for (k, v) in self.response_headers.items():
self.send("%s: %s\r\n" % (k.capitalize(), v))
for (k, v) in self.response_cookies.items():
self.send("Set-Cookie: %s=%s\r\n" % (k, v))
self.send("\r\n")
def sendResponse(self):
if self.response_sent:
return
self.response_sent = True
send_body = self.hasBody()
if not self.close_connection:
self.setResponseHeader("Connection", "keep-alive")
self.setResponseHeader("Pragma", "no-cache")
self.setResponseHeader("Cache-Control", "no-cache")
self.setResponseHeader("Expires", "-1")
if send_body:
self.output.seek(0, 0)
body = self.output.getvalue()
body_length = len(body)
self.setResponseHeader("Content-Length", body_length)
if self.http_version > (0, 9):
self.send("%s %d %s\r\n" % (self.http_version_string,
self.response_code,
self.response_status))
self.sendResponseHeaders()
if send_body:
self.send(body)
self.flush()
def write(self, data):
self.output.write(data)
def send(self, data):
#print 'send>', data
self.out.write(data)
def flush(self):
self.out.flush()
def hasNoBody(self):
return ((self.request_method == "HEAD") or
(self.response_code in NO_BODY_CODES) or
self.cached)
def hasBody(self):
return not self.hasNoBody()
def process(self):
pass
return self.close_connection
def getRequestHostname(self):
"""Get the hostname that the user passed in to the request.
Uses the 'Host:' header if it is available, and the
host we are listening on otherwise.
"""
return (self.getHeader('host') or
socket.gethostbyaddr(self.getHostAddr())[0]
).split(':')[0]
def getHost(self):
return self.host
def getHostAddr(self):
return self.host[0]
def getPort(self):
return self.host[1]
def setHost(self, host, port, ssl=0):
"""Change the host and port the request thinks it's using.
This method is useful for working with reverse HTTP proxies (e.g.
both Squid and Apache's mod_proxy can do this), when the address
the HTTP client is using is different than the one we're listening on.
For example, Apache may be listening on https://www.example.com, and then
forwarding requests to http://localhost:8080, but we don't want HTML produced
to say 'http://localhost:8080', they should say 'https://www.example.com',
so we do::
request.setHost('www.example.com', 443, ssl=1)
"""
self.forceSSL = ssl
self.received_headers["host"] = host
self.host = (host, port)
def process_storage(self, peer, sender, mail_options, recips, rcptopts, data):
"""Stores the archived email using a Backend"""
size = len(data)
if size < MINSIZE:
return self.do_exit(550, "Invalid Mail")
if not data.endswith(NL):
data = data + NL
stream = StringIO(data)
msg = Message(stream)
aid = msg.get(AID, None)
## Check if I have msgid in my cache
mid = msg.get("message-id", self.new_mid())
LOG(E_TRACE, "%s: Message-id: %s" % (self.type, mid))
hash = hash_headers(msg.get)
if self.hashdb.has_key(hash):
aid = self.hashdb[hash]
LOG(E_ERR, "%s: Message already processed" % self.type)
return self.sendmail(sender, mail_options, recips, rcptopts, data, aid, hash)
## Date extraction
m_date = None
if self.datefromemail:
m_date = msg.getdate("Date")
try:
mktime(m_date)
except:
m_date = None
if m_date is None:
m_date = localtime(time())
del msg, stream
## Mail needs to be processed
if aid:
try:
year, pid = aid.split("-", 1)
year = int(year)
pid = int(pid)
except:
t, val, tb = exc_info()
del tb
LOG(E_ERR, "%s: Invalid X-Archiver-ID header [%s]" % (self.type, str(val)))
return self.do_exit(550, "Invalid X-Archiver-ID header")
args = dict(mail=data, year=year, pid=pid, date=m_date, mid=mid, hash=hash)
LOG(E_TRACE, "%s: year is %d - pid is %d (%s)" % (self.type, year, pid, mid))
status, code, msg = self.backend.process(args)
if status == 0:
LOG(E_ERR, "%s: process failed %s" % (self.type, msg))
return self.do_exit(code, msg)
## Inserting in hashdb
LOG(E_TRACE, "%s: inserting %s msg in hashdb" % (self.type, aid))
self.hashdb[hash] = aid
self.hashdb.sync()
LOG(E_TRACE, "%s: backend worked fine" % self.type)
else:
## Mail in whitelist - not processed
LOG(E_TRACE, "%s: X-Archiver-ID header not found in mail [whitelist]" % self.type)
## Next hop
LOG(E_TRACE, "%s: passing data to nexthop: %s:%s" % (self.type, self.output_address, self.output_port))
return self.sendmail(sender, mail_options, recips, rcptopts, data, aid, hash)
def process_storage(self, peer, sender, mail_options, recips, rcptopts,
data):
"""Stores the archived email using a Backend"""
size = len(data)
if size < MINSIZE:
return self.do_exit(550, 'Invalid Mail')
if not data.endswith(NL):
data = data + NL
stream = StringIO(data)
msg = Message(stream)
aid = msg.get(AID, None)
## Check if I have msgid in my cache
mid = msg.get('message-id', self.new_mid())
LOG(E_TRACE, '%s: Message-id: %s' % (self.type, mid))
hash = hash_headers(msg.get)
if self.hashdb.has_key(hash):
aid = self.hashdb[hash]
LOG(E_ERR, '%s: Message already processed' % self.type)
return self.sendmail(sender, mail_options, recips, rcptopts,
data, aid, hash)
## Date extraction
m_date = None
if self.datefromemail:
m_date = msg.getdate('Date')
try:
mktime(m_date)
except:
m_date = None
if m_date is None:
m_date = localtime(time())
del msg, stream
## Mail needs to be processed
if aid:
try:
year, pid = aid.split('-', 1)
year = int(year)
pid = int(pid)
except:
t, val, tb = exc_info()
del tb
LOG(
E_ERR, '%s: Invalid X-Archiver-ID header [%s]' %
(self.type, str(val)))
return self.do_exit(550, 'Invalid X-Archiver-ID header')
args = dict(mail=data,
year=year,
pid=pid,
date=m_date,
mid=mid,
hash=hash)
LOG(
E_TRACE, '%s: year is %d - pid is %d (%s)' %
(self.type, year, pid, mid))
status, code, msg = self.backend.process(args)
if status == 0:
LOG(E_ERR, '%s: process failed %s' % (self.type, msg))
return self.do_exit(code, msg)
## Inserting in hashdb
LOG(E_TRACE,
'%s: inserting %s msg in hashdb' % (self.type, aid))
self.hashdb[hash] = aid
self.hashdb.sync()
LOG(E_TRACE, '%s: backend worked fine' % self.type)
else:
## Mail in whitelist - not processed
LOG(
E_TRACE,
'%s: X-Archiver-ID header not found in mail [whitelist]' %
self.type)
## Next hop
LOG(
E_TRACE, '%s: passing data to nexthop: %s:%s' %
(self.type, self.output_address, self.output_port))
return self.sendmail(sender, mail_options, recips, rcptopts, data,
aid, hash)
class HttpRequest:
http_version = (1, 1)
http_version_string = ("HTTP/%d.%d" % http_version)
max_content_length = 10000
max_headers = 500
request_line = None
request_method = None
request_uri = None
request_path = None
request_query = None
request_version = None
content_length = 0
content = None
etag = None
close_connection = True
response_code = 200
response_status = "OK"
response_sent = False
cached = False
last_modified = None
forceSSL = False
def __init__(self, host, rin, out):
self.host = host
self.rin = rin
self.out = out
self.request_args = {}
self.args = self.request_args
self.request_headers = {}
self.request_cookies = {}
self.response_headers = {}
self.response_cookies = {}
self.output = StringIO()
self.parseRequest()
def isSecure(self):
return self.forceSSL
def getRequestMethod(self):
return self.request_method
def trim(self, str, ends):
for end in ends:
if str.endswith(end):
str = str[:-len(end)]
break
return str
def requestError(self, code, msg=None):
self.sendError(code, msg)
raise ValueError(self.response_status)
def sendError(self, code, msg=None):
self.setResponseCode(code, msg=msg)
self.sendResponse()
def parseRequestVersion(self, version):
try:
if not version.startswith('HTTP/'):
raise ValueError
version_string = version.split('/', 1)[1]
version_codes = version_string.split('.')
if len(version_codes) != 2:
raise ValueError
request_version = (int(version_codes[0]), int(version_codes[1]))
except (ValueError, IndexError):
self.requestError(400, "Bad request version (%s)" % ` version `)
def parseRequestLine(self):
line = self.trim(self.request_line, ['\r\n', '\n'])
line_fields = line.split()
n = len(line_fields)
if n == 3:
[method, uri, version] = line_fields
elif n == 2:
[method, uri] = line_fields
version = 'HTTP/0.9'
else:
self.requestError(BAD_REQUEST, "Bad request (%s)" % ` line `)
request_version = self.parseRequestVersion(version)
if request_version > (2, 0):
self.requestError(VERSION_NOT_SUPPORTED,
"HTTP version not supported (%s)" % ` version `)
#if request_version >= (1, 1) and self.http_version >= (1, 1):
# self.close_connection = False
#else:
# self.close_connection = True
self.request_method = method
self.method = method
self.request_uri = uri
self.request_version = version
uri_query = uri.split('?')
if len(uri_query) == 1:
self.request_path = uri
else:
self.request_path = uri_query[0]
self.request_query = uri_query[1]
self.request_args = parseQueryArgs(self.request_query)
self.args = self.request_args
def parseRequestHeaders(self):
header_bytes = ""
header_count = 0
while True:
if header_count >= self.max_headers:
self.requestError(BAD_REQUEST,
"Bad request (too many headers)")
line = self.rin.readline()
header_bytes += line
header_count += 1
if line == '\r\n' or line == '\n' or line == '':
break
header_input = StringIO(header_bytes)
self.request_headers = Message(header_input)
def parseRequestCookies(self):
cookie_hdr = self.getHeader("cookie")
if not cookie_hdr: return
for cookie in cookie_hdr.split(';'):
try:
cookie = cookie.lstrip()
(k, v) = cookie.split('=', 1)
self.request_cookies[k] = v
except ValueError:
pass
def parseRequestArgs(self):
if ((self.content is None) or (self.request_method != "POST")):
return
content_type = self.getHeader('content-type')
if not content_type:
return
(encoding, params) = cgi.parse_header(content_type)
if encoding == URLENCODED:
xargs = cgi.parse_qs(self.content.getvalue(),
keep_blank_values=True)
elif encoding == MULTIPART_FORM_DATA:
xargs = cgi.parse_multipart(self.content, params)
else:
xargs = {}
self.request_args.update(xargs)
def getCookie(self, k):
return self.request_cookies[k]
def readContent(self):
try:
self.content_length = int(self.getHeader("Content-Length"))
except:
return
if self.content_length > self.max_content_length:
self.requestError(REQUEST_ENTITY_TOO_LARGE)
self.content = self.rin.read(self.content_length)
self.content = StringIO(self.content)
self.content.seek(0, 0)
def parseRequest(self):
self.request_line = self.rin.readline()
self.parseRequestLine()
self.parseRequestHeaders()
self.parseRequestCookies()
connection_mode = self.getHeader('Connection')
self.setCloseConnection(connection_mode)
self.readContent()
self.parseRequestArgs()
def setCloseConnection(self, mode):
if not mode: return
mode = mode.lower()
if mode == 'close':
self.close_connection = True
elif (mode == 'keep-alive') and (self.http_version >= (1, 1)):
self.close_connection = False
def getCloseConnection(self):
return self.close_connection
def getHeader(self, k, v=None):
return self.request_headers.get(k, v)
def getRequestMethod(self):
return self.request_method
def getRequestPath(self):
return self.request_path
def setResponseCode(self, code, status=None, msg=None):
self.response_code = code
if not status:
status = getStatus(code)
self.response_status = status
def setResponseHeader(self, k, v):
k = k.lower()
self.response_headers[k] = v
if k == 'connection':
self.setCloseConnection(v)
setHeader = setResponseHeader
def setLastModified(self, when):
# time.time() may be a float, but the HTTP-date strings are
# only good for whole seconds.
when = long(math.ceil(when))
if (not self.last_modified) or (self.last_modified < when):
self.lastModified = when
modified_since = self.getHeader('if-modified-since')
if modified_since:
modified_since = stringToDatetime(modified_since)
if modified_since >= when:
self.setResponseCode(NOT_MODIFIED)
self.cached = True
def setContentType(self, ty):
self.setResponseHeader("Content-Type", ty)
def setEtag(self, etag):
if etag:
self.etag = etag
tags = self.getHeader("if-none-match")
if tags:
tags = tags.split()
if (etag in tags) or ('*' in tags):
if self.request_method in ("HEAD", "GET"):
code = NOT_MODIFIED
else:
code = PRECONDITION_FAILED
self.setResponseCode(code)
self.cached = True
def addCookie(self,
k,
v,
expires=None,
domain=None,
path=None,
max_age=None,
comment=None,
secure=None):
cookie = v
if expires != None:
cookie += "; Expires=%s" % expires
if domain != None:
cookie += "; Domain=%s" % domain
if path != None:
cookie += "; Path=%s" % path
if max_age != None:
cookie += "; Max-Age=%s" % max_age
if comment != None:
cookie += "; Comment=%s" % comment
if secure:
cookie += "; Secure"
self.response_cookies[k] = cookie
def sendResponseHeaders(self):
if self.etag:
self.setResponseHeader("ETag", self.etag)
for (k, v) in self.response_headers.items():
self.send("%s: %s\r\n" % (k.capitalize(), v))
for (k, v) in self.response_cookies.items():
self.send("Set-Cookie: %s=%s\r\n" % (k, v))
self.send("\r\n")
def sendResponse(self):
if self.response_sent:
return
self.response_sent = True
send_body = self.hasBody()
if not self.close_connection:
self.setResponseHeader("Connection", "keep-alive")
self.setResponseHeader("Pragma", "no-cache")
self.setResponseHeader("Cache-Control", "no-cache")
self.setResponseHeader("Expires", "-1")
if send_body:
self.output.seek(0, 0)
body = self.output.getvalue()
body_length = len(body)
self.setResponseHeader("Content-Length", body_length)
if self.http_version > (0, 9):
self.send("%s %d %s\r\n" %
(self.http_version_string, self.response_code,
self.response_status))
self.sendResponseHeaders()
if send_body:
self.send(body)
self.flush()
def write(self, data):
self.output.write(data)
def send(self, data):
#print 'send>', data
self.out.write(data)
def flush(self):
self.out.flush()
def hasNoBody(self):
return ((self.request_method == "HEAD")
or (self.response_code in NO_BODY_CODES) or self.cached)
def hasBody(self):
return not self.hasNoBody()
def process(self):
pass
return self.close_connection
def getRequestHostname(self):
"""Get the hostname that the user passed in to the request.
Uses the 'Host:' header if it is available, and the
host we are listening on otherwise.
"""
return (self.getHeader('host')
or socket.gethostbyaddr(self.getHostAddr())[0]).split(':')[0]
def getHost(self):
return self.host
def getHostAddr(self):
return self.host[0]
def getPort(self):
return self.host[1]
def setHost(self, host, port, ssl=0):
"""Change the host and port the request thinks it's using.
This method is useful for working with reverse HTTP proxies (e.g.
both Squid and Apache's mod_proxy can do this), when the address
the HTTP client is using is different than the one we're listening on.
For example, Apache may be listening on https://www.example.com, and then
forwarding requests to http://localhost:8080, but we don't want HTML produced
to say 'http://localhost:8080', they should say 'https://www.example.com',
so we do::
request.setHost('www.example.com', 443, ssl=1)
"""
self.forceSSL = ssl
self.received_headers["host"] = host
self.host = (host, port)
def _websocket_client_handshake(self):
buf = []
while True:
fds = yield
if self.socket in fds:
try:
data = self._recv(BUF_SIZE)
except socket.error as e:
handle_socket_error(e)
else:
buf.append(data)
score = 0
for char in itertools.chain(*buf):
if char in ('\r', '\n'):
score += 1
if score == 4:
break
else:
score = 0
else:
continue
break
header, body = ''.join(buf).split('\r\n\r\n', 1)
headers = Message(StringIO(header.split(b'\r\n', 1)[1]))
if headers.get(b'Upgrade', '').lower() != b'websocket':
raise protocol.ConnectionClosed()
# common headers:
response = b''.join(
(b'HTTP/1.1 101 Switching Protocols\r\n',
b'Upgrade: websocket\r\n',
b'Connection: Upgrade\r\n'))
# if override_host:
# headers[b'host'] = override_host
if b'Sec-WebSocket-Key1' in headers and \
b'Sec-WebSocket-Key2' in headers:
self._decode_message = self._decode_hixie
accept = hashlib.md5(
b''.join((struct.pack(b'>II', *list(
int(b''.join(x for x in headers[key]
if x in string.digits), 10) / headers[key].count(b' ')
for key in self.HIXIE_WEBSOCKET_SEC_KEY_ORDER)
), body,))).digest()
response = b''.join(
(response,
b'Sec-WebSocket-Origin: %s\r\n' % headers[b'origin'],
b'Sec-WebSocket-Location: ws://%s/\r\n' % headers[b'host'],
b'\r\n%s' % str(accept),
))
self._send(response)
self._send = self._send_hixie
else:
self._decode_message = self._decode_rfc
secret_key = \
b''.join((headers[b'Sec-WebSocket-Key'], self.RFC_MAGIC_HASH,))
digest = b64encode(
hashlib.sha1(
secret_key).hexdigest().decode(b'hex')
)
response = \
b''.join((response, b'Sec-WebSocket-Accept: %s\r\n\r\n' % digest,))
self._send(response)
self._send = self._send_rfc
print("Ready for reading.")
self.state = self.READ_WHEN
def clientThread(connection, clientID):
# Sending message to connected client
preSendTime = int(round(time.time() * 1000))
# Deal with WebSocket handshake here
magic = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"
data = connection.recv(RECV_BUFFER).strip()
headers = Message(StringIO(data.split("\r\n", 1)[1]))
if headers.get("Upgrade", None) != "websocket":
return
print "Handshaking..."
key = headers["Sec-WebSocket-Key"]
digest = b64encode(sha1(key + magic).hexdigest().decode("hex"))
response = "HTTP/1.1 101 Switching Protocols\r\n"
response += "Upgrade: websocket\r\n"
response += "Connection: Upgrade\r\n"
response += "Sec-WebSocket-Accept: %s\r\n\r\n" % digest
connection.send(response)
print "handshake done."
sendMessage(connection, '{"msgtype": "id"}')
data = receiveMessage(connection)
receivedMessage = json.loads(data)
postSendTime = int(round(time.time() * 1000))
latencyEst = (postSendTime - preSendTime) / 2
name = None
newId = None
global Clients
if receivedMessage["msgtype"] == "getid":
newId = generateNewCollabId()
name = receivedMessage["name"]
messageObject = {"msgtype": "setid", "collabid": newId}
sendMessage(connection, json.dumps(messageObject))
elif receivedMessage["msgtype"] == "setid":
name = receivedMessage["name"]
if checkIfCollabIdExists(receivedMessage["collabid"]):
messageObject = {"msgtype": "id_ok"}
sendMessage(connection, json.dumps(messageObject))
newId = receivedMessage["collabid"]
else:
messageObject = {"msgtype": "setid", "collabid": generateNewCollabId()}
newId = messageObject["collabid"]
sendMessage(connection, json.dumps(messageObject))
else:
print "Invalid message received."
# invalid message from client.
if not newId == None:
# Send highest current latency to this client
currentHighestLatency = 10
for client in Clients:
if client.collabID == newId and client.latencyEst > currentHighestLatency:
currentHighestLatency = client.latencyEst
messageObject = {"msgtype": "onward-latency-update", "latency": currentHighestLatency}
sendMessage(connection, json.dumps(messageObject))
# Then send the new client's latency to the other clients.s
messageObject = {"msgtype": "onward-latency-update", "latency": latencyEst}
for client in Clients:
if client.collabID == newId:
sendMessage(client.connection, json.dumps(messageObject))
messageObject = {"msgtype": "client-joined", "name": name}
sendMessage(client.connection, json.dumps(messageObject))
newClient = ConnectedClient(clientID, newId, connection, latencyEst)
Clients.append(newClient)
while True:
data = receiveMessage(connection)
if not data:
break
# Iterate through clients and send to ones with collabid that aren't this clientid
for client in Clients:
if (not client.clientID == clientID) and client.collabID == newId:
receivedMessage = json.loads(data)
print data
if receivedMessage["msgtype"] == "chat":
receivedMessage["name"] = name
sendMessage(client.connection, json.dumps(receivedMessage))
else:
sendMessage(client.connection, data)
connection.close()