def test_spamc_tcp_process(self):
with open(self.filename) as handle:
result = self.spamc_tcp.process(handle)
self.assertIn('message', result)
with open(self.filename) as headerhandle:
headers1 = Message(headerhandle)
headers2 = Message(StringIO(result['message']))
self.assertEqual(
headers1.get('Subject'),
headers2.get('Subject')
)
def populate_headers(self):
line = None
while not line:
line = self.mr.buf_reader.readline().strip()
while line != "":
self.headers.update(Message(StringIO(line)))
line = self.mr.buf_reader.readline().strip()
def parseData(self, data):
data = data.split("|", 2)
dataDict = {"from": data[0], "to": data[1]}
path, headers = data[2].split('\r\n', 1)
payload = Message(StringIO(headers))
url = "http://" + payload['host'] + path.split(" ")[1]
if url.lower().endswith(('.png', '.ico', '.jpeg', '.jpg', '.gif', '.svg')):
dataDict['image'] = url
else:
dataDict['url'] = url
if 'cookie' in payload:
dataDict['cookie'] = payload['cookie']
postData = data[2].split('\r\n\r\n')
if len(postData) == 2:
if postData[1].strip():
dataDict['post'] = postData[1]
print dataDict;
return dataDict
def resolve_uri(uri, accept="application/rdf+xml", include_info=False):
""" Resolve an RDF URI and return the RDF/XML. """
logging.debug("resolve uri: " + uri)
if uri.startswith("http:"):
opener = urllib2.build_opener(urllib2.HTTPHandler)
elif uri.startswith("https:"):
opener = urllib2.build_opener(urllib2.HTTPSHandler)
request = urllib2.Request(uri)
request.add_header('Accept', accept)
request.get_method = lambda: 'GET'
url = opener.open(request)
try:
data = url.read()
#data = threads.blockingCallFromThread(reactor, url.read)
except Exception as e:
logging.debug("Error in resolve_uri: " + str(e))
if include_info:
headers = Message(StringIO("".join(url.info().headers)))
return {"data": data, "info": url.info(), "headers": headers}
else:
return data
def mk_header(self): rfile = StringIO(self.rfile) requestline = rfile.readline()[:-2] if not requestline: self.shutdown() return words = requestline.split() if len(words) == 3: [method, self.path, self.version] = words if self.version[:5] != 'HTTP/': self.shutdown() return elif len(words) == 2: [method, self.path] = words else: self.shutdown() return self.method = method.lower() self.headers, self.rfile = Message(rfile, 0), '' self.__getitem__ = self.headers.getheader try: tasklet(controller)(self) except: print_exc(file=stderr)
class FTPRangeHandler(urllib2.FTPHandler):
"""
FTP Range support..
"""
def ftp_open(self, req):
host = req.get_host()
host, port = urllib.splitport(host)
if port is None:
port = ftplib.FTP_PORT
try:
host = socket.gethostbyname(host)
except socket.error, msg:
raise FetchError(msg)
path, attrs = urllib.splitattr(req.get_selector())
dirs = path.split('/')
dirs = map(urllib.unquote, dirs)
dirs, file = dirs[:-1], dirs[-1]
if dirs and not dirs[0]:
dirs = dirs[1:]
try:
fw = self.connect_ftp('', '', host, port, dirs)
type = file and 'I' or 'D'
for attr in attrs:
attr, value = urllib.splitattr(attr)
if attr.lower() == 'type' and \
value in ('a', 'A', 'i', 'I', 'd', 'D'):
type = value.upper()
rawr = req.headers.get('Range', None)
if rawr:
rest = int(rawr.split("=")[1].rstrip("-"))
else:
rest = 0
fp, retrlen = fw.retrfile(file, type, rest)
fb, lb = rest, retrlen
if retrlen is None or retrlen == 0:
raise RangeError
retrlen = lb - fb
if retrlen < 0:
# beginning of range is larger than file
raise RangeError
headers = ''
mtype = guess_type(req.get_full_url())[0]
if mtype:
headers += 'Content-Type: %s\n' % mtype
if retrlen is not None and retrlen >= 0:
headers += 'Content-Length: %d\n' % retrlen
try:
from cStringIO import StringIO
except ImportError, msg:
from StringIO import StringIO
return urllib.addinfourl(fp, Message(StringIO(headers)),
req.get_full_url())
def handshake(self):
try:
data = self.client.recv(1024).strip()
headers = Message(StringIO(data.split('\r\n', 1)[1]))
if headers.get("Upgrade", None) != "websocket":
return
print 'Handshaking with web application...'
key = headers['Sec-WebSocket-Key']
digest = b64encode(
sha1(key + self.magic).hexdigest().decode('hex'))
response = 'HTTP/1.1 101 Switching Protocols\r\n'
response += 'Upgrade: websocket\r\n'
response += 'Connection: Upgrade\r\n'
response += 'Sec-WebSocket-Accept: %s\r\n\r\n' % digest
self.client.send(response)
# Have to make sure the handshake is successful before we add
# it as a client thread or it will get lots of data dumps before
# connection is ready. Doing it here so it doesn't hold up the main thread.
self.server.serverLock.acquire()
self.server.webthreads.append(self)
self.server.serverLock.release()
print "added thread"
self.handshake_done = "websocket"
except Exception as e:
print "Error in handshake: %s" % (str(e))
print traceback.format_exc()
self.kill()
def _getJson(self, page, query=None):
response = self._get(page, query)
headers = Message(StringIO(response.split('\n')[0]))
self.assertEqual(headers['content-type'], 'application/json')
body = ''.join(response.split('\n')[2:])
self.assertTrue(len(body) > 0)
return json.loads(body)
def _perform_handshake(self):
"""Perform The WebSocket Handshake"""
try:
Log.add("Got To Handshake")
data = self.recv(1024).strip()
# Log.add("Data: %s" % data)
headers = Message(StringIO(data.split('\r\n', 1)[1]))
Log.add("Parsed Headers:")
# Log.add(headers)
if headers.get('Upgrade', None) == 'websocket':
Log.add("Attempting Handshake")
# create response key
key = b64encode(
sha1(headers['Sec-WebSocket-Key'] + self.SALT).digest())
# create response headers
response = ("HTTP/1.1 101 Web Socket Protocol Handshake\r\n"
"Upgrade: websocket\r\n"
"Connection: Upgrade\r\n"
"Sec-WebSocket-Origin: %s\r\n"
"Sec-WebSocket-Accept: %s\r\n\r\n" %
(headers["Origin"], key))
if self.send_bytes(response):
Log.add("Handshake successful")
self._assign_room(data)
self._ready_state = "authenticating"
except Exception as e:
Log.add(e.args)
def process_response_textblock(response_text_block):
response_line, header_alone = response_text_block.split('\r\n', 1)
version, code = response_line.split()[0:2]
headers = Message(StringIO(header_alone))
response = {}
response.update(headers.dict)
response['code'] = code
return response
def wait_for_sip_read(self):
"""
This function waits for events and dispatches event handlers.
This function should run inside a greenlet.
"""
while True:
fileno = self.socket.fileno()
gsocket.wait_read(fileno)
message = self.socket.recv(8192)
header, body = message.split("\r\n\r\n", 1)
sip_status_line, parse_headers = header.split("\r\n", 1)
sip_status = sip_status_line.split(" ")
# FIXME Debug info
print sip_status_line
if sip_status[0] != "SIP/2.0":
print "WARNING: Got:", sip_status_line
print header
print body
continue
headers = Message(StringIO(parse_headers))
if sip_status[1] == '100' and headers['cseq'] == "20 INVITE":
if 'trying' in self.handlers:
for fn in self.handlers['trying']:
fn(headers, body)
elif sip_status[1] == '180' and headers['cseq'] == "20 INVITE":
if 'ringing' in self.handlers:
for fn in self.handlers['ringing']:
fn(headers, body)
elif sip_status[1] == '200' and headers['cseq'] == "20 INVITE":
self.contact = headers['contact'][1:-1]
self.to_hdr = headers['to']
self.from_hdr = headers['from']
if 'invite_ok' in self.handlers:
for fn in self.handlers['invite_ok']:
fn(headers, body)
elif sip_status[1] == '404' and headers['cseq'] == "20 INVITE":
if 'not_found' in self.handlers:
for fn in self.handlers['not_found']:
fn(headers, body)
elif sip_status[1] == '415' and headers['cseq'] == "20 INVITE":
if 'media-error' in self.handlers:
for fn in self.handlers['media-error']:
fn(headers, body)
else:
print header, body
if 'other' in self.handlers:
for fn in self.handlers['other']:
fn(headers, body)
def process_reqeust_textblock(request_text_block):
request_line, header_alone = request_text_block.split('\r\n', 1)
headers = Message(StringIO(header_alone))
method, uri, http_version = request_line.split()
request = {}
request.update(headers.dict)
request['request_method'] = method
request['request_uri'] = uri
return request
def product_appid(prod_name, prod_dict):
url = product_appid_url(prod_name, prod_dict)
raw_response = check_output([CURL_PATH, '-is', url])
request, body = raw_response.split('\r\n\r\n', 1)
response, headers = request.split('\r\n', 1)
header_dict = dict(Message(StringIO(headers)))
location = header_dict['location']
query = urlparse(location).query
return parse_qs(query)['appIdKey'][0]
def print_http_message(s, color):
req_or_res, headers_and_body = s.split('\r\n', 1)
fp_headers_and_body = StringIO(headers_and_body)
m = Message(fp_headers_and_body)
headers_part = fp_headers_and_body.getvalue(True)
compressed, out = format_body(m, fp_headers_and_body)
cprint(req_or_res, color, attrs=['dark'])
cprint(headers_part, color, attrs=['bold'])
cprint(out, color)
def request_with_data(post_request, content_type, req_type, retries, time_sleep, timeout_sec, payload,
condition, output, sample_event, message, log_in_file, thread_tmp_filename, language,
targets, ports, default_ports, socks_proxy):
"""
this function extracts the data, headers and url for the POST type request which is to be sent to
the __http_request_maker function
Args:
post_request: the returned data from __http_requests_generator function
req_type: GET, POST, PUT, DELETE or PATCH
content_type: application/json or application/x-www-form-urlencoded
payload: the payload corresponding to which the request is made
condition: the condition to be evaluated. eg: response.status_code == 200
other args: retries, time_sleep, timeout_sec, output, sample_event, message, log_in_file,
thread_tmp_filename, language
Returns:
the list of outputs in the format
[
{
"payload": payload,
"condition": condition,
"result": rule_evaluator(response, condition),
"response": response
},......
]
"""
post_data_format = ""
request_line, headers_alone = post_request.split('\r\n', 1)
headers = Message(StringIO(headers_alone)).dict
clean_headers = {x.strip(): y for x, y in headers.items()}
headers = clean_headers
if "content-type" in headers:
content_type = headers['content-type']
if content_type == 'application/x-www-form-urlencoded':
post_data_format = post_data_parser(post_request.split('\r\n')[-1])
elif content_type == 'application/json':
post_data_format = json.loads(post_request[post_request.find('{'):post_request.find('}') + 1])
headers.pop("Content-Length", None)
url_sample = request_line.strip().split(' ')[1]
for target in targets:
url = url_sample.replace('__target_locat_here__', str(target))
port = url[url.find(':', 7) + 1:url.find('/', 7)]
response = __http_request_maker(req_type, url, headers, retries, time_sleep, timeout_sec,
post_data_format, content_type, socks_proxy)
if rule_evaluator(response, condition):
__log_into_file(thread_tmp_filename, 'w', '0', language)
sample_event['PORT'] = port
event_parser(message, sample_event, response, payload, log_in_file, language)
output.append({
"payload": payload,
"condition": condition,
"result": rule_evaluator(response, condition),
"response": response
})
return output
def test_spamc_unix_headers(self):
with open(self.filename) as handle:
result = self.spamc_unix.headers(handle)
self.assertIn('message', result)
with open(self.filename) as headerhandle:
headers = Message(headerhandle)
org_subject = "Subject: %s" % headers.get('Subject')
new_subject = "Subject: %s" % result['headers'].get('Subject')
self.assertEqual(org_subject, new_subject)
def __init__(self, text, totalSize):
self.totalSize = totalSize
head, body = text.split('\r\n\r\n', 1)
top_line, headers_alone = head.split('\r\n', 1)
self.methodStr = top_line
# headers is a dict
self.headers = Message(StringIO(headers_alone))
self.headersSize = len(headers_alone)
# ASCII, so 1 byte per char
self.body = body # UTF-8 most likely...
self.bodySize = totalSize - self.headersSize
def test_spamc_tcp_headers(self):
with open(self.filename) as _handle:
data = _handle.read()
handle = StringIO(data)
result = self.spamc_tcp.headers(handle)
self.assertIn('message', result)
with open(self.filename) as headerhandle:
headers = Message(headerhandle)
org_subject = "Subject: %s" % headers.get('Subject')
new_subject = "Subject: %s" % result['headers'].get('Subject')
self.assertEqual(org_subject, new_subject)
def assert_headers(self, header, body):
headers = Message(StringIO(header))
if 'content-length' in headers:
content_length = int(headers['content-length'])
body_length = len(body)
self.assertEqual(
content_length,
body_length,
msg=
"Header reported content-length: %d Actual body length was: %d"
% (content_length, body_length))
def handshake(self):
data = self.request.recv(1024).strip()
headers = Message(StringIO(data.split('\r\n', 1)[1]))
if headers.get("Upgrade", None) != "websocket":
return
print 'Handshaking...'
key = headers['Sec-WebSocket-Key']
digest = b64encode(sha1(key + self.magic).hexdigest().decode('hex'))
response = 'HTTP/1.1 101 Switching Protocols\r\n'
response += 'Upgrade: websocket\r\n'
response += 'Connection: Upgrade\r\n'
response += 'Sec-WebSocket-Accept: %s\r\n\r\n' % digest
self.handshake_done = self.request.send(response)
def do_handshake(self):
request = self._sock.recv(1024)
# print("[Weblink] Received handshake:", request)
request_line, header_lines = request.split('\r\n', 1)
headers = Message(StringIO(header_lines))
key = headers['Sec-WebSocket-Key']
# Generate the hash value for the accept header
accept = b64encode(sha1(key + MAGICAL_STRING).digest())
response = HANDSHAKE_RESPONSE_RFC6455 % accept
# print("[Weblink] Completed handshake:", response)
self._sock.send(response)
return
def parse_headers(header_string):
""" Parse a header-string into individual headers
Implementation based on: http://stackoverflow.com/a/5955949/95122
"""
# First line is request line, strip it out
if not header_string:
return dict()
request, headers = header_string.split('\r\n', 1)
if not headers:
return dict()
else:
header_msg = Message(StringIO(headers))
return dict(header_msg.items())
def decode_request(self):
"""
Parses json from POST
"""
self.send_header('Content-type', 'text/html')
self.end_headers()
headers = Message(StringIO(self.headers))
try:
self.raw_data = self.rfile.read(int(headers["Content-Length"]))
self.json_data = simplejson.loads(self.raw_data)
except Exception, e:
logging.warning("Cannot parse %s" % self.raw_data)
return
def msg_handle(self, socket, address):
try:
rfile = socket.makefile('rb', self.rbufsize)
wfile = socket.makefile('wb', self.wbufsize)
headers = Message(rfile).dict
INFO('get a connection from:%s,headers:%s' %
(str(address), headers))
if 'module' in headers and headers['module'] in MODULES:
MODULES[headers['module']].handle(wfile, headers)
except Exception:
ERROR('msg_handle exception,please check')
def get(url):
url = parse_url(url)
hostname = url["hostname"]
port = url["port"] or 80
path = url["path"]
logging.info("[Host: %s port: %s path: %s]", hostname, port, path)
# Establish Socket Connection
with SocketConnection(hostname, port) as sock:
# Send HEAD Request to fetch Headers
sock.send("HEAD %s HTTP/1.0\r\nHost: %s\r\n\r\n" % (path, hostname))
data = sock.recv(1024)
recv = ""
while len(data):
recv = recv + data
data = sock.recv(1024)
# Parse Headers using MIMETools to Fetch Content-Length
request_line, headers_alone = recv.split('\r\n', 1)
headers = Message(StringIO(headers_alone))
content_length = headers['content-length']
logging.info("[Content-Length: %s]", content_length)
# initialize threads for download
thread_count = THREAD_COUNT
bytes_per_thread = int(content_length) / thread_count
download_threads = []
for x in xrange(0, thread_count):
start_range = (x) * bytes_per_thread
if x is not 0:
start_range += 1
end_range = (x + 1) * bytes_per_thread
if x == (thread_count - 1):
end_range = int(content_length) - 1
download_thread = DownloadThread(x, hostname, port, path, start_range,
end_range)
download_threads.append(download_thread)
for x in xrange(0, thread_count):
download_threads[x].start()
while threading.active_count() > 1:
time.sleep(1)
logging.info("Download Thread Running")
logging.info("Joining Files")
# Joining partfiles
file_name = os.path.basename(os.path.basename(path))
with open(file_name, 'w') as outfile:
for x in xrange(0, thread_count):
with open("partfile" + str(x)) as infile:
for line in infile:
outfile.write(line)
# Removing Partfiles
for x in xrange(0, thread_count):
os.remove("partfile" + str(x))
logging.info("Exiting")
def handshake(self):
print "handshake"
data = self.request.recv(1024).strip()
dSplit = data.split('\r\n', 1)
if len(dSplit) > 1:
headers = Message(StringIO(data.split('\r\n', 1)[1]))
else:
headers = Message(StringIO(data.split('\r\n', 1)))
if headers.get("Upgrade", None) == None:
return
if headers.get("Upgrade", None).lower() != "websocket":
return
try:
key = headers['Sec-WebSocket-Key']
digest = b64encode(
sha1(key + self.magic).hexdigest().decode('hex'))
print "has key"
except KeyError:
self.hasSecKey = False
print "no Sec-WebSocket-Key"
response = 'HTTP/1.1 101 Switching Protocols\r\n'
response += 'Upgrade: websocket\r\n'
response += 'Connection: Upgrade\r\n'
#this is also where we can distinguish a wifly from a browers
if (self.hasSecKey):
response += 'Sec-WebSocket-Accept: %s\r\n\r\n' % digest
print "sending back handshake"
self.handshake_done = self.request.send(response)
if (self.hasSecKey):
self.server.addBrowser(self)
else:
self.server.addWiFly(self)
def parseRequestHeaders(self):
header_bytes = ""
header_count = 0
while True:
if header_count >= self.max_headers:
self.requestError(BAD_REQUEST,
"Bad request (too many headers)")
line = self.rin.readline()
header_bytes += line
header_count += 1
if line == '\r\n' or line == '\n' or line == '':
break
header_input = StringIO(header_bytes)
self.request_headers = Message(header_input)
def handshake(self):
data = self.request.recv(1024).strip()
headers = Message(StringIO(data.split('\r\n', 1)[1]))
try:
uri = data.split('\r\n')[0].split(' ')[1]
authtoken = dict(
p.split('=') for p in uri.split('/')[-1].split('?')[1].split(
'&'))['authtoken']
if not wsauth.verify_token(authtoken):
print "[!] Invalid auth token: {}".format(authtoken)
return
else:
print "[+] Valid auth token: {}".format(authtoken)
except Exception, e:
print "Error checking token: {}".format(e)
def get_contacted_HTTP_Address(self):
HTTP_Request_Packets = self.applyFilters(self.capture,
self.HTTP_request)
urls = []
for pkt in HTTP_Request_Packets:
header_raw = pkt.load
try:
header_line, header_data = header_raw.split("\r\n", 1)
except ValueError:
pass
headers = Message(StringIO(header_data))
uri = header_line.strip("HTTP/1.1").strip("GET").lstrip().rstrip()
url = headers["host"] + uri
urls.append(url)
return urls
def getheaders(headers):
try:
import http.client
import io
except ImportError:
from mimetools import Message
from StringIO import StringIO
return Message(StringIO(headers))
class _FakeSocket(io.BytesIO):
def makefile(self, *args, **kw):
return self
return http.client.parse_headers(_FakeSocket(headers))
