def test_request(self):
"""
L{Agent.request} establishes a new connection to the host indicated by
the host part of the URI passed to it and issues a request using the
method, the path portion of the URI, the headers, and the body producer
passed to it. It returns a L{Deferred} which fires with a L{Response}
from the server.
"""
self.agent._connect = self._dummyConnect
headers = http_headers.Headers({'foo': ['bar']})
# Just going to check the body for identity, so it doesn't need to be
# real.
body = object()
self.agent.request('GET', 'http://example.com:1234/foo?bar', headers,
body)
protocol = self.protocol
# The request should be issued.
self.assertEquals(len(protocol.requests), 1)
req, res = protocol.requests.pop()
self.assertIsInstance(req, Request)
self.assertEquals(req.method, 'GET')
self.assertEquals(req.uri, '/foo?bar')
self.assertEquals(
req.headers,
http_headers.Headers({
'foo': ['bar'],
'host': ['example.com:1234']
}))
self.assertIdentical(req.bodyProducer, body)
def __init__(self,
args=None,
body=None,
method="GET",
prePathURL=BASE_URL,
requestHeaders=None):
self.args = args if args is not None else {}
if body is not None:
self.content = StringIO(body)
else:
self.content = StringIO('')
self.prePathURL = lambda: prePathURL
# we're always directly aimed at a resource and nobody is doing any
# postpath-related stuff, so let's just pretend it's always emtpy...
self.postpath = []
self.code = http.OK
if requestHeaders is not None:
self.requestHeaders = requestHeaders
else:
self.requestHeaders = http_headers.Headers()
self.responseHeaders = http_headers.Headers()
self.method = method
def handleStartSession(self, method):
def procResponse_cb(response):
defer = Deferred()
defer.addCallback(method)
response.deliverBody(DataPrinter(defer, "bool"))
return NOT_DONE_YET
def startSession_cb((signedNonce, nonceid)):
agent = CookieAgent(Agent(reactor), self.cookie_jar)
dataq = []
dataq.append(signedNonce)
body = _FileProducer(
StringIO(self.client_id.encryptData(self.client_id.password)),
dataq)
headers = http_headers.Headers()
d = agent.request(
'PUT',
'http://localhost:8000/session/?method=startsession&ccid=' +
self.ccid + '&nonceid=' + str(nonceid), headers, body)
d.addCallback(procResponse_cb)
return NOT_DONE_YET
def getNonce_cb(response):
defer = Deferred()
defer.addCallback(startSession_cb)
response.deliverBody(getNonce(defer, self.client_id, self.pin))
return NOT_DONE_YET
if self.pin != None:
agent = Agent(reactor)
body = FileBodyProducer(
StringIO(self.client_id.pub_key.exportKey('PEM')))
headers = http_headers.Headers()
d = agent.request(
'GET', 'http://localhost:8000/session/?method=getnonce',
headers, body)
d.addCallback(getNonce_cb)
return NOT_DONE_YET
agent = CookieAgent(Agent(reactor), self.cookie_jar)
body = FileBodyProducer(
StringIO(self.client_id.encryptData(self.client_id.password)))
headers = http_headers.Headers()
d = agent.request(
'PUT', 'http://localhost:8000/session/?method=startsession&ccid=' +
self.ccid + '&nonceid=' + str(-1), headers, body)
d.addCallback(procResponse_cb)
return NOT_DONE_YET
def postentry(self, entry):
headers = http_headers.Headers({
b"Content-Type": [b"application/json"],
})
body = FileBodyProducer(BytesIO(json.dumps(entry).encode("utf8")))
self.agent.request(b"POST", self.url, headers, body)
def updateShared_cb(iv):
print "Updating file..."
args = ("updateshared", str(self.ccid), os.path.basename(s[3]),
s[2])
salt = self.processCookie("/shares")
dataq = []
dataq.append(self.client_id.genHashArgs(args, salt))
dataq.append(iv)
# print "debugging:ticket, iv updatefile"
# print dataq[0]
# print dataq[1]
# print len(dataq[1])
print "Uploading file..."
agent = CookieAgent(Agent(reactor), self.cookie_jar)
enc_file = open("enc_fileout", 'r')
body = _FileProducer(enc_file, dataq)
headers = http_headers.Headers()
d = agent.request(
'POST',
'http://localhost:8000/shares/?method=updateshared&ccid=' +
self.ccid + "&name=" + os.path.basename(s[3]) +
"&fileid=" + s[2], headers, body)
d.addCallback(self.printPutReply_cb)
return NOT_DONE_YET
def handlePutFile(self, line):
print "Encrypting file..."
s = line.split()
file = open(s[2], 'r')
enc_file = open("enc_fileout", 'w')
crd = self.client_id.encryptFileSym(file, enc_file)
args = ("putfile", str(self.ccid), os.path.basename(s[2]))
salt = self.processCookie("/files")
dataq = []
dataq.append(self.client_id.genHashArgs(args, salt))
dataq.append(self.client_id.encryptData(crd[0],
self.client_id.pub_key))
dataq.append(self.client_id.encryptData(crd[1]))
agent = CookieAgent(Agent(reactor), self.cookie_jar)
#print crd[1]
# print "debugging:key, iv putfile"
# print dataq[1]
# print len(dataq[1])
# print dataq[2]
# print len(dataq[2])
print "Uploading file..."
enc_file = open("enc_fileout", 'r')
body = _FileProducer(enc_file, dataq)
headers = http_headers.Headers()
d = agent.request(
'PUT', 'http://localhost:8000/files/?method=putfile&ccid=' +
self.ccid + "&name=" + os.path.basename(s[2]), headers, body)
d.addCallback(self.printPutReply_cb)
return NOT_DONE_YET
def test_noProxyPassthrough(self):
"""
The CGI script is never called with the Proxy header passed through.
"""
cgiFilename = self.writeCGI(HEADER_OUTPUT_CGI)
portnum = self.startServer(cgiFilename)
url = "http://localhost:%d/cgi" % (portnum, )
agent = client.Agent(reactor)
headers = http_headers.Headers({
"Proxy": ["foo"],
"X-Innocent-Header": ["bar"]
})
d = agent.request("GET", url, headers=headers)
def checkResponse(response):
headers = json.loads(response)
self.assertEqual(
set(headers.keys()),
{"HTTP_HOST", "HTTP_CONNECTION", "HTTP_X_INNOCENT_HEADER"})
d.addCallback(client.readBody)
d.addCallback(checkResponse)
return d
def updateFile_cb(ticket, iv):
#data = (key,)
print "Updating file..."
s = line.split()
agent = Agent(reactor)
dataq = []
dataq.append(ticket)
dataq.append( iv )
# print "debugging:ticket, iv updatefile"
# print dataq[0]
# print dataq[1]
# print len(dataq[1])
print "Uploading file..."
enc_file = open("enc_fileout", 'r')
body = _FileProducer(enc_file ,dataq)
headers = http_headers.Headers()
d = agent.request(
'POST',
'http://localhost:8000/files/?method=updatefile&ccid='
+ self.ccid + "&name=" + os.path.basename(s[3]) + "&fileid=" + s[2] ,
headers,
body)
d.addCallback(self.printPutReply_cb)
return NOT_DONE_YET
def putFile_cb(ticket):
print "Encrypting file..."
s = line.split()
file = open(s[2], 'r')
enc_file = open("enc_fileout", 'w')
crd = self.client_id.encryptFileSym(file, enc_file)
agent = Agent(reactor)
dataq = []
dataq.append(ticket)
dataq.append( self.client_id.encryptData(crd[0], self.client_id.pub_key))
dataq.append( self.client_id.encryptData(crd[1]) )
#print crd[1]
# print "debugging:key, iv putfile"
# print dataq[1]
# print len(dataq[1])
# print dataq[2]
# print len(dataq[2])
print "Uploading file..."
enc_file = open("enc_fileout", 'r')
body = _FileProducer(enc_file ,dataq)
headers = http_headers.Headers()
d = agent.request(
'PUT',
'http://localhost:8000/files/?method=putfile&ccid='
+ self.ccid + "&name=" + os.path.basename(s[2]),
headers,
body)
d.addCallback(self.printPutReply_cb)
return NOT_DONE_YET
def build_twisted_request(self, method, url, extra_headers={}, body_producer=None, full_url=False):
""" Build a request for twisted
Args:
method (str): Request method (GET/POST/PUT/DELETE/etc.) If not specified, it will be POST if post_data is not None
url (str): Destination URL (full, or relative)
Kwargs:
extra_headers (dict): Headers (override default connection headers, if any)
body_producer (:class:`twisted.web.iweb.IBodyProducer`): Object producing request body
full_url (bool): If False, URL is relative
Returns:
tuple. Tuple with two elements: reactor, and request
"""
uri = url if full_url else self._url(url)
raw_headers = self.get_headers()
if extra_headers:
raw_headers.update(extra_headers)
headers = http_headers.Headers()
for header in raw_headers:
headers.addRawHeader(header, raw_headers[header])
agent = client.Agent(reactor)
request = agent.request(method, uri, headers, body_producer)
return (reactor, request)
def lookup(self):
headers = {}
headers['User-Agent'] = [random.choice(userAgents)]
d = self.agent.request("GET", self.url, http_headers.Headers(headers))
d.addCallback(self._response)
d.addErrback(self.failed)
return d
def test_headersUnmodified(self):
"""
If a I{Host} header must be added to the request, the L{Headers}
instance passed to L{Agent.request} is not modified.
"""
self.agent._connect = self._dummyConnect
headers = http_headers.Headers()
body = object()
self.agent.request('GET', 'http://example.com/foo', headers, body)
protocol = self.protocol
# The request should have been issued.
self.assertEquals(len(protocol.requests), 1)
# And the headers object passed in should not have changed.
self.assertEquals(headers, http_headers.Headers())
def test_missingElement_bogusAcceptHeader(self):
"""
Requesting a missing element results in a response that the element
was not found, even with a bogus Accept header.
"""
headers = http_headers.Headers()
headers.setRawHeaders("Accept", ["text/bogus"])
self._test_badElementRequest("bogus", None, headers, http.NOT_FOUND)
def test_badAcceptHeader(self):
"""
Providing a bogus Accept header when requesting a page results in an
error.
"""
headers = http_headers.Headers()
headers.setRawHeaders("Accept", ["text/bogus"])
self._test_badCollectionRequest(None, headers, http.NOT_ACCEPTABLE)
def post(self, url_path, dataMapping):
headers = http_headers.Headers()
headers.setRawHeaders(
b'Content-Type',
[b'application/x-www-form-urlencoded'],
)
bodyProducer = XWWWFormUrlencodedProducer(dataMapping)
return self._request(b'POST', url_path, headers, bodyProducer)
def postfile(self, artifact, fileName):
"""
Send a file to VirusTotal
"""
vtUrl = '{0}file/scan'.format(VTAPI_URL).encode('utf8')
fields = {('apikey', self.apiKey)}
files = {('file', fileName, open(artifact, 'rb'))}
if self.debug:
log.msg("submitting to VT: {0}".format(repr(files)))
contentType, body = encode_multipart_formdata(fields, files)
producer = StringProducer(body)
headers = http_headers.Headers({
'User-Agent': [COWRIE_USER_AGENT],
'Accept': ['*/*'],
'Content-Type': [contentType]
})
d = self.agent.request(b'POST', vtUrl, headers, producer)
def cbBody(body):
return processResult(body)
def cbPartial(failure):
"""
Google HTTP Server does not set Content-Length. Twisted marks it as partial
"""
return processResult(failure.value.response)
def cbResponse(response):
if response.code == 200:
d = client.readBody(response)
d.addCallback(cbBody)
d.addErrback(cbPartial)
return d
else:
log.msg("VT Request failed: {} {}".format(
response.code, response.phrase))
return
def cbError(failure):
failure.printTraceback()
def processResult(result):
if self.debug:
log.msg("VT postfile result: {}".format(result))
result = result.decode('utf8')
j = json.loads(result)
# This is always a new resource, since we did the scan before
# so always create the comment
log.msg("response=0: posting comment")
if self.comment is True:
return self.postcomment(j['resource'])
else:
return
d.addCallback(cbResponse)
d.addErrback(cbError)
return d
def postfile(self, artifact, fileName):
"""
Send a file to VirusTotal
"""
vtUrl = f"{VTAPI_URL}file/scan".encode("utf8")
fields = {("apikey", self.apiKey)}
files = {("file", fileName, open(artifact, "rb"))}
if self.debug:
log.msg(f"submitting to VT: {repr(files)}")
contentType, body = encode_multipart_formdata(fields, files)
producer = StringProducer(body)
headers = http_headers.Headers(
{
"User-Agent": [COWRIE_USER_AGENT],
"Accept": ["*/*"],
"Content-Type": [contentType],
}
)
d = self.agent.request(b"POST", vtUrl, headers, producer)
def cbBody(body):
return processResult(body)
def cbPartial(failure):
"""
Google HTTP Server does not set Content-Length. Twisted marks it as partial
"""
return processResult(failure.value.response)
def cbResponse(response):
if response.code == 200:
d = client.readBody(response)
d.addCallback(cbBody)
d.addErrback(cbPartial)
return d
else:
log.msg(f"VT Request failed: {response.code} {response.phrase}")
def cbError(failure):
failure.printTraceback()
def processResult(result):
if self.debug:
log.msg(f"VT postfile result: {result}")
result = result.decode("utf8")
j = json.loads(result)
# This is always a new resource, since we did the scan before
# so always create the comment
log.msg("response=0: posting comment")
if self.comment is True:
return self.postcomment(j["resource"])
else:
return
d.addCallback(cbResponse)
d.addErrback(cbError)
return d
def postfile(self, artifact, fileName):
"""
Send a file to VirusTotal
"""
vtUrl = "https://www.virustotal.com/vtapi/v2/file/scan"
contextFactory = WebClientContextFactory()
fields = {('apikey', self.apiKey)}
files = {('file', fileName, open(artifact, 'rb'))}
contentType, body = encode_multipart_formdata(fields, files)
producer = StringProducer(body)
headers = http_headers.Headers({
'User-Agent': ['Cowrie SSH Honeypot'],
'Accept': ['*/*'],
'Content-Type': [contentType]
})
agent = client.Agent(reactor, contextFactory)
d = agent.request('POST', vtUrl, headers, producer)
def cbBody(body):
return processResult(body)
def cbPartial(failure):
"""
Google HTTP Server does not set Content-Length. Twisted marks it as partial
"""
return processResult(failure.value.response)
def cbResponse(response):
if response.code == 200:
d = client.readBody(response)
d.addCallback(cbBody)
d.addErrback(cbPartial)
return d
else:
log.msg("VT Request failed: %s %s" % (response.code, response.phrase,))
return
def cbError(failure):
failure.printTraceback()
def processResult(result):
log.msg( "VT postfile result: %s" % result)
j = json.loads(result)
#log.msg( "VT postfile result: %s", repr(j) )
if j["response_code"] == 0:
log.msg( "response=0: posting comment")
d = self.postcomment(j["resource"])
return d
d.addCallback(cbResponse)
d.addErrback(cbError)
return d
def posturl(self, scanUrl):
"""
Send a URL to VirusTotal with Twisted
response_code:
If the item you searched for was not present in VirusTotal's dataset this result will be 0.
If the requested item is still queued for analysis it will be -2.
If the item was indeed present and it could be retrieved it will be 1.
"""
vtUrl = "https://www.virustotal.com/vtapi/v2/url/scan"
headers = http_headers.Headers({'User-Agent': ['Cowrie SSH Honeypot']})
fields = {"apikey": self.apiKey, "url": scanUrl}
data = urllib.urlencode(fields)
body = StringProducer(data)
contextFactory = WebClientContextFactory()
agent = client.Agent(reactor, contextFactory)
d = agent.request('POST', vtUrl, headers, body)
def cbBody(body):
return processResult(body)
def cbPartial(failure):
"""
Google HTTP Server does not set Content-Length. Twisted marks it as partial
"""
#failure.printTraceback()
return processResult(failure.value.response)
def cbResponse(response):
if response.code == 200:
d = client.readBody(response)
d.addCallback(cbBody)
d.addErrback(cbPartial)
return d
else:
log.msg("VT Request failed: %s %s" % (response.code, response.phrase,))
return
def cbError(failure):
failure.printTraceback()
def processResult(result):
j = json.loads(result)
log.msg( "VT posturl result: %s", repr(j) )
if j["response_code"] == 0:
log.msg( "response=0: posting comment")
d = self.postcomment(j["resource"])
return d
d.addCallback(cbResponse)
d.addErrback(cbError)
return d
def test_getElement_bogusAcceptHeader(self):
"""
Requesting an element with a bogus Accept header results in a response
that the resquest was not acceptable.
"""
headers = http_headers.Headers()
headers.setRawHeaders("Accept", ["text/bogus"])
name, _, _ = self.elementArgs[0]
self._test_badElementRequest(name, None, headers, http.NOT_ACCEPTABLE)
def postcomment(self, resource):
"""
Send a comment to VirusTotal with Twisted
"""
vtUrl = VTAPI_URL+b'comments/put'
parameters = { "resource": resource,
"comment": self.commenttext,
"apikey": self.apiKey}
headers = http_headers.Headers({'User-Agent': [COWRIE_USER_AGENT]})
body = StringProducer(urlencode(parameters).encode("utf-8"))
d = self.agent.request(b'POST', vtUrl, headers, body.encode('utf-8'))
def cbBody(body):
"""
"""
return processResult(body)
def cbPartial(failure):
"""
Google HTTP Server does not set Content-Length. Twisted marks it as partial
"""
return processResult(failure.value.response)
def cbResponse(response):
"""
"""
if response.code == 200:
d = client.readBody(response)
d.addCallback(cbBody)
d.addErrback(cbPartial)
return d
else:
log.msg("VT Request failed: {} {}".format(response.code, response.phrase))
return
def cbError(failure):
"""
"""
failure.printTraceback()
def processResult(result):
"""
"""
if self.debug:
log.msg("VT postcomment result: {}".format(result))
j = json.loads(result)
return j["response_code"]
d.addCallback(cbResponse)
d.addErrback(cbError)
return d
def handleListShares(signedTicket):
agent = Agent(reactor)
body = FileBodyProducer(StringIO(signedTicket))
headers = http_headers.Headers()
d = agent.request(
'GET',
'http://localhost:8000/shares/?method=list&ccid='
+ self.ccid,
headers,
body)
d.addCallback(handleList_cb)
return NOT_DONE_YET
def __init__(self, *args, **kwargs):
_TwistedDummyRequest.__init__(self, *args, **kwargs)
self.startedWriting = False
self._disconnected = False
# This is needed because _BaseHTTPTransport does
# self.request.channel.transport.setTcpNoDelay(True)
self.channel = DummyChannel()
self.requestHeaders = http_headers.Headers()
self.received_cookies = {}
def deleteShare_cb(ticket):
agent = Agent(reactor)
body = FileBodyProducer(StringIO(ticket))
headers = http_headers.Headers()
d = agent.request(
'DELETE',
'http://localhost:8000/shares/?method=delete&ccid='
+ self.ccid + "&fileid=" + s[2] + "&rccid=" + s[3],
headers,
body)
d.addCallback(printDeleteReply_cb)
def getURL(url, method="GET", redirect=0):
if isinstance(url, unicode):
url = url.encode("utf-8")
agent = Agent(reactor)
headers = http_headers.Headers({})
try:
response = (yield agent.request(method, url, headers, None))
except Exception, e:
log.error(str(e))
response = None
def postcomment(self, resource):
"""
Send a comment to VirusTotal with Twisted
"""
vtUrl = "https://www.virustotal.com/vtapi/v2/comments/put"
parameters = {
"resource": resource,
"comment":
"First seen by Cowrie SSH honeypot http://github.com/micheloosterhof/cowrie",
"apikey": self.apiKey
}
headers = http_headers.Headers({'User-Agent': ['Cowrie SSH Honeypot']})
data = urllib.urlencode(parameters)
body = StringProducer(data)
contextFactory = WebClientContextFactory()
agent = client.Agent(reactor, contextFactory)
d = agent.request('POST', vtUrl, headers, body)
def cbBody(body):
return processResult(body)
def cbPartial(failure):
"""
Google HTTP Server does not set Content-Length. Twisted marks it as partial
"""
return processResult(failure.value.response)
def cbResponse(response):
if response.code == 200:
d = client.readBody(response)
d.addCallback(cbBody)
d.addErrback(cbPartial)
return d
else:
log.msg("VT Request failed: %s %s" % (
response.code,
response.phrase,
))
return
def cbError(failure):
failure.printTraceback()
def processResult(result):
j = json.loads(result)
log.msg("VT postcomment result: %s", repr(j))
return j["response_code"]
d.addCallback(cbResponse)
d.addErrback(cbError)
return d
def postfile(self, artifact, fileName):
"""
Send a file to VirusTotal
"""
vtUrl = VTAPI_URL + b'file/scan'
fields = {('apikey', self.apiKey)}
files = {('file', fileName, open(artifact, 'rb'))}
print(repr(files))
contentType, body = encode_multipart_formdata(fields, files)
producer = StringProducer(body)
headers = http_headers.Headers({
'User-Agent': [COWRIE_USER_AGENT],
'Accept': ['*/*'],
'Content-Type': [contentType]
})
d = self.agent.request(b'POST', vtUrl, headers, producer)
def cbBody(body):
return processResult(body)
def cbPartial(failure):
"""
Google HTTP Server does not set Content-Length. Twisted marks it as partial
"""
return processResult(failure.value.response)
def cbResponse(response):
if response.code == 200:
d = client.readBody(response)
d.addCallback(cbBody)
d.addErrback(cbPartial)
return d
else:
log.msg("VT Request failed: {} {}".format(
response.code, response.phrase))
return
def cbError(failure):
failure.printTraceback()
def processResult(result):
log.msg("VT postfile result: {}".format(result))
j = json.loads(result)
if j["response_code"] == 0:
log.msg("response=0: posting comment")
d = self.postcomment(j["resource"])
return d
d.addCallback(cbResponse)
d.addErrback(cbError)
return d
def deleteFile_cb():
args = ("delete", str(self.ccid), s[2])
salt = self.processCookie("/files")
body = FileBodyProducer(
StringIO(self.client_id.genHashArgs(args, salt)))
agent = CookieAgent(Agent(reactor), self.cookie_jar)
headers = http_headers.Headers()
d = agent.request(
'DELETE', 'http://localhost:8000/files/?method=delete&ccid=' +
self.ccid + "&fileid=" + s[2], headers, body)
d.addCallback(printDeleteReply_cb)
def __init__(self, postpath, session=None):
self.sitepath = []
self.written = []
self.finished = 0
self.postpath = postpath
self.prepath = []
self.session = None
self.protoSession = session or server.Session(0, self)
self.args = {}
self.outgoingHeaders = {}
self.responseHeaders = http_headers.Headers()
self.responseCode = None
self.headers = {}
self._finishedDeferreds = []
def startSession_cb((signedNonce, nonceid)):
agent = CookieAgent(Agent(reactor), self.cookie_jar)
dataq = []
dataq.append(signedNonce)
body = _FileProducer(
StringIO(self.client_id.encryptData(self.client_id.password)),
dataq)
headers = http_headers.Headers()
d = agent.request(
'PUT',
'http://localhost:8000/session/?method=startsession&ccid=' +
self.ccid + '&nonceid=' + str(nonceid), headers, body)
d.addCallback(procResponse_cb)
return NOT_DONE_YET
