def test_request(self): """ L{Agent.request} establishes a new connection to the host indicated by the host part of the URI passed to it and issues a request using the method, the path portion of the URI, the headers, and the body producer passed to it. It returns a L{Deferred} which fires with a L{Response} from the server. """ self.agent._connect = self._dummyConnect headers = http_headers.Headers({'foo': ['bar']}) # Just going to check the body for identity, so it doesn't need to be # real. body = object() self.agent.request('GET', 'http://example.com:1234/foo?bar', headers, body) protocol = self.protocol # The request should be issued. self.assertEquals(len(protocol.requests), 1) req, res = protocol.requests.pop() self.assertIsInstance(req, Request) self.assertEquals(req.method, 'GET') self.assertEquals(req.uri, '/foo?bar') self.assertEquals( req.headers, http_headers.Headers({ 'foo': ['bar'], 'host': ['example.com:1234'] })) self.assertIdentical(req.bodyProducer, body)
def __init__(self, args=None, body=None, method="GET", prePathURL=BASE_URL, requestHeaders=None): self.args = args if args is not None else {} if body is not None: self.content = StringIO(body) else: self.content = StringIO('') self.prePathURL = lambda: prePathURL # we're always directly aimed at a resource and nobody is doing any # postpath-related stuff, so let's just pretend it's always emtpy... self.postpath = [] self.code = http.OK if requestHeaders is not None: self.requestHeaders = requestHeaders else: self.requestHeaders = http_headers.Headers() self.responseHeaders = http_headers.Headers() self.method = method
def handleStartSession(self, method): def procResponse_cb(response): defer = Deferred() defer.addCallback(method) response.deliverBody(DataPrinter(defer, "bool")) return NOT_DONE_YET def startSession_cb((signedNonce, nonceid)): agent = CookieAgent(Agent(reactor), self.cookie_jar) dataq = [] dataq.append(signedNonce) body = _FileProducer( StringIO(self.client_id.encryptData(self.client_id.password)), dataq) headers = http_headers.Headers() d = agent.request( 'PUT', 'http://localhost:8000/session/?method=startsession&ccid=' + self.ccid + '&nonceid=' + str(nonceid), headers, body) d.addCallback(procResponse_cb) return NOT_DONE_YET def getNonce_cb(response): defer = Deferred() defer.addCallback(startSession_cb) response.deliverBody(getNonce(defer, self.client_id, self.pin)) return NOT_DONE_YET if self.pin != None: agent = Agent(reactor) body = FileBodyProducer( StringIO(self.client_id.pub_key.exportKey('PEM'))) headers = http_headers.Headers() d = agent.request( 'GET', 'http://localhost:8000/session/?method=getnonce', headers, body) d.addCallback(getNonce_cb) return NOT_DONE_YET agent = CookieAgent(Agent(reactor), self.cookie_jar) body = FileBodyProducer( StringIO(self.client_id.encryptData(self.client_id.password))) headers = http_headers.Headers() d = agent.request( 'PUT', 'http://localhost:8000/session/?method=startsession&ccid=' + self.ccid + '&nonceid=' + str(-1), headers, body) d.addCallback(procResponse_cb) return NOT_DONE_YET
def postentry(self, entry): headers = http_headers.Headers({ b"Content-Type": [b"application/json"], }) body = FileBodyProducer(BytesIO(json.dumps(entry).encode("utf8"))) self.agent.request(b"POST", self.url, headers, body)
def updateShared_cb(iv): print "Updating file..." args = ("updateshared", str(self.ccid), os.path.basename(s[3]), s[2]) salt = self.processCookie("/shares") dataq = [] dataq.append(self.client_id.genHashArgs(args, salt)) dataq.append(iv) # print "debugging:ticket, iv updatefile" # print dataq[0] # print dataq[1] # print len(dataq[1]) print "Uploading file..." agent = CookieAgent(Agent(reactor), self.cookie_jar) enc_file = open("enc_fileout", 'r') body = _FileProducer(enc_file, dataq) headers = http_headers.Headers() d = agent.request( 'POST', 'http://localhost:8000/shares/?method=updateshared&ccid=' + self.ccid + "&name=" + os.path.basename(s[3]) + "&fileid=" + s[2], headers, body) d.addCallback(self.printPutReply_cb) return NOT_DONE_YET
def handlePutFile(self, line): print "Encrypting file..." s = line.split() file = open(s[2], 'r') enc_file = open("enc_fileout", 'w') crd = self.client_id.encryptFileSym(file, enc_file) args = ("putfile", str(self.ccid), os.path.basename(s[2])) salt = self.processCookie("/files") dataq = [] dataq.append(self.client_id.genHashArgs(args, salt)) dataq.append(self.client_id.encryptData(crd[0], self.client_id.pub_key)) dataq.append(self.client_id.encryptData(crd[1])) agent = CookieAgent(Agent(reactor), self.cookie_jar) #print crd[1] # print "debugging:key, iv putfile" # print dataq[1] # print len(dataq[1]) # print dataq[2] # print len(dataq[2]) print "Uploading file..." enc_file = open("enc_fileout", 'r') body = _FileProducer(enc_file, dataq) headers = http_headers.Headers() d = agent.request( 'PUT', 'http://localhost:8000/files/?method=putfile&ccid=' + self.ccid + "&name=" + os.path.basename(s[2]), headers, body) d.addCallback(self.printPutReply_cb) return NOT_DONE_YET
def test_noProxyPassthrough(self): """ The CGI script is never called with the Proxy header passed through. """ cgiFilename = self.writeCGI(HEADER_OUTPUT_CGI) portnum = self.startServer(cgiFilename) url = "http://localhost:%d/cgi" % (portnum, ) agent = client.Agent(reactor) headers = http_headers.Headers({ "Proxy": ["foo"], "X-Innocent-Header": ["bar"] }) d = agent.request("GET", url, headers=headers) def checkResponse(response): headers = json.loads(response) self.assertEqual( set(headers.keys()), {"HTTP_HOST", "HTTP_CONNECTION", "HTTP_X_INNOCENT_HEADER"}) d.addCallback(client.readBody) d.addCallback(checkResponse) return d
def updateFile_cb(ticket, iv): #data = (key,) print "Updating file..." s = line.split() agent = Agent(reactor) dataq = [] dataq.append(ticket) dataq.append( iv ) # print "debugging:ticket, iv updatefile" # print dataq[0] # print dataq[1] # print len(dataq[1]) print "Uploading file..." enc_file = open("enc_fileout", 'r') body = _FileProducer(enc_file ,dataq) headers = http_headers.Headers() d = agent.request( 'POST', 'http://localhost:8000/files/?method=updatefile&ccid=' + self.ccid + "&name=" + os.path.basename(s[3]) + "&fileid=" + s[2] , headers, body) d.addCallback(self.printPutReply_cb) return NOT_DONE_YET
def putFile_cb(ticket): print "Encrypting file..." s = line.split() file = open(s[2], 'r') enc_file = open("enc_fileout", 'w') crd = self.client_id.encryptFileSym(file, enc_file) agent = Agent(reactor) dataq = [] dataq.append(ticket) dataq.append( self.client_id.encryptData(crd[0], self.client_id.pub_key)) dataq.append( self.client_id.encryptData(crd[1]) ) #print crd[1] # print "debugging:key, iv putfile" # print dataq[1] # print len(dataq[1]) # print dataq[2] # print len(dataq[2]) print "Uploading file..." enc_file = open("enc_fileout", 'r') body = _FileProducer(enc_file ,dataq) headers = http_headers.Headers() d = agent.request( 'PUT', 'http://localhost:8000/files/?method=putfile&ccid=' + self.ccid + "&name=" + os.path.basename(s[2]), headers, body) d.addCallback(self.printPutReply_cb) return NOT_DONE_YET
def build_twisted_request(self, method, url, extra_headers={}, body_producer=None, full_url=False): """ Build a request for twisted Args: method (str): Request method (GET/POST/PUT/DELETE/etc.) If not specified, it will be POST if post_data is not None url (str): Destination URL (full, or relative) Kwargs: extra_headers (dict): Headers (override default connection headers, if any) body_producer (:class:`twisted.web.iweb.IBodyProducer`): Object producing request body full_url (bool): If False, URL is relative Returns: tuple. Tuple with two elements: reactor, and request """ uri = url if full_url else self._url(url) raw_headers = self.get_headers() if extra_headers: raw_headers.update(extra_headers) headers = http_headers.Headers() for header in raw_headers: headers.addRawHeader(header, raw_headers[header]) agent = client.Agent(reactor) request = agent.request(method, uri, headers, body_producer) return (reactor, request)
def lookup(self): headers = {} headers['User-Agent'] = [random.choice(userAgents)] d = self.agent.request("GET", self.url, http_headers.Headers(headers)) d.addCallback(self._response) d.addErrback(self.failed) return d
def test_headersUnmodified(self): """ If a I{Host} header must be added to the request, the L{Headers} instance passed to L{Agent.request} is not modified. """ self.agent._connect = self._dummyConnect headers = http_headers.Headers() body = object() self.agent.request('GET', 'http://example.com/foo', headers, body) protocol = self.protocol # The request should have been issued. self.assertEquals(len(protocol.requests), 1) # And the headers object passed in should not have changed. self.assertEquals(headers, http_headers.Headers())
def test_missingElement_bogusAcceptHeader(self): """ Requesting a missing element results in a response that the element was not found, even with a bogus Accept header. """ headers = http_headers.Headers() headers.setRawHeaders("Accept", ["text/bogus"]) self._test_badElementRequest("bogus", None, headers, http.NOT_FOUND)
def test_badAcceptHeader(self): """ Providing a bogus Accept header when requesting a page results in an error. """ headers = http_headers.Headers() headers.setRawHeaders("Accept", ["text/bogus"]) self._test_badCollectionRequest(None, headers, http.NOT_ACCEPTABLE)
def post(self, url_path, dataMapping): headers = http_headers.Headers() headers.setRawHeaders( b'Content-Type', [b'application/x-www-form-urlencoded'], ) bodyProducer = XWWWFormUrlencodedProducer(dataMapping) return self._request(b'POST', url_path, headers, bodyProducer)
def postfile(self, artifact, fileName): """ Send a file to VirusTotal """ vtUrl = '{0}file/scan'.format(VTAPI_URL).encode('utf8') fields = {('apikey', self.apiKey)} files = {('file', fileName, open(artifact, 'rb'))} if self.debug: log.msg("submitting to VT: {0}".format(repr(files))) contentType, body = encode_multipart_formdata(fields, files) producer = StringProducer(body) headers = http_headers.Headers({ 'User-Agent': [COWRIE_USER_AGENT], 'Accept': ['*/*'], 'Content-Type': [contentType] }) d = self.agent.request(b'POST', vtUrl, headers, producer) def cbBody(body): return processResult(body) def cbPartial(failure): """ Google HTTP Server does not set Content-Length. Twisted marks it as partial """ return processResult(failure.value.response) def cbResponse(response): if response.code == 200: d = client.readBody(response) d.addCallback(cbBody) d.addErrback(cbPartial) return d else: log.msg("VT Request failed: {} {}".format( response.code, response.phrase)) return def cbError(failure): failure.printTraceback() def processResult(result): if self.debug: log.msg("VT postfile result: {}".format(result)) result = result.decode('utf8') j = json.loads(result) # This is always a new resource, since we did the scan before # so always create the comment log.msg("response=0: posting comment") if self.comment is True: return self.postcomment(j['resource']) else: return d.addCallback(cbResponse) d.addErrback(cbError) return d
def postfile(self, artifact, fileName): """ Send a file to VirusTotal """ vtUrl = f"{VTAPI_URL}file/scan".encode("utf8") fields = {("apikey", self.apiKey)} files = {("file", fileName, open(artifact, "rb"))} if self.debug: log.msg(f"submitting to VT: {repr(files)}") contentType, body = encode_multipart_formdata(fields, files) producer = StringProducer(body) headers = http_headers.Headers( { "User-Agent": [COWRIE_USER_AGENT], "Accept": ["*/*"], "Content-Type": [contentType], } ) d = self.agent.request(b"POST", vtUrl, headers, producer) def cbBody(body): return processResult(body) def cbPartial(failure): """ Google HTTP Server does not set Content-Length. Twisted marks it as partial """ return processResult(failure.value.response) def cbResponse(response): if response.code == 200: d = client.readBody(response) d.addCallback(cbBody) d.addErrback(cbPartial) return d else: log.msg(f"VT Request failed: {response.code} {response.phrase}") def cbError(failure): failure.printTraceback() def processResult(result): if self.debug: log.msg(f"VT postfile result: {result}") result = result.decode("utf8") j = json.loads(result) # This is always a new resource, since we did the scan before # so always create the comment log.msg("response=0: posting comment") if self.comment is True: return self.postcomment(j["resource"]) else: return d.addCallback(cbResponse) d.addErrback(cbError) return d
def postfile(self, artifact, fileName): """ Send a file to VirusTotal """ vtUrl = "https://www.virustotal.com/vtapi/v2/file/scan" contextFactory = WebClientContextFactory() fields = {('apikey', self.apiKey)} files = {('file', fileName, open(artifact, 'rb'))} contentType, body = encode_multipart_formdata(fields, files) producer = StringProducer(body) headers = http_headers.Headers({ 'User-Agent': ['Cowrie SSH Honeypot'], 'Accept': ['*/*'], 'Content-Type': [contentType] }) agent = client.Agent(reactor, contextFactory) d = agent.request('POST', vtUrl, headers, producer) def cbBody(body): return processResult(body) def cbPartial(failure): """ Google HTTP Server does not set Content-Length. Twisted marks it as partial """ return processResult(failure.value.response) def cbResponse(response): if response.code == 200: d = client.readBody(response) d.addCallback(cbBody) d.addErrback(cbPartial) return d else: log.msg("VT Request failed: %s %s" % (response.code, response.phrase,)) return def cbError(failure): failure.printTraceback() def processResult(result): log.msg( "VT postfile result: %s" % result) j = json.loads(result) #log.msg( "VT postfile result: %s", repr(j) ) if j["response_code"] == 0: log.msg( "response=0: posting comment") d = self.postcomment(j["resource"]) return d d.addCallback(cbResponse) d.addErrback(cbError) return d
def posturl(self, scanUrl): """ Send a URL to VirusTotal with Twisted response_code: If the item you searched for was not present in VirusTotal's dataset this result will be 0. If the requested item is still queued for analysis it will be -2. If the item was indeed present and it could be retrieved it will be 1. """ vtUrl = "https://www.virustotal.com/vtapi/v2/url/scan" headers = http_headers.Headers({'User-Agent': ['Cowrie SSH Honeypot']}) fields = {"apikey": self.apiKey, "url": scanUrl} data = urllib.urlencode(fields) body = StringProducer(data) contextFactory = WebClientContextFactory() agent = client.Agent(reactor, contextFactory) d = agent.request('POST', vtUrl, headers, body) def cbBody(body): return processResult(body) def cbPartial(failure): """ Google HTTP Server does not set Content-Length. Twisted marks it as partial """ #failure.printTraceback() return processResult(failure.value.response) def cbResponse(response): if response.code == 200: d = client.readBody(response) d.addCallback(cbBody) d.addErrback(cbPartial) return d else: log.msg("VT Request failed: %s %s" % (response.code, response.phrase,)) return def cbError(failure): failure.printTraceback() def processResult(result): j = json.loads(result) log.msg( "VT posturl result: %s", repr(j) ) if j["response_code"] == 0: log.msg( "response=0: posting comment") d = self.postcomment(j["resource"]) return d d.addCallback(cbResponse) d.addErrback(cbError) return d
def test_getElement_bogusAcceptHeader(self): """ Requesting an element with a bogus Accept header results in a response that the resquest was not acceptable. """ headers = http_headers.Headers() headers.setRawHeaders("Accept", ["text/bogus"]) name, _, _ = self.elementArgs[0] self._test_badElementRequest(name, None, headers, http.NOT_ACCEPTABLE)
def postcomment(self, resource): """ Send a comment to VirusTotal with Twisted """ vtUrl = VTAPI_URL+b'comments/put' parameters = { "resource": resource, "comment": self.commenttext, "apikey": self.apiKey} headers = http_headers.Headers({'User-Agent': [COWRIE_USER_AGENT]}) body = StringProducer(urlencode(parameters).encode("utf-8")) d = self.agent.request(b'POST', vtUrl, headers, body.encode('utf-8')) def cbBody(body): """ """ return processResult(body) def cbPartial(failure): """ Google HTTP Server does not set Content-Length. Twisted marks it as partial """ return processResult(failure.value.response) def cbResponse(response): """ """ if response.code == 200: d = client.readBody(response) d.addCallback(cbBody) d.addErrback(cbPartial) return d else: log.msg("VT Request failed: {} {}".format(response.code, response.phrase)) return def cbError(failure): """ """ failure.printTraceback() def processResult(result): """ """ if self.debug: log.msg("VT postcomment result: {}".format(result)) j = json.loads(result) return j["response_code"] d.addCallback(cbResponse) d.addErrback(cbError) return d
def handleListShares(signedTicket): agent = Agent(reactor) body = FileBodyProducer(StringIO(signedTicket)) headers = http_headers.Headers() d = agent.request( 'GET', 'http://localhost:8000/shares/?method=list&ccid=' + self.ccid, headers, body) d.addCallback(handleList_cb) return NOT_DONE_YET
def __init__(self, *args, **kwargs): _TwistedDummyRequest.__init__(self, *args, **kwargs) self.startedWriting = False self._disconnected = False # This is needed because _BaseHTTPTransport does # self.request.channel.transport.setTcpNoDelay(True) self.channel = DummyChannel() self.requestHeaders = http_headers.Headers() self.received_cookies = {}
def deleteShare_cb(ticket): agent = Agent(reactor) body = FileBodyProducer(StringIO(ticket)) headers = http_headers.Headers() d = agent.request( 'DELETE', 'http://localhost:8000/shares/?method=delete&ccid=' + self.ccid + "&fileid=" + s[2] + "&rccid=" + s[3], headers, body) d.addCallback(printDeleteReply_cb)
def getURL(url, method="GET", redirect=0): if isinstance(url, unicode): url = url.encode("utf-8") agent = Agent(reactor) headers = http_headers.Headers({}) try: response = (yield agent.request(method, url, headers, None)) except Exception, e: log.error(str(e)) response = None
def postcomment(self, resource): """ Send a comment to VirusTotal with Twisted """ vtUrl = "https://www.virustotal.com/vtapi/v2/comments/put" parameters = { "resource": resource, "comment": "First seen by Cowrie SSH honeypot http://github.com/micheloosterhof/cowrie", "apikey": self.apiKey } headers = http_headers.Headers({'User-Agent': ['Cowrie SSH Honeypot']}) data = urllib.urlencode(parameters) body = StringProducer(data) contextFactory = WebClientContextFactory() agent = client.Agent(reactor, contextFactory) d = agent.request('POST', vtUrl, headers, body) def cbBody(body): return processResult(body) def cbPartial(failure): """ Google HTTP Server does not set Content-Length. Twisted marks it as partial """ return processResult(failure.value.response) def cbResponse(response): if response.code == 200: d = client.readBody(response) d.addCallback(cbBody) d.addErrback(cbPartial) return d else: log.msg("VT Request failed: %s %s" % ( response.code, response.phrase, )) return def cbError(failure): failure.printTraceback() def processResult(result): j = json.loads(result) log.msg("VT postcomment result: %s", repr(j)) return j["response_code"] d.addCallback(cbResponse) d.addErrback(cbError) return d
def postfile(self, artifact, fileName): """ Send a file to VirusTotal """ vtUrl = VTAPI_URL + b'file/scan' fields = {('apikey', self.apiKey)} files = {('file', fileName, open(artifact, 'rb'))} print(repr(files)) contentType, body = encode_multipart_formdata(fields, files) producer = StringProducer(body) headers = http_headers.Headers({ 'User-Agent': [COWRIE_USER_AGENT], 'Accept': ['*/*'], 'Content-Type': [contentType] }) d = self.agent.request(b'POST', vtUrl, headers, producer) def cbBody(body): return processResult(body) def cbPartial(failure): """ Google HTTP Server does not set Content-Length. Twisted marks it as partial """ return processResult(failure.value.response) def cbResponse(response): if response.code == 200: d = client.readBody(response) d.addCallback(cbBody) d.addErrback(cbPartial) return d else: log.msg("VT Request failed: {} {}".format( response.code, response.phrase)) return def cbError(failure): failure.printTraceback() def processResult(result): log.msg("VT postfile result: {}".format(result)) j = json.loads(result) if j["response_code"] == 0: log.msg("response=0: posting comment") d = self.postcomment(j["resource"]) return d d.addCallback(cbResponse) d.addErrback(cbError) return d
def deleteFile_cb(): args = ("delete", str(self.ccid), s[2]) salt = self.processCookie("/files") body = FileBodyProducer( StringIO(self.client_id.genHashArgs(args, salt))) agent = CookieAgent(Agent(reactor), self.cookie_jar) headers = http_headers.Headers() d = agent.request( 'DELETE', 'http://localhost:8000/files/?method=delete&ccid=' + self.ccid + "&fileid=" + s[2], headers, body) d.addCallback(printDeleteReply_cb)
def __init__(self, postpath, session=None): self.sitepath = [] self.written = [] self.finished = 0 self.postpath = postpath self.prepath = [] self.session = None self.protoSession = session or server.Session(0, self) self.args = {} self.outgoingHeaders = {} self.responseHeaders = http_headers.Headers() self.responseCode = None self.headers = {} self._finishedDeferreds = []
def startSession_cb((signedNonce, nonceid)): agent = CookieAgent(Agent(reactor), self.cookie_jar) dataq = [] dataq.append(signedNonce) body = _FileProducer( StringIO(self.client_id.encryptData(self.client_id.password)), dataq) headers = http_headers.Headers() d = agent.request( 'PUT', 'http://localhost:8000/session/?method=startsession&ccid=' + self.ccid + '&nonceid=' + str(nonceid), headers, body) d.addCallback(procResponse_cb) return NOT_DONE_YET