def __init__(self, *args, **kwargs):
self.proxy_username = None
if "proxy_host" in kwargs:
port = 80
if "proxy_port" in kwargs:
port = kwargs["proxy_port"]
del kwargs["proxy_port"]
if "proxy_username" in kwargs:
self.proxy_username = kwargs["proxy_username"]
del kwargs["proxy_username"]
if "proxy_password" in kwargs:
self.proxy_password = kwargs["proxy_password"]
del kwargs["proxy_password"]
endpoint = endpoints.TCP4ClientEndpoint(reactor,
kwargs["proxy_host"], port)
self.agent = client.ProxyAgent(endpoint)
del kwargs["proxy_host"]
else:
from twisted.web import version as twisted_version
if twisted_version.major < 14:
# FIXME: for Ubuntu 12.04 Twisted 11? (until 2017/04)
self.agent = client.Agent(reactor)
else:
contextFactory = WebClientContextFactory()
self.agent = client.Agent(reactor, contextFactory)
Twitter.__init__(self, *args, **kwargs)
def _getAgent(reactor=reactor, url=API_SSL_VERIFY_URL, connectTimeout=30,
**kwargs):
"""Create a :api:`twisted.web.client.Agent` which will verify the
certificate chain and hostname for the given **url**.
:param reactor: A provider of the
:api:`twisted.internet.interface.IReactorTCP` interface.
:param str url: The full URL which will be requested with the
``Agent``. (default: :attr:`API_SSL_VERIFY_URL`)
:param pool: An :api:`twisted.web.client.HTTPConnectionPool`
instance. (default: :attr:`_pool`)
:type connectTimeout: None or int
:param connectTimeout: If not ``None``, the timeout passed to
:api:`twisted.internet.reactor.connectTCP` or
:api:`twisted.internet.reactor.connectSSL` for specifying the
connection timeout. (default: ``30``)
"""
# Twisted>=14.0.0 changed the way in which hostname verification works.
if _twistedversion >= Version('twisted', 14, 0, 0):
contextFactory = RecaptchaPolicyForHTTPS()
else:
contextFactory = SSLVerifyingContextFactory(url)
if _connectionPoolAvailable:
return client.Agent(reactor,
contextFactory=contextFactory,
connectTimeout=connectTimeout,
pool=_pool,
**kwargs)
else:
return client.Agent(reactor,
contextFactory=contextFactory,
connectTimeout=connectTimeout,
**kwargs)
def setUp(self):
# set up a full master serving HTTP
yield self.setUpRealDatabase(table_names=['masters', 'objects', 'object_state'],
sqlite_memory=False)
master = fakemaster.FakeMaster(reactor)
master.config.db = dict(db_url=self.db_url)
master.db = dbconnector.DBConnector('basedir')
yield master.db.setServiceParent(master)
yield master.db.setup(check_version=False)
master.config.mq = dict(type='simple')
master.mq = mqconnector.MQConnector()
yield master.mq.setServiceParent(master)
yield master.mq.setup()
master.data = dataconnector.DataConnector()
yield master.data.setServiceParent(master)
master.config.www = dict(
port='tcp:0:interface=127.0.0.1',
debug=True,
auth=auth.NoAuth(),
authz=authz.Authz(),
avatar_methods=[],
logfileName='http.log')
master.www = wwwservice.WWWService()
yield master.www.setServiceParent(master)
yield master.www.startService()
yield master.www.reconfigServiceWithBuildbotConfig(master.config)
session = mock.Mock()
session.uid = "0"
master.www.site.sessionFactory = mock.Mock(return_value=session)
# now that we have a port, construct the real URL and insert it into
# the config. The second reconfig isn't really required, but doesn't
# hurt.
self.url = 'http://127.0.0.1:%d/' % master.www.getPortnum()
self.url = unicode2bytes(self.url)
master.config.buildbotURL = self.url
yield master.www.reconfigServiceWithBuildbotConfig(master.config)
self.master = master
# build an HTTP agent, using an explicit connection pool if Twisted
# supports it (Twisted 13.0.0 and up)
if hasattr(client, 'HTTPConnectionPool'):
self.pool = client.HTTPConnectionPool(reactor)
self.agent = client.Agent(reactor, pool=self.pool)
else:
self.pool = None
self.agent = client.Agent(reactor)
def __init__(self, relay, appid, side, handle_welcome):
assert isinstance(relay, type(u""))
self._relay = relay
self._appid = appid
self._side = side
self._handle_welcome = handle_welcome
self._agent = web_client.Agent(reactor)
def build_twisted_request(self, method, url, extra_headers={}, body_producer=None, full_url=False):
""" Build a request for twisted
Args:
method (str): Request method (GET/POST/PUT/DELETE/etc.) If not specified, it will be POST if post_data is not None
url (str): Destination URL (full, or relative)
Kwargs:
extra_headers (dict): Headers (override default connection headers, if any)
body_producer (:class:`twisted.web.iweb.IBodyProducer`): Object producing request body
full_url (bool): If False, URL is relative
Returns:
tuple. Tuple with two elements: reactor, and request
"""
uri = url if full_url else self._url(url)
raw_headers = self.get_headers()
if extra_headers:
raw_headers.update(extra_headers)
headers = http_headers.Headers()
for header in raw_headers:
headers.addRawHeader(header, raw_headers[header])
agent = client.Agent(reactor)
request = agent.request(method, uri, headers, body_producer)
return (reactor, request)
def test_malformedHeaderCGI(self):
"""
Check for the error message in the duplicated header
"""
cgiFilename = self.writeCGI(BROKEN_HEADER_CGI)
portnum = self.startServer(cgiFilename)
url = "http://localhost:%d/cgi" % (portnum, )
url = url.encode("ascii")
agent = client.Agent(reactor)
d = agent.request(b"GET", url)
d.addCallback(discardBody)
loggedMessages = []
def addMessage(eventDict):
loggedMessages.append(log.textFromEventDict(eventDict))
log.addObserver(addMessage)
self.addCleanup(log.removeObserver, addMessage)
def checkResponse(ignored):
self.assertIn("ignoring malformed CGI header: " + repr(b'XYZ'),
loggedMessages)
d.addCallback(checkResponse)
return d
def _requestAgentTest(self, child, **kwargs):
"""
Set up a resource on a distrib site using L{ResourcePublisher} and
then retrieve it from a L{ResourceSubscription} via an HTTP client.
@param child: The resource to publish using distrib.
@param **kwargs: Extra keyword arguments to pass to L{Agent.request} when
requesting the resource.
@return: A L{Deferred} which fires with a tuple consisting of a
L{twisted.test.proto_helpers.AccumulatingProtocol} containing the
body of the response and an L{IResponse} with the response itself.
"""
mainPort, mainAddr = self._setupDistribServer(child)
d = client.Agent(reactor).request("GET", "http://%s:%s/child" % (
mainAddr.host, mainAddr.port), **kwargs)
def cbCollectBody(response):
protocol = proto_helpers.AccumulatingProtocol()
response.deliverBody(protocol)
d = protocol.closedDeferred = defer.Deferred()
d.addCallback(lambda _: (protocol, response))
return d
d.addCallback(cbCollectBody)
return d
def http_post(reactor, url, value):
agent = client.Agent(reactor)
d = agent.request('POST', url,
headers=client.Headers({'Content-Type': ['application/json']}),
# in principle this could be streaming if we had a pipe-thing to glue between json.dump and FileBodyProducer
bodyProducer=client.FileBodyProducer(StringIO.StringIO(json.dumps(value))))
return _handle_agent_response(d)
def build_agent(self, proxy, headers):
"""Build an agent for this request
"""
fragments = common.parse_proxy(proxy)
pool = self.build_pool()
if fragments.host:
# add proxy authentication header
auth = base64.b64encode("%s:%s" %
(fragments.username, fragments.password))
headers['Proxy-Authorization'] = ["Basic " + auth.strip()]
# generate the agent
endpoint = endpoints.TCP4ClientEndpoint(
reactor,
fragments.host,
int(fragments.port),
timeout=self.settings.timeout)
agent = client.ProxyAgent(endpoint, reactor=reactor, pool=pool)
else:
agent = client.Agent(reactor,
connectTimeout=self.settings.timeout,
pool=pool)
agent = client.ContentDecoderAgent(agent,
[('gzip', client.GzipDecoder)])
# XXX if use same cookie for all then works...
# cookies usually empty
if proxy in self.cookiejars:
cj = self.cookiejars[proxy]
else:
cj = cookielib.CookieJar()
self.cookiejars[proxy] = cj
agent = client.CookieAgent(agent, cj)
return agent
def make_http_pool_and_agent(params):
cp_size = params.get('cp_size', 5)
c_timeout = params.get('c_timeout', 30.0)
# XXX: more extensibility
auth = params.get('auth', 'authhmac')
assert not auth or auth.lower() in ['none', 'authhmac', 'basic', 'digest']
http_pool = client.HTTPConnectionPool(reactor)
http_pool._factory = _NoiselessHTTP11ClientFactory
http_pool.retryAutomatically = False
http_pool.maxPersistentPerHost = cp_size
agent = client.Agent(reactor, pool=http_pool, connectTimeout=c_timeout)
if not auth or auth.lower() == 'none':
pass
elif auth.lower() == 'authhmac':
access_key = params['access_key']
secret_key = params.get('secret_key')
agent = authhmac.AuthHMACAgent(agent, access_key, secret_key)
elif auth.lower() == 'basic':
username = params['username']
password = params.get('password')
agent = httprpc.BasicAuthAgent(agent, username, password)
else:
raise AssertionError("unknown %r auth" % auth)
return http_pool, agent
def http_get(reactor, url, accept=None):
agent = client.Agent(reactor)
headers = Headers()
if accept is not None:
headers.addRawHeader('Accept', str(accept))
d = agent.request(b'GET', str(url), headers=headers)
return _handle_agent_response(d)
def test_connectHTTPSCustomContextFactory(self):
"""
If a context factory is passed to L{Agent.__init__} it will be used to
determine the SSL parameters for HTTPS requests. When an HTTPS request
is made, the hostname and port number of the request URL will be passed
to the context factory's C{getContext} method. The resulting context
object will be used to establish the SSL connection.
"""
expectedHost = 'example.org'
expectedPort = 20443
expectedContext = object()
contextArgs = []
class StubWebContextFactory(object):
def getContext(self, hostname, port):
contextArgs.append((hostname, port))
return expectedContext
agent = client.Agent(self.reactor, StubWebContextFactory())
d = agent._connect('https', expectedHost, expectedPort)
host, port, factory, contextFactory = self.reactor.sslClients.pop()[:4]
context = contextFactory.getContext()
self.assertEquals(context, expectedContext)
self.assertEquals(contextArgs, [(expectedHost, expectedPort)])
protocol = factory.buildProtocol(IPv4Address('TCP', '10.0.0.1', port))
self.assertIsInstance(protocol, HTTP11ClientProtocol)
self.completeConnection()
d.addCallback(self.assertIdentical, protocol)
return d
def test_noProxyPassthrough(self):
"""
The CGI script is never called with the Proxy header passed through.
"""
cgiFilename = self.writeCGI(HEADER_OUTPUT_CGI)
portnum = self.startServer(cgiFilename)
url = "http://localhost:%d/cgi" % (portnum, )
agent = client.Agent(reactor)
headers = http_headers.Headers({
"Proxy": ["foo"],
"X-Innocent-Header": ["bar"]
})
d = agent.request("GET", url, headers=headers)
def checkResponse(response):
headers = json.loads(response)
self.assertEqual(
set(headers.keys()),
{"HTTP_HOST", "HTTP_CONNECTION", "HTTP_X_INNOCENT_HEADER"})
d.addCallback(client.readBody)
d.addCallback(checkResponse)
return d
def __init__(self, appid, relay):
self.appid = appid
self.relay = relay
self.agent = web_client.Agent(reactor)
self.side = None
self.code = None
self.key = None
self._started_get_code = False
def get_sth():
agent = twisted_client.Agent(reactor)
client = log_client.AsyncLogClient(agent, "https://ct.googleapis.com/pilot")
d = client.get_sth()
# Print the STH on success.
d.addCallback(sth_callback)
# Stop the reactor whether we succeeded or not.
d.addBoth(stop_callback)
def __init__(self, db_filename, workerm, sched):
self.workerm = workerm
self.database = DBWrapper(db_filename)
self.scheduler = sched
self.inProgress = {}
# If a connection pool and/or keepalive is necessary
# in the future, add it here.
self.agent = client.Agent(reactor)
def startService(self):
self.agent = client.Agent(reactor, connectTimeout=5)
self.agent._pool._factory.noisy = False
self.db = adbapi.ConnectionPool("MySQLdb",
db="servrhe",
cp_reconnect=True)
yield self.loadModules()
service.MultiService.startService(self)
def posturl(self, scanUrl):
"""
Send a URL to VirusTotal with Twisted
response_code:
If the item you searched for was not present in VirusTotal's dataset this result will be 0.
If the requested item is still queued for analysis it will be -2.
If the item was indeed present and it could be retrieved it will be 1.
"""
vtUrl = "https://www.virustotal.com/vtapi/v2/url/scan"
headers = http_headers.Headers({'User-Agent': ['Cowrie SSH Honeypot']})
fields = {"apikey": self.apiKey, "url": scanUrl}
data = urllib.urlencode(fields)
body = StringProducer(data)
contextFactory = WebClientContextFactory()
agent = client.Agent(reactor, contextFactory)
d = agent.request('POST', vtUrl, headers, body)
def cbBody(body):
return processResult(body)
def cbPartial(failure):
"""
Google HTTP Server does not set Content-Length. Twisted marks it as partial
"""
#failure.printTraceback()
return processResult(failure.value.response)
def cbResponse(response):
if response.code == 200:
d = client.readBody(response)
d.addCallback(cbBody)
d.addErrback(cbPartial)
return d
else:
log.msg("VT Request failed: %s %s" % (response.code, response.phrase,))
return
def cbError(failure):
failure.printTraceback()
def processResult(result):
j = json.loads(result)
log.msg( "VT posturl result: %s", repr(j) )
if j["response_code"] == 0:
log.msg( "response=0: posting comment")
d = self.postcomment(j["resource"])
return d
d.addCallback(cbResponse)
d.addErrback(cbError)
return d
def postfile(self, artifact, fileName):
"""
Send a file to VirusTotal
"""
vtUrl = "https://www.virustotal.com/vtapi/v2/file/scan"
contextFactory = WebClientContextFactory()
fields = {('apikey', self.apiKey)}
files = {('file', fileName, open(artifact, 'rb'))}
contentType, body = encode_multipart_formdata(fields, files)
producer = StringProducer(body)
headers = http_headers.Headers({
'User-Agent': ['Cowrie SSH Honeypot'],
'Accept': ['*/*'],
'Content-Type': [contentType]
})
agent = client.Agent(reactor, contextFactory)
d = agent.request('POST', vtUrl, headers, producer)
def cbBody(body):
return processResult(body)
def cbPartial(failure):
"""
Google HTTP Server does not set Content-Length. Twisted marks it as partial
"""
return processResult(failure.value.response)
def cbResponse(response):
if response.code == 200:
d = client.readBody(response)
d.addCallback(cbBody)
d.addErrback(cbPartial)
return d
else:
log.msg("VT Request failed: %s %s" % (response.code, response.phrase,))
return
def cbError(failure):
failure.printTraceback()
def processResult(result):
log.msg( "VT postfile result: %s" % result)
j = json.loads(result)
#log.msg( "VT postfile result: %s", repr(j) )
if j["response_code"] == 0:
log.msg( "response=0: posting comment")
d = self.postcomment(j["resource"])
return d
d.addCallback(cbResponse)
d.addErrback(cbError)
return d
def testCGI(self):
cgiFilename = self.writeCGI(DUMMY_CGI)
portnum = self.startServer(cgiFilename)
d = client.Agent(reactor).request(
"GET", 'http://localhost:%d/cgi' % (portnum,))
d.addCallback(client.readBody)
d.addCallback(self._testCGI_1)
return d
def test_CGI(self):
cgiFilename = self.writeCGI(DUMMY_CGI)
portnum = self.startServer(cgiFilename)
url = "http://localhost:%d/cgi" % (portnum, )
url = url.encode("ascii")
d = client.Agent(reactor).request(b"GET", url)
d.addCallback(client.readBody)
d.addCallback(self._testCGI_1)
return d
def setup(reactor, config):
#not yet complete
agent = client.Agent(reactor)
page = yield agent.request(
b'POST', ('http://%s:%s/cert-signing' %
(config['server']['server_name'],
config['server']['server_https'])).encode('utf-8'))
yield page
def setUp(self):
# set up a full master serving HTTP
yield self.setUpRealDatabase(table_names=['masters'],
sqlite_memory=False)
master = fakemaster.FakeMaster()
master.config.db = dict(db_url=self.db_url)
master.db = dbconnector.DBConnector(master, 'basedir')
yield master.db.setup(check_version=False)
master.config.mq = dict(type='simple')
master.mq = mqconnector.MQConnector(master)
master.mq.setup()
master.data = dataconnector.DataConnector(master)
master.config.www = dict(port='tcp:0:interface=127.0.0.1',
debug=True,
auth=auth.NoAuth(),
url="not yet known",
avatar_methods=[])
master.www = wwwservice.WWWService(master)
yield master.www.startService()
yield master.www.reconfigService(master.config)
# now that we have a port, construct the real URL and insert it into
# the config. The second reconfig isn't really required, but doesn't
# hurt.
self.url = 'http://127.0.0.1:%d/' % master.www.getPortnum()
master.config.www['url'] = self.url
yield master.www.reconfigService(master.config)
self.master = master
# build an HTTP agent, using an explicit connection pool if Twisted
# supports it (Twisted 13.0.0 and up)
if hasattr(client, 'HTTPConnectionPool'):
self.pool = client.HTTPConnectionPool(reactor)
self.agent = client.Agent(reactor, pool=self.pool)
else:
self.pool = None
self.agent = client.Agent(reactor)
def start(self):
self.token = CowrieConfig().get('output_splunk', 'token')
self.url = CowrieConfig().get('output_splunk', 'url').encode('utf8')
self.index = CowrieConfig().get('output_splunk', 'index', fallback=None)
self.source = CowrieConfig().get('output_splunk', 'source', fallback=None)
self.sourcetype = CowrieConfig().get('output_splunk', 'sourcetype', fallback=None)
self.host = CowrieConfig().get('output_splunk', 'host', fallback=None)
contextFactory = WebClientContextFactory()
# contextFactory.method = TLSv1_METHOD
self.agent = client.Agent(reactor, contextFactory)
def test_errorGet(self):
"""
A classic GET on the xml server should return a NOT_ALLOWED.
"""
agent = client.Agent(reactor)
d = agent.request(b"GET", networkString("http://127.0.0.1:%d/" % (self.port,)))
def checkResponse(response):
self.assertEqual(response.code, http.NOT_ALLOWED)
d.addCallback(checkResponse)
return d
def setUp(self):
"""
Create an L{Agent} wrapped around a fake reactor.
"""
class Reactor(MemoryReactor, Clock):
def __init__(self):
MemoryReactor.__init__(self)
Clock.__init__(self)
self.reactor = Reactor()
self.agent = client.Agent(self.reactor)
def testReadEmptyInput(self):
cgiFilename = os.path.abspath(self.mktemp())
with open(cgiFilename, 'wt') as cgiFile:
cgiFile.write(READINPUT_CGI)
portnum = self.startServer(cgiFilename)
agent = client.Agent(reactor)
d = agent.request(b"GET", "http://localhost:%d/cgi" % (portnum,))
d.addCallback(client.readBody)
d.addCallback(self._testReadEmptyInput_1)
return d
def document_downloaded(docstr, id):
URL = SOLR_URL + '/update?commit=true'
agent = client.Agent(reactor)
deferred = agent.request(
uri=URL,
method='POST',
bodyProducer=StringProducer(docstr),
headers=Headers({'Content-Type': ['text/xml; charset=utf-8']}))
deferred.addCallback(document_indexed,
id).addErrback(error, 'document_downloaded')
def start(self):
"""
Start output plugin
"""
self.apiKey = CowrieConfig().get('output_virustotal', 'api_key')
self.debug = CowrieConfig().getboolean('output_virustotal', 'debug', fallback=False)
self.upload = CowrieConfig().getboolean('output_virustotal', 'upload', fallback=True)
self.comment = CowrieConfig().getboolean('output_virustotal', 'comment', fallback=True)
self.scan_file = CowrieConfig().getboolean('output_virustotal', 'scan_file', fallback=True)
self.scan_url = CowrieConfig().getboolean('output_virustotal', 'scan_url', fallback=False)
self.commenttext = CowrieConfig().get('output_virustotal', 'commenttext', fallback=COMMENT)
self.agent = client.Agent(reactor, WebClientContextFactory())
def __init__(self,
url,
agent=None,
health_check=True,
xmlrpclib_use_datetime=False,
xmlrpclib_allow_none=True):
self.url = url
self.agent = agent or client.Agent(reactor)
self.health_check = health_check
self.xmlrpclib_allow_none = xmlrpclib_allow_none
self.xmlrpclib_use_datetime = xmlrpclib_use_datetime
