def createQueue(port=8787,
name=None,
app=None,
consumers=[],
timerStep=1000,
withRss=False,
rssPort=6666):
assert name is not None
assert app is not None
assert len(consumers) > 0
queueModel = __getQueue(name)
scheduler = JobScheduler(queueModel)
for chost, cport in consumers:
scheduler.addConsumer(TwistedJobConsumer(chost, cport))
queue = internet.TCPServer(port, QueueFactory(scheduler))
queue.setServiceParent(app)
timer = internet.TimerService(timerStep, scheduler.dispatchPending)
timer.setServiceParent(app)
if (withRss):
site = server.Site(ErrorRssResource())
rss = strports.service('tcp:%s' % str(rssPort),
channel.HTTPFactory(site))
rss.setServiceParent(app)
def dance(options):
from angel_app.config import config
AngelConfig = config.getConfig()
providerport = AngelConfig.getint("provider", "listenPort")
repository = AngelConfig.get("common", "repository")
from angel_app.resource.local.external import resource
root = resource.External(repository)
from twisted.web2 import server
from twisted.web2 import channel
from twisted.internet import reactor
if AngelConfig.getboolean("provider", "useIPv6"):
from angel_app.ipv6 import reactor as ignored
site = server.Site(root)
getLogger().growl("User", "ETERNITY SERVICE",
"Started serving on port %s." % providerport)
reactor.listenTCP(providerport, channel.HTTPFactory(site), 50)
getLogger().info("Listening on port %d and serving content from %s" %
(providerport, repository))
# initial test version to integrate a dyndns client into the provider loop
dyndnsclient = getCallLaterDynDnsClientForAngelConfig(
AngelConfig,
callLaterMethod=reactor.callLater,
logger=getLogger('dyndns'))
if not dyndnsclient is None:
reactor.callLater(1, dyndnsclient.check)
reactor.run()
getLogger().info("Quit")
def makeService(config):
if config['logfile']:
logObserver = log.FileAccessLoggingObserver(config['logfile'])
else:
logObserver = log.DefaultCommonAccessLoggingObserver()
if config['root']:
if config['indexes']:
config['root'].indexNames = config['indexes']
root = log.LogWrapperResource(config['root'])
s = Web2Service(logObserver)
site = server.Site(root)
chan = channel.HTTPFactory(site)
if config['https']:
from twisted.internet.ssl import DefaultOpenSSLContextFactory
i = internet.SSLServer(
int(config['https']), chan,
DefaultOpenSSLContextFactory(config['privkey'],
config['certificate']))
i.setServiceParent(s)
strports.service(config['port'], chan).setServiceParent(s)
return s
def test_maxFields(self):
"""
Check that the C{maxSize} parameter makes the parsing raise an
exception if the data contains too many fields.
"""
ctype = http_headers.MimeType('multipart', 'form-data',
(('boundary', '---xyz'),))
content = """-----xyz\r
Content-Disposition: form-data; name="foo"\r
\r
Foo Bar\r
-----xyz\r
Content-Disposition: form-data; name="foo"\r
\r
Baz\r
-----xyz\r
Content-Disposition: form-data; name="file"; filename="filename"\r
Content-Type: text/html\r
\r
blah\r
-----xyz\r
Content-Disposition: form-data; name="file"; filename="filename"\r
Content-Type: text/plain\r
\r
bleh\r
-----xyz--\r
"""
root = resource.Resource()
request = SimpleRequest(server.Site(root), "GET", "/",
http_headers.Headers({'content-type': ctype}), content)
def cb(res):
self.assertEquals(res.response.description,
"Maximum number of fields 3 exceeded")
return self.assertFailure(server.parsePOSTData(request, maxFields=3),
http.HTTPError).addCallback(cb)
def getStatsFactory(self):
"""See L{apt_p2p.interfaces.IDHTStatsFactory}."""
assert _web2, "NOT IMPLEMENTED: twisted.web2 must be installed to use the stats factory."
if self.factory is None:
# Create a simple HTTP factory for stats
class StatsResource(resource.Resource):
def __init__(self, manager):
self.manager = manager
def render(self, ctx):
return http.Response(
200, {
'content-type': http_headers.MimeType(
'text', 'html')
}, '<html><body>\n\n' + self.manager.getStats() +
'\n</body></html>\n')
def locateChild(self, request, segments):
log.msg('Got HTTP stats request from %s' %
(request.remoteAddr, ))
return self, ()
self.factory = channel.HTTPFactory(server.Site(
StatsResource(self)))
return self.factory
def test_multipart(self):
"""
Test parsing data in multipart format: it should fill the C{files}
attribute.
"""
ctype = http_headers.MimeType('multipart', 'form-data',
(('boundary', '---weeboundary'),))
content="""-----weeboundary\r
Content-Disposition: form-data; name="FileNameOne"; filename="myfilename"\r
Content-Type: text/html\r
\r
my great content wooo\r
-----weeboundary--\r
"""
root = resource.Resource()
request = SimpleRequest(server.Site(root), "GET", "/",
http_headers.Headers({'content-type': ctype}), content)
def cb(ign):
self.assertEquals(request.args, {})
self.assertEquals(request.files.keys(), ['FileNameOne'])
self.assertEquals(request.files.values()[0][0][:2],
('myfilename', http_headers.MimeType('text', 'html', {})))
f = request.files.values()[0][0][2]
self.assertEquals(f.read(), "my great content wooo")
return server.parsePOSTData(request).addCallback(cb)
def main():
from twisted.web2 import server
from twisted.web2 import channel
from twisted.internet import reactor
site = server.Site(Toplevel())
reactor.listenTCP(TRACKER_PORT, channel.HTTPFactory(site), 50)
reactor.run()
def test_unknownResource(self):
"""
Test urlForResource() on unknown resource.
"""
root = resource.Resource()
child = resource.Resource()
request = SimpleRequest(server.Site(root), "GET", "/")
self.assertRaises(server.NoURLForResourceError, request.urlForResource, child)
def twisted_adapter(host, port):
# Experimental (Untested).
from twisted.application import service, strports
from twisted.web2 import server, channel, wsgi
ittyResource = wsgi.WSGIResource(handle_request)
site = server.Site(ittyResource)
application = service.Application('web')
s = strports.service('tcp:%s' % post, channel.HTTPFactory(site))
s.setServiceParent(application)
def test_noContentType(self):
"""
Parsing a request without content-type should succeed but should not
fill the C{args} and C{files} attributes of the request.
"""
root = resource.Resource()
request = SimpleRequest(server.Site(root), "GET", "/", content="foo")
def cb(ign):
self.assertEquals(request.args, {})
self.assertEquals(request.files, {})
return server.parsePOSTData(request).addCallback(cb)
def test_wrongContentType(self):
"""
Check that a content-type not handled raise a C{http.HTTPError}.
"""
ctype = http_headers.MimeType('application', 'foobar')
content = "key=value&multiple=two+words&multiple=more%20words"
root = resource.Resource()
request = SimpleRequest(server.Site(root), "GET", "/",
http_headers.Headers({'content-type': ctype}), content)
return self.assertFailure(server.parsePOSTData(request),
http.HTTPError)
def startService(self):
self.script_runner_thing = script_runner.ScriptRunner(
debug_level=self.debug_level, mbus=self.messageBus)
if config_data.getByPath('script_runner',
'web_interface_active') == True:
self.webIf = ScriptRunnerWebInterface(
debug_level=self.debug_level,
script_runner=self.script_runner_thing)
root = resource.Resource()
root.putChild('newjob', self.webIf)
site = server.Site(root)
reactor.listenTCP(self.port, channel.HTTPFactory(site))
def getHTTPFactory(self):
"""Initialize and get the factory for this HTTP server."""
if self.factory is None:
self.factory = channel.HTTPFactory(
server.Site(self), **{
'maxPipeline': 10,
'betweenRequestsTimeOut': 60
})
self.factory = ThrottlingFactory(self.factory,
writeLimit=self.uploadLimit)
self.factory.protocol = UploadThrottlingProtocol
if self.manager:
self.factory.protocol.stats = self.manager.stats
return self.factory
def test_scriptsExecute(self):
"""
Verify that CGI scripts within a CGIDirectory can actually be executed
"""
cgiBinDir = os.path.abspath(self.mktemp())
os.mkdir(cgiBinDir)
root = twcgi.CGIDirectory(cgiBinDir)
self.createScript(os.path.join(cgiBinDir, 'dummy'))
cgiSubDir = os.path.join(cgiBinDir, 'sub')
os.mkdir(cgiSubDir)
self.createScript(os.path.join(cgiSubDir, 'dummy'))
site = server.Site(root)
request = SimpleRequest(site, "GET", "/dummy")
d = request.locateResource('/dummy')
def _firstResponse(res):
self.failUnlessEqual(res, "cgi output%s" % os.linesep)
def _firstRequest(resource):
d1 = self.getPage(request, resource)
d1.addCallback(_firstResponse)
return d1
d.addCallback(_firstRequest)
def _secondResponse(res):
self.failUnlessEqual(res, "cgi output%s" % os.linesep)
def _secondRequest(ign):
request = SimpleRequest(site, "GET", '/sub/dummy')
d2 = request.locateResource('/sub/dummy')
d2.addCallback(lambda resource: self.getPage(request, resource))
d2.addCallback(_secondResponse)
return d2
d.addCallback(_secondRequest)
return d
def test_urlencoded(self):
"""
Test parsing data in urlencoded format: it should end in the C{args}
attribute.
"""
ctype = http_headers.MimeType('application', 'x-www-form-urlencoded')
content = "key=value&multiple=two+words&multiple=more%20words"
root = resource.Resource()
request = SimpleRequest(server.Site(root), "GET", "/",
http_headers.Headers({'content-type': ctype}), content)
def cb(ign):
self.assertEquals(request.files, {})
self.assertEquals(request.args,
{'multiple': ['two words', 'more words'], 'key': ['value']})
return server.parsePOSTData(request).addCallback(cb)
def test_locateResource(self):
"""
Test urlForResource() on resource looked up via a locateResource() call.
"""
root = resource.Resource()
child = resource.Resource()
root.putChild("foo", child)
request = SimpleRequest(server.Site(root), "GET", "/")
def gotResource(resource):
self.assertEquals("/foo", request.urlForResource(resource))
d = defer.maybeDeferred(request.locateResource, "/foo")
d.addCallback(gotResource)
return d
def test_otherErrors(self):
"""
Test that errors durign parsing other than C{MimeFormatError} are
propagated.
"""
ctype = http_headers.MimeType('multipart', 'form-data',
(('boundary', '---weeboundary'),))
# XXX: maybe this is not a good example
# parseContentDispositionFormData could handle this problem
content="""-----weeboundary\r
Content-Disposition: form-data; name="FileNameOne"; filename="myfilename and invalid data \r
-----weeboundary--\r
"""
root = resource.Resource()
request = SimpleRequest(server.Site(root), "GET", "/",
http_headers.Headers({'content-type': ctype}), content)
return self.assertFailure(server.parsePOSTData(request),
ValueError)
def test_multipartWithNoBoundary(self):
"""
If the boundary type is not specified, parsing should fail with a
C{http.HTTPError}.
"""
ctype = http_headers.MimeType('multipart', 'form-data')
content="""-----weeboundary\r
Content-Disposition: form-data; name="FileNameOne"; filename="myfilename"\r
Content-Type: text/html\r
\r
my great content wooo\r
-----weeboundary--\r
"""
root = resource.Resource()
request = SimpleRequest(server.Site(root), "GET", "/",
http_headers.Headers({'content-type': ctype}), content)
return self.assertFailure(server.parsePOSTData(request),
http.HTTPError)
def main(argv):
options = parse_options(argv[1:])
if options.log:
log.startLogging(sys.stdout)
# This is supposed to match the SF site so it's easy to run a functional
# test over the internet and against Apache.
# TODO: Remove bizarre structure and strings expected by functional tests.
root = Page()
root.text = ROOT_HTML
mechanize = make_page(root, "mechanize", MECHANIZE_HTML)
make_leaf_page(root, "robots.txt", "User-Agent: *\nDisallow: /norobots",
"text/plain")
make_leaf_page(root, "robots", "Hello, robots.", "text/plain")
make_leaf_page(root, "norobots", "Hello, non-robots.", "text/plain")
test_fixtures = make_page(
root,
"test_fixtures",
# satisfy stupid assertions in functional tests
html("Python bits", extra_content="GeneralFAQ.html"))
make_leaf_page(test_fixtures, "cctest2.txt",
"Hello ClientCookie functional test suite.", "text/plain")
make_leaf_page(test_fixtures, "referertest.html", REFERER_TEST_HTML)
make_leaf_page(test_fixtures, "mechanize_reload_test.html",
RELOAD_TEST_HTML)
make_redirect(root, "redirected", "/doesnotexist")
cgi_bin = make_dir(root, "cgi-bin")
project_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
make_cgi_script(cgi_bin, "cookietest.cgi",
os.path.join(project_dir, "test-tools", "cookietest.cgi"))
example_html = open(os.path.join("examples", "forms",
"example.html")).read()
make_leaf_page(mechanize, "example.html", example_html)
make_cgi_script(cgi_bin, "echo.cgi",
os.path.join(project_dir, "examples", "forms", "echo.cgi"))
make_page(root, "basic_auth", BASIC_AUTH_PAGE, wrapper=require_basic_auth)
make_page(root,
"digest_auth",
DIGEST_AUTH_PAGE,
wrapper=require_digest_auth)
site = server.Site(root)
reactor.listenTCP(options.port, channel.HTTPFactory(site))
reactor.run()
def main():
root = Page()
root.text = ROOT_HTML
make_page(root, "mechanize", MECHANIZE_HTML)
make_leaf_page(root, "robots.txt", "User-Agent: *\nDisallow: /norobots",
"text/plain")
make_leaf_page(root, "robots", "Hello, robots.", "text/plain")
make_leaf_page(root, "norobots", "Hello, non-robots.", "text/plain")
bits = make_page(root, "bits", "GeneralFAQ.html")
make_leaf_page(bits, "cctest2.txt",
"Hello ClientCookie functional test suite.", "text/plain")
make_leaf_page(bits, "referertest.html", REFERER_TEST_HTML)
make_leaf_page(bits, "mechanize_reload_test.html", RELOAD_TEST_HTML)
make_redirect(root, "redirected", "/doesnotexist")
make_cgi_bin(root, "cgi-bin", "test-tools")
site = server.Site(root)
reactor.listenTCP(int(sys.argv[1]), channel.HTTPFactory(site))
reactor.run()
def test_mimeParsingError(self):
"""
A malformed content should result in a C{http.HTTPError}.
The tested content has an invalid closing boundary.
"""
ctype = http_headers.MimeType('multipart', 'form-data',
(('boundary', '---weeboundary'),))
content="""-----weeboundary\r
Content-Disposition: form-data; name="FileNameOne"; filename="myfilename"\r
Content-Type: text/html\r
\r
my great content wooo\r
-----weeoundary--\r
"""
root = resource.Resource()
request = SimpleRequest(server.Site(root), "GET", "/",
http_headers.Headers({'content-type': ctype}), content)
return self.assertFailure(server.parsePOSTData(request),
http.HTTPError)
def test_deferredLocateChild(self):
"""
Test deferred value from locateChild()
"""
class DeferredLocateChild(resource.Resource):
def locateChild(self, req, segments):
return defer.maybeDeferred(
super(DeferredLocateChild, self).locateChild, req,
segments)
root = DeferredLocateChild()
child = resource.Resource()
root.putChild("foo", child)
request = SimpleRequest(server.Site(root), "GET", "/foo")
def gotResource(resource):
self.assertEquals("/foo", request.urlForResource(resource))
d = request.locateResource("/foo")
d.addCallback(gotResource)
return d
def test_locateChildResource(self):
"""
Test urlForResource() on deeply nested resource looked up via
locateChildResource().
"""
root = EmptyResource(self)
root.expectedURI = "/"
foo = EmptyResource(self)
foo.expectedURI = "/foo"
root.putChild("foo", foo)
bar = EmptyResource(self)
bar.expectedURI = "/foo/bar"
foo.putChild("bar", bar)
baz = EmptyResource(self)
baz.expectedURI = "/foo/bar/b%20a%20z"
bar.putChild("b a z", baz)
request = SimpleRequest(server.Site(root), "GET", "/")
def gotResource(resource):
# Make sure locateChildResource() gave us the right answer
self.assertEquals(resource, bar)
return request.locateChildResource(
resource, "b a z").addCallback(gotChildResource)
def gotChildResource(resource):
# Make sure locateChildResource() gave us the right answer
self.assertEquals(resource, baz)
self.assertEquals(resource.expectedURI,
request.urlForResource(resource))
d = request.locateResource(bar.expectedURI)
d.addCallback(gotResource)
return d
def dance(options):
from angel_app.log import getLogger
from angel_app.config import config
AngelConfig = config.getConfig()
port = AngelConfig.getint("presenter", "listenPort")
interface = AngelConfig.get("presenter", "listenInterface")
repository = AngelConfig.get("common", "repository")
from angel_app.resource.local.internal.resource import Crypto
Crypto.rootDirectory = repository
root = Crypto(repository)
from twisted.web2 import server
from twisted.web2 import channel
from twisted.internet import reactor
site = server.Site(root)
reactor.listenTCP(port, channel.HTTPFactory(site), 50, interface)
getLogger().info('Listening on IP %s port %d and serving content from %s',
interface, port, repository)
getLogger().growl("User", "ETERNITY FILE SYSTEM",
"Available on port %s." % port)
reactor.run()
getLogger().info("Quit")
def test_multipartMaxSize(self):
"""
Check that the C{maxSize} parameter makes the parsing raise an
exception if the data is too big.
"""
ctype = http_headers.MimeType('multipart', 'form-data',
(('boundary', '---weeboundary'),))
content="""-----weeboundary\r
Content-Disposition: form-data; name="FileNameOne"; filename="myfilename"\r
Content-Type: text/html\r
\r
my great content wooo
and even more and more\r
-----weeboundary--\r
"""
root = resource.Resource()
request = SimpleRequest(server.Site(root), "GET", "/",
http_headers.Headers({'content-type': ctype}), content)
def cb(res):
self.assertEquals(res.response.description,
"Maximum length of 10 bytes exceeded.")
return self.assertFailure(server.parsePOSTData(request, maxSize=10),
http.HTTPError).addCallback(cb)
def run(engine):
"Gateway´s Runloop"
config = engine.config
#os.environ['DJANGO_SETTINGS_MODULE'] = 'pyisis.web.settings'
# needed to use django templates
if config.HTTP_PORT:
from twisted.web2 import server, vhost, channel
from twisted.web2 import wsgi, log
from django.core.handlers.wsgi import WSGIHandler
from django.core.servers.basehttp import AdminMediaHandler
from django.conf import settings
settings.configure(
DEBUG=True,
TEMPLATE_DEBUG=config.HTML_DEBUG,
DEFAULT_CHARSET=config.OUTPUT_ENCODING,
DEFAULT_CONTENT_TYPE='text/html',
ROOT_URLCONF='pyisis.web.urls',
INSTALLED_APPS=(
#'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.sites',
#'django.contrib.admin',
'pyisis.web.isis'),
MIDDLEWARE_CLASSES=(
'django.middleware.common.CommonMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.middleware.doc.XViewMiddleware',
),
TEMPLATE_LOADERS=(
'django.template.loaders.filesystem.load_template_source',
'django.template.loaders.app_directories.load_template_source',
),
SECRET_KEY='b(+ukc38349u0reu_8j)@iwpm017e(c#=0nmdn%s2u=$+*t@vo',
MEDIA_URL='',
MEDIA_ROOT=config.MEDIA_ROOT,
USE_I18N=False,
LANGUAGE_CODE='en-us',
SITE_ID=1,
TIME_ZONE='America/Sao_Paulo',
TEMPLATE_CONTEXT_PROCESSORS=(
"django.core.context_processors.debug",
#"django.core.context_processors.i18n",
"django.core.context_processors.media",
),
TEMPLATE_DIRS=config.HTML_TEMPLATE_DIRS)
# Glue code to plug Django into Twisted
wsgires = wsgi.WSGIResource(AdminMediaHandler(WSGIHandler()))
res = log.LogWrapperResource(wsgires)
log.FileAccessLoggingObserver(config.WEB_LOG).start()
site = server.Site(res)
reactor.listenTCP(config.HTTP_PORT, channel.HTTPFactory(site))
if config.SSH_PORT:
from twisted.cred import portal, checkers
from twisted.conch import manhole, manhole_ssh
def getManholeFactory(namespace):
realm = manhole_ssh.TerminalRealm()
def getManhole(_):
return manhole.Manhole(namespace)
realm.chainedProtocolFactory.protocolFactory = getManhole
p = portal.Portal(realm)
checker = checkers.InMemoryUsernamePasswordDatabaseDontUse()
checker.addUser(config.USERNAME, config.PASSWORD)
p.registerChecker(checker)
f = manhole_ssh.ConchFactory(p)
return f
reactor.listenTCP(config.SSH_PORT,
getManholeFactory(engine.collection))
def showBanner(config):
print "PyISIS Cell", __version__
print "Python", sys.version
if config.HTTP_PORT:
print _("Gateway ready to handle HTTP requests at port %s"
) % config.HTTP_PORT
if config.SSH_PORT:
print _("Gateway ready to handle SSH requests at port %s"
) % config.SSH_PORT
reactor.callWhenRunning(showBanner, config)
reactor.run()
def render(self, ctx):
return http.Response(
responsecode.OK, {
'last-modified': self.creation_time,
'etag': http_headers.ETag(str(hash(self.text))),
'content-type': self.content_type
}, self.text)
class Toplevel(resource.Resource):
addSlash = True
child_monkey = static.File(os.path.dirname(static.__file__) + '/static.py')
child_elephant = Child()
def render(self, ctx):
return http.Response(
200, {'content-type': http_headers.MimeType('text', 'html')},
"""<html><body>
<a href="monkey">The source code of twisted.web2.static</a><br>
<a href="elephant">A defined child</a></body></html>""")
site = server.Site(Toplevel())
# Standard twisted application Boilerplate
from twisted.application import service, strports
application = service.Application("demoserver")
s = strports.service('tcp:8080', channel.HTTPFactory(site))
s.setServiceParent(application)
from twisted.python import log
from jsonrpc import jsonrpc
import roll
class JsonService(jsonrpc):
def jsonrpc_mean(self, request, ticker, window=20):
'''
Call roll.mean function
'''
return roll.mean.get(ticker, window=window)
if __name__ == '__main__':
'''
Start the server
'''
from twisted.internet import reactor
site = server.Site(JsonService())
port = 8080
if len(sys.argv) > 1:
try:
port = int(sys.argv[1])
except:
pass
reactor.listenTCP(port, channel.HTTPFactory(site))
log.startLogging(sys.stdout)
log.msg("LondonR Nov-09 simple JSON RPC server running on localhost:%s" %
port)
reactor.run()
def run():
site = server.Site(Root())
factory = channel.HTTPFactory(site)
reactor.listenTCP(8000, factory)
print "\nOpen your browser to 'http://localhost:8000'\n"
reactor.run()
def chanrequest(self, root, uri, length, headers, method, version, prepath, content):
site = server.Site(root)
return TestChanRequest(site, method, prepath, uri, length, headers, version, content)
