lecture = Lecture(
"Perspective Broker: Translucent RPC in Twisted",
# intro
Raw(
"Title",
"""
<h1>Perspective Broker: Translucent RPC in Twisted</h1>
<h2>PyCon 2003</h2>
<h2>Brian Warner < warner @ lothar . com > </h2>
""",
),
Slide(
"Introduction",
Bullet("Overview/definition of RPC"),
Bullet("What is Perspective Broker?"),
Bullet("How do I use it?"),
Bullet("Security Issues"),
Bullet("Future Directions"),
),
Slide(
"Remote Procedure Calls",
Bullet(
"Action at a distance: separate processes, safely telling each other what to do",
SubBullet("Separate memory spaces"),
SubBullet("Usually on different machines"),
),
Bullet("Frequently called RMI these days: Remote Method Invocation"),
Bullet("Three basic parts: Addressing, Serialization, Waiting"),
),
Slide(
"Addressing",
Bullet(
"What program are you talking to?",
SubBullet("hostname, port number"),
SubBullet("Some systems use other namespaces: sunrpc"),
),
Bullet("Which object in that program?"),
Bullet("Which method do you want to run?"),
Bullet(
"Related issues",
SubBullet("How do you know what the arguments are?"),
SubBullet("(do you care?)"),
SubBullet("How do you know what methods are available?"),
SubBullet("(do you care?)"),
),
),
Slide(
"Serialization",
Bullet("What happens to the arguments you send in?"),
Bullet(
"What happens to the results that are returned?",
SubBullet("Representation differences: endianness, word length"),
SubBullet("Dealing with user-defined types"),
),
Bullet("How to deal with references"),
),
Slide(
"The Waiting (is the hardest part)",
Bullet("Asynchronous: results come later, or not at all"),
Bullet("Need to do other work while waiting"),
),
Slide(
"Whither Translucence?",
Bullet(
"Not 'Transparent': don't pretend remote objects are really local",
SubBullet(
"CORBA (in C) does this, makes remote calls look like local calls"
),
SubBullet("makes it hard to deal with the async nature of RPC"),
),
Bullet(
"Not 'Opaque': make it easy to deal with the differences",
SubBullet("Including extra failure modes, delayed results"),
),
Bullet("Exceptions and Deferreds to the rescue"),
),
Slide(
"Other RPC protocols",
Bullet("HTML"),
Bullet("XML-RPC"),
Bullet("CORBA"),
Bullet("when you control both ends of the wire, use PB"),
),
Raw(
"Where does PB fit?",
"""<h2>PB sits on top of <span class=\"py-src-identifier\">twisted.internet</span></h2>
<img src=\"twisted-overview.png\" />
""",
),
Slide(
"pb.RemoteReference",
Bullet(
HTML(
'<span class="py-src-identifier">pb.Referenceable</span>: Object which can be accessed by remote systems.'
),
SubBullet(
HTML(
'Defines methods like <span class="py-src-identifier">remote_foo</span> and <span class="py-src-identifier">remote_bar</span> which can be invoked remotely.'
)
),
SubBullet(
HTML(
'Methods without the <span class="py-src-identifier">remote_</span> prefix are local-only.'
)
),
),
Bullet(
HTML(
'<span class="py-src-identifier">pb.RemoteReference</span>: Used by distant program to invoke methods.'
),
SubBullet(
HTML(
'Offers <span class="py-src-identifier">.callRemote()</span> to trigger remote method on a corresponding <span class="py-src-identifier">pb.Referenceable</span>.'
)
),
),
),
Raw("Sample code", "<h2>Sample Code</h2>" + server_lore + client_lore),
# Slide("Simple Demo"),
# "better demo: manhole, or reactor running in another thread"
# build up from callRemote?
Slide(
"What happens to those arguments?",
Bullet(
"Basic structures should travel transparently",
SubBullet("Actually quite difficult in some languages"),
),
Bullet(
"Object graph should remain the same",
SubBullet("Serialization context"),
SubBullet("(same issues as Pickle)"),
),
Bullet(
"Instances of user-defined classes require more care",
SubBullet("User-controlled unjellying"),
),
),
# serialization (skip banana)
Slide(
"40% More Sandwich Puns Than The Leading Brand",
Bullet("twisted.spread: python package holding other modules"),
Bullet("PB: remote method invocation"),
Bullet("Jelly: mid-level object serialization"),
Bullet("Banana: low-level serialization of s-expressions"),
Bullet("Taster: security context, decides what may be received"),
Bullet("Marmalade: like Jelly, but involves XML, so it's bitter"),
Bullet(
"better than the competition",
SubBullet("CORBA: few or no sandwich puns"),
SubBullet("XML-RPC: barely pronounceable"),
),
),
Slide(
"Jellying objects",
Bullet("'Jellying' vs 'Unjellying'"),
Bullet("Immutable objects are copied whole"),
Bullet(
"Mutable objects get reference IDs to insure shared references remain shared",
SubBullet("(within the same Jellying context)"),
),
),
Slide(
"Jellying instances",
Bullet(
HTML(
'User classes inherit from one of the <span class="py-src-identifier">pb.flavor</span> classes'
)
),
Bullet(
HTML(
'<span class="py-src-identifier">pb.Referenceable</span>: methods can be called remotely'
)
),
Bullet(
HTML(
'<span class="py-src-identifier">pb.Copyable</span>: contents are selectively copied'
)
),
Bullet(
HTML(
'<span class="py-src-identifier">pb.Cacheable</span>: contents are copied and kept up to date'
)
),
Bullet(
HTML(
'Classes define <span class="py-src-identifier">.getStateToCopy</span> and other methods to restrict exported state'
)
),
),
Slide(
"pb.Copyable example",
PRE(
"""class SenderPond(FrogPond, pb.Copyable):
def getStateToCopy(self):
d = self.__dict__.copy()
d['frogsAndToads'] = d['numFrogs'] + d['numToads']
del d['numFrogs']
del d['numToads']
return d
class ReceiverPond(pb.RemoteCopy):
def setCopyableState(self, state):
self.__dict__ = state
self.localCount = 12
def count(self):
return self.frogsAndToads
pb.setUnjellyableForClass(SenderPond, ReceiverPond)
"""
),
),
Slide(
"Secure Unjellying",
Bullet(
"Pickle has security problems",
SubBullet("Pickle will import any module the sender requests."),
SubBullet(
HTML(
'2.3 gave up, removed safety checks like <span class="py-src-identifier">__safe_for_unpickling__</span> .'
)
),
),
Bullet(
"Jelly attempts to be safe in the face of hostile clients",
SubBullet("All classes rejected by default"),
SubBullet(
HTML(
'<span class="py-src-identifier">registerUnjellyable()</span> used to accept safe ones'
)
),
SubBullet(
HTML(
'Registered classes define <span class="py-src-identifier">.setCopyableState</span> and others to process remote state'
)
),
),
Bullet("Must mark (by subclassing) to transmit"),
),
Slide(
"Transformation of references in transit",
Bullet(
"All referenced objects get turned into their counterparts as they go over the wire"
),
Bullet(
"References are followed recursively",
SubBullet(
"Sending a reference to a tree of objects will cause the whole thing to be transferred"
),
SubBullet("(subject to security restrictions)"),
),
Bullet(
HTML('<span class="py-src-identifier">pb.flavors</span> get reference ids'),
SubBullet(
"They are recognized when they return, transformed into the original reference"
),
SubBullet("Reference ids are scoped to the connection"),
SubBullet("One side-effect: no 'third party' references"),
),
),
Slide(
"Perspectives: pb.cred and the Identity/Service model",
Bullet(
"A layer to provide common authentication services to Twisted applications"
),
Bullet(
HTML(
'<span class="py-src-identifier">Identity</span>: named user accounts with passwords'
)
),
Bullet(
HTML(
'<span class="py-src-identifier">Service</span>: something a user can request access to'
)
),
Bullet(
HTML(
'<span class="py-src-identifier">Perspective</span>: user accessing a service'
)
),
Bullet(
HTML(
'<span class="py-src-identifier">pb.Perspective</span>: first object, a <span class="py-src-identifier">pb.Referenceable</span> used to access everything else'
)
),
),
# picture would help
Slide(
"Future directions",
Bullet(
"Other language bindings: Java, elisp, Haskell, Scheme, JavaScript, OCaml"
),
# donovan is doing the JavaScript port
Bullet("Other transports: UDP, Airhook"),
Bullet("Componentization"),
Bullet("Performance improvements: C extension for Jelly"),
Bullet("Refactor addressing model: PB URLs"),
),
Slide("Questions", Bullet("???")),
)
lecture = Lecture(
"Dealing with stale code",
Slide("The Problem",
Bullet("Developers update code"),
Bullet("In a long-running process, old modules stay in memory after "
"their source file has been updated"),
Bullet("Code has various dependencies -- modules import each other"),
Bullet("Stale code causes crazy behavior (who among us has not "
"pulled out hair as their code has resisted change?)")),
Slide("Clever hacks don't work",
Bullet("If you reload modules when changes are detected, old references persist"),
Bullet("Especially references to old classes"),
Bullet("Which might mean just some instances have stale class references (even worse!)"),
Bullet("You can't just check modules when they are used through certain mechanisms. ",
"Because those modules depend on other modules, and so on..."),
Bullet("And sometimes there's non-Python files out there as well (e.g., configuration files)"),
Bullet("You can fix these, but there's oodles of corner cases"),
Bullet("Worse, every corner case causes hair-pulling bugs"),
),
Slide("Webware's shotgun approach",
Bullet("Implemented by Jason Hildebrand (", URL("http://peaceworks.ca"), ")"),
Bullet("Put in a hook to keep track of files being used.",
SubBullet(Bullet("you can also manually add files"))),
Bullet("Poll all files regularly (like every second)",
SubBullet(Bullet("Or uses libfam to receive change events instead of polling"))),
Bullet("When any file has changed, restart the system"),
Bullet("Look in ",
TT("Webware.WebKit.ImportSpy")),
Bullet("Doesn't have any Webware dependencies"),
),
Slide("Webware's shotgun restart",
Bullet("The application server is started with a shell script:",
PRE("""\
retcode=3
while test $retcode -eq 3; do
/usr/bin/env python $OPTIONS Launch.py ThreadedAppServer $*
retcode=$?
done
""")),
Bullet("So we exit with error code 3 when we want to restart"),
Bullet("Restarts are fast, so by the time you've saved your file and returned to the "
"browser, you're all ready"),
Bullet("No corner cases, no cleverness, It Just Works"),
),
)
lecture = Lecture(
"The twisted.internet Tutorial of Doom",
Slide("Part 1 - Introduction"),
# there are different ways to do networking
# mention processes are not cross-platform
Slide(
"Choosing a networking paradigm for the enterprise",
Bullet("Event driven"),
Bullet(Bad("Threads")),
Bullet("Others which we will ignore (processes, SEDA, ...)"),
),
# it's a metaphor!
Slide(
"Applied Bistromathics 101",
Bullet("Consider a restaurant as a network application"),
Bullet("Clients come in, make requests to the waiters"),
Bullet("Waiters act on clients' choices"),
),
# an event loop is efficient, doesn't waste time
# event loop is also used for GUIs
Slide(
"The event driven waiter",
Bullet("One waiter, serving all tables"),
Bullet("Waiter takes orders from tables to kitchen"),
Bullet("Waiter takes food from kitchen to tables"),
),
# not accurate, but the problems are real. avoid threads if you can
Slide(
"Threads (a caricature)",
Bullet(Bad("One waiter per table")),
SubBullet(
"Problems:",
Bullet(Bad("Expensive")),
Bullet(Bad("Waiters need to be careful not bump into each other")),
),
),
# why threads are sometimes necessary
Slide(
"When do we want threads?",
Bullet("Long running, blocking operations"),
Bullet("Classic example: database access"),
),
# today we will discuss only (parts of) twisted.internet
Slide("Twisted: The Framework of Your Internet", Image("twisted-overview.png")),
Slide(
"Project Stats",
Bullet("URL: ", URL("http://www.twistedmatrix.com")),
Bullet("License: LGPL"),
Bullet("Number of developers: approximately 20"),
Bullet("Version: 1.0.3"),
Bullet("Platforms: Unix, Win32"),
Bullet("Started in January 2000 by Glyph Lefkowitz"),
),
Slide("Part 2 - Basic Networking With Twisted"),
# quick review of how the internet works
Slide(
"Internet!",
Bullet("Network of interconnected machines"),
Bullet("Each machine has one (or more) IP addresses"),
Bullet("DNS maps names ('www.yahoo.com') to IPs (216.109.125.69)"),
Bullet(
"TCP runs on top of IP, servers listen on of of 65536 ports,"
" e.g. HTTP on port 80"
),
),
# we need to understand certain basic terms before we continue.
# the event loop is the last thing we run - it waits until
# an event occurs, then calls the appropriate handler.
Slide(
"Basic Definitions - Reactor",
Bullet("An object implementing the event loop", PRE(EVENT_LOOP_CODE)),
),
Slide(
"Basic Definitions - Transport",
Bullet("Moves data from one location to another"),
Bullet("Main focus of talk are ordered, reliable byte stream transports"),
Bullet("Examples: TCP, SSL, Unix sockets"),
Bullet("UDP is a different kind of transport"),
),
# the client is the side which initiated the connection
# HTTP and SSH run on TCP-like transports, DNS runs on UDP or TCP
Slide(
"Basic Definitions - Protocol",
Bullet("Defines the rules for communication between two hosts"),
Bullet("Protocols communicate using a transport"),
Bullet("Typically there is a client, and server"),
Bullet("Examples: HTTP, SSH, DNS"),
),
Slide(
"All Together Now",
Bullet(
"The reactor gets events from the transports (read from network, write to network)"
),
Bullet(
"The reactor passes events to protocol (connection lost, data received)"
),
Bullet(
"The protocol tells the transport to do stuff (write data, lose connection)"
),
),
# designing a new protocol is usually a bad idea, there are lots of
# things you can get wrong, both in design and in implementation
Slide("How To Implement A Protocol", Bullet("Hopefully, you don't.")),
# XXX split into three expanded slides?
NumSlide(
"How To Not Implement A Protocol",
Bullet("Use an existing Twisted implementation of the protocol"),
Bullet("Use XML-RPC"),
Bullet("Use Perspective Broker, a remote object protocol"),
),
# connectionMade is called when connection is made
# dataReceived is called every time we receive data from the network
# connectionLost is called when the connection is lost
Slide("How To Really Implement A Protocol", PRE(PROTOCOL_CODE)),
# factories - why?
Slide(
"Factories",
Bullet("A protocol instance only exists as long as the connection is there"),
Bullet("Protocols want to share state"),
Bullet("Solution: a factory object that creates protocol instances"),
),
# factory code - notice how protocol instances have access to the factory
# instance, for shared state. buildProtocol can return None if we don't
# want to accept connections from that address.
Slide("A Server Factory", PRE(SERVER_CODE)),
# running the server we just wrote
Slide("Connecting A Factory To A TCP Port", PRE(RUNNING_SERVER_CODE)),
# transport independence - using listenUNIX as example
Slide(
"Transport Independence",
Bullet("Notice how none of the protocol code was TCP specific"),
Bullet("We can reuse same protocol with different transports"),
Bullet("We could use listenUNIX for unix sockets with same code"),
Bullet("Likewise listenSSL for SSL or TLS"),
),
Slide("Client Side Protocol", PRE(CLIENT_PROTOCOL_CODE)),
# client connections are different
Slide(
"Client Side Factories",
Bullet("Different requirements than server"),
Bullet("Failure to connect"),
Bullet("Automatic reconnecting"),
Bullet("Cancelling and timing out connections"),
),
# example client factory - explain use of default buildProtocol
Slide("Client Side Factories 2", PRE(CLIENT_FACTORY_CODE)),
# connectTCP
Slide("Connection API", PRE(CLIENT_CONNECT_CODE)),
# explain how transports buffer the output
Slide(
"Buffering",
Bullet("When we write to transport, data is buffered"),
Bullet(
"loseConnection will wait until all buffered data is sent, and producer (if any) is finished"
),
),
# start/stopFactory
Slide(
"Factory Resources",
Bullet("Factories may want to create/clean up resources"),
Bullet("startFactory() - called on start of listening/connect"),
Bullet("stopFactory() - called on end of listening/connect"),
Bullet("Called once even if factory listening/connecting multiple ports"),
),
# example of restartable factory
Slide("Factory Resources 2", PRE(FACTORY_START_CODE)),
Slide(
"Producers and Consumers",
Bullet("What if we want to send out lots of data?"),
Bullet("Can't write it out all at once"),
Bullet("We don't want to write too fast"),
),
Slide(
"Producers",
Bullet(
"Produce data for a consumer, in this case by calling transport's write()"
),
Bullet(
"Pausable (should implement pauseProducing and resumeProducing methods)"
),
Bullet("Push - keeps producing unless told to pause"),
Bullet("Pull - produces only when consumer tells it to"),
),
Slide(
"Consumers",
Bullet("registerProducer(producer, streaming)"),
Bullet("Will notify producer to pause if buffers are full"),
),
Slide("Sample Pull Producer", PRE(PULL_PRODUCER_CODE)),
Slide("Sample Push Producer", PRE(PUSH_PRODUCER_CODE)),
# scheduling events
Slide("Scheduling", PRE(SCHEDULING_CODE)),
# pluggable reactors - why?
Slide(
"Choosing a Reactor - Why?",
Bullet("GUI toolkits have their own event loop"),
Bullet("Platform specific event loops"),
),
Slide(
"Choosing a Reactor",
Bullet("Twisted supports multiple reactors"),
Bullet("Default, gtk, gtk2, qt, win32 and others"),
Bullet("Tk and wxPython as non-reactors"),
Bullet("Reactor installation should be first thing code does"),
),
# example GUI client
Slide("Example GTK Program", PRE(GUI_CODE)),
# you can learn more about
Slide(
"Learning more about networking and scheduling",
Bullet("twisted.internet.interfaces"),
Bullet("http://twistedmatrix.com/document/howtos/"),
),
Slide("Part 3 - Building Applications With Twisted"),
# the concept of the application
Slide(
"Applications",
Bullet("Reactor is a concept of event loop"),
Bullet("Application is higher-level"),
Bullet("Configuration, services, persistence"),
Bullet("Like reactor, you can listenTCP, connectTCP, etc."),
),
# services concept
Slide(
"Services",
Bullet("Services can be registered with Application"),
Bullet("A service encapsulates 'business logic'"),
Bullet("Infrastructure outside the scope of protocols"),
Bullet("Examples: authentication, mail storage"),
),
# service example code
Slide("Services 2", PRE(SERVICE_CODE)),
# logging
Slide("Logging", PRE(LOGGING_CODE)),
# logging errors
# explain why this is good idea (twistd -b)
Slide("Logging Errors", PRE(LOGGING_ERRORS_CODE)),
# twistd idea
Slide(
"twistd - Application Runner",
Bullet("Single access point for running applications"),
Bullet("Separate configuration from deployment"),
),
# twistd features
Slide(
"twistd Features",
Bullet("Daemonization"),
Bullet("Log file selection (including to syslog)"),
Bullet("Choosing reactor"),
Bullet("Running under debugger"),
Bullet("Profiling"),
Bullet("uid, gid"),
Bullet("Future: WinNT Services"),
),
# making modules for twistd -y
Slide("Making a runnable application", PRE(RUNNABLE_APP_CODE)),
# running the server
Slide("Running twistd", PRE(TWISTD_CODE)),
Slide("Part 4: Further Bits and Pieces"),
Slide(
"Other twisted.internet Features",
Bullet("UDP, Multicast, Unix sockets, Serial"),
Bullet("Thread integration"),
),
Slide(
"Deferreds",
Bullet("Deferred - a promise of a result"),
Bullet("Supports callback chains for results and exceptions"),
Bullet("Used across the whole framework"),
Bullet("Make event-driven programming much easier"),
Bullet("Can work with asyncore too, not just Twisted"),
),
Slide(
"Protocol implementations",
Bullet("Low-level implementations, without policies"),
Bullet(
"SSH, HTTP, SMTP, IRC, POP3, telnet, FTP, TOC, OSCAR, SOCKSv4, finger, DNS, NNTP, IMAP, LDAP"
),
Bullet("Common GPS modem protocols"),
),
Slide(
"Frameworks",
Bullet("twisted.web - Web server framework"),
Bullet("twisted.news - NNTP server framework"),
Bullet("twisted.words - messaging framework"),
Bullet("twisted.names - DNS server"),
),
Slide(
"Perspective Broker",
Bullet("Object publishing protocol"),
Bullet("Fast, efficient and extendable"),
Bullet("Two-way, asynchronous"),
Bullet("Secure and encourages secure model"),
Bullet("Implemented in Python for Twisted, and Java"),
),
Slide(
"Lore",
Bullet("Simple documentation system"),
Bullet("Simple subset of XHTML"),
Bullet("Generates LaTeX, XHTML"),
),
Slide(
"Reality",
Bullet("Multiplayer text simulation framework"),
Bullet("Original source of Twisted project"),
Bullet("Now a totally different project"),
),
)
lecture = Lecture(
"BuildBot: Build/Test Automation",
Slide("The BuildBot: Build/Test Automation",
Bullet("Home page: ",
URL("http://buildbot.sourceforge.net/"),
),
Bullet("Brian Warner < warner-buildbot @ lothar . com >"),
),
# quick description of what it does
# motivation: what is the problem it is trying to solve
# architecture: master+slaves
# design
# starting the build: changes, tree-stable timers
# doing the build: builders, processes
# describing the build: status
# status clients
# configuration: examples of Step, setup script
# demos: glib, show twisted status page
# future directions
# current status, availability, more info
Slide("Overview",
Bullet("What is the BuildBot?"),
Bullet("Motivation: What problem is it trying to solve?"),
Bullet("Architecture: What does it look like?"),
Bullet("Design: How does it work?"),
Bullet("Configuration: How do I make it do my bidding?"),
Bullet("Demo: Show me the code!"),
Bullet("Future Directons: What will it do in the future?"),
Bullet("More Information: How do I find out more?"),
),
# description
Slide("Automating the Compile/Test cycle",
Bullet("CVS commits trigger new builds"),
Bullet("Builds run on multiple machines to cover various platforms and environments"),
Bullet("Builds include running tests"),
Bullet("Build status easy to retrieve, can be pushed to developers"),
Bullet("inspired by Tinderbox"),
),
# motivation
Slide("Distributed cross-platform projects are tricky",
Bullet("Distance means poor communication"),
Bullet("Size leads to specialization: " + \
"Not everyone knows the whole tree"),
Bullet("Multiple platforms: hard to do sufficient testing"),
Bullet("Somebody has to keep it all working"),
),
# personal experience as chief harassment officer
Slide("Automating away the job of 'Build Sheriff'",
Bullet("Give quick feedback about build problems"),
Bullet("minimize inconvenience to other developers"),
Bullet("provide (gentle) incentive to fix bugs"),
Bullet("provide (less gentle) encouragement to fix bugs"),
Bullet("provide (outright hostile) harassment to STOP BREAKING STUFF!"),
Bullet("Help developers to Do The Right Thing"),
),
Slide("Features",
Bullet("Runs builds (and tests) when code is changed, report failures quickly"),
Bullet("Performs tests on a variety of slave platforms"),
Bullet("Handles arbitrary build processes: C, Python, Perl, etc"),
Bullet("Status delivery through web page, email, other protocols"),
Bullet("Track builds in progress, provide estimated completion time"),
Bullet("Flexible configuration by subclassing generic build process classes"),
Bullet("Minimal host requirements: Python and Twisted"),
Bullet("Released under the GPL"),
),
# master and slaves
# slaves can be behind a firewall if they can still do checkout
Raw("Architecture",
"""<h2>Architecture</h2>
<img src=\"../overview.png\" />
"""
),
# design
Slide("Starting the Build",
Bullet("Changes come from the version control system",
SubBullet("CVSToys listener"),
SubBullet("Parse cvs-commit email"),
),
Bullet("Builders wait for tree to be stable"),
Bullet("Builders can ignore some files which won't affect the build",
SubBullet("Documentation files"),
SubBullet("Example code"),
),
),
Slide("Running a Build",
Bullet("Build class defines each kind of build process",
SubBullet("Quick vs. clobber"),
SubBullet("Optimized vs. debug"),
SubBullet("Different versions of python, gcc, etc"),
),
Bullet("Sequence of Steps: CVS, Compile, Test",
SubBullet("Steps defined by subclasses of BuildStep"),
SubBullet("Steps invoke RemoteCommands on a connected slave"),
),
Bullet("Each Builder attaches to a BuildSlave (by name)"),
),
Slide("Describing the Build",
Bullet("Overall Status: success, fail, in-between"),
Bullet("Each step has status text, color, log files"),
Bullet("Progress events are distributed to HTML logger and status clients"),
),
Raw("HTML Build Status",
"""
<img src="../waterfall.png" alt="waterfall display" width="323" height="457" align="right" />
<h2>HTML Build Status</h2>
<ul>
<li>web server listens on arbitrary port</li>
<li>waterfall display shows time, commits, build results</li>
<li>Log files and build information are available through links</li>
<li>Alternate displays with different URLs</li>
</ul>
"""
),
Slide("Status Delivery",
Bullet("Build progress and status is given to a delivery object ..",
SubBullet(".. which can distribute it through arbitrary protocols"),
),
Bullet("HTML logger stores events, uses them to build waterfall display"),
Bullet("PB-based real-time status client",
SubBullet("Shows current activity, Time-Remaining for current build"),
),
),
Slide("Configuration",
Bullet("Everything driven by the buildmaster"),
# minimal slave setup: buildmaster location, dir, name/pw
Bullet("Classes provided for common build processes",
SubBullet("autoconf (C)"),
SubBullet("Makefile.PL (perl)"),
),
Bullet("Other BuildProcesses created by making subclasses"),
),
Slide("Demo",
Bullet("glib-1.2.10: simple C module with several self-tests"),
Bullet("python: Twisted BuildBot instance"),
),
Slide("Future Projects",
Bullet("Status Delivery through other protocols",
SubBullet("Email with build results and logfiles"),
SubBullet("IRC notification, interactive status queries"),
SubBullet("panel applet with highly summarized status"),
),
Bullet("Tracking test failures, history of each test"),
Bullet("Project-specific blame assignment, owner notification"),
Bullet("Web-based Builder Configuration"),
Bullet("bug-tracking integration"),
Bullet("'try': run tests on not-yet-committed changes"),
),
Slide("More Information",
Bullet("Home Page: ",
URL("http://buildbot.sourceforge.net/")),
Bullet("Sourceforge project page",
SubBullet("This paper and slides"),
SubBullet("Source available in CVS"),
SubBullet("Mailing list"),
SubBullet("Pointers to existing installations"),
),
Bullet("Please join the mailing list to find out about releases"),
),
)
lecture = Lecture(
"High-Level Database Interaction with SQLObject",
Slide(
"What is SQLObject?",
Bullet("YAORM -- Yet Another Object-Relational Mapper"),
Bullet(
"Allows you to map your Python classes to your database schema"),
Bullet("Takes the 'SQL' out of 'Database Programming'"),
Bullet("No special XML files to create, just normal Python classes")),
Slide(
"Why Do I Care?",
Bullet(
"Storing things in databases is fairly common in day-to-day programming"
),
Bullet(
"SQL is the standard language used to manipulate data in a database"
), Bullet("Writing SQL is boring, repetitive and depressing"),
Bullet("SQLObject relieves you of the burden of writing SQL"),
Bullet("...but still lets you write SQL when you need to")),
Slide(
"How does SQLObject differ from other ORM's?",
Bullet(
"Simple is better than complex; SQLObject is very simple to use"),
Bullet("Mappings are defined using normal Python classes"),
Bullet(
"Uses properties instead of wordy get/set methods for column attributes"
),
Bullet(
"No awkward auto-generation of Python classes/files from an external format"
),
Bullet(
"Can generate the database schema directly from your Python classes, and vice versa"
),
Bullet(
"Overloading magic provides convenient high-level SQL conditionals (dot q magic, stay tuned)"
)),
Slide(
"A Simple Example",
Bullet(
"To create a table that looks like this in SQL:",
PRE("""\
CREATE TABLE person (
id SERIAL,
first_name VARCHAR(100) NOT NULL,
middle_initial CHAR(1),
last_name VARCHAR(150) NOT NULL
);""")),
Bullet(
"You would write this Python code:",
PRE("""\
class Person(SQLObject):
firstName = StringCol(length = 100)
middleInitial = StringCol(length = 1)
lastName = StringCol(length = 150)""")),
Bullet("No, you're not dreaming :P")),
Slide(
"Declaring the Class",
PRE("""\
from SQLObject import *
conn = PostgresConnection(db = 'testdb', user = '******', password = '******')
class Person(SQLObject):
_connection = conn
firstName = StringCol(length = 100)
middleInitial = StringCol(length = 1)
lastName = StringCol(length = 150)"""),
Bullet(
"Use one of MySQLConnection, PostgresConnection, SQLiteConnection or DBMConnection as your _connection"
),
Bullet(
"Use StudlyCaps for your classes and mixedCase for your columns"),
Bullet(
"SQLObject will map TableName to table_name and columnName to column_name"
),
Bullet(
"In the above example, class Person becomes table person with columns first_name, middle_initial and last_name"
)),
Slide(
"Creating and Dropping Tables",
Bullet(
"Use the createTable class method to create the table, and two optional keyword arguments",
SubBullet(
Bullet(
"ifNotExists: only try to create the table if it doesn't exist"
),
Bullet(
"createJoinTables: will create the intermediate tables for many-to-many relationships"
))),
Bullet(
"Conversely, use dropTable, passing in optional ifExists and dropJoinTables keyword arguments"
)),
Slide(
"More on Column Syntax",
Bullet("Columns are implemented as properties"),
Bullet(
"Columns supported: StringCol, IntCol, FloatCol, EnumCol, DateTimeCol, ForeignKey, DecimalCol, CurrencyCol"
), Bullet("The first argument is the column name"),
Bullet(
"Keyword arguments specify additional information (e.g. notNone, default, length, etc)"
), Bullet("SQLObject lets the database do type checking and coercion"),
Bullet("An id column is implicitly created")),
Slide(
"Keyword Arguments for Col Classes",
Bullet("dbName: the column name in the database"),
Bullet(
"default: the default value (can be a callable that returns a value)"
),
Bullet(
"alternateID: set this to True if you want a byColumnName method to lookup rows based on this column"
), Bullet("unique: declare this column as UNIQUE in the database"),
Bullet("notNone: when True, column cannot be None/NULL"),
Bullet("sqlType: to specify the column type manually")),
Slide(
"Special Class Attributes",
Bullet("_connection: the database connection for this class"),
Bullet("_table: the database name of the table behind this class"),
Bullet("_joins: a list of join relationships to other classes"),
Bullet(
"_cacheValues: you'll want to set this false if you're using SQLObject classes from multiple processes"
), Bullet("_idName: the name of the PK (defaults to id)"),
Bullet(
"_style: a style object that provides a custom Python to DB name mapping algorithm"
)),
Slide(
"Using SQLObject classes",
Bullet(
"Create a new row with the .new() class method:",
PRE("""Person.new(firstName = "Brad", middleInitial = "E", lastName = "Bollenbach")"""
)),
Bullet(
"The row is inserted into the database as soon as you call .new()"
),
Bullet(
"Access an existing row by passing an ID to the class constructor",
PRE("""\
>>> me = Person(1)
>>> me.firstName
'Brad'
>>> me.lastName
'Bollenbach'""")), Bullet("Modify column values by modifying property values"),
Bullet(
"Changes to your object's properties are updated immediately in the database"
),
Bullet(
"...but transactions are there if you need them (as long as the database supports them)"
)),
Slide(
"Relating Your Classes with Joins",
Bullet(
"Use a ForeignKey to point a column's value at an instance of another class"
),
Bullet(
"To relate the PK class back to the FK class, use MultipleJoin"),
Bullet(
"Let's give a Person some PhoneNumbers:",
PRE("""\
from SQLObject import *
conn = PostgresConnection(db = 'testdb', user = '******', password = '******')
class Person(SQLObject):
_connection = conn
firstName = StringCol(length = 100)
middleInitial = StringCol(length = 1)
lastName = StringCol(length = 150)
phoneNumbers = MultipleJoin("PhoneNumber")
class PhoneNumber(SQLObject):
_connection = conn
person = ForeignKey('Person')
phoneNumber = StringCol(length = 10)"""))),
Slide(
"Many-to-Many Relationships", Bullet("A Person might have many Roles"),
Bullet("A Role might be associated to more than one Person"),
Bullet(
"Use a RelatedJoin to specify this many-to-many relation:",
PRE("""\
class Role(SQLObject):
_connection = conn
name = StringCol(length = 20)
people = RelatedJoin('Person')
class Person(SQLObject):
_connection = conn
firstName = StringCol(length = 100)
middleInitial = StringCol(length = 1)
lastName = StringCol(length = 150)
phoneNumbers = MultipleJoin("PhoneNumber")
roles = RelatedJoin('Role')
me = Person.new(firstName = "Brad", middleInitial = "E", lastName = "Bollenbach")
pg = Role.new(name = "Python Geek")
me.addRole(pg)
""")),
Bullet(
"SQLObject added .addRole() and .removeRole() methods to Person, and .addPerson() and .removePerson() methods to Role"
),
Bullet(
"...and created the person_role table by combining the table names of the classes alphabetically"
)),
Slide(
"Selecting Multiple Objects",
Bullet(
"The class's .select() method can be used to return multiple objects of a given class"
), Bullet("Here comes that dot q magic!"),
Bullet(
"Every SQLObject derived class has a special .q attribute, which uses some funky operator overloading to construct queries using Python:",
PRE("""\
>>> from person import Person
>>> brads = Person.select(Person.q.firstName == 'Brad')
>>> list(brads)
[<Person 1 lastName='Bollenbach' middleInitial='E' firstName='Brad'>]
>>> brads[0].lastName
'Bollenbach'
""")),
Bullet(
"select accepts orderBy and groupBy arguments, which are strings (or lists of strings) referring to the database column name"
)),
Slide(
"Selects using AND",
Bullet(
"Use the AND function for more specific queries:",
PRE("""\
>>> from SQLObject import AND
>>> from person import Person
>>> bradbs = Person.select(AND(
... Person.q.firstName == "Brad",
... Person.q.lastName.startswith("B")
... ))
""")),
Bullet(
"Notice that columns-as-properties behave much like their regular Python brethren (e.g. the startswith method used above)"
)),
Slide(
"Customizing Column Behaviour",
Bullet(
"SQLObject uses metaclasses to generate the classes that map to your database tables"
),
Bullet(
"Because of this, you need to take care when overriding property get/set methods"
),
Bullet(
"You cannot simply override the get or set method and then call the parent's get/set"
),
Bullet(
"Your class was created on-the-fly by SQLObject, so there is no parent! :)"
),
Bullet(
"SQLObject gives you special _SO_set_foo and _SO_get_foo methods to call after you've done something special in the normal setter/getter"
),
Bullet(
"An (admittedly contrived) example:",
PRE("""\
class Person(SQLObject):
...
def _set_lastName(self, value):
print "you changed the lastName"
self._SO_set_lastName(value)
"""))),
Slide(
"Better Living Through Transactions",
Bullet("Sometimes you need indivisibility"),
Bullet(
"SQLObject lets you use transactions when the database supports it",
PRE("""\
trans = Person._connection.transaction()
me = Person(1, trans)
me.firstName = "Bradley"
trans.commit()
me.firstName = "Yada"
trans.rollback()"""))),
Slide(
"Automatic Class Generation", Bullet("Laziness is a virtue"),
Bullet(
"If you've already got a database schema, why not make SQLObject create the classes for you?",
PRE("""\
class Person(SQLObject):
_fromDatabase = True""")),
Bullet("You can still specify additional columns, as normal"),
Bullet("Currently this only works with MySQL")),
Slide(
"SQLObject in the Wild -- Who's Using It?",
Bullet(
"BBnet.ca used SQLObject to develop a time tracking and invoicing system for a consultancy firm"
),
Bullet(
"XSOLI (my current employer) is using SQLObject to drive an online merchant proxy system"
),
Bullet(
"Future projects will probably include porting a sales application for a fixed-rate long distance provider onto the SQLObject framework"
),
Bullet(
"There's an active community of SQLObject developers on the mailing list"
)),
Slide(
"Future Plans for SQLObject", Bullet("Add support for more databases"),
Bullet(
"Improved transaction handling, so that even non-transaction aware backends can support them"
), Bullet("Hooks into a validation and form generation package"),
Bullet("More powerful joins")),
Slide(
"Summing Up",
Bullet(
"SQLObject is a framework that can make your (programming) life easier"
),
Bullet(
"It provides a high-level Python interface to things you'd normally suffer through in SQL"
),
Bullet(
"It offers advanced features like automatic class generation, dot q magic, and various ways of relating your classes"
),
Bullet(
"There's an active developer base, so you're sure to find help when you need it"
)),
Slide(
"Resources",
Bullet("SQLObject homepage -- ", URL("http://www.sqlobject.org")),
Bullet(
"Join the mailing list -- ",
URL("http://lists.sourceforge.net/lists/listinfo/sqlobject-discuss"
)), Bullet("Contact me -- [email protected]")),
Slide("Questions?", Bullet("Questions?"), Bullet("Comments?"),
Bullet("Free beer?")))
lecture = Lecture(
"Perspective Broker: Translucent RPC in Twisted",
# intro
Raw("Title", """
<h1>Perspective Broker: Translucent RPC in Twisted</h1>
<h2>PyCon 2003</h2>
<h2>Brian Warner < warner @ lothar . com > </h2>
"""),
Slide("Introduction",
Bullet("Overview/definition of RPC"),
Bullet("What is Perspective Broker?"),
Bullet("How do I use it?"),
Bullet("Security Issues"),
Bullet("Future Directions"),
),
Slide("Remote Procedure Calls",
Bullet("Action at a distance: separate processes, safely telling each other what to do",
SubBullet("Separate memory spaces"),
SubBullet("Usually on different machines"),
),
Bullet("Frequently called RMI these days: Remote Method Invocation"),
Bullet("Three basic parts: Addressing, Serialization, Waiting"),
),
Slide("Addressing",
Bullet("What program are you talking to?",
SubBullet("hostname, port number"),
SubBullet("Some systems use other namespaces: sunrpc")
),
Bullet("Which object in that program?"),
Bullet("Which method do you want to run?"),
Bullet("Related issues",
SubBullet("How do you know what the arguments are?"),
SubBullet("(do you care?)"),
SubBullet("How do you know what methods are available?"),
SubBullet("(do you care?)"),
),
),
Slide("Serialization",
Bullet("What happens to the arguments you send in?"),
Bullet("What happens to the results that are returned?",
SubBullet("Representation differences: endianness, word length"),
SubBullet("Dealing with user-defined types"),
),
Bullet("How to deal with references"),
),
Slide("The Waiting (is the hardest part)",
Bullet("Asynchronous: results come later, or not at all"),
Bullet("Need to do other work while waiting"),
),
Slide("Whither Translucence?",
Bullet("Not 'Transparent': don't pretend remote objects are really local",
SubBullet("CORBA (in C) does this, makes remote calls look like local calls"),
SubBullet("makes it hard to deal with the async nature of RPC"),
),
Bullet("Not 'Opaque': make it easy to deal with the differences",
SubBullet("Including extra failure modes, delayed results"),
),
Bullet("Exceptions and Deferreds to the rescue")),
Slide("Other RPC protocols",
Bullet("HTML"),
Bullet("XML-RPC"),
Bullet("CORBA"),
Bullet("when you control both ends of the wire, use PB"),
),
Raw("Where does PB fit?",
"""<h2>PB sits on top of <span class=\"py-src-identifier\">twisted.internet</span></h2>
<img src=\"twisted-overview.png\" />
"""),
Slide("pb.RemoteReference",
Bullet(HTML("<span class=\"py-src-identifier\">pb.Referenceable</span>: Object which can be accessed by remote systems."),
SubBullet(HTML("Defines methods like <span class=\"py-src-identifier\">remote_foo</span> and <span class=\"py-src-identifier\">remote_bar</span> which can be invoked remotely.")),
SubBullet(HTML("Methods without the <span class=\"py-src-identifier\">remote_</span> prefix are local-only.")),
),
Bullet(HTML("<span class=\"py-src-identifier\">pb.RemoteReference</span>: Used by distant program to invoke methods."),
SubBullet(HTML("Offers <span class=\"py-src-identifier\">.callRemote()</span> to trigger remote method on a corresponding <span class=\"py-src-identifier\">pb.Referenceable</span>.")),
),
),
Raw("Sample code",
"<h2>Sample Code</h2>" + server_lore + client_lore),
#Slide("Simple Demo"),
# "better demo: manhole, or reactor running in another thread"
#build up from callRemote?
Slide("What happens to those arguments?",
Bullet("Basic structures should travel transparently",
SubBullet("Actually quite difficult in some languages"),
),
Bullet("Object graph should remain the same",
SubBullet("Serialization context"),
SubBullet("(same issues as Pickle)")),
Bullet("Instances of user-defined classes require more care",
SubBullet("User-controlled unjellying"),)
),
#serialization (skip banana)
Slide("40% More Sandwich Puns Than The Leading Brand",
Bullet("twisted.spread: python package holding other modules"),
Bullet("PB: remote method invocation"),
Bullet("Jelly: mid-level object serialization"),
Bullet("Banana: low-level serialization of s-expressions"),
Bullet("Taster: security context, decides what may be received"),
Bullet("Marmalade: like Jelly, but involves XML, so it's bitter"),
Bullet("better than the competition",
SubBullet("CORBA: few or no sandwich puns"),
SubBullet("XML-RPC: barely pronounceable"),
),
),
Slide("Jellying objects",
Bullet("'Jellying' vs 'Unjellying'"),
Bullet("Immutable objects are copied whole"),
Bullet("Mutable objects get reference IDs to insure shared references remain shared",
SubBullet("(within the same Jellying context)")),
),
Slide("Jellying instances",
Bullet(HTML("User classes inherit from one of the <span class=\"py-src-identifier\">pb.flavor</span> classes")),
Bullet(HTML("<span class=\"py-src-identifier\">pb.Referenceable</span>: methods can be called remotely")),
Bullet(HTML("<span class=\"py-src-identifier\">pb.Copyable</span>: contents are selectively copied")),
Bullet(HTML("<span class=\"py-src-identifier\">pb.Cacheable</span>: contents are copied and kept up to date")),
Bullet(HTML("Classes define <span class=\"py-src-identifier\">.getStateToCopy</span> and other methods to restrict exported state")),
),
Slide("pb.Copyable example",
PRE("""class SenderPond(FrogPond, pb.Copyable):
def getStateToCopy(self):
d = self.__dict__.copy()
d['frogsAndToads'] = d['numFrogs'] + d['numToads']
del d['numFrogs']
del d['numToads']
return d
class ReceiverPond(pb.RemoteCopy):
def setCopyableState(self, state):
self.__dict__ = state
self.localCount = 12
def count(self):
return self.frogsAndToads
pb.setUnjellyableForClass(SenderPond, ReceiverPond)
""")),
Slide("Secure Unjellying",
Bullet("Pickle has security problems",
SubBullet("Pickle will import any module the sender requests."),
SubBullet(HTML("2.3 gave up, removed safety checks like <span class=\"py-src-identifier\">__safe_for_unpickling__</span> .")),
),
Bullet("Jelly attempts to be safe in the face of hostile clients",
SubBullet("All classes rejected by default"),
SubBullet(HTML("<span class=\"py-src-identifier\">registerUnjellyable()</span> used to accept safe ones")),
SubBullet(HTML("Registered classes define <span class=\"py-src-identifier\">.setCopyableState</span> and others to process remote state")),
),
Bullet("Must mark (by subclassing) to transmit"),
),
Slide("Transformation of references in transit",
Bullet("All referenced objects get turned into their counterparts as they go over the wire"),
Bullet("References are followed recursively",
SubBullet("Sending a reference to a tree of objects will cause the whole thing to be transferred"),
SubBullet("(subject to security restrictions)"),
),
Bullet(HTML("<span class=\"py-src-identifier\">pb.flavors</span> get reference ids"),
SubBullet("They are recognized when they return, transformed into the original reference"),
SubBullet("Reference ids are scoped to the connection"),
SubBullet("One side-effect: no 'third party' references"),
),
),
Slide("Perspectives: pb.cred and the Identity/Service model",
Bullet("A layer to provide common authentication services to Twisted applications"),
Bullet(HTML("<span class=\"py-src-identifier\">Identity</span>: named user accounts with passwords")),
Bullet(HTML("<span class=\"py-src-identifier\">Service</span>: something a user can request access to")),
Bullet(HTML("<span class=\"py-src-identifier\">Perspective</span>: user accessing a service")),
Bullet(HTML("<span class=\"py-src-identifier\">pb.Perspective</span>: first object, a <span class=\"py-src-identifier\">pb.Referenceable</span> used to access everything else")),
),
#picture would help
Slide("Future directions",
Bullet("Other language bindings: Java, elisp, Haskell, Scheme, JavaScript, OCaml"),
# donovan is doing the JavaScript port
Bullet("Other transports: UDP, Airhook"),
Bullet("Componentization"),
Bullet("Performance improvements: C extension for Jelly"),
Bullet("Refactor addressing model: PB URLs"),
),
Slide("Questions", Bullet("???")),
)
lecture = Lecture(
"PyLogo: the Logo Language implemented in Python",
Slide(
"PyLogo",
Bullet("Website: <b><tt>http://pylogo.org</tt></b>"),
Bullet("Logo is an educational language targeted at young children"),
Bullet("Logo isn't about teaching programming, but about teaching mathematical ideas "
"using programming"),
Bullet("PyLogo is a Logo interpreter written in Python"),
),
Slide(
"Why Logo?",
Bullet("Most people don't need the skills to become professional programmers"),
Bullet("Especially when they are 9 years old"),
Bullet("We shouldn't confuse our ideas about programming with the education "
"goals of programming"),
),
Slide(
"How Logo is different from Python",
Bullet("It doesn't put effort into code reuse"),
Bullet("Commands can be abbreviated, and can be very short, "
"because children type very, very slowly"),
Bullet("Very little punctuation"),
Bullet("Structure is casual with respect to command separation and "
"whitespace, also case-insensitive"),
Bullet("It uses dynamic typing"),
),
Slide(
"Examples",
PRE('''\
? REPEAT 4 [FD 100 RT 90]
? PR [Hello world!]
Hello world!
'''),
"The Python equivalent:",
PRE("""\
>>> from turtle import *
>>> for i in range(4):
... forward(100)
... right(90)
>>> print 'Hello world!'
Hello world!
""")),
Slide(
"What's neat about PyLogo",
Bullet("It's free, of course (and cross-platform and everything else you get from "
"implementing it in Python)"),
Bullet("It's traditional Logo (lots of naive non-traditional implementations of "
"logo exist)"),
Bullet("It integrates well with Python")),
Slide(
"Python Integration",
Bullet("All Python functions are made easily available"),
Bullet("Procedures can be easily annotated with function attributes, "
"allowing renaming, or allowing access to the Logo interpreter "
"object."),
Bullet("Making a procedure available is easy:"),
PRE("""\
# in turtle.py...
from turtle import forward
forward.aliases = ['fd']
"""),
PRE('''\
; In logo...
? import "turtle
? FD 100
''')),
Slide(
"Integration Example 2",
Bullet("Even control structures are easy:"),
PRE("""\
def logoWhile(interp, test, block):
lastVal = None
try:
while logoEval(interp, test):
try:
lastVal = logoEval(interp, block)
except LogoContinue:
pass
except LogoBreak:
lastVal = None
pass
return lastVal
logoWhile.logoAware = 1
logoWhile.logoName = 'while'
""")),
Slide(
"Why you might want to work on PyLogo",
Bullet("Python has a bunch of libraries that would be nice to make available "
"to a Logo user"),
Bullet("The PyLogo interpreter is really easy to play with (Logo is a <i>very</i> "
" simple language)"),
Bullet("Unfortunately, I get really distracted with different projects, so I "
"haven't been able to keep moving it forward."),
Bullet("Also, I'm not very experienced with GUI programming, and a Logo environment "
"should have a good interface"),
Bullet("But PyLogo has almost no professional potential, so it's just plain "
"fun"),
Bullet("Writing languages is fun. But hard. Playing around with a toy language "
"is educational, but ultimately not useful for anyone else."),
Bullet("But Logo is a toy language that isn't just a toy!"),
Bullet("Reminder: <b><tt>http://pylogo.org</tt></b>"),
))
lecture = Lecture(
"The twisted.internet Tutorial of Doom",
Slide("Part 1 - Introduction"),
# there are different ways to do networking
# mention processes are not cross-platform
Slide("Choosing a networking paradigm for the enterprise",
Bullet("Event driven"),
Bullet(Bad("Threads")),
Bullet("Others which we will ignore (processes, SEDA, ...)")),
# it's a metaphor!
Slide("Applied Bistromathics 101",
Bullet("Consider a restaurant as a network application"),
Bullet("Clients come in, make requests to the waiters"),
Bullet("Waiters act on clients' choices")),
# an event loop is efficient, doesn't waste time
# event loop is also used for GUIs
Slide("The event driven waiter",
Bullet("One waiter, serving all tables"),
Bullet("Waiter takes orders from tables to kitchen"),
Bullet("Waiter takes food from kitchen to tables")),
# not accurate, but the problems are real. avoid threads if you can
Slide("Threads (a caricature)",
Bullet(Bad("One waiter per table")),
SubBullet("Problems:",
Bullet(Bad("Expensive")),
Bullet(Bad("Waiters need to be careful not bump into each other")),
)),
# why threads are sometimes necessary
Slide("When do we want threads?",
Bullet("Long running, blocking operations"),
Bullet("Classic example: database access")),
# today we will discuss only (parts of) twisted.internet
Slide("Twisted: The Framework of Your Internet",
Image("twisted-overview.png")),
Slide("Project Stats",
Bullet("URL: ", URL("http://www.twistedmatrix.com")),
Bullet("License: LGPL"),
Bullet("Number of developers: approximately 20"),
Bullet("Version: 1.0.3"),
Bullet("Platforms: Unix, Win32"),
Bullet("Started in January 2000 by Glyph Lefkowitz")),
Slide("Part 2 - Basic Networking With Twisted"),
# quick review of how the internet works
Slide("Internet!",
Bullet("Network of interconnected machines"),
Bullet("Each machine has one (or more) IP addresses"),
Bullet("DNS maps names ('www.yahoo.com') to IPs (216.109.125.69)"),
Bullet("TCP runs on top of IP, servers listen on of of 65536 ports,"
" e.g. HTTP on port 80"),),
# we need to understand certain basic terms before we continue.
# the event loop is the last thing we run - it waits until
# an event occurs, then calls the appropriate handler.
Slide("Basic Definitions - Reactor",
Bullet("An object implementing the event loop",
PRE(EVENT_LOOP_CODE))),
Slide("Basic Definitions - Transport",
Bullet("Moves data from one location to another"),
Bullet("Main focus of talk are ordered, reliable byte stream transports"),
Bullet("Examples: TCP, SSL, Unix sockets"),
Bullet("UDP is a different kind of transport")),
# the client is the side which initiated the connection
# HTTP and SSH run on TCP-like transports, DNS runs on UDP or TCP
Slide("Basic Definitions - Protocol",
Bullet("Defines the rules for communication between two hosts"),
Bullet("Protocols communicate using a transport"),
Bullet("Typically there is a client, and server"),
Bullet("Examples: HTTP, SSH, DNS")),
Slide("All Together Now",
Bullet("The reactor gets events from the transports (read from network, write to network)"),
Bullet("The reactor passes events to protocol (connection lost, data received)"),
Bullet("The protocol tells the transport to do stuff (write data, lose connection)")),
# designing a new protocol is usually a bad idea, there are lots of
# things you can get wrong, both in design and in implementation
Slide("How To Implement A Protocol",
Bullet("Hopefully, you don't.")),
# XXX split into three expanded slides?
NumSlide("How To Not Implement A Protocol",
Bullet("Use an existing Twisted implementation of the protocol"),
Bullet("Use XML-RPC"),
Bullet("Use Perspective Broker, a remote object protocol")),
# connectionMade is called when connection is made
# dataReceived is called every time we receive data from the network
# connectionLost is called when the connection is lost
Slide("How To Really Implement A Protocol",
PRE(PROTOCOL_CODE)),
# factories - why?
Slide("Factories",
Bullet("A protocol instance only exists as long as the connection is there"),
Bullet("Protocols want to share state"),
Bullet("Solution: a factory object that creates protocol instances")),
# factory code - notice how protocol instances have access to the factory
# instance, for shared state. buildProtocol can return None if we don't
# want to accept connections from that address.
Slide("A Server Factory",
PRE(SERVER_CODE)),
# running the server we just wrote
Slide("Connecting A Factory To A TCP Port",
PRE(RUNNING_SERVER_CODE)),
# transport independence - using listenUNIX as example
Slide("Transport Independence",
Bullet("Notice how none of the protocol code was TCP specific"),
Bullet("We can reuse same protocol with different transports"),
Bullet("We could use listenUNIX for unix sockets with same code"),
Bullet("Likewise listenSSL for SSL or TLS")),
Slide("Client Side Protocol",
PRE(CLIENT_PROTOCOL_CODE)),
# client connections are different
Slide("Client Side Factories",
Bullet("Different requirements than server"),
Bullet("Failure to connect"),
Bullet("Automatic reconnecting"),
Bullet("Cancelling and timing out connections")),
# example client factory - explain use of default buildProtocol
Slide("Client Side Factories 2",
PRE(CLIENT_FACTORY_CODE)),
# connectTCP
Slide("Connection API",
PRE(CLIENT_CONNECT_CODE)),
# explain how transports buffer the output
Slide("Buffering",
Bullet("When we write to transport, data is buffered"),
Bullet("loseConnection will wait until all buffered data is sent, and producer (if any) is finished")),
# start/stopFactory
Slide("Factory Resources",
Bullet("Factories may want to create/clean up resources"),
Bullet("startFactory() - called on start of listening/connect"),
Bullet("stopFactory() - called on end of listening/connect"),
Bullet("Called once even if factory listening/connecting multiple ports")),
# example of restartable factory
Slide("Factory Resources 2",
PRE(FACTORY_START_CODE)),
Slide("Producers and Consumers",
Bullet("What if we want to send out lots of data?"),
Bullet("Can't write it out all at once"),
Bullet("We don't want to write too fast")),
Slide("Producers",
Bullet("Produce data for a consumer, in this case by calling transport's write()"),
Bullet("Pausable (should implement pauseProducing and resumeProducing methods)"),
Bullet("Push - keeps producing unless told to pause"),
Bullet("Pull - produces only when consumer tells it to")),
Slide("Consumers",
Bullet("registerProducer(producer, streaming)"),
Bullet("Will notify producer to pause if buffers are full")),
Slide("Sample Pull Producer",
PRE(PULL_PRODUCER_CODE)),
Slide("Sample Push Producer",
PRE(PUSH_PRODUCER_CODE)),
# scheduling events
Slide("Scheduling",
PRE(SCHEDULING_CODE)),
# pluggable reactors - why?
Slide("Choosing a Reactor - Why?",
Bullet("GUI toolkits have their own event loop"),
Bullet("Platform specific event loops")),
Slide("Choosing a Reactor",
Bullet("Twisted supports multiple reactors"),
Bullet("Default, gtk, gtk2, qt, win32 and others"),
Bullet("Tk and wxPython as non-reactors"),
Bullet("Reactor installation should be first thing code does")),
# example GUI client
Slide("Example GTK Program",
PRE(GUI_CODE)),
# you can learn more about
Slide("Learning more about networking and scheduling",
Bullet("twisted.internet.interfaces"),
Bullet("http://twistedmatrix.com/document/howtos/")),
Slide("Part 3 - Building Applications With Twisted"),
# the concept of the application
Slide("Applications",
Bullet("Reactor is a concept of event loop"),
Bullet("Application is higher-level"),
Bullet("Configuration, services, persistence"),
Bullet("Like reactor, you can listenTCP, connectTCP, etc.")),
# services concept
Slide("Services",
Bullet("Services can be registered with Application"),
Bullet("A service encapsulates 'business logic'"),
Bullet("Infrastructure outside the scope of protocols"),
Bullet("Examples: authentication, mail storage")),
# service example code
Slide("Services 2",
PRE(SERVICE_CODE)),
# logging
Slide("Logging",
PRE(LOGGING_CODE)),
# logging errors
# explain why this is good idea (twistd -b)
Slide("Logging Errors",
PRE(LOGGING_ERRORS_CODE)),
# twistd idea
Slide("twistd - Application Runner",
Bullet("Single access point for running applications"),
Bullet("Separate configuration from deployment")),
# twistd features
Slide("twistd Features",
Bullet("Daemonization"),
Bullet("Log file selection (including to syslog)"),
Bullet("Choosing reactor"),
Bullet("Running under debugger"),
Bullet("Profiling"),
Bullet("uid, gid"),
Bullet("Future: WinNT Services")),
# making modules for twistd -y
Slide("Making a runnable application",
PRE(RUNNABLE_APP_CODE)),
# running the server
Slide("Running twistd",
PRE(TWISTD_CODE)),
Slide("Part 4: Further Bits and Pieces"),
Slide("Other twisted.internet Features",
Bullet("UDP, Multicast, Unix sockets, Serial"),
Bullet("Thread integration")),
Slide("Deferreds",
Bullet("Deferred - a promise of a result"),
Bullet("Supports callback chains for results and exceptions"),
Bullet("Used across the whole framework"),
Bullet("Make event-driven programming much easier"),
Bullet("Can work with asyncore too, not just Twisted")),
Slide("Protocol implementations",
Bullet("Low-level implementations, without policies"),
Bullet("SSH, HTTP, SMTP, IRC, POP3, telnet, FTP, TOC, OSCAR, SOCKSv4, finger, DNS, NNTP, IMAP, LDAP"),
Bullet("Common GPS modem protocols")),
Slide("Frameworks",
Bullet("twisted.web - Web server framework"),
Bullet("twisted.news - NNTP server framework"),
Bullet("twisted.words - messaging framework"),
Bullet("twisted.names - DNS server")),
Slide("Perspective Broker",
Bullet("Object publishing protocol"),
Bullet("Fast, efficient and extendable"),
Bullet("Two-way, asynchronous"),
Bullet("Secure and encourages secure model"),
Bullet("Implemented in Python for Twisted, and Java")),
Slide("Lore",
Bullet("Simple documentation system"),
Bullet("Simple subset of XHTML"),
Bullet("Generates LaTeX, XHTML")),
Slide("Reality",
Bullet("Multiplayer text simulation framework"),
Bullet("Original source of Twisted project"),
Bullet("Now a totally different project")),
)
lecture = Lecture(
"BuildBot: Build/Test Automation",
Slide("The BuildBot: Build/Test Automation",
Bullet("Home page: ",
URL("http://buildbot.sourceforge.net/"),
),
Bullet("Brian Warner < warner-buildbot @ lothar . com >"),
),
# quick description of what it does
# motivation: what is the problem it is trying to solve
# architecture: master+slaves
# design
# starting the build: changes, tree-stable timers
# doing the build: builders, processes
# describing the build: status
# status clients
# configuration: examples of Step, setup script
# demos: glib, show twisted status page
# future directions
# current status, availability, more info
Slide("Overview",
Bullet("What is the BuildBot?"),
Bullet("Motivation: What problem is it trying to solve?"),
Bullet("Architecture: What does it look like?"),
Bullet("Design: How does it work?"),
Bullet("Configuration: How do I make it do my bidding?"),
Bullet("Demo: Show me the code!"),
Bullet("Future Directons: What will it do in the future?"),
Bullet("More Information: How do I find out more?"),
),
# description
Slide("Automating the Compile/Test cycle",
Bullet("CVS commits trigger new builds"),
Bullet("Builds run on multiple machines to cover various platforms and environments"),
Bullet("Builds include running tests"),
Bullet("Build status easy to retrieve, can be pushed to developers"),
Bullet("inspired by Tinderbox"),
),
# motivation
Slide("Distributed cross-platform projects are tricky",
Bullet("Distance means poor communication"),
Bullet("Size leads to specialization: " + \
"Not everyone knows the whole tree"),
Bullet("Multiple platforms: hard to do sufficient testing"),
Bullet("Somebody has to keep it all working"),
),
# personal experience as chief harassment officer
Slide("Automating away the job of 'Build Sheriff'",
Bullet("Give quick feedback about build problems"),
Bullet("minimize inconvenience to other developers"),
Bullet("provide (gentle) incentive to fix bugs"),
Bullet("provide (less gentle) encouragement to fix bugs"),
Bullet("provide (outright hostile) harassment to STOP BREAKING STUFF!"),
Bullet("Help developers to Do The Right Thing"),
),
Slide("Features",
Bullet("Runs builds (and tests) when code is changed, report failures quickly"),
Bullet("Performs tests on a variety of slave platforms"),
Bullet("Handles arbitrary build processes: C, Python, Perl, etc"),
Bullet("Status delivery through web page, email, other protocols"),
Bullet("Track builds in progress, provide estimated completion time"),
Bullet("Flexible configuration by subclassing generic build process classes"),
Bullet("Minimal host requirements: Python and Twisted"),
Bullet("Released under the GPL"),
),
# master and slaves
# slaves can be behind a firewall if they can still do checkout
Raw("Architecture",
"""<h2>Architecture</h2>
<img src=\"../overview.png\" />
"""
),
# design
Slide("Starting the Build",
Bullet("Changes come from the version control system",
SubBullet("CVSToys listener"),
SubBullet("Parse cvs-commit email"),
),
Bullet("Builders wait for tree to be stable"),
Bullet("Builders can ignore some files which won't affect the build",
SubBullet("Documentation files"),
SubBullet("Example code"),
),
),
Slide("Running a Build",
Bullet("Build class defines each kind of build process",
SubBullet("Quick vs. clobber"),
SubBullet("Optimized vs. debug"),
SubBullet("Different versions of python, gcc, etc"),
),
Bullet("Sequence of Steps: CVS, Compile, Test",
SubBullet("Steps defined by subclasses of BuildStep"),
SubBullet("Steps invoke RemoteCommands on a connected slave"),
),
Bullet("Each Builder attaches to a BuildSlave (by name)"),
),
Slide("Describing the Build",
Bullet("Overall Status: success, fail, in-between"),
Bullet("Each step has status text, color, log files"),
Bullet("Progress events are distributed to HTML logger and status clients"),
),
Raw("HTML Build Status",
"""
<img src="../waterfall.png" alt="waterfall display" width="323" height="457" align="right" />
<h2>HTML Build Status</h2>
<ul>
<li>web server listens on arbitrary port</li>
<li>waterfall display shows time, commits, build results</li>
<li>Log files and build information are available through links</li>
<li>Alternate displays with different URLs</li>
</ul>
"""
),
Slide("Status Delivery",
Bullet("Build progress and status is given to a delivery object ..",
SubBullet(".. which can distribute it through arbitrary protocols"),
),
Bullet("HTML logger stores events, uses them to build waterfall display"),
Bullet("PB-based real-time status client",
SubBullet("Shows current activity, Time-Remaining for current build"),
),
),
Slide("Configuration",
Bullet("Everything driven by the buildmaster"),
# minimal slave setup: buildmaster location, dir, name/pw
Bullet("Classes provided for common build processes",
SubBullet("autoconf (C)"),
SubBullet("Makefile.PL (perl)"),
),
Bullet("Other BuildProcesses created by making subclasses"),
),
Slide("Demo",
Bullet("glib-1.2.10: simple C module with several self-tests"),
Bullet("python: Twisted BuildBot instance"),
),
Slide("Future Projects",
Bullet("Status Delivery through other protocols",
SubBullet("Email with build results and logfiles"),
SubBullet("IRC notification, interactive status queries"),
SubBullet("panel applet with highly summarized status"),
),
Bullet("Tracking test failures, history of each test"),
Bullet("Project-specific blame assignment, owner notification"),
Bullet("Web-based Builder Configuration"),
Bullet("bug-tracking integration"),
Bullet("'try': run tests on not-yet-committed changes"),
),
Slide("More Information",
Bullet("Home Page: ",
URL("http://buildbot.sourceforge.net/")),
Bullet("Sourceforge project page",
SubBullet("This paper and slides"),
SubBullet("Source available in CVS"),
SubBullet("Mailing list"),
SubBullet("Pointers to existing installations"),
),
Bullet("Please join the mailing list to find out about releases"),
),
)
