def main():
import random
from twisted.internet import reactor
from twisted.python.log import addObserver
from twisted.python.failure import startDebugMode
startDebugMode()
report = ReportStatistics()
addObserver(SimpleStatistics().observe)
addObserver(report.observe)
addObserver(RequestLogger().observe)
r = random.Random()
r.seed(100)
populator = Populator(r)
parameters = PopulationParameters()
parameters.addClient(
1, ClientType(OS_X_10_6, [Eventer, Inviter, Accepter]))
simulator = CalendarClientSimulator(
populator, parameters, reactor, '127.0.0.1', 8008)
arrivalPolicy = SmoothRampUp(groups=10, groupSize=1, interval=3)
arrivalPolicy.run(reactor, simulator)
reactor.run()
report.report()
def process_options(self, args, opts):
try:
self.settings.setdict(arglist_to_dict(opts.set),
priority='cmdline')
except ValueError:
raise UsageError("Invalid -s value, use -s NAME=VALUE",
print_help=False)
if opts.logfile:
self.settings.set('LOG_ENABLED', True, priority='cmdline')
self.settings.set('LOG_FILE', opts.logfile, priority='cmdline')
if opts.loglevel:
self.settings.set('LOG_ENABLED', True, priority='cmdline')
self.settings.set('LOG_LEVEL', opts.loglevel, priority='cmdline')
if opts.nolog:
self.settings.set('LOG_ENABLED', False, priority='cmdline')
if opts.pidfile:
with open(opts.pidfile, "w") as f:
f.write(str(os.getpid()) + os.linesep)
if opts.pdb:
failure.startDebugMode()
def main():
from twisted.python.failure import startDebugMode
startDebugMode()
d = collect(sys.argv[1])
d.addErrback(err, "Problem collecting SQL")
d.addBoth(lambda ign: reactor.stop())
reactor.run(installSignalHandlers=False)
def main():
import random
from twisted.internet import reactor
from twisted.python.log import addObserver
from twisted.python.failure import startDebugMode
startDebugMode()
report = ReportStatistics()
addObserver(SimpleStatistics().observe)
addObserver(report.observe)
addObserver(RequestLogger().observe)
r = random.Random()
r.seed(100)
populator = Populator(r)
parameters = PopulationParameters()
parameters.addClient(1, ClientType(OS_X_10_6,
[Eventer, Inviter, Accepter]))
simulator = CalendarClientSimulator(populator, parameters, reactor,
'127.0.0.1', 8008)
arrivalPolicy = SmoothRampUp(groups=10, groupSize=1, interval=3)
arrivalPolicy.run(reactor, simulator)
reactor.run()
report.report()
def main():
from twisted.python.failure import startDebugMode
startDebugMode()
d = collect(sys.argv[1])
d.addErrback(err, "Problem collecting SQL")
d.addBoth(lambda ign: reactor.stop())
reactor.run(installSignalHandlers=False)
def opt_debug(self):
"""
Run the application in the Python Debugger (implies nodaemon),
sending SIGUSR2 will drop into debugger
"""
defer.setDebugging(True)
failure.startDebugMode()
self['debug'] = True
def opt_debug(self):
"""
Run the application in the Python Debugger (implies nodaemon),
sending SIGUSR2 will drop into debugger
"""
defer.setDebugging(True)
failure.startDebugMode()
self['debug'] = True
def opt_debug(self):
"""
run the application in the Python Debugger (implies nodaemon),
sending SIGUSR2 will drop into debugger
"""
from twisted.internet import defer
defer.setDebugging(True)
failure.startDebugMode()
self['debug'] = True
def cli(ctx, loglevel, config, pdb):
"""
feeds creates feeds for pages that don't have feeds.
"""
if pdb:
failure.startDebugMode()
os.chdir(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
settings = load_feeds_settings(config)
settings.set("LOG_LEVEL", loglevel.upper())
ctx.obj["settings"] = settings
def cli(ctx, loglevel, config, pdb):
"""
feeds creates feeds for pages that don't have feeds.
"""
if pdb:
failure.startDebugMode()
os.chdir(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
settings = get_feeds_settings(config)
settings.set('LOG_LEVEL', loglevel.upper())
ctx.obj['settings'] = settings
def cli(ctx, loglevel, config, pdb):
"""
feeds creates feeds for pages that don't have feeds.
"""
if pdb:
failure.startDebugMode()
# A pip-installed Feeds does not have a scrapy.cfg in its project root.
os.environ["SCRAPY_SETTINGS_MODULE"] = "feeds.default_settings"
settings = load_feeds_settings(config)
settings.set("LOG_LEVEL", loglevel.upper())
ctx.obj["settings"] = settings
def setUp(self):
"""
Override pdb.post_mortem so we can make sure it's called.
"""
# Make sure any changes we make are reversed:
post_mortem = pdb.post_mortem
origInit = failure.Failure.__dict__['__init__']
def restore():
pdb.post_mortem = post_mortem
failure.Failure.__dict__['__init__'] = origInit
self.addCleanup(restore)
self.result = []
pdb.post_mortem = self.result.append
failure.startDebugMode()
def setUp(self):
"""
Override pdb.post_mortem so we can make sure it's called.
"""
# Make sure any changes we make are reversed:
post_mortem = pdb.post_mortem
origInit = failure.Failure.__init__
def restore():
pdb.post_mortem = post_mortem
failure.Failure.__init__ = origInit
self.addCleanup(restore)
self.result = []
pdb.post_mortem = self.result.append
failure.startDebugMode()
def main():
credChecker = CredChecker()
credChecker.addUser('foo', 'bar')
print 'Defined Users:', credChecker.users
realm = DummyRealm()
f = Factory()
f.protocol = POP3
f.portal = Portal(realm, (credChecker,))
f.protocol.portal = f.portal
print 'Portal Credential Interfaces', f.portal.listCredentialsInterfaces()
PORT = 2110
print 'PORT', PORT
tpf.startDebugMode()
reactor.listenTCP(PORT, f)
reactor.run()
def setUp(self):
"""
Override pdb.post_mortem so we can make sure it's called.
"""
# Make sure any changes we make are reversed:
post_mortem = pdb.post_mortem
if _shouldEnableNewStyle:
origInit = failure.Failure.__init__
else:
origInit = failure.Failure.__dict__['__init__']
def restore():
pdb.post_mortem = post_mortem
if _shouldEnableNewStyle:
failure.Failure.__init__ = origInit
else:
failure.Failure.__dict__['__init__'] = origInit
self.addCleanup(restore)
self.result = []
pdb.post_mortem = self.result.append
failure.startDebugMode()
def process_options(self, args, opts):
try:
self.settings.overrides.update(arglist_to_dict(opts.set))
except ValueError:
raise UsageError("Invalid -s value, use -s NAME=VALUE", print_help=False)
if opts.logfile:
self.settings.overrides['LOG_ENABLED'] = True
self.settings.overrides['LOG_FILE'] = opts.logfile
if opts.loglevel:
self.settings.overrides['LOG_ENABLED'] = True
self.settings.overrides['LOG_LEVEL'] = opts.loglevel
if opts.nolog:
self.settings.overrides['LOG_ENABLED'] = False
if opts.pidfile:
with open(opts.pidfile, "w") as f:
f.write(str(os.getpid()) + os.linesep)
if opts.pdb:
failure.startDebugMode()
def process_options(self, args, opts):
try:
self.settings.overrides.update(arglist_to_dict(opts.set))
except ValueError:
raise UsageError("Invalid -s value, use -s NAME=VALUE", print_help=False)
if opts.logfile:
self.settings.overrides['LOG_ENABLED'] = True
self.settings.overrides['LOG_FILE'] = opts.logfile
if opts.loglevel:
self.settings.overrides['LOG_ENABLED'] = True
self.settings.overrides['LOG_LEVEL'] = opts.loglevel
if opts.nolog:
self.settings.overrides['LOG_ENABLED'] = False
if opts.pidfile:
with open(opts.pidfile, "w") as f:
f.write(str(os.getpid()) + os.linesep)
if opts.pdb:
failure.startDebugMode()
def process_options(self, args, opts):
try:
self.settings.setdict(arglist_to_dict(opts.set),
priority='cmdline')
except ValueError:
raise UsageError("Invalid -s value, use -s NAME=VALUE", print_help=False)
if opts.logfile:
self.settings.set('LOG_ENABLED', True, priority='cmdline')
self.settings.set('LOG_FILE', opts.logfile, priority='cmdline')
if opts.loglevel:
self.settings.set('LOG_ENABLED', True, priority='cmdline')
self.settings.set('LOG_LEVEL', opts.loglevel, priority='cmdline')
if opts.nolog:
self.settings.set('LOG_ENABLED', False, priority='cmdline')
if opts.pidfile:
with open(opts.pidfile, "w") as f:
f.write(str(os.getpid()) + os.linesep)
if opts.pdb:
failure.startDebugMode()
def set_twisted_debug():
DelayedCall.debug = True
failure.startDebugMode()
defer.setDebugging(True)
raise ValueError("Unknown benchmark: %r" % (name, ))
def main():
from twisted.python.log import startLogging, err
options = BenchmarkOptions()
try:
options.parseOptions(sys.argv[1:])
except UsageError, e:
print(e)
return 1
if options['debug']:
from twisted.python.failure import startDebugMode
startDebugMode()
if options['source-directory']:
source = options['source-directory']
conf = source.child('conf').child('caldavd-dev.plist')
pids = whichPIDs(source, plistlib.PlistParser().parse(conf.open()))
else:
pids = []
msg("Using dtrace to monitor pids %r" % (pids, ))
startLogging(file('benchmark.log', 'a'), False)
d = benchmark(options['host'], options['port'], pids, options['label'],
options['parameters'], [(arg, resolveBenchmark(arg).measure)
for arg in options['benchmarks']])
d.addErrback(err, "Failure at benchmark runner top-level")
# -*- coding:utf-8 -*-
from __future__ import absolute_import, division, print_function, unicode_literals
import os
import uniout
import json
from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings
from scrapy.utils.log import configure_logging
from twisted.python import failure
from school_info_spider import spiders, models
configure_logging()
failure.startDebugMode()
process = CrawlerProcess(get_project_settings())
process.crawl(spiders.SchoolSpider)
process.crawl(spiders.MajorSpider)
process.crawl(spiders.RetrialAcceptingLineSpider)
process.crawl(spiders.AcceptanceRateSpider)
process.start() # the script will block here until all crawling jobs are finished
# session = models.DBSession()
# with open(os.path.join('JsonLinesExport', spiders.SchoolSpider.name), 'rb') as fp:
# for l in fp:
# d = json.loads(l)
# s = models.School(**d)
# session.add(s)
# session.commit()
# with open(os.path.join('JsonLinesExport', spiders.MajorSpider.name), 'rb') as fp:
# for l in fp:
def _initialDebugSetup(config):
# do this part of debug setup first for easy debugging of import failures
if config['debug']:
failure.startDebugMode()
if config['debug'] or config['debug-stacktraces']:
defer.setDebugging(True)
def _initialDebugSetup(config):
if config['debug']:
failure.startDebugMode()
if config['debug'] or config['debug-stacktraces']:
defer.setDebugging(True)
raise ValueError("Unknown benchmark: %r" % (name,))
def main():
from twisted.python.log import startLogging, err
options = BenchmarkOptions()
try:
options.parseOptions(sys.argv[1:])
except UsageError, e:
print(e)
return 1
if options['debug']:
from twisted.python.failure import startDebugMode
startDebugMode()
if options['source-directory']:
source = options['source-directory']
conf = source.child('conf').child('caldavd-dev.plist')
pids = whichPIDs(source, plistlib.PlistParser().parse(conf.open()))
else:
pids = []
msg("Using dtrace to monitor pids %r" % (pids,))
startLogging(file('benchmark.log', 'a'), False)
d = benchmark(
options['host'], options['port'], pids, options['label'],
options['parameters'],
[(arg, resolveBenchmark(arg).measure) for arg in options['benchmarks']])
def _initialDebugSetup(config):
# do this part of debug setup first for easy debugging of import failures
if config['debug']:
failure.startDebugMode()
if config['debug'] or config['debug-stacktraces']:
defer.setDebugging(True)
def _initialDebugSetup(config):
# do this part of debug setup first for easy debugging of import failures
if config['debug']:
from twisted.internet import defer
defer.setDebugging(True)
failure.startDebugMode()
