def parse_args(self):
self.options, self.args = optparse.OptionParser.parse_args(self, args=[unicode(x, "utf-8") for x in sys.argv[1:]])
for c in self.config_values:
if not hasattr(c, "pages"):
c.pages = []
if self.options.logfile:
start_logging(self.options.logfile)
if self.options.metabook:
self.metabook = json.loads(unicode(open(self.options.metabook, 'rb').read(), 'utf-8'))
try:
self.options.imagesize = int(self.options.imagesize)
assert self.options.imagesize > 0
except (ValueError, AssertionError):
self.error('Argument for --imagesize must be an integer > 0.')
for title in self.args:
if self.metabook is None:
self.metabook = metabook.collection()
self.metabook.append_article(title)
return self.options, self.args
def post():
parser = optparse.OptionParser(usage="%prog OPTIONS")
parser.add_option("-i", "--input", help="ZIP file to POST")
parser.add_option('-l', '--logfile',
help='log output to LOGFILE',
)
parser.add_option("-p", "--posturl", help="HTTP POST ZIP file to POSTURL")
parser.add_option("-g", "--getposturl",
help='get POST URL from PediaPress.com, open upload page in webbrowser',
action='store_true',
)
parser.add_option("-d", "--daemonize", action="store_true",
help='become a daemon process as soon as possible')
parser.add_option('--pid-file',
help='write PID of daemonized process to this file',
)
options, args = parser.parse_args()
use_help = 'Use --help for usage information.'
if not options.input:
parser.error('Specify --input.\n' + use_help)
if (options.posturl and options.getposturl)\
or (not options.posturl and not options.getposturl):
parser.error('Specify either --posturl or --getposturl.\n' + use_help)
if options.posturl:
from mwlib.podclient import PODClient
podclient = PODClient(options.posturl)
elif options.getposturl:
import webbrowser
from mwlib.podclient import podclient_from_serviceurl
podclient = podclient_from_serviceurl('http://pediapress.com/api/collections/')
webbrowser.open(podclient.redirecturl)
from mwlib import utils
from mwlib.status import Status
if options.logfile:
utils.start_logging(options.logfile)
if options.daemonize:
utils.daemonize()
if options.pid_file:
open(options.pid_file, 'wb').write('%d\n' % os.getpid())
status = Status(podclient=podclient)
try:
try:
status(status='uploading', progress=0)
podclient.post_zipfile(options.input)
status(status='finished', progress=100)
except Exception, e:
status(status='error')
raise
finally:
if options.pid_file:
utils.safe_unlink(options.pid_file)
def watch():
parser = optparse.OptionParser(usage="%prog [OPTIONS]")
parser.add_option('-l', '--logfile',
help='log output to LOGFILE',
)
parser.add_option('-d', '--daemonize',
action='store_true',
help='become daemon as soon as possible',
)
parser.add_option('--pid-file',
help='write PID of daemonized process to this file',
)
parser.add_option('-q', '--queue-dir',
help='queue directory, where new job files are written to (default: /var/cache/mw-watch/q/)',
default='/var/cache/mw-watch/q/',
)
parser.add_option('-p', '--processing-dir',
help='processing directory, where active job files are moved to (must be on same filesystem as --queue-dir, default: /var/cache/mw-watch/p/)',
default='/var/cache/mw-watch/p/',
)
parser.add_option('-n', '--num-jobs',
help='maximum number of simulataneous jobs (default: 5)',
default='5',
)
options, args = parser.parse_args()
try:
options.num_jobs = int(options.num_jobs)
except ValueError:
parser.error('--num-jobs value must be an integer')
from mwlib import filequeue, utils
if options.logfile:
utils.start_logging(options.logfile)
if options.daemonize:
utils.daemonize()
if options.pid_file:
open(options.pid_file, 'wb').write('%d\n' % os.getpid())
poller = filequeue.FileJobPoller(
queue_dir=options.queue_dir,
processing_dir=options.processing_dir,
max_num_jobs=options.num_jobs,
).run_forever()
if options.pid_file:
utils.safe_unlink(options.pid_file)
def parse_args(self):
self.options, self.args = optparse.OptionParser.parse_args(self)
if self.options.logfile:
start_logging(self.options.logfile)
if self.options.config is None:
if not self.config_optional:
self.error(
'Please specify --config option. See --help for all options.'
)
return self.options, self.args
if self.options.metabook:
self.metabook = json.loads(
open(self.options.metabook, 'rb').read())
if self.options.login is not None and ':' not in self.options.login:
self.error(
'Please specify username and password as USERNAME:PASSWORD.')
try:
self.options.imagesize = int(self.options.imagesize)
assert self.options.imagesize > 0
except (ValueError, AssertionError):
self.error('Argument for --imagesize must be an integer > 0.')
try:
self.options.num_threads = int(self.options.num_threads)
assert self.options.num_threads >= 0
except (ValueError, AssertionError):
self.error('Argument for --num-threads must be an integer >= 0.')
if self.options.no_threads:
self.options.num_threads = 0
if self.args:
if self.metabook is None:
self.metabook = metabook.make_metabook()
for title in self.args:
self.metabook['items'].append(
metabook.make_article(title=unicode(title, 'utf-8'), ))
return self.options, self.args
def post():
parser = optparse.OptionParser(usage="%prog OPTIONS")
parser.add_option("-i", "--input", help="ZIP file to POST")
parser.add_option('-l', '--logfile', help='log output to LOGFILE')
parser.add_option("-p", "--posturl", help="HTTP POST ZIP file to POSTURL")
parser.add_option(
"-g",
"--getposturl",
help='get POST URL from PediaPress.com, open upload page in webbrowser',
action='store_true')
options, args = parser.parse_args()
use_help = 'Use --help for usage information.'
if not options.input:
parser.error('Specify --input.\n' + use_help)
if (options.posturl and options.getposturl)\
or (not options.posturl and not options.getposturl):
parser.error('Specify either --posturl or --getposturl.\n' + use_help)
if options.posturl:
from mwlib.podclient import PODClient
podclient = PODClient(options.posturl)
elif options.getposturl:
import webbrowser
from mwlib.podclient import podclient_from_serviceurl
podclient = podclient_from_serviceurl(
'http://pediapress.com/api/collections/')
webbrowser.open(podclient.redirecturl)
from mwlib import utils
from mwlib.status import Status
if options.logfile:
utils.start_logging(options.logfile)
status = Status(podclient=podclient)
try:
status(status='uploading', progress=0)
podclient.post_zipfile(options.input)
status(status='finished', progress=100)
except Exception as e:
status(status='error')
raise
def post():
parser = optparse.OptionParser(usage="%prog OPTIONS")
parser.add_option("-i", "--input", help="ZIP file to POST")
parser.add_option('-l', '--logfile',
help='log output to LOGFILE')
parser.add_option("-p", "--posturl", help="HTTP POST ZIP file to POSTURL")
parser.add_option("-g", "--getposturl",
help='get POST URL from PediaPress.com, open upload page in webbrowser',
action='store_true')
options, args = parser.parse_args()
use_help = 'Use --help for usage information.'
if not options.input:
parser.error('Specify --input.\n' + use_help)
if (options.posturl and options.getposturl)\
or (not options.posturl and not options.getposturl):
parser.error('Specify either --posturl or --getposturl.\n' + use_help)
if options.posturl:
from mwlib.podclient import PODClient
podclient = PODClient(options.posturl)
elif options.getposturl:
import webbrowser
from mwlib.podclient import podclient_from_serviceurl
podclient = podclient_from_serviceurl('http://pediapress.com/api/collections/')
webbrowser.open(podclient.redirecturl)
from mwlib import utils
from mwlib.status import Status
if options.logfile:
utils.start_logging(options.logfile)
status = Status(podclient=podclient)
try:
status(status='uploading', progress=0)
podclient.post_zipfile(options.input)
status(status='finished', progress=100)
except Exception as e:
status(status='error')
raise
def parse_args(self):
self.options, self.args = optparse.OptionParser.parse_args(self)
if self.options.logfile:
start_logging(self.options.logfile)
if self.options.config is None:
if not self.config_optional:
self.error('Please specify --config option. See --help for all options.')
return self.options, self.args
if self.options.metabook:
self.metabook = json.loads(open(self.options.metabook, 'rb').read())
if self.options.login is not None and ':' not in self.options.login:
self.error('Please specify username and password as USERNAME:PASSWORD.')
try:
self.options.imagesize = int(self.options.imagesize)
assert self.options.imagesize > 0
except (ValueError, AssertionError):
self.error('Argument for --imagesize must be an integer > 0.')
try:
self.options.num_threads = int(self.options.num_threads)
assert self.options.num_threads >= 0
except (ValueError, AssertionError):
self.error('Argument for --num-threads must be an integer >= 0.')
if self.options.no_threads:
self.options.num_threads = 0
if self.args:
if self.metabook is None:
self.metabook = metabook.make_metabook()
for title in self.args:
self.metabook['items'].append(metabook.make_article(
title=unicode(title, 'utf-8'),
))
return self.options, self.args
def parse_args(self):
self.options, self.args = optparse.OptionParser.parse_args(self, args=[unicode(x, "utf-8") for x in sys.argv[1:]])
for c in self.config_values:
if not hasattr(c, "pages"):
c.pages = []
if self.options.logfile:
start_logging(self.options.logfile)
if self.options.metabook:
self.metabook = json.loads(unicode(open(self.options.metabook, 'rb').read(), 'utf-8'))
try:
self.options.imagesize = int(self.options.imagesize)
assert self.options.imagesize > 0
except (ValueError, AssertionError):
self.error('Argument for --imagesize must be an integer > 0.')
for title in self.args:
if self.metabook is None:
self.metabook = metabook.collection()
self.metabook.append_article(title)
if self.options.print_template_pattern and "$1" not in self.options.print_template_pattern:
self.error("bad --print-template-pattern argument [must contain $1, but %r does not]" % (self.options.print_template_pattern,))
if self.options.print_template_prefix and self.options.print_template_pattern:
log.warn('Both --print-template-pattern and --print-template-prefix (deprecated) specified. Using --print-template-pattern only.')
elif self.options.print_template_prefix:
self.options.print_template_pattern = '%s$1' % self.options.print_template_prefix
del self.options.print_template_prefix
return self.options, self.args
def parse_args(self):
self.options, self.args = optparse.OptionParser.parse_args(self, args=[unicode(x, "utf-8") for x in sys.argv[1:]])
for c in self.config_values:
if not hasattr(c, "pages"):
c.pages = []
if self.options.logfile:
start_logging(self.options.logfile)
if self.options.metabook:
self.metabook = json.loads(unicode(open(self.options.metabook, 'rb').read(), 'utf-8'))
try:
self.options.imagesize = int(self.options.imagesize)
assert self.options.imagesize > 0
except (ValueError, AssertionError):
self.error('Argument for --imagesize must be an integer > 0.')
for title in self.args:
if self.metabook is None:
self.metabook = metabook.collection()
self.metabook.append_article(title)
if self.options.print_template_pattern and "$1" not in self.options.print_template_pattern:
self.error("bad --print-template-pattern argument [must contain $1, but %r does not]" % (self.options.print_template_pattern,))
if self.options.print_template_prefix and self.options.print_template_pattern:
log.warn('Both --print-template-pattern and --print-template-prefix (deprecated) specified. Using --print-template-pattern only.')
elif self.options.print_template_prefix:
self.options.print_template_pattern = '%s$1' % self.options.print_template_prefix
del self.options.print_template_prefix
return self.options, self.args
return urllib2.urlopen(posturl, urllib.urlencode({'status': status})).read()
except Exception, e:
print 'ERROR posting status %r to %r' % (status, posturl)
def post_progress(progress):
print 'progress', progress
if not posturl:
return
try:
return urllib2.urlopen(posturl, urllib.urlencode({'progress': int(progress)})).read()
except Exception, e:
print 'ERROR posting progress %r to %r' % (progress, posturl)
try:
if options.logfile:
utils.start_logging(options.logfile)
output = options.output
from mwlib import wiki, recorddb, metabook
mb = metabook.MetaBook()
if conf:
from ConfigParser import ConfigParser
w = wiki.makewiki(conf)
cp = ConfigParser()
cp.read(conf)
license = {
'name': cp.get('wiki', 'defaultarticlelicense')
}
def main():
parser = OptionParser(usage="%prog [OPTIONS]")
parser.add_option("-b", "--baseurl", help="baseurl of wiki")
parser.add_option("-w", "--writer", help="writer to use")
parser.add_option('-l', '--logfile', help='log output to LOGFILE')
parser.add_option('-f', '--from-email',
help='From: email address for error mails',
)
parser.add_option('-r', '--mail-recipients',
help='To: email addresses ("," separated) for error mails',
)
parser.add_option('-m', '--max-narticles',
help='maximum number of articles for random collections (min is 1)',
default=10,
)
parser.add_option('-s', '--serviceurl',
help="location of the mw-serve server to test",
default='http://tools.pediapress.com/mw-serve/',
# default='http://localhost:8899/mw-serve/',
)
use_help = 'Use --help for usage information.'
options, args = parser.parse_args()
assert options.from_email
if options.logfile:
utils.start_logging(options.logfile)
baseurl2api = {}
baseurls = options.baseurl.split()
for baseurl in baseurls:
baseurl2api[baseurl] = mwapidb.APIHelper(baseurl)
maxarts = int(options.max_narticles)
mail_recipients = None
if options.mail_recipients:
mail_recipients = options.mail_recipients.split(',')
ok_count = 0
fail_count = 0
while True:
baseurl = random.choice(baseurls)
try:
ok = checkservice(baseurl2api[baseurl],
options.serviceurl,
baseurl,
options.writer,
maxarts,
from_email=options.from_email,
mail_recipients=mail_recipients,
)
if ok:
ok_count += 1
log.check('OK')
else:
fail_count += 1
log.check('FAIL!')
except KeyboardInterrupt:
break
except BaseException:
fail_count += 1
log.check('EPIC FAIL!!!')
utils.report(
system=system,
subject='checkservice() failed, waiting 60seconds',
from_email=options.from_email,
mail_recipients=mail_recipients,
)
sys.exc_clear()
time.sleep(60)
log.info('%s, %s\tok: %d, failed: %d' % (
baseurl, options.writer, ok_count, fail_count,
))
def main():
from SocketServer import ForkingMixIn, ThreadingMixIn
from wsgiref.simple_server import make_server, WSGIServer
from flup.server import fcgi, fcgi_fork, scgi, scgi_fork
class ForkingWSGIServer(ForkingMixIn, WSGIServer):
pass
class ThreadingWSGIServer(ThreadingMixIn, WSGIServer):
pass
proto2server = {
'http': ForkingWSGIServer,
'http_threaded': ThreadingWSGIServer,
'fcgi': fcgi_fork.WSGIServer,
'fcgi_threaded': fcgi.WSGIServer,
'scgi': scgi_fork.WSGIServer,
'scgi_threaded': scgi.WSGIServer,
}
parser = optparse.OptionParser(usage="%prog [OPTIONS]")
parser.add_option('-l', '--logfile',
help='log output to LOGFILE',
)
parser.add_option('-d', '--daemonize',
action='store_true',
help='become daemon as soon as possible',
)
parser.add_option('--pid-file',
help='write PID of daemonized process to this file',
)
parser.add_option('-P', '--protocol',
help='one of %s (default: http)' % ', '.join(proto2server.keys()),
default='http',
)
parser.add_option('-p', '--port',
help='port to listen on (default: 8899)',
default='8899',
)
parser.add_option('-i', '--interface',
help='interface to listen on (default: 0.0.0.0)',
default='0.0.0.0',
)
parser.add_option('--cache-dir',
help='cache directory (default: /var/cache/mw-serve/)',
default='/var/cache/mw-serve/',
)
parser.add_option('--mwrender',
help='(path to) mw-render executable',
default='mw-render',
)
parser.add_option('--mwrender-logfile',
help='global logfile for mw-render',
metavar='LOGFILE',
)
parser.add_option('--mwzip',
help='(path to) mw-zip executable',
default='mw-zip',
)
parser.add_option('--mwzip-logfile',
help='global logfile for mw-zip',
metavar='LOGFILE',
)
parser.add_option('--mwpost',
help='(path to) mw-post executable',
default='mw-post',
)
parser.add_option('--mwpost-logfile',
help='global logfile for mw-post',
metavar='LOGFILE',
)
parser.add_option('-q', '--queue-dir',
help='queue dir of mw-watch (if not specified, no queue is used)',
)
parser.add_option('-m', '--method',
help='prefork or threaded (default: prefork)',
default='prefork',
)
parser.add_option('--max-requests',
help='maximum number of requests a child process can handle before it is killed, irrelevant for --method=threaded (default: 0 = no limit)',
default='0',
metavar='NUM',
)
parser.add_option('--min-spare',
help='minimum number of spare processes/threads (default: 2)',
default='2',
metavar='NUM',
)
parser.add_option('--max-spare',
help='maximum number of spare processes/threads (default: 5)',
default='5',
metavar='NUM',
)
parser.add_option('--max-children',
help='maximum number of processes/threads (default: 50)',
default='50',
metavar='NUM',
)
parser.add_option('--report-from-mail',
help='sender of error mails (--report-recipient also needed)',
metavar='EMAIL',
)
parser.add_option('--report-recipient',
help='recipient of error mails (--report-from-mail also needed)',
metavar='EMAIL',
)
options, args = parser.parse_args()
if args:
parser.error('no arguments supported')
if options.protocol not in proto2server:
parser.error('unsupported protocol (must be one of %s)' % (
', '.join(proto2server.keys()),
))
def to_int(opt_name):
try:
setattr(options, opt_name, int(getattr(options, opt_name)))
except ValueError:
parser.error('--%s value must be an integer' % opt_name.replace('_', '-'))
to_int('port')
to_int('max_requests')
to_int('min_spare')
to_int('max_spare')
to_int('max_children')
if options.method not in ('prefork', 'threaded'):
parser.error('the only supported values for --method are "prefork" and "threaded"')
from mwlib import serve, log, utils
log = log.Log('mw-serve')
if options.logfile:
utils.start_logging(options.logfile)
if options.daemonize:
utils.daemonize()
if options.pid_file:
open(options.pid_file, 'wb').write('%d\n' % os.getpid())
if options.method == 'threaded':
options.protocol += '_threaded'
flup_kwargs = {
'maxThreads': options.max_children,
}
else:
flup_kwargs = {
'maxChildren': options.max_children,
'maxRequests': options.max_requests,
}
log.info("serving %s on %s:%s" % (options.protocol, options.interface, options.port))
if options.report_recipient and options.report_from_mail:
report_from_mail = options.report_from_mail.encode('utf-8')
report_recipients = [options.report_recipient.encode('utf-8')]
else:
report_from_mail = None
report_recipients = None
app = serve.Application(
cache_dir=options.cache_dir,
mwrender_cmd=options.mwrender,
mwrender_logfile=options.mwrender_logfile,
mwzip_cmd=options.mwzip,
mwzip_logfile=options.mwzip_logfile,
mwpost_cmd=options.mwpost,
mwpost_logfile=options.mwpost_logfile,
queue_dir=options.queue_dir,
report_from_mail=report_from_mail,
report_recipients=report_recipients,
)
if options.protocol.startswith('http'):
server = make_server(options.interface, options.port, app,
server_class=proto2server[options.protocol],
)
try:
server.serve_forever()
except KeyboardInterrupt:
pass
else:
serverclass = proto2server[options.protocol]
serverclass(app,
bindAddress=(options.interface, options.port),
minSpare=options.min_spare,
maxSpare=options.max_spare,
**flup_kwargs
).run()
if options.pid_file:
utils.safe_unlink(options.pid_file)
log.info('exit.')
def check_service():
import sys
import time
from mwlib.client import Client
from mwlib.log import Log
from mwlib import utils
log = Log('mw-check-service')
parser = optparse.OptionParser(usage="%prog [OPTIONS] BASEURL METABOOK")
default_url = 'http://localhost:8899/'
parser.add_option('-u', '--url',
help='URL of HTTP interface to mw-serve (default: %r)' % default_url,
default=default_url,
)
parser.add_option('-w', '--writer',
help='writer to use for rendering (default: rl)',
default='rl',
)
parser.add_option('--max-render-time',
help='maximum number of seconds rendering may take (default: 120)',
default='120',
metavar='SECONDS',
)
parser.add_option('--save-output',
help='if specified, save rendered file with given filename',
metavar='FILENAME',
)
parser.add_option('-l', '--logfile',
help='log output to LOGFILE',
)
parser.add_option('--report-from-mail',
help='sender of error mails (--report-recipient also needed)',
metavar='EMAIL',
)
parser.add_option('--report-recipient',
help='recipient of error mails (--report-from-mail also needed)',
metavar='EMAIL',
)
options, args = parser.parse_args()
if len(args) != 2:
parser.error('exactly 2 arguments required')
base_url = args[0]
metabook = open(args[1], 'rb').read()
max_render_time = int(options.max_render_time)
if options.report_recipient and options.report_from_mail:
def report(msg):
utils.report(
system='mw-check-service',
subject='mw-check-service error',
from_email=options.report_from_mail.encode('utf-8'),
mail_recipients=[options.report_recipient.encode('utf-8')],
msg=msg,
)
else:
report = log.ERROR
writer = options.writer
if options.logfile:
utils.start_logging(options.logfile)
client = Client(options.url)
def check_req(command, **kwargs):
try:
success = client.request(command, kwargs, is_json=(command != 'download'))
except Exception, exc:
report('request failed: %s' % exc)
sys.exit(1)
if success:
return client.response
if client.error is not None:
report('request failed: %s' % client.error)
sys.exit(1)
else:
report('request failed: got response code %d' % client.response_code)
sys.exit(1)
except Exception, e:
print 'ERROR posting status %r to %r' % (status, posturl)
def post_progress(progress):
print 'progress', progress
if not posturl:
return
try:
return urllib2.urlopen(
posturl, urllib.urlencode({'progress': int(progress)})).read()
except Exception, e:
print 'ERROR posting progress %r to %r' % (progress, posturl)
try:
if options.logfile:
utils.start_logging(options.logfile)
output = options.output
from mwlib import wiki, recorddb, metabook
mb = metabook.MetaBook()
if conf:
from ConfigParser import ConfigParser
w = wiki.makewiki(conf)
cp = ConfigParser()
cp.read(conf)
license = {'name': cp.get('wiki', 'defaultarticlelicense')}
if license['name'] is not None:
license['wikitext'] = w['wiki'].getRawArticle(license['name'])
def check_service():
import sys
import time
from mwlib.client import Client
from mwlib.log import Log
from mwlib import utils
log = Log('mw-check-service')
parser = optparse.OptionParser(usage="%prog [OPTIONS] BASEURL METABOOK")
default_url = 'http://localhost:8899/'
parser.add_option('-u', '--url',
help='URL of HTTP interface to mw-serve (default: %r)' % default_url,
default=default_url,
)
parser.add_option('-w', '--writer',
help='writer to use for rendering (default: rl)',
default='rl',
)
parser.add_option('--max-render-time',
help='maximum number of seconds rendering may take (default: 120)',
default='120',
metavar='SECONDS',
)
parser.add_option('--save-output',
help='if specified, save rendered file with given filename',
metavar='FILENAME',
)
parser.add_option('-l', '--logfile',
help='log output to LOGFILE',
)
parser.add_option('--report-from-mail',
help='sender of error mails (--report-recipient also needed)',
metavar='EMAIL',
)
parser.add_option('--report-recipient',
help='recipient of error mails (--report-from-mail also needed)',
metavar='EMAIL',
)
options, args = parser.parse_args()
if len(args) != 2:
parser.error('exactly 2 arguments required')
base_url = args[0]
metabook = open(args[1], 'rb').read()
max_render_time = int(options.max_render_time)
if options.report_recipient and options.report_from_mail:
def report(msg):
utils.report(
system='mw-check-service',
subject='mw-check-service error',
from_email=options.report_from_mail.encode('utf-8'),
mail_recipients=[options.report_recipient.encode('utf-8')],
msg=msg,
)
else:
report = log.ERROR
writer = options.writer
if options.logfile:
utils.start_logging(options.logfile)
client = Client(options.url)
def check_req(command, **kwargs):
try:
success = client.request(command, kwargs, is_json=(command != 'download'))
except Exception, exc:
report('request failed: %s' % exc)
sys.exit(1)
if success:
return client.response
if client.error is not None:
report('request failed: %s' % client.error)
sys.exit(1)
else:
report('request failed: got response code %d' % client.response_code)
sys.exit(1)
def post():
parser = optparse.OptionParser(usage="%prog OPTIONS")
parser.add_option("-i", "--input", help="ZIP file to POST")
parser.add_option(
'-l',
'--logfile',
help='log output to LOGFILE',
)
parser.add_option("-p", "--posturl", help="HTTP POST ZIP file to POSTURL")
parser.add_option(
"-g",
"--getposturl",
help='get POST URL from PediaPress.com, open upload page in webbrowser',
action='store_true',
)
parser.add_option("-d",
"--daemonize",
action="store_true",
help='become a daemon process as soon as possible')
parser.add_option(
'--pid-file',
help='write PID of daemonized process to this file',
)
options, args = parser.parse_args()
use_help = 'Use --help for usage information.'
if not options.input:
parser.error('Specify --input.\n' + use_help)
if (options.posturl and options.getposturl)\
or (not options.posturl and not options.getposturl):
parser.error('Specify either --posturl or --getposturl.\n' + use_help)
if options.posturl:
from mwlib.podclient import PODClient
podclient = PODClient(options.posturl)
elif options.getposturl:
import webbrowser
from mwlib.podclient import podclient_from_serviceurl
podclient = podclient_from_serviceurl(
'http://pediapress.com/api/collections/')
webbrowser.open(podclient.redirecturl)
from mwlib import utils
from mwlib.status import Status
if options.logfile:
utils.start_logging(options.logfile)
if options.daemonize:
utils.daemonize()
if options.pid_file:
open(options.pid_file, 'wb').write('%d\n' % os.getpid())
status = Status(podclient=podclient)
try:
try:
status(status='uploading', progress=0)
podclient.post_zipfile(options.input)
status(status='finished', progress=100)
except Exception, e:
status(status='error')
raise
finally:
if options.pid_file:
utils.safe_unlink(options.pid_file)
def check_service():
import sys
import time
from mwlib.client import Client
from mwlib.log import Log
from mwlib import utils
log = Log('mw-check-service')
parser = optparse.OptionParser(usage="%prog [OPTIONS] BASEURL METABOOK")
default_url = 'http://localhost:8899/'
parser.add_option(
'-u',
'--url',
help='URL of HTTP interface to mw-serve (default: %r)' % default_url,
default=default_url,
)
parser.add_option(
'-w',
'--writer',
help='writer to use for rendering (default: rl)',
default='rl',
)
parser.add_option(
'--max-render-time',
help='maximum number of seconds rendering may take (default: 120)',
default='120',
metavar='SECONDS',
)
parser.add_option(
'--save-output',
help='if specified, save rendered file with given filename',
metavar='FILENAME',
)
parser.add_option(
'-l',
'--logfile',
help='log output to LOGFILE',
)
parser.add_option(
'--report-from-mail',
help='sender of error mails (--report-recipient also needed)',
metavar='EMAIL',
)
parser.add_option(
'--report-recipient',
help='recipient of error mails (--report-from-mail also needed)',
metavar='EMAIL',
)
options, args = parser.parse_args()
if len(args) != 2:
parser.error('exactly 2 arguments required')
base_url = args[0]
metabook = open(args[1], 'rb').read()
max_render_time = int(options.max_render_time)
if options.report_recipient and options.report_from_mail:
def report(msg):
utils.report(
system='mw-check-service',
subject='mw-check-service error',
from_email=options.report_from_mail.encode('utf-8'),
mail_recipients=[options.report_recipient.encode('utf-8')],
msg=msg,
)
else:
report = log.ERROR
writer = options.writer
if options.logfile:
utils.start_logging(options.logfile)
client = Client(options.url)
def check_req(command, **kwargs):
try:
success = client.request(command,
kwargs,
is_json=(command != 'download'))
except Exception as exc:
report('request failed: %s' % exc)
sys.exit(1)
if success:
return client.response
if client.error is not None:
report('request failed: %s' % client.error)
sys.exit(1)
else:
report('request failed: got response code %d' %
client.response_code)
sys.exit(1)
start_time = time.time()
log.info('sending render command')
response = check_req(
'render',
base_url=base_url,
metabook=metabook,
writer=writer,
force_render=True,
)
collection_id = response['collection_id']
while True:
time.sleep(1)
if time.time() - start_time > max_render_time:
report('rendering exceeded allowed time of %d s' % max_render_time)
sys.exit(2)
log.info('checking status')
response = check_req(
'render_status',
collection_id=collection_id,
writer=writer,
)
if response['state'] == 'finished':
break
log.info('downloading')
response = check_req(
'download',
collection_id=collection_id,
writer=writer,
)
if len(response) < 100:
report('got suspiciously small file from download: size is %d Bytes' %
len(response))
sys.exit(3)
log.info('resulting file is %d Bytes' % len(response))
if options.save_output:
log.info('saving to %r' % options.save_output)
open(options.save_output, 'wb').write(response)
render_time = time.time() - start_time
log.info('rendering ok, took %fs' % render_time)
def watch():
parser = optparse.OptionParser(usage="%prog [OPTIONS]")
parser.add_option(
'-l',
'--logfile',
help='log output to LOGFILE',
)
parser.add_option(
'-d',
'--daemonize',
action='store_true',
help='become daemon as soon as possible',
)
parser.add_option(
'--pid-file',
help='write PID of daemonized process to this file',
)
parser.add_option(
'-q',
'--queue-dir',
help=
'queue directory, where new job files are written to (default: /var/cache/mw-watch/q/)',
default='/var/cache/mw-watch/q/',
)
parser.add_option(
'-p',
'--processing-dir',
help=
'processing directory, where active job files are moved to (must be on same filesystem as --queue-dir, default: /var/cache/mw-watch/p/)',
default='/var/cache/mw-watch/p/',
)
parser.add_option(
'-n',
'--num-jobs',
help='maximum number of simulataneous jobs (default: 5)',
default='5',
)
options, args = parser.parse_args()
try:
options.num_jobs = int(options.num_jobs)
except ValueError:
parser.error('--num-jobs value must be an integer')
from mwlib import filequeue, utils
if options.logfile:
utils.start_logging(options.logfile)
if options.daemonize:
utils.daemonize()
if options.pid_file:
open(options.pid_file, 'wb').write('%d\n' % os.getpid())
poller = filequeue.FileJobPoller(
queue_dir=options.queue_dir,
processing_dir=options.processing_dir,
max_num_jobs=options.num_jobs,
).run_forever()
if options.pid_file:
utils.safe_unlink(options.pid_file)
def serve():
from SocketServer import ForkingMixIn, ThreadingMixIn
from wsgiref.simple_server import make_server, WSGIServer
from flup.server import fcgi, fcgi_fork, scgi, scgi_fork
class ForkingWSGIServer(ForkingMixIn, WSGIServer):
pass
class ThreadingWSGIServer(ThreadingMixIn, WSGIServer):
pass
proto2server = {
'http': ForkingWSGIServer,
'http_threaded': ThreadingWSGIServer,
'fcgi': fcgi_fork.WSGIServer,
'fcgi_threaded': fcgi.WSGIServer,
'scgi': scgi_fork.WSGIServer,
'scgi_threaded': scgi.WSGIServer,
}
parser = optparse.OptionParser(usage="%prog [OPTIONS]")
parser.add_option(
'-l',
'--logfile',
help='log output to LOGFILE',
)
parser.add_option(
'-d',
'--daemonize',
action='store_true',
help='become daemon as soon as possible',
)
parser.add_option(
'--pid-file',
help='write PID of daemonized process to this file',
)
parser.add_option(
'-P',
'--protocol',
help='one of %s (default: http)' % ', '.join(proto2server.keys()),
default='http',
)
parser.add_option(
'-p',
'--port',
help='port to listen on (default: 8899)',
default='8899',
)
parser.add_option(
'-i',
'--interface',
help='interface to listen on (default: 0.0.0.0)',
default='0.0.0.0',
)
parser.add_option(
'--cache-dir',
help='cache directory (default: /var/cache/mw-serve/)',
default='/var/cache/mw-serve/',
)
parser.add_option(
'--mwrender',
help='(path to) mw-render executable',
default='mw-render',
)
parser.add_option(
'--mwrender-logfile',
help='global logfile for mw-render',
metavar='LOGFILE',
)
parser.add_option(
'--mwzip',
help='(path to) mw-zip executable',
default='mw-zip',
)
parser.add_option(
'--mwzip-logfile',
help='global logfile for mw-zip',
metavar='LOGFILE',
)
parser.add_option(
'--mwpost',
help='(path to) mw-post executable',
default='mw-post',
)
parser.add_option(
'--mwpost-logfile',
help='global logfile for mw-post',
metavar='LOGFILE',
)
parser.add_option(
'-q',
'--queue-dir',
help='queue dir of mw-watch (if not specified, no queue is used)',
)
parser.add_option(
'-m',
'--method',
help='prefork or threaded (default: prefork)',
default='prefork',
)
parser.add_option(
'--max-requests',
help=
'maximum number of requests a child process can handle before it is killed, irrelevant for --method=threaded (default: 0 = no limit)',
default='0',
metavar='NUM',
)
parser.add_option(
'--min-spare',
help='minimum number of spare processes/threads (default: 2)',
default='2',
metavar='NUM',
)
parser.add_option(
'--max-spare',
help='maximum number of spare processes/threads (default: 5)',
default='5',
metavar='NUM',
)
parser.add_option(
'--max-children',
help='maximum number of processes/threads (default: 50)',
default='50',
metavar='NUM',
)
parser.add_option(
'--report-from-mail',
help='sender of error mails (--report-recipient also needed)',
metavar='EMAIL',
)
parser.add_option(
'--report-recipient',
help='recipient of error mails (--report-from-mail also needed)',
metavar='EMAIL',
)
parser.add_option(
'--clean-cache',
help=
'clean cache files that have not been touched for at least HOURS hours and exit',
metavar='HOURS',
)
options, args = parser.parse_args()
if options.clean_cache:
try:
options.clean_cache = int(options.clean_cache)
except ValueError:
parser.error('--clean-cache value must be an integer')
from mwlib.serve import clean_cache
clean_cache(options.clean_cache * 60 * 60, cache_dir=options.cache_dir)
return
if options.protocol not in proto2server:
parser.error('unsupported protocol (must be one of %s)' %
(', '.join(proto2server.keys()), ))
def to_int(opt_name):
try:
setattr(options, opt_name, int(getattr(options, opt_name)))
except ValueError:
parser.error('--%s value must be an integer' %
opt_name.replace('_', '-'))
to_int('port')
to_int('max_requests')
to_int('min_spare')
to_int('max_spare')
to_int('max_children')
if options.method not in ('prefork', 'threaded'):
parser.error(
'the only supported values for --method are "prefork" and "threaded"'
)
from mwlib import serve, log, utils
log = log.Log('mw-serve')
if options.logfile:
utils.start_logging(options.logfile)
if options.daemonize:
utils.daemonize()
if options.pid_file:
open(options.pid_file, 'wb').write('%d\n' % os.getpid())
if options.method == 'threaded':
options.protocol += '_threaded'
flup_kwargs = {
'maxThreads': options.max_children,
}
else:
flup_kwargs = {
'maxChildren': options.max_children,
'maxRequests': options.max_requests,
}
log.info("serving %s on %s:%s" %
(options.protocol, options.interface, options.port))
if options.report_recipient and options.report_from_mail:
report_from_mail = options.report_from_mail.encode('utf-8')
report_recipients = [options.report_recipient.encode('utf-8')]
else:
report_from_mail = None
report_recipients = None
app = serve.Application(
cache_dir=options.cache_dir,
mwrender_cmd=options.mwrender,
mwrender_logfile=options.mwrender_logfile,
mwzip_cmd=options.mwzip,
mwzip_logfile=options.mwzip_logfile,
mwpost_cmd=options.mwpost,
mwpost_logfile=options.mwpost_logfile,
queue_dir=options.queue_dir,
report_from_mail=report_from_mail,
report_recipients=report_recipients,
)
if options.protocol.startswith('http'):
server = make_server(
options.interface,
options.port,
app,
server_class=proto2server[options.protocol],
)
try:
server.serve_forever()
except KeyboardInterrupt:
pass
else:
serverclass = proto2server[options.protocol]
serverclass(app,
bindAddress=(options.interface, options.port),
minSpare=options.min_spare,
maxSpare=options.max_spare,
**flup_kwargs).run()
if options.pid_file:
utils.safe_unlink(options.pid_file)
log.info('exit.')
