import subprocess
import sys
try:
import simplejson as json
except ImportError:
import json
from mwlib import mwapidb, utils, log, bookshelf
import mwlib.metabook
RENDER_TIMEOUT_DEFAULT = 60 * 60 # 60 minutes
LICENSE_URL = 'http://en.wikipedia.org/w/index.php?title=Wikipedia:Text_of_the_GNU_Free_Documentation_License&action=raw'
system = 'mw-serve-stresser'
log = log.Log('mw-serve-stresser')
# disable fetch cache
utils.fetch_url_orig = utils.fetch_url
def fetch_url(*args, **kargs):
kargs["fetch_cache"] = {}
return utils.fetch_url_orig(*args, **kargs)
utils.fetch_url = fetch_url
writer_options = {
'rl': 'strict',
}
import optparse
import json
from mwlib.utils import start_logging
from mwlib import wiki, metabook, log
log = log.Log('mwlib.options')
class OptionParser(optparse.OptionParser):
def __init__(self, usage=None, config_optional=False):
self.config_optional = config_optional
if usage is None:
usage = '%prog [OPTIONS] [ARTICLETITLE...]'
optparse.OptionParser.__init__(self, usage=usage)
self.metabook = None
self.add_option(
"-c",
"--config",
help="configuration file, ZIP file or base URL",
)
self.add_option(
"-i",
"--imagesize",
help="max. pixel size (width or height) for images (default: 800)",
default=800,
)
self.add_option(
"-m",
"--metabook",
help="JSON encoded text file with article collection",
from __future__ import division
import re
import math
from mwlib import log
from mwlib.advtree import Text, ItemList, Table, Row, Cell
from mwlib.writer import styleutils
from reportlab.lib import colors
from customflowables import Figure
#import debughelper
from mwlib.rl import pdfstyles
from mwlib import advtree
log = log.Log('rlwriter')
def scaleImages(data):
for row in data:
for cell in row:
for (i,e) in enumerate(cell):
if isinstance(e,Figure): # scale image to half size
cell[i] = Figure(imgFile = e.imgPath, captionTxt=e.captionTxt, captionStyle=e.cs, imgWidth=e.imgWidth/2.0,imgHeight=e.imgHeight/2.0, margin=e.margin, padding=e.padding,align=e.align)
def getColWidths(data, table=None, recursionDepth=0, nestingLevel=1):
"""
the widths for the individual columns are calculated. if the horizontal size exceeds the pagewidth
the fontsize is reduced
"""
import gevent.monkey
if __name__ == "__main__":
gevent.monkey.patch_all()
import sys, re, StringIO, urllib2, urlparse, traceback, urllib, unicodedata
from hashlib import md5
from gevent import pool, pywsgi
from qs.misc import call_in_loop
from mwlib import myjson as json, log, _version
from mwlib.metabook import calc_checksum
from mwlib.async import rpcclient
log = log.Log('mwlib.serve')
class bunch(object):
def __init__(self, **kw):
self.__dict__.update(kw)
def __repr__(self):
return "bunch(%s)" % (", ".join(["%s=%r" % (k, v) for k, v in self.__dict__.items()]), )
# -- we try to load all writers here but also keep a list of known writers
# -- these writers do not have to be installed on the machine that's running the server
# -- and we also like to speedup the get_writers method
name2writer = {'odf': bunch(file_extension='odt', name='odf', content_type='application/vnd.oasis.opendocument.text'),
# Copyright (c) 2007-2009 PediaPress GmbH
# See README.rst for additional licensing information.
import os
import tempfile
import shutil
from subprocess import Popen, PIPE
try:
import xml.etree.ElementTree as ET
except:
from elementtree import ElementTree as ET
from mwlib import log
log = log.Log('mwlib.mathutils')
def try_system(cmd):
n = os.path.devnull
cmd += " >%s 2>%s" % (n, n)
return os.system(cmd)
texvc_available = not try_system("texvc")
blahtexml_available = not try_system("blahtexml")
def _renderMathBlahtex(latex, output_path, output_mode):
if not blahtexml_available:
return None
#! /usr/bin/env python
# Copyright (c) 2007-2008 PediaPress GmbH
# See README.txt for additional licensing information.
import urllib
from mwlib import parser, log, metabook
# import functions needed by most writers that should be accessible through writerbase
from mathutils import renderMath
log = log.Log('mwlib.writerbase')
class WriterError(RuntimeError):
pass
def build_book(env, status_callback=None, progress_range=None):
book = parser.Book()
if status_callback is not None:
progress = progress_range[0]
num_articles = float(len(metabook.get_item_list(env.metabook,
filter_type='article',
)))
if num_articles > 0:
progress_step = int(
(progress_range[1] - progress_range[0])/num_articles
)
for item in metabook.get_item_list(env.metabook):
if item['type'] == 'chapter':
book.children.append(parser.Chapter(item['title'].strip()))
def serve():
from SocketServer import ForkingMixIn, ThreadingMixIn
from wsgiref.simple_server import make_server, WSGIServer
from flup.server import fcgi, fcgi_fork, scgi, scgi_fork
class ForkingWSGIServer(ForkingMixIn, WSGIServer):
pass
class ThreadingWSGIServer(ThreadingMixIn, WSGIServer):
pass
proto2server = {
'http': ForkingWSGIServer,
'http_threaded': ThreadingWSGIServer,
'fcgi': fcgi_fork.WSGIServer,
'fcgi_threaded': fcgi.WSGIServer,
'scgi': scgi_fork.WSGIServer,
'scgi_threaded': scgi.WSGIServer,
}
parser = optparse.OptionParser(usage="%prog [OPTIONS]")
parser.add_option(
'-l',
'--logfile',
help='log output to LOGFILE',
)
parser.add_option(
'-d',
'--daemonize',
action='store_true',
help='become daemon as soon as possible',
)
parser.add_option(
'--pid-file',
help='write PID of daemonized process to this file',
)
parser.add_option(
'-P',
'--protocol',
help='one of %s (default: http)' % ', '.join(proto2server.keys()),
default='http',
)
parser.add_option(
'-p',
'--port',
help='port to listen on (default: 8899)',
default='8899',
)
parser.add_option(
'-i',
'--interface',
help='interface to listen on (default: 0.0.0.0)',
default='0.0.0.0',
)
parser.add_option(
'--cache-dir',
help='cache directory (default: /var/cache/mw-serve/)',
default='/var/cache/mw-serve/',
)
parser.add_option(
'--mwrender',
help='(path to) mw-render executable',
default='mw-render',
)
parser.add_option(
'--mwrender-logfile',
help='global logfile for mw-render',
metavar='LOGFILE',
)
parser.add_option(
'--mwzip',
help='(path to) mw-zip executable',
default='mw-zip',
)
parser.add_option(
'--mwzip-logfile',
help='global logfile for mw-zip',
metavar='LOGFILE',
)
parser.add_option(
'--mwpost',
help='(path to) mw-post executable',
default='mw-post',
)
parser.add_option(
'--mwpost-logfile',
help='global logfile for mw-post',
metavar='LOGFILE',
)
parser.add_option(
'-q',
'--queue-dir',
help='queue dir of mw-watch (if not specified, no queue is used)',
)
parser.add_option(
'-m',
'--method',
help='prefork or threaded (default: prefork)',
default='prefork',
)
parser.add_option(
'--max-requests',
help=
'maximum number of requests a child process can handle before it is killed, irrelevant for --method=threaded (default: 0 = no limit)',
default='0',
metavar='NUM',
)
parser.add_option(
'--min-spare',
help='minimum number of spare processes/threads (default: 2)',
default='2',
metavar='NUM',
)
parser.add_option(
'--max-spare',
help='maximum number of spare processes/threads (default: 5)',
default='5',
metavar='NUM',
)
parser.add_option(
'--max-children',
help='maximum number of processes/threads (default: 50)',
default='50',
metavar='NUM',
)
parser.add_option(
'--report-from-mail',
help='sender of error mails (--report-recipient also needed)',
metavar='EMAIL',
)
parser.add_option(
'--report-recipient',
help='recipient of error mails (--report-from-mail also needed)',
metavar='EMAIL',
)
parser.add_option(
'--clean-cache',
help=
'clean cache files that have not been touched for at least HOURS hours and exit',
metavar='HOURS',
)
options, args = parser.parse_args()
if options.clean_cache:
try:
options.clean_cache = int(options.clean_cache)
except ValueError:
parser.error('--clean-cache value must be an integer')
from mwlib.serve import clean_cache
clean_cache(options.clean_cache * 60 * 60, cache_dir=options.cache_dir)
return
if options.protocol not in proto2server:
parser.error('unsupported protocol (must be one of %s)' %
(', '.join(proto2server.keys()), ))
def to_int(opt_name):
try:
setattr(options, opt_name, int(getattr(options, opt_name)))
except ValueError:
parser.error('--%s value must be an integer' %
opt_name.replace('_', '-'))
to_int('port')
to_int('max_requests')
to_int('min_spare')
to_int('max_spare')
to_int('max_children')
if options.method not in ('prefork', 'threaded'):
parser.error(
'the only supported values for --method are "prefork" and "threaded"'
)
from mwlib import serve, log, utils
log = log.Log('mw-serve')
if options.logfile:
utils.start_logging(options.logfile)
if options.daemonize:
utils.daemonize()
if options.pid_file:
open(options.pid_file, 'wb').write('%d\n' % os.getpid())
if options.method == 'threaded':
options.protocol += '_threaded'
flup_kwargs = {
'maxThreads': options.max_children,
}
else:
flup_kwargs = {
'maxChildren': options.max_children,
'maxRequests': options.max_requests,
}
log.info("serving %s on %s:%s" %
(options.protocol, options.interface, options.port))
if options.report_recipient and options.report_from_mail:
report_from_mail = options.report_from_mail.encode('utf-8')
report_recipients = [options.report_recipient.encode('utf-8')]
else:
report_from_mail = None
report_recipients = None
app = serve.Application(
cache_dir=options.cache_dir,
mwrender_cmd=options.mwrender,
mwrender_logfile=options.mwrender_logfile,
mwzip_cmd=options.mwzip,
mwzip_logfile=options.mwzip_logfile,
mwpost_cmd=options.mwpost,
mwpost_logfile=options.mwpost_logfile,
queue_dir=options.queue_dir,
report_from_mail=report_from_mail,
report_recipients=report_recipients,
)
if options.protocol.startswith('http'):
server = make_server(
options.interface,
options.port,
app,
server_class=proto2server[options.protocol],
)
try:
server.serve_forever()
except KeyboardInterrupt:
pass
else:
serverclass = proto2server[options.protocol]
serverclass(app,
bindAddress=(options.interface, options.port),
minSpare=options.min_spare,
maxSpare=options.max_spare,
**flup_kwargs).run()
if options.pid_file:
utils.safe_unlink(options.pid_file)
log.info('exit.')