def post():
parser = optparse.OptionParser(usage="%prog OPTIONS")
parser.add_option("-i", "--input", help="ZIP file to POST")
parser.add_option('-l', '--logfile',
help='log output to LOGFILE',
)
parser.add_option("-p", "--posturl", help="HTTP POST ZIP file to POSTURL")
parser.add_option("-g", "--getposturl",
help='get POST URL from PediaPress.com, open upload page in webbrowser',
action='store_true',
)
parser.add_option("-d", "--daemonize", action="store_true",
help='become a daemon process as soon as possible')
parser.add_option('--pid-file',
help='write PID of daemonized process to this file',
)
options, args = parser.parse_args()
use_help = 'Use --help for usage information.'
if not options.input:
parser.error('Specify --input.\n' + use_help)
if (options.posturl and options.getposturl)\
or (not options.posturl and not options.getposturl):
parser.error('Specify either --posturl or --getposturl.\n' + use_help)
if options.posturl:
from mwlib.podclient import PODClient
podclient = PODClient(options.posturl)
elif options.getposturl:
import webbrowser
from mwlib.podclient import podclient_from_serviceurl
podclient = podclient_from_serviceurl('http://pediapress.com/api/collections/')
webbrowser.open(podclient.redirecturl)
from mwlib import utils
from mwlib.status import Status
if options.logfile:
utils.start_logging(options.logfile)
if options.daemonize:
utils.daemonize()
if options.pid_file:
open(options.pid_file, 'wb').write('%d\n' % os.getpid())
status = Status(podclient=podclient)
try:
try:
status(status='uploading', progress=0)
podclient.post_zipfile(options.input)
status(status='finished', progress=100)
except Exception, e:
status(status='error')
raise
finally:
if options.pid_file:
utils.safe_unlink(options.pid_file)
def watch():
parser = optparse.OptionParser(usage="%prog [OPTIONS]")
parser.add_option('-l', '--logfile',
help='log output to LOGFILE',
)
parser.add_option('-d', '--daemonize',
action='store_true',
help='become daemon as soon as possible',
)
parser.add_option('--pid-file',
help='write PID of daemonized process to this file',
)
parser.add_option('-q', '--queue-dir',
help='queue directory, where new job files are written to (default: /var/cache/mw-watch/q/)',
default='/var/cache/mw-watch/q/',
)
parser.add_option('-p', '--processing-dir',
help='processing directory, where active job files are moved to (must be on same filesystem as --queue-dir, default: /var/cache/mw-watch/p/)',
default='/var/cache/mw-watch/p/',
)
parser.add_option('-n', '--num-jobs',
help='maximum number of simulataneous jobs (default: 5)',
default='5',
)
options, args = parser.parse_args()
try:
options.num_jobs = int(options.num_jobs)
except ValueError:
parser.error('--num-jobs value must be an integer')
from mwlib import filequeue, utils
if options.logfile:
utils.start_logging(options.logfile)
if options.daemonize:
utils.daemonize()
if options.pid_file:
open(options.pid_file, 'wb').write('%d\n' % os.getpid())
poller = filequeue.FileJobPoller(
queue_dir=options.queue_dir,
processing_dir=options.processing_dir,
max_num_jobs=options.num_jobs,
).run_forever()
if options.pid_file:
utils.safe_unlink(options.pid_file)
from mwlib import utils, wiki
filename = None
status = None
try:
env = parser.makewiki()
assert env.metabook, "no metabook"
from mwlib.status import Status
status = Status(options.status_file,
podclient=podclient,
progress_range=(1, 90))
status(progress=0)
output = options.output
make_zip(output,
options,
env.metabook,
podclient=podclient,
status=status)
except Exception, e:
if status:
status(status='error')
raise
finally:
if options.output is None and filename is not None:
print 'removing %r' % filename
utils.safe_unlink(filename)
def make_zip_file(
output,
env,
status=None,
num_threads=10,
imagesize=800,
):
if status is None:
status = lambda **kwargs: None
if output is None:
fd, output = tempfile.mkstemp(suffix='.zip')
os.close(fd)
fd, tmpzip = tempfile.mkstemp(suffix='.zip', dir=os.path.dirname(output))
os.close(fd)
zf = zipfile.ZipFile(tmpzip, 'w')
try:
articles = metabook.get_item_list(env.metabook, filter_type='article')
if num_threads > 0:
z = ThreadedZipCreator(
zf,
imagesize=imagesize,
num_threads=num_threads,
status=status,
num_articles=len(articles),
)
else:
z = ZipCreator(
zf,
imagesize=imagesize,
status=status,
num_articles=len(articles),
)
# if articles:
# class IncProgress(object):
# inc = 100./len(articles)
# p = 0
# def __call__(self, title):
# self.p += self.inc
# status(progress=int(self.p), article=title)
# inc_progress = IncProgress()
# else:
# inc_progress = None
for item in articles:
d = mwapidb.parse_article_url(item['title'].encode('utf-8'))
if d is not None:
item['title'] = d['title']
item['revision'] = d['revision']
wikidb = mwapidb.WikiDB(api_helper=d['api_helper'])
imagedb = mwapidb.ImageDB(api_helper=d['api_helper'])
else:
wikidb = env.wiki
imagedb = env.images
z.addArticle(
item['title'],
revision=item.get('revision', None),
wikidb=wikidb,
imagedb=imagedb,
)
for license in env.get_licenses():
z.parseArticle(
title=license['title'],
raw=license['wikitext'],
wikidb=env.wiki,
imagedb=env.images,
)
z.join()
z.addObject('metabook.json', json.dumps(env.metabook))
zf.close()
if os.path.exists(output): # Windows...
os.unlink(output)
os.rename(tmpzip, output)
if env.images and hasattr(env.images, 'clear'):
env.images.clear()
status(progress=100)
return output
finally:
if os.path.exists(tmpzip):
utils.safe_unlink(tmpzip)
def buildzip():
from mwlib.options import OptionParser
parser = OptionParser()
parser.add_option("-o", "--output", help="write output to OUTPUT")
parser.add_option("-p", "--posturl", help="http post to POSTURL (directly)")
parser.add_option("-g", "--getposturl",
help='get POST URL from PediaPress.com, open upload page in webbrowser',
action='store_true',
)
options, args = parser.parse_args()
use_help = 'Use --help for usage information.'
if parser.metabook is None and options.collectionpage is None:
parser.error('Neither --metabook nor, --collectionpage or arguments specified.\n' + use_help)
if options.posturl and options.getposturl:
parser.error('Specify either --posturl or --getposturl.\n' + use_help)
if not options.posturl and not options.getposturl and not options.output:
parser.error('Neither --output, nor --posturl or --getposturl specified.\n' + use_help)
if options.posturl:
from mwlib.podclient import PODClient
podclient = PODClient(options.posturl)
elif options.getposturl:
import webbrowser
from mwlib.podclient import podclient_from_serviceurl
podclient = podclient_from_serviceurl('http://pediapress.com/api/collections/')
webbrowser.open(podclient.redirecturl)
else:
podclient = None
from mwlib import utils
if options.daemonize:
utils.daemonize()
if options.pid_file:
open(options.pid_file, 'wb').write('%d\n' % os.getpid())
filename = None
status = None
try:
try:
env = parser.makewiki()
from mwlib.status import Status
from mwlib import zipcreator
status = Status(podclient=podclient, progress_range=(1, 90))
status(progress=0)
filename = zipcreator.make_zip_file(options.output, env,
status=status,
num_threads=options.num_threads,
imagesize=options.imagesize,
)
status = Status(podclient=podclient, progress_range=(91, 100))
if podclient:
status(status='uploading', progress=0)
podclient.post_zipfile(filename)
status(status='finished', progress=100)
except Exception, e:
if status:
status(status='error')
raise
finally:
if options.output is None and filename is not None:
print 'removing %r' % filename
utils.safe_unlink(filename)
if options.pid_file:
utils.safe_unlink(options.pid_file)
response = {
'collection_id': collection_id,
'writer': writer,
'is_cached': False,
}
pid_path = self.get_path(collection_id, self.pid_filename, writer)
if os.path.exists(pid_path):
log.info('mw-render already running for collection %r' % collection_id)
return response
output_path = self.get_path(collection_id, self.output_filename, writer)
if os.path.exists(output_path):
if force_render:
log.info('removing rendered file %r (forced rendering)' % output_path)
utils.safe_unlink(output_path)
else:
log.info('re-using rendered file %r' % output_path)
response['is_cached'] = True
return response
error_path = self.get_path(collection_id, self.error_filename, writer)
if os.path.exists(error_path):
log.info('removing error file %r' % error_path)
utils.safe_unlink(error_path)
force_render = True
status_path = self.get_path(collection_id, self.status_filename, writer)
if os.path.exists(status_path):
if force_render:
log.info('removing status file %r (forced rendering)' % status_path)
def main():
from gevent import monkey
monkey.patch_all(thread=False)
from mwlib.options import OptionParser
from mwlib import conf
parser = OptionParser()
parser.add_option("-o", "--output", help="write output to OUTPUT")
parser.add_option("-p", "--posturl", help="http post to POSTURL (directly)")
parser.add_option("-g", "--getposturl",
help='get POST URL from PediaPress.com, open upload page in webbrowser',
action='count',
)
parser.add_option('--keep-tmpfiles',
action='store_true',
default=False,
help="don't remove temporary files like images",
)
parser.add_option("-s", "--status-file",
help='write status/progress info to this file')
options, args = parser.parse_args()
conf.readrc()
use_help = 'Use --help for usage information.'
if parser.metabook is None and options.collectionpage is None:
parser.error(
'Neither --metabook nor, --collectionpage or arguments specified.\n' +
use_help)
if options.posturl and options.getposturl:
parser.error('Specify either --posturl or --getposturl.\n' + use_help)
if not options.posturl and not options.getposturl and not options.output:
parser.error('Neither --output, nor --posturl or --getposturl specified.\n' + use_help)
if options.posturl:
from mwlib.podclient import PODClient
podclient = PODClient(options.posturl)
elif options.getposturl:
if options.getposturl > 1:
serviceurl = 'http://test.pediapress.com/api/collections/'
else:
serviceurl = 'http://pediapress.com/api/collections/'
import webbrowser
from mwlib.podclient import podclient_from_serviceurl
podclient = podclient_from_serviceurl(serviceurl)
pid = os.fork()
if not pid:
try:
webbrowser.open(podclient.redirecturl)
finally:
os._exit(0)
import time
time.sleep(1)
try:
os.kill(pid, 9)
except BaseException:
pass
else:
podclient = None
from mwlib import utils, wiki
filename = None
status = None
try:
env = parser.makewiki()
assert env.metabook, "no metabook"
from mwlib.status import Status
status = Status(options.status_file, podclient=podclient, progress_range=(1, 90))
status(progress=0)
output = options.output
make_zip(output, options, env.metabook, podclient=podclient, status=status)
except Exception as e:
if status:
status(status='error')
raise
finally:
if options.output is None and filename is not None:
print 'removing %r' % filename
utils.safe_unlink(filename)
def serve():
from SocketServer import ForkingMixIn, ThreadingMixIn
from wsgiref.simple_server import make_server, WSGIServer
from flup.server import fcgi, fcgi_fork, scgi, scgi_fork
class ForkingWSGIServer(ForkingMixIn, WSGIServer):
pass
class ThreadingWSGIServer(ThreadingMixIn, WSGIServer):
pass
proto2server = {
'http': ForkingWSGIServer,
'http_threaded': ThreadingWSGIServer,
'fcgi': fcgi_fork.WSGIServer,
'fcgi_threaded': fcgi.WSGIServer,
'scgi': scgi_fork.WSGIServer,
'scgi_threaded': scgi.WSGIServer,
}
parser = optparse.OptionParser(usage="%prog [OPTIONS]")
parser.add_option(
'-l',
'--logfile',
help='log output to LOGFILE',
)
parser.add_option(
'-d',
'--daemonize',
action='store_true',
help='become daemon as soon as possible',
)
parser.add_option(
'--pid-file',
help='write PID of daemonized process to this file',
)
parser.add_option(
'-P',
'--protocol',
help='one of %s (default: http)' % ', '.join(proto2server.keys()),
default='http',
)
parser.add_option(
'-p',
'--port',
help='port to listen on (default: 8899)',
default='8899',
)
parser.add_option(
'-i',
'--interface',
help='interface to listen on (default: 0.0.0.0)',
default='0.0.0.0',
)
parser.add_option(
'--cache-dir',
help='cache directory (default: /var/cache/mw-serve/)',
default='/var/cache/mw-serve/',
)
parser.add_option(
'--mwrender',
help='(path to) mw-render executable',
default='mw-render',
)
parser.add_option(
'--mwrender-logfile',
help='global logfile for mw-render',
metavar='LOGFILE',
)
parser.add_option(
'--mwzip',
help='(path to) mw-zip executable',
default='mw-zip',
)
parser.add_option(
'--mwzip-logfile',
help='global logfile for mw-zip',
metavar='LOGFILE',
)
parser.add_option(
'--mwpost',
help='(path to) mw-post executable',
default='mw-post',
)
parser.add_option(
'--mwpost-logfile',
help='global logfile for mw-post',
metavar='LOGFILE',
)
parser.add_option(
'-q',
'--queue-dir',
help='queue dir of mw-watch (if not specified, no queue is used)',
)
parser.add_option(
'-m',
'--method',
help='prefork or threaded (default: prefork)',
default='prefork',
)
parser.add_option(
'--max-requests',
help=
'maximum number of requests a child process can handle before it is killed, irrelevant for --method=threaded (default: 0 = no limit)',
default='0',
metavar='NUM',
)
parser.add_option(
'--min-spare',
help='minimum number of spare processes/threads (default: 2)',
default='2',
metavar='NUM',
)
parser.add_option(
'--max-spare',
help='maximum number of spare processes/threads (default: 5)',
default='5',
metavar='NUM',
)
parser.add_option(
'--max-children',
help='maximum number of processes/threads (default: 50)',
default='50',
metavar='NUM',
)
parser.add_option(
'--report-from-mail',
help='sender of error mails (--report-recipient also needed)',
metavar='EMAIL',
)
parser.add_option(
'--report-recipient',
help='recipient of error mails (--report-from-mail also needed)',
metavar='EMAIL',
)
parser.add_option(
'--clean-cache',
help=
'clean cache files that have not been touched for at least HOURS hours and exit',
metavar='HOURS',
)
options, args = parser.parse_args()
if options.clean_cache:
try:
options.clean_cache = int(options.clean_cache)
except ValueError:
parser.error('--clean-cache value must be an integer')
from mwlib.serve import clean_cache
clean_cache(options.clean_cache * 60 * 60, cache_dir=options.cache_dir)
return
if options.protocol not in proto2server:
parser.error('unsupported protocol (must be one of %s)' %
(', '.join(proto2server.keys()), ))
def to_int(opt_name):
try:
setattr(options, opt_name, int(getattr(options, opt_name)))
except ValueError:
parser.error('--%s value must be an integer' %
opt_name.replace('_', '-'))
to_int('port')
to_int('max_requests')
to_int('min_spare')
to_int('max_spare')
to_int('max_children')
if options.method not in ('prefork', 'threaded'):
parser.error(
'the only supported values for --method are "prefork" and "threaded"'
)
from mwlib import serve, log, utils
log = log.Log('mw-serve')
if options.logfile:
utils.start_logging(options.logfile)
if options.daemonize:
utils.daemonize()
if options.pid_file:
open(options.pid_file, 'wb').write('%d\n' % os.getpid())
if options.method == 'threaded':
options.protocol += '_threaded'
flup_kwargs = {
'maxThreads': options.max_children,
}
else:
flup_kwargs = {
'maxChildren': options.max_children,
'maxRequests': options.max_requests,
}
log.info("serving %s on %s:%s" %
(options.protocol, options.interface, options.port))
if options.report_recipient and options.report_from_mail:
report_from_mail = options.report_from_mail.encode('utf-8')
report_recipients = [options.report_recipient.encode('utf-8')]
else:
report_from_mail = None
report_recipients = None
app = serve.Application(
cache_dir=options.cache_dir,
mwrender_cmd=options.mwrender,
mwrender_logfile=options.mwrender_logfile,
mwzip_cmd=options.mwzip,
mwzip_logfile=options.mwzip_logfile,
mwpost_cmd=options.mwpost,
mwpost_logfile=options.mwpost_logfile,
queue_dir=options.queue_dir,
report_from_mail=report_from_mail,
report_recipients=report_recipients,
)
if options.protocol.startswith('http'):
server = make_server(
options.interface,
options.port,
app,
server_class=proto2server[options.protocol],
)
try:
server.serve_forever()
except KeyboardInterrupt:
pass
else:
serverclass = proto2server[options.protocol]
serverclass(app,
bindAddress=(options.interface, options.port),
minSpare=options.min_spare,
maxSpare=options.max_spare,
**flup_kwargs).run()
if options.pid_file:
utils.safe_unlink(options.pid_file)
log.info('exit.')
def watch():
parser = optparse.OptionParser(usage="%prog [OPTIONS]")
parser.add_option(
'-l',
'--logfile',
help='log output to LOGFILE',
)
parser.add_option(
'-d',
'--daemonize',
action='store_true',
help='become daemon as soon as possible',
)
parser.add_option(
'--pid-file',
help='write PID of daemonized process to this file',
)
parser.add_option(
'-q',
'--queue-dir',
help=
'queue directory, where new job files are written to (default: /var/cache/mw-watch/q/)',
default='/var/cache/mw-watch/q/',
)
parser.add_option(
'-p',
'--processing-dir',
help=
'processing directory, where active job files are moved to (must be on same filesystem as --queue-dir, default: /var/cache/mw-watch/p/)',
default='/var/cache/mw-watch/p/',
)
parser.add_option(
'-n',
'--num-jobs',
help='maximum number of simulataneous jobs (default: 5)',
default='5',
)
options, args = parser.parse_args()
try:
options.num_jobs = int(options.num_jobs)
except ValueError:
parser.error('--num-jobs value must be an integer')
from mwlib import filequeue, utils
if options.logfile:
utils.start_logging(options.logfile)
if options.daemonize:
utils.daemonize()
if options.pid_file:
open(options.pid_file, 'wb').write('%d\n' % os.getpid())
poller = filequeue.FileJobPoller(
queue_dir=options.queue_dir,
processing_dir=options.processing_dir,
max_num_jobs=options.num_jobs,
).run_forever()
if options.pid_file:
utils.safe_unlink(options.pid_file)
fd, tmpout = tempfile.mkstemp(dir=os.path.dirname(options.output))
os.close(fd)
writer(env,
output=tmpout,
status_callback=status,
**writer_options)
os.rename(tmpout, options.output)
kwargs = {}
if hasattr(writer, 'content_type'):
kwargs['content_type'] = writer.content_type
if hasattr(writer, 'file_extension'):
kwargs['file_extension'] = writer.file_extension
status(status='finished', progress=100, **kwargs)
if options.keep_zip is None and zip_filename is not None:
utils.safe_unlink(zip_filename)
except Exception, e:
status(status='error')
if options.error_file:
fd, tmpfile = tempfile.mkstemp(
dir=os.path.dirname(options.error_file))
f = os.fdopen(fd, 'wb')
if isinstance(e, WriterError) or isinstance(e, MWAPIError):
f.write(str(e))
else:
traceback.print_exc(file=f)
f.close()
os.rename(tmpfile, options.error_file)
raise
finally:
if env is not None and env.images is not None:
def post():
parser = optparse.OptionParser(usage="%prog OPTIONS")
parser.add_option("-i", "--input", help="ZIP file to POST")
parser.add_option(
'-l',
'--logfile',
help='log output to LOGFILE',
)
parser.add_option("-p", "--posturl", help="HTTP POST ZIP file to POSTURL")
parser.add_option(
"-g",
"--getposturl",
help='get POST URL from PediaPress.com, open upload page in webbrowser',
action='store_true',
)
parser.add_option("-d",
"--daemonize",
action="store_true",
help='become a daemon process as soon as possible')
parser.add_option(
'--pid-file',
help='write PID of daemonized process to this file',
)
options, args = parser.parse_args()
use_help = 'Use --help for usage information.'
if not options.input:
parser.error('Specify --input.\n' + use_help)
if (options.posturl and options.getposturl)\
or (not options.posturl and not options.getposturl):
parser.error('Specify either --posturl or --getposturl.\n' + use_help)
if options.posturl:
from mwlib.podclient import PODClient
podclient = PODClient(options.posturl)
elif options.getposturl:
import webbrowser
from mwlib.podclient import podclient_from_serviceurl
podclient = podclient_from_serviceurl(
'http://pediapress.com/api/collections/')
webbrowser.open(podclient.redirecturl)
from mwlib import utils
from mwlib.status import Status
if options.logfile:
utils.start_logging(options.logfile)
if options.daemonize:
utils.daemonize()
if options.pid_file:
open(options.pid_file, 'wb').write('%d\n' % os.getpid())
status = Status(podclient=podclient)
try:
try:
status(status='uploading', progress=0)
podclient.post_zipfile(options.input)
status(status='finished', progress=100)
except Exception, e:
status(status='error')
raise
finally:
if options.pid_file:
utils.safe_unlink(options.pid_file)
def make_zip_file(output, env,
status=None,
num_threads=10,
imagesize=800,
):
if status is None:
status = lambda **kwargs: None
if output is None:
fd, output = tempfile.mkstemp(suffix='.zip')
os.close(fd)
fd, tmpzip = tempfile.mkstemp(suffix='.zip', dir=os.path.dirname(output))
os.close(fd)
zf = zipfile.ZipFile(tmpzip, 'w')
try:
articles = metabook.get_item_list(env.metabook, filter_type='article')
if num_threads > 0:
z = ThreadedZipCreator(zf,
imagesize=imagesize,
num_threads=num_threads,
status=status,
num_articles=len(articles),
)
else:
z = ZipCreator(zf,
imagesize=imagesize,
status=status,
num_articles=len(articles),
)
# if articles:
# class IncProgress(object):
# inc = 100./len(articles)
# p = 0
# def __call__(self, title):
# self.p += self.inc
# status(progress=int(self.p), article=title)
# inc_progress = IncProgress()
# else:
# inc_progress = None
for item in articles:
d = mwapidb.parse_article_url(item['title'].encode('utf-8'))
if d is not None:
item['title'] = d['title']
item['revision'] = d['revision']
wikidb = mwapidb.WikiDB(api_helper=d['api_helper'])
imagedb = mwapidb.ImageDB(api_helper=d['api_helper'])
else:
wikidb = env.wiki
imagedb = env.images
z.addArticle(item['title'],
revision=item.get('revision', None),
wikidb=wikidb,
imagedb=imagedb,
)
for license in env.get_licenses():
z.parseArticle(
title=license['title'],
raw=license['wikitext'],
wikidb=env.wiki,
imagedb=env.images,
)
z.join()
z.addObject('metabook.json', json.dumps(env.metabook))
zf.close()
if os.path.exists(output): # Windows...
os.unlink(output)
os.rename(tmpzip, output)
if env.images and hasattr(env.images, 'clear'):
env.images.clear()
status(progress=100)
return output
finally:
if os.path.exists(tmpzip):
utils.safe_unlink(tmpzip)
def main():
from gevent import monkey
monkey.patch_all(thread=False)
from mwlib.options import OptionParser
from mwlib import conf
parser = OptionParser()
parser.add_option("-o", "--output", help="write output to OUTPUT")
parser.add_option("-p",
"--posturl",
help="http post to POSTURL (directly)")
parser.add_option(
"-g",
"--getposturl",
help='get POST URL from PediaPress.com, open upload page in webbrowser',
action='count',
)
parser.add_option(
'--keep-tmpfiles',
action='store_true',
default=False,
help="don't remove temporary files like images",
)
parser.add_option("-s",
"--status-file",
help='write status/progress info to this file')
options, args = parser.parse_args()
conf.readrc()
use_help = 'Use --help for usage information.'
if parser.metabook is None and options.collectionpage is None:
parser.error(
'Neither --metabook nor, --collectionpage or arguments specified.\n'
+ use_help)
if options.posturl and options.getposturl:
parser.error('Specify either --posturl or --getposturl.\n' + use_help)
if not options.posturl and not options.getposturl and not options.output:
parser.error(
'Neither --output, nor --posturl or --getposturl specified.\n' +
use_help)
if options.posturl:
from mwlib.podclient import PODClient
podclient = PODClient(options.posturl)
elif options.getposturl:
if options.getposturl > 1:
serviceurl = 'http://test.pediapress.com/api/collections/'
else:
serviceurl = 'http://pediapress.com/api/collections/'
import webbrowser
from mwlib.podclient import podclient_from_serviceurl
podclient = podclient_from_serviceurl(serviceurl)
pid = os.fork()
if not pid:
try:
webbrowser.open(podclient.redirecturl)
finally:
os._exit(0)
import time
time.sleep(1)
try:
os.kill(pid, 9)
except BaseException:
pass
else:
podclient = None
from mwlib import utils, wiki
filename = None
status = None
try:
env = parser.makewiki()
assert env.metabook, "no metabook"
from mwlib.status import Status
status = Status(options.status_file,
podclient=podclient,
progress_range=(1, 90))
status(progress=0)
output = options.output
make_zip(output,
options,
env.metabook,
podclient=podclient,
status=status)
except Exception as e:
if status:
status(status='error')
raise
finally:
if options.output is None and filename is not None:
print 'removing %r' % filename
utils.safe_unlink(filename)
def make_zip_file(output, env,
status=None,
num_threads=10,
imagesize=800,
):
if output is None:
fd, output = tempfile.mkstemp(suffix='.zip')
os.close(fd)
fd, tmpzip = tempfile.mkstemp(suffix='.zip', dir=os.path.dirname(output))
os.close(fd)
zf = zipfile.ZipFile(tmpzip, 'w')
try:
articles = metabook.get_item_list(env.metabook, filter_type='article')
z = ZipCreator(zf,
imagesize=imagesize,
num_threads=num_threads,
status=status,
num_articles=len(articles),
)
for item in articles:
d = mwapidb.parse_article_url(item['title'].encode('utf-8'))
if d is not None:
item['title'] = d['title']
item['revision'] = d['revision']
wikidb = mwapidb.WikiDB(api_helper=d['api_helper'])
imagedb = mwapidb.ImageDB(api_helper=d['api_helper'])
else:
wikidb = env.wiki
imagedb = env.images
z.addArticle(item['title'],
revision=item.get('revision', None),
wikidb=wikidb,
imagedb=imagedb,
)
for license in env.get_licenses():
z.parseArticle(
title=license['title'],
raw=license['wikitext'],
wikidb=env.wiki,
imagedb=env.images,
)
z.join()
# using check() is a bit rigorous: sometimes articles just cannot be
# fetched -- PDFs should be generated nevertheless
#z.check(articles)
z.addObject('metabook.json', json.dumps(env.metabook))
# add stats for later analysis
z.node_stats["Chapter"] = len(metabook.get_item_list(env.metabook, filter_type='chapter'))
z.addObject('node_stats.json', json.dumps(z.node_stats))
zf.close()
if os.path.exists(output): # Windows...
os.unlink(output)
os.rename(tmpzip, output)
if env.images and hasattr(env.images, 'clear'):
env.images.clear()
if status is not None:
status(progress=100)
return output
finally:
if os.path.exists(tmpzip):
utils.safe_unlink(tmpzip)
from mwlib import utils, wiki
if options.daemonize:
utils.daemonize()
if options.pid_file:
open(options.pid_file, 'wb').write('%d\n' % os.getpid())
filename = None
status = None
try:
env = parser.makewiki()
assert env.metabook, "no metabook"
from mwlib.status import Status
status = Status(options.status_file, podclient=podclient, progress_range=(1, 90))
status(progress=0)
output = options.output
make_zip(output, options, env.metabook, podclient=podclient, status=status)
except Exception, e:
if status:
status(status='error')
raise
finally:
if options.output is None and filename is not None:
print 'removing %r' % filename
utils.safe_unlink(filename)
if options.pid_file:
utils.safe_unlink(options.pid_file)
def buildzip():
from mwlib.options import OptionParser
parser = OptionParser()
parser.add_option("-o", "--output", help="write output to OUTPUT")
parser.add_option("-p",
"--posturl",
help="http post to POSTURL (directly)")
parser.add_option(
"-g",
"--getposturl",
help='get POST URL from PediaPress.com, open upload page in webbrowser',
action='store_true',
)
options, args = parser.parse_args()
use_help = 'Use --help for usage information.'
if parser.metabook is None and options.collectionpage is None:
parser.error(
'Neither --metabook nor, --collectionpage or arguments specified.\n'
+ use_help)
if options.posturl and options.getposturl:
parser.error('Specify either --posturl or --getposturl.\n' + use_help)
if not options.posturl and not options.getposturl and not options.output:
parser.error(
'Neither --output, nor --posturl or --getposturl specified.\n' +
use_help)
if options.posturl:
from mwlib.podclient import PODClient
podclient = PODClient(options.posturl)
elif options.getposturl:
import webbrowser
from mwlib.podclient import podclient_from_serviceurl
podclient = podclient_from_serviceurl(
'http://pediapress.com/api/collections/')
webbrowser.open(podclient.redirecturl)
else:
podclient = None
from mwlib import utils
if options.daemonize:
utils.daemonize()
if options.pid_file:
open(options.pid_file, 'wb').write('%d\n' % os.getpid())
filename = None
status = None
try:
try:
env = parser.makewiki()
from mwlib.status import Status
from mwlib import zipcreator
status = Status(podclient=podclient, progress_range=(1, 90))
status(status='parsing', progress=0)
filename = zipcreator.make_zip_file(
options.output,
env,
status=status,
num_threads=options.num_threads,
imagesize=options.imagesize,
)
status = Status(podclient=podclient, progress_range=(91, 100))
if podclient:
status(status='uploading', progress=0)
podclient.post_zipfile(filename)
status(status='finished', progress=100)
except Exception, e:
if status:
status(status='error')
raise
finally:
if options.output is None and filename is not None:
print 'removing %r' % filename
utils.safe_unlink(filename)
if options.pid_file:
utils.safe_unlink(options.pid_file)
except:
pass
else:
podclient = None
from mwlib import utils, wiki
filename = None
status = None
try:
env = parser.makewiki()
assert env.metabook, "no metabook"
from mwlib.status import Status
status = Status(options.status_file, podclient=podclient, progress_range=(1, 90))
status(progress=0)
output = options.output
make_zip(output, options, env.metabook, podclient=podclient, status=status)
except Exception, e:
if status:
status(status='error')
raise
finally:
if options.output is None and filename is not None:
print 'removing %r' % filename
utils.safe_unlink(filename)
response = {
'collection_id': collection_id,
'writer': writer,
'is_cached': False,
}
pid_path = self.get_path(collection_id, self.pid_filename, writer)
if os.path.exists(pid_path):
log.info('mw-render already running for collection %r' % collection_id)
return response
output_path = self.get_path(collection_id, self.output_filename, writer)
if os.path.exists(output_path):
if force_render:
log.info('removing rendered file %r (forced rendering)' % output_path)
utils.safe_unlink(output_path)
else:
log.info('re-using rendered file %r' % output_path)
response['is_cached'] = True
return response
status_path = self.get_path(collection_id, self.status_filename, writer)
if os.path.exists(status_path):
if force_render:
log.info('removing status file %r (forced rendering)' % status_path)
utils.safe_unlink(status_path)
else:
log.info('status file exists %r' % status_path)
return response
error_path = self.get_path(collection_id, self.error_filename, writer)
def main():
from SocketServer import ForkingMixIn, ThreadingMixIn
from wsgiref.simple_server import make_server, WSGIServer
from flup.server import fcgi, fcgi_fork, scgi, scgi_fork
class ForkingWSGIServer(ForkingMixIn, WSGIServer):
pass
class ThreadingWSGIServer(ThreadingMixIn, WSGIServer):
pass
proto2server = {
'http': ForkingWSGIServer,
'http_threaded': ThreadingWSGIServer,
'fcgi': fcgi_fork.WSGIServer,
'fcgi_threaded': fcgi.WSGIServer,
'scgi': scgi_fork.WSGIServer,
'scgi_threaded': scgi.WSGIServer,
}
parser = optparse.OptionParser(usage="%prog [OPTIONS]")
parser.add_option('-l', '--logfile',
help='log output to LOGFILE',
)
parser.add_option('-d', '--daemonize',
action='store_true',
help='become daemon as soon as possible',
)
parser.add_option('--pid-file',
help='write PID of daemonized process to this file',
)
parser.add_option('-P', '--protocol',
help='one of %s (default: http)' % ', '.join(proto2server.keys()),
default='http',
)
parser.add_option('-p', '--port',
help='port to listen on (default: 8899)',
default='8899',
)
parser.add_option('-i', '--interface',
help='interface to listen on (default: 0.0.0.0)',
default='0.0.0.0',
)
parser.add_option('--cache-dir',
help='cache directory (default: /var/cache/mw-serve/)',
default='/var/cache/mw-serve/',
)
parser.add_option('--mwrender',
help='(path to) mw-render executable',
default='mw-render',
)
parser.add_option('--mwrender-logfile',
help='global logfile for mw-render',
metavar='LOGFILE',
)
parser.add_option('--mwzip',
help='(path to) mw-zip executable',
default='mw-zip',
)
parser.add_option('--mwzip-logfile',
help='global logfile for mw-zip',
metavar='LOGFILE',
)
parser.add_option('--mwpost',
help='(path to) mw-post executable',
default='mw-post',
)
parser.add_option('--mwpost-logfile',
help='global logfile for mw-post',
metavar='LOGFILE',
)
parser.add_option('-q', '--queue-dir',
help='queue dir of mw-watch (if not specified, no queue is used)',
)
parser.add_option('-m', '--method',
help='prefork or threaded (default: prefork)',
default='prefork',
)
parser.add_option('--max-requests',
help='maximum number of requests a child process can handle before it is killed, irrelevant for --method=threaded (default: 0 = no limit)',
default='0',
metavar='NUM',
)
parser.add_option('--min-spare',
help='minimum number of spare processes/threads (default: 2)',
default='2',
metavar='NUM',
)
parser.add_option('--max-spare',
help='maximum number of spare processes/threads (default: 5)',
default='5',
metavar='NUM',
)
parser.add_option('--max-children',
help='maximum number of processes/threads (default: 50)',
default='50',
metavar='NUM',
)
parser.add_option('--report-from-mail',
help='sender of error mails (--report-recipient also needed)',
metavar='EMAIL',
)
parser.add_option('--report-recipient',
help='recipient of error mails (--report-from-mail also needed)',
metavar='EMAIL',
)
options, args = parser.parse_args()
if args:
parser.error('no arguments supported')
if options.protocol not in proto2server:
parser.error('unsupported protocol (must be one of %s)' % (
', '.join(proto2server.keys()),
))
def to_int(opt_name):
try:
setattr(options, opt_name, int(getattr(options, opt_name)))
except ValueError:
parser.error('--%s value must be an integer' % opt_name.replace('_', '-'))
to_int('port')
to_int('max_requests')
to_int('min_spare')
to_int('max_spare')
to_int('max_children')
if options.method not in ('prefork', 'threaded'):
parser.error('the only supported values for --method are "prefork" and "threaded"')
from mwlib import serve, log, utils
log = log.Log('mw-serve')
if options.logfile:
utils.start_logging(options.logfile)
if options.daemonize:
utils.daemonize()
if options.pid_file:
open(options.pid_file, 'wb').write('%d\n' % os.getpid())
if options.method == 'threaded':
options.protocol += '_threaded'
flup_kwargs = {
'maxThreads': options.max_children,
}
else:
flup_kwargs = {
'maxChildren': options.max_children,
'maxRequests': options.max_requests,
}
log.info("serving %s on %s:%s" % (options.protocol, options.interface, options.port))
if options.report_recipient and options.report_from_mail:
report_from_mail = options.report_from_mail.encode('utf-8')
report_recipients = [options.report_recipient.encode('utf-8')]
else:
report_from_mail = None
report_recipients = None
app = serve.Application(
cache_dir=options.cache_dir,
mwrender_cmd=options.mwrender,
mwrender_logfile=options.mwrender_logfile,
mwzip_cmd=options.mwzip,
mwzip_logfile=options.mwzip_logfile,
mwpost_cmd=options.mwpost,
mwpost_logfile=options.mwpost_logfile,
queue_dir=options.queue_dir,
report_from_mail=report_from_mail,
report_recipients=report_recipients,
)
if options.protocol.startswith('http'):
server = make_server(options.interface, options.port, app,
server_class=proto2server[options.protocol],
)
try:
server.serve_forever()
except KeyboardInterrupt:
pass
else:
serverclass = proto2server[options.protocol]
serverclass(app,
bindAddress=(options.interface, options.port),
minSpare=options.min_spare,
maxSpare=options.max_spare,
**flup_kwargs
).run()
if options.pid_file:
utils.safe_unlink(options.pid_file)
log.info('exit.')
def __call__(self):
options, args, parser = self.parse_options()
conf.readrc()
self.parser = parser
self.options = options
import tempfile
from mwlib.writerbase import WriterError
from mwlib.status import Status
use_help = 'Use --help for usage information.'
if options.list_writers:
self.list_writers()
return
if options.writer_info:
self.show_writer_info(options.writer_info)
return
if options.output is None:
parser.error('Please specify an output file with --output.\n' +
use_help)
options.output = os.path.abspath(options.output)
if options.writer is None:
parser.error('Please specify a writer with --writer.\n' + use_help)
writer = self.load_writer(options.writer)
writer_options = {}
if options.writer_options:
for wopt in options.writer_options.split(';'):
if '=' in wopt:
key, value = wopt.split('=', 1)
else:
key, value = wopt, True
writer_options[str(key)] = value
if options.language:
writer_options['lang'] = options.language
for option in writer_options.keys():
if option not in getattr(writer, 'options', {}):
print 'Warning: unknown writer option %r' % option
del writer_options[option]
init_tmp_cleaner()
self.status = Status(options.status_file, progress_range=(1, 33))
self.status(progress=0)
env = None
try:
env = self.get_environment()
try:
_locale.set_locale_from_lang(
env.wiki.siteinfo["general"]["lang"])
except BaseException as err:
print "Error: could not set locale", err
basename = os.path.basename(options.output)
if '.' in basename:
ext = '.' + basename.rsplit('.', 1)[-1]
else:
ext = ''
fd, tmpout = tempfile.mkstemp(dir=os.path.dirname(options.output),
suffix=ext)
os.close(fd)
writer(env,
output=tmpout,
status_callback=self.status,
**writer_options)
os.rename(tmpout, options.output)
kwargs = {}
if hasattr(writer, 'content_type'):
kwargs['content_type'] = writer.content_type
if hasattr(writer, 'file_extension'):
kwargs['file_extension'] = writer.file_extension
self.status(status='finished', progress=100, **kwargs)
if options.keep_zip is None and self.zip_filename is not None:
utils.safe_unlink(self.zip_filename)
except Exception as e:
import traceback
self.status(status='error')
if options.error_file:
fd, tmpfile = tempfile.mkstemp(
dir=os.path.dirname(options.error_file))
f = os.fdopen(fd, 'wb')
if isinstance(e, WriterError):
f.write(str(e))
else:
f.write('traceback\n')
traceback.print_exc(file=f)
f.write("sys.argv=%r\n" % (utils.garble_password(sys.argv), ))
f.close()
os.rename(tmpfile, options.error_file)
raise
finally:
if env is not None and env.images is not None:
try:
if not options.keep_tmpfiles:
env.images.clear()
except OSError as e:
if e.errno != errno.ENOENT:
print 'ERROR: Could not remove temporary images: %s' % e, e.errno