def main():
from pyramid.paster import get_app
from pyramid.scripting import get_root
from ..resources import BlogEntry
parser = OptionParser(description=__doc__, usage='usage: %prog [options]')
parser.add_option('-c', '--config', dest='config',
help='Specify a paster config file.')
parser.add_option('-n', '--name', dest='name', default='main',
help='The paster config file section indicating the app.')
parser.add_option('-i', '--num', dest='num', default='10000',
help='Specify the number of blog entries to add.')
options, args = parser.parse_args()
config = options.config
name = options.name
num = int(options.num)
if config is None:
raise ValueError('must supply config file name')
config = os.path.abspath(os.path.normpath(config))
app = get_app(config, name)
root, closer = get_root(app)
for n in range(0, num):
print ("adding", n)
entry = BlogEntry('title %s' % n, 'entry %s' % n,
'html', datetime.date.today())
id = 'blogentry_%s' % n
root[id] = entry
if n % 10000 == 0:
print ('committing')
transaction.commit()
print ('committing')
transaction.commit()
root._p_jar._db.close()
closer()
def _run_import_function(func: callable, app_router: Router, filename: str):
root, closer = get_root(app_router)
try:
func(root, app_router.registry, filename)
transaction.commit()
finally:
closer()
def run():
root, closer = get_root(app)
set_subsystem('mailin')
zodb_uri = get_setting(root, 'postoffice.zodb_uri')
zodb_path = get_setting(root, 'postoffice.zodb_path', '/postoffice')
queue = get_setting(root, 'postoffice.queue')
if zodb_uri is None:
parser.error("postoffice.zodb_uri must be set in config file")
if queue is None:
parser.error("postoffice.queue must be set in config file")
runner = MailinRunner2(root, zodb_uri, zodb_path, queue)
try:
runner()
if options.dry_run:
transaction.abort()
else:
transaction.commit()
p_jar = getattr(root, '_p_jar', None)
if p_jar is not None:
# Attempt to fix memory leak
p_jar.db().cacheMinimize()
except:
transaction.abort()
raise
finally:
closer()
runner.close()
def main(argv=sys.argv):
logging.basicConfig()
parser = optparse.OptionParser(
description=__doc__,
usage="%prog [options]",
)
parser.add_option('-C', '--config', dest='config',
help='Path to configuration file (defaults to $CWD/etc/karl.ini)',
metavar='FILE')
parser.add_option('-O', '--output', dest='output', default='.',
help="Path to the directory where reports should be written.",
metavar='DIR')
options, args = parser.parse_args(argv[1:])
if args:
parser.error("Too many arguments. " + str(args))
config = options.config
if config is None:
config = get_default_config()
app = loadapp('config:%s' % config, name='karl')
root, closer = get_root(app)
folder = os.path.abspath(options.output)
generate_reports(root, folder)
def main(argv=sys.argv):
logging.basicConfig()
parser = optparse.OptionParser(
description=__doc__,
usage="%prog [options]",
)
parser.add_option(
'-C',
'--config',
dest='config',
help='Path to configuration file (defaults to $CWD/etc/karl.ini)',
metavar='FILE')
parser.add_option(
'-O',
'--output',
dest='output',
default='.',
help="Path to the directory where reports should be written.",
metavar='DIR')
options, args = parser.parse_args(argv[1:])
if args:
parser.error("Too many arguments. " + str(args))
config = options.config
if config is None:
config = get_default_config()
app = loadapp('config:%s' % config, name='karl')
root, closer = get_root(app)
folder = os.path.abspath(options.output)
generate_reports(root, folder)
def _run_import_function(func: callable, app_router: Router, filename: str):
root, closer = get_root(app_router)
try:
func(root, app_router.registry, filename)
transaction.commit()
finally:
closer()
def main(argv=sys.argv):
parser = optparse.OptionParser(
description=__doc__,
usage="%prog [options] [user1, user2, ...]",
)
parser.add_option('-C', '--config', dest='config',
help='Path to configuration file (defaults to $CWD/etc/karl.ini)',
metavar='FILE')
parser.add_option('-f', '--filter', dest='filter', action='store_true',
help='Limit results to users specified by username in stdin. Users'
' should be whitespace delimited.', default=False)
options, args = parser.parse_args(argv[1:])
ids = None
if options.filter:
ids = sys.stdin.read().split()
if args:
if ids is not None:
ids.extend(args)
else:
ids = args
config = options.config
if config is None:
config = get_default_config()
app = loadapp('config:%s' % config, name='karl')
root, closer = get_root(app)
out_csv = csv.writer(sys.stdout)
out_csv.writerow(['username', 'community', 'last_activity'])
for row in stats.user_activity_report(root, ids):
username, community, last_activity = row
out_csv.writerow([username, community.__name__, last_activity.ctime()])
def main():
from pyramid.paster import get_app
from pyramid.scripting import get_root
from ..resources import BlogEntry
parser = OptionParser(description=__doc__, usage='usage: %prog [options]')
parser.add_option('-c',
'--config',
dest='config',
help='Specify a paster config file.')
parser.add_option('-n',
'--name',
dest='name',
default='zodb',
help='Specify a section name.')
options, args = parser.parse_args()
config = options.config
name = options.name
if config is None:
raise ValueError('must supply config file name')
config = os.path.abspath(os.path.normpath(config))
app = get_app(config, name)
root, closer = get_root(app)
for arg in args:
print("filename:", arg)
if not os.path.isfile(arg):
print('not a file')
continue
path, filename = os.path.split(arg)
id, ext = os.path.splitext(filename)
print("id:", id)
lines = open(arg, 'r').readlines()
title = lines[0]
print('title:', title)
entry = '\n'.join(lines[2:])
print('entry:', entry[:40])
pieces = id.split('-')
last = pieces[-1]
pubdate = None
if last.startswith('200'):
if len(last) == 8:
year, month, day = last[0:4], last[4:6], last[6:8]
pubdate = datetime.date(int(year), int(month), int(day))
if pubdate is None:
p1 = Popen(["svn", "info", arg], stdout=PIPE)
p2 = Popen(["grep", "Last Changed Date"],
stdin=p1.stdout,
stdout=PIPE)
output = p2.communicate()[0]
lines = output.split(':', 1)
datestr = lines[1].strip()
datestr = datestr.split(' ', 1)[0]
year, month, day = datestr[0:4], datestr[5:7], datestr[8:10]
pubdate = datetime.date(int(year), int(month), int(day))
print('pubdate:', pubdate)
entry = BlogEntry(title.decode('UTF-8'), entry.decode('UTF-8'),
id.decode('UTF-8'), pubdate, 'html', None, None)
root[id] = entry
transaction.commit()
def open_root(config, name='karl'): #pragma NO COVERAGE
"""Open the database root object, given a Paste Deploy config file name.
Returns (root, closer). Call the closer function to close the database
connection.
"""
config = os.path.abspath(os.path.normpath(config))
app = loadapp('config:%s' % config, name=name)
return get_root(app)
def namespace(self):
if hasattr(self, '_namespace'):
return self._namespace
from pyramid.scripting import get_root
innerapp = self.get_app_with_registry(self.manager.app)
root, closer = get_root(innerapp)
self._namespace = {'root': root,
'registry': innerapp.registry,
'app': self.manager.app}
return self._namespace
def main():
from pyramid.paster import get_app
from pyramid.scripting import get_root
from ..resources import BlogEntry
parser = OptionParser(description=__doc__, usage='usage: %prog [options]')
parser.add_option('-c', '--config', dest='config',
help='Specify a paster config file.')
parser.add_option('-n', '--name', dest='name', default='zodb',
help='Specify a section name.')
options, args = parser.parse_args()
config = options.config
name = options.name
if config is None:
raise ValueError('must supply config file name')
config = os.path.abspath(os.path.normpath(config))
app = get_app(config, name)
root, closer = get_root(app)
for arg in args:
print("filename:", arg)
if not os.path.isfile(arg):
print('not a file')
continue
path, filename = os.path.split(arg)
id, ext = os.path.splitext(filename)
print("id:", id)
lines = open(arg, 'r').readlines()
title = lines[0]
print('title:', title)
entry = '\n'.join(lines[2:])
print('entry:', entry[:40])
pieces = id.split('-')
last = pieces[-1]
pubdate = None
if last.startswith('200'):
if len(last) == 8:
year, month, day = last[0:4], last[4:6], last[6:8]
pubdate = datetime.date(int(year), int(month), int(day))
if pubdate is None:
p1 = Popen(["svn", "info", arg], stdout=PIPE)
p2 = Popen(["grep", "Last Changed Date"], stdin=p1.stdout,
stdout=PIPE)
output = p2.communicate()[0]
lines = output.split(':', 1)
datestr = lines[1].strip()
datestr = datestr.split(' ', 1)[0]
year, month, day = datestr[0:4], datestr[5:7], datestr[8:10]
pubdate = datetime.date(int(year), int(month), int(day))
print('pubdate:', pubdate)
entry = BlogEntry(title.decode('UTF-8'), entry.decode('UTF-8'),
id.decode('UTF-8'), pubdate, 'html', None, None)
root[id] = entry
transaction.commit()
def namespace(self):
if hasattr(self, '_namespace'):
return self._namespace
from pyramid.scripting import get_root
innerapp = self.get_app_with_registry(self.manager.app)
root, closer = get_root(innerapp)
self._namespace = {
'root': root,
'registry': innerapp.registry,
'app': self.manager.app
}
return self._namespace
def run(root=root):
closer = lambda: None # unit test
try:
if root is None:
root, closer = get_root(app)
#else: unit test
update_func(root, config, force=options.force)
except:
tx.abort()
closer()
raise
else:
if options.dryrun:
tx.abort()
else:
tx.commit()
closer()
def run(root=root):
closer = lambda: None # unit test
try:
if root is None:
root, closer = get_root(app)
#else: unit test
update_func(root, config, force=options.force)
except:
tx.abort()
closer()
raise
else:
if options.dryrun:
tx.abort()
else:
tx.commit()
closer()
def main():
from pyramid.paster import get_app
from pyramid.scripting import get_root
from ..resources import BlogEntry
parser = OptionParser(description=__doc__, usage='usage: %prog [options]')
parser.add_option('-c',
'--config',
dest='config',
help='Specify a paster config file.')
parser.add_option(
'-n',
'--name',
dest='name',
default='main',
help='The paster config file section indicating the app.')
parser.add_option('-i',
'--num',
dest='num',
default='10000',
help='Specify the number of blog entries to add.')
options, args = parser.parse_args()
config = options.config
name = options.name
num = int(options.num)
if config is None:
raise ValueError('must supply config file name')
config = os.path.abspath(os.path.normpath(config))
app = get_app(config, name)
root, closer = get_root(app)
for n in range(0, num):
print("adding", n)
entry = BlogEntry('title %s' % n, 'entry %s' % n, 'html',
datetime.date.today())
id = 'blogentry_%s' % n
root[id] = entry
if n % 10000 == 0:
print('committing')
transaction.commit()
print('committing')
transaction.commit()
root._p_jar._db.close()
closer()
def main(argv=sys.argv):
parser = optparse.OptionParser(
description=__doc__,
usage="%prog [options] [user1, user2, ...]",
)
parser.add_option(
'-C',
'--config',
dest='config',
help='Path to configuration file (defaults to $CWD/etc/karl.ini)',
metavar='FILE')
parser.add_option(
'-f',
'--filter',
dest='filter',
action='store_true',
help='Limit results to users specified by username in stdin. Users'
' should be whitespace delimited.',
default=False)
options, args = parser.parse_args(argv[1:])
ids = None
if options.filter:
ids = sys.stdin.read().split()
if args:
if ids is not None:
ids.extend(args)
else:
ids = args
config = options.config
if config is None:
config = get_default_config()
app = loadapp('config:%s' % config, name='karl')
root, closer = get_root(app)
out_csv = csv.writer(sys.stdout)
out_csv.writerow(['username', 'community', 'last_activity'])
for row in stats.user_activity_report(root, ids):
username, community, last_activity = row
out_csv.writerow([username, community.__name__, last_activity.ctime()])
def worker(app=None):
"""thread worker function"""
while True:
print('Worker alive.')
(root, closer) = get_root(app)
dependencies = Dependencies()
import transaction
noBuildsHandledYet = True
for build in root.builds:
if build.status == 'created' and noBuildsHandledYet:
build.status = 'cloning..'
transaction.get().commit()
noBuildsHandledYet = False
print('Handling build.')
processBuild(build, dependencies)
build.status = 'done.'
closer()
transaction.get().commit()
time.sleep(8)
def __init__(self, id):
""" Initialize worker.
id will be part of the pid-file.
"""
self.id = id
#Buildout path
me = sys.argv[0]
me = os.path.abspath(me)
self.buildoutpath = os.path.dirname(os.path.dirname(me))
if len(sys.argv) < 2:
sys.exit("Must specify paster.ini file to run")
#setup logging
#self._setup_log()
#PID file name
rel = os.path.join(self.buildoutpath, 'worker_%s.pid' % self.id)
self.pidfile = os.path.abspath(os.path.normpath(rel))
#Check if PID exists
if os.path.exists(self.pidfile):
#Is this correct?
msg = "PID-file already exists. Maybe the script is already running?"
self.logger.exception(msg)
sys.exit(msg)
#Start wsgi stuff
config = os.path.join(self.buildoutpath, sys.argv[1])
config = os.path.abspath(os.path.normpath(config))
self.app = loadapp('config:%s' % config, name='MadeToMeasure')
self.root, self.closer = get_root(self.app)
print 'Worker initialized'
#write pid
self._write_pid_file()
def __init__(self, id):
""" Initialize worker.
id will be part of the pid-file.
"""
#FIXME: There must be a smarter way to handle PID-files. I'm no console scripting expert :) /Robin
self.id = id
#Buildout path
me = sys.argv[0]
me = os.path.abspath(me)
self.buildoutpath = os.path.dirname(os.path.dirname(me))
if len(sys.argv) < 2:
sys.exit("Must specify paster.ini file to run")
#setup logging
self._setup_log()
#PID file name
rel = os.path.join(self.buildoutpath, 'var', 'worker_%s.pid' % self.id)
self.pidfile = os.path.abspath(os.path.normpath(rel))
#Check if PID exists
if os.path.exists(self.pidfile):
#Is this correct?
msg = "PID-file already exists. Maybe the script is already running?"
self.logger.exception(msg)
sys.exit(msg)
#Start wsgi stuff
config = os.path.join(self.buildoutpath, sys.argv[1])
self.app = loadapp('config:%s' % config, name='VoteIT')
self.root, self.closer = get_root(self.app)
print 'Worker initialized'
#write pid
self._write_pid_file()
def run(root=root):
closer = lambda: None # unit test
if options.run_draino:
draino_args = ['draino', '-vvv', '-p', '%s/Maildir' % maildir_root,
maildir_root]
if options.dry_run:
draino_args.insert(1, '--dry-run')
draino = Draino(draino_args)
if root is None:
root, closer = get_root(app)
#else: unit test
set_subsystem('mailin')
if options.run_draino:
draino.run()
runner = factory(root, maildir_root, options)
runner()
p_jar = getattr(root, '_p_jar', None)
if p_jar is not None:
# Attempt to fix memory leak
p_jar.db().cacheMinimize()
closer()
def set_acms_for_app_root(event):
"""Set/update :term:`acm`s for the root object of the pyramid application.
:param event: this function should be used as a subscriber for the
:class:`pyramid.interfaces.IApplicationCreated` event.
That way everytime the application starts the root `acm`
is updated.
The `root_acm` (:func:`root_acm_asset`) is extended by the :term:`acm`
returned by the :class:`adhocracy_core.authorization.IRootACMExtension`
adapter.
In addition all permissions are granted the god user.
"""
root, closer = get_root(event.app)
acl = [god_all_permission_ace]
acl += _get_root_extension_acl(root, event.app.registry)
acl += _get_root_base_acl()
old_acl = get_acl(root)
if old_acl == acl:
return
_set_acl(root, acl, registry=event.app.registry)
transaction.commit()
closer()
def get_site_root(args, name):
return get_root(get_site(args, name).site())
def _callFUT(self, app, request=None):
from pyramid.scripting import get_root
return get_root(app, request)
def _callFUT(self, app, request=None):
from pyramid.scripting import get_root
return get_root(app, request)
def run():
set_subsystem('digest')
root, closer = get_root(app)
alerts.send_digests(root)
closer()
def get_instance_root(name):
instance = get_instance(app, name)
if instance is None:
args.parser.error("Unknown Karl instance: %s" % name)
return get_root(instance.instance())
def run():
set_subsystem('digest')
root, closer = get_root(app)
alerts.send_digests(root)
closer()
def main(argv=sys.argv):
config = None
if len(argv) > 1:
config = sys.argv[1]
if not config:
# we assume that the console script lives in the 'bin' dir of a
# sandbox or buildout, and that the .ini file lives in the 'etc'
# directory of the sandbox or buildout
me = sys.argv[0]
me = os.path.abspath(me)
sandbox = os.path.dirname(os.path.dirname(me))
config = os.path.join(sandbox, "marlton.ini")
exe = sys.executable
sandbox = os.path.dirname(os.path.dirname(os.path.abspath(exe)))
app = get_app(config, name="website")
app_root, closer = get_root(app)
catalog = find_catalog(app_root)
for address in list(catalog.document_map.address_to_docid.keys()):
if address.startswith("external:") or address.startswith("sphinx:"):
docid = catalog.document_map.address_to_docid[address]
catalog.unindex_doc(docid)
catalog.document_map.remove_address(address)
settings = get_current_registry().settings
docroot = settings["sphinx_docroot"]
roots = settings["sphinx_roots"]
oldcwd = os.getcwd()
for root in roots:
package_dir = roots[root]["package_dir"]
docs_subpath = roots[root]["docs_subpath"]
url_prefix = roots[root]["url_prefix"]
virtual_home = os.path.join(docroot, root)
os.system("%s/bin/virtualenv --no-site-packages %s" % (sys.prefix, virtual_home))
os.system("%s/bin/easy_install Sphinx" % virtual_home)
os.system("%s/bin/easy_install docutils" % virtual_home)
os.system("%s/bin/easy_install repoze.sphinx.autointerface" % virtual_home)
try:
os.chdir(package_dir)
os.system("%s/bin/python setup.py develop" % virtual_home)
finally:
os.chdir(oldcwd)
docs_source = os.path.join(package_dir, docs_subpath)
os.system(
"%s/bin/sphinx-build -b html -d %s/doctrees "
"%s %s/html" % (virtual_home, virtual_home, docs_source, virtual_home)
)
os.system(
"%s/bin/sphinx-build -b text -d %s/doctrees "
"%s %s/text" % (virtual_home, virtual_home, docs_source, virtual_home)
)
textpath = "%s/text/" % virtual_home
gen = os.walk(textpath)
for path, directories, files in gen:
relpath = path[len(textpath) :]
for filename in files:
if filename.endswith(".txt"):
text = open(os.path.join(path, filename)).read()
text = text.decode("utf-8")
ob = SphinxDocument(text)
filename_no_ext = filename[:-4]
path_info = os.path.join(relpath, filename_no_ext)
address = "sphinx:%s/%s.html" % (url_prefix, path_info)
docid = catalog.document_map.add(address)
catalog.index_doc(docid, ob)
data = {"title": filename_no_ext, "type": "Documentation", "text": text}
catalog.document_map.add_metadata(docid, data)
transaction.commit()
def run():
set_subsystem('digest')
root, closer = get_root(app)
alerts.send_digests(root, options.frequency)
closer()
