def update_feeds_for_instance(args, instance):
log.info("Updating feeds for %s", instance)
root, closer = args.get_root(instance)
set_current_instance(instance)
set_subsystem('update_feeds')
feeds.update_feeds(root, log, args.force)
transaction.commit()
def main():
parser = optparse.OptionParser(description=__doc__, usage="%prog [options] queue_path")
parser.add_option(
"-C",
"--config",
dest="config",
default=None,
help="Path to configuration file (defaults to $CWD/etc/karl.ini)",
metavar="FILE",
)
parser.add_option("--daemon", "-D", dest="daemon", action="store_true", default=False, help="Run in daemon mode.")
parser.add_option(
"--interval",
"-i",
dest="interval",
type="int",
default=6 * 3600,
help="Interval, in seconds, between executions when in " "daemon mode.",
)
parser.add_option(
"--server", "-s", dest="hostname", default="localhost", help="SMTP server host name", metavar="HOST"
)
parser.add_option("--port", "-P", dest="port", type="int", default=25, help="Port of SMTP server", metavar="PORT")
parser.add_option("--username", "-u", dest="username", default=None, help="Username, if authentication is required")
parser.add_option("--password", "-p", dest="password", default=None, help="Password, if authentication is required")
parser.add_option(
"--force-tls", "-f", dest="force_tls", action="store_true", default=False, help="Require that TLS be used."
)
parser.add_option(
"--no-tls", "-n", dest="no_tls", action="store_true", default=False, help="Require that TLS not be used."
)
options, args = parser.parse_args()
if not args:
parser.error("Please specify queue path.")
elif len(args) > 1:
parser.error("Too many arguments.")
queue_path = args[0]
config = options.config
if config is None:
config = get_default_config()
app = loadapp("config:%s" % config, "karl")
set_subsystem("mailout")
mailer = SMTPMailer(
hostname=options.hostname,
port=options.port,
username=options.username,
password=options.password,
no_tls=options.no_tls,
force_tls=options.force_tls,
)
qp = QueueProcessor(mailer, queue_path)
if options.daemon:
run_daemon("digest", qp.send_messages, options.interval)
else:
qp.send_messages()
def digest(args, instance):
root, closer = args.get_root(instance)
set_current_instance(instance)
set_subsystem('digest')
alerts = Alerts()
alerts.send_digests(root)
closer()
def mailin(args, instance):
log.info("Processing mailin for %s" % instance)
root, closer = args.get_root(instance)
set_current_instance(instance)
set_subsystem('mailin')
zodb_uri = get_setting(root, 'postoffice.zodb_uri')
zodb_path = get_setting(root, 'postoffice.zodb_path', '/postoffice')
queue = get_setting(root, 'postoffice.queue')
if zodb_uri is None:
args.parser.error("postoffice.zodb_uri must be set in config file")
if queue is None:
args.parser.error("postoffice.queue must be set in config file")
runner = None
try:
runner = MailinRunner2(root, zodb_uri, zodb_path, queue)
runner()
transaction.commit()
p_jar = getattr(root, '_p_jar', None)
if p_jar is not None:
# Attempt to fix memory leak
p_jar.db().cacheMinimize()
except:
transaction.abort()
raise
finally:
closer()
if runner is not None:
runner.close()
def main(args):
queue_path = args.get_setting('mail_queue_path')
set_subsystem('mailout')
mailer = SMTPMailer(
hostname=args.server,
port=args.port,
username=args.username,
password=args.password,
no_tls=args.no_tls,
force_tls=args.force_tls
)
qp = QueueProcessor(mailer, queue_path)
qp.send_messages()
def dump_senders(args, instance):
root, closer = args.get_root(instance)
set_current_instance(instance)
set_subsystem('dump_senders')
pc = find_peopledirectory_catalog(root)
for email in pc['email']._fwd_index.keys():
if ',' in email:
for e in email.split(','):
if not e.isspace() and len(e) != 0:
print e.strip() + " permit_auth_destination"
else:
if not email.isspace() and len(email) != 0:
print email.strip() + " permit_auth_destination"
def usersync(args):
root, closer = args.get_root(args.inst)
set_current_instance(args.inst)
set_subsystem('usersync')
if args.password and args.password_file:
args.parser.error('cannot set both --password and --password-file')
if args.password_file:
with open(args.password_file) as f:
password = f.read().strip('\n')
else:
password = args.password
log.info("Syncing users at %s" % args.url)
sync = UserSync(root)
sync(args.url, args.username, password)
transaction.commit()
def run(root=root):
closer = lambda: None # unit test
try:
if root is None:
root, closer = get_root(app)
# else: unit test
set_subsystem("update_feeds")
update_func(root, config, force=options.force)
except:
tx.abort()
closer()
raise
else:
if options.dryrun:
tx.abort()
else:
tx.commit()
closer()
def run():
set_subsystem('digest')
root, closer = get_root(app)
alerts.send_digests(root, options.frequency)
closer()
def make_karl_instance(name, global_config, uri):
settings = _get_config(global_config, uri)
connstats_file = global_config.get('connection_stats_filename')
connstats_threshhold = float(global_config.get(
'connection_stats_threshhold', 0))
def root_factory(request, name='site'):
def finished(request):
# closing the primary also closes any secondaries opened
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
elapsed = time.time() - before
if elapsed > connstats_threshhold:
loads_after, stores_after = connection.getTransferCounts()
loads = loads_after - loads_before
stores = stores_after - stores_before
with open(connstats_file, 'a', 0) as f:
f.write('"%s", "%s", "%s", %f, %d, %d\n' %
(now,
request.method,
request.path_url,
elapsed,
loads,
stores,
)
)
f.flush()
if connstats_file is not None:
request.add_finished_callback(finished)
# NB: Finished callbacks are executed in the order they've been added
# to the request. pyramid_zodbconn's ``get_connection`` registers a
# finished callback which closes the ZODB database. Because the
# finished callback it registers closes the database, we need it to
# execute after the "finished" function above. As a result, the above
# call to ``request.add_finished_callback`` *must* be executed before
# we call ``get_connection`` below.
# Rationale: we want the call to getTransferCounts() above to happen
# before the ZODB database is closed, because closing the ZODB database
# has the side effect of clearing the transfer counts (the ZODB
# activity monitor clears the transfer counts when the database is
# closed). Having the finished callbacks called in the "wrong" order
# will result in the transfer counts being cleared before the above
# "finished" function has a chance to read their per-request values,
# and they will appear to always be zero.
connection = get_connection(request)
if connstats_file is not None:
before = time.time()
loads_before, stores_before = connection.getTransferCounts()
folder = connection.root()
if name not in folder:
bootstrapper = queryUtility(IBootstrapper, default=populate)
bootstrapper(folder, name, request)
# Use pgtextindex
if 'pgtextindex.dsn' in settings:
site = folder.get(name)
index = lookup(KarlPGTextIndex)(get_weighted_textrepr)
site.catalog['texts'] = index
transaction.commit()
return folder[name]
# Subsystem for logging
set_subsystem('karl')
# Make Pyramid app
#
# Do the configuration dance. If a 'package' setting is present, then
# that package is the customization package and should be used to configure
# the application. Configuration can be done either by loading ZCML or by
# calling a function for configuring Karl imperatively. Imperative
# configuration is preferred with loading of ZCML as a fallback.
# Find package and configuration
pkg_name = settings.get('package', None)
configure_karl = None
if pkg_name:
__import__(pkg_name)
package = sys.modules[pkg_name]
configure_karl = get_imperative_config(package)
if configure_karl is not None:
filename = None
else:
filename = 'configure.zcml'
# BBB Customization packages may be using ZCML style config but
# need configuration done imperatively in core Karl. These
# customizaton packages have generally been written before the
# introduction of imperative style config.
configure_karl = configure_default
else:
package = karl.includes
configure_karl = configure_default
filename = None
if 'tm.attempts' not in settings:
settings['tm.attempts'] = 5
config = Configurator(package=package, settings=settings,
root_factory=root_factory, autocommit=True)
config.begin()
config.include('pyramid_tm')
config.include('pyramid_zodbconn')
if filename is not None:
if configure_karl is not None: # BBB See above
configure_karl(config, load_zcml=False)
config.hook_zca()
config.include('pyramid_zcml')
config.load_zcml(filename)
else:
configure_karl(config)
config.end()
def closer():
registry = config.registry
dbs = getattr(registry, '_zodb_databases', None)
if dbs:
for db in dbs.values():
db.close()
del registry._zodb_databases
app = config.make_wsgi_app()
app.config = settings
app.close = closer
return app
def make_karl_instance(name, global_config, uri):
settings = _get_config(global_config, uri)
def appmaker(folder, name='site'):
if name not in folder:
bootstrapper = queryUtility(IBootstrapper, default=populate)
bootstrapper(folder, name)
# Use pgtextindex
if 'pgtextindex.dsn' in settings:
site = folder.get(name)
index = lookup(KarlPGTextIndex)(get_weighted_textrepr)
site.catalog['texts'] = index
transaction.commit()
return folder[name]
# paster app settings callback
finder = PersistentApplicationFinder(uri, appmaker)
def get_root(request):
return finder(request.environ)
def closer():
db = finder.db
if db is not None:
db.close()
finder.db = None
# Subsystem for logging
set_subsystem('karl')
# Make Pyramid app
#
# Do the configuration dance. If a 'package' setting is present, then
# that package is the customization package and should be used to configure
# the application. Configuration can be done either by loading ZCML or by
# calling a function for configuring Karl imperatively. Imperative
# configuration is preferred with loading of ZCML as a fallback.
# Find package and configuration
pkg_name = settings.get('package', None)
configure_karl = None
if pkg_name:
__import__(pkg_name)
package = sys.modules[pkg_name]
configure_karl = get_imperative_config(package)
if configure_karl is not None:
filename = None
else:
filename = 'configure.zcml'
# BBB Customization packages may be using ZCML style config but
# need configuration done imperatively in core Karl. These
# customizaton packages have generally been written before the
# introduction of imperative style config.
configure_karl = configure_default
else:
package = karl.includes
configure_karl = configure_default
filename = None
config = Configurator(package=package, settings=settings,
root_factory=get_root, autocommit=True)
config.begin()
if filename is not None:
if configure_karl is not None: # BBB See above
configure_karl(config, load_zcml=False)
config.hook_zca()
config.include('pyramid_zcml')
config.load_zcml(filename)
else:
configure_karl(config)
config.end()
app = config.make_wsgi_app()
app.config = settings
app.uri = uri
app.close = closer
return app
