def sync_with_canonical(url, dry_run=False, force=False, tree_directory=None):
zk = zc.zk.ZK(ZK_LOCATION)
zk_version = get_zk_version(zk)
if zk_version is None:
logger.critical("ALL STOP, cluster version is None")
if not force:
return
retries = 0
while True:
try:
if url.startswith('svn') or url.startswith('file://'):
vcs = SVN(url)
else:
vcs = GIT(url, tree_directory)
break
except RuntimeError:
retries += 1
if retries > MAX_VCS_RETRIES:
raise
time.sleep(3)
logger.info("VCS Version: " + str(vcs.version))
logger.info("ZK Version: " + str(zk_version))
if zk_version != vcs.version:
if not (force or zk_version is False):
for child in zk.get_children('/hosts'):
host_version = zk.properties('/hosts/' + child)['version']
if host_version != zk_version:
logger.error(
"Version mismatch detected, can't resync since " +
"host %s has not converged (%s -> %s)" %
(child, host_version, zk_version))
return
cluster_lock = zk.client.Lock('/hosts-lock', str(os.getpid()))
if cluster_lock.acquire(0):
try:
logger.info("Version mismatch detected, resyncing")
# Import changes
for fi, contents in vcs:
output = ' '.join(('Importing', fi))
if dry_run:
output += ' (dry run, no action taken)'
logger.info(output)
if not dry_run:
zk.import_tree(contents, trim=fi.endswith('.zk'))
# bump version number
if not dry_run:
zk.properties('/hosts').update(version=vcs.version)
finally:
cluster_lock.release()
else:
logger.error("Refused to update zookeeper tree, "
"couldn't obtain cluster lock")
zk.close()
def set_property(args=None):
if args is None:
args = sys.argv[1:]
connection = args.pop(0)
path = args.pop(0)
zk = zc.zk.ZooKeeper(connection)
def _property(arg):
name, expr = arg.split('=', 1)
return name, eval(expr, {})
zk.properties(path).update(dict(map(_property, args)))
zk.close()
def __init__(self, buildout, name, options):
super(Aggregate, self).__init__(buildout, name, options)
assert name.endswith('.0'), name # There can be only one.
name = name[:-2]
path = '/' + name.replace(',', '/')
name = name.replace(',', '_')
zk = zc.zk.ZK('zookeeper:2181')
options = zk.properties(path)
self['deployment'] = dict(
recipe='zc.recipe.deployment',
name=name,
user=options.get('user', 'zope'),
)
self[name + '.conf'] = dict(
recipe='zc.recipe.deployment:configuration',
deployment='deployment',
directory='/etc/syslog-ng/syslog-ng.d',
text=aggregate_template % dict(
port=options['port'],
name=name,
lpath=options.get(
'path', '/var/log/%s/$R_YEAR-$R_MONTH-$R_DAY.log' % name)),
)
def worker(app, global_conf, zookeeper, path, loggers=None, address=':0',
threads=None, backdoor=False, description=None, version=None,
run=True, **kw):
"""Paste deploy server runner
"""
if loggers:
if re.match(r'\d+$', loggers):
logging.basicConfig(level=int(loggers))
elif loggers in ('CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG'):
logging.basicConfig(level=getattr(logging, loggers))
else:
import ZConfig
ZConfig.configureLoggers(loggers)
zk = zc.zk.ZooKeeper(zookeeper)
address = zc.parse_addr.parse_addr(address)
from zc.resumelb.worker import Worker
worker = Worker(app, address, threads=threads and int(threads),
**kw)
# Set up notification of settings changes.
settings = zk.properties(path)
watcher = gevent.get_hub().loop.async()
watcher.start(lambda : worker.update_settings(settings))
settings(lambda _: watcher.send())
registration_data = {}
if backdoor == 'true':
from gevent import backdoor
bd = backdoor.BackdoorServer(('127.0.0.1', 0), locals())
bd.start()
registration_data['backdoor'] = '127.0.0.1:%s' % bd.server_port
worker.__bd = bd
if description:
registration_data['description'] = description
if version:
registration_data['version'] = version
zk.register_server(path+'/providers', worker.addr, **registration_data)
worker.zk = zk
worker.__zksettings = settings
def shutdown():
zk.close()
worker.shutdown()
gevent.signal(signal.SIGTERM, shutdown)
if run:
try:
worker.server.serve_forever()
finally:
logging.getLogger(__name__+'.worker').info('exiting')
zk.close()
else:
gevent.sleep(.01)
return worker
def flatten(zk, path):
l = len(path) + 1
for p in zk.walk(path):
prefix = p[l:]
if prefix:
prefix += '/'
for n, v in zk.properties(p).items():
if isinstance(v, unicode):
v = str(v)
yield (prefix + n, v)
def main(args=None):
if args is None:
args = sys.argv[1:]
args = parser.parse_args(args)
config = zc.zkdeployment.agent.Configuration(args.configuration)
zk = zc.zk.ZK(args.zookeeper)
try:
host_properties = dict(zk.properties('/hosts/' + config.host_id))
except kazoo.exceptions.NoNodeError:
return error('Host not registered')
zkversion = zk.properties('/hosts', False).get('version')
zk.close()
if zkversion is None:
return warn('Cluster version is None')
try:
with open(os.path.join(config.run_directory, 'status')) as f:
t, _, version, status = f.read().strip().split(None, 3)
except IOError, err:
return error(str(err))
def __init__(self, buildout, name, options):
super(Recipe, self).__init__(buildout, name, options)
assert name.endswith('.0'), name # There can be only one.
name = name[:-2]
zk = self.zk = zc.zk.ZK('zookeeper:2181')
path = '/' + name.replace(',', '/')
zkoptions = zk.properties(path, False)
region = zkoptions.get('region')
self['deployment'] = dict(
recipe = 'zc.recipe.deployment',
name=name,
user=zkoptions.get('user', 'zope'),
)
zim_text = ''
cimaa_text = ''
for k, v in sorted(zkoptions.items()):
if not re.match('[A-Z]', k):
continue
assert len(v) == 2, ("bad option", k, v)
warn, error = v
data = dict(
region=region,
name=(name + '-' + k),
metric=k,
warn=warn,
error=error,
)
zim_text += zim_template % data
cimaa_text += cimaa_template % data
self[name + '-zim.cfg'] = dict(
recipe = 'zc.recipe.deployment:configuration',
deployment = 'deployment',
text = zim_text,
directory = "/etc/zim/agent.d",
)
self[name + '.cfg'] = dict(
recipe = 'zc.recipe.deployment:configuration',
deployment = 'deployment',
text = cimaa_text,
directory = "/etc/cimaa/monitors.d",
)
def worker(path):
import logging
logging.basicConfig()
logger = logging.getLogger(__name__+'-worker')
try:
import zc.resumelb.zk
zk = zc.zk.ZooKeeper()
properties = zk.properties(path)
app = App(properties)
resume_file = 'resume%s.mar' % os.getpid()
if os.path.exists(resume_file):
os.remove(resume_file)
zc.resumelb.zk.worker(
app, {},
zookeeper='127.0.0.1:2181',
path=path+'/lb/workers',
address='127.0.0.1:0',
resume_file=resume_file,
)
except:
logger.exception('worker')
def main(args=None):
if args is None:
args = sys.argv[1:]
[path] = args
logging.basicConfig()
zk = zc.zk.ZooKeeper()
properties = zk.properties(path)
settings = zc.mappingobject.mappingobject(properties)
workers = [zc.thread.Process(worker, args=(path,))
for i in range(settings.workers)]
@zc.thread.Process(args=(path,))
def lb(path):
import logging
logging.basicConfig()
logger = logging.getLogger(__name__+'-lb')
try:
import zc.resumelb.zk
lb, server, logger = zc.resumelb.zk.lbmain(
['-a127.0.0.1:0', '-l', LBLogger(),
'127.0.0.1:2181', '/simul/lb'],
run=False)
logger.lb = lb
server.serve_forever()
except:
logger.exception('lb')
clients_process = zc.thread.Process(clients, args=(path,))
@properties
def update(*a):
while settings.workers > len(workers):
workers.append(zc.thread.Process(worker, args=(path,)))
while settings.workers < len(workers):
workers.pop().terminate()
threading.Event().wait() # sleep forever
def __init__(self, buildout, name, options):
super(Recipe, self).__init__(buildout, name, options)
assert name.endswith('.0'), name # There can be only one.
name = name[:-2]
zk = self.zk = zc.zk.ZK('zookeeper:2181')
path = '/' + name.replace(',', '/')
zkoptions = zk.properties(path, False)
user = zkoptions.get('user', 'zope')
dsn = zkoptions.get('dsn')
command = str(zkoptions['command'])
if dsn:
prefix = 'SENTRY_DSN=%s\n' % dsn
command = "/opt/zkcron/bin/raven_cron %r" % command
else:
prefix = ''
self['deployment'] = dict(
recipe='zc.recipe.deployment',
name=name,
user=user,
)
text = "%(schedule)s %(user)s %(command)s\n#\n" % dict(
user=user,
command=command,
schedule=zkoptions['schedule'],
)
self[name] = dict(
recipe='zc.recipe.deployment:configuration',
deployment='deployment',
text=prefix + text,
directory="/etc/cron.d",
)
def __init__(self, buildout, name, options):
zk = zc.zk.ZK('zookeeper:2181')
zk_options = zk.properties('/' +
name.replace(',', '/').rsplit('.', 1)[0])
port = zk_options['port']
buildout['deployment'] = dict(
recipe='zc.recipe.deployment',
name=name,
user='******',
)
buildout.parse("""
[paste.ini]
recipe = zc.recipe.deployment:configuration
dojo = ${buildout:parts-directory}/dojo
text =
[app:main]
use = egg:bobo
bobo_configure = zc.asanakanban.akb:config
zc.asanakanban.auth:config
bobo_resources = zc.asanakanban.akb
zc.asanakanban.auth
boboserver:static('/dojo', '${:dojo}')
api = us-east-1 zope kanban
url = http://kanban.nova.aws.zope.net
filter-with = reload
[filter:reload]
use = egg:bobo#reload
modules = zc.asanakanban.akb
filter-with = browserid
[filter:browserid]
use = egg:zc.wsgisessions
db-name =
filter-with = zodb
[filter:zodb]
use = egg:zc.zodbwsgi
filter-with = debug
configuration =
<zodb>
<mappingstorage>
</mappingstorage>
</zodb>
initializer = zc.asanakanban.auth:db_init
[filter:debug]
use = egg:bobo#debug
[server:main]
use = egg:zc.asanakanban
port = %(port)s
[web]
recipe = zc.zdaemonrecipe
deployment = deployment
program = ${buildout:bin-directory}/paster serve ${paste.ini:location}
zdaemon.conf =
<runner>
transcript ${deployment:log-directory}/web.log
</runner>
start-test-program = nc -z localhost %(port)s
[rc]
recipe = zc.recipe.rhrc
deployment = deployment
parts = web
chkconfig = 345 99 10
process-management = true
digest = %(digest)s
""" % dict(port=port,
digest=hashlib.sha224(pprint.pformat(
dict(zk_options))).hexdigest()))
def clients(path):
logging.basicConfig()
random.seed(0)
import zc.zk
zk = zc.zk.ZooKeeper()
properties = zk.properties(path)
settings = zc.mappingobject.mappingobject(properties)
siteids = []
@properties
def _(*a):
n = settings.sites
siteids[:] = [0]
for i in range(4):
if n:
siteids.extend(range(n))
n /= 2
wpath = path + '/lb/providers'
zope.testing.wait.wait(lambda : zk.get_children(wpath))
[waddr] = zk.get_children(wpath)
waddr = zc.parse_addr.parse_addr(waddr)
stats = zc.mappingobject.mappingobject(dict(
truncated = 0,
requests = 0,
bypid = {},
nobs = 0,
nhits = 0,
))
spath = path + '/stats'
if not zk.exists(spath):
zk.create(spath, '', zc.zk.OPEN_ACL_UNSAFE)
import gevent.socket
def do_request():
siteid = random.choice(siteids)
oids = set(
int(random.gauss(0, settings.objects_per_site/4))
for i in range(settings.objects_per_request)
)
socket = gevent.socket.create_connection(waddr)
try:
socket.sendall(
request_template % dict(
data='_'.join(map(str, oids)),
host='h%s' % siteid,
)
)
response = ''
while '\r\n\r\n' not in response:
data = socket.recv(9999)
if not data:
stats.truncated += 1
return
response += data
headers, body = response.split('\r\n\r\n')
headers = headers.split('\r\n')
status = headers.pop(0)
headers = dict(l.strip().lower().split(':', 1)
for l in headers if ':' in l)
content_length = int(headers['content-length'])
while len(body) < content_length:
data = socket.recv(9999)
if not data:
stats.truncated += 1
return
body += data
pid, n, nhit, nmiss, nevict = map(int, body.strip().split())
stats.requests += 1
stats.nobs += n
stats.nhits += nhit
bypid = stats.bypid.get(pid)
if bypid is None:
bypid = stats.bypid[pid] = dict(nr=0, n=0, nhit=0)
bypid['nr'] += 1
bypid['n'] += n
bypid['nhit'] += nhit
logger.info(' '.join(map(str, (
100*stats.nhits/stats.nobs,
pid, n, nhit, 100*nhit/n,
))))
finally:
socket.close()
def client():
try:
while 1:
do_request()
except:
print 'client error'
logging.getLogger(__name__+'-client').exception('client')
greenlets = [gevent.spawn(client) for i in range(settings.clients)]
# Set up notification of address changes.
watcher = gevent.get_hub().loop.async()
@watcher.start
def _():
print 'got update event'
while settings.clients > len(greenlets):
greenlets.append(gevent.spawn(client))
while settings.clients < len(greenlets):
greenlets.pop().kill()
properties(lambda a: watcher.send())
while 1:
gevent.sleep(60.0)
def __init__(self, buildout, name, options):
super(ZKRecipe, self).__init__(buildout, name, options)
assert name.endswith('.0'), name # There can be only one.
name = name[:-2]
user = self.user = options.get('user', 'zope')
self['deployment'] = dict(
recipe = 'zc.recipe.deployment',
name=name,
user=user,
)
zk = self.zk = zc.zk.ZK('zookeeper:2181')
path = '/' + name.replace(',', '/')
zk_options = zk.properties(path)
before = zk_options['before']
source_path = zk_options['path']
source_zookeeper = zk_options.get('zookeeper', 'zookeeper:2181')
blobs = zk_options.get('blobs', True)
ddir = '/home/databases'+path+'/'+before
self[name+'data-directory'] = dict(
recipe = 'z3c.recipe.mkdir',
paths = ddir,
user = self.user,
group = self.user,
**{'remove-on-update': 'true'})
self[name+'-storage'] = dict(
recipe = 'zc.zodbrecipes:server',
deployment = 'deployment',
**{
'zeo.conf': zeo_conf % dict(
ddir = ddir,
before = before,
zookeeper = source_zookeeper,
source_path = source_path,
zblob = (
'blob-dir %s/before.blobs\nblob-cache-size 100MB' % ddir
if blobs else ''),
cblob = (
'blob-dir %s/changes.blobs' % ddir
if blobs else ''),
path = path,
),
'shell-script': 'true',
'zdaemon.conf': zdaemon_conf % (
'${deployment:run-directory}/monitor.sock',
path),
})
self['rc'] = dict(
recipe = 'zc.recipe.rhrc',
deployment = 'deployment',
parts = name+'-storage',
chkconfig = '345 99 10',
digest = hashlib.sha1(repr(sorted(zk_options.items()))).hexdigest(),
**{'process-management': 'true'}
)
def lbmain(args=None, run=True):
"""%prog [options] zookeeper_connection path
Run a resume-based load balancer on addr.
"""
if args is None:
args = sys.argv[1:]
elif isinstance(args, str):
args = args.split()
run = False
import optparse
parser = optparse.OptionParser(lbmain.__doc__)
parser.add_option(
'-a', '--address', default=':0',
help="Address to listed on for web requests"
)
parser.add_option(
'-b', '--backlog', type='int',
help="Server backlog setting.")
parser.add_option(
'-d', '--backdoor', action='store_true',
help="Run a backdoor server. Use with caution!")
parser.add_option(
'-e', '--disconnect-message',
help="Path to error page to use when a request is lost due to "
"worker disconnection"
)
parser.add_option(
'-L', '--logger-configuration',
help=
"Read logger configuration from the given configuration file path.\n"
"\n"
"The configuration file must be in ZConfig logger configuration syntax."
"\n"
"Alternatively, you can give a Python logger level name or number."
)
parser.add_option('-l', '--access-logger', help='Access-log logger name.')
parser.add_option(
'-m', '--max-connections', type='int',
help="Maximum number of simultanious accepted connections.")
parser.add_option(
'-r', '--request-classifier', default='zc.resumelb.lb:host_classifier',
help="Request classification function (module:expr)"
)
parser.add_option(
'-s', '--status-server',
help=("Run a status server for getting pool information. "
"The argument is a unix-domain socket path to listen on."))
parser.add_option(
'-t', '--socket-timeout', type='float', default=99.,
help=('HTTP socket timeout.'))
parser.add_option(
'-v', '--single-version', action='store_true',
help=('Only use a single worker version.'))
try:
options, args = parser.parse_args(args)
if len(args) != 2:
print 'Error: must supply a zookeeper connection string and path.'
parser.parse_args(['-h'])
zookeeper, path = args
except SystemExit:
if run:
raise
else:
return
if options.logger_configuration:
logger_config = options.logger_configuration
if re.match(r'\d+$', logger_config):
logging.basicConfig(level=int(logger_config))
elif logger_config in ('CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG'):
logging.basicConfig(level=getattr(logging, logger_config))
else:
import ZConfig
with open(logger_config) as f:
ZConfig.configureLoggers(f.read())
zk = zc.zk.ZooKeeper(zookeeper)
addrs = zk.children(path+'/workers/providers')
rcmod, rcexpr = options.request_classifier.split(':')
__import__(rcmod)
rcmod = sys.modules[rcmod]
request_classifier = eval(rcexpr, rcmod.__dict__)
disconnect_message = options.disconnect_message
if disconnect_message:
with open(disconnect_message) as f:
disconnect_message = f.read()
else:
disconnect_message = zc.resumelb.lb.default_disconnect_message
from zc.resumelb.lb import LB
lb = LB(map(zc.parse_addr.parse_addr, ()),
request_classifier, disconnect_message,
single_version=options.single_version)
to_send = [[]]
# Set up notification of address changes.
awatcher = gevent.get_hub().loop.async()
@awatcher.start
def _():
lb.set_worker_addrs(to_send[0])
if options.single_version:
@addrs
def get_addrs(a):
to_send[0] = dict(
(zc.parse_addr.parse_addr(addr),
zk.get_properties(
path+'/workers/providers/'+addr).get('version')
)
for addr in addrs)
awatcher.send()
else:
@addrs
def get_addrs(a):
to_send[0] = map(zc.parse_addr.parse_addr, addrs)
awatcher.send()
# Set up notification of address changes.
settings = zk.properties(path)
swatcher = gevent.get_hub().loop.async()
swatcher.start(lambda : lb.update_settings(settings))
settings(lambda a: swatcher.send())
lb.zk = zk
lb.__zk = addrs, settings
# Now, start a wsgi server
addr = zc.parse_addr.parse_addr(options.address)
if options.max_connections:
spawn= gevent.pool.Pool(options.max_connections)
else:
spawn = 'default'
if options.access_logger:
accesslog = AccessLog(options.access_logger)
else:
accesslog = None
server = WSGIServer(
addr, lb.handle_wsgi, backlog=options.backlog,
spawn=spawn, log=accesslog, socket_timeout=options.socket_timeout)
server.start()
registration_data = {}
if options.backdoor:
from gevent import backdoor
bd = backdoor.BackdoorServer(('127.0.0.1', 0), locals())
bd.start()
registration_data['backdoor'] = '127.0.0.1:%s' % bd.server_port
status_server = None
if options.status_server:
def status(socket, addr):
pool = lb.pool
writer = socket.makefile('w')
writer.write(json.dumps(
dict(
backlog = pool.backlog,
mean_backlog = pool.mbacklog,
workers = [
(worker.__name__,
worker.backlog,
worker.mbacklog,
(int(worker.oldest_time)
if worker.oldest_time else None),
)
for worker in sorted(
pool.workers, key=lambda w: w.__name__)
]
))+'\n')
writer.close()
socket.close()
status_server_address = options.status_server
if os.path.exists(status_server_address):
os.remove(status_server_address)
sock = gevent.socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.bind(status_server_address)
sock.listen(5)
status_server = gevent.server.StreamServer(sock, status)
status_server.start()
zk.register_server(path+'/providers', (addr[0], server.server_port),
**registration_data)
def shutdown():
zk.close()
server.close()
if status_server is not None:
status_server.close()
lb.shutdown()
gevent.signal(signal.SIGTERM, shutdown)
if run:
try:
server.serve_forever()
finally:
logging.getLogger(__name__+'.lbmain').info('exiting')
zk.close()
else:
gevent.sleep(.01)
return lb, server
def __init__(self, buildout, name, options):
super(ZKRecipe, self).__init__(buildout, name, options)
assert name.endswith('.0'), name # There can be only one.
name = name[:-2]
user = self.user = options.get('user', 'zope')
self['deployment'] = dict(
recipe='zc.recipe.deployment',
name=name,
user=user,
)
zk = self.zk = zc.zk.ZK('zookeeper:2181')
path = '/' + name.replace(',', '/')
zk_options = zk.properties(path)
before = zk_options['before']
source_path = zk_options['path']
source_zookeeper = zk_options.get('zookeeper', 'zookeeper:2181')
blobs = zk_options.get('blobs', True)
ddir = '/home/databases' + path + '/' + before
self[name + 'data-directory'] = dict(recipe='z3c.recipe.mkdir',
paths=ddir,
user=self.user,
group=self.user,
**{'remove-on-update': 'true'})
self[name + '-storage'] = dict(
recipe='zc.zodbrecipes:server',
deployment='deployment',
**{
'zeo.conf':
zeo_conf % dict(
ddir=ddir,
before=before,
zookeeper=source_zookeeper,
source_path=source_path,
zblob=('blob-dir %s/before.blobs\nblob-cache-size 100MB' %
ddir if blobs else ''),
cblob=('blob-dir %s/changes.blobs' %
ddir if blobs else ''),
path=path,
),
'shell-script':
'true',
'zdaemon.conf':
zdaemon_conf %
('${deployment:run-directory}/monitor.sock', path),
})
self['rc'] = dict(recipe='zc.recipe.rhrc',
deployment='deployment',
parts=name + '-storage',
chkconfig='345 99 10',
digest=hashlib.sha1(repr(sorted(
zk_options.items()))).hexdigest(),
**{'process-management': 'true'})
