def testConcurrentThreads(self):
DB = testbase.DB
if DB.startswith('sqlite'):
return
conf = ConfigParser.ConfigParser()
conf.add_section('clusto')
conf.set('clusto', 'dsn', testbase.DB)
clusto.connect(conf, echo=testbase.ECHO)
clusto.init_clusto()
firstver = clusto.get_latest_version_number()
threadcount = 5
threads = []
barrier = barrier_creator(threadcount)
for i in range(threadcount):
threads.append(ClustoWorkThread(DB, testbase.ECHO, barrier))
for i in threads:
i.start()
for i in threads:
i.join()
self.assertEqual(clusto.get_latest_version_number(),
threadcount + firstver)
def testConcurrentThreads(self):
DB = testbase.DB
if DB.startswith('sqlite'):
return
conf = ConfigParser.ConfigParser()
conf.add_section('clusto')
conf.set('clusto', 'dsn', testbase.DB)
clusto.connect(conf, echo=testbase.ECHO)
clusto.init_clusto()
firstver = clusto.get_latest_version_number()
threadcount = 5
threads = []
barrier = barrier_creator(threadcount)
for i in range(threadcount):
threads.append(ClustoWorkThread(DB, testbase.ECHO,
barrier))
for i in threads:
i.start()
for i in threads:
i.join()
self.assertEqual(clusto.get_latest_version_number(),
threadcount+firstver)
def setUp(self):
clusto.SESSION.clusto_version = clusto.working_version()
clusto.connect(DB,echo=ECHO)
clusto.clear()
clusto.SESSION.close()
clusto.init_clusto()
self.data()
def setUp(self):
conf = ConfigParser.ConfigParser()
conf.add_section('clusto')
conf.set('clusto', 'dsn', testbase.DB)
clusto.SESSION.clusto_version = clusto.working_version()
clusto.connect(conf,echo=testbase.ECHO)
clusto.METADATA.drop_all(clusto.SESSION.bind)
clusto.clear()
clusto.SESSION.close()
def setUp(self):
conf = ConfigParser.ConfigParser()
conf.add_section('clusto')
conf.set('clusto', 'dsn', testbase.DB)
clusto.SESSION.clusto_version = clusto.working_version()
clusto.connect(conf, echo=testbase.ECHO)
clusto.METADATA.drop_all(clusto.SESSION.bind)
clusto.clear()
clusto.SESSION.close()
def setUp(self):
conf = ConfigParser.ConfigParser()
conf.add_section('clusto')
conf.set('clusto', 'dsn', DB)
clusto.connect(conf,echo=ECHO)
clusto.clear()
clusto.SESSION.close()
clusto.init_clusto()
self.data()
def setUp(self):
conf = ConfigParser.ConfigParser()
conf.add_section('clusto')
conf.set('clusto', 'dsn', DB)
clusto.SESSION.clusto_version = clusto.working_version()
clusto.connect(conf, echo=ECHO)
clusto.clear()
clusto.SESSION.close()
clusto.init_clusto()
self.data()
def setUp(self):
# Mostly cribbed from clusto's test framework
conf = ConfigParser.ConfigParser()
conf.add_section('clusto')
conf.set('clusto', 'dsn', 'sqlite:///:memory:')
clusto.connect(conf)
clusto.clear()
clusto.SESSION.close()
clusto.init_clusto()
self.data()
def init_script(name=os.path.basename(sys.argv[0]), configfile=None, initializedb=False):
"""Initialize the clusto environment for clusto scripts.
Connects to the clusto database, returns a python SafeConfigParser and a
logger.
Uses get_clusto_config and setup_logging
"""
config = get_clusto_config(filename=configfile)
clusto.connect(config=config)
if initializedb:
clusto.init_clusto()
logger = setup_logging(config=config, name=name)
return (config, logger)
def run(self):
clusto.connect(self.conf, echo=self.echo)
clusto.init_clusto()
try:
clusto.begin_transaction()
e = clusto.Entity('foo' + self.getName())
self.barrier()
clusto.commit()
except Exception, x:
clusto.rollback_transaction()
raise x
def run(self):
clusto.connect(self.conf,echo=self.echo)
clusto.init_clusto()
try:
clusto.begin_transaction()
e = clusto.Entity('foo'+self.getName())
self.barrier()
clusto.commit()
except Exception, x:
clusto.rollback_transaction()
raise x
def init_script(self, args, logger=None):
"""
Initialize the clusto environment for clusto scripts.
Connects to the clusto database, returns a python SafeConfigParser
"""
if logger:
self.set_logger(logger)
if "CLUSTOCONFIG" in os.environ:
filename = os.environ["CLUSTOCONFIG"]
else:
filename = args.config
self.config = load_config(filename, args.dsn, logger)
self.debug("Connecting to %s" % self.config.get("clusto", "dsn"))
clusto.connect(self.config)
return self.config
def init_script(self, args, logger=None):
'''
Initialize the clusto environment for clusto scripts.
Connects to the clusto database, returns a python SafeConfigParser
'''
if logger:
self.set_logger(logger)
if 'CLUSTOCONFIG' in os.environ:
filename = os.environ['CLUSTOCONFIG']
else:
filename = args.config
self.config = load_config(filename, args.dsn, logger)
self.debug('Connecting to %s' % self.config.get('clusto', 'dsn'))
clusto.connect(self.config)
return self.config
def init_script(self, args, logger=None):
'''
Initialize the clusto environment for clusto scripts.
Connects to the clusto database, returns a python SafeConfigParser
'''
if logger:
self.set_logger(logger)
if 'CLUSTOCONFIG' in os.environ:
filename = os.environ['CLUSTOCONFIG']
else:
filename = args.config
self.config = load_config(filename, args.dsn, logger)
self.debug('Connecting to %s' % self.config.get('clusto', 'dsn'))
clusto.connect(self.config)
return self.config
def init_script(
name=os.path.basename(sys.argv[0]), configfile=None,
initializedb=False):
"""Initialize the clusto environment for clusto scripts.
Connects to the clusto database, returns a python SafeConfigParser and a
logger.
Uses get_clusto_config and setup_logging
"""
config = get_clusto_config(filename=configfile)
clusto.connect(config=config)
if initializedb:
clusto.init_clusto()
logger = setup_logging(config=config, name=name)
return (config, logger)
from webob import Request, Response
from traceback import format_exc
from urllib import unquote_plus
import re
import os
from clusto.drivers import Driver, IPManager
import clusto.script_helper
import clusto
from clusto.services.config import conf, get_logger
log = get_logger('clusto.http', 'INFO')
conf = clusto.script_helper.load_config(os.environ.get('CLUSTOCONFIG', '/etc/clusto/clusto.conf'))
clusto.connect(conf)
def unclusto(obj, prefetch_attrs=None):
'''
Convert an object to a representation that can be safely serialized into
JSON.
'''
if type(obj) in (str, unicode, int) or obj is None:
return obj
if type(obj) in (list, dict):
return json.dumps(obj)
if isinstance(obj, clusto.Attribute):
return {
'key': obj.key,
'value': unclusto(obj.value),
def _configure(config={}, configfile=None, init_data={}):
"""
Configure the root app
"""
if configfile:
cfg = configfile
else:
cfg = os.environ.get(
'CLUSTOCONFIG',
'/etc/clusto/clusto.conf'
)
cfg = script_helper.load_config(cfg)
clusto.connect(cfg)
# This is an idempotent operation
clusto.init_clusto()
# If init_data is provided, populate it in the clusto database
if init_data:
for name, data in init_data.items():
ent = clusto.get_or_create(
name,
data['driver'],
**data.get('attrs', {})
)
for pool in data.get('member_of', []):
clusto.get_by_name(pool).insert(ent)
for attr in data.get('attr_list', []):
ent.set_attr(**attr)
kwargs = {}
kwargs['host'] = config.get(
'host',
script_helper.get_conf(
cfg, 'apiserver.host', default='127.0.0.1'
),
)
kwargs['port'] = config.get(
'port',
script_helper.get_conf(
cfg, 'apiserver.port', default=9664, datatype=int
),
)
kwargs['server'] = config.get(
'server',
script_helper.get_conf(
cfg, 'apiserver.server', default='wsgiref'
),
)
kwargs['server_kwargs'] = config.get(
'server_kwargs',
script_helper.get_conf(
cfg, 'apiserver.server_kwargs', default={}, datatype=dict
),
)
kwargs['debug'] = config.get(
'debug',
script_helper.get_conf(
cfg, 'apiserver.debug', default=False, datatype=bool
)
)
kwargs['quiet'] = config.get(
'quiet',
script_helper.get_conf(
cfg, 'apiserver.quiet', default=False, datatype=bool
)
)
kwargs['reloader'] = config.get(
'reloader',
script_helper.get_conf(
cfg, 'apiserver.reloader', default=False, datatype=bool
)
)
mount_apps = config.get(
'apps',
script_helper.get_conf(
cfg, 'apiserver.apps', default={}, datatype=dict
)
)
response_headers = config.get(
'response_headers',
script_helper.get_conf(
cfg, 'apiserver.response_headers', default={}, datatype=dict
)
)
root_app.route('/__doc__', 'GET', functools.partial(build_docs))
for mount_point, cls in mount_apps.items():
module = importlib.import_module(cls)
root_app.mount(mount_point, module.app)
# Documentation endpoints
module.app.route('/__doc__', 'GET', functools.partial(build_docs, cls))
module.app.route('/__doc__/', 'GET', functools.partial(build_docs, cls))
# OPTIONS dummy routers
module.app.route('/', 'OPTIONS', functools.partial(options))
module.app.route('/<url:re:.+>', 'OPTIONS', functools.partial(options))
@root_app.hook('before_request')
def enable_response_headers():
for header, value in response_headers.items():
bottle.response.headers[header] = value
return kwargs
def main():
global log
parser = optparse.OptionParser(usage="%prog [options] clusto_query_string",
version=__version__)
parser.add_option('-v', '--verbose', action='count', default=0)
parser.add_option('-f',
'--formatter',
default=r"%name",
help='Formatter to use for printing, default "%default"')
parser.add_option('--list-attributes',
default=False,
action='store_true',
help='Print all the queryable attributes')
parser.add_option('--clusto-config',
default='/etc/clusto/clusto.conf',
help='Path to clusto config file (default %default)')
parser.add_option('--man',
action="store_true",
help="Show more detailed help")
parser.add_option(
'-m',
'--merge-container-attrs',
action='store_true',
help="When showing attributes, merge in parents' attributes")
opts, args = parser.parse_args()
level = logging.WARNING
if opts.verbose == 1:
level = logging.INFO
elif opts.verbose > 1:
level = logging.DEBUG
if opts.man:
print(long_help)
return 0
settings.merge_container_attrs = opts.merge_container_attrs
conf = clusto.script_helper.load_config(opts.clusto_config)
clusto.connect(conf)
# clusto.connect screws up logging for all of its consumers, so we need
# to init logging *after* calling clusto.connect
log = logging.getLogger("clusto-query-logger")
log.propagate = 0
handler = logging.StreamHandler()
handler.setFormatter(
logging.Formatter("%(asctime)s %(levelname)s: %(message)s"))
handler.setLevel(level)
log.addHandler(handler)
log.setLevel(level)
if opts.list_attributes:
all_attrs = [it.attrs() for it in clusto.get_entities()]
print("\n".join(
sorted(
set([
".".join(map(str, (at.key, at.subkey)))
for at in itertools.chain.from_iterable(all_attrs)
]))))
return 0
if not args:
parser.error("No query provided")
raw_query = " ".join(args)
log.info("Going to parse %r", raw_query)
lexed_query = lex(raw_query)
log.info("Lexed into %r", lexed_query)
parsed_query, unparsed = parse_query(lexed_query)
log.info("Parsed into %r", parsed_query)
if unparsed:
log.warning("Unparsed content: %r", unparsed)
return 1
if os.environ.get('CLUSTO_TYPE_FILTER', None) is not None:
def look_for_type(node):
if isinstance(node, (Equality, Inequality)) and\
('type' in node.parameters or 'clusto_type' in node.parameters) and\
'server' in node.parameters:
return True
return False
if not any(look_for_type(node) for node in parsed_query.visit_iter()):
log.debug('Adding intersection with type=%s' %
os.environ['CLUSTO_TYPE_FILTER'])
parsed_query = Intersection(
parsed_query,
Equality('clusto_type', os.environ['CLUSTO_TYPE_FILTER']))
log.info('After intersection, parsed into %r', parsed_query)
else:
log.debug('Not adding type intersection')
# fetch all the hosts
format_template = EasierTemplate(opts.formatter)
context = Context(clusto)
for result_key in sorted(
parsed_query.run(context.entity_map.keys(), context)):
host = context.entity_map[result_key]
print(format_template.substitute(HostFormatter(host, context)))
return 0
def _configure(config={}, configfile=None, init_data={}):
"""
Configure the root app
"""
if configfile:
cfg = configfile
else:
cfg = os.environ.get(
'CLUSTOCONFIG',
'/etc/clusto/clusto.conf'
)
cfg = script_helper.load_config(cfg)
clusto.connect(cfg)
# This is an idempotent operation
clusto.init_clusto()
# If init_data is provided, populate it in the clusto database
if init_data:
for name, data in init_data.items():
ent = clusto.get_or_create(
name,
data['driver'],
**data.get('attrs', {})
)
for pool in data.get('member_of', []):
clusto.get_by_name(pool).insert(ent)
for attr in data.get('attr_list', []):
ent.set_attr(**attr)
kwargs = {}
kwargs['host'] = config.get(
'host',
script_helper.get_conf(
cfg, 'apiserver.host', default='127.0.0.1'
),
)
kwargs['port'] = config.get(
'port',
script_helper.get_conf(
cfg, 'apiserver.port', default=9664, datatype=int
),
)
kwargs['server'] = config.get(
'server',
script_helper.get_conf(
cfg, 'apiserver.server', default='wsgiref'
),
)
kwargs['server_kwargs'] = config.get(
'server_kwargs',
script_helper.get_conf(
cfg, 'apiserver.server_kwargs', default={}, datatype=dict
),
)
kwargs['debug'] = config.get(
'debug',
script_helper.get_conf(
cfg, 'apiserver.debug', default=False, datatype=bool
)
)
kwargs['quiet'] = config.get(
'quiet',
script_helper.get_conf(
cfg, 'apiserver.quiet', default=False, datatype=bool
)
)
kwargs['reloader'] = config.get(
'reloader',
script_helper.get_conf(
cfg, 'apiserver.reloader', default=False, datatype=bool
)
)
mount_apps = config.get(
'apps',
script_helper.get_conf(
cfg, 'apiserver.apps', default={}, datatype=dict
)
)
response_headers = config.get(
'response_headers',
script_helper.get_conf(
cfg, 'apiserver.response_headers', default={}, datatype=dict
)
)
root_app.route('/__doc__', 'GET', functools.partial(build_docs))
for mount_point, cls in mount_apps.items():
module = importlib.import_module(cls)
root_app.mount(mount_point, module.app)
# Documentation endpoints
module.app.route('/__doc__', 'GET', functools.partial(build_docs, cls))
module.app.route('/__doc__/', 'GET', functools.partial(build_docs, cls))
# OPTIONS dummy routers
module.app.route('/', 'OPTIONS', functools.partial(options))
module.app.route('/<url:re:.+>', 'OPTIONS', functools.partial(options))
@root_app.hook('before_request')
def enable_response_headers():
for header, value in response_headers.items():
bottle.response.headers[header] = value
return kwargs
def setUp(self):
clusto.SESSION.clusto_version = clusto.working_version()
clusto.connect(testbase.DB,echo=testbase.ECHO)
clusto.METADATA.drop_all(clusto.SESSION.bind)
clusto.clear()
clusto.SESSION.close()
from webob import Request, Response
from traceback import format_exc
from urllib import unquote_plus
import new
import re
import os
from clusto.drivers import Driver, IPManager
import clusto.script_helper
import clusto
from clusto.services.config import conf, get_logger
log = get_logger('clusto.http', 'INFO')
conf = clusto.script_helper.load_config(os.environ.get('CLUSTOCONFIG', '/etc/clusto/clusto.conf'))
clusto.connect(conf)
def unclusto(obj):
'''
Convert an object to a representation that can be safely serialized into
JSON.
'''
if type(obj) in (str, unicode, int) or obj == None:
return obj
if isinstance(obj, clusto.Attribute):
return {
'key': obj.key,
'value': unclusto(obj.value),
'subkey': obj.subkey,
'number': obj.number,
def _configure(config={}, configfile=None, init_data={}):
"""
Configure the root app
"""
if configfile:
cfg = configfile
else:
cfg = os.environ.get(
'CLUSTOCONFIG',
'/etc/clusto/clusto.conf'
)
cfg = script_helper.load_config(cfg)
clusto.connect(cfg)
# This is an idempotent operation
clusto.init_clusto()
# If init_data is provided, populate it in the clusto database
if init_data:
for name, data in init_data.items():
ent = clusto.get_or_create(
name,
data['driver'],
**data.get('attrs', {})
)
for pool in data.get('member_of', []):
clusto.get_by_name(pool).insert(ent)
kwargs = {}
kwargs['host'] = config.get(
'host',
script_helper.get_conf(
cfg, 'apiserver.host', default='127.0.0.1'
),
)
kwargs['port'] = config.get(
'port',
script_helper.get_conf(
cfg, 'apiserver.port', default=9664, datatype=int
),
)
kwargs['server'] = config.get(
'server',
script_helper.get_conf(
cfg, 'apiserver.server', default='wsgiref'
),
)
kwargs['debug'] = config.get(
'debug',
script_helper.get_conf(
cfg, 'apiserver.debug', default=False, datatype=bool
)
)
kwargs['quiet'] = config.get(
'quiet',
script_helper.get_conf(
cfg, 'apiserver.quiet', default=False, datatype=bool
)
)
kwargs['reloader'] = config.get(
'reloader',
script_helper.get_conf(
cfg, 'apiserver.reloader', default=False, datatype=bool
)
)
mount_apps = config.get(
'apps',
script_helper.get_conf(
cfg, 'apiserver.apps', default={}, datatype=dict
)
)
root_app.route('/__doc__', 'GET', functools.partial(build_docs, '/', __name__))
for mount_point, cls in mount_apps.items():
module = importlib.import_module(cls)
path = '/__doc__%s' % (mount_point,)
root_app.route(path, 'GET', functools.partial(build_docs, path, cls))
root_app.mount(mount_point, module.app)
return kwargs
def main():
global log
parser = optparse.OptionParser(usage="%prog [options] clusto_query_string", version=__version__)
parser.add_option('-v', '--verbose', action='count', default=0)
parser.add_option('-f', '--formatter', default=r"%name",
help='Formatter to use for printing, default "%default"')
parser.add_option('--list-attributes', default=False, action='store_true',
help='Print all the queryable attributes')
parser.add_option('--clusto-config', default='/etc/clusto/clusto.conf',
help='Path to clusto config file (default %default)')
parser.add_option('--man', action="store_true", help="Show more detailed help")
parser.add_option(
'-m', '--merge-container-attrs', action='store_true',
help="When showing attributes, merge in parents' attributes"
)
opts, args = parser.parse_args()
level = logging.WARNING
if opts.verbose == 1:
level = logging.INFO
elif opts.verbose > 1:
level = logging.DEBUG
if opts.man:
print(long_help)
return 0
settings.merge_container_attrs = opts.merge_container_attrs
conf = clusto.script_helper.load_config(opts.clusto_config)
clusto.connect(conf)
# clusto.connect screws up logging for all of its consumers, so we need
# to init logging *after* calling clusto.connect
log = logging.getLogger("clusto-query-logger")
log.propagate = 0
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s: %(message)s"))
handler.setLevel(level)
log.addHandler(handler)
log.setLevel(level)
if opts.list_attributes:
all_attrs = [it.attrs() for it in clusto.get_entities()]
print("\n".join(sorted(set([".".join(map(str, (at.key, at.subkey)))
for at in itertools.chain.from_iterable(all_attrs)]))))
return 0
if not args:
parser.error("No query provided")
raw_query = " ".join(args)
log.info("Going to parse %r", raw_query)
lexed_query = lex(raw_query)
log.info("Lexed into %r", lexed_query)
parsed_query, unparsed = parse_query(lexed_query)
log.info("Parsed into %r", parsed_query)
if unparsed:
log.warning("Unparsed content: %r", unparsed)
return 1
if os.environ.get('CLUSTO_TYPE_FILTER', None) is not None:
def look_for_type(node):
if isinstance(node, (Equality, Inequality)) and\
('type' in node.parameters or 'clusto_type' in node.parameters) and\
'server' in node.parameters:
return True
return False
if not any(look_for_type(node) for node in parsed_query.visit_iter()):
log.debug('Adding intersection with type=%s' % os.environ['CLUSTO_TYPE_FILTER'])
parsed_query = Intersection(
parsed_query,
Equality('clusto_type', os.environ['CLUSTO_TYPE_FILTER'])
)
log.info('After intersection, parsed into %r', parsed_query)
else:
log.debug('Not adding type intersection')
# fetch all the hosts
format_template = EasierTemplate(opts.formatter)
context = Context(clusto)
for result_key in sorted(parsed_query.run(context.entity_map.keys(), context)):
host = context.entity_map[result_key]
print(format_template.substitute(HostFormatter(host, context)))
return 0