def upgrade():
conf = cfg.ConfigOpts()
conf.register_cli_opts([
cfg.BoolOpt("skip-index", default=False, help="Skip index upgrade."),
cfg.BoolOpt("skip-storage",
default=False,
help="Skip storage upgrade."),
cfg.BoolOpt("skip-archive-policies-creation",
default=False,
help="Skip default archive policies creation."),
cfg.BoolOpt("create-legacy-resource-types",
default=False,
help="Creation of Ceilometer legacy resource types.")
])
conf = service.prepare_service(conf=conf)
index = indexer.get_driver(conf)
index.connect()
if not conf.skip_index:
LOG.info("Upgrading indexer %s" % index)
index.upgrade(
create_legacy_resource_types=conf.create_legacy_resource_types)
if not conf.skip_storage:
s = storage.get_driver(conf)
LOG.info("Upgrading storage %s" % s)
s.upgrade(index)
if (not conf.skip_archive_policies_creation
and not index.list_archive_policies()
and not index.list_archive_policy_rules()):
for name, ap in six.iteritems(archive_policy.DEFAULT_ARCHIVE_POLICIES):
index.create_archive_policy(ap)
index.create_archive_policy_rule("default", "*", "low")
def start():
conf = service.prepare_service()
for field in ["resource_id", "user_id", "project_id"]:
if conf.statsd[field] is None:
raise cfg.RequiredOptError(field, cfg.OptGroup("statsd"))
stats = Stats(conf)
loop = asyncio.get_event_loop()
# TODO(jd) Add TCP support
listen = loop.create_datagram_endpoint(
lambda: StatsdServer(stats),
local_addr=(conf.statsd.host, conf.statsd.port))
def _flush():
loop.call_later(conf.statsd.flush_delay, _flush)
stats.flush()
loop.call_later(conf.statsd.flush_delay, _flush)
transport, protocol = loop.run_until_complete(listen)
try:
loop.run_forever()
except KeyboardInterrupt:
pass
transport.close()
loop.close()
def metricd():
conf = service.prepare_service()
if (conf.storage.metric_reporting_delay <
conf.storage.metric_processing_delay):
LOG.error("Metric reporting must run less frequently then processing")
sys.exit(0)
signal.signal(signal.SIGTERM, _metricd_terminate)
try:
queues = []
workers = []
for worker in range(conf.metricd.workers):
queue = multiprocessing.Queue()
metric_worker = MetricProcessor(
conf, worker, conf.storage.metric_processing_delay, queue)
metric_worker.start()
queues.append(queue)
workers.append(metric_worker)
metric_report = MetricReporting(conf, 0,
conf.storage.metric_reporting_delay,
queues)
metric_report.start()
workers.append(metric_report)
for worker in workers:
worker.join()
except KeyboardInterrupt:
_metricd_cleanup(workers)
sys.exit(0)
except Exception:
LOG.warning("exiting", exc_info=True)
_metricd_cleanup(workers)
sys.exit(1)
def injector():
conf = cfg.ConfigOpts()
conf.register_cli_opts([
cfg.IntOpt("metrics", default=1, min=1),
cfg.StrOpt("archive-policy-name", default="low"),
cfg.StrOpt("creator", default="admin"),
cfg.IntOpt("batch-of-measures", default=1000),
cfg.IntOpt("measures-per-batch", default=10),
])
conf = service.prepare_service(conf=conf)
index = indexer.get_driver(conf)
instore = incoming.get_driver(conf)
def todo():
metric = index.create_metric(
uuid.uuid4(),
creator=conf.creator,
archive_policy_name=conf.archive_policy_name)
for _ in six.moves.range(conf.batch_of_measures):
measures = [
incoming.Measure(
utils.dt_in_unix_ns(utils.utcnow()), random.random())
for __ in six.moves.range(conf.measures_per_batch)]
instore.add_measures(metric, measures)
with futures.ThreadPoolExecutor(max_workers=conf.metrics) as executor:
for m in six.moves.range(conf.metrics):
executor.submit(todo)
def injector():
conf = cfg.ConfigOpts()
conf.register_cli_opts([
cfg.IntOpt("metrics", default=None),
cfg.IntOpt("batch-of-measures", default=1000),
cfg.IntOpt("measures-per-batch", default=10),
])
conf = service.prepare_service(conf=conf)
index = indexer.get_driver(conf)
index.connect()
s = storage.get_driver(conf)
metrics = index.list_metrics()
if conf.metrics:
metrics = metrics[:conf.metrics]
def todo(metric):
for _ in six.moves.range(conf.batch_of_measures):
measures = [
storage.Measure(utils.to_timestamp(datetime.datetime.now()),
random.random())
for __ in six.moves.range(conf.measures_per_batch)
]
s.add_measures(metric, measures)
with futures.ThreadPoolExecutor(max_workers=len(metrics)) as executor:
# We use 'list' to iterate all threads here to raise the first
# exception now, not much choice
list(executor.map(todo, metrics))
def upgrade():
conf = cfg.ConfigOpts()
conf.register_cli_opts([
cfg.BoolOpt("skip-index", default=False,
help="Skip index upgrade."),
cfg.BoolOpt("skip-storage", default=False,
help="Skip storage upgrade."),
cfg.BoolOpt("skip-archive-policies-creation", default=False,
help="Skip default archive policies creation.")
])
conf = service.prepare_service(conf=conf)
index = indexer.get_driver(conf)
index.connect()
if not conf.skip_index:
LOG.info("Upgrading indexer %s" % index)
index.upgrade()
if not conf.skip_storage:
s = storage.get_driver(conf)
LOG.info("Upgrading storage %s" % s)
s.upgrade(index)
if (not conf.skip_archive_policies_creation
and not index.list_archive_policies()
and not index.list_archive_policy_rules()):
for name, ap in six.iteritems(archive_policy.DEFAULT_ARCHIVE_POLICIES):
index.create_archive_policy(ap)
index.create_archive_policy_rule("default", "*", "low")
def metricd():
conf = service.prepare_service()
if (conf.storage.metric_reporting_delay <
conf.storage.metric_processing_delay):
LOG.error("Metric reporting must run less frequently then processing")
sys.exit(0)
MetricdServiceManager(conf).run()
def metricd():
conf = service.prepare_service()
if (conf.storage.metric_reporting_delay <
conf.storage.metric_processing_delay):
LOG.error("Metric reporting must run less frequently then processing")
sys.exit(0)
signal.signal(signal.SIGTERM, _metricd_terminate)
try:
queues = []
workers = []
for worker in range(conf.metricd.workers):
queue = multiprocessing.Queue()
metric_worker = MetricProcessor(
conf, worker, conf.storage.metric_processing_delay, queue)
metric_worker.start()
queues.append(queue)
workers.append(metric_worker)
metric_report = MetricReporting(
conf, 0, conf.storage.metric_reporting_delay, queues)
metric_report.start()
workers.append(metric_report)
for worker in workers:
worker.join()
except KeyboardInterrupt:
_metricd_cleanup(workers)
sys.exit(0)
except Exception:
LOG.warning("exiting", exc_info=True)
_metricd_cleanup(workers)
sys.exit(1)
def start():
conf = service.prepare_service()
if conf.statsd.resource_id is None:
raise cfg.RequiredOptError("resource_id", cfg.OptGroup("statsd"))
stats = Stats(conf)
loop = asyncio.get_event_loop()
# TODO(jd) Add TCP support
listen = loop.create_datagram_endpoint(
lambda: StatsdServer(stats),
local_addr=(conf.statsd.host, conf.statsd.port))
def _flush():
loop.call_later(conf.statsd.flush_delay, _flush)
stats.flush()
loop.call_later(conf.statsd.flush_delay, _flush)
transport, protocol = loop.run_until_complete(listen)
LOG.info("Started on %s:%d", conf.statsd.host, conf.statsd.port)
LOG.info("Flush delay: %d seconds", conf.statsd.flush_delay)
try:
loop.run_forever()
except KeyboardInterrupt:
pass
transport.close()
loop.close()
def start():
conf = service.prepare_service()
server = proton.reactor.Container(AMQP1Server(conf))
try:
server.run()
except KeyboardInterrupt:
pass
def metricd():
conf = service.prepare_service()
# Check that the storage driver actually needs this daemon to run
s = storage.get_driver_class(conf)
if s.process_measures == storage.StorageDriver.process_measures:
LOG.debug("This storage driver does not need metricd to run, exiting")
return 0
signal.signal(signal.SIGTERM, _metricd_terminate)
try:
workers = []
for worker in range(conf.metricd.workers):
metric_worker = MetricProcessor(conf, worker)
metric_worker.start()
workers.append(metric_worker)
for worker in workers:
worker.join()
except KeyboardInterrupt:
_metricd_cleanup(workers)
sys.exit(0)
except Exception:
LOG.warn("exiting", exc_info=True)
_metricd_cleanup(workers)
sys.exit(1)
def test_aggregation_methods(self):
conf = service.prepare_service([], default_config_files=[])
ap = archive_policy.ArchivePolicy("foobar", 0, [], ["*"])
self.assertEqual(
archive_policy.ArchivePolicy.VALID_AGGREGATION_METHODS,
ap.aggregation_methods)
ap = archive_policy.ArchivePolicy("foobar", 0, [], ["last"])
self.assertEqual(set(["last"]), ap.aggregation_methods)
ap = archive_policy.ArchivePolicy("foobar", 0, [], ["*", "-mean"])
self.assertEqual(
(archive_policy.ArchivePolicy.VALID_AGGREGATION_METHODS -
set(["mean"])), ap.aggregation_methods)
ap = archive_policy.ArchivePolicy("foobar", 0, [], ["-mean", "-last"])
self.assertEqual(
(set(conf.archive_policy.default_aggregation_methods) -
set(["mean", "last"])), ap.aggregation_methods)
ap = archive_policy.ArchivePolicy("foobar", 0, [], ["+12pct"])
self.assertEqual(
(set(conf.archive_policy.default_aggregation_methods).union(
set(["12pct"]))), ap.aggregation_methods)
def metricd():
conf = service.prepare_service()
signal.signal(signal.SIGTERM, _metricd_terminate)
try:
metric_report = MetricReporting(
conf, 0, conf.storage.metric_reporting_delay)
metric_report.start()
workers = [metric_report]
for worker in range(conf.metricd.workers):
metric_worker = MetricProcessor(
conf, worker, conf.storage.metric_processing_delay)
metric_worker.start()
workers.append(metric_worker)
for worker in workers:
worker.join()
except KeyboardInterrupt:
_metricd_cleanup(workers)
sys.exit(0)
except Exception:
LOG.warn("exiting", exc_info=True)
_metricd_cleanup(workers)
sys.exit(1)
def injector():
conf = cfg.ConfigOpts()
conf.register_cli_opts([
cfg.IntOpt("metrics", default=None),
cfg.IntOpt("batch-of-measures", default=1000),
cfg.IntOpt("measures-per-batch", default=10),
])
conf = service.prepare_service(conf=conf)
index = indexer.get_driver(conf)
index.connect()
s = storage.get_driver(conf)
metrics = index.list_metrics()
if conf.metrics:
metrics = metrics[:conf.metrics]
def todo(metric):
for _ in six.moves.range(conf.batch_of_measures):
measures = [
storage.Measure(utils.to_timestamp(datetime.datetime.now()),
random.random())
for __ in six.moves.range(conf.measures_per_batch)]
s.add_measures(metric, measures)
with futures.ThreadPoolExecutor(max_workers=len(metrics)) as executor:
# We use 'list' to iterate all threads here to raise the first
# exception now, not much choice
list(executor.map(todo, metrics))
def setUpClass(self):
super(TestCase, self).setUpClass()
self.conf = service.prepare_service(
[], conf=utils.prepare_conf(),
default_config_files=[],
logging_level=logging.DEBUG)
if not os.getenv("GNOCCHI_TEST_DEBUG"):
daiquiri.setup(outputs=[])
py_root = os.path.abspath(os.path.join(os.path.dirname(__file__),
'..',))
self.conf.set_override('paste_config',
os.path.join(py_root, 'rest', 'api-paste.ini'),
group="api")
self.conf.set_override('policy_file',
os.path.join(py_root, 'rest', 'policy.json'),
group="oslo_policy")
# NOTE(jd) This allows to test S3 on AWS
if not os.getenv("AWS_ACCESS_KEY_ID"):
self.conf.set_override('s3_endpoint_url',
os.getenv("GNOCCHI_STORAGE_HTTP_URL"),
group="storage")
self.conf.set_override('s3_access_key_id', "gnocchi",
group="storage")
self.conf.set_override('s3_secret_access_key', "anythingworks",
group="storage")
self.index = indexer.get_driver(self.conf)
self.coord = metricd.get_coordinator_and_start(
str(uuid.uuid4()),
self.conf.coordination_url)
# NOTE(jd) So, some driver, at least SQLAlchemy, can't create all
# their tables in a single transaction even with the
# checkfirst=True, so what we do here is we force the upgrade code
# path to be sequential to avoid race conditions as the tests run
# in parallel.
with self.coord.get_lock(b"gnocchi-tests-db-lock"):
self.index.upgrade()
self.archive_policies = self.ARCHIVE_POLICIES.copy()
for name, ap in six.iteritems(self.archive_policies):
# Create basic archive policies
try:
self.index.create_archive_policy(ap)
except indexer.ArchivePolicyAlreadyExists:
pass
storage_driver = os.getenv("GNOCCHI_TEST_STORAGE_DRIVER", "file")
self.conf.set_override('driver', storage_driver, 'storage')
if storage_driver == 'ceph':
self.conf.set_override('ceph_conffile',
os.getenv("CEPH_CONF"),
'storage')
def start_fixture(self):
"""Create necessary temp files and do the config dance."""
global CONF
data_tmp_dir = tempfile.mkdtemp(prefix='gnocchi')
coordination_dir = os.path.join(data_tmp_dir, 'tooz')
os.mkdir(coordination_dir)
coordination_url = 'file://%s' % coordination_dir
conf = service.prepare_service([])
CONF = self.conf = conf
self.tmp_dir = data_tmp_dir
# Use the indexer set in the conf, unless we have set an
# override via the environment.
if 'GNOCCHI_TEST_INDEXER_URL' in os.environ:
conf.set_override('url',
os.environ.get("GNOCCHI_TEST_INDEXER_URL"),
'indexer')
# TODO(jd) It would be cool if Gabbi was able to use the null://
# indexer, but this makes the API returns a lot of 501 error, which
# Gabbi does not want to see, so let's just disable it.
if conf.indexer.url is None or conf.indexer.url == "null://":
raise case.SkipTest("No indexer configured")
# Use the presence of DEVSTACK_GATE_TEMPEST as a semaphore
# to signal we are not in a gate driven functional test
# and thus should override conf settings.
if 'DEVSTACK_GATE_TEMPEST' not in os.environ:
conf.set_override('driver', 'file', 'storage')
conf.set_override('coordination_url', coordination_url, 'storage')
conf.set_override('policy_file',
os.path.abspath('etc/gnocchi/policy.json'),
group="oslo_policy")
conf.set_override('file_basepath', data_tmp_dir, 'storage')
# NOTE(jd) All of that is still very SQL centric but we only support
# SQL for now so let's say it's good enough.
url = sqlalchemy_url.make_url(conf.indexer.url)
url.database = url.database + str(uuid.uuid4()).replace('-', '')
db_url = str(url)
conf.set_override('url', db_url, 'indexer')
sqlalchemy_utils.create_database(db_url)
index = indexer.get_driver(conf)
index.connect()
index.upgrade()
conf.set_override('pecan_debug', False, 'api')
# Turn off any middleware.
conf.set_override('middlewares', [], 'api')
self.index = index
def test_aggregation_methods(self):
conf = service.prepare_service([],
default_config_files=[],
logging_level=logging.DEBUG,
skip_log_opts=True)
ap = archive_policy.ArchivePolicy("foobar",
0,
[],
["*"])
self.assertEqual(
archive_policy.ArchivePolicy.VALID_AGGREGATION_METHODS,
ap.aggregation_methods)
ap = archive_policy.ArchivePolicy("foobar",
0,
[],
["last"])
self.assertEqual(
set(["last"]),
ap.aggregation_methods)
ap = archive_policy.ArchivePolicy("foobar",
0,
[],
["*", "-mean"])
self.assertEqual(
(archive_policy.ArchivePolicy.VALID_AGGREGATION_METHODS
- set(["mean"])),
ap.aggregation_methods)
ap = archive_policy.ArchivePolicy("foobar",
0,
[],
["-mean", "-last"])
self.assertEqual(
(set(conf.archive_policy.default_aggregation_methods)
- set(["mean", "last"])),
ap.aggregation_methods)
ap = archive_policy.ArchivePolicy("foobar",
0,
[],
["+12pct"])
self.assertEqual(
(set(conf.archive_policy.default_aggregation_methods)
.union(set(["12pct"]))),
ap.aggregation_methods)
ap = archive_policy.ArchivePolicy("foobar",
0,
[],
["+rate:last"])
self.assertEqual(
(set(conf.archive_policy.default_aggregation_methods)
.union(set(["rate:last"]))),
ap.aggregation_methods)
def api():
# Compat with previous pbr script
try:
double_dash = sys.argv.index("--")
except ValueError:
double_dash = None
else:
sys.argv.pop(double_dash)
conf = cfg.ConfigOpts()
for opt in app.API_OPTS:
# NOTE(jd) Register the API options without a default, so they are only
# used to override the one in the config file
c = copy.copy(opt)
c.default = None
conf.register_cli_opt(c)
conf = service.prepare_service(conf=conf)
if double_dash is not None:
# NOTE(jd) Wait to this stage to log so we're sure the logging system
# is in place
LOG.warning(
"No need to pass `--' in gnocchi-api command line anymore, "
"please remove")
uwsgi = spawn.find_executable("uwsgi")
if not uwsgi:
LOG.error("Unable to find `uwsgi'.\n"
"Be sure it is installed and in $PATH.")
return 1
workers = utils.get_default_workers()
return os.execl(
uwsgi,
uwsgi,
"--http",
"%s:%d" % (conf.host or conf.api.host, conf.port or conf.api.port),
"--master",
"--enable-threads",
"--die-on-term",
# NOTE(jd) See https://github.com/gnocchixyz/gnocchi/issues/156
"--add-header",
"Connection: close",
"--processes",
str(math.floor(workers * 1.5)),
"--threads",
str(workers),
"--lazy-apps",
"--chdir",
"/",
"--wsgi",
"gnocchi.rest.wsgi",
"--pyargv",
" ".join(sys.argv[1:]),
)
def __init__(self, url, count, conf):
super(Amqp, self).__init__()
self.url = url
self.expected = count
self.received = 0
self.conf = conf
self.incoming = incoming.get_driver(self.conf)
self.indexer = indexer.get_driver(self.conf)
self.gauges = {}
self.counters = {}
self.absolute = {}
self.conf = service.prepare_service()
self.peer_close_is_error = True
def start():
conf = service.prepare_service()
if conf.amqp1d.resource_name is None:
raise cfg.RequiredOptError("resource_name", cfg.OptGroup("amqp1d"))
try:
if conf.amqp1d.data_source == "collectd":
stats = CollectdStats(conf)
Container(AMQP1Server(conf, stats)).run()
else:
raise ValueError(
"Unknown data source type '%s'" %
conf.amqp1d.data_source)
except KeyboardInterrupt:
pass
def metricd():
conf = cfg.ConfigOpts()
conf.register_cli_opts([
cfg.IntOpt("stop-after-processing-metrics",
default=0,
min=0,
help="Number of metrics to process without workers, "
"for testing purpose"),
])
conf = service.prepare_service(conf=conf)
if conf.stop_after_processing_metrics:
metricd_tester(conf)
else:
MetricdServiceManager(conf).run()
def metricd():
conf = cfg.ConfigOpts()
conf.register_cli_opts([
cfg.IntOpt("stop-after-processing-metrics",
default=0,
min=0,
help="Number of metrics to process without workers, "
"for testing purpose"),
])
conf = service.prepare_service(conf=conf)
if conf.stop_after_processing_metrics:
metricd_tester(conf)
else:
MetricdServiceManager(conf).run()
def prepare_service(conf=None):
if conf is None:
conf = cfg.ConfigOpts()
opts.set_defaults()
policy_opts.set_defaults(conf)
conf = service.prepare_service(conf=conf)
cfg_path = conf.oslo_policy.policy_file
if not os.path.isabs(cfg_path):
cfg_path = conf.find_file(cfg_path)
if cfg_path is None or not os.path.exists(cfg_path):
cfg_path = os.path.abspath(os.path.join(os.path.dirname(__file__),
'..', 'rest', 'policy.json'))
conf.set_default('policy_file', cfg_path, group='oslo_policy')
return conf
def prepare_service(conf=None):
if conf is None:
conf = cfg.ConfigOpts()
opts.set_defaults()
policy_opts.set_defaults(conf)
conf = service.prepare_service(conf=conf)
cfg_path = conf.oslo_policy.policy_file
if not os.path.isabs(cfg_path):
cfg_path = conf.find_file(cfg_path)
if cfg_path is None or not os.path.exists(cfg_path):
cfg_path = os.path.abspath(os.path.join(os.path.dirname(__file__),
'..', 'rest', 'policy.json'))
conf.set_default('policy_file', cfg_path, group='oslo_policy')
return conf
def test_aggregation_methods(self):
conf = service.prepare_service([])
ap = archive_policy.ArchivePolicy("foobar",
0,
[],
["*"])
self.assertEqual(
archive_policy.ArchivePolicy.VALID_AGGREGATION_METHODS,
ap.aggregation_methods)
ap = archive_policy.ArchivePolicy("foobar",
0,
[],
["last"])
self.assertEqual(
set(["last"]),
ap.aggregation_methods)
ap = archive_policy.ArchivePolicy("foobar",
0,
[],
["*", "-mean"])
self.assertEqual(
(archive_policy.ArchivePolicy.VALID_AGGREGATION_METHODS
- set(["mean"])),
ap.aggregation_methods)
ap = archive_policy.ArchivePolicy("foobar",
0,
[],
["-mean", "-last"])
self.assertEqual(
(set(conf.archive_policy.default_aggregation_methods)
- set(["mean", "last"])),
ap.aggregation_methods)
ap = archive_policy.ArchivePolicy("foobar",
0,
[],
["+12pct"])
self.assertEqual(
(set(conf.archive_policy.default_aggregation_methods)
.union(set(["12pct"]))),
ap.aggregation_methods)
def upgrade():
conf = cfg.ConfigOpts()
conf.register_cli_opts([
cfg.BoolOpt("skip-index", default=False,
help="Skip index upgrade."),
cfg.BoolOpt("skip-storage", default=False,
help="Skip storage upgrade.")
])
conf = service.prepare_service(conf=conf)
if not conf.skip_index:
index = indexer.get_driver(conf)
index.connect()
LOG.info("Upgrading indexer %s" % index)
index.upgrade()
if not conf.skip_storage:
s = storage.get_driver(conf)
LOG.info("Upgrading storage %s" % s)
s.upgrade(index)
def upgrade():
conf = cfg.ConfigOpts()
sack_number_opt = copy.copy(_SACK_NUMBER_OPT)
sack_number_opt.default = 128
conf.register_cli_opts([
cfg.BoolOpt("skip-index", default=False, help="Skip index upgrade."),
cfg.BoolOpt("skip-storage",
default=False,
help="Skip storage upgrade."),
cfg.BoolOpt("skip-incoming",
default=False,
help="Skip incoming storage upgrade."),
cfg.BoolOpt("skip-archive-policies-creation",
default=False,
help="Skip default archive policies creation."),
sack_number_opt,
])
conf = service.prepare_service(conf=conf, log_to_std=True)
if not conf.skip_index:
index = indexer.get_driver(conf)
LOG.info("Upgrading indexer %s", index)
index.upgrade()
if not conf.skip_storage:
# FIXME(jd) Pass None as coordinator because it's not needed in this
# case. This will be removed when the storage will stop requiring a
# coordinator object.
s = storage.get_driver(conf, None)
LOG.info("Upgrading storage %s", s)
s.upgrade()
if not conf.skip_incoming:
i = incoming.get_driver(conf)
LOG.info("Upgrading incoming storage %s", i)
i.upgrade(conf.sacks_number)
if (not conf.skip_archive_policies_creation
and not index.list_archive_policies()
and not index.list_archive_policy_rules()):
if conf.skip_index:
index = indexer.get_driver(conf)
for name, ap in six.iteritems(archive_policy.DEFAULT_ARCHIVE_POLICIES):
index.create_archive_policy(ap)
index.create_archive_policy_rule("default", "*", "low")
def injector():
conf = cfg.ConfigOpts()
conf.register_cli_opts([
cfg.IntOpt("--measures", help="Measures per metric."),
cfg.IntOpt("--metrics", help="Number of metrics to create."),
cfg.IntOpt("--archive-policy-name",
help="Name of archive policy to use.",
default="low"),
cfg.IntOpt("--interval",
help="Interval to sleep between metrics sending."),
cfg.BoolOpt("--process",
default=False,
help="Process the ingested measures."),
])
return _inject(service.prepare_service(conf=conf, log_to_std=True),
metrics=conf.metrics,
measures=conf.measures,
archive_policy_name=conf.archive_policy_name,
process=conf.process,
interval=conf.interval)
def change_sack_size():
conf = cfg.ConfigOpts()
conf.register_cli_opts([_SACK_NUMBER_OPT])
conf = service.prepare_service(conf=conf, log_to_std=True)
s = incoming.get_driver(conf)
try:
report = s.measures_report(details=False)
except incoming.SackDetectionError:
# issue is already logged by NUM_SACKS, abort.
return
remainder = report['summary']['measures']
if remainder:
LOG.error(
'Cannot change sack when non-empty backlog. Process '
'remaining %s measures and try again', remainder)
return
LOG.info("Changing sack size to: %s", conf.sacks_number)
old_num_sacks = s.get_storage_sacks()
s.set_storage_settings(conf.sacks_number)
s.remove_sack_group(old_num_sacks)
def change_sack_size():
conf = cfg.ConfigOpts()
conf.register_cli_opts([_SACK_NUMBER_OPT])
conf = service.prepare_service(conf=conf, log_to_std=True)
s = incoming.get_driver(conf)
try:
report = s.measures_report(details=False)
except incoming.SackDetectionError:
LOG.error('Unable to detect the number of storage sacks.\n'
'Ensure gnocchi-upgrade has been executed.')
return
remainder = report['summary']['measures']
if remainder:
LOG.error('Cannot change sack when non-empty backlog. Process '
'remaining %s measures and try again', remainder)
return
LOG.info("Removing current %d sacks", s.NUM_SACKS)
s.remove_sacks()
LOG.info("Creating new %d sacks", conf.sacks_number)
s.upgrade(conf.sacks_number)
def start():
conf = service.prepare_service()
stats = Stats(conf)
loop = asyncio.get_event_loop()
# TODO(jd) Add TCP support
listen = loop.create_datagram_endpoint(
# TODO(jd) Add config options for host/port
lambda: StatsdServer(stats), local_addr=('0.0.0.0', 8125))
loop.call_later(conf.statsd.flush_delay, stats.flush)
transport, protocol = loop.run_until_complete(listen)
try:
loop.run_forever()
except KeyboardInterrupt:
pass
transport.close()
loop.close()
def upgrade():
conf = cfg.ConfigOpts()
sack_number_opt = copy.copy(_SACK_NUMBER_OPT)
sack_number_opt.default = 128
conf.register_cli_opts([
cfg.BoolOpt("skip-index", default=False, help="Skip index upgrade."),
cfg.BoolOpt("skip-storage",
default=False,
help="Skip storage upgrade."),
cfg.BoolOpt("skip-incoming",
default=False,
help="Skip incoming storage upgrade."),
cfg.BoolOpt("skip-archive-policies-creation",
default=False,
help="Skip default archive policies creation."),
sack_number_opt,
])
conf = service.prepare_service(conf=conf, log_to_std=True)
if not conf.skip_index:
index = indexer.get_driver(conf)
index.connect()
LOG.info("Upgrading indexer %s", index)
index.upgrade()
if not conf.skip_storage:
s = storage.get_driver(conf)
LOG.info("Upgrading storage %s", s)
s.upgrade()
if not conf.skip_incoming:
i = incoming.get_driver(conf)
LOG.info("Upgrading incoming storage %s", i)
i.upgrade(conf.sacks_number)
if (not conf.skip_archive_policies_creation
and not index.list_archive_policies()
and not index.list_archive_policy_rules()):
if conf.skip_index:
index = indexer.get_driver(conf)
index.connect()
for name, ap in six.iteritems(archive_policy.DEFAULT_ARCHIVE_POLICIES):
index.create_archive_policy(ap)
index.create_archive_policy_rule("default", "*", "low")
def injector():
conf = cfg.ConfigOpts()
conf.register_cli_opts([
cfg.IntOpt("--measures",
help="Measures per metric."),
cfg.IntOpt("--metrics",
help="Number of metrics to create."),
cfg.IntOpt("--archive-policy-name",
help="Name of archive policy to use.",
default="low"),
cfg.IntOpt("--interval",
help="Interval to sleep between metrics sending."),
cfg.BoolOpt("--process", default=False,
help="Process the ingested measures."),
])
return _inject(service.prepare_service(conf=conf, log_to_std=True),
metrics=conf.metrics,
measures=conf.measures,
archive_policy_name=conf.archive_policy_name,
process=conf.process,
interval=conf.interval)
def change_sack_size():
conf = cfg.ConfigOpts()
conf.register_cli_opts([_SACK_NUMBER_OPT])
conf = service.prepare_service(conf=conf, log_to_std=True)
s = incoming.get_driver(conf)
try:
report = s.measures_report(details=False)
except incoming.SackDetectionError:
LOG.error('Unable to detect the number of storage sacks.\n'
'Ensure gnocchi-upgrade has been executed.')
return
remainder = report['summary']['measures']
if remainder:
LOG.error(
'Cannot change sack when non-empty backlog. Process '
'remaining %s measures and try again', remainder)
return
LOG.info("Removing current %d sacks", s.NUM_SACKS)
s.remove_sacks()
LOG.info("Creating new %d sacks", conf.sacks_number)
s.upgrade(conf.sacks_number)
def setUpClass(self):
super(TestCase, self).setUpClass()
self.conf = service.prepare_service([],
default_config_files=[])
self.conf.set_override('policy_file',
self.path_get('etc/gnocchi/policy.json'),
group="oslo_policy")
self.index = indexer.get_driver(self.conf)
self.index.connect()
# NOTE(jd) So, some driver, at least SQLAlchemy, can't create all
# their tables in a single transaction even with the
# checkfirst=True, so what we do here is we force the upgrade code
# path to be sequential to avoid race conditions as the tests run
# in parallel.
self.coord = coordination.get_coordinator(
self.conf.storage.coordination_url,
str(uuid.uuid4()).encode('ascii'))
self.coord.start(start_heart=True)
with self.coord.get_lock(b"gnocchi-tests-db-lock"):
self.index.upgrade()
self.coord.stop()
self.archive_policies = self.ARCHIVE_POLICIES.copy()
self.archive_policies.update(archive_policy.DEFAULT_ARCHIVE_POLICIES)
for name, ap in six.iteritems(self.archive_policies):
# Create basic archive policies
try:
self.index.create_archive_policy(ap)
except indexer.ArchivePolicyAlreadyExists:
pass
self.conf.set_override(
'driver',
os.getenv("GNOCCHI_TEST_STORAGE_DRIVER", "null"),
'storage')
def upgrade():
conf = cfg.ConfigOpts()
sack_number_opt = copy.copy(_SACK_NUMBER_OPT)
sack_number_opt.default = 128
conf.register_cli_opts([
cfg.BoolOpt("skip-index", default=False,
help="Skip index upgrade."),
cfg.BoolOpt("skip-storage", default=False,
help="Skip storage upgrade."),
cfg.BoolOpt("skip-incoming", default=False,
help="Skip incoming storage upgrade."),
cfg.BoolOpt("skip-archive-policies-creation", default=False,
help="Skip default archive policies creation."),
sack_number_opt,
])
conf = service.prepare_service(conf=conf, log_to_std=True)
if not conf.skip_index:
index = indexer.get_driver(conf)
LOG.info("Upgrading indexer %s", index)
index.upgrade()
if not conf.skip_storage:
s = storage.get_driver(conf)
LOG.info("Upgrading storage %s", s)
s.upgrade()
if not conf.skip_incoming:
i = incoming.get_driver(conf)
LOG.info("Upgrading incoming storage %s", i)
i.upgrade(conf.sacks_number)
if (not conf.skip_archive_policies_creation
and not index.list_archive_policies()
and not index.list_archive_policy_rules()):
if conf.skip_index:
index = indexer.get_driver(conf)
for name, ap in six.iteritems(archive_policy.DEFAULT_ARCHIVE_POLICIES):
index.create_archive_policy(ap)
index.create_archive_policy_rule("default", "*", "low")
def storage_dbsync():
service.prepare_service()
indexer = sql_db.SQLAlchemyIndexer(cfg.CONF)
indexer.upgrade()
def api():
service.prepare_service()
app.build_server()
def build_server():
conf = service.prepare_service()
serving.run_simple(conf.api.host,
conf.api.port,
WerkzeugApp(conf),
processes=conf.api.workers)
def metricd():
conf = service.prepare_service()
MetricdServiceManager(conf).run()
def start_fixture(self):
"""Create necessary temp files and do the config dance."""
global LOAD_APP_KWARGS
if not os.getenv("GNOCCHI_TEST_DEBUG"):
self.output = base.CaptureOutput()
self.output.setUp()
data_tmp_dir = tempfile.mkdtemp(prefix='gnocchi')
if os.getenv("GABBI_LIVE"):
dcf = None
else:
dcf = []
conf = service.prepare_service([],
conf=utils.prepare_conf(),
default_config_files=dcf,
logging_level=logging.DEBUG,
skip_log_opts=True)
py_root = os.path.abspath(
os.path.join(
os.path.dirname(__file__),
'..',
'..',
))
conf.set_override('paste_config',
os.path.join(py_root, 'rest', 'api-paste.ini'),
group="api")
conf.set_override('policy_file',
os.path.join(py_root, 'rest', 'policy.yaml'),
group="oslo_policy")
# NOTE(sileht): This is not concurrency safe, but only this tests file
# deal with cors, so we are fine. set_override don't work because cors
# group doesn't yet exists, and we the CORS middleware is created it
# register the option and directly copy value of all configurations
# options making impossible to override them properly...
cfg.set_defaults(cors.CORS_OPTS, allowed_origin="http://foobar.com")
self.conf = conf
self.tmp_dir = data_tmp_dir
if conf.indexer.url is None:
raise case.SkipTest("No indexer configured")
storage_driver = os.getenv("GNOCCHI_TEST_STORAGE_DRIVER", "file")
conf.set_override('driver', storage_driver, 'storage')
if conf.storage.driver == 'file':
conf.set_override('file_basepath', data_tmp_dir, 'storage')
elif conf.storage.driver == 'ceph':
conf.set_override('ceph_conffile', os.getenv("CEPH_CONF"),
'storage')
self.ceph_pool_name = uuid.uuid4().hex
with open(os.devnull, 'w') as f:
subprocess.call(("ceph -c %s osd pool create %s "
"16 16 replicated") %
(os.getenv("CEPH_CONF"), self.ceph_pool_name),
shell=True,
stdout=f,
stderr=subprocess.STDOUT)
subprocess.call(("ceph -c %s osd pool application "
"enable %s rbd") %
(os.getenv("CEPH_CONF"), self.ceph_pool_name),
shell=True,
stdout=f,
stderr=subprocess.STDOUT)
conf.set_override('ceph_pool', self.ceph_pool_name, 'storage')
elif conf.storage.driver == "s3":
conf.set_override('s3_endpoint_url',
os.getenv("GNOCCHI_STORAGE_HTTP_URL"),
group="storage")
conf.set_override('s3_access_key_id', "gnocchi", group="storage")
conf.set_override('s3_secret_access_key',
"anythingworks",
group="storage")
conf.set_override("s3_bucket_prefix",
str(uuid.uuid4())[:26], "storage")
elif conf.storage.driver == "swift":
# NOTE(sileht): This fixture must start before any driver stuff
swift_fixture = fixtures.MockPatch('swiftclient.client.Connection',
base.FakeSwiftClient)
swift_fixture.setUp()
# NOTE(jd) All of that is still very SQL centric but we only support
# SQL for now so let's say it's good enough.
conf.set_override(
'url',
sqlalchemy.SQLAlchemyIndexer._create_new_database(
conf.indexer.url), 'indexer')
index = indexer.get_driver(conf)
index.upgrade()
# Set pagination to a testable value
conf.set_override('max_limit', 7, 'api')
conf.set_override('enable_proxy_headers_parsing', True, group="api")
self.index = index
self.coord = metricd.get_coordinator_and_start(str(uuid.uuid4()),
conf.coordination_url)
s = storage.get_driver(conf)
i = incoming.get_driver(conf)
if conf.storage.driver == 'redis':
# Create one prefix per test
s.STORAGE_PREFIX = str(uuid.uuid4()).encode()
if conf.incoming.driver == 'redis':
i.SACK_NAME_FORMAT = (str(uuid.uuid4()) +
incoming.IncomingDriver.SACK_NAME_FORMAT)
self.fixtures = [
fixtures.MockPatch("gnocchi.storage.get_driver", return_value=s),
fixtures.MockPatch("gnocchi.incoming.get_driver", return_value=i),
fixtures.MockPatch("gnocchi.indexer.get_driver",
return_value=self.index),
fixtures.MockPatch("gnocchi.cli.metricd.get_coordinator_and_start",
return_value=self.coord),
]
for f in self.fixtures:
f.setUp()
if conf.storage.driver == 'swift':
self.fixtures.append(swift_fixture)
LOAD_APP_KWARGS = {
'conf': conf,
}
s.upgrade()
i.upgrade(128)
# start up a thread to async process measures
self.metricd_thread = MetricdThread(chef.Chef(self.coord, i, index, s))
self.metricd_thread.start()
def build_wsgi_app():
return load_app(service.prepare_service())
def app_factory(global_config, **local_conf):
cfg = service.prepare_service()
return setup_app(None, cfg=cfg)
def setUp(self):
super(TestCase, self).setUp()
self.conf = service.prepare_service([],
default_config_files=[])
self.conf.set_override('policy_file',
self.path_get('etc/gnocchi/policy.json'),
group="oslo_policy")
self.index = indexer.get_driver(self.conf)
self.index.connect()
# NOTE(jd) So, some driver, at least SQLAlchemy, can't create all
# their tables in a single transaction even with the
# checkfirst=True, so what we do here is we force the upgrade code
# path to be sequential to avoid race conditions as the tests run
# in parallel.
self.coord = coordination.get_coordinator(
self.conf.storage.coordination_url,
str(uuid.uuid4()).encode('ascii'))
self.coord.start()
with self.coord.get_lock(b"gnocchi-tests-db-lock"):
# Force upgrading using Alembic rather than creating the
# database from scratch so we are sure we don't miss anything
# in the Alembic upgrades. We have a test to check that
# upgrades == create but it misses things such as custom CHECK
# constraints.
self.index.upgrade(nocreate=True)
self.coord.stop()
self.archive_policies = self.ARCHIVE_POLICIES
# Used in gnocchi.gendoc
if not getattr(self, "skip_archive_policies_creation", False):
for name, ap in six.iteritems(self.ARCHIVE_POLICIES):
# Create basic archive policies
try:
self.index.create_archive_policy(ap)
except indexer.ArchivePolicyAlreadyExists:
pass
if swexc:
self.useFixture(mockpatch.Patch(
'swiftclient.client.Connection',
FakeSwiftClient))
self.useFixture(mockpatch.Patch('gnocchi.storage.ceph.rados',
FakeRadosModule()))
self.conf.set_override(
'driver',
os.getenv("GNOCCHI_TEST_STORAGE_DRIVER", "null"),
'storage')
if self.conf.storage.driver == 'file':
tempdir = self.useFixture(fixtures.TempDir())
self.conf.set_override('file_basepath',
tempdir.path,
'storage')
elif self.conf.storage.driver == 'influxdb':
self.conf.set_override('influxdb_block_until_data_ingested', True,
'storage')
self.conf.set_override('influxdb_database', 'test', 'storage')
self.conf.set_override('influxdb_password', 'root', 'storage')
self.conf.set_override('influxdb_port',
os.getenv("GNOCCHI_TEST_INFLUXDB_PORT",
51234), 'storage')
# NOTE(ityaptin) Creating unique database for every test may cause
# tests failing by timeout, but in may be useful in some cases
if os.getenv("GNOCCHI_TEST_INFLUXDB_UNIQUE_DATABASES"):
self.conf.set_override("influxdb_database",
"gnocchi_%s" % uuid.uuid4().hex,
'storage')
self.storage = storage.get_driver(self.conf)
# NOTE(jd) Do not upgrade the storage. We don't really need the storage
# upgrade for now, and the code that upgrade from pre-1.3
# (TimeSerieArchive) uses a lot of parallel lock, which makes tooz
# explodes because MySQL does not support that many connections in real
# life.
# self.storage.upgrade(self.index)
self.mgr = extension.ExtensionManager('gnocchi.aggregates',
invoke_on_load=True)
self.custom_agg = dict((x.name, x.obj) for x in self.mgr)
def setUp(self):
super(TestCase, self).setUp()
self.conf = service.prepare_service([])
self.conf.set_override('policy_file',
self.path_get('etc/gnocchi/policy.json'),
group="oslo_policy")
self.conf.set_override(
'url',
os.environ.get("GNOCCHI_TEST_INDEXER_URL", "null://"),
'indexer')
self.index = indexer.get_driver(self.conf)
self.index.connect()
self.conf.set_override('coordination_url',
os.getenv("GNOCCHI_COORDINATION_URL", "ipc://"),
'storage')
# NOTE(jd) So, some driver, at least SQLAlchemy, can't create all
# their tables in a single transaction even with the
# checkfirst=True, so what we do here is we force the upgrade code
# path to be sequential to avoid race conditions as the tests run
# in parallel.
self.coord = coordination.get_coordinator(
os.getenv("GNOCCHI_COORDINATION_URL", "ipc://"),
str(uuid.uuid4()).encode('ascii'))
with self.coord.get_lock(b"gnocchi-tests-db-lock"):
# Force upgrading using Alembic rather than creating the
# database from scratch so we are sure we don't miss anything
# in the Alembic upgrades. We have a test to check that
# upgrades == create but it misses things such as custom CHECK
# constraints.
self.index.upgrade(nocreate=True)
self.archive_policies = self.ARCHIVE_POLICIES
# Used in gnocchi.gendoc
if not getattr(self, "skip_archive_policies_creation", False):
for name, ap in six.iteritems(self.ARCHIVE_POLICIES):
# Create basic archive policies
try:
self.index.create_archive_policy(ap)
except indexer.ArchivePolicyAlreadyExists:
pass
self.useFixture(mockpatch.Patch(
'swiftclient.client.Connection',
FakeSwiftClient))
self.useFixture(mockpatch.Patch('gnocchi.storage.ceph.rados',
FakeRadosModule()))
self.conf.set_override(
'driver',
os.getenv("GNOCCHI_TEST_STORAGE_DRIVER", "null"),
'storage')
if self.conf.storage.driver == 'file':
tempdir = self.useFixture(fixtures.TempDir())
self.conf.set_override('file_basepath',
tempdir.path,
'storage')
self.storage = storage.get_driver(self.conf)
self.mgr = extension.ExtensionManager('gnocchi.aggregates',
invoke_on_load=True)
self.custom_agg = dict((x.name, x.obj) for x in self.mgr)
def start_fixture(self):
"""Create necessary temp files and do the config dance."""
global LOAD_APP_KWARGS
data_tmp_dir = tempfile.mkdtemp(prefix='gnocchi')
if os.getenv("GABBI_LIVE"):
dcf = None
else:
dcf = []
conf = service.prepare_service([], default_config_files=dcf)
conf.set_override('paste_config',
os.path.abspath('etc/gnocchi/api-paste.ini'), 'api')
self.conf = conf
self.tmp_dir = data_tmp_dir
# TODO(jd) It would be cool if Gabbi was able to use the null://
# indexer, but this makes the API returns a lot of 501 error, which
# Gabbi does not want to see, so let's just disable it.
if conf.indexer.url is None or conf.indexer.url == "null://":
raise case.SkipTest("No indexer configured")
# Use the presence of DEVSTACK_GATE_TEMPEST as a semaphore
# to signal we are not in a gate driven functional test
# and thus should override conf settings.
if 'DEVSTACK_GATE_TEMPEST' not in os.environ:
conf.set_override('driver', 'file', 'storage')
conf.set_override('policy_file',
os.path.abspath('etc/gnocchi/policy.json'),
group="oslo_policy")
conf.set_override('file_basepath', data_tmp_dir, 'storage')
# NOTE(jd) All of that is still very SQL centric but we only support
# SQL for now so let's say it's good enough.
url = sqlalchemy_url.make_url(conf.indexer.url)
url.database = url.database + str(uuid.uuid4()).replace('-', '')
db_url = str(url)
conf.set_override('url', db_url, 'indexer')
sqlalchemy_utils.create_database(db_url)
index = indexer.get_driver(conf)
index.connect()
index.upgrade()
conf.set_override('pecan_debug', False, 'api')
# Set pagination to a testable value
conf.set_override('max_limit', 7, 'api')
self.index = index
s = storage.get_driver(conf)
s.upgrade(index)
LOAD_APP_KWARGS = {
'appname': 'gnocchi+noauth',
'storage': s,
'indexer': index,
'conf': conf,
}
# start up a thread to async process measures
self.metricd_thread = MetricdThread(index, s)
self.metricd_thread.start()
def build_wsgi_app():
return load_app(service.prepare_service())
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
conf = config.conf
indexer = sqlalchemy.SQLAlchemyIndexer(conf)
with indexer.facade.writer_connection() as connectable:
with connectable.connect() as connection:
context.configure(connection=connection,
target_metadata=target_metadata)
with context.begin_transaction():
context.run_migrations()
indexer.disconnect()
# If `alembic' was used directly from the CLI
if not hasattr(config, "conf"):
from gnocchi import service
config.conf = service.prepare_service([])
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()
def storage_dbsync():
conf = service.prepare_service()
indexer = sql_db.SQLAlchemyIndexer(conf)
indexer.connect()
indexer.upgrade()
def setUp(self):
super(TestCase, self).setUp()
self.conf = service.prepare_service(
[], conf=utils.prepare_conf(),
default_config_files=[],
logging_level=logging.DEBUG,
skip_log_opts=True)
self.index = indexer.get_driver(self.conf)
self.coord = metricd.get_coordinator_and_start(
str(uuid.uuid4()),
self.conf.coordination_url)
# NOTE(jd) So, some driver, at least SQLAlchemy, can't create all
# their tables in a single transaction even with the
# checkfirst=True, so what we do here is we force the upgrade code
# path to be sequential to avoid race conditions as the tests run
# in parallel.
with self.coord.get_lock(b"gnocchi-tests-db-lock"):
self.index.upgrade()
self.archive_policies = self.ARCHIVE_POLICIES.copy()
for name, ap in six.iteritems(self.archive_policies):
# Create basic archive policies
try:
self.index.create_archive_policy(ap)
except indexer.ArchivePolicyAlreadyExists:
pass
py_root = os.path.abspath(os.path.join(os.path.dirname(__file__),
'..',))
self.conf.set_override('paste_config',
os.path.join(py_root, 'rest', 'api-paste.ini'),
group="api")
self.conf.set_override('policy_file',
os.path.join(py_root, 'rest', 'policy.json'),
group="oslo_policy")
# NOTE(jd) This allows to test S3 on AWS
if not os.getenv("AWS_ACCESS_KEY_ID"):
self.conf.set_override('s3_endpoint_url',
os.getenv("GNOCCHI_STORAGE_HTTP_URL"),
group="storage")
self.conf.set_override('s3_access_key_id', "gnocchi",
group="storage")
self.conf.set_override('s3_secret_access_key', "anythingworks",
group="storage")
storage_driver = os.getenv("GNOCCHI_TEST_STORAGE_DRIVER", "file")
self.conf.set_override('driver', storage_driver, 'storage')
if swexc:
self.useFixture(fixtures.MockPatch(
'swiftclient.client.Connection',
FakeSwiftClient))
if self.conf.storage.driver == 'file':
tempdir = self.useFixture(fixtures.TempDir())
self.conf.set_override('file_basepath',
tempdir.path,
'storage')
elif self.conf.storage.driver == 'ceph':
self.conf.set_override('ceph_conffile',
os.getenv("CEPH_CONF"),
'storage')
pool_name = uuid.uuid4().hex
with open(os.devnull, 'w') as f:
subprocess.call("rados -c %s mkpool %s" % (
os.getenv("CEPH_CONF"), pool_name), shell=True,
stdout=f, stderr=subprocess.STDOUT)
self.conf.set_override('ceph_pool', pool_name, 'storage')
# Override the bucket prefix to be unique to avoid concurrent access
# with any other test
self.conf.set_override("s3_bucket_prefix", str(uuid.uuid4())[:26],
"storage")
self.storage = storage.get_driver(self.conf)
self.incoming = incoming.get_driver(self.conf)
if self.conf.storage.driver == 'redis':
# Create one prefix per test
self.storage.STORAGE_PREFIX = str(uuid.uuid4()).encode()
if self.conf.incoming.driver == 'redis':
self.incoming.SACK_NAME_FORMAT = (
str(uuid.uuid4()) + incoming.IncomingDriver.SACK_NAME_FORMAT
)
self.storage.upgrade()
self.incoming.upgrade(3)
self.chef = chef.Chef(
self.coord, self.incoming, self.index, self.storage)
def start_fixture(self):
"""Create necessary temp files and do the config dance."""
global LOAD_APP_KWARGS
if not os.getenv("GNOCCHI_TEST_DEBUG"):
self.output = base.CaptureOutput()
self.output.setUp()
data_tmp_dir = tempfile.mkdtemp(prefix='gnocchi')
if os.getenv("GABBI_LIVE"):
dcf = None
else:
dcf = []
conf = service.prepare_service([],
conf=utils.prepare_conf(),
default_config_files=dcf)
if not os.getenv("GNOCCHI_TEST_DEBUG"):
daiquiri.setup(outputs=[])
py_root = os.path.abspath(
os.path.join(
os.path.dirname(__file__),
'..',
'..',
))
conf.set_override('paste_config',
os.path.join(py_root, 'rest', 'api-paste.ini'),
group="api")
conf.set_override('policy_file',
os.path.join(py_root, 'rest', 'policy.json'),
group="oslo_policy")
# NOTE(sileht): This is not concurrency safe, but only this tests file
# deal with cors, so we are fine. set_override don't work because cors
# group doesn't yet exists, and we the CORS middleware is created it
# register the option and directly copy value of all configurations
# options making impossible to override them properly...
cfg.set_defaults(cors.CORS_OPTS, allowed_origin="http://foobar.com")
self.conf = conf
self.tmp_dir = data_tmp_dir
if conf.indexer.url is None:
raise case.SkipTest("No indexer configured")
conf.set_override('driver', 'file', 'storage')
conf.set_override('file_basepath', data_tmp_dir, 'storage')
# NOTE(jd) All of that is still very SQL centric but we only support
# SQL for now so let's say it's good enough.
conf.set_override(
'url',
sqlalchemy.SQLAlchemyIndexer._create_new_database(
conf.indexer.url), 'indexer')
index = indexer.get_driver(conf)
index.upgrade()
# Set pagination to a testable value
conf.set_override('max_limit', 7, 'api')
self.index = index
self.coord = metricd.get_coordinator_and_start(str(uuid.uuid4()),
conf.coordination_url)
s = storage.get_driver(conf, self.coord)
s.upgrade()
i = incoming.get_driver(conf)
i.upgrade(128)
self.fixtures = [
fixtures.MockPatch("gnocchi.storage.get_driver", return_value=s),
fixtures.MockPatch("gnocchi.incoming.get_driver", return_value=i),
fixtures.MockPatch("gnocchi.indexer.get_driver",
return_value=self.index),
fixtures.MockPatch("gnocchi.cli.metricd.get_coordinator_and_start",
return_value=self.coord),
]
for f in self.fixtures:
f.setUp()
LOAD_APP_KWARGS = {
'conf': conf,
}
# start up a thread to async process measures
self.metricd_thread = MetricdThread(index, s, i)
self.metricd_thread.start()
def build_server():
conf = service.prepare_service()
serving.run_simple(conf.api.host, conf.api.port,
WerkzeugApp(conf),
processes=conf.api.workers)
def setUp(self):
super(TestCase, self).setUp()
self.conf = service.prepare_service([],
default_config_files=[])
self.conf.set_override('policy_file',
self.path_get('etc/gnocchi/policy.json'),
group="oslo_policy")
self.index = indexer.get_driver(self.conf)
self.index.connect()
# NOTE(jd) So, some driver, at least SQLAlchemy, can't create all
# their tables in a single transaction even with the
# checkfirst=True, so what we do here is we force the upgrade code
# path to be sequential to avoid race conditions as the tests run
# in parallel.
self.coord = coordination.get_coordinator(
self.conf.storage.coordination_url,
str(uuid.uuid4()).encode('ascii'))
self.coord.start()
with self.coord.get_lock(b"gnocchi-tests-db-lock"):
# Force upgrading using Alembic rather than creating the
# database from scratch so we are sure we don't miss anything
# in the Alembic upgrades. We have a test to check that
# upgrades == create but it misses things such as custom CHECK
# constraints.
self.index.upgrade(nocreate=True)
self.coord.stop()
self.archive_policies = self.ARCHIVE_POLICIES.copy()
self.archive_policies.update(archive_policy.DEFAULT_ARCHIVE_POLICIES)
# Used in gnocchi.gendoc
if not getattr(self, "skip_archive_policies_creation", False):
for name, ap in six.iteritems(self.archive_policies):
# Create basic archive policies
try:
self.index.create_archive_policy(ap)
except indexer.ArchivePolicyAlreadyExists:
pass
if swexc:
self.useFixture(mockpatch.Patch(
'swiftclient.client.Connection',
FakeSwiftClient))
self.useFixture(mockpatch.Patch('gnocchi.storage.ceph.rados',
FakeRadosModule()))
self.conf.set_override(
'driver',
os.getenv("GNOCCHI_TEST_STORAGE_DRIVER", "null"),
'storage')
if self.conf.storage.driver == 'file':
tempdir = self.useFixture(fixtures.TempDir())
self.conf.set_override('file_basepath',
tempdir.path,
'storage')
elif self.conf.storage.driver == 'influxdb':
self.conf.set_override('influxdb_block_until_data_ingested', True,
'storage')
self.conf.set_override('influxdb_database', 'test', 'storage')
self.conf.set_override('influxdb_password', 'root', 'storage')
self.conf.set_override('influxdb_port',
os.getenv("GNOCCHI_TEST_INFLUXDB_PORT",
51234), 'storage')
# NOTE(ityaptin) Creating unique database for every test may cause
# tests failing by timeout, but in may be useful in some cases
if os.getenv("GNOCCHI_TEST_INFLUXDB_UNIQUE_DATABASES"):
self.conf.set_override("influxdb_database",
"gnocchi_%s" % uuid.uuid4().hex,
'storage')
self.storage = storage.get_driver(self.conf)
# NOTE(jd) Do not upgrade the storage. We don't really need the storage
# upgrade for now, and the code that upgrade from pre-1.3
# (TimeSerieArchive) uses a lot of parallel lock, which makes tooz
# explodes because MySQL does not support that many connections in real
# life.
# self.storage.upgrade(self.index)
self.mgr = extension.ExtensionManager('gnocchi.aggregates',
invoke_on_load=True)
self.custom_agg = dict((x.name, x.obj) for x in self.mgr)
def metricd():
conf = service.prepare_service()
MetricdServiceManager(conf).run()
if self.conf.amqpd.archive_policy_name:
return self.conf.amqpd.archive_policy_name
# NOTE(sileht): We didn't catch NoArchivePolicyRuleMatch to log it
ap = self.indexer.get_archive_policy_for_metric(metric_name)
return ap.name
parser = optparse.OptionParser(usage="usage: %prog [options]")
parser.add_option(
"-a",
"--address",
default="localhost:5672/examples",
help="address from which messages are received (default %default)")
parser.add_option(
"-m",
"--messages",
type="int",
default=100,
help=
"number of messages to receive; 0 receives indefinitely (default %default)"
)
conf = service.prepare_service()
#if conf.statsd.resource_id is None:
#raise cfg.RequiredOptError("resource_id", cfg.OptGroup("amqpd"))
try:
Container(Amqp(opts.address, opts.messages, conf)).run()
except KeyboardInterrupt:
pass
def start_fixture(self):
"""Create necessary temp files and do the config dance."""
self.output = output.CaptureOutput()
self.output.setUp()
self.log = log.ConfigureLogging()
self.log.setUp()
global LOAD_APP_KWARGS
data_tmp_dir = tempfile.mkdtemp(prefix='gnocchi')
if os.getenv("GABBI_LIVE"):
dcf = None
else:
dcf = []
conf = service.prepare_service([],
default_config_files=dcf)
py_root = os.path.abspath(os.path.join(os.path.dirname(__file__),
'..', '..',))
conf.set_override('paste_config',
os.path.join(py_root, 'rest', 'api-paste.ini'),
group="api")
conf.set_override('policy_file',
os.path.join(py_root, 'rest', 'policy.json'),
group="oslo_policy")
# NOTE(sileht): This is not concurrency safe, but only this tests file
# deal with cors, so we are fine. set_override don't work because cors
# group doesn't yet exists, and we the CORS middleware is created it
# register the option and directly copy value of all configurations
# options making impossible to override them properly...
cfg.set_defaults(cors.CORS_OPTS, allowed_origin="http://foobar.com")
self.conf = conf
self.tmp_dir = data_tmp_dir
if conf.indexer.url is None:
raise case.SkipTest("No indexer configured")
# Use the presence of DEVSTACK_GATE_TEMPEST as a semaphore
# to signal we are not in a gate driven functional test
# and thus should override conf settings.
if 'DEVSTACK_GATE_TEMPEST' not in os.environ:
conf.set_override('driver', 'file', 'storage')
conf.set_override('file_basepath', data_tmp_dir, 'storage')
# NOTE(jd) All of that is still very SQL centric but we only support
# SQL for now so let's say it's good enough.
conf.set_override(
'url',
sqlalchemy.SQLAlchemyIndexer._create_new_database(
conf.indexer.url),
'indexer')
index = indexer.get_driver(conf)
index.connect()
index.upgrade()
# Set pagination to a testable value
conf.set_override('max_limit', 7, 'api')
# Those tests uses noauth mode
# TODO(jd) Rewrite them for basic
conf.set_override("auth_mode", "noauth", 'api')
self.index = index
s = storage.get_driver(conf)
s.upgrade(index)
LOAD_APP_KWARGS = {
'storage': s,
'indexer': index,
'conf': conf,
}
# start up a thread to async process measures
self.metricd_thread = MetricdThread(index, s)
self.metricd_thread.start()
def start_fixture(self):
"""Create necessary temp files and do the config dance."""
global LOAD_APP_KWARGS
data_tmp_dir = tempfile.mkdtemp(prefix='gnocchi')
if os.getenv("GABBI_LIVE"):
dcf = None
else:
dcf = []
conf = service.prepare_service([],
default_config_files=dcf)
conf.set_override('paste_config',
os.path.abspath('etc/gnocchi/api-paste.ini'),
'api')
self.conf = conf
self.tmp_dir = data_tmp_dir
# TODO(jd) It would be cool if Gabbi was able to use the null://
# indexer, but this makes the API returns a lot of 501 error, which
# Gabbi does not want to see, so let's just disable it.
if conf.indexer.url is None or conf.indexer.url == "null://":
raise case.SkipTest("No indexer configured")
# Use the presence of DEVSTACK_GATE_TEMPEST as a semaphore
# to signal we are not in a gate driven functional test
# and thus should override conf settings.
if 'DEVSTACK_GATE_TEMPEST' not in os.environ:
conf.set_override('driver', 'file', 'storage')
conf.set_override('policy_file',
os.path.abspath('etc/gnocchi/policy.json'),
group="oslo_policy")
conf.set_override('file_basepath', data_tmp_dir, 'storage')
# NOTE(jd) All of that is still very SQL centric but we only support
# SQL for now so let's say it's good enough.
conf.set_override(
'url',
sqlalchemy.SQLAlchemyIndexer._create_new_database(
conf.indexer.url),
'indexer')
index = indexer.get_driver(conf)
index.connect()
index.upgrade(create_legacy_resource_types=True)
conf.set_override('pecan_debug', False, 'api')
# Set pagination to a testable value
conf.set_override('max_limit', 7, 'api')
self.index = index
s = storage.get_driver(conf)
s.upgrade(index)
LOAD_APP_KWARGS = {
'appname': 'gnocchi+noauth',
'storage': s,
'indexer': index,
'conf': conf,
}
# start up a thread to async process measures
self.metricd_thread = MetricdThread(index, s)
self.metricd_thread.start()
def setUp(self):
super(TestCase, self).setUp()
self.conf = service.prepare_service(
[], conf=utils.prepare_conf(),
default_config_files=[],
logging_level=logging.DEBUG,
skip_log_opts=True)
self.index = indexer.get_driver(self.conf)
self.coord = metricd.get_coordinator_and_start(
str(uuid.uuid4()),
self.conf.coordination_url)
# NOTE(jd) So, some driver, at least SQLAlchemy, can't create all
# their tables in a single transaction even with the
# checkfirst=True, so what we do here is we force the upgrade code
# path to be sequential to avoid race conditions as the tests run
# in parallel.
with self.coord.get_lock(b"gnocchi-tests-db-lock"):
self.index.upgrade()
self.archive_policies = self.ARCHIVE_POLICIES.copy()
for name, ap in six.iteritems(self.archive_policies):
# Create basic archive policies
try:
self.index.create_archive_policy(ap)
except indexer.ArchivePolicyAlreadyExists:
pass
py_root = os.path.abspath(os.path.join(os.path.dirname(__file__),
'..',))
self.conf.set_override('paste_config',
os.path.join(py_root, 'rest', 'api-paste.ini'),
group="api")
self.conf.set_override('policy_file',
os.path.join(py_root, 'rest', 'policy.yaml'),
group="oslo_policy")
# NOTE(jd) This allows to test S3 on AWS
if not os.getenv("AWS_ACCESS_KEY_ID"):
self.conf.set_override('s3_endpoint_url',
os.getenv("GNOCCHI_STORAGE_HTTP_URL"),
group="storage")
self.conf.set_override('s3_access_key_id', "gnocchi",
group="storage")
self.conf.set_override('s3_secret_access_key', "anythingworks",
group="storage")
storage_driver = os.getenv("GNOCCHI_TEST_STORAGE_DRIVER", "file")
self.conf.set_override('driver', storage_driver, 'storage')
if swexc:
self.useFixture(fixtures.MockPatch(
'swiftclient.client.Connection',
FakeSwiftClient))
if self.conf.storage.driver == 'file':
tempdir = self.useFixture(fixtures.TempDir())
self.conf.set_override('file_basepath',
tempdir.path,
'storage')
elif self.conf.storage.driver == 'ceph':
self.conf.set_override('ceph_conffile',
os.getenv("CEPH_CONF"),
'storage')
self.ceph_pool_name = uuid.uuid4().hex
with open(os.devnull, 'w') as f:
subprocess.call(("ceph -c %s osd pool create %s "
"16 16 replicated") % (
os.getenv("CEPH_CONF"), self.ceph_pool_name), shell=True,
stdout=f, stderr=subprocess.STDOUT)
subprocess.call(("ceph -c %s osd pool application "
"enable %s rbd") % (
os.getenv("CEPH_CONF"), self.ceph_pool_name), shell=True,
stdout=f, stderr=subprocess.STDOUT)
self.conf.set_override('ceph_pool', self.ceph_pool_name, 'storage')
# Override the bucket prefix to be unique to avoid concurrent access
# with any other test
self.conf.set_override("s3_bucket_prefix", str(uuid.uuid4())[:26],
"storage")
self.storage = storage.get_driver(self.conf)
self.incoming = incoming.get_driver(self.conf)
if self.conf.storage.driver == 'redis':
# Create one prefix per test
self.storage.STORAGE_PREFIX = str(uuid.uuid4()).encode()
if self.conf.incoming.driver == 'redis':
self.incoming.SACK_NAME_FORMAT = (
str(uuid.uuid4()) + incoming.IncomingDriver.SACK_NAME_FORMAT
)
self.storage.upgrade()
self.incoming.upgrade(3)
self.chef = chef.Chef(
self.coord, self.incoming, self.index, self.storage)
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
conf = config.conf
indexer = sqlalchemy.SQLAlchemyIndexer(conf)
indexer.connect()
connectable = indexer.engine_facade.get_engine()
with connectable.connect() as connection:
context.configure(
connection=connection,
target_metadata=target_metadata
)
with context.begin_transaction():
context.run_migrations()
# If `alembic' was used directly from the CLI
if not hasattr(config, "conf"):
from gnocchi import service
config.conf = service.prepare_service([])
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()
