def main():
"""Start persister."""
config.parse_args()
metric_repository = simport.load(cfg.CONF.repositories.metrics_driver)
alarm_state_history_repository = simport.load(
cfg.CONF.repositories.alarm_state_history_driver)
# Add processors for metrics topic
for proc in range(0, cfg.CONF.kafka_metrics.num_processors):
processors.append(
multiprocessing.Process(target=start_process,
args=(metric_repository,
cfg.CONF.kafka_metrics)))
# Add processors for alarm history topic
for proc in range(0, cfg.CONF.kafka_alarm_history.num_processors):
processors.append(
multiprocessing.Process(target=start_process,
args=(alarm_state_history_repository,
cfg.CONF.kafka_alarm_history)))
# Start
try:
LOG.info('''
_____
/ \ ____ ____ _____ ______ ____ _____
/ \ / \ / _ \ / \\\__ \ / ___// ___\\\__ \\
/ Y ( <_> ) | \/ __ \_\___ \\ \___ / __ \\_
\____|__ /\____/|___| (____ /____ >\___ >____ /
\/ \/ \/ \/ \/ \/
__________ .__ __
\______ \ ___________ _____|__| _______/ |_ ___________
| ___// __ \_ __ \/ ___/ |/ ___/\ __\/ __ \_ __ \\
| | \ ___/| | \/\___ \| |\___ \ | | \ ___/| | \/
|____| \___ >__| /____ >__/____ > |__| \___ >__|
\/ \/ \/ \/
''')
for process in processors:
process.start()
# The signal handlers must be added after the processes start otherwise
# they run on all processes
signal.signal(signal.SIGCHLD, clean_exit)
signal.signal(signal.SIGINT, clean_exit)
signal.signal(signal.SIGTERM, clean_exit)
while True:
time.sleep(10)
except Exception:
LOG.exception('Error! Exiting.')
clean_exit(signal.SIGKILL)
def main():
"""Start persister."""
config.parse_args()
# Add processors for metrics topic
if cfg.CONF.kafka_metrics.enabled:
prepare_processes(cfg.CONF.kafka_metrics,
cfg.CONF.repositories.metrics_driver)
# Add processors for alarm history topic
if cfg.CONF.kafka_alarm_history.enabled:
prepare_processes(cfg.CONF.kafka_alarm_history,
cfg.CONF.repositories.alarm_state_history_driver)
# Add processors for events topic
if cfg.CONF.kafka_events.enabled:
prepare_processes(cfg.CONF.kafka_events,
cfg.CONF.repositories.events_driver)
if cfg.CONF.prometheus.enabled:
prepare_prometheus(cfg.CONF.prometheus.port,
cfg.CONF.prometheus.multiproc_dir)
# Start
try:
LOG.info('''
_____
/ \ ____ ____ _____ ______ ____ _____
/ \ / \ / _ \ / \\\__ \ / ___// ___\\\__ \\
/ Y ( <_> ) | \/ __ \_\___ \\ \___ / __ \\_
\____|__ /\____/|___| (____ /____ >\___ >____ /
\/ \/ \/ \/ \/ \/
__________ .__ __
\______ \ ___________ _____|__| _______/ |_ ___________
| ___// __ \_ __ \/ ___/ |/ ___/\ __\/ __ \_ __ \\
| | \ ___/| | \/\___ \| |\___ \ | | \ ___/| | \/
|____| \___ >__| /____ >__/____ > |__| \___ >__|
\/ \/ \/ \/
''')
for process in processors:
process.start()
# The signal handlers must be added after the processes start otherwise
# they run on all processes
signal.signal(signal.SIGCHLD, clean_exit)
signal.signal(signal.SIGINT, clean_exit)
signal.signal(signal.SIGTERM, clean_exit)
while True:
time.sleep(10)
except Exception:
LOG.exception('Error! Exiting.')
clean_exit(signal.SIGKILL)
def main():
CONF = cfg.CONF
cli_opts = [
cfg.StrOpt('migrate-time-unit', choices=['h', 'd', 'w'], default='w',
help='Unit of time, h=hour, d=day, w=week (default: "w").'),
cfg.IntOpt('migrate-start-time-offset', default=0,
help='Start time offset in the given unit of time (default: 0).'),
cfg.IntOpt('migrate-end-time-offset', default=520,
help='End time offset in the given unit of time (default: 520).'),
cfg.DictOpt('migrate-retention-policy', default={},
help=('Custom retention policy for projects in the provided'
'time unit. (e.g. project-id-x:2,project-id-y:4)')),
cfg.ListOpt('migrate-skip-regex', default=[],
help=('Skip metrics that match this comma separated list of regex patterns.'
'(e.g. ^log\\\\..+,^cpu\\\\..+ to skip metrics beginning with log.)')),
]
CONF.register_cli_opts(cli_opts)
config.parse_args("Monasca InfluxDB database per tenant migration tool")
# Configure custom retention policy for your existing projects. For
# example, rp2w is a retention policy of two weeks which we can assign to
# project example-project-id.
tenant_defaults = dict()
for k, v in CONF.migrate_retention_policy.items():
if v.isdigit():
rp = '{}{}'.format(v, CONF.migrate_time_unit)
tenant_defaults[k] = dict(
end_time_offset_override=int(v),
rp=dict(name=rp, duration=rp, replication='1', default=True),
)
LOG.info('Project {} will be applied retention policy: {}.'.format(k, rp))
else:
raise ValueError('Retention policy for project {} must be an'
'integer of given time unit. Current value:'
'{}.'.format(k, v))
skip_regex = []
for p in CONF.migrate_skip_regex:
skip_regex.append(re.compile(str(p)))
LOG.info('Metrics matching pattern "{}" will be skipped.'.format(p))
helper = MigrationHelper()
helper.migrate(skip_regex=skip_regex,
tenant_defaults=tenant_defaults,
default_end_time_offset=CONF.migrate_end_time_offset,
default_start_time_offset=CONF.migrate_start_time_offset,
time_unit=CONF.migrate_time_unit,
measurements_file='migrate-measurements',
success_file='migrate-success',
failure_file='migrate-failure')
return 0
def main():
"""Start persister."""
config.parse_args()
# Add processors for metrics topic
if cfg.CONF.kafka_metrics.enabled:
prepare_processes(cfg.CONF.kafka_metrics,
cfg.CONF.repositories.metrics_driver)
# Add processors for alarm history topic
if cfg.CONF.kafka_alarm_history.enabled:
prepare_processes(cfg.CONF.kafka_alarm_history,
cfg.CONF.repositories.alarm_state_history_driver)
# Add processors for events topic
if cfg.CONF.kafka_events.enabled:
prepare_processes(cfg.CONF.kafka_events,
cfg.CONF.repositories.events_driver)
# Start
try:
LOG.info('''
_____
/ \ ____ ____ _____ ______ ____ _____
/ \ / \ / _ \ / \\\__ \ / ___// ___\\\__ \\
/ Y ( <_> ) | \/ __ \_\___ \\ \___ / __ \\_
\____|__ /\____/|___| (____ /____ >\___ >____ /
\/ \/ \/ \/ \/ \/
__________ .__ __
\______ \ ___________ _____|__| _______/ |_ ___________
| ___// __ \_ __ \/ ___/ |/ ___/\ __\/ __ \_ __ \\
| | \ ___/| | \/\___ \| |\___ \ | | \ ___/| | \/
|____| \___ >__| /____ >__/____ > |__| \___ >__|
\/ \/ \/ \/
''')
for process in processors:
process.start()
# The signal handlers must be added after the processes start otherwise
# they run on all processes
signal.signal(signal.SIGCHLD, clean_exit)
signal.signal(signal.SIGINT, clean_exit)
signal.signal(signal.SIGTERM, clean_exit)
while True:
time.sleep(10)
except Exception:
LOG.exception('Error! Exiting.')
clean_exit(signal.SIGKILL)
def main():
"""persister recreate metric_id tool."""
config.parse_args()
conf = cfg.CONF
try:
LOG.info('Starting check and repair of metric_id consistency.')
# Connection setup
# rocky style - note that we don't deliver pike style
_cluster = connection_util.create_cluster()
_session = connection_util.create_session(_cluster)
_retention = conf.cassandra.retention_policy * 24 * 3600
metric_all_stmt = _session.prepare(METRIC_ALL_CQL)
metric_repair_stmt = _session.prepare(METRICS_INSERT_CQL)
rows = _session.execute(metric_all_stmt)
# if rows:
# LOG.info('First - {}'.format(rows[0]))
# # LOG.info('First name {} and id {}'.format(
# # rows[0].metric_name, rows[0].metric_id)) # metric_id can't be logged raw
# Bit of a misnomer - "null" is not in the cassandra db
missing_value_rows = []
for row in rows:
if row.metric_id is None:
LOG.info('Row with missing metric_id - {}'.format(row))
missing_value_rows.append(row)
# check created_at
fixed_created_at = row.created_at
if row.created_at is None and row.updated_at is not None:
LOG.info("Metric created_at was also None, repairing.")
fixed_created_at = row.updated_at
# TODO(joadavis) update the updated_at timestamp to now
# recreate metric id
# copied from metrics_repository.py
hash_string = '%s\0%s\0%s\0%s' % (row.region, row.tenant_id,
row.metric_name,
'\0'.join(row.dimensions))
metric_id = hashlib.sha1(hash_string.encode('utf8')).hexdigest()
id_bytes = bytearray.fromhex(metric_id)
LOG.info("Recreated hash for metric id: {}".format(hash_string))
# LOG.info("new id_bytes {}".format(id_bytes)) # can't unicode decode for logging
# execute cql
metric_repair_bound_stmt = metric_repair_stmt.bind((_retention,
id_bytes,
fixed_created_at,
row.updated_at,
row.region,
row.tenant_id,
row.metric_name,
row.dimensions,
row.dimension_names))
_session.execute(metric_repair_bound_stmt)
# LOG.info("of {} rows there are {} missing metric_id".format(len(rows), len(null_rows)))
if len(missing_value_rows) > 0:
LOG.warning("--> There were {} rows missing metric_id.".format(
len(missing_value_rows)))
LOG.warning(" Those rows have been updated.")
else:
LOG.info("No missing metric_ids were found, no changes made.")
LOG.info('Done with metric_id consistency check and repair.')
return 0
except Exception:
LOG.exception('Error! Exiting.')
def main():
"""persister check for missing metric_id tool."""
config.parse_args()
try:
LOG.info('Starting check of metric_id consistency.')
# Connection setup
# rocky style - note that we don't deliver pike style
_cluster = connection_util.create_cluster()
_session = connection_util.create_session(_cluster)
metric_all_stmt = _session.prepare(METRIC_ALL_CQL)
rows = _session.execute(metric_all_stmt)
# if rows:
# LOG.info('First - {}'.format(rows[0]))
# # LOG.info('First name {} and id {}'.format(
# # rows[0].metric_name, rows[0].metric_id)) # metric_id can't be logged raw
# Bit of a misnomer - "null" is not in the cassandra db
missing_value_rows = []
for row in rows:
if row.metric_id is None:
LOG.info('Row with missing metric_id - {}'.format(row))
missing_value_rows.append(row)
# check created_at
if row.created_at is None and row.updated_at is not None:
LOG.info("Metric created_at was also None.")
# TODO(joadavis) update the updated_at timestamp to now
# recreate metric id
# copied from metrics_repository.py
hash_string = '%s\0%s\0%s\0%s' % (row.region, row.tenant_id,
row.metric_name,
'\0'.join(row.dimensions))
# metric_id = hashlib.sha1(hash_string.encode('utf8')).hexdigest()
# id_bytes = bytearray.fromhex(metric_id)
LOG.info("Recreated hash for metric id: {}".format(hash_string))
# LOG.info("new id_bytes {}".format(id_bytes)) # can't unicode decode for logging
# LOG.info("of {} rows there are {} missing metric_id".format(len(rows), len(null_rows)))
if len(missing_value_rows) > 0:
LOG.warning("--> There were {} rows missing metric_id.".format(
len(missing_value_rows)))
LOG.warning(" Those rows have NOT been updated.\n"
" Please run the persister-recreate-metric-id "
"tool to repair the rows.")
else:
LOG.info("No missing metric_ids were found, no changes made.")
LOG.info('Done with metric_id consistency check.')
return 0
except Exception:
LOG.exception('Error! Exiting.')