def main():
# init conf and logging
conf = cfg.CONF
conf.register_cli_opts(config.OPTS)
conf.register_opts(config.OPTS)
conf()
logging.setup('stackalytics')
LOG.info('Logging enabled')
runtime_storage_inst = runtime_storage.get_runtime_storage(
cfg.CONF.runtime_storage_uri)
default_data = utils.read_json_from_uri(cfg.CONF.default_data_uri)
if not default_data:
LOG.critical('Unable to load default data')
return not 0
default_data_processor.process(runtime_storage_inst,
default_data,
cfg.CONF.sources_root)
update_pids(runtime_storage_inst)
update_repos(runtime_storage_inst)
apply_corrections(cfg.CONF.corrections_uri, runtime_storage_inst)
def main():
utils.init_config_and_logging(config.CONNECTION_OPTS + config.PROCESSOR_OPTS)
runtime_storage_inst = runtime_storage.get_runtime_storage(cfg.CONF.runtime_storage_uri)
default_data = utils.read_json_from_uri(cfg.CONF.default_data_uri)
if not default_data:
LOG.critical("Unable to load default data")
return not 0
default_data_processor.process(runtime_storage_inst, default_data, cfg.CONF.driverlog_data_uri)
process_project_list(runtime_storage_inst, cfg.CONF.project_list_uri)
update_pids(runtime_storage_inst)
record_processor_inst = record_processor.RecordProcessor(runtime_storage_inst)
process(runtime_storage_inst, record_processor_inst)
apply_corrections(cfg.CONF.corrections_uri, runtime_storage_inst)
# long operation should be the last
update_members(runtime_storage_inst, record_processor_inst)
runtime_storage_inst.set_by_key("runtime_storage_update_time", utils.date_to_timestamp("now"))
def get_vault():
vault = getattr(app, 'stackalytics_vault', None)
if not vault:
vault = {}
runtime_storage_inst = runtime_storage.get_runtime_storage(
cfg.CONF.runtime_storage_uri)
vault['runtime_storage'] = runtime_storage_inst
vault['memory_storage'] = memory_storage.get_memory_storage(
memory_storage.MEMORY_STORAGE_CACHED,
vault['runtime_storage'].get_update(os.getpid()))
releases = list(runtime_storage_inst.get_by_key('releases'))
vault['start_date'] = releases[0]['end_date']
vault['end_date'] = releases[-1]['end_date']
start_date = releases[0]['end_date']
for r in releases[1:]:
r['start_date'] = start_date
start_date = r['end_date']
vault['releases'] = dict((r['release_name'].lower(), r)
for r in releases[1:])
modules = runtime_storage_inst.get_by_key('repos')
vault['modules'] = dict((r['module'].lower(),
r['project_type'].lower()) for r in modules)
app.stackalytics_vault = vault
init_project_types(vault)
else:
if not getattr(flask.request, 'stackalytics_updated', None):
flask.request.stackalytics_updated = True
memory_storage_inst = vault['memory_storage']
memory_storage_inst.update(
vault['runtime_storage'].get_update(os.getpid()))
return vault
def main():
# init conf and logging
conf = cfg.CONF
conf.register_cli_opts(config.OPTS)
conf.register_cli_opts(OPTS)
conf.register_opts(config.OPTS)
conf.register_opts(OPTS)
conf()
logging.setup('stackalytics')
LOG.info('Logging enabled')
runtime_storage_inst = runtime_storage.get_runtime_storage(
cfg.CONF.runtime_storage_uri)
filename = cfg.CONF.file
if cfg.CONF.reverse:
if filename:
fd = open(filename, 'r')
else:
fd = sys.stdin
import_data(runtime_storage_inst, fd)
else:
if filename:
fd = open(filename, 'w')
else:
fd = sys.stdout
export_data(runtime_storage_inst, fd)
def main():
# init conf and logging
conf = cfg.CONF
conf.register_cli_opts(config.OPTS)
conf.register_cli_opts(OPTS)
conf.register_opts(config.OPTS)
conf.register_opts(OPTS)
conf()
logging.setup('stackalytics')
LOG.info('Logging enabled')
runtime_storage_inst = runtime_storage.get_runtime_storage(
cfg.CONF.runtime_storage_uri)
filename = cfg.CONF.file
if cfg.CONF.reverse:
if filename:
fd = open(filename, 'r')
else:
fd = sys.stdin
import_data(runtime_storage_inst, fd)
else:
if filename:
fd = open(filename, 'w')
else:
fd = sys.stdout
export_data(runtime_storage_inst, fd)
def main():
# init conf and logging
conf = cfg.CONF
conf.register_cli_opts(config.OPTS)
conf.register_opts(config.OPTS)
conf()
logging.setup('stackalytics')
LOG.info('Logging enabled')
runtime_storage_inst = runtime_storage.get_runtime_storage(
cfg.CONF.runtime_storage_uri)
default_data = utils.read_json_from_uri(cfg.CONF.default_data_uri)
if not default_data:
LOG.critical('Unable to load default data')
return not 0
default_data_processor.process(runtime_storage_inst, default_data,
cfg.CONF.sources_root)
update_pids(runtime_storage_inst)
update_repos(runtime_storage_inst)
apply_corrections(cfg.CONF.corrections_uri, runtime_storage_inst)
def main():
# init conf and logging
conf = cfg.CONF
conf.register_cli_opts(config.OPTS)
conf.register_opts(config.OPTS)
conf()
logging.setup('stackalytics')
LOG.info('Logging enabled')
runtime_storage_inst = runtime_storage.get_runtime_storage(
cfg.CONF.runtime_storage_uri)
default_data = utils.read_json_from_uri(cfg.CONF.default_data_uri)
if not default_data:
LOG.critical('Unable to load default data')
return not 0
default_data_processor.process(runtime_storage_inst,
default_data,
cfg.CONF.sources_root,
cfg.CONF.force_update)
process_program_list(runtime_storage_inst, cfg.CONF.program_list_uri)
update_pids(runtime_storage_inst)
record_processor_inst = record_processor.RecordProcessor(
runtime_storage_inst)
update_records(runtime_storage_inst, record_processor_inst)
apply_corrections(cfg.CONF.corrections_uri, runtime_storage_inst)
# long operation should be the last
update_members(runtime_storage_inst, record_processor_inst)
def main():
utils.init_config_and_logging(config.CONNECTION_OPTS +
config.PROCESSOR_OPTS)
runtime_storage_inst = runtime_storage.get_runtime_storage(
cfg.CONF.runtime_storage_uri)
default_data = utils.read_json_from_uri(cfg.CONF.default_data_uri)
if not default_data:
LOG.critical('Unable to load default data')
return not 0
default_data_processor.process(runtime_storage_inst, default_data,
cfg.CONF.driverlog_data_uri)
process_project_list(runtime_storage_inst)
update_pids(runtime_storage_inst)
record_processor_inst = record_processor.RecordProcessor(
runtime_storage_inst)
process(runtime_storage_inst, record_processor_inst)
apply_corrections(cfg.CONF.corrections_uri, runtime_storage_inst)
# long operation should be the last
update_members(runtime_storage_inst, record_processor_inst)
runtime_storage_inst.set_by_key('runtime_storage_update_time',
utils.date_to_timestamp('now'))
def main():
utils.init_config_and_logging(config.CONNECTION_OPTS +
config.PROCESSOR_OPTS)
runtime_storage_inst = runtime_storage.get_runtime_storage(
CONF.runtime_storage_uri)
default_data = utils.read_json_from_uri(CONF.default_data_uri)
if not default_data:
LOG.critical('Unable to load default data')
return not 0
try:
jsonschema.validate(default_data, schema.default_data)
except jsonschema.ValidationError as e:
LOG.critical('The default data is invalid: %s' % e)
return not 0
default_data_processor.process(runtime_storage_inst, default_data)
process_project_list(runtime_storage_inst)
update_pids(runtime_storage_inst)
record_processor_inst = record_processor.RecordProcessor(
runtime_storage_inst)
process(runtime_storage_inst, record_processor_inst)
runtime_storage_inst.set_by_key('runtime_storage_update_time',
utils.date_to_timestamp('now'))
LOG.info('stackalytics-processor succeeded.')
def get_vault():
vault = getattr(flask.current_app, 'stackalytics_vault', None)
if not vault:
try:
vault = {}
runtime_storage_inst = runtime_storage.get_runtime_storage(
cfg.CONF.runtime_storage_uri)
vault['runtime_storage'] = runtime_storage_inst
vault['memory_storage'] = memory_storage.get_memory_storage(
memory_storage.MEMORY_STORAGE_CACHED)
init_project_types(vault)
init_releases(vault)
flask.current_app.stackalytics_vault = vault
except Exception as e:
LOG.critical('Failed to initialize application: %s', e)
LOG.exception(e)
flask.abort(500)
if not getattr(flask.request, 'stackalytics_updated', None):
flask.request.stackalytics_updated = True
memory_storage_inst = vault['memory_storage']
have_updates = memory_storage_inst.update(
vault['runtime_storage'].get_update(os.getpid()))
if have_updates:
init_project_types(vault)
init_releases(vault)
init_module_groups(vault)
return vault
def get_vault():
vault = getattr(app, 'stackalytics_vault', None)
if not vault:
try:
vault = {}
runtime_storage_inst = runtime_storage.get_runtime_storage(
cfg.CONF.runtime_storage_uri)
vault['runtime_storage'] = runtime_storage_inst
vault['memory_storage'] = memory_storage.get_memory_storage(
memory_storage.MEMORY_STORAGE_CACHED)
init_project_types(vault)
init_releases(vault)
app.stackalytics_vault = vault
except Exception as e:
LOG.critical('Failed to initialize application: %s', e)
LOG.exception(e)
flask.abort(500)
if not getattr(flask.request, 'stackalytics_updated', None):
flask.request.stackalytics_updated = True
memory_storage_inst = vault['memory_storage']
have_updates = memory_storage_inst.update(
vault['runtime_storage'].get_update(os.getpid()))
if have_updates:
init_project_types(vault)
init_releases(vault)
init_module_groups(vault)
return vault
def main():
# init conf and logging
conf = cfg.CONF
conf.register_cli_opts(config.OPTS)
conf.register_opts(config.OPTS)
conf()
logging.setup('stackalytics')
LOG.info('Logging enabled')
runtime_storage_inst = runtime_storage.get_runtime_storage(
cfg.CONF.runtime_storage_uri)
default_data = utils.read_json_from_uri(cfg.CONF.default_data_uri)
if not default_data:
LOG.critical('Unable to load default data')
return not 0
default_data_processor.process(runtime_storage_inst, default_data,
cfg.CONF.sources_root,
cfg.CONF.force_update)
process_program_list(runtime_storage_inst, cfg.CONF.program_list_uri)
update_pids(runtime_storage_inst)
record_processor_inst = record_processor.RecordProcessor(
runtime_storage_inst)
update_records(runtime_storage_inst, record_processor_inst)
apply_corrections(cfg.CONF.corrections_uri, runtime_storage_inst)
# long operation should be the last
update_members(runtime_storage_inst, record_processor_inst)
def get_vault():
vault = getattr(app, 'stackalytics_vault', None)
if not vault:
vault = {}
runtime_storage_inst = runtime_storage.get_runtime_storage(
cfg.CONF.runtime_storage_uri)
vault['runtime_storage'] = runtime_storage_inst
vault['memory_storage'] = memory_storage.get_memory_storage(
memory_storage.MEMORY_STORAGE_CACHED,
vault['runtime_storage'].get_update(os.getpid()))
releases = list(runtime_storage_inst.get_by_key('releases'))
vault['start_date'] = releases[0]['end_date']
vault['end_date'] = releases[-1]['end_date']
start_date = releases[0]['end_date']
for r in releases[1:]:
r['start_date'] = start_date
start_date = r['end_date']
vault['releases'] = dict(
(r['release_name'].lower(), r) for r in releases[1:])
modules = runtime_storage_inst.get_by_key('repos')
vault['modules'] = dict(
(r['module'].lower(), r['project_type'].lower()) for r in modules)
app.stackalytics_vault = vault
init_project_types(vault)
else:
if not getattr(flask.request, 'stackalytics_updated', None):
flask.request.stackalytics_updated = True
memory_storage_inst = vault['memory_storage']
memory_storage_inst.update(vault['runtime_storage'].get_update(
os.getpid()))
return vault
def get_vault():
vault = getattr(app, 'stackalytics_vault', None)
if not vault:
vault = {}
vault['runtime_storage'] = runtime_storage.get_runtime_storage(
cfg.CONF.runtime_storage_uri)
vault['persistent_storage'] = (
persistent_storage.get_persistent_storage(
cfg.CONF.persistent_storage_uri))
vault['memory_storage'] = memory_storage.get_memory_storage(
memory_storage.MEMORY_STORAGE_CACHED,
vault['runtime_storage'].get_update(os.getpid()))
releases = vault['persistent_storage'].get_releases()
vault['releases'] = dict((r['release_name'].lower(), r)
for r in releases)
modules = vault['persistent_storage'].get_repos()
vault['modules'] = dict((r['module'].lower(),
r['project_type'].lower()) for r in modules)
app.stackalytics_vault = vault
init_project_types(vault)
else:
if not getattr(flask.request, 'stackalytics_updated', None):
flask.request.stackalytics_updated = True
memory_storage_inst = vault['memory_storage']
memory_storage_inst.update(
vault['runtime_storage'].get_update(os.getpid()))
return vault
def get_vault():
vault = getattr(flask.current_app, 'stackalytics_vault', None)
if not vault:
try:
vault = {}
runtime_storage_inst = runtime_storage.get_runtime_storage(
cfg.CONF.runtime_storage_uri)
vault['runtime_storage'] = runtime_storage_inst
vault['memory_storage'] = memory_storage.get_memory_storage(
memory_storage.MEMORY_STORAGE_CACHED)
flask.current_app.stackalytics_vault = vault
except Exception as e:
LOG.critical('Failed to initialize application: %s',
e,
exc_info=True)
flask.abort(500)
if not getattr(flask.request, 'stackalytics_updated', None):
time_now = utils.date_to_timestamp('now')
may_update_by_time = time_now > vault.get('vault_next_update_time', 0)
if may_update_by_time:
flask.request.stackalytics_updated = True
vault['vault_update_time'] = time_now
vault['vault_next_update_time'] = (
time_now + cfg.CONF.dashboard_update_interval)
memory_storage_inst = vault['memory_storage']
have_updates = memory_storage_inst.update(
compact_records(vault['runtime_storage'].get_update(
os.getpid())))
vault['runtime_storage_update_time'] = (
vault['runtime_storage'].get_by_key(
'runtime_storage_update_time'))
if have_updates:
vault['cache'] = {}
vault['cache_size'] = 0
_init_releases(vault)
_init_module_groups(vault)
_init_project_types(vault)
_init_repos(vault)
_init_user_index(vault)
return vault
def main():
# init conf and logging
conf = cfg.CONF
conf.register_cli_opts(config.OPTS)
conf.register_opts(config.OPTS)
conf(project='stackalytics')
logging.setup('stackalytics')
LOG.info('Logging enabled')
runtime_storage_inst = runtime_storage.get_runtime_storage(
cfg.CONF.runtime_storage_uri)
default_data = utils.read_json_from_uri(cfg.CONF.default_data_uri)
if not default_data:
LOG.critical('Unable to load default data')
return not 0
gerrit = rcs.get_rcs(None, cfg.CONF.review_uri)
gerrit.setup(key_filename=cfg.CONF.ssh_key_filename,
username=cfg.CONF.ssh_username)
default_data_processor.process(runtime_storage_inst,
default_data,
cfg.CONF.git_base_uri,
gerrit,
cfg.CONF.driverlog_data_uri)
process_program_list(runtime_storage_inst, cfg.CONF.program_list_uri)
update_pids(runtime_storage_inst)
record_processor_inst = record_processor.RecordProcessor(
runtime_storage_inst)
process(runtime_storage_inst, record_processor_inst)
apply_corrections(cfg.CONF.corrections_uri, runtime_storage_inst)
# long operation should be the last
update_members(runtime_storage_inst, record_processor_inst)
runtime_storage_inst.set_by_key('runtime_storage_update_time',
utils.date_to_timestamp('now'))
def get_vault():
vault = getattr(flask.current_app, 'stackalytics_vault', None)
if not vault:
try:
vault = {}
runtime_storage_inst = runtime_storage.get_runtime_storage(
cfg.CONF.runtime_storage_uri)
vault['runtime_storage'] = runtime_storage_inst
vault['memory_storage'] = memory_storage.get_memory_storage(
memory_storage.MEMORY_STORAGE_CACHED)
flask.current_app.stackalytics_vault = vault
except Exception as e:
LOG.critical('Failed to initialize application: %s', e,
exc_info=True)
flask.abort(500)
if not getattr(flask.request, 'stackalytics_updated', None):
time_now = utils.date_to_timestamp('now')
may_update_by_time = time_now > vault.get('vault_next_update_time', 0)
if may_update_by_time:
flask.request.stackalytics_updated = True
vault['vault_update_time'] = time_now
vault['vault_next_update_time'] = (
time_now + cfg.CONF.dashboard_update_interval)
memory_storage_inst = vault['memory_storage']
have_updates = memory_storage_inst.update(compact_records(
vault['runtime_storage'].get_update(os.getpid())))
vault['runtime_storage_update_time'] = (
vault['runtime_storage'].get_by_key(
'runtime_storage_update_time'))
if have_updates:
vault['cache'] = {}
vault['cache_size'] = 0
_init_releases(vault)
_init_module_groups(vault)
_init_project_types(vault)
_init_repos(vault)
_init_user_index(vault)
return vault
def main():
# init conf and logging
conf = cfg.CONF
conf.register_cli_opts(config.OPTS)
conf.register_opts(config.OPTS)
conf(project='stackalytics')
logging.setup('stackalytics')
LOG.info('Logging enabled')
runtime_storage_inst = runtime_storage.get_runtime_storage(
cfg.CONF.runtime_storage_uri)
default_data = utils.read_json_from_uri(cfg.CONF.default_data_uri)
if not default_data:
LOG.critical('Unable to load default data')
return not 0
gerrit = rcs.get_rcs(None, cfg.CONF.review_uri)
gerrit.setup(key_filename=cfg.CONF.ssh_key_filename,
username=cfg.CONF.ssh_username)
default_data_processor.process(runtime_storage_inst, default_data,
cfg.CONF.git_base_uri, gerrit,
cfg.CONF.driverlog_data_uri)
process_program_list(runtime_storage_inst, cfg.CONF.program_list_uri)
update_pids(runtime_storage_inst)
record_processor_inst = record_processor.RecordProcessor(
runtime_storage_inst)
process(runtime_storage_inst, record_processor_inst)
apply_corrections(cfg.CONF.corrections_uri, runtime_storage_inst)
# long operation should be the last
update_members(runtime_storage_inst, record_processor_inst)
runtime_storage_inst.set_by_key('runtime_storage_update_time',
utils.date_to_timestamp('now'))
def main():
# init conf and logging
conf = cfg.CONF
conf.register_cli_opts(config.OPTS)
conf.register_opts(config.OPTS)
logging.register_options(conf)
logging.set_defaults()
conf(project='stackalytics')
logging.setup(conf, 'stackalytics')
LOG.info('Logging enabled')
conf.log_opt_values(LOG, std_logging.DEBUG)
runtime_storage_inst = runtime_storage.get_runtime_storage(
cfg.CONF.runtime_storage_uri)
default_data = utils.read_json_from_uri(cfg.CONF.default_data_uri)
if not default_data:
LOG.critical('Unable to load default data')
return not 0
default_data_processor.process(runtime_storage_inst,
default_data,
cfg.CONF.driverlog_data_uri)
process_project_list(runtime_storage_inst, cfg.CONF.project_list_uri)
update_pids(runtime_storage_inst)
record_processor_inst = record_processor.RecordProcessor(
runtime_storage_inst)
process(runtime_storage_inst, record_processor_inst)
apply_corrections(cfg.CONF.corrections_uri, runtime_storage_inst)
# long operation should be the last
update_members(runtime_storage_inst, record_processor_inst)
runtime_storage_inst.set_by_key('runtime_storage_update_time',
utils.date_to_timestamp('now'))
def main():
# init conf and logging
conf = cfg.CONF
conf.register_cli_opts(config.OPTS)
conf.register_opts(config.OPTS)
logging.register_options(conf)
logging.set_defaults()
conf(project='stackalytics')
logging.setup(conf, 'stackalytics')
LOG.info('Logging enabled')
conf.log_opt_values(LOG, std_logging.DEBUG)
runtime_storage_inst = runtime_storage.get_runtime_storage(
cfg.CONF.runtime_storage_uri)
default_data = utils.read_json_from_uri(cfg.CONF.default_data_uri)
if not default_data:
LOG.critical('Unable to load default data')
return not 0
default_data_processor.process(runtime_storage_inst, default_data,
cfg.CONF.driverlog_data_uri)
process_project_list(runtime_storage_inst, cfg.CONF.project_list_uri)
update_pids(runtime_storage_inst)
record_processor_inst = record_processor.RecordProcessor(
runtime_storage_inst)
process(runtime_storage_inst, record_processor_inst)
apply_corrections(cfg.CONF.corrections_uri, runtime_storage_inst)
# long operation should be the last
update_members(runtime_storage_inst, record_processor_inst)
runtime_storage_inst.set_by_key('runtime_storage_update_time',
utils.date_to_timestamp('now'))
def main():
utils.init_config_and_logging(config.CONNECTION_OPTS +
config.PROCESSOR_OPTS)
runtime_storage_inst = runtime_storage.get_runtime_storage(
CONF.runtime_storage_uri)
default_data = utils.read_json_from_uri(CONF.default_data_uri)
if not default_data:
LOG.critical('Unable to load default data')
return not 0
try:
jsonschema.validate(default_data, schema.default_data)
except jsonschema.ValidationError as e:
LOG.critical('The default data is invalid: %s' % e)
return not 0
default_data_processor.process(runtime_storage_inst,
default_data)
process_project_list(runtime_storage_inst)
update_pids(runtime_storage_inst)
record_processor_inst = record_processor.RecordProcessor(
runtime_storage_inst)
process(runtime_storage_inst, record_processor_inst)
apply_corrections(CONF.corrections_uri, runtime_storage_inst)
# long operation should be the last
update_members(runtime_storage_inst, record_processor_inst)
runtime_storage_inst.set_by_key('runtime_storage_update_time',
utils.date_to_timestamp('now'))
LOG.info('stackalytics-processor succeeded.')
