def main(): # init conf and logging conf = cfg.CONF conf.register_cli_opts(config.OPTS) conf.register_opts(config.OPTS) conf() logging.setup('stackalytics') LOG.info('Logging enabled') runtime_storage_inst = runtime_storage.get_runtime_storage( cfg.CONF.runtime_storage_uri) default_data = utils.read_json_from_uri(cfg.CONF.default_data_uri) if not default_data: LOG.critical('Unable to load default data') return not 0 default_data_processor.process(runtime_storage_inst, default_data, cfg.CONF.sources_root) update_pids(runtime_storage_inst) update_repos(runtime_storage_inst) apply_corrections(cfg.CONF.corrections_uri, runtime_storage_inst)
def main(): utils.init_config_and_logging(config.CONNECTION_OPTS + config.PROCESSOR_OPTS) runtime_storage_inst = runtime_storage.get_runtime_storage(cfg.CONF.runtime_storage_uri) default_data = utils.read_json_from_uri(cfg.CONF.default_data_uri) if not default_data: LOG.critical("Unable to load default data") return not 0 default_data_processor.process(runtime_storage_inst, default_data, cfg.CONF.driverlog_data_uri) process_project_list(runtime_storage_inst, cfg.CONF.project_list_uri) update_pids(runtime_storage_inst) record_processor_inst = record_processor.RecordProcessor(runtime_storage_inst) process(runtime_storage_inst, record_processor_inst) apply_corrections(cfg.CONF.corrections_uri, runtime_storage_inst) # long operation should be the last update_members(runtime_storage_inst, record_processor_inst) runtime_storage_inst.set_by_key("runtime_storage_update_time", utils.date_to_timestamp("now"))
def get_vault(): vault = getattr(app, 'stackalytics_vault', None) if not vault: vault = {} runtime_storage_inst = runtime_storage.get_runtime_storage( cfg.CONF.runtime_storage_uri) vault['runtime_storage'] = runtime_storage_inst vault['memory_storage'] = memory_storage.get_memory_storage( memory_storage.MEMORY_STORAGE_CACHED, vault['runtime_storage'].get_update(os.getpid())) releases = list(runtime_storage_inst.get_by_key('releases')) vault['start_date'] = releases[0]['end_date'] vault['end_date'] = releases[-1]['end_date'] start_date = releases[0]['end_date'] for r in releases[1:]: r['start_date'] = start_date start_date = r['end_date'] vault['releases'] = dict((r['release_name'].lower(), r) for r in releases[1:]) modules = runtime_storage_inst.get_by_key('repos') vault['modules'] = dict((r['module'].lower(), r['project_type'].lower()) for r in modules) app.stackalytics_vault = vault init_project_types(vault) else: if not getattr(flask.request, 'stackalytics_updated', None): flask.request.stackalytics_updated = True memory_storage_inst = vault['memory_storage'] memory_storage_inst.update( vault['runtime_storage'].get_update(os.getpid())) return vault
def main(): # init conf and logging conf = cfg.CONF conf.register_cli_opts(config.OPTS) conf.register_cli_opts(OPTS) conf.register_opts(config.OPTS) conf.register_opts(OPTS) conf() logging.setup('stackalytics') LOG.info('Logging enabled') runtime_storage_inst = runtime_storage.get_runtime_storage( cfg.CONF.runtime_storage_uri) filename = cfg.CONF.file if cfg.CONF.reverse: if filename: fd = open(filename, 'r') else: fd = sys.stdin import_data(runtime_storage_inst, fd) else: if filename: fd = open(filename, 'w') else: fd = sys.stdout export_data(runtime_storage_inst, fd)
def main(): # init conf and logging conf = cfg.CONF conf.register_cli_opts(config.OPTS) conf.register_opts(config.OPTS) conf() logging.setup('stackalytics') LOG.info('Logging enabled') runtime_storage_inst = runtime_storage.get_runtime_storage( cfg.CONF.runtime_storage_uri) default_data = utils.read_json_from_uri(cfg.CONF.default_data_uri) if not default_data: LOG.critical('Unable to load default data') return not 0 default_data_processor.process(runtime_storage_inst, default_data, cfg.CONF.sources_root, cfg.CONF.force_update) process_program_list(runtime_storage_inst, cfg.CONF.program_list_uri) update_pids(runtime_storage_inst) record_processor_inst = record_processor.RecordProcessor( runtime_storage_inst) update_records(runtime_storage_inst, record_processor_inst) apply_corrections(cfg.CONF.corrections_uri, runtime_storage_inst) # long operation should be the last update_members(runtime_storage_inst, record_processor_inst)
def main(): utils.init_config_and_logging(config.CONNECTION_OPTS + config.PROCESSOR_OPTS) runtime_storage_inst = runtime_storage.get_runtime_storage( cfg.CONF.runtime_storage_uri) default_data = utils.read_json_from_uri(cfg.CONF.default_data_uri) if not default_data: LOG.critical('Unable to load default data') return not 0 default_data_processor.process(runtime_storage_inst, default_data, cfg.CONF.driverlog_data_uri) process_project_list(runtime_storage_inst) update_pids(runtime_storage_inst) record_processor_inst = record_processor.RecordProcessor( runtime_storage_inst) process(runtime_storage_inst, record_processor_inst) apply_corrections(cfg.CONF.corrections_uri, runtime_storage_inst) # long operation should be the last update_members(runtime_storage_inst, record_processor_inst) runtime_storage_inst.set_by_key('runtime_storage_update_time', utils.date_to_timestamp('now'))
def main(): utils.init_config_and_logging(config.CONNECTION_OPTS + config.PROCESSOR_OPTS) runtime_storage_inst = runtime_storage.get_runtime_storage( CONF.runtime_storage_uri) default_data = utils.read_json_from_uri(CONF.default_data_uri) if not default_data: LOG.critical('Unable to load default data') return not 0 try: jsonschema.validate(default_data, schema.default_data) except jsonschema.ValidationError as e: LOG.critical('The default data is invalid: %s' % e) return not 0 default_data_processor.process(runtime_storage_inst, default_data) process_project_list(runtime_storage_inst) update_pids(runtime_storage_inst) record_processor_inst = record_processor.RecordProcessor( runtime_storage_inst) process(runtime_storage_inst, record_processor_inst) runtime_storage_inst.set_by_key('runtime_storage_update_time', utils.date_to_timestamp('now')) LOG.info('stackalytics-processor succeeded.')
def get_vault(): vault = getattr(flask.current_app, 'stackalytics_vault', None) if not vault: try: vault = {} runtime_storage_inst = runtime_storage.get_runtime_storage( cfg.CONF.runtime_storage_uri) vault['runtime_storage'] = runtime_storage_inst vault['memory_storage'] = memory_storage.get_memory_storage( memory_storage.MEMORY_STORAGE_CACHED) init_project_types(vault) init_releases(vault) flask.current_app.stackalytics_vault = vault except Exception as e: LOG.critical('Failed to initialize application: %s', e) LOG.exception(e) flask.abort(500) if not getattr(flask.request, 'stackalytics_updated', None): flask.request.stackalytics_updated = True memory_storage_inst = vault['memory_storage'] have_updates = memory_storage_inst.update( vault['runtime_storage'].get_update(os.getpid())) if have_updates: init_project_types(vault) init_releases(vault) init_module_groups(vault) return vault
def get_vault(): vault = getattr(app, 'stackalytics_vault', None) if not vault: try: vault = {} runtime_storage_inst = runtime_storage.get_runtime_storage( cfg.CONF.runtime_storage_uri) vault['runtime_storage'] = runtime_storage_inst vault['memory_storage'] = memory_storage.get_memory_storage( memory_storage.MEMORY_STORAGE_CACHED) init_project_types(vault) init_releases(vault) app.stackalytics_vault = vault except Exception as e: LOG.critical('Failed to initialize application: %s', e) LOG.exception(e) flask.abort(500) if not getattr(flask.request, 'stackalytics_updated', None): flask.request.stackalytics_updated = True memory_storage_inst = vault['memory_storage'] have_updates = memory_storage_inst.update( vault['runtime_storage'].get_update(os.getpid())) if have_updates: init_project_types(vault) init_releases(vault) init_module_groups(vault) return vault
def get_vault(): vault = getattr(app, 'stackalytics_vault', None) if not vault: vault = {} runtime_storage_inst = runtime_storage.get_runtime_storage( cfg.CONF.runtime_storage_uri) vault['runtime_storage'] = runtime_storage_inst vault['memory_storage'] = memory_storage.get_memory_storage( memory_storage.MEMORY_STORAGE_CACHED, vault['runtime_storage'].get_update(os.getpid())) releases = list(runtime_storage_inst.get_by_key('releases')) vault['start_date'] = releases[0]['end_date'] vault['end_date'] = releases[-1]['end_date'] start_date = releases[0]['end_date'] for r in releases[1:]: r['start_date'] = start_date start_date = r['end_date'] vault['releases'] = dict( (r['release_name'].lower(), r) for r in releases[1:]) modules = runtime_storage_inst.get_by_key('repos') vault['modules'] = dict( (r['module'].lower(), r['project_type'].lower()) for r in modules) app.stackalytics_vault = vault init_project_types(vault) else: if not getattr(flask.request, 'stackalytics_updated', None): flask.request.stackalytics_updated = True memory_storage_inst = vault['memory_storage'] memory_storage_inst.update(vault['runtime_storage'].get_update( os.getpid())) return vault
def get_vault(): vault = getattr(app, 'stackalytics_vault', None) if not vault: vault = {} vault['runtime_storage'] = runtime_storage.get_runtime_storage( cfg.CONF.runtime_storage_uri) vault['persistent_storage'] = ( persistent_storage.get_persistent_storage( cfg.CONF.persistent_storage_uri)) vault['memory_storage'] = memory_storage.get_memory_storage( memory_storage.MEMORY_STORAGE_CACHED, vault['runtime_storage'].get_update(os.getpid())) releases = vault['persistent_storage'].get_releases() vault['releases'] = dict((r['release_name'].lower(), r) for r in releases) modules = vault['persistent_storage'].get_repos() vault['modules'] = dict((r['module'].lower(), r['project_type'].lower()) for r in modules) app.stackalytics_vault = vault init_project_types(vault) else: if not getattr(flask.request, 'stackalytics_updated', None): flask.request.stackalytics_updated = True memory_storage_inst = vault['memory_storage'] memory_storage_inst.update( vault['runtime_storage'].get_update(os.getpid())) return vault
def get_vault(): vault = getattr(flask.current_app, 'stackalytics_vault', None) if not vault: try: vault = {} runtime_storage_inst = runtime_storage.get_runtime_storage( cfg.CONF.runtime_storage_uri) vault['runtime_storage'] = runtime_storage_inst vault['memory_storage'] = memory_storage.get_memory_storage( memory_storage.MEMORY_STORAGE_CACHED) flask.current_app.stackalytics_vault = vault except Exception as e: LOG.critical('Failed to initialize application: %s', e, exc_info=True) flask.abort(500) if not getattr(flask.request, 'stackalytics_updated', None): time_now = utils.date_to_timestamp('now') may_update_by_time = time_now > vault.get('vault_next_update_time', 0) if may_update_by_time: flask.request.stackalytics_updated = True vault['vault_update_time'] = time_now vault['vault_next_update_time'] = ( time_now + cfg.CONF.dashboard_update_interval) memory_storage_inst = vault['memory_storage'] have_updates = memory_storage_inst.update( compact_records(vault['runtime_storage'].get_update( os.getpid()))) vault['runtime_storage_update_time'] = ( vault['runtime_storage'].get_by_key( 'runtime_storage_update_time')) if have_updates: vault['cache'] = {} vault['cache_size'] = 0 _init_releases(vault) _init_module_groups(vault) _init_project_types(vault) _init_repos(vault) _init_user_index(vault) return vault
def main(): # init conf and logging conf = cfg.CONF conf.register_cli_opts(config.OPTS) conf.register_opts(config.OPTS) conf(project='stackalytics') logging.setup('stackalytics') LOG.info('Logging enabled') runtime_storage_inst = runtime_storage.get_runtime_storage( cfg.CONF.runtime_storage_uri) default_data = utils.read_json_from_uri(cfg.CONF.default_data_uri) if not default_data: LOG.critical('Unable to load default data') return not 0 gerrit = rcs.get_rcs(None, cfg.CONF.review_uri) gerrit.setup(key_filename=cfg.CONF.ssh_key_filename, username=cfg.CONF.ssh_username) default_data_processor.process(runtime_storage_inst, default_data, cfg.CONF.git_base_uri, gerrit, cfg.CONF.driverlog_data_uri) process_program_list(runtime_storage_inst, cfg.CONF.program_list_uri) update_pids(runtime_storage_inst) record_processor_inst = record_processor.RecordProcessor( runtime_storage_inst) process(runtime_storage_inst, record_processor_inst) apply_corrections(cfg.CONF.corrections_uri, runtime_storage_inst) # long operation should be the last update_members(runtime_storage_inst, record_processor_inst) runtime_storage_inst.set_by_key('runtime_storage_update_time', utils.date_to_timestamp('now'))
def get_vault(): vault = getattr(flask.current_app, 'stackalytics_vault', None) if not vault: try: vault = {} runtime_storage_inst = runtime_storage.get_runtime_storage( cfg.CONF.runtime_storage_uri) vault['runtime_storage'] = runtime_storage_inst vault['memory_storage'] = memory_storage.get_memory_storage( memory_storage.MEMORY_STORAGE_CACHED) flask.current_app.stackalytics_vault = vault except Exception as e: LOG.critical('Failed to initialize application: %s', e, exc_info=True) flask.abort(500) if not getattr(flask.request, 'stackalytics_updated', None): time_now = utils.date_to_timestamp('now') may_update_by_time = time_now > vault.get('vault_next_update_time', 0) if may_update_by_time: flask.request.stackalytics_updated = True vault['vault_update_time'] = time_now vault['vault_next_update_time'] = ( time_now + cfg.CONF.dashboard_update_interval) memory_storage_inst = vault['memory_storage'] have_updates = memory_storage_inst.update(compact_records( vault['runtime_storage'].get_update(os.getpid()))) vault['runtime_storage_update_time'] = ( vault['runtime_storage'].get_by_key( 'runtime_storage_update_time')) if have_updates: vault['cache'] = {} vault['cache_size'] = 0 _init_releases(vault) _init_module_groups(vault) _init_project_types(vault) _init_repos(vault) _init_user_index(vault) return vault
def main(): # init conf and logging conf = cfg.CONF conf.register_cli_opts(config.OPTS) conf.register_opts(config.OPTS) logging.register_options(conf) logging.set_defaults() conf(project='stackalytics') logging.setup(conf, 'stackalytics') LOG.info('Logging enabled') conf.log_opt_values(LOG, std_logging.DEBUG) runtime_storage_inst = runtime_storage.get_runtime_storage( cfg.CONF.runtime_storage_uri) default_data = utils.read_json_from_uri(cfg.CONF.default_data_uri) if not default_data: LOG.critical('Unable to load default data') return not 0 default_data_processor.process(runtime_storage_inst, default_data, cfg.CONF.driverlog_data_uri) process_project_list(runtime_storage_inst, cfg.CONF.project_list_uri) update_pids(runtime_storage_inst) record_processor_inst = record_processor.RecordProcessor( runtime_storage_inst) process(runtime_storage_inst, record_processor_inst) apply_corrections(cfg.CONF.corrections_uri, runtime_storage_inst) # long operation should be the last update_members(runtime_storage_inst, record_processor_inst) runtime_storage_inst.set_by_key('runtime_storage_update_time', utils.date_to_timestamp('now'))
def main(): utils.init_config_and_logging(config.CONNECTION_OPTS + config.PROCESSOR_OPTS) runtime_storage_inst = runtime_storage.get_runtime_storage( CONF.runtime_storage_uri) default_data = utils.read_json_from_uri(CONF.default_data_uri) if not default_data: LOG.critical('Unable to load default data') return not 0 try: jsonschema.validate(default_data, schema.default_data) except jsonschema.ValidationError as e: LOG.critical('The default data is invalid: %s' % e) return not 0 default_data_processor.process(runtime_storage_inst, default_data) process_project_list(runtime_storage_inst) update_pids(runtime_storage_inst) record_processor_inst = record_processor.RecordProcessor( runtime_storage_inst) process(runtime_storage_inst, record_processor_inst) apply_corrections(CONF.corrections_uri, runtime_storage_inst) # long operation should be the last update_members(runtime_storage_inst, record_processor_inst) runtime_storage_inst.set_by_key('runtime_storage_update_time', utils.date_to_timestamp('now')) LOG.info('stackalytics-processor succeeded.')