def project_syncer(logger, loglevel, uri, config_path, sync_period, numworkers, env): """ Wrapper for running opengrok-sync. To be run in a thread/process in the background. """ wait_for_tomcat(logger, uri) set_config_value(logger, 'projectsEnabled', 'true', uri) while True: refresh_projects(logger, uri) if os.environ.get('OPENGROK_SYNC_YML'): # debug only config_file = os.environ.get('OPENGROK_SYNC_YML') else: config_file = os.path.join(fs_root, 'scripts', 'sync.yml') config = read_config(logger, config_file) if config is None: logger.error("Cannot read config file from {}".format(config_file)) raise Exception("no sync config") projects = list_projects(logger, uri) # # The driveon=True is needed for the initial indexing of newly # added project, otherwise the incoming check in the opengrok-mirror # program would short circuit it. # if env: logger.info('Merging commands with environment') commands = merge_commands_env(config["commands"], env) logger.debug(config['commands']) else: commands = config["commands"] logger.info("Sync starting") do_sync(loglevel, commands, config.get('cleanup'), projects, config.get("ignore_errors"), uri, numworkers, driveon=True, logger=logger, print_output=True) logger.info("Sync done") # Workaround for https://github.com/oracle/opengrok/issues/1670 Path(os.path.join(OPENGROK_DATA_ROOT, 'timestamp')).touch() save_config(logger, uri, config_path) sleep_seconds = sync_period * 60 logger.info("Sleeping for {} seconds".format(sleep_seconds)) time.sleep(sleep_seconds)
def test_read_config_empty_yaml(): tmpf = tempfile.NamedTemporaryFile(mode='w+t', delete=False) tmpf.file.write('#foo\n') tmpf.close() res = read_config(mock(spec=logging.Logger), tmpf.name) os.remove(tmpf.name) assert res is not None assert type(res) == dict assert len(res) == 0
def main(): log_level = os.environ.get('OPENGROK_LOG_LEVEL') if log_level: log_level = get_log_level(log_level) else: log_level = logging.INFO logger = get_console_logger(get_class_basename(), log_level) uri, url_root = set_url_root(logger, os.environ.get('URL_ROOT')) logger.debug("URL_ROOT = {}".format(url_root)) logger.debug("URI = {}".format(uri)) sync_period = get_num_from_env(logger, 'SYNC_PERIOD_MINUTES', 10) if sync_period == 0: logger.info("periodic synchronization disabled") else: logger.info("synchronization period = {} minutes".format(sync_period)) # Note that deploy is done before Tomcat is started. deploy(logger, url_root) if url_root != '/source': setup_redirect_source(logger, url_root) env = {} extra_indexer_options = os.environ.get('INDEXER_OPT', '') if extra_indexer_options: logger.info("extra indexer options: {}".format(extra_indexer_options)) env['OPENGROK_INDEXER_OPTIONAL_ARGS'] = extra_indexer_options if os.environ.get(NOMIRROR_ENV_NAME): env['OPENGROK_NO_MIRROR'] = os.environ.get(NOMIRROR_ENV_NAME) logger.debug('Extra environment: {}'.format(env)) use_projects = True if os.environ.get('AVOID_PROJECTS'): use_projects = False # # Create empty configuration to avoid the non existent file exception # in the web app during the first web app startup. # if not os.path.exists(OPENGROK_CONFIG_FILE) or \ os.path.getsize(OPENGROK_CONFIG_FILE) == 0: create_bare_config(logger, use_projects, extra_indexer_options.split()) # # Index check needs read-only configuration so it is placed # right after create_bare_config(). # check_index_and_wipe_out(logger) # # If there is read-only configuration file, merge it with current # configuration. # read_only_config_file = os.environ.get('READONLY_CONFIG_FILE') if read_only_config_file and os.path.exists(read_only_config_file): logger.info('Merging read-only configuration from \'{}\' with current ' 'configuration in \'{}\''.format(read_only_config_file, OPENGROK_CONFIG_FILE)) out_file = None with tempfile.NamedTemporaryFile(mode='w+', delete=False, prefix='merged_config') as tmp_out: out_file = tmp_out.name merge_config_files(read_only_config_file, OPENGROK_CONFIG_FILE, tmp_out, jar=OPENGROK_JAR, loglevel=log_level) if out_file and os.path.getsize(out_file) > 0: shutil.move(tmp_out.name, OPENGROK_CONFIG_FILE) else: logger.warning('Failed to merge read-only configuration, ' 'leaving the original in place') if out_file: os.remove(out_file) sync_enabled = True if use_projects: num_workers = get_num_from_env(logger, 'WORKERS', multiprocessing.cpu_count()) logger.info('Number of sync workers: {}'.format(num_workers)) if not os.environ.get(NOMIRROR_ENV_NAME): mirror_config = os.path.join(OPENGROK_CONFIG_DIR, "mirror.yml") if not os.path.exists(mirror_config): with open(mirror_config, 'w') as fp: fp.write("# Empty config file for opengrok-mirror\n") else: conf = read_config(logger, mirror_config) logger.info("Checking mirror configuration in '{}'".format( mirror_config)) if not check_configuration(conf): logger.error("Mirror configuration in '{}' is invalid, " "disabling sync".format(mirror_config)) sync_enabled = False worker_function = project_syncer syncer_args = (logger, log_level, uri, OPENGROK_CONFIG_FILE, num_workers, env) else: worker_function = indexer_no_projects syncer_args = (logger, uri, OPENGROK_CONFIG_FILE, extra_indexer_options) if sync_enabled: logger.debug("Starting sync thread") sync_thread = threading.Thread(target=worker_function, name="Sync thread", args=syncer_args, daemon=True) sync_thread.start() start_rest_thread(logger) if sync_period > 0: start_timeout_thread(logger, sync_period) # Start Tomcat last. It will be the foreground process. logger.info("Starting Tomcat") global tomcat_popen tomcat_popen = subprocess.Popen( [os.path.join(tomcat_root, 'bin', 'catalina.sh'), 'run']) tomcat_popen.wait()