def project_syncer(logger, loglevel, uri, config_path, sync_period, numworkers,
env):
"""
Wrapper for running opengrok-sync.
To be run in a thread/process in the background.
"""
wait_for_tomcat(logger, uri)
set_config_value(logger, 'projectsEnabled', 'true', uri)
while True:
refresh_projects(logger, uri)
if os.environ.get('OPENGROK_SYNC_YML'): # debug only
config_file = os.environ.get('OPENGROK_SYNC_YML')
else:
config_file = os.path.join(fs_root, 'scripts', 'sync.yml')
config = read_config(logger, config_file)
if config is None:
logger.error("Cannot read config file from {}".format(config_file))
raise Exception("no sync config")
projects = list_projects(logger, uri)
#
# The driveon=True is needed for the initial indexing of newly
# added project, otherwise the incoming check in the opengrok-mirror
# program would short circuit it.
#
if env:
logger.info('Merging commands with environment')
commands = merge_commands_env(config["commands"], env)
logger.debug(config['commands'])
else:
commands = config["commands"]
logger.info("Sync starting")
do_sync(loglevel,
commands,
config.get('cleanup'),
projects,
config.get("ignore_errors"),
uri,
numworkers,
driveon=True,
logger=logger,
print_output=True)
logger.info("Sync done")
# Workaround for https://github.com/oracle/opengrok/issues/1670
Path(os.path.join(OPENGROK_DATA_ROOT, 'timestamp')).touch()
save_config(logger, uri, config_path)
sleep_seconds = sync_period * 60
logger.info("Sleeping for {} seconds".format(sleep_seconds))
time.sleep(sleep_seconds)
def test_read_config_empty_yaml():
tmpf = tempfile.NamedTemporaryFile(mode='w+t', delete=False)
tmpf.file.write('#foo\n')
tmpf.close()
res = read_config(mock(spec=logging.Logger), tmpf.name)
os.remove(tmpf.name)
assert res is not None
assert type(res) == dict
assert len(res) == 0
def main():
log_level = os.environ.get('OPENGROK_LOG_LEVEL')
if log_level:
log_level = get_log_level(log_level)
else:
log_level = logging.INFO
logger = get_console_logger(get_class_basename(), log_level)
uri, url_root = set_url_root(logger, os.environ.get('URL_ROOT'))
logger.debug("URL_ROOT = {}".format(url_root))
logger.debug("URI = {}".format(uri))
sync_period = get_num_from_env(logger, 'SYNC_PERIOD_MINUTES', 10)
if sync_period == 0:
logger.info("periodic synchronization disabled")
else:
logger.info("synchronization period = {} minutes".format(sync_period))
# Note that deploy is done before Tomcat is started.
deploy(logger, url_root)
if url_root != '/source':
setup_redirect_source(logger, url_root)
env = {}
extra_indexer_options = os.environ.get('INDEXER_OPT', '')
if extra_indexer_options:
logger.info("extra indexer options: {}".format(extra_indexer_options))
env['OPENGROK_INDEXER_OPTIONAL_ARGS'] = extra_indexer_options
if os.environ.get(NOMIRROR_ENV_NAME):
env['OPENGROK_NO_MIRROR'] = os.environ.get(NOMIRROR_ENV_NAME)
logger.debug('Extra environment: {}'.format(env))
use_projects = True
if os.environ.get('AVOID_PROJECTS'):
use_projects = False
#
# Create empty configuration to avoid the non existent file exception
# in the web app during the first web app startup.
#
if not os.path.exists(OPENGROK_CONFIG_FILE) or \
os.path.getsize(OPENGROK_CONFIG_FILE) == 0:
create_bare_config(logger, use_projects, extra_indexer_options.split())
#
# Index check needs read-only configuration so it is placed
# right after create_bare_config().
#
check_index_and_wipe_out(logger)
#
# If there is read-only configuration file, merge it with current
# configuration.
#
read_only_config_file = os.environ.get('READONLY_CONFIG_FILE')
if read_only_config_file and os.path.exists(read_only_config_file):
logger.info('Merging read-only configuration from \'{}\' with current '
'configuration in \'{}\''.format(read_only_config_file,
OPENGROK_CONFIG_FILE))
out_file = None
with tempfile.NamedTemporaryFile(mode='w+',
delete=False,
prefix='merged_config') as tmp_out:
out_file = tmp_out.name
merge_config_files(read_only_config_file,
OPENGROK_CONFIG_FILE,
tmp_out,
jar=OPENGROK_JAR,
loglevel=log_level)
if out_file and os.path.getsize(out_file) > 0:
shutil.move(tmp_out.name, OPENGROK_CONFIG_FILE)
else:
logger.warning('Failed to merge read-only configuration, '
'leaving the original in place')
if out_file:
os.remove(out_file)
sync_enabled = True
if use_projects:
num_workers = get_num_from_env(logger, 'WORKERS',
multiprocessing.cpu_count())
logger.info('Number of sync workers: {}'.format(num_workers))
if not os.environ.get(NOMIRROR_ENV_NAME):
mirror_config = os.path.join(OPENGROK_CONFIG_DIR, "mirror.yml")
if not os.path.exists(mirror_config):
with open(mirror_config, 'w') as fp:
fp.write("# Empty config file for opengrok-mirror\n")
else:
conf = read_config(logger, mirror_config)
logger.info("Checking mirror configuration in '{}'".format(
mirror_config))
if not check_configuration(conf):
logger.error("Mirror configuration in '{}' is invalid, "
"disabling sync".format(mirror_config))
sync_enabled = False
worker_function = project_syncer
syncer_args = (logger, log_level, uri, OPENGROK_CONFIG_FILE,
num_workers, env)
else:
worker_function = indexer_no_projects
syncer_args = (logger, uri, OPENGROK_CONFIG_FILE,
extra_indexer_options)
if sync_enabled:
logger.debug("Starting sync thread")
sync_thread = threading.Thread(target=worker_function,
name="Sync thread",
args=syncer_args,
daemon=True)
sync_thread.start()
start_rest_thread(logger)
if sync_period > 0:
start_timeout_thread(logger, sync_period)
# Start Tomcat last. It will be the foreground process.
logger.info("Starting Tomcat")
global tomcat_popen
tomcat_popen = subprocess.Popen(
[os.path.join(tomcat_root, 'bin', 'catalina.sh'), 'run'])
tomcat_popen.wait()