def start_arcyd(daemonize=True, loop=True, restart=False, stop_message=''):
# exit gracefully if this process is killed
phlsys_signal.set_exit_on_sigterm()
fs = abdt_fs.make_default_accessor()
with fs.lockfile_context():
pid = fs.get_pid_or_none()
if pid is not None and phlsys_pid.is_running(pid):
if restart:
stop_arcyd_pid(pid, fs.layout.killfile, stop_message)
else:
raise Exception("already running")
if daemonize:
phlsys_daemonize.do(
stdout_path=fs.layout.stdout,
stderr_path=fs.layout.stderr)
# important that we do this *after* daemonizing
pid = phlsys_pid.get()
fs.set_pid(pid)
parser = argparse.ArgumentParser()
params = []
for line in open(fs.layout.root_config):
params.append(line.strip())
if not loop:
params.append('--no-loop')
repo_configs = abdi_repoargs.parse_config_file_list(
fs.repo_config_path_list())
abdi_processrepos.setupParser(parser)
args = parser.parse_args(params)
def logger_config():
_setup_logger(fs, daemonize)
with phlsys_multiprocessing.logging_context(logger_config):
_LOGGER.debug("start with args: {}".format(args))
while True:
_LOGGER.info("arcyd started")
try:
exit_code = abdi_processrepos.process(args, repo_configs)
_LOGGER.debug("arcyd process loop exit_code: %s" % exit_code)
if exit_code == abdi_processexitcodes.ExitCodes.ec_exit:
break
finally:
_LOGGER.info("arcyd stopped")
_LOGGER.debug("reloading arcyd configuration")
try:
with fs.lockfile_context():
repo_configs = abdi_repoargs.parse_config_file_list(
fs.repo_config_path_list())
except phlsys_fs.LockfileExistsError:
_LOGGER.error("couldn't acquire lockfile, reload failed")
def _determine_name_from_url(fs, repo_url):
"""Return the string name of the repository for 'repo_url' or raise.
If there is not exactly one repository config that refers to 'repo_url'
then raise.
:fs: abdt_fs.Accessor to use to determine the repo name from the url
:repo_url: string clone url of the repository we fetch from / push to
:returns: string name of the matching config
"""
repo_config_path_list = fs.repo_config_path_list()
repo_name_config_list = abdi_repoargs.parse_config_file_list(
repo_config_path_list)
candidate_names = []
for repo_name, repo_config in repo_name_config_list:
url = abdi_repoargs.get_repo_url(repo_config)
if repo_url == url:
candidate_names.append(repo_name)
if not candidate_names:
raise Exception("url '{url}' didn't match any names".format(
url=repo_url))
if len(candidate_names) > 1:
raise Exception("url '{url}' matches many names:\n{names}\n".format(
url=repo_url,
names=candidate_names))
return candidate_names[0]
def process(args):
fs = abdt_fs.make_default_accessor()
exit_code = 0
with fs.lockfile_context():
pid = fs.get_pid_or_none()
if pid is not None and phlsys_pid.is_running(pid):
raise Exception("cannot fsck whilst arcyd is running.")
repo_config_path_list = _determine_repo_config_path_list(
fs, args.repos)
if not _check_repo_config_path_list(repo_config_path_list):
exit_code = 1
repo_name_config_list = abdi_repoargs.parse_config_file_list(
repo_config_path_list)
if not _check_repo_name_config_list(args, repo_name_config_list):
exit_code = 1
if exit_code != 0 and not args.fix:
print("use '--fix' to attempt to fix the issues")
return exit_code
def _determine_name_from_url(fs, repo_url):
"""Return the string name of the repository for 'repo_url' or raise.
If there is not exactly one repository config that refers to 'repo_url'
then raise.
:fs: abdt_fs.Accessor to use to determine the repo name from the url
:repo_url: string clone url of the repository we fetch from / push to
:returns: string name of the matching config
"""
repo_config_path_list = fs.repo_config_path_list()
repo_name_config_list = abdi_repoargs.parse_config_file_list(
repo_config_path_list)
candidate_names = []
for repo_name, repo_config in repo_name_config_list:
url = abdi_repoargs.get_repo_url(repo_config)
if repo_url == url:
candidate_names.append(repo_name)
if not candidate_names:
raise Exception(
"url '{url}' didn't match any names".format(url=repo_url))
if len(candidate_names) > 1:
raise Exception("url '{url}' matches many names:\n{names}\n".format(
url=repo_url, names=candidate_names))
return candidate_names[0]
def process(args):
_ = args # NOQA
fs = abdt_fs.make_default_accessor()
repo_config_path_list = fs.repo_config_path_list()
repo_name_config_list = abdi_repoargs.parse_config_file_list(
repo_config_path_list)
url_watcher_wrapper = phlurl_watcher.FileCacheWatcherWrapper(
fs.layout.urlwatcher_cache_path)
url_watcher_wrapper.watcher.refresh()
for repo_name, repo_config in repo_name_config_list:
print(repo_name + '..', end=' ')
snoop_url = abdi_repoargs.get_repo_snoop_url(repo_config)
abd_repo = abdt_git.Repo(
phlsys_git.Repo(repo_config.repo_path),
"origin",
repo_config.repo_desc)
did_fetch = abdi_processrepoargs.fetch_if_needed(
url_watcher_wrapper.watcher,
snoop_url,
abd_repo,
repo_config.repo_desc)
if did_fetch:
print('fetched')
else:
print('skipped')
url_watcher_wrapper.save()
def process(args):
_ = args # NOQA
fs = abdt_fs.make_default_accessor()
with fs.lockfile_context():
pid = fs.get_pid_or_none()
if pid is not None and phlsys_pid.is_running(pid):
raise Exception("cannot fetch whilst arcyd is running.")
repo_config_path_list = fs.repo_config_path_list()
repo_name_config_list = abdi_repoargs.parse_config_file_list(
repo_config_path_list)
url_watcher_wrapper = phlurl_watcher.FileCacheWatcherWrapper(
fs.layout.urlwatcher_cache_path)
# Let the user know what's happening before potentially blocking for a
# while.
print('Refreshing repository snoop status ..', end=' ')
# Make sure that the output is actually visible by flushing stdout
# XXX: Will use 'flush' parameter to 'print()' in Python 3.3
sys.stdout.flush()
print("done")
url_watcher_wrapper.watcher.refresh()
for repo_name, repo_config in repo_name_config_list:
print(repo_name + ' ..', end=' ')
# Make sure that the output is actually visible by flushing stdout
# XXX: Will use 'flush' parameter to 'print()' in Python 3.3
sys.stdout.flush()
snoop_url = abdi_repoargs.get_repo_snoop_url(repo_config)
sys_repo = phlsys_git.Repo(repo_config.repo_path)
refcache_repo = phlgitx_refcache.Repo(sys_repo)
differ_cache = abdt_differresultcache.Cache(refcache_repo)
abd_repo = abdt_git.Repo(
refcache_repo,
differ_cache,
"origin",
repo_config.repo_desc)
did_fetch = abdi_processrepoarglist.fetch_if_needed(
url_watcher_wrapper.watcher,
snoop_url,
abd_repo,
repo_config.repo_desc)
if did_fetch:
print('fetched')
else:
print('skipped')
url_watcher_wrapper.save()
def process(args):
_ = args # NOQA
fs = abdt_fs.make_default_accessor()
with fs.lockfile_context():
pid = fs.get_pid_or_none()
if pid is not None and phlsys_pid.is_running(pid):
raise Exception("cannot fetch whilst arcyd is running.")
repo_config_path_list = fs.repo_config_path_list()
repo_name_config_list = abdi_repoargs.parse_config_file_list(
repo_config_path_list)
url_watcher_wrapper = phlurl_watcher.FileCacheWatcherWrapper(
fs.layout.urlwatcher_cache_path)
# Let the user know what's happening before potentially blocking for a
# while.
print('Refreshing repository snoop status ..', end=' ')
# Make sure that the output is actually visible by flushing stdout
# XXX: Will use 'flush' parameter to 'print()' in Python 3.3
sys.stdout.flush()
print("done")
url_watcher_wrapper.watcher.refresh()
for repo_name, repo_config in repo_name_config_list:
print(repo_name + ' ..', end=' ')
# Make sure that the output is actually visible by flushing stdout
# XXX: Will use 'flush' parameter to 'print()' in Python 3.3
sys.stdout.flush()
snoop_url = abdi_repoargs.get_repo_snoop_url(repo_config)
sys_repo = phlsys_git.Repo(repo_config.repo_path)
refcache_repo = phlgitx_refcache.Repo(sys_repo)
differ_cache = abdt_differresultcache.Cache(refcache_repo)
abd_repo = abdt_git.Repo(refcache_repo, differ_cache, "origin",
repo_config.repo_desc)
did_fetch = abdi_processrepoarglist.fetch_if_needed(
url_watcher_wrapper.watcher, snoop_url, abd_repo,
repo_config.repo_desc)
if did_fetch:
print('fetched')
else:
print('skipped')
url_watcher_wrapper.save()
def do(
repo_config_path_list,
sys_admin_emails,
kill_file,
reset_file,
pause_file,
sleep_secs,
is_no_loop,
reporter):
repo_configs = abdi_repoargs.parse_config_file_list(repo_config_path_list)
# TODO: test write access to repos here
operations = []
conduits = {}
fs_accessor = abdt_fs.make_default_accessor()
url_watcher_wrapper = phlurl_watcher.FileCacheWatcherWrapper(
fs_accessor.layout.urlwatcher_cache_path)
# refresh cache after loading and before any repos are processed, otherwise
# we may not pull when we need to on the first run around the loop.
# TODO: wrap in usual retry handlers so that we can start up in unstable
# environments
url_watcher_wrapper.watcher.refresh()
_append_operations_for_repos(
operations,
reporter,
conduits,
url_watcher_wrapper,
sys_admin_emails,
repo_configs)
_append_interrupt_operations(
operations,
sys_admin_emails,
kill_file,
reset_file,
pause_file,
sleep_secs,
reporter)
operations.append(
abdi_operation.RefreshCaches(
conduits, url_watcher_wrapper.watcher, reporter))
_process_operations(
is_no_loop, operations, sys_admin_emails, reporter)
def process(args):
fs = abdt_fs.make_default_accessor()
with fs.lockfile_context():
repo_config_path_list = fs.repo_config_path_list()
repo_name_config_list = abdi_repoargs.parse_config_file_list(
repo_config_path_list)
for _, repo_config in repo_name_config_list:
if args.only_formatted_repo_urls:
print(abdi_repoargs.get_repo_url(repo_config))
else:
print(repo_config)
def start_arcyd(daemonize=True, loop=True, restart=False):
# exit gracefully if this process is killed
phlsys_signal.set_exit_on_sigterm()
fs = abdt_fs.make_default_accessor()
with fs.lockfile_context():
pid = fs.get_pid_or_none()
if pid is not None and phlsys_pid.is_running(pid):
if restart:
stop_arcyd_pid(pid)
else:
raise Exception("already running")
if daemonize:
phlsys_daemonize.do(
stdout_path=fs.layout.stdout,
stderr_path=fs.layout.stderr)
# important that we do this *after* daemonizing
pid = phlsys_pid.get()
fs.set_pid(pid)
parser = argparse.ArgumentParser()
params = []
for line in open(fs.layout.root_config):
params.append(line.strip())
if not loop:
params.append('--no-loop')
repo_configs = abdi_repoargs.parse_config_file_list(
fs.repo_config_path_list())
abdi_processrepos.setupParser(parser)
args = parser.parse_args(params)
def logger_config():
_setup_logger(fs)
with phlsys_multiprocessing.logging_context(logger_config):
_LOGGER.debug("start with args: {}".format(args))
_LOGGER.info("arcyd started")
try:
abdi_processrepos.process(args, repo_configs)
finally:
_LOGGER.info("arcyd stopped")
def process(args):
# exit gracefully if this process is killed
phlsys_signal.set_exit_on_sigterm()
fs = abdt_fs.make_default_accessor()
with fs.lockfile_context():
pid = fs.get_pid_or_none()
if pid is not None and phlsys_pid.is_running(pid):
raise Exception("already running")
if not args.foreground:
phlsys_daemonize.do(
stdout_path=fs.layout.stdout,
stderr_path=fs.layout.stderr)
# important that we do this *after* daemonizing
pid = phlsys_pid.get()
fs.set_pid(pid)
parser = argparse.ArgumentParser()
params = []
for line in open(fs.layout.root_config):
params.append(line.strip())
if args.no_loop:
params.append('--no-loop')
repo_configs = abdi_repoargs.parse_config_file_list(
fs.repo_config_path_list())
abdi_processrepos.setupParser(parser)
args = parser.parse_args(params)
# setup to log everything to fs.layout.log_info, with a timestamp
logging.Formatter.converter = time.gmtime
logging.basicConfig(
format='%(asctime)s UTC: %(levelname)s: %(message)s',
level=logging.INFO,
filename=fs.layout.log_info)
_LOGGER.info("arcyd started")
try:
abdi_processrepos.process(args, repo_configs)
finally:
_LOGGER.info("arcyd stopped")
def process(args):
# exit gracefully if this process is killed
phlsys_signal.set_exit_on_sigterm()
fs = abdt_fs.make_default_accessor()
with fs.lockfile_context():
pid = fs.get_pid_or_none()
if pid is not None and phlsys_pid.is_running(pid):
raise Exception("already running")
if not args.foreground:
phlsys_daemonize.do(
stdout_path=fs.layout.stdout,
stderr_path=fs.layout.stderr)
# important that we do this *after* daemonizing
pid = phlsys_pid.get()
fs.set_pid(pid)
parser = argparse.ArgumentParser()
params = []
for line in open(fs.layout.root_config):
params.append(line.strip())
if args.no_loop:
params.append('--no-loop')
repo_configs = abdi_repoargs.parse_config_file_list(
fs.repo_config_path_list())
abdi_processrepos.setupParser(parser)
args = parser.parse_args(params)
abdi_processrepos.process(args, repo_configs)
