if __name__ == '__main__':
# Force no log init since we use separate logger
configuration = get_configuration_object(skip_log=True)
log_level = configuration.loglevel
if sys.argv[1:] and sys.argv[1] in ['debug', 'info', 'warning', 'error']:
log_level = sys.argv[1]
# Use separate logger
logger = daemon_logger("monitor", configuration.user_monitor_log,
log_level)
configuration.logger = logger
# Allow e.g. logrotate to force log re-open after rotates
register_hangup_handler(configuration)
if not configuration.site_enable_jobs:
err_msg = "Job support is disabled in configuration!"
logger.error(err_msg)
print(err_msg)
sys.exit(1)
print("""
Running grid monitor generator.
Set the MIG_CONF environment to the server configuration path
unless it is available in mig/server/MiGserver.conf
""")
# Make sure that the default VGrid home used by monitor exists
def monitor(configuration):
"""Monitors the filesystem for crontab changes"""
pid = multiprocessing.current_process().pid
print('Starting global crontab monitor process')
logger.info('Starting global crontab monitor process')
# Set base_dir and base_dir_len
shared_state['base_dir'] = os.path.join(configuration.user_settings)
shared_state['base_dir_len'] = len(shared_state['base_dir'])
# Allow e.g. logrotate to force log re-open after rotates
register_hangup_handler(configuration)
# Monitor crontab configurations
crontab_monitor_home = shared_state['base_dir']
recursive_crontab_monitor = True
crontab_monitor = Observer()
crontab_pattern = os.path.join(crontab_monitor_home, '*', crontab_name)
atjobs_pattern = os.path.join(crontab_monitor_home, '*', atjobs_name)
shared_state['crontab_handler'] = MiGCrontabEventHandler(
patterns=[crontab_pattern, atjobs_pattern], ignore_directories=False,
case_sensitive=True)
crontab_monitor.schedule(shared_state['crontab_handler'],
configuration.user_settings,
recursive=recursive_crontab_monitor)
crontab_monitor.start()
if len(crontab_monitor._emitters) != 1:
logger.error('(%s) Number of crontab_monitor._emitters != 1' % pid)
return 1
crontab_monitor_emitter = min(crontab_monitor._emitters)
if not hasattr(crontab_monitor_emitter, '_inotify'):
logger.error('(%s) crontab_monitor_emitter require inotify' % pid)
return 1
shared_state['crontab_inotify'] = crontab_monitor_emitter._inotify._inotify
logger.info('(%s) trigger crontab and atjobs refresh' % (pid, ))
# Fake touch event on all crontab files to load initial crontabs
# logger.info('(%s) trigger load on all files (greedy) matching %s or %s' \
# % (pid, crontab_pattern, atjobs_pattern))
# We manually walk and test to get the greedy "*" directory match behaviour
# of the PatternMatchingEventHandler
all_crontab_files, all_atjobs_files = [], []
for (root, _, files) in walk(crontab_monitor_home):
if crontab_name in files:
crontab_path = os.path.join(root, crontab_name)
all_crontab_files.append(crontab_path)
if atjobs_name in files:
atjobs_path = os.path.join(root, atjobs_name)
all_atjobs_files.append(atjobs_path)
for target_path in all_crontab_files + all_atjobs_files:
logger.debug('(%s) trigger load on cron/at file in %s' %
(pid, target_path))
shared_state['crontab_handler'].dispatch(
FileModifiedEvent(target_path))
# logger.debug('(%s) loaded initial crontabs:\n%s' % (pid,
# all_crontab_files))
while not stop_running.is_set():
try:
loop_start = datetime.datetime.now()
loop_minute = loop_start.replace(second=0, microsecond=0)
logger.debug('main loop started with %d crontabs and %d atjobs' %
(len(all_crontabs), len(all_atjobs)))
for crontab_path, user_crontab in all_crontabs.items():
client_dir = os.path.basename(os.path.dirname(crontab_path))
client_id = client_dir_id(client_dir)
for entry in user_crontab:
logger.debug('inspect cron entry for %s: %s' %
(client_id, entry))
if cron_match(configuration, loop_minute, entry):
logger.info('run matching cron entry: %s' % entry)
run_handler(configuration, client_id, loop_minute,
entry)
for atjobs_path, user_atjobs in all_atjobs.items():
client_dir = os.path.basename(os.path.dirname(atjobs_path))
client_id = client_dir_id(client_dir)
remaining = []
for entry in user_atjobs:
logger.debug('inspect atjobs entry for %s: %s' %
(client_id, entry))
remain_mins = at_remain(configuration, loop_minute, entry)
if remain_mins == 0:
logger.info('run matching at entry: %s' % entry)
run_handler(configuration, client_id, loop_minute,
entry)
elif remain_mins > 0:
remaining.append(entry)
else:
logger.info('removing expired at job: %s' % entry)
# Update remaining jobs to clean up expired
if remaining:
all_atjobs[atjobs_path] = remaining
else:
del all_atjobs[atjobs_path]
except KeyboardInterrupt:
print('(%s) caught interrupt' % pid)
stop_running.set()
except Exception as exc:
logger.error('unexpected exception in monitor: %s' % exc)
import traceback
print(traceback.format_exc())
# Throttle down until next minute
loop_time = (datetime.datetime.now() - loop_start).seconds
if loop_time > 60:
logger.warning('(%s) loop did not finish before next tick: %s' %
(os.getpid(), loop_time))
loop_time = 59
# Target sleep until start of next minute
sleep_time = max(60 - (loop_time + loop_start.second), 1)
# TODO: this debug log never shows up - conflict with user info log?
# at least it does if changed to info.
logger.debug('main loop sleeping %ds' % sleep_time)
# print('main loop sleeping %ds' % sleep_time)
time.sleep(sleep_time)
print('(%s) Exiting crontab monitor' % pid)
logger.info('(%s) Exiting crontab monitor' % pid)
return 0