def update_crontabs(self, event):
"""Handle all crontab updates"""
pid = multiprocessing.current_process().pid
state = event.event_type
src_path = event.src_path
if event.is_directory:
self.__update_crontab_monitor(configuration, src_path, state)
elif os.path.basename(src_path) == crontab_name:
logger.debug('(%s) %s -> Updating crontab for: %s' % (pid,
state, src_path))
rel_path = src_path[len(configuration.user_settings):]
client_dir = os.path.basename(os.path.dirname(src_path))
client_id = client_dir_id(client_dir)
user_home = os.path.join(configuration.user_home, client_dir)
logger.info('(%s) refresh %s crontab from %s' % (pid,
client_id, src_path))
if state == 'deleted':
cur_crontab = []
logger.debug("(%s) deleted crontab from '%s'" %
(pid, src_path))
else:
cur_crontab = parse_crontab(configuration, client_id, src_path)
logger.debug("(%s) loaded new crontab from '%s':\n%s" %
(pid, src_path, cur_crontab))
# Replace crontabs for this user
all_crontabs[src_path] = cur_crontab
logger.debug('(%s) all crontabs: %s' % (pid, all_crontabs))
elif os.path.basename(src_path) == atjobs_name:
logger.debug('(%s) %s -> Updating atjobs for: %s' % (pid,
state, src_path))
rel_path = src_path[len(configuration.user_settings):]
client_dir = os.path.basename(os.path.dirname(src_path))
client_id = client_dir_id(client_dir)
user_home = os.path.join(configuration.user_home, client_dir)
logger.info('(%s) refresh %s atjobs from %s' % (pid,
client_id, src_path))
if state == 'deleted':
cur_atjobs = []
logger.debug("(%s) deleted atjobs from '%s'" %
(pid, src_path))
else:
cur_atjobs = parse_atjobs(configuration, client_id, src_path)
logger.debug("(%s) loaded new atjobs from '%s':\n%s" %
(pid, src_path, cur_atjobs))
# Replace atjobs for this user
all_atjobs[src_path] = cur_atjobs
logger.debug('(%s) all atjobs: %s' % (pid, all_atjobs))
else:
logger.debug('(%s) %s skipping non-cron file: %s' % (pid,
state, src_path))
def manage_transfers(configuration):
"""Manage all updates of saved user data transfer requests"""
logger.debug('manage transfers')
old_transfers = {}
src_pattern = os.path.join(configuration.user_settings, '*',
datatransfers_filename)
for transfers_path in glob.glob(src_pattern):
if os.path.getmtime(transfers_path) < last_update:
# logger.debug('skip transfer update for unchanged path: %s' % \
# transfers_path)
continue
logger.debug('handling update of transfers file: %s' % transfers_path)
abs_client_dir = os.path.dirname(transfers_path)
client_dir = os.path.basename(abs_client_dir)
logger.debug('extracted client dir: %s' % client_dir)
client_id = client_dir_id(client_dir)
logger.debug('loading transfers for: %s' % client_id)
(load_status, transfers) = load_data_transfers(configuration,
client_id)
if not load_status:
logger.error('could not load transfer for path: %s' %
transfers_path)
continue
old_transfers[client_id] = all_transfers.get(client_id, {})
all_transfers[client_id] = transfers
for (client_id, transfers) in all_transfers.items():
for (transfer_id, transfer_dict) in transfers.items():
#logger.debug('inspecting transfer:\n%s' % blind_pw(transfer_dict))
transfer_status = transfer_dict['status']
if transfer_status in ("DONE", "FAILED", "PAUSED"):
# logger.debug('skip %(status)s transfer %(transfer_id)s' % \
# transfer_dict)
continue
if transfer_status in ("ACTIVE", ):
if get_worker_transfer(configuration, all_workers, client_id,
transfer_id):
logger.debug('wait for transfer %(transfer_id)s' %
transfer_dict)
continue
else:
logger.info('restart transfer %(transfer_id)s' %
transfer_dict)
logger.info('handle %(status)s transfer %(transfer_id)s' %
transfer_dict)
handle_transfer(configuration, client_id, transfer_dict)
def list_users(user_home):
"""Return a list of all users by listing the user homes in user_home.
Uses dircache for efficiency when used more than once per session.
"""
users = []
children = dircache.listdir(user_home)
for name in children:
path = os.path.join(user_home, name)
# skip all files and dot dirs - they are _not_ users
if not os.path.isdir(path):
continue
if path.find(os.sep + '.') != -1:
continue
if path.find('no_grid_jobs_in_grid_scheduler') != -1:
continue
users.append(client_dir_id(name))
return users
def list_users(configuration):
"""Return a list of all users by listing the user homes in user_home.
Uses dircache for efficiency when used more than once per session.
"""
users = []
children = dircache.listdir(configuration.user_home)
for name in children:
path = os.path.join(configuration.user_home, name)
# skip all files and dot dirs - they are _not_ users
if os.path.islink(path) or not os.path.isdir(path):
continue
if name.startswith('.'):
continue
# We assume user IDs on the form /A=bla/B=bla/... here
if not '=' in name:
continue
if name in [configuration.empty_job_name, litmus_id]:
continue
users.append(client_dir_id(name))
return users
def monitor(configuration):
"""Monitors the filesystem for crontab changes"""
pid = multiprocessing.current_process().pid
print 'Starting global crontab monitor process'
logger.info('Starting global crontab monitor process')
# Set base_dir and base_dir_len
shared_state['base_dir'] = os.path.join(configuration.user_settings)
shared_state['base_dir_len'] = len(shared_state['base_dir'])
# Allow e.g. logrotate to force log re-open after rotates
register_hangup_handler(configuration)
# Monitor crontab configurations
crontab_monitor_home = shared_state['base_dir']
recursive_crontab_monitor = True
crontab_monitor = Observer()
crontab_pattern = os.path.join(crontab_monitor_home, '*', crontab_name)
atjobs_pattern = os.path.join(crontab_monitor_home, '*', atjobs_name)
shared_state['crontab_handler'] = MiGCrontabEventHandler(
patterns=[crontab_pattern, atjobs_pattern], ignore_directories=False,
case_sensitive=True)
crontab_monitor.schedule(shared_state['crontab_handler'],
configuration.user_settings,
recursive=recursive_crontab_monitor)
crontab_monitor.start()
if len(crontab_monitor._emitters) != 1:
logger.error('(%s) Number of crontab_monitor._emitters != 1' % pid)
return 1
crontab_monitor_emitter = min(crontab_monitor._emitters)
if not hasattr(crontab_monitor_emitter, '_inotify'):
logger.error('(%s) crontab_monitor_emitter require inotify' % pid)
return 1
shared_state['crontab_inotify'] = crontab_monitor_emitter._inotify._inotify
logger.info('(%s) trigger crontab and atjobs refresh' % (pid, ))
# Fake touch event on all crontab files to load initial crontabs
# logger.info('(%s) trigger load on all files (greedy) matching %s or %s' \
# % (pid, crontab_pattern, atjobs_pattern))
# We manually walk and test to get the greedy "*" directory match behaviour
# of the PatternMatchingEventHandler
all_crontab_files, all_atjobs_files = [], []
for (root, _, files) in walk(crontab_monitor_home):
if crontab_name in files:
crontab_path = os.path.join(root, crontab_name)
all_crontab_files.append(crontab_path)
if atjobs_name in files:
atjobs_path = os.path.join(root, atjobs_name)
all_atjobs_files.append(atjobs_path)
for target_path in all_crontab_files + all_atjobs_files:
logger.debug('(%s) trigger load on cron/at file in %s' %
(pid, target_path))
shared_state['crontab_handler'].dispatch(
FileModifiedEvent(target_path))
# logger.debug('(%s) loaded initial crontabs:\n%s' % (pid,
# all_crontab_files))
while not stop_running.is_set():
try:
loop_start = datetime.datetime.now()
loop_minute = loop_start.replace(second=0, microsecond=0)
logger.debug('main loop started with %d crontabs and %d atjobs' %
(len(all_crontabs), len(all_atjobs)))
for crontab_path, user_crontab in all_crontabs.items():
client_dir = os.path.basename(os.path.dirname(crontab_path))
client_id = client_dir_id(client_dir)
for entry in user_crontab:
logger.debug('inspect cron entry for %s: %s' %
(client_id, entry))
if cron_match(configuration, loop_minute, entry):
logger.info('run matching cron entry: %s' % entry)
run_handler(configuration, client_id, loop_minute,
entry)
for atjobs_path, user_atjobs in all_atjobs.items():
client_dir = os.path.basename(os.path.dirname(atjobs_path))
client_id = client_dir_id(client_dir)
remaining = []
for entry in user_atjobs:
logger.debug('inspect atjobs entry for %s: %s' %
(client_id, entry))
remain_mins = at_remain(configuration, loop_minute, entry)
if remain_mins == 0:
logger.info('run matching at entry: %s' % entry)
run_handler(configuration, client_id, loop_minute,
entry)
elif remain_mins > 0:
remaining.append(entry)
else:
logger.info('removing expired at job: %s' % entry)
# Update remaining jobs to clean up expired
if remaining:
all_atjobs[atjobs_path] = remaining
else:
del all_atjobs[atjobs_path]
except KeyboardInterrupt:
print '(%s) caught interrupt' % pid
stop_running.set()
except Exception, exc:
logger.error('unexpected exception in monitor: %s' % exc)
import traceback
print traceback.format_exc()
# Throttle down until next minute
loop_time = (datetime.datetime.now() - loop_start).seconds
if loop_time > 60:
logger.warning('(%s) loop did not finish before next tick: %s' %
(os.getpid(), loop_time))
loop_time = 59
# Target sleep until start of next minute
sleep_time = max(60 - (loop_time + loop_start.second), 1)
# TODO: this debug log never shows up - conflict with user info log?
# at least it does if changed to info.
logger.debug('main loop sleeping %ds' % sleep_time)
# print('main loop sleeping %ds' % sleep_time)
time.sleep(sleep_time)
def refresh_users(configuration, protocol):
"""Reload all users from auth confs if they changed on disk.
Add user entries to configuration.daemon_conf['users']
for all active keys and passwords enabled in configuration.
Optionally add short ID username alias entries for all users
if that is enabled in the configuration.
Removes all the user entries no longer active, too.
The protocol argument specifies which auth files to use.
Returns a tuple with the updated daemon_conf and the list of changed user
IDs.
NOTE: Deprecated due to severe system load, use refresh_user_creds instead
"""
changed_users = []
conf = configuration.daemon_conf
logger = conf.get("logger", logging.getLogger())
creds_lock = conf.get('creds_lock', None)
last_update = conf['time_stamp']
if creds_lock:
creds_lock.acquire()
old_usernames = [i.username for i in conf['users']]
if creds_lock:
creds_lock.release()
cur_usernames = []
private_auth_file = True
if protocol in ('ssh', 'sftp', 'scp', 'rsync'):
proto_authkeys = ssh_authkeys
proto_authpasswords = ssh_authpasswords
proto_authdigests = ssh_authdigests
elif protocol in ('dav', 'davs'):
proto_authkeys = davs_authkeys
proto_authpasswords = davs_authpasswords
proto_authdigests = davs_authdigests
elif protocol in ('ftp', 'ftps'):
proto_authkeys = ftps_authkeys
proto_authpasswords = ftps_authpasswords
proto_authdigests = ftps_authdigests
elif protocol in ('https', 'openid'):
private_auth_file = False
proto_authkeys = https_authkeys
proto_authpasswords = https_authpasswords
proto_authdigests = https_authdigests
else:
logger.error("invalid protocol: %s" % protocol)
return (conf, changed_users)
auth_protos = (proto_authkeys, proto_authpasswords, proto_authdigests)
authkeys_pattern = os.path.join(conf['root_dir'], '*', proto_authkeys)
authpasswords_pattern = os.path.join(conf['root_dir'], '*',
proto_authpasswords)
authdigests_pattern = os.path.join(conf['root_dir'], '*',
proto_authdigests)
short_id, short_alias = None, None
# TODO: support private_auth_file == False here?
matches = []
if conf['allow_publickey']:
matches += [(proto_authkeys, i) for i in glob.glob(authkeys_pattern)]
if conf['allow_password']:
matches += [(proto_authpasswords, i)
for i in glob.glob(authpasswords_pattern)]
if conf['allow_digest']:
matches += [(proto_authdigests, i)
for i in glob.glob(authdigests_pattern)]
for (auth_file, path) in matches:
user_home = path.replace(os.sep + auth_file, '')
# Skip OpenID alias symlinks
if os.path.islink(user_home):
continue
user_dir = user_home.replace(conf['root_dir'] + os.sep, '')
user_id = client_dir_id(user_dir)
user_alias = client_alias(user_id)
# we always accept asciified distinguished name
cur_usernames.append(user_alias)
if conf['user_alias']:
short_id = get_short_id(configuration, user_id, conf['user_alias'])
# Allow both raw alias field value and asciified alias
cur_usernames.append(short_id)
# logger.debug("find short_alias for %s" % short_id)
short_alias = client_alias(short_id)
cur_usernames.append(short_alias)
if last_update >= os.path.getmtime(path):
continue
# In GDP-mode user must be chrooted to project home for IO daemons
# but obviously not for the OpenID login prior to project login.
if configuration.site_enable_gdp and private_auth_file and \
protocol != 'openid':
project_name = get_project_from_user_id(configuration, user_id)
user_dir = os.path.join(user_dir, project_name)
user_vars = (user_id, user_alias, user_dir, short_id, short_alias)
update_user_objects(configuration,
auth_file,
path,
user_vars,
auth_protos,
private_auth_file)
changed_users += [user_id, user_alias]
if short_id is not None:
changed_users += [short_id, short_alias]
removed = [i for i in old_usernames if not i in cur_usernames]
if removed:
logger.info("Removing login for %d deleted users" % len(removed))
if creds_lock:
creds_lock.acquire()
conf['users'] = [i for i in conf['users'] if not i.username in removed]
if creds_lock:
creds_lock.release()
changed_users += removed
logger.info("Refreshed users from configuration (%d users)" %
len(cur_usernames))
conf['time_stamp'] = time.time()
return (conf, changed_users)
def refresh_user_creds(configuration, protocol, username):
"""Reload user credentials for username if they changed on disk. That is,
add user entries in configuration.daemon_conf['users'] for all active keys
and passwords enabled in configuration. Optionally add short ID username
alias entries for user if that is enabled in the configuration.
Removes all aliased user entries if the user is no longer active, too.
The protocol argument specifies which auth files to use.
Returns a tuple with the updated daemon_conf and the list of changed user
IDs.
NOTE: username must be the direct username used in home dir or an OpenID
alias with associated symlink there. Encoded username aliases must be
decoded before use here.
"""
changed_users = []
conf = configuration.daemon_conf
logger = conf.get("logger", logging.getLogger())
private_auth_file = True
if protocol in ('ssh', 'sftp', 'scp', 'rsync'):
proto_authkeys = ssh_authkeys
proto_authpasswords = ssh_authpasswords
proto_authdigests = ssh_authdigests
elif protocol in ('dav', 'davs'):
proto_authkeys = davs_authkeys
proto_authpasswords = davs_authpasswords
proto_authdigests = davs_authdigests
elif protocol in ('ftp', 'ftps'):
proto_authkeys = ftps_authkeys
proto_authpasswords = ftps_authpasswords
proto_authdigests = ftps_authdigests
elif protocol in ('https', 'openid'):
private_auth_file = False
proto_authkeys = https_authkeys
proto_authpasswords = https_authpasswords
proto_authdigests = https_authdigests
else:
logger.error("Invalid protocol: %s" % protocol)
return (conf, changed_users)
auth_protos = (proto_authkeys, proto_authpasswords, proto_authdigests)
# We support direct and symlinked usernames for now
# NOTE: entries are gracefully removed if user no longer exists
if private_auth_file:
authkeys_path = os.path.realpath(os.path.join(conf['root_dir'],
username,
proto_authkeys))
authpasswords_path = os.path.realpath(os.path.join(
conf['root_dir'], username, proto_authpasswords))
authdigests_path = os.path.realpath(os.path.join(conf['root_dir'],
username,
proto_authdigests))
else:
authkeys_path = authpasswords_path = authdigests_path = conf['db_path']
# logger.debug("Updating user creds for %s" % username)
changed_paths = get_creds_changes(conf, username, authkeys_path,
authpasswords_path, authdigests_path)
if not changed_paths:
# logger.debug("No user creds changes for %s" % username)
return (conf, changed_users)
short_id, short_alias = None, None
matches = []
if conf['allow_publickey']:
matches += [(proto_authkeys, authkeys_path)]
if conf['allow_password']:
matches += [(proto_authpasswords, authpasswords_path)]
if conf['allow_digest']:
matches += [(proto_authdigests, authdigests_path)]
for (auth_file, path) in matches:
if not path in changed_paths:
# logger.debug("Skipping %s without changes" % path)
continue
# Missing alias symlink - should be fixed for user instead
if not os.path.exists(path):
logger.warning("Skipping non-existant auth path %s" % path)
continue
# logger.debug("Checking %s" % path)
if private_auth_file:
user_home = path.replace(os.sep + auth_file, '')
user_dir = user_home.replace(conf['root_dir'] + os.sep, '')
else:
# Expand actual user home from alias
user_home = os.path.realpath(os.path.join(configuration.user_home,
username))
user_dir = os.path.basename(user_home)
# Check that user home exists
if not os.path.exists(user_home):
logger.warning("Skipping user without home %s" % user_home)
continue
user_id = client_dir_id(user_dir)
user_alias = client_alias(user_id)
if conf['user_alias']:
short_id = get_short_id(configuration, user_id, conf['user_alias'])
# Allow both raw alias field value and asciified alias
# logger.debug("find short_alias for %s" % short_id)
short_alias = client_alias(short_id)
# In GDP-mode user must be chrooted to project home for IO daemons
# but obviously not for the OpenID login prior to project login.
if configuration.site_enable_gdp and private_auth_file and \
protocol != 'openid':
project_name = get_project_from_user_id(configuration, user_id)
if not project_name:
logger.warning("Skipping invalid GDP user %s" % user_id)
continue
user_dir = os.path.join(user_dir, project_name)
user_vars = (user_id, user_alias, user_dir, short_id, short_alias)
update_user_objects(configuration,
auth_file,
path,
user_vars,
auth_protos,
private_auth_file)
if changed_paths:
logger.info("Refreshed user %s from configuration: %s" %
(username, changed_paths))
changed_users.append(username)
return (conf, changed_users)