def update_crontabs(self, event): """Handle all crontab updates""" pid = multiprocessing.current_process().pid state = event.event_type src_path = event.src_path if event.is_directory: self.__update_crontab_monitor(configuration, src_path, state) elif os.path.basename(src_path) == crontab_name: logger.debug('(%s) %s -> Updating crontab for: %s' % (pid, state, src_path)) rel_path = src_path[len(configuration.user_settings):] client_dir = os.path.basename(os.path.dirname(src_path)) client_id = client_dir_id(client_dir) user_home = os.path.join(configuration.user_home, client_dir) logger.info('(%s) refresh %s crontab from %s' % (pid, client_id, src_path)) if state == 'deleted': cur_crontab = [] logger.debug("(%s) deleted crontab from '%s'" % (pid, src_path)) else: cur_crontab = parse_crontab(configuration, client_id, src_path) logger.debug("(%s) loaded new crontab from '%s':\n%s" % (pid, src_path, cur_crontab)) # Replace crontabs for this user all_crontabs[src_path] = cur_crontab logger.debug('(%s) all crontabs: %s' % (pid, all_crontabs)) elif os.path.basename(src_path) == atjobs_name: logger.debug('(%s) %s -> Updating atjobs for: %s' % (pid, state, src_path)) rel_path = src_path[len(configuration.user_settings):] client_dir = os.path.basename(os.path.dirname(src_path)) client_id = client_dir_id(client_dir) user_home = os.path.join(configuration.user_home, client_dir) logger.info('(%s) refresh %s atjobs from %s' % (pid, client_id, src_path)) if state == 'deleted': cur_atjobs = [] logger.debug("(%s) deleted atjobs from '%s'" % (pid, src_path)) else: cur_atjobs = parse_atjobs(configuration, client_id, src_path) logger.debug("(%s) loaded new atjobs from '%s':\n%s" % (pid, src_path, cur_atjobs)) # Replace atjobs for this user all_atjobs[src_path] = cur_atjobs logger.debug('(%s) all atjobs: %s' % (pid, all_atjobs)) else: logger.debug('(%s) %s skipping non-cron file: %s' % (pid, state, src_path))
def manage_transfers(configuration): """Manage all updates of saved user data transfer requests""" logger.debug('manage transfers') old_transfers = {} src_pattern = os.path.join(configuration.user_settings, '*', datatransfers_filename) for transfers_path in glob.glob(src_pattern): if os.path.getmtime(transfers_path) < last_update: # logger.debug('skip transfer update for unchanged path: %s' % \ # transfers_path) continue logger.debug('handling update of transfers file: %s' % transfers_path) abs_client_dir = os.path.dirname(transfers_path) client_dir = os.path.basename(abs_client_dir) logger.debug('extracted client dir: %s' % client_dir) client_id = client_dir_id(client_dir) logger.debug('loading transfers for: %s' % client_id) (load_status, transfers) = load_data_transfers(configuration, client_id) if not load_status: logger.error('could not load transfer for path: %s' % transfers_path) continue old_transfers[client_id] = all_transfers.get(client_id, {}) all_transfers[client_id] = transfers for (client_id, transfers) in all_transfers.items(): for (transfer_id, transfer_dict) in transfers.items(): #logger.debug('inspecting transfer:\n%s' % blind_pw(transfer_dict)) transfer_status = transfer_dict['status'] if transfer_status in ("DONE", "FAILED", "PAUSED"): # logger.debug('skip %(status)s transfer %(transfer_id)s' % \ # transfer_dict) continue if transfer_status in ("ACTIVE", ): if get_worker_transfer(configuration, all_workers, client_id, transfer_id): logger.debug('wait for transfer %(transfer_id)s' % transfer_dict) continue else: logger.info('restart transfer %(transfer_id)s' % transfer_dict) logger.info('handle %(status)s transfer %(transfer_id)s' % transfer_dict) handle_transfer(configuration, client_id, transfer_dict)
def list_users(user_home): """Return a list of all users by listing the user homes in user_home. Uses dircache for efficiency when used more than once per session. """ users = [] children = dircache.listdir(user_home) for name in children: path = os.path.join(user_home, name) # skip all files and dot dirs - they are _not_ users if not os.path.isdir(path): continue if path.find(os.sep + '.') != -1: continue if path.find('no_grid_jobs_in_grid_scheduler') != -1: continue users.append(client_dir_id(name)) return users
def list_users(configuration): """Return a list of all users by listing the user homes in user_home. Uses dircache for efficiency when used more than once per session. """ users = [] children = dircache.listdir(configuration.user_home) for name in children: path = os.path.join(configuration.user_home, name) # skip all files and dot dirs - they are _not_ users if os.path.islink(path) or not os.path.isdir(path): continue if name.startswith('.'): continue # We assume user IDs on the form /A=bla/B=bla/... here if not '=' in name: continue if name in [configuration.empty_job_name, litmus_id]: continue users.append(client_dir_id(name)) return users
def monitor(configuration): """Monitors the filesystem for crontab changes""" pid = multiprocessing.current_process().pid print 'Starting global crontab monitor process' logger.info('Starting global crontab monitor process') # Set base_dir and base_dir_len shared_state['base_dir'] = os.path.join(configuration.user_settings) shared_state['base_dir_len'] = len(shared_state['base_dir']) # Allow e.g. logrotate to force log re-open after rotates register_hangup_handler(configuration) # Monitor crontab configurations crontab_monitor_home = shared_state['base_dir'] recursive_crontab_monitor = True crontab_monitor = Observer() crontab_pattern = os.path.join(crontab_monitor_home, '*', crontab_name) atjobs_pattern = os.path.join(crontab_monitor_home, '*', atjobs_name) shared_state['crontab_handler'] = MiGCrontabEventHandler( patterns=[crontab_pattern, atjobs_pattern], ignore_directories=False, case_sensitive=True) crontab_monitor.schedule(shared_state['crontab_handler'], configuration.user_settings, recursive=recursive_crontab_monitor) crontab_monitor.start() if len(crontab_monitor._emitters) != 1: logger.error('(%s) Number of crontab_monitor._emitters != 1' % pid) return 1 crontab_monitor_emitter = min(crontab_monitor._emitters) if not hasattr(crontab_monitor_emitter, '_inotify'): logger.error('(%s) crontab_monitor_emitter require inotify' % pid) return 1 shared_state['crontab_inotify'] = crontab_monitor_emitter._inotify._inotify logger.info('(%s) trigger crontab and atjobs refresh' % (pid, )) # Fake touch event on all crontab files to load initial crontabs # logger.info('(%s) trigger load on all files (greedy) matching %s or %s' \ # % (pid, crontab_pattern, atjobs_pattern)) # We manually walk and test to get the greedy "*" directory match behaviour # of the PatternMatchingEventHandler all_crontab_files, all_atjobs_files = [], [] for (root, _, files) in walk(crontab_monitor_home): if crontab_name in files: crontab_path = os.path.join(root, crontab_name) all_crontab_files.append(crontab_path) if atjobs_name in files: atjobs_path = os.path.join(root, atjobs_name) all_atjobs_files.append(atjobs_path) for target_path in all_crontab_files + all_atjobs_files: logger.debug('(%s) trigger load on cron/at file in %s' % (pid, target_path)) shared_state['crontab_handler'].dispatch( FileModifiedEvent(target_path)) # logger.debug('(%s) loaded initial crontabs:\n%s' % (pid, # all_crontab_files)) while not stop_running.is_set(): try: loop_start = datetime.datetime.now() loop_minute = loop_start.replace(second=0, microsecond=0) logger.debug('main loop started with %d crontabs and %d atjobs' % (len(all_crontabs), len(all_atjobs))) for crontab_path, user_crontab in all_crontabs.items(): client_dir = os.path.basename(os.path.dirname(crontab_path)) client_id = client_dir_id(client_dir) for entry in user_crontab: logger.debug('inspect cron entry for %s: %s' % (client_id, entry)) if cron_match(configuration, loop_minute, entry): logger.info('run matching cron entry: %s' % entry) run_handler(configuration, client_id, loop_minute, entry) for atjobs_path, user_atjobs in all_atjobs.items(): client_dir = os.path.basename(os.path.dirname(atjobs_path)) client_id = client_dir_id(client_dir) remaining = [] for entry in user_atjobs: logger.debug('inspect atjobs entry for %s: %s' % (client_id, entry)) remain_mins = at_remain(configuration, loop_minute, entry) if remain_mins == 0: logger.info('run matching at entry: %s' % entry) run_handler(configuration, client_id, loop_minute, entry) elif remain_mins > 0: remaining.append(entry) else: logger.info('removing expired at job: %s' % entry) # Update remaining jobs to clean up expired if remaining: all_atjobs[atjobs_path] = remaining else: del all_atjobs[atjobs_path] except KeyboardInterrupt: print '(%s) caught interrupt' % pid stop_running.set() except Exception, exc: logger.error('unexpected exception in monitor: %s' % exc) import traceback print traceback.format_exc() # Throttle down until next minute loop_time = (datetime.datetime.now() - loop_start).seconds if loop_time > 60: logger.warning('(%s) loop did not finish before next tick: %s' % (os.getpid(), loop_time)) loop_time = 59 # Target sleep until start of next minute sleep_time = max(60 - (loop_time + loop_start.second), 1) # TODO: this debug log never shows up - conflict with user info log? # at least it does if changed to info. logger.debug('main loop sleeping %ds' % sleep_time) # print('main loop sleeping %ds' % sleep_time) time.sleep(sleep_time)
def refresh_users(configuration, protocol): """Reload all users from auth confs if they changed on disk. Add user entries to configuration.daemon_conf['users'] for all active keys and passwords enabled in configuration. Optionally add short ID username alias entries for all users if that is enabled in the configuration. Removes all the user entries no longer active, too. The protocol argument specifies which auth files to use. Returns a tuple with the updated daemon_conf and the list of changed user IDs. NOTE: Deprecated due to severe system load, use refresh_user_creds instead """ changed_users = [] conf = configuration.daemon_conf logger = conf.get("logger", logging.getLogger()) creds_lock = conf.get('creds_lock', None) last_update = conf['time_stamp'] if creds_lock: creds_lock.acquire() old_usernames = [i.username for i in conf['users']] if creds_lock: creds_lock.release() cur_usernames = [] private_auth_file = True if protocol in ('ssh', 'sftp', 'scp', 'rsync'): proto_authkeys = ssh_authkeys proto_authpasswords = ssh_authpasswords proto_authdigests = ssh_authdigests elif protocol in ('dav', 'davs'): proto_authkeys = davs_authkeys proto_authpasswords = davs_authpasswords proto_authdigests = davs_authdigests elif protocol in ('ftp', 'ftps'): proto_authkeys = ftps_authkeys proto_authpasswords = ftps_authpasswords proto_authdigests = ftps_authdigests elif protocol in ('https', 'openid'): private_auth_file = False proto_authkeys = https_authkeys proto_authpasswords = https_authpasswords proto_authdigests = https_authdigests else: logger.error("invalid protocol: %s" % protocol) return (conf, changed_users) auth_protos = (proto_authkeys, proto_authpasswords, proto_authdigests) authkeys_pattern = os.path.join(conf['root_dir'], '*', proto_authkeys) authpasswords_pattern = os.path.join(conf['root_dir'], '*', proto_authpasswords) authdigests_pattern = os.path.join(conf['root_dir'], '*', proto_authdigests) short_id, short_alias = None, None # TODO: support private_auth_file == False here? matches = [] if conf['allow_publickey']: matches += [(proto_authkeys, i) for i in glob.glob(authkeys_pattern)] if conf['allow_password']: matches += [(proto_authpasswords, i) for i in glob.glob(authpasswords_pattern)] if conf['allow_digest']: matches += [(proto_authdigests, i) for i in glob.glob(authdigests_pattern)] for (auth_file, path) in matches: user_home = path.replace(os.sep + auth_file, '') # Skip OpenID alias symlinks if os.path.islink(user_home): continue user_dir = user_home.replace(conf['root_dir'] + os.sep, '') user_id = client_dir_id(user_dir) user_alias = client_alias(user_id) # we always accept asciified distinguished name cur_usernames.append(user_alias) if conf['user_alias']: short_id = get_short_id(configuration, user_id, conf['user_alias']) # Allow both raw alias field value and asciified alias cur_usernames.append(short_id) # logger.debug("find short_alias for %s" % short_id) short_alias = client_alias(short_id) cur_usernames.append(short_alias) if last_update >= os.path.getmtime(path): continue # In GDP-mode user must be chrooted to project home for IO daemons # but obviously not for the OpenID login prior to project login. if configuration.site_enable_gdp and private_auth_file and \ protocol != 'openid': project_name = get_project_from_user_id(configuration, user_id) user_dir = os.path.join(user_dir, project_name) user_vars = (user_id, user_alias, user_dir, short_id, short_alias) update_user_objects(configuration, auth_file, path, user_vars, auth_protos, private_auth_file) changed_users += [user_id, user_alias] if short_id is not None: changed_users += [short_id, short_alias] removed = [i for i in old_usernames if not i in cur_usernames] if removed: logger.info("Removing login for %d deleted users" % len(removed)) if creds_lock: creds_lock.acquire() conf['users'] = [i for i in conf['users'] if not i.username in removed] if creds_lock: creds_lock.release() changed_users += removed logger.info("Refreshed users from configuration (%d users)" % len(cur_usernames)) conf['time_stamp'] = time.time() return (conf, changed_users)
def refresh_user_creds(configuration, protocol, username): """Reload user credentials for username if they changed on disk. That is, add user entries in configuration.daemon_conf['users'] for all active keys and passwords enabled in configuration. Optionally add short ID username alias entries for user if that is enabled in the configuration. Removes all aliased user entries if the user is no longer active, too. The protocol argument specifies which auth files to use. Returns a tuple with the updated daemon_conf and the list of changed user IDs. NOTE: username must be the direct username used in home dir or an OpenID alias with associated symlink there. Encoded username aliases must be decoded before use here. """ changed_users = [] conf = configuration.daemon_conf logger = conf.get("logger", logging.getLogger()) private_auth_file = True if protocol in ('ssh', 'sftp', 'scp', 'rsync'): proto_authkeys = ssh_authkeys proto_authpasswords = ssh_authpasswords proto_authdigests = ssh_authdigests elif protocol in ('dav', 'davs'): proto_authkeys = davs_authkeys proto_authpasswords = davs_authpasswords proto_authdigests = davs_authdigests elif protocol in ('ftp', 'ftps'): proto_authkeys = ftps_authkeys proto_authpasswords = ftps_authpasswords proto_authdigests = ftps_authdigests elif protocol in ('https', 'openid'): private_auth_file = False proto_authkeys = https_authkeys proto_authpasswords = https_authpasswords proto_authdigests = https_authdigests else: logger.error("Invalid protocol: %s" % protocol) return (conf, changed_users) auth_protos = (proto_authkeys, proto_authpasswords, proto_authdigests) # We support direct and symlinked usernames for now # NOTE: entries are gracefully removed if user no longer exists if private_auth_file: authkeys_path = os.path.realpath(os.path.join(conf['root_dir'], username, proto_authkeys)) authpasswords_path = os.path.realpath(os.path.join( conf['root_dir'], username, proto_authpasswords)) authdigests_path = os.path.realpath(os.path.join(conf['root_dir'], username, proto_authdigests)) else: authkeys_path = authpasswords_path = authdigests_path = conf['db_path'] # logger.debug("Updating user creds for %s" % username) changed_paths = get_creds_changes(conf, username, authkeys_path, authpasswords_path, authdigests_path) if not changed_paths: # logger.debug("No user creds changes for %s" % username) return (conf, changed_users) short_id, short_alias = None, None matches = [] if conf['allow_publickey']: matches += [(proto_authkeys, authkeys_path)] if conf['allow_password']: matches += [(proto_authpasswords, authpasswords_path)] if conf['allow_digest']: matches += [(proto_authdigests, authdigests_path)] for (auth_file, path) in matches: if not path in changed_paths: # logger.debug("Skipping %s without changes" % path) continue # Missing alias symlink - should be fixed for user instead if not os.path.exists(path): logger.warning("Skipping non-existant auth path %s" % path) continue # logger.debug("Checking %s" % path) if private_auth_file: user_home = path.replace(os.sep + auth_file, '') user_dir = user_home.replace(conf['root_dir'] + os.sep, '') else: # Expand actual user home from alias user_home = os.path.realpath(os.path.join(configuration.user_home, username)) user_dir = os.path.basename(user_home) # Check that user home exists if not os.path.exists(user_home): logger.warning("Skipping user without home %s" % user_home) continue user_id = client_dir_id(user_dir) user_alias = client_alias(user_id) if conf['user_alias']: short_id = get_short_id(configuration, user_id, conf['user_alias']) # Allow both raw alias field value and asciified alias # logger.debug("find short_alias for %s" % short_id) short_alias = client_alias(short_id) # In GDP-mode user must be chrooted to project home for IO daemons # but obviously not for the OpenID login prior to project login. if configuration.site_enable_gdp and private_auth_file and \ protocol != 'openid': project_name = get_project_from_user_id(configuration, user_id) if not project_name: logger.warning("Skipping invalid GDP user %s" % user_id) continue user_dir = os.path.join(user_dir, project_name) user_vars = (user_id, user_alias, user_dir, short_id, short_alias) update_user_objects(configuration, auth_file, path, user_vars, auth_protos, private_auth_file) if changed_paths: logger.info("Refreshed user %s from configuration: %s" % (username, changed_paths)) changed_users.append(username) return (conf, changed_users)