def post(self, errors, current_time=None):
"""
Runs in main after do
If errors evals to true, this is indicates a handled failure
If errors evals to false, and this is not a dry_run
it is considered success and creates the cache file with current time
"""
if current_time is None:
current_time = self.current_time
if errors:
logging.warning("Encountered errors: %s", errors)
self.warning("Failed to complete without errors.")
elif not self.options.dry_run:
# don't update the timestamp on dryrun
timestamp = -1 # handle failing convert_timestamp
try:
_, timestamp = convert_timestamp(current_time)
write_timestamp(self.options.timestamp_file, timestamp)
except Exception as err:
txt = "Writing timestamp %s to %s failed: %s" % (
timestamp, self.options.timestamp_file, err)
self.critical_exception(txt, err)
def main():
"""
Main script.
- build the filter
- fetches the users
- process the users
- write the new timestamp if everything went OK
- write the nagios check file
"""
options = {
'nagios-check-interval-threshold':
NAGIOS_CHECK_INTERVAL_THRESHOLD,
'storage': ('storage systems on which to deploy users and vos', None,
'extend', []),
'user': ('process users', None, 'store_true', False),
'vo': ('process vos', None, 'store_true', False),
'access_token': ('OAuth2 token to access the account page REST API',
None, 'store', None),
'account_page_url':
('URL of the account page where we can find the REST API', None,
'store', None),
'host_institute':
('Name of the institute where this script is being run', str, 'store',
GENT),
}
opts = ExtendedSimpleOption(options)
stats = {}
try:
now = datetime.utcnow()
client = AccountpageClient(token=opts.options.access_token,
url=opts.options.account_page_url + "/api/")
try:
last_timestamp = read_timestamp(SYNC_TIMESTAMP_FILENAME)
except Exception:
logger.exception("Something broke reading the timestamp from %s" %
SYNC_TIMESTAMP_FILENAME)
last_timestamp = "200901010000Z"
logger.info("Last recorded timestamp was %s" % (last_timestamp))
last_timestamp = convert_to_unix_timestamp(last_timestamp)
(users_ok, users_fail) = ([], [])
(quota_ok, quota_fail) = ([], [])
if opts.options.user:
ugent_changed_accounts = client.account.institute['gent'].modified[
last_timestamp].get()[1]
logger.info(
"Found %d UGent accounts that have changed in the accountpage since %s"
% (len(ugent_changed_accounts), last_timestamp))
ugent_accounts = [u['vsc_id'] for u in ugent_changed_accounts]
ugent_accounts = nub(ugent_accounts)
for storage_name in opts.options.storage:
(users_ok,
users_fail) = process_users(opts.options, ugent_accounts,
storage_name, client,
opts.options.host_institute)
stats["%s_users_sync" % (storage_name, )] = len(users_ok)
stats["%s_users_sync_fail" %
(storage_name, )] = len(users_fail)
stats["%s_users_sync_fail_warning" %
(storage_name, )] = STORAGE_USERS_LIMIT_WARNING
stats["%s_users_sync_fail_critical" %
(storage_name, )] = STORAGE_USERS_LIMIT_CRITICAL
for storage_name in opts.options.storage:
storage_changed_quota = [
mkVscUserSizeQuota(q) for q in client.quota.user.
storage[storage_name].modified[last_timestamp].get()[1]
]
storage_changed_quota = [
q for q in storage_changed_quota
if q.fileset.startswith('vsc')
]
logger.info(
"Found %d accounts that have changed quota on storage %s in the accountpage since %s",
len(storage_changed_quota), storage_name, last_timestamp)
(quota_ok, quota_fail) = process_users_quota(
opts.options, storage_changed_quota, storage_name, client,
opts.options.host_institute)
stats["%s_quota_sync" % (storage_name, )] = len(quota_ok)
stats["%s_quota_sync_fail" %
(storage_name, )] = len(quota_fail)
stats["%s_quota_sync_fail_warning" %
(storage_name, )] = STORAGE_QUOTA_LIMIT_WARNING
stats["%s_quota_sync_fail_critical" %
(storage_name, )] = STORAGE_QUOTA_LIMIT_CRITICAL
(vos_ok, vos_fail) = ([], [])
if opts.options.vo:
ugent_changed_vos = client.vo.modified[last_timestamp].get()[1]
ugent_changed_vo_quota = client.quota.vo.modified[
last_timestamp].get()[1]
ugent_vos = sorted(
set([v['vsc_id'] for v in ugent_changed_vos] + [
v['virtual_organisation'] for v in ugent_changed_vo_quota
]))
logger.info(
"Found %d UGent VOs that have changed in the accountpage since %s"
% (len(ugent_changed_vos), last_timestamp))
logger.info(
"Found %d UGent VOs that have changed quota in the accountpage since %s"
% (len(ugent_changed_vo_quota), last_timestamp))
logger.debug(
"Found the following UGent VOs: {vos}".format(vos=ugent_vos))
for storage_name in opts.options.storage:
(vos_ok, vos_fail) = process_vos(opts.options, ugent_vos,
storage_name, client,
last_timestamp,
opts.options.host_institute)
stats["%s_vos_sync" % (storage_name, )] = len(vos_ok)
stats["%s_vos_sync_fail" % (storage_name, )] = len(vos_fail)
stats["%s_vos_sync_fail_warning" %
(storage_name, )] = STORAGE_VO_LIMIT_WARNING
stats["%s_vos_sync_fail_critical" %
(storage_name, )] = STORAGE_VO_LIMIT_CRITICAL
if not (users_fail or quota_fail or vos_fail):
(_, ldap_timestamp) = convert_timestamp(now)
if not opts.options.dry_run:
write_timestamp(SYNC_TIMESTAMP_FILENAME, ldap_timestamp)
except Exception as err:
logger.exception("critical exception caught: %s" % (err))
opts.critical("Script failed in a horrible way")
sys.exit(NAGIOS_EXIT_CRITICAL)
opts.epilogue("UGent users and VOs synchronised", stats)
def main():
"""
Main script. The usual.
"""
options = {
"nagios-check-interval-threshold":
NAGIOS_CHECK_INTERVAL_THRESHOLD,
"access_token": ("OAuth2 token to access the account page REST API",
None, "store", None),
"account_page_url": (
"URL of the account page where we can find the REST API",
str,
"store",
"https://apivsc.ugent.be/django",
),
'host_institute':
('Name of the institute where this script is being run', str, 'store',
GENT),
"clusters": (
"Cluster(s) (comma-separated) to sync for. "
"Overrides <host_institute>_SLURM_COMPUTE_CLUSTERS that are in production.",
"strlist",
"store",
[],
),
'start_timestamp':
('Timestamp to start the sync from', str, 'store', None),
'cluster_classes':
('Classes of clusters that should be synced, comma-separated',
"strlist", 'store', [PRODUCTION, PILOT])
}
opts = ExtendedSimpleOption(options)
stats = {}
(last_timestamp, start_time) = retrieve_timestamp_with_default(
SYNC_TIMESTAMP_FILENAME, start_timestamp=opts.options.start_timestamp)
logging.info("Using timestamp %s", last_timestamp)
logging.info("Using startime %s", start_time)
try:
client = AccountpageClient(token=opts.options.access_token,
url=opts.options.account_page_url + "/api/")
host_institute = opts.options.host_institute
slurm_account_info = get_slurm_acct_info(SyncTypes.accounts)
slurm_user_info = get_slurm_acct_info(SyncTypes.users)
logging.debug("%d accounts found", len(slurm_account_info))
logging.debug("%d users found", len(slurm_user_info))
if opts.options.clusters:
clusters = opts.options.clusters
else:
clusters = [
cs for p in opts.options.cluster_classes
for cs in VSC_SLURM_CLUSTERS[host_institute][p]
]
sacctmgr_commands = []
# All users belong to a VO, so fetching the VOs is necessary/
account_page_vos = [
mkVo(v)
for v in client.vo.institute[opts.options.host_institute].get()[1]
]
# make sure the institutes and the default accounts (VOs) are there for each cluster
institute_vos = dict([
(v.vsc_id, v) for v in account_page_vos
if v.vsc_id in INSTITUTE_VOS_BY_INSTITUTE[host_institute].values()
])
sacctmgr_commands += slurm_institute_accounts(slurm_account_info,
clusters, host_institute,
institute_vos)
# The VOs do not track active state of users, so we need to fetch all accounts as well
active_accounts = set(
[a["vsc_id"] for a in client.account.get()[1] if a["isactive"]])
# dictionary mapping the VO vsc_id on a tuple with the VO members and the VO itself
account_page_members = dict([(vo.vsc_id, (set(vo.members), vo))
for vo in account_page_vos])
# process all regular VOs
sacctmgr_commands += slurm_vo_accounts(account_page_vos,
slurm_account_info, clusters,
host_institute)
# process VO members
sacctmgr_commands += slurm_user_accounts(account_page_members,
active_accounts,
slurm_user_info, clusters,
opts.options.dry_run)
logging.info("Executing %d commands", len(sacctmgr_commands))
if opts.options.dry_run:
print("Commands to be executed:\n")
print("\n".join([" ".join(c) for c in sacctmgr_commands]))
else:
execute_commands(sacctmgr_commands)
if not opts.options.dry_run:
(_, ldap_timestamp) = convert_timestamp(start_time)
write_timestamp(SYNC_TIMESTAMP_FILENAME, ldap_timestamp)
opts.epilogue("Accounts synced to slurm", stats)
else:
logging.info("Dry run done")
except Exception as err:
logging.exception("critical exception caught: %s", err)
opts.critical("Script failed in a horrible way")
sys.exit(NAGIOS_EXIT_CRITICAL)
def main():
options = {
'nagios-check-interval-threshold':
NAGIOS_CHECK_INTERVAL_THRESHOLD,
'start-timestamp':
("The timestamp form which to start, otherwise use the cached value",
None, "store", None),
'access_token':
('OAuth2 token identifying the user with the accountpage', None,
'store', None),
'account_page_url': ('url for the account page', None, 'store', None),
'start_timestamp':
('Timestamp to start the sync from', str, 'store', None),
}
# get access_token from conf file
ExtendedSimpleOption.CONFIGFILES_INIT = ['/etc/account_page.conf']
opts = ExtendedSimpleOption(options)
stats = {}
# Creating this here because this is a singleton class
_ = LdapQuery(VscConfiguration(VSC_CONF_DEFAULT_FILENAME))
(last_timestamp, start_time) = retrieve_timestamp_with_default(
SYNC_TIMESTAMP_FILENAME, start_timestamp=opts.options.start_timestamp)
logging.info("Using timestamp %s", last_timestamp)
logging.info("Using startime %s", start_time)
try:
parent_pid = os.fork()
logging.info("Forked.")
except OSError:
logging.exception("Could not fork")
parent_pid = 1
except Exception:
logging.exception("Oops")
parent_pid = 1
if parent_pid == 0:
try:
global logger
logger = fancylogger.getLogger(NAGIOS_HEADER)
# drop privileges in the child
try:
apache_uid = pwd.getpwnam('apache').pw_uid
apache_gid = grp.getgrnam('apache').gr_gid
os.setgroups([])
os.setgid(apache_gid)
os.setuid(apache_uid)
logging.info("Now running as %s" % (os.geteuid(), ))
except OSError:
logger.raiseException("Could not drop privileges")
client = AccountpageClient(token=opts.options.access_token,
url=opts.options.account_page_url +
'/api/')
syncer = LdapSyncer(client)
last = last_timestamp
altered_accounts = syncer.sync_altered_accounts(
last, opts.options.dry_run)
logging.debug("Altered accounts: %s", altered_accounts)
altered_groups = syncer.sync_altered_groups(
last, opts.options.dry_run)
logging.debug("Altered groups: %s" % altered_groups)
if not altered_accounts[ERROR] \
and not altered_groups[ERROR]:
logging.info("Child process exiting correctly")
sys.exit(0)
else:
logging.info("Child process exiting with status -1")
logging.warning("Error occured in %s" % ([
"%s: %s\n" % (k, v) for (k, v) in [
("altered accounts", altered_accounts[ERROR]),
("altered groups", altered_groups[ERROR]),
]
]))
sys.exit(-1)
except Exception:
logging.exception("Child caught an exception")
sys.exit(-1)
else:
# parent
(_, result) = os.waitpid(parent_pid, 0)
logging.info("Child exited with exit code %d" % (result, ))
if not result and not opts.options.dry_run:
(_, ldap_timestamp) = convert_timestamp(start_time)
write_timestamp(SYNC_TIMESTAMP_FILENAME, ldap_timestamp)
opts.epilogue("Synchronised LDAP users to the Django DB", stats)
else:
sys.exit(NAGIOS_EXIT_CRITICAL)
def main():
options = {
'nagios-check-interval-threshold':
NAGIOS_CHECK_INTERVAL_THRESHOLD,
'start-timestamp':
("The timestamp form which to start, otherwise use the cached value",
None, "store", None),
'access_token':
('OAuth2 token identifying the user with the accountpage', None,
'store', None),
'account_page_url': ('url for the account page', None, 'store', None),
}
# get access_token from conf file
ExtendedSimpleOption.CONFIGFILES_INIT = ['/etc/account_page.conf']
opts = ExtendedSimpleOption(options)
stats = {}
# Creating this here because this is a singleton class
_ = LdapQuery(VscConfiguration(VSC_CONF_DEFAULT_FILENAME))
last_timestamp = opts.options.start_timestamp
if not last_timestamp:
try:
last_timestamp = read_timestamp(SYNC_TIMESTAMP_FILENAME)
except Exception:
_log.warning("Something broke reading the timestamp from %s",
SYNC_TIMESTAMP_FILENAME)
last_timestamp = "201710230000Z"
_log.warning(
"We will resync from a hardcoded know working sync a while back : %s",
last_timestamp)
_log.info("Using timestamp %s", last_timestamp)
# record starttime before starting, and take a 10 sec safety buffer so we don't get gaps where users are approved
# in between the requesting of modified users and writing out the start time
start_time = datetime.datetime.now() + datetime.timedelta(seconds=-10)
_log.info("startime %s", start_time)
try:
parent_pid = os.fork()
_log.info("Forked.")
except OSError:
_log.exception("Could not fork")
parent_pid = 1
except Exception:
_log.exception("Oops")
parent_pid = 1
if parent_pid == 0:
try:
global _log
_log = fancylogger.getLogger(NAGIOS_HEADER)
# drop privileges in the child
try:
apache_uid = pwd.getpwnam('apache').pw_uid
apache_gid = grp.getgrnam('apache').gr_gid
os.setgroups([])
os.setgid(apache_gid)
os.setuid(apache_uid)
_log.info("Now running as %s" % (os.geteuid(), ))
except OSError:
_log.raiseException("Could not drop privileges")
client = AccountpageClient(token=opts.options.access_token,
url=opts.options.account_page_url +
'/api/')
syncer = LdapSyncer(client)
last = int(
(datetime.datetime.strptime(last_timestamp, "%Y%m%d%H%M%SZ") -
datetime.datetime(1970, 1, 1)).total_seconds())
altered_accounts = syncer.sync_altered_accounts(
last, opts.options.dry_run)
_log.debug("Altered accounts: %s", altered_accounts)
altered_groups = syncer.sync_altered_groups(
last, opts.options.dry_run)
_log.debug("Altered groups: %s" % altered_groups)
if not altered_accounts[ERROR] \
and not altered_groups[ERROR]:
_log.info("Child process exiting correctly")
sys.exit(0)
else:
_log.info("Child process exiting with status -1")
_log.warning("Error occured in %s" % ([
"%s: %s\n" % (k, v) for (k, v) in [
("altered accounts", altered_accounts[ERROR]),
("altered groups", altered_groups[ERROR]),
]
]))
sys.exit(-1)
except Exception:
_log.exception("Child caught an exception")
sys.exit(-1)
else:
# parent
(_, result) = os.waitpid(parent_pid, 0)
_log.info("Child exited with exit code %d" % (result, ))
if not result:
if not opts.options.start_timestamp:
(_, ldap_timestamp) = convert_timestamp(start_time)
if not opts.options.dry_run:
write_timestamp(SYNC_TIMESTAMP_FILENAME, ldap_timestamp)
else:
_log.info(
"Not updating the timestamp, since one was provided on the command line"
)
opts.epilogue("Synchronised LDAP users to the Django DB", stats)
else:
_log.info(
"Not updating the timestamp, since it was given on the command line for this run"
)
sys.exit(NAGIOS_EXIT_CRITICAL)
def main():
"""
Main script.
- build the filter
- fetches the users
- process the users
- write the new timestamp if everything went OK
- write the nagios check file
"""
options = {
'nagios-check-interval-threshold':
NAGIOS_CHECK_INTERVAL_THRESHOLD,
'storage': ('storage systems on which to deploy users and vos', None,
'extend', []),
'user': ('process users', None, 'store_true', False),
'vo': ('process vos', None, 'store_true', False),
'access_token': ('OAuth2 token to access the account page REST API',
None, 'store', None),
'account_page_url':
('URL of the account page where we can find the REST API', None,
'store', None),
'host_institute':
('Name of the institute where this script is being run', str, 'store',
GENT),
'start_timestamp':
('Timestamp to start the sync from', str, 'store', None),
}
opts = ExtendedSimpleOption(options)
stats = {}
(last_timestamp, start_time) = retrieve_timestamp_with_default(
SYNC_TIMESTAMP_FILENAME, start_timestamp=opts.options.start_timestamp)
logging.info("Using timestamp %s", last_timestamp)
logging.info("Using startime %s", start_time)
try:
client = AccountpageClient(token=opts.options.access_token,
url=opts.options.account_page_url + "/api/")
institute = opts.options.host_institute
(users_ok, users_fail) = ([], [])
(quota_ok, quota_fail) = ([], [])
if opts.options.user:
changed_accounts = client.account.institute[institute].modified[
last_timestamp].get()[1]
logging.info(
"Found %d %s accounts that have changed in the accountpage since %s"
% (len(changed_accounts), institute, last_timestamp))
accounts = nub([u['vsc_id'] for u in changed_accounts])
for storage_name in opts.options.storage:
(users_ok, users_fail) = process_users(opts.options, accounts,
storage_name, client,
institute)
stats["%s_users_sync" % (storage_name, )] = len(users_ok)
stats["%s_users_sync_fail" %
(storage_name, )] = len(users_fail)
stats["%s_users_sync_fail_warning" %
(storage_name, )] = STORAGE_USERS_LIMIT_WARNING
stats["%s_users_sync_fail_critical" %
(storage_name, )] = STORAGE_USERS_LIMIT_CRITICAL
for storage_name in opts.options.storage:
storage_changed_quota = [
mkVscUserSizeQuota(q) for q in client.quota.user.
storage[storage_name].modified[last_timestamp].get()[1]
]
storage_changed_quota = [
q for q in storage_changed_quota
if q.fileset.startswith('vsc')
]
logging.info(
"Found %d accounts that have changed quota on storage %s in the accountpage since %s",
len(storage_changed_quota), storage_name, last_timestamp)
(quota_ok, quota_fail) = process_users_quota(
opts.options, storage_changed_quota, storage_name, client,
institute)
stats["%s_quota_sync" % (storage_name, )] = len(quota_ok)
stats["%s_quota_sync_fail" %
(storage_name, )] = len(quota_fail)
stats["%s_quota_sync_fail_warning" %
(storage_name, )] = STORAGE_QUOTA_LIMIT_WARNING
stats["%s_quota_sync_fail_critical" %
(storage_name, )] = STORAGE_QUOTA_LIMIT_CRITICAL
(vos_ok, vos_fail) = ([], [])
if opts.options.vo:
changed_vos = client.vo.institute[institute].modified[
last_timestamp].get()[1]
changed_vo_quota = client.quota.vo.modified[last_timestamp].get(
)[1]
vos = sorted(
set([v['vsc_id'] for v in changed_vos] +
[v['virtual_organisation'] for v in changed_vo_quota]))
logging.info(
"Found %d %s VOs that have changed in the accountpage since %s"
% (len(changed_vos), institute, last_timestamp))
logging.info(
"Found %d %s VOs that have changed quota in the accountpage since %s"
% (len(changed_vo_quota), institute, last_timestamp))
logging.debug("Found the following {institute} VOs: {vos}".format(
institute=institute, vos=vos))
for storage_name in opts.options.storage:
(vos_ok, vos_fail) = process_vos(opts.options, vos,
storage_name, client,
last_timestamp, institute)
stats["%s_vos_sync" % (storage_name, )] = len(vos_ok)
stats["%s_vos_sync_fail" % (storage_name, )] = len(vos_fail)
stats["%s_vos_sync_fail_warning" %
(storage_name, )] = STORAGE_VO_LIMIT_WARNING
stats["%s_vos_sync_fail_critical" %
(storage_name, )] = STORAGE_VO_LIMIT_CRITICAL
if not (users_fail or quota_fail
or vos_fail) and not opts.options.dry_run:
(_, ldap_timestamp) = convert_timestamp(start_time)
write_timestamp(SYNC_TIMESTAMP_FILENAME, ldap_timestamp)
except Exception as err:
logger.exception("critical exception caught: %s" % (err))
opts.critical("Script failed in a horrible way")
sys.exit(NAGIOS_EXIT_CRITICAL)
opts.epilogue("%s users and VOs synchronised" % institute, stats)