def mail_admins(critical_filesets, dry_run=True, host_institute=GENT):
"""Send email to the HPC admin about the inodes running out soonish."""
mail = VscMail(mail_host=INSTITUTE_SMTP_SERVER[host_institute])
message = CRITICAL_INODE_COUNT_MESSAGE
fileset_info = []
for (fs_name, fs_info) in critical_filesets.items():
for (fileset_name, inode_info) in fs_info.items():
fileset_info.append(
"%s - %s: used %d (%d%%) of max %d [allocated: %d]" %
(fs_name, fileset_name, inode_info.used,
int(inode_info.used * 100 / inode_info.maxinodes),
inode_info.maxinodes, inode_info.allocated))
message = message % ({'fileset_info': "\n".join(fileset_info)})
if dry_run:
logging.info("Would have sent this message: %s", message)
else:
mail.sendTextMail(mail_to=INSTITUTE_ADMIN_EMAIL[host_institute],
mail_from=INSTITUTE_ADMIN_EMAIL[host_institute],
reply_to=INSTITUTE_ADMIN_EMAIL[host_institute],
mail_subject="Inode space(s) running out on %s" %
(socket.gethostname()),
message=message)
def mail_admins(critical_filesets, dry_run=True):
"""Send email to the HPC admin about the inodes running out soonish."""
mail = VscMail(mail_host="smtp.ugent.be")
message = CRITICAL_INODE_COUNT_MESSAGE
fileset_info = []
for (fs_name, fs_info) in critical_filesets.items():
for (fileset_name, inode_info) in fs_info.items():
fileset_info.append(
"%s - %s: used %d (%d%%) of max %d [allocated: %d]" %
(fs_name, fileset_name, inode_info.used,
int(inode_info.used * 100 / inode_info.maxinodes),
inode_info.maxinodes, inode_info.allocated))
message = message % ({'fileset_info': "\n".join(fileset_info)})
if dry_run:
logging.info("Would have sent this message: %s" % (message, ))
else:
mail.sendTextMail(mail_to="*****@*****.**",
mail_from="*****@*****.**",
reply_to="*****@*****.**",
mail_subject="Inode space(s) running out on %s" %
(socket.gethostname()),
message=message)
def mail_admins(critical_filesets, dry_run):
"""Send email to the HPC admin about the inodes running out soonish."""
mail = VscMail(mail_host="smtp.ugent.be")
message = """
Dear HPC admins,
The following filesets will be running out of inodes soon (or may already have run out).
%(fileset_info)s
Kind regards,
Your friendly inode-watching script
"""
fileset_info = []
for (fs_name, fs_info) in critical_filesets.items():
for (fileset_name, inode_info) in fs_info.items():
fileset_info.append("%s - %s: used %d (%d%%) of %d" % (fs_name,
fileset_name,
inode_info.allocated,
int(inode_info.allocated * 100 / inode_info.maxinodes),
inode_info.maxinodes))
message = message % ({'fileset_info': "\n".join(fileset_info)})
if dry_run:
logger.info("Would have sent this message: %s" % (message,))
else:
mail.sendTextMail(mail_to="*****@*****.**",
mail_from="*****@*****.**",
reply_to="*****@*****.**",
mail_subject="Inode space(s) running out on %s" % (socket.gethostname()),
message=message)
def mail_report(t, queued_jobs, running_jobs):
"""Mail report to [email protected].
@type t: string representing the time when the job list was fetched
@type queued_jobs: list of queued job tuples (name, PBS job entry)
@type running_jobs: list of running job tuples (name, PBS job entry)
"""
message_queued_jobs = "\n".join(
["Queued jobs belonging to gracing or inactive users", 50 * "-"]
+ [
"{user_name} - {job_name} queued at {queue_time}".format(
user_name=job["euser"][0], queue_time=job["qtime"][0], job_name=job_name
)
for (job_name, job) in queued_jobs
]
)
message_running_jobs = "\n".join(
["Running jobs belonging to inactive users", 40 * "-"]
+ [
"{user_name} - {job_name} running on {nodes}".format(
user_name=job["euser"][0], job_name=job_name, nodes=str(job["exec_host"])
)
for (job_name, job) in running_jobs
]
)
mail_to = "*****@*****.**"
mail = VscMail()
message = """Dear admins,
These are the jobs on belonging to users who have entered their grace period or have become inactive, as indicated by the
LDAP replica on {master} at {time}.
{message_queued_jobs}
{message_running_jobs}
Kind regards,
Your friendly pbs job checking script
""".format(
master=socket.gethostname(),
time=time.ctime(),
message_queued_jobs=message_queued_jobs,
message_running_jobs=message_running_jobs,
)
try:
logger.info("Sending report mail to %s" % (mail_to))
mail.sendTextMail(
mail_to=mail_to,
mail_from="*****@*****.**",
reply_to="*****@*****.**",
subject="PBS check for jobs belonging to gracing or inactive users",
message=message,
)
except Exception, err:
logger.error("Failed in sending mail to %s (%s)." % (mail_to, err))
def mail_report(t, queued_jobs, running_jobs):
"""Mail report to [email protected].
@type t: string representing the time when the job list was fetched
@type queued_jobs: list of queued job tuples (name, PBS job entry)
@type running_jobs: list of running job tuples (name, PBS job entry)
"""
message_queued_jobs = '\n'.join(
['Queued jobs belonging to gracing or inactive users', 50 * '-'] + [
"{user_name} - {job_name} queued at {queue_time}".format(
user_name=job['euser'][0],
queue_time=job['qtime'][0],
job_name=job_name) for (job_name, job) in queued_jobs
])
message_running_jobs = '\n'.join(
['Running jobs belonging to inactive users', 40 * '-'] + [
"{user_name} - {job_name} running on {nodes}".format(
user_name=job['euser'][0],
job_name=job_name,
nodes=str(job['exec_host']))
for (job_name, job) in running_jobs
])
mail_to = '*****@*****.**'
mail = VscMail()
message = """Dear admins,
These are the jobs on belonging to users who have entered their grace period or have become inactive, as indicated by
the LDAP replica on {master} at {time}.
{message_queued_jobs}
{message_running_jobs}
Kind regards,
Your friendly pbs job checking script
""".format(master=socket.gethostname(),
time=t,
message_queued_jobs=message_queued_jobs,
message_running_jobs=message_running_jobs)
try:
logger.info("Sending report mail to %s" % (mail_to))
mail.sendTextMail(
mail_to=mail_to,
mail_from='*****@*****.**',
reply_to='*****@*****.**',
mail_subject=
'PBS check for jobs belonging to gracing or inactive users',
message=message)
except Exception, err:
logger.error("Failed in sending mail to %s (%s)." % (mail_to, err))
def notify(storage_name, item, quota, dry_run=False):
"""Send out the notification"""
mail = VscMail(mail_host="smtp.ugent.be")
if item.startswith("gvo"): # VOs
vo = VscVo(item)
for user in [VscUser(m) for m in vo.moderator]:
message = VO_QUOTA_EXCEEDED_MAIL_TEXT_TEMPLATE.safe_substitute(user_name=user.gecos,
vo_name=item,
storage_name=storage_name,
quota_info="%s" % (quota,),
time=time.ctime())
if dry_run:
logger.info("Dry-run, would send the following message: %s" % (message,))
else:
mail.sendTextMail(mail_to=user.mail,
mail_from="*****@*****.**",
reply_to="*****@*****.**",
mail_subject="Quota on %s exceeded" % (storage_name,),
message=message)
logger.info("notification: recipient %s storage %s quota_string %s" %
(user.cn, storage_name, "%s" % (quota,)))
elif item.startswith("gpr"): # projects
pass
elif item.startswith("vsc"): # users
user = VscUser(item)
exceeding_filesets = [fs for (fs, q) in quota.quota_map.items() if q.expired[0]]
storage_names = []
if [ef for ef in exceeding_filesets if not ef.startswith("gvo")]:
storage_names.append(storage_name)
if [ef for ef in exceeding_filesets if ef.startswith("gvo")]:
storage_names.append(storage_name + "_VO")
storage_names = ", ".join(["$" + sn for sn in storage_names])
message = QUOTA_EXCEEDED_MAIL_TEXT_TEMPLATE.safe_substitute(user_name=user.gecos,
storage_name=storage_names,
quota_info="%s" % (quota,),
time=time.ctime())
if dry_run:
logger.info("Dry-run, would send the following message: %s" % (message,))
else:
mail.sendTextMail(mail_to=user.mail,
mail_from="*****@*****.**",
reply_to="*****@*****.**",
mail_subject="Quota on %s exceeded" % (storage_name,),
message=message)
logger.info("notification: recipient %s storage %s quota_string %s" %
(user.cn, storage_name, "%s" % (quota,)))
else:
logger.error("Should send a mail, but cannot process item %s" % (item,))
def mail_report(t, queued_jobs, running_jobs):
"""Mail report to [email protected].
@type t: string representing the time when the job list was fetched
@type queued_jobs: list of queued job tuples (name, PBS job entry)
@type running_jobs: list of running job tuples (name, PBS job entry)
"""
message_queued_jobs = '\n'.join(['Queued jobs belonging to gracing or inactive users', 50 * '-'] +
["{user_name} - {job_name} queued at {queue_time}".format(user_name=job['euser'][0],
queue_time=job['qtime'][0],
job_name=job_name)
for (job_name, job) in queued_jobs])
message_running_jobs = '\n'.join(['Running jobs belonging to inactive users', 40 * '-'] +
["{user_name} - {job_name} running on {nodes}".format(user_name=job['euser'][0],
job_name=job_name,
nodes=str(job['exec_host']))
for (job_name, job) in running_jobs])
mail_to = '*****@*****.**'
mail = VscMail()
message = """Dear admins,
These are the jobs on belonging to users who have entered their grace period or have become inactive.
{message_queued_jobs}
{message_running_jobs}
Kind regards,
Your friendly pbs job checking script
""".format(master=socket.gethostname(),
time=t,
message_queued_jobs=message_queued_jobs,
message_running_jobs=message_running_jobs)
try:
logger.info("Sending report mail to %s" % (mail_to))
mail.sendTextMail(mail_to=mail_to,
mail_from='*****@*****.**',
reply_to='*****@*****.**',
mail_subject='PBS check for jobs belonging to gracing or inactive users',
message=message)
except Exception, err:
logger.error("Failed in sending mail to %s (%s)." % (mail_to, err))
def do(self, dry_run):
"""Get the information and mail it"""
clusters = GENT_PRODUCTION_COMPUTE_CLUSTERS
if self.options.cluster:
clusters = (self.options.cluster, )
self.load_users()
body = []
for cluster in clusters:
body += self.report(cluster)
mail_body = "\n".join(body)
mail_subject = "HPC: usage report %s - %s" % (self.options.start,
self.options.end)
mail_to = self.options.recipient
mail = VscMail("loudred.gastly.os")
mail.sendTextMail(mail_to=mail_to,
mail_from="*****@*****.**",
reply_to="*****@*****.**",
mail_subject=mail_subject,
message=mail_body)
