def mail_admins(critical_filesets, dry_run=True, host_institute=GENT): """Send email to the HPC admin about the inodes running out soonish.""" mail = VscMail(mail_host=INSTITUTE_SMTP_SERVER[host_institute]) message = CRITICAL_INODE_COUNT_MESSAGE fileset_info = [] for (fs_name, fs_info) in critical_filesets.items(): for (fileset_name, inode_info) in fs_info.items(): fileset_info.append( "%s - %s: used %d (%d%%) of max %d [allocated: %d]" % (fs_name, fileset_name, inode_info.used, int(inode_info.used * 100 / inode_info.maxinodes), inode_info.maxinodes, inode_info.allocated)) message = message % ({'fileset_info': "\n".join(fileset_info)}) if dry_run: logging.info("Would have sent this message: %s", message) else: mail.sendTextMail(mail_to=INSTITUTE_ADMIN_EMAIL[host_institute], mail_from=INSTITUTE_ADMIN_EMAIL[host_institute], reply_to=INSTITUTE_ADMIN_EMAIL[host_institute], mail_subject="Inode space(s) running out on %s" % (socket.gethostname()), message=message)
def mail_admins(critical_filesets, dry_run=True): """Send email to the HPC admin about the inodes running out soonish.""" mail = VscMail(mail_host="smtp.ugent.be") message = CRITICAL_INODE_COUNT_MESSAGE fileset_info = [] for (fs_name, fs_info) in critical_filesets.items(): for (fileset_name, inode_info) in fs_info.items(): fileset_info.append( "%s - %s: used %d (%d%%) of max %d [allocated: %d]" % (fs_name, fileset_name, inode_info.used, int(inode_info.used * 100 / inode_info.maxinodes), inode_info.maxinodes, inode_info.allocated)) message = message % ({'fileset_info': "\n".join(fileset_info)}) if dry_run: logging.info("Would have sent this message: %s" % (message, )) else: mail.sendTextMail(mail_to="*****@*****.**", mail_from="*****@*****.**", reply_to="*****@*****.**", mail_subject="Inode space(s) running out on %s" % (socket.gethostname()), message=message)
def mail_admins(critical_filesets, dry_run): """Send email to the HPC admin about the inodes running out soonish.""" mail = VscMail(mail_host="smtp.ugent.be") message = """ Dear HPC admins, The following filesets will be running out of inodes soon (or may already have run out). %(fileset_info)s Kind regards, Your friendly inode-watching script """ fileset_info = [] for (fs_name, fs_info) in critical_filesets.items(): for (fileset_name, inode_info) in fs_info.items(): fileset_info.append("%s - %s: used %d (%d%%) of %d" % (fs_name, fileset_name, inode_info.allocated, int(inode_info.allocated * 100 / inode_info.maxinodes), inode_info.maxinodes)) message = message % ({'fileset_info': "\n".join(fileset_info)}) if dry_run: logger.info("Would have sent this message: %s" % (message,)) else: mail.sendTextMail(mail_to="*****@*****.**", mail_from="*****@*****.**", reply_to="*****@*****.**", mail_subject="Inode space(s) running out on %s" % (socket.gethostname()), message=message)
def mail_report(t, queued_jobs, running_jobs): """Mail report to [email protected]. @type t: string representing the time when the job list was fetched @type queued_jobs: list of queued job tuples (name, PBS job entry) @type running_jobs: list of running job tuples (name, PBS job entry) """ message_queued_jobs = "\n".join( ["Queued jobs belonging to gracing or inactive users", 50 * "-"] + [ "{user_name} - {job_name} queued at {queue_time}".format( user_name=job["euser"][0], queue_time=job["qtime"][0], job_name=job_name ) for (job_name, job) in queued_jobs ] ) message_running_jobs = "\n".join( ["Running jobs belonging to inactive users", 40 * "-"] + [ "{user_name} - {job_name} running on {nodes}".format( user_name=job["euser"][0], job_name=job_name, nodes=str(job["exec_host"]) ) for (job_name, job) in running_jobs ] ) mail_to = "*****@*****.**" mail = VscMail() message = """Dear admins, These are the jobs on belonging to users who have entered their grace period or have become inactive, as indicated by the LDAP replica on {master} at {time}. {message_queued_jobs} {message_running_jobs} Kind regards, Your friendly pbs job checking script """.format( master=socket.gethostname(), time=time.ctime(), message_queued_jobs=message_queued_jobs, message_running_jobs=message_running_jobs, ) try: logger.info("Sending report mail to %s" % (mail_to)) mail.sendTextMail( mail_to=mail_to, mail_from="*****@*****.**", reply_to="*****@*****.**", subject="PBS check for jobs belonging to gracing or inactive users", message=message, ) except Exception, err: logger.error("Failed in sending mail to %s (%s)." % (mail_to, err))
def mail_report(t, queued_jobs, running_jobs): """Mail report to [email protected]. @type t: string representing the time when the job list was fetched @type queued_jobs: list of queued job tuples (name, PBS job entry) @type running_jobs: list of running job tuples (name, PBS job entry) """ message_queued_jobs = '\n'.join( ['Queued jobs belonging to gracing or inactive users', 50 * '-'] + [ "{user_name} - {job_name} queued at {queue_time}".format( user_name=job['euser'][0], queue_time=job['qtime'][0], job_name=job_name) for (job_name, job) in queued_jobs ]) message_running_jobs = '\n'.join( ['Running jobs belonging to inactive users', 40 * '-'] + [ "{user_name} - {job_name} running on {nodes}".format( user_name=job['euser'][0], job_name=job_name, nodes=str(job['exec_host'])) for (job_name, job) in running_jobs ]) mail_to = '*****@*****.**' mail = VscMail() message = """Dear admins, These are the jobs on belonging to users who have entered their grace period or have become inactive, as indicated by the LDAP replica on {master} at {time}. {message_queued_jobs} {message_running_jobs} Kind regards, Your friendly pbs job checking script """.format(master=socket.gethostname(), time=t, message_queued_jobs=message_queued_jobs, message_running_jobs=message_running_jobs) try: logger.info("Sending report mail to %s" % (mail_to)) mail.sendTextMail( mail_to=mail_to, mail_from='*****@*****.**', reply_to='*****@*****.**', mail_subject= 'PBS check for jobs belonging to gracing or inactive users', message=message) except Exception, err: logger.error("Failed in sending mail to %s (%s)." % (mail_to, err))
def notify(storage_name, item, quota, dry_run=False): """Send out the notification""" mail = VscMail(mail_host="smtp.ugent.be") if item.startswith("gvo"): # VOs vo = VscVo(item) for user in [VscUser(m) for m in vo.moderator]: message = VO_QUOTA_EXCEEDED_MAIL_TEXT_TEMPLATE.safe_substitute(user_name=user.gecos, vo_name=item, storage_name=storage_name, quota_info="%s" % (quota,), time=time.ctime()) if dry_run: logger.info("Dry-run, would send the following message: %s" % (message,)) else: mail.sendTextMail(mail_to=user.mail, mail_from="*****@*****.**", reply_to="*****@*****.**", mail_subject="Quota on %s exceeded" % (storage_name,), message=message) logger.info("notification: recipient %s storage %s quota_string %s" % (user.cn, storage_name, "%s" % (quota,))) elif item.startswith("gpr"): # projects pass elif item.startswith("vsc"): # users user = VscUser(item) exceeding_filesets = [fs for (fs, q) in quota.quota_map.items() if q.expired[0]] storage_names = [] if [ef for ef in exceeding_filesets if not ef.startswith("gvo")]: storage_names.append(storage_name) if [ef for ef in exceeding_filesets if ef.startswith("gvo")]: storage_names.append(storage_name + "_VO") storage_names = ", ".join(["$" + sn for sn in storage_names]) message = QUOTA_EXCEEDED_MAIL_TEXT_TEMPLATE.safe_substitute(user_name=user.gecos, storage_name=storage_names, quota_info="%s" % (quota,), time=time.ctime()) if dry_run: logger.info("Dry-run, would send the following message: %s" % (message,)) else: mail.sendTextMail(mail_to=user.mail, mail_from="*****@*****.**", reply_to="*****@*****.**", mail_subject="Quota on %s exceeded" % (storage_name,), message=message) logger.info("notification: recipient %s storage %s quota_string %s" % (user.cn, storage_name, "%s" % (quota,))) else: logger.error("Should send a mail, but cannot process item %s" % (item,))
def mail_report(t, queued_jobs, running_jobs): """Mail report to [email protected]. @type t: string representing the time when the job list was fetched @type queued_jobs: list of queued job tuples (name, PBS job entry) @type running_jobs: list of running job tuples (name, PBS job entry) """ message_queued_jobs = '\n'.join(['Queued jobs belonging to gracing or inactive users', 50 * '-'] + ["{user_name} - {job_name} queued at {queue_time}".format(user_name=job['euser'][0], queue_time=job['qtime'][0], job_name=job_name) for (job_name, job) in queued_jobs]) message_running_jobs = '\n'.join(['Running jobs belonging to inactive users', 40 * '-'] + ["{user_name} - {job_name} running on {nodes}".format(user_name=job['euser'][0], job_name=job_name, nodes=str(job['exec_host'])) for (job_name, job) in running_jobs]) mail_to = '*****@*****.**' mail = VscMail() message = """Dear admins, These are the jobs on belonging to users who have entered their grace period or have become inactive. {message_queued_jobs} {message_running_jobs} Kind regards, Your friendly pbs job checking script """.format(master=socket.gethostname(), time=t, message_queued_jobs=message_queued_jobs, message_running_jobs=message_running_jobs) try: logger.info("Sending report mail to %s" % (mail_to)) mail.sendTextMail(mail_to=mail_to, mail_from='*****@*****.**', reply_to='*****@*****.**', mail_subject='PBS check for jobs belonging to gracing or inactive users', message=message) except Exception, err: logger.error("Failed in sending mail to %s (%s)." % (mail_to, err))
def do(self, dry_run): """Get the information and mail it""" clusters = GENT_PRODUCTION_COMPUTE_CLUSTERS if self.options.cluster: clusters = (self.options.cluster, ) self.load_users() body = [] for cluster in clusters: body += self.report(cluster) mail_body = "\n".join(body) mail_subject = "HPC: usage report %s - %s" % (self.options.start, self.options.end) mail_to = self.options.recipient mail = VscMail("loudred.gastly.os") mail.sendTextMail(mail_to=mail_to, mail_from="*****@*****.**", reply_to="*****@*****.**", mail_subject=mail_subject, message=mail_body)