def email(self, alerts): node = alert_node() dismisseds = [a.message_id for a in mAlert.objects.filter(node=node)] msgs = [] for alert in alerts: if alert.getId() not in dismisseds: """ This is all to allow <a> tags in alert messages. We need to strip out all the tags so we can send a plain text email. """ msg = str(alert) msgnode = etree.fromstring('<msg>{}</msg>'.format(msg)) for i in msgnode.xpath('//a'): new = etree.Element('span') new.text = '{} ({})'.format(i.text, i.attrib['href']) msgnode.replace(i, new) etree.strip_tags(msgnode, '*') msgs.append(msgnode.text) if len(msgs) == 0: return hostname = socket.gethostname() send_mail( subject='%s: %s' % ( hostname, _("Critical Alerts"), ), text='\n'.join(msgs) )
def do_sendmail(msg, to_addrs=None, parse_recipients=False): if to_addrs is None: if not parse_recipients: raise ValueError('Do not know who to send the message to.') to_addrs = [] # XXX: this should probably be a FeedParser because reading from sys.stdin # is blocking. em_parser = email.parser.Parser() em = em_parser.parsestr(msg) if parse_recipients: # Strip away the comma based delimiters and whitespace. to_addrs = map(str.strip, em.get('To').split(',')) if not to_addrs or not to_addrs[0]: to_addrs = ['root'] if to_addrs: aliases = get_aliases() to_addrs_repl = [] for to_addr in to_addrs: for to_addr in to_addr.split(','): if to_addr.find('@') == -1 and to_addr in aliases: to_addr = aliases[to_addr] to_addrs_repl.append(to_addr) margs = {} margs['extra_headers'] = dict(em) margs['extra_headers'].update({ 'X-Mailer': get_sw_name(), 'X-%s-Host' % get_sw_name(): socket.gethostname(), 'To': to_addr, }) margs['subject'] = em.get('Subject') # abusive use of querysets lemail = Email.objects.all() for obj in lemail: if obj.em_fromemail != '': margs['extra_headers'].update({ 'From': obj.em_fromemail }) if em.is_multipart(): margs['attachments'] = filter( lambda part: part.get_content_maintype() != 'multipart', em.walk() ) margs['text'] = u"%s" % _( 'This is a MIME formatted message. If you see ' 'this text it means that your email software ' 'does not support MIME formatted messages.') else: margs['text'] = ''.join(email.iterators.body_line_iterator(em)) if to_addrs_repl: margs['to'] = to_addrs_repl send_mail(**margs)
def email(self): """ Use alert.last to hold a sha256 hash of the last sent alerts If the hash is the same do not resend the email """ dismisseds = [a.message_id for a in mAlert.objects.filter(dismiss=True) ] msgs = [] for msg in self.__logs[self.LOG_CRIT]: if msg._msgid not in dismisseds: msgs.append(unicode(msg)) if len(msgs) == 0: if os.path.exists(LAST_ALERT_FILE): os.unlink(LAST_ALERT_FILE) return try: with open(LAST_ALERT_FILE) as f: sha256 = f.read() except: sha256 = '' newsha = hashlib.sha256(repr(msgs)).hexdigest() if newsha != sha256: send_mail(subject=_("Critical Alerts"), text='\n'.join(msgs)) with open(LAST_ALERT_FILE, 'w') as f: f.write(newsha)
def email(dataset, threshold, avail): send_mail(subject="Volume threshold", text="""Hi, Your volume %s has reached the threshold of %s. Currently there is %s of available space. """ % (dataset, threshold, avail))
def email(self, alerts): node = alert_node() dismisseds = [a.message_id for a in mAlert.objects.filter(node=node)] msgs = [] for alert in alerts: if alert.getId() not in dismisseds: """ This is all to allow <a> tags in alert messages. We need to strip out all the tags so we can send a plain text email. """ msg = str(alert) msgnode = etree.fromstring('<msg>{}</msg>'.format(msg)) for i in msgnode.xpath('//a'): new = etree.Element('span') new.text = '{} ({})'.format(i.text, i.attrib['href']) msgnode.replace(i, new) etree.strip_tags(msgnode, '*') msgs.append(msgnode.text) if len(msgs) == 0: return hostname = socket.gethostname() send_mail(subject='%s: %s' % ( hostname, _("Critical Alerts"), ), text='\n'.join(msgs))
def email(self): """ Use alert.last to hold a sha256 hash of the last sent alerts If the hash is the same do not resend the email """ dismisseds = [ a.message_id for a in mAlert.objects.filter(dismiss=True) ] msgs = [] for msg in self.__logs[self.LOG_CRIT]: if msg._msgid not in dismisseds: msgs.append(unicode(msg)) if len(msgs) == 0: if os.path.exists(LAST_ALERT_FILE): os.unlink(LAST_ALERT_FILE) return try: with open(LAST_ALERT_FILE) as f: sha256 = f.read() except: sha256 = '' newsha = hashlib.sha256(repr(msgs)).hexdigest() if newsha != sha256: send_mail(subject=_("Critical Alerts"), text='\n'.join(msgs)) with open(LAST_ALERT_FILE, 'w') as f: f.write(newsha)
def do_sendmail(msg, to_addrs=None, parse_recipients=False): if to_addrs is None: if not parse_recipients: raise ValueError('Do not know who to send the message to.') to_addrs = [] # XXX: this should probably be a FeedParser because reading from sys.stdin # is blocking. em_parser = email.parser.Parser() em = em_parser.parsestr(msg) if parse_recipients: # Strip away the comma based delimiters and whitespace. to_addrs = map(str.strip, em.get('To').split(',')) if not to_addrs or not to_addrs[0]: to_addrs = ['root'] if to_addrs: aliases = get_aliases() to_addrs_repl = [] for to_addr in to_addrs: for to_addr in to_addr.split(','): if to_addr.find('@') == -1 and to_addr in aliases: to_addr = aliases[to_addr] to_addrs_repl.append(to_addr) margs = {} margs['extra_headers'] = dict(em) margs['extra_headers'].update({ 'X-Mailer': get_sw_name(), 'X-%s-Host' % get_sw_name(): socket.gethostname(), 'To': to_addr, }) margs['subject'] = em.get('Subject') # abusive use of querysets lemail = Email.objects.all() for obj in lemail: if obj.em_fromemail != '': margs['extra_headers'].update({'From': obj.em_fromemail}) if em.is_multipart(): margs['attachments'] = filter( lambda part: part.get_content_maintype() != 'multipart', em.walk()) margs['text'] = u"%s" % _( 'This is a MIME formatted message. If you see ' 'this text it means that your email software ' 'does not support MIME formatted messages.') else: margs['text'] = ''.join(email.iterators.body_line_iterator(em)) if to_addrs_repl: margs['to'] = to_addrs_repl send_mail(**margs)
def main(): try: updateobj = mUpdate.objects.order_by('-id')[0] except IndexError: updateobj = mUpdate.objects.create() if updateobj.upd_autocheck is False: return location = notifier().get_update_location() Update.DownloadUpdate(updateobj.get_train(), location) update = Update.CheckForUpdates( train=updateobj.get_train(), cache_dir=location, ) if not update: return conf = Configuration.Configuration() sys_mani = conf.SystemManifest() if sys_mani: sequence = sys_mani.Sequence() else: sequence = '' changelog = get_changelog( updateobj.get_train(), start=sequence, end=update.Sequence(), ) hostname = socket.gethostname() send_mail( subject='%s: %s' % ( hostname, _('Update Available'), ), extra_headers={ 'X-Mailer': get_sw_name(), 'X-%s-Host' % get_sw_name(): socket.gethostname() }, text=_('''A new update is available for the %(train)s train. Version: %(version)s Changelog: %(changelog)s ''') % { 'train': updateobj.get_train(), 'version': update.Version(), 'changelog': changelog, }, )
def email(self, alerts): dismisseds = [ a.message_id for a in mAlert.objects.filter(dismiss=True) ] msgs = [] for alert in alerts: if alert.getId() not in dismisseds: msgs.append(unicode(alert)) if len(msgs) == 0: return send_mail(subject=_("Critical Alerts"), text='\n'.join(msgs))
def email(self, alerts): dismisseds = [a.message_id for a in mAlert.objects.filter(dismiss=True)] msgs = [] for alert in alerts: if alert.getId() not in dismisseds: msgs.append(unicode(alert)) if len(msgs) == 0: return send_mail(subject=_("Critical Alerts"), text='\n'.join(msgs))
def main(): try: updateobj = mUpdate.objects.order_by('-id')[0] except IndexError: updateobj = mUpdate.objects.create() if updateobj.upd_autocheck is False: return location = notifier().get_update_location() Update.DownloadUpdate(updateobj.get_train(), location) update = Update.CheckForUpdates( train=updateobj.get_train(), cache_dir=location, ) if not update: return conf = Configuration.Configuration() sys_mani = conf.SystemManifest() if sys_mani: sequence = sys_mani.Sequence() else: sequence = '' changelog = get_changelog( updateobj.get_train(), start=sequence, end=update.Sequence(), ) hostname = socket.gethostname() send_mail( subject='%s: %s' % ( hostname, _('Update Available'), ), extra_headers={'X-Mailer': get_sw_name(), 'X-%s-Host' % get_sw_name(): socket.gethostname()}, text=_('''A new update is available for the %(train)s train. Changelog: %(changelog)s ''') % { 'train': updateobj.get_train(), 'changelog': changelog, }, )
def testmail(request): try: kwargs = dict(instance=models.Email.objects.order_by('-id')[0]) except IndexError: kwargs = {} form = forms.EmailForm(request.POST, **kwargs) if not form.is_valid(): return JsonResp(request, form=form) email = bsdUsers.objects.get(bsdusr_username='******').bsdusr_email if not email: return JsonResp( request, error=True, message=_( "You must configure the root email (Accounts->Users->root)"), ) sid = transaction.savepoint() form.save() error = False if request.is_ajax(): sw_name = get_sw_name() error, errmsg = send_mail( subject=_('Test message from %s' % (sw_name)), text=_('This is a message test from %s' % (sw_name, ))) if error: errmsg = _("Your test email could not be sent: %s") % errmsg else: errmsg = _('Your test email has been sent!') transaction.savepoint_rollback(sid) return JsonResp(request, error=error, message=errmsg)
def testmail(request): try: kwargs = dict(instance=models.Email.objects.order_by("-id")[0]) except IndexError: kwargs = {} form = forms.EmailForm(request.POST, **kwargs) if not form.is_valid(): return JsonResp(request, form=form) email = bsdUsers.objects.get(bsdusr_username="******").bsdusr_email if not email: return JsonResp(request, error=True, message=_("You must configure the root email" " (Accounts->Users->root)")) sid = transaction.savepoint() form.save() error = False if request.is_ajax(): sw_name = get_sw_name() error, errmsg = send_mail( subject=_("Test message from %s" % (sw_name)), text=_("This is a message test from %s" % (sw_name,)) ) if error: errmsg = _("Your test email could not be sent: %s") % errmsg else: errmsg = _("Your test email has been sent!") transaction.savepoint_rollback(sid) return JsonResp(request, error=error, message=errmsg)
def send(self, message): """ Sends mail using configured mail settings. """ # TODO: For now this is just a wrapper for freenasUI send_mail, # when the time comes we will do the reverse, logic here # and calling this method from freenasUI. return send_mail(**message)
def email(self, alerts): dismisseds = [ a.message_id for a in mAlert.objects.filter(dismiss=True) ] msgs = [] for alert in alerts: if alert.getId() not in dismisseds: msgs.append(unicode(alert).encode('utf8')) if len(msgs) == 0: return hostname = socket.gethostname() send_mail(subject='%s: %s' % ( hostname, _("Critical Alerts").encode('utf8'), ), text='\n'.join(msgs))
def testmail(request): try: kwargs = dict(instance=models.Email.objects.order_by('-id')[0]) except IndexError: kwargs = {} fromwizard = False data = request.POST.copy() for key, value in data.items(): if key.startswith('system-'): fromwizard = True data[key.replace('system-', '')] = value form = forms.EmailForm(data, **kwargs) if not form.is_valid(): return JsonResp(request, form=form) if fromwizard: allfield = 'system-__all__' else: allfield = '__all__' if fromwizard: email = request.POST.get('system-sys_email') errmsg = _('You must provide a Root E-mail') else: email = bsdUsers.objects.get(bsdusr_username='******').bsdusr_email errmsg = _('You must configure the root email (Accounts->Users->root)') if not email: form.errors[allfield] = form.error_class([errmsg]) return JsonResp( request, form=form, ) sid = transaction.savepoint() form.save() error = False if request.is_ajax(): sw_name = get_sw_name() error, errmsg = send_mail( subject=_('Test message from %s') % sw_name, text=_('This is a message test from %s') % sw_name, to=[email]) if error: errmsg = _("Your test email could not be sent: %s") % errmsg else: errmsg = _('Your test email has been sent!') transaction.savepoint_rollback(sid) form.errors[allfield] = form.error_class([errmsg]) return JsonResp( request, form=form, )
def testmail(request): try: kwargs = dict(instance=models.Email.objects.order_by('-id')[0]) except IndexError: kwargs = {} fromwizard = False data = request.POST.copy() for key, value in data.items(): if key.startswith('system-'): fromwizard = True data[key.replace('system-', '')] = value form = forms.EmailForm(data, **kwargs) if not form.is_valid(): return JsonResp(request, form=form) if fromwizard: allfield = 'system-__all__' else: allfield = '__all__' if fromwizard: email = request.POST.get('system-sys_email') errmsg = _('You must provide a Root E-mail') else: email = bsdUsers.objects.get(bsdusr_username='******').bsdusr_email errmsg = _('You must configure the root email (Accounts->Users->root)') if not email: form.errors[allfield] = form.error_class([errmsg]) return JsonResp( request, form=form, ) sid = transaction.savepoint() form.save() error = False if request.is_ajax(): sw_name = get_sw_name() error, errmsg = send_mail(subject=_('Test message from %s') % sw_name, text=_('This is a message test from %s') % sw_name, to=[email]) if error: errmsg = _("Your test email could not be sent: %s") % errmsg else: errmsg = _('Your test email has been sent!') transaction.savepoint_rollback(sid) form.errors[allfield] = form.error_class([errmsg]) return JsonResp( request, form=form, )
def email(self, alerts): dismisseds = [a.message_id for a in mAlert.objects.filter(dismiss=True)] msgs = [] for alert in alerts: if alert.getId() not in dismisseds: msgs.append(unicode(alert).encode('utf8')) if len(msgs) == 0: return hostname = socket.gethostname() send_mail( subject='%s: %s' % ( hostname, _("Critical Alerts").encode('utf8'), ), text='\n'.join(msgs) )
def email(self, alerts): dismisseds = [a.message_id for a in mAlert.objects.filter(dismiss=True)] msgs = [] for alert in alerts: if alert.getId() not in dismisseds: msgs.append(unicode(alert).encode('utf8')) if len(msgs) == 0: return hostname = socket.gethostname() UUID = subprocess.check_output("dmidecode | grep UUID | sed 's/.*: //'", shell=True) send_mail( subject='%s: %s' % ( hostname, _("Critical Alerts").encode('utf8'), ), text='\n UUID: %s \n'.join(msgs) % (UUID) )
last_snapshot, state = output.split("\n")[0].split("\t") log.info("Marking %s as latest snapshot" % (last_snapshot)) if state == "-": system("/sbin/zfs inherit freenas:state %s" % (known_latest_snapshot)) system("/sbin/zfs release -r freenas:repl %s" % (snapshot)) system("/sbin/zfs set freenas:state=LATEST %s" % (last_snapshot)) known_latest_snapshot = last_snapshot else: log.warn("Can not locate a proper local snapshot for %s" % (localfs)) # Can NOT proceed any further. Report this situation. error, errmsg = send_mail( subject="Replication failed! (%s)" % remote, text=""" Hello, The replication failed for the local ZFS %s because the remote system has diverged snapshots with us. """ % (localfs), interval=datetime.timedelta(hours=2), channel="autorepl", ) MNTLOCK.unlock() results[replication.id] = "Remote system has diverged snapshots with us" continue MNTLOCK.unlock() elif sshproc.returncode == 0: log.log(logging.NOTICE, "Can not locate %s on remote system, starting from there" % (known_latest_snapshot)) # Reset the "latest" snapshot to a new one. system("/sbin/zfs set freenas:state=NEW %s" % (known_latest_snapshot)) system("/sbin/zfs hold -r freenas:repl %s" % (known_latest_snapshot)) wanted_list.insert(0, known_latest_snapshot)
try: with LockFile(VMWARE_FAILS) as lock: with open(VMWARE_FAILS, 'rb') as f: fails = pickle.load(f) except: fails = {} fails[snapname] = [str(vm) for vm in snapvmfails] with LockFile(VMWARE_FAILS) as lock: with open(VMWARE_FAILS, 'wb') as f: pickle.dump(fails, f) send_mail( subject="VMware Snapshot failed! (%s)" % snapname, text=""" Hello, The following VM failed to snapshot %s: %s """ % (snapname, ' \n'.join([str(vm) for vm in snapvmfails])), channel='snapvmware' ) if len(snapvms) > 0 and len(snapvmfails) == 0: vmflag = '-o freenas:vmsynced=Y ' else: vmflag = '' # If there is associated replication task, mark the snapshots as 'NEW'. if Replication.objects.filter(repl_filesystem=fs, repl_enabled=True).count() > 0: MNTLOCK.lock() snapcmd = '/sbin/zfs snapshot%s -o freenas:state=NEW %s%s' % (rflag, vmflag, snapname) proc = pipeopen(snapcmd, logger=log)
for snapshot in snaplist: rzfscmd = "\"zfs destroy '%s'\"" % (snapshot) sshproc = pipeopen("%s %s" % (sshcmd, rzfscmd)) output, error = sshproc.communicate() if sshproc.returncode: log.warn("Unable to destroy snapshot %s on remote system" % (snapshot)) failed_snapshots.append(snapshot) if len(failed_snapshots) > 0: # We can't proceed in this situation, report error, errmsg = send_mail( subject="Replication failed! (%s)" % remote, text=""" Hello, The replication failed for the local ZFS %s because the remote system has diverged snapshots with us and we were unable to remove them, including: %s """ % (localfs, failed_snapshots), interval=datetime.timedelta(hours=2), channel="autorepl", ) results[replication.id] = "Unable to destroy remote snapshot: %s" % (failed_snapshots) ### rzfs destroy %s psnap = tasklist[1] success = sendzfs(None, psnap, dataset, localfs, remotefs_final, throttle, replication) if success: for nsnap in tasklist[2:]: success = sendzfs(psnap, nsnap, dataset, localfs, remotefs_final, throttle, replication) if not success: # Report the situation
rzfscmd = '"zfs destroy \'%s\'"' % (snapshot) sshproc = pipeopen('%s %s' % (sshcmd, rzfscmd)) output, error = sshproc.communicate() if sshproc.returncode: log.warn( "Unable to destroy snapshot %s on remote system" % (snapshot)) failed_snapshots.append(snapshot) if len(failed_snapshots) > 0: # We can't proceed in this situation, report error, errmsg = send_mail( subject="Replication failed! (%s)" % remote, text=""" Hello, The replication failed for the local ZFS %s because the remote system has diverged snapshots with us and we were unable to remove them, including: %s """ % (localfs, failed_snapshots), interval=datetime.timedelta(hours=2), channel='autorepl') results[replication. id] = 'Unable to destroy remote snapshot: %s' % ( failed_snapshots) ### rzfs destroy %s psnap = tasklist[1] success = sendzfs(None, psnap, dataset, localfs, remotefs_final, throttle, replication) if success: for nsnap in tasklist[2:]: success = sendzfs(psnap, nsnap, dataset, localfs,
else: vmflag = '' # Take the ZFS snapshot MNTLOCK.lock() snapcmd = 'zfs snapshot%s %s"%s"' % (rflag, vmflag, snapname) proc = pipeopen(snapcmd, logger=log) err = proc.communicate()[1] MNTLOCK.unlock() if proc.returncode != 0: log.error("Failed to create snapshot '%s': %s", snapname, err) send_mail( subject="Snapshot failed! (%s)" % snapname, text=""" Hello, Snapshot %s failed with the following error: %s""" % (snapname, err), interval=timedelta(hours=1), channel='autosnap', ) # Delete all the VMWare snapshots we just took. # This can be a list instead of a dict because we really don't care # which VMWare task the failed deletion was fron. snapdeletefails = [] for vmsnapobj in qs: try: ssl_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) ssl_context.verify_mode = ssl.CERT_NONE si = connect.SmartConnect(host=vmsnapobj.hostname,
else: if output != '': if output.find('off') == -1: may_proceed = True if not may_proceed: # Report the problem and continue results[replication.id] = 'Remote destination must be set readonly' log.debug("dataset %s and it's children must be readonly." % remotefs_final) if ("on" in output or "off" in output) and len(output) > 0: error, errmsg = send_mail( subject="Replication denied! (%s)" % remote, text=""" Hello, The remote system have denied our replication from local ZFS %s to remote ZFS %s. Please change the 'readonly' property of: %s as well as its children to 'on' to allow receiving replication. """ % (localfs, remotefs_final, remotefs_final), interval=datetime.timedelta(hours=24), channel='autorepl') else: if len(output) > 0: error, errmsg = send_mail( subject="Replication failed! (%s)" % remote, text=""" Hello, Replication of local ZFS %s to remote ZFS %s failed.""" % (localfs, remotefs_final), interval=datetime.timedelta(hours=24), channel='autorepl')
fails = pickle.load(f) except: fails = {} # vmitem.get_property('path') is the reverse of server.get_vm_by_path(vm) fails[snapname] = [ vmitem.get_property('path') for vmitem in snapvmfails[vmsnapobj] ] with LockFile(VMWARE_FAILS) as lock: with open(VMWARE_FAILS, 'wb') as f: pickle.dump(fails, f) send_mail(subject="VMware Snapshot failed! (%s)" % snapname, text=""" Hello, The following VM failed to snapshot %s: %s """ % (snapname, ' \n'.join(fails[snapname])), channel='snapvmware') # At this point we have finished sending alerts out # If there were no failures and we successfully took some VMWare snapshots # set the ZFS property to show the snapshot has consistent VM snapshots # inside it. if qs: sentinel = True for vmsnapobj in qs: if not (len(snapvms[vmsnapobj]) > 0 and len(snapvmfails[vmsnapobj]) == 0): sentinel = False
log.info("Marking %s as latest snapshot" % (last_snapshot)) if state == '-': system('/sbin/zfs inherit freenas:state %s' % (known_latest_snapshot)) system('/sbin/zfs release -r freenas:repl %s' % (snapshot)) system('/sbin/zfs set freenas:state=LATEST %s' % (last_snapshot)) known_latest_snapshot = last_snapshot else: log.warn("Can not locate a proper local snapshot for %s" % (localfs)) # Can NOT proceed any further. Report this situation. error, errmsg = send_mail(subject="Replication failed!", text=\ """ Hello, The replication failed for the local ZFS %s because the remote system has diverged snapshots with us. """ % (localfs), interval=datetime.timedelta(hours=2), channel='autorepl') MNTLOCK.unlock() results[ replication. id] = 'Remote system has diverged snapshots with us' continue MNTLOCK.unlock() elif sshproc.returncode == 0: log.log( logging.NOTICE, "Can not locate %s on remote system, starting from there" % (known_latest_snapshot)) # Reset the "latest" snapshot to a new one. system('/sbin/zfs set freenas:state=NEW %s' %
if output != '': last_snapshot, state = output.split('\n')[0].split('\t') log.info("Marking %s as latest snapshot" % (last_snapshot)) if state == '-': system('/sbin/zfs inherit freenas:state %s' % (known_latest_snapshot)) system('/sbin/zfs set freenas:state=LATEST %s' % (last_snapshot)) known_latest_snapshot = last_snapshot else: log.warn("Can not locate a proper local snapshot for %s" % (localfs)) # Can NOT proceed any further. Report this situation. error, errmsg = send_mail(subject="Replication failed!", text=\ """ Hello, The replication failed for the local ZFS %s because the remote system have diverged snapshot with us. """ % (localfs), interval=datetime.timedelta(hours=2), channel='autorepl') MNTLOCK.unlock() continue MNTLOCK.unlock() else: log.log( logging.NOTICE, "Can not locate %s on remote system, starting from there" % (known_latest_snapshot)) # Reset the "latest" snapshot to a new one. system('/sbin/zfs set freenas:state=NEW %s' % (known_latest_snapshot)) wanted_list.insert(0, known_latest_snapshot) last_snapshot = ''
try: with LockFile(VMWARE_FAILS) as lock: with open(VMWARE_FAILS, 'rb') as f: fails = pickle.load(f) except: fails = {} fails[snapname] = [str(vm) for vm in snapvmfails] with LockFile(VMWARE_FAILS) as lock: with open(VMWARE_FAILS, 'wb') as f: pickle.dump(fails, f) send_mail( subject="VMware Snapshot failed! (%s)" % snapname, text=""" Hello, The following VM failed to snapshot %s: %s """ % (snapname, ' \n'.join([str(vm) for vm in snapvmfails])), channel='snapvmware' ) if len(snapvms) > 0 and len(snapvmfails) == 0: vmflag = '-o freenas:vmsynced=Y ' else: vmflag = '' # If there is associated replication task, mark the snapshots as 'NEW'. if Replication.objects.filter(repl_filesystem=fs, repl_enabled=True).count() > 0: MNTLOCK.lock() snapcmd = '/sbin/zfs snapshot%s %s"%s"' % (rflag, vmflag, snapname) proc = pipeopen(snapcmd, logger=log)
try: with LockFile(VMWARE_FAILS) as lock: with open(VMWARE_FAILS, 'rb') as f: fails = pickle.load(f) except: fails = {} fails[snapname] = [vm.get_property('path') for vm in snapvmfails] with LockFile(VMWARE_FAILS) as lock: with open(VMWARE_FAILS, 'wb') as f: pickle.dump(fails, f) send_mail( subject="VMware Snapshot failed! (%s)" % snapname, text=""" Hello, The following VM failed to snapshot %s: %s """ % (snapname, ' \n'.join([vm.get_property('path') for vm in snapvmfails])), channel='snapvmware' ) if len(snapvms) > 0 and len(snapvmfails) == 0: vmflag = '-o freenas:vmsynced=Y ' else: vmflag = '' # If there is associated replication task, mark the snapshots as 'NEW'. if Replication.objects.filter(repl_filesystem=fs, repl_enabled=True).count() > 0: MNTLOCK.lock() snapcmd = '/sbin/zfs snapshot%s %s"%s"' % (rflag, vmflag, snapname) proc = pipeopen(snapcmd, logger=log)
with LockFile(VMWARE_FAILS) as lock: with open(VMWARE_FAILS, 'rb') as f: fails = pickle.load(f) except: fails = {} # vmitem.get_property('path') is the reverse of server.get_vm_by_path(vm) fails[snapname] = [f'{i[1]}: {i[2]}' for i in snapvmfails[vmsnapobj]] with LockFile(VMWARE_FAILS) as lock: with open(VMWARE_FAILS, 'wb') as f: pickle.dump(fails, f) send_mail( subject="VMware Snapshot failed! (%s)" % snapname, text=""" Hello, The following VM failed to snapshot %s: %s """ % (snapname, ' \n'.join(fails[snapname])), channel='snapvmware' ) # At this point we have finished sending alerts out # If there were no failures and we successfully took some VMWare snapshots # set the ZFS property to show the snapshot has consistent VM snapshots # inside it. if qs: sentinel = True for vmsnapobj in qs: if not (len(snapvms[vmsnapobj]) > 0 and len(snapvmfails[vmsnapobj]) == 0): sentinel = False
MNTLOCK.lock() zfsproc = pipeopen('/sbin/zfs list -Ht snapshot -o name,freenas:state %s' % (expected_local_snapshot), debug) output = zfsproc.communicate()[0] if output != '': last_snapshot, state = output.split('\n')[0].split('\t') log.info("Marking %s as latest snapshot" % (last_snapshot)) if state == '-': system('/sbin/zfs inherit freenas:state %s' % (known_latest_snapshot)) system('/sbin/zfs set freenas:state=LATEST %s' % (last_snapshot)) known_latest_snapshot = last_snapshot else: log.warn("Can not locate a proper local snapshot for %s" % (localfs)) # Can NOT proceed any further. Report this situation. error, errmsg = send_mail(subject="Replication failed!", text=\ """ Hello, The replication failed for the local ZFS %s because the remote system have diverged snapshot with us. """ % (localfs), interval=datetime.timedelta(hours=2), channel='autorepl') MNTLOCK.unlock() continue MNTLOCK.unlock() else: log.log(logging.NOTICE, "Can not locate %s on remote system, starting from there" % (known_latest_snapshot)) # Reset the "latest" snapshot to a new one. system('/sbin/zfs set freenas:state=NEW %s' % (known_latest_snapshot)) wanted_list.insert(0, known_latest_snapshot) last_snapshot = '' known_latest_snapshot = '' if resetonce: log.log(logging.NOTICE, "Destroying remote %s" % (remotefs_final)) destroycmd = '%s -p %d %s /sbin/zfs destroy -rRf %s' % (sshcmd, remote_port, remote, remotefs_final)
if sshproc.returncode: # Be conservative: only consider it's Okay when we see the expected result. if error != '': if output.split('\n')[0] == ("cannot open '%s': dataset does not exist" % (remotefs_final)): may_proceed = True else: if output != '': if output.find('off') == -1: may_proceed = True if not may_proceed: # Report the problem and continue error, errmsg = send_mail( subject="Replication denied! (%s)" % remote, text=""" Hello, The remote system have denied our replication from local ZFS %s to remote ZFS %s. Please change the 'readonly' property of: %s as well as its children to 'on' to allow receiving replication. """ % (localfs, remotefs_final, remotefs_final), interval=datetime.timedelta(hours=24), channel='autorepl') results[replication.id] = 'Remote system denied receiving of snapshot on %s' % (remotefs_final) continue # Remote filesystem is the root dataset # Make sure it has no .system dataset over there because zfs receive will try to # remove it and fail (because its mounted and being used) if '/' not in remotefs_final: rzfscmd = '"mount | grep ^%s/.system"' % (remotefs_final) sshproc = pipeopen('%s %s' % (sshcmd, rzfscmd), debug) output = sshproc.communicate()[0].strip() if output != '':