def email(self, alerts):
node = alert_node()
dismisseds = [a.message_id for a in mAlert.objects.filter(node=node)]
msgs = []
for alert in alerts:
if alert.getId() not in dismisseds:
"""
This is all to allow <a> tags in alert messages.
We need to strip out all the tags so we can send a
plain text email.
"""
msg = str(alert)
msgnode = etree.fromstring('<msg>{}</msg>'.format(msg))
for i in msgnode.xpath('//a'):
new = etree.Element('span')
new.text = '{} ({})'.format(i.text, i.attrib['href'])
msgnode.replace(i, new)
etree.strip_tags(msgnode, '*')
msgs.append(msgnode.text)
if len(msgs) == 0:
return
hostname = socket.gethostname()
send_mail(
subject='%s: %s' % (
hostname,
_("Critical Alerts"),
),
text='\n'.join(msgs)
)
def do_sendmail(msg, to_addrs=None, parse_recipients=False):
if to_addrs is None:
if not parse_recipients:
raise ValueError('Do not know who to send the message to.')
to_addrs = []
# XXX: this should probably be a FeedParser because reading from sys.stdin
# is blocking.
em_parser = email.parser.Parser()
em = em_parser.parsestr(msg)
if parse_recipients:
# Strip away the comma based delimiters and whitespace.
to_addrs = map(str.strip, em.get('To').split(','))
if not to_addrs or not to_addrs[0]:
to_addrs = ['root']
if to_addrs:
aliases = get_aliases()
to_addrs_repl = []
for to_addr in to_addrs:
for to_addr in to_addr.split(','):
if to_addr.find('@') == -1 and to_addr in aliases:
to_addr = aliases[to_addr]
to_addrs_repl.append(to_addr)
margs = {}
margs['extra_headers'] = dict(em)
margs['extra_headers'].update({
'X-Mailer': get_sw_name(),
'X-%s-Host' % get_sw_name(): socket.gethostname(),
'To': to_addr,
})
margs['subject'] = em.get('Subject')
# abusive use of querysets
lemail = Email.objects.all()
for obj in lemail:
if obj.em_fromemail != '':
margs['extra_headers'].update({
'From': obj.em_fromemail
})
if em.is_multipart():
margs['attachments'] = filter(
lambda part: part.get_content_maintype() != 'multipart',
em.walk()
)
margs['text'] = u"%s" % _(
'This is a MIME formatted message. If you see '
'this text it means that your email software '
'does not support MIME formatted messages.')
else:
margs['text'] = ''.join(email.iterators.body_line_iterator(em))
if to_addrs_repl:
margs['to'] = to_addrs_repl
send_mail(**margs)
def email(self):
"""
Use alert.last to hold a sha256 hash of the last sent alerts
If the hash is the same do not resend the email
"""
dismisseds = [a.message_id
for a in mAlert.objects.filter(dismiss=True)
]
msgs = []
for msg in self.__logs[self.LOG_CRIT]:
if msg._msgid not in dismisseds:
msgs.append(unicode(msg))
if len(msgs) == 0:
if os.path.exists(LAST_ALERT_FILE):
os.unlink(LAST_ALERT_FILE)
return
try:
with open(LAST_ALERT_FILE) as f:
sha256 = f.read()
except:
sha256 = ''
newsha = hashlib.sha256(repr(msgs)).hexdigest()
if newsha != sha256:
send_mail(subject=_("Critical Alerts"),
text='\n'.join(msgs))
with open(LAST_ALERT_FILE, 'w') as f:
f.write(newsha)
def email(dataset, threshold, avail):
send_mail(subject="Volume threshold",
text="""Hi,
Your volume %s has reached the threshold of %s.
Currently there is %s of available space.
""" % (dataset, threshold, avail))
def email(self, alerts):
node = alert_node()
dismisseds = [a.message_id for a in mAlert.objects.filter(node=node)]
msgs = []
for alert in alerts:
if alert.getId() not in dismisseds:
"""
This is all to allow <a> tags in alert messages.
We need to strip out all the tags so we can send a
plain text email.
"""
msg = str(alert)
msgnode = etree.fromstring('<msg>{}</msg>'.format(msg))
for i in msgnode.xpath('//a'):
new = etree.Element('span')
new.text = '{} ({})'.format(i.text, i.attrib['href'])
msgnode.replace(i, new)
etree.strip_tags(msgnode, '*')
msgs.append(msgnode.text)
if len(msgs) == 0:
return
hostname = socket.gethostname()
send_mail(subject='%s: %s' % (
hostname,
_("Critical Alerts"),
),
text='\n'.join(msgs))
def email(dataset, threshold, avail):
send_mail(subject="Volume threshold",
text="""Hi,
Your volume %s has reached the threshold of %s.
Currently there is %s of available space.
""" % (dataset, threshold, avail))
def email(self):
"""
Use alert.last to hold a sha256 hash of the last sent alerts
If the hash is the same do not resend the email
"""
dismisseds = [
a.message_id for a in mAlert.objects.filter(dismiss=True)
]
msgs = []
for msg in self.__logs[self.LOG_CRIT]:
if msg._msgid not in dismisseds:
msgs.append(unicode(msg))
if len(msgs) == 0:
if os.path.exists(LAST_ALERT_FILE):
os.unlink(LAST_ALERT_FILE)
return
try:
with open(LAST_ALERT_FILE) as f:
sha256 = f.read()
except:
sha256 = ''
newsha = hashlib.sha256(repr(msgs)).hexdigest()
if newsha != sha256:
send_mail(subject=_("Critical Alerts"), text='\n'.join(msgs))
with open(LAST_ALERT_FILE, 'w') as f:
f.write(newsha)
def do_sendmail(msg, to_addrs=None, parse_recipients=False):
if to_addrs is None:
if not parse_recipients:
raise ValueError('Do not know who to send the message to.')
to_addrs = []
# XXX: this should probably be a FeedParser because reading from sys.stdin
# is blocking.
em_parser = email.parser.Parser()
em = em_parser.parsestr(msg)
if parse_recipients:
# Strip away the comma based delimiters and whitespace.
to_addrs = map(str.strip, em.get('To').split(','))
if not to_addrs or not to_addrs[0]:
to_addrs = ['root']
if to_addrs:
aliases = get_aliases()
to_addrs_repl = []
for to_addr in to_addrs:
for to_addr in to_addr.split(','):
if to_addr.find('@') == -1 and to_addr in aliases:
to_addr = aliases[to_addr]
to_addrs_repl.append(to_addr)
margs = {}
margs['extra_headers'] = dict(em)
margs['extra_headers'].update({
'X-Mailer':
get_sw_name(),
'X-%s-Host' % get_sw_name():
socket.gethostname(),
'To':
to_addr,
})
margs['subject'] = em.get('Subject')
# abusive use of querysets
lemail = Email.objects.all()
for obj in lemail:
if obj.em_fromemail != '':
margs['extra_headers'].update({'From': obj.em_fromemail})
if em.is_multipart():
margs['attachments'] = filter(
lambda part: part.get_content_maintype() != 'multipart', em.walk())
margs['text'] = u"%s" % _(
'This is a MIME formatted message. If you see '
'this text it means that your email software '
'does not support MIME formatted messages.')
else:
margs['text'] = ''.join(email.iterators.body_line_iterator(em))
if to_addrs_repl:
margs['to'] = to_addrs_repl
send_mail(**margs)
def main():
try:
updateobj = mUpdate.objects.order_by('-id')[0]
except IndexError:
updateobj = mUpdate.objects.create()
if updateobj.upd_autocheck is False:
return
location = notifier().get_update_location()
Update.DownloadUpdate(updateobj.get_train(), location)
update = Update.CheckForUpdates(
train=updateobj.get_train(),
cache_dir=location,
)
if not update:
return
conf = Configuration.Configuration()
sys_mani = conf.SystemManifest()
if sys_mani:
sequence = sys_mani.Sequence()
else:
sequence = ''
changelog = get_changelog(
updateobj.get_train(),
start=sequence,
end=update.Sequence(),
)
hostname = socket.gethostname()
send_mail(
subject='%s: %s' % (
hostname,
_('Update Available'),
),
extra_headers={
'X-Mailer': get_sw_name(),
'X-%s-Host' % get_sw_name(): socket.gethostname()
},
text=_('''A new update is available for the %(train)s train.
Version: %(version)s
Changelog:
%(changelog)s
''') % {
'train': updateobj.get_train(),
'version': update.Version(),
'changelog': changelog,
},
)
def email(self, alerts):
dismisseds = [
a.message_id for a in mAlert.objects.filter(dismiss=True)
]
msgs = []
for alert in alerts:
if alert.getId() not in dismisseds:
msgs.append(unicode(alert))
if len(msgs) == 0:
return
send_mail(subject=_("Critical Alerts"), text='\n'.join(msgs))
def email(self, alerts):
dismisseds = [a.message_id
for a in mAlert.objects.filter(dismiss=True)]
msgs = []
for alert in alerts:
if alert.getId() not in dismisseds:
msgs.append(unicode(alert))
if len(msgs) == 0:
return
send_mail(subject=_("Critical Alerts"),
text='\n'.join(msgs))
def main():
try:
updateobj = mUpdate.objects.order_by('-id')[0]
except IndexError:
updateobj = mUpdate.objects.create()
if updateobj.upd_autocheck is False:
return
location = notifier().get_update_location()
Update.DownloadUpdate(updateobj.get_train(), location)
update = Update.CheckForUpdates(
train=updateobj.get_train(),
cache_dir=location,
)
if not update:
return
conf = Configuration.Configuration()
sys_mani = conf.SystemManifest()
if sys_mani:
sequence = sys_mani.Sequence()
else:
sequence = ''
changelog = get_changelog(
updateobj.get_train(),
start=sequence,
end=update.Sequence(),
)
hostname = socket.gethostname()
send_mail(
subject='%s: %s' % (
hostname,
_('Update Available'),
),
extra_headers={'X-Mailer': get_sw_name(),
'X-%s-Host' % get_sw_name(): socket.gethostname()},
text=_('''A new update is available for the %(train)s train.
Changelog:
%(changelog)s
''') % {
'train': updateobj.get_train(),
'changelog': changelog,
},
)
def testmail(request):
try:
kwargs = dict(instance=models.Email.objects.order_by('-id')[0])
except IndexError:
kwargs = {}
form = forms.EmailForm(request.POST, **kwargs)
if not form.is_valid():
return JsonResp(request, form=form)
email = bsdUsers.objects.get(bsdusr_username='******').bsdusr_email
if not email:
return JsonResp(
request,
error=True,
message=_(
"You must configure the root email (Accounts->Users->root)"),
)
sid = transaction.savepoint()
form.save()
error = False
if request.is_ajax():
sw_name = get_sw_name()
error, errmsg = send_mail(
subject=_('Test message from %s' % (sw_name)),
text=_('This is a message test from %s' % (sw_name, )))
if error:
errmsg = _("Your test email could not be sent: %s") % errmsg
else:
errmsg = _('Your test email has been sent!')
transaction.savepoint_rollback(sid)
return JsonResp(request, error=error, message=errmsg)
def testmail(request):
try:
kwargs = dict(instance=models.Email.objects.order_by("-id")[0])
except IndexError:
kwargs = {}
form = forms.EmailForm(request.POST, **kwargs)
if not form.is_valid():
return JsonResp(request, form=form)
email = bsdUsers.objects.get(bsdusr_username="******").bsdusr_email
if not email:
return JsonResp(request, error=True, message=_("You must configure the root email" " (Accounts->Users->root)"))
sid = transaction.savepoint()
form.save()
error = False
if request.is_ajax():
sw_name = get_sw_name()
error, errmsg = send_mail(
subject=_("Test message from %s" % (sw_name)), text=_("This is a message test from %s" % (sw_name,))
)
if error:
errmsg = _("Your test email could not be sent: %s") % errmsg
else:
errmsg = _("Your test email has been sent!")
transaction.savepoint_rollback(sid)
return JsonResp(request, error=error, message=errmsg)
def send(self, message):
"""
Sends mail using configured mail settings.
"""
# TODO: For now this is just a wrapper for freenasUI send_mail,
# when the time comes we will do the reverse, logic here
# and calling this method from freenasUI.
return send_mail(**message)
def email(self, alerts):
dismisseds = [
a.message_id for a in mAlert.objects.filter(dismiss=True)
]
msgs = []
for alert in alerts:
if alert.getId() not in dismisseds:
msgs.append(unicode(alert).encode('utf8'))
if len(msgs) == 0:
return
hostname = socket.gethostname()
send_mail(subject='%s: %s' % (
hostname,
_("Critical Alerts").encode('utf8'),
),
text='\n'.join(msgs))
def send(self, message):
"""
Sends mail using configured mail settings.
"""
# TODO: For now this is just a wrapper for freenasUI send_mail,
# when the time comes we will do the reverse, logic here
# and calling this method from freenasUI.
return send_mail(**message)
def testmail(request):
try:
kwargs = dict(instance=models.Email.objects.order_by('-id')[0])
except IndexError:
kwargs = {}
fromwizard = False
data = request.POST.copy()
for key, value in data.items():
if key.startswith('system-'):
fromwizard = True
data[key.replace('system-', '')] = value
form = forms.EmailForm(data, **kwargs)
if not form.is_valid():
return JsonResp(request, form=form)
if fromwizard:
allfield = 'system-__all__'
else:
allfield = '__all__'
if fromwizard:
email = request.POST.get('system-sys_email')
errmsg = _('You must provide a Root E-mail')
else:
email = bsdUsers.objects.get(bsdusr_username='******').bsdusr_email
errmsg = _('You must configure the root email (Accounts->Users->root)')
if not email:
form.errors[allfield] = form.error_class([errmsg])
return JsonResp(
request,
form=form,
)
sid = transaction.savepoint()
form.save()
error = False
if request.is_ajax():
sw_name = get_sw_name()
error, errmsg = send_mail(
subject=_('Test message from %s') % sw_name,
text=_('This is a message test from %s') % sw_name,
to=[email])
if error:
errmsg = _("Your test email could not be sent: %s") % errmsg
else:
errmsg = _('Your test email has been sent!')
transaction.savepoint_rollback(sid)
form.errors[allfield] = form.error_class([errmsg])
return JsonResp(
request,
form=form,
)
def testmail(request):
try:
kwargs = dict(instance=models.Email.objects.order_by('-id')[0])
except IndexError:
kwargs = {}
fromwizard = False
data = request.POST.copy()
for key, value in data.items():
if key.startswith('system-'):
fromwizard = True
data[key.replace('system-', '')] = value
form = forms.EmailForm(data, **kwargs)
if not form.is_valid():
return JsonResp(request, form=form)
if fromwizard:
allfield = 'system-__all__'
else:
allfield = '__all__'
if fromwizard:
email = request.POST.get('system-sys_email')
errmsg = _('You must provide a Root E-mail')
else:
email = bsdUsers.objects.get(bsdusr_username='******').bsdusr_email
errmsg = _('You must configure the root email (Accounts->Users->root)')
if not email:
form.errors[allfield] = form.error_class([errmsg])
return JsonResp(
request,
form=form,
)
sid = transaction.savepoint()
form.save()
error = False
if request.is_ajax():
sw_name = get_sw_name()
error, errmsg = send_mail(subject=_('Test message from %s') % sw_name,
text=_('This is a message test from %s') %
sw_name,
to=[email])
if error:
errmsg = _("Your test email could not be sent: %s") % errmsg
else:
errmsg = _('Your test email has been sent!')
transaction.savepoint_rollback(sid)
form.errors[allfield] = form.error_class([errmsg])
return JsonResp(
request,
form=form,
)
def email(self, alerts):
dismisseds = [a.message_id
for a in mAlert.objects.filter(dismiss=True)]
msgs = []
for alert in alerts:
if alert.getId() not in dismisseds:
msgs.append(unicode(alert).encode('utf8'))
if len(msgs) == 0:
return
hostname = socket.gethostname()
send_mail(
subject='%s: %s' % (
hostname,
_("Critical Alerts").encode('utf8'),
),
text='\n'.join(msgs)
)
def email(self, alerts):
dismisseds = [a.message_id
for a in mAlert.objects.filter(dismiss=True)]
msgs = []
for alert in alerts:
if alert.getId() not in dismisseds:
msgs.append(unicode(alert).encode('utf8'))
if len(msgs) == 0:
return
hostname = socket.gethostname()
UUID = subprocess.check_output("dmidecode | grep UUID | sed 's/.*: //'", shell=True)
send_mail(
subject='%s: %s' % (
hostname,
_("Critical Alerts").encode('utf8'),
),
text='\n UUID: %s \n'.join(msgs) % (UUID)
)
last_snapshot, state = output.split("\n")[0].split("\t")
log.info("Marking %s as latest snapshot" % (last_snapshot))
if state == "-":
system("/sbin/zfs inherit freenas:state %s" % (known_latest_snapshot))
system("/sbin/zfs release -r freenas:repl %s" % (snapshot))
system("/sbin/zfs set freenas:state=LATEST %s" % (last_snapshot))
known_latest_snapshot = last_snapshot
else:
log.warn("Can not locate a proper local snapshot for %s" % (localfs))
# Can NOT proceed any further. Report this situation.
error, errmsg = send_mail(
subject="Replication failed! (%s)" % remote,
text="""
Hello,
The replication failed for the local ZFS %s because the remote system
has diverged snapshots with us.
"""
% (localfs),
interval=datetime.timedelta(hours=2),
channel="autorepl",
)
MNTLOCK.unlock()
results[replication.id] = "Remote system has diverged snapshots with us"
continue
MNTLOCK.unlock()
elif sshproc.returncode == 0:
log.log(logging.NOTICE, "Can not locate %s on remote system, starting from there" % (known_latest_snapshot))
# Reset the "latest" snapshot to a new one.
system("/sbin/zfs set freenas:state=NEW %s" % (known_latest_snapshot))
system("/sbin/zfs hold -r freenas:repl %s" % (known_latest_snapshot))
wanted_list.insert(0, known_latest_snapshot)
try:
with LockFile(VMWARE_FAILS) as lock:
with open(VMWARE_FAILS, 'rb') as f:
fails = pickle.load(f)
except:
fails = {}
fails[snapname] = [str(vm) for vm in snapvmfails]
with LockFile(VMWARE_FAILS) as lock:
with open(VMWARE_FAILS, 'wb') as f:
pickle.dump(fails, f)
send_mail(
subject="VMware Snapshot failed! (%s)" % snapname,
text="""
Hello,
The following VM failed to snapshot %s:
%s
""" % (snapname, ' \n'.join([str(vm) for vm in snapvmfails])),
channel='snapvmware'
)
if len(snapvms) > 0 and len(snapvmfails) == 0:
vmflag = '-o freenas:vmsynced=Y '
else:
vmflag = ''
# If there is associated replication task, mark the snapshots as 'NEW'.
if Replication.objects.filter(repl_filesystem=fs, repl_enabled=True).count() > 0:
MNTLOCK.lock()
snapcmd = '/sbin/zfs snapshot%s -o freenas:state=NEW %s%s' % (rflag, vmflag, snapname)
proc = pipeopen(snapcmd, logger=log)
for snapshot in snaplist:
rzfscmd = "\"zfs destroy '%s'\"" % (snapshot)
sshproc = pipeopen("%s %s" % (sshcmd, rzfscmd))
output, error = sshproc.communicate()
if sshproc.returncode:
log.warn("Unable to destroy snapshot %s on remote system" % (snapshot))
failed_snapshots.append(snapshot)
if len(failed_snapshots) > 0:
# We can't proceed in this situation, report
error, errmsg = send_mail(
subject="Replication failed! (%s)" % remote,
text="""
Hello,
The replication failed for the local ZFS %s because the remote system
has diverged snapshots with us and we were unable to remove them,
including:
%s
"""
% (localfs, failed_snapshots),
interval=datetime.timedelta(hours=2),
channel="autorepl",
)
results[replication.id] = "Unable to destroy remote snapshot: %s" % (failed_snapshots)
### rzfs destroy %s
psnap = tasklist[1]
success = sendzfs(None, psnap, dataset, localfs, remotefs_final, throttle, replication)
if success:
for nsnap in tasklist[2:]:
success = sendzfs(psnap, nsnap, dataset, localfs, remotefs_final, throttle, replication)
if not success:
# Report the situation
rzfscmd = '"zfs destroy \'%s\'"' % (snapshot)
sshproc = pipeopen('%s %s' % (sshcmd, rzfscmd))
output, error = sshproc.communicate()
if sshproc.returncode:
log.warn(
"Unable to destroy snapshot %s on remote system" %
(snapshot))
failed_snapshots.append(snapshot)
if len(failed_snapshots) > 0:
# We can't proceed in this situation, report
error, errmsg = send_mail(
subject="Replication failed! (%s)" % remote,
text="""
Hello,
The replication failed for the local ZFS %s because the remote system
has diverged snapshots with us and we were unable to remove them,
including:
%s
""" % (localfs, failed_snapshots),
interval=datetime.timedelta(hours=2),
channel='autorepl')
results[replication.
id] = 'Unable to destroy remote snapshot: %s' % (
failed_snapshots)
### rzfs destroy %s
psnap = tasklist[1]
success = sendzfs(None, psnap, dataset, localfs, remotefs_final,
throttle, replication)
if success:
for nsnap in tasklist[2:]:
success = sendzfs(psnap, nsnap, dataset, localfs,
else:
vmflag = ''
# Take the ZFS snapshot
MNTLOCK.lock()
snapcmd = 'zfs snapshot%s %s"%s"' % (rflag, vmflag, snapname)
proc = pipeopen(snapcmd, logger=log)
err = proc.communicate()[1]
MNTLOCK.unlock()
if proc.returncode != 0:
log.error("Failed to create snapshot '%s': %s", snapname, err)
send_mail(
subject="Snapshot failed! (%s)" % snapname,
text="""
Hello,
Snapshot %s failed with the following error: %s""" % (snapname, err),
interval=timedelta(hours=1),
channel='autosnap',
)
# Delete all the VMWare snapshots we just took.
# This can be a list instead of a dict because we really don't care
# which VMWare task the failed deletion was fron.
snapdeletefails = []
for vmsnapobj in qs:
try:
ssl_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
ssl_context.verify_mode = ssl.CERT_NONE
si = connect.SmartConnect(host=vmsnapobj.hostname,
else:
if output != '':
if output.find('off') == -1:
may_proceed = True
if not may_proceed:
# Report the problem and continue
results[replication.id] = 'Remote destination must be set readonly'
log.debug("dataset %s and it's children must be readonly." %
remotefs_final)
if ("on" in output or "off" in output) and len(output) > 0:
error, errmsg = send_mail(
subject="Replication denied! (%s)" % remote,
text="""
Hello,
The remote system have denied our replication from local ZFS
%s to remote ZFS %s. Please change the 'readonly' property
of:
%s
as well as its children to 'on' to allow receiving replication.
""" % (localfs, remotefs_final, remotefs_final),
interval=datetime.timedelta(hours=24),
channel='autorepl')
else:
if len(output) > 0:
error, errmsg = send_mail(
subject="Replication failed! (%s)" % remote,
text="""
Hello,
Replication of local ZFS %s to remote ZFS %s failed.""" %
(localfs, remotefs_final),
interval=datetime.timedelta(hours=24),
channel='autorepl')
fails = pickle.load(f)
except:
fails = {}
# vmitem.get_property('path') is the reverse of server.get_vm_by_path(vm)
fails[snapname] = [
vmitem.get_property('path')
for vmitem in snapvmfails[vmsnapobj]
]
with LockFile(VMWARE_FAILS) as lock:
with open(VMWARE_FAILS, 'wb') as f:
pickle.dump(fails, f)
send_mail(subject="VMware Snapshot failed! (%s)" % snapname,
text="""
Hello,
The following VM failed to snapshot %s:
%s
""" % (snapname, ' \n'.join(fails[snapname])),
channel='snapvmware')
# At this point we have finished sending alerts out
# If there were no failures and we successfully took some VMWare snapshots
# set the ZFS property to show the snapshot has consistent VM snapshots
# inside it.
if qs:
sentinel = True
for vmsnapobj in qs:
if not (len(snapvms[vmsnapobj]) > 0
and len(snapvmfails[vmsnapobj]) == 0):
sentinel = False
log.info("Marking %s as latest snapshot" % (last_snapshot))
if state == '-':
system('/sbin/zfs inherit freenas:state %s' %
(known_latest_snapshot))
system('/sbin/zfs release -r freenas:repl %s' %
(snapshot))
system('/sbin/zfs set freenas:state=LATEST %s' %
(last_snapshot))
known_latest_snapshot = last_snapshot
else:
log.warn("Can not locate a proper local snapshot for %s" %
(localfs))
# Can NOT proceed any further. Report this situation.
error, errmsg = send_mail(subject="Replication failed!", text=\
"""
Hello,
The replication failed for the local ZFS %s because the remote system
has diverged snapshots with us.
""" % (localfs), interval=datetime.timedelta(hours=2), channel='autorepl')
MNTLOCK.unlock()
results[
replication.
id] = 'Remote system has diverged snapshots with us'
continue
MNTLOCK.unlock()
elif sshproc.returncode == 0:
log.log(
logging.NOTICE,
"Can not locate %s on remote system, starting from there" %
(known_latest_snapshot))
# Reset the "latest" snapshot to a new one.
system('/sbin/zfs set freenas:state=NEW %s' %
if output != '':
last_snapshot, state = output.split('\n')[0].split('\t')
log.info("Marking %s as latest snapshot" % (last_snapshot))
if state == '-':
system('/sbin/zfs inherit freenas:state %s' %
(known_latest_snapshot))
system('/sbin/zfs set freenas:state=LATEST %s' %
(last_snapshot))
known_latest_snapshot = last_snapshot
else:
log.warn("Can not locate a proper local snapshot for %s" %
(localfs))
# Can NOT proceed any further. Report this situation.
error, errmsg = send_mail(subject="Replication failed!", text=\
"""
Hello,
The replication failed for the local ZFS %s because the remote system
have diverged snapshot with us.
""" % (localfs), interval=datetime.timedelta(hours=2), channel='autorepl')
MNTLOCK.unlock()
continue
MNTLOCK.unlock()
else:
log.log(
logging.NOTICE,
"Can not locate %s on remote system, starting from there" %
(known_latest_snapshot))
# Reset the "latest" snapshot to a new one.
system('/sbin/zfs set freenas:state=NEW %s' %
(known_latest_snapshot))
wanted_list.insert(0, known_latest_snapshot)
last_snapshot = ''
try:
with LockFile(VMWARE_FAILS) as lock:
with open(VMWARE_FAILS, 'rb') as f:
fails = pickle.load(f)
except:
fails = {}
fails[snapname] = [str(vm) for vm in snapvmfails]
with LockFile(VMWARE_FAILS) as lock:
with open(VMWARE_FAILS, 'wb') as f:
pickle.dump(fails, f)
send_mail(
subject="VMware Snapshot failed! (%s)" % snapname,
text="""
Hello,
The following VM failed to snapshot %s:
%s
""" % (snapname, ' \n'.join([str(vm) for vm in snapvmfails])),
channel='snapvmware'
)
if len(snapvms) > 0 and len(snapvmfails) == 0:
vmflag = '-o freenas:vmsynced=Y '
else:
vmflag = ''
# If there is associated replication task, mark the snapshots as 'NEW'.
if Replication.objects.filter(repl_filesystem=fs, repl_enabled=True).count() > 0:
MNTLOCK.lock()
snapcmd = '/sbin/zfs snapshot%s %s"%s"' % (rflag, vmflag, snapname)
proc = pipeopen(snapcmd, logger=log)
try:
with LockFile(VMWARE_FAILS) as lock:
with open(VMWARE_FAILS, 'rb') as f:
fails = pickle.load(f)
except:
fails = {}
fails[snapname] = [vm.get_property('path') for vm in snapvmfails]
with LockFile(VMWARE_FAILS) as lock:
with open(VMWARE_FAILS, 'wb') as f:
pickle.dump(fails, f)
send_mail(
subject="VMware Snapshot failed! (%s)" % snapname,
text="""
Hello,
The following VM failed to snapshot %s:
%s
""" % (snapname, ' \n'.join([vm.get_property('path') for vm in snapvmfails])),
channel='snapvmware'
)
if len(snapvms) > 0 and len(snapvmfails) == 0:
vmflag = '-o freenas:vmsynced=Y '
else:
vmflag = ''
# If there is associated replication task, mark the snapshots as 'NEW'.
if Replication.objects.filter(repl_filesystem=fs, repl_enabled=True).count() > 0:
MNTLOCK.lock()
snapcmd = '/sbin/zfs snapshot%s %s"%s"' % (rflag, vmflag, snapname)
proc = pipeopen(snapcmd, logger=log)
with LockFile(VMWARE_FAILS) as lock:
with open(VMWARE_FAILS, 'rb') as f:
fails = pickle.load(f)
except:
fails = {}
# vmitem.get_property('path') is the reverse of server.get_vm_by_path(vm)
fails[snapname] = [f'{i[1]}: {i[2]}' for i in snapvmfails[vmsnapobj]]
with LockFile(VMWARE_FAILS) as lock:
with open(VMWARE_FAILS, 'wb') as f:
pickle.dump(fails, f)
send_mail(
subject="VMware Snapshot failed! (%s)" % snapname,
text="""
Hello,
The following VM failed to snapshot %s:
%s
""" % (snapname, ' \n'.join(fails[snapname])),
channel='snapvmware'
)
# At this point we have finished sending alerts out
# If there were no failures and we successfully took some VMWare snapshots
# set the ZFS property to show the snapshot has consistent VM snapshots
# inside it.
if qs:
sentinel = True
for vmsnapobj in qs:
if not (len(snapvms[vmsnapobj]) > 0 and len(snapvmfails[vmsnapobj]) == 0):
sentinel = False
MNTLOCK.lock()
zfsproc = pipeopen('/sbin/zfs list -Ht snapshot -o name,freenas:state %s' % (expected_local_snapshot), debug)
output = zfsproc.communicate()[0]
if output != '':
last_snapshot, state = output.split('\n')[0].split('\t')
log.info("Marking %s as latest snapshot" % (last_snapshot))
if state == '-':
system('/sbin/zfs inherit freenas:state %s' % (known_latest_snapshot))
system('/sbin/zfs set freenas:state=LATEST %s' % (last_snapshot))
known_latest_snapshot = last_snapshot
else:
log.warn("Can not locate a proper local snapshot for %s" % (localfs))
# Can NOT proceed any further. Report this situation.
error, errmsg = send_mail(subject="Replication failed!", text=\
"""
Hello,
The replication failed for the local ZFS %s because the remote system
have diverged snapshot with us.
""" % (localfs), interval=datetime.timedelta(hours=2), channel='autorepl')
MNTLOCK.unlock()
continue
MNTLOCK.unlock()
else:
log.log(logging.NOTICE, "Can not locate %s on remote system, starting from there" % (known_latest_snapshot))
# Reset the "latest" snapshot to a new one.
system('/sbin/zfs set freenas:state=NEW %s' % (known_latest_snapshot))
wanted_list.insert(0, known_latest_snapshot)
last_snapshot = ''
known_latest_snapshot = ''
if resetonce:
log.log(logging.NOTICE, "Destroying remote %s" % (remotefs_final))
destroycmd = '%s -p %d %s /sbin/zfs destroy -rRf %s' % (sshcmd, remote_port, remote, remotefs_final)
if sshproc.returncode:
# Be conservative: only consider it's Okay when we see the expected result.
if error != '':
if output.split('\n')[0] == ("cannot open '%s': dataset does not exist" % (remotefs_final)):
may_proceed = True
else:
if output != '':
if output.find('off') == -1:
may_proceed = True
if not may_proceed:
# Report the problem and continue
error, errmsg = send_mail(
subject="Replication denied! (%s)" % remote,
text="""
Hello,
The remote system have denied our replication from local ZFS
%s to remote ZFS %s. Please change the 'readonly' property
of:
%s
as well as its children to 'on' to allow receiving replication.
""" % (localfs, remotefs_final, remotefs_final), interval=datetime.timedelta(hours=24), channel='autorepl')
results[replication.id] = 'Remote system denied receiving of snapshot on %s' % (remotefs_final)
continue
# Remote filesystem is the root dataset
# Make sure it has no .system dataset over there because zfs receive will try to
# remove it and fail (because its mounted and being used)
if '/' not in remotefs_final:
rzfscmd = '"mount | grep ^%s/.system"' % (remotefs_final)
sshproc = pipeopen('%s %s' % (sshcmd, rzfscmd), debug)
output = sshproc.communicate()[0].strip()
if output != '':
