def run(self):
#0. mount destination share
dest_mnt_pt = ('%s/%s' % (settings.MNT_PT, self.dest_share))
msg = ('Destination share(%s) not mounted' % self.dest_share)
with self._update_trail_and_quit(msg):
if (not is_share_mounted(self.dest_share)):
mount_share(self.dest_share, self.dest_pool, dest_mnt_pt)
if (not is_share_mounted(self.dest_share)):
raise Exception(msg)
#1. mount source share
msg = ('Failed to mount source(%s:%s)' %
(self.source_ip, self.source_path))
with self._update_trail_and_quit(msg):
mount_source(self.source_ip, self.source_path)
#2. create a snapshot
snap_name = ('snap-%d' % self.tid)
msg = ('Failed to create snapshot(%s) for share(%s)' %
(snap_name, self.dest_share))
data = {
'status': 'snapshot created',
}
with self._update_trail_and_quit(msg, data=data):
create_snapshot(self.dest_share, snap_name, logger)
#3. rsync
src_mnt = ('/mnt/backup/%s_%s' % (self.source_ip, self.source_path))
cmd = [RSYNC, '-az', src_mnt, dest_mnt_pt]
msg = ('Unable to start sync')
data = {
'status': 'sync started',
}
with self._update_trail_and_quit(msg, data=data):
rp = subprocess.Popen(cmd,
shell=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
while True:
if (os.getppid() != self.ppid):
msg = ('Backup plugin scheduler exited. Aborting...')
with self._update_trail_and_quit(msg):
rp.terminate()
if (rp.poll() is not None):
msg = ('sync finished. But failed to update trail status')
data = {
'status': 'succeeded',
}
with self._update_trail_and_quit(msg, data):
logger.debug('sync finished')
break
time.sleep(1)
def post(self, request, command):
if (command == 'bootstrap'):
self._refresh_pool_state()
for p in Pool.objects.all():
import_shares(p, request)
for share in Share.objects.all():
try:
if (share.pqgroup == settings.MODEL_DEFS['pqgroup']):
share.pqgroup = qgroup_create(share.pool)
share.save()
if (not is_share_mounted(share.name)):
mnt_pt = ('%s%s' % (settings.MNT_PT, share.name))
mount_share(share, mnt_pt)
except Exception, e:
e_msg = ('Exception while mounting a share(%s) during '
'bootstrap: %s' % (share.name, e.__str__()))
logger.error(e_msg)
logger.exception(e)
try:
import_snapshots(share)
except Exception, e:
e_msg = ('Exception while importing Snapshots of '
'Share(%s): %s' % (share.name, e.__str__()))
logger.error(e_msg)
logger.exception(e)
def remove_share(pool, share_name, pqgroup, force=False):
"""
umount share if its mounted.
mount root pool
btrfs subvolume delete root_mnt/vol_name
umount root pool
"""
if (is_share_mounted(share_name)):
mnt_pt = ('%s%s' % (DEFAULT_MNT_DIR, share_name))
umount_root(mnt_pt)
root_pool_mnt = mount_root(pool)
subvol_mnt_pt = root_pool_mnt + '/' + share_name
if (not is_subvol(subvol_mnt_pt)):
return
if (force):
o, e, rc = run_command(
[BTRFS, 'subvolume', 'list', '-o', subvol_mnt_pt])
for l in o:
if (re.match('ID ', l) is not None):
subvol = root_pool_mnt + '/' + l.split()[-1]
run_command([BTRFS, 'subvolume', 'delete', subvol], log=True)
qgroup = ('0/%s' % share_id(pool, share_name))
delete_cmd = [BTRFS, 'subvolume', 'delete', subvol_mnt_pt]
run_command(delete_cmd, log=True)
qgroup_destroy(qgroup, root_pool_mnt)
return qgroup_destroy(pqgroup, root_pool_mnt)
def mount_root(pool):
device = pool.disk_set.first().name
root_pool_mnt = DEFAULT_MNT_DIR + pool.name
if (is_share_mounted(pool.name)):
return root_pool_mnt
create_tmp_dir(root_pool_mnt)
mnt_device = '/dev/disk/by-label/%s' % pool.name
if (not os.path.exists(mnt_device)):
mnt_device = device
if (re.match('/dev/', device) is None):
mnt_device = ('/dev/%s' % mnt_device)
mnt_cmd = [
MOUNT,
mnt_device,
root_pool_mnt,
]
mnt_options = ''
if (pool.mnt_options is not None):
mnt_options = pool.mnt_options
if (pool.compression is not None):
if (re.search('compress', mnt_options) is None):
mnt_options = ('%s,compress=%s' % (mnt_options, pool.compression))
if (len(mnt_options) > 0):
mnt_cmd.extend(['-o', mnt_options])
run_command(mnt_cmd)
return root_pool_mnt
def remove_share(pool, share_name, pqgroup, force=False):
"""
umount share if its mounted.
mount root pool
btrfs subvolume delete root_mnt/vol_name
umount root pool
"""
if (is_share_mounted(share_name)):
mnt_pt = ('%s%s' % (DEFAULT_MNT_DIR, share_name))
umount_root(mnt_pt)
root_pool_mnt = mount_root(pool)
subvol_mnt_pt = root_pool_mnt + '/' + share_name
if (not is_subvol(subvol_mnt_pt)):
return
if (force):
o, e, rc = run_command([BTRFS, 'subvolume', 'list', '-o', subvol_mnt_pt])
for l in o:
if (re.match('ID ', l) is not None):
subvol = root_pool_mnt + '/' + l.split()[-1]
run_command([BTRFS, 'subvolume', 'delete', subvol], log=True)
qgroup = ('0/%s' % share_id(pool, share_name))
delete_cmd = [BTRFS, 'subvolume', 'delete', subvol_mnt_pt]
run_command(delete_cmd, log=True)
qgroup_destroy(qgroup, root_pool_mnt)
return qgroup_destroy(pqgroup, root_pool_mnt)
def post(self, request, command):
if (command == 'bootstrap'):
self._refresh_pool_state()
for p in Pool.objects.all():
import_shares(p, request)
for share in Share.objects.all():
try:
if (share.pqgroup == settings.MODEL_DEFS['pqgroup']):
share.pqgroup = qgroup_create(share.pool)
share.save()
if (not is_share_mounted(share.name)):
mnt_pt = ('%s%s' % (settings.MNT_PT, share.name))
mount_share(share, mnt_pt)
except Exception, e:
e_msg = ('Exception while mounting a share(%s) during '
'bootstrap: %s' % (share.name, e.__str__()))
logger.error(e_msg)
logger.exception(e)
try:
import_snapshots(share)
except Exception, e:
e_msg = ('Exception while importing Snapshots of '
'Share(%s): %s' % (share.name, e.__str__()))
logger.error(e_msg)
logger.exception(e)
def remove_snap(pool_name, pool_device, share_name, snap_name):
full_name = ('%s/%s' % (share_name, snap_name))
if (is_share_mounted(full_name)):
umount_root('%s%s' % (DEFAULT_MNT_DIR, full_name))
root_pool_mnt = mount_root(pool_name, pool_device)
subvol_mnt_pt = ('%s/%s_%s' % (root_pool_mnt, share_name, snap_name))
return run_command([BTRFS, 'subvolume', 'delete', subvol_mnt_pt])
def remove_snap(pool_name, pool_device, share_name, snap_name):
full_name = ('%s/%s' % (share_name, snap_name))
if (is_share_mounted(full_name)):
umount_root('%s%s' % (DEFAULT_MNT_DIR, full_name))
root_pool_mnt = mount_root(pool_name, pool_device)
subvol_mnt_pt = ('%s/%s_%s' % (root_pool_mnt, share_name, snap_name))
return run_command([BTRFS, 'subvolume', 'delete', subvol_mnt_pt])
def mount_root(pool_name, device):
root_pool_mnt = DEFAULT_MNT_DIR + pool_name
if (is_share_mounted(pool_name)):
return root_pool_mnt
create_tmp_dir(root_pool_mnt)
mnt_cmd = [MOUNT, '-t', 'btrfs', device, root_pool_mnt]
run_command(mnt_cmd)
return root_pool_mnt
def mount_root(pool_name, device):
root_pool_mnt = DEFAULT_MNT_DIR + pool_name
if (is_share_mounted(pool_name)):
return root_pool_mnt
create_tmp_dir(root_pool_mnt)
mnt_cmd = [MOUNT, '-t', 'btrfs', device, root_pool_mnt]
run_command(mnt_cmd)
return root_pool_mnt
def run(self):
#0. mount destination share
dest_mnt_pt = ('%s/%s' % (settings.MNT_PT, self.dest_share))
msg = ('Destination share(%s) not mounted' % self.dest_share)
with self._update_trail_and_quit(msg):
if (not is_share_mounted(self.dest_share)):
mount_share(self.dest_share, self.dest_pool, dest_mnt_pt)
if (not is_share_mounted(self.dest_share)):
raise Exception(msg)
#1. mount source share
msg = ('Failed to mount source(%s:%s)' % (self.source_ip,
self.source_path))
with self._update_trail_and_quit(msg):
mount_source(self.source_ip, self.source_path)
#2. create a snapshot
snap_name = ('snap-%d' % self.tid)
msg = ('Failed to create snapshot(%s) for share(%s)' %
(snap_name, self.dest_share))
data = {'status': 'snapshot created',}
with self._update_trail_and_quit(msg, data=data):
create_snapshot(self.dest_share, snap_name, logger)
#3. rsync
src_mnt = ('/mnt/backup/%s_%s' % (self.source_ip, self.source_path))
cmd = [RSYNC, '-az', src_mnt, dest_mnt_pt]
msg = ('Unable to start sync')
data = {'status': 'sync started',}
with self._update_trail_and_quit(msg, data=data):
rp = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
while True:
if (os.getppid() != self.ppid):
msg = ('Backup plugin scheduler exited. Aborting...')
with self._update_trail_and_quit(msg):
rp.terminate()
if (rp.poll() is not None):
msg = ('sync finished. But failed to update trail status')
data = {'status': 'succeeded',}
with self._update_trail_and_quit(msg, data):
logger.debug('sync finished')
break
time.sleep(1)
def remove_snap_old(pool, pool_device, share_name, snap_name):
full_name = "%s/%s" % (share_name, snap_name)
if is_share_mounted(full_name):
umount_root("%s%s" % (DEFAULT_MNT_DIR, full_name))
root_pool_mnt = mount_root(pool, pool_device)
subvol_mnt_pt = "%s/%s_%s" % (root_pool_mnt, share_name, snap_name)
if is_subvol(subvol_mnt_pt):
return run_command([BTRFS, "subvolume", "delete", subvol_mnt_pt])
return True
def mount_root(pool):
"""
Mounts a given pool at the default mount root (usually /mnt2/) using the
pool.name as the final path entry. Ie pool.name = test-pool will be mounted
at /mnt2/test-pool. Any mount options held in pool.mnt_options will be added
to the mount command via the -o option as will a compress=pool.compression
entry.
N.B. Initially the mount target is defined by /dev/disk/by-label/pool.name,
if this fails then an attempt to mount by each member of
/dev/disk/by-id/pool.disk_set.all() but only if there are any members.
If this second method also fails then an exception is raised, currently all
but the last failed mount by device name is logged. If no disk members were
reported by pool.disk_set.count() a separate Exception is raised.
:param pool: pool object
:return: either the relevant mount point or an Exception which either
indicates 'no disks in pool' or 'Unknown Reason'
"""
root_pool_mnt = DEFAULT_MNT_DIR + pool.name
if (is_share_mounted(pool.name)):
return root_pool_mnt
# Creates a directory to act as the mount point.
create_tmp_dir(root_pool_mnt)
mnt_device = '/dev/disk/by-label/%s' % pool.name
mnt_cmd = [MOUNT, mnt_device, root_pool_mnt, ]
mnt_options = ''
if (pool.mnt_options is not None):
mnt_options = pool.mnt_options
if (pool.compression is not None):
if (re.search('compress', mnt_options) is None):
mnt_options = ('%s,compress=%s' % (mnt_options, pool.compression))
if (os.path.exists(mnt_device)):
if (len(mnt_options) > 0):
mnt_cmd.extend(['-o', mnt_options])
run_command(mnt_cmd)
return root_pool_mnt
# If we cannot mount by-label, let's try mounting by device; one by one
# until we get our first success.
if (pool.disk_set.count() < 1):
raise Exception('Cannot mount Pool(%s) as it has no disks in it.' % pool.name)
last_device = pool.disk_set.last()
for device in pool.disk_set.all():
mnt_device = ('/dev/disk/by-id/%s' % device.name)
if (os.path.exists(mnt_device)):
mnt_cmd = [MOUNT, mnt_device, root_pool_mnt, ]
if (len(mnt_options) > 0):
mnt_cmd.extend(['-o', mnt_options])
try:
run_command(mnt_cmd)
return root_pool_mnt
except Exception, e:
if (device.name == last_device.name):
# exhausted mounting using all devices in the pool
raise e
logger.error('Error mounting: %s. Will try using another device.' % mnt_cmd)
logger.exception(e)
def post(self, request):
with self._handle_exception(request):
pool_name = request.data.get('pool', None)
try:
pool = Pool.objects.get(name=pool_name)
except:
e_msg = ('Pool(%s) does not exist.' % pool_name)
handle_exception(Exception(e_msg), request)
compression = self._validate_compression(request)
size = self._validate_share_size(request, pool)
sname = request.data.get('sname', None)
if ((sname is None
or re.match('%s$' % settings.SHARE_REGEX, sname) is None)):
e_msg = ('Share name must start with a alphanumeric(a-z0-9) '
'character and can be followed by any of the '
'following characters: letter(a-z), digits(0-9), '
'hyphen(-), underscore(_) or a period(.).')
handle_exception(Exception(e_msg), request)
if (Share.objects.filter(name=sname).exists()):
e_msg = ('Share(%s) already exists. Choose a different name' %
sname)
handle_exception(Exception(e_msg), request)
if (Pool.objects.filter(name=sname).exists()):
e_msg = (
'A Pool with this name(%s) exists. Share and Pool names '
'must be distinct. Choose a different name' % sname)
handle_exception(Exception(e_msg), request)
disk = Disk.objects.filter(pool=pool)[0]
replica = False
if ('replica' in request.data):
replica = request.data['replica']
if (type(replica) != bool):
e_msg = ('replica must be a boolean, not %s' %
type(replica))
handle_exception(Exception(e_msg), request)
add_share(pool, disk.name, sname)
qid = qgroup_id(pool, disk.name, sname)
update_quota(pool, disk.name, qid, size * 1024)
s = Share(pool=pool,
qgroup=qid,
name=sname,
size=size,
subvol_name=sname,
replica=replica,
compression_algo=compression)
s.save()
mnt_pt = '%s%s' % (settings.MNT_PT, sname)
if (not is_share_mounted(sname)):
disk = Disk.objects.filter(pool=pool)[0].name
mount_share(s, disk, mnt_pt)
if (compression != 'no'):
set_property(mnt_pt, 'compression', compression)
return Response(ShareSerializer(s).data)
def post(self, request, sname, command):
with self._handle_exception(request):
share = self._validate_share(request, sname)
if command == "clone":
new_name = request.DATA.get("name", "")
return create_clone(share, new_name, request, logger)
if command == "rollback":
snap = self._validate_snapshot(request, share)
if NFSExport.objects.filter(share=share).exists():
e_msg = (
"Share(%s) cannot be rolled back as it is "
"exported via nfs. Delete nfs exports and "
"try again" % sname
)
handle_exception(Exception(e_msg), request)
if SambaShare.objects.filter(share=share).exists():
e_msg = (
"Share(%s) cannot be rolled back as it is shared" " via Samba. Unshare and try again" % sname
)
handle_exception(Exception(e_msg), request)
pool_device = Disk.objects.filter(pool=share.pool)[0].name
rollback_snap(snap.real_name, share.name, share.subvol_name, share.pool, pool_device)
update_quota(share.pool, pool_device, snap.qgroup, share.size * 1024)
share.qgroup = snap.qgroup
share.save()
snap.delete()
return Response()
if command == "compress":
algo = request.DATA.get("compress", None)
if algo is None:
e_msg = (
"Compression algorithm must be specified. Valid " "options are: %s" % settings.COMPRESSION_TYPES
)
handle_exception(Exception(e_msg), request)
if algo not in settings.COMPRESSION_TYPES:
e_msg = (
"Compression algorithm(%s) is invalid. Valid " "options are: %s" % settings.COMPRESSION_TYPES
)
handle_exception(Exception(e_msg), request)
mnt_pt = "%s%s" % (settings.MNT_PT, share.name)
if not is_share_mounted(share.name):
disk = Disk.objects.filter(pool=share.pool)[0].name
mount_share(share, disk, mnt_pt)
share.compression_algo = algo
share.save()
if algo == "no":
algo = ""
set_property(mnt_pt, "compression", algo)
return Response(ShareSerializer(share).data)
def rollback_snap(snap_name, sname, subvol_name, pool_name, pool_device):
"""
1. umount the share
2. mount the snap as the share
3. remove the share
"""
mnt_pt = ('%s%s' % (DEFAULT_MNT_DIR, sname))
if (is_share_mounted(sname)):
umount_root(mnt_pt)
mount_share(snap_name, pool_device, mnt_pt)
remove_share(pool_name, pool_device, subvol_name)
def rollback_snap(snap_name, sname, subvol_name, pool_name, pool_device):
"""
1. umount the share
2. mount the snap as the share
3. remove the share
"""
mnt_pt = ('%s%s' % (DEFAULT_MNT_DIR, sname))
if (is_share_mounted(sname)):
umount_root(mnt_pt)
mount_share(snap_name, pool_device, mnt_pt)
remove_share(pool_name, pool_device, subvol_name)
def mount_root(pool):
root_pool_mnt = DEFAULT_MNT_DIR + pool["name"]
if (is_share_mounted(pool["name"])):
return root_pool_mnt
create_tmp_dir(root_pool_mnt)
mnt_device = '/dev/disk/by-label/%s' % pool["name"]
mnt_cmd = [
MOUNT,
mnt_device,
root_pool_mnt,
]
mnt_options = ''
if (pool["mnt_options"] is not None):
mnt_options = pool["mnt_options"]
if (pool["compression"] is not None):
if (re.search('compress', mnt_options) is None):
mnt_options = ('%s,compress=%s' %
(mnt_options, pool["compression"]))
if (os.path.exists(mnt_device)):
if (len(mnt_options) > 0):
mnt_cmd.extend(['-o', mnt_options])
run_command(mnt_cmd)
return root_pool_mnt
# If we cannot mount by-label, let's try mounting by device one by one
# until we get our first success.
# ==============================================================================#
if (len(pool["disks"]) < 1):
raise Exception('Cannot mount Pool(%s) as it has no disks in it.' %
pool["name"])
last_device = pool["disks"][-1]
for device in pool["disks"]:
mnt_device = ('/dev/%s' % device["name"])
if (os.path.exists(mnt_device)):
mnt_cmd = [
MOUNT,
mnt_device,
root_pool_mnt,
]
if (len(mnt_options) > 0):
mnt_cmd.extend(['-o', mnt_options])
try:
run_command(mnt_cmd)
return root_pool_mnt
except Exception, e:
if (device["name"] == last_device["name"]):
# exhausted mounting using all devices in the pool
raise e
logger.error(
'Error mouting: %s. Will try using another device.' %
mnt_cmd)
logger.exception(e)
def post(self, request):
with self._handle_exception(request):
pool_name = request.DATA.get('pool', None)
try:
pool = Pool.objects.get(name=pool_name)
except:
e_msg = ('Pool(%s) does not exist.' % pool_name)
handle_exception(Exception(e_msg), request)
compression = self._validate_compression(request)
size = self._validate_share_size(request, pool)
sname = request.DATA.get('sname', None)
if ((sname is None or
re.match('%s$' % settings.SHARE_REGEX, sname) is None)):
e_msg = ('Share name must start with a letter(a-z) and can '
'be followed by any of the following characters: '
'letter(a-z), digits(0-9), hyphen(-), underscore'
'(_) or a period(.).')
handle_exception(Exception(e_msg), request)
if (Share.objects.filter(name=sname).exists()):
e_msg = ('Share(%s) already exists.' % sname)
handle_exception(Exception(e_msg), request)
try:
disk = Disk.objects.filter(pool=pool)[0]
except:
e_msg = ('Pool(%s) does not have any disks in it.' %
pool_name)
handle_exception(Exception(e_msg), request)
replica = False
if ('replica' in request.DATA):
replica = request.DATA['replica']
if (type(replica) != bool):
e_msg = ('replica must be a boolean, not %s' %
type(replica))
handle_exception(Exception(e_msg), request)
add_share(pool, disk.name, sname)
qgroup_id = self._update_quota(pool, disk.name, sname, size)
s = Share(pool=pool, qgroup=qgroup_id, name=sname, size=size,
subvol_name=sname, replica=replica,
compression_algo=compression)
s.save()
mnt_pt = '%s%s' % (settings.MNT_PT, sname)
if (not is_share_mounted(sname)):
disk = Disk.objects.filter(pool=pool)[0].name
mount_share(s, disk, mnt_pt)
if (compression != 'no'):
set_property(mnt_pt, 'compression', compression)
return Response(ShareSerializer(s).data)
def post(self, request):
with self._handle_exception(request):
pool_name = request.data.get('pool', None)
try:
pool = Pool.objects.get(name=pool_name)
except:
e_msg = ('Pool(%s) does not exist.' % pool_name)
handle_exception(Exception(e_msg), request)
compression = self._validate_compression(request)
size = self._validate_share_size(request, pool)
sname = request.data.get('sname', None)
if ((sname is None or
re.match('%s$' % settings.SHARE_REGEX, sname) is None)):
e_msg = ('Share name must start with a alphanumeric(a-z0-9) '
'character and can be followed by any of the '
'following characters: letter(a-z), digits(0-9), '
'hyphen(-), underscore(_) or a period(.).')
handle_exception(Exception(e_msg), request)
if (Share.objects.filter(name=sname).exists()):
e_msg = ('Share(%s) already exists. Choose a different name' % sname)
handle_exception(Exception(e_msg), request)
if (Pool.objects.filter(name=sname).exists()):
e_msg = ('A Pool with this name(%s) exists. Share and Pool names '
'must be distinct. Choose a different name' % sname)
handle_exception(Exception(e_msg), request)
disk = Disk.objects.filter(pool=pool)[0]
replica = False
if ('replica' in request.data):
replica = request.data['replica']
if (type(replica) != bool):
e_msg = ('replica must be a boolean, not %s' %
type(replica))
handle_exception(Exception(e_msg), request)
add_share(pool, disk.name, sname)
qid = qgroup_id(pool, disk.name, sname)
update_quota(pool, disk.name, qid, size * 1024)
s = Share(pool=pool, qgroup=qgroup_id, name=sname, size=size,
subvol_name=sname, replica=replica,
compression_algo=compression)
s.save()
mnt_pt = '%s%s' % (settings.MNT_PT, sname)
if (not is_share_mounted(sname)):
disk = Disk.objects.filter(pool=pool)[0].name
mount_share(s, disk, mnt_pt)
if (compression != 'no'):
set_property(mnt_pt, 'compression', compression)
return Response(ShareSerializer(s).data)
def remove_share(pool_name, pool_device, share_name):
"""
umount share if its mounted.
mount root pool
btrfs subvolume delete root_mnt/vol_name
umount root pool
"""
if (is_share_mounted(share_name)):
mnt_pt = ('%s%s' % (DEFAULT_MNT_DIR, share_name))
umount_root(mnt_pt)
pool_device = '/dev/' + pool_device
root_pool_mnt = mount_root(pool_name, pool_device)
subvol_mnt_pt = root_pool_mnt + '/' + share_name
delete_cmd = [BTRFS, 'subvolume', 'delete', subvol_mnt_pt]
run_command(delete_cmd)
def remove_share(pool_name, pool_device, share_name):
"""
umount share if its mounted.
mount root pool
btrfs subvolume delete root_mnt/vol_name
umount root pool
"""
if (is_share_mounted(share_name)):
mnt_pt = ('%s%s' % (DEFAULT_MNT_DIR, share_name))
umount_root(mnt_pt)
pool_device = '/dev/' + pool_device
root_pool_mnt = mount_root(pool_name, pool_device)
subvol_mnt_pt = root_pool_mnt + '/' + share_name
delete_cmd = [BTRFS, 'subvolume', 'delete', subvol_mnt_pt]
run_command(delete_cmd)
def remove_share(pool, pool_device, share_name):
"""
umount share if its mounted.
mount root pool
btrfs subvolume delete root_mnt/vol_name
umount root pool
"""
if is_share_mounted(share_name):
mnt_pt = "%s%s" % (DEFAULT_MNT_DIR, share_name)
umount_root(mnt_pt)
pool_device = "/dev/" + pool_device
root_pool_mnt = mount_root(pool, pool_device)
subvol_mnt_pt = root_pool_mnt + "/" + share_name
if not is_subvol(subvol_mnt_pt):
return
delete_cmd = [BTRFS, "subvolume", "delete", subvol_mnt_pt]
run_command(delete_cmd)
class CommandView(APIView):
authentication_classes = (
DigestAuthentication,
SessionAuthentication,
BasicAuthentication,
OAuth2Authentication,
)
permission_classes = (IsAuthenticated, )
def post(self, request, command):
if (command == 'bootstrap'):
try:
device_scan()
except Exception, e:
e_msg = ('Unabled to scan disk drives on the system.')
logger.error(e_msg)
logger.exception(e)
handle_exception(Exception(e_msg), request)
for pool in Pool.objects.all():
disk = Disk.objects.filter(pool=pool)[0].name
try:
mount_root(pool, '/dev/%s' % disk)
except Exception, e:
e_msg = ('Unable to mount a pool(%s) during bootstrap.' %
pool.name)
logger.exception(e)
for share in Share.objects.all():
try:
if (share.pqgroup == settings.MODEL_DEFS['pqgroup']):
share.pqgroup = qgroup_create(share.pool)
share.save()
if (not is_share_mounted(share.name)):
mnt_pt = ('%s%s' % (settings.MNT_PT, share.name))
pool_device = Disk.objects.filter(
pool=share.pool)[0].name
mount_share(share, pool_device, mnt_pt)
except Exception, e:
e_msg = (
'Unable to mount a share(%s, %s) during bootstrap.' %
(pool_device, mnt_pt))
logger.error(e_msg)
logger.exception(e)
def mount_root(pool, device):
root_pool_mnt = DEFAULT_MNT_DIR + pool.name
if is_share_mounted(pool.name):
return root_pool_mnt
create_tmp_dir(root_pool_mnt)
mnt_device = "/dev/disk/by-label/%s" % pool.name
if not os.path.exists(mnt_device):
mnt_device = device
mnt_cmd = [MOUNT, mnt_device, root_pool_mnt]
mnt_options = ""
if pool.mnt_options is not None:
mnt_options = pool.mnt_options
if pool.compression is not None:
if re.search("compress", mnt_options) is None:
mnt_options = "%s,compress=%s" % (mnt_options, pool.compression)
if len(mnt_options) > 0:
mnt_cmd.extend(["-o", mnt_options])
run_command(mnt_cmd)
return root_pool_mnt
def remove_share(pool, share_name, pqgroup):
"""
umount share if its mounted.
mount root pool
btrfs subvolume delete root_mnt/vol_name
umount root pool
"""
if is_share_mounted(share_name):
mnt_pt = "%s%s" % (DEFAULT_MNT_DIR, share_name)
umount_root(mnt_pt)
root_pool_mnt = mount_root(pool)
subvol_mnt_pt = root_pool_mnt + "/" + share_name
if not is_subvol(subvol_mnt_pt):
return
qgroup = "0/%s" % share_id(pool, share_name)
delete_cmd = [BTRFS, "subvolume", "delete", subvol_mnt_pt]
run_command(delete_cmd, log=True)
qgroup_destroy(qgroup, root_pool_mnt)
return qgroup_destroy(pqgroup, root_pool_mnt)
def remove_share(pool, share_name, pqgroup):
"""
umount share if its mounted.
mount root pool
btrfs subvolume delete root_mnt/vol_name
umount root pool
"""
if (is_share_mounted(share_name)):
mnt_pt = ('%s%s' % (DEFAULT_MNT_DIR, share_name))
umount_root(mnt_pt)
root_pool_mnt = mount_root(pool)
subvol_mnt_pt = root_pool_mnt + '/' + share_name
if (not is_subvol(subvol_mnt_pt)):
return
qgroup = ('0/%s' % share_id(pool, share_name))
delete_cmd = [BTRFS, 'subvolume', 'delete', subvol_mnt_pt]
run_command(delete_cmd, log=True)
qgroup_destroy(qgroup, root_pool_mnt)
return qgroup_destroy(pqgroup, root_pool_mnt)
def remove_share(pool, share_name, pqgroup):
"""
umount share if its mounted.
mount root pool
btrfs subvolume delete root_mnt/vol_name
umount root pool
"""
if (is_share_mounted(share_name)):
mnt_pt = ('%s%s' % (DEFAULT_MNT_DIR, share_name))
umount_root(mnt_pt)
root_pool_mnt = mount_root(pool)
subvol_mnt_pt = root_pool_mnt + '/' + share_name
if (not is_subvol(subvol_mnt_pt)):
return
qgroup = ('0/%s' % share_id(pool, share_name))
delete_cmd = [BTRFS, 'subvolume', 'delete', subvol_mnt_pt]
run_command(delete_cmd, log=True)
qgroup_destroy(qgroup, root_pool_mnt)
return qgroup_destroy(pqgroup, root_pool_mnt)
def shares_usage(pool, share_map, snap_map):
# don't mount the pool if at least one share in the map is mounted.
usage_map = {}
mnt_pt = None
for s in share_map.keys():
if (is_share_mounted(share_map[s])):
mnt_pt = ('%s%s' % (DEFAULT_MNT_DIR, share_map[s]))
break
if (mnt_pt is None):
mnt_pt = mount_root(pool)
cmd = [BTRFS, 'qgroup', 'show', mnt_pt]
out, err, rc = run_command(cmd, log=True)
combined_map = dict(share_map, **snap_map)
for line in out:
fields = line.split()
if (len(fields) > 0 and fields[0] in combined_map):
r_usage = convert_to_KiB(fields[-2])
e_usage = convert_to_KiB(fields[-1])
usage_map[combined_map[fields[0]]] = (r_usage, e_usage)
return usage_map
def shares_usage(pool_name, pool_device, share_map, snap_map):
#don't mount the pool if at least one share in the map is mounted.
usage_map = {}
mnt_pt = None
for s in share_map.keys():
if (is_share_mounted(share_map[s])):
mnt_pt = ('%s%s' % (DEFAULT_MNT_DIR, share_map[s]))
break
if (mnt_pt is None):
mnt_pt = mount_root(pool_name, '/dev/' + pool_device)
cmd = [BTRFS, 'qgroup', 'show', mnt_pt]
out, err, rc = run_command(cmd)
combined_map = dict(share_map, **snap_map)
for line in out:
fields = line.split()
if (len(fields) > 0 and fields[0] in combined_map):
r_usage = int(fields[-2]) / 1024 # referenced usage in KB
e_usage = int(fields[-1]) / 1024 # exclusive usage in KB
usage_map[combined_map[fields[0]]] = (r_usage, e_usage)
return usage_map
def rollback_snap(snap_name, sname, subvol_name, pool, pool_device):
"""
1. validate destination snapshot and umount the share
2. remove the share
3. move the snapshot to share location and mount it.
"""
mnt_pt = "%s%s" % (DEFAULT_MNT_DIR, sname)
snap_fp = "%s/%s/.snapshots/%s/%s" % (DEFAULT_MNT_DIR, pool.name, sname, snap_name)
if not is_subvol(snap_fp):
raise Exception("Snapshot(%s) does not exist. Rollback is not " "possible" % snap_fp)
dpath = "/dev/%s" % pool_device
mount_root(pool, dpath)
if is_share_mounted(sname):
umount_root(mnt_pt)
remove_share(pool, pool_device, subvol_name)
shutil.move(snap_fp, "%s/%s/%s" % (DEFAULT_MNT_DIR, pool.name, sname))
create_tmp_dir(mnt_pt)
subvol_str = "subvol=%s" % sname
mnt_cmd = [MOUNT, "-t", "btrfs", "-o", subvol_str, dpath, mnt_pt]
run_command(mnt_cmd)
def shares_usage(pool, share_map, snap_map):
# don't mount the pool if at least one share in the map is mounted.
usage_map = {}
mnt_pt = None
for s in share_map.keys():
if (is_share_mounted(share_map[s])):
mnt_pt = ('%s%s' % (DEFAULT_MNT_DIR, share_map[s]))
break
if (mnt_pt is None):
mnt_pt = mount_root(pool)
cmd = [BTRFS, 'qgroup', 'show', mnt_pt]
out, err, rc = run_command(cmd, log=True)
combined_map = dict(share_map, **snap_map)
for line in out:
fields = line.split()
if (len(fields) > 0 and fields[0] in combined_map):
r_usage = convert_to_KiB(fields[-2])
e_usage = convert_to_KiB(fields[-1])
usage_map[combined_map[fields[0]]] = (r_usage, e_usage)
return usage_map
def shares_usage(pool_name, pool_device, share_map, snap_map):
#don't mount the pool if at least one share in the map is mounted.
usage_map = {}
mnt_pt = None
for s in share_map.keys():
if (is_share_mounted(share_map[s])):
mnt_pt = ('%s%s' % (DEFAULT_MNT_DIR, share_map[s]))
break
if (mnt_pt is None):
mnt_pt = mount_root(pool_name, '/dev/' + pool_device)
cmd = [BTRFS, 'qgroup', 'show', mnt_pt]
out, err, rc = run_command(cmd)
combined_map = dict(share_map, **snap_map)
for line in out:
fields = line.split()
if (len(fields) > 0 and fields[0] in combined_map):
r_usage = int(fields[-2]) / 1024 # referenced usage in KB
e_usage = int(fields[-1]) / 1024 # exclusive usage in KB
usage_map[combined_map[fields[0]]] = (r_usage, e_usage)
return usage_map
def mount_root(pool, device):
root_pool_mnt = DEFAULT_MNT_DIR + pool.name
if (is_share_mounted(pool.name)):
return root_pool_mnt
create_tmp_dir(root_pool_mnt)
mnt_device = '/dev/disk/by-label/%s' % pool.name
if (not os.path.exists(mnt_device)):
mnt_device = device
if (re.match('/dev/', device) is None):
mnt_device = ('/dev/%s' % mnt_device)
mnt_cmd = [MOUNT, mnt_device, root_pool_mnt, ]
mnt_options = ''
if (pool.mnt_options is not None):
mnt_options = pool.mnt_options
if (pool.compression is not None):
if (re.search('compress', mnt_options) is None):
mnt_options = ('%s,compress=%s' % (mnt_options, pool.compression))
if (len(mnt_options) > 0):
mnt_cmd.extend(['-o', mnt_options])
run_command(mnt_cmd)
return root_pool_mnt
def mount_root(pool):
root_pool_mnt = DEFAULT_MNT_DIR + pool["name"]
if (is_share_mounted(pool["name"])):
return root_pool_mnt
create_tmp_dir(root_pool_mnt)
mnt_device = '/dev/disk/by-label/%s' % pool["name"]
mnt_cmd = [MOUNT, mnt_device, root_pool_mnt, ]
mnt_options = ''
if (pool["mnt_options"] is not None):
mnt_options = pool["mnt_options"]
if (pool["compression"] is not None):
if (re.search('compress', mnt_options) is None):
mnt_options = ('%s,compress=%s' % (mnt_options, pool["compression"]))
if (os.path.exists(mnt_device)):
if (len(mnt_options) > 0):
mnt_cmd.extend(['-o', mnt_options])
run_command(mnt_cmd)
return root_pool_mnt
# If we cannot mount by-label, let's try mounting by device one by one
# until we get our first success.
# ==============================================================================#
if (len(pool["disks"]) < 1):
raise Exception('Cannot mount Pool(%s) as it has no disks in it.' % pool["name"])
last_device = pool["disks"][-1]
for device in pool["disks"]:
mnt_device = ('/dev/%s' % device["name"])
if (os.path.exists(mnt_device)):
mnt_cmd = [MOUNT, mnt_device, root_pool_mnt, ]
if (len(mnt_options) > 0):
mnt_cmd.extend(['-o', mnt_options])
try:
run_command(mnt_cmd)
return root_pool_mnt
except Exception, e:
if (device["name"] == last_device["name"]):
# exhausted mounting using all devices in the pool
raise e
logger.error('Error mouting: %s. Will try using another device.' % mnt_cmd)
logger.exception(e)
def rollback_snap(snap_name, sname, subvol_name, pool):
"""
1. validate destination snapshot and umount the share
2. remove the share
3. move the snapshot to share location and mount it.
"""
mnt_pt = ('%s%s' % (DEFAULT_MNT_DIR, sname))
snap_fp = ('%s/%s/.snapshots/%s/%s' % (DEFAULT_MNT_DIR, pool.name, sname,
snap_name))
if (not is_subvol(snap_fp)):
raise Exception('Snapshot(%s) does not exist. Rollback is not '
'possible' % snap_fp)
mount_root(pool)
if (is_share_mounted(sname)):
umount_root(mnt_pt)
remove_share(pool, subvol_name, '-1/-1')
shutil.move(snap_fp, '%s/%s/%s' % (DEFAULT_MNT_DIR, pool.name, sname))
create_tmp_dir(mnt_pt)
subvol_str = 'subvol=%s' % sname
dpath = '/dev/%s' % pool.disk_set.first().name
mnt_cmd = [MOUNT, '-t', 'btrfs', '-o', subvol_str, dpath, mnt_pt]
run_command(mnt_cmd)
def post(self, request, command):
if (command == 'bootstrap'):
for pool in Pool.objects.all():
try:
mount_root(pool)
except Exception, e:
e_msg = ('Exception while mounting a pool(%s) during '
'bootstrap: %s' % (pool.name, e.__str__()))
logger.error(e_msg)
for share in Share.objects.all():
try:
if (share.pqgroup == settings.MODEL_DEFS['pqgroup']):
share.pqgroup = qgroup_create(share.pool)
share.save()
if (not is_share_mounted(share.name)):
mnt_pt = ('%s%s' % (settings.MNT_PT, share.name))
mount_share(share, mnt_pt)
except Exception, e:
e_msg = ('Exception while mounting a share(%s) during '
'bootstrap: %s' % (share.name, e.__str__()))
logger.error(e_msg)
def rollback_snap(snap_name, sname, subvol_name, pool):
"""
1. validate destination snapshot and umount the share
2. remove the share
3. move the snapshot to share location and mount it.
"""
mnt_pt = ('%s%s' % (DEFAULT_MNT_DIR, sname))
snap_fp = ('%s/%s/.snapshots/%s/%s' %
(DEFAULT_MNT_DIR, pool.name, sname, snap_name))
if (not is_subvol(snap_fp)):
raise Exception('Snapshot(%s) does not exist. Rollback is not '
'possible' % snap_fp)
mount_root(pool)
if (is_share_mounted(sname)):
umount_root(mnt_pt)
remove_share(pool, subvol_name, '-1/-1')
shutil.move(snap_fp, '%s/%s/%s' % (DEFAULT_MNT_DIR, pool.name, sname))
create_tmp_dir(mnt_pt)
subvol_str = 'subvol=%s' % sname
dpath = '/dev/%s' % pool.disk_set.first().name
mnt_cmd = [MOUNT, '-t', 'btrfs', '-o', subvol_str, dpath, mnt_pt]
run_command(mnt_cmd)
def post(self, request, command):
if (command == 'bootstrap'):
for pool in Pool.objects.all():
try:
mount_root(pool)
except Exception, e:
e_msg = ('Exception while mounting a pool(%s) during '
'bootstrap: %s' % (pool.name, e.__str__()))
logger.error(e_msg)
for share in Share.objects.all():
try:
if (share.pqgroup == settings.MODEL_DEFS['pqgroup']):
share.pqgroup = qgroup_create(share.pool)
share.save()
if (not is_share_mounted(share.name)):
mnt_pt = ('%s%s' % (settings.MNT_PT, share.name))
mount_share(share, mnt_pt)
except Exception, e:
e_msg = ('Exception while mounting a share(%s) during '
'bootstrap: %s' % (share.name, e.__str__()))
logger.error(e_msg)
def post(self, request, command):
if (command == 'bootstrap'):
try:
device_scan()
except Exception, e:
e_msg = ('Unabled to scan disk drives on the system.')
logger.error(e_msg)
logger.exception(e)
handle_exception(Exception(e_msg), request)
try:
device_scan()
for share in Share.objects.all():
if (not is_share_mounted(share.name)):
mnt_pt = ('%s%s' % (settings.MNT_PT, share.name))
pool_device = Disk.objects.filter(
pool=share.pool)[0].name
mount_share(share.subvol_name, pool_device, mnt_pt)
except Exception, e:
e_msg = ('Unable to mount a share(%s, %s) during bootstrap.' %
(pool_device, mnt_pt))
logger.error(e_msg)
logger.exception(e)
handle_exception(Exception(e_msg), request)
def post(self, request):
#qgroup notes for shares. we need to create a qgroup prior to share
#creation. qgroup ids 0/<subvol_id> automatically get created when a
#subvolume(i.e., a Share or a Snapshot) is created. So let's create a
#new qgroup: 2015/<some_number> whenever a Share is
#created. <some_number> starts from 1 and is incremented as more Shares
#are created. So, for the very first Share in a pool, it's qgroup will
#be 1/1. 2015 is arbitrarily chose.
#Before creating a new Share, we create the qgroup for it. And during
#it's creation, we assign this qgroup to it. During it's creation a 0/x
#qgroup will automatically be created, but it will become the child of
#our explicitly-created qgroup(2015/x).
#We will set the qgroup limit on our qgroup and it will enforce the
#quota on every subvolume(i.e., Share and Snapshot) in that qgroup.
#When a Share is deleted, we need to destroy two qgroups. One is it's
#auto 0/x qgroup and the other is our explicitly-created 2015/y qgroup.
with self._handle_exception(request):
pool_name = request.data.get('pool', None)
try:
pool = Pool.objects.get(name=pool_name)
except:
e_msg = ('Pool(%s) does not exist.' % pool_name)
handle_exception(Exception(e_msg), request)
compression = self._validate_compression(request)
size = self._validate_share_size(request, pool)
sname = request.data.get('sname', None)
if ((sname is None or
re.match('%s$' % settings.SHARE_REGEX, sname) is None)):
e_msg = ('Share name must start with a alphanumeric(a-z0-9) '
'character and can be followed by any of the '
'following characters: letter(a-z), digits(0-9), '
'hyphen(-), underscore(_) or a period(.).')
handle_exception(Exception(e_msg), request)
if (len(sname) > 254):
#btrfs subvolume names cannot exceed 254 characters.
e_msg = ('Share name length cannot exceed 254 characters')
handle_exception(Exception(e_msg), request)
if (Share.objects.filter(name=sname).exists()):
e_msg = ('Share(%s) already exists. Choose a different name' % sname)
handle_exception(Exception(e_msg), request)
if (Pool.objects.filter(name=sname).exists()):
e_msg = ('A Pool with this name(%s) exists. Share and Pool names '
'must be distinct. Choose a different name' % sname)
handle_exception(Exception(e_msg), request)
disk = Disk.objects.filter(pool=pool)[0]
replica = False
if ('replica' in request.data):
replica = request.data['replica']
if (type(replica) != bool):
e_msg = ('replica must be a boolean, not %s' %
type(replica))
handle_exception(Exception(e_msg), request)
pqid = qgroup_create(pool)
add_share(pool, sname, pqid)
qid = qgroup_id(pool, sname)
update_quota(pool, pqid, size * 1024)
s = Share(pool=pool, qgroup=qid, pqgroup=pqid, name=sname,
size=size, subvol_name=sname, replica=replica,
compression_algo=compression)
s.save()
mnt_pt = '%s%s' % (settings.MNT_PT, sname)
if (not is_share_mounted(sname)):
mount_share(s, mnt_pt)
if (compression != 'no'):
set_property(mnt_pt, 'compression', compression)
return Response(ShareSerializer(s).data)
def post(self, request, command):
if (command == 'bootstrap'):
self._refresh_pool_state()
for p in Pool.objects.all():
import_shares(p, request)
for share in Share.objects.all():
try:
if (share.pqgroup == settings.MODEL_DEFS['pqgroup']):
share.pqgroup = qgroup_create(share.pool)
share.save()
if (not is_share_mounted(share.name)):
mnt_pt = ('%s%s' % (settings.MNT_PT, share.name))
mount_share(share, mnt_pt)
except Exception as e:
e_msg = ('Exception while mounting a share(%s) during '
'bootstrap: %s' % (share.name, e.__str__()))
logger.error(e_msg)
logger.exception(e)
try:
import_snapshots(share)
except Exception as e:
e_msg = ('Exception while importing Snapshots of '
'Share(%s): %s' % (share.name, e.__str__()))
logger.error(e_msg)
logger.exception(e)
for snap in Snapshot.objects.all():
if (snap.uvisible):
try:
mount_snap(snap.share, snap.real_name)
except Exception as e:
e_msg = ('Failed to make the Snapshot(%s) visible. '
'Exception: %s' %
(snap.real_name, e.__str__()))
logger.error(e_msg)
mnt_map = sftp_mount_map(settings.SFTP_MNT_ROOT)
for sftpo in SFTP.objects.all():
try:
sftp_mount(sftpo.share, settings.MNT_PT,
settings.SFTP_MNT_ROOT, mnt_map, sftpo.editable)
sftp_snap_toggle(sftpo.share)
except Exception as e:
e_msg = ('Exception while exportin a sftp share during '
'bootstrap: %s' % e.__str__())
logger.error(e_msg)
try:
adv_entries = [
a.export_str for a in AdvancedNFSExport.objects.all()
]
exports_d = self.create_adv_nfs_export_input(
adv_entries, request)
exports = self.create_nfs_export_input(NFSExport.objects.all())
exports.update(exports_d)
self.refresh_wrapper(exports, request, logger)
except Exception as e:
e_msg = ('Exception while bootstrapping NFS: %s' % e.__str__())
logger.error(e_msg)
# bootstrap services
try:
systemctl('firewalld', 'stop')
systemctl('firewalld', 'disable')
systemctl('nginx', 'stop')
systemctl('nginx', 'disable')
systemctl('atd', 'enable')
systemctl('atd', 'start')
except Exception as e:
e_msg = ('Exception while setting service statuses during '
'bootstrap: %s' % e.__str__())
logger.error(e_msg)
handle_exception(Exception(e_msg), request)
logger.debug('Bootstrap operations completed')
return Response()
if (command == 'utcnow'):
return Response(datetime.utcnow().replace(tzinfo=utc))
if (command == 'uptime'):
return Response(uptime())
if (command == 'kernel'):
try:
return Response(kernel_info(settings.SUPPORTED_KERNEL_VERSION))
except Exception as e:
handle_exception(e, request)
if (command == 'update-check'):
try:
subo = None
try:
subo = UpdateSubscription.objects.get(name='Stable',
status='active')
except UpdateSubscription.DoesNotExist:
try:
subo = UpdateSubscription.objects.get(name='Testing',
status='active')
except UpdateSubscription.DoesNotExist:
pass
return Response(update_check(subscription=subo))
except Exception as e:
e_msg = ('Unable to check update due to a system error: %s' %
e.__str__())
handle_exception(Exception(e_msg), request)
if (command == 'update'):
try:
update_run()
return Response('Done')
except Exception as e:
e_msg = ('Update failed due to this exception: %s' %
e.__str__())
handle_exception(Exception(e_msg), request)
if (command == 'current-version'):
try:
return Response(current_version())
except Exception as e:
e_msg = ('Unable to check current version due to this '
'exception: ' % e.__str__())
handle_exception(Exception(e_msg), request)
if (command == 'shutdown'):
msg = ('The system will now be shutdown')
try:
request.session.flush()
system_shutdown()
except Exception as e:
msg = ('Failed to shutdown the system due to a low level '
'error: %s' % e.__str__())
handle_exception(Exception(msg), request)
finally:
return Response(msg)
if (command == 'reboot'):
msg = ('The system will now reboot')
try:
request.session.flush()
system_reboot()
except Exception as e:
msg = ('Failed to reboot the system due to a low level error: '
'%s' % e.__str__())
handle_exception(Exception(msg), request)
finally:
return Response(msg)
if (command == 'current-user'):
return Response(request.user.username)
if (command == 'auto-update-status'):
status = True
try:
status = auto_update_status()
except:
status = False
finally:
return Response({
'enabled': status,
})
if (command == 'enable-auto-update'):
try:
auto_update(enable=True)
return Response({
'enabled': True,
})
except Exception as e:
msg = ('Failed to enable auto update due to this exception: '
'%s' % e.__str__())
handle_exception(Exception(msg), request)
if (command == 'disable-auto-update'):
try:
auto_update(enable=False)
return Response({
'enabled': False,
})
except Exception as e:
msg = ('Failed to disable auto update due to this exception: '
'%s' % e.__str__())
handle_exception(Exception(msg), request)
if (command == 'refresh-pool-state'):
self._refresh_pool_state()
return Response()
if (command == 'refresh-share-state'):
for p in Pool.objects.all():
import_shares(p, request)
return Response()
if (command == 'refresh-snapshot-state'):
for share in Share.objects.all():
import_snapshots(share)
return Response()
def post(self, request):
#qgroup notes for shares. we need to create a qgroup prior to share
#creation. qgroup ids 0/<subvol_id> automatically get created when a
#subvolume(i.e., a Share or a Snapshot) is created. So let's create a
#new qgroup: 2015/<some_number> whenever a Share is
#created. <some_number> starts from 1 and is incremented as more Shares
#are created. So, for the very first Share in a pool, it's qgroup will
#be 1/1. 2015 is arbitrarily chose.
#Before creating a new Share, we create the qgroup for it. And during
#it's creation, we assign this qgroup to it. During it's creation a 0/x
#qgroup will automatically be created, but it will become the child of
#our explicitly-created qgroup(2015/x).
#We will set the qgroup limit on our qgroup and it will enforce the
#quota on every subvolume(i.e., Share and Snapshot) in that qgroup.
#When a Share is deleted, we need to destroy two qgroups. One is it's
#auto 0/x qgroup and the other is our explicitly-created 2015/y qgroup.
with self._handle_exception(request):
pool_name = request.data.get('pool', None)
try:
pool = Pool.objects.get(name=pool_name)
except:
e_msg = ('Pool(%s) does not exist.' % pool_name)
handle_exception(Exception(e_msg), request)
compression = self._validate_compression(request)
size = self._validate_share_size(request, pool)
sname = request.data.get('sname', None)
if ((sname is None
or re.match('%s$' % settings.SHARE_REGEX, sname) is None)):
e_msg = ('Share name must start with a alphanumeric(a-z0-9) '
'character and can be followed by any of the '
'following characters: letter(a-z), digits(0-9), '
'hyphen(-), underscore(_) or a period(.).')
handle_exception(Exception(e_msg), request)
if (len(sname) > 254):
#btrfs subvolume names cannot exceed 254 characters.
e_msg = ('Share name length cannot exceed 254 characters')
handle_exception(Exception(e_msg), request)
if (Share.objects.filter(name=sname).exists()):
e_msg = ('Share(%s) already exists. Choose a different name' %
sname)
handle_exception(Exception(e_msg), request)
if (Pool.objects.filter(name=sname).exists()):
e_msg = (
'A Pool with this name(%s) exists. Share and Pool names '
'must be distinct. Choose a different name' % sname)
handle_exception(Exception(e_msg), request)
disk = Disk.objects.filter(pool=pool)[0]
replica = False
if ('replica' in request.data):
replica = request.data['replica']
if (type(replica) != bool):
e_msg = ('replica must be a boolean, not %s' %
type(replica))
handle_exception(Exception(e_msg), request)
pqid = qgroup_create(pool)
add_share(pool, sname, pqid)
qid = qgroup_id(pool, sname)
update_quota(pool, pqid, size * 1024)
s = Share(pool=pool,
qgroup=qid,
pqgroup=pqid,
name=sname,
size=size,
subvol_name=sname,
replica=replica,
compression_algo=compression)
s.save()
mnt_pt = '%s%s' % (settings.MNT_PT, sname)
if (not is_share_mounted(sname)):
mount_share(s, mnt_pt)
if (compression != 'no'):
set_property(mnt_pt, 'compression', compression)
return Response(ShareSerializer(s).data)
def mount_root(pool):
"""
Mounts a given pool at the default mount root (usually /mnt2/) using the
pool.name as the final path entry. Ie pool.name = test-pool will be mounted
at /mnt2/test-pool. Any mount options held in pool.mnt_options will be
added to the mount command via the -o option as will a compress =
pool.compression entry.
N.B. Initially the mount target is defined by /dev/disk/by-label/pool.name,
if this fails then an attempt to mount by each member of
/dev/disk/by-id/pool.disk_set.all() but only if there are any members.
If this second method also fails then an exception is raised, currently all
but the last failed mount by device name is logged. If no disk members were
reported by pool.disk_set.count() a separate Exception is raised.
:param pool: pool object
:return: either the relevant mount point or an Exception which either
indicates 'no disks in pool' or 'Unknown Reason'
"""
root_pool_mnt = DEFAULT_MNT_DIR + pool.name
if (is_share_mounted(pool.name)):
return root_pool_mnt
# Creates a directory to act as the mount point.
create_tmp_dir(root_pool_mnt)
toggle_path_rw(root_pool_mnt, rw=False)
mnt_device = '/dev/disk/by-label/%s' % pool.name
mnt_cmd = [
MOUNT,
mnt_device,
root_pool_mnt,
]
mnt_options = ''
if (pool.mnt_options is not None):
mnt_options = pool.mnt_options
if (pool.compression is not None):
if (re.search('compress', mnt_options) is None):
mnt_options = ('%s,compress=%s' % (mnt_options, pool.compression))
if (os.path.exists(mnt_device)):
if (len(mnt_options) > 0):
mnt_cmd.extend(['-o', mnt_options])
run_command(mnt_cmd)
return root_pool_mnt
# If we cannot mount by-label, let's try mounting by device; one by one
# until we get our first success.
if (pool.disk_set.count() < 1):
raise Exception('Cannot mount Pool(%s) as it has no disks in it.' %
pool.name)
last_device = pool.disk_set.last()
for device in pool.disk_set.all():
mnt_device = ('/dev/disk/by-id/%s' % device.name)
if (os.path.exists(mnt_device)):
mnt_cmd = [
MOUNT,
mnt_device,
root_pool_mnt,
]
if (len(mnt_options) > 0):
mnt_cmd.extend(['-o', mnt_options])
try:
run_command(mnt_cmd)
return root_pool_mnt
except Exception as e:
if (device.name == last_device.name):
# exhausted mounting using all devices in the pool
raise e
logger.error('Error mouting: %s. '
'Will try using another device.' % mnt_cmd)
logger.exception(e)
raise Exception('Failed to mount Pool(%s) due to an unknown reason.' %
pool.name)
def post(self, request, command, rtcepoch=None):
if (command == 'bootstrap'):
self._refresh_pool_state()
for p in Pool.objects.all():
import_shares(p, request)
for share in Share.objects.all():
try:
if (share.pqgroup == settings.MODEL_DEFS['pqgroup']):
share.pqgroup = qgroup_create(share.pool)
share.save()
if (not is_share_mounted(share.name)):
mnt_pt = ('%s%s' % (settings.MNT_PT, share.name))
mount_share(share, mnt_pt)
except Exception as e:
e_msg = ('Exception while mounting a share(%s) during '
'bootstrap: %s' % (share.name, e.__str__()))
logger.error(e_msg)
logger.exception(e)
try:
import_snapshots(share)
except Exception as e:
e_msg = ('Exception while importing Snapshots of '
'Share(%s): %s' % (share.name, e.__str__()))
logger.error(e_msg)
logger.exception(e)
for snap in Snapshot.objects.all():
if (snap.uvisible):
try:
mount_snap(snap.share, snap.real_name)
except Exception as e:
e_msg = ('Failed to make the Snapshot(%s) visible. '
'Exception: %s' %
(snap.real_name, e.__str__()))
logger.error(e_msg)
mnt_map = sftp_mount_map(settings.SFTP_MNT_ROOT)
for sftpo in SFTP.objects.all():
try:
sftp_mount(sftpo.share, settings.MNT_PT,
settings.SFTP_MNT_ROOT, mnt_map, sftpo.editable)
sftp_snap_toggle(sftpo.share)
except Exception as e:
e_msg = ('Exception while exportin a sftp share during '
'bootstrap: %s' % e.__str__())
logger.error(e_msg)
try:
adv_entries = [
a.export_str for a in AdvancedNFSExport.objects.all()
]
exports_d = self.create_adv_nfs_export_input(
adv_entries, request)
exports = self.create_nfs_export_input(NFSExport.objects.all())
exports.update(exports_d)
self.refresh_wrapper(exports, request, logger)
except Exception as e:
e_msg = ('Exception while bootstrapping NFS: %s' % e.__str__())
logger.error(e_msg)
# bootstrap services
try:
systemctl('firewalld', 'stop')
systemctl('firewalld', 'disable')
systemctl('nginx', 'stop')
systemctl('nginx', 'disable')
systemctl('atd', 'enable')
systemctl('atd', 'start')
except Exception as e:
e_msg = ('Exception while setting service statuses during '
'bootstrap: %s' % e.__str__())
logger.error(e_msg)
handle_exception(Exception(e_msg), request)
logger.debug('Bootstrap operations completed')
return Response()
if (command == 'utcnow'):
return Response(datetime.utcnow().replace(tzinfo=utc))
if (command == 'uptime'):
return Response(uptime())
if (command == 'kernel'):
try:
return Response(kernel_info(settings.SUPPORTED_KERNEL_VERSION))
except Exception as e:
handle_exception(e, request)
if (command == 'update-check'):
try:
subo = None
try:
subo = UpdateSubscription.objects.get(name='Stable',
status='active')
except UpdateSubscription.DoesNotExist:
try:
subo = UpdateSubscription.objects.get(name='Testing',
status='active')
except UpdateSubscription.DoesNotExist:
pass
return Response(update_check(subscription=subo))
except Exception as e:
e_msg = ('Unable to check update due to a system error: %s' %
e.__str__())
handle_exception(Exception(e_msg), request)
if (command == 'update'):
try:
# Once again, like on system shutdown/reboot, we filter
# incoming requests with request.auth: every update from
# WebUI misses request.auth, while yum update requests from
# data_collector APIWrapper have it, so we can avoid
# an additional command for yum updates
if request.auth is None:
update_run()
else:
update_run(yum_update=True)
return Response('Done')
except Exception as e:
e_msg = ('Update failed due to this exception: %s' %
e.__str__())
handle_exception(Exception(e_msg), request)
if (command == 'current-version'):
try:
return Response(current_version())
except Exception as e:
e_msg = ('Unable to check current version due to this '
'exception: ' % e.__str__())
handle_exception(Exception(e_msg), request)
# default has shutdown and reboot with delay set to now
# having normal sytem power off with now = 1 min
# reboot and shutdown requests from WebUI don't have request.auth
# while same requests over rest api (ex. scheduled tasks) have
# an auth token, so if we detect a token we delay with 3 mins
# to grant connected WebUI user to close it or cancel shutdown/reboot
delay = 'now'
if request.auth is not None:
delay = 3
if (command == 'shutdown'):
msg = ('The system will now be shutdown')
try:
# if shutdown request coming from a scheduled task
# with rtc wake up time on we set it before
# system shutdown starting
if rtcepoch is not None:
set_system_rtc_wake(rtcepoch)
request.session.flush()
system_shutdown(delay)
except Exception as e:
msg = ('Failed to shutdown the system due to a low level '
'error: %s' % e.__str__())
handle_exception(Exception(msg), request)
finally:
return Response(msg)
if (command == 'reboot'):
msg = ('The system will now reboot')
try:
request.session.flush()
system_reboot(delay)
except Exception as e:
msg = ('Failed to reboot the system due to a low level error: '
'%s' % e.__str__())
handle_exception(Exception(msg), request)
finally:
return Response(msg)
if (command == 'suspend'):
msg = ('The system will now be suspended to RAM')
try:
request.session.flush()
set_system_rtc_wake(rtcepoch)
system_suspend()
except Exception as e:
msg = ('Failed to suspend the system due to a low level '
'error: %s' % e.__str__())
handle_exception(Exception(msg), request)
finally:
return Response(msg)
if (command == 'current-user'):
return Response(request.user.username)
if (command == 'auto-update-status'):
status = True
try:
status = auto_update_status()
except:
status = False
finally:
return Response({
'enabled': status,
})
if (command == 'enable-auto-update'):
try:
auto_update(enable=True)
return Response({
'enabled': True,
})
except Exception as e:
msg = ('Failed to enable auto update due to this exception: '
'%s' % e.__str__())
handle_exception(Exception(msg), request)
if (command == 'disable-auto-update'):
try:
auto_update(enable=False)
return Response({
'enabled': False,
})
except Exception as e:
msg = ('Failed to disable auto update due to this exception: '
'%s' % e.__str__())
handle_exception(Exception(msg), request)
if (command == 'refresh-pool-state'):
self._refresh_pool_state()
return Response()
if (command == 'refresh-share-state'):
for p in Pool.objects.all():
import_shares(p, request)
return Response()
if (command == 'refresh-snapshot-state'):
for share in Share.objects.all():
import_snapshots(share)
return Response()