Пример #1
0
    def post(self, request, command):
        if (command == 'bootstrap'):

            self._refresh_pool_state()
            for p in Pool.objects.all():
                import_shares(p, request)

            for share in Share.objects.all():
                try:
                    if (share.pqgroup == settings.MODEL_DEFS['pqgroup']):
                        share.pqgroup = qgroup_create(share.pool)
                        share.save()
                    if (not is_share_mounted(share.name)):
                        mnt_pt = ('%s%s' % (settings.MNT_PT, share.name))
                        mount_share(share, mnt_pt)
                except Exception, e:
                    e_msg = ('Exception while mounting a share(%s) during '
                             'bootstrap: %s' % (share.name, e.__str__()))
                    logger.error(e_msg)
                    logger.exception(e)

                try:
                    import_snapshots(share)
                except Exception, e:
                    e_msg = ('Exception while importing Snapshots of '
                             'Share(%s): %s' % (share.name, e.__str__()))
                    logger.error(e_msg)
                    logger.exception(e)
Пример #2
0
    def post(self, request, command):
        if (command == 'bootstrap'):

            self._refresh_pool_state()
            for p in Pool.objects.all():
                import_shares(p, request)

            for share in Share.objects.all():
                try:
                    if (share.pqgroup == settings.MODEL_DEFS['pqgroup']):
                        share.pqgroup = qgroup_create(share.pool)
                        share.save()
                    if (not is_share_mounted(share.name)):
                        mnt_pt = ('%s%s' % (settings.MNT_PT, share.name))
                        mount_share(share, mnt_pt)
                except Exception, e:
                    e_msg = ('Exception while mounting a share(%s) during '
                             'bootstrap: %s' % (share.name, e.__str__()))
                    logger.error(e_msg)
                    logger.exception(e)

                try:
                    import_snapshots(share)
                except Exception, e:
                    e_msg = ('Exception while importing Snapshots of '
                             'Share(%s): %s' % (share.name, e.__str__()))
                    logger.error(e_msg)
                    logger.exception(e)
Пример #3
0
def import_shares(pool, request):
    disk = Disk.objects.filter(pool=pool)[0].name
    shares = [s.name for s in Share.objects.filter(pool=pool)]
    shares_d = shares_info('%s%s' % (settings.MNT_PT, pool.name))
    for s in shares:
        if (s not in shares_d):
            Share.objects.get(pool=pool, name=s).delete()
    for s in shares_d:
        if (s in shares):
            share = Share.objects.get(name=s)
            share.qgroup = shares_d[s]
            rusage, eusage = share_usage(pool, share.qgroup)
            ts = datetime.utcnow().replace(tzinfo=utc)
            if (rusage != share.rusage or eusage != share.eusage):
                share.rusage = rusage
                share.eusage = eusage
                su = ShareUsage(name=s, r_usage=rusage, e_usage=eusage,
                                ts=ts)
                su.save()
            else:
                try:
                    su = ShareUsage.objects.filter(name=s).latest('id')
                    su.ts = ts
                    su.count += 1
                except ShareUsage.DoesNotExist:
                    su = ShareUsage(name=s, r_usage=rusage,
                                    e_usage=eusage, ts=ts)
                finally:
                    su.save()
            share.save()
            continue
        try:
            cshare = Share.objects.get(name=s)
            cshares_d = shares_info('%s%s' % (settings.MNT_PT,
                                              cshare.pool.name))
            if (s in cshares_d):
                e_msg = ('Another pool(%s) has a Share with this same '
                         'name(%s) as this pool(%s). This configuration is not supported.'
                         ' You can delete one of them manually with this command: '
                         'btrfs subvol delete %s[pool name]/%s' %
                         (cshare.pool.name, s, pool.name, settings.MNT_PT, s))
                handle_exception(Exception(e_msg), request)
            else:
                cshare.pool = pool
                cshare.qgroup = shares_d[s]
                cshare.size = pool.size
                cshare.subvol_name = s
                cshare.rusage, cshare.eusage = share_usage(pool, cshare.qgroup)
                cshare.save()
        except Share.DoesNotExist:
            pqid = qgroup_create(pool)
            update_quota(pool, pqid, pool.size * 1024)
            nso = Share(pool=pool, qgroup=shares_d[s], pqgroup=pqid, name=s,
                        size=pool.size, subvol_name=s)
            nso.save()
        mount_share(nso, '%s%s' % (settings.MNT_PT, s))
Пример #4
0
def import_shares(pool, request):
    disk = Disk.objects.filter(pool=pool)[0].name
    shares = [s.name for s in Share.objects.filter(pool=pool)]
    shares_d = shares_info(pool)
    for s in shares:
        if (s not in shares_d):
            Share.objects.get(pool=pool, name=s).delete()
    for s in shares_d:
        if (s in shares):
            share = Share.objects.get(name=s)
            share.qgroup = shares_d[s]
            rusage, eusage = share_usage(pool, share.qgroup)
            ts = datetime.utcnow().replace(tzinfo=utc)
            if (rusage != share.rusage or eusage != share.eusage):
                share.rusage = rusage
                share.eusage = eusage
                su = ShareUsage(name=s, r_usage=rusage, e_usage=eusage,
                                ts=ts)
                su.save()
            else:
                try:
                    su = ShareUsage.objects.filter(name=s).latest('id')
                    su.ts = ts
                    su.count += 1
                except ShareUsage.DoesNotExist:
                    su = ShareUsage(name=s, r_usage=rusage,
                                    e_usage=eusage, ts=ts)
                finally:
                    su.save()
            share.save()
            continue
        try:
            cshare = Share.objects.get(name=s)
            cshares_d = shares_info('%s%s' % (settings.MNT_PT,
                                              cshare.pool.name))
            if (s in cshares_d):
                e_msg = ('Another pool(%s) has a Share with this same '
                         'name(%s) as this pool(%s). This configuration is not supported.'
                         ' You can delete one of them manually with this command: '
                         'btrfs subvol delete %s[pool name]/%s' %
                         (cshare.pool.name, s, pool.name, settings.MNT_PT, s))
                handle_exception(Exception(e_msg), request)
            else:
                cshare.pool = pool
                cshare.qgroup = shares_d[s]
                cshare.size = pool.size
                cshare.subvol_name = s
                cshare.rusage, cshare.eusage = share_usage(pool, cshare.qgroup)
                cshare.save()
        except Share.DoesNotExist:
            pqid = qgroup_create(pool)
            update_quota(pool, pqid, pool.size * 1024)
            nso = Share(pool=pool, qgroup=shares_d[s], pqgroup=pqid, name=s,
                        size=pool.size, subvol_name=s)
            nso.save()
        mount_share(nso, '%s%s' % (settings.MNT_PT, s))
Пример #5
0
 def put(self, request, sid):
     with self._handle_exception(request):
         share = self._validate_share(request, sid)
         if ('size' in request.data):
             new_size = self._validate_share_size(request, share.pool)
             qid = qgroup_id(share.pool, share.subvol_name)
             cur_rusage, cur_eusage = volume_usage(share.pool, qid)
             if (new_size < cur_rusage):
                 e_msg = ('Unable to resize because requested new '
                          'size {} KB is less than current usage {} KB '
                          'of the share.').format(new_size, cur_rusage)
                 handle_exception(Exception(e_msg), request)
             # quota maintenance
             if share.pool.quotas_enabled:
                 # Only try create / update quotas if they are enabled,
                 # pqgroup of PQGROUP_DEFAULT (-1/-1) indicates no pqgroup,
                 # ie quotas were disabled when update was requested.
                 if share.pqgroup == PQGROUP_DEFAULT or \
                         not share.pqgroup_exist:
                     # if quotas were disabled or pqgroup non-existent.
                     share.pqgroup = qgroup_create(share.pool)
                     share.save()
                 if share.pqgroup is not PQGROUP_DEFAULT:
                     # Only update quota and assign if now non default as
                     # default can also indicate Read-only fs at this point.
                     update_quota(share.pool, share.pqgroup,
                                  new_size * 1024)
                     share_pqgroup_assign(share.pqgroup, share)
             else:
                 # Our pool's quotas are disabled so reset pqgroup to -1/-1.
                 if share.pqgroup != PQGROUP_DEFAULT:
                     # Only reset if necessary
                     share.pqgroup = PQGROUP_DEFAULT
                     share.save()
             share.size = new_size
         if ('compression' in request.data):
             new_compression = self._validate_compression(request)
             if (share.compression_algo != new_compression):
                 share.compression_algo = new_compression
                 mnt_pt = '%s%s' % (settings.MNT_PT, share.name)
                 if (new_compression == 'no'):
                     new_compression = ''
                 set_property(mnt_pt, 'compression', new_compression)
         share.save()
         return Response(ShareSerializer(share).data)
Пример #6
0
class CommandView(APIView):
    authentication_classes = (
        DigestAuthentication,
        SessionAuthentication,
        BasicAuthentication,
        OAuth2Authentication,
    )
    permission_classes = (IsAuthenticated, )

    def post(self, request, command):
        if (command == 'bootstrap'):
            try:
                device_scan()
            except Exception, e:
                e_msg = ('Unabled to scan disk drives on the system.')
                logger.error(e_msg)
                logger.exception(e)
                handle_exception(Exception(e_msg), request)

            for pool in Pool.objects.all():
                disk = Disk.objects.filter(pool=pool)[0].name
                try:
                    mount_root(pool, '/dev/%s' % disk)
                except Exception, e:
                    e_msg = ('Unable to mount a pool(%s) during bootstrap.' %
                             pool.name)
                    logger.exception(e)

            for share in Share.objects.all():
                try:
                    if (share.pqgroup == settings.MODEL_DEFS['pqgroup']):
                        share.pqgroup = qgroup_create(share.pool)
                        share.save()
                    if (not is_share_mounted(share.name)):
                        mnt_pt = ('%s%s' % (settings.MNT_PT, share.name))
                        pool_device = Disk.objects.filter(
                            pool=share.pool)[0].name
                        mount_share(share, pool_device, mnt_pt)

                except Exception, e:
                    e_msg = (
                        'Unable to mount a share(%s, %s) during bootstrap.' %
                        (pool_device, mnt_pt))
                    logger.error(e_msg)
                    logger.exception(e)
Пример #7
0
def create_clone(share, new_name, request, logger, snapshot=None):
    # if snapshot is None, create clone of the share.
    # If it's not, then clone it.
    if re.match(settings.SHARE_REGEX + "$", new_name) is None:
        e_msg = ("Clone name is invalid. It must start with a letter and can "
                 "contain letters, digits, _, . and - characters.")
        handle_exception(Exception(e_msg), request)
    if Share.objects.filter(name=new_name).exists():
        e_msg = "Another share with name ({}) already exists.".format(new_name)
        handle_exception(Exception(e_msg), request)
    if Snapshot.objects.filter(share=share, name=new_name).exists():
        e_msg = ("Snapshot with name ({}) already exists for the "
                 "share ({}). Choose a different name.").format(
                     new_name, share.name)
        handle_exception(Exception(e_msg), request)

    try:
        share_name = share.subvol_name
        snap = None
        if snapshot is not None:
            snap = snapshot.real_name
        add_clone(share.pool, share_name, new_name, snapshot=snap)
        snap_id = share_id(share.pool, new_name)
        qgroup_id = "0/{}".format(snap_id)
        pqid = qgroup_create(share.pool)
        new_share = Share(
            pool=share.pool,
            qgroup=qgroup_id,
            pqgroup=pqid,
            name=new_name,
            size=share.size,
            subvol_name=new_name,
        )
        new_share.save()
        if pqid != PQGROUP_DEFAULT:
            update_quota(new_share.pool, pqid, new_share.size * 1024)
            share_pqgroup_assign(pqid, new_share)
        # Mount our new clone share.
        # We independently mount all shares, data pool or system pool, in /mnt2/name
        mnt_pt = "{}{}".format(settings.MNT_PT, new_name)
        mount_share(new_share, mnt_pt)
        return Response(ShareSerializer(new_share).data)
    except Exception as e:
        handle_exception(e, request)
Пример #8
0
def btrfs_add_share(share):
	pools = btrfs_pool_scan()
	share_s = {}
	for pool in pools:
		if pool["name"] == share["pool"]:
			pqid = qgroup_create(pool)
			add_share(pool, share["sname"], pqid)
			qid = qgroup_id(pool, share["sname"])
			update_quota(pool, pqid, share["size"] * 1024)
			mnt_pt = '%s%s' % (MNT_PT, share["sname"])
			replica = False
			share_s["name"] = share["sname"]
			share_s["subvol_name"] = share["sname"]
			share_s["size"] = share["size"]
			share_s["qgroup"] = qid
			share_s["pqgroup"] = pqid
			share_s["pool"] = pool
			share_s["replica"] = replica
			share_s["compression_algo"] = share["compression"]
			mount_share(share_s, mnt_pt)
	return share_s
Пример #9
0
def btrfs_add_share(share):
    pools = btrfs_pool_scan()
    share_s = {}
    for pool in pools:
        if pool["name"] == share["pool"]:
            pqid = qgroup_create(pool)
            add_share(pool, share["sname"], pqid)
            qid = qgroup_id(pool, share["sname"])
            update_quota(pool, pqid, share["size"] * 1024)
            mnt_pt = '%s%s' % (MNT_PT, share["sname"])
            replica = False
            share_s["name"] = share["sname"]
            share_s["subvol_name"] = share["sname"]
            share_s["size"] = share["size"]
            share_s["qgroup"] = qid
            share_s["pqgroup"] = pqid
            share_s["pool"] = pool
            share_s["replica"] = replica
            share_s["compression_algo"] = share["compression"]
            mount_share(share_s, mnt_pt)
    return share_s
Пример #10
0
def create_clone(share, new_name, request, logger, snapshot=None):
    # if snapshot is None, create clone of the share.
    # If it's not, then clone it.
    if (re.match(settings.SHARE_REGEX + '$', new_name) is None):
        e_msg = ('Clone name is invalid. It must start with a letter and can'
                 ' contain letters, digits, _, . and - characters')
        handle_exception(Exception(e_msg), request)
    if (Share.objects.filter(name=new_name).exists()):
        e_msg = ('Another Share with name: %s already exists.' % new_name)
        handle_exception(Exception(e_msg), request)
    if (Snapshot.objects.filter(share=share, name=new_name).exists()):
        e_msg = ('Snapshot with name: %s already exists for the '
                 'share: %s. Choose a different name' % (new_name, share.name))
        handle_exception(Exception(e_msg), request)

    try:
        share_name = share.subvol_name
        snap = None
        if (snapshot is not None):
            snap = snapshot.real_name
        add_clone(share.pool, share_name, new_name, snapshot=snap)
        snap_id = share_id(share.pool, new_name)
        qgroup_id = ('0/%s' % snap_id)
        pqid = qgroup_create(share.pool)
        new_share = Share(pool=share.pool,
                          qgroup=qgroup_id,
                          pqgroup=pqid,
                          name=new_name,
                          size=share.size,
                          subvol_name=new_name)
        new_share.save()
        if pqid is not settings.MODEL_DEFS['pqgroup']:
            update_quota(new_share.pool, pqid, new_share.size * 1024)
            share_pqgroup_assign(pqid, new_share)
        # Mount our new clone share.
        mnt_pt = '{}{}'.format(settings.MNT_PT, new_name)
        mount_share(new_share, mnt_pt)
        return Response(ShareSerializer(new_share).data)
    except Exception as e:
        handle_exception(e, request)
Пример #11
0
    def post(self, request, command):
        if (command == 'bootstrap'):
            for pool in Pool.objects.all():
                try:
                    mount_root(pool)
                except Exception, e:
                    e_msg = ('Exception while mounting a pool(%s) during '
                             'bootstrap: %s' % (pool.name, e.__str__()))
                    logger.error(e_msg)

            for share in Share.objects.all():
                try:
                    if (share.pqgroup == settings.MODEL_DEFS['pqgroup']):
                        share.pqgroup = qgroup_create(share.pool)
                        share.save()
                    if (not is_share_mounted(share.name)):
                        mnt_pt = ('%s%s' % (settings.MNT_PT, share.name))
                        mount_share(share, mnt_pt)
                except Exception, e:
                    e_msg = ('Exception while mounting a share(%s) during '
                             'bootstrap: %s' % (share.name, e.__str__()))
                    logger.error(e_msg)
Пример #12
0
    def post(self, request, command):
        if (command == 'bootstrap'):

            for pool in Pool.objects.all():
                try:
                    mount_root(pool)
                except Exception, e:
                    e_msg = ('Exception while mounting a pool(%s) during '
                             'bootstrap: %s' % (pool.name, e.__str__()))
                    logger.error(e_msg)

            for share in Share.objects.all():
                try:
                    if (share.pqgroup == settings.MODEL_DEFS['pqgroup']):
                        share.pqgroup = qgroup_create(share.pool)
                        share.save()
                    if (not is_share_mounted(share.name)):
                        mnt_pt = ('%s%s' % (settings.MNT_PT, share.name))
                        mount_share(share, mnt_pt)
                except Exception, e:
                    e_msg = ('Exception while mounting a share(%s) during '
                             'bootstrap: %s' % (share.name, e.__str__()))
                    logger.error(e_msg)
Пример #13
0
def create_clone(share, new_name, request, logger, snapshot=None):
    # if snapshot is None, create clone of the share.
    # If it's not, then clone it.
    if (re.match(settings.SHARE_REGEX + '$', new_name) is None):
        e_msg = ('Clone name is invalid. It must start with a letter and can '
                 'contain letters, digits, _, . and - characters.')
        handle_exception(Exception(e_msg), request)
    if (Share.objects.filter(name=new_name).exists()):
        e_msg = 'Another share with name ({}) already exists.'.format(new_name)
        handle_exception(Exception(e_msg), request)
    if (Snapshot.objects.filter(share=share, name=new_name).exists()):
        e_msg = ('Snapshot with name ({}) already exists for the '
                 'share ({}). Choose a different name.').format(new_name,
                                                                share.name)
        handle_exception(Exception(e_msg), request)

    try:
        share_name = share.subvol_name
        snap = None
        if (snapshot is not None):
            snap = snapshot.real_name
        add_clone(share.pool, share_name, new_name, snapshot=snap)
        snap_id = share_id(share.pool, new_name)
        qgroup_id = ('0/%s' % snap_id)
        pqid = qgroup_create(share.pool)
        new_share = Share(pool=share.pool, qgroup=qgroup_id, pqgroup=pqid,
                          name=new_name, size=share.size, subvol_name=new_name)
        new_share.save()
        if pqid is not PQGROUP_DEFAULT:
            update_quota(new_share.pool, pqid, new_share.size * 1024)
            share_pqgroup_assign(pqid, new_share)
        # Mount our new clone share.
        mnt_pt = '{}{}'.format(settings.MNT_PT, new_name)
        mount_share(new_share, mnt_pt)
        return Response(ShareSerializer(new_share).data)
    except Exception as e:
        handle_exception(e, request)
Пример #14
0
    def post(self, request):
        # qgroup notes for shares. we need to create a qgroup prior to share
        # creation. qgroup ids 0/<subvol_id> automatically get created when a
        # subvolume(i.e., a Share or a Snapshot) is created. So let's create a
        # new qgroup: 2015/<some_number> whenever a Share is
        # created. <some_number> starts from 1 and is incremented as more
        # Shares are created. So, for the very first Share in a pool, it's
        # qgroup will be 1/1. 2015 is arbitrarily chose.

        # Before creating a new Share, we create the qgroup for it. And during
        # it's creation, we assign this qgroup to it. During it's creation a
        # 0/x qgroup will automatically be created, but it will become the
        # child of our explicitly-created qgroup(2015/x).

        # We will set the qgroup limit on our qgroup and it will enforce the
        # quota on every subvolume(i.e., Share and Snapshot) in that qgroup.

        # When a Share is deleted, we need to destroy two qgroups. One is it's
        # auto 0/x qgroup and the other is our explicitly-created 2015/y
        # qgroup.

        with self._handle_exception(request):
            pool_name = request.data.get('pool', None)
            try:
                pool = Pool.objects.get(name=pool_name)
            except:
                e_msg = 'Pool ({}) does not exist.'.format(pool_name)
                handle_exception(Exception(e_msg), request)
            compression = self._validate_compression(request)
            size = self._validate_share_size(request, pool)
            sname = request.data.get('sname', None)
            if ((sname is None or
                 re.match('%s$' % settings.SHARE_REGEX, sname) is None)):
                e_msg = ('Invalid characters in share name. Following are '
                         'allowed: letter(a-z or A-Z), digit(0-9), '
                         'hyphen(-), underscore(_) or a period(.).')
                handle_exception(Exception(e_msg), request)

            if (len(sname) > 254):
                # btrfs subvolume names cannot exceed 254 characters.
                e_msg = ('Share name length cannot exceed 254 characters.')
                handle_exception(Exception(e_msg), request)

            if (Share.objects.filter(name=sname).exists()):
                # Note e_msg is consumed by replication/util.py create_share()
                e_msg = ('Share ({}) already exists. Choose a '
                         'different name.').format(sname)
                handle_exception(Exception(e_msg), request)

            if (Pool.objects.filter(name=sname).exists()):
                e_msg = ('A pool with this name ({}) exists. Share '
                         'and pool names must be distinct. Choose '
                         'a different name.').format(sname)
                handle_exception(Exception(e_msg), request)
            replica = False
            if ('replica' in request.data):
                replica = request.data['replica']
                if (type(replica) != bool):
                    # TODO: confirm this 'type' call works as format parameter.
                    e_msg = ('Replica must be a boolean, '
                             'not ({}).').format(type(replica))
                    handle_exception(Exception(e_msg), request)
            pqid = qgroup_create(pool)
            add_share(pool, sname, pqid)
            qid = qgroup_id(pool, sname)
            s = Share(pool=pool, qgroup=qid, pqgroup=pqid, name=sname,
                      size=size, subvol_name=sname, replica=replica,
                      compression_algo=compression)
            # The following pool.save() was informed by test_share.py
            pool.save()
            s.save()
            if pqid is not PQGROUP_DEFAULT:
                update_quota(pool, pqid, size * 1024)
                share_pqgroup_assign(pqid, s)
            mnt_pt = '%s%s' % (settings.MNT_PT, sname)
            if not s.is_mounted:
                mount_share(s, mnt_pt)
            if (compression != 'no'):
                set_property(mnt_pt, 'compression', compression)
            return Response(ShareSerializer(s).data)
Пример #15
0
def import_shares(pool, request):
    # Establish known shares/subvols within our db for the given pool:
    shares_in_pool_db = [s.name for s in Share.objects.filter(pool=pool)]
    # Find the actual/current shares/subvols within the given pool:
    # Limited to Rockstor relevant subvols ie shares and clones.
    shares_in_pool = shares_info(pool)
    # List of pool's share.pqgroups so we can remove inadvertent duplication.
    # All pqgroups are removed when quotas are disabled, combined with a part
    # refresh we could have duplicates within the db.
    share_pqgroups_used = []
    # Delete db Share object if it is no longer found on disk.
    for s_in_pool_db in shares_in_pool_db:
        if s_in_pool_db not in shares_in_pool:
            logger.debug("Removing, missing on disk, share db entry ({}) from "
                         "pool ({}).".format(s_in_pool_db, pool.name))
            Share.objects.get(pool=pool, name=s_in_pool_db).delete()
    # Check if each share in pool also has a db counterpart.
    for s_in_pool in shares_in_pool:
        logger.debug("---- Share name = {}.".format(s_in_pool))
        if s_in_pool in shares_in_pool_db:
            logger.debug("Updating pre-existing same pool db share entry.")
            # We have a pool db share counterpart so retrieve and update it.
            share = Share.objects.get(name=s_in_pool, pool=pool)
            # Initially default our pqgroup value to db default of '-1/-1'
            # This way, unless quotas are enabled, all pqgroups will be
            # returned to db default.
            pqgroup = PQGROUP_DEFAULT
            if share.pool.quotas_enabled:
                # Quotas are enabled on our pool so we can validate pqgroup.
                if (share.pqgroup == pqgroup or not share.pqgroup_exist
                        or share.pqgroup in share_pqgroups_used):
                    # we have a void '-1/-1' or non existent pqgroup or
                    # this pqgroup has already been seen / used in this pool.
                    logger.debug(
                        "#### replacing void, non-existent, or duplicate pqgroup."
                    )
                    pqgroup = qgroup_create(pool)
                    if pqgroup != PQGROUP_DEFAULT:
                        update_quota(pool, pqgroup, share.size * 1024)
                        share_pqgroup_assign(pqgroup, share)
                else:
                    # Our share's pqgroup looks OK so use it.
                    pqgroup = share.pqgroup
                # Record our use of this pqgroup to spot duplicates later.
                share_pqgroups_used.append(deepcopy(share.pqgroup))
            if share.pqgroup != pqgroup:
                # we need to update our share.pqgroup
                share.pqgroup = pqgroup
                share.save()
            share.qgroup = shares_in_pool[s_in_pool]
            rusage, eusage, pqgroup_rusage, pqgroup_eusage = volume_usage(
                pool, share.qgroup, pqgroup)
            if (rusage != share.rusage or eusage != share.eusage
                    or pqgroup_rusage != share.pqgroup_rusage
                    or pqgroup_eusage != share.pqgroup_eusage):
                share.rusage = rusage
                share.eusage = eusage
                share.pqgroup_rusage = pqgroup_rusage
                share.pqgroup_eusage = pqgroup_eusage
                update_shareusage_db(s_in_pool, rusage, eusage)
            else:
                update_shareusage_db(s_in_pool, rusage, eusage, UPDATE_TS)
            share.save()
            continue
        try:
            logger.debug("No prior entries in scanned pool trying all pools.")
            # Test (Try) for an existing system wide Share db entry.
            cshare = Share.objects.get(name=s_in_pool)
            # Get a list of Rockstor relevant subvols (ie shares and clones)
            # for the prior existing db share entry's pool.
            cshares_d = shares_info(cshare.pool)
            if s_in_pool in cshares_d:
                e_msg = ("Another pool ({}) has a share with this same "
                         "name ({}) as this pool ({}). This configuration "
                         "is not supported. You can delete one of them "
                         "manually with the following command: "
                         '"btrfs subvol delete {}[pool name]/{}" WARNING this '
                         "will remove the entire contents of that "
                         "subvolume.").format(cshare.pool.name, s_in_pool,
                                              pool.name, settings.MNT_PT,
                                              s_in_pool)
                handle_exception(Exception(e_msg), request)
            else:
                # Update the prior existing db share entry previously
                # associated with another pool.
                logger.debug("Updating prior db entry from another pool.")
                cshare.pool = pool
                cshare.qgroup = shares_in_pool[s_in_pool]
                cshare.size = pool.size
                cshare.subvol_name = s_in_pool
                (
                    cshare.rusage,
                    cshare.eusage,
                    cshare.pqgroup_rusage,
                    cshare.pqgroup_eusage,
                ) = volume_usage(pool, cshare.qgroup, cshare.pqgroup)
                cshare.save()
                update_shareusage_db(s_in_pool, cshare.rusage, cshare.eusage)
        except Share.DoesNotExist:
            logger.debug("Db share entry does not exist - creating.")
            # We have a share on disk that has no db counterpart so create one.
            # Retrieve new pool quota id for use in db Share object creation.
            # As the replication receive share is 'special' we tag it as such.
            replica = False
            share_name = s_in_pool
            if re.match(".snapshot", s_in_pool) is not None:
                # We have an initial replication share, non snap in .snapshots.
                # We could change it's name here but still a little mixing
                # of name and subvol throughout project.
                replica = True
                logger.debug("Initial receive quirk-subvol found: Importing "
                             "as share and setting replica flag.")
            qid = shares_in_pool[s_in_pool]
            pqid = qgroup_create(pool)
            if pqid != PQGROUP_DEFAULT:
                update_quota(pool, pqid, pool.size * 1024)
                qgroup_assign(qid, pqid, pool.mnt_pt)
            rusage, eusage, pqgroup_rusage, pqgroup_eusage = volume_usage(
                pool, qid, pqid)
            nso = Share(
                pool=pool,
                qgroup=qid,
                pqgroup=pqid,
                name=share_name,
                size=pool.size,
                subvol_name=s_in_pool,
                rusage=rusage,
                eusage=eusage,
                pqgroup_rusage=pqgroup_rusage,
                pqgroup_eusage=pqgroup_eusage,
                replica=replica,
            )
            nso.save()
            update_shareusage_db(s_in_pool, rusage, eusage)
            mount_share(nso, "{}{}".format(settings.MNT_PT, s_in_pool))
Пример #16
0
    def post(self, request):
        #qgroup notes for shares. we need to create a qgroup prior to share
        #creation. qgroup ids 0/<subvol_id> automatically get created when a
        #subvolume(i.e., a Share or a Snapshot) is created. So let's create a
        #new qgroup: 2015/<some_number> whenever a Share is
        #created. <some_number> starts from 1 and is incremented as more Shares
        #are created. So, for the very first Share in a pool, it's qgroup will
        #be 1/1. 2015 is arbitrarily chose.

        #Before creating a new Share, we create the qgroup for it. And during
        #it's creation, we assign this qgroup to it. During it's creation a 0/x
        #qgroup will automatically be created, but it will become the child of
        #our explicitly-created qgroup(2015/x).

        #We will set the qgroup limit on our qgroup and it will enforce the
        #quota on every subvolume(i.e., Share and Snapshot) in that qgroup.

        #When a Share is deleted, we need to destroy two qgroups. One is it's
        #auto 0/x qgroup and the other is our explicitly-created 2015/y qgroup.

        with self._handle_exception(request):
            pool_name = request.data.get('pool', None)
            try:
                pool = Pool.objects.get(name=pool_name)
            except:
                e_msg = ('Pool(%s) does not exist.' % pool_name)
                handle_exception(Exception(e_msg), request)
            compression = self._validate_compression(request)
            size = self._validate_share_size(request, pool)
            sname = request.data.get('sname', None)
            if ((sname is None or
                 re.match('%s$' % settings.SHARE_REGEX, sname) is None)):
                e_msg = ('Share name must start with a alphanumeric(a-z0-9) '
                         'character and can be followed by any of the '
                         'following characters: letter(a-z), digits(0-9), '
                         'hyphen(-), underscore(_) or a period(.).')
                handle_exception(Exception(e_msg), request)

            if (len(sname) > 254):
                #btrfs subvolume names cannot exceed 254 characters.
                e_msg = ('Share name length cannot exceed 254 characters')
                handle_exception(Exception(e_msg), request)

            if (Share.objects.filter(name=sname).exists()):
                e_msg = ('Share(%s) already exists. Choose a different name' % sname)
                handle_exception(Exception(e_msg), request)

            if (Pool.objects.filter(name=sname).exists()):
                e_msg = ('A Pool with this name(%s) exists. Share and Pool names '
                         'must be distinct. Choose a different name' % sname)
                handle_exception(Exception(e_msg), request)
            disk = Disk.objects.filter(pool=pool)[0]
            replica = False
            if ('replica' in request.data):
                replica = request.data['replica']
                if (type(replica) != bool):
                    e_msg = ('replica must be a boolean, not %s' %
                             type(replica))
                    handle_exception(Exception(e_msg), request)
            pqid = qgroup_create(pool)
            add_share(pool, sname, pqid)
            qid = qgroup_id(pool, sname)
            update_quota(pool, pqid, size * 1024)
            s = Share(pool=pool, qgroup=qid, pqgroup=pqid, name=sname,
                      size=size, subvol_name=sname, replica=replica,
                      compression_algo=compression)
            s.save()
            mnt_pt = '%s%s' % (settings.MNT_PT, sname)
            if (not is_share_mounted(sname)):
                mount_share(s, mnt_pt)
            if (compression != 'no'):
                set_property(mnt_pt, 'compression', compression)
            return Response(ShareSerializer(s).data)
Пример #17
0
    def post(self, request):
        #qgroup notes for shares. we need to create a qgroup prior to share
        #creation. qgroup ids 0/<subvol_id> automatically get created when a
        #subvolume(i.e., a Share or a Snapshot) is created. So let's create a
        #new qgroup: 2015/<some_number> whenever a Share is
        #created. <some_number> starts from 1 and is incremented as more Shares
        #are created. So, for the very first Share in a pool, it's qgroup will
        #be 1/1. 2015 is arbitrarily chose.

        #Before creating a new Share, we create the qgroup for it. And during
        #it's creation, we assign this qgroup to it. During it's creation a 0/x
        #qgroup will automatically be created, but it will become the child of
        #our explicitly-created qgroup(2015/x).

        #We will set the qgroup limit on our qgroup and it will enforce the
        #quota on every subvolume(i.e., Share and Snapshot) in that qgroup.

        #When a Share is deleted, we need to destroy two qgroups. One is it's
        #auto 0/x qgroup and the other is our explicitly-created 2015/y qgroup.

        with self._handle_exception(request):
            pool_name = request.data.get('pool', None)
            try:
                pool = Pool.objects.get(name=pool_name)
            except:
                e_msg = ('Pool(%s) does not exist.' % pool_name)
                handle_exception(Exception(e_msg), request)
            compression = self._validate_compression(request)
            size = self._validate_share_size(request, pool)
            sname = request.data.get('sname', None)
            if ((sname is None
                 or re.match('%s$' % settings.SHARE_REGEX, sname) is None)):
                e_msg = ('Share name must start with a alphanumeric(a-z0-9) '
                         'character and can be followed by any of the '
                         'following characters: letter(a-z), digits(0-9), '
                         'hyphen(-), underscore(_) or a period(.).')
                handle_exception(Exception(e_msg), request)

            if (len(sname) > 254):
                #btrfs subvolume names cannot exceed 254 characters.
                e_msg = ('Share name length cannot exceed 254 characters')
                handle_exception(Exception(e_msg), request)

            if (Share.objects.filter(name=sname).exists()):
                e_msg = ('Share(%s) already exists. Choose a different name' %
                         sname)
                handle_exception(Exception(e_msg), request)

            if (Pool.objects.filter(name=sname).exists()):
                e_msg = (
                    'A Pool with this name(%s) exists. Share and Pool names '
                    'must be distinct. Choose a different name' % sname)
                handle_exception(Exception(e_msg), request)
            disk = Disk.objects.filter(pool=pool)[0]
            replica = False
            if ('replica' in request.data):
                replica = request.data['replica']
                if (type(replica) != bool):
                    e_msg = ('replica must be a boolean, not %s' %
                             type(replica))
                    handle_exception(Exception(e_msg), request)
            pqid = qgroup_create(pool)
            add_share(pool, sname, pqid)
            qid = qgroup_id(pool, sname)
            update_quota(pool, pqid, size * 1024)
            s = Share(pool=pool,
                      qgroup=qid,
                      pqgroup=pqid,
                      name=sname,
                      size=size,
                      subvol_name=sname,
                      replica=replica,
                      compression_algo=compression)
            s.save()
            mnt_pt = '%s%s' % (settings.MNT_PT, sname)
            if (not is_share_mounted(sname)):
                mount_share(s, mnt_pt)
            if (compression != 'no'):
                set_property(mnt_pt, 'compression', compression)
            return Response(ShareSerializer(s).data)
Пример #18
0
def import_shares(pool, request):
    # Establish known shares/subvols within our db for the given pool:
    shares_in_pool_db = [s.name for s in Share.objects.filter(pool=pool)]
    # Find the actual/current shares/subvols within the given pool:
    # Limited to Rockstor relevant subvols ie shares and clones.
    shares_in_pool = shares_info(pool)
    # Delete db Share object if it is no longer found on disk.
    for s_in_pool_db in shares_in_pool_db:
        if s_in_pool_db not in shares_in_pool:
            Share.objects.get(pool=pool, name=s_in_pool_db).delete()
    # Check if each share in pool also has a db counterpart.
    for s_in_pool in shares_in_pool:
        logger.debug('Share name = {}.'.format(s_in_pool))
        if s_in_pool in shares_in_pool_db:
            logger.debug('Updating pre-existing same pool db share entry.')
            # We have a pool db share counterpart so retrieve and update it.
            share = Share.objects.get(name=s_in_pool, pool=pool)
            share.qgroup = shares_in_pool[s_in_pool]
            rusage, eusage, pqgroup_rusage, pqgroup_eusage = \
                volume_usage(pool, share.qgroup, share.pqgroup)
            if (rusage != share.rusage or eusage != share.eusage
                    or pqgroup_rusage != share.pqgroup_rusage
                    or pqgroup_eusage != share.pqgroup_eusage):
                share.rusage = rusage
                share.eusage = eusage
                share.pqgroup_rusage = pqgroup_rusage
                share.pqgroup_eusage = pqgroup_eusage
                update_shareusage_db(s_in_pool, rusage, eusage)
            else:
                update_shareusage_db(s_in_pool, rusage, eusage, UPDATE_TS)
            share.save()
            continue
        try:
            logger.debug('No prior entries in scanned pool trying all pools.')
            # Test (Try) for an existing system wide Share db entry.
            cshare = Share.objects.get(name=s_in_pool)
            # Get a list of Rockstor relevant subvols (ie shares and clones)
            # for the prior existing db share entry's pool.
            cshares_d = shares_info(cshare.pool)
            if s_in_pool in cshares_d:
                e_msg = ('Another pool ({}) has a Share with this same '
                         'name ({}) as this pool ({}). This configuration '
                         'is not supported. You can delete one of them '
                         'manually with the following command: '
                         '"btrfs subvol delete {}[pool name]/{}" WARNING this '
                         'will remove the entire contents of that subvolume.'.
                         format(cshare.pool.name, s_in_pool, pool.name,
                                settings.MNT_PT, s_in_pool))
                handle_exception(Exception(e_msg), request)
            else:
                # Update the prior existing db share entry previously
                # associated with another pool.
                logger.debug('Updating prior db entry from another pool.')
                cshare.pool = pool
                cshare.qgroup = shares_in_pool[s_in_pool]
                cshare.size = pool.size
                cshare.subvol_name = s_in_pool
                (cshare.rusage, cshare.eusage, cshare.pqgroup_rusage,
                 cshare.pqgroup_eusage) = volume_usage(pool, cshare.qgroup,
                                                       cshare.pqgroup)
                cshare.save()
                update_shareusage_db(s_in_pool, cshare.rusage, cshare.eusage)
        except Share.DoesNotExist:
            logger.debug('Db share entry does not exist - creating.')
            # We have a share on disk that has no db counterpart so create one.
            # Retrieve pool quota id for use in db Share object creation.
            pqid = qgroup_create(pool)
            update_quota(pool, pqid, pool.size * 1024)
            rusage, eusage, pqgroup_rusage, pqgroup_eusage = \
                volume_usage(pool, shares_in_pool[s_in_pool], pqid)
            nso = Share(pool=pool,
                        qgroup=shares_in_pool[s_in_pool],
                        pqgroup=pqid,
                        name=s_in_pool,
                        size=pool.size,
                        subvol_name=s_in_pool,
                        rusage=rusage,
                        eusage=eusage,
                        pqgroup_rusage=pqgroup_rusage,
                        pqgroup_eusage=pqgroup_eusage)
            nso.save()
            update_shareusage_db(s_in_pool, rusage, eusage)
            mount_share(nso, '%s%s' % (settings.MNT_PT, s_in_pool))
Пример #19
0
    def post(self, request, command, rtcepoch=None):
        if (command == 'bootstrap'):
            self._update_disk_state()
            self._refresh_pool_state()
            for p in Pool.objects.all():
                if p.disk_set.attached().count() == 0:
                    continue
                import_shares(p, request)

            for share in Share.objects.all():
                if share.pool.disk_set.attached().count() == 0:
                    continue
                try:
                    if (share.pqgroup == settings.MODEL_DEFS['pqgroup']):
                        share.pqgroup = qgroup_create(share.pool)
                        share.save()
                    if not share.is_mounted:
                        mnt_pt = ('%s%s' % (settings.MNT_PT, share.name))
                        mount_share(share, mnt_pt)
                except Exception as e:
                    e_msg = ('Exception while mounting a share(%s) during '
                             'bootstrap: %s' % (share.name, e.__str__()))
                    logger.error(e_msg)
                    logger.exception(e)

                try:
                    import_snapshots(share)
                except Exception as e:
                    e_msg = ('Exception while importing Snapshots of '
                             'Share(%s): %s' % (share.name, e.__str__()))
                    logger.error(e_msg)
                    logger.exception(e)

            for snap in Snapshot.objects.all():
                if (snap.uvisible):
                    try:
                        mount_snap(snap.share, snap.real_name)
                    except Exception as e:
                        e_msg = ('Failed to make the Snapshot(%s) visible. '
                                 'Exception: %s' %
                                 (snap.real_name, e.__str__()))
                        logger.error(e_msg)

            mnt_map = sftp_mount_map(settings.SFTP_MNT_ROOT)
            for sftpo in SFTP.objects.all():
                try:
                    sftp_mount(sftpo.share, settings.MNT_PT,
                               settings.SFTP_MNT_ROOT, mnt_map, sftpo.editable)
                    sftp_snap_toggle(sftpo.share)
                except Exception as e:
                    e_msg = ('Exception while exportin a sftp share during '
                             'bootstrap: %s' % e.__str__())
                    logger.error(e_msg)

            try:
                adv_entries = [a.export_str for a in
                               AdvancedNFSExport.objects.all()]
                exports_d = self.create_adv_nfs_export_input(adv_entries,
                                                             request)
                exports = self.create_nfs_export_input(NFSExport.objects.all())
                exports.update(exports_d)
                self.refresh_wrapper(exports, request, logger)
            except Exception as e:
                e_msg = ('Exception while bootstrapping NFS: %s' % e.__str__())
                logger.error(e_msg)

            #  bootstrap services
            try:
                systemctl('firewalld', 'stop')
                systemctl('firewalld', 'disable')
                systemctl('nginx', 'stop')
                systemctl('nginx', 'disable')
                systemctl('atd', 'enable')
                systemctl('atd', 'start')
            except Exception as e:
                e_msg = ('Exception while setting service statuses during '
                         'bootstrap: %s' % e.__str__())
                logger.error(e_msg)
                handle_exception(Exception(e_msg), request)

            logger.debug('Bootstrap operations completed')
            return Response()

        if (command == 'utcnow'):
            return Response(datetime.utcnow().replace(tzinfo=utc))

        if (command == 'uptime'):
            return Response(uptime())

        if (command == 'kernel'):
            try:
                return Response(kernel_info(settings.SUPPORTED_KERNEL_VERSION))
            except Exception as e:
                handle_exception(e, request)

        if (command == 'update-check'):
            try:
                subo = None
                try:
                    subo = UpdateSubscription.objects.get(name='Stable',
                                                          status='active')
                except UpdateSubscription.DoesNotExist:
                    try:
                        subo = UpdateSubscription.objects.get(name='Testing',
                                                              status='active')
                    except UpdateSubscription.DoesNotExist:
                        pass
                return Response(update_check(subscription=subo))
            except Exception as e:
                e_msg = ('Unable to check update due to a system error: %s'
                         % e.__str__())
                handle_exception(Exception(e_msg), request)

        if (command == 'update'):
            try:
                # Once again, like on system shutdown/reboot, we filter
                # incoming requests with request.auth: every update from
                # WebUI misses request.auth, while yum update requests from
                # data_collector APIWrapper have it, so we can avoid
                # an additional command for yum updates
                if request.auth is None:
                    update_run()
                else:
                    update_run(yum_update=True)
                return Response('Done')
            except Exception as e:
                e_msg = ('Update failed due to this exception: %s'
                         % e.__str__())
                handle_exception(Exception(e_msg), request)

        if (command == 'current-version'):
            try:
                return Response(current_version())
            except Exception as e:
                e_msg = ('Unable to check current version due to this '
                         'exception: ' % e.__str__())
                handle_exception(Exception(e_msg), request)

        # default has shutdown and reboot with delay set to now
        # having normal sytem power off with now = 1 min
        # reboot and shutdown requests from WebUI don't have request.auth
        # while same requests over rest api (ex. scheduled tasks) have
        # an auth token, so if we detect a token we delay with 3 mins
        # to grant connected WebUI user to close it or cancel shutdown/reboot
        delay = 'now'
        if request.auth is not None:
            delay = 3

        if (command == 'shutdown'):
            msg = ('The system will now be shutdown')
            try:
                # if shutdown request coming from a scheduled task
                # with rtc wake up time on we set it before
                # system shutdown starting
                if rtcepoch is not None:
                    set_system_rtc_wake(rtcepoch)
                request.session.flush()
                system_shutdown(delay)
            except Exception as e:
                msg = ('Failed to shutdown the system due to a low level '
                       'error: %s' % e.__str__())
                handle_exception(Exception(msg), request)
            finally:
                return Response(msg)

        if (command == 'reboot'):
            msg = ('The system will now reboot')
            try:
                request.session.flush()
                system_reboot(delay)
            except Exception as e:
                msg = ('Failed to reboot the system due to a low level error: '
                       '%s' % e.__str__())
                handle_exception(Exception(msg), request)
            finally:
                return Response(msg)

        if (command == 'suspend'):
            msg = ('The system will now be suspended to RAM')
            try:
                request.session.flush()
                set_system_rtc_wake(rtcepoch)
                system_suspend()
            except Exception as e:
                msg = ('Failed to suspend the system due to a low level '
                       'error: %s' % e.__str__())
                handle_exception(Exception(msg), request)
            finally:
                return Response(msg)

        if (command == 'current-user'):
            return Response(request.user.username)

        if (command == 'auto-update-status'):
            status = True
            try:
                status = auto_update_status()
            except:
                status = False
            finally:
                return Response({'enabled': status, })

        if (command == 'enable-auto-update'):
            try:
                auto_update(enable=True)
                return Response({'enabled': True, })
            except Exception as e:
                msg = ('Failed to enable auto update due to this exception: '
                       '%s' % e.__str__())
                handle_exception(Exception(msg), request)

        if (command == 'disable-auto-update'):
            try:
                auto_update(enable=False)
                return Response({'enabled': False, })
            except Exception as e:
                msg = ('Failed to disable auto update due to this exception:  '
                       '%s' % e.__str__())
                handle_exception(Exception(msg), request)

        if (command == 'refresh-disk-state'):
            self._update_disk_state()
            return Response()

        if (command == 'refresh-pool-state'):
            self._refresh_pool_state()
            return Response()

        if (command == 'refresh-share-state'):
            for p in Pool.objects.all():
                import_shares(p, request)
            return Response()

        if (command == 'refresh-snapshot-state'):
            for share in Share.objects.all():
                import_snapshots(share)
            return Response()
Пример #20
0
def import_shares(pool, request):
    # Establish known shares/subvols within our db for the given pool:
    shares_in_pool_db = [s.name for s in Share.objects.filter(pool=pool)]
    # Find the actual/current shares/subvols within the given pool:
    # Limited to Rockstor relevant subvols ie shares and clones.
    shares_in_pool = shares_info(pool)
    # List of pool's share.pqgroups so we can remove inadvertent duplication.
    # All pqgroups are removed when quotas are disabled, combined with a part
    # refresh we could have duplicates within the db.
    share_pqgroups_used = []
    # Delete db Share object if it is no longer found on disk.
    for s_in_pool_db in shares_in_pool_db:
        if s_in_pool_db not in shares_in_pool:
            logger.debug('Removing, missing on disk, share db entry ({}) from '
                         'pool ({}).'.format(s_in_pool_db, pool.name))
            Share.objects.get(pool=pool, name=s_in_pool_db).delete()
    # Check if each share in pool also has a db counterpart.
    for s_in_pool in shares_in_pool:
        logger.debug('---- Share name = {}.'.format(s_in_pool))
        if s_in_pool in shares_in_pool_db:
            logger.debug('Updating pre-existing same pool db share entry.')
            # We have a pool db share counterpart so retrieve and update it.
            share = Share.objects.get(name=s_in_pool, pool=pool)
            # Initially default our pqgroup value to db default of '-1/-1'
            # This way, unless quotas are enabled, all pqgroups will be
            # returned to db default.
            pqgroup = PQGROUP_DEFAULT
            if share.pool.quotas_enabled:
                # Quotas are enabled on our pool so we can validate pqgroup.
                if share.pqgroup == pqgroup or not share.pqgroup_exist \
                        or share.pqgroup in share_pqgroups_used:
                    # we have a void '-1/-1' or non existent pqgroup or
                    # this pqgroup has already been seen / used in this pool.
                    logger.debug('#### replacing void, non-existent, or '
                                 'duplicate pqgroup.')
                    pqgroup = qgroup_create(pool)
                    if pqgroup is not PQGROUP_DEFAULT:
                        update_quota(pool, pqgroup, share.size * 1024)
                        share_pqgroup_assign(pqgroup, share)
                else:
                    # Our share's pqgroup looks OK so use it.
                    pqgroup = share.pqgroup
                # Record our use of this pqgroup to spot duplicates later.
                share_pqgroups_used.append(deepcopy(share.pqgroup))
            if share.pqgroup != pqgroup:
                # we need to update our share.pqgroup
                share.pqgroup = pqgroup
                share.save()
            share.qgroup = shares_in_pool[s_in_pool]
            rusage, eusage, pqgroup_rusage, pqgroup_eusage = \
                volume_usage(pool, share.qgroup, pqgroup)
            if (rusage != share.rusage or eusage != share.eusage or
               pqgroup_rusage != share.pqgroup_rusage or
               pqgroup_eusage != share.pqgroup_eusage):
                share.rusage = rusage
                share.eusage = eusage
                share.pqgroup_rusage = pqgroup_rusage
                share.pqgroup_eusage = pqgroup_eusage
                update_shareusage_db(s_in_pool, rusage, eusage)
            else:
                update_shareusage_db(s_in_pool, rusage, eusage, UPDATE_TS)
            share.save()
            continue
        try:
            logger.debug('No prior entries in scanned pool trying all pools.')
            # Test (Try) for an existing system wide Share db entry.
            cshare = Share.objects.get(name=s_in_pool)
            # Get a list of Rockstor relevant subvols (ie shares and clones)
            # for the prior existing db share entry's pool.
            cshares_d = shares_info(cshare.pool)
            if s_in_pool in cshares_d:
                e_msg = ('Another pool ({}) has a share with this same '
                         'name ({}) as this pool ({}). This configuration '
                         'is not supported. You can delete one of them '
                         'manually with the following command: '
                         '"btrfs subvol delete {}[pool name]/{}" WARNING this '
                         'will remove the entire contents of that '
                         'subvolume.').format(cshare.pool.name, s_in_pool,
                                              pool.name, settings.MNT_PT,
                                              s_in_pool)
                handle_exception(Exception(e_msg), request)
            else:
                # Update the prior existing db share entry previously
                # associated with another pool.
                logger.debug('Updating prior db entry from another pool.')
                cshare.pool = pool
                cshare.qgroup = shares_in_pool[s_in_pool]
                cshare.size = pool.size
                cshare.subvol_name = s_in_pool
                (cshare.rusage, cshare.eusage, cshare.pqgroup_rusage,
                 cshare.pqgroup_eusage) = volume_usage(pool, cshare.qgroup,
                                                       cshare.pqgroup)
                cshare.save()
                update_shareusage_db(s_in_pool, cshare.rusage, cshare.eusage)
        except Share.DoesNotExist:
            logger.debug('Db share entry does not exist - creating.')
            # We have a share on disk that has no db counterpart so create one.
            # Retrieve new pool quota id for use in db Share object creation.
            # As the replication receive share is 'special' we tag it as such.
            replica = False
            share_name = s_in_pool
            if re.match('.snapshot', s_in_pool) is not None:
                # We have an initial replication share, non snap in .snapshots.
                # We could change it's name here but still a little mixing
                # of name and subvol throughout project.
                replica = True
                logger.debug('Initial receive quirk-subvol found: Importing '
                             'as share and setting replica flag.')
            qid = shares_in_pool[s_in_pool]
            pqid = qgroup_create(pool)
            if pqid is not PQGROUP_DEFAULT:
                update_quota(pool, pqid, pool.size * 1024)
                pool_mnt_pt = '{}{}'.format(settings.MNT_PT, pool.name)
                qgroup_assign(qid, pqid, pool_mnt_pt)
            rusage, eusage, pqgroup_rusage, pqgroup_eusage = \
                volume_usage(pool, qid, pqid)
            nso = Share(pool=pool, qgroup=qid, pqgroup=pqid, name=share_name,
                        size=pool.size, subvol_name=s_in_pool, rusage=rusage,
                        eusage=eusage, pqgroup_rusage=pqgroup_rusage,
                        pqgroup_eusage=pqgroup_eusage,
                        replica=replica)
            nso.save()
            update_shareusage_db(s_in_pool, rusage, eusage)
            mount_share(nso, '%s%s' % (settings.MNT_PT, s_in_pool))
Пример #21
0
    def post(self, request, command):
        if (command == 'bootstrap'):

            self._refresh_pool_state()
            for p in Pool.objects.all():
                import_shares(p, request)

            for share in Share.objects.all():
                try:
                    if (share.pqgroup == settings.MODEL_DEFS['pqgroup']):
                        share.pqgroup = qgroup_create(share.pool)
                        share.save()
                    if (not is_share_mounted(share.name)):
                        mnt_pt = ('%s%s' % (settings.MNT_PT, share.name))
                        mount_share(share, mnt_pt)
                except Exception as e:
                    e_msg = ('Exception while mounting a share(%s) during '
                             'bootstrap: %s' % (share.name, e.__str__()))
                    logger.error(e_msg)
                    logger.exception(e)

                try:
                    import_snapshots(share)
                except Exception as e:
                    e_msg = ('Exception while importing Snapshots of '
                             'Share(%s): %s' % (share.name, e.__str__()))
                    logger.error(e_msg)
                    logger.exception(e)

            for snap in Snapshot.objects.all():
                if (snap.uvisible):
                    try:
                        mount_snap(snap.share, snap.real_name)
                    except Exception as e:
                        e_msg = ('Failed to make the Snapshot(%s) visible. '
                                 'Exception: %s' %
                                 (snap.real_name, e.__str__()))
                        logger.error(e_msg)

            mnt_map = sftp_mount_map(settings.SFTP_MNT_ROOT)
            for sftpo in SFTP.objects.all():
                try:
                    sftp_mount(sftpo.share, settings.MNT_PT,
                               settings.SFTP_MNT_ROOT, mnt_map, sftpo.editable)
                    sftp_snap_toggle(sftpo.share)
                except Exception as e:
                    e_msg = ('Exception while exportin a sftp share during '
                             'bootstrap: %s' % e.__str__())
                    logger.error(e_msg)

            try:
                adv_entries = [
                    a.export_str for a in AdvancedNFSExport.objects.all()
                ]
                exports_d = self.create_adv_nfs_export_input(
                    adv_entries, request)
                exports = self.create_nfs_export_input(NFSExport.objects.all())
                exports.update(exports_d)
                self.refresh_wrapper(exports, request, logger)
            except Exception as e:
                e_msg = ('Exception while bootstrapping NFS: %s' % e.__str__())
                logger.error(e_msg)

            #  bootstrap services
            try:
                systemctl('firewalld', 'stop')
                systemctl('firewalld', 'disable')
                systemctl('nginx', 'stop')
                systemctl('nginx', 'disable')
                systemctl('atd', 'enable')
                systemctl('atd', 'start')
            except Exception as e:
                e_msg = ('Exception while setting service statuses during '
                         'bootstrap: %s' % e.__str__())
                logger.error(e_msg)
                handle_exception(Exception(e_msg), request)

            logger.debug('Bootstrap operations completed')
            return Response()

        if (command == 'utcnow'):
            return Response(datetime.utcnow().replace(tzinfo=utc))

        if (command == 'uptime'):
            return Response(uptime())

        if (command == 'kernel'):
            try:
                return Response(kernel_info(settings.SUPPORTED_KERNEL_VERSION))
            except Exception as e:
                handle_exception(e, request)

        if (command == 'update-check'):
            try:
                subo = None
                try:
                    subo = UpdateSubscription.objects.get(name='Stable',
                                                          status='active')
                except UpdateSubscription.DoesNotExist:
                    try:
                        subo = UpdateSubscription.objects.get(name='Testing',
                                                              status='active')
                    except UpdateSubscription.DoesNotExist:
                        pass
                return Response(update_check(subscription=subo))
            except Exception as e:
                e_msg = ('Unable to check update due to a system error: %s' %
                         e.__str__())
                handle_exception(Exception(e_msg), request)

        if (command == 'update'):
            try:
                update_run()
                return Response('Done')
            except Exception as e:
                e_msg = ('Update failed due to this exception: %s' %
                         e.__str__())
                handle_exception(Exception(e_msg), request)

        if (command == 'current-version'):
            try:
                return Response(current_version())
            except Exception as e:
                e_msg = ('Unable to check current version due to this '
                         'exception: ' % e.__str__())
                handle_exception(Exception(e_msg), request)

        if (command == 'shutdown'):
            msg = ('The system will now be shutdown')
            try:
                request.session.flush()
                system_shutdown()
            except Exception as e:
                msg = ('Failed to shutdown the system due to a low level '
                       'error: %s' % e.__str__())
                handle_exception(Exception(msg), request)
            finally:
                return Response(msg)

        if (command == 'reboot'):
            msg = ('The system will now reboot')
            try:
                request.session.flush()
                system_reboot()
            except Exception as e:
                msg = ('Failed to reboot the system due to a low level error: '
                       '%s' % e.__str__())
                handle_exception(Exception(msg), request)
            finally:
                return Response(msg)

        if (command == 'current-user'):
            return Response(request.user.username)

        if (command == 'auto-update-status'):
            status = True
            try:
                status = auto_update_status()
            except:
                status = False
            finally:
                return Response({
                    'enabled': status,
                })

        if (command == 'enable-auto-update'):
            try:
                auto_update(enable=True)
                return Response({
                    'enabled': True,
                })
            except Exception as e:
                msg = ('Failed to enable auto update due to this exception: '
                       '%s' % e.__str__())
                handle_exception(Exception(msg), request)

        if (command == 'disable-auto-update'):
            try:
                auto_update(enable=False)
                return Response({
                    'enabled': False,
                })
            except Exception as e:
                msg = ('Failed to disable auto update due to this exception:  '
                       '%s' % e.__str__())
                handle_exception(Exception(msg), request)

        if (command == 'refresh-pool-state'):
            self._refresh_pool_state()
            return Response()

        if (command == 'refresh-share-state'):
            for p in Pool.objects.all():
                import_shares(p, request)
            return Response()

        if (command == 'refresh-snapshot-state'):
            for share in Share.objects.all():
                import_snapshots(share)
            return Response()
Пример #22
0
def import_shares(pool, request):
    # Establish known shares/subvols within our db for the given pool:
    shares_in_pool_db = [s.name for s in Share.objects.filter(pool=pool)]
    # Find the actual/current shares/subvols within the given pool:
    # Limited to Rockstor relevant subvols ie shares and clones.
    shares_in_pool = shares_info(pool)
    # List of pool's share.pqgroups so we can remove inadvertent duplication.
    # All pqgroups are removed when quotas are disabled, combined with a part
    # refresh we could have duplicates within the db.
    share_pqgroups_used = []
    # Delete db Share object if it is no longer found on disk.
    for s_in_pool_db in shares_in_pool_db:
        if s_in_pool_db not in shares_in_pool:
            Share.objects.get(pool=pool, name=s_in_pool_db).delete()
    # Check if each share in pool also has a db counterpart.
    for s_in_pool in shares_in_pool:
        logger.debug('---- Share name = {}.'.format(s_in_pool))
        if s_in_pool in shares_in_pool_db:
            logger.debug('Updating pre-existing same pool db share entry.')
            # We have a pool db share counterpart so retrieve and update it.
            share = Share.objects.get(name=s_in_pool, pool=pool)
            # Initially default our pqgroup value to db default of '-1/-1'
            # This way, unless quotas are enabled, all pqgroups will be
            # returned to db default.
            pqgroup = settings.MODEL_DEFS['pqgroup']
            if share.pool.quotas_enabled:
                # Quotas are enabled on our pool so we can validate pqgroup.
                if share.pqgroup == pqgroup or not share.pqgroup_exist \
                        or share.pqgroup in share_pqgroups_used:
                    # we have a void '-1/-1' or non existent pqgroup or
                    # this pqgroup has already been seen / used in this pool.
                    logger.debug('#### replacing void, non-existent, or '
                                 'duplicate pqgroup')
                    pqgroup = qgroup_create(pool)
                    update_quota(pool, pqgroup, share.size * 1024)
                    share_pqgroup_assign(pqgroup, share)
                else:
                    # Our share's pqgroup looks OK so use it.
                    pqgroup = share.pqgroup
                # Record our use of this pqgroup to spot duplicates later.
                share_pqgroups_used.append(deepcopy(share.pqgroup))
            if share.pqgroup != pqgroup:
                # we need to update our share.pqgroup
                share.pqgroup = pqgroup
                share.save()
            share.qgroup = shares_in_pool[s_in_pool]
            rusage, eusage, pqgroup_rusage, pqgroup_eusage = \
                volume_usage(pool, share.qgroup, pqgroup)
            if (rusage != share.rusage or eusage != share.eusage
                    or pqgroup_rusage != share.pqgroup_rusage
                    or pqgroup_eusage != share.pqgroup_eusage):
                share.rusage = rusage
                share.eusage = eusage
                share.pqgroup_rusage = pqgroup_rusage
                share.pqgroup_eusage = pqgroup_eusage
                update_shareusage_db(s_in_pool, rusage, eusage)
            else:
                update_shareusage_db(s_in_pool, rusage, eusage, UPDATE_TS)
            share.save()
            continue
        try:
            logger.debug('No prior entries in scanned pool trying all pools.')
            # Test (Try) for an existing system wide Share db entry.
            cshare = Share.objects.get(name=s_in_pool)
            # Get a list of Rockstor relevant subvols (ie shares and clones)
            # for the prior existing db share entry's pool.
            cshares_d = shares_info(cshare.pool)
            if s_in_pool in cshares_d:
                e_msg = ('Another pool ({}) has a Share with this same '
                         'name ({}) as this pool ({}). This configuration '
                         'is not supported. You can delete one of them '
                         'manually with the following command: '
                         '"btrfs subvol delete {}[pool name]/{}" WARNING this '
                         'will remove the entire contents of that subvolume.'.
                         format(cshare.pool.name, s_in_pool, pool.name,
                                settings.MNT_PT, s_in_pool))
                handle_exception(Exception(e_msg), request)
            else:
                # Update the prior existing db share entry previously
                # associated with another pool.
                logger.debug('Updating prior db entry from another pool.')
                cshare.pool = pool
                cshare.qgroup = shares_in_pool[s_in_pool]
                cshare.size = pool.size
                cshare.subvol_name = s_in_pool
                (cshare.rusage, cshare.eusage, cshare.pqgroup_rusage,
                 cshare.pqgroup_eusage) = volume_usage(pool, cshare.qgroup,
                                                       cshare.pqgroup)
                cshare.save()
                update_shareusage_db(s_in_pool, cshare.rusage, cshare.eusage)
        except Share.DoesNotExist:
            logger.debug('Db share entry does not exist - creating.')
            # We have a share on disk that has no db counterpart so create one.
            # Retrieve new pool quota id for use in db Share object creation.
            pqid = qgroup_create(pool)
            update_quota(pool, pqid, pool.size * 1024)
            rusage, eusage, pqgroup_rusage, pqgroup_eusage = \
                volume_usage(pool, shares_in_pool[s_in_pool], pqid)
            nso = Share(pool=pool,
                        qgroup=shares_in_pool[s_in_pool],
                        pqgroup=pqid,
                        name=s_in_pool,
                        size=pool.size,
                        subvol_name=s_in_pool,
                        rusage=rusage,
                        eusage=eusage,
                        pqgroup_rusage=pqgroup_rusage,
                        pqgroup_eusage=pqgroup_eusage)
            nso.save()
            update_shareusage_db(s_in_pool, rusage, eusage)
            mount_share(nso, '%s%s' % (settings.MNT_PT, s_in_pool))