def import_shares(pool, request): disk = Disk.objects.filter(pool=pool)[0].name shares = [s.name for s in Share.objects.filter(pool=pool)] shares_d = shares_info('%s%s' % (settings.MNT_PT, pool.name)) for s in shares: if (s not in shares_d): Share.objects.get(pool=pool, name=s).delete() for s in shares_d: if (s in shares): share = Share.objects.get(name=s) share.qgroup = shares_d[s] rusage, eusage = share_usage(pool, share.qgroup) ts = datetime.utcnow().replace(tzinfo=utc) if (rusage != share.rusage or eusage != share.eusage): share.rusage = rusage share.eusage = eusage su = ShareUsage(name=s, r_usage=rusage, e_usage=eusage, ts=ts) su.save() else: try: su = ShareUsage.objects.filter(name=s).latest('id') su.ts = ts su.count += 1 except ShareUsage.DoesNotExist: su = ShareUsage(name=s, r_usage=rusage, e_usage=eusage, ts=ts) finally: su.save() share.save() continue try: cshare = Share.objects.get(name=s) cshares_d = shares_info('%s%s' % (settings.MNT_PT, cshare.pool.name)) if (s in cshares_d): e_msg = ('Another pool(%s) has a Share with this same ' 'name(%s) as this pool(%s). This configuration is not supported.' ' You can delete one of them manually with this command: ' 'btrfs subvol delete %s[pool name]/%s' % (cshare.pool.name, s, pool.name, settings.MNT_PT, s)) handle_exception(Exception(e_msg), request) else: cshare.pool = pool cshare.qgroup = shares_d[s] cshare.size = pool.size cshare.subvol_name = s cshare.rusage, cshare.eusage = share_usage(pool, cshare.qgroup) cshare.save() except Share.DoesNotExist: pqid = qgroup_create(pool) update_quota(pool, pqid, pool.size * 1024) nso = Share(pool=pool, qgroup=shares_d[s], pqgroup=pqid, name=s, size=pool.size, subvol_name=s) nso.save() mount_share(nso, '%s%s' % (settings.MNT_PT, s))
def import_shares(pool, request): disk = Disk.objects.filter(pool=pool)[0].name shares = [s.name for s in Share.objects.filter(pool=pool)] shares_d = shares_info(pool) for s in shares: if (s not in shares_d): Share.objects.get(pool=pool, name=s).delete() for s in shares_d: if (s in shares): share = Share.objects.get(name=s) share.qgroup = shares_d[s] rusage, eusage = share_usage(pool, share.qgroup) ts = datetime.utcnow().replace(tzinfo=utc) if (rusage != share.rusage or eusage != share.eusage): share.rusage = rusage share.eusage = eusage su = ShareUsage(name=s, r_usage=rusage, e_usage=eusage, ts=ts) su.save() else: try: su = ShareUsage.objects.filter(name=s).latest('id') su.ts = ts su.count += 1 except ShareUsage.DoesNotExist: su = ShareUsage(name=s, r_usage=rusage, e_usage=eusage, ts=ts) finally: su.save() share.save() continue try: cshare = Share.objects.get(name=s) cshares_d = shares_info('%s%s' % (settings.MNT_PT, cshare.pool.name)) if (s in cshares_d): e_msg = ('Another pool(%s) has a Share with this same ' 'name(%s) as this pool(%s). This configuration is not supported.' ' You can delete one of them manually with this command: ' 'btrfs subvol delete %s[pool name]/%s' % (cshare.pool.name, s, pool.name, settings.MNT_PT, s)) handle_exception(Exception(e_msg), request) else: cshare.pool = pool cshare.qgroup = shares_d[s] cshare.size = pool.size cshare.subvol_name = s cshare.rusage, cshare.eusage = share_usage(pool, cshare.qgroup) cshare.save() except Share.DoesNotExist: pqid = qgroup_create(pool) update_quota(pool, pqid, pool.size * 1024) nso = Share(pool=pool, qgroup=shares_d[s], pqgroup=pqid, name=s, size=pool.size, subvol_name=s) nso.save() mount_share(nso, '%s%s' % (settings.MNT_PT, s))
def btrfs_share_scan(): pools = btrfs_pool_scan() shares = [] for pool in pools: info = shares_info(pool) for i in range(len(info.keys())): share = {} share["name"] = info.keys()[i] share["subvol_name"] = info.keys()[i] share["group"] = info[info.keys()[i]] share["pool"] = pool qid = qgroup_id(pool, share["subvol_name"]) cur_rusage, cur_eusage = share_usage(pool, qid) share["rusage"] = cur_rusage share["eusage"] = cur_eusage share["pqgroup"] = "xxx" # print qgroup_info(pool) shares.append(share) return shares
def import_shares(pool, request): # Establish known shares/subvols within our db for the given pool: shares_in_pool_db = [s.name for s in Share.objects.filter(pool=pool)] # Find the actual/current shares/subvols within the given pool: # Limited to Rockstor relevant subvols ie shares and clones. shares_in_pool = shares_info(pool) # List of pool's share.pqgroups so we can remove inadvertent duplication. # All pqgroups are removed when quotas are disabled, combined with a part # refresh we could have duplicates within the db. share_pqgroups_used = [] # Delete db Share object if it is no longer found on disk. for s_in_pool_db in shares_in_pool_db: if s_in_pool_db not in shares_in_pool: logger.debug("Removing, missing on disk, share db entry ({}) from " "pool ({}).".format(s_in_pool_db, pool.name)) Share.objects.get(pool=pool, name=s_in_pool_db).delete() # Check if each share in pool also has a db counterpart. for s_in_pool in shares_in_pool: logger.debug("---- Share name = {}.".format(s_in_pool)) if s_in_pool in shares_in_pool_db: logger.debug("Updating pre-existing same pool db share entry.") # We have a pool db share counterpart so retrieve and update it. share = Share.objects.get(name=s_in_pool, pool=pool) # Initially default our pqgroup value to db default of '-1/-1' # This way, unless quotas are enabled, all pqgroups will be # returned to db default. pqgroup = PQGROUP_DEFAULT if share.pool.quotas_enabled: # Quotas are enabled on our pool so we can validate pqgroup. if (share.pqgroup == pqgroup or not share.pqgroup_exist or share.pqgroup in share_pqgroups_used): # we have a void '-1/-1' or non existent pqgroup or # this pqgroup has already been seen / used in this pool. logger.debug( "#### replacing void, non-existent, or duplicate pqgroup." ) pqgroup = qgroup_create(pool) if pqgroup != PQGROUP_DEFAULT: update_quota(pool, pqgroup, share.size * 1024) share_pqgroup_assign(pqgroup, share) else: # Our share's pqgroup looks OK so use it. pqgroup = share.pqgroup # Record our use of this pqgroup to spot duplicates later. share_pqgroups_used.append(deepcopy(share.pqgroup)) if share.pqgroup != pqgroup: # we need to update our share.pqgroup share.pqgroup = pqgroup share.save() share.qgroup = shares_in_pool[s_in_pool] rusage, eusage, pqgroup_rusage, pqgroup_eusage = volume_usage( pool, share.qgroup, pqgroup) if (rusage != share.rusage or eusage != share.eusage or pqgroup_rusage != share.pqgroup_rusage or pqgroup_eusage != share.pqgroup_eusage): share.rusage = rusage share.eusage = eusage share.pqgroup_rusage = pqgroup_rusage share.pqgroup_eusage = pqgroup_eusage update_shareusage_db(s_in_pool, rusage, eusage) else: update_shareusage_db(s_in_pool, rusage, eusage, UPDATE_TS) share.save() continue try: logger.debug("No prior entries in scanned pool trying all pools.") # Test (Try) for an existing system wide Share db entry. cshare = Share.objects.get(name=s_in_pool) # Get a list of Rockstor relevant subvols (ie shares and clones) # for the prior existing db share entry's pool. cshares_d = shares_info(cshare.pool) if s_in_pool in cshares_d: e_msg = ("Another pool ({}) has a share with this same " "name ({}) as this pool ({}). This configuration " "is not supported. You can delete one of them " "manually with the following command: " '"btrfs subvol delete {}[pool name]/{}" WARNING this ' "will remove the entire contents of that " "subvolume.").format(cshare.pool.name, s_in_pool, pool.name, settings.MNT_PT, s_in_pool) handle_exception(Exception(e_msg), request) else: # Update the prior existing db share entry previously # associated with another pool. logger.debug("Updating prior db entry from another pool.") cshare.pool = pool cshare.qgroup = shares_in_pool[s_in_pool] cshare.size = pool.size cshare.subvol_name = s_in_pool ( cshare.rusage, cshare.eusage, cshare.pqgroup_rusage, cshare.pqgroup_eusage, ) = volume_usage(pool, cshare.qgroup, cshare.pqgroup) cshare.save() update_shareusage_db(s_in_pool, cshare.rusage, cshare.eusage) except Share.DoesNotExist: logger.debug("Db share entry does not exist - creating.") # We have a share on disk that has no db counterpart so create one. # Retrieve new pool quota id for use in db Share object creation. # As the replication receive share is 'special' we tag it as such. replica = False share_name = s_in_pool if re.match(".snapshot", s_in_pool) is not None: # We have an initial replication share, non snap in .snapshots. # We could change it's name here but still a little mixing # of name and subvol throughout project. replica = True logger.debug("Initial receive quirk-subvol found: Importing " "as share and setting replica flag.") qid = shares_in_pool[s_in_pool] pqid = qgroup_create(pool) if pqid != PQGROUP_DEFAULT: update_quota(pool, pqid, pool.size * 1024) qgroup_assign(qid, pqid, pool.mnt_pt) rusage, eusage, pqgroup_rusage, pqgroup_eusage = volume_usage( pool, qid, pqid) nso = Share( pool=pool, qgroup=qid, pqgroup=pqid, name=share_name, size=pool.size, subvol_name=s_in_pool, rusage=rusage, eusage=eusage, pqgroup_rusage=pqgroup_rusage, pqgroup_eusage=pqgroup_eusage, replica=replica, ) nso.save() update_shareusage_db(s_in_pool, rusage, eusage) mount_share(nso, "{}{}".format(settings.MNT_PT, s_in_pool))
def import_shares(pool, request): # Establish known shares/subvols within our db for the given pool: shares_in_pool_db = [s.name for s in Share.objects.filter(pool=pool)] # Find the actual/current shares/subvols within the given pool: # Limited to Rockstor relevant subvols ie shares and clones. shares_in_pool = shares_info(pool) # List of pool's share.pqgroups so we can remove inadvertent duplication. # All pqgroups are removed when quotas are disabled, combined with a part # refresh we could have duplicates within the db. share_pqgroups_used = [] # Delete db Share object if it is no longer found on disk. for s_in_pool_db in shares_in_pool_db: if s_in_pool_db not in shares_in_pool: logger.debug('Removing, missing on disk, share db entry ({}) from ' 'pool ({}).'.format(s_in_pool_db, pool.name)) Share.objects.get(pool=pool, name=s_in_pool_db).delete() # Check if each share in pool also has a db counterpart. for s_in_pool in shares_in_pool: logger.debug('---- Share name = {}.'.format(s_in_pool)) if s_in_pool in shares_in_pool_db: logger.debug('Updating pre-existing same pool db share entry.') # We have a pool db share counterpart so retrieve and update it. share = Share.objects.get(name=s_in_pool, pool=pool) # Initially default our pqgroup value to db default of '-1/-1' # This way, unless quotas are enabled, all pqgroups will be # returned to db default. pqgroup = PQGROUP_DEFAULT if share.pool.quotas_enabled: # Quotas are enabled on our pool so we can validate pqgroup. if share.pqgroup == pqgroup or not share.pqgroup_exist \ or share.pqgroup in share_pqgroups_used: # we have a void '-1/-1' or non existent pqgroup or # this pqgroup has already been seen / used in this pool. logger.debug('#### replacing void, non-existent, or ' 'duplicate pqgroup.') pqgroup = qgroup_create(pool) if pqgroup is not PQGROUP_DEFAULT: update_quota(pool, pqgroup, share.size * 1024) share_pqgroup_assign(pqgroup, share) else: # Our share's pqgroup looks OK so use it. pqgroup = share.pqgroup # Record our use of this pqgroup to spot duplicates later. share_pqgroups_used.append(deepcopy(share.pqgroup)) if share.pqgroup != pqgroup: # we need to update our share.pqgroup share.pqgroup = pqgroup share.save() share.qgroup = shares_in_pool[s_in_pool] rusage, eusage, pqgroup_rusage, pqgroup_eusage = \ volume_usage(pool, share.qgroup, pqgroup) if (rusage != share.rusage or eusage != share.eusage or pqgroup_rusage != share.pqgroup_rusage or pqgroup_eusage != share.pqgroup_eusage): share.rusage = rusage share.eusage = eusage share.pqgroup_rusage = pqgroup_rusage share.pqgroup_eusage = pqgroup_eusage update_shareusage_db(s_in_pool, rusage, eusage) else: update_shareusage_db(s_in_pool, rusage, eusage, UPDATE_TS) share.save() continue try: logger.debug('No prior entries in scanned pool trying all pools.') # Test (Try) for an existing system wide Share db entry. cshare = Share.objects.get(name=s_in_pool) # Get a list of Rockstor relevant subvols (ie shares and clones) # for the prior existing db share entry's pool. cshares_d = shares_info(cshare.pool) if s_in_pool in cshares_d: e_msg = ('Another pool ({}) has a share with this same ' 'name ({}) as this pool ({}). This configuration ' 'is not supported. You can delete one of them ' 'manually with the following command: ' '"btrfs subvol delete {}[pool name]/{}" WARNING this ' 'will remove the entire contents of that ' 'subvolume.').format(cshare.pool.name, s_in_pool, pool.name, settings.MNT_PT, s_in_pool) handle_exception(Exception(e_msg), request) else: # Update the prior existing db share entry previously # associated with another pool. logger.debug('Updating prior db entry from another pool.') cshare.pool = pool cshare.qgroup = shares_in_pool[s_in_pool] cshare.size = pool.size cshare.subvol_name = s_in_pool (cshare.rusage, cshare.eusage, cshare.pqgroup_rusage, cshare.pqgroup_eusage) = volume_usage(pool, cshare.qgroup, cshare.pqgroup) cshare.save() update_shareusage_db(s_in_pool, cshare.rusage, cshare.eusage) except Share.DoesNotExist: logger.debug('Db share entry does not exist - creating.') # We have a share on disk that has no db counterpart so create one. # Retrieve new pool quota id for use in db Share object creation. # As the replication receive share is 'special' we tag it as such. replica = False share_name = s_in_pool if re.match('.snapshot', s_in_pool) is not None: # We have an initial replication share, non snap in .snapshots. # We could change it's name here but still a little mixing # of name and subvol throughout project. replica = True logger.debug('Initial receive quirk-subvol found: Importing ' 'as share and setting replica flag.') qid = shares_in_pool[s_in_pool] pqid = qgroup_create(pool) if pqid is not PQGROUP_DEFAULT: update_quota(pool, pqid, pool.size * 1024) pool_mnt_pt = '{}{}'.format(settings.MNT_PT, pool.name) qgroup_assign(qid, pqid, pool_mnt_pt) rusage, eusage, pqgroup_rusage, pqgroup_eusage = \ volume_usage(pool, qid, pqid) nso = Share(pool=pool, qgroup=qid, pqgroup=pqid, name=share_name, size=pool.size, subvol_name=s_in_pool, rusage=rusage, eusage=eusage, pqgroup_rusage=pqgroup_rusage, pqgroup_eusage=pqgroup_eusage, replica=replica) nso.save() update_shareusage_db(s_in_pool, rusage, eusage) mount_share(nso, '%s%s' % (settings.MNT_PT, s_in_pool))
def import_shares(pool, request): # Establish known shares/subvols within our db for the given pool: shares_in_pool_db = [s.name for s in Share.objects.filter(pool=pool)] # Find the actual/current shares/subvols within the given pool: # Limited to Rockstor relevant subvols ie shares and clones. shares_in_pool = shares_info(pool) # Delete db Share object if it is no longer found on disk. for s_in_pool_db in shares_in_pool_db: if s_in_pool_db not in shares_in_pool: Share.objects.get(pool=pool, name=s_in_pool_db).delete() # Check if each share in pool also has a db counterpart. for s_in_pool in shares_in_pool: logger.debug('Share name = {}.'.format(s_in_pool)) if s_in_pool in shares_in_pool_db: logger.debug('Updating pre-existing same pool db share entry.') # We have a pool db share counterpart so retrieve and update it. share = Share.objects.get(name=s_in_pool, pool=pool) share.qgroup = shares_in_pool[s_in_pool] rusage, eusage, pqgroup_rusage, pqgroup_eusage = \ volume_usage(pool, share.qgroup, share.pqgroup) if (rusage != share.rusage or eusage != share.eusage or pqgroup_rusage != share.pqgroup_rusage or pqgroup_eusage != share.pqgroup_eusage): share.rusage = rusage share.eusage = eusage share.pqgroup_rusage = pqgroup_rusage share.pqgroup_eusage = pqgroup_eusage update_shareusage_db(s_in_pool, rusage, eusage) else: update_shareusage_db(s_in_pool, rusage, eusage, UPDATE_TS) share.save() continue try: logger.debug('No prior entries in scanned pool trying all pools.') # Test (Try) for an existing system wide Share db entry. cshare = Share.objects.get(name=s_in_pool) # Get a list of Rockstor relevant subvols (ie shares and clones) # for the prior existing db share entry's pool. cshares_d = shares_info(cshare.pool) if s_in_pool in cshares_d: e_msg = ('Another pool ({}) has a Share with this same ' 'name ({}) as this pool ({}). This configuration ' 'is not supported. You can delete one of them ' 'manually with the following command: ' '"btrfs subvol delete {}[pool name]/{}" WARNING this ' 'will remove the entire contents of that subvolume.'. format(cshare.pool.name, s_in_pool, pool.name, settings.MNT_PT, s_in_pool)) handle_exception(Exception(e_msg), request) else: # Update the prior existing db share entry previously # associated with another pool. logger.debug('Updating prior db entry from another pool.') cshare.pool = pool cshare.qgroup = shares_in_pool[s_in_pool] cshare.size = pool.size cshare.subvol_name = s_in_pool (cshare.rusage, cshare.eusage, cshare.pqgroup_rusage, cshare.pqgroup_eusage) = volume_usage(pool, cshare.qgroup, cshare.pqgroup) cshare.save() update_shareusage_db(s_in_pool, cshare.rusage, cshare.eusage) except Share.DoesNotExist: logger.debug('Db share entry does not exist - creating.') # We have a share on disk that has no db counterpart so create one. # Retrieve pool quota id for use in db Share object creation. pqid = qgroup_create(pool) update_quota(pool, pqid, pool.size * 1024) rusage, eusage, pqgroup_rusage, pqgroup_eusage = \ volume_usage(pool, shares_in_pool[s_in_pool], pqid) nso = Share(pool=pool, qgroup=shares_in_pool[s_in_pool], pqgroup=pqid, name=s_in_pool, size=pool.size, subvol_name=s_in_pool, rusage=rusage, eusage=eusage, pqgroup_rusage=pqgroup_rusage, pqgroup_eusage=pqgroup_eusage) nso.save() update_shareusage_db(s_in_pool, rusage, eusage) mount_share(nso, '%s%s' % (settings.MNT_PT, s_in_pool))
def import_shares(pool, request): # Establish known shares/subvols within our db for the given pool: shares_in_pool_db = [s.name for s in Share.objects.filter(pool=pool)] # Find the actual/current shares/subvols within the given pool: # Limited to Rockstor relevant subvols ie shares and clones. shares_in_pool = shares_info(pool) # List of pool's share.pqgroups so we can remove inadvertent duplication. # All pqgroups are removed when quotas are disabled, combined with a part # refresh we could have duplicates within the db. share_pqgroups_used = [] # Delete db Share object if it is no longer found on disk. for s_in_pool_db in shares_in_pool_db: if s_in_pool_db not in shares_in_pool: Share.objects.get(pool=pool, name=s_in_pool_db).delete() # Check if each share in pool also has a db counterpart. for s_in_pool in shares_in_pool: logger.debug('---- Share name = {}.'.format(s_in_pool)) if s_in_pool in shares_in_pool_db: logger.debug('Updating pre-existing same pool db share entry.') # We have a pool db share counterpart so retrieve and update it. share = Share.objects.get(name=s_in_pool, pool=pool) # Initially default our pqgroup value to db default of '-1/-1' # This way, unless quotas are enabled, all pqgroups will be # returned to db default. pqgroup = settings.MODEL_DEFS['pqgroup'] if share.pool.quotas_enabled: # Quotas are enabled on our pool so we can validate pqgroup. if share.pqgroup == pqgroup or not share.pqgroup_exist \ or share.pqgroup in share_pqgroups_used: # we have a void '-1/-1' or non existent pqgroup or # this pqgroup has already been seen / used in this pool. logger.debug('#### replacing void, non-existent, or ' 'duplicate pqgroup') pqgroup = qgroup_create(pool) update_quota(pool, pqgroup, share.size * 1024) share_pqgroup_assign(pqgroup, share) else: # Our share's pqgroup looks OK so use it. pqgroup = share.pqgroup # Record our use of this pqgroup to spot duplicates later. share_pqgroups_used.append(deepcopy(share.pqgroup)) if share.pqgroup != pqgroup: # we need to update our share.pqgroup share.pqgroup = pqgroup share.save() share.qgroup = shares_in_pool[s_in_pool] rusage, eusage, pqgroup_rusage, pqgroup_eusage = \ volume_usage(pool, share.qgroup, pqgroup) if (rusage != share.rusage or eusage != share.eusage or pqgroup_rusage != share.pqgroup_rusage or pqgroup_eusage != share.pqgroup_eusage): share.rusage = rusage share.eusage = eusage share.pqgroup_rusage = pqgroup_rusage share.pqgroup_eusage = pqgroup_eusage update_shareusage_db(s_in_pool, rusage, eusage) else: update_shareusage_db(s_in_pool, rusage, eusage, UPDATE_TS) share.save() continue try: logger.debug('No prior entries in scanned pool trying all pools.') # Test (Try) for an existing system wide Share db entry. cshare = Share.objects.get(name=s_in_pool) # Get a list of Rockstor relevant subvols (ie shares and clones) # for the prior existing db share entry's pool. cshares_d = shares_info(cshare.pool) if s_in_pool in cshares_d: e_msg = ('Another pool ({}) has a Share with this same ' 'name ({}) as this pool ({}). This configuration ' 'is not supported. You can delete one of them ' 'manually with the following command: ' '"btrfs subvol delete {}[pool name]/{}" WARNING this ' 'will remove the entire contents of that subvolume.'. format(cshare.pool.name, s_in_pool, pool.name, settings.MNT_PT, s_in_pool)) handle_exception(Exception(e_msg), request) else: # Update the prior existing db share entry previously # associated with another pool. logger.debug('Updating prior db entry from another pool.') cshare.pool = pool cshare.qgroup = shares_in_pool[s_in_pool] cshare.size = pool.size cshare.subvol_name = s_in_pool (cshare.rusage, cshare.eusage, cshare.pqgroup_rusage, cshare.pqgroup_eusage) = volume_usage(pool, cshare.qgroup, cshare.pqgroup) cshare.save() update_shareusage_db(s_in_pool, cshare.rusage, cshare.eusage) except Share.DoesNotExist: logger.debug('Db share entry does not exist - creating.') # We have a share on disk that has no db counterpart so create one. # Retrieve new pool quota id for use in db Share object creation. pqid = qgroup_create(pool) update_quota(pool, pqid, pool.size * 1024) rusage, eusage, pqgroup_rusage, pqgroup_eusage = \ volume_usage(pool, shares_in_pool[s_in_pool], pqid) nso = Share(pool=pool, qgroup=shares_in_pool[s_in_pool], pqgroup=pqid, name=s_in_pool, size=pool.size, subvol_name=s_in_pool, rusage=rusage, eusage=eusage, pqgroup_rusage=pqgroup_rusage, pqgroup_eusage=pqgroup_eusage) nso.save() update_shareusage_db(s_in_pool, rusage, eusage) mount_share(nso, '%s%s' % (settings.MNT_PT, s_in_pool))