def import_snapshots(share): snaps_d = snaps_info('%s%s' % (settings.MNT_PT, share.pool.name), share.name) snaps = [s.name for s in Snapshot.objects.filter(share=share)] for s in snaps: if (s not in snaps_d): Snapshot.objects.get(share=share, name=s).delete() for s in snaps_d: if (s in snaps): so = Snapshot.objects.get(share=share, name=s) else: so = Snapshot(share=share, name=s, real_name=s, writable=snaps_d[s][1], qgroup=snaps_d[s][0]) rusage, eusage = volume_usage(share.pool, snaps_d[s][0]) ts = datetime.utcnow().replace(tzinfo=utc) if (rusage != so.rusage or eusage != so.eusage): so.rusage = rusage so.eusage = eusage su = ShareUsage(name=s, r_usage=rusage, e_usage=eusage, ts=ts) su.save() else: try: su = ShareUsage.objects.filter(name=s).latest('id') su.ts = ts su.count += 1 except ShareUsage.DoesNotExist: su = ShareUsage(name=s, r_usage=rusage, e_usage=eusage, ts=ts) finally: su.save() so.save()
def update_shareusage_db(subvol_name, rusage, eusage, new_entry=True): """ Creates a new share/subvol db usage entry, or updates an existing one with a new time stamp and count increment. The 'create new entry' mode is expected to be faster. :param subvol_name: share/subvol name :param rusage: Referenced/shared usage :param eusage: Exclusive usage :param new_entry: If True create a new entry with the passed params, otherwise attempt to update the latest (by id) entry with time and count. """ ts = datetime.utcnow().replace(tzinfo=utc) if new_entry: su = ShareUsage(name=subvol_name, r_usage=rusage, e_usage=eusage, ts=ts) su.save() else: try: su = ShareUsage.objects.filter(name=subvol_name).latest("id") su.ts = ts su.count += 1 except ShareUsage.DoesNotExist: su = ShareUsage(name=subvol_name, r_usage=rusage, e_usage=eusage, ts=ts) finally: su.save()
def update_shareusage_db(subvol_name, rusage, eusage, new_entry=True): """ Creates a new share/subvol db usage entry, or updates an existing one with a new time stamp and count increment. The 'create new entry' mode is expected to be faster. :param subvol_name: share/subvol name :param rusage: Referenced/shared usage :param eusage: Exclusive usage :param new_entry: If True create a new entry with the passed params, otherwise attempt to update the latest (by id) entry with time and count. """ ts = datetime.utcnow().replace(tzinfo=utc) if new_entry: su = ShareUsage(name=subvol_name, r_usage=rusage, e_usage=eusage, ts=ts) su.save() else: try: su = ShareUsage.objects.filter(name=subvol_name).latest('id') su.ts = ts su.count += 1 except ShareUsage.DoesNotExist: su = ShareUsage(name=subvol_name, r_usage=rusage, e_usage=eusage, ts=ts) finally: su.save()
def import_shares(pool, request): disk = Disk.objects.filter(pool=pool)[0].name shares = [s.name for s in Share.objects.filter(pool=pool)] shares_d = shares_info('%s%s' % (settings.MNT_PT, pool.name)) for s in shares: if (s not in shares_d): Share.objects.get(pool=pool, name=s).delete() for s in shares_d: if (s in shares): share = Share.objects.get(name=s) share.qgroup = shares_d[s] rusage, eusage = share_usage(pool, share.qgroup) ts = datetime.utcnow().replace(tzinfo=utc) if (rusage != share.rusage or eusage != share.eusage): share.rusage = rusage share.eusage = eusage su = ShareUsage(name=s, r_usage=rusage, e_usage=eusage, ts=ts) su.save() else: try: su = ShareUsage.objects.filter(name=s).latest('id') su.ts = ts su.count += 1 except ShareUsage.DoesNotExist: su = ShareUsage(name=s, r_usage=rusage, e_usage=eusage, ts=ts) finally: su.save() share.save() continue try: cshare = Share.objects.get(name=s) cshares_d = shares_info('%s%s' % (settings.MNT_PT, cshare.pool.name)) if (s in cshares_d): e_msg = ('Another pool(%s) has a Share with this same ' 'name(%s) as this pool(%s). This configuration is not supported.' ' You can delete one of them manually with this command: ' 'btrfs subvol delete %s[pool name]/%s' % (cshare.pool.name, s, pool.name, settings.MNT_PT, s)) handle_exception(Exception(e_msg), request) else: cshare.pool = pool cshare.qgroup = shares_d[s] cshare.size = pool.size cshare.subvol_name = s cshare.rusage, cshare.eusage = share_usage(pool, cshare.qgroup) cshare.save() except Share.DoesNotExist: pqid = qgroup_create(pool) update_quota(pool, pqid, pool.size * 1024) nso = Share(pool=pool, qgroup=shares_d[s], pqgroup=pqid, name=s, size=pool.size, subvol_name=s) nso.save() mount_share(nso, '%s%s' % (settings.MNT_PT, s))
def import_shares(pool, request): disk = Disk.objects.filter(pool=pool)[0].name shares = [s.name for s in Share.objects.filter(pool=pool)] shares_d = shares_info(pool) for s in shares: if (s not in shares_d): Share.objects.get(pool=pool, name=s).delete() for s in shares_d: if (s in shares): share = Share.objects.get(name=s) share.qgroup = shares_d[s] rusage, eusage = share_usage(pool, share.qgroup) ts = datetime.utcnow().replace(tzinfo=utc) if (rusage != share.rusage or eusage != share.eusage): share.rusage = rusage share.eusage = eusage su = ShareUsage(name=s, r_usage=rusage, e_usage=eusage, ts=ts) su.save() else: try: su = ShareUsage.objects.filter(name=s).latest('id') su.ts = ts su.count += 1 except ShareUsage.DoesNotExist: su = ShareUsage(name=s, r_usage=rusage, e_usage=eusage, ts=ts) finally: su.save() share.save() continue try: cshare = Share.objects.get(name=s) cshares_d = shares_info('%s%s' % (settings.MNT_PT, cshare.pool.name)) if (s in cshares_d): e_msg = ('Another pool(%s) has a Share with this same ' 'name(%s) as this pool(%s). This configuration is not supported.' ' You can delete one of them manually with this command: ' 'btrfs subvol delete %s[pool name]/%s' % (cshare.pool.name, s, pool.name, settings.MNT_PT, s)) handle_exception(Exception(e_msg), request) else: cshare.pool = pool cshare.qgroup = shares_d[s] cshare.size = pool.size cshare.subvol_name = s cshare.rusage, cshare.eusage = share_usage(pool, cshare.qgroup) cshare.save() except Share.DoesNotExist: pqid = qgroup_create(pool) update_quota(pool, pqid, pool.size * 1024) nso = Share(pool=pool, qgroup=shares_d[s], pqgroup=pqid, name=s, size=pool.size, subvol_name=s) nso.save() mount_share(nso, '%s%s' % (settings.MNT_PT, s))
if ((ts - su.ts).total_seconds() > 90): su = None except Exception, e: e_msg = ('Unable to get latest share usage object ' 'for share(%s). A new one will be created.' % s) logger.error(e_msg) # we check for changed in both referenced and exclusive # usage because in rare cases it's possible for only one # to change. if (su is None or su.r_usage != usaged[s][0] or su.e_usage != usaged[s][1]): su = ShareUsage(name=s, r_usage=usaged[s][0], e_usage=usaged[s][1], ts=ts) else: su.ts = ts su.count = su.count + 1 self._save_wrapper(su) except Exception, e: logger.debug('command exception while getting shares usage ' 'for pool: %s' % (p.name)) logger.exception(e) return now def main(): pr = ProcRetreiver() pr.start() logger.debug('Started Proc Retreiver') pr.join()
def pools_usage(self, last_ts): """ This info is not from proc atm, but will eventually be. """ # collect usage only if the data is more than 30 seconds old now = time.mktime(time.gmtime()) if (now - last_ts < 30): return last_ts ts = datetime.utcnow().replace(tzinfo=utc) for p in Pool.objects.all(): total_reclaimable = 0 try: # get usage of all shares in this pool pool_device = Disk.objects.filter(pool=p)[0].name share_map = {} snap_map = {} for share in Share.objects.filter(pool=p): share_map[share.qgroup] = share.name for snap in Snapshot.objects.filter(share=share): snap_map[snap.qgroup] = snap.real_name usaged = shares_usage(p, pool_device, share_map, snap_map) for s in usaged.keys(): try: total_reclaimable += ( Share.objects.get(name=s).size - usaged[s][1]) except: pass su = None try: su = ShareUsage.objects.filter(name=s).latest('id') if ((ts - su.ts).total_seconds() > 90): su = None except Exception, e: e_msg = ('Unable to get latest share usage object ' 'for share(%s). A new one will be created.' % s) logger.error(e_msg) # we check for changed in both referenced and exclusive # usage because in rare cases it's possible for only one # to change. if ((su is None or su.r_usage != usaged[s][0] or su.e_usage != usaged[s][1])): su = ShareUsage(name=s, r_usage=usaged[s][0], e_usage=usaged[s][1], ts=ts) else: su.ts = ts su.count = su.count + 1 self._save_wrapper(su) except Exception, e: logger.debug('command exception while getting shares usage ' 'for pool: %s' % (p.name)) logger.exception(e) try: usage = pool_usage('/%s/%s' % (settings.MNT_PT, p.name)) total_free = usage[2] # free + reclaimable pu = None try: pu = PoolUsage.objects.filter(pool=p.name).latest('id') if ((ts - pu.ts).total_seconds() > 90): pu = None except Exception, e: e_msg = ('Unable to get latest pool usage object for ' 'pool(%s). A new one will be created.' % p.name) logger.error(e_msg) if ((pu is None or p.size - (pu.free + pu.reclaimable) != usage[1])): pu = PoolUsage(pool=p.name, free=total_free-total_reclaimable, reclaimable=total_reclaimable, ts=ts) else: pu.ts = ts pu.count = pu.count + 1 self._save_wrapper(pu)
except Exception, e: e_msg = ('Unable to get latest share usage object ' 'for share(%s). A new one will be created.' % s) logger.error(e_msg) #we check for changed in both referenced and exclusive #usage because in rare cases it's possible for only one to #change. if (su is None or su.r_usage != usaged[s][0] or su.e_usage != usaged[s][1]): su = ShareUsage(name=s, r_usage=usaged[s][0], e_usage=usaged[s][1], ts=ts) else: su.ts = ts su.count = su.count + 1 self._save_wrapper(su) except Exception, e: logger.debug('command exception while getting shares usage ' 'for pool: %s' % (p.name)) logger.exception(e) return now def main(): pr = ProcRetreiver() pr.start() logger.debug('Started Proc Retreiver') pr.join()