def import_snapshots(share):
snaps_d = snaps_info('%s%s' % (settings.MNT_PT, share.pool.name),
share.name)
snaps = [s.name for s in Snapshot.objects.filter(share=share)]
for s in snaps:
if (s not in snaps_d):
Snapshot.objects.get(share=share, name=s).delete()
for s in snaps_d:
if (s in snaps):
so = Snapshot.objects.get(share=share, name=s)
else:
so = Snapshot(share=share,
name=s,
real_name=s,
writable=snaps_d[s][1],
qgroup=snaps_d[s][0])
rusage, eusage = volume_usage(share.pool, snaps_d[s][0])
ts = datetime.utcnow().replace(tzinfo=utc)
if (rusage != so.rusage or eusage != so.eusage):
so.rusage = rusage
so.eusage = eusage
su = ShareUsage(name=s, r_usage=rusage, e_usage=eusage, ts=ts)
su.save()
else:
try:
su = ShareUsage.objects.filter(name=s).latest('id')
su.ts = ts
su.count += 1
except ShareUsage.DoesNotExist:
su = ShareUsage(name=s, r_usage=rusage, e_usage=eusage, ts=ts)
finally:
su.save()
so.save()
def update_shareusage_db(subvol_name, rusage, eusage, new_entry=True):
"""
Creates a new share/subvol db usage entry, or updates an existing one with
a new time stamp and count increment.
The 'create new entry' mode is expected to be faster.
:param subvol_name: share/subvol name
:param rusage: Referenced/shared usage
:param eusage: Exclusive usage
:param new_entry: If True create a new entry with the passed params,
otherwise attempt to update the latest (by id) entry with time and count.
"""
ts = datetime.utcnow().replace(tzinfo=utc)
if new_entry:
su = ShareUsage(name=subvol_name,
r_usage=rusage,
e_usage=eusage,
ts=ts)
su.save()
else:
try:
su = ShareUsage.objects.filter(name=subvol_name).latest("id")
su.ts = ts
su.count += 1
except ShareUsage.DoesNotExist:
su = ShareUsage(name=subvol_name,
r_usage=rusage,
e_usage=eusage,
ts=ts)
finally:
su.save()
def import_shares(pool, request):
disk = Disk.objects.filter(pool=pool)[0].name
shares = [s.name for s in Share.objects.filter(pool=pool)]
shares_d = shares_info(pool)
for s in shares:
if (s not in shares_d):
Share.objects.get(pool=pool, name=s).delete()
for s in shares_d:
if (s in shares):
share = Share.objects.get(name=s)
share.qgroup = shares_d[s]
rusage, eusage = share_usage(pool, share.qgroup)
ts = datetime.utcnow().replace(tzinfo=utc)
if (rusage != share.rusage or eusage != share.eusage):
share.rusage = rusage
share.eusage = eusage
su = ShareUsage(name=s, r_usage=rusage, e_usage=eusage,
ts=ts)
su.save()
else:
try:
su = ShareUsage.objects.filter(name=s).latest('id')
su.ts = ts
su.count += 1
except ShareUsage.DoesNotExist:
su = ShareUsage(name=s, r_usage=rusage,
e_usage=eusage, ts=ts)
finally:
su.save()
share.save()
continue
try:
cshare = Share.objects.get(name=s)
cshares_d = shares_info('%s%s' % (settings.MNT_PT,
cshare.pool.name))
if (s in cshares_d):
e_msg = ('Another pool(%s) has a Share with this same '
'name(%s) as this pool(%s). This configuration is not supported.'
' You can delete one of them manually with this command: '
'btrfs subvol delete %s[pool name]/%s' %
(cshare.pool.name, s, pool.name, settings.MNT_PT, s))
handle_exception(Exception(e_msg), request)
else:
cshare.pool = pool
cshare.qgroup = shares_d[s]
cshare.size = pool.size
cshare.subvol_name = s
cshare.rusage, cshare.eusage = share_usage(pool, cshare.qgroup)
cshare.save()
except Share.DoesNotExist:
pqid = qgroup_create(pool)
update_quota(pool, pqid, pool.size * 1024)
nso = Share(pool=pool, qgroup=shares_d[s], pqgroup=pqid, name=s,
size=pool.size, subvol_name=s)
nso.save()
mount_share(nso, '%s%s' % (settings.MNT_PT, s))
def import_snapshots(share):
snaps_d = snaps_info('%s%s' % (settings.MNT_PT, share.pool.name),
share.name)
snaps = [s.name for s in Snapshot.objects.filter(share=share)]
for s in snaps:
if (s not in snaps_d):
Snapshot.objects.get(share=share, name=s).delete()
for s in snaps_d:
if (s in snaps):
so = Snapshot.objects.get(share=share, name=s)
else:
so = Snapshot(share=share, name=s, real_name=s,
writable=snaps_d[s][1], qgroup=snaps_d[s][0])
rusage, eusage = volume_usage(share.pool, snaps_d[s][0])
ts = datetime.utcnow().replace(tzinfo=utc)
if (rusage != so.rusage or eusage != so.eusage):
so.rusage = rusage
so.eusage = eusage
su = ShareUsage(name=s, r_usage=rusage, e_usage=eusage, ts=ts)
su.save()
else:
try:
su = ShareUsage.objects.filter(name=s).latest('id')
su.ts = ts
su.count += 1
except ShareUsage.DoesNotExist:
su = ShareUsage(name=s, r_usage=rusage, e_usage=eusage,
ts=ts)
finally:
su.save()
so.save()
def update_shareusage_db(subvol_name, rusage, eusage, new_entry=True):
"""
Creates a new share/subvol db usage entry, or updates an existing one with
a new time stamp and count increment.
The 'create new entry' mode is expected to be faster.
:param subvol_name: share/subvol name
:param rusage: Referenced/shared usage
:param eusage: Exclusive usage
:param new_entry: If True create a new entry with the passed params,
otherwise attempt to update the latest (by id) entry with time and count.
"""
ts = datetime.utcnow().replace(tzinfo=utc)
if new_entry:
su = ShareUsage(name=subvol_name, r_usage=rusage, e_usage=eusage,
ts=ts)
su.save()
else:
try:
su = ShareUsage.objects.filter(name=subvol_name).latest('id')
su.ts = ts
su.count += 1
except ShareUsage.DoesNotExist:
su = ShareUsage(name=subvol_name, r_usage=rusage, e_usage=eusage,
ts=ts)
finally:
su.save()
def import_shares(pool, request):
disk = Disk.objects.filter(pool=pool)[0].name
shares = [s.name for s in Share.objects.filter(pool=pool)]
shares_d = shares_info('%s%s' % (settings.MNT_PT, pool.name))
for s in shares:
if (s not in shares_d):
Share.objects.get(pool=pool, name=s).delete()
for s in shares_d:
if (s in shares):
share = Share.objects.get(name=s)
share.qgroup = shares_d[s]
rusage, eusage = share_usage(pool, share.qgroup)
ts = datetime.utcnow().replace(tzinfo=utc)
if (rusage != share.rusage or eusage != share.eusage):
share.rusage = rusage
share.eusage = eusage
su = ShareUsage(name=s, r_usage=rusage, e_usage=eusage,
ts=ts)
su.save()
else:
try:
su = ShareUsage.objects.filter(name=s).latest('id')
su.ts = ts
su.count += 1
except ShareUsage.DoesNotExist:
su = ShareUsage(name=s, r_usage=rusage,
e_usage=eusage, ts=ts)
finally:
su.save()
share.save()
continue
try:
cshare = Share.objects.get(name=s)
cshares_d = shares_info('%s%s' % (settings.MNT_PT,
cshare.pool.name))
if (s in cshares_d):
e_msg = ('Another pool(%s) has a Share with this same '
'name(%s) as this pool(%s). This configuration is not supported.'
' You can delete one of them manually with this command: '
'btrfs subvol delete %s[pool name]/%s' %
(cshare.pool.name, s, pool.name, settings.MNT_PT, s))
handle_exception(Exception(e_msg), request)
else:
cshare.pool = pool
cshare.qgroup = shares_d[s]
cshare.size = pool.size
cshare.subvol_name = s
cshare.rusage, cshare.eusage = share_usage(pool, cshare.qgroup)
cshare.save()
except Share.DoesNotExist:
pqid = qgroup_create(pool)
update_quota(pool, pqid, pool.size * 1024)
nso = Share(pool=pool, qgroup=shares_d[s], pqgroup=pqid, name=s,
size=pool.size, subvol_name=s)
nso.save()
mount_share(nso, '%s%s' % (settings.MNT_PT, s))
su = None
try:
su = ShareUsage.objects.filter(name=s).latest('id')
if ((ts - su.ts).total_seconds() > 90):
su = None
except Exception, e:
e_msg = ('Unable to get latest share usage object '
'for share(%s). A new one will be created.'
% s)
logger.error(e_msg)
# we check for changed in both referenced and exclusive
# usage because in rare cases it's possible for only one
# to change.
if (su is None or su.r_usage != usaged[s][0] or
su.e_usage != usaged[s][1]):
su = ShareUsage(name=s, r_usage=usaged[s][0],
e_usage=usaged[s][1], ts=ts)
else:
su.ts = ts
su.count = su.count + 1
self._save_wrapper(su)
except Exception, e:
logger.debug('command exception while getting shares usage '
'for pool: %s' % (p.name))
logger.exception(e)
return now
def main():
pr = ProcRetreiver()
pr.start()
logger.debug('Started Proc Retreiver')
def pools_usage(self, last_ts):
"""
This info is not from proc atm, but will eventually be.
"""
# collect usage only if the data is more than 30 seconds old
now = time.mktime(time.gmtime())
if (now - last_ts < 30):
return last_ts
ts = datetime.utcnow().replace(tzinfo=utc)
for p in Pool.objects.all():
total_reclaimable = 0
try:
# get usage of all shares in this pool
pool_device = Disk.objects.filter(pool=p)[0].name
share_map = {}
snap_map = {}
for share in Share.objects.filter(pool=p):
share_map[share.qgroup] = share.name
for snap in Snapshot.objects.filter(share=share):
snap_map[snap.qgroup] = snap.real_name
usaged = shares_usage(p, pool_device, share_map, snap_map)
for s in usaged.keys():
try:
total_reclaimable += (
Share.objects.get(name=s).size - usaged[s][1])
except:
pass
su = None
try:
su = ShareUsage.objects.filter(name=s).latest('id')
if ((ts - su.ts).total_seconds() > 90):
su = None
except Exception, e:
e_msg = ('Unable to get latest share usage object '
'for share(%s). A new one will be created.'
% s)
logger.error(e_msg)
# we check for changed in both referenced and exclusive
# usage because in rare cases it's possible for only one
# to change.
if ((su is None or su.r_usage != usaged[s][0] or
su.e_usage != usaged[s][1])):
su = ShareUsage(name=s, r_usage=usaged[s][0],
e_usage=usaged[s][1], ts=ts)
else:
su.ts = ts
su.count = su.count + 1
self._save_wrapper(su)
except Exception, e:
logger.debug('command exception while getting shares usage '
'for pool: %s' % (p.name))
logger.exception(e)
try:
usage = pool_usage('/%s/%s' % (settings.MNT_PT, p.name))
total_free = usage[2] # free + reclaimable
pu = None
try:
pu = PoolUsage.objects.filter(pool=p.name).latest('id')
if ((ts - pu.ts).total_seconds() > 90):
pu = None
except Exception, e:
e_msg = ('Unable to get latest pool usage object for '
'pool(%s). A new one will be created.' % p.name)
logger.error(e_msg)
if ((pu is None or
p.size - (pu.free + pu.reclaimable) != usage[1])):
pu = PoolUsage(pool=p.name,
free=total_free-total_reclaimable,
reclaimable=total_reclaimable, ts=ts)
else:
pu.ts = ts
pu.count = pu.count + 1
self._save_wrapper(pu)
try:
su = ShareUsage.objects.filter(name=s).latest('id')
if ((ts - su.ts).total_seconds() > 90):
su = None
except Exception, e:
e_msg = ('Unable to get latest share usage object '
'for share(%s). A new one will be created.' %
s)
logger.error(e_msg)
#we check for changed in both referenced and exclusive
#usage because in rare cases it's possible for only one to
#change.
if (su is None or su.r_usage != usaged[s][0]
or su.e_usage != usaged[s][1]):
su = ShareUsage(name=s,
r_usage=usaged[s][0],
e_usage=usaged[s][1],
ts=ts)
else:
su.ts = ts
su.count = su.count + 1
self._save_wrapper(su)
except Exception, e:
logger.debug('command exception while getting shares usage '
'for pool: %s' % (p.name))
logger.exception(e)
return now
def main():
pr = ProcRetreiver()
pr.start()
