def archive_drivespace_bar(request):
'''Displays as a horizontal bar chart, the free vs used space
on the archive drive. Will only display if it is mounted'''
try:
directory = request.GET.get('path')
# create figure
fig, ax = bargraph()
try:
totalSpace, availSpace, f, b = disk_attributes(directory)
used_frac = (float(totalSpace - availSpace) / float(totalSpace))
free_frac = 1 - used_frac
title = '%s' % directory
labels = ['Used', 'Free']
except:
used_frac = 1
free_frac = 0
title = '%s' % directory
labels = ["Error: Could not get drive statistics",""]
frac = [used_frac * 100, free_frac * 100]
if float(frac[0] + frac[1]) > 100.0:
frac[1] = frac[1] - (float(frac[0] + frac[1]) - 100.0)
color = IONYELLOW
i = 0
for fr in frac:
ax.barh(bottom=0, left=i, width=fr, color=color)
color = IONBLUE
i += fr
if int(frac[0]) >= 10:
ax.text(float(frac[0]) / 200,
0.5,
labels[0],
transform=ax.transAxes,
fontsize=10,
verticalalignment='center',
horizontalalignment='center')
if int(frac[1]) >= 10:
ax.text((100 - float(frac[0])) / 200 + float(frac[0]) / 100,
0.5,
labels[1],
transform=ax.transAxes,
fontsize=10,
verticalalignment='center',
horizontalalignment='center')
ax.set_title(title)
ax.set_xlabel('% Capacity')
canvas = FigureCanvas(fig)
response = http.HttpResponse(content_type='image/png')
canvas.print_png(response)
return response
except Exception as inst:
open('/tmp/graphProblemLog.txt', 'w').write('problem: %s\n' % (inst))
def disk_usage_stats():
stats = {}
def diskspace_used(objects):
'''
Sums diskusage field for given Experiment or Result objects
diskusage stores MB
'''
tot_disk_used = 0
diskusage_list = objects.values_list('diskusage', flat=True)
for diskusage in diskusage_list:
tot_disk_used += diskusage if diskusage is not None else 0
return tot_disk_used
servers = get_servers()
for server in servers:
# Statistics for this location
errormsg = None
try:
total, availSpace, freeSpace, bsize = disk_attributes(server.filesPrefix) # bytes
total_gb = float(total*bsize)/(1024*1024*1024)
avail_gb = float(availSpace*bsize)/(1024*1024*1024)
free_gb = float(freeSpace*bsize)/(1024*1024*1024)
percentfull = 100-(float(availSpace)/float(total)*100) if total > 0 else 0
reserved = free_gb - avail_gb
except:
errormsg = "Error accessing %s filesystem." % server.filesPrefix
percentfull = '0'
total_gb = ''
avail_gb = ''
free_gb = ''
reserved = ''
other = ''
logger.error(traceback.format_exc())
# for experiment data:
experiments = Experiment.objects.filter(expDir__startswith=server.filesPrefix)
rawused = float(diskspace_used(experiments))/1024 #gbytes
# for results data:
results = Results.objects.filter(reportstorage__dirPath__startswith=server.filesPrefix)
reportsused = float(diskspace_used(results))/1024 #gbytes
other = total_gb - (reserved + avail_gb + rawused + reportsused) #gbytes
stats[server.filesPrefix] = {
'statusmsg':errormsg,
'percentfull': percentfull,
'disksize': total_gb,
'diskfree': avail_gb,
'rawused': rawused,
'reportsused': reportsused,
'other': other,
}
return stats
def archive_drivespace_bar(request):
'''Displays as a horizontal bar chart, the free vs used space
on the archive drive. Will only display if it is mounted'''
try:
directory = request.GET.get('path')
# create figure
fig, ax = bargraph()
try:
totalSpace, availSpace, f, b = disk_attributes(directory)
used_frac = (float(totalSpace - availSpace) / float(totalSpace))
free_frac = 1 - used_frac
title = '%s' % directory
labels = ['Used', 'Free']
except:
used_frac = 1
free_frac = 0
title = '%s' % directory
labels = ["Error: Could not get drive statistics", ""]
frac = [used_frac * 100, free_frac * 100]
if float(frac[0] + frac[1]) > 100.0:
frac[1] = frac[1] - (float(frac[0] + frac[1]) - 100.0)
color = IONYELLOW
i = 0
for fr in frac:
ax.barh(bottom=0, left=i, width=fr, color=color)
color = IONBLUE
i += fr
if int(frac[0]) >= 10:
ax.text(float(frac[0]) / 200,
0.5,
labels[0],
transform=ax.transAxes,
fontsize=10,
verticalalignment='center',
horizontalalignment='center')
if int(frac[1]) >= 10:
ax.text((100 - float(frac[0])) / 200 + float(frac[0]) / 100,
0.5,
labels[1],
transform=ax.transAxes,
fontsize=10,
verticalalignment='center',
horizontalalignment='center')
ax.set_title(title)
ax.set_xlabel('% Capacity')
canvas = FigureCanvas(fig)
response = http.HttpResponse(content_type='image/png')
canvas.print_png(response)
return response
except Exception as inst:
open('/tmp/graphProblemLog.txt', 'w').write('problem: %s\n' % (inst))
def disk_usage_stats():
stats = {}
def diskspace_used(objects):
'''
Sums diskusage field for given Experiment or Result objects
diskusage stores MB
'''
tot_disk_used = 0
diskusage_list = objects.values_list('diskusage', flat=True)
for diskusage in diskusage_list:
tot_disk_used += diskusage if diskusage is not None else 0
return tot_disk_used
servers = get_servers()
for server in servers:
# Statistics for this location
errormsg = None
try:
total, availSpace, freeSpace, bsize = disk_attributes(server.filesPrefix) # bytes
total_gb = float(total*bsize)/(1024*1024*1024)
avail_gb = float(availSpace*bsize)/(1024*1024*1024)
free_gb = float(freeSpace*bsize)/(1024*1024*1024)
percentfull = 100-(float(availSpace)/float(total)*100) if total > 0 else 0
reserved = free_gb - avail_gb
except:
errormsg = "Error accessing %s filesystem." % server.filesPrefix
percentfull = '0'
total_gb = ''
avail_gb = ''
free_gb = ''
reserved = ''
other = ''
logger.error(traceback.format_exc())
# get space used by data marked Keep
keeper_used = get_keepers_diskspace(server.filesPrefix)
keeper_used = float(sum(keeper_used.values()))/1024 #gbytes
percentkeep = 100*(keeper_used/total_gb if total_gb and total_gb > 0 else 0)
stats[server.filesPrefix] = {
'statusmsg':errormsg,
'percentfull': percentfull,
'disksize': total_gb,
'diskfree': avail_gb,
'keeper_used': keeper_used,
'percentkeep': percentkeep,
}
return stats
def exp_list(backup_object):
# make dictionary, one array per file server of archiveExperiment objects
def rawdata_used(experiments, serverprefix):
"""
Sums diskusage field for given experiment objects
diskusage stores MB
"""
tot_disk_used = 0
experiments = experiments.filter(expDir__startswith=serverprefix).values_list("diskusage", flat=True)
for diskusage in experiments:
tot_disk_used += diskusage if diskusage is not None else 0
return tot_disk_used
def reportstorage_used(results, serverprefix):
"""
Sums diskusage field for given results objects
diskusage stores MB
"""
tot_disk_used = 0
reportstorage = ReportStorage.objects.filter(dirPath__startswith=serverprefix)
results = results.filter(reportstorage=reportstorage).values_list("diskusage", flat=True)
for result in results:
tot_disk_used += result if result is not None else 0
return tot_disk_used
to_archive = {}
server_stats = {}
try:
# populate dictionary with experiments ready to be archived
# Get all experiments in database sorted by date
experiments = Experiment.objects.all().order_by("date")
# Ignore experiments already archived/deleted
experiments = experiments.exclude(expName__in=Backup.objects.all().values("backupName"))
except:
logger.error(traceback.print_exc())
raise
results = Results.objects.all().order_by("pk")
storage_stats = [0, 0, 0] # [ Keep, Archive, Delete]
servers = get_servers()
for server in servers:
# Statistics for this server
# TODO: store these in database??
total, availSpace, freeSpace, bsize = disk_attributes(server.filesPrefix) # bytes
total_gb = float(total * bsize) / (1024 * 1024 * 1024)
avail_gb = float(availSpace * bsize) / (1024 * 1024 * 1024)
free_gb = float(freeSpace * bsize) / (1024 * 1024 * 1024)
rawused = float(rawdata_used(experiments, server.filesPrefix)) / 1024 # gbytes
reportsused = float(reportstorage_used(results, server.filesPrefix)) / 1024 # gbytes
reserved = free_gb - avail_gb
other = total_gb - (reserved + avail_gb + rawused + reportsused) # gbytes
server_stats[server.filesPrefix] = {
"percentfull": server.percentfull,
"disksize": total_gb,
"diskfree": avail_gb,
"rawused": rawused,
"reportsused": reportsused,
"other": other,
}
# only generate list of experiments if Archiving is enabled
if backup_object.online:
# Experiments for this server
explist = []
experiments_fs = experiments.filter(expDir__startswith=server.filesPrefix)
for exp in experiments_fs:
archive_experiment = ArchiveExperiment(
exp,
str(exp.expName),
exp.date.strftime("%Y-%m-%d"),
str(exp.star),
str(exp.storage_options),
str(exp.user_ack),
str(exp.expDir),
exp.pk,
str(exp.rawdatastyle),
str(exp.diskusage) if exp.diskusage is not None else "Unknown",
)
explist.append(archive_experiment)
if "KI" in exp.storage_options:
storage_stats[0] += 1
elif "A" in exp.storage_options:
storage_stats[1] += 1
elif "D" in exp.storage_options:
storage_stats[2] += 1
to_archive[server.filesPrefix] = explist
return to_archive, server_stats, storage_stats
def disk_usage_stats():
stats = {}
def diskspace_used(objects):
"""
Sums diskusage field for given Experiment or Result objects
diskusage stores MB
"""
tot_disk_used = 0
diskusage_list = objects.values_list("diskusage", flat=True)
for diskusage in diskusage_list:
tot_disk_used += diskusage if diskusage is not None else 0
return tot_disk_used
servers = get_servers()
for server in servers:
# Statistics for this location
errormsg = None
try:
total, availSpace, freeSpace, bsize = disk_attributes(server.filesPrefix) # bytes
total_gb = float(total * bsize) / (1024 * 1024 * 1024)
avail_gb = float(availSpace * bsize) / (1024 * 1024 * 1024)
free_gb = float(freeSpace * bsize) / (1024 * 1024 * 1024)
percentfull = 100 - (float(availSpace) / float(total) * 100) if total > 0 else 0
reserved = free_gb - avail_gb
except:
errormsg = "Error accessing %s filesystem." % server.filesPrefix
percentfull = "0"
total_gb = ""
avail_gb = ""
free_gb = ""
reserved = ""
other = ""
logger.error(traceback.format_exc())
# for experiment data:
experiments = Experiment.objects.filter(expDir__startswith=server.filesPrefix)
rawused = float(diskspace_used(experiments)) / 1024 # gbytes
# for results data:
results = Results.objects.filter(reportstorage__dirPath__startswith=server.filesPrefix)
reportsused = float(diskspace_used(results)) / 1024 # gbytes
other = total_gb - (reserved + avail_gb + rawused + reportsused) # gbytes
# get space used by data marked Keep
keeper_used = get_keepers_diskspace(server.filesPrefix)
keeper_used = float(sum(keeper_used.values())) / 1024 # gbytes
percentkeep = 100 * (keeper_used / total_gb if total_gb and total_gb > 0 else 0)
stats[server.filesPrefix] = {
"statusmsg": errormsg,
"percentfull": percentfull,
"disksize": total_gb,
"diskfree": avail_gb,
"rawused": rawused,
"reportsused": reportsused,
"other": other,
"keeper_used": keeper_used,
"percentkeep": percentkeep,
}
return stats
def exp_list(backup_object):
# make dictionary, one array per file server of archiveExperiment objects
def rawdata_used(experiments, serverprefix):
'''
Sums diskusage field for given experiment objects
diskusage stores MB
'''
tot_disk_used = 0
experiments = experiments.filter(
expDir__startswith=serverprefix).values_list('diskusage',
flat=True)
for diskusage in experiments:
tot_disk_used += diskusage if diskusage is not None else 0
return tot_disk_used
def reportstorage_used(results, serverprefix):
'''
Sums diskusage field for given results objects
diskusage stores MB
'''
tot_disk_used = 0
reportstorage = ReportStorage.objects.filter(
dirPath__startswith=serverprefix)
results = results.filter(reportstorage=reportstorage).values_list(
'diskusage', flat=True)
for result in results:
tot_disk_used += result if result is not None else 0
return tot_disk_used
to_archive = {}
server_stats = {}
try:
# populate dictionary with experiments ready to be archived
# Get all experiments in database sorted by date
experiments = Experiment.objects.all().order_by('date')
# Ignore experiments already archived/deleted
experiments = experiments.exclude(
expName__in=Backup.objects.all().values('backupName'))
except:
logger.error(traceback.print_exc())
raise
results = Results.objects.all().order_by('pk')
storage_stats = [0, 0, 0] # [ Keep, Archive, Delete]
servers = get_servers()
for server in servers:
# Statistics for this server
#TODO: store these in database??
total, availSpace, freeSpace, bsize = disk_attributes(
server.filesPrefix) # bytes
total_gb = float(total * bsize) / (1024 * 1024 * 1024)
avail_gb = float(availSpace * bsize) / (1024 * 1024 * 1024)
free_gb = float(freeSpace * bsize) / (1024 * 1024 * 1024)
rawused = float(rawdata_used(experiments,
server.filesPrefix)) / 1024 #gbytes
reportsused = float(reportstorage_used(
results, server.filesPrefix)) / 1024 #gbytes
reserved = free_gb - avail_gb
other = total_gb - (reserved + avail_gb + rawused + reportsused
) #gbytes
server_stats[server.filesPrefix] = {
'percentfull': server.percentfull,
'disksize': total_gb,
'diskfree': avail_gb,
'rawused': rawused,
'reportsused': reportsused,
'other': other,
}
# only generate list of experiments if Archiving is enabled
if backup_object.online:
# Experiments for this server
explist = []
experiments_fs = experiments.filter(
expDir__startswith=server.filesPrefix)
for exp in experiments_fs:
archive_experiment = ArchiveExperiment(
exp,
str(exp.expName),
exp.date.strftime("%Y-%m-%d"),
str(exp.star),
str(exp.storage_options),
str(exp.user_ack),
str(exp.expDir),
exp.pk,
str(exp.rawdatastyle),
str(exp.diskusage)
if exp.diskusage is not None else "Unknown",
)
explist.append(archive_experiment)
if 'KI' in exp.storage_options:
storage_stats[0] += 1
elif 'A' in exp.storage_options:
storage_stats[1] += 1
elif 'D' in exp.storage_options:
storage_stats[2] += 1
to_archive[server.filesPrefix] = explist
return to_archive, server_stats, storage_stats
def disk_usage_stats():
stats = {}
def diskspace_used(objects):
'''
Sums diskusage field for given Experiment or Result objects
diskusage stores MB
'''
tot_disk_used = 0
diskusage_list = objects.values_list('diskusage', flat=True)
for diskusage in diskusage_list:
tot_disk_used += diskusage if diskusage is not None else 0
return tot_disk_used
servers = get_servers()
for server in servers:
# Statistics for this location
errormsg = None
try:
total, availSpace, freeSpace, bsize = disk_attributes(
server.filesPrefix) # bytes
total_gb = float(total * bsize) / (1024 * 1024 * 1024)
avail_gb = float(availSpace * bsize) / (1024 * 1024 * 1024)
free_gb = float(freeSpace * bsize) / (1024 * 1024 * 1024)
percentfull = 100 - (float(availSpace) / float(total) *
100) if total > 0 else 0
reserved = free_gb - avail_gb
except:
errormsg = "Error accessing %s filesystem." % server.filesPrefix
percentfull = '0'
total_gb = ''
avail_gb = ''
free_gb = ''
reserved = ''
other = ''
logger.error(traceback.format_exc())
# for experiment data:
experiments = Experiment.objects.filter(
expDir__startswith=server.filesPrefix)
rawused = float(diskspace_used(experiments)) / 1024 #gbytes
# for results data:
results = Results.objects.filter(
reportstorage__dirPath__startswith=server.filesPrefix)
reportsused = float(diskspace_used(results)) / 1024 #gbytes
other = total_gb - (reserved + avail_gb + rawused + reportsused
) #gbytes
stats[server.filesPrefix] = {
'statusmsg': errormsg,
'percentfull': percentfull,
'disksize': total_gb,
'diskfree': avail_gb,
'rawused': rawused,
'reportsused': reportsused,
'other': other,
}
return stats
