def archive_drivespace_bar(request): '''Displays as a horizontal bar chart, the free vs used space on the archive drive. Will only display if it is mounted''' try: directory = request.GET.get('path') # create figure fig, ax = bargraph() try: totalSpace, availSpace, f, b = disk_attributes(directory) used_frac = (float(totalSpace - availSpace) / float(totalSpace)) free_frac = 1 - used_frac title = '%s' % directory labels = ['Used', 'Free'] except: used_frac = 1 free_frac = 0 title = '%s' % directory labels = ["Error: Could not get drive statistics",""] frac = [used_frac * 100, free_frac * 100] if float(frac[0] + frac[1]) > 100.0: frac[1] = frac[1] - (float(frac[0] + frac[1]) - 100.0) color = IONYELLOW i = 0 for fr in frac: ax.barh(bottom=0, left=i, width=fr, color=color) color = IONBLUE i += fr if int(frac[0]) >= 10: ax.text(float(frac[0]) / 200, 0.5, labels[0], transform=ax.transAxes, fontsize=10, verticalalignment='center', horizontalalignment='center') if int(frac[1]) >= 10: ax.text((100 - float(frac[0])) / 200 + float(frac[0]) / 100, 0.5, labels[1], transform=ax.transAxes, fontsize=10, verticalalignment='center', horizontalalignment='center') ax.set_title(title) ax.set_xlabel('% Capacity') canvas = FigureCanvas(fig) response = http.HttpResponse(content_type='image/png') canvas.print_png(response) return response except Exception as inst: open('/tmp/graphProblemLog.txt', 'w').write('problem: %s\n' % (inst))
def disk_usage_stats(): stats = {} def diskspace_used(objects): ''' Sums diskusage field for given Experiment or Result objects diskusage stores MB ''' tot_disk_used = 0 diskusage_list = objects.values_list('diskusage', flat=True) for diskusage in diskusage_list: tot_disk_used += diskusage if diskusage is not None else 0 return tot_disk_used servers = get_servers() for server in servers: # Statistics for this location errormsg = None try: total, availSpace, freeSpace, bsize = disk_attributes(server.filesPrefix) # bytes total_gb = float(total*bsize)/(1024*1024*1024) avail_gb = float(availSpace*bsize)/(1024*1024*1024) free_gb = float(freeSpace*bsize)/(1024*1024*1024) percentfull = 100-(float(availSpace)/float(total)*100) if total > 0 else 0 reserved = free_gb - avail_gb except: errormsg = "Error accessing %s filesystem." % server.filesPrefix percentfull = '0' total_gb = '' avail_gb = '' free_gb = '' reserved = '' other = '' logger.error(traceback.format_exc()) # for experiment data: experiments = Experiment.objects.filter(expDir__startswith=server.filesPrefix) rawused = float(diskspace_used(experiments))/1024 #gbytes # for results data: results = Results.objects.filter(reportstorage__dirPath__startswith=server.filesPrefix) reportsused = float(diskspace_used(results))/1024 #gbytes other = total_gb - (reserved + avail_gb + rawused + reportsused) #gbytes stats[server.filesPrefix] = { 'statusmsg':errormsg, 'percentfull': percentfull, 'disksize': total_gb, 'diskfree': avail_gb, 'rawused': rawused, 'reportsused': reportsused, 'other': other, } return stats
def archive_drivespace_bar(request): '''Displays as a horizontal bar chart, the free vs used space on the archive drive. Will only display if it is mounted''' try: directory = request.GET.get('path') # create figure fig, ax = bargraph() try: totalSpace, availSpace, f, b = disk_attributes(directory) used_frac = (float(totalSpace - availSpace) / float(totalSpace)) free_frac = 1 - used_frac title = '%s' % directory labels = ['Used', 'Free'] except: used_frac = 1 free_frac = 0 title = '%s' % directory labels = ["Error: Could not get drive statistics", ""] frac = [used_frac * 100, free_frac * 100] if float(frac[0] + frac[1]) > 100.0: frac[1] = frac[1] - (float(frac[0] + frac[1]) - 100.0) color = IONYELLOW i = 0 for fr in frac: ax.barh(bottom=0, left=i, width=fr, color=color) color = IONBLUE i += fr if int(frac[0]) >= 10: ax.text(float(frac[0]) / 200, 0.5, labels[0], transform=ax.transAxes, fontsize=10, verticalalignment='center', horizontalalignment='center') if int(frac[1]) >= 10: ax.text((100 - float(frac[0])) / 200 + float(frac[0]) / 100, 0.5, labels[1], transform=ax.transAxes, fontsize=10, verticalalignment='center', horizontalalignment='center') ax.set_title(title) ax.set_xlabel('% Capacity') canvas = FigureCanvas(fig) response = http.HttpResponse(content_type='image/png') canvas.print_png(response) return response except Exception as inst: open('/tmp/graphProblemLog.txt', 'w').write('problem: %s\n' % (inst))
def disk_usage_stats(): stats = {} def diskspace_used(objects): ''' Sums diskusage field for given Experiment or Result objects diskusage stores MB ''' tot_disk_used = 0 diskusage_list = objects.values_list('diskusage', flat=True) for diskusage in diskusage_list: tot_disk_used += diskusage if diskusage is not None else 0 return tot_disk_used servers = get_servers() for server in servers: # Statistics for this location errormsg = None try: total, availSpace, freeSpace, bsize = disk_attributes(server.filesPrefix) # bytes total_gb = float(total*bsize)/(1024*1024*1024) avail_gb = float(availSpace*bsize)/(1024*1024*1024) free_gb = float(freeSpace*bsize)/(1024*1024*1024) percentfull = 100-(float(availSpace)/float(total)*100) if total > 0 else 0 reserved = free_gb - avail_gb except: errormsg = "Error accessing %s filesystem." % server.filesPrefix percentfull = '0' total_gb = '' avail_gb = '' free_gb = '' reserved = '' other = '' logger.error(traceback.format_exc()) # get space used by data marked Keep keeper_used = get_keepers_diskspace(server.filesPrefix) keeper_used = float(sum(keeper_used.values()))/1024 #gbytes percentkeep = 100*(keeper_used/total_gb if total_gb and total_gb > 0 else 0) stats[server.filesPrefix] = { 'statusmsg':errormsg, 'percentfull': percentfull, 'disksize': total_gb, 'diskfree': avail_gb, 'keeper_used': keeper_used, 'percentkeep': percentkeep, } return stats
def exp_list(backup_object): # make dictionary, one array per file server of archiveExperiment objects def rawdata_used(experiments, serverprefix): """ Sums diskusage field for given experiment objects diskusage stores MB """ tot_disk_used = 0 experiments = experiments.filter(expDir__startswith=serverprefix).values_list("diskusage", flat=True) for diskusage in experiments: tot_disk_used += diskusage if diskusage is not None else 0 return tot_disk_used def reportstorage_used(results, serverprefix): """ Sums diskusage field for given results objects diskusage stores MB """ tot_disk_used = 0 reportstorage = ReportStorage.objects.filter(dirPath__startswith=serverprefix) results = results.filter(reportstorage=reportstorage).values_list("diskusage", flat=True) for result in results: tot_disk_used += result if result is not None else 0 return tot_disk_used to_archive = {} server_stats = {} try: # populate dictionary with experiments ready to be archived # Get all experiments in database sorted by date experiments = Experiment.objects.all().order_by("date") # Ignore experiments already archived/deleted experiments = experiments.exclude(expName__in=Backup.objects.all().values("backupName")) except: logger.error(traceback.print_exc()) raise results = Results.objects.all().order_by("pk") storage_stats = [0, 0, 0] # [ Keep, Archive, Delete] servers = get_servers() for server in servers: # Statistics for this server # TODO: store these in database?? total, availSpace, freeSpace, bsize = disk_attributes(server.filesPrefix) # bytes total_gb = float(total * bsize) / (1024 * 1024 * 1024) avail_gb = float(availSpace * bsize) / (1024 * 1024 * 1024) free_gb = float(freeSpace * bsize) / (1024 * 1024 * 1024) rawused = float(rawdata_used(experiments, server.filesPrefix)) / 1024 # gbytes reportsused = float(reportstorage_used(results, server.filesPrefix)) / 1024 # gbytes reserved = free_gb - avail_gb other = total_gb - (reserved + avail_gb + rawused + reportsused) # gbytes server_stats[server.filesPrefix] = { "percentfull": server.percentfull, "disksize": total_gb, "diskfree": avail_gb, "rawused": rawused, "reportsused": reportsused, "other": other, } # only generate list of experiments if Archiving is enabled if backup_object.online: # Experiments for this server explist = [] experiments_fs = experiments.filter(expDir__startswith=server.filesPrefix) for exp in experiments_fs: archive_experiment = ArchiveExperiment( exp, str(exp.expName), exp.date.strftime("%Y-%m-%d"), str(exp.star), str(exp.storage_options), str(exp.user_ack), str(exp.expDir), exp.pk, str(exp.rawdatastyle), str(exp.diskusage) if exp.diskusage is not None else "Unknown", ) explist.append(archive_experiment) if "KI" in exp.storage_options: storage_stats[0] += 1 elif "A" in exp.storage_options: storage_stats[1] += 1 elif "D" in exp.storage_options: storage_stats[2] += 1 to_archive[server.filesPrefix] = explist return to_archive, server_stats, storage_stats
def disk_usage_stats(): stats = {} def diskspace_used(objects): """ Sums diskusage field for given Experiment or Result objects diskusage stores MB """ tot_disk_used = 0 diskusage_list = objects.values_list("diskusage", flat=True) for diskusage in diskusage_list: tot_disk_used += diskusage if diskusage is not None else 0 return tot_disk_used servers = get_servers() for server in servers: # Statistics for this location errormsg = None try: total, availSpace, freeSpace, bsize = disk_attributes(server.filesPrefix) # bytes total_gb = float(total * bsize) / (1024 * 1024 * 1024) avail_gb = float(availSpace * bsize) / (1024 * 1024 * 1024) free_gb = float(freeSpace * bsize) / (1024 * 1024 * 1024) percentfull = 100 - (float(availSpace) / float(total) * 100) if total > 0 else 0 reserved = free_gb - avail_gb except: errormsg = "Error accessing %s filesystem." % server.filesPrefix percentfull = "0" total_gb = "" avail_gb = "" free_gb = "" reserved = "" other = "" logger.error(traceback.format_exc()) # for experiment data: experiments = Experiment.objects.filter(expDir__startswith=server.filesPrefix) rawused = float(diskspace_used(experiments)) / 1024 # gbytes # for results data: results = Results.objects.filter(reportstorage__dirPath__startswith=server.filesPrefix) reportsused = float(diskspace_used(results)) / 1024 # gbytes other = total_gb - (reserved + avail_gb + rawused + reportsused) # gbytes # get space used by data marked Keep keeper_used = get_keepers_diskspace(server.filesPrefix) keeper_used = float(sum(keeper_used.values())) / 1024 # gbytes percentkeep = 100 * (keeper_used / total_gb if total_gb and total_gb > 0 else 0) stats[server.filesPrefix] = { "statusmsg": errormsg, "percentfull": percentfull, "disksize": total_gb, "diskfree": avail_gb, "rawused": rawused, "reportsused": reportsused, "other": other, "keeper_used": keeper_used, "percentkeep": percentkeep, } return stats
def exp_list(backup_object): # make dictionary, one array per file server of archiveExperiment objects def rawdata_used(experiments, serverprefix): ''' Sums diskusage field for given experiment objects diskusage stores MB ''' tot_disk_used = 0 experiments = experiments.filter( expDir__startswith=serverprefix).values_list('diskusage', flat=True) for diskusage in experiments: tot_disk_used += diskusage if diskusage is not None else 0 return tot_disk_used def reportstorage_used(results, serverprefix): ''' Sums diskusage field for given results objects diskusage stores MB ''' tot_disk_used = 0 reportstorage = ReportStorage.objects.filter( dirPath__startswith=serverprefix) results = results.filter(reportstorage=reportstorage).values_list( 'diskusage', flat=True) for result in results: tot_disk_used += result if result is not None else 0 return tot_disk_used to_archive = {} server_stats = {} try: # populate dictionary with experiments ready to be archived # Get all experiments in database sorted by date experiments = Experiment.objects.all().order_by('date') # Ignore experiments already archived/deleted experiments = experiments.exclude( expName__in=Backup.objects.all().values('backupName')) except: logger.error(traceback.print_exc()) raise results = Results.objects.all().order_by('pk') storage_stats = [0, 0, 0] # [ Keep, Archive, Delete] servers = get_servers() for server in servers: # Statistics for this server #TODO: store these in database?? total, availSpace, freeSpace, bsize = disk_attributes( server.filesPrefix) # bytes total_gb = float(total * bsize) / (1024 * 1024 * 1024) avail_gb = float(availSpace * bsize) / (1024 * 1024 * 1024) free_gb = float(freeSpace * bsize) / (1024 * 1024 * 1024) rawused = float(rawdata_used(experiments, server.filesPrefix)) / 1024 #gbytes reportsused = float(reportstorage_used( results, server.filesPrefix)) / 1024 #gbytes reserved = free_gb - avail_gb other = total_gb - (reserved + avail_gb + rawused + reportsused ) #gbytes server_stats[server.filesPrefix] = { 'percentfull': server.percentfull, 'disksize': total_gb, 'diskfree': avail_gb, 'rawused': rawused, 'reportsused': reportsused, 'other': other, } # only generate list of experiments if Archiving is enabled if backup_object.online: # Experiments for this server explist = [] experiments_fs = experiments.filter( expDir__startswith=server.filesPrefix) for exp in experiments_fs: archive_experiment = ArchiveExperiment( exp, str(exp.expName), exp.date.strftime("%Y-%m-%d"), str(exp.star), str(exp.storage_options), str(exp.user_ack), str(exp.expDir), exp.pk, str(exp.rawdatastyle), str(exp.diskusage) if exp.diskusage is not None else "Unknown", ) explist.append(archive_experiment) if 'KI' in exp.storage_options: storage_stats[0] += 1 elif 'A' in exp.storage_options: storage_stats[1] += 1 elif 'D' in exp.storage_options: storage_stats[2] += 1 to_archive[server.filesPrefix] = explist return to_archive, server_stats, storage_stats
def disk_usage_stats(): stats = {} def diskspace_used(objects): ''' Sums diskusage field for given Experiment or Result objects diskusage stores MB ''' tot_disk_used = 0 diskusage_list = objects.values_list('diskusage', flat=True) for diskusage in diskusage_list: tot_disk_used += diskusage if diskusage is not None else 0 return tot_disk_used servers = get_servers() for server in servers: # Statistics for this location errormsg = None try: total, availSpace, freeSpace, bsize = disk_attributes( server.filesPrefix) # bytes total_gb = float(total * bsize) / (1024 * 1024 * 1024) avail_gb = float(availSpace * bsize) / (1024 * 1024 * 1024) free_gb = float(freeSpace * bsize) / (1024 * 1024 * 1024) percentfull = 100 - (float(availSpace) / float(total) * 100) if total > 0 else 0 reserved = free_gb - avail_gb except: errormsg = "Error accessing %s filesystem." % server.filesPrefix percentfull = '0' total_gb = '' avail_gb = '' free_gb = '' reserved = '' other = '' logger.error(traceback.format_exc()) # for experiment data: experiments = Experiment.objects.filter( expDir__startswith=server.filesPrefix) rawused = float(diskspace_used(experiments)) / 1024 #gbytes # for results data: results = Results.objects.filter( reportstorage__dirPath__startswith=server.filesPrefix) reportsused = float(diskspace_used(results)) / 1024 #gbytes other = total_gb - (reserved + avail_gb + rawused + reportsused ) #gbytes stats[server.filesPrefix] = { 'statusmsg': errormsg, 'percentfull': percentfull, 'disksize': total_gb, 'diskfree': avail_gb, 'rawused': rawused, 'reportsused': reportsused, 'other': other, } return stats