def main(): p = subprocess.Popen(["df", "-P", MOUNT], stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdoutdata, stderrdata) = p.communicate() for line in stdoutdata.split("\n"): split_line = line.split() if len(split_line) < 5: continue if split_line[5] == MOUNT: dash = dashing.DashingImport('viz.unl.edu', auth_token = '542221b1-b765-4cd9-a9e6-0c0727870375') send_dict = { 'min': 0, 'max': float("%.1f" % (float(split_line[1]) / terabyte)) , 'value': float("%.1f" % (float(split_line[2]) / terabyte)), 'moreinfo': "Capacity: %s" % sizeof_fmt(int(split_line[1])) } dash.SendEvent('CraneStorage', send_dict) dash.SendEvent('HCCAmazonPrice', {'craneStorage': send_dict['value']}) # Send the number of jobs running command = "squeue -t R -O numcpus,account -h".split(" ") p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = p.communicate() sum_running_cores = 0 per_user_cores = {} for line in stdout.split("\n"): try: (cores, username) = line.split() sum_running_cores += int(cores) username.strip() if username not in per_user_cores: per_user_cores[username] = 0 per_user_cores[username] += int(cores) except: print "Error parsing line: %s" % line pass dash.SendEvent('CraneRunning', {'current': sum_running_cores, 'last': sum_running_cores}) dash.SendEvent('HCCAmazonPrice', {'CraneCores': sum_running_cores})
def getUNOTimeData(timeDelta, SQLItems, scoreBox, numEntries): xdmoddb = SQL.connect(host=SQLItems["xdmodmysql_host"], user=SQLItems["xdmodmysql_username"], password=SQLItems["xdmodmysql_pass"], db=SQLItems["xdmodmysql_db"], cursorclass=pymysql.cursors.DictCursor) with open(sys.argv[2], 'r') as file: auth_key = file.read().strip() dash = dashing.DashingImport('viz.unl.edu', auth_token=auth_key) # Dashboard Data Structure # Array of dict with keys of label, value, dept, campus dataToDash = [] with xdmoddb: cur = xdmoddb.cursor() stmt = "SELECT j.person_id,username, SUM(cpu_time), group_name,department,campus FROM jobfact as j INNER JOIN modw.systemaccount as h ON j.person_id=h.person_id INNER JOIN mod_shredder.ldapGroups as s ON j.group_name=s.GroupName WHERE campus not like 'UNL' and campus not like 'IANR' and start_time_ts > " + timeDelta + " GROUP BY j.person_id order by SUM(cpu_time) DESC LIMIT " + numEntries + ";" print(stmt) cur.execute(stmt) result = cur.fetchall() print(result) for i in result: print(i["SUM(cpu_time)"]) cpuHour = str(round(int(i["SUM(cpu_time)"]) / 3600)) if i["department"] == i["campus"]: i["campus"] = "" dataToDash.append({ "label": i["username"], "value": cpuHour, "dept": i["department"].replace("College of", ""), "campus": i["campus"] }) print(dataToDash) ### dostuff dash.SendEvent(scoreBox, {'items': dataToDash}) cur.close()
import urllib.request import re import dashing import json ### Import SQL Login f = open(sys.argv[1], 'r') lines = f.readlines() SQLItems = {} for i in lines: SQLItems[i.split(" ")[0]] = i.split(" ")[1][:-1] print(i.split(" ")[0], i.split(" ")[1][:-1]) f.close() with open('key.txt', 'r') as file: auth_key = file.read().strip() dash = dashing.DashingImport('viz.unl.edu', auth_token=auth_key) ### Cloud Service Pricing ###AWS -- 4 Core Instance with 8GB Memory for 1 hour #Storage: per GB Hour // 0.045 per GB/Month #Network: per GB ###GCP -- Retrieved from Google Cloud Platform #CPU: per CPU Hour #Storage: per GB Hour #Memory: per GB Hour // 0.040 per GB/Month #Network: per GB ###Azure -- 4 Core Instance with 8GB Memory for 1 hour #Storage: per GB Hour // 0.048 per GB/Month #Network: per GB ###DO -- 4 Core Instance with 8GB Memory for 1 hour
def main(): with open('key.txt', 'r') as file: auth_key = file.read().strip() p = subprocess.Popen(["df", "-P", MOUNT], stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdoutdata, stderrdata) = p.communicate() for line in stdoutdata.split("\n"): split_line = line.split() if len(split_line) < 5: continue if split_line[5] == MOUNT: dash = dashing.DashingImport('viz.unl.edu', port=3030, auth_token=auth_key) send_dict = { 'min': 0, 'max': float("%.1f" % (float(split_line[1]) / terabyte)), 'value': float("%.1f" % (float(split_line[2]) / terabyte)), 'moreinfo': "Capacity: %s" % sizeof_fmt(int(split_line[1])) } dash.SendEvent(CLUSTER + 'Storage', send_dict) dash.SendEvent('HCCAmazonPrice', {CLUSTER.lower() + 'Storage': send_dict['value']}) # Send the number of jobs running command = "squeue -t R -O numcpus,account -h".split(" ") p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = p.communicate() sum_running_cores = 0 per_user_cores = {} for line in stdout.split("\n"): if line == "": break try: (cores, username) = line.split() sum_running_cores += int(cores) username.strip() if username not in per_user_cores: per_user_cores[username] = 0 per_user_cores[username] += int(cores) except: print("Error parsing line: " + line) pass if os.path.isfile('dashing.txt') and os.access('dashing.txt', os.R_OK): date = os.path.getmtime('dashing.txt') with open('dashing.txt', 'r') as file: last_running_cores = int(file.read()) else: print("Error reading from dashing.txt") date = time.time() + 3600 last_running_cores = sum_running_cores with open('dashing.txt', 'w') as file: file.write(str(sum_running_cores)) date = time.strftime('%m-%d-%Y %H:%M:%S', time.localtime(date)) dash.SendEvent( CLUSTER + 'Running', { 'current': sum_running_cores, 'last': last_running_cores, 'last_period': date }) dash.SendEvent('HCCAmazonPrice', {CLUSTER + 'Cores': sum_running_cores}) # send number of completed jobs current_time = time.strftime('%m/%d/%y-%H:%M:%S', time.localtime(time.time())) start_time = time.strftime('%m/%d/%y', time.localtime(time.time())) command = "sacct -a -E " + current_time + " -S " + start_time + " -s CD -o JobID -X" command = command.split(" ") p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = p.communicate() jobs_completed = len(stdout.split()) path = '/common/swanson/.dashing/' files = ['crane_jobs.txt', 'rhino_jobs.txt'] filename = path + files[0] with open(filename, 'w') as file: file.write(str(jobs_completed)) total_jobs = 0 for filename in files: filename = path + filename if os.path.isfile(filename) and os.access(filename, os.R_OK): with open(filename, 'r') as file: total_jobs += int(file.read()) else: print("Error reading from " + filename) dash.SendEvent('JobsCompleted', {'current': total_jobs}) #Send number of CPU Hours for Today command = "sacct -a -o CPUTimeRaw -n -T" command = command.split(" ") p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = p.communicate() stdout = map(int, stdout.split()) hours_completed = sum(stdout) / 3600 files = ['crane_hours.txt', 'rhino_hours.txt'] filename = path + files[0] with open(path + CLUSTER.lower() + '_hours.txt', 'w') as file: file.write(str(hours_completed)) total_hours = 0 for filename in files: filename = path + filename if os.path.isfile(filename) and os.access(filename, os.R_OK): with open(filename, 'r') as file: total_hours += int(file.read()) else: print("Error reading from " + filename) dash.SendEvent('HoursToday', {'current': total_hours}) # Send Anvil Information ## Time Delay is to allow dashing to keep up with POST f = urllib2.urlopen("http://anvil-beta.unl.edu:8123/") rawData = json.load(f) dash.SendEvent('AnvilTile', {'current_vm': rawData["vm_count"]}) time.sleep(1) dash.SendEvent('AnvilTile', {'current_cores': rawData["core_count"]}) time.sleep(1) dash.SendEvent('AnvilTile', { 'current_mem': str(round(int(rawData["mem_count"]) / (1024.0**2), 2)) }) time.sleep(1) dash.SendEvent( 'AnvilTile', {'current_vol': str(round(int(rawData["volume_gb"]) / 1024.0, 2))}) time.sleep(1) dash.SendEvent( 'AnvilTile', {'current_disk': str(round(int(rawData["disk_gb"]) / 1024.0, 2))}) # Red Storage redT2 = urllib2.urlopen("http://t2.unl.edu:8088/dfshealth.jsp") redData = re.findall("\d+\.\d+", str(redT2.read())) dash.SendEvent( 'RedStorage', { 'min': 0, 'max': float(redData[9]) * 1024, 'value': float(redData[10]) * 1024, 'Capacity': redData[9] + " PB" }) dash.SendEvent('HCCAmazonPrice', {'redStorage': float(redData[10]) * 1024}) # Top Users UNL dbFile = open('db.yml', 'r') lines = dbFile.readlines() SQLItems = {} for i in lines: SQLItems[i.split(" ")[0]] = i.split(" ")[1][:-1] f.close() rcfdb = SQL.connect(host=SQLItems["rcfmysql_host"], user=SQLItems["rcfmysql_username"], passwd=SQLItems["rcfmysql_pass"], db=SQLItems["rcfmysql_db"], cursorclass=pymysql.cursors.DictCursor) ## Grab this clusters squeue command = "squeue -h -t R -o '%u %C'" p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = p.communicate() file = open(path + CLUSTER.lower() + '_users.txt', 'w') file.write(stdout) file.close() ## Pull top Users files = ['crane_users.txt', 'rhino_users.txt'] topUsers = {} for file in files: filename = path + file if os.path.isfile(filename) and os.access(filename, os.R_OK): userFile = open(filename, 'r').readlines() for line in userFile: if line.split(' ')[0] in topUsers: topUsers[line.split(' ')[0]] += int(line.split(' ')[1]) else: topUsers[line.split(' ')[0]] = int(line.split(' ')[1]) topUsers25 = sorted(topUsers.items(), key=operator.itemgetter(1), reverse=True)[:25] ## The real magic of sql begins dataToDash = [] cur = rcfdb.cursor() for k, v in topUsers25: stmt = "select Department, Campus from Personal where LoginID = \"" + k + "\";" cur.execute(stmt) result = cur.fetchall()[0] if result["Department"] == None: result["Department"] = "" if result["Campus"] == None: result["Campus"] = "" dataToDash.append({ "label": k[:9], "value": v, "dept": result["Department"][:14], "campus": result["Campus"] }) dash.SendEvent('BiggestUsers', {'items': dataToDash}) cur.close()
import rrdtool as rt import sys from time import sleep, gmtime, strftime from subprocess import call from sortedcontainers import SortedDict import dashing TOKEN = open("key.txt", 'r').readline()[:-1] dash = dashing.DashingImport('viz.unl.edu', auth_token=TOKEN) def rrdToArrayPoint(rrdFile): result = rt.fetch(rrdFile, "AVERAGE") start, end, step = result[0] legend = result[1] rows = result[2] endTime = end - 00 points = [] counter = 0 try: for i in range(20, 0, -1): #print(rrdFile,i,rows[len(rows)-(3+i)][1]) points.append({ "y": int(rows[len(rows) - (3 + i)][0] + rows[len(rows) - (3 + i)][1]) * 8, "x": endTime }) last_point = (rows[len(rows) - (3 + i)][0] + rows[len(rows) - (3 + i)][1]) endTime = endTime - 300