#!/usr/bin/env python import os import sys sys.path.insert(0, os.path.expandvars("../")) import alogger print alogger.__file__ from alogger import log_to_dict fd = open('examples/torque') lines = fd.readlines() fd.close() for line in lines: try: print log_to_dict(line, 'PBS') except KeyError, ValueError: pass
#!/usr/bin/env python import os import sys sys.path.insert(0, os.path.expandvars("../")) import alogger print alogger.__file__ from alogger import log_to_dict fd = open('examples/winhpc') lines = fd.readlines() fd.close() for line in lines: #try: print log_to_dict(line, 'WINHPC') #except KeyError, ValueError: # pass
#!/usr/bin/env python import os import sys sys.path.insert(0, os.path.expandvars("../")) import alogger print alogger.__file__ from alogger import log_to_dict fd = open('examples/sge') lines = fd.readlines() fd.close() for line in lines: try: print log_to_dict(line, 'SGE') except KeyError, ValueError: pass
def parse_logs(log_list, date, machine_name, log_type): """ filename format YYYYMMDD """ output = [] count = fail = skip = updated = 0 line_no = 0 # Check things are setup correctly try: machine = Machine.objects.get(name=machine_name) except Machine.DoesNotExist: return "ERROR: Couldn't find machine named: %s" % machine_name # Process each line for line in log_list: line_no = line_no + 1 try: data = log_to_dict(line, log_type) except ValueError: output.append("Error reading line") except Exception, e: skip = skip + 1 continue try: account = Account.objects.get(username=data['user'], machine_category=machine.category, date_deleted__isnull=True) except Account.DoesNotExist: # Couldn't find user account - Assign to user None account = None output.append("Couldn't find user account for username=%s and machine category=%s. Assigned to None" % (data['user'], machine.category.name)) fail = fail + 1 except Account.MultipleObjectsReturned: account = None output.append("Username %s has multiple active accounts on machine category %s. Assigned to None" % (data['user'], machine.category.name)) fail = fail + 1 if 'project' in data: try: project = Project.objects.get(pid=data['project']) except Project.DoesNotExist: output.append("Couldn't find specified project %s, using default project" % data['project']) fail = fail + 1 try: project = account.default_project except: output.append("Couldn't find default project %s, using None" % account.default_project) project = None else: try: project = account.default_project except: # Couldn't find project - Assign to None output.append("Couldn't find default project for username=%s and machine category=%s" % (data['user'], machine.category.name)) project = None fail += 1 if project is not None and account is not None: if account.user not in project.users.all(): output.append("%s is not in project %s, cpu usage: %s" % (account.user, project, data['cpu_usage'])) fail += 1 # Everything is good so add entry queue, created = Queue.objects.get_or_create(name=data['queue']) if machine.mem_per_core: avail_mem_per_core = machine.mem_per_core * 1024 avail_mem_for_job = avail_mem_per_core * data['cores'] if data['list_pmem'] * data['cores'] > data['list_mem']: memory_used_per_core = data['list_pmem'] memory_used_for_job = data['list_pmem'] * data['cores'] else: memory_used_per_core = data['list_mem'] / data['cores'] memory_used_for_job = data['list_mem'] if memory_used_for_job > avail_mem_for_job: data['cpu_usage'] = ceil(memory_used_per_core/avail_mem_per_core * data['act_wall_time'] * data['cores']) data['cpu_usage'] = data['cpu_usage'] * machine.scaling_factor try: cpujob, created = CPUJob.objects.get_or_create(jobid=data['jobid']) cpujob.user=account cpujob.username=data['user'] cpujob.project=project cpujob.machine=machine cpujob.date=date cpujob.queue=queue cpujob.cpu_usage=data['cpu_usage'] cpujob.est_wall_time=data['est_wall_time'] cpujob.act_wall_time=data['act_wall_time'] cpujob.mem = data['mem'] cpujob.vmem = data['vmem'] cpujob.ctime = data['ctime'] cpujob.qtime = data['qtime'] cpujob.etime = data['etime'] cpujob.start = data['start'] cpujob.cores = data['cores'] cpujob.exit_status = data['exit_status'] cpujob.jobname = data['jobname'] cpujob.list_mem = data['list_mem'] cpujob.list_vmem = data['list_vmem'] cpujob.list_pmem = data['list_pmem'] cpujob.list_pvmem = data['list_pvmem'] cpujob.save() if created: count += 1 else: updated += 1 except Exception, e: output.append("Failed to insert a line - %s" % e) fail = fail + 1 continue