def moveJobs(t): #get start_time global JOB_PATH global TACC_STATS_PATH global RANGER_WORK_PATH start_time=long(t) end_time=start_time + ONEDAY #open job archive and put in list f = open(job.sge_acct_path) rd = sge_acct.reader(f, start_time=start_time, end_time=end_time) print "Making list: Start time = " +str(datetime.datetime.fromtimestamp(start_time))+"..." lis = list(rd) f.close() #pickle every job for acct in lis: try: j=job.from_acct(acct) except TypeError: log(acct) #TODO: Handle this better...just skips job if it can't get the job #thisPath=os.path.join([JOB_PATH,str(j.id)+".pkl"]) thisPath=JOB_PATH+"/"+str(j.id)+".pkl" f=open(thisPath,'wb') pickle.dump(j,f,-1) f.close() #compress dir containing .pkl files subprocess.call(["tar","zcvf",RANGER_WORK_PATH+"/"+str(start_time)+".tar.gz","-C",RANGER_WORK_PATH,str(start_time)]) #transfer to tacc-stats subprocess.call(["scp",RANGER_WORK_PATH+"/"+str(start_time)+".tar.gz",TACC_STATS_PATH]) #remove data from /work subprocess.call(["mv",RANGER_WORK_PATH+"/"+str(start_time)+".tar.gz",RANGER_WORK_PATH+"/"+str(start_time)+"/"]) subprocess.call(["rm","-rf",RANGER_WORK_PATH+"/"+str(start_time)])
def test(): jobid = '2255593' acct_file = open('accounting', 'r') acct_lis = [a for a in sge_acct.reader(acct_file)] acct = filter(lambda acct: acct['id'] == jobid, acct_lis)[0] job = Job(acct) job.gather_stats() job.munge_times() job.process_stats() return job
def moveJobs(t): #get start_time global JOB_PATH global TACC_STATS_PATH global RANGER_WORK_PATH start_time = long(t) end_time = start_time + ONEDAY #open job archive and put in list f = open(job.sge_acct_path) rd = sge_acct.reader(f, start_time=start_time, end_time=end_time) print "Making list: Start time = " + str( datetime.datetime.fromtimestamp(start_time)) + "..." lis = list(rd) f.close() #pickle every job for acct in lis: try: j = job.from_acct(acct) except TypeError: log(acct) #TODO: Handle this better...just skips job if it can't get the job #thisPath=os.path.join([JOB_PATH,str(j.id)+".pkl"]) thisPath = JOB_PATH + "/" + str(j.id) + ".pkl" f = open(thisPath, 'wb') pickle.dump(j, f, -1) f.close() #compress dir containing .pkl files subprocess.call([ "tar", "zcvf", RANGER_WORK_PATH + "/" + str(start_time) + ".tar.gz", "-C", RANGER_WORK_PATH, str(start_time) ]) #transfer to tacc-stats subprocess.call([ "scp", RANGER_WORK_PATH + "/" + str(start_time) + ".tar.gz", TACC_STATS_PATH ]) #remove data from /work subprocess.call([ "mv", RANGER_WORK_PATH + "/" + str(start_time) + ".tar.gz", RANGER_WORK_PATH + "/" + str(start_time) + "/" ]) subprocess.call(["rm", "-rf", RANGER_WORK_PATH + "/" + str(start_time)])
USAGE("START_DATE END_DATE [HOST]...") start = getdate(sys.argv[1]) end = getdate(sys.argv[2]) host_set = set(short_host_name(arg) for arg in sys.argv[3:]) def print_acct(acct, host='-'): start_time = time.strftime(' %b %d %T ', time.localtime(acct['start_time'])) end_time = time.strftime(' %b %d %T ', time.localtime(acct['end_time'])) owner = acct['owner'].ljust(10) slots = str(acct['slots']).rjust(5) print acct['id'], start_time, end_time, host, owner, slots # Run though all jobs that ended after start and before end + 3 days. seek = 600 << 20 # XXX for acct in sge_acct.reader(open(acct_path), start_time=start, end_time=end + 3 * 86400, seek=seek): if acct['end_time'] == 0: continue if max(acct['start_time'], start) <= min(acct['end_time'], end): if host_set: for host in get_host_list(acct): host = short_host_name(host) if host in host_set: print_acct(acct, host) else: print_acct(acct)
end = getdate(sys.argv[2]) host_set = set(short_host_name(arg) for arg in sys.argv[3:]) def print_acct(acct, host='-'): start_time = time.strftime(' %b %d %T ', time.localtime(acct['start_time'])) end_time = time.strftime(' %b %d %T ', time.localtime(acct['end_time'])) owner = acct['owner'].ljust(10) slots = str(acct['slots']).rjust(5) print acct['id'], start_time, end_time, host, owner, slots # Run though all jobs that ended after start and before end + 3 days. seek = 600 << 20 # XXX for acct in sge_acct.reader(open(acct_path), start_time=start, end_time=end + 3 * 86400, seek=seek): if acct['end_time'] == 0: continue if max(acct['start_time'], start) <= min(acct['end_time'], end): if host_set: for host in get_host_list(acct): host = short_host_name(host) if host in host_set: print_acct(acct, host) else: print_acct(acct)