Exemplos de reader em Python

Exemplo n.º 1

Arquivo: move_jobs.py Projeto: Jlong591/tacc_stats

def moveJobs(t):
#get start_time
 global JOB_PATH
 global TACC_STATS_PATH
 global RANGER_WORK_PATH
 start_time=long(t)
 end_time=start_time + ONEDAY

#open job archive and put in list
 f = open(job.sge_acct_path)
 rd = sge_acct.reader(f, start_time=start_time, end_time=end_time)
 print "Making list: Start time = " +str(datetime.datetime.fromtimestamp(start_time))+"..."
 lis = list(rd)
 f.close()
#pickle every job
 for acct in lis:
  try:
   j=job.from_acct(acct)
  except TypeError:
   log(acct)
  #TODO: Handle this better...just skips job if it can't get the job
  #thisPath=os.path.join([JOB_PATH,str(j.id)+".pkl"])
  thisPath=JOB_PATH+"/"+str(j.id)+".pkl"
  f=open(thisPath,'wb')
  pickle.dump(j,f,-1)
  f.close()
#compress dir containing .pkl files
 subprocess.call(["tar","zcvf",RANGER_WORK_PATH+"/"+str(start_time)+".tar.gz","-C",RANGER_WORK_PATH,str(start_time)])
#transfer to tacc-stats
 subprocess.call(["scp",RANGER_WORK_PATH+"/"+str(start_time)+".tar.gz",TACC_STATS_PATH])
#remove data from /work
 subprocess.call(["mv",RANGER_WORK_PATH+"/"+str(start_time)+".tar.gz",RANGER_WORK_PATH+"/"+str(start_time)+"/"])
 subprocess.call(["rm","-rf",RANGER_WORK_PATH+"/"+str(start_time)])

Exemplo n.º 2

def test():
    jobid = '2255593'
    acct_file = open('accounting', 'r')
    acct_lis = [a for a in sge_acct.reader(acct_file)]
    acct = filter(lambda acct: acct['id'] == jobid, acct_lis)[0]
    job = Job(acct)
    job.gather_stats()
    job.munge_times()
    job.process_stats()
    return job

Exemplo n.º 3

Arquivo: job.py Projeto: TACCProjects/tacc_stats

def test():
    jobid = '2255593'
    acct_file = open('accounting', 'r')
    acct_lis = [a for a in sge_acct.reader(acct_file)]
    acct = filter(lambda acct: acct['id'] == jobid, acct_lis)[0]
    job = Job(acct)
    job.gather_stats()
    job.munge_times()
    job.process_stats()
    return job

Exemplo n.º 4

def moveJobs(t):
    #get start_time
    global JOB_PATH
    global TACC_STATS_PATH
    global RANGER_WORK_PATH
    start_time = long(t)
    end_time = start_time + ONEDAY

    #open job archive and put in list
    f = open(job.sge_acct_path)
    rd = sge_acct.reader(f, start_time=start_time, end_time=end_time)
    print "Making list: Start time = " + str(
        datetime.datetime.fromtimestamp(start_time)) + "..."
    lis = list(rd)
    f.close()
    #pickle every job
    for acct in lis:
        try:
            j = job.from_acct(acct)
        except TypeError:
            log(acct)
        #TODO: Handle this better...just skips job if it can't get the job
        #thisPath=os.path.join([JOB_PATH,str(j.id)+".pkl"])
        thisPath = JOB_PATH + "/" + str(j.id) + ".pkl"
        f = open(thisPath, 'wb')
        pickle.dump(j, f, -1)
        f.close()


#compress dir containing .pkl files
    subprocess.call([
        "tar", "zcvf", RANGER_WORK_PATH + "/" + str(start_time) + ".tar.gz",
        "-C", RANGER_WORK_PATH,
        str(start_time)
    ])
    #transfer to tacc-stats
    subprocess.call([
        "scp", RANGER_WORK_PATH + "/" + str(start_time) + ".tar.gz",
        TACC_STATS_PATH
    ])
    #remove data from /work
    subprocess.call([
        "mv", RANGER_WORK_PATH + "/" + str(start_time) + ".tar.gz",
        RANGER_WORK_PATH + "/" + str(start_time) + "/"
    ])
    subprocess.call(["rm", "-rf", RANGER_WORK_PATH + "/" + str(start_time)])

Exemplo n.º 5

Arquivo: find_jobs_between.py Projeto: Lezval/tacc_stats

    USAGE("START_DATE END_DATE [HOST]...")

start = getdate(sys.argv[1])
end = getdate(sys.argv[2])
host_set = set(short_host_name(arg) for arg in sys.argv[3:])

def print_acct(acct, host='-'):
    start_time = time.strftime(' %b %d %T ', time.localtime(acct['start_time']))
    end_time = time.strftime(' %b %d %T ', time.localtime(acct['end_time']))
    owner = acct['owner'].ljust(10)
    slots = str(acct['slots']).rjust(5)
    print acct['id'], start_time, end_time, host, owner, slots

# Run though all jobs that ended after start and before end + 3 days.
seek = 600 << 20 # XXX

for acct in sge_acct.reader(open(acct_path),
                            start_time=start,
                            end_time=end + 3 * 86400,
                            seek=seek):
    if acct['end_time'] == 0:
        continue
    if max(acct['start_time'], start) <= min(acct['end_time'], end):
        if host_set:
            for host in get_host_list(acct):
                host = short_host_name(host)
                if host in host_set:
                    print_acct(acct, host)
        else:
            print_acct(acct)

Exemplo n.º 6

Arquivo: find_jobs_between.py Projeto: kylemarcus/tacc_stats

end = getdate(sys.argv[2])
host_set = set(short_host_name(arg) for arg in sys.argv[3:])


def print_acct(acct, host='-'):
    start_time = time.strftime(' %b %d %T ',
                               time.localtime(acct['start_time']))
    end_time = time.strftime(' %b %d %T ', time.localtime(acct['end_time']))
    owner = acct['owner'].ljust(10)
    slots = str(acct['slots']).rjust(5)
    print acct['id'], start_time, end_time, host, owner, slots


# Run though all jobs that ended after start and before end + 3 days.
seek = 600 << 20  # XXX

for acct in sge_acct.reader(open(acct_path),
                            start_time=start,
                            end_time=end + 3 * 86400,
                            seek=seek):
    if acct['end_time'] == 0:
        continue
    if max(acct['start_time'], start) <= min(acct['end_time'], end):
        if host_set:
            for host in get_host_list(acct):
                host = short_host_name(host)
                if host in host_set:
                    print_acct(acct, host)
        else:
            print_acct(acct)