Ejemplo n.º 1
0
def main():
    totalbytes = 24*1024*1024
    justshow = False

    # ndir
    exps = [0] 
    ndir = [2**x for x in exps] # 1
    
    # nfile_per_dir
    exps = [2]
    nfile_per_dir = [2**x for x in exps] #2

    # nops_per_file
    nops_per_file = [0] #3, decided by totalbytes
    #size_per_op = [1, 1024, 4096, 1024*1024, 4*1024*1024], reverse=True) #4 

    # others
    size_per_op = [1] #4 
    do_fsync = [1] #5
    do_write = [0] #6
    do_read = [0]  #7
    topdir = ["/l0"] #8 
    do_append = [0,1]

    NDIR = 1
    NFILE_PER_DIR = 2
    NOPS_PER_FILE = 3
    SIZE_PER_OP = 4
    DO_FSYNC = 5
    DO_WRITE = 6
    DO_READ = 7
    TOPDIR = 8
    DO_APPEND = 9
    
    parameters = [ndir, nfile_per_dir, nops_per_file, 
                  size_per_op, do_fsync, do_write, do_read, topdir, do_append]
    paralist = list(itertools.product(*parameters))

    partition = "/dev/sda4"

    jobid = strftime("%Y-%m-%d-%H-%M-%S", localtime())
    resultname = jobid + ".result"

    result_file = open(jobid+".result", 'w')
    for rep in range(3):
        for para in paralist:
            para = list(para)
            cmd = ['./fsbench'] + para
        
            #########################
            # get a clean file system
            if not justshow:
                FormatFS.remakeExt4(partition, cmd[8], "jhe", "plfs", 
                    blockscount=2*1024*1024, blocksize=4096)

            #########################
            # Create files for later reads
            filesize = totalbytes / (cmd[NDIR]*cmd[NFILE_PER_DIR])
            opsize = min(1024, filesize) # in case the file is too small, like 1 byte
            _nops_per_file = filesize / opsize
            if _nops_per_file < 1:
                # we don't accept number that is not 2^x
                continue
            cmd[SIZE_PER_OP] = opsize
            cmd[NOPS_PER_FILE] = _nops_per_file

            cmd[DO_WRITE] = 1
            cmd[DO_READ] = 0

            # Now we do the two tests:
            #   1. write all, then close
            #   2. write one opsize, close, write next one, close ...
            if  cmd[DO_APPEND] == 0:
                cmd[NOPS_PER_FILE] = _nops_per_file
                cmd[DO_FSYNC] = 0

                print "For writing:", cmd
                cmd = [str(x) for x in cmd]
                if not justshow:
                    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
                    proc.wait()

            else:
                # do the appending thing
                # size per operation does not change, 
                # we just do one operation per file open
                cmd[DO_APPEND] = 1
                cmd[DO_FSYNC] = 1
                print "repeat:", _nops_per_file, "For writing (append):", cmd
                for i in range(_nops_per_file):
                    cmd[NOPS_PER_FILE] = 1
                    cmd = [str(x) for x in cmd]
                    if not justshow:
                        proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
                        proc.wait()

            ########################
            # Read the files in different ways, re-mount before 
            # each reading


            opsizes = sorted([1024], reverse=True)
            
            # do read for each operation size if it is valid
            for _size_per_op in opsizes:
                #re-mount the file system to drop caches
                if not justshow:
                    FormatFS.umountFS(cmd[8])
                    FormatFS.mountExt4(partition, cmd[8])

                _nops_per_file = filesize / _size_per_op

                if _nops_per_file < 1:
                    # we don't accept number that is not 2^x
                    continue
                cmd[NOPS_PER_FILE] = _nops_per_file
                cmd[SIZE_PER_OP] = _size_per_op
                
                cmd[DO_WRITE] = 0
                cmd[DO_READ] = 1

                cmd = [str(x) for x in cmd]
                print "For read:", cmd

                # Run it
                if not justshow:
                    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
                    proc.wait()
                    for line in proc.stdout:
                        print line,
                        result_file.write(line)

    result_file.close()
Ejemplo n.º 2
0
def main():
    totalbytes = 1*1024*1024*1024

    # ndir
    exps = [0, 1, 9] 
    #exps = [9] 
    ndir = [2**x for x in exps] # 1
    
    # nfile_per_dir
    exps = [0, 1, 9]
    #exps = [9]
    nfile_per_dir = [2**x for x in exps] #2

    # nops_per_file
    nops_per_file = [0] #3, decided by totalbytes
    #size_per_op = [1, 1024, 4096, 1024*1024, 4*1024*1024], reverse=True) #4 

    # others
    size_per_op = [1] #4 
    do_fsync = [0] #5
    do_write = [0] #6
    do_read = [0]  #7
    topdir = ["/l0"] #8 

    NDIR = 1
    NFILE_PER_DIR = 2
    NOPS_PER_FILE = 3
    SIZE_PER_OP = 4
    DO_FSYNC = 5
    DO_WRITE = 6
    DO_READ = 7
    TOPDIR = 8
    
    #parameters = [ndir, nfile_per_dir, nops_per_file, 
    #              size_per_op, do_fsync, do_write, do_read, topdir]
    #paralist = list(itertools.product(*parameters))
    paralist = [ \
                 [ 1, 1, 1048576, 1024, 0, 0, 0, '/l0'],
                 [ 512, 512, 4096, 1, 0, 0, 0, '/l0'] \
               ]

    partition = "/dev/sda4"

    jobid = strftime("%Y-%m-%d-%H-%M-%S", localtime())
    resultname = jobid + ".result"

    result_file = open(jobid+".result", 'w')
    for rep in range(5):
        for para in paralist:
            para = list(para)
            cmd = ['./fsbench'] + para
        
            #########################
            # get a clean file system
            FormatFS.remakeExt4(partition, cmd[8], "jhe", "plfs", 
                blockscount=2*1024*1024, blocksize=4096)

            #########################
            # Create files for later reads
            filesize = totalbytes / (cmd[NDIR]*cmd[NFILE_PER_DIR])
            opsize = min(4096, filesize) # in case the file is too small, like 1 byte
            _nops_per_file = filesize / opsize
            if _nops_per_file < 1:
                # we don't accept number that is not 2^x
                continue
            cmd[SIZE_PER_OP] = opsize
            cmd[NOPS_PER_FILE] = _nops_per_file

            cmd[DO_WRITE] = 1
            cmd[DO_READ] = 0
            cmd = [str(x) for x in cmd]
            print "For writing:", cmd


            cmd[NOPS_PER_FILE] = _nops_per_file
            cmd = [str(x) for x in cmd]

            #print cmd
            #continue

            proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
            proc.wait()


            ########################
            # Read the files in different ways, re-mount before 
            # each reading


            opsizes = sorted([1, 1024, 4*1024], reverse=True)
            
            # do read for each operation size if it is valid
            for _size_per_op in opsizes:
                if cmd[NDIR] == "512" and _size_per_op != 1:
                    continue
                if cmd[NDIR] == "1" and _size_per_op not in [1024, 4096]:
                    continue
                #re-mount the file system to drop caches
                FormatFS.umountFS(cmd[8])
                FormatFS.mountExt4(partition, cmd[8])

                _nops_per_file = filesize / _size_per_op

                if _nops_per_file < 1:
                    # we don't accept number that is not 2^x
                    continue
                cmd[NOPS_PER_FILE] = _nops_per_file
                cmd[SIZE_PER_OP] = _size_per_op
                
                cmd[DO_WRITE] = 0
                cmd[DO_READ] = 1

                cmd = [str(x) for x in cmd]
                print "For read:", cmd

                # Run it
                proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
                proc.wait()
                for line in proc.stdout:
                    print line,
                    result_file.write(line)
                    result_file.flush()

    result_file.close()
def main():
    # justshow = True
    justshow = False

    ndir = [1]  # 1
    nfile_per_dir = [1]  # 2
    nops_per_file = [0]  # 3, decided by totalbytes
    size_per_op = [1]  # 4
    do_fsync = [0]  # 5
    do_write = [0]  # 6
    do_read = [0]  # 7
    topdir = ["/l0"]  # 8
    do_append = [1]  # 9
    totalbytes = [128 * 1024 * 4 * 1024]  # 10
    switchcpu = [0]

    NDIR = 1
    NFILE_PER_DIR = 2
    NOPS_PER_FILE = 3
    SIZE_PER_OP = 4
    DO_FSYNC = 5
    DO_WRITE = 6
    DO_READ = 7
    TOPDIR = 8
    DO_APPEND = 9
    TOTALBYTES = 10
    SWITCHCPU = 11

    parameters = [
        ndir,
        nfile_per_dir,
        nops_per_file,
        size_per_op,
        do_fsync,
        do_write,
        do_read,
        topdir,
        do_append,
        totalbytes,
        switchcpu,
    ]
    paralist = list(itertools.product(*parameters))

    partition = "/dev/sda4"

    jobid = strftime("%Y-%m-%d-%H-%M-%S", localtime())
    resultname = jobid + ".result"

    result_file = open(jobid + ".result", "w")
    for rep in range(5):
        for para in paralist:
            para = list(para)
            cmd = ["./fsbench"] + para

            #########################
            # get a clean file system
            if not justshow:
                FormatFS.remakeExt4(partition, cmd[8], "jhe", "plfs", blockscount=2 * 1024 * 1024, blocksize=4096)
                subprocess.Popen("echo 262144 > /sys/fs/ext4/sda4/mb_stream_req", shell=True)
                sleep(3)

            #########################
            # Create files for later reads
            filesize = cmd[TOTALBYTES] / (cmd[NDIR] * cmd[NFILE_PER_DIR])
            opsize = min(4096, filesize)  # in case the file is too small, like 1 byte
            _nops_per_file = filesize / opsize
            if _nops_per_file < 1:
                # we don't accept number that is not 2^x
                continue
            cmd[SIZE_PER_OP] = opsize
            cmd[NOPS_PER_FILE] = _nops_per_file

            cmd[DO_WRITE] = 1
            cmd[DO_READ] = 0

            if cmd[SWITCHCPU] == 0:
                # do not switch cpu
                cmd[DO_APPEND] = 1
                cmd[DO_FSYNC] = 0
                print "repeat:", _nops_per_file, "For writing NOSWITCH (append):", trimcmd(cmd)
                for i in range(_nops_per_file):
                    cmd[NOPS_PER_FILE] = 1
                    cmd = [str(x) for x in cmd]
                    if not justshow:
                        proc = subprocess.Popen(trimcmd(cmd), stdout=subprocess.PIPE)
                        proc.wait()

            else:
                # do the appending thing
                # size per operation does not change,
                # we just do one operation per file open
                cmd[DO_APPEND] = 1
                cmd[DO_FSYNC] = 1
                print "repeat:", _nops_per_file, "For writing (append):", trimcmd(cmd)
                for i in range(_nops_per_file):
                    cmd[NOPS_PER_FILE] = 1
                    cmd = [str(x) for x in cmd]
                    taskset_wrapper = ["taskset", "0x0000000" + str((i % 2) + 1)]
                    # print taskset_wrapper + trimcmd(cmd)

                    if not justshow:
                        proc = subprocess.Popen(taskset_wrapper + trimcmd(cmd), stdout=subprocess.PIPE)
                        proc.wait()

            ########################
            # Read the files in different ways, re-mount before
            # each reading

            opsizes = sorted([4096], reverse=True)

            # do read for each operation size if it is valid
            for _size_per_op in opsizes:
                # re-mount the file system to drop caches
                if not justshow:
                    FormatFS.umountFS(cmd[8])
                    FormatFS.mountExt4(partition, cmd[8])

                _nops_per_file = filesize / _size_per_op

                if _nops_per_file < 1:
                    # we don't accept number that is not 2^x
                    continue
                cmd[NOPS_PER_FILE] = _nops_per_file
                cmd[SIZE_PER_OP] = _size_per_op

                cmd[DO_WRITE] = 0
                cmd[DO_READ] = 1

                cmd = [str(x) for x in cmd]
                print "For read:", trimcmd(cmd)

                # Run it
                if not justshow:
                    proc = subprocess.Popen(trimcmd(cmd), stdout=subprocess.PIPE)
                    proc.wait()
                    for line in proc.stdout:
                        line = decrate(line, "switchcpu", cmd[SWITCHCPU])
                        print line,
                        result_file.write(line)

    result_file.close()
def main():

    # ndir
    exps = [2] 
    ndir = [2**x for x in exps] # 1
    
    # nfile_per_dir
    exps = [2]
    nfile_per_dir = [2**x for x in exps] #2

    # nops_per_file
    nops_per_file = [0] #3, decided by totalbytes
    #size_per_op = [1, 1024, 4096, 1024*1024, 4*1024*1024], reverse=True) #4 

    # others
    size_per_op = [0] #4 
    do_fsync = [0] #5
    do_write = [0] #6
    do_read = [0]  #7
    topdir = ["/l0"] #8 

    NDIR = 1
    NFILE_PER_DIR = 2
    NOPS_PER_FILE = 3
    SIZE_PER_OP = 4
    DO_FSYNC = 5
    DO_WRITE = 6
    DO_READ = 7
    TOPDIR = 8
    
    parameters = [ndir, nfile_per_dir, nops_per_file, 
                  size_per_op, do_fsync, do_write, do_read, topdir]
    paralist = list(itertools.product(*parameters))

    partition = "/dev/sda4"

    jobid = strftime("%Y-%m-%d-%H-%M-%S", localtime())
    resultname = jobid + ".result"

    result_file = open(jobid+".result", 'w')
    for rep in range(5):
        for para in paralist:
            para = list(para)
            cmd = ['./fsbench'] + para
        
            filesizes = range(64-4, 64+4)
            filesizes = [x*1024 for x in filesizes]

            for filesize in filesizes:    
                #########################
                # get a clean file system
                FormatFS.remakeExt4(partition, cmd[8], "jhe", "plfs", 
                    blockscount=2*1024*1024, blocksize=4096)

                #########################
                # Create files for later reads
                cmd[SIZE_PER_OP] = filesize
                cmd[NOPS_PER_FILE] = 1

                cmd[DO_WRITE] = 1
                cmd[DO_READ] = 0
                cmd = [str(x) for x in cmd]
                print "For writing:", cmd

                proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
                proc.wait()


                ########################
                # Read the files in different ways, re-mount before 
                # each reading

                #re-mount the file system to drop caches
                FormatFS.umountFS(cmd[8])
                FormatFS.mountExt4(partition, cmd[8])
                
                cmd[DO_WRITE] = 0
                cmd[DO_READ] = 1
                # make the total amount of bytes read the same
                cmd[SIZE_PER_OP] = min(filesizes) 

                cmd = [str(x) for x in cmd]
                print "For read:", cmd

                # Run it
                proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
                proc.wait()
                for line in proc.stdout:
                    if "HEADERMARKER" in line:
                        elems = line.split()
                        elems.append("original_file_size")
                    elif "DATAMARKER" in line:
                        elems = line.split()
                        elems.append(filesize)
                    else:
                        continue
                    line = [str(x) for x in elems]
                    print line,
                    result_file.write(line)

    result_file.close()