def main(): totalbytes = 24*1024*1024 justshow = False # ndir exps = [0] ndir = [2**x for x in exps] # 1 # nfile_per_dir exps = [2] nfile_per_dir = [2**x for x in exps] #2 # nops_per_file nops_per_file = [0] #3, decided by totalbytes #size_per_op = [1, 1024, 4096, 1024*1024, 4*1024*1024], reverse=True) #4 # others size_per_op = [1] #4 do_fsync = [1] #5 do_write = [0] #6 do_read = [0] #7 topdir = ["/l0"] #8 do_append = [0,1] NDIR = 1 NFILE_PER_DIR = 2 NOPS_PER_FILE = 3 SIZE_PER_OP = 4 DO_FSYNC = 5 DO_WRITE = 6 DO_READ = 7 TOPDIR = 8 DO_APPEND = 9 parameters = [ndir, nfile_per_dir, nops_per_file, size_per_op, do_fsync, do_write, do_read, topdir, do_append] paralist = list(itertools.product(*parameters)) partition = "/dev/sda4" jobid = strftime("%Y-%m-%d-%H-%M-%S", localtime()) resultname = jobid + ".result" result_file = open(jobid+".result", 'w') for rep in range(3): for para in paralist: para = list(para) cmd = ['./fsbench'] + para ######################### # get a clean file system if not justshow: FormatFS.remakeExt4(partition, cmd[8], "jhe", "plfs", blockscount=2*1024*1024, blocksize=4096) ######################### # Create files for later reads filesize = totalbytes / (cmd[NDIR]*cmd[NFILE_PER_DIR]) opsize = min(1024, filesize) # in case the file is too small, like 1 byte _nops_per_file = filesize / opsize if _nops_per_file < 1: # we don't accept number that is not 2^x continue cmd[SIZE_PER_OP] = opsize cmd[NOPS_PER_FILE] = _nops_per_file cmd[DO_WRITE] = 1 cmd[DO_READ] = 0 # Now we do the two tests: # 1. write all, then close # 2. write one opsize, close, write next one, close ... if cmd[DO_APPEND] == 0: cmd[NOPS_PER_FILE] = _nops_per_file cmd[DO_FSYNC] = 0 print "For writing:", cmd cmd = [str(x) for x in cmd] if not justshow: proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) proc.wait() else: # do the appending thing # size per operation does not change, # we just do one operation per file open cmd[DO_APPEND] = 1 cmd[DO_FSYNC] = 1 print "repeat:", _nops_per_file, "For writing (append):", cmd for i in range(_nops_per_file): cmd[NOPS_PER_FILE] = 1 cmd = [str(x) for x in cmd] if not justshow: proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) proc.wait() ######################## # Read the files in different ways, re-mount before # each reading opsizes = sorted([1024], reverse=True) # do read for each operation size if it is valid for _size_per_op in opsizes: #re-mount the file system to drop caches if not justshow: FormatFS.umountFS(cmd[8]) FormatFS.mountExt4(partition, cmd[8]) _nops_per_file = filesize / _size_per_op if _nops_per_file < 1: # we don't accept number that is not 2^x continue cmd[NOPS_PER_FILE] = _nops_per_file cmd[SIZE_PER_OP] = _size_per_op cmd[DO_WRITE] = 0 cmd[DO_READ] = 1 cmd = [str(x) for x in cmd] print "For read:", cmd # Run it if not justshow: proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) proc.wait() for line in proc.stdout: print line, result_file.write(line) result_file.close()
def main(): totalbytes = 1*1024*1024*1024 # ndir exps = [0, 1, 9] #exps = [9] ndir = [2**x for x in exps] # 1 # nfile_per_dir exps = [0, 1, 9] #exps = [9] nfile_per_dir = [2**x for x in exps] #2 # nops_per_file nops_per_file = [0] #3, decided by totalbytes #size_per_op = [1, 1024, 4096, 1024*1024, 4*1024*1024], reverse=True) #4 # others size_per_op = [1] #4 do_fsync = [0] #5 do_write = [0] #6 do_read = [0] #7 topdir = ["/l0"] #8 NDIR = 1 NFILE_PER_DIR = 2 NOPS_PER_FILE = 3 SIZE_PER_OP = 4 DO_FSYNC = 5 DO_WRITE = 6 DO_READ = 7 TOPDIR = 8 #parameters = [ndir, nfile_per_dir, nops_per_file, # size_per_op, do_fsync, do_write, do_read, topdir] #paralist = list(itertools.product(*parameters)) paralist = [ \ [ 1, 1, 1048576, 1024, 0, 0, 0, '/l0'], [ 512, 512, 4096, 1, 0, 0, 0, '/l0'] \ ] partition = "/dev/sda4" jobid = strftime("%Y-%m-%d-%H-%M-%S", localtime()) resultname = jobid + ".result" result_file = open(jobid+".result", 'w') for rep in range(5): for para in paralist: para = list(para) cmd = ['./fsbench'] + para ######################### # get a clean file system FormatFS.remakeExt4(partition, cmd[8], "jhe", "plfs", blockscount=2*1024*1024, blocksize=4096) ######################### # Create files for later reads filesize = totalbytes / (cmd[NDIR]*cmd[NFILE_PER_DIR]) opsize = min(4096, filesize) # in case the file is too small, like 1 byte _nops_per_file = filesize / opsize if _nops_per_file < 1: # we don't accept number that is not 2^x continue cmd[SIZE_PER_OP] = opsize cmd[NOPS_PER_FILE] = _nops_per_file cmd[DO_WRITE] = 1 cmd[DO_READ] = 0 cmd = [str(x) for x in cmd] print "For writing:", cmd cmd[NOPS_PER_FILE] = _nops_per_file cmd = [str(x) for x in cmd] #print cmd #continue proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) proc.wait() ######################## # Read the files in different ways, re-mount before # each reading opsizes = sorted([1, 1024, 4*1024], reverse=True) # do read for each operation size if it is valid for _size_per_op in opsizes: if cmd[NDIR] == "512" and _size_per_op != 1: continue if cmd[NDIR] == "1" and _size_per_op not in [1024, 4096]: continue #re-mount the file system to drop caches FormatFS.umountFS(cmd[8]) FormatFS.mountExt4(partition, cmd[8]) _nops_per_file = filesize / _size_per_op if _nops_per_file < 1: # we don't accept number that is not 2^x continue cmd[NOPS_PER_FILE] = _nops_per_file cmd[SIZE_PER_OP] = _size_per_op cmd[DO_WRITE] = 0 cmd[DO_READ] = 1 cmd = [str(x) for x in cmd] print "For read:", cmd # Run it proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) proc.wait() for line in proc.stdout: print line, result_file.write(line) result_file.flush() result_file.close()
def main(): # justshow = True justshow = False ndir = [1] # 1 nfile_per_dir = [1] # 2 nops_per_file = [0] # 3, decided by totalbytes size_per_op = [1] # 4 do_fsync = [0] # 5 do_write = [0] # 6 do_read = [0] # 7 topdir = ["/l0"] # 8 do_append = [1] # 9 totalbytes = [128 * 1024 * 4 * 1024] # 10 switchcpu = [0] NDIR = 1 NFILE_PER_DIR = 2 NOPS_PER_FILE = 3 SIZE_PER_OP = 4 DO_FSYNC = 5 DO_WRITE = 6 DO_READ = 7 TOPDIR = 8 DO_APPEND = 9 TOTALBYTES = 10 SWITCHCPU = 11 parameters = [ ndir, nfile_per_dir, nops_per_file, size_per_op, do_fsync, do_write, do_read, topdir, do_append, totalbytes, switchcpu, ] paralist = list(itertools.product(*parameters)) partition = "/dev/sda4" jobid = strftime("%Y-%m-%d-%H-%M-%S", localtime()) resultname = jobid + ".result" result_file = open(jobid + ".result", "w") for rep in range(5): for para in paralist: para = list(para) cmd = ["./fsbench"] + para ######################### # get a clean file system if not justshow: FormatFS.remakeExt4(partition, cmd[8], "jhe", "plfs", blockscount=2 * 1024 * 1024, blocksize=4096) subprocess.Popen("echo 262144 > /sys/fs/ext4/sda4/mb_stream_req", shell=True) sleep(3) ######################### # Create files for later reads filesize = cmd[TOTALBYTES] / (cmd[NDIR] * cmd[NFILE_PER_DIR]) opsize = min(4096, filesize) # in case the file is too small, like 1 byte _nops_per_file = filesize / opsize if _nops_per_file < 1: # we don't accept number that is not 2^x continue cmd[SIZE_PER_OP] = opsize cmd[NOPS_PER_FILE] = _nops_per_file cmd[DO_WRITE] = 1 cmd[DO_READ] = 0 if cmd[SWITCHCPU] == 0: # do not switch cpu cmd[DO_APPEND] = 1 cmd[DO_FSYNC] = 0 print "repeat:", _nops_per_file, "For writing NOSWITCH (append):", trimcmd(cmd) for i in range(_nops_per_file): cmd[NOPS_PER_FILE] = 1 cmd = [str(x) for x in cmd] if not justshow: proc = subprocess.Popen(trimcmd(cmd), stdout=subprocess.PIPE) proc.wait() else: # do the appending thing # size per operation does not change, # we just do one operation per file open cmd[DO_APPEND] = 1 cmd[DO_FSYNC] = 1 print "repeat:", _nops_per_file, "For writing (append):", trimcmd(cmd) for i in range(_nops_per_file): cmd[NOPS_PER_FILE] = 1 cmd = [str(x) for x in cmd] taskset_wrapper = ["taskset", "0x0000000" + str((i % 2) + 1)] # print taskset_wrapper + trimcmd(cmd) if not justshow: proc = subprocess.Popen(taskset_wrapper + trimcmd(cmd), stdout=subprocess.PIPE) proc.wait() ######################## # Read the files in different ways, re-mount before # each reading opsizes = sorted([4096], reverse=True) # do read for each operation size if it is valid for _size_per_op in opsizes: # re-mount the file system to drop caches if not justshow: FormatFS.umountFS(cmd[8]) FormatFS.mountExt4(partition, cmd[8]) _nops_per_file = filesize / _size_per_op if _nops_per_file < 1: # we don't accept number that is not 2^x continue cmd[NOPS_PER_FILE] = _nops_per_file cmd[SIZE_PER_OP] = _size_per_op cmd[DO_WRITE] = 0 cmd[DO_READ] = 1 cmd = [str(x) for x in cmd] print "For read:", trimcmd(cmd) # Run it if not justshow: proc = subprocess.Popen(trimcmd(cmd), stdout=subprocess.PIPE) proc.wait() for line in proc.stdout: line = decrate(line, "switchcpu", cmd[SWITCHCPU]) print line, result_file.write(line) result_file.close()
def main(): # ndir exps = [2] ndir = [2**x for x in exps] # 1 # nfile_per_dir exps = [2] nfile_per_dir = [2**x for x in exps] #2 # nops_per_file nops_per_file = [0] #3, decided by totalbytes #size_per_op = [1, 1024, 4096, 1024*1024, 4*1024*1024], reverse=True) #4 # others size_per_op = [0] #4 do_fsync = [0] #5 do_write = [0] #6 do_read = [0] #7 topdir = ["/l0"] #8 NDIR = 1 NFILE_PER_DIR = 2 NOPS_PER_FILE = 3 SIZE_PER_OP = 4 DO_FSYNC = 5 DO_WRITE = 6 DO_READ = 7 TOPDIR = 8 parameters = [ndir, nfile_per_dir, nops_per_file, size_per_op, do_fsync, do_write, do_read, topdir] paralist = list(itertools.product(*parameters)) partition = "/dev/sda4" jobid = strftime("%Y-%m-%d-%H-%M-%S", localtime()) resultname = jobid + ".result" result_file = open(jobid+".result", 'w') for rep in range(5): for para in paralist: para = list(para) cmd = ['./fsbench'] + para filesizes = range(64-4, 64+4) filesizes = [x*1024 for x in filesizes] for filesize in filesizes: ######################### # get a clean file system FormatFS.remakeExt4(partition, cmd[8], "jhe", "plfs", blockscount=2*1024*1024, blocksize=4096) ######################### # Create files for later reads cmd[SIZE_PER_OP] = filesize cmd[NOPS_PER_FILE] = 1 cmd[DO_WRITE] = 1 cmd[DO_READ] = 0 cmd = [str(x) for x in cmd] print "For writing:", cmd proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) proc.wait() ######################## # Read the files in different ways, re-mount before # each reading #re-mount the file system to drop caches FormatFS.umountFS(cmd[8]) FormatFS.mountExt4(partition, cmd[8]) cmd[DO_WRITE] = 0 cmd[DO_READ] = 1 # make the total amount of bytes read the same cmd[SIZE_PER_OP] = min(filesizes) cmd = [str(x) for x in cmd] print "For read:", cmd # Run it proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) proc.wait() for line in proc.stdout: if "HEADERMARKER" in line: elems = line.split() elems.append("original_file_size") elif "DATAMARKER" in line: elems = line.split() elems.append(filesize) else: continue line = [str(x) for x in elems] print line, result_file.write(line) result_file.close()