def usage(msg): # call if CLI syntax error or invalid parameter opnames = ' --operation ' for op in smf_invocation.all_op_names: opnames += op + '|' opnames = opnames[:-1] dflts = smf_invocation() print() print('ERROR: ' + msg) print('usage: smallfile_cli.py ') print(opnames) print(' --top top-dir | top-dir1,top-dir2,...,top-dirN (default: %s)'%smf_invocation.tmp_dir) print(' --host-set h1,h2,...,hN') print(' --network-sync-dir directory-path (default: %s/network_shared)'%smf_invocation.tmp_dir) print(' --files positive-integer (default: %d)'%dflts.iterations) print(' --files-per-dir positive-integer (default: %d)'%dflts.files_per_dir) print(' --dirs-per-dir positive-integer (default: %d)'%dflts.dirs_per_dir) print(' --threads positive-integer (default: %d)'%2) print(' --record-size non-negative-integer-KB (default: %d)'%dflts.record_sz_kb) print(' --xattr-size non-negative-integer-bytes (default: %d)'%dflts.xattr_size) print(' --xattr-count non-negative-integer-bytes (default: %d)'%dflts.xattr_count) print(' --file-size-distribution exponential (default: fixed-size)') print(' --permute-host-dirs Y|N (default: N)') print(' --hash-into-dirs Y|N (default: %s)'%bool2YN(dflts.hash_to_dir)) print(' --file-size non-negative-integer-KB (default: %d)'%dflts.total_sz_kb) print(' --prefix alphanumeric-string') print(' --suffix alphanumeric-string') print(' --fsync Y|N (default: %s)'%bool2YN(dflts.fsync)) print(' --finish Y|N (default: %s)'%bool2YN(dflts.finish_all_rq)) print(' --verify-read Y|N (default: %s)'%bool2YN(dflts.verify_read)) print(' --response-times Y|N (default: %s)'%bool2YN(dflts.measure_rsptimes)) print(' --same-dir Y|N (default: %s)'%bool2YN(dflts.is_shared_dir)) print(' --pause microsec (default: %d)'%dflts.pause_between_files) print(' --remote-pgm-dir directory-pathname (default: %s)'%os.getcwd()) sys.exit(1)
def setUp(self): self.invok = smallfile.smf_invocation() self.invok.debug = True self.invok.verbose = True self.invok.tid = "regtest" self.invok.start_log() deltree(self.invok.src_dirs[0]) os.makedirs(self.invok.src_dirs[0], 0o644)
def setUp(self): self.invok = smallfile.smf_invocation() self.invok.debug = True self.invok.verbose = True self.invok.tid = "regtest" self.invok.tmp_dir = self.topdir self.invok.src_dir = self.topdir + os.sep + 'src' self.invok.dest_dir = self.topdir + os.sep + 'dst' self.invok.start_log()
def usage(msg): # call if CLI syntax error or invalid parameter opnames = ' --operation ' for op in smf_invocation.all_op_names: opnames += op + '|' opnames = opnames[:-1] dflts = smf_invocation() print() print('ERROR: ' + msg) print('usage: smallfile_cli.py ') print(opnames) print(' --top top-dir | top-dir1,top-dir2,...,top-dirN (default: %s)' % smf_invocation.tmp_dir) print(' --host-set h1,h2,...,hN') print( ' --network-sync-dir directory-path (default: %s/network_shared)' % smf_invocation.tmp_dir) print(' --files positive-integer (default: %d)' % dflts.iterations) print(' --files-per-dir positive-integer (default: %d)' % dflts.files_per_dir) print(' --dirs-per-dir positive-integer (default: %d)' % dflts.dirs_per_dir) print(' --threads positive-integer (default: %d)' % 2) print(' --record-size non-negative-integer-KB (default: %d)' % dflts.record_sz_kb) print(' --xattr-size non-negative-integer-bytes (default: %d)' % dflts.xattr_size) print(' --xattr-count non-negative-integer-bytes (default: %d)' % dflts.xattr_count) print( ' --file-size-distribution exponential (default: fixed-size)' ) print(' --permute-host-dirs Y|N (default: N)') print(' --hash-into-dirs Y|N (default: %s)' % bool2YN(dflts.hash_to_dir)) print(' --file-size non-negative-integer-KB (default: %d)' % dflts.total_sz_kb) print(' --prefix alphanumeric-string') print(' --suffix alphanumeric-string') print(' --fsync Y|N (default: %s)' % bool2YN(dflts.fsync)) print(' --finish Y|N (default: %s)' % bool2YN(dflts.finish_all_rq)) print(' --verify-read Y|N (default: %s)' % bool2YN(dflts.verify_read)) print(' --response-times Y|N (default: %s)' % bool2YN(dflts.measure_rsptimes)) print(' --same-dir Y|N (default: %s)' % bool2YN(dflts.is_shared_dir)) print(' --pause microsec (default: %d)' % dflts.pause_between_files) print(' --remote-pgm-dir directory-pathname (default: %s)' % os.getcwd()) sys.exit(1)
def test_multiproc_stonewall(self): self.invok.log.info('starting stonewall test') thread_ready_timeout = 4 thread_count = 4 for tree in self.invok.top_dirs: deltree(tree) os.mkdir(tree) for dir in self.invok.src_dirs: os.mkdir(dir) for dir in self.invok.dest_dirs: os.mkdir(dir) os.mkdir(self.invok.network_dir) self.invok.starting_gate = os.path.join(self.invok.network_dir, 'starting-gate') sgate_file = self.invok.starting_gate invokeList = [] for j in range(0, thread_count): s = smallfile.smf_invocation() #s.log_to_stderr = True s.verbose = True s.tid = str(j) s.prefix = "thr_" s.suffix = "foo" s.iterations = 10 s.stonewall = False s.starting_gate = sgate_file invokeList.append(s) threadList = [] for s in invokeList: threadList.append(subprocess(s)) for t in threadList: t.start() threads_ready = True for i in range(0, thread_ready_timeout): threads_ready = True for s in invokeList: thread_ready_file = s.gen_thread_ready_fname(s.tid) if not os.path.exists(thread_ready_file): threads_ready = False if threads_ready: break time.sleep(1) if not threads_ready: raise Exception("threads did not show up within %d seconds" % thread_ready_timeout) time.sleep(1) smallfile.touch(sgate_file) for t in threadList: rtnd_invok = t.receiver.recv() t.join() self.invok.log.info(str(rtnd_invok)) if rtnd_invok.status != ok: raise Exception("subprocess failure for %s invocation %s: " % (str(t), str(rtnd_invok)))
def test_multiproc_stonewall(self): self.invok.start_log() self.invok.log.info('starting stonewall test') thread_ready_timeout = 4 thread_count = 4 self.deltree() os.mkdir(self.topdir) os.mkdir(self.invok.src_dir) os.mkdir(self.invok.dest_dir) sgate_file = self.invok.tmp_dir + os.sep + "start" smallfile.ensure_deleted(sgate_file) invokeList = [] for j in range(0, thread_count): s = smallfile.smf_invocation() #s.log_to_stderr = True s.verbose = True s.tid = str(j) s.src_dir = self.invok.src_dir s.dst_dir = self.invok.dest_dir s.prefix = "thr_" s.suffix = "foo" s.iterations=10 s.stonewall = True s.starting_gate = sgate_file s.do_sync_at_end = True; invokeList.append(s) threadList=[] for s in invokeList: threadList.append(subprocess(s)) for t in threadList: t.start() threads_ready = True for i in range(0, thread_ready_timeout): threads_ready = True for s in invokeList: thread_ready_file = s.gen_thread_ready_fname(s.tid) if not os.access(thread_ready_file, os.R_OK): threads_ready = False if threads_ready: break time.sleep(1) if not threads_ready: raise Exception("threads did not show up within %d seconds"%thread_ready_timeout) time.sleep(1) f = open(sgate_file, "w") f.close() for t in threadList: rtnd_invok = t.receiver.recv() t.join() self.invok.log.info(str(rtnd_invok)) if rtnd_invok.status != ok: raise Exception("subprocess failure: " + str(t))
def test_multiproc_stonewall(self): self.invok.log.info('starting stonewall test') thread_ready_timeout = 4 thread_count = 4 for tree in self.invok.top_dirs: deltree(tree) os.mkdir(tree) for dir in self.invok.src_dirs: os.mkdir(dir) for dir in self.invok.dest_dirs: os.mkdir(dir) os.mkdir(self.invok.network_dir) self.invok.starting_gate = os.path.join(self.invok.network_dir, 'starting-gate') sgate_file = self.invok.starting_gate invokeList = [] for j in range(0, thread_count): s = smallfile.smf_invocation() #s.log_to_stderr = True s.verbose = True s.tid = str(j) s.prefix = "thr_" s.suffix = "foo" s.iterations=10 s.stonewall = False s.starting_gate = sgate_file invokeList.append(s) threadList=[] for s in invokeList: threadList.append(subprocess(s)) for t in threadList: t.start() threads_ready = True for i in range(0, thread_ready_timeout): threads_ready = True for s in invokeList: thread_ready_file = s.gen_thread_ready_fname(s.tid) if not os.path.exists(thread_ready_file): threads_ready = False if threads_ready: break time.sleep(1) if not threads_ready: raise Exception("threads did not show up within %d seconds"%thread_ready_timeout) time.sleep(1) smallfile.touch(sgate_file) for t in threadList: rtnd_invok = t.receiver.recv() t.join() self.invok.log.info(str(rtnd_invok)) if rtnd_invok.status != ok: raise Exception("subprocess failure for %s invocation %s: " %(str(t), str(rtnd_invok)))
def parse(): # define parameter variables # default does short test in /var/tmp so you can see the program run # store as much as you can in smf_invocation object so per-thread invocations inherit inv = smf_invocation() # parameters that can't be stored in a smf_invocation # describe how the smf_invocation threads work together prm_thread_count = 2 prm_host_set = None prm_slave = False prm_permute_host_dirs = False prm_remote_pgm_dir = os.path.abspath(os.path.dirname(sys.argv[0])) prm_top_dirs = None prm_network_sync_dir = None # parse command line argc = len(sys.argv) pass_on_prm_list = '' # parameters passed to remote hosts if needed if argc == 1: print('\nfor additional help add the parameter "--help" to the command\n') j=1 while j < argc: rawprm = sys.argv[j] if rawprm == '-h' or rawprm == '--help': usage('ok, so you need help, we all knew that ;-)') if rawprm[0:2] != '--': usage('parameter names begin with "--"') prm = rawprm[2:] if (j == argc - 1) and (argc%2 != 1): usage('all parameters consist of a name and a value') val = sys.argv[j+1] if len(rawprm) < 3: usage('parameter name not long enough' ) pass_on_prm = rawprm + ' ' + val j += 2 if prm == 'files': chkPositiveInt(val, rawprm) inv.iterations = int(val) elif prm == 'threads': chkPositiveInt(val, rawprm) prm_thread_count = int(val) elif prm == 'files-per-dir': chkPositiveInt(val, rawprm) inv.files_per_dir = int(val) elif prm == 'dirs-per-dir': chkPositiveInt(val, rawprm) inv.dirs_per_dir = int(val) elif prm == 'record-size': chkNonNegInt(val, rawprm) inv.record_sz_kb = int(val) elif prm == 'file-size': chkNonNegInt(val, rawprm) inv.total_sz_kb = int(val) elif prm == 'file-size-distribution': if val != 'exponential': usage('unrecognized file size distribution: %s'%val) inv.filesize_distr = smf_invocation.filesize_distr_random_exponential elif prm == 'xattr-size': chkNonNegInt(val, rawprm) inv.xattr_size = int(val) elif prm == 'xattr-count': chkNonNegInt(val, rawprm) inv.xattr_count = int(val) elif prm == 'prefix': inv.prefix = val elif prm == 'suffix': inv.suffix = val elif prm == 'hash-into-dirs': inv.hash_to_dir = str2bool(val, rawprm) elif prm == 'operation': if not smf_invocation.all_op_names.__contains__(val): usage('unrecognized operation name: %s'%val) inv.opname = val elif prm == 'top': prm_top_dirs = val.split(',') elif prm == 'pause': chkPositiveInt(val, rawprm) inv.pause_between_files = int(val) elif prm == 'stonewall': inv.stonewall = str2bool(val, rawprm) elif prm == 'finish': inv.finish_all_rq = str2bool(val, rawprm) elif prm == 'fsync': inv.fsync = str2bool(val, rawprm) elif prm == 'record-ctime-size': inv.record_ctime_size = str2bool(val, rawprm) elif prm == 'permute-host-dirs': prm_permute_host_dirs = str2bool(val, rawprm) pass_on_prm = '' elif prm == 'response-times': inv.measure_rsptimes = str2bool(val, rawprm) elif prm == 'incompressible': inv.incompressible = str2bool(val, rawprm) elif prm == 'verify-read': inv.verify_read = str2bool(val, rawprm) elif prm == 'same-dir': inv.is_shared_dir = str2bool(val, rawprm) elif prm == 'verbose': inv.verbose = str2bool(val, rawprm) elif prm == 'log-to-stderr': inv.log_to_stderr = str2bool(val, rawprm) elif prm == 'host-set': prm_host_set = val.split(",") if len(prm_host_set) < 2: prm_host_set = val.strip().split() if len(prm_host_set) == 0: usage('host list must be non-empty when --host-set option used') pass_on_prm = '' elif prm == 'remote-pgm-dir': prm_remote_pgm_dir = val elif prm == 'network-sync-dir': prm_network_sync_dir = val # --slave should not be used by end-user elif prm == 'slave': prm_slave = str2bool(val, rawprm) # --ashost should not be used by end-user elif prm == 'as-host': inv.onhost = smallfile.get_hostname(val) else: usage('unrecognized parameter name') pass_on_prm_list += ' ' + pass_on_prm # parameter options that workload generators will need # validate parameters further now that we know what they all are if inv.record_sz_kb > inv.total_sz_kb and inv.total_sz_kb != 0: usage('record size cannot exceed file size') if (inv.record_sz_kb != 0) and ((inv.total_sz_kb % inv.record_sz_kb) != 0): usage('file size must be multiple of record size if record size is non-zero') if prm_top_dirs: for d in prm_top_dirs: if len(d) < 6: usage('directory less than 6 characters, cannot use top of filesystem, too dangerous') if prm_top_dirs: inv.set_top(prm_top_dirs) else: prm_top_dirs = inv.top_dirs if prm_network_sync_dir: inv.network_dir = prm_network_sync_dir else: prm_network_sync_dir = inv.network_dir inv.starting_gate = os.path.join(inv.network_dir, 'starting_gate.tmp') # location of file that signals start, end of test if inv.iterations < 10: inv.stonewall = False # display results of parse so user knows what default values are # most important parameters come first # display host set first because this can be very long, # this way the rest of the parameters appear together on the screen size_distribution_string = 'fixed' if inv.filesize_distr == smf_invocation.filesize_distr_random_exponential: size_distribution_string = 'random exponential' prm_list = [ \ ('hosts in test', '%s'%prm_host_set), \ ('top test directory(s)', str(prm_top_dirs)), \ ('operation', inv.opname), \ ('files/thread', '%d'%inv.iterations), \ ('threads', '%d'%prm_thread_count), \ ('record size (KB)', '%d'%inv.record_sz_kb), \ ('file size (KB)', '%d'%inv.total_sz_kb), \ ('file size distribution', size_distribution_string), \ ('files per dir', '%d'%inv.files_per_dir), \ ('dirs per dir', '%d'%inv.dirs_per_dir), \ ('threads share directories?', '%s'%bool2YN(inv.is_shared_dir)), \ ('filename prefix', inv.prefix), \ ('filename suffix', inv.suffix), \ ('hash file number into dir.?', bool2YN(inv.hash_to_dir)), \ ('fsync after modify?', bool2YN(inv.fsync)), \ ('pause between files (microsec)', '%d'%inv.pause_between_files), \ ('finish all requests?', '%s'%bool2YN(inv.finish_all_rq)), \ ('stonewall?', '%s'%bool2YN(inv.stonewall)), \ ('measure response times?', '%s'%bool2YN(inv.measure_rsptimes)), \ ('verify read?', '%s'%bool2YN(inv.verify_read)), \ ('verbose?', inv.verbose), \ ('log to stderr?', inv.log_to_stderr) ] if (not smallfile.xattr_not_installed): prm_list.extend( [ ('ext.attr.size', '%d'%inv.xattr_size), ('ext.attr.count', '%d'%inv.xattr_count) ] ) if prm_host_set: prm_list.extend( [ \ ('permute host directories?', '%s'%bool2YN(prm_permute_host_dirs)) ] ) if prm_remote_pgm_dir: prm_list.append( ('remote program directory', prm_remote_pgm_dir) ) if prm_network_sync_dir: prm_list.append( ('network thread sync. dir.', prm_network_sync_dir) ) if inv.record_sz_kb == 0 and inv.verbose: print('record size not specified, large files will default to record size %d KB'%(smf_invocation.biggest_buf_size/inv.BYTES_PER_KB)) if not prm_slave: print('smallfile version %s'%version) for (prm_name, prm_value) in prm_list: print('%40s : %s'%(prm_name, prm_value)) # construct command to run remote slave process using CLI parameters, we have them all here if not prm_remote_pgm_dir: prm_remote_pgm_dir = os.getcwd() remote_cmd = prm_remote_pgm_dir + os.sep + 'smallfile_cli.py ' + pass_on_prm_list # "inv" contains all per-thread parameters params = smf_test_params.smf_test_params(prm_host_set, prm_thread_count, inv, prm_remote_pgm_dir, prm_top_dirs, \ prm_network_sync_dir, prm_slave, prm_permute_host_dirs) #print params return params
def parse(): # define parameter variables and default parameter values # default does short test in /var/tmp so you can see the program run # store as much as you can in smf_invocation object so per-thread invocations inherit inv = smallfile.smf_invocation() inv.iterations = 3 inv.record_sz_kb = 4 inv.total_sz_kb = 64 inv.files_per_dir = 200 inv.dirs_per_dir = 20 inv.prefix = '' inv.log_to_stderr = False inv.verbose = False inv.opname = 'create' inv.pause_between_files = 0 inv.verify_read = True inv.stonewall = True inv.finish_all_rq = True inv.measure_rsptimes = False inv.is_shared_dir = False inv.do_sync_at_end = False # parameters that can't be stored in a smf_invocation # describe how the smf_invocation threads work together prm_thread_count = 2 prm_host_set = None prm_slave = False prm_permute_host_dirs = False prm_remote_pgm_dir = os.getcwd() prm_top_dir = os.getenv("TMPDIR") if prm_top_dir == None: prm_top_dir = os.getenv("TEMP") if prm_top_dir == None: prm_top_dir = "/var/tmp" # parse command line argc = len(sys.argv) if argc%2 != 1: usage('all parameters consist of a name and a value') pass_on_prm_list = '' # parameters passed to remote hosts if needed j=1 while j < argc: rawprm = sys.argv[j] val = sys.argv[j+1] if len(rawprm) < 3: usage('parameter name not long enough' ) if rawprm[0:2] != '--': usage('parameter names begin with "--"') prm = rawprm[2:] pass_on_prm = rawprm + ' ' + val j += 2 if prm == 'files': chkPositiveInt(val, rawprm) inv.iterations = int(val) elif prm == 'threads': chkPositiveInt(val, rawprm) prm_thread_count = int(val) elif prm == 'files-per-dir': chkPositiveInt(val, rawprm) inv.files_per_dir = int(val) elif prm == 'dirs-per-dir': chkPositiveInt(val, rawprm) inv.dirs_per_dir = int(val) elif prm == 'record-size': chkNonNegInt(val, rawprm) inv.record_sz_kb = int(val) elif prm == 'file-size': chkNonNegInt(val, rawprm) inv.total_sz_kb = int(val) elif prm == 'xattr-size': chkNonNegInt(val, rawprm) inv.xattr_size = int(val) elif prm == 'xattr-count': chkNonNegInt(val, rawprm) inv.xattr_count = int(val) elif prm == 'prefix': inv.prefix = val elif prm == 'operation': if not smallfile.smf_invocation.all_op_names.__contains__(val): usage('unrecognized operation name: %s'%val) inv.opname = val elif prm == 'top': prm_top_dir = val elif prm == 'pause': chkPositiveInt(val, rawprm) inv.pause_between_files = int(val) elif prm == 'stonewall': inv.stonewall = str2bool(val, rawprm) elif prm == 'finish': inv.finish_all_rq = str2bool(val, rawprm) elif prm == 'permute-host-dirs': prm_permute_host_dirs = str2bool(val, rawprm) pass_on_prm = '' elif prm == 'response-times': inv.measure_rsptimes = str2bool(val, rawprm) elif prm == 'verify-read': inv.verify_read = str2bool(val, rawprm) elif prm == 'same-dir': inv.is_shared_dir = str2bool(val, rawprm) elif prm == 'verbose': inv.verbose = str2bool(val, rawprm) elif prm == 'log-to-stderr': inv.log_to_stderr = str2bool(val, rawprm) elif prm == 'host-set': prm_host_set = val.split(",") if len(prm_host_set) < 2: prm_host_set = val.strip().split() pass_on_prm = '' elif prm == 'remote-pgm-dir': prm_remote_pgm_dir = val # --slave should not be used by end-user elif prm == 'slave': prm_slave = str2bool(val, rawprm) # --ashost should not be used by end-user elif prm == 'as-host': inv.onhost = smallfile.short_hostname(val) else: usage('unrecognized parameter name') pass_on_prm_list += ' ' + pass_on_prm # parameter options that workload generators will need if inv.record_sz_kb > inv.total_sz_kb and inv.total_sz_kb != 0: usage('record size cannot exceed file size') if (inv.record_sz_kb != 0) and ((inv.total_sz_kb % inv.record_sz_kb) != 0): usage('file size must be multiple of record size if record size is non-zero') if (prm_top_dir == '/'): usage('cannot use filesystem root, too dangerous') if inv.iterations < 10: inv.stonewall = False # display results of parse so user knows what default values are prm_list = [ ('files/thread', '%d'%inv.iterations), \ ('threads', '%d'%prm_thread_count), \ ('record size (KB)', '%d'%inv.record_sz_kb), \ ('file size (KB)', '%d'%inv.total_sz_kb), \ ('files per dir', '%d'%inv.files_per_dir), \ ('dirs per dir', '%d'%inv.dirs_per_dir), \ ('ext.attr.size', '%d'%inv.xattr_size), \ ('ext.attr.count', '%d'%inv.xattr_count), \ ('finish all requests?', '%s'%bool2YN(inv.finish_all_rq)), \ ('stonewall?', '%s'%bool2YN(inv.stonewall)), \ ('measure response times?', '%s'%bool2YN(inv.measure_rsptimes)), \ ('verify read?', '%s'%bool2YN(inv.verify_read)), \ ('permute host directories?', '%s'%bool2YN(prm_permute_host_dirs)), \ ('pause between files (microsec)', '%d'%inv.pause_between_files), \ ('files in same directory?', '%s'%bool2YN(inv.is_shared_dir)), \ ('hosts in test', '%s'%prm_host_set), \ ('filename prefix', inv.prefix), \ ('operation', inv.opname), \ ('top test directory', prm_top_dir), \ ('remote program directory', prm_remote_pgm_dir), \ ('verbose?', inv.verbose), \ ('log to stderr?', inv.log_to_stderr) ] if not prm_slave: print 'smallfile version %s'%version for (prm_name, prm_value) in prm_list: print '%40s : %s'%(prm_name, prm_value) inv.starting_gate = prm_top_dir + os.sep + 'starting_gate' # location of file that signals start, end of test inv.src_dir = prm_top_dir + os.sep + 'src' inv.dest_dir = prm_top_dir + os.sep + 'dst' # construct command to run remote slave process using CLI parameters, we have them all here remote_cmd = prm_remote_pgm_dir + os.sep + 'smallfile_cli.py ' + pass_on_prm_list # "inv" contains all per-thread parameters return (prm_host_set, prm_thread_count, inv, remote_cmd, prm_slave, prm_permute_host_dirs)
def parse(): # define parameter variables # default does short test in /var/tmp so you can see the program run # store as much as you can in smf_invocation object so per-thread invocations inherit inv = smf_invocation() # parameters that can't be stored in a smf_invocation # describe how the smf_invocation threads work together prm_thread_count = 2 prm_host_set = None prm_slave = False prm_permute_host_dirs = False prm_remote_pgm_dir = None prm_top_dirs = None prm_network_sync_dir = None # parse command line argc = len(sys.argv) pass_on_prm_list = '' # parameters passed to remote hosts if needed if argc == 1: print( '\nfor additional help add the parameter "--help" to the command\n' ) j = 1 while j < argc: rawprm = sys.argv[j] if rawprm == '-h' or rawprm == '--help': usage('ok, so you need help, we all knew that ;-)') if rawprm[0:2] != '--': usage('parameter names begin with "--"') prm = rawprm[2:] if (j == argc - 1) and (argc % 2 != 1): usage('all parameters consist of a name and a value') val = sys.argv[j + 1] if len(rawprm) < 3: usage('parameter name not long enough') pass_on_prm = rawprm + ' ' + val j += 2 if prm == 'files': chkPositiveInt(val, rawprm) inv.iterations = int(val) elif prm == 'threads': chkPositiveInt(val, rawprm) prm_thread_count = int(val) elif prm == 'files-per-dir': chkPositiveInt(val, rawprm) inv.files_per_dir = int(val) elif prm == 'dirs-per-dir': chkPositiveInt(val, rawprm) inv.dirs_per_dir = int(val) elif prm == 'record-size': chkNonNegInt(val, rawprm) inv.record_sz_kb = int(val) elif prm == 'file-size': chkNonNegInt(val, rawprm) inv.total_sz_kb = int(val) elif prm == 'file-size-distribution': if val != 'exponential': usage('unrecognized file size distribution: %s' % val) inv.filesize_distr = smf_invocation.filesize_distr_random_exponential elif prm == 'xattr-size': chkNonNegInt(val, rawprm) inv.xattr_size = int(val) elif prm == 'xattr-count': chkNonNegInt(val, rawprm) inv.xattr_count = int(val) elif prm == 'prefix': inv.prefix = val elif prm == 'suffix': inv.suffix = val elif prm == 'hash-into-dirs': inv.hash_to_dir = str2bool(val, rawprm) elif prm == 'operation': if not smf_invocation.all_op_names.__contains__(val): usage('unrecognized operation name: %s' % val) inv.opname = val elif prm == 'top': prm_top_dirs = val.split(',') elif prm == 'pause': chkPositiveInt(val, rawprm) inv.pause_between_files = int(val) elif prm == 'stonewall': inv.stonewall = str2bool(val, rawprm) elif prm == 'finish': inv.finish_all_rq = str2bool(val, rawprm) elif prm == 'fsync': inv.fsync = str2bool(val, rawprm) elif prm == 'permute-host-dirs': prm_permute_host_dirs = str2bool(val, rawprm) pass_on_prm = '' elif prm == 'response-times': inv.measure_rsptimes = str2bool(val, rawprm) elif prm == 'verify-read': inv.verify_read = str2bool(val, rawprm) elif prm == 'same-dir': inv.is_shared_dir = str2bool(val, rawprm) elif prm == 'verbose': inv.verbose = str2bool(val, rawprm) elif prm == 'log-to-stderr': inv.log_to_stderr = str2bool(val, rawprm) elif prm == 'host-set': prm_host_set = val.split(",") if len(prm_host_set) < 2: prm_host_set = val.strip().split() if len(prm_host_set) == 0: usage( 'host list must be non-empty when --host-set option used') pass_on_prm = '' elif prm == 'remote-pgm-dir': prm_remote_pgm_dir = val elif prm == 'network-sync-dir': prm_network_sync_dir = val # --slave should not be used by end-user elif prm == 'slave': prm_slave = str2bool(val, rawprm) # --ashost should not be used by end-user elif prm == 'as-host': inv.onhost = smallfile.get_hostname(val) else: usage('unrecognized parameter name') pass_on_prm_list += ' ' + pass_on_prm # parameter options that workload generators will need # validate parameters further now that we know what they all are if inv.record_sz_kb > inv.total_sz_kb and inv.total_sz_kb != 0: usage('record size cannot exceed file size') if (inv.record_sz_kb != 0) and ((inv.total_sz_kb % inv.record_sz_kb) != 0): usage( 'file size must be multiple of record size if record size is non-zero' ) if prm_top_dirs: for d in prm_top_dirs: if len(d) < 6: usage( 'directory less than 6 characters, cannot use top of filesystem, too dangerous' ) if prm_top_dirs: inv.set_top(prm_top_dirs) else: prm_top_dirs = inv.top_dirs if prm_network_sync_dir: if not prm_host_set and not prm_slave: usage( 'you do not need to specify a network thread synchronization directory unless you use multiple hosts' ) inv.network_dir = prm_network_sync_dir else: prm_network_sync_dir = inv.network_dir if prm_remote_pgm_dir: if not prm_host_set and not prm_slave: usage( 'you do not need to specify a remote program directory unless you use multiple hosts' ) inv.starting_gate = os.path.join( inv.network_dir, 'starting_gate.tmp' ) # location of file that signals start, end of test if inv.iterations < 10: inv.stonewall = False # display results of parse so user knows what default values are # most important parameters come first # display host set first because this can be very long, # this way the rest of the parameters appear together on the screen size_distribution_string = 'fixed' if inv.filesize_distr == smf_invocation.filesize_distr_random_exponential: size_distribution_string = 'random exponential' prm_list = [ \ ('hosts in test', '%s'%prm_host_set), \ ('top test directory(s)', str(prm_top_dirs)), \ ('operation', inv.opname), \ ('files/thread', '%d'%inv.iterations), \ ('threads', '%d'%prm_thread_count), \ ('record size (KB)', '%d'%inv.record_sz_kb), \ ('file size (KB)', '%d'%inv.total_sz_kb), \ ('file size distribution', size_distribution_string), \ ('files per dir', '%d'%inv.files_per_dir), \ ('dirs per dir', '%d'%inv.dirs_per_dir), \ ('threads share directories?', '%s'%bool2YN(inv.is_shared_dir)), \ ('filename prefix', inv.prefix), \ ('filename suffix', inv.suffix), \ ('hash file number into dir.?', bool2YN(inv.hash_to_dir)), \ ('fsync after modify?', bool2YN(inv.fsync)), \ ('pause between files (microsec)', '%d'%inv.pause_between_files), \ ('finish all requests?', '%s'%bool2YN(inv.finish_all_rq)), \ ('stonewall?', '%s'%bool2YN(inv.stonewall)), \ ('measure response times?', '%s'%bool2YN(inv.measure_rsptimes)), \ ('verify read?', '%s'%bool2YN(inv.verify_read)), \ ('verbose?', inv.verbose), \ ('log to stderr?', inv.log_to_stderr) ] if (not smallfile.xattr_not_installed): prm_list.extend([('ext.attr.size', '%d' % inv.xattr_size), ('ext.attr.count', '%d' % inv.xattr_count)]) if prm_host_set: prm_list.extend( [ \ ('permute host directories?', '%s'%bool2YN(prm_permute_host_dirs)) ] ) if prm_remote_pgm_dir: prm_list.append(('remote program directory', prm_remote_pgm_dir)) if prm_network_sync_dir: prm_list.append( ('network thread sync. dir.', prm_network_sync_dir)) if inv.record_sz_kb == 0 and inv.verbose: print( 'record size not specified, large files will default to record size %d KB' % (smf_invocation.biggest_buf_size / inv.BYTES_PER_KB)) if not prm_slave: print('smallfile version %s' % version) for (prm_name, prm_value) in prm_list: print('%40s : %s' % (prm_name, prm_value)) # construct command to run remote slave process using CLI parameters, we have them all here if not prm_remote_pgm_dir: prm_remote_pgm_dir = os.getcwd() remote_cmd = prm_remote_pgm_dir + os.sep + 'smallfile_cli.py ' + pass_on_prm_list # "inv" contains all per-thread parameters params = smf_test_params.smf_test_params(prm_host_set, prm_thread_count, inv, prm_remote_pgm_dir, prm_top_dirs, \ prm_network_sync_dir, prm_slave, prm_permute_host_dirs) #print params return params
# # python program used by profile.sh to generate profile of smallfile workloads # import os import socket import smallfile from smallfile import smf_invocation top=os.getenv('TOP') count=int(os.getenv('COUNT')) invk = smf_invocation() invk.tid = '00' invk.src_dirs = [ top + os.sep + 'file_srcdir' + os.sep + socket.gethostname() + os.sep + 'thrd_' + invk.tid ] invk.dest_dirs = [ top + os.sep + 'file_dstdir' + os.sep + socket.gethostname() + os.sep + 'thrd_' + invk.tid ] invk.network_dir = top + os.sep + 'network_shared' invk.record_sz_kb = 0 invk.total_sz_kb = 1 invk.starting_gate = os.path.join(invk.network_dir,'starting_gate') invk.stonewall = True invk.finish_all_rq = True invk.opname = os.getenv('OPNAME') invk.iterations = count print(invk) invk.do_workload() print(invk)
# # python program used by profile.sh to generate profile of smallfile workloads # import os import socket import smallfile from smallfile import smf_invocation top = os.getenv("TOP") count = int(os.getenv("COUNT")) invk = smf_invocation() invk.tid = "00" invk.src_dirs = [top + os.sep + "file_srcdir" + os.sep + socket.gethostname() + os.sep + "thrd_" + invk.tid] invk.dest_dirs = [top + os.sep + "file_dstdir" + os.sep + socket.gethostname() + os.sep + "thrd_" + invk.tid] invk.network_dir = top + os.sep + "network_shared" invk.record_sz_kb = 0 invk.total_sz_kb = 1 invk.starting_gate = os.path.join(invk.network_dir, "starting_gate") invk.stonewall = True invk.finish_all_rq = True invk.opname = os.getenv("OPNAME") invk.iterations = count print(invk) invk.do_workload() print(invk)