def parse(): prm_network_sync_dir = None argc = len(sys.argv) if argc == 1: print('for additional help add the parameter "--help" to the command') sys.exit(1) j = 1 while j < argc: rawprm = sys.argv[j] if rawprm == '-h' or rawprm == '--help': usage('normally this process is run automatically ' + 'by smallfile_cli.py') if rawprm[0:2] != '--': usage('parameter names begin with "--"') prm = rawprm[2:] if j == argc - 1 and argc % 2 != 1: usage('all parameters consist of a name and a value') val = sys.argv[j + 1] if len(rawprm) < 3: usage('parameter name not long enough') j += 2 if prm == 'network-sync-dir': prm_network_sync_dir = val elif prm == 'as-host': # --ashost should not be used by end-user prm_as_host = smallfile.get_hostname(val) else: usage('unrecognized parameter name') param_pickle_fname = os.path.join(prm_network_sync_dir, 'param.pickle') if not os.path.exists(param_pickle_fname): time.sleep(1.1) params = None try: with open(param_pickle_fname, 'rb') as pickled_params: params = pickle.load(pickled_params) params.is_slave = True params.as_host = prm_as_host params.master_invoke.onhost = prm_as_host except IOError as e: if e.errno != errno.ENOENT: raise e usage('could not read parameter pickle file %s' % param_pickle_fname) return params
def parse(): parser = argparse.ArgumentParser( description='parse remote smallfile parameters') parser.add_argument('--network-sync-dir', help='directory used to synchronize with test driver') parser.add_argument('--as-host', default=smallfile.get_hostname(None), help='directory used to synchronize with test driver') args = parser.parse_args() param_pickle_fname = os.path.join(args.network_sync_dir, 'param.pickle') if not os.path.exists(param_pickle_fname): time.sleep(1.1) params = None with open(param_pickle_fname, 'rb') as pickled_params: params = pickle.load(pickled_params) params.is_slave = True params.as_host = args.as_host params.master_invoke.onhost = args.as_host return params
def parse(): parser = argparse.ArgumentParser( description='parse remote smallfile parameters') parser.add_argument( '--network-sync-dir', help='directory used to synchronize with test driver') parser.add_argument( '--as-host', default=smallfile.get_hostname(None), help='directory used to synchronize with test driver') args = parser.parse_args() param_pickle_fname = os.path.join(args.network_sync_dir, 'param.pickle') if not os.path.exists(param_pickle_fname): time.sleep(1.1) params = None with open(param_pickle_fname, 'rb') as pickled_params: params = pickle.load(pickled_params) params.is_slave = True params.as_host = args.as_host params.master_invoke.onhost = args.as_host return params
def parse(): prm_network_sync_dir = None argc = len(sys.argv) if argc == 1: print('\nfor additional help add the parameter "--help" to the command\n') j=1 while j < argc: rawprm = sys.argv[j] if rawprm == '-h' or rawprm == '--help': usage('normally this process is run automatically by smallfile_cli.py') if rawprm[0:2] != '--': usage('parameter names begin with "--"') prm = rawprm[2:] if (j == argc - 1) and (argc%2 != 1): usage('all parameters consist of a name and a value') val = sys.argv[j+1] if len(rawprm) < 3: usage('parameter name not long enough' ) pass_on_prm = rawprm + ' ' + val j += 2 if prm == 'network-sync-dir': prm_network_sync_dir = val # --ashost should not be used by end-user elif prm == 'as-host': prm_as_host = smallfile.get_hostname(val) else: usage('unrecognized parameter name') param_pickle_fname = os.path.join(prm_network_sync_dir, 'param.pickle') if not os.path.exists(param_pickle_fname): time.sleep(1.1) params = None try: with open(param_pickle_fname, 'rb') as pickled_params: params = pickle.load(pickled_params) params.is_slave = True params.as_host = prm_as_host params.master_invoke.onhost = prm_as_host except IOError as e: if e.errno != errno.ENOENT: raise e usage('could not read parameter pickle file %s'%param_pickle_fname) return params
def parse(): prm_network_sync_dir = None argc = len(sys.argv) if argc == 1: print('\nfor additional help add the parameter "--help" to the command\n') j=1 while j < argc: rawprm = sys.argv[j] if rawprm == '-h' or rawprm == '--help': usage('normally this process is run automatically by smallfile_cli.py') if rawprm[0:2] != '--': usage('parameter names begin with "--"') prm = rawprm[2:] if (j == argc - 1) and (argc%2 != 1): usage('all parameters consist of a name and a value') val = sys.argv[j+1] if len(rawprm) < 3: usage('parameter name not long enough' ) pass_on_prm = rawprm + ' ' + val j += 2 if prm == 'network-sync-dir': prm_network_sync_dir = val # --ashost should not be used by end-user elif prm == 'as-host': prm_as_host = smallfile.get_hostname(val) else: usage('unrecognized parameter name') params = None param_pickle_fname = os.path.join(prm_network_sync_dir, 'param.pickle') if not os.path.exists(param_pickle_fname): time.sleep(1.1) pickle_fn = os.path.join(prm_network_sync_dir, 'param.pickle') #print(pickle_fn) with open(pickle_fn, 'rb') as pickled_params: params = pickle.load(pickled_params) params.is_slave = True params.as_host = prm_as_host params.master_invoke.onhost = prm_as_host return params
def parse(): # parse command line argc = len(sys.argv) if argc == 1: print(''' for additional help add the parameter "--help" to the command ''') j = 1 while j < argc: rawprm = sys.argv[j] if rawprm == '-h' or rawprm == '--help': usage('ok, so you need help, we all knew that ;-)') if rawprm[0:2] != '--': usage('parameter names begin with "--"') prm = rawprm[2:] if j == argc - 1 and argc % 2 != 1: usage('all parameters consist of a name and a value') val = sys.argv[j + 1] if len(rawprm) < 3: usage('parameter name not long enough') pass_on_prm = rawprm + ' ' + val j += 2 if prm == 'files': chkPositiveInt(val, rawprm) inv.iterations = int(val) elif prm == 'threads': chkPositiveInt(val, rawprm) test_params.thread_count = int(val) elif prm == 'files-per-dir': chkPositiveInt(val, rawprm) inv.files_per_dir = int(val) elif prm == 'dirs-per-dir': chkPositiveInt(val, rawprm) inv.dirs_per_dir = int(val) elif prm == 'record-size': chkNonNegInt(val, rawprm) inv.record_sz_kb = int(val) elif prm == 'file-size': chkNonNegInt(val, rawprm) inv.total_sz_kb = int(val) elif prm == 'file-size-distribution': if val != 'exponential': usage('unrecognized file size distribution: %s' % val) inv.filesize_distr = \ SmallfileWorkload.fsdistr_random_exponential test_params.size_distribution = 'random exponential' elif prm == 'xattr-size': chkNonNegInt(val, rawprm) inv.xattr_size = int(val) elif prm == 'xattr-count': chkNonNegInt(val, rawprm) inv.xattr_count = int(val) elif prm == 'prefix': inv.prefix = val elif prm == 'suffix': inv.suffix = val elif prm == 'hash-into-dirs': inv.hash_to_dir = str2bool(val, rawprm) elif prm == 'operation': if not SmallfileWorkload.all_op_names.__contains__(val): usage('unrecognized operation name: %s' % val) inv.opname = val elif prm == 'top': test_params.top_dirs = [os.path.abspath(p) for p in val.split(',')] elif prm == 'pause': chkPositiveInt(val, rawprm) inv.pause_between_files = int(val) elif prm == 'stonewall': inv.stonewall = str2bool(val, rawprm) elif prm == 'finish': inv.finish_all_rq = str2bool(val, rawprm) elif prm == 'fsync': inv.fsync = str2bool(val, rawprm) elif prm == 'record-ctime-size': inv.record_ctime_size = str2bool(val, rawprm) elif prm == 'permute-host-dirs': test_params.permute_host_dirs = str2bool(val, rawprm) pass_on_prm = '' elif prm == 'output-json': test_params.output_json = val elif prm == 'response-times': inv.measure_rsptimes = str2bool(val, rawprm) elif prm == 'incompressible': inv.incompressible = str2bool(val, rawprm) elif prm == 'verify-read': inv.verify_read = str2bool(val, rawprm) elif prm == 'min-dirs-per-sec': chkPositiveInt(val, rawprm) test_params.min_directories_per_sec = int(val) elif prm == 'same-dir': inv.is_shared_dir = str2bool(val, rawprm) elif prm == 'verbose': inv.verbose = str2bool(val, rawprm) elif prm == 'log-to-stderr': inv.log_to_stderr = str2bool(val, rawprm) elif prm == 'host-set': if os.path.isfile(val): with open(val, 'r') as f: test_params.host_set = [ record.strip() for record in f.readlines()] else: test_params.host_set = val.split(',') if len(test_params.host_set) < 2: test_params.host_set = val.strip().split() if len(test_params.host_set) == 0: usage('host list must be non-empty when ' + '--host-set option used') pass_on_prm = '' elif prm == 'remote-pgm-dir': test_params.remote_pgm_dir = val elif prm == 'network-sync-dir': test_params.network_sync_dir = val if not os.path.isdir(test_params.network_sync_dir): usage('you must ensure that shared directory ' + test_params.network_sync_dir + ' exists on this host and every remote host in test') elif prm == 'slave': # --slave should not be used by end-user test_params.is_slave = str2bool(val, rawprm) elif prm == 'as-host': # --ashost should not be used by end-user inv.onhost = smallfile.get_hostname(val) else: usage('unrecognized parameter name: %s' % prm) # validate parameters further now that we know what they all are if inv.record_sz_kb > inv.total_sz_kb and inv.total_sz_kb != 0: usage('record size cannot exceed file size') if inv.record_sz_kb == 0 and inv.verbose: print(('record size not specified, ' + 'large files will default to record size %d KB') % (SmallfileWorkload.biggest_buf_size / inv.BYTES_PER_KB)) if test_params.top_dirs: for d in test_params.top_dirs: if len(d) < 6: usage('directory less than 6 characters, ' + 'cannot use top of filesystem, too dangerous') if not os.path.isdir(d) and not test_params.network_sync_dir: usage('you must ensure that shared directory ' + d + ' is accessible ' + 'from this host and every remote host in test') if test_params.top_dirs: inv.set_top(test_params.top_dirs) else: test_params.top_dirs = inv.top_dirs if test_params.network_sync_dir: inv.network_dir = test_params.network_sync_dir else: test_params.network_sync_dir = inv.network_dir inv.starting_gate = os.path.join(inv.network_dir, 'starting_gate.tmp') if inv.iterations < 10: inv.stonewall = False if not test_params.is_slave: prm_list = test_params.human_readable() for (prm_name, prm_value) in prm_list: print('%40s : %s' % (prm_name, prm_value)) test_params.recalculate_timeouts() return test_params
print(msg) print('usage: python launch_smf_host.py' '--top top-directory ' '[ --substitute-top synonym-directory ]' '[ --as-host as-host-name ] ') sys.exit(NOTOK) # parse command line if len(sys.argv) < 3: usage('required command line arguments missing') substitute_dir = None top_dir = None as_host = smallfile.get_hostname(None) j = 1 while j < len(sys.argv): if len(sys.argv) == j + 1: usage('every parameter name must have a value') nm = sys.argv[j] if len(nm) < 3: usage('parameter name must be > 3 characters long and start with --') nm = nm[2:] val = sys.argv[j + 1] j += 2 if nm == 'substitute-top': substitute_dir = val elif nm == 'top': top_dir = val elif nm == 'as-host':
def parse(): # define parameter variables # default does short test in /var/tmp so you can see the program run # store as much as you can in smf_invocation object so per-thread invocations inherit inv = smf_invocation() # parameters that can't be stored in a smf_invocation # describe how the smf_invocation threads work together prm_thread_count = 2 prm_host_set = None prm_slave = False prm_permute_host_dirs = False prm_remote_pgm_dir = os.path.abspath(os.path.dirname(sys.argv[0])) prm_top_dirs = None prm_network_sync_dir = None # parse command line argc = len(sys.argv) pass_on_prm_list = '' # parameters passed to remote hosts if needed if argc == 1: print('\nfor additional help add the parameter "--help" to the command\n') j=1 while j < argc: rawprm = sys.argv[j] if rawprm == '-h' or rawprm == '--help': usage('ok, so you need help, we all knew that ;-)') if rawprm[0:2] != '--': usage('parameter names begin with "--"') prm = rawprm[2:] if (j == argc - 1) and (argc%2 != 1): usage('all parameters consist of a name and a value') val = sys.argv[j+1] if len(rawprm) < 3: usage('parameter name not long enough' ) pass_on_prm = rawprm + ' ' + val j += 2 if prm == 'files': chkPositiveInt(val, rawprm) inv.iterations = int(val) elif prm == 'threads': chkPositiveInt(val, rawprm) prm_thread_count = int(val) elif prm == 'files-per-dir': chkPositiveInt(val, rawprm) inv.files_per_dir = int(val) elif prm == 'dirs-per-dir': chkPositiveInt(val, rawprm) inv.dirs_per_dir = int(val) elif prm == 'record-size': chkNonNegInt(val, rawprm) inv.record_sz_kb = int(val) elif prm == 'file-size': chkNonNegInt(val, rawprm) inv.total_sz_kb = int(val) elif prm == 'file-size-distribution': if val != 'exponential': usage('unrecognized file size distribution: %s'%val) inv.filesize_distr = smf_invocation.filesize_distr_random_exponential elif prm == 'xattr-size': chkNonNegInt(val, rawprm) inv.xattr_size = int(val) elif prm == 'xattr-count': chkNonNegInt(val, rawprm) inv.xattr_count = int(val) elif prm == 'prefix': inv.prefix = val elif prm == 'suffix': inv.suffix = val elif prm == 'hash-into-dirs': inv.hash_to_dir = str2bool(val, rawprm) elif prm == 'operation': if not smf_invocation.all_op_names.__contains__(val): usage('unrecognized operation name: %s'%val) inv.opname = val elif prm == 'top': prm_top_dirs = val.split(',') elif prm == 'pause': chkPositiveInt(val, rawprm) inv.pause_between_files = int(val) elif prm == 'stonewall': inv.stonewall = str2bool(val, rawprm) elif prm == 'finish': inv.finish_all_rq = str2bool(val, rawprm) elif prm == 'fsync': inv.fsync = str2bool(val, rawprm) elif prm == 'record-ctime-size': inv.record_ctime_size = str2bool(val, rawprm) elif prm == 'permute-host-dirs': prm_permute_host_dirs = str2bool(val, rawprm) pass_on_prm = '' elif prm == 'response-times': inv.measure_rsptimes = str2bool(val, rawprm) elif prm == 'incompressible': inv.incompressible = str2bool(val, rawprm) elif prm == 'verify-read': inv.verify_read = str2bool(val, rawprm) elif prm == 'same-dir': inv.is_shared_dir = str2bool(val, rawprm) elif prm == 'verbose': inv.verbose = str2bool(val, rawprm) elif prm == 'log-to-stderr': inv.log_to_stderr = str2bool(val, rawprm) elif prm == 'host-set': prm_host_set = val.split(",") if len(prm_host_set) < 2: prm_host_set = val.strip().split() if len(prm_host_set) == 0: usage('host list must be non-empty when --host-set option used') pass_on_prm = '' elif prm == 'remote-pgm-dir': prm_remote_pgm_dir = val elif prm == 'network-sync-dir': prm_network_sync_dir = val # --slave should not be used by end-user elif prm == 'slave': prm_slave = str2bool(val, rawprm) # --ashost should not be used by end-user elif prm == 'as-host': inv.onhost = smallfile.get_hostname(val) else: usage('unrecognized parameter name') pass_on_prm_list += ' ' + pass_on_prm # parameter options that workload generators will need # validate parameters further now that we know what they all are if inv.record_sz_kb > inv.total_sz_kb and inv.total_sz_kb != 0: usage('record size cannot exceed file size') if (inv.record_sz_kb != 0) and ((inv.total_sz_kb % inv.record_sz_kb) != 0): usage('file size must be multiple of record size if record size is non-zero') if prm_top_dirs: for d in prm_top_dirs: if len(d) < 6: usage('directory less than 6 characters, cannot use top of filesystem, too dangerous') if prm_top_dirs: inv.set_top(prm_top_dirs) else: prm_top_dirs = inv.top_dirs if prm_network_sync_dir: inv.network_dir = prm_network_sync_dir else: prm_network_sync_dir = inv.network_dir inv.starting_gate = os.path.join(inv.network_dir, 'starting_gate.tmp') # location of file that signals start, end of test if inv.iterations < 10: inv.stonewall = False # display results of parse so user knows what default values are # most important parameters come first # display host set first because this can be very long, # this way the rest of the parameters appear together on the screen size_distribution_string = 'fixed' if inv.filesize_distr == smf_invocation.filesize_distr_random_exponential: size_distribution_string = 'random exponential' prm_list = [ \ ('hosts in test', '%s'%prm_host_set), \ ('top test directory(s)', str(prm_top_dirs)), \ ('operation', inv.opname), \ ('files/thread', '%d'%inv.iterations), \ ('threads', '%d'%prm_thread_count), \ ('record size (KB)', '%d'%inv.record_sz_kb), \ ('file size (KB)', '%d'%inv.total_sz_kb), \ ('file size distribution', size_distribution_string), \ ('files per dir', '%d'%inv.files_per_dir), \ ('dirs per dir', '%d'%inv.dirs_per_dir), \ ('threads share directories?', '%s'%bool2YN(inv.is_shared_dir)), \ ('filename prefix', inv.prefix), \ ('filename suffix', inv.suffix), \ ('hash file number into dir.?', bool2YN(inv.hash_to_dir)), \ ('fsync after modify?', bool2YN(inv.fsync)), \ ('pause between files (microsec)', '%d'%inv.pause_between_files), \ ('finish all requests?', '%s'%bool2YN(inv.finish_all_rq)), \ ('stonewall?', '%s'%bool2YN(inv.stonewall)), \ ('measure response times?', '%s'%bool2YN(inv.measure_rsptimes)), \ ('verify read?', '%s'%bool2YN(inv.verify_read)), \ ('verbose?', inv.verbose), \ ('log to stderr?', inv.log_to_stderr) ] if (not smallfile.xattr_not_installed): prm_list.extend( [ ('ext.attr.size', '%d'%inv.xattr_size), ('ext.attr.count', '%d'%inv.xattr_count) ] ) if prm_host_set: prm_list.extend( [ \ ('permute host directories?', '%s'%bool2YN(prm_permute_host_dirs)) ] ) if prm_remote_pgm_dir: prm_list.append( ('remote program directory', prm_remote_pgm_dir) ) if prm_network_sync_dir: prm_list.append( ('network thread sync. dir.', prm_network_sync_dir) ) if inv.record_sz_kb == 0 and inv.verbose: print('record size not specified, large files will default to record size %d KB'%(smf_invocation.biggest_buf_size/inv.BYTES_PER_KB)) if not prm_slave: print('smallfile version %s'%version) for (prm_name, prm_value) in prm_list: print('%40s : %s'%(prm_name, prm_value)) # construct command to run remote slave process using CLI parameters, we have them all here if not prm_remote_pgm_dir: prm_remote_pgm_dir = os.getcwd() remote_cmd = prm_remote_pgm_dir + os.sep + 'smallfile_cli.py ' + pass_on_prm_list # "inv" contains all per-thread parameters params = smf_test_params.smf_test_params(prm_host_set, prm_thread_count, inv, prm_remote_pgm_dir, prm_top_dirs, \ prm_network_sync_dir, prm_slave, prm_permute_host_dirs) #print params return params
def parse(): # store as much as you can in SmallfileWorkload object # so per-thread invocations inherit test_params = smf_test_params.smf_test_params() inv = test_params.master_invoke # for convenience parser = argparse.ArgumentParser( description='parse smallfile CLI parameters') add = parser.add_argument add('--yaml-input-file', help='input YAML file containing all parameters below') add('--output-json', default=test_params.output_json, help='if true then output JSON-format version of results') add('--response-times', type=boolean, default=inv.measure_rsptimes, help='if true then record response time of each file op') add('--network-sync-dir', help='if --top not shared filesystem, provide shared filesystem directory') add('--operation', default='cleanup', choices=SmallfileWorkload.all_op_names, help='type of operation to perform on each file') add('--top', type=directory_list, default=inv.top_dirs, help='top directory or directories used by smallfile') add('--host-set', type=host_set, default=test_params.host_set, help='list of workload generator hosts (or file containing it) ') add('--launch-by-daemon', type=boolean, default=test_params.launch_by_daemon, help='use non-ssh launcher to get test running') add('--files', type=positive_integer, default=inv.iterations, help='files processed per thread') add('--threads', type=positive_integer, default=test_params.thread_count, help='threads per client') add('--files-per-dir', type=positive_integer, default=inv.files_per_dir, help='files per (sub)directory') add('--dirs-per-dir', type=positive_integer, default=inv.dirs_per_dir, help='subdirectories per directory') add('--record-size', type=positive_integer, default=inv.record_sz_kb, help='record size (KB)') add('--file-size', type=non_negative_integer, default=inv.total_sz_kb, help='subdirectories per directory') add('--file-size-distribution', type=file_size_distrib, default=inv.filesize_distr, help='file size can be constant ("fixed") or random ("exponential")') add('--fsync', type=boolean, default=inv.fsync, help='call fsync() after each file is written/modified') add('--xattr-size', type=non_negative_integer, default=inv.xattr_size, help='extended attribute size (bytes)') add('--xattr-count', type=non_negative_integer, default=inv.xattr_count, help='number of extended attributes per file') add('--pause', type=non_negative_integer, default=inv.pause_between_files, help='pause between each file (microsec)') add('--stonewall', type=boolean, default=inv.stonewall, help='stop measuring as soon as first thread is done') add('--finish', type=boolean, default=inv.finish_all_rq, help='stop processing files as soon as first thread is done') add('--prefix', default=inv.prefix, help='filename prefix') add('--suffix', default=inv.suffix, help='filename suffix') add('--hash-into-dirs', type=boolean, default=inv.hash_to_dir, help='if true then pseudo-randomly place files into directories') add('--same-dir', type=boolean, default=inv.is_shared_dir, help='if true then all threads share the same directories') add('--verbose', type=boolean, default=inv.verbose, help='if true then log extra messages about test') add('--permute-host-dirs', type=boolean, default=test_params.permute_host_dirs, help='if true then shift clients to different host directories') add('--record-ctime-size', type=boolean, default=inv.record_ctime_size, help='if true then update file xattr with ctime+size') add('--verify-read', type=boolean, default=inv.verify_read, help='if true then check that data read = data written') add('--incompressible', type=boolean, default=inv.incompressible, help='if true then non-compressible data written') # these parameters shouldn't be used by mere mortals add('--min-dirs-per-sec', type=positive_integer, default=test_params.min_directories_per_sec, help=argparse.SUPPRESS) add('--log-to-stderr', type=boolean, default=inv.log_to_stderr, help=argparse.SUPPRESS) add('--remote-pgm-dir', default=test_params.remote_pgm_dir, help=argparse.SUPPRESS) add('--slave', help=argparse.SUPPRESS) add('--as-host', help=argparse.SUPPRESS) args = parser.parse_args() inv.opname = args.operation test_params.top_dirs = [ os.path.abspath(p) for p in args.top ] test_params.launch_by_daemon = args.launch_by_daemon inv.iterations = args.files test_params.thread_count = args.threads inv.files_per_dir = args.files_per_dir inv.dirs_per_dir = args.dirs_per_dir inv.record_sz_kb = args.record_size inv.total_sz_kb = args.file_size test_params.size_distribution = inv.filesize_distr = args.file_size_distribution inv.xattr_size = args.xattr_size inv.xattr_count = args.xattr_count inv.prefix = args.prefix inv.suffix = args.suffix inv.hash_to_dir = args.hash_into_dirs inv.pause_between_files = args.pause inv.stonewall = args.stonewall inv.finish_all_rq = args.finish inv.measure_rsptimes = args.response_times inv.fsync = args.fsync inv.record_ctime_size = args.record_ctime_size test_params.permute_host_dirs = args.permute_host_dirs test_params.output_json = args.output_json inv.incompressible = args.incompressible inv.verify_read = args.verify_read test_params.min_directories_per_sec = args.min_dirs_per_sec inv.is_shared_dir = args.same_dir inv.verbose = args.verbose inv.log_to_stderr = args.log_to_stderr test_params.remote_pgm_dir = args.remote_pgm_dir test_params.network_sync_dir = args.network_sync_dir test_params.is_slave = args.slave inv.onhost = smallfile.get_hostname(args.as_host) test_params.host_set = args.host_set # if YAML input was used, update test_params object with this # YAML parameters override CLI parameters if args.yaml_input_file: if not yaml_parser_installed: raise SmfParseException('python yaml module not available - is this PyPy?') yaml_parser.parse_yaml(test_params, args.yaml_input_file) if not test_params.network_sync_dir: test_params.network_sync_dir = os.path.join(test_params.top_dirs[0], 'network_shared') # validate parameters further now that we know what they all are sdmsg = 'directory %s containing network sync dir. must exist on all hosts (including this one)' parentdir = os.path.dirname(test_params.network_sync_dir) if not os.path.isdir(parentdir) and args.host_set != None: raise SmfParseException(sdmsg % parentdir) if inv.record_sz_kb > inv.total_sz_kb and inv.total_sz_kb != 0: raise SmfParseException('record size cannot exceed file size') if inv.record_sz_kb == 0 and inv.verbose: print(('record size not specified, ' + 'large files will default to record size %d KB') % (SmallfileWorkload.biggest_buf_size / inv.BYTES_PER_KB)) if test_params.top_dirs: for d in test_params.top_dirs: if len(d) < 6: raise SmfParseException( 'directory less than 6 characters, ' + 'cannot use top of filesystem, too dangerous') if not os.path.isdir(d) and test_params.network_sync_dir != None: raise SmfParseException( 'you must ensure that shared directory ' + d + ' is accessible ' + 'from this host and every remote host in test') if test_params.top_dirs: inv.set_top(test_params.top_dirs) else: test_params.top_dirs = inv.top_dirs if test_params.network_sync_dir: inv.network_dir = test_params.network_sync_dir else: test_params.network_sync_dir = inv.network_dir inv.starting_gate = os.path.join(inv.network_dir, 'starting_gate.tmp') if inv.iterations < 10: inv.stonewall = False if not test_params.is_slave: prm_list = test_params.human_readable() for (prm_name, prm_value) in prm_list: print('%40s : %s' % (prm_name, prm_value)) test_params.recalculate_timeouts() return test_params
def parse(): # define parameter variables # default does short test in /var/tmp so you can see the program run # store as much as you can in smf_invocation object so per-thread invocations inherit inv = smf_invocation() # parameters that can't be stored in a smf_invocation # describe how the smf_invocation threads work together prm_thread_count = 2 prm_host_set = None prm_slave = False prm_permute_host_dirs = False prm_remote_pgm_dir = None prm_top_dirs = None prm_network_sync_dir = None # parse command line argc = len(sys.argv) pass_on_prm_list = '' # parameters passed to remote hosts if needed if argc == 1: print( '\nfor additional help add the parameter "--help" to the command\n' ) j = 1 while j < argc: rawprm = sys.argv[j] if rawprm == '-h' or rawprm == '--help': usage('ok, so you need help, we all knew that ;-)') if rawprm[0:2] != '--': usage('parameter names begin with "--"') prm = rawprm[2:] if (j == argc - 1) and (argc % 2 != 1): usage('all parameters consist of a name and a value') val = sys.argv[j + 1] if len(rawprm) < 3: usage('parameter name not long enough') pass_on_prm = rawprm + ' ' + val j += 2 if prm == 'files': chkPositiveInt(val, rawprm) inv.iterations = int(val) elif prm == 'threads': chkPositiveInt(val, rawprm) prm_thread_count = int(val) elif prm == 'files-per-dir': chkPositiveInt(val, rawprm) inv.files_per_dir = int(val) elif prm == 'dirs-per-dir': chkPositiveInt(val, rawprm) inv.dirs_per_dir = int(val) elif prm == 'record-size': chkNonNegInt(val, rawprm) inv.record_sz_kb = int(val) elif prm == 'file-size': chkNonNegInt(val, rawprm) inv.total_sz_kb = int(val) elif prm == 'file-size-distribution': if val != 'exponential': usage('unrecognized file size distribution: %s' % val) inv.filesize_distr = smf_invocation.filesize_distr_random_exponential elif prm == 'xattr-size': chkNonNegInt(val, rawprm) inv.xattr_size = int(val) elif prm == 'xattr-count': chkNonNegInt(val, rawprm) inv.xattr_count = int(val) elif prm == 'prefix': inv.prefix = val elif prm == 'suffix': inv.suffix = val elif prm == 'hash-into-dirs': inv.hash_to_dir = str2bool(val, rawprm) elif prm == 'operation': if not smf_invocation.all_op_names.__contains__(val): usage('unrecognized operation name: %s' % val) inv.opname = val elif prm == 'top': prm_top_dirs = val.split(',') elif prm == 'pause': chkPositiveInt(val, rawprm) inv.pause_between_files = int(val) elif prm == 'stonewall': inv.stonewall = str2bool(val, rawprm) elif prm == 'finish': inv.finish_all_rq = str2bool(val, rawprm) elif prm == 'fsync': inv.fsync = str2bool(val, rawprm) elif prm == 'permute-host-dirs': prm_permute_host_dirs = str2bool(val, rawprm) pass_on_prm = '' elif prm == 'response-times': inv.measure_rsptimes = str2bool(val, rawprm) elif prm == 'verify-read': inv.verify_read = str2bool(val, rawprm) elif prm == 'same-dir': inv.is_shared_dir = str2bool(val, rawprm) elif prm == 'verbose': inv.verbose = str2bool(val, rawprm) elif prm == 'log-to-stderr': inv.log_to_stderr = str2bool(val, rawprm) elif prm == 'host-set': prm_host_set = val.split(",") if len(prm_host_set) < 2: prm_host_set = val.strip().split() if len(prm_host_set) == 0: usage( 'host list must be non-empty when --host-set option used') pass_on_prm = '' elif prm == 'remote-pgm-dir': prm_remote_pgm_dir = val elif prm == 'network-sync-dir': prm_network_sync_dir = val # --slave should not be used by end-user elif prm == 'slave': prm_slave = str2bool(val, rawprm) # --ashost should not be used by end-user elif prm == 'as-host': inv.onhost = smallfile.get_hostname(val) else: usage('unrecognized parameter name') pass_on_prm_list += ' ' + pass_on_prm # parameter options that workload generators will need # validate parameters further now that we know what they all are if inv.record_sz_kb > inv.total_sz_kb and inv.total_sz_kb != 0: usage('record size cannot exceed file size') if (inv.record_sz_kb != 0) and ((inv.total_sz_kb % inv.record_sz_kb) != 0): usage( 'file size must be multiple of record size if record size is non-zero' ) if prm_top_dirs: for d in prm_top_dirs: if len(d) < 6: usage( 'directory less than 6 characters, cannot use top of filesystem, too dangerous' ) if prm_top_dirs: inv.set_top(prm_top_dirs) else: prm_top_dirs = inv.top_dirs if prm_network_sync_dir: if not prm_host_set and not prm_slave: usage( 'you do not need to specify a network thread synchronization directory unless you use multiple hosts' ) inv.network_dir = prm_network_sync_dir else: prm_network_sync_dir = inv.network_dir if prm_remote_pgm_dir: if not prm_host_set and not prm_slave: usage( 'you do not need to specify a remote program directory unless you use multiple hosts' ) inv.starting_gate = os.path.join( inv.network_dir, 'starting_gate.tmp' ) # location of file that signals start, end of test if inv.iterations < 10: inv.stonewall = False # display results of parse so user knows what default values are # most important parameters come first # display host set first because this can be very long, # this way the rest of the parameters appear together on the screen size_distribution_string = 'fixed' if inv.filesize_distr == smf_invocation.filesize_distr_random_exponential: size_distribution_string = 'random exponential' prm_list = [ \ ('hosts in test', '%s'%prm_host_set), \ ('top test directory(s)', str(prm_top_dirs)), \ ('operation', inv.opname), \ ('files/thread', '%d'%inv.iterations), \ ('threads', '%d'%prm_thread_count), \ ('record size (KB)', '%d'%inv.record_sz_kb), \ ('file size (KB)', '%d'%inv.total_sz_kb), \ ('file size distribution', size_distribution_string), \ ('files per dir', '%d'%inv.files_per_dir), \ ('dirs per dir', '%d'%inv.dirs_per_dir), \ ('threads share directories?', '%s'%bool2YN(inv.is_shared_dir)), \ ('filename prefix', inv.prefix), \ ('filename suffix', inv.suffix), \ ('hash file number into dir.?', bool2YN(inv.hash_to_dir)), \ ('fsync after modify?', bool2YN(inv.fsync)), \ ('pause between files (microsec)', '%d'%inv.pause_between_files), \ ('finish all requests?', '%s'%bool2YN(inv.finish_all_rq)), \ ('stonewall?', '%s'%bool2YN(inv.stonewall)), \ ('measure response times?', '%s'%bool2YN(inv.measure_rsptimes)), \ ('verify read?', '%s'%bool2YN(inv.verify_read)), \ ('verbose?', inv.verbose), \ ('log to stderr?', inv.log_to_stderr) ] if (not smallfile.xattr_not_installed): prm_list.extend([('ext.attr.size', '%d' % inv.xattr_size), ('ext.attr.count', '%d' % inv.xattr_count)]) if prm_host_set: prm_list.extend( [ \ ('permute host directories?', '%s'%bool2YN(prm_permute_host_dirs)) ] ) if prm_remote_pgm_dir: prm_list.append(('remote program directory', prm_remote_pgm_dir)) if prm_network_sync_dir: prm_list.append( ('network thread sync. dir.', prm_network_sync_dir)) if inv.record_sz_kb == 0 and inv.verbose: print( 'record size not specified, large files will default to record size %d KB' % (smf_invocation.biggest_buf_size / inv.BYTES_PER_KB)) if not prm_slave: print('smallfile version %s' % version) for (prm_name, prm_value) in prm_list: print('%40s : %s' % (prm_name, prm_value)) # construct command to run remote slave process using CLI parameters, we have them all here if not prm_remote_pgm_dir: prm_remote_pgm_dir = os.getcwd() remote_cmd = prm_remote_pgm_dir + os.sep + 'smallfile_cli.py ' + pass_on_prm_list # "inv" contains all per-thread parameters params = smf_test_params.smf_test_params(prm_host_set, prm_thread_count, inv, prm_remote_pgm_dir, prm_top_dirs, \ prm_network_sync_dir, prm_slave, prm_permute_host_dirs) #print params return params