def parse():
prm_network_sync_dir = None
argc = len(sys.argv)
if argc == 1:
print('for additional help add the parameter "--help" to the command')
sys.exit(1)
j = 1
while j < argc:
rawprm = sys.argv[j]
if rawprm == '-h' or rawprm == '--help':
usage('normally this process is run automatically ' +
'by smallfile_cli.py')
if rawprm[0:2] != '--':
usage('parameter names begin with "--"')
prm = rawprm[2:]
if j == argc - 1 and argc % 2 != 1:
usage('all parameters consist of a name and a value')
val = sys.argv[j + 1]
if len(rawprm) < 3:
usage('parameter name not long enough')
j += 2
if prm == 'network-sync-dir':
prm_network_sync_dir = val
elif prm == 'as-host':
# --ashost should not be used by end-user
prm_as_host = smallfile.get_hostname(val)
else:
usage('unrecognized parameter name')
param_pickle_fname = os.path.join(prm_network_sync_dir, 'param.pickle')
if not os.path.exists(param_pickle_fname):
time.sleep(1.1)
params = None
try:
with open(param_pickle_fname, 'rb') as pickled_params:
params = pickle.load(pickled_params)
params.is_slave = True
params.as_host = prm_as_host
params.master_invoke.onhost = prm_as_host
except IOError as e:
if e.errno != errno.ENOENT:
raise e
usage('could not read parameter pickle file %s' % param_pickle_fname)
return params
def parse():
parser = argparse.ArgumentParser(
description='parse remote smallfile parameters')
parser.add_argument('--network-sync-dir',
help='directory used to synchronize with test driver')
parser.add_argument('--as-host',
default=smallfile.get_hostname(None),
help='directory used to synchronize with test driver')
args = parser.parse_args()
param_pickle_fname = os.path.join(args.network_sync_dir, 'param.pickle')
if not os.path.exists(param_pickle_fname):
time.sleep(1.1)
params = None
with open(param_pickle_fname, 'rb') as pickled_params:
params = pickle.load(pickled_params)
params.is_slave = True
params.as_host = args.as_host
params.master_invoke.onhost = args.as_host
return params
def parse():
parser = argparse.ArgumentParser(
description='parse remote smallfile parameters')
parser.add_argument( '--network-sync-dir',
help='directory used to synchronize with test driver')
parser.add_argument( '--as-host',
default=smallfile.get_hostname(None),
help='directory used to synchronize with test driver')
args = parser.parse_args()
param_pickle_fname = os.path.join(args.network_sync_dir, 'param.pickle')
if not os.path.exists(param_pickle_fname):
time.sleep(1.1)
params = None
with open(param_pickle_fname, 'rb') as pickled_params:
params = pickle.load(pickled_params)
params.is_slave = True
params.as_host = args.as_host
params.master_invoke.onhost = args.as_host
return params
def parse():
prm_network_sync_dir = None
argc = len(sys.argv)
if argc == 1:
print('\nfor additional help add the parameter "--help" to the command\n')
j=1
while j < argc:
rawprm = sys.argv[j]
if rawprm == '-h' or rawprm == '--help':
usage('normally this process is run automatically by smallfile_cli.py')
if rawprm[0:2] != '--': usage('parameter names begin with "--"')
prm = rawprm[2:]
if (j == argc - 1) and (argc%2 != 1): usage('all parameters consist of a name and a value')
val = sys.argv[j+1]
if len(rawprm) < 3: usage('parameter name not long enough' )
pass_on_prm = rawprm + ' ' + val
j += 2
if prm == 'network-sync-dir': prm_network_sync_dir = val
# --ashost should not be used by end-user
elif prm == 'as-host':
prm_as_host = smallfile.get_hostname(val)
else: usage('unrecognized parameter name')
param_pickle_fname = os.path.join(prm_network_sync_dir, 'param.pickle')
if not os.path.exists(param_pickle_fname):
time.sleep(1.1)
params = None
try:
with open(param_pickle_fname, 'rb') as pickled_params:
params = pickle.load(pickled_params)
params.is_slave = True
params.as_host = prm_as_host
params.master_invoke.onhost = prm_as_host
except IOError as e:
if e.errno != errno.ENOENT: raise e
usage('could not read parameter pickle file %s'%param_pickle_fname)
return params
def parse():
prm_network_sync_dir = None
argc = len(sys.argv)
if argc == 1:
print('\nfor additional help add the parameter "--help" to the command\n')
j=1
while j < argc:
rawprm = sys.argv[j]
if rawprm == '-h' or rawprm == '--help':
usage('normally this process is run automatically by smallfile_cli.py')
if rawprm[0:2] != '--': usage('parameter names begin with "--"')
prm = rawprm[2:]
if (j == argc - 1) and (argc%2 != 1): usage('all parameters consist of a name and a value')
val = sys.argv[j+1]
if len(rawprm) < 3: usage('parameter name not long enough' )
pass_on_prm = rawprm + ' ' + val
j += 2
if prm == 'network-sync-dir': prm_network_sync_dir = val
# --ashost should not be used by end-user
elif prm == 'as-host':
prm_as_host = smallfile.get_hostname(val)
else: usage('unrecognized parameter name')
params = None
param_pickle_fname = os.path.join(prm_network_sync_dir, 'param.pickle')
if not os.path.exists(param_pickle_fname):
time.sleep(1.1)
pickle_fn = os.path.join(prm_network_sync_dir, 'param.pickle')
#print(pickle_fn)
with open(pickle_fn, 'rb') as pickled_params:
params = pickle.load(pickled_params)
params.is_slave = True
params.as_host = prm_as_host
params.master_invoke.onhost = prm_as_host
return params
def parse():
# parse command line
argc = len(sys.argv)
if argc == 1:
print('''
for additional help add the parameter "--help" to the command
''')
j = 1
while j < argc:
rawprm = sys.argv[j]
if rawprm == '-h' or rawprm == '--help':
usage('ok, so you need help, we all knew that ;-)')
if rawprm[0:2] != '--':
usage('parameter names begin with "--"')
prm = rawprm[2:]
if j == argc - 1 and argc % 2 != 1:
usage('all parameters consist of a name and a value')
val = sys.argv[j + 1]
if len(rawprm) < 3:
usage('parameter name not long enough')
pass_on_prm = rawprm + ' ' + val
j += 2
if prm == 'files':
chkPositiveInt(val, rawprm)
inv.iterations = int(val)
elif prm == 'threads':
chkPositiveInt(val, rawprm)
test_params.thread_count = int(val)
elif prm == 'files-per-dir':
chkPositiveInt(val, rawprm)
inv.files_per_dir = int(val)
elif prm == 'dirs-per-dir':
chkPositiveInt(val, rawprm)
inv.dirs_per_dir = int(val)
elif prm == 'record-size':
chkNonNegInt(val, rawprm)
inv.record_sz_kb = int(val)
elif prm == 'file-size':
chkNonNegInt(val, rawprm)
inv.total_sz_kb = int(val)
elif prm == 'file-size-distribution':
if val != 'exponential':
usage('unrecognized file size distribution: %s' % val)
inv.filesize_distr = \
SmallfileWorkload.fsdistr_random_exponential
test_params.size_distribution = 'random exponential'
elif prm == 'xattr-size':
chkNonNegInt(val, rawprm)
inv.xattr_size = int(val)
elif prm == 'xattr-count':
chkNonNegInt(val, rawprm)
inv.xattr_count = int(val)
elif prm == 'prefix':
inv.prefix = val
elif prm == 'suffix':
inv.suffix = val
elif prm == 'hash-into-dirs':
inv.hash_to_dir = str2bool(val, rawprm)
elif prm == 'operation':
if not SmallfileWorkload.all_op_names.__contains__(val):
usage('unrecognized operation name: %s' % val)
inv.opname = val
elif prm == 'top':
test_params.top_dirs = [os.path.abspath(p) for p in val.split(',')]
elif prm == 'pause':
chkPositiveInt(val, rawprm)
inv.pause_between_files = int(val)
elif prm == 'stonewall':
inv.stonewall = str2bool(val, rawprm)
elif prm == 'finish':
inv.finish_all_rq = str2bool(val, rawprm)
elif prm == 'fsync':
inv.fsync = str2bool(val, rawprm)
elif prm == 'record-ctime-size':
inv.record_ctime_size = str2bool(val, rawprm)
elif prm == 'permute-host-dirs':
test_params.permute_host_dirs = str2bool(val, rawprm)
pass_on_prm = ''
elif prm == 'output-json':
test_params.output_json = val
elif prm == 'response-times':
inv.measure_rsptimes = str2bool(val, rawprm)
elif prm == 'incompressible':
inv.incompressible = str2bool(val, rawprm)
elif prm == 'verify-read':
inv.verify_read = str2bool(val, rawprm)
elif prm == 'min-dirs-per-sec':
chkPositiveInt(val, rawprm)
test_params.min_directories_per_sec = int(val)
elif prm == 'same-dir':
inv.is_shared_dir = str2bool(val, rawprm)
elif prm == 'verbose':
inv.verbose = str2bool(val, rawprm)
elif prm == 'log-to-stderr':
inv.log_to_stderr = str2bool(val, rawprm)
elif prm == 'host-set':
if os.path.isfile(val):
with open(val, 'r') as f:
test_params.host_set = [
record.strip() for record in f.readlines()]
else:
test_params.host_set = val.split(',')
if len(test_params.host_set) < 2:
test_params.host_set = val.strip().split()
if len(test_params.host_set) == 0:
usage('host list must be non-empty when ' +
'--host-set option used')
pass_on_prm = ''
elif prm == 'remote-pgm-dir':
test_params.remote_pgm_dir = val
elif prm == 'network-sync-dir':
test_params.network_sync_dir = val
if not os.path.isdir(test_params.network_sync_dir):
usage('you must ensure that shared directory ' +
test_params.network_sync_dir +
' exists on this host and every remote host in test')
elif prm == 'slave':
# --slave should not be used by end-user
test_params.is_slave = str2bool(val, rawprm)
elif prm == 'as-host':
# --ashost should not be used by end-user
inv.onhost = smallfile.get_hostname(val)
else:
usage('unrecognized parameter name: %s' % prm)
# validate parameters further now that we know what they all are
if inv.record_sz_kb > inv.total_sz_kb and inv.total_sz_kb != 0:
usage('record size cannot exceed file size')
if inv.record_sz_kb == 0 and inv.verbose:
print(('record size not specified, ' +
'large files will default to record size %d KB') %
(SmallfileWorkload.biggest_buf_size / inv.BYTES_PER_KB))
if test_params.top_dirs:
for d in test_params.top_dirs:
if len(d) < 6:
usage('directory less than 6 characters, ' +
'cannot use top of filesystem, too dangerous')
if not os.path.isdir(d) and not test_params.network_sync_dir:
usage('you must ensure that shared directory ' + d +
' is accessible ' +
'from this host and every remote host in test')
if test_params.top_dirs:
inv.set_top(test_params.top_dirs)
else:
test_params.top_dirs = inv.top_dirs
if test_params.network_sync_dir:
inv.network_dir = test_params.network_sync_dir
else:
test_params.network_sync_dir = inv.network_dir
inv.starting_gate = os.path.join(inv.network_dir, 'starting_gate.tmp')
if inv.iterations < 10:
inv.stonewall = False
if not test_params.is_slave:
prm_list = test_params.human_readable()
for (prm_name, prm_value) in prm_list:
print('%40s : %s' % (prm_name, prm_value))
test_params.recalculate_timeouts()
return test_params
print(msg)
print('usage: python launch_smf_host.py'
'--top top-directory '
'[ --substitute-top synonym-directory ]'
'[ --as-host as-host-name ] ')
sys.exit(NOTOK)
# parse command line
if len(sys.argv) < 3:
usage('required command line arguments missing')
substitute_dir = None
top_dir = None
as_host = smallfile.get_hostname(None)
j = 1
while j < len(sys.argv):
if len(sys.argv) == j + 1:
usage('every parameter name must have a value')
nm = sys.argv[j]
if len(nm) < 3:
usage('parameter name must be > 3 characters long and start with --')
nm = nm[2:]
val = sys.argv[j + 1]
j += 2
if nm == 'substitute-top':
substitute_dir = val
elif nm == 'top':
top_dir = val
elif nm == 'as-host':
def parse():
# define parameter variables
# default does short test in /var/tmp so you can see the program run
# store as much as you can in smf_invocation object so per-thread invocations inherit
inv = smf_invocation()
# parameters that can't be stored in a smf_invocation
# describe how the smf_invocation threads work together
prm_thread_count = 2
prm_host_set = None
prm_slave = False
prm_permute_host_dirs = False
prm_remote_pgm_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
prm_top_dirs = None
prm_network_sync_dir = None
# parse command line
argc = len(sys.argv)
pass_on_prm_list = '' # parameters passed to remote hosts if needed
if argc == 1:
print('\nfor additional help add the parameter "--help" to the command\n')
j=1
while j < argc:
rawprm = sys.argv[j]
if rawprm == '-h' or rawprm == '--help':
usage('ok, so you need help, we all knew that ;-)')
if rawprm[0:2] != '--': usage('parameter names begin with "--"')
prm = rawprm[2:]
if (j == argc - 1) and (argc%2 != 1): usage('all parameters consist of a name and a value')
val = sys.argv[j+1]
if len(rawprm) < 3: usage('parameter name not long enough' )
pass_on_prm = rawprm + ' ' + val
j += 2
if prm == 'files':
chkPositiveInt(val, rawprm)
inv.iterations = int(val)
elif prm == 'threads':
chkPositiveInt(val, rawprm)
prm_thread_count = int(val)
elif prm == 'files-per-dir':
chkPositiveInt(val, rawprm)
inv.files_per_dir = int(val)
elif prm == 'dirs-per-dir':
chkPositiveInt(val, rawprm)
inv.dirs_per_dir = int(val)
elif prm == 'record-size':
chkNonNegInt(val, rawprm)
inv.record_sz_kb = int(val)
elif prm == 'file-size':
chkNonNegInt(val, rawprm)
inv.total_sz_kb = int(val)
elif prm == 'file-size-distribution':
if val != 'exponential':
usage('unrecognized file size distribution: %s'%val)
inv.filesize_distr = smf_invocation.filesize_distr_random_exponential
elif prm == 'xattr-size':
chkNonNegInt(val, rawprm)
inv.xattr_size = int(val)
elif prm == 'xattr-count':
chkNonNegInt(val, rawprm)
inv.xattr_count = int(val)
elif prm == 'prefix': inv.prefix = val
elif prm == 'suffix': inv.suffix = val
elif prm == 'hash-into-dirs': inv.hash_to_dir = str2bool(val, rawprm)
elif prm == 'operation':
if not smf_invocation.all_op_names.__contains__(val):
usage('unrecognized operation name: %s'%val)
inv.opname = val
elif prm == 'top':
prm_top_dirs = val.split(',')
elif prm == 'pause':
chkPositiveInt(val, rawprm)
inv.pause_between_files = int(val)
elif prm == 'stonewall': inv.stonewall = str2bool(val, rawprm)
elif prm == 'finish': inv.finish_all_rq = str2bool(val, rawprm)
elif prm == 'fsync': inv.fsync = str2bool(val, rawprm)
elif prm == 'record-ctime-size': inv.record_ctime_size = str2bool(val, rawprm)
elif prm == 'permute-host-dirs':
prm_permute_host_dirs = str2bool(val, rawprm)
pass_on_prm = ''
elif prm == 'response-times': inv.measure_rsptimes = str2bool(val, rawprm)
elif prm == 'incompressible': inv.incompressible = str2bool(val, rawprm)
elif prm == 'verify-read': inv.verify_read = str2bool(val, rawprm)
elif prm == 'same-dir': inv.is_shared_dir = str2bool(val, rawprm)
elif prm == 'verbose': inv.verbose = str2bool(val, rawprm)
elif prm == 'log-to-stderr': inv.log_to_stderr = str2bool(val, rawprm)
elif prm == 'host-set':
prm_host_set = val.split(",")
if len(prm_host_set) < 2: prm_host_set = val.strip().split()
if len(prm_host_set) == 0:
usage('host list must be non-empty when --host-set option used')
pass_on_prm = ''
elif prm == 'remote-pgm-dir': prm_remote_pgm_dir = val
elif prm == 'network-sync-dir': prm_network_sync_dir = val
# --slave should not be used by end-user
elif prm == 'slave': prm_slave = str2bool(val, rawprm)
# --ashost should not be used by end-user
elif prm == 'as-host':
inv.onhost = smallfile.get_hostname(val)
else: usage('unrecognized parameter name')
pass_on_prm_list += ' ' + pass_on_prm # parameter options that workload generators will need
# validate parameters further now that we know what they all are
if inv.record_sz_kb > inv.total_sz_kb and inv.total_sz_kb != 0:
usage('record size cannot exceed file size')
if (inv.record_sz_kb != 0) and ((inv.total_sz_kb % inv.record_sz_kb) != 0):
usage('file size must be multiple of record size if record size is non-zero')
if prm_top_dirs:
for d in prm_top_dirs:
if len(d) < 6:
usage('directory less than 6 characters, cannot use top of filesystem, too dangerous')
if prm_top_dirs:
inv.set_top(prm_top_dirs)
else:
prm_top_dirs = inv.top_dirs
if prm_network_sync_dir:
inv.network_dir = prm_network_sync_dir
else:
prm_network_sync_dir = inv.network_dir
inv.starting_gate = os.path.join(inv.network_dir, 'starting_gate.tmp') # location of file that signals start, end of test
if inv.iterations < 10: inv.stonewall = False
# display results of parse so user knows what default values are
# most important parameters come first
# display host set first because this can be very long,
# this way the rest of the parameters appear together on the screen
size_distribution_string = 'fixed'
if inv.filesize_distr == smf_invocation.filesize_distr_random_exponential:
size_distribution_string = 'random exponential'
prm_list = [ \
('hosts in test', '%s'%prm_host_set), \
('top test directory(s)', str(prm_top_dirs)), \
('operation', inv.opname), \
('files/thread', '%d'%inv.iterations), \
('threads', '%d'%prm_thread_count), \
('record size (KB)', '%d'%inv.record_sz_kb), \
('file size (KB)', '%d'%inv.total_sz_kb), \
('file size distribution', size_distribution_string), \
('files per dir', '%d'%inv.files_per_dir), \
('dirs per dir', '%d'%inv.dirs_per_dir), \
('threads share directories?', '%s'%bool2YN(inv.is_shared_dir)), \
('filename prefix', inv.prefix), \
('filename suffix', inv.suffix), \
('hash file number into dir.?', bool2YN(inv.hash_to_dir)), \
('fsync after modify?', bool2YN(inv.fsync)), \
('pause between files (microsec)', '%d'%inv.pause_between_files), \
('finish all requests?', '%s'%bool2YN(inv.finish_all_rq)), \
('stonewall?', '%s'%bool2YN(inv.stonewall)), \
('measure response times?', '%s'%bool2YN(inv.measure_rsptimes)), \
('verify read?', '%s'%bool2YN(inv.verify_read)), \
('verbose?', inv.verbose), \
('log to stderr?', inv.log_to_stderr) ]
if (not smallfile.xattr_not_installed):
prm_list.extend( [ ('ext.attr.size', '%d'%inv.xattr_size), ('ext.attr.count', '%d'%inv.xattr_count) ] )
if prm_host_set:
prm_list.extend( [ \
('permute host directories?', '%s'%bool2YN(prm_permute_host_dirs)) ] )
if prm_remote_pgm_dir: prm_list.append( ('remote program directory', prm_remote_pgm_dir) )
if prm_network_sync_dir: prm_list.append( ('network thread sync. dir.', prm_network_sync_dir) )
if inv.record_sz_kb == 0 and inv.verbose:
print('record size not specified, large files will default to record size %d KB'%(smf_invocation.biggest_buf_size/inv.BYTES_PER_KB))
if not prm_slave:
print('smallfile version %s'%version)
for (prm_name, prm_value) in prm_list:
print('%40s : %s'%(prm_name, prm_value))
# construct command to run remote slave process using CLI parameters, we have them all here
if not prm_remote_pgm_dir: prm_remote_pgm_dir = os.getcwd()
remote_cmd = prm_remote_pgm_dir + os.sep + 'smallfile_cli.py ' + pass_on_prm_list
# "inv" contains all per-thread parameters
params = smf_test_params.smf_test_params(prm_host_set, prm_thread_count, inv, prm_remote_pgm_dir, prm_top_dirs, \
prm_network_sync_dir, prm_slave, prm_permute_host_dirs)
#print params
return params
def parse():
# store as much as you can in SmallfileWorkload object
# so per-thread invocations inherit
test_params = smf_test_params.smf_test_params()
inv = test_params.master_invoke # for convenience
parser = argparse.ArgumentParser(
description='parse smallfile CLI parameters')
add = parser.add_argument
add('--yaml-input-file',
help='input YAML file containing all parameters below')
add('--output-json',
default=test_params.output_json,
help='if true then output JSON-format version of results')
add('--response-times',
type=boolean, default=inv.measure_rsptimes,
help='if true then record response time of each file op')
add('--network-sync-dir',
help='if --top not shared filesystem, provide shared filesystem directory')
add('--operation',
default='cleanup', choices=SmallfileWorkload.all_op_names,
help='type of operation to perform on each file')
add('--top',
type=directory_list, default=inv.top_dirs,
help='top directory or directories used by smallfile')
add('--host-set',
type=host_set, default=test_params.host_set,
help='list of workload generator hosts (or file containing it) ')
add('--launch-by-daemon',
type=boolean, default=test_params.launch_by_daemon,
help='use non-ssh launcher to get test running')
add('--files',
type=positive_integer, default=inv.iterations,
help='files processed per thread')
add('--threads',
type=positive_integer, default=test_params.thread_count,
help='threads per client')
add('--files-per-dir',
type=positive_integer, default=inv.files_per_dir,
help='files per (sub)directory')
add('--dirs-per-dir',
type=positive_integer, default=inv.dirs_per_dir,
help='subdirectories per directory')
add('--record-size',
type=positive_integer, default=inv.record_sz_kb,
help='record size (KB)')
add('--file-size',
type=non_negative_integer, default=inv.total_sz_kb,
help='subdirectories per directory')
add('--file-size-distribution',
type=file_size_distrib, default=inv.filesize_distr,
help='file size can be constant ("fixed") or random ("exponential")')
add('--fsync',
type=boolean, default=inv.fsync,
help='call fsync() after each file is written/modified')
add('--xattr-size',
type=non_negative_integer, default=inv.xattr_size,
help='extended attribute size (bytes)')
add('--xattr-count',
type=non_negative_integer, default=inv.xattr_count,
help='number of extended attributes per file')
add('--pause',
type=non_negative_integer, default=inv.pause_between_files,
help='pause between each file (microsec)')
add('--stonewall',
type=boolean, default=inv.stonewall,
help='stop measuring as soon as first thread is done')
add('--finish',
type=boolean, default=inv.finish_all_rq,
help='stop processing files as soon as first thread is done')
add('--prefix',
default=inv.prefix,
help='filename prefix')
add('--suffix',
default=inv.suffix,
help='filename suffix')
add('--hash-into-dirs',
type=boolean, default=inv.hash_to_dir,
help='if true then pseudo-randomly place files into directories')
add('--same-dir',
type=boolean, default=inv.is_shared_dir,
help='if true then all threads share the same directories')
add('--verbose',
type=boolean, default=inv.verbose,
help='if true then log extra messages about test')
add('--permute-host-dirs',
type=boolean, default=test_params.permute_host_dirs,
help='if true then shift clients to different host directories')
add('--record-ctime-size',
type=boolean, default=inv.record_ctime_size,
help='if true then update file xattr with ctime+size')
add('--verify-read',
type=boolean, default=inv.verify_read,
help='if true then check that data read = data written')
add('--incompressible',
type=boolean, default=inv.incompressible,
help='if true then non-compressible data written')
# these parameters shouldn't be used by mere mortals
add('--min-dirs-per-sec',
type=positive_integer, default=test_params.min_directories_per_sec,
help=argparse.SUPPRESS)
add('--log-to-stderr', type=boolean, default=inv.log_to_stderr,
help=argparse.SUPPRESS)
add('--remote-pgm-dir', default=test_params.remote_pgm_dir,
help=argparse.SUPPRESS)
add('--slave',
help=argparse.SUPPRESS)
add('--as-host',
help=argparse.SUPPRESS)
args = parser.parse_args()
inv.opname = args.operation
test_params.top_dirs = [ os.path.abspath(p) for p in args.top ]
test_params.launch_by_daemon = args.launch_by_daemon
inv.iterations = args.files
test_params.thread_count = args.threads
inv.files_per_dir = args.files_per_dir
inv.dirs_per_dir = args.dirs_per_dir
inv.record_sz_kb = args.record_size
inv.total_sz_kb = args.file_size
test_params.size_distribution = inv.filesize_distr = args.file_size_distribution
inv.xattr_size = args.xattr_size
inv.xattr_count = args.xattr_count
inv.prefix = args.prefix
inv.suffix = args.suffix
inv.hash_to_dir = args.hash_into_dirs
inv.pause_between_files = args.pause
inv.stonewall = args.stonewall
inv.finish_all_rq = args.finish
inv.measure_rsptimes = args.response_times
inv.fsync = args.fsync
inv.record_ctime_size = args.record_ctime_size
test_params.permute_host_dirs = args.permute_host_dirs
test_params.output_json = args.output_json
inv.incompressible = args.incompressible
inv.verify_read = args.verify_read
test_params.min_directories_per_sec = args.min_dirs_per_sec
inv.is_shared_dir = args.same_dir
inv.verbose = args.verbose
inv.log_to_stderr = args.log_to_stderr
test_params.remote_pgm_dir = args.remote_pgm_dir
test_params.network_sync_dir = args.network_sync_dir
test_params.is_slave = args.slave
inv.onhost = smallfile.get_hostname(args.as_host)
test_params.host_set = args.host_set
# if YAML input was used, update test_params object with this
# YAML parameters override CLI parameters
if args.yaml_input_file:
if not yaml_parser_installed:
raise SmfParseException('python yaml module not available - is this PyPy?')
yaml_parser.parse_yaml(test_params, args.yaml_input_file)
if not test_params.network_sync_dir:
test_params.network_sync_dir = os.path.join(test_params.top_dirs[0], 'network_shared')
# validate parameters further now that we know what they all are
sdmsg = 'directory %s containing network sync dir. must exist on all hosts (including this one)'
parentdir = os.path.dirname(test_params.network_sync_dir)
if not os.path.isdir(parentdir) and args.host_set != None:
raise SmfParseException(sdmsg % parentdir)
if inv.record_sz_kb > inv.total_sz_kb and inv.total_sz_kb != 0:
raise SmfParseException('record size cannot exceed file size')
if inv.record_sz_kb == 0 and inv.verbose:
print(('record size not specified, ' +
'large files will default to record size %d KB') %
(SmallfileWorkload.biggest_buf_size / inv.BYTES_PER_KB))
if test_params.top_dirs:
for d in test_params.top_dirs:
if len(d) < 6:
raise SmfParseException(
'directory less than 6 characters, ' +
'cannot use top of filesystem, too dangerous')
if not os.path.isdir(d) and test_params.network_sync_dir != None:
raise SmfParseException(
'you must ensure that shared directory ' + d +
' is accessible ' +
'from this host and every remote host in test')
if test_params.top_dirs:
inv.set_top(test_params.top_dirs)
else:
test_params.top_dirs = inv.top_dirs
if test_params.network_sync_dir:
inv.network_dir = test_params.network_sync_dir
else:
test_params.network_sync_dir = inv.network_dir
inv.starting_gate = os.path.join(inv.network_dir, 'starting_gate.tmp')
if inv.iterations < 10:
inv.stonewall = False
if not test_params.is_slave:
prm_list = test_params.human_readable()
for (prm_name, prm_value) in prm_list:
print('%40s : %s' % (prm_name, prm_value))
test_params.recalculate_timeouts()
return test_params
print(msg)
print('usage: python launch_smf_host.py'
'--top top-directory '
'[ --substitute-top synonym-directory ]'
'[ --as-host as-host-name ] ')
sys.exit(NOTOK)
# parse command line
if len(sys.argv) < 3:
usage('required command line arguments missing')
substitute_dir = None
top_dir = None
as_host = smallfile.get_hostname(None)
j = 1
while j < len(sys.argv):
if len(sys.argv) == j + 1:
usage('every parameter name must have a value')
nm = sys.argv[j]
if len(nm) < 3:
usage('parameter name must be > 3 characters long and start with --')
nm = nm[2:]
val = sys.argv[j + 1]
j += 2
if nm == 'substitute-top':
substitute_dir = val
elif nm == 'top':
top_dir = val
elif nm == 'as-host':
def parse():
# define parameter variables
# default does short test in /var/tmp so you can see the program run
# store as much as you can in smf_invocation object so per-thread invocations inherit
inv = smf_invocation()
# parameters that can't be stored in a smf_invocation
# describe how the smf_invocation threads work together
prm_thread_count = 2
prm_host_set = None
prm_slave = False
prm_permute_host_dirs = False
prm_remote_pgm_dir = None
prm_top_dirs = None
prm_network_sync_dir = None
# parse command line
argc = len(sys.argv)
pass_on_prm_list = '' # parameters passed to remote hosts if needed
if argc == 1:
print(
'\nfor additional help add the parameter "--help" to the command\n'
)
j = 1
while j < argc:
rawprm = sys.argv[j]
if rawprm == '-h' or rawprm == '--help':
usage('ok, so you need help, we all knew that ;-)')
if rawprm[0:2] != '--': usage('parameter names begin with "--"')
prm = rawprm[2:]
if (j == argc - 1) and (argc % 2 != 1):
usage('all parameters consist of a name and a value')
val = sys.argv[j + 1]
if len(rawprm) < 3: usage('parameter name not long enough')
pass_on_prm = rawprm + ' ' + val
j += 2
if prm == 'files':
chkPositiveInt(val, rawprm)
inv.iterations = int(val)
elif prm == 'threads':
chkPositiveInt(val, rawprm)
prm_thread_count = int(val)
elif prm == 'files-per-dir':
chkPositiveInt(val, rawprm)
inv.files_per_dir = int(val)
elif prm == 'dirs-per-dir':
chkPositiveInt(val, rawprm)
inv.dirs_per_dir = int(val)
elif prm == 'record-size':
chkNonNegInt(val, rawprm)
inv.record_sz_kb = int(val)
elif prm == 'file-size':
chkNonNegInt(val, rawprm)
inv.total_sz_kb = int(val)
elif prm == 'file-size-distribution':
if val != 'exponential':
usage('unrecognized file size distribution: %s' % val)
inv.filesize_distr = smf_invocation.filesize_distr_random_exponential
elif prm == 'xattr-size':
chkNonNegInt(val, rawprm)
inv.xattr_size = int(val)
elif prm == 'xattr-count':
chkNonNegInt(val, rawprm)
inv.xattr_count = int(val)
elif prm == 'prefix':
inv.prefix = val
elif prm == 'suffix':
inv.suffix = val
elif prm == 'hash-into-dirs':
inv.hash_to_dir = str2bool(val, rawprm)
elif prm == 'operation':
if not smf_invocation.all_op_names.__contains__(val):
usage('unrecognized operation name: %s' % val)
inv.opname = val
elif prm == 'top':
prm_top_dirs = val.split(',')
elif prm == 'pause':
chkPositiveInt(val, rawprm)
inv.pause_between_files = int(val)
elif prm == 'stonewall':
inv.stonewall = str2bool(val, rawprm)
elif prm == 'finish':
inv.finish_all_rq = str2bool(val, rawprm)
elif prm == 'fsync':
inv.fsync = str2bool(val, rawprm)
elif prm == 'permute-host-dirs':
prm_permute_host_dirs = str2bool(val, rawprm)
pass_on_prm = ''
elif prm == 'response-times':
inv.measure_rsptimes = str2bool(val, rawprm)
elif prm == 'verify-read':
inv.verify_read = str2bool(val, rawprm)
elif prm == 'same-dir':
inv.is_shared_dir = str2bool(val, rawprm)
elif prm == 'verbose':
inv.verbose = str2bool(val, rawprm)
elif prm == 'log-to-stderr':
inv.log_to_stderr = str2bool(val, rawprm)
elif prm == 'host-set':
prm_host_set = val.split(",")
if len(prm_host_set) < 2: prm_host_set = val.strip().split()
if len(prm_host_set) == 0:
usage(
'host list must be non-empty when --host-set option used')
pass_on_prm = ''
elif prm == 'remote-pgm-dir':
prm_remote_pgm_dir = val
elif prm == 'network-sync-dir':
prm_network_sync_dir = val
# --slave should not be used by end-user
elif prm == 'slave':
prm_slave = str2bool(val, rawprm)
# --ashost should not be used by end-user
elif prm == 'as-host':
inv.onhost = smallfile.get_hostname(val)
else:
usage('unrecognized parameter name')
pass_on_prm_list += ' ' + pass_on_prm # parameter options that workload generators will need
# validate parameters further now that we know what they all are
if inv.record_sz_kb > inv.total_sz_kb and inv.total_sz_kb != 0:
usage('record size cannot exceed file size')
if (inv.record_sz_kb != 0) and ((inv.total_sz_kb % inv.record_sz_kb) != 0):
usage(
'file size must be multiple of record size if record size is non-zero'
)
if prm_top_dirs:
for d in prm_top_dirs:
if len(d) < 6:
usage(
'directory less than 6 characters, cannot use top of filesystem, too dangerous'
)
if prm_top_dirs:
inv.set_top(prm_top_dirs)
else:
prm_top_dirs = inv.top_dirs
if prm_network_sync_dir:
if not prm_host_set and not prm_slave:
usage(
'you do not need to specify a network thread synchronization directory unless you use multiple hosts'
)
inv.network_dir = prm_network_sync_dir
else:
prm_network_sync_dir = inv.network_dir
if prm_remote_pgm_dir:
if not prm_host_set and not prm_slave:
usage(
'you do not need to specify a remote program directory unless you use multiple hosts'
)
inv.starting_gate = os.path.join(
inv.network_dir, 'starting_gate.tmp'
) # location of file that signals start, end of test
if inv.iterations < 10: inv.stonewall = False
# display results of parse so user knows what default values are
# most important parameters come first
# display host set first because this can be very long,
# this way the rest of the parameters appear together on the screen
size_distribution_string = 'fixed'
if inv.filesize_distr == smf_invocation.filesize_distr_random_exponential:
size_distribution_string = 'random exponential'
prm_list = [ \
('hosts in test', '%s'%prm_host_set), \
('top test directory(s)', str(prm_top_dirs)), \
('operation', inv.opname), \
('files/thread', '%d'%inv.iterations), \
('threads', '%d'%prm_thread_count), \
('record size (KB)', '%d'%inv.record_sz_kb), \
('file size (KB)', '%d'%inv.total_sz_kb), \
('file size distribution', size_distribution_string), \
('files per dir', '%d'%inv.files_per_dir), \
('dirs per dir', '%d'%inv.dirs_per_dir), \
('threads share directories?', '%s'%bool2YN(inv.is_shared_dir)), \
('filename prefix', inv.prefix), \
('filename suffix', inv.suffix), \
('hash file number into dir.?', bool2YN(inv.hash_to_dir)), \
('fsync after modify?', bool2YN(inv.fsync)), \
('pause between files (microsec)', '%d'%inv.pause_between_files), \
('finish all requests?', '%s'%bool2YN(inv.finish_all_rq)), \
('stonewall?', '%s'%bool2YN(inv.stonewall)), \
('measure response times?', '%s'%bool2YN(inv.measure_rsptimes)), \
('verify read?', '%s'%bool2YN(inv.verify_read)), \
('verbose?', inv.verbose), \
('log to stderr?', inv.log_to_stderr) ]
if (not smallfile.xattr_not_installed):
prm_list.extend([('ext.attr.size', '%d' % inv.xattr_size),
('ext.attr.count', '%d' % inv.xattr_count)])
if prm_host_set:
prm_list.extend( [ \
('permute host directories?', '%s'%bool2YN(prm_permute_host_dirs)) ] )
if prm_remote_pgm_dir:
prm_list.append(('remote program directory', prm_remote_pgm_dir))
if prm_network_sync_dir:
prm_list.append(
('network thread sync. dir.', prm_network_sync_dir))
if inv.record_sz_kb == 0 and inv.verbose:
print(
'record size not specified, large files will default to record size %d KB'
% (smf_invocation.biggest_buf_size / inv.BYTES_PER_KB))
if not prm_slave:
print('smallfile version %s' % version)
for (prm_name, prm_value) in prm_list:
print('%40s : %s' % (prm_name, prm_value))
# construct command to run remote slave process using CLI parameters, we have them all here
if not prm_remote_pgm_dir: prm_remote_pgm_dir = os.getcwd()
remote_cmd = prm_remote_pgm_dir + os.sep + 'smallfile_cli.py ' + pass_on_prm_list
# "inv" contains all per-thread parameters
params = smf_test_params.smf_test_params(prm_host_set, prm_thread_count, inv, prm_remote_pgm_dir, prm_top_dirs, \
prm_network_sync_dir, prm_slave, prm_permute_host_dirs)
#print params
return params
