def __init__(self,
host_set = None,
thread_count = 2,
remote_pgm_dir = os.path.abspath(os.path.dirname(sys.argv[0])),
top_dirs = None,
network_sync_dir = None,
slave = False,
size_distribution = 'fixed',
permute_host_dirs = False,
output_json = None):
# this field used to calculate timeouts
self.min_directories_per_sec = 50
self.output_json = output_json
self.version = '3.1'
self.as_host = None
self.host_set = host_set
self.thread_count = thread_count
self.master_invoke = smallfile.SmallfileWorkload()
self.remote_pgm_dir = remote_pgm_dir
self.top_dirs = top_dirs
if top_dirs:
self.master_invoke.set_top(top_dirs)
self.network_sync_dir = network_sync_dir
if network_sync_dir:
self.master_invoke.network_dir = network_sync_dir
self.is_slave = slave
self.size_distribution = size_distribution
self.permute_host_dirs = permute_host_dirs
self.startup_timeout = 0
self.host_startup_timeout = 0
def setUp(self):
self.invok = smallfile.SmallfileWorkload()
self.invok.debug = True
self.invok.verbose = True
self.invok.tid = 'regtest'
self.invok.start_log()
deltree(self.invok.src_dirs[0])
os.makedirs(self.invok.src_dirs[0], 0o644)
def test_multiproc_stonewall(self):
self.invok.log.info('starting stonewall test')
thread_ready_timeout = 4
thread_count = 4
for tree in self.invok.top_dirs:
deltree(tree)
os.mkdir(tree)
for dir in self.invok.src_dirs:
os.mkdir(dir)
for dir in self.invok.dest_dirs:
os.mkdir(dir)
os.mkdir(self.invok.network_dir)
self.invok.starting_gate = os.path.join(self.invok.network_dir,
'starting-gate')
sgate_file = self.invok.starting_gate
invokeList = []
for j in range(0, thread_count):
s = smallfile.SmallfileWorkload()
# s.log_to_stderr = True
s.verbose = True
s.tid = str(j)
s.prefix = 'thr_'
s.suffix = 'foo'
s.iterations = 10
s.stonewall = False
s.starting_gate = sgate_file
invokeList.append(s)
threadList = []
for s in invokeList:
threadList.append(subprocess(s))
for t in threadList:
t.start()
threads_ready = True
for i in range(0, thread_ready_timeout):
threads_ready = True
for s in invokeList:
thread_ready_file = s.gen_thread_ready_fname(s.tid)
if not os.path.exists(thread_ready_file):
threads_ready = False
if threads_ready:
break
time.sleep(1)
if not threads_ready:
raise Exception('threads did not show up within %d seconds' %
thread_ready_timeout)
time.sleep(1)
smallfile.touch(sgate_file)
for t in threadList:
rtnd_invok = t.receiver.recv()
t.join()
self.invok.log.info(str(rtnd_invok))
if rtnd_invok.status != ok:
raise Exception('subprocess failure for %s invocation %s: ' %
(str(t), str(rtnd_invok)))
def __init__(self,
host_set=None,
thread_count=2,
remote_pgm_dir=os.path.abspath(os.path.dirname(sys.argv[0])),
top_dirs=None,
network_sync_dir=None,
slave=False,
size_distribution='fixed',
permute_host_dirs=False,
output_json=None):
self.output_json = output_json
self.version = '3.1'
self.as_host = None
self.host_set = host_set
self.thread_count = thread_count
self.master_invoke = smallfile.SmallfileWorkload()
self.remote_pgm_dir = remote_pgm_dir
self.top_dirs = top_dirs
if top_dirs:
self.master_invoke.set_top(top_dirs)
self.network_sync_dir = network_sync_dir
if network_sync_dir:
self.master_invoke.network_dir = network_sync_dir
self.is_slave = slave
self.size_distribution = size_distribution
self.permute_host_dirs = permute_host_dirs
# calculate timeouts
# make sure dirs is never zero
# or you will break host_startup_timeout calculation
self.startup_timeout = 10
dirs = 1
dirs += (self.master_invoke.iterations * self.thread_count //
self.master_invoke.files_per_dir)
self.startup_timeout += dirs // 3
self.host_startup_timeout = self.startup_timeout
if self.host_set:
self.host_startup_timeout += 5 + dirs * len(self.host_set) // 3
# -*- coding: utf-8 -*-
#
# python program used by profile.sh to generate profile of smallfile workloads
#
import os
import socket
import smallfile
top = os.getenv('TOP')
count = int(os.getenv('COUNT'))
invk = smallfile.SmallfileWorkload()
invk.tid = '00'
invk.src_dirs = [
top + os.sep + 'file_srcdir' + os.sep + socket.gethostname() + os.sep +
'thrd_' + invk.tid
]
invk.dest_dirs = [
top + os.sep + 'file_dstdir' + os.sep + socket.gethostname() + os.sep +
'thrd_' + invk.tid
]
invk.network_dir = top + os.sep + 'network_shared'
invk.record_sz_kb = 0
invk.total_sz_kb = 1
invk.starting_gate = os.path.join(invk.network_dir, 'starting_gate')
invk.stonewall = True
invk.finish_all_rq = True
invk.opname = os.getenv('OPNAME')
invk.iterations = count
print(invk)
invk.do_workload()
print(invk)