def create_top_dirs(master_invoke, is_multi_host): if os.path.exists( master_invoke.network_dir ): shutil.rmtree( master_invoke.network_dir ) if is_multi_host: time.sleep(2.1) # hack to ensure that all remote clients can see that directory has been recreated ensure_dir_exists( master_invoke.network_dir ) for dlist in [ master_invoke.src_dirs, master_invoke.dest_dirs ]: for d in dlist: ensure_dir_exists(d) if is_multi_host: os.listdir(master_invoke.network_dir) # workaround to attempt to force synchronization time.sleep(1.1) # hack to let NFS mount option actimeo=1 have its effect
def create_top_dirs(master_invoke, is_multi_host): if os.path.exists(master_invoke.network_dir): shutil.rmtree(master_invoke.network_dir) if is_multi_host: # so all remote clients see that directory was recreated time.sleep(2.1) smallfile.ensure_dir_exists(master_invoke.network_dir) for dlist in [master_invoke.src_dirs, master_invoke.dest_dirs]: for d in dlist: smallfile.ensure_dir_exists(d) if is_multi_host: # workaround to force cross-host synchronization os.listdir(master_invoke.network_dir) time.sleep(1.1) # lets NFS mount option actimeo=1 take effect
def create_top_dirs(master_invoke, is_multi_host): if os.path.exists(master_invoke.network_dir): shutil.rmtree(master_invoke.network_dir) if is_multi_host: time.sleep( 2.1 ) # hack to ensure that all remote clients can see that directory has been recreated ensure_dir_exists(master_invoke.network_dir) for dlist in [master_invoke.src_dirs, master_invoke.dest_dirs]: for d in dlist: ensure_dir_exists(d) if is_multi_host: os.listdir(master_invoke.network_dir ) # workaround to attempt to force synchronization time.sleep( 1.1) # hack to let NFS mount option actimeo=1 have its effect
def run_multi_thread_workload(prm): master_invoke = prm.master_invoke prm_slave = prm.is_slave verbose = master_invoke.verbose host = master_invoke.onhost if not prm_slave: sync_files.create_top_dirs(master_invoke, False) if prm_slave: time.sleep(1.1) for d in master_invoke.top_dirs: ensure_dir_exists(d) for dlist in [master_invoke.src_dirs, master_invoke.dest_dirs]: for d in dlist: ensure_dir_exists(d) if verbose: print(host + ' saw ' + str(d)) # for each thread set up SmallfileWorkload instance, # create a thread instance, and delete the thread-ready file thread_list = create_worker_list(prm) my_host_invoke = thread_list[0].invoke # start threads, wait for them to reach starting gate # to do this, look for thread-ready files for t in thread_list: ensure_deleted(t.invoke.gen_thread_ready_fname(t.invoke.tid)) for t in thread_list: t.start() if verbose: print('started %d worker threads on host %s' % (len(thread_list), host)) # wait for all threads to reach the starting gate # this makes it more likely that they will start simultaneously startup_timeout = prm.startup_timeout if smallfile.is_windows_os: print('adding time for Windows synchronization') startup_timeout += 30 abort_fname = my_host_invoke.abort_fn() thread_count = len(thread_list) thread_to_wait_for = 0 for sec in range(0, startup_timeout * 2): for k in range(thread_to_wait_for, thread_count): t = thread_list[k] fn = t.invoke.gen_thread_ready_fname(t.invoke.tid) if not os.path.exists(fn): if verbose: print('thread %d thread-ready file %s not found...' % (k, fn)) break thread_to_wait_for = k + 1 if thread_to_wait_for == thread_count: break if os.path.exists(abort_fname): break time.sleep(0.5) # if all threads didn't make it to the starting gate if thread_to_wait_for < thread_count: abort_test(abort_fname, thread_list) raise Exception('only %d threads reached starting gate within %d sec' % (thread_to_wait_for, startup_timeout)) # declare that this host is at the starting gate if prm_slave: host_ready_fn = my_host_invoke.gen_host_ready_fname() if my_host_invoke.verbose: print('host %s creating ready file %s' % (my_host_invoke.onhost, host_ready_fn)) smallfile.touch(host_ready_fn) sg = my_host_invoke.starting_gate if not prm_slave: # special case of no --host-set parameter try: sync_files.write_sync_file(sg, 'hi there') if verbose: print('wrote starting gate file') except IOError as e: print('error writing starting gate for threads: %s' % str(e)) # wait for starting_gate file to be created by test driver # every second we resume scan from last host file not found if verbose: print('awaiting ' + sg) if prm_slave: for sec in range(0, prm.host_startup_timeout + 10): # hack to ensure that directory is up to date # ndlist = os.listdir(my_host_invoke.network_dir) # if verbose: print(str(ndlist)) if os.path.exists(sg): break time.sleep(0.5) if not os.path.exists(sg): abort_test(my_host_invoke.abort_fn(), thread_list) raise Exception('starting signal not seen within %d seconds' % prm.host_startup_timeout) if verbose: print('starting test on host ' + host + ' in 2 seconds') time.sleep(2 + random.random()) # let other hosts see starting gate file # FIXME: don't timeout the test, # instead check thread progress and abort if you see any of them stalled # but if servers are heavily loaded you can't rely on filesystem # wait for all threads on this host to finish for t in thread_list: if verbose: print('waiting for thread %s' % t.invoke.tid) t.invoke = t.receiver.recv() # to get results from sub-process t.join() # if not a slave of some other host, print results (for this host) exit_status = OK if not prm_slave: try: invoke_list = [t.invoke for t in thread_list] output_results.output_results(invoke_list, prm) except SMFResultException as e: print('ERROR: ' + str(e)) exit_status = NOTOK else: # if we are participating in a multi-host test # then write out this host's result in pickle format # so test driver can pick up result result_filename = \ master_invoke.host_result_filename(prm.as_host) if verbose: print('writing invokes to: ' + result_filename) invok_list = [t.invoke for t in thread_list] if verbose: print('saving result to filename %s' % result_filename) for ivk in invok_list: ivk.buf = None ivk.biggest_buf = None sync_files.write_pickle(result_filename, invok_list) time.sleep(1.2) # for benefit of NFS with actimeo=1 sys.exit(exit_status)
nextinv.dest_dir += "/" + master_invoke.onhost + "/d" + nextinv.tid t = invoke_process.subprocess(nextinv) thread_list.append(t) starting_gate = thread_list[0].invoke.starting_gate my_host_invoke = thread_list[0].invoke host = short_hostname(None) if not prm_slave: smallfile.ensure_deleted(starting_gate) # start threads, wait for them to reach starting gate # to do this, look for thread-ready files print "starting worker threads on host " + host smallfile.ensure_deleted(my_host_invoke.gen_host_ready_fname(host)) for t in thread_list: smallfile.ensure_dir_exists(t.invoke.src_dir) smallfile.ensure_dir_exists(t.invoke.dest_dir) ensure_deleted(t.invoke.gen_thread_ready_fname(t.invoke.tid)) time.sleep(1) for t in thread_list: t.start() # wait for all threads to reach the starting gate # this makes it more likely that they will start simultaneously threads_ready = False # really just to set scope of variable k=0 for sec in range(0, startup_timeout): threads_ready = True for t in thread_list: