Ejemplo n.º 1
0
def create_top_dirs(master_invoke, is_multi_host):
    if os.path.exists( master_invoke.network_dir ): 
      shutil.rmtree( master_invoke.network_dir )
      if is_multi_host: time.sleep(2.1) # hack to ensure that all remote clients can see that directory has been recreated
    ensure_dir_exists( master_invoke.network_dir )
    for dlist in [ master_invoke.src_dirs, master_invoke.dest_dirs ]:
      for d in dlist:
         ensure_dir_exists(d)
    if is_multi_host: 
      os.listdir(master_invoke.network_dir)  # workaround to attempt to force synchronization
      time.sleep(1.1) # hack to let NFS mount option actimeo=1 have its effect
Ejemplo n.º 2
0
def create_top_dirs(master_invoke, is_multi_host):
    if os.path.exists(master_invoke.network_dir):
        shutil.rmtree(master_invoke.network_dir)
        if is_multi_host:
            # so all remote clients see that directory was recreated
            time.sleep(2.1)
    smallfile.ensure_dir_exists(master_invoke.network_dir)
    for dlist in [master_invoke.src_dirs, master_invoke.dest_dirs]:
        for d in dlist:
            smallfile.ensure_dir_exists(d)
    if is_multi_host:
        # workaround to force cross-host synchronization
        os.listdir(master_invoke.network_dir)
        time.sleep(1.1)  # lets NFS mount option actimeo=1 take effect
Ejemplo n.º 3
0
def create_top_dirs(master_invoke, is_multi_host):
    if os.path.exists(master_invoke.network_dir):
        shutil.rmtree(master_invoke.network_dir)
        if is_multi_host:
            # so all remote clients see that directory was recreated
            time.sleep(2.1)
    smallfile.ensure_dir_exists(master_invoke.network_dir)
    for dlist in [master_invoke.src_dirs, master_invoke.dest_dirs]:
        for d in dlist:
            smallfile.ensure_dir_exists(d)
    if is_multi_host:
        # workaround to force cross-host synchronization
        os.listdir(master_invoke.network_dir)
        time.sleep(1.1)  # lets NFS mount option actimeo=1 take effect
Ejemplo n.º 4
0
def create_top_dirs(master_invoke, is_multi_host):
    if os.path.exists(master_invoke.network_dir):
        shutil.rmtree(master_invoke.network_dir)
        if is_multi_host:
            time.sleep(
                2.1
            )  # hack to ensure that all remote clients can see that directory has been recreated
    ensure_dir_exists(master_invoke.network_dir)
    for dlist in [master_invoke.src_dirs, master_invoke.dest_dirs]:
        for d in dlist:
            ensure_dir_exists(d)
    if is_multi_host:
        os.listdir(master_invoke.network_dir
                   )  # workaround to attempt to force synchronization
        time.sleep(
            1.1)  # hack to let NFS mount option actimeo=1 have its effect
Ejemplo n.º 5
0
def run_multi_thread_workload(prm):

    master_invoke = prm.master_invoke
    prm_slave = prm.is_slave
    verbose = master_invoke.verbose
    host = master_invoke.onhost

    if not prm_slave:
        sync_files.create_top_dirs(master_invoke, False)

    if prm_slave:
        time.sleep(1.1)
        for d in master_invoke.top_dirs:
            ensure_dir_exists(d)
        for dlist in [master_invoke.src_dirs, master_invoke.dest_dirs]:
            for d in dlist:
                ensure_dir_exists(d)
                if verbose:
                    print(host + ' saw ' + str(d))

    # for each thread set up SmallfileWorkload instance,
    # create a thread instance, and delete the thread-ready file

    thread_list = create_worker_list(prm)
    my_host_invoke = thread_list[0].invoke

    # start threads, wait for them to reach starting gate
    # to do this, look for thread-ready files

    for t in thread_list:
        ensure_deleted(t.invoke.gen_thread_ready_fname(t.invoke.tid))
    for t in thread_list:
        t.start()
    if verbose:
        print('started %d worker threads on host %s' %
              (len(thread_list), host))

    # wait for all threads to reach the starting gate
    # this makes it more likely that they will start simultaneously

    startup_timeout = prm.startup_timeout
    if smallfile.is_windows_os:
        print('adding time for Windows synchronization')
        startup_timeout += 30
    abort_fname = my_host_invoke.abort_fn()
    thread_count = len(thread_list)
    thread_to_wait_for = 0
    for sec in range(0, startup_timeout * 2):
        for k in range(thread_to_wait_for, thread_count):
            t = thread_list[k]
            fn = t.invoke.gen_thread_ready_fname(t.invoke.tid)
            if not os.path.exists(fn):
                if verbose:
                    print('thread %d thread-ready file %s not found...' %
                          (k, fn))
                break
            thread_to_wait_for = k + 1
        if thread_to_wait_for == thread_count:
            break
        if os.path.exists(abort_fname):
            break
        time.sleep(0.5)

    # if all threads didn't make it to the starting gate

    if thread_to_wait_for < thread_count:
        abort_test(abort_fname, thread_list)
        raise Exception('only %d threads reached starting gate within %d sec' %
                        (thread_to_wait_for, startup_timeout))

    # declare that this host is at the starting gate

    if prm_slave:
        host_ready_fn = my_host_invoke.gen_host_ready_fname()
        if my_host_invoke.verbose:
            print('host %s creating ready file %s' %
                  (my_host_invoke.onhost, host_ready_fn))
        smallfile.touch(host_ready_fn)

    sg = my_host_invoke.starting_gate
    if not prm_slave:  # special case of no --host-set parameter
        try:
            sync_files.write_sync_file(sg, 'hi there')
            if verbose:
                print('wrote starting gate file')
        except IOError as e:
            print('error writing starting gate for threads: %s' % str(e))

    # wait for starting_gate file to be created by test driver
    # every second we resume scan from last host file not found

    if verbose:
        print('awaiting ' + sg)
    if prm_slave:
        for sec in range(0, prm.host_startup_timeout + 10):
            # hack to ensure that directory is up to date
            #   ndlist = os.listdir(my_host_invoke.network_dir)
            # if verbose: print(str(ndlist))
            if os.path.exists(sg):
                break
            time.sleep(0.5)
        if not os.path.exists(sg):
            abort_test(my_host_invoke.abort_fn(), thread_list)
            raise Exception('starting signal not seen within %d seconds' %
                            prm.host_startup_timeout)
    if verbose:
        print('starting test on host ' + host + ' in 2 seconds')
    time.sleep(2 + random.random())  # let other hosts see starting gate file

    # FIXME: don't timeout the test,
    # instead check thread progress and abort if you see any of them stalled
    # but if servers are heavily loaded you can't rely on filesystem

    # wait for all threads on this host to finish

    for t in thread_list:
        if verbose:
            print('waiting for thread %s' % t.invoke.tid)
        t.invoke = t.receiver.recv()  # to get results from sub-process
        t.join()

    # if not a slave of some other host, print results (for this host)

    exit_status = OK
    if not prm_slave:
        try:
            invoke_list = [t.invoke for t in thread_list]
            output_results.output_results(invoke_list, prm)
        except SMFResultException as e:
            print('ERROR: ' + str(e))
            exit_status = NOTOK
    else:

        # if we are participating in a multi-host test
        # then write out this host's result in pickle format
        # so test driver can pick up result

        result_filename = \
            master_invoke.host_result_filename(prm.as_host)
        if verbose:
            print('writing invokes to: ' + result_filename)
        invok_list = [t.invoke for t in thread_list]
        if verbose:
            print('saving result to filename %s' % result_filename)
        for ivk in invok_list:
            ivk.buf = None
            ivk.biggest_buf = None
        sync_files.write_pickle(result_filename, invok_list)
        time.sleep(1.2)  # for benefit of NFS with actimeo=1

    sys.exit(exit_status)
Ejemplo n.º 6
0
def run_multi_thread_workload(prm):

    master_invoke = prm.master_invoke
    prm_slave = prm.is_slave
    verbose = master_invoke.verbose
    host = master_invoke.onhost

    if not prm_slave:
        sync_files.create_top_dirs(master_invoke, False)

    if prm_slave:
        time.sleep(1.1)
        for d in master_invoke.top_dirs:
            ensure_dir_exists(d)
        for dlist in [master_invoke.src_dirs, master_invoke.dest_dirs]:
            for d in dlist:
                ensure_dir_exists(d)
                if verbose:
                    print(host + ' saw ' + str(d))

    # for each thread set up SmallfileWorkload instance,
    # create a thread instance, and delete the thread-ready file

    thread_list = create_worker_list(prm)
    my_host_invoke = thread_list[0].invoke

    # start threads, wait for them to reach starting gate
    # to do this, look for thread-ready files

    for t in thread_list:
        ensure_deleted(t.invoke.gen_thread_ready_fname(t.invoke.tid))
    for t in thread_list:
        t.start()
    if verbose:
        print('started %d worker threads on host %s' %
              (len(thread_list), host))

    # wait for all threads to reach the starting gate
    # this makes it more likely that they will start simultaneously

    startup_timeout = prm.startup_timeout
    if smallfile.is_windows_os:
        print('adding time for Windows synchronization')
        startup_timeout += 30
    abort_fname = my_host_invoke.abort_fn()
    thread_count = len(thread_list)
    thread_to_wait_for = 0
    for sec in range(0, startup_timeout * 2):
        for k in range(thread_to_wait_for, thread_count):
            t = thread_list[k]
            fn = t.invoke.gen_thread_ready_fname(t.invoke.tid)
            if not os.path.exists(fn):
                if verbose:
                    print('thread %d thread-ready file %s not found...' % (k, fn))
                break
            thread_to_wait_for = k + 1
        if thread_to_wait_for == thread_count:
            break
        if os.path.exists(abort_fname):
            break
        time.sleep(0.5)

    # if all threads didn't make it to the starting gate

    if thread_to_wait_for < thread_count:
        abort_test(abort_fname, thread_list)
        raise Exception('only %d threads reached starting gate within %d sec'
                        % (thread_to_wait_for, startup_timeout))

    # declare that this host is at the starting gate

    if prm_slave:
        host_ready_fn = my_host_invoke.gen_host_ready_fname()
        if my_host_invoke.verbose:
            print('host %s creating ready file %s' %
                  (my_host_invoke.onhost, host_ready_fn))
        smallfile.touch(host_ready_fn)

    sg = my_host_invoke.starting_gate
    if not prm_slave:  # special case of no --host-set parameter
        try:
            sync_files.write_sync_file(sg, 'hi there')
            if verbose:
                print('wrote starting gate file')
        except IOError as e:
            print('error writing starting gate for threads: %s' % str(e))

    # wait for starting_gate file to be created by test driver
    # every second we resume scan from last host file not found

    if verbose:
        print('awaiting ' + sg)
    if prm_slave:
        for sec in range(0, prm.host_startup_timeout + 10):
            # hack to ensure that directory is up to date
            #   ndlist = os.listdir(my_host_invoke.network_dir)
            # if verbose: print(str(ndlist))
            if os.path.exists(sg):
                break
            time.sleep(0.5)
        if not os.path.exists(sg):
            abort_test(my_host_invoke.abort_fn(), thread_list)
            raise Exception('starting signal not seen within %d seconds'
                            % prm.host_startup_timeout)
    if verbose:
        print('starting test on host ' + host + ' in 2 seconds')
    time.sleep(2 + random.random())  # let other hosts see starting gate file

    # FIXME: don't timeout the test,
    # instead check thread progress and abort if you see any of them stalled
    # but if servers are heavily loaded you can't rely on filesystem

    # wait for all threads on this host to finish

    for t in thread_list:
        if verbose:
            print('waiting for thread %s' % t.invoke.tid)
        t.invoke = t.receiver.recv()  # to get results from sub-process
        t.join()

    # if not a slave of some other host, print results (for this host)

    exit_status = OK
    if not prm_slave:
        try:
            invoke_list = [t.invoke for t in thread_list]
            output_results.output_results(invoke_list, prm)
        except SMFResultException as e:
            print('ERROR: ' + str(e))
            exit_status = NOTOK
    else:

        # if we are participating in a multi-host test
        # then write out this host's result in pickle format
        # so test driver can pick up result

        result_filename = \
            master_invoke.host_result_filename(prm.as_host)
        if verbose:
            print('writing invokes to: ' + result_filename)
        invok_list = [t.invoke for t in thread_list]
        if verbose:
            print('saving result to filename %s' % result_filename)
        for ivk in invok_list:
            ivk.buf = None
            ivk.biggest_buf = None
        sync_files.write_pickle(result_filename, invok_list)
        time.sleep(1.2)  # for benefit of NFS with actimeo=1

    sys.exit(exit_status)
Ejemplo n.º 7
0
        nextinv.dest_dir += "/" + master_invoke.onhost + "/d" + nextinv.tid
    t = invoke_process.subprocess(nextinv)
    thread_list.append(t)

starting_gate = thread_list[0].invoke.starting_gate
my_host_invoke = thread_list[0].invoke
host = short_hostname(None)
if not prm_slave: smallfile.ensure_deleted(starting_gate)

# start threads, wait for them to reach starting gate
# to do this, look for thread-ready files 

print "starting worker threads on host " + host
smallfile.ensure_deleted(my_host_invoke.gen_host_ready_fname(host))
for t in thread_list:
    smallfile.ensure_dir_exists(t.invoke.src_dir)
    smallfile.ensure_dir_exists(t.invoke.dest_dir)
    ensure_deleted(t.invoke.gen_thread_ready_fname(t.invoke.tid))
time.sleep(1)

for t in thread_list:
    t.start()

# wait for all threads to reach the starting gate
# this makes it more likely that they will start simultaneously

threads_ready = False  # really just to set scope of variable
k=0
for sec in range(0, startup_timeout):
    threads_ready = True
    for t in thread_list: