def inner(srv, run_id, accum):
script = 'runner.py'
root = getRoot(accum)
parameters = {
'root': getRoot(accum),
'fortune' : fortune,
'program': 'progs/Millionaires.txt',
'run_id' : run_id,
'player': player}
if host_slavetag:
host = getHosts(accum).get(host_slavetag)
parameters['addr_of_host'] = host
return execute_injected_python_script(srv, script, root, parameters)
def inner(srv, run_id, accum):
root = getRoot(accum)
path = rjoin(root, inject_folder_name, "run")
cmd = "cd %s; chmod -R 777 *; ./%s -c %s" % (path, command, program)
logger.info("execute: " + cmd)
print_remote_out(srv.execute(cmd))
def execute_runner_1(srv, run_id, accum):
''' helper method for hierarchy 1 '''
script = 'runner.py'
root = getRoot(accum)
parameters = { 'tag' : getTag(accum), 'run_id': run_id}
return execute_injected_python_script(srv, script, root, parameters)
def assertDataFromSlave1(srv, id, accum):
'''
assert the structure and content of slave1's data,
this includes both accum and the results file
'''
_assert('assert fetched results', id)
tag = getTag(accum) # assert on accum from here
root = getRoot(accum)
hosts = getHosts(accum)
_assert(len(hosts), 4)
_assert('test_framework/slave1', root)
_assert('slave1', tag)
with open(local_results_file_path_F(tag), 'rb') as f:
results = json.load(f)
_assert_contain('result_0', results) # assert on results file from here
result_0 = results.get('result_0')
_assert_contain( 'tmsp', result_0)
_assert_contain( 'parameters', result_0)
_assert_contain( 'query', result_0.get('parameters'))
_assert_contain( 'Oryx and Crake', result_0.get('parameters').get('query'))
_assert_contain( 'result', result_0)
_assert(len(result_0.get('result')), 4)
def assertDataFromSlave4(srv, id, accum):
'''
assert the structure and content of slave4's data,
this includes both accum and the results file
'''
_assert('assert fetched results', id) # assert on accum from here
tag = getTag(accum)
root = getRoot(accum)
hosts = getHosts(accum)
_assert(len(hosts), 4)
_assert('test_framework/slave4', root)
_assert('slave4', tag)
with open(local_results_file_path_F(tag), 'rb') as f:
results = json.load(f) # assert on results file here
_assert_contain('result_0', results)
result_0 = results.get('result_0')
_assert_contain( 'tmsp', result_0)
_assert_contain( 'parameters', result_0)
_assert_contain( 'The Jabberwocky', result_0.get('parameters').get('Dragon_no_1'))
_assert_contain( 'The Tarasque', result_0.get('parameters').get('Dragon_no_2'))
_assert_contain( 'result', result_0)
_assert(result_0.get('result'), 42)
def inner(srv, run_id, accum):
root = getRoot(accum)
script = script_ # seems necessary for maintaining script in scope
if not script.endswith('.java'): script += '.java'
path = rjoin(root, inject_folder_name)
cmd = 'javac -cp %s/*.jar %s' % (path, rjoin(path, script))
logger.info('execute: ' + cmd)
print_remote_out( srv.execute(cmd))
def collect_context_stat(srv, run_id, accum):
''' Helper method for hierarchy 3 '''
hosts = getHosts(accum)
me = getTag(accum)
tag2hosts = {}
for tag, host in hosts.iteritems():
if tag != me:
tag2hosts[tag] = host
script = 'runner.py'
root = getRoot(accum)
parameters = {'tag2hosts':tag2hosts, 'run_id': run_id}
execute_injected_python_script(srv, script, root, parameters)
def assertDataFromSlave2(srv, id, accum):
'''
assert the structure and content of slave2's data,
this includes both accum and the results file
'''
_assert('assert fetched results', id) # assert on accum from here
tag = getTag(accum)
root = getRoot(accum)
hosts = getHosts(accum)
_assert(len(hosts), 4)
_assert('test_framework/slave2', root)
_assert('slave2', tag)
assert os.path.exists(local_results_file_path_F(tag)) == False, \
'slave 2 should not have generated any results' # assert on results file here
def inner(srv, run_id, accum):
root = getRoot(accum)
logger.info("bob on host: " + getMetadata(accum).get("host"))
command = "run_bob"
path = rjoin(root, inject_folder_name, "run")
cmd = r"cd %s; echo %d | ./%s -r %s abcdefghijk 4" % (path, fortune, command, program)
logger.info("execute: " + cmd)
t1 = clock()
(stdout, stderr) = srv.execute(cmd)
t2 = clock()
print_remote_out((None, stderr))
logger.info("".join(stdout).replace("\n", ". "))
logger.info("execution time took %d seconds" % (t2 - t1))
def put_my_init_file(srv, run_id, accum): my_tag = getTag(accum) root = getRoot(accum) lpath = ljoin(ctx.local_output_dir, my_tag+'.ini') put(srv, lpath, rjoin(root, 'inject'))
def let_my_millionaire_participate(srv, run_id, accum):
tag = getTag(accum)
root = getRoot(accum)
params = '--no-ssl {my_initfile}'.format(my_initfile=(rjoin(root,'inject', tag+'.ini')))
execute_injected_python_script(srv, 'millionaires', root, params)
def inner(srv, run_id, accum):
root = getRoot(accum)
path = rjoin(root, inject_folder_name)
cmd = 'java -cp %s/.:%s/* %s %s' % (path, path, script, remote_results_dir_F(root))
logger.info('execute: ' + cmd)
print_remote_out( srv.execute(cmd) )
# if none is specified, a unique temporary folder is created
{
slave1 : ('setup script 1.', deploy(rjoin(ipath, 'slave_scripts', 'slave_script_1.py'), 'test_framework/slave1')),
slave2 : ('setup script 2.', deploy(rjoin(ipath, 'slave_scripts', 'slave_script_2.py'), 'test_framework/slave2')),
slave3 : ('setup script 2. again ', deploy(rjoin(ipath, 'slave_scripts', 'slave_script_2.py'), 'test_framework/slave3')),
slave4 : ('setup folder with script 3, 4, 5', deploy(rjoin(ipath, 'some_other_slave_scripts'), 'test_framework/slave4')),
},
# tier 4: call a script and see how the logger handles remote prints
# call slave_script 3: which simply prints to stdout what parameters is gets
# this is an example of directly executing a remote script.
# note the necessity of supplying a function which takes srv, run_id and accum
# note also how to get the name of the root folder (as it might be generated on runtime, we need this)
# should simply be evident in the log output
{
slave4 : ('echo', lambda srv, run_id, accum :
execute_injected_python_script(srv, 'slave_script_3.py', getRoot(accum),
{'Sherlock Holmes':'Arthur Conan Doyle', 'Conan the barbarian': 'Robert E. Howard'})),
slave1 : ('google search', lambda srv, run_id, accum :
execute_injected_python_script(srv, 'slave_script_1.py', getRoot(accum),
{'query':'Oryx and Crake'})),
},
# tier 5: call a script and see how remote results can be saved for later retrival
# which simply uses another script to generate a random number
# which is saved as a result i output/results/myresults along with benchmarking data
{
slave4 : ('guess a number', lambda srv, a, accum :
execute_injected_python_script(srv, 'slave_script_4.py', getRoot(accum),
{'Dragon' : 'The Jabberwocky', 'Dragon' : 'The Tarasque'})),
},
# tier 6: see how accum works by accumulating results from locally executed scripts
def inner(srv, run_id, accum):
script = 'runner.py'
root = getRoot(accum)
parameters = { 'x': x,
'timeout': 300}
return execute_injected_python_script(srv, script, root, parameters)
def calculate(srv, run_id, accum):
root = getRoot(accum)
tag = getTag(accum)
params = "--no-ssl {my_initfile}".format(my_initfile=rjoin(root, "inject", tag + ".ini"))
execute_injected_python_script(srv, "benchmark.py", root, params)
slave3 : ('hello world', lambda srv, run_id, accum: logger.info('hello world from %s' % run_id)),
slave4 : ('hello world', lambda srv, run_id, accum: logger.info('hello world from %s' % run_id)), },
{
slave1 : ('purge remote', lambda srv, run_id, accum: remove_remote_location(srv, 'test_framework')),
slave2 : ('purge remote', lambda srv, run_id, accum: remove_remote_location(srv, 'test_framework')),
slave3 : ('purge remote', lambda srv, run_id, accum: remove_remote_location(srv, 'test_framework')),
slave4 : ('purge remote', lambda srv, run_id, accum: remove_remote_location(srv, 'test_framework'))
},
{ slave1 : ('setup script 1.', deploy(rjoin(ipath, 'slave_scripts', 'slave_script_1.py'), 'test_framework/slave1')),
slave2 : ('setup script 2.', deploy(rjoin(ipath, 'slave_scripts', 'slave_script_2.py'), 'test_framework/slave2')),
slave3 : ('setup script 2. again ', deploy(rjoin(ipath, 'slave_scripts', 'slave_script_2.py'), 'test_framework/slave3')),
slave4 : ('setup folder with script 3, 4, 5', deploy(rjoin(ipath, 'some_other_slave_scripts'), 'test_framework/slave4')), },
{ slave4 : ('echo', lambda srv, run_id, accum :
execute_injected_python_script(srv, 'slave_script_3.py', getRoot(accum),
{'Sherlock Holmes':'Arthur Conan Doyle', 'Conan the barbarian': 'Robert E. Howard'})),
slave1 : ('google search', lambda srv, run_id, accum :
execute_injected_python_script(srv, 'slave_script_1.py', getRoot(accum),
{'query':'Oryx and Crake'})), },
{ slave4 : ('guess a number', lambda srv, a, accum :
execute_injected_python_script(srv, 'slave_script_4.py', getRoot(accum),
{'Dragon_no_1' : 'The Jabberwocky', 'Dragon_no_2' : 'The Tarasque'})), },
{ slave3: ('produce local results', lambda a, b, c: ('some local results', [42,43,44,45])), },
{ slave1 : ('fetch results and cleanup remote', fetch_and_teardown),
slave2 : ('fetch results and cleanup remote', fetch_and_teardown),
slave3 : ('fetch results and cleanup remote', fetch_and_teardown),
