def test2(self):
"""
In this test we want to make sure that when we run "tether_task_runner.py"
as our main script everything works as expected. We do this by using subprocess to run it
in a separate thread.
"""
from word_count_task import WordCountTask
from avro.tether import TaskRunner, find_port,HTTPRequestor,inputProtocol, TaskType
from avro.tether import tether_task_runner
from avro import io as avio
import mock_tether_parent
import subprocess
import StringIO
proc=None
runnerproc=None
try:
#launch the server in a separate process
env=dict()
env["PYTHONPATH"]=':'.join(sys.path)
parent_port=find_port()
pyfile=mock_tether_parent.__file__
proc=subprocess.Popen(["python", pyfile,"start_server","{0}".format(parent_port)])
#Possible race condition? when we start tether_task_runner it will call
# open tries to connect to the subprocess before the subprocess is fully started
#so we give the subprocess time to start up
time.sleep(1)
#start the tether_task_runner in a separate process
env={"AVRO_TETHER_OUTPUT_PORT":"{0}".format(parent_port)}
env["PYTHONPATH"]=':'.join(sys.path)
runnerproc=subprocess.Popen(["python",tether_task_runner.__file__,"word_count_task.WordCountTask"],env=env)
#possible race condition wait for the process to start
time.sleep(1)
print "Mock server started process pid={0}".format(proc.pid)
#Possible race condition? open tries to connect to the subprocess before the subprocess is fully started
#so we give the subprocess time to start up
time.sleep(1)
except Exception as e:
raise
finally:
#close the process
if not(runnerproc is None):
runnerproc.kill()
if not(proc is None):
proc.kill()
def test2(self):
"""
In this test we want to make sure that when we run "tether_task_runner.py"
as our main script everything works as expected. We do this by using subprocess to run it
in a separate thread.
"""
from word_count_task import WordCountTask
from avro.tether import TaskRunner, find_port, HTTPRequestor, inputProtocol, TaskType
from avro.tether import tether_task_runner
from avro import io as avio
import mock_tether_parent
import subprocess
import StringIO
proc = None
runnerproc = None
try:
#launch the server in a separate process
env = dict()
env["PYTHONPATH"] = ':'.join(sys.path)
parent_port = find_port()
pyfile = mock_tether_parent.__file__
proc = subprocess.Popen(
["python", pyfile, "start_server", "{0}".format(parent_port)])
#Possible race condition? when we start tether_task_runner it will call
# open tries to connect to the subprocess before the subprocess is fully started
#so we give the subprocess time to start up
time.sleep(1)
#start the tether_task_runner in a separate process
env = {"AVRO_TETHER_OUTPUT_PORT": "{0}".format(parent_port)}
env["PYTHONPATH"] = ':'.join(sys.path)
runnerproc = subprocess.Popen([
"python", tether_task_runner.__file__,
"word_count_task.WordCountTask"
],
env=env)
#possible race condition wait for the process to start
time.sleep(1)
print "Mock server started process pid={0}".format(proc.pid)
#Possible race condition? open tries to connect to the subprocess before the subprocess is fully started
#so we give the subprocess time to start up
time.sleep(1)
except Exception as e:
raise
finally:
#close the process
if not (runnerproc is None):
runnerproc.kill()
if not (proc is None):
proc.kill()
def start(self, outputport=None, join=True):
"""
Start the server
Parameters
-------------------------------------------------------------------
outputport - (optional) The port on which the parent process is listening
for requests from the task.
- This will typically be supplied by an environment variable
we allow it to be supplied as an argument mainly for debugging
join - (optional) If set to fault then we don't issue a join to block
until the thread excecuting the server terminates.
This is mainly for debugging. By setting it to false,
we can resume execution in this thread so that we can do additional
testing
"""
port = find_port()
address = ("localhost", port)
def thread_run(task_runner=None):
task_runner.server = HTTPServer(address,
HTTPHandlerGen(task_runner))
task_runner.server.allow_reuse_address = True
task_runner.server.serve_forever()
# create a separate thread for the http server
sthread = threading.Thread(target=thread_run,
kwargs={"task_runner": self})
sthread.start()
self.sthread = sthread
# This needs to run in a separat thread b\c serve_forever() blocks
self.task.open(port, clientPort=outputport)
# wait for the other thread to finish
if (join):
self.task.ready_for_shutdown.wait()
self.server.shutdown()
# should we do some kind of check to make sure it exits
self.log.info("Shutdown the logger")
# shutdown the logging
logging.shutdown()
def start(self,outputport=None,join=True):
"""
Start the server
Parameters
-------------------------------------------------------------------
outputport - (optional) The port on which the parent process is listening
for requests from the task.
- This will typically be supplied by an environment variable
we allow it to be supplied as an argument mainly for debugging
join - (optional) If set to fault then we don't issue a join to block
until the thread excecuting the server terminates.
This is mainly for debugging. By setting it to false,
we can resume execution in this thread so that we can do additional
testing
"""
port=find_port()
address=("localhost",port)
def thread_run(task_runner=None):
task_runner.server = HTTPServer(address, HTTPHandlerGen(task_runner))
task_runner.server.allow_reuse_address = True
task_runner.server.serve_forever()
# create a separate thread for the http server
sthread=threading.Thread(target=thread_run,kwargs={"task_runner":self})
sthread.start()
self.sthread=sthread
# This needs to run in a separat thread b\c serve_forever() blocks
self.task.open(port,clientPort=outputport)
# wait for the other thread to finish
if (join):
self.task.ready_for_shutdown.wait()
self.server.shutdown()
# should we do some kind of check to make sure it exits
self.log.info("Shutdown the logger")
# shutdown the logging
logging.shutdown()
def test1(self):
from word_count_task import WordCountTask
from avro.tether import TaskRunner, find_port, HTTPRequestor, inputProtocol, TaskType
from avro import io as avio
import mock_tether_parent
import subprocess
import StringIO
import logging
# set the logging level to debug so that debug messages are printed
logging.basicConfig(level=logging.DEBUG)
proc = None
try:
# launch the server in a separate process
env = dict()
env["PYTHONPATH"] = ':'.join(sys.path)
parent_port = find_port()
pyfile = mock_tether_parent.__file__
proc = subprocess.Popen(
["python", pyfile, "start_server", "{0}".format(parent_port)])
input_port = find_port()
print "Mock server started process pid={0}".format(proc.pid)
# Possible race condition? open tries to connect to the subprocess before the subprocess is fully started
# so we give the subprocess time to start up
time.sleep(1)
runner = TaskRunner(WordCountTask())
runner.start(outputport=parent_port, join=False)
# Test sending various messages to the server and ensuring they are
# processed correctly
requestor = HTTPRequestor("localhost",
runner.server.server_address[1],
inputProtocol)
# TODO: We should validate that open worked by grabbing the STDOUT of the subproces
# and ensuring that it outputted the correct message.
# Test the mapper
requestor.request(
"configure", {
"taskType": TaskType.MAP,
"inSchema": str(runner.task.inschema),
"outSchema": str(runner.task.midschema)
})
# Serialize some data so we can send it to the input function
datum = "This is a line of text"
writer = StringIO.StringIO()
encoder = avio.BinaryEncoder(writer)
datum_writer = avio.DatumWriter(runner.task.inschema)
datum_writer.write(datum, encoder)
writer.seek(0)
data = writer.read()
# Call input to simulate calling map
requestor.request("input", {"data": data, "count": 1})
#Test the reducer
requestor.request(
"configure", {
"taskType": TaskType.REDUCE,
"inSchema": str(runner.task.midschema),
"outSchema": str(runner.task.outschema)
})
#Serialize some data so we can send it to the input function
datum = {"key": "word", "value": 2}
writer = StringIO.StringIO()
encoder = avio.BinaryEncoder(writer)
datum_writer = avio.DatumWriter(runner.task.midschema)
datum_writer.write(datum, encoder)
writer.seek(0)
data = writer.read()
#Call input to simulate calling reduce
requestor.request("input", {"data": data, "count": 1})
requestor.request("complete", {})
runner.task.ready_for_shutdown.wait()
runner.server.shutdown()
#time.sleep(2)
#runner.server.shutdown()
sthread = runner.sthread
#Possible race condition?
time.sleep(1)
#make sure the other thread terminated
self.assertFalse(sthread.isAlive())
#shutdown the logging
logging.shutdown()
except Exception as e:
raise
finally:
#close the process
if not (proc is None):
proc.kill()
def test1(self):
"""
Test that the thether_task is working. We run the mock_tether_parent in a separate
subprocess
"""
from avro import tether
from avro import io as avio
from avro import schema
from avro.tether import HTTPRequestor, inputProtocol, find_port
import StringIO
import mock_tether_parent
from word_count_task import WordCountTask
task = WordCountTask()
proc = None
try:
# launch the server in a separate process
# env["AVRO_TETHER_OUTPUT_PORT"]=output_port
env = dict()
env["PYTHONPATH"] = ':'.join(sys.path)
server_port = find_port()
pyfile = mock_tether_parent.__file__
proc = subprocess.Popen(
["python", pyfile, "start_server", "{0}".format(server_port)])
input_port = find_port()
print "Mock server started process pid={0}".format(proc.pid)
# Possible race condition? open tries to connect to the subprocess before the subprocess is fully started
# so we give the subprocess time to start up
time.sleep(1)
task.open(input_port, clientPort=server_port)
# TODO: We should validate that open worked by grabbing the STDOUT of the subproces
# and ensuring that it outputted the correct message.
#***************************************************************
# Test the mapper
task.configure(tether.TaskType.MAP, str(task.inschema),
str(task.midschema))
# Serialize some data so we can send it to the input function
datum = "This is a line of text"
writer = StringIO.StringIO()
encoder = avio.BinaryEncoder(writer)
datum_writer = avio.DatumWriter(task.inschema)
datum_writer.write(datum, encoder)
writer.seek(0)
data = writer.read()
# Call input to simulate calling map
task.input(data, 1)
# Test the reducer
task.configure(tether.TaskType.REDUCE, str(task.midschema),
str(task.outschema))
# Serialize some data so we can send it to the input function
datum = {"key": "word", "value": 2}
writer = StringIO.StringIO()
encoder = avio.BinaryEncoder(writer)
datum_writer = avio.DatumWriter(task.midschema)
datum_writer.write(datum, encoder)
writer.seek(0)
data = writer.read()
# Call input to simulate calling reduce
task.input(data, 1)
task.complete()
# try a status
task.status("Status message")
except Exception as e:
raise
finally:
# close the process
if not (proc is None):
proc.kill()
pass
def test1(self):
"""
Test that the thether_task is working. We run the mock_tether_parent in a separate
subprocess
"""
from avro import tether
from avro import io as avio
from avro import schema
from avro.tether import HTTPRequestor,inputProtocol, find_port
import StringIO
import mock_tether_parent
from word_count_task import WordCountTask
task=WordCountTask()
proc=None
try:
# launch the server in a separate process
# env["AVRO_TETHER_OUTPUT_PORT"]=output_port
env=dict()
env["PYTHONPATH"]=':'.join(sys.path)
server_port=find_port()
pyfile=mock_tether_parent.__file__
proc=subprocess.Popen(["python", pyfile,"start_server","{0}".format(server_port)])
input_port=find_port()
print "Mock server started process pid={0}".format(proc.pid)
# Possible race condition? open tries to connect to the subprocess before the subprocess is fully started
# so we give the subprocess time to start up
time.sleep(1)
task.open(input_port,clientPort=server_port)
# TODO: We should validate that open worked by grabbing the STDOUT of the subproces
# and ensuring that it outputted the correct message.
#***************************************************************
# Test the mapper
task.configure(tether.TaskType.MAP,str(task.inschema),str(task.midschema))
# Serialize some data so we can send it to the input function
datum="This is a line of text"
writer = StringIO.StringIO()
encoder = avio.BinaryEncoder(writer)
datum_writer = avio.DatumWriter(task.inschema)
datum_writer.write(datum, encoder)
writer.seek(0)
data=writer.read()
# Call input to simulate calling map
task.input(data,1)
# Test the reducer
task.configure(tether.TaskType.REDUCE,str(task.midschema),str(task.outschema))
# Serialize some data so we can send it to the input function
datum={"key":"word","value":2}
writer = StringIO.StringIO()
encoder = avio.BinaryEncoder(writer)
datum_writer = avio.DatumWriter(task.midschema)
datum_writer.write(datum, encoder)
writer.seek(0)
data=writer.read()
# Call input to simulate calling reduce
task.input(data,1)
task.complete()
# try a status
task.status("Status message")
except Exception as e:
raise
finally:
# close the process
if not(proc is None):
proc.kill()
pass
def test1(self):
from word_count_task import WordCountTask
from avro.tether import TaskRunner, find_port,HTTPRequestor,inputProtocol, TaskType
from avro import io as avio
import mock_tether_parent
import subprocess
import StringIO
import logging
# set the logging level to debug so that debug messages are printed
logging.basicConfig(level=logging.DEBUG)
proc=None
try:
# launch the server in a separate process
env=dict()
env["PYTHONPATH"]=':'.join(sys.path)
parent_port=find_port()
pyfile=mock_tether_parent.__file__
proc=subprocess.Popen(["python", pyfile,"start_server","{0}".format(parent_port)])
input_port=find_port()
print "Mock server started process pid={0}".format(proc.pid)
# Possible race condition? open tries to connect to the subprocess before the subprocess is fully started
# so we give the subprocess time to start up
time.sleep(1)
runner=TaskRunner(WordCountTask())
runner.start(outputport=parent_port,join=False)
# Wait for the server to start
time.sleep(1)
# Test sending various messages to the server and ensuring they are
# processed correctly
requestor=HTTPRequestor("localhost",runner.server.server_address[1],inputProtocol)
# TODO: We should validate that open worked by grabbing the STDOUT of the subproces
# and ensuring that it outputted the correct message.
# Test the mapper
requestor.request("configure",{"taskType":TaskType.MAP,"inSchema":str(runner.task.inschema),"outSchema":str(runner.task.midschema)})
# Serialize some data so we can send it to the input function
datum="This is a line of text"
writer = StringIO.StringIO()
encoder = avio.BinaryEncoder(writer)
datum_writer = avio.DatumWriter(runner.task.inschema)
datum_writer.write(datum, encoder)
writer.seek(0)
data=writer.read()
# Call input to simulate calling map
requestor.request("input",{"data":data,"count":1})
#Test the reducer
requestor.request("configure",{"taskType":TaskType.REDUCE,"inSchema":str(runner.task.midschema),"outSchema":str(runner.task.outschema)})
#Serialize some data so we can send it to the input function
datum={"key":"word","value":2}
writer = StringIO.StringIO()
encoder = avio.BinaryEncoder(writer)
datum_writer = avio.DatumWriter(runner.task.midschema)
datum_writer.write(datum, encoder)
writer.seek(0)
data=writer.read()
#Call input to simulate calling reduce
requestor.request("input",{"data":data,"count":1})
requestor.request("complete",{})
runner.task.ready_for_shutdown.wait()
runner.server.shutdown()
#time.sleep(2)
#runner.server.shutdown()
sthread=runner.sthread
#Possible race condition?
time.sleep(1)
#make sure the other thread terminated
self.assertFalse(sthread.isAlive())
#shutdown the logging
logging.shutdown()
except Exception as e:
raise
finally:
#close the process
if not(proc is None):
proc.kill()