def test1(self): """ Test that the thether_task is working. We run the mock_tether_parent in a separate subprocess """ from avro import tether from avro import io as avio from avro import schema from avro.tether import HTTPRequestor, inputProtocol, find_port import StringIO import mock_tether_parent from word_count_task import WordCountTask task = WordCountTask() proc = None try: # launch the server in a separate process # env["AVRO_TETHER_OUTPUT_PORT"]=output_port env = dict() env["PYTHONPATH"] = ':'.join(sys.path) server_port = find_port() pyfile = mock_tether_parent.__file__ proc = subprocess.Popen( ["python", pyfile, "start_server", "{0}".format(server_port)]) input_port = find_port() print "Mock server started process pid={0}".format(proc.pid) # Possible race condition? open tries to connect to the subprocess before the subprocess is fully started # so we give the subprocess time to start up time.sleep(1) task.open(input_port, clientPort=server_port) # TODO: We should validate that open worked by grabbing the STDOUT of the subproces # and ensuring that it outputted the correct message. #*************************************************************** # Test the mapper task.configure(tether.TaskType.MAP, str(task.inschema), str(task.midschema)) # Serialize some data so we can send it to the input function datum = "This is a line of text" writer = StringIO.StringIO() encoder = avio.BinaryEncoder(writer) datum_writer = avio.DatumWriter(task.inschema) datum_writer.write(datum, encoder) writer.seek(0) data = writer.read() # Call input to simulate calling map task.input(data, 1) # Test the reducer task.configure(tether.TaskType.REDUCE, str(task.midschema), str(task.outschema)) # Serialize some data so we can send it to the input function datum = {"key": "word", "value": 2} writer = StringIO.StringIO() encoder = avio.BinaryEncoder(writer) datum_writer = avio.DatumWriter(task.midschema) datum_writer.write(datum, encoder) writer.seek(0) data = writer.read() # Call input to simulate calling reduce task.input(data, 1) task.complete() # try a status task.status("Status message") except Exception as e: raise finally: # close the process if not (proc is None): proc.kill() pass
def test1(self): from word_count_task import WordCountTask from avro.tether import TaskRunner, find_port, HTTPRequestor, inputProtocol, TaskType from avro import io as avio import mock_tether_parent import subprocess import StringIO import logging # set the logging level to debug so that debug messages are printed logging.basicConfig(level=logging.DEBUG) proc = None try: # launch the server in a separate process env = dict() env["PYTHONPATH"] = ':'.join(sys.path) parent_port = find_port() pyfile = mock_tether_parent.__file__ proc = subprocess.Popen( ["python", pyfile, "start_server", "{0}".format(parent_port)]) input_port = find_port() print "Mock server started process pid={0}".format(proc.pid) # Possible race condition? open tries to connect to the subprocess before the subprocess is fully started # so we give the subprocess time to start up time.sleep(1) runner = TaskRunner(WordCountTask()) runner.start(outputport=parent_port, join=False) # Test sending various messages to the server and ensuring they are # processed correctly requestor = HTTPRequestor("localhost", runner.server.server_address[1], inputProtocol) # TODO: We should validate that open worked by grabbing the STDOUT of the subproces # and ensuring that it outputted the correct message. # Test the mapper requestor.request( "configure", { "taskType": TaskType.MAP, "inSchema": str(runner.task.inschema), "outSchema": str(runner.task.midschema) }) # Serialize some data so we can send it to the input function datum = "This is a line of text" writer = StringIO.StringIO() encoder = avio.BinaryEncoder(writer) datum_writer = avio.DatumWriter(runner.task.inschema) datum_writer.write(datum, encoder) writer.seek(0) data = writer.read() # Call input to simulate calling map requestor.request("input", {"data": data, "count": 1}) #Test the reducer requestor.request( "configure", { "taskType": TaskType.REDUCE, "inSchema": str(runner.task.midschema), "outSchema": str(runner.task.outschema) }) #Serialize some data so we can send it to the input function datum = {"key": "word", "value": 2} writer = StringIO.StringIO() encoder = avio.BinaryEncoder(writer) datum_writer = avio.DatumWriter(runner.task.midschema) datum_writer.write(datum, encoder) writer.seek(0) data = writer.read() #Call input to simulate calling reduce requestor.request("input", {"data": data, "count": 1}) requestor.request("complete", {}) runner.task.ready_for_shutdown.wait() runner.server.shutdown() #time.sleep(2) #runner.server.shutdown() sthread = runner.sthread #Possible race condition? time.sleep(1) #make sure the other thread terminated self.assertFalse(sthread.isAlive()) #shutdown the logging logging.shutdown() except Exception as e: raise finally: #close the process if not (proc is None): proc.kill()
def test1(self): """ Test that the thether_task is working. We run the mock_tether_parent in a separate subprocess """ from avro import tether from avro import io as avio from avro import schema from avro.tether import HTTPRequestor,inputProtocol, find_port import StringIO import mock_tether_parent from word_count_task import WordCountTask task=WordCountTask() proc=None try: # launch the server in a separate process # env["AVRO_TETHER_OUTPUT_PORT"]=output_port env=dict() env["PYTHONPATH"]=':'.join(sys.path) server_port=find_port() pyfile=mock_tether_parent.__file__ proc=subprocess.Popen(["python", pyfile,"start_server","{0}".format(server_port)]) input_port=find_port() print "Mock server started process pid={0}".format(proc.pid) # Possible race condition? open tries to connect to the subprocess before the subprocess is fully started # so we give the subprocess time to start up time.sleep(1) task.open(input_port,clientPort=server_port) # TODO: We should validate that open worked by grabbing the STDOUT of the subproces # and ensuring that it outputted the correct message. #*************************************************************** # Test the mapper task.configure(tether.TaskType.MAP,str(task.inschema),str(task.midschema)) # Serialize some data so we can send it to the input function datum="This is a line of text" writer = StringIO.StringIO() encoder = avio.BinaryEncoder(writer) datum_writer = avio.DatumWriter(task.inschema) datum_writer.write(datum, encoder) writer.seek(0) data=writer.read() # Call input to simulate calling map task.input(data,1) # Test the reducer task.configure(tether.TaskType.REDUCE,str(task.midschema),str(task.outschema)) # Serialize some data so we can send it to the input function datum={"key":"word","value":2} writer = StringIO.StringIO() encoder = avio.BinaryEncoder(writer) datum_writer = avio.DatumWriter(task.midschema) datum_writer.write(datum, encoder) writer.seek(0) data=writer.read() # Call input to simulate calling reduce task.input(data,1) task.complete() # try a status task.status("Status message") except Exception as e: raise finally: # close the process if not(proc is None): proc.kill() pass