def test_subprocess_controller(self): """ Controller process runs two workers and collects their output. """ logger = 'spreadflow_core.scripts.spreadflow_twistd.StderrLogger' config = os.path.join(FIXTURE_DIRECTORY, 'spreadflow-partitions.conf') with fixtures.TempDir() as fix: rundir = fix.path # FIXME: subprocess controller currently does not propagate the # configuration file path to its child processess. shutil.copy(config, os.path.join(rundir, 'spreadflow.conf')) pidfile = os.path.join(rundir, 'twistd.pid') argv = ['-n', '-d', rundir, '--logger', logger] argv += ['--pidfile', pidfile, '--multiprocess'] proc = subprocess.Popen(['spreadflow-twistd'] + argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stream_data = {proc.stdout: b'', proc.stderr: b''} reader = StreamsReader([proc.stdout, proc.stderr]) reader.start() marker = b'[spreadflow_core.proc.DebugLog#debug] ' \ b'Item received: hello world' for stream, data in reader.drain(): stream_data[stream] += data if marker in stream_data[proc.stderr]: break else: self.fail('Worker process is expected to emit a message to stderr{0}'.format(self._format_stream(stream_data[proc.stdout], stream_data[proc.stderr]))) # Send SIGTERM to controller. proc.terminate() for stream, data in reader.drain(0): stream_data[stream] += data proc.wait() self.assertEqual(proc.returncode, 0, self._format_stream(stream_data[proc.stdout], stream_data[proc.stderr])) for stream, data in reader.drain(0): stream_data[stream] += data reader.join()
def test_subprocess_worker_consumer(self): """ Worker process reads messages from stdin and writes results to stdout. """ logger = 'spreadflow_core.scripts.spreadflow_twistd.StderrLogger' config = os.path.join(FIXTURE_DIRECTORY, 'spreadflow-partitions.conf') with fixtures.TempDir() as fix: rundir = fix.path pidfile = os.path.join(rundir, 'twistd.pid') argv = ['-n', '-d', rundir, '-c', config, '--logger', logger] argv += ['--pidfile', pidfile, '--multiprocess', '--partition'] argv += ['consumer'] proc = subprocess.Popen(['spreadflow-twistd'] + argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stream_data = {proc.stdout: b'', proc.stderr: b''} reader = StreamsReader([proc.stdout, proc.stderr]) reader.start() msg = b"I51\n.(dp0\nS'item'\np1\nS'hello world'\np2\nsS'port'\np3\nI0\ns." proc.stdin.write(msg) proc.stdin.flush() marker = b'[spreadflow_core.proc.DebugLog#debug] ' \ b'Item received: hello world' for stream, data in reader.drain(): stream_data[stream] += data if marker in stream_data[proc.stderr]: break else: self.fail('Worker process is expected to emit a message to stderr{0}'.format(self._format_stream(stream_data[proc.stdout], stream_data[proc.stderr]))) # Close stdin, this signals the worker process to terminate. proc.stdin.close() proc.wait() self.assertEqual(proc.returncode, 0, self._format_stream(stream_data[proc.stdout], stream_data[proc.stderr])) for stream, data in reader.drain(0): stream_data[stream] += data reader.join()
def test_subprocess_worker_producer(self): """ Worker process reads messages from stdin and writes results to stdout. """ logger = 'spreadflow_core.scripts.spreadflow_twistd.StderrLogger' config = os.path.join(FIXTURE_DIRECTORY, 'spreadflow-partitions.conf') with fixtures.TempDir() as fix: rundir = fix.path pidfile = os.path.join(rundir, 'twistd.pid') argv = ['-n', '-d', rundir, '-c', config, '--logger', logger] argv += ['--pidfile', pidfile, '--multiprocess', '--partition'] argv += ['producer'] proc = subprocess.Popen(['spreadflow-twistd'] + argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stream_data = {proc.stdout: b'', proc.stderr: b''} reader = StreamsReader([proc.stdout, proc.stderr]) reader.start() for stream, data in reader.drain(): stream_data[stream] += data if stream_data[proc.stdout]: break else: self.fail('Worker process is expected to emit a message to stdout{0}'.format(self._format_stream('*** BINARY ***', stream_data[proc.stderr]))) # Close stdin, this signals the worker process to terminate. proc.stdin.close() proc.wait() self.assertEqual(proc.returncode, 0, self._format_stream('*** BINARY ***', stream_data[proc.stderr])) for stream, data in reader.drain(0): stream_data[stream] += data reader.join()
def test_observer_process(self): """ Observer watches a directory for file changes and reports on stdout. """ with fixtures.TempDir() as fix: rundir = fix.path argv = [rundir, '*.txt'] proc = subprocess.Popen(['spreadflow-observer-fs-default'] + argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stream_data = {proc.stdout: b'', proc.stderr: b''} reader = StreamsReader([proc.stdout, proc.stderr]) reader.start() # Write a file. tmppath = os.path.join(rundir, 'test.tmp') txtpath = os.path.join(rundir, 'test.txt') with open(tmppath, 'w') as stream: stream.write('5WpWC30X') os.rename(tmppath, txtpath) for stream, data in reader.drain(): stream_data[stream] += data if stream_data[proc.stdout]: break else: self.fail('Observer process is expected to emit a message to stdout{0}'.format(self._format_stream('*** BINARY ***', stream_data[proc.stderr]))) # Verify output (create operation). msg = BSON(stream_data[proc.stdout]).decode() self.assertIn('item', msg) self.assertIn('port', msg) item = msg['item'] self.assertEqual(msg['port'], 'default') self.assertIn('data', item) self.assertIn('inserts', item) self.assertIn('deletes', item) self.assertIsInstance(item['data'], collections.Mapping) self.assertIsInstance(item['inserts'], collections.Sequence) self.assertIsInstance(item['deletes'], collections.Sequence) self.assertEqual(len(item['data']), 1) self.assertEqual(len(item['inserts']), 1) self.assertEqual(len(item['deletes']), 0) origkey = item['inserts'][0] self.assertIn(origkey, item['data']) self.assertEqual(item['data'][origkey]['path'], txtpath) # Reset stdout buffer. stream_data[proc.stdout] = b'' # Move a file. newpath = os.path.join(rundir, 'test-2.txt') os.rename(txtpath, newpath) for stream, data in reader.drain(): stream_data[stream] += data if stream_data[proc.stdout]: break else: self.fail('Observer process is expected to emit a message to stdout{0}'.format(self._format_stream('*** BINARY ***', stream_data[proc.stderr]))) # Verify output (mv operation). msg = BSON(stream_data[proc.stdout]).decode() self.assertIn('item', msg) self.assertIn('port', msg) item = msg['item'] self.assertEqual(msg['port'], 'default') self.assertIn('data', item) self.assertIn('inserts', item) self.assertIn('deletes', item) self.assertIsInstance(item['data'], collections.Mapping) self.assertIsInstance(item['inserts'], collections.Sequence) self.assertIsInstance(item['deletes'], collections.Sequence) # FIXME: Apparently there is a data-entry also for deletes. Figure # out whether this is really expected/necessary. Also compare to # the behavior of spreadflow-observer-fs-spotlight. self.assertEqual(len(item['data']), 2) self.assertEqual(len(item['inserts']), 1) self.assertEqual(len(item['deletes']), 1) oldkey = item['deletes'][0] newkey = item['inserts'][0] self.assertIn(newkey, item['data']) self.assertEqual(item['data'][newkey]['path'], newpath) self.assertEqual(origkey, oldkey) self.assertNotEqual(origkey, newkey) # Reset stdout buffer. stream_data[proc.stdout] = b'' # Close stdin, this signals the observer process to terminate. proc.stdin.close() proc.wait() self.assertEqual(proc.returncode, 0, self._format_stream('*** BINARY ***', stream_data[proc.stderr])) for stream, data in reader.drain(0): stream_data[stream] += data reader.join()