def test_subprocess_controller(self):
"""
Controller process runs two workers and collects their output.
"""
logger = 'spreadflow_core.scripts.spreadflow_twistd.StderrLogger'
config = os.path.join(FIXTURE_DIRECTORY, 'spreadflow-partitions.conf')
with fixtures.TempDir() as fix:
rundir = fix.path
# FIXME: subprocess controller currently does not propagate the
# configuration file path to its child processess.
shutil.copy(config, os.path.join(rundir, 'spreadflow.conf'))
pidfile = os.path.join(rundir, 'twistd.pid')
argv = ['-n', '-d', rundir, '--logger', logger]
argv += ['--pidfile', pidfile, '--multiprocess']
proc = subprocess.Popen(['spreadflow-twistd'] + argv,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stream_data = {proc.stdout: b'', proc.stderr: b''}
reader = StreamsReader([proc.stdout, proc.stderr])
reader.start()
marker = b'[spreadflow_core.proc.DebugLog#debug] ' \
b'Item received: hello world'
for stream, data in reader.drain():
stream_data[stream] += data
if marker in stream_data[proc.stderr]:
break
else:
self.fail('Worker process is expected to emit a message to stderr{0}'.format(self._format_stream(stream_data[proc.stdout], stream_data[proc.stderr])))
# Send SIGTERM to controller.
proc.terminate()
for stream, data in reader.drain(0):
stream_data[stream] += data
proc.wait()
self.assertEqual(proc.returncode, 0, self._format_stream(stream_data[proc.stdout], stream_data[proc.stderr]))
for stream, data in reader.drain(0):
stream_data[stream] += data
reader.join()
def test_subprocess_worker_consumer(self):
"""
Worker process reads messages from stdin and writes results to stdout.
"""
logger = 'spreadflow_core.scripts.spreadflow_twistd.StderrLogger'
config = os.path.join(FIXTURE_DIRECTORY, 'spreadflow-partitions.conf')
with fixtures.TempDir() as fix:
rundir = fix.path
pidfile = os.path.join(rundir, 'twistd.pid')
argv = ['-n', '-d', rundir, '-c', config, '--logger', logger]
argv += ['--pidfile', pidfile, '--multiprocess', '--partition']
argv += ['consumer']
proc = subprocess.Popen(['spreadflow-twistd'] + argv,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stream_data = {proc.stdout: b'', proc.stderr: b''}
reader = StreamsReader([proc.stdout, proc.stderr])
reader.start()
msg = b"I51\n.(dp0\nS'item'\np1\nS'hello world'\np2\nsS'port'\np3\nI0\ns."
proc.stdin.write(msg)
proc.stdin.flush()
marker = b'[spreadflow_core.proc.DebugLog#debug] ' \
b'Item received: hello world'
for stream, data in reader.drain():
stream_data[stream] += data
if marker in stream_data[proc.stderr]:
break
else:
self.fail('Worker process is expected to emit a message to stderr{0}'.format(self._format_stream(stream_data[proc.stdout], stream_data[proc.stderr])))
# Close stdin, this signals the worker process to terminate.
proc.stdin.close()
proc.wait()
self.assertEqual(proc.returncode, 0, self._format_stream(stream_data[proc.stdout], stream_data[proc.stderr]))
for stream, data in reader.drain(0):
stream_data[stream] += data
reader.join()
def test_subprocess_worker_producer(self):
"""
Worker process reads messages from stdin and writes results to stdout.
"""
logger = 'spreadflow_core.scripts.spreadflow_twistd.StderrLogger'
config = os.path.join(FIXTURE_DIRECTORY, 'spreadflow-partitions.conf')
with fixtures.TempDir() as fix:
rundir = fix.path
pidfile = os.path.join(rundir, 'twistd.pid')
argv = ['-n', '-d', rundir, '-c', config, '--logger', logger]
argv += ['--pidfile', pidfile, '--multiprocess', '--partition']
argv += ['producer']
proc = subprocess.Popen(['spreadflow-twistd'] + argv,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stream_data = {proc.stdout: b'', proc.stderr: b''}
reader = StreamsReader([proc.stdout, proc.stderr])
reader.start()
for stream, data in reader.drain():
stream_data[stream] += data
if stream_data[proc.stdout]:
break
else:
self.fail('Worker process is expected to emit a message to stdout{0}'.format(self._format_stream('*** BINARY ***', stream_data[proc.stderr])))
# Close stdin, this signals the worker process to terminate.
proc.stdin.close()
proc.wait()
self.assertEqual(proc.returncode, 0, self._format_stream('*** BINARY ***', stream_data[proc.stderr]))
for stream, data in reader.drain(0):
stream_data[stream] += data
reader.join()
def test_observer_process(self):
"""
Observer watches a directory for file changes and reports on stdout.
"""
with fixtures.TempDir() as fix:
rundir = fix.path
argv = [rundir, '*.txt']
proc = subprocess.Popen(['spreadflow-observer-fs-default'] + argv,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stream_data = {proc.stdout: b'', proc.stderr: b''}
reader = StreamsReader([proc.stdout, proc.stderr])
reader.start()
# Write a file.
tmppath = os.path.join(rundir, 'test.tmp')
txtpath = os.path.join(rundir, 'test.txt')
with open(tmppath, 'w') as stream:
stream.write('5WpWC30X')
os.rename(tmppath, txtpath)
for stream, data in reader.drain():
stream_data[stream] += data
if stream_data[proc.stdout]:
break
else:
self.fail('Observer process is expected to emit a message to stdout{0}'.format(self._format_stream('*** BINARY ***', stream_data[proc.stderr])))
# Verify output (create operation).
msg = BSON(stream_data[proc.stdout]).decode()
self.assertIn('item', msg)
self.assertIn('port', msg)
item = msg['item']
self.assertEqual(msg['port'], 'default')
self.assertIn('data', item)
self.assertIn('inserts', item)
self.assertIn('deletes', item)
self.assertIsInstance(item['data'], collections.Mapping)
self.assertIsInstance(item['inserts'], collections.Sequence)
self.assertIsInstance(item['deletes'], collections.Sequence)
self.assertEqual(len(item['data']), 1)
self.assertEqual(len(item['inserts']), 1)
self.assertEqual(len(item['deletes']), 0)
origkey = item['inserts'][0]
self.assertIn(origkey, item['data'])
self.assertEqual(item['data'][origkey]['path'], txtpath)
# Reset stdout buffer.
stream_data[proc.stdout] = b''
# Move a file.
newpath = os.path.join(rundir, 'test-2.txt')
os.rename(txtpath, newpath)
for stream, data in reader.drain():
stream_data[stream] += data
if stream_data[proc.stdout]:
break
else:
self.fail('Observer process is expected to emit a message to stdout{0}'.format(self._format_stream('*** BINARY ***', stream_data[proc.stderr])))
# Verify output (mv operation).
msg = BSON(stream_data[proc.stdout]).decode()
self.assertIn('item', msg)
self.assertIn('port', msg)
item = msg['item']
self.assertEqual(msg['port'], 'default')
self.assertIn('data', item)
self.assertIn('inserts', item)
self.assertIn('deletes', item)
self.assertIsInstance(item['data'], collections.Mapping)
self.assertIsInstance(item['inserts'], collections.Sequence)
self.assertIsInstance(item['deletes'], collections.Sequence)
# FIXME: Apparently there is a data-entry also for deletes. Figure
# out whether this is really expected/necessary. Also compare to
# the behavior of spreadflow-observer-fs-spotlight.
self.assertEqual(len(item['data']), 2)
self.assertEqual(len(item['inserts']), 1)
self.assertEqual(len(item['deletes']), 1)
oldkey = item['deletes'][0]
newkey = item['inserts'][0]
self.assertIn(newkey, item['data'])
self.assertEqual(item['data'][newkey]['path'], newpath)
self.assertEqual(origkey, oldkey)
self.assertNotEqual(origkey, newkey)
# Reset stdout buffer.
stream_data[proc.stdout] = b''
# Close stdin, this signals the observer process to terminate.
proc.stdin.close()
proc.wait()
self.assertEqual(proc.returncode, 0, self._format_stream('*** BINARY ***', stream_data[proc.stderr]))
for stream, data in reader.drain(0):
stream_data[stream] += data
reader.join()
