def test_pipeline_multi_modules(self):
def test_func(bundle):
bundle.data['func'] = True
return bundle
def test_func2(bundle):
bundle.data['func2'] = True
return bundle
pipelines = dict()
pipe = FakePipeline('multi')
pipe.add_func(test_func)
pipe.add_func(test_func2)
pipelines['multi'] = pipe
thread = self._start_server(pipelines=pipelines)
client = ProcessClient(thread.server._transport)
data = client.process_url(str(uuid.uuid4()),
'multi',
dict(),
'fake://blank',
autosave=False,
wait_for_result=True)
self._stop_server(client, thread)
self.assertFalse('exceptions' in data)
self.assertTrue('func' in data)
self.assertTrue('func2' in data)
self.assertTrue(data['func'])
self.assertTrue(data['func'])
def test_fileinfo_result(self):
pipelines = dict()
pipe = FakePipeline('fileinfo')
pipe.add_module(FileInfoModule(dict()))
pipelines['fileinfo'] = pipe
thread = self._start_server(pipelines=pipelines)
client = ProcessClient(thread.server._transport)
for filename, desc, mime in TESTFILES:
filepath = os.path.join(os.path.dirname(test.__file__),
"data", filename)
with open(filepath, 'rb') as fp:
data = fp.read()
result = client.process_raw(str(uuid.uuid4()), 'fileinfo',
{'path': filepath, 'name': filename}, data, autosave=False, wait_for_result=True)
self.assertEqual(desc, result['filetype'])
self.assertEqual(mime, result['mimetype'])
self.assertEqual(len(data), result['filesize'])
self.assertEqual(os.path.splitext(filename)[1][1:], result['extension'])
self._stop_server(client, thread)
def test_pipeline_multi_modules(self):
def test_func(bundle):
bundle.data['func'] = True
return bundle
def test_func2(bundle):
bundle.data['func2'] = True
return bundle
pipelines = dict()
pipe = FakePipeline('multi')
pipe.add_func(test_func)
pipe.add_func(test_func2)
pipelines['multi'] = pipe
thread = self._start_server(pipelines=pipelines)
client = ProcessClient(thread.server._transport)
data = client.process_url(str(uuid.uuid4()), 'multi', dict(), 'fake://blank',
autosave=False, wait_for_result=True)
self._stop_server(client, thread)
self.assertFalse('exceptions' in data)
self.assertTrue('func' in data)
self.assertTrue('func2' in data)
self.assertTrue(data['func'])
self.assertTrue(data['func'])
def test_fileinfo_result(self):
pipelines = dict()
pipe = FakePipeline('fileinfo')
pipe.add_module(FileInfoModule(dict()))
pipelines['fileinfo'] = pipe
thread = self._start_server(pipelines=pipelines)
client = ProcessClient(thread.server._transport)
for filename, desc, mime in TESTFILES:
filepath = os.path.join(os.path.dirname(test.__file__), "data",
filename)
with open(filepath, 'rb') as fp:
data = fp.read()
result = client.process_raw(str(uuid.uuid4()),
'fileinfo', {
'path': filepath,
'name': filename
},
data,
autosave=False,
wait_for_result=True)
self.assertEqual(desc, result['filetype'])
self.assertEqual(mime, result['mimetype'])
self.assertEqual(len(data), result['filesize'])
self.assertEqual(
os.path.splitext(filename)[1][1:], result['extension'])
self._stop_server(client, thread)
def test_result(self):
pipelines = dict()
pipe = FakePipeline('hashing')
pipe.add_module(HashModule(dict()))
pipelines['hashing'] = pipe
thread = self._start_server(pipelines=pipelines)
client = ProcessClient(thread.server._transport)
data = client.process_raw(str(uuid.uuid4()), 'hashing', dict(), TEST_DATA,
autosave=False, wait_for_result=True)
self._stop_server(client, thread)
self.assertTrue('hash-md5' in data)
md5 = hashlib.md5()
md5.update(TEST_DATA)
self.assertEqual(md5.hexdigest(), data['hash-md5'])
self.assertTrue('hash-sha1' in data)
sha1 = hashlib.sha1()
sha1.update(TEST_DATA)
self.assertEqual(sha1.hexdigest(), data['hash-sha1'])
self.assertTrue('hash-sha256' in data)
sha256 = hashlib.sha256()
sha256.update(TEST_DATA)
self.assertEqual(sha256.hexdigest(), data['hash-sha256'])
self.assertTrue('hash-sha512' in data)
sha512 = hashlib.sha512()
sha512.update(TEST_DATA)
self.assertEqual(sha512.hexdigest(), data['hash-sha512'])
def test_pipeline_exceptions(self):
def empty_func(bundle):
raise Exception("test")
pipelines = dict()
pipelines['empty'] = FakePipeline('empty', empty_func)
thread = self._start_server(pipelines=pipelines)
client = ProcessClient(thread.server._transport)
data = client.process_url(str(uuid.uuid4()), 'empty', dict(), 'fake://blank',
autosave=False, wait_for_result=True)
self._stop_server(client, thread)
self.assertTrue('exceptions' in data)
self.assertEqual(data['exceptions'][0]['name'], "Exception")
self.assertEqual(data['exceptions'][0]['msg'], "test")
def test_pipeline_process(self):
def empty_func(bundle):
bundle.data['called'] = True
return bundle
pipelines = dict()
pipelines['empty'] = FakePipeline('empty', empty_func)
thread = self._start_server(pipelines=pipelines)
client = ProcessClient(thread.server._transport)
data = client.process_url(str(uuid.uuid4()), 'empty', dict(), 'fake://blank',
autosave=False, wait_for_result=True)
self._stop_server(client, thread)
self.assertTrue('called' in data)
self.assertTrue(data['called'])
def test_default(self):
pipelines = dict()
pipe = FakePipeline('hashing')
pipe.add_module(HashModule(dict()))
pipelines['hashing'] = pipe
thread = self._start_server(pipelines=pipelines)
client = ProcessClient(thread.server._transport)
data = client.process_url(str(uuid.uuid4()), 'hashing', dict(), 'fake://blank',
autosave=False, wait_for_result=True)
self._stop_server(client, thread)
self.assertTrue('hash-md5' in data)
self.assertTrue('hash-sha1' in data)
self.assertTrue('hash-sha256' in data)
self.assertTrue('hash-sha512' in data)
def test_pipeline_exceptions(self):
def empty_func(bundle):
raise Exception("test")
pipelines = dict()
pipelines['empty'] = FakePipeline('empty', empty_func)
thread = self._start_server(pipelines=pipelines)
client = ProcessClient(thread.server._transport)
data = client.process_url(str(uuid.uuid4()),
'empty',
dict(),
'fake://blank',
autosave=False,
wait_for_result=True)
self._stop_server(client, thread)
self.assertTrue('exceptions' in data)
self.assertEqual(data['exceptions'][0]['name'], "Exception")
self.assertEqual(data['exceptions'][0]['msg'], "test")
def test_pipeline_process(self):
def empty_func(bundle):
bundle.data['called'] = True
return bundle
pipelines = dict()
pipelines['empty'] = FakePipeline('empty', empty_func)
thread = self._start_server(pipelines=pipelines)
client = ProcessClient(thread.server._transport)
data = client.process_url(str(uuid.uuid4()),
'empty',
dict(),
'fake://blank',
autosave=False,
wait_for_result=True)
self._stop_server(client, thread)
self.assertTrue('called' in data)
self.assertTrue(data['called'])
def test_prefix(self):
pipelines = dict()
pipe = FakePipeline('hashing')
pipe.add_module(HashModule({'prefix': 'verify'}))
pipelines['hashing'] = pipe
thread = self._start_server(pipelines=pipelines)
client = ProcessClient(thread.server._transport)
data = client.process_url(str(uuid.uuid4()),
'hashing',
dict(),
'fake://blank',
autosave=False,
wait_for_result=True)
self._stop_server(client, thread)
self.assertTrue('verify-hash-md5' in data)
self.assertTrue('verify-hash-sha1' in data)
self.assertTrue('verify-hash-sha256' in data)
self.assertTrue('verify-hash-sha512' in data)
def test_result(self):
pipelines = dict()
pipe = FakePipeline('hashing')
pipe.add_module(HashModule(dict()))
pipelines['hashing'] = pipe
thread = self._start_server(pipelines=pipelines)
client = ProcessClient(thread.server._transport)
data = client.process_raw(str(uuid.uuid4()),
'hashing',
dict(),
TEST_DATA,
autosave=False,
wait_for_result=True)
self._stop_server(client, thread)
self.assertTrue('hash-md5' in data)
md5 = hashlib.md5()
md5.update(TEST_DATA)
self.assertEqual(md5.hexdigest(), data['hash-md5'])
self.assertTrue('hash-sha1' in data)
sha1 = hashlib.sha1()
sha1.update(TEST_DATA)
self.assertEqual(sha1.hexdigest(), data['hash-sha1'])
self.assertTrue('hash-sha256' in data)
sha256 = hashlib.sha256()
sha256.update(TEST_DATA)
self.assertEqual(sha256.hexdigest(), data['hash-sha256'])
self.assertTrue('hash-sha512' in data)
sha512 = hashlib.sha512()
sha512.update(TEST_DATA)
self.assertEqual(sha512.hexdigest(), data['hash-sha512'])
def __init__(self, host, storage_driver=None, topic=None, allow_stop=False):
topic = topic or 'engine'
self.host = host
self.topic = 'gate.%s' % topic
self._storage_driver = storage_driver
if not self._storage_driver:
self._storage_driver = get_storage_driver()
self._check_indexes()
self._transport = messaging.get_transport(cfg.CONF)
self._target = messaging.Target(topic=self.topic, server=self.host)
self._process_client = ProcessClient(self._transport)
self._endpoints = [
EngineAPI(self, process_client=self._process_client)
]
if allow_stop:
self._endpoints.append(self)
self._server = messaging.get_rpc_server(
self._transport, self._target, self._endpoints)
def test_custom_topic(self):
thread = self._start_server(topic='custom')
client = ProcessClient(thread.server._transport, topic='custom')
self.assertEqual(thread.server.topic, client.topic)
self._stop_server(client, thread)
def test_default_topic(self):
thread = self._start_server()
client = ProcessClient(thread.server._transport)
self.assertEqual(thread.server.topic, client.topic)
self._stop_server(client, thread)