def test_setup_doc_store_no_doc_store_class(self):
"""
Test with no "doc_store_class" entry in JSON configuration.
@param self Object reference.
"""
with self.assertRaisesRegexp(KeyError, ".*"):
DocumentStoreUtilities.setup_doc_store({})
def test_setup_doc_store_no_doc_store_class(self):
"""
Test with no "doc_store_class" entry in JSON configuration.
@param self Object reference.
"""
with self.assertRaisesRegexp(KeyError,
".*"):
DocumentStoreUtilities.setup_doc_store({})
def test_create_doc_store_collection_exception(self):
"""
Test where document store throws an exception.
@param self Object reference.
"""
doc_store = MockDocumentStore()
with self.assertRaisesRegexp(DocumentStoreException, ".*"):
DocumentStoreUtilities.create_doc_store_collection( \
doc_store, "fail", 1)
def test_setup_doc_store_no_module(self):
"""
Test where "doc_store_class" specifies a class but
no module prefix.
@param self Object reference.
"""
config = {}
config["doc_store_class"] = "ClassOnly"
with self.assertRaisesRegexp(ValueError, ".*"):
DocumentStoreUtilities.setup_doc_store(config)
def test_setup_doc_store_bad_module(self):
"""
Test where "doc_store_class" specifies a non-existant
module.
@param self Object reference.
"""
config = {}
config["doc_store_class"] = "nonexistant.NonExistantModule"
with self.assertRaisesRegexp(ValueError, ".*"):
DocumentStoreUtilities.setup_doc_store(config)
def test_setup_doc_store_bad_module(self):
"""
Test where "doc_store_class" specifies a non-existant
module.
@param self Object reference.
"""
config = {}
config["doc_store_class"] = "nonexistant.NonExistantModule"
with self.assertRaisesRegexp(ValueError, ".*"):
DocumentStoreUtilities.setup_doc_store(config)
def test_setup_doc_store_no_module(self):
"""
Test where "doc_store_class" specifies a class but
no module prefix.
@param self Object reference.
"""
config = {}
config["doc_store_class"] = "ClassOnly"
with self.assertRaisesRegexp(ValueError, ".*"):
DocumentStoreUtilities.setup_doc_store(config)
def test_create_doc_store_collection_exception(self):
"""
Test where document store throws an exception.
@param self Object reference.
"""
doc_store = MockDocumentStore()
with self.assertRaisesRegexp(DocumentStoreException,
".*"):
DocumentStoreUtilities.create_doc_store_collection( \
doc_store, "fail", 1)
def execute(self, _job_header, _job_footer): # pylint: disable = R0914, R0912, R0915, C0301
"""
Set up MAUS input tasks and, on receipt of spills, submit
to transform tasks accessed via a distributed task queue.
@param self Object reference.
@throws WorkerBirthFailedException if inputer.birth returns False.
@throws WorkerDeathFailedException if inputer death returns False.
@throws RabbitMQException if RabbitMQ cannot be contacted.
@throws NoCeleryNodeException if no Celery nodes.
@throws CeleryNodeException if Celery nodes fail to
configure.
@throws DocumentStoreException if there is a problem
using the document store.
"""
# Purge the document store.
print("Purging document store")
DocumentStoreUtilities.create_doc_store_collection(self.doc_store,
self.collection, self.config["doc_store_event_cache_size"])
# Do an initial check for active Celery nodes.
print("Checking for active Celery nodes...")
num_nodes = CeleryUtilities.ping_celery_nodes()
print("Number of active Celery nodes: %d" % num_nodes)
# reset input
print("INPUT: Birth")
if self.inputer.birth(self.config_doc) == False:
raise WorkerBirthFailedException(self.inputer.__class__)
emitter = self.inputer.emitter()
map_buffer = DataflowUtilities.buffer_input(emitter, 1)
# reset counters
self.spill_input_count = 0
self.spill_process_count = 0
self.spill_fail_count = 0
self.celery_tasks = []
# process events
while (len(map_buffer) != 0) or (len(self.celery_tasks) != 0):
for spill in map_buffer:
self.process_event(spill)
map_buffer = DataflowUtilities.buffer_input(emitter, 1)
if (len(map_buffer) != 0):
print("%d spills left in buffer" % (len(map_buffer)))
# Go through current transform tasks and see if any's completed.
self.poll_celery_tasks()
print "--------------------"
# Invoke death
print "Requesting Celery nodes death transforms..."
self.end_of_run(self.run_number)
print "Celery node transforms deathed!"
print("TRANSFORM: transform tasks completed")
print("INPUT: Death")
if (not self.inputer.death()):
raise WorkerDeathFailedException(self.inputer.__class__)
print "--------------------"
def test_setup_doc_store_not_subclass(self):
"""
Test where "doc_store_class" specifies a class that does
not subclass docstore.DocumentStore.
@param self Object reference.
"""
config = {}
config["doc_store_class"] = \
"framework.utilities.DocumentStoreUtilities"
with self.assertRaisesRegexp(TypeError, ".*"):
DocumentStoreUtilities.setup_doc_store(config)
def test_setup_doc_store_bad_class(self):
"""
Test where "doc_store_class" specifies a non-existant
class.
@param self Object reference.
"""
config = {}
config["doc_store_class"] = \
"docstore.InMemoryDocumentStore.NonExistantClass"
with self.assertRaisesRegexp(ValueError, ".*"):
DocumentStoreUtilities.setup_doc_store(config)
def test_setup_doc_store_bad_class(self):
"""
Test where "doc_store_class" specifies a non-existant
class.
@param self Object reference.
"""
config = {}
config["doc_store_class"] = \
"docstore.InMemoryDocumentStore.NonExistantClass"
with self.assertRaisesRegexp(ValueError, ".*"):
DocumentStoreUtilities.setup_doc_store(config)
def test_setup_doc_store_connect_fails(self):
"""
Test where "doc_store_class" specifies a class whose
connect method throws an exception.
@param self Object reference.
"""
config = {}
config["doc_store_class"] = \
"test_framework.test_utilities.MockDocumentStore"
config["fail"] = "fail"
with self.assertRaisesRegexp(DocumentStoreException, ".*"):
DocumentStoreUtilities.setup_doc_store(config)
def test_setup_doc_store_not_subclass(self):
"""
Test where "doc_store_class" specifies a class that does
not subclass docstore.DocumentStore.
@param self Object reference.
"""
config = {}
config["doc_store_class"] = \
"framework.utilities.DocumentStoreUtilities"
with self.assertRaisesRegexp(TypeError,
".*"):
DocumentStoreUtilities.setup_doc_store(config)
def test_setup_doc_store_connect_fails(self):
"""
Test where "doc_store_class" specifies a class whose
connect method throws an exception.
@param self Object reference.
"""
config = {}
config["doc_store_class"] = \
"test_framework.test_utilities.MockDocumentStore"
config["fail"] = "fail"
with self.assertRaisesRegexp(DocumentStoreException,
".*"):
DocumentStoreUtilities.setup_doc_store(config)
def test_create_doc_store_collection(self):
"""
Test where arguments are valid.
@param self Object reference.
"""
doc_store = InMemoryDocumentStore()
doc_store.create_collection("test", -1)
doc_store.put("test", "1", {"a": "a"})
doc_store.put("test", "2", {"a": "a"})
self.assertEquals(2, doc_store.count("test"), "Expected 2 documents")
DocumentStoreUtilities.create_doc_store_collection(
doc_store, "test", -1)
# Collection should have been deleted and recreated.
self.assertEquals(0, doc_store.count("test"), "Expected 0 documents")
def __init__(self, inputer, transformer, merger, outputer, config_doc): # pylint: disable=R0913,C0301
"""
Save references to arguments and parse the JSON configuration
document, then connect to the document store.
@param self Object reference.
@param inputer Input task.
@param transformer Transformer task.
@param merger Merger task.
@param outputer Output task.
@param config_doc JSON configuration document.
@throws KeyError If any required configuration parameters are
missing.
@throws ValueError If any configuration values are invalid.
@throws TypeError If any dynamic class does not implement
a required interface.
@throws DocumentStoreException if there is a problem
connecting to the document store.
"""
configuration = json.loads(config_doc)
self.doc_store = \
DocumentStoreUtilities.setup_doc_store(configuration)
self.input_transform = InputTransformExecutor( \
inputer, transformer, config_doc, self.doc_store)
self.merge_output = MergeOutputExecutor( \
merger, outputer, config_doc, self.doc_store,
self.input_transform.collection)
def test_create_doc_store_collection(self):
"""
Test where arguments are valid.
@param self Object reference.
"""
doc_store = InMemoryDocumentStore()
doc_store.create_collection("test", -1)
doc_store.put("test", "1", {"a":"a"})
doc_store.put("test", "2", {"a":"a"})
self.assertEquals(2, doc_store.count("test"),
"Expected 2 documents")
DocumentStoreUtilities.create_doc_store_collection(doc_store,
"test", -1)
# Collection should have been deleted and recreated.
self.assertEquals(0, doc_store.count("test"),
"Expected 0 documents")
def test_setup_doc_store(self):
"""
Test where arguments are valid.
@param self Object reference.
"""
config = {}
config["doc_store_class"] = \
"docstore.InMemoryDocumentStore.InMemoryDocumentStore"
doc_store = DocumentStoreUtilities.setup_doc_store(config)
self.assertTrue(isinstance(doc_store, InMemoryDocumentStore),
"Document store isn't instance of given class")
def test_setup_doc_store(self):
"""
Test where arguments are valid.
@param self Object reference.
"""
config = {}
config["doc_store_class"] = \
"docstore.InMemoryDocumentStore.InMemoryDocumentStore"
doc_store = DocumentStoreUtilities.setup_doc_store(config)
self.assertTrue(isinstance(doc_store, InMemoryDocumentStore),
"Document store isn't instance of given class")
def __init__(self,
merger,
outputer,
config_doc,
doc_store=None,
collection_name=None): # pylint: disable=R0913,C0301
"""
Save references to arguments and parse the JSON configuration
document, then connect to the document store.
@param self Object reference.
@param merger Merger task.
@param outputer Output task.
@param config_doc JSON configuration document.
@param doc_store Document store.
@param collection_name Collection name in document store.
@throws ValueError if collection_name is None and there is no
"doc_collection_name" entry in config_doc.
@throws KeyError If any required configuration parameters are
missing.
@throws ValueError If any configuration values are invalid.
@throws TypeError If any dynamic class does not implement
a required interface.
@throws DocumentStoreException if there is a problem
connecting to the document store.
"""
self.last_time = datetime(1970, 01, 01)
self.job_footer = None
self.merger = merger
self.outputer = outputer
self.config_doc = config_doc
self.config = json.loads(self.config_doc)
# Current run number (from spills).
self.run_number = None
# Last end_of_run spill received.
self.end_of_run_spill = None
# Counts of spills processed.
self.spill_process_count = 0
self.write_headers = self.config["header_and_footer_mode"] == "append"
# Connect to doc store.
if (doc_store == None):
self.doc_store = DocumentStoreUtilities.setup_doc_store(\
self.config)
else:
self.doc_store = doc_store
# Get collection name.
if (collection_name == None):
if (not self.config.has_key("doc_collection_name")
or (self.config["doc_collection_name"] in [None, ""])):
raise ValueError("collection is not specified")
else:
self.collection = self.config["doc_collection_name"]
else:
self.collection = collection_name
def __init__(self, merger, outputer, config_doc, doc_store = None, collection_name = None): # pylint: disable=R0913,C0301
"""
Save references to arguments and parse the JSON configuration
document, then connect to the document store.
@param self Object reference.
@param merger Merger task.
@param outputer Output task.
@param config_doc JSON configuration document.
@param doc_store Document store.
@param collection_name Collection name in document store.
@throws ValueError if collection_name is None and there is no
"doc_collection_name" entry in config_doc.
@throws KeyError If any required configuration parameters are
missing.
@throws ValueError If any configuration values are invalid.
@throws TypeError If any dynamic class does not implement
a required interface.
@throws DocumentStoreException if there is a problem
connecting to the document store.
"""
self.last_time = datetime(1970, 01, 01)
self.job_footer = None
self.merger = merger
self.outputer = outputer
self.config_doc = config_doc
self.config = json.loads(self.config_doc)
# Current run number (from spills).
self.run_number = None
# Last end_of_run spill received.
self.end_of_run_spill = None
# Counts of spills processed.
self.spill_process_count = 0
self.write_headers = self.config["header_and_footer_mode"] == "append"
# Connect to doc store.
if (doc_store == None):
self.doc_store = DocumentStoreUtilities.setup_doc_store(\
self.config)
else:
self.doc_store = doc_store
# Get collection name.
if (collection_name == None):
if (not self.config.has_key("doc_collection_name") or
(self.config["doc_collection_name"] in [None, ""])):
raise ValueError("collection is not specified")
else:
self.collection = self.config["doc_collection_name"]
else:
self.collection = collection_name
def __init__(self, inputer, transformer, config_doc, doc_store = None): # pylint: disable=R0913,C0301
"""
Save references to arguments and parse the JSON configuration
document, then connect to the document store.
@param self Object reference.
@param inputer Input task.
@param transformer Transformer task.
@param config_doc JSON configuration document.
@param doc_store Document store.
@throws KeyError If any required configuration parameters are
missing.
@throws ValueError If any configuration values are invalid.
@throws TypeError If any dynamic class does not implement
a required interface.
@throws DocumentStoreException if there is a problem
connecting to the document store.
"""
self.inputer = inputer
self.transformer = transformer
self.config_doc = config_doc
self.config = json.loads(self.config_doc)
# Maximum number of tasks in queue
self.timeout = self.config['reconstruction_timeout']
# Unique ID for execution.
self.config_id = "%s-%s" % (socket.gethostname(), os.getpid())
# Current run number (from spills).
self.run_number = "first"
# Counts of spills input, processed, failed.
self.spill_input_count = 0
self.spill_process_count = 0
self.spill_fail_count = 0
# Active Celery jobs - Celery AsyncResult objects.
self.celery_tasks = []
# Connect to doc store.
if (doc_store == None):
self.doc_store = DocumentStoreUtilities.setup_doc_store( \
self.config)
else:
self.doc_store = doc_store
# Get collection name.
if (not self.config.has_key("doc_collection_name") or
(self.config["doc_collection_name"] in
[None, "", "auto"])):
self.collection = self.config_id
else:
self.collection = self.config["doc_collection_name"]