def step_impl(context):
"""
:type context behave.runner.Context
"""
try:
SchemaTools.check_schema_fields(context.schema_file, "test file")
except Exception as e:
if str(e) == 'MongoBackend.load_schemas_from_directory: ' \
'The "namespace" field is not in the schema-"test file"':
ok_(True)
else:
ok_(False)
def __init__(self, _database, _json_schema_folders=None, _uri_handlers = None, _schema_tools = None):
"""
Initialize a database access object
:param _database: A MongoDB client object
:param _json_schema_folders: A list of application specific JSON schema folders
"""
self.database = _database
self.logging = Logging(_database=self.database)
# Database access is dependent on schema tools for validation.
if _schema_tools:
self.schema_tools = _schema_tools
else:
self.schema_tools = SchemaTools(_json_schema_folders = _json_schema_folders, _uri_handlers=_uri_handlers)
class DatabaseAccess():
"""
The database access class handles all communication with the database
It provides logging, validation against a schema and hiding ObjectIds.
"""
database = None
schema_tools = None
logging = None
def __init__(self, _database, _json_schema_folders=None, _uri_handlers = None, _schema_tools = None):
"""
Initialize a database access object
:param _database: A MongoDB client object
:param _json_schema_folders: A list of application specific JSON schema folders
"""
self.database = _database
self.logging = Logging(_database=self.database)
# Database access is dependent on schema tools for validation.
if _schema_tools:
self.schema_tools = _schema_tools
else:
self.schema_tools = SchemaTools(_json_schema_folders = _json_schema_folders, _uri_handlers=_uri_handlers)
def verify_document(self, _data, _collection_name, _caller_name):
"""
Check so that the collection for the specified schema in the input data match the supplied collection in
_collection_name.
:param _data: The data to check.
:param _collection_name: Name of the collection that it should be checked against
:param _caller_name: The name of the calling function for logging
"""
if "schemaRef" in _data:
try:
_curr_schema = self.schema_tools.json_schema_objects[_data["schemaRef"]]
except:
raise Exception(
_caller_name + ": Error - invalid data structure, schemaRef not found: " + _data["schemaRef"])
if _curr_schema["collection"] != _collection_name:
raise Exception(
_caller_name + ": Invalid schemaRef: \"" + _curr_schema["collection"] +"\". " + _caller_name + " is restricted to the \"" +
_collection_name + "\"-collection.")
else:
# An MBE document must *always* have a schemaRef property
raise Exception(_caller_name + ": Missing schemaRef.")
@staticmethod
def verify_condition(_data, _collection_name, _caller_name):
"""
Validates the conditions' structure and the collection it queries.
:param _data: The data to check.
:param _collection_name: Name of the collection that it should be checked against
:param _caller_name: The name of the calling function for logging
"""
if "conditions" not in _data:
raise Exception(_caller_name + ": Error - Not a valid condition, \"conditions\"-property missing")
if "collection" in _data and _data["collection"] != _collection_name:
raise Exception(_caller_name + ": Error - collection omitted or wrong, only queries against"
" the " + _collection_name + " collection is permitted.")
def manage_input(self, _input, _schema_ref=None):
"""
Validate, convert _id to objectId instances and parse collection from input, whether it is data to save or a\
condition.
Note: This should not be run twice on the same data.
:param _input: Data to handle
:param _schema_ref: If set the schema to validate against
:return: A tuple with the data and the database collection specified in the schema.
"""
try:
# Apply(if _schema_ref is set, validate against that schema instead)
_input, _schema_obj = self.schema_tools.apply(_input, _schema_ref)
except ValidationError as e:
raise ValidationError("handle_input: Validation error:" + str(e) + ". Data: \n" + str(_input))
except Exception as e:
raise Exception(
"handle_input: Non-validation error from validation: Type: " + e.__class__.__name__ + " Message:" + str(
e) + ". Data: \n" + str(_input))
# In a condition, the input data is under the conditions-field
if _schema_ref == "ref://of.conditions":
_data = copy.deepcopy(_input["conditions"])
_collection = _input["collection"]
else:
_collection = _schema_obj["collection"]
_data = copy.deepcopy(_input)
return _data, self.database[_collection]
@staticmethod
def load_exactly_one_document(_collection, _id, _zero_error=None, _duplicate_error=None):
"""
Raises provided errors if not exactly one row in result.
:param _collection: The collection to load from
:param _id: The _id of the document
:param _zero_error: Error to report if no documents are found
:param _duplicate_error: Error to report when duplicate documents are found
:return: The document
"""
_document_cursor = _collection.find({"_id": _id})
if _document_cursor.count() == 0:
if _zero_error is not None:
raise Exception(_zero_error + "_id: " + str(
_id) + " could not be found.")
else:
return None
elif _document_cursor.count() > 1:
if _duplicate_error is not None:
raise Exception(
"Access.save: Tried saving existing, but found duplicate documents with(try to remove either) _id: "
+ str(_id) + ".")
else:
return None
elif _document_cursor.count() == 1:
return _document_cursor[0]
def save(self, _document, _user, _old_document=None, _allow_save_id=False):
"""
Save a document to a collection
:param _document: The data to save
:param _user: A user object
:param _old_document: If available, the existing data, used to avoid an extra read when logging changes.
:param _allow_save_id: If set, to not assume that _id being set means updating an existing document.
:return: The object id of the saved document
"""
if _user is not None:
_user_id = _user["_id"]
else:
_user_id = None
_document, _collection = self.manage_input(_document)
if _old_document is None and "_id" in _document:
# Load the existing data
if not _allow_save_id:
_zero_error = "Access.save: Tried to save data over existing but didn't find an existing node."
else:
_zero_error = None
_old_document = self.load_exactly_one_document(_collection, _document["_id"], _zero_error=_zero_error,
_duplicate_error="Access.save: Tried to save data over "
"existing but found duplicate nodes")
if (_old_document is not None) and (str(_old_document["schemaRef"]) != str(_document["schemaRef"])):
raise Exception(
"Access.save: Cannot change schema of an existing node, remove and add. _id: " + str(
_document["_id"]) + ", new:" + str(_old_document["schemaRef"]) + ", old:" + str(
_old_document["schemaRef"]))
if _old_document is not None:
_old_document, _dummy_collection = self.manage_input(_old_document)
_result = str(_collection.save(_document))
if _collection.name != "log":
self.logging.log_save(_document, _user_id, _old_document)
return _result
def remove_documents(self, _documents, _user, _collection_name=None):
"""
Remove the documents in the documents list
:param _documents: The list of documents
:param _user: A user object
:param _collection_name: The collection from where to remove them.
:return: Nothing
"""
if _user is not None:
_user_id = _user["_id"]
else:
_user_id = None
_result = []
# Loop through documents and remove them
_collection = self.database[_collection_name]
for _document in _documents:
# noinspection PyUnusedLocal
_result.append(_collection.remove(_document))
self.logging.log_remove(_document, _user_id)
return _result
def remove_condition(self, _condition, _user):
"""
Remove documents that match the supplied MBE condition
:param _condition: A MongoDB search criteria
:return:
"""
if _user is not None:
_user_id = _user["_id"]
else:
_user_id = None
_raw_condition, _collection = self.manage_input(_condition, "ref://of.conditions")
_removed_documents_cursor = _collection.find(_raw_condition)
_documents = [x for x in _removed_documents_cursor]
_result = []
for _document in _documents:
# noinspection PyUnusedLocal
_result.append(_collection.remove(_raw_condition))
self.logging.log_remove(_document, _user_id)
return _result
def find(self, _conditions, _do_not_fix_object_ids=False):
"""
Return a list of documents that match the supplied MBE condition.
:param _conditions: An MBE condition
:return: A list of matching documents
"""
def _recurse_object_ids(_data):
if isinstance(_data, list):
_destination = []
for _curr_row in _data:
_destination.append(_recurse_object_ids(_curr_row))
return _destination
elif isinstance(_data, dict):
_destination = {}
for _curr_key, _curr_value in _data.items():
_destination[_curr_key] = _recurse_object_ids(_curr_value)
return _destination
elif isinstance(_data, ObjectId):
return str(_data)
else:
return _data
_raw_conditions, _collection = self.manage_input(_conditions, "ref://of.conditions")
_result = list(_collection.find(_raw_conditions))
if not _do_not_fix_object_ids:
return _recurse_object_ids(_result)
else:
return _result
# noinspection PyMethodMayBeStatic
def transform_collection(self, _destination_schema, _map, _row_callback):
"""
NOT IMPLEMENTED.
:param _destination_schema:
:param _map:
:param _row_callback:
:return:
"""
pass
# Iterate all documents
# Copy _id to new document
# Copy all data using map and field id:s to document
# If set, use callback to finish what has to be manually done
# Write cumulative log entry using map changes and results from _row_callback
def start_agent(_cfg_filename = None):
"""
Starts the agent; Loads settings, connects to database, registers process and starts the web server.
"""
global process_id, _control_monitor, _terminated, _address, _process_queue_manager, _broker_url, \
_username, _password, _peers, _log_to_database_severity, _verify_SSL
_process_id = str(ObjectId())
of.common.logging.callback = log_locally
_terminated = False
# Handle multiprocessing on windows
freeze_support()
write_srvc_dbg("=====start_agent===============================")
try:
if _cfg_filename is None:
_cfg_filename = resolve_config_path()
_settings = JSONXPath(_cfg_filename)
except Exception as e:
write_to_log("Error loading settings: " + str(e), _category=EC_SERVICE, _severity=SEV_FATAL,
_process_id=_process_id)
return
of.common.logging.severity = of.common.logging.severity_identifiers.index(
_settings.get("agent/logging/severityLevel", _default="warning"))
_log_to_database_severity = of.common.logging.severity_identifiers.index(
_settings.get("agent/logging/brokerLevel", _default="warning"))
write_srvc_dbg("===register signal handlers===")
register_signals(stop_agent)
# An _address is completely necessary.
_address = _settings.get("agent/address", _default=None)
if not _address or _address == "":
raise Exception(write_to_log(
"Fatal error: Agent cannot start, missing [agent] _address setting in configuration file.",
_category=EC_SERVICE, _severity=SEV_FATAL))
# An _address is completely necessary.
_verify_SSL = _settings.get("agent/verifySSL", _default=True)
# Gather credentials
_broker_url = _settings.get("agent/brokerUrl", _default="127.0.0.1:8080")
_username = _settings.get("agent/username")
if not _username:
raise Exception(write_to_log("Username must be configured", _category=EC_SERVICE, _severity=SEV_FATAL))
_password = _settings.get("agent/password")
if not _password:
raise Exception(write_to_log("Password must be configured", _category=EC_SERVICE, _severity=SEV_FATAL))
_retries = int(_settings.get("agent/connectionRetries", 5))
# Register at the broker
if not register_agent(_retries):
raise Exception(write_to_log("Fatal: The agent failed to register with the broker, tried " + str(
_retries + 1) + " time(s), quitting.", _category=EC_SERVICE, _severity=SEV_FATAL))
os._exit(1)
of.common.logging.callback = log_to_database
_repository_base_folder = _settings.get_path("agent/repositoryFolder",
_default=os.path.join(os.path.dirname(__file__), "repositories"))
write_srvc_dbg("Load schema tool")
try:
# Initiate a schema tools instance for validation other purposes.
_schema_tools = SchemaTools(_json_schema_folders=[of_schema_folder(),
os.path.abspath(os.path.join(script_dir, "..", "schemas", "namespaces"))
],
_uri_handlers={"ref": None})
except Exception as e:
raise Exception(write_to_log("An error occurred while loading schema tools:" + str(e),
_category=EC_SERVICE, _severity=SEV_FATAL))
os._exit(1)
return
write_srvc_dbg("Load schema tool done")
try:
write_srvc_dbg("Initializing monitors")
# Init the monitor for incoming messages
_message_monitor = Monitor(
_handler=AgentWebSocketHandler(_process_id=_process_id,
_peers=_peers,
_schema_tools=_schema_tools,
_address=_address,
_broker_address="broker"))
# The manager for the process queue
_process_queue_manager = multiprocessing.Manager()
# Init the monitor for the worker queue
_worker_monitor = Monitor(
_handler=WorkerSupervisor(_process_id=_process_id,
_message_monitor=_message_monitor,
_repo_base_folder=_repository_base_folder,
_severity=of.common.logging.severity),
_queue=_process_queue_manager.Queue())
# Init the monitor for the agent queue
_control_monitor = Monitor(
_handler=ControlHandler(_process_id=_process_id,
_message_monitor=_message_monitor,
_worker_monitor=_worker_monitor,
_stop_agent=stop_agent
))
# The global variable for handling websockets. TODO: Could this be done without globals? (PROD-33)
of.common.messaging.websocket.monitor = _message_monitor
write_srvc_dbg("Initializing monitors done")
except Exception as e:
raise Exception(write_to_log("Fatal: An error occurred while initiating the monitors and handlers:" + str(e),
_category=EC_SERVICE, _severity=SEV_FATAL))
os._exit(1)
# Try to connect to websocket, quit on failure
if not connect_to_websocket():
os._exit(1)
write_srvc_dbg("Register agent system process")
_control_monitor.handler.message_monitor.queue.put(
[None, store_process_system_document(_process_id=_process_id,
_name="Agent instance(" + _address + ")")])
write_srvc_dbg("Log agent system state")
_control_monitor.handler.message_monitor.queue.put([None,
log_process_state_message(_changed_by=zero_object_id,
_state="running",
_process_id=_process_id,
_reason="Agent starting up at " +
_address)])
# Security check to remind broker if it is unsecured
if not _verify_SSL:
try:
call_api("https://"+ _broker_url + "/status", _data={}, _session_id= _session_id, _verify_SSL=True)
except SSLError as e:
write_to_log("There is a problem with the security certificate:\n" + str(e) + "\n"
"This is a security risk, and and important thing to _address.",
_category=EC_NOTIFICATION, _severity=SEV_WARNING)
except Exception as e:
write_to_log("An error occured while checking status of broker and SSL certificate:" + str(e),
_category=EC_NOTIFICATION, _severity=SEV_ERROR)
write_srvc_dbg("Agent up and running.")
while not _terminated:
time.sleep(0.1)
write_srvc_dbg("Exiting main thread")
