def test_get_by_id_model_same_dir(self):
test_dir = "test-dir"
datadriver = BlitzDBDALDriver("file", test_dir)
dal = LocalDAL(datadriver)
model1 = dal.model.create(Model(self.model_input_dict))
del datadriver
del dal
datadriver = BlitzDBDALDriver("file", test_dir)
dal = LocalDAL(datadriver)
model2 = dal.model.create(Model(self.model_input_dict))
del datadriver
del dal
datadriver = BlitzDBDALDriver("file", test_dir)
dal = LocalDAL(datadriver)
model3 = dal.model.create(Model(self.model_input_dict))
model1 = dal.model.get_by_id(model1.id)
model2 = dal.model.get_by_id(model2.id)
model3 = dal.model.get_by_id(model3.id)
assert model1
assert model2
assert model3
shutil.rmtree(test_dir)
def test_query_models_range_query(self):
_ = self.dal.model.create(Model(self.model_input_dict))
_ = self.dal.model.create(Model(self.model_input_dict))
_ = self.dal.model.create(Model(self.model_input_dict))
models = self.dal.model.query({},
sort_key="created_at",
sort_order="descending")
result = self.dal.model.query({
"created_at": {
"$lt": models[1].created_at.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
}
})
assert len(models) == 3
assert len(result) == 1
def test_query_models_multiple(self):
# Query for multiple models
model_1 = self.dal.model.create(Model(self.model_input_dict))
model_2 = self.dal.model.create(Model(self.model_input_dict))
model_3 = self.dal.model.create(Model(self.model_input_dict))
results = self.dal.model.query({},
sort_key="created_at",
sort_order="ascending")
assert len(results) == 3
assert results[0].created_at == model_1.created_at
assert results[1].created_at == model_2.created_at
results = self.dal.model.query({},
sort_key="created_at",
sort_order="descending")
assert len(results) == 3
assert results[0].created_at == model_3.created_at
assert results[1].created_at == model_2.created_at
# Wrong order being passed in
failed = False
try:
_ = self.dal.model.query({},
sort_key='created_at',
sort_order='wrong_order')
except InvalidArgumentType:
failed = True
assert failed
# Wrong key and order being passed in
failed = False
try:
_ = self.dal.model.query({},
sort_key='wrong_key',
sort_order='wrong_order')
except InvalidArgumentType:
failed = True
assert failed
# wrong key and right order being passed in
expected_items = self.dal.model.query({},
sort_key='created_at',
sort_order='ascending')
items = self.dal.model.query({},
sort_key='wrong_key',
sort_order='ascending')
expected_ids = [item.id for item in expected_items]
ids = [item.id for item in items]
assert set(expected_ids) == set(ids)
def test_create_model_by_dictionary(self):
model = self.dal.model.create(Model(self.model_input_dict))
assert model.id
assert model.name == self.model_input_dict['name']
assert model.created_at
assert model.updated_at
model_2 = self.dal.model.create(Model(self.model_input_dict))
assert model.id != model_2.id
test_model_input_dict = self.model_input_dict.copy()
test_model_input_dict['id'] = "cool"
model_3 = self.dal.model.create(Model(test_model_input_dict))
assert model_3.id == test_model_input_dict['id']
def setup_method(self):
# provide mountable tmp directory for docker
tempfile.tempdir = "/tmp" if not platform.system(
) == "Windows" else None
test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
tempfile.gettempdir())
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
self.datadriver = BlitzDBDALDriver("file", self.temp_dir)
self.dal = LocalDAL(self.datadriver)
model_name = "model_1"
model = self.dal.model.create(Model({"name": model_name}))
session_name = "session_1"
session = self.dal.session.create(
Session({
"name": session_name,
"model_id": model.id
}))
self.snapshot_input_dict = {
"model_id": model.id,
"session_id": session.id,
"message": "my message",
"code_id": "code_id",
"environment_id": "environment_id",
"file_collection_id": "file_collection_id",
"config": {
"test": 0.45
},
"stats": {
"test": 0.98
}
}
def setup_method(self):
# provide mountable tmp directory for docker
tempfile.tempdir = "/tmp" if not platform.system(
) == "Windows" else None
test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
tempfile.gettempdir())
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
self.datadriver = BlitzDBDALDriver("file", self.temp_dir)
self.dal = LocalDAL(self.datadriver)
model_name = "model_1"
model = self.dal.model.create(Model({"name": model_name}))
session_name = "session_1"
session = self.dal.session.create(
Session({
"name": session_name,
"model_id": model.id
}))
self.task_input_dict = {
"model_id": model.id,
"session_id": session.id,
"command": "task_1",
"start_time": datetime.utcnow(),
"end_time": datetime.utcnow(),
"duration": 0.004,
}
def test_current_session(self):
# Test failure case
failed = False
try:
_ = self.base_controller.current_session
except DatmoModelNotInitialized:
failed = True
assert failed
# Test success case
self.base_controller.dal.model.create(
Model({
"name": "test",
"description": "test"
}))
_ = self.base_controller.model
self.base_controller.dal.session.create(
Session({
"name": "test",
"model_id": "test",
"current": True
}))
session = self.base_controller.current_session
assert session.id
assert session.name == "test"
assert session.model_id == "test"
assert session.current == True
def setup_method(self):
# provide mountable tmp directory for docker
tempfile.tempdir = "/tmp" if not platform.system(
) == "Windows" else None
test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
tempfile.gettempdir())
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
self.driver_type = "blitzdb"
self.driver_options = {
"driver_type": "file",
"connection_string": self.temp_dir
}
self.dal = LocalDAL(self.driver_type, self.driver_options)
model_name = "model_1"
model = self.dal.model.create(Model({"name": model_name}))
self.environment_input_dict = {
"model_id": model.id,
"driver_type": "docker",
"file_collection_id": "test_file_id",
"definition_filename": "Dockerfile",
"hardware_info": {
"system": "macosx"
},
"unique_hash": "slkdjfa23dk",
"language": "python3"
}
def test_init_with_id(self):
self.input_dict['id'] = "test"
model_entity = Model(self.input_dict)
for k, v in self.input_dict.items():
assert getattr(model_entity, k) == v
assert model_entity.description == None
assert model_entity.created_at
assert model_entity.updated_at
def test_get_by_id_model_new_driver_instance(self):
model = self.dal.model.create(Model(self.model_input_dict))
# create new dal with new driver instance (success)
new_driver_instance = BlitzDBDALDriver("file", self.temp_dir)
new_dal_instance = LocalDAL(new_driver_instance)
new_model_1 = new_dal_instance.model.get_by_id(model.id)
assert new_model_1.id == model.id
# create new dal instance with same driver (success)
new_dal_instance = LocalDAL(self.datadriver)
new_model_2 = new_dal_instance.model.get_by_id(model.id)
assert new_model_2.id == model.id
def test_delete_model(self):
model = self.dal.model.create(Model(self.model_input_dict))
self.dal.model.delete(model.id)
deleted = False
try:
self.dal.model.get_by_id(model.id)
except EntityNotFound:
deleted = True
assert deleted
def test_update_model(self):
model = self.dal.model.create(Model(self.model_input_dict))
# Update required and optional parameters
updated_model_input_dict = self.model_input_dict.copy()
updated_model_input_dict['id'] = model.id
updated_model_input_dict['name'] = "model_4a"
updated_model_input_dict['description'] = "this is a cool model"
updated_model = self.dal.model.update(updated_model_input_dict)
assert model.id == updated_model.id
assert model.updated_at < updated_model.updated_at
assert updated_model.name == updated_model_input_dict['name']
assert updated_model.description == updated_model_input_dict[
'description']
def setup_class(self):
# provide mountable tmp directory for docker
tempfile.tempdir = "/tmp" if not platform.system(
) == "Windows" else None
test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
tempfile.gettempdir())
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
self.datadriver = BlitzDBDALDriver("file", self.temp_dir)
self.dal = LocalDAL(self.datadriver)
model_name = "model_1"
model = self.dal.model.create(Model({"name": model_name}))
self.session_input_dict = {"name": "session_1", "model_id": model.id}
def test_datmo_model(self):
# Test failure case
assert self.base_controller.model == None
# Test success case
self.base_controller.dal.model.create(
Model({
"name": "test",
"description": "test"
}))
model = self.base_controller.model
assert model.id
assert model.name == "test"
assert model.description == "test"
def setup_method(self):
# provide mountable tmp directory for docker
tempfile.tempdir = "/tmp" if not platform.system(
) == "Windows" else None
test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
tempfile.gettempdir())
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
self.datadriver = BlitzDBDALDriver("file", self.temp_dir)
self.dal = LocalDAL(self.datadriver)
model_name = "model_1"
model = self.dal.model.create(Model({"name": model_name}))
self.file_collection_input_dict = {
"model_id": model.id,
"driver_type": "local",
"filehash": "myhash",
"path": "test_path",
}
def setup_method(self):
# provide mountable tmp directory for docker
tempfile.tempdir = "/tmp" if not platform.system(
) == "Windows" else None
test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
tempfile.gettempdir())
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
self.driver_type = "blitzdb"
self.driver_options = {
"driver_type": "file",
"connection_string": self.temp_dir
}
self.dal = LocalDAL(self.driver_type, self.driver_options)
model_name = "model_1"
model = self.dal.model.create(Model({"name": model_name}))
self.code_input_dict = {
"model_id": model.id,
"driver_type": "git",
"commit_id": "commit_id"
}
def init(self, name, description):
""" Initialize the project
This function will initialize the project or reinitialize it the project is
already initialized.
Parameters
----------
name : str
description : str
Returns
-------
bool
"""
is_new_model = False
old_model = self.model
if not self.model:
is_new_model = True
try:
# Always validate inputs to the init function
validate("create_project", {
"name": name,
"description": description
})
# Initialize File Driver if needed
if not self.file_driver.is_initialized:
self.file_driver.init()
# Initialize the dal
if not self.dal.is_initialized:
self.dal.init()
# Initialize Code Driver if needed
if not self.code_driver.is_initialized:
self.code_driver.init()
# Initialize Environment Driver if needed
if not self.environment_driver.is_initialized:
self.environment_driver.init()
# Initialize the config JSON store
self.config_store = JSONStore(
os.path.join(self.home,
Config().datmo_directory_name, ".config"))
# Create model if new else update
if is_new_model:
_ = self.dal.model.create(
Model({
"name": name,
"description": description
}))
else:
self._model = self.dal.model.update({
"id": self.model.id,
"name": name,
"description": description
})
# Connect Environment Driver if needed
# (not required but will warn if not present)
try:
if not self.environment_driver.is_connected:
self.environment_driver.connect()
except EnvironmentConnectFailed:
self.logger.warning(
__("warn", "controller.general.environment.failed"))
# Build the initial default Environment (NOT NECESSARY)
# self.environment_driver.build_image(tag="datmo-" + \
# self.model.name)
return True
except Exception:
# if any error occurred with new model, ensure no initialize occurs and raise previous error
# if any error occurred with existing model, ensure no updates were made, raise previous error
if is_new_model:
self.cleanup()
else:
self._model = self.dal.model.update({
"id": old_model.id,
"name": old_model.name,
"description": old_model.description
})
raise
def init(self, name, description):
# Error if name is not given
if not name:
raise RequiredArgumentMissing(
__("error", "controller.project.init.arg", "name"))
# Create the Model, is it new or update?
is_new_model = False
if not self.model:
_ = self.dal.model.create(
Model({
"name": name,
"description": description
}))
is_new_model = True
else:
self._model = self.dal.model.update({
"id": self.model.id,
"name": name,
"description": description
})
# Initialize Code Manager if needed
if not self.code_driver.is_initialized:
self.code_driver.init()
# Initialize File Manager if needed
if not self.file_driver.is_initialized:
self.file_driver.init()
# Initialize Environment Manager if needed
if not self.environment_driver.is_initialized:
self.environment_driver.init()
# Build the initial default Environment (NOT NECESSARY)
# self.environment_driver.build_image(tag="datmo-" + \
# self.model.name)
# Add in Project template files if specified
# TODO: Add in project template files
# Create and set current session
if is_new_model:
# Create new default session
_ = self.dal.session.create(
Session({
"name": "default",
"model_id": self.model.id,
"current": True
}))
else:
if not self.current_session:
default_session_obj = self.dal.session.query({
"name":
"default",
"model_id":
self.model.id
})
if not default_session_obj:
raise SessionDoesNotExistException(
__("error", "controller.project.init"))
# Update default session to be current
self.dal.session.update({
"id": default_session_obj.id,
"current": True
})
return True
def test_eq(self):
model_entity_1 = Model(self.input_dict)
model_entity_2 = Model(self.input_dict)
assert model_entity_1 == model_entity_2
def init(self, name, description):
""" Initialize the project
This function will initialize the project or reinitialize it the project is
already initialized.
Parameters
----------
name : str
description : str
Returns
-------
bool
Raises
------
SessionDoesNotExist
"""
# Create the Model, is it new or update?
is_new_model = False
old_model = self.model
if not self.model:
is_new_model = True
try:
# Always validate inputs to the init function
validate("create_project", {
"name": name,
"description": description
})
# Create model if new else update
if is_new_model:
_ = self.dal.model.create(
Model({
"name": name,
"description": description
}))
else:
self._model = self.dal.model.update({
"id": self.model.id,
"name": name,
"description": description
})
# Initialize Code Driver if needed
if not self.code_driver.is_initialized:
self.code_driver.init()
# Initialize File Driver if needed
if not self.file_driver.is_initialized:
self.file_driver.init()
# Initialize Environment Driver if needed
# (not required but will warn if not present)
try:
if not self.environment_driver.is_initialized:
self.environment_driver.init()
except EnvironmentInitFailed:
self.logger.warning(
__("warn", "controller.general.environment.failed"))
# Build the initial default Environment (NOT NECESSARY)
# self.environment_driver.build_image(tag="datmo-" + \
# self.model.name)
# Create and set current session
if is_new_model:
# Create new default session
_ = self.dal.session.create(
Session({
"name": "default",
"model_id": self.model.id,
"current": True
}))
else:
if not self.current_session:
default_session_objs = self.dal.session.query({
"name": "default",
"model_id": self.model.id
})
if not default_session_objs:
# Creating a default session since none exists
_ = self.dal.session.create(
Session({
"name": "default",
"model_id": self.model.id,
"current": True
}))
else:
# Update default session to be current
default_session_obj = default_session_objs[0]
self.dal.session.update({
"id": default_session_obj.id,
"current": True
})
return True
except Exception:
# if any error occurred with new model, ensure no initialize occurs and raise previous error
# if any error occurred with existing model, ensure no updates were made, raise previous error
if is_new_model:
self.cleanup()
else:
self._model = self.dal.model.update({
"id": old_model.id,
"name": old_model.name,
"description": old_model.description
})
raise
def test_to_dictionary(self):
model_entity = Model(self.input_dict)
output_dict = model_entity.to_dictionary()
for k, v in output_dict.items():
assert v == getattr(model_entity, k)
def test_sanity_check_for_dal(self):
model = self.base_controller.dal.model.create(Model({"name": "test"}))
model2 = self.base_controller.dal.model.get_by_id(model.id)
assert model and model2
assert model.id == model2.id
def test_query_models(self):
model = self.dal.model.create(Model(self.model_input_dict))
assert len(self.dal.model.query({"id": model.id})) == 1
assert len(
self.dal.model.query({"name": self.model_input_dict['name']})) == 6
def test_get_by_id_model(self):
model = self.dal.model.create(Model(self.model_input_dict))
result = self.dal.model.get_by_id(model.id)
assert model.id == result.id