def remote_credentials(self):
"""
Returns credentials if present
Returns
-------
MASTER_SERVER_IP : str
return if present else None
DATMO_API_KEY : str
return if present else None
END_POINT : str
return if present else None
"""
# 1) Load from the environment if datmo config not already saved globally
MASTER_SERVER_IP = os.environ.get('MASTER_SERVER_IP', None)
DATMO_API_KEY = os.environ.get('DATMO_API_KEY', None)
# 2) loading the datmo config if present
datmo_config_filepath = os.path.join(os.path.expanduser("~"),
".datmo", "config")
if os.path.isfile(datmo_config_filepath):
datmo_config = JSONStore(datmo_config_filepath)
config_dict = datmo_config.to_dict()
if MASTER_SERVER_IP is None:
MASTER_SERVER_IP = config_dict.get('MASTER_SERVER_IP',
None)
if DATMO_API_KEY is None:
DATMO_API_KEY = config_dict.get('DATMO_API_KEY', None)
if MASTER_SERVER_IP:
END_POINT = 'http://' + MASTER_SERVER_IP + ':2083/api/v1'
else:
END_POINT = None
return MASTER_SERVER_IP, DATMO_API_KEY, END_POINT
def test_get_obj(self):
storage = JSONStore(self.storage_file)
key = 'foobar1'
value = {"does this work": "noway"}
storage.save(key, value)
return_value = storage.get(key)
assert return_value == value
def test_save(self):
storage = JSONStore(self.storage_file)
storage.save('foobar', 'yep')
found_it = False
if 'foobar' in open(self.storage_file).read():
found_it = True
assert found_it
def test_get_string(self):
storage = JSONStore(self.storage_file)
key = 'foobar1'
value = 'disco'
storage.save(key, value)
return_value = storage.get(key)
assert return_value == value
def _find_in_filecollection(self, file_to_find, file_collection_id):
""" Attempts to find a file within the file collection
Returns
-------
dict
output dictionary of the JSON file
"""
file_collection_obj = self.file_collection.dal.file_collection.\
get_by_id(file_collection_id)
file_collection_path = \
self.file_collection.file_driver.get_collection_path(
file_collection_obj.filehash)
# find all of the possible paths it could exist
possible_paths = [os.path.join(self.home, file_to_find)] + \
[os.path.join(self.home, item[0], file_to_find)
for item in os.walk(file_collection_path)]
existing_possible_paths = [
possible_path for possible_path in possible_paths
if os.path.isfile(possible_path)
]
if not existing_possible_paths:
# TODO: Add some info / warning that no file was found
# create some default stats
return {}
else:
# If any such path exists, transform file to stats dict
json_file = JSONStore(existing_possible_paths[0])
return json_file.to_dict()
def _config_setup(self, incoming_dictionary, create_dict):
""" Fills in snapshot config by having one of the following:
1. config = JSON object
2. config_filepath = some location where a json file exists
3. config_filename = just the file name
Parameters
----------
incoming_dictionary : dict
dictionary for the create function defined above
create_dict : dict
dictionary for creating the Snapshot entity
Raises
------
FileIOError
"""
if "config" in incoming_dictionary:
create_dict['config'] = incoming_dictionary['config']
elif "config_filepath" in incoming_dictionary:
if not os.path.isfile(incoming_dictionary['config_filepath']):
raise FileIOError(
__("error", "controller.snapshot.create.file_config"))
# If path exists transform file to config dict
config_json_driver = JSONStore(
incoming_dictionary['config_filepath'])
create_dict['config'] = config_json_driver.to_dict()
elif "config_filename" in incoming_dictionary:
config_filename = incoming_dictionary['config_filename']
create_dict['config'] = self._find_in_filecollection(
config_filename, create_dict['file_collection_id'])
else:
config_filename = "config.json"
create_dict['config'] = self._find_in_filecollection(
config_filename, create_dict['file_collection_id'])
def test_load_new_json_file(self):
new_json_filepath = os.path.join(self.temp_dir, "test.json")
with open(new_json_filepath, "wb") as f:
f.write(to_bytes(""))
# Test returns empty dict for empty file
storage = JSONStore(new_json_filepath)
assert storage.to_dict() == {}
# Test FileIOError if not JSON decodeable
with open(new_json_filepath, "wb") as f:
f.write(to_bytes("this is not json decodeable"))
failed = False
try:
storage = JSONStore(new_json_filepath)
storage.to_dict()
except FileIOError:
failed = True
assert failed
def __init__(self):
self._home = None
self.datmo_directory_name = ".datmo"
self.logging_level = logging.DEBUG
DatmoLogger.get_logger(__name__).info("initializing")
self.data_cache = JSONStore(
os.path.join(os.path.expanduser("~"), ".datmo", "cache.json"))
self.docker_cli = '/usr/bin/docker'
def _setup_compatible_environment(self,
create_dict,
paths,
directory,
save_hardware_file=True):
"""Setup compatible environment from user paths. Creates the necessary datmo files if
they are not already present
Parameters
----------
create_dict : dict
dictionary for entity creation, this is mutated in the function (not returned)
paths : list
list of absolute or relative filepaths and/or dirpaths to collect with destination names
(e.g. "/path/to/file>hello", "/path/to/file2", "/path/to/dir>newdir")
directory : str
path of directory to save additional files to
save_hardware_file : bool
boolean to save hardware file along with other files
(default is True to save the file and create distinct hashes based on software and hardware)
Returns
-------
paths : list
returns the input paths with the paths of the new files created appended
"""
# a. look for the default definition, if not present add it to the directory, and add it to paths
original_definition_filepath = ""
if all(create_dict['definition_filename'] not in path
for path in paths):
self.environment_driver.create_default_definition(directory)
original_definition_filepath = os.path.join(
directory, create_dict['definition_filename'])
paths.append(original_definition_filepath)
else:
for idx, path in enumerate(paths):
if create_dict['definition_filename'] in path:
src_path, dest_path = parse_path(path)
original_definition_filepath = src_path
# b. use the default definition and create a datmo definition in the directory, and add to paths
datmo_definition_filepath = \
os.path.join(directory, "datmo" + create_dict['definition_filename'])
if not os.path.isfile(datmo_definition_filepath):
_, original_definition_filepath, datmo_definition_filepath = \
self.environment_driver.create(path=original_definition_filepath, output_path=datmo_definition_filepath)
paths.append(datmo_definition_filepath)
# c. get the hardware info and save it to the entity, if save_hardware_file is True
# then save it to file and add it to the paths
create_dict[
'hardware_info'] = self.environment_driver.get_hardware_info()
if save_hardware_file:
hardware_info_filepath = os.path.join(directory, "hardware_info")
_ = JSONStore(hardware_info_filepath,
initial_dict=create_dict['hardware_info'])
paths.append(hardware_info_filepath)
return paths
def __init__(self):
self.home = Config().home
if not os.path.isdir(self.home):
raise InvalidProjectPath(
__("error", "controller.base.__init__", self.home))
self.config_store = JSONStore(
os.path.join(self.home, ".datmo", ".config"))
self.logger = DatmoLogger.get_logger(__name__)
# property caches and initial values
self._dal = None
self._model = None
self._current_session = None
self._code_driver = None
self._file_driver = None
self._environment_driver = None
self._is_initialized = False
def __init__(self, home):
self.home = home
if not os.path.isdir(self.home):
raise InvalidProjectPathException(__("error",
"controller.base.__init__",
home))
self.config = JSONStore(os.path.join(self.home,
".datmo",
".config"))
# property caches and initial values
self._dal = None
self._model = None
self._current_session = None
self._code_driver = None
self._file_driver = None
self._environment_driver = None
self._is_initialized = False
def configure(self):
"""
Configure datmo installation
"""
# General setup
setup_remote_bool = self.cli_helper.prompt_bool(
"Would you like to setup your remote credentials? [yN]")
if setup_remote_bool:
datmo_api_key = None
while not datmo_api_key:
datmo_api_key = self.cli_helper.prompt(
"Enter API key for datmo")
# Initialize remote API to get master ip address
remote_api = RemoteAPI(datmo_api_key)
response = remote_api.get_deployment_info()
master_system_info = response['body']['master_system_info']
master_server_ip = str(master_system_info.get('datmo_master_ip')) if isinstance(master_system_info, dict)\
else None
# Create a config file
self.datmo_config = JSONStore(
os.path.join(os.path.expanduser("~"), ".datmo", "config"))
config = dict()
if master_server_ip and datmo_api_key:
config["MASTER_SERVER_IP"] = master_server_ip
config["DATMO_API_KEY"] = datmo_api_key
self.datmo_config.to_file(config)
else:
self.cli_helper.echo(
"Datmo API key could not be saved. Please try again")
# Setup project specific things
if self.project_controller.model:
pass
else:
self.cli_helper.echo(
"No datmo project found. Skipping configuration for project.")
def _store_hardware_info(self, dictionary, create_dict, definition_path):
if "hardware_info" in dictionary:
create_dict['hardware_info'] = dictionary['hardware_info']
else:
# Extract hardware info of the container (currently taking from system platform)
# TODO: extract hardware information directly from the container
(system, node, release, version, machine,
processor) = platform.uname()
create_dict['hardware_info'] = {
'system': system,
'node': node,
'release': release,
'version': version,
'machine': machine,
'processor': processor
}
# Create hardware info file in definition path
hardware_info_filepath = os.path.join(definition_path, "hardware_info")
_ = JSONStore(hardware_info_filepath,
initial_dict=create_dict['hardware_info'])
return hardware_info_filepath
def init(self, name, description):
""" Initialize the project
This function will initialize the project or reinitialize it the project is
already initialized.
Parameters
----------
name : str
description : str
Returns
-------
bool
"""
is_new_model = False
old_model = self.model
if not self.model:
is_new_model = True
try:
# Always validate inputs to the init function
validate("create_project", {
"name": name,
"description": description
})
# Initialize File Driver if needed
if not self.file_driver.is_initialized:
self.file_driver.init()
# Initialize the dal
if not self.dal.is_initialized:
self.dal.init()
# Initialize Code Driver if needed
if not self.code_driver.is_initialized:
self.code_driver.init()
# Initialize Environment Driver if needed
if not self.environment_driver.is_initialized:
self.environment_driver.init()
# Initialize the config JSON store
self.config_store = JSONStore(
os.path.join(self.home,
Config().datmo_directory_name, ".config"))
# Create model if new else update
if is_new_model:
_ = self.dal.model.create(
Model({
"name": name,
"description": description
}))
else:
self._model = self.dal.model.update({
"id": self.model.id,
"name": name,
"description": description
})
# Connect Environment Driver if needed
# (not required but will warn if not present)
try:
if not self.environment_driver.is_connected:
self.environment_driver.connect()
except EnvironmentConnectFailed:
self.logger.warning(
__("warn", "controller.general.environment.failed"))
# Build the initial default Environment (NOT NECESSARY)
# self.environment_driver.build_image(tag="datmo-" + \
# self.model.name)
return True
except Exception:
# if any error occurred with new model, ensure no initialize occurs and raise previous error
# if any error occurred with existing model, ensure no updates were made, raise previous error
if is_new_model:
self.cleanup()
else:
self._model = self.dal.model.update({
"id": old_model.id,
"name": old_model.name,
"description": old_model.description
})
raise
def save_stats(self, filepath):
JSONStore(os.path.join(filepath, 'stats.json'), self.stats)
return
def save_config(self, filepath):
JSONStore(os.path.join(filepath, 'config.json'), self.config)
return
def __init__(self):
self._home = None
self.logging_level = logging.DEBUG
DatmoLogger.get_logger(__name__).info("initalizing")
self.data_cache = JSONStore(
os.path.join(os.path.expanduser("~"), ".datmo", "cache.json"))
def __save_dictionary(self, dictionary, path):
json_obj = JSONStore(path)
data = json_obj.to_dict()
data.update(dictionary)
json_obj.to_file(data)
return data
def test_init(self):
storage = JSONStore(self.storage_file)
assert storage.filepath == self.storage_file
