def test_multiple_loggers(self):
l1 = DatmoLogger.get_logger("logger1")
l2 = DatmoLogger.get_logger("logger2")
l1.info("pizza")
l2.info("party")
assert len(DatmoLogger.find_text_in_logs("pizza")) == 1
assert len(DatmoLogger.find_text_in_logs("logger2")) == 1
def __init__(self):
self._home = None
self.datmo_directory_name = ".datmo"
self.logging_level = logging.DEBUG
DatmoLogger.get_logger(__name__).info("initializing")
self.data_cache = JSONStore(
os.path.join(os.path.expanduser("~"), ".datmo", "cache.json"))
self.docker_cli = '/usr/bin/docker'
def test_multiple_loggers(self):
l1 = DatmoLogger.get_logger("logger1")
l2 = DatmoLogger.get_logger("logger2")
l1.info("pizza")
l2.info("party")
assert len(DatmoLogger.find_text_in_logs("pizza")) == 1
assert len(DatmoLogger.find_text_in_logs("logger2")) == 1
logpath = os.path.join(self.temp_dir, ".datmo", "logs", "log.txt")
assert os.path.isfile(logpath)
def test_find_text_in_logs(self):
logger = DatmoLogger.get_logger()
logger.info("can you find this")
results = DatmoLogger.find_text_in_logs("can you find this")
assert len(results) == 1
for r in results:
assert r["file"]
assert r["line_number"]
assert r["line"]
def test_new_logger_and_default(self):
random_name = str(uuid.uuid1())
logger = DatmoLogger.get_logger("logger3", random_name)
logger.info("testing")
assert len(DatmoLogger.find_text_in_logs("testing")) == 1
default_logger = DatmoLogger.get_logger()
default_logger.info("default-logger")
assert len(DatmoLogger.find_text_in_logs("default-logger")) == 1
logpath = os.path.join(self.temp_dir, ".datmo", "logs", random_name)
assert os.path.isfile(logpath)
def test_autocreate_dir(self):
log_path = os.path.join(os.path.expanduser("~"), '.datmo')
shutil.rmtree(log_path)
assert not os.path.exists(log_path)
logger = DatmoLogger.get_logger("logger3", "new.txt")
logger.info("testing")
assert len(DatmoLogger.find_text_in_logs("testing")) == 1
default_logger = DatmoLogger.get_logger()
default_logger.info("default-logger")
assert len(DatmoLogger.find_text_in_logs("default-logger")) == 1
def test_multiple_log_files(self):
l1 = DatmoLogger.get_logger("logger3", "debug.txt")
l2 = DatmoLogger.get_logger("logger3", "info.txt")
l1.info("green")
l2.info("purple")
r = DatmoLogger.find_text_in_logs("green")
assert len(r) == 1
assert r[0]["file"].find("debug.txt")
r = DatmoLogger.find_text_in_logs("purple")
assert len(r) == 1
assert r[0]["file"].find("info.txt")
def __init__(self):
from datmo.core.util.logger import DatmoLogger
from datmo.config import Config
self.config = Config()
self.docker_cli = self.config.docker_cli
self.log = DatmoLogger.get_logger(__name__)
self.log.info("handling command %s", self.config.home)
def test_multiple_log_files(self):
random_name_1 = str(uuid.uuid1())
random_name_2 = str(uuid.uuid1())
l1 = DatmoLogger.get_logger("logger3", random_name_1)
l2 = DatmoLogger.get_logger("logger3", random_name_2)
l1.info("green")
l2.info("purple")
r = DatmoLogger.find_text_in_logs("green")
assert len(r) == 1
assert r[0]["file"].find(random_name_1)
r = DatmoLogger.find_text_in_logs("purple")
assert len(r) == 1
assert r[0]["file"].find(random_name_2)
logpath = os.path.join(self.temp_dir, ".datmo", "logs", random_name_1)
assert os.path.isfile(logpath)
logpath = os.path.join(self.temp_dir, ".datmo", "logs", random_name_2)
assert os.path.isfile(logpath)
def test_timeit_decorator(self):
# NOTE: If this test is failing be sure to add
# LOGGING_LEVEL=DEBUG before pytest
# or add as an environment variable
@DatmoLogger.timeit
def slow_fn():
sleep(1)
slow_fn()
assert len(DatmoLogger.find_text_in_logs("slow_fn")) == 1
def main():
cli_helper = Helper()
# Config is required to run first so it can
# initialize/find datmo home directory (.datmo)
# This is required for logging to place the logs in a
# place for the user.
config = Config()
log = DatmoLogger.get_logger(__name__)
log.info("handling command %s", config.home)
# parse_args defaults to [1:] for args, but you need to
# exclude the rest of the args too, or validation will fail
# args = parser.parse_args(sys.argv[1:2])
if len(sys.argv) > 1 and \
sys.argv[1] in cli_helper.get_command_choices():
command_name = sys.argv[1]
if command_name == "init":
command_name = "project"
elif command_name == "version" or \
command_name == "--version" or \
command_name == "-v":
command_name = "project"
sys.argv[1] = "version"
elif command_name == "status":
command_name = "project"
sys.argv[1] = "status"
elif command_name == "cleanup":
command_name = "project"
sys.argv[1] = "cleanup"
command_class = cli_helper.get_command_class(command_name)
else:
command_class = BaseCommand
# instantiate the command class
try:
command_instance = command_class(os.getcwd(), cli_helper)
except TypeError as ex:
cli_helper.echo(__("error", "cli.general", str(ex)))
return 1
# parse the command line arguments
try:
command_instance.parse(sys.argv[1:])
except CLIArgumentException as ex:
cli_helper.echo(__("error", "cli.general", str(ex)))
return 1
try:
command_instance.execute()
return 0
except Exception as ex:
cli_helper.echo(__("error", "cli.general", str(ex)))
return 1
def setup_class(self):
# provide mountable tmp directory for docker
tempfile.tempdir = "/tmp" if not platform.system(
) == "Windows" else None
test_datmo_dir = os.environ.get('TEST_DATMO_DIR',
tempfile.gettempdir())
self.temp_dir = tempfile.mkdtemp(dir=test_datmo_dir)
# Delete previous singletons for logger if present
if hasattr(DatmoLogger, "instance"):
del DatmoLogger.instance
DatmoLogger(dirpath=self.temp_dir)
def test_timeit_decorator(self):
# NOTE: If this test is failing be sure to add
# LOGGING_LEVEL=DEBUG before pytest
# or add as an environment variable
@DatmoLogger.timeit
def slow_fn():
sleep(1)
slow_fn()
assert len(DatmoLogger.find_text_in_logs("slow_fn")) >= 1
logpath = os.path.join(self.temp_dir, ".datmo", "logs", "timers.log")
assert os.path.isfile(logpath)
def __init__(self, home=None):
self.home = Config().home if not home else home
if not os.path.isdir(self.home):
raise InvalidProjectPath(
__("error", "controller.base.__init__", self.home))
self.logger = DatmoLogger.get_logger(__name__)
# property caches and initial values
self._is_initialized = False
self._dal = None
self._model = None
self._code_driver = None
self._file_driver = None
self._environment_driver = None
def __init__(self):
self.home = Config().home
if not os.path.isdir(self.home):
raise InvalidProjectPath(
__("error", "controller.base.__init__", self.home))
self.config_store = JSONStore(
os.path.join(self.home, ".datmo", ".config"))
self.logger = DatmoLogger.get_logger(__name__)
# property caches and initial values
self._dal = None
self._model = None
self._current_session = None
self._code_driver = None
self._file_driver = None
self._environment_driver = None
self._is_initialized = False
def __init__(self, cli_helper):
self.home = Config().home
self.cli_helper = cli_helper
self.logger = DatmoLogger.get_logger(__name__)
self.parser = get_datmo_parser()
"""
Bring in all of Datmo's public python interfaces
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
from datmo.core.util.logger import DatmoLogger
from datmo.config import Config
datmo_root = os.path.dirname(os.path.abspath(__file__))
with open(os.path.join(datmo_root, 'VERSION')) as file:
__version__ = file.read()
# Config is required to run first so it can
# initialize/find datmo home directory (.datmo)
# This is required for logging to place the logs in a
# place for the user.
config = Config()
config.set_home(os.getcwd())
log = DatmoLogger.get_logger(__name__)
log.info("handling command %s", config.home)
import datmo.snapshot
import datmo.config
def __init__(self, home, cli_helper):
super(TaskCommand, self).__init__(home, cli_helper)
self.logger = DatmoLogger.get_logger(__name__)
self.task_controller = TaskController(home=home)
def __init__(self):
self._home = None
self.logging_level = logging.DEBUG
DatmoLogger.get_logger(__name__).info("initalizing")
self.data_cache = JSONStore(
os.path.join(os.path.expanduser("~"), ".datmo", "cache.json"))
def test_logger(self):
logger = DatmoLogger.get_logger()
random_text = str(uuid.uuid1())
logger.warning(random_text)
logger.error("WHAT?")
assert logger.level == DatmoLogger(dirpath=self.temp_dir).logging_level
def test_get_logger_should_return_same_logger(self):
f = DatmoLogger.get_logger("foobar")
f2 = DatmoLogger.get_logger("foobar")
assert f == f2
logpath = os.path.join(self.temp_dir, ".datmo", "logs", "log.txt")
assert os.path.isfile(logpath)
def test_datmo_logger_get_logfiles(self):
files = DatmoLogger.get_logfiles()
assert len(list(files)) > 0
for f in files:
assert DatmoLogger().logging_path in f
def test_get_logger_should_return_same_logger(self):
f = DatmoLogger.get_logger("foobar")
f2 = DatmoLogger.get_logger("foobar")
assert f == f2
def main():
cli_helper = Helper()
# Config is required to run first so it can
# initialize/find datmo home directory (.datmo)
# This is required for logging to place the logs in a
# place for the user.
config = Config()
config.set_home(os.getcwd())
log = DatmoLogger.get_logger(__name__)
log.info("handling command %s", config.home)
# parse_args defaults to [1:] for args, but you need to
# exclude the rest of the args too, or validation will fail
# args = parser.parse_args(sys.argv[1:2])
if len(sys.argv) > 1 and \
sys.argv[1] in cli_helper.get_command_choices():
command_name = sys.argv[1]
# commands in project.py
if command_name == "init":
command_name = "project"
elif command_name == "version" or \
command_name == "--version" or \
command_name == "-v":
command_name = "project"
sys.argv[1] = "version"
elif command_name == "status":
command_name = "project"
sys.argv[1] = "status"
elif command_name == "cleanup":
command_name = "project"
sys.argv[1] = "cleanup"
# commands in workspace.py
elif command_name in ["notebook", "jupyterlab", "terminal", "rstudio"]:
sys.argv[1] = command_name
command_name = "workspace"
# commands in run.py
elif command_name == "rerun":
command_name = "run"
sys.argv[1] = "rerun"
elif command_name == "run":
if len(sys.argv) == 2:
command_name = "run"
sys.argv.append("--help")
else:
command_name = "run"
elif command_name == "stop": # stop command in run.py
if len(sys.argv) == 2:
command_name = "run"
sys.argv.append("--help")
else:
command_name = "run"
elif command_name == "ls": # ls command in run.py
command_name = "run"
elif command_name == "delete": # delete command in run.py
command_name = "run"
command_class = cli_helper.get_command_class(command_name)
elif len(sys.argv) == 1:
command_name = "datmo_command"
command_class = cli_helper.get_command_class(command_name)
else:
command_class = BaseCommand
# instantiate the command class
try:
command_instance = command_class(cli_helper)
except TypeError as ex:
cli_helper.echo(__("error", "cli.general", "%s %s" % (type(ex), ex)))
return 1
# parse the command line arguments
try:
command_instance.parse(sys.argv[1:])
except CLIArgumentError as ex:
cli_helper.echo(__("error", "cli.general", "%s %s" % (type(ex), ex)))
return 1
try:
command_instance.execute()
return 0
except Exception as ex:
cli_helper.echo(__("error", "cli.general", "%s %s" % (type(ex), ex)))
return 1
def test_logger(self):
logger = DatmoLogger.get_logger()
logger.warning("testing")
logger.error("WHAT?")
assert logger.level == DatmoLogger().logging_level