def setUp(self):
ccdb_path = get_ccdb_home_path()
self.sqlite_connection_str = "sqlite:///" + os.path.join(
ccdb_path, "sql", "ccdb.sqlite")
self.mysql_connection_str = "mysql://[email protected]:3306/ccdb"
self.provider = AlchemyProvider()
def __init__(self):
self._prov = AlchemyProvider()
self._is_interactive = False
self.user_name = self.anonymous_user_name
self._is_interactive = False
self._current_run = 0
self._current_path = "/"
self._current_variation = "default"
self._utils = {}
self._verbose = False
self._ls = None
self._connection_string = ""
self.silent_exceptions = True # rethrow happened exceptions
self._theme = themes.NoColorTheme()
self.log_sqlite = False
class AuthenticationTest(unittest.TestCase):
def setUp(self):
ccdb_path = get_ccdb_home_path()
self.sqlite_connection_str = "sqlite:///" + os.path.join(
ccdb_path, "sql", "ccdb.sqlite")
self.mysql_connection_str = "mysql://[email protected]:3306/ccdb"
self.provider = AlchemyProvider()
def test_environment_auth(self):
self.provider.connect(self.sqlite_connection_str)
if "CCDB_USER" in os.environ: del os.environ["CCDB_USER"]
if "USER" in os.environ: del os.environ["USER"]
os.environ["CCDB_USER"] = "******"
auth = EnvironmentAuthentication(self.provider)
self.assertEqual(auth.current_user_name, "test_user")
self.assertTrue(auth.validate_current_user())
class AuthenticationTest(unittest.TestCase):
def setUp(self):
ccdb_path = get_ccdb_home_path()
self.sqlite_connection_str = "sqlite:///" + os.path.join(ccdb_path, "sql", "ccdb.sqlite")
self.mysql_connection_str = "mysql://[email protected]:3306/ccdb"
self.provider = AlchemyProvider()
def test_environment_auth(self):
self.provider.connect(self.sqlite_connection_str)
if "CCDB_USER" in os.environ: del os.environ["CCDB_USER"]
if "USER" in os.environ: del os.environ["USER"]
os.environ["CCDB_USER"] = "******"
auth = EnvironmentAuthentication(self.provider)
self.assertEqual(auth.current_user_name, "test_user")
self.assertTrue(auth.validate_current_user())
def __init__(self):
self._prov = AlchemyProvider()
self._is_interactive = False
self.user_name = self.anonymous_user_name
self._is_interactive = False
self._current_run = 0
self._current_path = "/"
self._current_variation = "default"
self._utils = {}
self._verbose = False
self._ls = None
self._connection_string = ""
self.silent_exceptions = True # rethrow happened exceptions
self._theme = themes.NoColorTheme()
self.log_sqlite = False
def setUp(self):
self._provider = AlchemyProvider()
self.provider.logging_enabled = False
self.provider.authentication.current_user_name = "test_user"
class AlchemyProviderTest(unittest.TestCase):
ccdb_path = get_ccdb_home_path()
_connection_str = helper.sqlite_test_connection_str
_provider = AlchemyProvider()
@property
def provider(self):
return self._provider
@property
def connection_str(self):
return self._connection_str
@connection_str.setter
def connection_str(self, connection_str):
self._connection_str = connection_str
def setUp(self):
self._provider = AlchemyProvider()
self.provider.logging_enabled = False
self.provider.authentication.current_user_name = "test_user"
def test_connection(self):
""" Tests that provider connects successfully"""
self.provider.connect(self.connection_str)
def test_connect_to_old_schema(self):
""" Test connection to schema with wrong version """
ccdb_path = get_ccdb_home_path()
old_schema_cs = "sqlite:///" + os.path.join(ccdb_path, "python", "tests", "old_schema.ccdb.sqlite")
self.assertRaises(DatabaseStructureError, self.provider.connect, old_schema_cs)
def test_directories(self):
""" Test of directories"""
self.provider.connect(self.connection_str) # this test requires the connection
# simple get directory
dir_obj = self.provider.get_directory("/test")
self.assertIsNotNone(dir_obj)
self.assertMultiLineEqual(dir_obj.path, "/test")
self.assertMultiLineEqual(dir_obj.name, "test")
# search directories
dirs = self.provider.search_directories("t??t_va*", "/test")
assert (len(dirs) != 0)
dirs = self.provider.search_directories("*", "/test")
assert (len(dirs) >= 2)
dirs = self.provider.search_directories("*", "")
assert (len(dirs) >= 2)
# cleanup directories
# Ok, lets check if directory for the next text exists...
try:
self.provider.delete_directory("/test/testdir/constants")
except DirectoryNotFound:
pass
try:
self.provider.delete_directory("/test/testdir")
except DirectoryNotFound:
pass
# cleanup directories
# Ok, lets check if directory for the next text exists...
dir_obj = self.provider.create_directory("testdir", "/test")
self.assertIsNotNone(dir_obj)
self.provider.logging_enabled = True # enable logging to test log too
# create subdirectory
constants_subdir = self.provider.create_directory("constants", "/test/testdir", "My constants")
self.assertIsNotNone(constants_subdir)
self.assertEqual(constants_subdir.comment, "My constants")
# check log
log = self.provider.get_log_records(limit=1)[0]
assert (isinstance(log, LogRecord))
self.assertEqual(log.action, "create")
self.assertEqual(log.affected_ids, "|directories" + str(constants_subdir.id) + "|")
self.assertEqual(log.comment, "My constants")
self.assertIn("Created directory", log.description)
self.provider.logging_enabled = False
# cannot recreate subdirectory
self.assertRaises(ValueError, self.provider.create_directory, "constants", "/test/testdir", "My constants")
# create another subdirectory
variables_subdir = self.provider.create_directory("variables", "/test/testdir", "My constants")
# test delete
self.provider.delete_directory("/test/testdir/constants")
# test can't delete dir with sub dirs
self.assertRaises(ValueError, self.provider.delete_directory, "/test/testdir")
# test delete by object
self.provider.delete_directory(variables_subdir)
# now, when dir doesn't have sub dirs and sub tables, it can be deleted
self.provider.delete_directory("/test/testdir")
# noinspection PyBroadException
def test_type_tables(self):
"""
Test type table operation
@return: None
"""
self.provider.connect(self.connection_str) # this test requires the connection
table = self.provider.get_type_table("/test/test_vars/test_table")
assert table is not None
self.assertEqual(len(table.columns), 3)
assert table.name == "test_table"
assert table.path == "/test/test_vars/test_table"
assert table.parent_dir
assert table.parent_dir.name == "test_vars"
assert table.columns[0].name == "x"
# get all tables in directory
tables = self.provider.get_type_tables("/test/test_vars")
assert len(tables) >= 2 # at least 2 tables are located in "/test/test_vars"
# count tables in a directory
assert self.provider.count_type_tables("/test/test_vars") >= 2
# SEARCH TYPE TABLES
# basic search type table functional
tables = self.provider.search_type_tables("t??t_tab*")
self.assertNotEqual(len(tables), 0)
self.assertIn("/", tables[0].path)
# now lets get all tables from the directory.
tables = self.provider.search_type_tables("*", "/test/test_vars")
self.assertNotEqual(len(tables), 0)
for table in tables:
self.assertEqual(table.path, "/test/test_vars" + "/" + table.name)
# now lets get all tables from root directory.
tables = self.provider.search_type_tables("t*", "/")
self.assertEquals(len(tables), 0)
# CREATE AND DELETE
try:
# if such type table already exists.. probably from last failed test...
# we haven't test it yet, but we should try to delete it
table = self.provider.get_type_table("/test/test_vars/new_table")
self.provider.delete_type_table(table)
except:
pass
table = self.provider.create_type_table(
name="new_table",
dir_obj_or_path="/test/test_vars",
rows_num=5,
columns=[('c', 'double'), ('a', 'double'), ('b', 'int')],
comment="This is temporary created table for test reasons")
self.assertIsNotNone(table)
table = self.provider.get_type_table("/test/test_vars/new_table")
self.assertEqual(table.rows_count, 5)
self.assertEqual(table.columns_count, 3)
self.assertEqual(table.name, 'new_table')
self.assertEqual(table.columns[0].name, 'c')
self.assertEqual(table.columns[0].type, 'double')
self.assertEqual(table.columns[1].name, 'a')
self.assertEqual(table.columns[1].type, 'double')
self.assertEqual(table.columns[2].name, 'b')
self.assertEqual(table.columns[2].type, 'int')
self.assertEqual(table.comment, "This is temporary created table for test reasons")
# delete
self.provider.delete_type_table(table)
self.assertRaises(TypeTableNotFound, self.provider.get_type_table, "/test/test_vars/new_table")
def test_run_ranges(self):
"""Test run ranges """
self.provider.connect(self.connection_str) # this test requires the connection
# Get run range by name, test "all" run range
rr = self.provider.get_named_run_range("all")
self.assertIsNotNone(rr)
# Get run range by min and max run values
rr = self.provider.get_run_range(0, 2000)
self.assertIsNotNone(rr)
# NON EXISTENT RUN RANGE
# ----------------------------------------------------
# Get run range that is not defined
try:
rr = self.provider.get_run_range(0, 2001)
# oh... such run range exists? It shouldn't be... Maybe it is left because of the last tests...
print ("WARNING provider.get_run_range(0, 2001) found run range (should not be there)")
print ("trying to delete run range and run the test one more time... ")
self.provider.delete_run_range(rr) # (!) <-- test of this function is further
rr = self.provider.get_run_range(0, 2001)
self.assertIsNotNone(rr)
except RunRangeNotFound:
pass # test passed
# GET OR CREATE RUNRANGE
# ----------------------------------------------------
# Get or create run-range is the main function to get RunRange without name
# 0-2001 should be absent or deleted so this function will create run-range
rr = self.provider.get_or_create_run_range(0, 2001)
self.assertIsNotNone(rr)
self.assertNotEquals(rr.id, 0)
self.assertEquals(rr.min, 0)
self.assertEquals(rr.max, 2001)
# DELETE RUN-RANGE TEST
# ----------------------------------------------------
self.provider.delete_run_range(rr)
self.assertRaises(RunRangeNotFound, self.provider.get_run_range, 0, 2001)
def test_variations(self):
"""Test variations"""
self.provider.connect(self.connection_str) # this test requires the connection
# Get variation by name, test "all" run range
v = self.provider.get_variation("default")
self.assertIsNotNone(v)
# Get variations by type table
table = self.provider.get_type_table("/test/test_vars/test_table")
vs = self.provider.search_variations(table)
self.assertIsNotNone(vs)
self.assertNotEquals(len(vs), 0)
# Get variations by name
vs = self.provider.get_variations("def*")
var_names = [var.name for var in vs]
self.assertIn("default", var_names)
# NON EXISTENT VARIATION
# ----------------------------------------------------
# Get run range that is not defined
try:
v = self.provider.get_variation("abra_kozyabra")
# oh... such run range exists? It shouldn't be... Maybe it is left because of the last tests...
print ("WARNING provider.get_variation('abra_kozyabra') found but should not be there")
print ("trying to delete variation and run the test one more time... ")
self.provider.delete_variation(v) # (!) <-- test of this function is further
v = self.provider.get_variation("abra_kozyabra")
self.assertIsNotNone(v)
except VariationNotFound:
pass # test passed
# create variation
# ----------------------------------------------------
# Get or create run-range is the main function to get RunRange without name
# 0-2001 should be absent or deleted so this function will create run-range
v = self.provider.create_variation("abra_kozyabra")
self.assertIsNotNone(v)
self.assertNotEquals(v.id, 0)
self.assertEquals(v.parent_id, 1)
self.assertEquals(v.name, "abra_kozyabra")
# DELETE RUN-RANGE TEST
# ----------------------------------------------------
self.provider.delete_variation(v)
self.assertRaises(VariationNotFound, self.provider.get_variation, "abra_kozyabra")
# Now create with comment and parent
v = self.provider.create_variation("abra_kozyabra", "Abra!!!", "test")
self.assertEquals(v.parent.name, "test")
self.assertEquals(v.comment, "Abra!!!")
# cleanup
self.provider.delete_variation(v)
def test_variation_backup(self):
"""Test Backup of """
self.provider.connect(self.connection_str) # this test requires the connection
a = self.provider.get_assignment("/test/test_vars/test_table", 100, "test")
self.assertEqual(a.constant_set.data_list[0], "2.2")
# No such calibration exist in test variation run 100, but constants should fallback to variation default
def test_assignments(self):
"""Test Assignments"""
self.provider.connect(self.connection_str) # this test requires the connection
assignment = self.provider.get_assignment("/test/test_vars/test_table", 100, "default")
self.assertIsNotNone(assignment)
# Check that everything is loaded
tabled_data = assignment.constant_set.data_table
self.assertEquals(len(tabled_data), 2)
self.assertEquals(len(tabled_data[0]), 3)
self.assertEquals(tabled_data[0][0], "2.2")
self.assertEquals(tabled_data[0][1], "2.3")
self.assertEquals(tabled_data[0][2], "2.4")
self.assertEquals(tabled_data[1][0], "2.5")
self.assertEquals(tabled_data[1][1], "2.6")
self.assertEquals(tabled_data[1][2], "2.7")
# Ok! Lets get all assignments for current types table
assignments = self.provider.get_assignments("/test/test_vars/test_table")
self.assertNotEquals(len(assignments), 0)
# Ok! Lets get all assignments for current types table and variation
assignments = self.provider.get_assignments("/test/test_vars/test_table", variation="default")
self.assertNotEquals(len(assignments), 0)
assignment = self.provider.create_assignment([[0, 1, 2], [3, 4, 5]], "/test/test_vars/test_table", 0, 1000,
"default", "Test assignment")
self.assertEqual(assignment.constant_set.type_table.path, "/test/test_vars/test_table")
self.assertEqual(assignment.variation.name, "default")
self.assertEqual(assignment.run_range.min, 0)
self.assertEqual(assignment.run_range.max, 1000)
self.assertEqual(assignment.comment, "Test assignment")
tabled_data = assignment.constant_set.data_table
self.assertEquals(len(tabled_data), 2)
self.assertEquals(len(tabled_data[0]), 3)
self.assertEquals(tabled_data[0][0], "0")
self.assertEquals(tabled_data[0][1], "1")
self.assertEquals(tabled_data[0][2], "2")
self.assertEquals(tabled_data[1][0], "3")
self.assertEquals(tabled_data[1][1], "4")
self.assertEquals(tabled_data[1][2], "5")
self.provider.delete_assignment(assignment)
def test_users(self):
"""Test users"""
self.provider.connect(self.connection_str) # this test requires the connection
user = self.provider.get_user("anonymous")
self.assertIsNotNone(user)
self.assertEqual(user.name, "anonymous")
user = self.provider.get_user("test_user")
isinstance(user, User)
self.assertIsNotNone(user)
self.assertEqual(user.password, "test")
self.assertEqual(user.roles, ["runrange_crate", "runrange_delete"])
# self.assertEqual(user.)
# test that with wrong user we can't create anything
self.provider.authentication.current_user_name = "non_exist_user_ever"
self.assertRaises(UserNotFoundError, self.provider.create_directory, "some_strange_dir", "/")
self.assertEqual(0, len(self.provider.search_directories("some_strange_dir")))
self.assertRaises(UserNotFoundError, self.provider.update_directory, self.provider.get_directory("/test"))
self.assertRaises(UserNotFoundError, self.provider.delete_directory, self.provider.get_directory("/test"))
self.assertIsNotNone(self.provider.get_directory("/test"))
@staticmethod
def test_gen_flatten_data():
source = [[1, 2], [3, "444"]]
result = list(gen_flatten_data(source))
assert result[0] == 1
assert result[1] == 2
assert result[2] == 3
assert result[3] == "444"
def test_list_to_blob(self):
self.assertMultiLineEqual("1|2|33", list_to_blob([1, 2, "33"]))
self.assertMultiLineEqual("strings|with&delimiter;surprise", list_to_blob(["strings", "with|surprise"]))
def test_blob_to_list(self):
self.assertItemsEqual(["1", "2", "str"], blob_to_list("1|2|str"))
self.assertItemsEqual(["strings", "with|surprise"], blob_to_list("strings|with&delimiter;surprise"))
def test_list_to_table(self):
self.assertRaises(ValueError, list_to_table, [1, 2, 3], 2)
self.assertItemsEqual([[1, 2, 3], [4, 5, 6]], list_to_table([1, 2, 3, 4, 5, 6], 3))
def test_get_users(self):
self.provider.connect(self.connection_str) # this test requires the connection
users = self.provider.get_users()
self.assertGreater(len(users), 0)
def test_validate_data(self):
column = TypeTableColumn()
# int type
column.type = 'int'
self.assertEqual(self.provider.validate_data_value('1', column), 1)
self.assertRaises(ValueError, self.provider.validate_data_value, 'hren', column)
# lets check bool type
column.type = 'bool'
self.assertEqual(self.provider.validate_data_value('TrUe', column), True)
self.assertEqual(self.provider.validate_data_value('FalSe', column), False)
self.assertEqual(self.provider.validate_data_value('1', column), True)
self.assertEqual(self.provider.validate_data_value('0', column), False)
self.assertRaises(ValueError, self.provider.validate_data_value, 'hren', column)
# uint!
column.type = 'uint'
self.assertEqual(self.provider.validate_data_value('1', column), 1)
self.assertRaises(ValueError, self.provider.validate_data_value, '-1', column)
class ConsoleContext(object):
"""
Class to manage console commands
This class uses console_utilities from utils directories
"""
anonymous_user_name = "anonymous"
prefix = None
words = []
def __init__(self):
self._prov = AlchemyProvider()
self._is_interactive = False
self.user_name = self.anonymous_user_name
self._is_interactive = False
self._current_run = 0
self._current_path = "/"
self._current_variation = "default"
self._utils = {}
self._verbose = False
self._ls = None
self._connection_string = ""
self.silent_exceptions = True # rethrow happened exceptions
self._theme = themes.NoColorTheme()
self.log_sqlite = False
# prop verbose
# _______________________
def _get_verbose(self):
"""Sets or gets verbose behaviour for this class"""
return self._verbose
def _set_verbose(self, is_verbose):
self._verbose = is_verbose
verbose = property(_get_verbose, _set_verbose)
@property
def utils(self):
""":rtype: {}"""
return self._utils
@property
def connection_string(self):
""":rtype: str"""
return self._connection_string
@connection_string.setter
def connection_string(self, con_str):
self._connection_string = con_str
@property
def provider(self):
return self._prov
@property
def current_path(self):
return self._current_path
@current_path.setter
def current_path(self, new_path):
self._current_path = new_path
@property
def current_run(self):
"""
Sets or gets current working run
:rtype: int
"""
return self._current_run
@current_run.setter
def current_run(self, new_run):
self._current_run = new_run
@property
def current_variation(self):
"""
Sets or gets current working variation
:rtype: str
"""
return self._current_variation
@current_variation.setter
def current_variation(self, new_var):
self._current_variation = new_var
@property
def user_name(self):
return self._prov.authentication.current_user_name
@user_name.setter
def user_name(self, name):
self._prov.authentication.current_user_name = name
@property
def is_interactive(self):
return self._is_interactive
@is_interactive.setter
def is_interactive(self, value):
self._is_interactive = value
@property
def theme(self):
""":rtype: themes.NoColorTheme"""
return self._theme
@theme.setter
def theme(self, value):
"""
:param value: new theme to set
:type value: themes.NoColorTheme
"""
assert (isinstance(value, themes.NoColorTheme))
for key in self._utils.keys():
self._utils[key].theme = value
log.debug(
lfm(" |- theme(value) {0} | \\{0} | |- theme switched to : '{1}'",
os.linesep, value))
self._theme = value
# =====================================================================================
# ------------------ P L U G I N M A N A G E M E N T -------------------------------
# =====================================================================================
# --------------------------------
# register_utilities
# --------------------------------
def register_utilities(self, path=""):
""" Function to auto find and register utilities"""
path = path or os.path.join(ccdb.cmd.__path__[0], "utils")
# search modules
modules = self.search_utils(path)
self._utils = {}
# register each module
for module in modules:
try:
registerFunc = getattr(module, "create_util_instance")
util = registerFunc()
if util:
self._utils[util.command] = util
util.context = self
util.theme = self.theme
if util.command == "ls":
self._ls = util
except AttributeError as ex:
log.debug("Error registering module : " + repr(ex))
except Exception as ex:
log.debug("Error registering module : " + repr(ex))
if log.isEnabledFor(logging.DEBUG):
log.debug(
lfm("{0}Utils found and registered in directory '{1}' are:",
os.linesep, path))
log.debug("{0:<10} {1:<15} {2}:".format("(command)", "(name)",
"(description)"))
log.debug("\n".join([
"{0:<10} {1:<15} {2}:".format(command, util.name,
util.short_descr)
for command, util in self._utils.items()
]))
# --------------------------------
# search_utils
# --------------------------------
def search_utils(self, path):
"""Load plugins from directory and return list of modules
:rtype: []
"""
# >oO debug output
log.debug(lfm("{0}search_utils{0}\\", os.linesep))
log.debug(
lfm(" |- searching modules in directory:{0} | '{1}{2}{3}'",
os.linesep, self.theme.Directories, path, self.theme.Reset))
# get list of files and module names
files = os.listdir(path)
test = re.compile(".py$", re.IGNORECASE)
files = filter(test.search, files)
filenameToModuleName = lambda f: os.path.splitext(f)[0]
moduleNames = sorted(map(filenameToModuleName, files))
log.debug(
lfm(
" |- found '{0}' modules.{1} |- proceed loading each module:{1}",
len(moduleNames), os.linesep))
modules = []
for m in moduleNames:
# skip any files starting with '__', such as __init__.py
if m.startswith('__'):
continue
try:
f, filename, desc = imp.find_module(m, [path])
modules.append(imp.load_module(m, f, filename, desc))
except ImportError as ex:
log.debug(lfm(" |- error importing module: {0}", m))
log.debug(lfm(" |\\{0} ||-{1}", os.linesep, repr(ex)))
continue
return modules
# --------------------------------
# processes the arguments
# --------------------------------
def process(self, args, startIndex=1):
# check if there is enough arguments...
if len(args) < (startIndex + 1):
self.print_general_usage()
return
# get working arguments list
workargs = args[startIndex:]
# parse loop
i = 0
while i < len(workargs):
token = workargs[i]
assert isinstance(token, str)
i += 1
if token.startswith('-'):
# it is some command, lets parse what is the command
if (token == "-c"
or token == "--connection") and (i < len(workargs)):
# connection string
self.connection_string = workargs[i]
i += 1
elif token == "-I" or token == "-i" or token == "--interactive":
# it is an interactive mode
self._is_interactive = True
elif token == "-r" or token == "--run":
# working run
try:
self.current_run = int(workargs[i])
log.info("Working run is %i", self.current_run)
except ValueError:
log.warning(
"(!) Warning. Cannot read run from %s command" %
token)
i += 1
elif token == "-v" or token == "--variation":
# working variation
if not ccdb.path_utils.validate_name(workargs[i]):
log.warning(
"(!) Warning. Cannot read variation from --variation flag. "
"Variation name should consist of A-Z, a-z, 0-9, _"
)
else:
self._current_variation = workargs[i]
i += 1
else:
# looks like is is a command
command = token
commandArgs = workargs[i:]
try:
self.process_command(command, commandArgs)
break
except Exception as ex:
log.error(ex)
if not self.silent_exceptions:
raise
else:
return 1
# we have to ask mysql password
if "--mysql-pwd" in args[
startIndex:] and self.connection_string.startswith("mysql"):
print("Enter MySql password: "******"@", ":" + password + "@")
if self._is_interactive:
self.interactive_loop()
# --------------------------------
# executes text as command
# --------------------------------
def process_command_line(self, command_line):
"""tries to execute a line of text"""
log.debug("{0}Process command line: {1}{0}\\".format(
os.linesep, command_line))
# execute shell command if input starts with '!'
if command_line.startswith('!'):
command_line = command_line[1:]
# noinspection PyBroadException
try:
os.system(command_line)
except:
pass
return # everything is done
# split command to arguments
is_posix = os.name == 'posix'
tokens = shlex.split(command_line, posix=is_posix)
# validate
if len(tokens) == 0:
return
# get our command
command = tokens[0]
log.debug(" |- command is : {0}".format(command))
log.debug(" |- tokens : {0}".format(" ".join([("'" + t + "'")
for t in tokens])))
# get command arguments
arguments = []
if len(tokens) > 1:
arguments = tokens[1:]
# execute everything
return self.process_command(command, arguments)
# --------------------------------
#
# --------------------------------
def process_command(self, command, args):
# >oO debug
if log.isEnabledFor(logging.DEBUG):
log.debug(
lfm("{0}Processing command: '{1}'{0}\\", os.linesep, command))
log.debug(lfm(" |- arguments : '{0}'", "' '".join(args)))
# try to find function...
try:
util = self._utils[command]
except KeyError:
log.error(
"Command " + command +
" is unknown! Please, use help to see available commands")
if not self.silent_exceptions:
raise
else:
return 1
# check connection and connect if needed
if util.uses_db and (not self.provider.is_connected):
if not self.check_connection(util):
return False
# is there file redirect?
redir_to_file = False # should we redirect to file?
redir_file = None # file to redirect
redir_stream_backup = sys.stdout
redir_theme_backup = self.theme
if ">" in args and args.index(">") == len(args) - 2:
redir_fname = args[-1]
redir_to_file = True
args = args[:-2]
log.debug(" |- redirecting to file : {0}".format(redir_fname))
# open file
try:
redir_file = file(redir_fname, 'w')
except Exception as ex:
log.error("Cannot open file '{0}' {1} ".format(
redir_fname, ex))
if not self.silent_exceptions:
raise
else:
return 1
# execute command
try:
if redir_to_file:
colorama.deinit()
sys.stdout = redir_file
self.theme = themes.NoColorTheme()
result = util.process(args)
except Exception as ex:
log.error(ex)
if not self.silent_exceptions:
raise
else:
return 1
finally:
if redir_to_file:
sys.stdout = redir_stream_backup
redir_file.close()
self.theme = redir_theme_backup
colorama.reinit()
return result
# --------------------------------
#
# --------------------------------
def check_connection(self, util):
# maybe there is nothing to do?
if self._prov.is_connected or (not util.uses_db):
return
if log.isEnabledFor(logging.DEBUG):
log.debug(" |- check_connection(util){0} | \\".format(os.linesep))
log.debug(
" | |- utility '{}' requires DB. CCDB not yet connected. Connecting."
.format(util.name))
log.debug(" | |- connection string is: '{}{}{}'".format(
self.theme.Accent, self.connection_string, self.theme.Reset))
# connecting
try:
self._prov.connect(self.connection_string)
except Exception as ex:
log.critical(
"CCDB provider unable to connect to {0}. Aborting command. Exception details: {1}"
.format(self.connection_string, ex))
return False
# skip user for sqlite
if (not self.log_sqlite) and (
self.provider.connection_string.startswith("sqlite://")):
log.debug(
" | |- log_sqlite == False, set user to anonymous and disable logging'"
)
self.provider.logging_enabled = False
# self.user_name = self.anonymous_user_name
# connected
if log.isEnabledFor(logging.DEBUG):
if self.provider.connection_string.startswith(
"mysql+mysqlconnector://"):
log.debug(
" | |- no module MySQLdb. Fallback to mysql-connector used"
)
log.debug(" | |- connection string used: '" + self.theme.Accent +
self.provider.connection_string + self.theme.Reset + "'")
log.debug(" | |- connection: " + self.theme.Success +
" successful " + self.theme.Reset)
return True
# ----------------------------------------------------
# interactive_loop
# ----------------------------------------------------
# noinspection PyBroadException
def interactive_loop(self):
import readline
self.print_interactive_intro()
# initialise autocomplete
self.words = self._utils.keys()
# completer = Completer(words)
colorama.deinit() # make colorama to release stderr and stdout
readline.parse_and_bind("tab: complete")
readline.set_completer(self.complete)
try:
readline.read_history_file()
except:
pass # eat it
# readline..set_completion_display_matches_hook(self.show_completions)
# readline.set_completion_display_matches_hook([function])
# Set or remove the completion display function. If function is specified, it will be used as the new completion
# display function; if omitted or None, any completion display function already installed is removed.
# The completion display function is called as function(substitution, [matches], longest_match_length) each
# time matches need to be displayed.
# Begin user commands read loop
while 1:
colorama.deinit()
# read command from user
try:
user_input = raw_input(self.current_path + "> ")
except EOFError:
log.debug("EOF sequence received. Ending interactive loop")
break
except KeyboardInterrupt:
log.debug("Break sequence received. Ending interactive loop")
break
colorama.reinit()
# exit if user wishes so
if user_input in ("quit", "q", "exit"):
break # quit!
# process user input
self.process_command_line(user_input)
# loop ended
try:
readline.write_history_file()
except:
log.warn("unable to write history file")
# =====================================================================================
# ------------------------ C O M P L E T I O N ----------------------------------------
# =====================================================================================
# region Description
# --------------------------------
# show_completions
# --------------------------------
def show_completions(self, substitution, matches, longest_match_length):
print(self)
print(substitution)
print(matches)
print(longest_match_length)
# --------------------------------
# generate_completion_words
# --------------------------------
# noinspection PyBroadException
def generate_completion_words(self, prefix):
# find all words that start with this prefix
self.matching_words = [w for w in self.words if w.startswith(prefix)]
# get name patches
path_prefix = posixpath.join(self.current_path, prefix)
log.debug("getting path completion for: " + path_prefix)
try:
self.check_connection(self._ls)
try:
result = self._ls.get_name_pathes(path_prefix)
except:
result = None
if result is None or (len(result[0]) == 0 and len(result[1]) == 0):
result = self._ls.get_name_pathes(path_prefix + "*")
dir_list = [subdir.name for subdir in result[0]]
table_list = [table.name for table in result[1]]
self.matching_words.extend(dir_list)
self.matching_words.extend(table_list)
except Exception as ex:
log.debug("error getting competition paths: " + ex.message)
# --------------------------------
# complete
# --------------------------------
def complete(self, prefix, index):
# print "prefix ", prefix, " index ", index
# readline.insert_text("bla bla bla")
# colorama.deinit()
# print "ha ha ha"
# colorama.reinit()
if prefix != self.prefix:
# get new completions
self.generate_completion_words(prefix)
# self.matching_words.append(
self.prefix = prefix
else:
# print "prefix the same, matching:", len(self.matching_words)
# print " ".join([word for word in self.matching_words])
# readline.redisplay()
pass
try:
return self.matching_words[index]
except IndexError:
return None
# endregion
# =====================================================================================
# -------- G E T T I N G O B J E C T S -------------------------------------------
# =====================================================================================
# --------------------------------
# prepare_path
# --------------------------------
def prepare_path(self, path):
# correct ending /
if path.endswith("/"):
path = path[:-1]
# local or absolute path?
if not path.startswith("/"):
path = posixpath.join(self.current_path, path)
# normalize
path = posixpath.normpath(path)
return path
# --------------------------------
# parse_run_range
# --------------------------------
def parse_run_range(self, run_range_str):
""" @brief parse run range string in form of <run_min>-<run-max>
if one inputs '<run_min>-' this means <run_min>-<infinite run>
if one inputs '-<run_max>' this means <0>-<run_max>
@return (run_min, run_max, run_min_set, run_max_set)
run_min_set, run_max_set - are flags indicating that values was set by user
"""
assert isinstance(run_range_str, str)
if not "-" in run_range_str:
return None
# split <>-<>
(str_min, str_max) = run_range_str.split("-")
run_min = 0
run_min_set = False
run_max = ccdb.INFINITE_RUN
run_max_set = False
# parse run min
try:
run_min = int(str_min)
run_min_set = True
except ValueError:
self.run_min = 0
# parse run max
try:
run_max = int(str_max)
run_max_set = True
except ValueError:
self.run_max = ccdb.INFINITE_RUN
return run_min, run_max, run_min_set, run_max_set
# =====================================================================================
# ------------------------- H E L P A N D I N F O ---------------------------------
# =====================================================================================
def print_general_usage(self):
print("Use '-i' option to enter interactive shell")
print("Use 'help' option for help")
print("Use 'help command' to get help for particular command")
def print_info(self):
log.info("Login as : '" + self.user_name + "'")
log.info("Connect to : '" + self.connection_string + "'")
log.info("Variation : '" + self.current_variation + "'")
log.info("Deflt. run : '" + str(self.current_run) + "'")
def print_interactive_intro(self):
print("""
+--------------------------+
CCDB shell v.1.07.00
+--------------------------+
""")
print(self.theme.Title + "Interactive mode")
print("print " + self.theme.Accent + "help" + self.theme.Reset +
" to get help")
print("print " + self.theme.Accent + "quit" + self.theme.Reset +
" or " + self.theme.Accent + "q" + self.theme.Reset + " to exit")
print("print !<command> to execute shell command")
print()
self.print_info()
def setUp(self):
self._connection_str = self.sqlite_connection_str
self._provider = AlchemyProvider()
self.provider.logging_enabled = False
class ConsoleContext(object):
"""
Class to manage console commands
This class uses console_utilities from utils directories
"""
anonymous_user_name = "anonymous"
prefix = None
words = []
def __init__(self):
self._prov = AlchemyProvider()
self._is_interactive = False
self.user_name = self.anonymous_user_name
self._is_interactive = False
self._current_run = 0
self._current_path = "/"
self._current_variation = "default"
self._utils = {}
self._verbose = False
self._ls = None
self._connection_string = ""
self.silent_exceptions = True # rethrow happened exceptions
self._theme = themes.NoColorTheme()
self.log_sqlite = False
# prop verbose
# _______________________
def _get_verbose(self):
"""Sets or gets verbose behaviour for this class"""
return self._verbose
def _set_verbose(self, is_verbose):
self._verbose = is_verbose
verbose = property(_get_verbose, _set_verbose)
@property
def utils(self):
""":rtype: {}"""
return self._utils
@property
def connection_string(self):
""":rtype: str"""
return self._connection_string
@connection_string.setter
def connection_string(self, con_str):
self._connection_string = con_str
@property
def provider(self):
return self._prov
@property
def current_path(self):
return self._current_path
@current_path.setter
def current_path(self, new_path):
self._current_path = new_path
@property
def current_run(self):
"""
Sets or gets current working run
:rtype: int
"""
return self._current_run
@current_run.setter
def current_run(self, new_run):
self._current_run = new_run
@property
def current_variation(self):
"""
Sets or gets current working variation
:rtype: str
"""
return self._current_variation
@current_variation.setter
def current_variation(self, new_var):
self._current_variation = new_var
@property
def user_name(self):
return self._prov.authentication.current_user_name
@user_name.setter
def user_name(self, name):
self._prov.authentication.current_user_name = name
@property
def is_interactive(self):
return self._is_interactive
@is_interactive.setter
def is_interactive(self, value):
self._is_interactive = value
@property
def theme(self):
""":rtype: themes.NoColorTheme"""
return self._theme
@theme.setter
def theme(self, value):
"""
:param value: new theme to set
:type value: themes.NoColorTheme
"""
assert isinstance(value, themes.NoColorTheme)
for key in self._utils.keys():
self._utils[key].theme = value
log.debug(lfm(" |- theme(value) {0} | \\{0} | |- theme switched to : '{1}'", os.linesep, value))
self._theme = value
# =====================================================================================
# ------------------ P L U G I N M A N A G E M E N T -------------------------------
# =====================================================================================
# --------------------------------
# register_utilities
# --------------------------------
def register_utilities(self, path=""):
""" Function to auto find and register utilities"""
path = path or os.path.join(ccdb.cmd.__path__[0], "utils")
# search modules
modules = self.search_utils(path)
self._utils = {}
# register each module
for module in modules:
try:
registerFunc = getattr(module, "create_util_instance")
util = registerFunc()
if util:
self._utils[util.command] = util
util.context = self
util.theme = self.theme
if util.command == "ls":
self._ls = util
except AttributeError as ex:
log.debug("Error registering module : " + repr(ex))
except Exception as ex:
log.debug("Error registering module : " + repr(ex))
if log.isEnabledFor(logging.DEBUG):
log.debug(lfm("{0}Utils found and registered in directory '{1}' are:", os.linesep, path))
log.debug("{0:<10} {1:<15} {2}:".format("(command)", "(name)", "(description)"))
log.debug(
"\n".join(
[
"{0:<10} {1:<15} {2}:".format(command, util.name, util.short_descr)
for command, util in self._utils.items()
]
)
)
# --------------------------------
# search_utils
# --------------------------------
def search_utils(self, path):
"""Load plugins from directory and return list of modules
:rtype: []
"""
# >oO debug output
log.debug(lfm("{0}search_utils{0}\\", os.linesep))
log.debug(
lfm(
" |- searching modules in directory:{0} | '{1}{2}{3}'",
os.linesep,
self.theme.Directories,
path,
self.theme.Reset,
)
)
# get list of files and module names
files = os.listdir(path)
test = re.compile(".py$", re.IGNORECASE)
files = filter(test.search, files)
filenameToModuleName = lambda f: os.path.splitext(f)[0]
moduleNames = sorted(map(filenameToModuleName, files))
log.debug(lfm(" |- found '{0}' modules.{1} |- proceed loading each module:{1}", len(moduleNames), os.linesep))
modules = []
for m in moduleNames:
# skip any files starting with '__', such as __init__.py
if m.startswith("__"):
continue
try:
f, filename, desc = imp.find_module(m, [path])
modules.append(imp.load_module(m, f, filename, desc))
except ImportError as ex:
log.debug(lfm(" |- error importing module: {0}", m))
log.debug(lfm(" |\\{0} ||-{1}", os.linesep, repr(ex)))
continue
return modules
# --------------------------------
#
# --------------------------------
def process(self, args, startIndex=1):
# check if there is enough arguments...
if len(args) < (startIndex + 1):
self.print_general_usage()
return
# get working arguments list
workargs = args[startIndex:]
# parse loop
i = 0
while i < len(workargs):
token = workargs[i]
assert isinstance(token, str)
i += 1
if token.startswith("-"):
# it is some command, lets parse what is the command
if (token == "-c" or token == "--connection") and (i < len(workargs)):
# connection string
self.connection_string = workargs[i]
i += 1
elif token == "-I" or token == "-i" or token == "--interactive":
# it is an interactive mode
self._is_interactive = True
elif token == "-r" or token == "--run":
# working run
try:
self.current_run = int(workargs[i])
log.info("Working run is %i", self.current_run)
except ValueError:
log.warning("(!) Warning. Cannot read run from %s command" % token)
i += 1
elif token == "-v" or token == "--variation":
# working variation
if not ccdb.path_utils.validate_name(workargs[i]):
log.warning(
"(!) Warning. Cannot read variation from --variation flag. "
"Variation name should consist of A-Z, a-z, 0-9, _"
)
else:
self._current_variation = workargs[i]
i += 1
else:
# looks like is is a command
command = token
commandArgs = workargs[i:]
try:
self.process_command(command, commandArgs)
break
except Exception as ex:
log.error(ex)
if not self.silent_exceptions:
raise
else:
return 1
# we have to ask mysql password
if "-p" in workargs and self.connection_string.startswith("mysql"):
print("Enter MySql password: "******"@", ":" + password + "@")
if self._is_interactive:
self.interactive_loop()
# --------------------------------
# executes text as command
# --------------------------------
def process_command_line(self, command_line):
"""tries to execute a line of text"""
log.debug("{0}Process command line: {1}{0}\\".format(os.linesep, command_line))
# execute shell command if input starts with '!'
if command_line.startswith("!"):
command_line = command_line[1:]
# noinspection PyBroadException
try:
os.system(command_line)
except:
pass
return # everything is done
# split command to arguments
tokens = shlex.split(command_line)
# validate
if len(tokens) == 0:
return
# get our command
command = tokens[0]
log.debug(" |- command is : {0}".format(command))
log.debug(" |- tokens : {0}".format(" ".join([("'" + t + "'") for t in tokens])))
# get command arguments
arguments = []
if len(tokens) > 1:
arguments = tokens[1:]
# execute everything
return self.process_command(command, arguments)
# --------------------------------
#
# --------------------------------
def process_command(self, command, args):
# >oO debug
if log.isEnabledFor(logging.DEBUG):
log.debug(lfm("{0}Processing command: '{1}'{0}\\", os.linesep, command))
log.debug(lfm(" |- arguments : '{0}'", "' '".join(args)))
# try to find function...
try:
util = self._utils[command]
except KeyError:
log.error("Command " + command + " is unknown! Please, use help to see available commands")
if not self.silent_exceptions:
raise
else:
return 1
# check connection and connect if needed
if util.uses_db and (not self.provider.is_connected):
if not self.check_connection(util):
return False
# is there file redirect?
redir_to_file = False # should we redirect to file?
redir_file = None # file to redirect
redir_stream_backup = sys.stdout
redir_theme_backup = self.theme
if ">" in args and args.index(">") == len(args) - 2:
redir_fname = args[-1]
redir_to_file = True
args = args[:-2]
log.debug(" |- redirecting to file : {0}".format(redir_fname))
# open file
try:
redir_file = file(redir_fname, "w")
except Exception as ex:
log.error("Cannot open file '{0}' {1} ".format(redir_fname, ex))
if not self.silent_exceptions:
raise
else:
return 1
# execute command
try:
if redir_to_file:
colorama.deinit()
sys.stdout = redir_file
self.theme = themes.NoColorTheme()
result = util.process(args)
except Exception as ex:
log.error(ex)
if not self.silent_exceptions:
raise
else:
return 1
finally:
if redir_to_file:
sys.stdout = redir_stream_backup
redir_file.close()
self.theme = redir_theme_backup
colorama.reinit()
return result
# --------------------------------
#
# --------------------------------
def check_connection(self, util):
# maybe there is nothing to do?
if self._prov.is_connected or (not util.uses_db):
return
if log.isEnabledFor(logging.DEBUG):
log.debug(" |- check_connection(util){0} | \\".format(os.linesep))
log.debug(
" | |- utility '" + util.name + "' requires the database "
"and there is no connection yet. Connecting."
)
log.debug(
" | |- connection string is: '" + self.theme.Accent + self.connection_string + self.theme.Reset + "'"
)
# connecting
try:
self._prov.connect(self.connection_string)
except Exception as ex:
log.critical(
"CCDB provider unable to connect to {0}. Aborting command. Exception details: {1}".format(
self.connection_string, ex
)
)
return False
# skip user for sqlite
if (not self.log_sqlite) and (self.provider.connection_string.startswith("sqlite://")):
log.debug(" | |- log_sqlite == False, set user to anonymous and disable logging'")
self.provider.logging_enabled = False
self.user_name = self.anonymous_user_name
# connected
if log.isEnabledFor(logging.DEBUG):
if self.provider.connection_string.startswith("mysql+mysqlconnector://"):
log.debug(" | |- no module MySQLdb. Fallback to mysql-connector used")
log.debug(
" | |- connection string used: '"
+ self.theme.Accent
+ self.provider.connection_string
+ self.theme.Reset
+ "'"
)
log.debug(" | |- connection: " + self.theme.Success + " successful " + self.theme.Reset)
return True
# ----------------------------------------------------
# interactive_loop
# ----------------------------------------------------
# noinspection PyBroadException
def interactive_loop(self):
self.print_interactive_intro()
# initialise autocomplete
self.words = self._utils.keys()
# completer = Completer(words)
colorama.deinit() # make colorama to release stderr and stdout
readline.parse_and_bind("tab: complete")
readline.set_completer(self.complete)
try:
readline.read_history_file()
except:
pass # eat it
# readline..set_completion_display_matches_hook(self.show_completions)
# readline.set_completion_display_matches_hook([function])
# Set or remove the completion display function. If function is specified, it will be used as the new completion
# display function; if omitted or None, any completion display function already installed is removed.
# The completion display function is called as function(substitution, [matches], longest_match_length) each
# time matches need to be displayed.
# Begin user commands read loop
while 1:
colorama.deinit()
# read command from user
try:
user_input = raw_input(self.current_path + "> ")
except EOFError:
log.debug("EOF sequence received. Ending interactive loop")
break
except KeyboardInterrupt:
log.debug("Break sequence received. Ending interactive loop")
break
colorama.reinit()
# exit if user wishes so
if user_input in ("quit", "q", "exit"):
break # quit!
# process user input
self.process_command_line(user_input)
# loop ended
try:
readline.write_history_file()
except:
log.warn("unable to write history file")
# =====================================================================================
# ------------------------ C O M P L E T I O N ----------------------------------------
# =====================================================================================
# region Description
# --------------------------------
# show_completions
# --------------------------------
def show_completions(self, substitution, matches, longest_match_length):
print(self)
print(substitution)
print(matches)
print(longest_match_length)
# --------------------------------
# generate_completion_words
# --------------------------------
# noinspection PyBroadException
def generate_completion_words(self, prefix):
# find all words that start with this prefix
self.matching_words = [w for w in self.words if w.startswith(prefix)]
# get name patches
path_prefix = posixpath.join(self.current_path, prefix)
log.debug("getting path completion for: " + path_prefix)
try:
self.check_connection(self._ls)
try:
result = self._ls.get_name_pathes(path_prefix)
except:
result = None
if result is None or (len(result[0]) == 0 and len(result[1]) == 0):
result = self._ls.get_name_pathes(path_prefix + "*")
dir_list = [subdir.name for subdir in result[0]]
table_list = [table.name for table in result[1]]
self.matching_words.extend(dir_list)
self.matching_words.extend(table_list)
except Exception as ex:
log.debug("error getting competition paths: " + ex.message)
# --------------------------------
# complete
# --------------------------------
def complete(self, prefix, index):
# print "prefix ", prefix, " index ", index
# readline.insert_text("bla bla bla")
# colorama.deinit()
# print "ha ha ha"
# colorama.reinit()
if prefix != self.prefix:
# get new completions
self.generate_completion_words(prefix)
# self.matching_words.append(
self.prefix = prefix
else:
# print "prefix the same, matching:", len(self.matching_words)
# print " ".join([word for word in self.matching_words])
# readline.redisplay()
pass
try:
return self.matching_words[index]
except IndexError:
return None
# endregion
# =====================================================================================
# -------- G E T T I N G O B J E C T S -------------------------------------------
# =====================================================================================
# --------------------------------
# prepare_path
# --------------------------------
def prepare_path(self, path):
# correct ending /
if path.endswith("/"):
path = path[:-1]
# local or absolute path?
if not path.startswith("/"):
path = posixpath.join(self.current_path, path)
# normalize
path = posixpath.normpath(path)
return path
# --------------------------------
# parse_run_range
# --------------------------------
def parse_run_range(self, run_range_str):
""" @brief parse run range string in form of <run_min>-<run-max>
if one inputs '<run_min>-' this means <run_min>-<infinite run>
if one inputs '-<run_max>' this means <0>-<run_max>
@return (run_min, run_max, run_min_set, run_max_set)
run_min_set, run_max_set - are flags indicating that values was set by user
"""
assert isinstance(run_range_str, str)
if not "-" in run_range_str:
return None
# split <>-<>
(str_min, str_max) = run_range_str.split("-")
run_min = 0
run_min_set = False
run_max = ccdb.INFINITE_RUN
run_max_set = False
# parse run min
try:
run_min = int(str_min)
run_min_set = True
except ValueError:
self.run_min = 0
# parse run max
try:
run_max = int(str_max)
run_max_set = True
except ValueError:
self.run_max = ccdb.INFINITE_RUN
return run_min, run_max, run_min_set, run_max_set
# =====================================================================================
# ------------------------- H E L P A N D I N F O ---------------------------------
# =====================================================================================
def print_general_usage(self):
print("Use '-i' option to enter interactive shell")
print("Use 'help' option for help")
print("Use 'help command' to get help for particular command")
def print_info(self):
log.info("Login as : '" + self.user_name + "'")
log.info("Connect to : '" + self.connection_string + "'")
log.info("Variation : '" + self.current_variation + "'")
log.info("Deflt. run : '" + str(self.current_run) + "'")
def print_interactive_intro(self):
print(
"""
+--------------------------+
CCDB shell v.1.05b
+--------------------------+
"""
)
print(self.theme.Title + "Interactive mode")
print("print " + self.theme.Accent + "help" + self.theme.Reset + " to get help")
print(
"print "
+ self.theme.Accent
+ "quit"
+ self.theme.Reset
+ " or "
+ self.theme.Accent
+ "q"
+ self.theme.Reset
+ " to exit"
)
print("print !<command> to execute shell command")
print()
self.print_info()
def setUp(self):
ccdb_path = get_ccdb_home_path()
self.sqlite_connection_str = "sqlite:///" + os.path.join(ccdb_path, "sql", "ccdb.sqlite")
self.provider = AlchemyProvider()
def setUp(self):
ccdb_path = get_ccdb_home_path()
self.sqlite_connection_str = "sqlite:///" + os.path.join(ccdb_path, "sql", "ccdb.sqlite")
self.mysql_connection_str = "mysql://[email protected]:3306/ccdb"
self.provider = AlchemyProvider()
