def setUp(self): ccdb_path = get_ccdb_home_path() self.sqlite_connection_str = "sqlite:///" + os.path.join( ccdb_path, "sql", "ccdb.sqlite") self.mysql_connection_str = "mysql://[email protected]:3306/ccdb" self.provider = AlchemyProvider()
def __init__(self): self._prov = AlchemyProvider() self._is_interactive = False self.user_name = self.anonymous_user_name self._is_interactive = False self._current_run = 0 self._current_path = "/" self._current_variation = "default" self._utils = {} self._verbose = False self._ls = None self._connection_string = "" self.silent_exceptions = True # rethrow happened exceptions self._theme = themes.NoColorTheme() self.log_sqlite = False
class AuthenticationTest(unittest.TestCase): def setUp(self): ccdb_path = get_ccdb_home_path() self.sqlite_connection_str = "sqlite:///" + os.path.join( ccdb_path, "sql", "ccdb.sqlite") self.mysql_connection_str = "mysql://[email protected]:3306/ccdb" self.provider = AlchemyProvider() def test_environment_auth(self): self.provider.connect(self.sqlite_connection_str) if "CCDB_USER" in os.environ: del os.environ["CCDB_USER"] if "USER" in os.environ: del os.environ["USER"] os.environ["CCDB_USER"] = "******" auth = EnvironmentAuthentication(self.provider) self.assertEqual(auth.current_user_name, "test_user") self.assertTrue(auth.validate_current_user())
class AuthenticationTest(unittest.TestCase): def setUp(self): ccdb_path = get_ccdb_home_path() self.sqlite_connection_str = "sqlite:///" + os.path.join(ccdb_path, "sql", "ccdb.sqlite") self.mysql_connection_str = "mysql://[email protected]:3306/ccdb" self.provider = AlchemyProvider() def test_environment_auth(self): self.provider.connect(self.sqlite_connection_str) if "CCDB_USER" in os.environ: del os.environ["CCDB_USER"] if "USER" in os.environ: del os.environ["USER"] os.environ["CCDB_USER"] = "******" auth = EnvironmentAuthentication(self.provider) self.assertEqual(auth.current_user_name, "test_user") self.assertTrue(auth.validate_current_user())
def setUp(self): self._provider = AlchemyProvider() self.provider.logging_enabled = False self.provider.authentication.current_user_name = "test_user"
class AlchemyProviderTest(unittest.TestCase): ccdb_path = get_ccdb_home_path() _connection_str = helper.sqlite_test_connection_str _provider = AlchemyProvider() @property def provider(self): return self._provider @property def connection_str(self): return self._connection_str @connection_str.setter def connection_str(self, connection_str): self._connection_str = connection_str def setUp(self): self._provider = AlchemyProvider() self.provider.logging_enabled = False self.provider.authentication.current_user_name = "test_user" def test_connection(self): """ Tests that provider connects successfully""" self.provider.connect(self.connection_str) def test_connect_to_old_schema(self): """ Test connection to schema with wrong version """ ccdb_path = get_ccdb_home_path() old_schema_cs = "sqlite:///" + os.path.join(ccdb_path, "python", "tests", "old_schema.ccdb.sqlite") self.assertRaises(DatabaseStructureError, self.provider.connect, old_schema_cs) def test_directories(self): """ Test of directories""" self.provider.connect(self.connection_str) # this test requires the connection # simple get directory dir_obj = self.provider.get_directory("/test") self.assertIsNotNone(dir_obj) self.assertMultiLineEqual(dir_obj.path, "/test") self.assertMultiLineEqual(dir_obj.name, "test") # search directories dirs = self.provider.search_directories("t??t_va*", "/test") assert (len(dirs) != 0) dirs = self.provider.search_directories("*", "/test") assert (len(dirs) >= 2) dirs = self.provider.search_directories("*", "") assert (len(dirs) >= 2) # cleanup directories # Ok, lets check if directory for the next text exists... try: self.provider.delete_directory("/test/testdir/constants") except DirectoryNotFound: pass try: self.provider.delete_directory("/test/testdir") except DirectoryNotFound: pass # cleanup directories # Ok, lets check if directory for the next text exists... dir_obj = self.provider.create_directory("testdir", "/test") self.assertIsNotNone(dir_obj) self.provider.logging_enabled = True # enable logging to test log too # create subdirectory constants_subdir = self.provider.create_directory("constants", "/test/testdir", "My constants") self.assertIsNotNone(constants_subdir) self.assertEqual(constants_subdir.comment, "My constants") # check log log = self.provider.get_log_records(limit=1)[0] assert (isinstance(log, LogRecord)) self.assertEqual(log.action, "create") self.assertEqual(log.affected_ids, "|directories" + str(constants_subdir.id) + "|") self.assertEqual(log.comment, "My constants") self.assertIn("Created directory", log.description) self.provider.logging_enabled = False # cannot recreate subdirectory self.assertRaises(ValueError, self.provider.create_directory, "constants", "/test/testdir", "My constants") # create another subdirectory variables_subdir = self.provider.create_directory("variables", "/test/testdir", "My constants") # test delete self.provider.delete_directory("/test/testdir/constants") # test can't delete dir with sub dirs self.assertRaises(ValueError, self.provider.delete_directory, "/test/testdir") # test delete by object self.provider.delete_directory(variables_subdir) # now, when dir doesn't have sub dirs and sub tables, it can be deleted self.provider.delete_directory("/test/testdir") # noinspection PyBroadException def test_type_tables(self): """ Test type table operation @return: None """ self.provider.connect(self.connection_str) # this test requires the connection table = self.provider.get_type_table("/test/test_vars/test_table") assert table is not None self.assertEqual(len(table.columns), 3) assert table.name == "test_table" assert table.path == "/test/test_vars/test_table" assert table.parent_dir assert table.parent_dir.name == "test_vars" assert table.columns[0].name == "x" # get all tables in directory tables = self.provider.get_type_tables("/test/test_vars") assert len(tables) >= 2 # at least 2 tables are located in "/test/test_vars" # count tables in a directory assert self.provider.count_type_tables("/test/test_vars") >= 2 # SEARCH TYPE TABLES # basic search type table functional tables = self.provider.search_type_tables("t??t_tab*") self.assertNotEqual(len(tables), 0) self.assertIn("/", tables[0].path) # now lets get all tables from the directory. tables = self.provider.search_type_tables("*", "/test/test_vars") self.assertNotEqual(len(tables), 0) for table in tables: self.assertEqual(table.path, "/test/test_vars" + "/" + table.name) # now lets get all tables from root directory. tables = self.provider.search_type_tables("t*", "/") self.assertEquals(len(tables), 0) # CREATE AND DELETE try: # if such type table already exists.. probably from last failed test... # we haven't test it yet, but we should try to delete it table = self.provider.get_type_table("/test/test_vars/new_table") self.provider.delete_type_table(table) except: pass table = self.provider.create_type_table( name="new_table", dir_obj_or_path="/test/test_vars", rows_num=5, columns=[('c', 'double'), ('a', 'double'), ('b', 'int')], comment="This is temporary created table for test reasons") self.assertIsNotNone(table) table = self.provider.get_type_table("/test/test_vars/new_table") self.assertEqual(table.rows_count, 5) self.assertEqual(table.columns_count, 3) self.assertEqual(table.name, 'new_table') self.assertEqual(table.columns[0].name, 'c') self.assertEqual(table.columns[0].type, 'double') self.assertEqual(table.columns[1].name, 'a') self.assertEqual(table.columns[1].type, 'double') self.assertEqual(table.columns[2].name, 'b') self.assertEqual(table.columns[2].type, 'int') self.assertEqual(table.comment, "This is temporary created table for test reasons") # delete self.provider.delete_type_table(table) self.assertRaises(TypeTableNotFound, self.provider.get_type_table, "/test/test_vars/new_table") def test_run_ranges(self): """Test run ranges """ self.provider.connect(self.connection_str) # this test requires the connection # Get run range by name, test "all" run range rr = self.provider.get_named_run_range("all") self.assertIsNotNone(rr) # Get run range by min and max run values rr = self.provider.get_run_range(0, 2000) self.assertIsNotNone(rr) # NON EXISTENT RUN RANGE # ---------------------------------------------------- # Get run range that is not defined try: rr = self.provider.get_run_range(0, 2001) # oh... such run range exists? It shouldn't be... Maybe it is left because of the last tests... print ("WARNING provider.get_run_range(0, 2001) found run range (should not be there)") print ("trying to delete run range and run the test one more time... ") self.provider.delete_run_range(rr) # (!) <-- test of this function is further rr = self.provider.get_run_range(0, 2001) self.assertIsNotNone(rr) except RunRangeNotFound: pass # test passed # GET OR CREATE RUNRANGE # ---------------------------------------------------- # Get or create run-range is the main function to get RunRange without name # 0-2001 should be absent or deleted so this function will create run-range rr = self.provider.get_or_create_run_range(0, 2001) self.assertIsNotNone(rr) self.assertNotEquals(rr.id, 0) self.assertEquals(rr.min, 0) self.assertEquals(rr.max, 2001) # DELETE RUN-RANGE TEST # ---------------------------------------------------- self.provider.delete_run_range(rr) self.assertRaises(RunRangeNotFound, self.provider.get_run_range, 0, 2001) def test_variations(self): """Test variations""" self.provider.connect(self.connection_str) # this test requires the connection # Get variation by name, test "all" run range v = self.provider.get_variation("default") self.assertIsNotNone(v) # Get variations by type table table = self.provider.get_type_table("/test/test_vars/test_table") vs = self.provider.search_variations(table) self.assertIsNotNone(vs) self.assertNotEquals(len(vs), 0) # Get variations by name vs = self.provider.get_variations("def*") var_names = [var.name for var in vs] self.assertIn("default", var_names) # NON EXISTENT VARIATION # ---------------------------------------------------- # Get run range that is not defined try: v = self.provider.get_variation("abra_kozyabra") # oh... such run range exists? It shouldn't be... Maybe it is left because of the last tests... print ("WARNING provider.get_variation('abra_kozyabra') found but should not be there") print ("trying to delete variation and run the test one more time... ") self.provider.delete_variation(v) # (!) <-- test of this function is further v = self.provider.get_variation("abra_kozyabra") self.assertIsNotNone(v) except VariationNotFound: pass # test passed # create variation # ---------------------------------------------------- # Get or create run-range is the main function to get RunRange without name # 0-2001 should be absent or deleted so this function will create run-range v = self.provider.create_variation("abra_kozyabra") self.assertIsNotNone(v) self.assertNotEquals(v.id, 0) self.assertEquals(v.parent_id, 1) self.assertEquals(v.name, "abra_kozyabra") # DELETE RUN-RANGE TEST # ---------------------------------------------------- self.provider.delete_variation(v) self.assertRaises(VariationNotFound, self.provider.get_variation, "abra_kozyabra") # Now create with comment and parent v = self.provider.create_variation("abra_kozyabra", "Abra!!!", "test") self.assertEquals(v.parent.name, "test") self.assertEquals(v.comment, "Abra!!!") # cleanup self.provider.delete_variation(v) def test_variation_backup(self): """Test Backup of """ self.provider.connect(self.connection_str) # this test requires the connection a = self.provider.get_assignment("/test/test_vars/test_table", 100, "test") self.assertEqual(a.constant_set.data_list[0], "2.2") # No such calibration exist in test variation run 100, but constants should fallback to variation default def test_assignments(self): """Test Assignments""" self.provider.connect(self.connection_str) # this test requires the connection assignment = self.provider.get_assignment("/test/test_vars/test_table", 100, "default") self.assertIsNotNone(assignment) # Check that everything is loaded tabled_data = assignment.constant_set.data_table self.assertEquals(len(tabled_data), 2) self.assertEquals(len(tabled_data[0]), 3) self.assertEquals(tabled_data[0][0], "2.2") self.assertEquals(tabled_data[0][1], "2.3") self.assertEquals(tabled_data[0][2], "2.4") self.assertEquals(tabled_data[1][0], "2.5") self.assertEquals(tabled_data[1][1], "2.6") self.assertEquals(tabled_data[1][2], "2.7") # Ok! Lets get all assignments for current types table assignments = self.provider.get_assignments("/test/test_vars/test_table") self.assertNotEquals(len(assignments), 0) # Ok! Lets get all assignments for current types table and variation assignments = self.provider.get_assignments("/test/test_vars/test_table", variation="default") self.assertNotEquals(len(assignments), 0) assignment = self.provider.create_assignment([[0, 1, 2], [3, 4, 5]], "/test/test_vars/test_table", 0, 1000, "default", "Test assignment") self.assertEqual(assignment.constant_set.type_table.path, "/test/test_vars/test_table") self.assertEqual(assignment.variation.name, "default") self.assertEqual(assignment.run_range.min, 0) self.assertEqual(assignment.run_range.max, 1000) self.assertEqual(assignment.comment, "Test assignment") tabled_data = assignment.constant_set.data_table self.assertEquals(len(tabled_data), 2) self.assertEquals(len(tabled_data[0]), 3) self.assertEquals(tabled_data[0][0], "0") self.assertEquals(tabled_data[0][1], "1") self.assertEquals(tabled_data[0][2], "2") self.assertEquals(tabled_data[1][0], "3") self.assertEquals(tabled_data[1][1], "4") self.assertEquals(tabled_data[1][2], "5") self.provider.delete_assignment(assignment) def test_users(self): """Test users""" self.provider.connect(self.connection_str) # this test requires the connection user = self.provider.get_user("anonymous") self.assertIsNotNone(user) self.assertEqual(user.name, "anonymous") user = self.provider.get_user("test_user") isinstance(user, User) self.assertIsNotNone(user) self.assertEqual(user.password, "test") self.assertEqual(user.roles, ["runrange_crate", "runrange_delete"]) # self.assertEqual(user.) # test that with wrong user we can't create anything self.provider.authentication.current_user_name = "non_exist_user_ever" self.assertRaises(UserNotFoundError, self.provider.create_directory, "some_strange_dir", "/") self.assertEqual(0, len(self.provider.search_directories("some_strange_dir"))) self.assertRaises(UserNotFoundError, self.provider.update_directory, self.provider.get_directory("/test")) self.assertRaises(UserNotFoundError, self.provider.delete_directory, self.provider.get_directory("/test")) self.assertIsNotNone(self.provider.get_directory("/test")) @staticmethod def test_gen_flatten_data(): source = [[1, 2], [3, "444"]] result = list(gen_flatten_data(source)) assert result[0] == 1 assert result[1] == 2 assert result[2] == 3 assert result[3] == "444" def test_list_to_blob(self): self.assertMultiLineEqual("1|2|33", list_to_blob([1, 2, "33"])) self.assertMultiLineEqual("strings|with&delimiter;surprise", list_to_blob(["strings", "with|surprise"])) def test_blob_to_list(self): self.assertItemsEqual(["1", "2", "str"], blob_to_list("1|2|str")) self.assertItemsEqual(["strings", "with|surprise"], blob_to_list("strings|with&delimiter;surprise")) def test_list_to_table(self): self.assertRaises(ValueError, list_to_table, [1, 2, 3], 2) self.assertItemsEqual([[1, 2, 3], [4, 5, 6]], list_to_table([1, 2, 3, 4, 5, 6], 3)) def test_get_users(self): self.provider.connect(self.connection_str) # this test requires the connection users = self.provider.get_users() self.assertGreater(len(users), 0) def test_validate_data(self): column = TypeTableColumn() # int type column.type = 'int' self.assertEqual(self.provider.validate_data_value('1', column), 1) self.assertRaises(ValueError, self.provider.validate_data_value, 'hren', column) # lets check bool type column.type = 'bool' self.assertEqual(self.provider.validate_data_value('TrUe', column), True) self.assertEqual(self.provider.validate_data_value('FalSe', column), False) self.assertEqual(self.provider.validate_data_value('1', column), True) self.assertEqual(self.provider.validate_data_value('0', column), False) self.assertRaises(ValueError, self.provider.validate_data_value, 'hren', column) # uint! column.type = 'uint' self.assertEqual(self.provider.validate_data_value('1', column), 1) self.assertRaises(ValueError, self.provider.validate_data_value, '-1', column)
class ConsoleContext(object): """ Class to manage console commands This class uses console_utilities from utils directories """ anonymous_user_name = "anonymous" prefix = None words = [] def __init__(self): self._prov = AlchemyProvider() self._is_interactive = False self.user_name = self.anonymous_user_name self._is_interactive = False self._current_run = 0 self._current_path = "/" self._current_variation = "default" self._utils = {} self._verbose = False self._ls = None self._connection_string = "" self.silent_exceptions = True # rethrow happened exceptions self._theme = themes.NoColorTheme() self.log_sqlite = False # prop verbose # _______________________ def _get_verbose(self): """Sets or gets verbose behaviour for this class""" return self._verbose def _set_verbose(self, is_verbose): self._verbose = is_verbose verbose = property(_get_verbose, _set_verbose) @property def utils(self): """:rtype: {}""" return self._utils @property def connection_string(self): """:rtype: str""" return self._connection_string @connection_string.setter def connection_string(self, con_str): self._connection_string = con_str @property def provider(self): return self._prov @property def current_path(self): return self._current_path @current_path.setter def current_path(self, new_path): self._current_path = new_path @property def current_run(self): """ Sets or gets current working run :rtype: int """ return self._current_run @current_run.setter def current_run(self, new_run): self._current_run = new_run @property def current_variation(self): """ Sets or gets current working variation :rtype: str """ return self._current_variation @current_variation.setter def current_variation(self, new_var): self._current_variation = new_var @property def user_name(self): return self._prov.authentication.current_user_name @user_name.setter def user_name(self, name): self._prov.authentication.current_user_name = name @property def is_interactive(self): return self._is_interactive @is_interactive.setter def is_interactive(self, value): self._is_interactive = value @property def theme(self): """:rtype: themes.NoColorTheme""" return self._theme @theme.setter def theme(self, value): """ :param value: new theme to set :type value: themes.NoColorTheme """ assert (isinstance(value, themes.NoColorTheme)) for key in self._utils.keys(): self._utils[key].theme = value log.debug( lfm(" |- theme(value) {0} | \\{0} | |- theme switched to : '{1}'", os.linesep, value)) self._theme = value # ===================================================================================== # ------------------ P L U G I N M A N A G E M E N T ------------------------------- # ===================================================================================== # -------------------------------- # register_utilities # -------------------------------- def register_utilities(self, path=""): """ Function to auto find and register utilities""" path = path or os.path.join(ccdb.cmd.__path__[0], "utils") # search modules modules = self.search_utils(path) self._utils = {} # register each module for module in modules: try: registerFunc = getattr(module, "create_util_instance") util = registerFunc() if util: self._utils[util.command] = util util.context = self util.theme = self.theme if util.command == "ls": self._ls = util except AttributeError as ex: log.debug("Error registering module : " + repr(ex)) except Exception as ex: log.debug("Error registering module : " + repr(ex)) if log.isEnabledFor(logging.DEBUG): log.debug( lfm("{0}Utils found and registered in directory '{1}' are:", os.linesep, path)) log.debug("{0:<10} {1:<15} {2}:".format("(command)", "(name)", "(description)")) log.debug("\n".join([ "{0:<10} {1:<15} {2}:".format(command, util.name, util.short_descr) for command, util in self._utils.items() ])) # -------------------------------- # search_utils # -------------------------------- def search_utils(self, path): """Load plugins from directory and return list of modules :rtype: [] """ # >oO debug output log.debug(lfm("{0}search_utils{0}\\", os.linesep)) log.debug( lfm(" |- searching modules in directory:{0} | '{1}{2}{3}'", os.linesep, self.theme.Directories, path, self.theme.Reset)) # get list of files and module names files = os.listdir(path) test = re.compile(".py$", re.IGNORECASE) files = filter(test.search, files) filenameToModuleName = lambda f: os.path.splitext(f)[0] moduleNames = sorted(map(filenameToModuleName, files)) log.debug( lfm( " |- found '{0}' modules.{1} |- proceed loading each module:{1}", len(moduleNames), os.linesep)) modules = [] for m in moduleNames: # skip any files starting with '__', such as __init__.py if m.startswith('__'): continue try: f, filename, desc = imp.find_module(m, [path]) modules.append(imp.load_module(m, f, filename, desc)) except ImportError as ex: log.debug(lfm(" |- error importing module: {0}", m)) log.debug(lfm(" |\\{0} ||-{1}", os.linesep, repr(ex))) continue return modules # -------------------------------- # processes the arguments # -------------------------------- def process(self, args, startIndex=1): # check if there is enough arguments... if len(args) < (startIndex + 1): self.print_general_usage() return # get working arguments list workargs = args[startIndex:] # parse loop i = 0 while i < len(workargs): token = workargs[i] assert isinstance(token, str) i += 1 if token.startswith('-'): # it is some command, lets parse what is the command if (token == "-c" or token == "--connection") and (i < len(workargs)): # connection string self.connection_string = workargs[i] i += 1 elif token == "-I" or token == "-i" or token == "--interactive": # it is an interactive mode self._is_interactive = True elif token == "-r" or token == "--run": # working run try: self.current_run = int(workargs[i]) log.info("Working run is %i", self.current_run) except ValueError: log.warning( "(!) Warning. Cannot read run from %s command" % token) i += 1 elif token == "-v" or token == "--variation": # working variation if not ccdb.path_utils.validate_name(workargs[i]): log.warning( "(!) Warning. Cannot read variation from --variation flag. " "Variation name should consist of A-Z, a-z, 0-9, _" ) else: self._current_variation = workargs[i] i += 1 else: # looks like is is a command command = token commandArgs = workargs[i:] try: self.process_command(command, commandArgs) break except Exception as ex: log.error(ex) if not self.silent_exceptions: raise else: return 1 # we have to ask mysql password if "--mysql-pwd" in args[ startIndex:] and self.connection_string.startswith("mysql"): print("Enter MySql password: "******"@", ":" + password + "@") if self._is_interactive: self.interactive_loop() # -------------------------------- # executes text as command # -------------------------------- def process_command_line(self, command_line): """tries to execute a line of text""" log.debug("{0}Process command line: {1}{0}\\".format( os.linesep, command_line)) # execute shell command if input starts with '!' if command_line.startswith('!'): command_line = command_line[1:] # noinspection PyBroadException try: os.system(command_line) except: pass return # everything is done # split command to arguments is_posix = os.name == 'posix' tokens = shlex.split(command_line, posix=is_posix) # validate if len(tokens) == 0: return # get our command command = tokens[0] log.debug(" |- command is : {0}".format(command)) log.debug(" |- tokens : {0}".format(" ".join([("'" + t + "'") for t in tokens]))) # get command arguments arguments = [] if len(tokens) > 1: arguments = tokens[1:] # execute everything return self.process_command(command, arguments) # -------------------------------- # # -------------------------------- def process_command(self, command, args): # >oO debug if log.isEnabledFor(logging.DEBUG): log.debug( lfm("{0}Processing command: '{1}'{0}\\", os.linesep, command)) log.debug(lfm(" |- arguments : '{0}'", "' '".join(args))) # try to find function... try: util = self._utils[command] except KeyError: log.error( "Command " + command + " is unknown! Please, use help to see available commands") if not self.silent_exceptions: raise else: return 1 # check connection and connect if needed if util.uses_db and (not self.provider.is_connected): if not self.check_connection(util): return False # is there file redirect? redir_to_file = False # should we redirect to file? redir_file = None # file to redirect redir_stream_backup = sys.stdout redir_theme_backup = self.theme if ">" in args and args.index(">") == len(args) - 2: redir_fname = args[-1] redir_to_file = True args = args[:-2] log.debug(" |- redirecting to file : {0}".format(redir_fname)) # open file try: redir_file = file(redir_fname, 'w') except Exception as ex: log.error("Cannot open file '{0}' {1} ".format( redir_fname, ex)) if not self.silent_exceptions: raise else: return 1 # execute command try: if redir_to_file: colorama.deinit() sys.stdout = redir_file self.theme = themes.NoColorTheme() result = util.process(args) except Exception as ex: log.error(ex) if not self.silent_exceptions: raise else: return 1 finally: if redir_to_file: sys.stdout = redir_stream_backup redir_file.close() self.theme = redir_theme_backup colorama.reinit() return result # -------------------------------- # # -------------------------------- def check_connection(self, util): # maybe there is nothing to do? if self._prov.is_connected or (not util.uses_db): return if log.isEnabledFor(logging.DEBUG): log.debug(" |- check_connection(util){0} | \\".format(os.linesep)) log.debug( " | |- utility '{}' requires DB. CCDB not yet connected. Connecting." .format(util.name)) log.debug(" | |- connection string is: '{}{}{}'".format( self.theme.Accent, self.connection_string, self.theme.Reset)) # connecting try: self._prov.connect(self.connection_string) except Exception as ex: log.critical( "CCDB provider unable to connect to {0}. Aborting command. Exception details: {1}" .format(self.connection_string, ex)) return False # skip user for sqlite if (not self.log_sqlite) and ( self.provider.connection_string.startswith("sqlite://")): log.debug( " | |- log_sqlite == False, set user to anonymous and disable logging'" ) self.provider.logging_enabled = False # self.user_name = self.anonymous_user_name # connected if log.isEnabledFor(logging.DEBUG): if self.provider.connection_string.startswith( "mysql+mysqlconnector://"): log.debug( " | |- no module MySQLdb. Fallback to mysql-connector used" ) log.debug(" | |- connection string used: '" + self.theme.Accent + self.provider.connection_string + self.theme.Reset + "'") log.debug(" | |- connection: " + self.theme.Success + " successful " + self.theme.Reset) return True # ---------------------------------------------------- # interactive_loop # ---------------------------------------------------- # noinspection PyBroadException def interactive_loop(self): import readline self.print_interactive_intro() # initialise autocomplete self.words = self._utils.keys() # completer = Completer(words) colorama.deinit() # make colorama to release stderr and stdout readline.parse_and_bind("tab: complete") readline.set_completer(self.complete) try: readline.read_history_file() except: pass # eat it # readline..set_completion_display_matches_hook(self.show_completions) # readline.set_completion_display_matches_hook([function]) # Set or remove the completion display function. If function is specified, it will be used as the new completion # display function; if omitted or None, any completion display function already installed is removed. # The completion display function is called as function(substitution, [matches], longest_match_length) each # time matches need to be displayed. # Begin user commands read loop while 1: colorama.deinit() # read command from user try: user_input = raw_input(self.current_path + "> ") except EOFError: log.debug("EOF sequence received. Ending interactive loop") break except KeyboardInterrupt: log.debug("Break sequence received. Ending interactive loop") break colorama.reinit() # exit if user wishes so if user_input in ("quit", "q", "exit"): break # quit! # process user input self.process_command_line(user_input) # loop ended try: readline.write_history_file() except: log.warn("unable to write history file") # ===================================================================================== # ------------------------ C O M P L E T I O N ---------------------------------------- # ===================================================================================== # region Description # -------------------------------- # show_completions # -------------------------------- def show_completions(self, substitution, matches, longest_match_length): print(self) print(substitution) print(matches) print(longest_match_length) # -------------------------------- # generate_completion_words # -------------------------------- # noinspection PyBroadException def generate_completion_words(self, prefix): # find all words that start with this prefix self.matching_words = [w for w in self.words if w.startswith(prefix)] # get name patches path_prefix = posixpath.join(self.current_path, prefix) log.debug("getting path completion for: " + path_prefix) try: self.check_connection(self._ls) try: result = self._ls.get_name_pathes(path_prefix) except: result = None if result is None or (len(result[0]) == 0 and len(result[1]) == 0): result = self._ls.get_name_pathes(path_prefix + "*") dir_list = [subdir.name for subdir in result[0]] table_list = [table.name for table in result[1]] self.matching_words.extend(dir_list) self.matching_words.extend(table_list) except Exception as ex: log.debug("error getting competition paths: " + ex.message) # -------------------------------- # complete # -------------------------------- def complete(self, prefix, index): # print "prefix ", prefix, " index ", index # readline.insert_text("bla bla bla") # colorama.deinit() # print "ha ha ha" # colorama.reinit() if prefix != self.prefix: # get new completions self.generate_completion_words(prefix) # self.matching_words.append( self.prefix = prefix else: # print "prefix the same, matching:", len(self.matching_words) # print " ".join([word for word in self.matching_words]) # readline.redisplay() pass try: return self.matching_words[index] except IndexError: return None # endregion # ===================================================================================== # -------- G E T T I N G O B J E C T S ------------------------------------------- # ===================================================================================== # -------------------------------- # prepare_path # -------------------------------- def prepare_path(self, path): # correct ending / if path.endswith("/"): path = path[:-1] # local or absolute path? if not path.startswith("/"): path = posixpath.join(self.current_path, path) # normalize path = posixpath.normpath(path) return path # -------------------------------- # parse_run_range # -------------------------------- def parse_run_range(self, run_range_str): """ @brief parse run range string in form of <run_min>-<run-max> if one inputs '<run_min>-' this means <run_min>-<infinite run> if one inputs '-<run_max>' this means <0>-<run_max> @return (run_min, run_max, run_min_set, run_max_set) run_min_set, run_max_set - are flags indicating that values was set by user """ assert isinstance(run_range_str, str) if not "-" in run_range_str: return None # split <>-<> (str_min, str_max) = run_range_str.split("-") run_min = 0 run_min_set = False run_max = ccdb.INFINITE_RUN run_max_set = False # parse run min try: run_min = int(str_min) run_min_set = True except ValueError: self.run_min = 0 # parse run max try: run_max = int(str_max) run_max_set = True except ValueError: self.run_max = ccdb.INFINITE_RUN return run_min, run_max, run_min_set, run_max_set # ===================================================================================== # ------------------------- H E L P A N D I N F O --------------------------------- # ===================================================================================== def print_general_usage(self): print("Use '-i' option to enter interactive shell") print("Use 'help' option for help") print("Use 'help command' to get help for particular command") def print_info(self): log.info("Login as : '" + self.user_name + "'") log.info("Connect to : '" + self.connection_string + "'") log.info("Variation : '" + self.current_variation + "'") log.info("Deflt. run : '" + str(self.current_run) + "'") def print_interactive_intro(self): print(""" +--------------------------+ CCDB shell v.1.07.00 +--------------------------+ """) print(self.theme.Title + "Interactive mode") print("print " + self.theme.Accent + "help" + self.theme.Reset + " to get help") print("print " + self.theme.Accent + "quit" + self.theme.Reset + " or " + self.theme.Accent + "q" + self.theme.Reset + " to exit") print("print !<command> to execute shell command") print() self.print_info()
def setUp(self): self._connection_str = self.sqlite_connection_str self._provider = AlchemyProvider() self.provider.logging_enabled = False
class ConsoleContext(object): """ Class to manage console commands This class uses console_utilities from utils directories """ anonymous_user_name = "anonymous" prefix = None words = [] def __init__(self): self._prov = AlchemyProvider() self._is_interactive = False self.user_name = self.anonymous_user_name self._is_interactive = False self._current_run = 0 self._current_path = "/" self._current_variation = "default" self._utils = {} self._verbose = False self._ls = None self._connection_string = "" self.silent_exceptions = True # rethrow happened exceptions self._theme = themes.NoColorTheme() self.log_sqlite = False # prop verbose # _______________________ def _get_verbose(self): """Sets or gets verbose behaviour for this class""" return self._verbose def _set_verbose(self, is_verbose): self._verbose = is_verbose verbose = property(_get_verbose, _set_verbose) @property def utils(self): """:rtype: {}""" return self._utils @property def connection_string(self): """:rtype: str""" return self._connection_string @connection_string.setter def connection_string(self, con_str): self._connection_string = con_str @property def provider(self): return self._prov @property def current_path(self): return self._current_path @current_path.setter def current_path(self, new_path): self._current_path = new_path @property def current_run(self): """ Sets or gets current working run :rtype: int """ return self._current_run @current_run.setter def current_run(self, new_run): self._current_run = new_run @property def current_variation(self): """ Sets or gets current working variation :rtype: str """ return self._current_variation @current_variation.setter def current_variation(self, new_var): self._current_variation = new_var @property def user_name(self): return self._prov.authentication.current_user_name @user_name.setter def user_name(self, name): self._prov.authentication.current_user_name = name @property def is_interactive(self): return self._is_interactive @is_interactive.setter def is_interactive(self, value): self._is_interactive = value @property def theme(self): """:rtype: themes.NoColorTheme""" return self._theme @theme.setter def theme(self, value): """ :param value: new theme to set :type value: themes.NoColorTheme """ assert isinstance(value, themes.NoColorTheme) for key in self._utils.keys(): self._utils[key].theme = value log.debug(lfm(" |- theme(value) {0} | \\{0} | |- theme switched to : '{1}'", os.linesep, value)) self._theme = value # ===================================================================================== # ------------------ P L U G I N M A N A G E M E N T ------------------------------- # ===================================================================================== # -------------------------------- # register_utilities # -------------------------------- def register_utilities(self, path=""): """ Function to auto find and register utilities""" path = path or os.path.join(ccdb.cmd.__path__[0], "utils") # search modules modules = self.search_utils(path) self._utils = {} # register each module for module in modules: try: registerFunc = getattr(module, "create_util_instance") util = registerFunc() if util: self._utils[util.command] = util util.context = self util.theme = self.theme if util.command == "ls": self._ls = util except AttributeError as ex: log.debug("Error registering module : " + repr(ex)) except Exception as ex: log.debug("Error registering module : " + repr(ex)) if log.isEnabledFor(logging.DEBUG): log.debug(lfm("{0}Utils found and registered in directory '{1}' are:", os.linesep, path)) log.debug("{0:<10} {1:<15} {2}:".format("(command)", "(name)", "(description)")) log.debug( "\n".join( [ "{0:<10} {1:<15} {2}:".format(command, util.name, util.short_descr) for command, util in self._utils.items() ] ) ) # -------------------------------- # search_utils # -------------------------------- def search_utils(self, path): """Load plugins from directory and return list of modules :rtype: [] """ # >oO debug output log.debug(lfm("{0}search_utils{0}\\", os.linesep)) log.debug( lfm( " |- searching modules in directory:{0} | '{1}{2}{3}'", os.linesep, self.theme.Directories, path, self.theme.Reset, ) ) # get list of files and module names files = os.listdir(path) test = re.compile(".py$", re.IGNORECASE) files = filter(test.search, files) filenameToModuleName = lambda f: os.path.splitext(f)[0] moduleNames = sorted(map(filenameToModuleName, files)) log.debug(lfm(" |- found '{0}' modules.{1} |- proceed loading each module:{1}", len(moduleNames), os.linesep)) modules = [] for m in moduleNames: # skip any files starting with '__', such as __init__.py if m.startswith("__"): continue try: f, filename, desc = imp.find_module(m, [path]) modules.append(imp.load_module(m, f, filename, desc)) except ImportError as ex: log.debug(lfm(" |- error importing module: {0}", m)) log.debug(lfm(" |\\{0} ||-{1}", os.linesep, repr(ex))) continue return modules # -------------------------------- # # -------------------------------- def process(self, args, startIndex=1): # check if there is enough arguments... if len(args) < (startIndex + 1): self.print_general_usage() return # get working arguments list workargs = args[startIndex:] # parse loop i = 0 while i < len(workargs): token = workargs[i] assert isinstance(token, str) i += 1 if token.startswith("-"): # it is some command, lets parse what is the command if (token == "-c" or token == "--connection") and (i < len(workargs)): # connection string self.connection_string = workargs[i] i += 1 elif token == "-I" or token == "-i" or token == "--interactive": # it is an interactive mode self._is_interactive = True elif token == "-r" or token == "--run": # working run try: self.current_run = int(workargs[i]) log.info("Working run is %i", self.current_run) except ValueError: log.warning("(!) Warning. Cannot read run from %s command" % token) i += 1 elif token == "-v" or token == "--variation": # working variation if not ccdb.path_utils.validate_name(workargs[i]): log.warning( "(!) Warning. Cannot read variation from --variation flag. " "Variation name should consist of A-Z, a-z, 0-9, _" ) else: self._current_variation = workargs[i] i += 1 else: # looks like is is a command command = token commandArgs = workargs[i:] try: self.process_command(command, commandArgs) break except Exception as ex: log.error(ex) if not self.silent_exceptions: raise else: return 1 # we have to ask mysql password if "-p" in workargs and self.connection_string.startswith("mysql"): print("Enter MySql password: "******"@", ":" + password + "@") if self._is_interactive: self.interactive_loop() # -------------------------------- # executes text as command # -------------------------------- def process_command_line(self, command_line): """tries to execute a line of text""" log.debug("{0}Process command line: {1}{0}\\".format(os.linesep, command_line)) # execute shell command if input starts with '!' if command_line.startswith("!"): command_line = command_line[1:] # noinspection PyBroadException try: os.system(command_line) except: pass return # everything is done # split command to arguments tokens = shlex.split(command_line) # validate if len(tokens) == 0: return # get our command command = tokens[0] log.debug(" |- command is : {0}".format(command)) log.debug(" |- tokens : {0}".format(" ".join([("'" + t + "'") for t in tokens]))) # get command arguments arguments = [] if len(tokens) > 1: arguments = tokens[1:] # execute everything return self.process_command(command, arguments) # -------------------------------- # # -------------------------------- def process_command(self, command, args): # >oO debug if log.isEnabledFor(logging.DEBUG): log.debug(lfm("{0}Processing command: '{1}'{0}\\", os.linesep, command)) log.debug(lfm(" |- arguments : '{0}'", "' '".join(args))) # try to find function... try: util = self._utils[command] except KeyError: log.error("Command " + command + " is unknown! Please, use help to see available commands") if not self.silent_exceptions: raise else: return 1 # check connection and connect if needed if util.uses_db and (not self.provider.is_connected): if not self.check_connection(util): return False # is there file redirect? redir_to_file = False # should we redirect to file? redir_file = None # file to redirect redir_stream_backup = sys.stdout redir_theme_backup = self.theme if ">" in args and args.index(">") == len(args) - 2: redir_fname = args[-1] redir_to_file = True args = args[:-2] log.debug(" |- redirecting to file : {0}".format(redir_fname)) # open file try: redir_file = file(redir_fname, "w") except Exception as ex: log.error("Cannot open file '{0}' {1} ".format(redir_fname, ex)) if not self.silent_exceptions: raise else: return 1 # execute command try: if redir_to_file: colorama.deinit() sys.stdout = redir_file self.theme = themes.NoColorTheme() result = util.process(args) except Exception as ex: log.error(ex) if not self.silent_exceptions: raise else: return 1 finally: if redir_to_file: sys.stdout = redir_stream_backup redir_file.close() self.theme = redir_theme_backup colorama.reinit() return result # -------------------------------- # # -------------------------------- def check_connection(self, util): # maybe there is nothing to do? if self._prov.is_connected or (not util.uses_db): return if log.isEnabledFor(logging.DEBUG): log.debug(" |- check_connection(util){0} | \\".format(os.linesep)) log.debug( " | |- utility '" + util.name + "' requires the database " "and there is no connection yet. Connecting." ) log.debug( " | |- connection string is: '" + self.theme.Accent + self.connection_string + self.theme.Reset + "'" ) # connecting try: self._prov.connect(self.connection_string) except Exception as ex: log.critical( "CCDB provider unable to connect to {0}. Aborting command. Exception details: {1}".format( self.connection_string, ex ) ) return False # skip user for sqlite if (not self.log_sqlite) and (self.provider.connection_string.startswith("sqlite://")): log.debug(" | |- log_sqlite == False, set user to anonymous and disable logging'") self.provider.logging_enabled = False self.user_name = self.anonymous_user_name # connected if log.isEnabledFor(logging.DEBUG): if self.provider.connection_string.startswith("mysql+mysqlconnector://"): log.debug(" | |- no module MySQLdb. Fallback to mysql-connector used") log.debug( " | |- connection string used: '" + self.theme.Accent + self.provider.connection_string + self.theme.Reset + "'" ) log.debug(" | |- connection: " + self.theme.Success + " successful " + self.theme.Reset) return True # ---------------------------------------------------- # interactive_loop # ---------------------------------------------------- # noinspection PyBroadException def interactive_loop(self): self.print_interactive_intro() # initialise autocomplete self.words = self._utils.keys() # completer = Completer(words) colorama.deinit() # make colorama to release stderr and stdout readline.parse_and_bind("tab: complete") readline.set_completer(self.complete) try: readline.read_history_file() except: pass # eat it # readline..set_completion_display_matches_hook(self.show_completions) # readline.set_completion_display_matches_hook([function]) # Set or remove the completion display function. If function is specified, it will be used as the new completion # display function; if omitted or None, any completion display function already installed is removed. # The completion display function is called as function(substitution, [matches], longest_match_length) each # time matches need to be displayed. # Begin user commands read loop while 1: colorama.deinit() # read command from user try: user_input = raw_input(self.current_path + "> ") except EOFError: log.debug("EOF sequence received. Ending interactive loop") break except KeyboardInterrupt: log.debug("Break sequence received. Ending interactive loop") break colorama.reinit() # exit if user wishes so if user_input in ("quit", "q", "exit"): break # quit! # process user input self.process_command_line(user_input) # loop ended try: readline.write_history_file() except: log.warn("unable to write history file") # ===================================================================================== # ------------------------ C O M P L E T I O N ---------------------------------------- # ===================================================================================== # region Description # -------------------------------- # show_completions # -------------------------------- def show_completions(self, substitution, matches, longest_match_length): print(self) print(substitution) print(matches) print(longest_match_length) # -------------------------------- # generate_completion_words # -------------------------------- # noinspection PyBroadException def generate_completion_words(self, prefix): # find all words that start with this prefix self.matching_words = [w for w in self.words if w.startswith(prefix)] # get name patches path_prefix = posixpath.join(self.current_path, prefix) log.debug("getting path completion for: " + path_prefix) try: self.check_connection(self._ls) try: result = self._ls.get_name_pathes(path_prefix) except: result = None if result is None or (len(result[0]) == 0 and len(result[1]) == 0): result = self._ls.get_name_pathes(path_prefix + "*") dir_list = [subdir.name for subdir in result[0]] table_list = [table.name for table in result[1]] self.matching_words.extend(dir_list) self.matching_words.extend(table_list) except Exception as ex: log.debug("error getting competition paths: " + ex.message) # -------------------------------- # complete # -------------------------------- def complete(self, prefix, index): # print "prefix ", prefix, " index ", index # readline.insert_text("bla bla bla") # colorama.deinit() # print "ha ha ha" # colorama.reinit() if prefix != self.prefix: # get new completions self.generate_completion_words(prefix) # self.matching_words.append( self.prefix = prefix else: # print "prefix the same, matching:", len(self.matching_words) # print " ".join([word for word in self.matching_words]) # readline.redisplay() pass try: return self.matching_words[index] except IndexError: return None # endregion # ===================================================================================== # -------- G E T T I N G O B J E C T S ------------------------------------------- # ===================================================================================== # -------------------------------- # prepare_path # -------------------------------- def prepare_path(self, path): # correct ending / if path.endswith("/"): path = path[:-1] # local or absolute path? if not path.startswith("/"): path = posixpath.join(self.current_path, path) # normalize path = posixpath.normpath(path) return path # -------------------------------- # parse_run_range # -------------------------------- def parse_run_range(self, run_range_str): """ @brief parse run range string in form of <run_min>-<run-max> if one inputs '<run_min>-' this means <run_min>-<infinite run> if one inputs '-<run_max>' this means <0>-<run_max> @return (run_min, run_max, run_min_set, run_max_set) run_min_set, run_max_set - are flags indicating that values was set by user """ assert isinstance(run_range_str, str) if not "-" in run_range_str: return None # split <>-<> (str_min, str_max) = run_range_str.split("-") run_min = 0 run_min_set = False run_max = ccdb.INFINITE_RUN run_max_set = False # parse run min try: run_min = int(str_min) run_min_set = True except ValueError: self.run_min = 0 # parse run max try: run_max = int(str_max) run_max_set = True except ValueError: self.run_max = ccdb.INFINITE_RUN return run_min, run_max, run_min_set, run_max_set # ===================================================================================== # ------------------------- H E L P A N D I N F O --------------------------------- # ===================================================================================== def print_general_usage(self): print("Use '-i' option to enter interactive shell") print("Use 'help' option for help") print("Use 'help command' to get help for particular command") def print_info(self): log.info("Login as : '" + self.user_name + "'") log.info("Connect to : '" + self.connection_string + "'") log.info("Variation : '" + self.current_variation + "'") log.info("Deflt. run : '" + str(self.current_run) + "'") def print_interactive_intro(self): print( """ +--------------------------+ CCDB shell v.1.05b +--------------------------+ """ ) print(self.theme.Title + "Interactive mode") print("print " + self.theme.Accent + "help" + self.theme.Reset + " to get help") print( "print " + self.theme.Accent + "quit" + self.theme.Reset + " or " + self.theme.Accent + "q" + self.theme.Reset + " to exit" ) print("print !<command> to execute shell command") print() self.print_info()
def setUp(self): ccdb_path = get_ccdb_home_path() self.sqlite_connection_str = "sqlite:///" + os.path.join(ccdb_path, "sql", "ccdb.sqlite") self.provider = AlchemyProvider()
def setUp(self): ccdb_path = get_ccdb_home_path() self.sqlite_connection_str = "sqlite:///" + os.path.join(ccdb_path, "sql", "ccdb.sqlite") self.mysql_connection_str = "mysql://[email protected]:3306/ccdb" self.provider = AlchemyProvider()