def get_schema(self, con, table_name): # this should really accept repo and table name as seperate try: manager = DataHubManager(user=con.user, repo_base=con.repo_base, is_app=con.is_app) # thrift isn't upgraded to use seperate repo and tokens yet, # so they have to be split here, and then passed to the manager. tokens = table_name.split('.') repo = tokens[0] table = tokens[1] res = manager.get_schema(repo=repo, table=table) # apps expect get_schema to return in a backwards way. It's made # to be backwards here. thrift_crazy_result = { 'status': True, 'row_count': 0, 'tuples': res, 'fields': [{ 'type': 1043, 'name': 'column_name' }, { 'type': 1043, 'name': 'data_type' }] } return construct_result_set(thrift_crazy_result) except Exception as e: raise DBException(message=str(e))
def get_schema(self, con, table_name): try: manager = DataHubManager(user=con.user, repo_base=con.repo_base) res = manager.get_schema(table=table_name) return construct_result_set(res) except Exception, e: raise DBException(message=str(e))
def get_schema(self, con, table_name): try: manager = DataHubManager(user=con.repo_base, repo_base=con.repo_base) res = manager.get_schema(table=table_name) return construct_result_set(res) except Exception, e: raise DBException(message=str(e))
def schema(request, repo_name, table_name): username = request.user.get_username() manager = DataHubManager(username) repos = get_repos(manager) if repos is not None and repo_name in repos: tables = get_tables(manager, repo_name) if tables is not None and table_name in tables: schema = manager.get_schema(repo_name, table_name) if len(schema) > 0: return json_response({"schema": schema}) return error_response()
def schema(request, repo_name, table_name): username = get_login(request) manager = DataHubManager(username) repos = get_repos(manager) if repos is not None and repo_name in repos: tables = get_tables(manager, repo_name) if tables is not None and table_name in tables: schema = manager.get_schema(repo_name + "." + table_name) if schema is not None and 'tuples' in schema: return json_response({"schema": schema["tuples"]}) return error_response()
def schema(request, repo, table): username = get_login(request) manager = DataHubManager(username) repos = get_repos(manager) if repos is not None and repo in repos: tables = get_tables(manager, repo) if tables is not None and table in tables: schema = manager.get_schema(repo + "." + table) if schema is not None and 'tuples' in schema: return json_response({"schema": schema["tuples"]}) return error_response()
def schema(request, repo, table): username = request.user.get_username() manager = DataHubManager(username) repos = get_repos(manager) if repos is not None and repo in repos: tables = get_tables(manager, repo) if tables is not None and table in tables: schema = manager.get_schema(repo, table) if len(schema) > 0: return json_response({"schema": schema}) return error_response()
def get_schema(repo, table, username): manager = DataHubManager(user=username) pairs = manager.get_schema(repo, table) schema = {} for col, typ in pairs: if typ == 'text': typ = 'str' if 'double' in typ: typ = 'num' if 'int' in typ: typ = 'num' if 'date' in typ or 'time' in typ: typ = 'time' schema[str(col)] = str(typ) return schema
def get_schema(repo, table, username): manager = DataHubManager(user=username) pairs = manager.get_schema(repo, table) schema = {} for col, typ in pairs: if typ == "text": typ = "str" if "double" in typ: typ = "num" if "int" in typ: typ = "num" if "date" in typ or "time" in typ: typ = "time" schema[str(col)] = str(typ) return schema
def get_columns_and_types(self): manager = DataHubManager(user=self.username) rows = manager.get_schema(self.tablename)['tuples'] ret = [] for col, typ in rows: if typ == 'text': typ = 'str' if 'double' in typ: typ = 'num' if 'int' in typ: typ = 'num' if 'date' in typ or 'time' in typ: typ = 'time' ret.append((str(col), str(typ)) ) return ret
def get_columns_and_types(self): manager = DataHubManager(user=self.username) rows = manager.get_schema(self.tablename)['tuples'] ret = [] for col, typ in rows: if typ == 'text': typ = 'str' if 'double' in typ: typ = 'num' if 'int' in typ: typ = 'num' if 'date' in typ or 'time' in typ: typ = 'time' ret.append((str(col), str(typ))) return ret
def get_schema(repo, table, username): full_tablename = "%s.%s" % (repo, table) manager = DataHubManager(user=username) pairs = manager.get_schema(full_tablename)['tuples'] schema = {} for col, typ in pairs: if typ == 'text': typ = 'str' if 'double' in typ: typ = 'num' if 'int' in typ: typ = 'num' if 'date' in typ or 'time' in typ: typ = 'time' schema[str(col)] = str(typ) return schema
def get_schema(repo, table, username, repo_base): """ gets the schema of a given repo.table in a repo_base """ manager = DataHubManager(user=username, repo_base=repo_base) pairs = manager.get_schema(repo, table) schema = {} for col, typ in pairs: if typ == 'text': typ = 'str' if 'double' in typ: typ = 'num' if 'int' in typ: typ = 'num' if 'date' in typ or 'time' in typ: typ = 'time' schema[str(col)] = str(typ) return schema
def aggregate(request, repo, table, agg_type, col_name): username = get_login(request) manager = DataHubManager(username) repos = get_repos(manager) # Ensure that the repo exists. if repos is not None and repo in repos: tables = get_tables(manager, repo) # Ensure that the table exists. if tables is not None and table in tables: schema = manager.get_schema(repo + "." + table) # Ensure that the schema for the repo.table exists. if schema is not None and 'tuples' in schema: for c_name, c_type in schema["tuples"]: if c_name == col_name and can_apply(agg_type, col_name): result = manager.execute_sql("SELECT %s(%s) FROM %s.%s" % (agg_type.lower(), col_name.lower(), repo, table)) if "tuples" in result and len(result["tuples"]) > 0 and len(result["tuples"][0]) > 0: return json_response({"value": result["tuples"][0]}) return error_response()
def aggregate(request, repo, table, agg_type, col_name): username = request.user.get_username() manager = DataHubManager(username) repos = get_repos(manager) # Ensure that the repo exists. if repos is not None and repo in repos: tables = get_tables(manager, repo) # Ensure that the table exists. if tables is not None and table in tables: schema = manager.get_schema(repo, table) # Ensure that the schema for the repo.table exists. if len(schema) > 0: for c_name, c_type in schema: if c_name == col_name and can_apply(agg_type, col_name): result = manager.execute_sql( "SELECT %s(%s) FROM %s.%s" % (agg_type.lower(), col_name.lower(), repo, table)) if "tuples" in result and len(result["tuples"]) > 0 and len(result["tuples"][0]) > 0: return json_response({"value": result["tuples"][0]}) return error_response()
def get_columns_and_types(self): manager = DataHubManager(user=self.username) tokens = self.tablename.split('.') repo = tokens[0] table = tokens[1] rows = manager.get_schema(repo, table) ret = [] for col, typ in rows: if typ == 'text': typ = 'str' if 'double' in typ: typ = 'num' if 'int' in typ: typ = 'num' if 'date' in typ or 'time' in typ: typ = 'time' ret.append((str(col), str(typ)) ) return ret
def get_columns_and_types(self): manager = DataHubManager(user=self.username) tokens = self.tablename.split('.') repo = tokens[0] table = tokens[1] rows = manager.get_schema(repo, table) ret = [] for col, typ in rows: if typ == 'text': typ = 'str' if 'double' in typ: typ = 'num' if 'int' in typ: typ = 'num' if 'date' in typ or 'time' in typ: typ = 'time' ret.append((str(col), str(typ))) return ret
def does_cache_exist(repo_base): """ check to see if the cache exists for the repo base""" manager = DataHubManager(repo_base) repo_exists = False table_exists = False schema_correct = False # check for repo if dbwipes_repo in manager.list_repos(): repo_exists = True # check for table if repo_exists and dbwipes_table in manager.list_tables(dbwipes_repo): table_exists = True # check for schema schema = None if table_exists: schema = manager.get_schema(dbwipes_repo, dbwipes_table) if schema == [('key', 'character varying'), ('val', 'text')]: schema_correct = True return repo_exists and table_exists and schema_correct
def get_schema(self, con, table_name): # this should really accept repo and table name as seperate try: manager = DataHubManager( user=con.user, repo_base=con.repo_base, is_app=con.is_app) # thrift isn't upgraded to use seperate repo and tokens yet, # so they have to be split here, and then passed to the manager. tokens = table_name.split('.') repo = tokens[0] table = tokens[1] res = manager.get_schema(repo=repo, table=table) # apps expect get_schema to return in a backwards way. It's made # to be backwards here. thrift_crazy_result = { 'status': True, 'row_count': 0, 'tuples': res, 'fields': [ {'type': 1043, 'name': 'column_name'}, {'type': 1043, 'name': 'data_type'}]} return construct_result_set(thrift_crazy_result) except Exception, e: raise DBException(message=str(e))
class BasicOperations(TestCase): """Tests basic operations in manager.py.""" @factory.django.mute_signals(signals.pre_save) def setUp(self): self.username = "******" self.password = "******" self.email = "*****@*****.**" self.user = User.objects.create_user( self.username, self.email, self.password) self.mock_connection = self.create_patch( 'core.db.manager.DataHubConnection') self.manager = DataHubManager(user=self.username) def create_patch(self, name): # helper method for creating patches patcher = patch(name) thing = patcher.start() self.addCleanup(patcher.stop) return thing def test_create_table(self): con_create_table = self.mock_connection.return_value.create_table repo = 'repo' table = 'table' params = {'id': 'integer', 'words': 'text'} self.manager.create_table(repo, table, params) self.assertTrue(con_create_table.called) def test_list_tables(self): con_list_tables = self.mock_connection.return_value.list_tables self.manager.list_tables('repo') self.assertTrue(con_list_tables.called) def test_describe_table(self): con_describe_table = self.mock_connection.return_value.describe_table self.manager.describe_table(repo='repo', table='table', detail=False) self.assertTrue(con_describe_table.called) def test_list_table_permissions(self): con_tbl_perm = self.mock_connection.return_value.list_table_permissions self.manager.list_table_permissions(repo='repo', table='table') self.assertTrue(con_tbl_perm.called) def test_delete_table(self): con_delete_table = self.mock_connection.return_value.delete_table self.manager.delete_table(repo='repo', table='table', force=False) self.assertTrue(con_delete_table.called) def test_clone_table(self): con_clone_table = self.mock_connection.return_value.clone_table self.manager.clone_table( repo='repo_name', table='table_name', new_table='new_table_name') self.assertTrue(con_clone_table.called) def test_list_views(self): con_list_views = self.mock_connection.return_value.list_views self.manager.list_views('repo') self.assertTrue(con_list_views.called) def test_create_view(self): con_create_view = self.mock_connection.return_value.create_view repo = 'repo_name' view = 'view_name' sql = 'SELECT * FROM table_name' self.manager.create_view(repo, view, sql) self.assertTrue(con_create_view.called) def test_describe_view(self): con_describe_view = self.mock_connection.return_value.describe_view self.manager.describe_view( repo='repo_name', view='_name', detail=False) self.assertTrue(con_describe_view.called) def test_delete_view(self): con_delete_view = self.mock_connection.return_value.delete_view self.manager.delete_view( repo='repo_name', view='view_name', force=False) self.assertTrue(con_delete_view.called) def test_list_repos(self): con_list_repos = self.mock_connection.return_value.list_repos self.manager.list_repos() self.assertTrue(con_list_repos.called) def test_rename_repo(self): con_rename_repo = self.mock_connection.return_value.rename_repo # self.mock_Collaborator = self.create_patch( # 'inventory.models.Collaborator') self.manager = DataHubManager( user=self.username, repo_base=self.username) self.manager.Collaborator = MagicMock self.manager.rename_repo('old_name', 'new_name') self.assertTrue(con_rename_repo.called) def test_create_repo(self): con_create_repo = self.mock_connection.return_value.create_repo self.manager.create_repo('repo') self.assertTrue(con_create_repo.called) def test_delete_repo(self): con_delete_repo = self.mock_connection.return_value.delete_repo self.manager = DataHubManager( user=self.username, repo_base=self.username) self.manager.delete_repo('repo') self.assertTrue(con_delete_repo.called) self.assertEqual(con_delete_repo.call_args[1]['force'], False) def test_add_collaborator(self): con_add_collab = self.mock_connection.return_value.add_collaborator self.create_patch('core.db.manager.User') mock_Collaborator = self.create_patch( 'core.db.manager.Collaborator') mock_Collaborator.objects.get_or_create.return_value = ( MagicMock(), True) self.manager.add_collaborator( repo='reponame', collaborator='new_collaborator', db_privileges=['select'], file_privileges=['read', 'write']) self.assertTrue(con_add_collab.called) self.assertEqual(con_add_collab.call_args[1]['repo'], 'reponame') self.assertEqual( con_add_collab.call_args[1]['collaborator'], 'new_collaborator') self.assertEqual( con_add_collab.call_args[1]['db_privileges'], ['SELECT']) def test_delete_collaborator(self): self.mock_connection.return_value.list_collaborators.return_value = [ {'username': '******', 'permissions': 'U'}, {'username': self.username, 'permissions': 'UC'} ] self.mock_User = self.create_patch('core.db.manager.User') self.mock_Collaborator = self.create_patch( 'core.db.manager.Collaborator') con_delete_collab = (self.mock_connection .return_value.delete_collaborator) self.manager.delete_collaborator('reponame', 'old_collaborator') self.assertTrue(con_delete_collab.called) self.assertEqual(con_delete_collab.call_args[1]['repo'], 'reponame') self.assertEqual( con_delete_collab.call_args[1]['collaborator'], 'old_collaborator') def test_get_schema(self): con_get_schema = self.mock_connection.return_value.get_schema self.manager.get_schema('reponame', 'tablename') self.assertTrue(con_get_schema.called) self.assertEqual(con_get_schema.call_args[1]['repo'], 'reponame') self.assertEqual(con_get_schema.call_args[1]['table'], 'tablename')
class BasicOperations(TestCase): """Tests basic operations in manager.py.""" @factory.django.mute_signals(signals.pre_save) def setUp(self): self.username = "******" self.password = "******" self.email = "*****@*****.**" self.user = User.objects.create_user( self.username, self.email, self.password) self.mock_connection = self.create_patch( 'core.db.manager.DataHubConnection') self.manager = DataHubManager(user=self.username) def create_patch(self, name): # helper method for creating patches patcher = patch(name) thing = patcher.start() self.addCleanup(patcher.stop) return thing def test_list_tables(self): con_list_tables = self.mock_connection.return_value.list_tables self.manager.list_tables('repo') self.assertTrue(con_list_tables.called) def test_list_views(self): con_list_views = self.mock_connection.return_value.list_views self.manager.list_views('repo') self.assertTrue(con_list_views.called) def test_list_repos(self): con_list_repos = self.mock_connection.return_value.list_repos self.manager.list_repos() self.assertTrue(con_list_repos.called) def test_create_repo(self): con_create_repo = self.mock_connection.return_value.create_repo self.manager.create_repo('repo') self.assertTrue(con_create_repo.called) def test_delete_repo(self): con_delete_repo = self.mock_connection.return_value.delete_repo self.manager = DataHubManager( user=self.username, repo_base=self.username) self.manager.delete_repo('repo') self.assertTrue(con_delete_repo.called) self.assertEqual(con_delete_repo.call_args[1]['force'], False) def test_add_collaborator(self): con_add_collab = self.mock_connection.return_value.add_collaborator mock_User = self.create_patch('core.db.manager.User') mock_Collaborator = self.create_patch( 'core.db.manager.Collaborator') mock_Collaborator.objects.get_or_create.return_value = ( MagicMock(), True) self.manager.add_collaborator( repo='reponame', collaborator='new_collaborator', privileges='select') self.assertTrue(con_add_collab.called) self.assertEqual(con_add_collab.call_args[1]['repo'], 'reponame') self.assertEqual( con_add_collab.call_args[1]['collaborator'], 'new_collaborator') self.assertEqual(con_add_collab.call_args[1]['privileges'], 'select') def test_delete_collaborator(self): self.mock_connection.return_value.list_collaborators.return_value = [ 'old_collaborator'] self.mock_User = self.create_patch('core.db.manager.User') self.mock_Collaborator = self.create_patch( 'core.db.manager.Collaborator') con_delete_collab = (self.mock_connection .return_value.delete_collaborator) self.manager.delete_collaborator('reponame', 'old_collaborator') self.assertTrue(con_delete_collab.called) self.assertEqual(con_delete_collab.call_args[1]['repo'], 'reponame') self.assertEqual( con_delete_collab.call_args[1]['collaborator'], 'old_collaborator') def test_get_schema(self): con_get_schema = self.mock_connection.return_value.get_schema self.manager.get_schema('reponame', 'tablename') self.assertTrue(con_get_schema.called) self.assertEqual(con_get_schema.call_args[1]['repo'], 'reponame') self.assertEqual(con_get_schema.call_args[1]['table'], 'tablename')
class BasicOperations(TestCase): """Tests basic operations in manager.py.""" @factory.django.mute_signals(signals.pre_save) def setUp(self): self.username = "******" self.password = "******" self.email = "*****@*****.**" self.user = User.objects.create_user( self.username, self.email, self.password) self.mock_connection = self.create_patch( 'core.db.manager.DataHubConnection') self.manager = DataHubManager(user=self.username) def create_patch(self, name): # helper method for creating patches patcher = patch(name) thing = patcher.start() self.addCleanup(patcher.stop) return thing def test_set_search_paths(self): con_set_spaths = self.mock_connection.return_value.set_search_paths search_paths = ['reponame', 'username'] self.manager.set_search_paths(search_paths) self.assertTrue(con_set_spaths.called) def test_create_table(self): con_create_table = self.mock_connection.return_value.create_table repo = 'repo' table = 'table' params = {'id': 'integer', 'words': 'text'} self.manager.create_table(repo, table, params) self.assertTrue(con_create_table.called) def test_list_tables(self): con_list_tables = self.mock_connection.return_value.list_tables self.manager.list_tables('repo') self.assertTrue(con_list_tables.called) def test_describe_table(self): con_describe_table = self.mock_connection.return_value.describe_table self.manager.describe_table(repo='repo', table='table', detail=False) self.assertTrue(con_describe_table.called) def test_list_table_permissions(self): con_tbl_perm = self.mock_connection.return_value.list_table_permissions self.manager.list_table_permissions(repo='repo', table='table') self.assertTrue(con_tbl_perm.called) def test_delete_table(self): con_delete_table = self.mock_connection.return_value.delete_table self.manager.delete_table(repo='repo', table='table', force=False) self.assertTrue(con_delete_table.called) def test_list_views(self): con_list_views = self.mock_connection.return_value.list_views self.manager.list_views('repo') self.assertTrue(con_list_views.called) def test_create_view(self): con_create_view = self.mock_connection.return_value.create_view repo = 'repo_name' view = 'view_name' sql = 'SELECT * FROM table_name' self.manager.create_view(repo, view, sql) self.assertTrue(con_create_view.called) def test_describe_view(self): con_describe_view = self.mock_connection.return_value.describe_view self.manager.describe_view( repo='repo_name', view='_name', detail=False) self.assertTrue(con_describe_view.called) def test_delete_view(self): con_delete_view = self.mock_connection.return_value.delete_view self.manager.delete_view( repo='repo_name', view='view_name', force=False) self.assertTrue(con_delete_view.called) def test_list_repos(self): con_list_repos = self.mock_connection.return_value.list_repos self.manager.list_repos() self.assertTrue(con_list_repos.called) def test_rename_repo(self): con_rename_repo = self.mock_connection.return_value.rename_repo # self.mock_Collaborator = self.create_patch( # 'inventory.models.Collaborator') self.manager = DataHubManager( user=self.username, repo_base=self.username) self.manager.Collaborator = MagicMock self.manager.rename_repo('old_name', 'new_name') self.assertTrue(con_rename_repo.called) def test_create_repo(self): con_create_repo = self.mock_connection.return_value.create_repo self.manager.create_repo('repo') self.assertTrue(con_create_repo.called) def test_delete_repo(self): con_delete_repo = self.mock_connection.return_value.delete_repo self.manager = DataHubManager( user=self.username, repo_base=self.username) self.manager.delete_repo('repo') self.assertTrue(con_delete_repo.called) self.assertEqual(con_delete_repo.call_args[1]['force'], False) def test_add_collaborator(self): con_add_collab = self.mock_connection.return_value.add_collaborator self.create_patch('core.db.manager.User') mock_Collaborator = self.create_patch( 'core.db.manager.Collaborator') mock_Collaborator.objects.get_or_create.return_value = ( MagicMock(), True) self.manager.add_collaborator( repo='reponame', collaborator='new_collaborator', db_privileges=['select'], file_privileges=['read', 'write']) self.assertTrue(con_add_collab.called) self.assertEqual(con_add_collab.call_args[1]['repo'], 'reponame') self.assertEqual( con_add_collab.call_args[1]['collaborator'], 'new_collaborator') self.assertEqual( con_add_collab.call_args[1]['db_privileges'], ['SELECT']) def test_delete_collaborator(self): self.mock_connection.return_value.list_collaborators.return_value = [ {'username': '******', 'permissions': 'U'}, {'username': self.username, 'permissions': 'UC'} ] self.mock_User = self.create_patch('core.db.manager.User') self.mock_Collaborator = self.create_patch( 'core.db.manager.Collaborator') con_delete_collab = (self.mock_connection .return_value.delete_collaborator) self.manager.delete_collaborator('reponame', 'old_collaborator') self.assertTrue(con_delete_collab.called) self.assertEqual(con_delete_collab.call_args[1]['repo'], 'reponame') self.assertEqual( con_delete_collab.call_args[1]['collaborator'], 'old_collaborator') def test_get_schema(self): con_get_schema = self.mock_connection.return_value.get_schema self.manager.get_schema('reponame', 'tablename') self.assertTrue(con_get_schema.called) self.assertEqual(con_get_schema.call_args[1]['repo'], 'reponame') self.assertEqual(con_get_schema.call_args[1]['table'], 'tablename')