def repo(request, repo_base, repo): try: login = get_login(request) res = DataHubManager.has_repo_privilege(login, repo_base, repo, 'USAGE') if not (res and res['tuples'][0][0]): raise Exception('Access denied. Missing required privileges.') manager = DataHubManager(user=repo_base) res = manager.list_tables(repo) tables = [t[0] for t in res['tuples']] repo_dir = '/user_data/%s/%s' %(repo_base, repo) if not os.path.exists(repo_dir): os.makedirs(repo_dir) uploaded_files = [f for f in os.listdir(repo_dir)] res = { 'login': get_login(request), 'repo_base': repo_base, 'repo': repo, 'tables': tables, 'files': uploaded_files} res.update(csrf(request)) return render_to_response("repo.html", res) except Exception, e: return HttpResponse(json.dumps( {'error': str(e)}), mimetype="application/json")
def repo_tables(request, repo_base, repo): try: login = get_login(request) res = DataHubManager.has_repo_privilege(login, repo_base, repo, 'USAGE') if not (res and res['tuples'][0][0]): raise Exception('Access denied. Missing required privileges.') manager = DataHubManager(user=repo_base) # get base_tables for a given repo res = manager.list_tables(repo) base_tables = [t[0] for t in res['tuples']] # get views for a given repo res = manager.list_views(repo) views = [t[0] for t in res['tuples']] res = { 'login': get_login(request), 'repo_base': repo_base, 'repo': repo, 'base_tables': base_tables, 'views': views} res.update(csrf(request)) return render_to_response("repo-browse-tables.html", res) except Exception, e: return HttpResponse(json.dumps( {'error': str(e)}), content_type="application/json")
def repo_tables(request, repo_base, repo): ''' shows the tables under a repo. ''' username = request.user.get_username() # get the base tables and views of the user's repo manager = DataHubManager(user=username, repo_base=repo_base) try: base_tables = manager.list_tables(repo) views = manager.list_views(repo) except LookupError: base_tables = [] views = [] res = { 'login': username, 'repo_base': repo_base, 'repo': repo, 'base_tables': base_tables, 'views': views } res.update(csrf(request)) return render_to_response("repo-browse-tables.html", res)
def list_tables(self, con, repo_name): try: ''' res = DataHubManager.has_repo_privilege( con.user, con.repo_base, repo_name, 'USAGE') if not (res and res['tuples'][0][0]): raise Exception('Access denied. Missing required privileges.') ''' manager = DataHubManager(user=con.user, repo_base=con.repo_base, is_app=con.is_app) res = manager.list_tables(repo=repo_name) # create the crazy thrift tuple, since this is for consumption from # 3rd party apps, and they haven't upgraded to the new list_tables # ARC 2015-12-15 tuple_list = zip(res) length = len(res) thrift_crazy_result = { 'status': True, 'row_count': length, 'tuples': tuple_list, 'fields': [{ 'type': 1043, 'name': 'table_name' }] } return construct_result_set(thrift_crazy_result) except Exception as e: raise DBException(message=str(e))
def repo_tables(request, repo_base, repo): try: login = get_login(request) res = DataHubManager.has_repo_privilege(login, repo_base, repo, 'USAGE') if not (res and res['tuples'][0][0]): raise Exception('Access denied. Missing required privileges.') manager = DataHubManager(user=repo_base) # get base_tables for a given repo res = manager.list_tables(repo) base_tables = [t[0] for t in res['tuples']] # get views for a given repo res = manager.list_views(repo) views = [t[0] for t in res['tuples']] res = { 'login': get_login(request), 'repo_base': repo_base, 'repo': repo, 'base_tables': base_tables, 'views': views } res.update(csrf(request)) return render_to_response("repo-browse-tables.html", res) except Exception, e: return HttpResponse(json.dumps({'error': str(e)}), content_type="application/json")
def list_tables(self, con, repo_name): try: ''' res = DataHubManager.has_repo_privilege( con.user, con.repo_base, repo_name, 'USAGE') if not (res and res['tuples'][0][0]): raise Exception('Access denied. Missing required privileges.') ''' manager = DataHubManager( user=con.user, repo_base=con.repo_base, is_app=con.is_app) res = manager.list_tables(repo=repo_name) # create the crazy thrift tuple, since this is for consumption from # 3rd party apps, and they haven't upgraded to the new list_tables # ARC 2015-12-15 tuple_list = zip(res) length = len(res) thrift_crazy_result = { 'status': True, 'row_count': length, 'tuples': tuple_list, 'fields': [{'type': 1043, 'name': 'table_name'}] } return construct_result_set(thrift_crazy_result) except Exception, e: raise DBException(message=str(e))
def list_tables(self, con, repo_name): try: manager = DataHubManager(user=con.user, repo_base=con.repo_base) res = manager.list_tables(repo=repo_name) return construct_result_set(res) except Exception, e: raise DBException(message=str(e))
def tables(request): """ Returns the tables in a user's repo. I think this is only used for (disabled) scorpion features. """ username = request.user.get_username() repo_base = request.GET.get('username', None) manager = DataHubManager(username, repo_base) repo = request.GET.get('repo') tables = manager.list_tables(repo) return {'tables': tables}
def list_tables(self, con, repo_name): try: ''' res = DataHubManager.has_repo_privilege(con.user, con.repo_base, repo_name, 'USAGE') if not (res and res['tuples'][0][0]): raise Exception('Access denied. Missing required privileges.') ''' manager = DataHubManager(user=con.repo_base, repo_base=con.repo_base) res = manager.list_tables(repo=repo_name) return construct_result_set(res) except Exception, e: raise DBException(message=str(e))
def does_cache_exist(repo_base): """ check to see if the cache exists for the repo base""" manager = DataHubManager(repo_base) repo_exists = False table_exists = False schema_correct = False # check for repo if dbwipes_repo in manager.list_repos(): repo_exists = True # check for table if repo_exists and dbwipes_table in manager.list_tables(dbwipes_repo): table_exists = True # check for schema schema = None if table_exists: schema = manager.get_schema(dbwipes_repo, dbwipes_table) if schema == [('key', 'character varying'), ('val', 'text')]: schema_correct = True return repo_exists and table_exists and schema_correct
def repo_tables(request, repo_base, repo): ''' shows the tables under a repo. ''' username = request.user.get_username() # get the base tables and views of the user's repo manager = DataHubManager(user=username, repo_base=repo_base) try: base_tables = manager.list_tables(repo) views = manager.list_views(repo) except LookupError: base_tables = [] views = [] res = { 'login': username, 'repo_base': repo_base, 'repo': repo, 'base_tables': base_tables, 'views': views} res.update(csrf(request)) return render_to_response("repo-browse-tables.html", res)
def tables(request): username = request.user.get_username() manager = DataHubManager(user=username) repo = request.GET.get("repo") tables = manager.list_tables(repo) return {"tables": tables}
class BasicOperations(TestCase): """Tests basic operations in manager.py.""" @factory.django.mute_signals(signals.pre_save) def setUp(self): self.username = "******" self.password = "******" self.email = "*****@*****.**" self.user = User.objects.create_user( self.username, self.email, self.password) self.mock_connection = self.create_patch( 'core.db.manager.DataHubConnection') self.manager = DataHubManager(user=self.username) def create_patch(self, name): # helper method for creating patches patcher = patch(name) thing = patcher.start() self.addCleanup(patcher.stop) return thing def test_create_table(self): con_create_table = self.mock_connection.return_value.create_table repo = 'repo' table = 'table' params = {'id': 'integer', 'words': 'text'} self.manager.create_table(repo, table, params) self.assertTrue(con_create_table.called) def test_list_tables(self): con_list_tables = self.mock_connection.return_value.list_tables self.manager.list_tables('repo') self.assertTrue(con_list_tables.called) def test_describe_table(self): con_describe_table = self.mock_connection.return_value.describe_table self.manager.describe_table(repo='repo', table='table', detail=False) self.assertTrue(con_describe_table.called) def test_list_table_permissions(self): con_tbl_perm = self.mock_connection.return_value.list_table_permissions self.manager.list_table_permissions(repo='repo', table='table') self.assertTrue(con_tbl_perm.called) def test_delete_table(self): con_delete_table = self.mock_connection.return_value.delete_table self.manager.delete_table(repo='repo', table='table', force=False) self.assertTrue(con_delete_table.called) def test_clone_table(self): con_clone_table = self.mock_connection.return_value.clone_table self.manager.clone_table( repo='repo_name', table='table_name', new_table='new_table_name') self.assertTrue(con_clone_table.called) def test_list_views(self): con_list_views = self.mock_connection.return_value.list_views self.manager.list_views('repo') self.assertTrue(con_list_views.called) def test_create_view(self): con_create_view = self.mock_connection.return_value.create_view repo = 'repo_name' view = 'view_name' sql = 'SELECT * FROM table_name' self.manager.create_view(repo, view, sql) self.assertTrue(con_create_view.called) def test_describe_view(self): con_describe_view = self.mock_connection.return_value.describe_view self.manager.describe_view( repo='repo_name', view='_name', detail=False) self.assertTrue(con_describe_view.called) def test_delete_view(self): con_delete_view = self.mock_connection.return_value.delete_view self.manager.delete_view( repo='repo_name', view='view_name', force=False) self.assertTrue(con_delete_view.called) def test_list_repos(self): con_list_repos = self.mock_connection.return_value.list_repos self.manager.list_repos() self.assertTrue(con_list_repos.called) def test_rename_repo(self): con_rename_repo = self.mock_connection.return_value.rename_repo # self.mock_Collaborator = self.create_patch( # 'inventory.models.Collaborator') self.manager = DataHubManager( user=self.username, repo_base=self.username) self.manager.Collaborator = MagicMock self.manager.rename_repo('old_name', 'new_name') self.assertTrue(con_rename_repo.called) def test_create_repo(self): con_create_repo = self.mock_connection.return_value.create_repo self.manager.create_repo('repo') self.assertTrue(con_create_repo.called) def test_delete_repo(self): con_delete_repo = self.mock_connection.return_value.delete_repo self.manager = DataHubManager( user=self.username, repo_base=self.username) self.manager.delete_repo('repo') self.assertTrue(con_delete_repo.called) self.assertEqual(con_delete_repo.call_args[1]['force'], False) def test_add_collaborator(self): con_add_collab = self.mock_connection.return_value.add_collaborator self.create_patch('core.db.manager.User') mock_Collaborator = self.create_patch( 'core.db.manager.Collaborator') mock_Collaborator.objects.get_or_create.return_value = ( MagicMock(), True) self.manager.add_collaborator( repo='reponame', collaborator='new_collaborator', db_privileges=['select'], file_privileges=['read', 'write']) self.assertTrue(con_add_collab.called) self.assertEqual(con_add_collab.call_args[1]['repo'], 'reponame') self.assertEqual( con_add_collab.call_args[1]['collaborator'], 'new_collaborator') self.assertEqual( con_add_collab.call_args[1]['db_privileges'], ['SELECT']) def test_delete_collaborator(self): self.mock_connection.return_value.list_collaborators.return_value = [ {'username': '******', 'permissions': 'U'}, {'username': self.username, 'permissions': 'UC'} ] self.mock_User = self.create_patch('core.db.manager.User') self.mock_Collaborator = self.create_patch( 'core.db.manager.Collaborator') con_delete_collab = (self.mock_connection .return_value.delete_collaborator) self.manager.delete_collaborator('reponame', 'old_collaborator') self.assertTrue(con_delete_collab.called) self.assertEqual(con_delete_collab.call_args[1]['repo'], 'reponame') self.assertEqual( con_delete_collab.call_args[1]['collaborator'], 'old_collaborator') def test_get_schema(self): con_get_schema = self.mock_connection.return_value.get_schema self.manager.get_schema('reponame', 'tablename') self.assertTrue(con_get_schema.called) self.assertEqual(con_get_schema.call_args[1]['repo'], 'reponame') self.assertEqual(con_get_schema.call_args[1]['table'], 'tablename')
def tables(request): username = get_login(request) manager = DataHubManager(user=username) repo = request.GET.get('repo') tables = [str(row[0]) for row in manager.list_tables(repo)['tuples']] return {'tables': tables}
def tables(request): username = request.user.get_username() manager = DataHubManager(user=username) repo = request.GET.get('repo') tables = manager.list_tables(repo) return {'tables': tables}
class BasicOperations(TestCase): """Tests basic operations in manager.py.""" @factory.django.mute_signals(signals.pre_save) def setUp(self): self.username = "******" self.password = "******" self.email = "*****@*****.**" self.user = User.objects.create_user( self.username, self.email, self.password) self.mock_connection = self.create_patch( 'core.db.manager.DataHubConnection') self.manager = DataHubManager(user=self.username) def create_patch(self, name): # helper method for creating patches patcher = patch(name) thing = patcher.start() self.addCleanup(patcher.stop) return thing def test_list_tables(self): con_list_tables = self.mock_connection.return_value.list_tables self.manager.list_tables('repo') self.assertTrue(con_list_tables.called) def test_list_views(self): con_list_views = self.mock_connection.return_value.list_views self.manager.list_views('repo') self.assertTrue(con_list_views.called) def test_list_repos(self): con_list_repos = self.mock_connection.return_value.list_repos self.manager.list_repos() self.assertTrue(con_list_repos.called) def test_create_repo(self): con_create_repo = self.mock_connection.return_value.create_repo self.manager.create_repo('repo') self.assertTrue(con_create_repo.called) def test_delete_repo(self): con_delete_repo = self.mock_connection.return_value.delete_repo self.manager = DataHubManager( user=self.username, repo_base=self.username) self.manager.delete_repo('repo') self.assertTrue(con_delete_repo.called) self.assertEqual(con_delete_repo.call_args[1]['force'], False) def test_add_collaborator(self): con_add_collab = self.mock_connection.return_value.add_collaborator mock_User = self.create_patch('core.db.manager.User') mock_Collaborator = self.create_patch( 'core.db.manager.Collaborator') mock_Collaborator.objects.get_or_create.return_value = ( MagicMock(), True) self.manager.add_collaborator( repo='reponame', collaborator='new_collaborator', privileges='select') self.assertTrue(con_add_collab.called) self.assertEqual(con_add_collab.call_args[1]['repo'], 'reponame') self.assertEqual( con_add_collab.call_args[1]['collaborator'], 'new_collaborator') self.assertEqual(con_add_collab.call_args[1]['privileges'], 'select') def test_delete_collaborator(self): self.mock_connection.return_value.list_collaborators.return_value = [ 'old_collaborator'] self.mock_User = self.create_patch('core.db.manager.User') self.mock_Collaborator = self.create_patch( 'core.db.manager.Collaborator') con_delete_collab = (self.mock_connection .return_value.delete_collaborator) self.manager.delete_collaborator('reponame', 'old_collaborator') self.assertTrue(con_delete_collab.called) self.assertEqual(con_delete_collab.call_args[1]['repo'], 'reponame') self.assertEqual( con_delete_collab.call_args[1]['collaborator'], 'old_collaborator') def test_get_schema(self): con_get_schema = self.mock_connection.return_value.get_schema self.manager.get_schema('reponame', 'tablename') self.assertTrue(con_get_schema.called) self.assertEqual(con_get_schema.call_args[1]['repo'], 'reponame') self.assertEqual(con_get_schema.call_args[1]['table'], 'tablename')
class BasicOperations(TestCase): """Tests basic operations in manager.py.""" @factory.django.mute_signals(signals.pre_save) def setUp(self): self.username = "******" self.password = "******" self.email = "*****@*****.**" self.user = User.objects.create_user( self.username, self.email, self.password) self.mock_connection = self.create_patch( 'core.db.manager.DataHubConnection') self.manager = DataHubManager(user=self.username) def create_patch(self, name): # helper method for creating patches patcher = patch(name) thing = patcher.start() self.addCleanup(patcher.stop) return thing def test_set_search_paths(self): con_set_spaths = self.mock_connection.return_value.set_search_paths search_paths = ['reponame', 'username'] self.manager.set_search_paths(search_paths) self.assertTrue(con_set_spaths.called) def test_create_table(self): con_create_table = self.mock_connection.return_value.create_table repo = 'repo' table = 'table' params = {'id': 'integer', 'words': 'text'} self.manager.create_table(repo, table, params) self.assertTrue(con_create_table.called) def test_list_tables(self): con_list_tables = self.mock_connection.return_value.list_tables self.manager.list_tables('repo') self.assertTrue(con_list_tables.called) def test_describe_table(self): con_describe_table = self.mock_connection.return_value.describe_table self.manager.describe_table(repo='repo', table='table', detail=False) self.assertTrue(con_describe_table.called) def test_list_table_permissions(self): con_tbl_perm = self.mock_connection.return_value.list_table_permissions self.manager.list_table_permissions(repo='repo', table='table') self.assertTrue(con_tbl_perm.called) def test_delete_table(self): con_delete_table = self.mock_connection.return_value.delete_table self.manager.delete_table(repo='repo', table='table', force=False) self.assertTrue(con_delete_table.called) def test_list_views(self): con_list_views = self.mock_connection.return_value.list_views self.manager.list_views('repo') self.assertTrue(con_list_views.called) def test_create_view(self): con_create_view = self.mock_connection.return_value.create_view repo = 'repo_name' view = 'view_name' sql = 'SELECT * FROM table_name' self.manager.create_view(repo, view, sql) self.assertTrue(con_create_view.called) def test_describe_view(self): con_describe_view = self.mock_connection.return_value.describe_view self.manager.describe_view( repo='repo_name', view='_name', detail=False) self.assertTrue(con_describe_view.called) def test_delete_view(self): con_delete_view = self.mock_connection.return_value.delete_view self.manager.delete_view( repo='repo_name', view='view_name', force=False) self.assertTrue(con_delete_view.called) def test_list_repos(self): con_list_repos = self.mock_connection.return_value.list_repos self.manager.list_repos() self.assertTrue(con_list_repos.called) def test_rename_repo(self): con_rename_repo = self.mock_connection.return_value.rename_repo # self.mock_Collaborator = self.create_patch( # 'inventory.models.Collaborator') self.manager = DataHubManager( user=self.username, repo_base=self.username) self.manager.Collaborator = MagicMock self.manager.rename_repo('old_name', 'new_name') self.assertTrue(con_rename_repo.called) def test_create_repo(self): con_create_repo = self.mock_connection.return_value.create_repo self.manager.create_repo('repo') self.assertTrue(con_create_repo.called) def test_delete_repo(self): con_delete_repo = self.mock_connection.return_value.delete_repo self.manager = DataHubManager( user=self.username, repo_base=self.username) self.manager.delete_repo('repo') self.assertTrue(con_delete_repo.called) self.assertEqual(con_delete_repo.call_args[1]['force'], False) def test_add_collaborator(self): con_add_collab = self.mock_connection.return_value.add_collaborator self.create_patch('core.db.manager.User') mock_Collaborator = self.create_patch( 'core.db.manager.Collaborator') mock_Collaborator.objects.get_or_create.return_value = ( MagicMock(), True) self.manager.add_collaborator( repo='reponame', collaborator='new_collaborator', db_privileges=['select'], file_privileges=['read', 'write']) self.assertTrue(con_add_collab.called) self.assertEqual(con_add_collab.call_args[1]['repo'], 'reponame') self.assertEqual( con_add_collab.call_args[1]['collaborator'], 'new_collaborator') self.assertEqual( con_add_collab.call_args[1]['db_privileges'], ['SELECT']) def test_delete_collaborator(self): self.mock_connection.return_value.list_collaborators.return_value = [ {'username': '******', 'permissions': 'U'}, {'username': self.username, 'permissions': 'UC'} ] self.mock_User = self.create_patch('core.db.manager.User') self.mock_Collaborator = self.create_patch( 'core.db.manager.Collaborator') con_delete_collab = (self.mock_connection .return_value.delete_collaborator) self.manager.delete_collaborator('reponame', 'old_collaborator') self.assertTrue(con_delete_collab.called) self.assertEqual(con_delete_collab.call_args[1]['repo'], 'reponame') self.assertEqual( con_delete_collab.call_args[1]['collaborator'], 'old_collaborator') def test_get_schema(self): con_get_schema = self.mock_connection.return_value.get_schema self.manager.get_schema('reponame', 'tablename') self.assertTrue(con_get_schema.called) self.assertEqual(con_get_schema.call_args[1]['repo'], 'reponame') self.assertEqual(con_get_schema.call_args[1]['table'], 'tablename')