def repo(request, repo_base, repo):
try:
login = get_login(request)
res = DataHubManager.has_repo_privilege(login, repo_base, repo, 'USAGE')
if not (res and res['tuples'][0][0]):
raise Exception('Access denied. Missing required privileges.')
manager = DataHubManager(user=repo_base)
res = manager.list_tables(repo)
tables = [t[0] for t in res['tuples']]
repo_dir = '/user_data/%s/%s' %(repo_base, repo)
if not os.path.exists(repo_dir):
os.makedirs(repo_dir)
uploaded_files = [f for f in os.listdir(repo_dir)]
res = {
'login': get_login(request),
'repo_base': repo_base,
'repo': repo,
'tables': tables,
'files': uploaded_files}
res.update(csrf(request))
return render_to_response("repo.html", res)
except Exception, e:
return HttpResponse(json.dumps(
{'error': str(e)}),
mimetype="application/json")
def repo_tables(request, repo_base, repo):
try:
login = get_login(request)
res = DataHubManager.has_repo_privilege(login, repo_base, repo, 'USAGE')
if not (res and res['tuples'][0][0]):
raise Exception('Access denied. Missing required privileges.')
manager = DataHubManager(user=repo_base)
# get base_tables for a given repo
res = manager.list_tables(repo)
base_tables = [t[0] for t in res['tuples']]
# get views for a given repo
res = manager.list_views(repo)
views = [t[0] for t in res['tuples']]
res = {
'login': get_login(request),
'repo_base': repo_base,
'repo': repo,
'base_tables': base_tables,
'views': views}
res.update(csrf(request))
return render_to_response("repo-browse-tables.html", res)
except Exception, e:
return HttpResponse(json.dumps(
{'error': str(e)}),
content_type="application/json")
def repo_tables(request, repo_base, repo):
'''
shows the tables under a repo.
'''
username = request.user.get_username()
# get the base tables and views of the user's repo
manager = DataHubManager(user=username, repo_base=repo_base)
try:
base_tables = manager.list_tables(repo)
views = manager.list_views(repo)
except LookupError:
base_tables = []
views = []
res = {
'login': username,
'repo_base': repo_base,
'repo': repo,
'base_tables': base_tables,
'views': views
}
res.update(csrf(request))
return render_to_response("repo-browse-tables.html", res)
def list_tables(self, con, repo_name):
try:
'''
res = DataHubManager.has_repo_privilege(
con.user, con.repo_base, repo_name, 'USAGE')
if not (res and res['tuples'][0][0]):
raise Exception('Access denied. Missing required privileges.')
'''
manager = DataHubManager(user=con.user,
repo_base=con.repo_base,
is_app=con.is_app)
res = manager.list_tables(repo=repo_name)
# create the crazy thrift tuple, since this is for consumption from
# 3rd party apps, and they haven't upgraded to the new list_tables
# ARC 2015-12-15
tuple_list = zip(res)
length = len(res)
thrift_crazy_result = {
'status': True,
'row_count': length,
'tuples': tuple_list,
'fields': [{
'type': 1043,
'name': 'table_name'
}]
}
return construct_result_set(thrift_crazy_result)
except Exception as e:
raise DBException(message=str(e))
def repo_tables(request, repo_base, repo):
try:
login = get_login(request)
res = DataHubManager.has_repo_privilege(login, repo_base, repo,
'USAGE')
if not (res and res['tuples'][0][0]):
raise Exception('Access denied. Missing required privileges.')
manager = DataHubManager(user=repo_base)
# get base_tables for a given repo
res = manager.list_tables(repo)
base_tables = [t[0] for t in res['tuples']]
# get views for a given repo
res = manager.list_views(repo)
views = [t[0] for t in res['tuples']]
res = {
'login': get_login(request),
'repo_base': repo_base,
'repo': repo,
'base_tables': base_tables,
'views': views
}
res.update(csrf(request))
return render_to_response("repo-browse-tables.html", res)
except Exception, e:
return HttpResponse(json.dumps({'error': str(e)}),
content_type="application/json")
def list_tables(self, con, repo_name):
try:
'''
res = DataHubManager.has_repo_privilege(
con.user, con.repo_base, repo_name, 'USAGE')
if not (res and res['tuples'][0][0]):
raise Exception('Access denied. Missing required privileges.')
'''
manager = DataHubManager(
user=con.user, repo_base=con.repo_base, is_app=con.is_app)
res = manager.list_tables(repo=repo_name)
# create the crazy thrift tuple, since this is for consumption from
# 3rd party apps, and they haven't upgraded to the new list_tables
# ARC 2015-12-15
tuple_list = zip(res)
length = len(res)
thrift_crazy_result = {
'status': True, 'row_count': length,
'tuples': tuple_list,
'fields': [{'type': 1043, 'name': 'table_name'}]
}
return construct_result_set(thrift_crazy_result)
except Exception, e:
raise DBException(message=str(e))
def list_tables(self, con, repo_name):
try:
manager = DataHubManager(user=con.user, repo_base=con.repo_base)
res = manager.list_tables(repo=repo_name)
return construct_result_set(res)
except Exception, e:
raise DBException(message=str(e))
def tables(request):
""" Returns the tables in a user's repo.
I think this is only used for (disabled) scorpion features.
"""
username = request.user.get_username()
repo_base = request.GET.get('username', None)
manager = DataHubManager(username, repo_base)
repo = request.GET.get('repo')
tables = manager.list_tables(repo)
return {'tables': tables}
def list_tables(self, con, repo_name):
try:
'''
res = DataHubManager.has_repo_privilege(con.user, con.repo_base, repo_name, 'USAGE')
if not (res and res['tuples'][0][0]):
raise Exception('Access denied. Missing required privileges.')
'''
manager = DataHubManager(user=con.repo_base, repo_base=con.repo_base)
res = manager.list_tables(repo=repo_name)
return construct_result_set(res)
except Exception, e:
raise DBException(message=str(e))
def list_tables(self, con, repo_name):
try:
'''
res = DataHubManager.has_repo_privilege(con.user, con.repo_base, repo_name, 'USAGE')
if not (res and res['tuples'][0][0]):
raise Exception('Access denied. Missing required privileges.')
'''
manager = DataHubManager(user=con.repo_base,
repo_base=con.repo_base)
res = manager.list_tables(repo=repo_name)
return construct_result_set(res)
except Exception, e:
raise DBException(message=str(e))
def does_cache_exist(repo_base):
""" check to see if the cache exists for the repo base"""
manager = DataHubManager(repo_base)
repo_exists = False
table_exists = False
schema_correct = False
# check for repo
if dbwipes_repo in manager.list_repos():
repo_exists = True
# check for table
if repo_exists and dbwipes_table in manager.list_tables(dbwipes_repo):
table_exists = True
# check for schema
schema = None
if table_exists:
schema = manager.get_schema(dbwipes_repo, dbwipes_table)
if schema == [('key', 'character varying'), ('val', 'text')]:
schema_correct = True
return repo_exists and table_exists and schema_correct
def repo_tables(request, repo_base, repo):
'''
shows the tables under a repo.
'''
username = request.user.get_username()
# get the base tables and views of the user's repo
manager = DataHubManager(user=username, repo_base=repo_base)
try:
base_tables = manager.list_tables(repo)
views = manager.list_views(repo)
except LookupError:
base_tables = []
views = []
res = {
'login': username,
'repo_base': repo_base,
'repo': repo,
'base_tables': base_tables,
'views': views}
res.update(csrf(request))
return render_to_response("repo-browse-tables.html", res)
def tables(request):
username = request.user.get_username()
manager = DataHubManager(user=username)
repo = request.GET.get("repo")
tables = manager.list_tables(repo)
return {"tables": tables}
class BasicOperations(TestCase):
"""Tests basic operations in manager.py."""
@factory.django.mute_signals(signals.pre_save)
def setUp(self):
self.username = "******"
self.password = "******"
self.email = "*****@*****.**"
self.user = User.objects.create_user(
self.username, self.email, self.password)
self.mock_connection = self.create_patch(
'core.db.manager.DataHubConnection')
self.manager = DataHubManager(user=self.username)
def create_patch(self, name):
# helper method for creating patches
patcher = patch(name)
thing = patcher.start()
self.addCleanup(patcher.stop)
return thing
def test_create_table(self):
con_create_table = self.mock_connection.return_value.create_table
repo = 'repo'
table = 'table'
params = {'id': 'integer', 'words': 'text'}
self.manager.create_table(repo, table, params)
self.assertTrue(con_create_table.called)
def test_list_tables(self):
con_list_tables = self.mock_connection.return_value.list_tables
self.manager.list_tables('repo')
self.assertTrue(con_list_tables.called)
def test_describe_table(self):
con_describe_table = self.mock_connection.return_value.describe_table
self.manager.describe_table(repo='repo', table='table', detail=False)
self.assertTrue(con_describe_table.called)
def test_list_table_permissions(self):
con_tbl_perm = self.mock_connection.return_value.list_table_permissions
self.manager.list_table_permissions(repo='repo', table='table')
self.assertTrue(con_tbl_perm.called)
def test_delete_table(self):
con_delete_table = self.mock_connection.return_value.delete_table
self.manager.delete_table(repo='repo', table='table', force=False)
self.assertTrue(con_delete_table.called)
def test_clone_table(self):
con_clone_table = self.mock_connection.return_value.clone_table
self.manager.clone_table(
repo='repo_name', table='table_name', new_table='new_table_name')
self.assertTrue(con_clone_table.called)
def test_list_views(self):
con_list_views = self.mock_connection.return_value.list_views
self.manager.list_views('repo')
self.assertTrue(con_list_views.called)
def test_create_view(self):
con_create_view = self.mock_connection.return_value.create_view
repo = 'repo_name'
view = 'view_name'
sql = 'SELECT * FROM table_name'
self.manager.create_view(repo, view, sql)
self.assertTrue(con_create_view.called)
def test_describe_view(self):
con_describe_view = self.mock_connection.return_value.describe_view
self.manager.describe_view(
repo='repo_name', view='_name', detail=False)
self.assertTrue(con_describe_view.called)
def test_delete_view(self):
con_delete_view = self.mock_connection.return_value.delete_view
self.manager.delete_view(
repo='repo_name', view='view_name', force=False)
self.assertTrue(con_delete_view.called)
def test_list_repos(self):
con_list_repos = self.mock_connection.return_value.list_repos
self.manager.list_repos()
self.assertTrue(con_list_repos.called)
def test_rename_repo(self):
con_rename_repo = self.mock_connection.return_value.rename_repo
# self.mock_Collaborator = self.create_patch(
# 'inventory.models.Collaborator')
self.manager = DataHubManager(
user=self.username, repo_base=self.username)
self.manager.Collaborator = MagicMock
self.manager.rename_repo('old_name', 'new_name')
self.assertTrue(con_rename_repo.called)
def test_create_repo(self):
con_create_repo = self.mock_connection.return_value.create_repo
self.manager.create_repo('repo')
self.assertTrue(con_create_repo.called)
def test_delete_repo(self):
con_delete_repo = self.mock_connection.return_value.delete_repo
self.manager = DataHubManager(
user=self.username, repo_base=self.username)
self.manager.delete_repo('repo')
self.assertTrue(con_delete_repo.called)
self.assertEqual(con_delete_repo.call_args[1]['force'], False)
def test_add_collaborator(self):
con_add_collab = self.mock_connection.return_value.add_collaborator
self.create_patch('core.db.manager.User')
mock_Collaborator = self.create_patch(
'core.db.manager.Collaborator')
mock_Collaborator.objects.get_or_create.return_value = (
MagicMock(), True)
self.manager.add_collaborator(
repo='reponame',
collaborator='new_collaborator',
db_privileges=['select'], file_privileges=['read', 'write'])
self.assertTrue(con_add_collab.called)
self.assertEqual(con_add_collab.call_args[1]['repo'], 'reponame')
self.assertEqual(
con_add_collab.call_args[1]['collaborator'], 'new_collaborator')
self.assertEqual(
con_add_collab.call_args[1]['db_privileges'], ['SELECT'])
def test_delete_collaborator(self):
self.mock_connection.return_value.list_collaborators.return_value = [
{'username': '******', 'permissions': 'U'},
{'username': self.username, 'permissions': 'UC'}
]
self.mock_User = self.create_patch('core.db.manager.User')
self.mock_Collaborator = self.create_patch(
'core.db.manager.Collaborator')
con_delete_collab = (self.mock_connection
.return_value.delete_collaborator)
self.manager.delete_collaborator('reponame', 'old_collaborator')
self.assertTrue(con_delete_collab.called)
self.assertEqual(con_delete_collab.call_args[1]['repo'], 'reponame')
self.assertEqual(
con_delete_collab.call_args[1]['collaborator'], 'old_collaborator')
def test_get_schema(self):
con_get_schema = self.mock_connection.return_value.get_schema
self.manager.get_schema('reponame', 'tablename')
self.assertTrue(con_get_schema.called)
self.assertEqual(con_get_schema.call_args[1]['repo'], 'reponame')
self.assertEqual(con_get_schema.call_args[1]['table'], 'tablename')
def tables(request):
username = get_login(request)
manager = DataHubManager(user=username)
repo = request.GET.get('repo')
tables = [str(row[0]) for row in manager.list_tables(repo)['tuples']]
return {'tables': tables}
def tables(request):
username = get_login(request)
manager = DataHubManager(user=username)
repo = request.GET.get('repo')
tables = [str(row[0]) for row in manager.list_tables(repo)['tuples']]
return {'tables': tables}
def tables(request):
username = request.user.get_username()
manager = DataHubManager(user=username)
repo = request.GET.get('repo')
tables = manager.list_tables(repo)
return {'tables': tables}
class BasicOperations(TestCase):
"""Tests basic operations in manager.py."""
@factory.django.mute_signals(signals.pre_save)
def setUp(self):
self.username = "******"
self.password = "******"
self.email = "*****@*****.**"
self.user = User.objects.create_user(
self.username, self.email, self.password)
self.mock_connection = self.create_patch(
'core.db.manager.DataHubConnection')
self.manager = DataHubManager(user=self.username)
def create_patch(self, name):
# helper method for creating patches
patcher = patch(name)
thing = patcher.start()
self.addCleanup(patcher.stop)
return thing
def test_list_tables(self):
con_list_tables = self.mock_connection.return_value.list_tables
self.manager.list_tables('repo')
self.assertTrue(con_list_tables.called)
def test_list_views(self):
con_list_views = self.mock_connection.return_value.list_views
self.manager.list_views('repo')
self.assertTrue(con_list_views.called)
def test_list_repos(self):
con_list_repos = self.mock_connection.return_value.list_repos
self.manager.list_repos()
self.assertTrue(con_list_repos.called)
def test_create_repo(self):
con_create_repo = self.mock_connection.return_value.create_repo
self.manager.create_repo('repo')
self.assertTrue(con_create_repo.called)
def test_delete_repo(self):
con_delete_repo = self.mock_connection.return_value.delete_repo
self.manager = DataHubManager(
user=self.username, repo_base=self.username)
self.manager.delete_repo('repo')
self.assertTrue(con_delete_repo.called)
self.assertEqual(con_delete_repo.call_args[1]['force'], False)
def test_add_collaborator(self):
con_add_collab = self.mock_connection.return_value.add_collaborator
mock_User = self.create_patch('core.db.manager.User')
mock_Collaborator = self.create_patch(
'core.db.manager.Collaborator')
mock_Collaborator.objects.get_or_create.return_value = (
MagicMock(), True)
self.manager.add_collaborator(
repo='reponame', collaborator='new_collaborator', privileges='select')
self.assertTrue(con_add_collab.called)
self.assertEqual(con_add_collab.call_args[1]['repo'], 'reponame')
self.assertEqual(
con_add_collab.call_args[1]['collaborator'], 'new_collaborator')
self.assertEqual(con_add_collab.call_args[1]['privileges'], 'select')
def test_delete_collaborator(self):
self.mock_connection.return_value.list_collaborators.return_value = [
'old_collaborator']
self.mock_User = self.create_patch('core.db.manager.User')
self.mock_Collaborator = self.create_patch(
'core.db.manager.Collaborator')
con_delete_collab = (self.mock_connection
.return_value.delete_collaborator)
self.manager.delete_collaborator('reponame', 'old_collaborator')
self.assertTrue(con_delete_collab.called)
self.assertEqual(con_delete_collab.call_args[1]['repo'], 'reponame')
self.assertEqual(
con_delete_collab.call_args[1]['collaborator'], 'old_collaborator')
def test_get_schema(self):
con_get_schema = self.mock_connection.return_value.get_schema
self.manager.get_schema('reponame', 'tablename')
self.assertTrue(con_get_schema.called)
self.assertEqual(con_get_schema.call_args[1]['repo'], 'reponame')
self.assertEqual(con_get_schema.call_args[1]['table'], 'tablename')
class BasicOperations(TestCase):
"""Tests basic operations in manager.py."""
@factory.django.mute_signals(signals.pre_save)
def setUp(self):
self.username = "******"
self.password = "******"
self.email = "*****@*****.**"
self.user = User.objects.create_user(
self.username, self.email, self.password)
self.mock_connection = self.create_patch(
'core.db.manager.DataHubConnection')
self.manager = DataHubManager(user=self.username)
def create_patch(self, name):
# helper method for creating patches
patcher = patch(name)
thing = patcher.start()
self.addCleanup(patcher.stop)
return thing
def test_set_search_paths(self):
con_set_spaths = self.mock_connection.return_value.set_search_paths
search_paths = ['reponame', 'username']
self.manager.set_search_paths(search_paths)
self.assertTrue(con_set_spaths.called)
def test_create_table(self):
con_create_table = self.mock_connection.return_value.create_table
repo = 'repo'
table = 'table'
params = {'id': 'integer', 'words': 'text'}
self.manager.create_table(repo, table, params)
self.assertTrue(con_create_table.called)
def test_list_tables(self):
con_list_tables = self.mock_connection.return_value.list_tables
self.manager.list_tables('repo')
self.assertTrue(con_list_tables.called)
def test_describe_table(self):
con_describe_table = self.mock_connection.return_value.describe_table
self.manager.describe_table(repo='repo', table='table', detail=False)
self.assertTrue(con_describe_table.called)
def test_list_table_permissions(self):
con_tbl_perm = self.mock_connection.return_value.list_table_permissions
self.manager.list_table_permissions(repo='repo', table='table')
self.assertTrue(con_tbl_perm.called)
def test_delete_table(self):
con_delete_table = self.mock_connection.return_value.delete_table
self.manager.delete_table(repo='repo', table='table', force=False)
self.assertTrue(con_delete_table.called)
def test_list_views(self):
con_list_views = self.mock_connection.return_value.list_views
self.manager.list_views('repo')
self.assertTrue(con_list_views.called)
def test_create_view(self):
con_create_view = self.mock_connection.return_value.create_view
repo = 'repo_name'
view = 'view_name'
sql = 'SELECT * FROM table_name'
self.manager.create_view(repo, view, sql)
self.assertTrue(con_create_view.called)
def test_describe_view(self):
con_describe_view = self.mock_connection.return_value.describe_view
self.manager.describe_view(
repo='repo_name', view='_name', detail=False)
self.assertTrue(con_describe_view.called)
def test_delete_view(self):
con_delete_view = self.mock_connection.return_value.delete_view
self.manager.delete_view(
repo='repo_name', view='view_name', force=False)
self.assertTrue(con_delete_view.called)
def test_list_repos(self):
con_list_repos = self.mock_connection.return_value.list_repos
self.manager.list_repos()
self.assertTrue(con_list_repos.called)
def test_rename_repo(self):
con_rename_repo = self.mock_connection.return_value.rename_repo
# self.mock_Collaborator = self.create_patch(
# 'inventory.models.Collaborator')
self.manager = DataHubManager(
user=self.username, repo_base=self.username)
self.manager.Collaborator = MagicMock
self.manager.rename_repo('old_name', 'new_name')
self.assertTrue(con_rename_repo.called)
def test_create_repo(self):
con_create_repo = self.mock_connection.return_value.create_repo
self.manager.create_repo('repo')
self.assertTrue(con_create_repo.called)
def test_delete_repo(self):
con_delete_repo = self.mock_connection.return_value.delete_repo
self.manager = DataHubManager(
user=self.username, repo_base=self.username)
self.manager.delete_repo('repo')
self.assertTrue(con_delete_repo.called)
self.assertEqual(con_delete_repo.call_args[1]['force'], False)
def test_add_collaborator(self):
con_add_collab = self.mock_connection.return_value.add_collaborator
self.create_patch('core.db.manager.User')
mock_Collaborator = self.create_patch(
'core.db.manager.Collaborator')
mock_Collaborator.objects.get_or_create.return_value = (
MagicMock(), True)
self.manager.add_collaborator(
repo='reponame',
collaborator='new_collaborator',
db_privileges=['select'], file_privileges=['read', 'write'])
self.assertTrue(con_add_collab.called)
self.assertEqual(con_add_collab.call_args[1]['repo'], 'reponame')
self.assertEqual(
con_add_collab.call_args[1]['collaborator'], 'new_collaborator')
self.assertEqual(
con_add_collab.call_args[1]['db_privileges'], ['SELECT'])
def test_delete_collaborator(self):
self.mock_connection.return_value.list_collaborators.return_value = [
{'username': '******', 'permissions': 'U'},
{'username': self.username, 'permissions': 'UC'}
]
self.mock_User = self.create_patch('core.db.manager.User')
self.mock_Collaborator = self.create_patch(
'core.db.manager.Collaborator')
con_delete_collab = (self.mock_connection
.return_value.delete_collaborator)
self.manager.delete_collaborator('reponame', 'old_collaborator')
self.assertTrue(con_delete_collab.called)
self.assertEqual(con_delete_collab.call_args[1]['repo'], 'reponame')
self.assertEqual(
con_delete_collab.call_args[1]['collaborator'], 'old_collaborator')
def test_get_schema(self):
con_get_schema = self.mock_connection.return_value.get_schema
self.manager.get_schema('reponame', 'tablename')
self.assertTrue(con_get_schema.called)
self.assertEqual(con_get_schema.call_args[1]['repo'], 'reponame')
self.assertEqual(con_get_schema.call_args[1]['table'], 'tablename')
