def user(request, repo_base=None):
username = request.user.get_username()
if not repo_base:
repo_base = username
manager = DataHubManager(user=username, repo_base=repo_base)
repos = manager.list_repos()
visible_repos = []
for repo in repos:
collaborators = manager.list_collaborators(repo)
collaborators = filter(
lambda x: x != '' and x != repo_base, collaborators)
visible_repos.append({
'name': repo,
'owner': repo_base,
'public': True if 'PUBLIC' in collaborators else False,
'collaborators': collaborators,
})
collaborator_repos = manager.list_collaborator_repos()
return render_to_response("user-browse.html", {
'login': username,
'repo_base': repo_base,
'repos': visible_repos,
'collaborator_repos': collaborator_repos})
def list_repos(self, con):
try:
manager = DataHubManager(user=con.user, repo_base=con.repo_base)
res = manager.list_repos()
return construct_result_set(res)
except Exception, e:
raise DBException(message=str(e))
def dbs(request):
username = get_login(request)
manager = DataHubManager(user=username)
dbnames = pick(manager.list_repos()['tuples'], 0)
#q = "SELECT datname FROM pg_database where datistemplate = false;"
#dbnames = [str(row[0]) for row in manager.execute_sql(query=q)['tuples']]
return {'databases': dbnames}
def dbs(request):
username = request.user.get_username()
manager = DataHubManager(user=username)
dbnames = manager.list_repos()
# q = "SELECT datname FROM pg_database where datistemplate = false;"
# dbnames = [str(row[0]) for row in manager.execute_sql(query=q)['tuples']]
return {"databases": dbnames}
def list_repos(self, con):
try:
'''
res = DataHubManager.has_base_privilege(
con.user, con.repo_base, 'CONNECT')
if not (res and res['tuples'][0][0]):
raise Exception('Access denied. Missing required privileges.')
'''
manager = DataHubManager(user=con.user,
repo_base=con.repo_base,
is_app=con.is_app)
res = manager.list_repos()
# prepare dumb thrift stuff
tuple_list = zip(res)
length = len(res)
thrift_crazy_result = {
'status': True,
'row_count': length,
'tuples': tuple_list,
'fields': [{
'type': 1043,
'name': 'repo_name'
}]
}
return construct_result_set(thrift_crazy_result)
except Exception as e:
raise DBException(message=str(e))
def user(request, repo_base=None):
username = request.user.get_username()
if not repo_base:
repo_base = username
manager = DataHubManager(user=username, repo_base=repo_base)
repos = manager.list_repos()
visible_repos = []
for repo in repos:
collaborators = manager.list_collaborators(repo)
collaborators = filter(lambda x: x != '' and x != repo_base,
collaborators)
visible_repos.append({
'name': repo,
'owner': repo_base,
'public': True if 'PUBLIC' in collaborators else False,
'collaborators': collaborators,
})
collaborator_repos = manager.list_collaborator_repos()
return render_to_response(
"user-browse.html", {
'login': username,
'repo_base': repo_base,
'repos': visible_repos,
'collaborator_repos': collaborator_repos
})
def dbs(request):
username = get_login(request)
manager = DataHubManager(user=username)
dbnames = pick(manager.list_repos()['tuples'], 0)
#q = "SELECT datname FROM pg_database where datistemplate = false;"
#dbnames = [str(row[0]) for row in manager.execute_sql(query=q)['tuples']]
return {'databases': dbnames}
def repos(request):
""" return the repos in the user's database.
I think this is only used for (disabled) scorpion features.
"""
username = request.user.get_username()
repo_base = request.GET.get('username', None)
manager = DataHubManager(username, repo_base)
dbnames = manager.list_repos()
# q = "SELECT datname FROM pg_database where datistemplate = false;"
# dbnames = [str(row[0]) for row in manager.execute_sql(query=q)['tuples']]
return {'databases': dbnames}
def test_create_repo_creates_a_repo(self):
# create the new repo
post_object = {'repo': 'repo_name'}
response = self.client.post('/create/' + self.username + '/repo', post_object)
# get a list of repos that the user owns
manager = DataHubManager(user=self.username)
res = manager.list_repos()
repos = [t[0] for t in res['tuples']]
# make sure that it's in there
self.assertTrue('repo_name' in repos)
def list_repos(self, con):
try:
'''
res = DataHubManager.has_base_privilege(con.user, con.repo_base, 'CONNECT')
if not (res and res['tuples'][0][0]):
raise Exception('Access denied. Missing required privileges.')
'''
manager = DataHubManager(user=con.repo_base, repo_base=con.repo_base)
res = manager.list_repos()
return construct_result_set(res)
except Exception, e:
raise DBException(message=str(e))
def test_create_repo_creates_a_repo(self):
# create the new repo
post_object = {'repo': 'repo_name'}
response = self.client.post('/create/' + self.username + '/repo',
post_object)
# get a list of repos that the user owns
manager = DataHubManager(user=self.username)
res = manager.list_repos()
repos = [t[0] for t in res['tuples']]
# make sure that it's in there
self.assertTrue('repo_name' in repos)
def list_repos(self, con):
try:
'''
res = DataHubManager.has_base_privilege(con.user, con.repo_base, 'CONNECT')
if not (res and res['tuples'][0][0]):
raise Exception('Access denied. Missing required privileges.')
'''
manager = DataHubManager(user=con.repo_base,
repo_base=con.repo_base)
res = manager.list_repos()
return construct_result_set(res)
except Exception, e:
raise DBException(message=str(e))
def handle(self, *args, **options):
# Throw out all of these changes if it fails somehow
with transaction.atomic():
for base in (user.get_username() for user in User.objects.all()):
try:
manager = DataHubManager(user=base, repo_base=base)
except OperationalError as e:
print(e)
continue
repos = manager.list_repos()
for repo in repos:
self._sync_repo_collaborators(manager, base, repo)
print("Added missing {0} users, {1} missing apps.".format(
self.user_count, self.app_count))
def handle(self, *args, **options):
# Throw out all of these changes if it fails somehow
with transaction.atomic():
for base in (user.get_username() for user in User.objects.all()):
try:
manager = DataHubManager(user=base, repo_base=base)
except OperationalError as e:
print(e)
continue
repos = manager.list_repos()
for repo in repos:
self._sync_repo_collaborators(manager, base, repo)
print("Added missing {0} users, {1} missing apps.".format(
self.user_count, self.app_count))
def user(request, repo_base):
try:
login = get_login(request)
res = DataHubManager.has_base_privilege(login, repo_base, 'CONNECT')
if not (res and res['tuples'][0][0]):
raise Exception('Access denied. Missing required privileges.')
manager = DataHubManager(user=repo_base)
res = manager.list_repos()
repos = [t[0] for t in res['tuples']]
visible_repos = []
for repo in repos:
res = manager.list_collaborators(repo_base, repo)
collaborators = [(c[0].split('=')[0]).strip()
for c in res['tuples']]
collaborators = filter(lambda x: x != '' and x != repo_base,
collaborators)
if login not in collaborators and login != repo_base:
continue
visible_repos.append({
'name':
repo,
'owner':
repo_base,
'public':
True if 'PUBLIC' in collaborators else False,
'collaborators':
collaborators,
'collaborators_str':
', '.join(collaborators),
'num_collaborators':
len(collaborators)
})
return render_to_response(
"user-browse.html", {
'login': get_login(request),
'repo_base': repo_base,
'repos': visible_repos
})
except Exception, e:
return HttpResponse(json.dumps({'error': str(e)}),
content_type="application/json")
def user(request, repo_base):
try:
login = get_login(request)
res = DataHubManager.has_base_privilege(login, repo_base, 'CONNECT')
if not (res and res['tuples'][0][0]):
raise Exception('Access denied. Missing required privileges.')
manager = DataHubManager(user=repo_base)
res = manager.list_repos()
repos = [t[0] for t in res['tuples']]
visible_repos = []
for repo in repos:
res = manager.list_collaborators(repo_base, repo)
collaborators = [(c[0].split('=')[0]).strip() for c in res['tuples']]
collaborators = filter(lambda x: x!='' and x!=repo_base, collaborators)
if login not in collaborators and login != repo_base:
continue
visible_repos.append({
'name':repo,
'owner': repo_base,
'public': True if 'PUBLIC' in collaborators else False,
'collaborators': collaborators,
'collaborators_str': ', '.join(collaborators),
'num_collaborators': len(collaborators)
})
return render_to_response("user-browse.html", {
'login': get_login(request),
'repo_base': repo_base,
'repos': visible_repos})
except Exception, e:
return HttpResponse(json.dumps(
{'error': str(e)}),
content_type="application/json")
def does_cache_exist(repo_base):
""" check to see if the cache exists for the repo base"""
manager = DataHubManager(repo_base)
repo_exists = False
table_exists = False
schema_correct = False
# check for repo
if dbwipes_repo in manager.list_repos():
repo_exists = True
# check for table
if repo_exists and dbwipes_table in manager.list_tables(dbwipes_repo):
table_exists = True
# check for schema
schema = None
if table_exists:
schema = manager.get_schema(dbwipes_repo, dbwipes_table)
if schema == [('key', 'character varying'), ('val', 'text')]:
schema_correct = True
return repo_exists and table_exists and schema_correct
def list_repos(self, con):
try:
'''
res = DataHubManager.has_base_privilege(
con.user, con.repo_base, 'CONNECT')
if not (res and res['tuples'][0][0]):
raise Exception('Access denied. Missing required privileges.')
'''
manager = DataHubManager(
user=con.user, repo_base=con.repo_base, is_app=con.is_app)
res = manager.list_repos()
# prepare dumb thrift stuff
tuple_list = zip(res)
length = len(res)
thrift_crazy_result = {
'status': True, 'row_count': length,
'tuples': tuple_list,
'fields': [{'type': 1043, 'name': 'repo_name'}]
}
return construct_result_set(thrift_crazy_result)
except Exception, e:
raise DBException(message=str(e))
class BasicOperations(TestCase):
"""Tests basic operations in manager.py."""
@factory.django.mute_signals(signals.pre_save)
def setUp(self):
self.username = "******"
self.password = "******"
self.email = "*****@*****.**"
self.user = User.objects.create_user(
self.username, self.email, self.password)
self.mock_connection = self.create_patch(
'core.db.manager.DataHubConnection')
self.manager = DataHubManager(user=self.username)
def create_patch(self, name):
# helper method for creating patches
patcher = patch(name)
thing = patcher.start()
self.addCleanup(patcher.stop)
return thing
def test_create_table(self):
con_create_table = self.mock_connection.return_value.create_table
repo = 'repo'
table = 'table'
params = {'id': 'integer', 'words': 'text'}
self.manager.create_table(repo, table, params)
self.assertTrue(con_create_table.called)
def test_list_tables(self):
con_list_tables = self.mock_connection.return_value.list_tables
self.manager.list_tables('repo')
self.assertTrue(con_list_tables.called)
def test_describe_table(self):
con_describe_table = self.mock_connection.return_value.describe_table
self.manager.describe_table(repo='repo', table='table', detail=False)
self.assertTrue(con_describe_table.called)
def test_list_table_permissions(self):
con_tbl_perm = self.mock_connection.return_value.list_table_permissions
self.manager.list_table_permissions(repo='repo', table='table')
self.assertTrue(con_tbl_perm.called)
def test_delete_table(self):
con_delete_table = self.mock_connection.return_value.delete_table
self.manager.delete_table(repo='repo', table='table', force=False)
self.assertTrue(con_delete_table.called)
def test_clone_table(self):
con_clone_table = self.mock_connection.return_value.clone_table
self.manager.clone_table(
repo='repo_name', table='table_name', new_table='new_table_name')
self.assertTrue(con_clone_table.called)
def test_list_views(self):
con_list_views = self.mock_connection.return_value.list_views
self.manager.list_views('repo')
self.assertTrue(con_list_views.called)
def test_create_view(self):
con_create_view = self.mock_connection.return_value.create_view
repo = 'repo_name'
view = 'view_name'
sql = 'SELECT * FROM table_name'
self.manager.create_view(repo, view, sql)
self.assertTrue(con_create_view.called)
def test_describe_view(self):
con_describe_view = self.mock_connection.return_value.describe_view
self.manager.describe_view(
repo='repo_name', view='_name', detail=False)
self.assertTrue(con_describe_view.called)
def test_delete_view(self):
con_delete_view = self.mock_connection.return_value.delete_view
self.manager.delete_view(
repo='repo_name', view='view_name', force=False)
self.assertTrue(con_delete_view.called)
def test_list_repos(self):
con_list_repos = self.mock_connection.return_value.list_repos
self.manager.list_repos()
self.assertTrue(con_list_repos.called)
def test_rename_repo(self):
con_rename_repo = self.mock_connection.return_value.rename_repo
# self.mock_Collaborator = self.create_patch(
# 'inventory.models.Collaborator')
self.manager = DataHubManager(
user=self.username, repo_base=self.username)
self.manager.Collaborator = MagicMock
self.manager.rename_repo('old_name', 'new_name')
self.assertTrue(con_rename_repo.called)
def test_create_repo(self):
con_create_repo = self.mock_connection.return_value.create_repo
self.manager.create_repo('repo')
self.assertTrue(con_create_repo.called)
def test_delete_repo(self):
con_delete_repo = self.mock_connection.return_value.delete_repo
self.manager = DataHubManager(
user=self.username, repo_base=self.username)
self.manager.delete_repo('repo')
self.assertTrue(con_delete_repo.called)
self.assertEqual(con_delete_repo.call_args[1]['force'], False)
def test_add_collaborator(self):
con_add_collab = self.mock_connection.return_value.add_collaborator
self.create_patch('core.db.manager.User')
mock_Collaborator = self.create_patch(
'core.db.manager.Collaborator')
mock_Collaborator.objects.get_or_create.return_value = (
MagicMock(), True)
self.manager.add_collaborator(
repo='reponame',
collaborator='new_collaborator',
db_privileges=['select'], file_privileges=['read', 'write'])
self.assertTrue(con_add_collab.called)
self.assertEqual(con_add_collab.call_args[1]['repo'], 'reponame')
self.assertEqual(
con_add_collab.call_args[1]['collaborator'], 'new_collaborator')
self.assertEqual(
con_add_collab.call_args[1]['db_privileges'], ['SELECT'])
def test_delete_collaborator(self):
self.mock_connection.return_value.list_collaborators.return_value = [
{'username': '******', 'permissions': 'U'},
{'username': self.username, 'permissions': 'UC'}
]
self.mock_User = self.create_patch('core.db.manager.User')
self.mock_Collaborator = self.create_patch(
'core.db.manager.Collaborator')
con_delete_collab = (self.mock_connection
.return_value.delete_collaborator)
self.manager.delete_collaborator('reponame', 'old_collaborator')
self.assertTrue(con_delete_collab.called)
self.assertEqual(con_delete_collab.call_args[1]['repo'], 'reponame')
self.assertEqual(
con_delete_collab.call_args[1]['collaborator'], 'old_collaborator')
def test_get_schema(self):
con_get_schema = self.mock_connection.return_value.get_schema
self.manager.get_schema('reponame', 'tablename')
self.assertTrue(con_get_schema.called)
self.assertEqual(con_get_schema.call_args[1]['repo'], 'reponame')
self.assertEqual(con_get_schema.call_args[1]['table'], 'tablename')
class BasicOperations(TestCase):
"""Tests basic operations in manager.py."""
@factory.django.mute_signals(signals.pre_save)
def setUp(self):
self.username = "******"
self.password = "******"
self.email = "*****@*****.**"
self.user = User.objects.create_user(
self.username, self.email, self.password)
self.mock_connection = self.create_patch(
'core.db.manager.DataHubConnection')
self.manager = DataHubManager(user=self.username)
def create_patch(self, name):
# helper method for creating patches
patcher = patch(name)
thing = patcher.start()
self.addCleanup(patcher.stop)
return thing
def test_set_search_paths(self):
con_set_spaths = self.mock_connection.return_value.set_search_paths
search_paths = ['reponame', 'username']
self.manager.set_search_paths(search_paths)
self.assertTrue(con_set_spaths.called)
def test_create_table(self):
con_create_table = self.mock_connection.return_value.create_table
repo = 'repo'
table = 'table'
params = {'id': 'integer', 'words': 'text'}
self.manager.create_table(repo, table, params)
self.assertTrue(con_create_table.called)
def test_list_tables(self):
con_list_tables = self.mock_connection.return_value.list_tables
self.manager.list_tables('repo')
self.assertTrue(con_list_tables.called)
def test_describe_table(self):
con_describe_table = self.mock_connection.return_value.describe_table
self.manager.describe_table(repo='repo', table='table', detail=False)
self.assertTrue(con_describe_table.called)
def test_list_table_permissions(self):
con_tbl_perm = self.mock_connection.return_value.list_table_permissions
self.manager.list_table_permissions(repo='repo', table='table')
self.assertTrue(con_tbl_perm.called)
def test_delete_table(self):
con_delete_table = self.mock_connection.return_value.delete_table
self.manager.delete_table(repo='repo', table='table', force=False)
self.assertTrue(con_delete_table.called)
def test_list_views(self):
con_list_views = self.mock_connection.return_value.list_views
self.manager.list_views('repo')
self.assertTrue(con_list_views.called)
def test_create_view(self):
con_create_view = self.mock_connection.return_value.create_view
repo = 'repo_name'
view = 'view_name'
sql = 'SELECT * FROM table_name'
self.manager.create_view(repo, view, sql)
self.assertTrue(con_create_view.called)
def test_describe_view(self):
con_describe_view = self.mock_connection.return_value.describe_view
self.manager.describe_view(
repo='repo_name', view='_name', detail=False)
self.assertTrue(con_describe_view.called)
def test_delete_view(self):
con_delete_view = self.mock_connection.return_value.delete_view
self.manager.delete_view(
repo='repo_name', view='view_name', force=False)
self.assertTrue(con_delete_view.called)
def test_list_repos(self):
con_list_repos = self.mock_connection.return_value.list_repos
self.manager.list_repos()
self.assertTrue(con_list_repos.called)
def test_rename_repo(self):
con_rename_repo = self.mock_connection.return_value.rename_repo
# self.mock_Collaborator = self.create_patch(
# 'inventory.models.Collaborator')
self.manager = DataHubManager(
user=self.username, repo_base=self.username)
self.manager.Collaborator = MagicMock
self.manager.rename_repo('old_name', 'new_name')
self.assertTrue(con_rename_repo.called)
def test_create_repo(self):
con_create_repo = self.mock_connection.return_value.create_repo
self.manager.create_repo('repo')
self.assertTrue(con_create_repo.called)
def test_delete_repo(self):
con_delete_repo = self.mock_connection.return_value.delete_repo
self.manager = DataHubManager(
user=self.username, repo_base=self.username)
self.manager.delete_repo('repo')
self.assertTrue(con_delete_repo.called)
self.assertEqual(con_delete_repo.call_args[1]['force'], False)
def test_add_collaborator(self):
con_add_collab = self.mock_connection.return_value.add_collaborator
self.create_patch('core.db.manager.User')
mock_Collaborator = self.create_patch(
'core.db.manager.Collaborator')
mock_Collaborator.objects.get_or_create.return_value = (
MagicMock(), True)
self.manager.add_collaborator(
repo='reponame',
collaborator='new_collaborator',
db_privileges=['select'], file_privileges=['read', 'write'])
self.assertTrue(con_add_collab.called)
self.assertEqual(con_add_collab.call_args[1]['repo'], 'reponame')
self.assertEqual(
con_add_collab.call_args[1]['collaborator'], 'new_collaborator')
self.assertEqual(
con_add_collab.call_args[1]['db_privileges'], ['SELECT'])
def test_delete_collaborator(self):
self.mock_connection.return_value.list_collaborators.return_value = [
{'username': '******', 'permissions': 'U'},
{'username': self.username, 'permissions': 'UC'}
]
self.mock_User = self.create_patch('core.db.manager.User')
self.mock_Collaborator = self.create_patch(
'core.db.manager.Collaborator')
con_delete_collab = (self.mock_connection
.return_value.delete_collaborator)
self.manager.delete_collaborator('reponame', 'old_collaborator')
self.assertTrue(con_delete_collab.called)
self.assertEqual(con_delete_collab.call_args[1]['repo'], 'reponame')
self.assertEqual(
con_delete_collab.call_args[1]['collaborator'], 'old_collaborator')
def test_get_schema(self):
con_get_schema = self.mock_connection.return_value.get_schema
self.manager.get_schema('reponame', 'tablename')
self.assertTrue(con_get_schema.called)
self.assertEqual(con_get_schema.call_args[1]['repo'], 'reponame')
self.assertEqual(con_get_schema.call_args[1]['table'], 'tablename')
class BasicOperations(TestCase):
"""Tests basic operations in manager.py."""
@factory.django.mute_signals(signals.pre_save)
def setUp(self):
self.username = "******"
self.password = "******"
self.email = "*****@*****.**"
self.user = User.objects.create_user(
self.username, self.email, self.password)
self.mock_connection = self.create_patch(
'core.db.manager.DataHubConnection')
self.manager = DataHubManager(user=self.username)
def create_patch(self, name):
# helper method for creating patches
patcher = patch(name)
thing = patcher.start()
self.addCleanup(patcher.stop)
return thing
def test_list_tables(self):
con_list_tables = self.mock_connection.return_value.list_tables
self.manager.list_tables('repo')
self.assertTrue(con_list_tables.called)
def test_list_views(self):
con_list_views = self.mock_connection.return_value.list_views
self.manager.list_views('repo')
self.assertTrue(con_list_views.called)
def test_list_repos(self):
con_list_repos = self.mock_connection.return_value.list_repos
self.manager.list_repos()
self.assertTrue(con_list_repos.called)
def test_create_repo(self):
con_create_repo = self.mock_connection.return_value.create_repo
self.manager.create_repo('repo')
self.assertTrue(con_create_repo.called)
def test_delete_repo(self):
con_delete_repo = self.mock_connection.return_value.delete_repo
self.manager = DataHubManager(
user=self.username, repo_base=self.username)
self.manager.delete_repo('repo')
self.assertTrue(con_delete_repo.called)
self.assertEqual(con_delete_repo.call_args[1]['force'], False)
def test_add_collaborator(self):
con_add_collab = self.mock_connection.return_value.add_collaborator
mock_User = self.create_patch('core.db.manager.User')
mock_Collaborator = self.create_patch(
'core.db.manager.Collaborator')
mock_Collaborator.objects.get_or_create.return_value = (
MagicMock(), True)
self.manager.add_collaborator(
repo='reponame', collaborator='new_collaborator', privileges='select')
self.assertTrue(con_add_collab.called)
self.assertEqual(con_add_collab.call_args[1]['repo'], 'reponame')
self.assertEqual(
con_add_collab.call_args[1]['collaborator'], 'new_collaborator')
self.assertEqual(con_add_collab.call_args[1]['privileges'], 'select')
def test_delete_collaborator(self):
self.mock_connection.return_value.list_collaborators.return_value = [
'old_collaborator']
self.mock_User = self.create_patch('core.db.manager.User')
self.mock_Collaborator = self.create_patch(
'core.db.manager.Collaborator')
con_delete_collab = (self.mock_connection
.return_value.delete_collaborator)
self.manager.delete_collaborator('reponame', 'old_collaborator')
self.assertTrue(con_delete_collab.called)
self.assertEqual(con_delete_collab.call_args[1]['repo'], 'reponame')
self.assertEqual(
con_delete_collab.call_args[1]['collaborator'], 'old_collaborator')
def test_get_schema(self):
con_get_schema = self.mock_connection.return_value.get_schema
self.manager.get_schema('reponame', 'tablename')
self.assertTrue(con_get_schema.called)
self.assertEqual(con_get_schema.call_args[1]['repo'], 'reponame')
self.assertEqual(con_get_schema.call_args[1]['table'], 'tablename')
