def get_schema(self, con, table_name):
# this should really accept repo and table name as seperate
try:
manager = DataHubManager(user=con.user,
repo_base=con.repo_base,
is_app=con.is_app)
# thrift isn't upgraded to use seperate repo and tokens yet,
# so they have to be split here, and then passed to the manager.
tokens = table_name.split('.')
repo = tokens[0]
table = tokens[1]
res = manager.get_schema(repo=repo, table=table)
# apps expect get_schema to return in a backwards way. It's made
# to be backwards here.
thrift_crazy_result = {
'status':
True,
'row_count':
0,
'tuples':
res,
'fields': [{
'type': 1043,
'name': 'column_name'
}, {
'type': 1043,
'name': 'data_type'
}]
}
return construct_result_set(thrift_crazy_result)
except Exception as e:
raise DBException(message=str(e))
def get_schema(self, con, table_name):
try:
manager = DataHubManager(user=con.user, repo_base=con.repo_base)
res = manager.get_schema(table=table_name)
return construct_result_set(res)
except Exception, e:
raise DBException(message=str(e))
def get_schema(self, con, table_name):
try:
manager = DataHubManager(user=con.repo_base,
repo_base=con.repo_base)
res = manager.get_schema(table=table_name)
return construct_result_set(res)
except Exception, e:
raise DBException(message=str(e))
def schema(request, repo_name, table_name):
username = request.user.get_username()
manager = DataHubManager(username)
repos = get_repos(manager)
if repos is not None and repo_name in repos:
tables = get_tables(manager, repo_name)
if tables is not None and table_name in tables:
schema = manager.get_schema(repo_name, table_name)
if len(schema) > 0:
return json_response({"schema": schema})
return error_response()
def schema(request, repo_name, table_name):
username = get_login(request)
manager = DataHubManager(username)
repos = get_repos(manager)
if repos is not None and repo_name in repos:
tables = get_tables(manager, repo_name)
if tables is not None and table_name in tables:
schema = manager.get_schema(repo_name + "." + table_name)
if schema is not None and 'tuples' in schema:
return json_response({"schema": schema["tuples"]})
return error_response()
def schema(request, repo, table):
username = get_login(request)
manager = DataHubManager(username)
repos = get_repos(manager)
if repos is not None and repo in repos:
tables = get_tables(manager, repo)
if tables is not None and table in tables:
schema = manager.get_schema(repo + "." + table)
if schema is not None and 'tuples' in schema:
return json_response({"schema": schema["tuples"]})
return error_response()
def schema(request, repo, table):
username = request.user.get_username()
manager = DataHubManager(username)
repos = get_repos(manager)
if repos is not None and repo in repos:
tables = get_tables(manager, repo)
if tables is not None and table in tables:
schema = manager.get_schema(repo, table)
if len(schema) > 0:
return json_response({"schema": schema})
return error_response()
def get_schema(repo, table, username):
manager = DataHubManager(user=username)
pairs = manager.get_schema(repo, table)
schema = {}
for col, typ in pairs:
if typ == 'text':
typ = 'str'
if 'double' in typ:
typ = 'num'
if 'int' in typ:
typ = 'num'
if 'date' in typ or 'time' in typ:
typ = 'time'
schema[str(col)] = str(typ)
return schema
def get_schema(repo, table, username):
manager = DataHubManager(user=username)
pairs = manager.get_schema(repo, table)
schema = {}
for col, typ in pairs:
if typ == "text":
typ = "str"
if "double" in typ:
typ = "num"
if "int" in typ:
typ = "num"
if "date" in typ or "time" in typ:
typ = "time"
schema[str(col)] = str(typ)
return schema
def get_columns_and_types(self):
manager = DataHubManager(user=self.username)
rows = manager.get_schema(self.tablename)['tuples']
ret = []
for col, typ in rows:
if typ == 'text':
typ = 'str'
if 'double' in typ:
typ = 'num'
if 'int' in typ:
typ = 'num'
if 'date' in typ or 'time' in typ:
typ = 'time'
ret.append((str(col), str(typ)) )
return ret
def get_columns_and_types(self):
manager = DataHubManager(user=self.username)
rows = manager.get_schema(self.tablename)['tuples']
ret = []
for col, typ in rows:
if typ == 'text':
typ = 'str'
if 'double' in typ:
typ = 'num'
if 'int' in typ:
typ = 'num'
if 'date' in typ or 'time' in typ:
typ = 'time'
ret.append((str(col), str(typ)))
return ret
def get_schema(repo, table, username):
full_tablename = "%s.%s" % (repo, table)
manager = DataHubManager(user=username)
pairs = manager.get_schema(full_tablename)['tuples']
schema = {}
for col, typ in pairs:
if typ == 'text':
typ = 'str'
if 'double' in typ:
typ = 'num'
if 'int' in typ:
typ = 'num'
if 'date' in typ or 'time' in typ:
typ = 'time'
schema[str(col)] = str(typ)
return schema
def get_schema(repo, table, username, repo_base):
""" gets the schema of a given repo.table in a repo_base """
manager = DataHubManager(user=username, repo_base=repo_base)
pairs = manager.get_schema(repo, table)
schema = {}
for col, typ in pairs:
if typ == 'text':
typ = 'str'
if 'double' in typ:
typ = 'num'
if 'int' in typ:
typ = 'num'
if 'date' in typ or 'time' in typ:
typ = 'time'
schema[str(col)] = str(typ)
return schema
def aggregate(request, repo, table, agg_type, col_name):
username = get_login(request)
manager = DataHubManager(username)
repos = get_repos(manager)
# Ensure that the repo exists.
if repos is not None and repo in repos:
tables = get_tables(manager, repo)
# Ensure that the table exists.
if tables is not None and table in tables:
schema = manager.get_schema(repo + "." + table)
# Ensure that the schema for the repo.table exists.
if schema is not None and 'tuples' in schema:
for c_name, c_type in schema["tuples"]:
if c_name == col_name and can_apply(agg_type, col_name):
result = manager.execute_sql("SELECT %s(%s) FROM %s.%s" % (agg_type.lower(), col_name.lower(), repo, table))
if "tuples" in result and len(result["tuples"]) > 0 and len(result["tuples"][0]) > 0:
return json_response({"value": result["tuples"][0]})
return error_response()
def aggregate(request, repo, table, agg_type, col_name):
username = request.user.get_username()
manager = DataHubManager(username)
repos = get_repos(manager)
# Ensure that the repo exists.
if repos is not None and repo in repos:
tables = get_tables(manager, repo)
# Ensure that the table exists.
if tables is not None and table in tables:
schema = manager.get_schema(repo, table)
# Ensure that the schema for the repo.table exists.
if len(schema) > 0:
for c_name, c_type in schema:
if c_name == col_name and can_apply(agg_type, col_name):
result = manager.execute_sql(
"SELECT %s(%s) FROM %s.%s" % (agg_type.lower(), col_name.lower(), repo, table))
if "tuples" in result and len(result["tuples"]) > 0 and len(result["tuples"][0]) > 0:
return json_response({"value": result["tuples"][0]})
return error_response()
def get_columns_and_types(self):
manager = DataHubManager(user=self.username)
tokens = self.tablename.split('.')
repo = tokens[0]
table = tokens[1]
rows = manager.get_schema(repo, table)
ret = []
for col, typ in rows:
if typ == 'text':
typ = 'str'
if 'double' in typ:
typ = 'num'
if 'int' in typ:
typ = 'num'
if 'date' in typ or 'time' in typ:
typ = 'time'
ret.append((str(col), str(typ)) )
return ret
def get_columns_and_types(self):
manager = DataHubManager(user=self.username)
tokens = self.tablename.split('.')
repo = tokens[0]
table = tokens[1]
rows = manager.get_schema(repo, table)
ret = []
for col, typ in rows:
if typ == 'text':
typ = 'str'
if 'double' in typ:
typ = 'num'
if 'int' in typ:
typ = 'num'
if 'date' in typ or 'time' in typ:
typ = 'time'
ret.append((str(col), str(typ)))
return ret
def does_cache_exist(repo_base):
""" check to see if the cache exists for the repo base"""
manager = DataHubManager(repo_base)
repo_exists = False
table_exists = False
schema_correct = False
# check for repo
if dbwipes_repo in manager.list_repos():
repo_exists = True
# check for table
if repo_exists and dbwipes_table in manager.list_tables(dbwipes_repo):
table_exists = True
# check for schema
schema = None
if table_exists:
schema = manager.get_schema(dbwipes_repo, dbwipes_table)
if schema == [('key', 'character varying'), ('val', 'text')]:
schema_correct = True
return repo_exists and table_exists and schema_correct
def get_schema(self, con, table_name):
# this should really accept repo and table name as seperate
try:
manager = DataHubManager(
user=con.user, repo_base=con.repo_base, is_app=con.is_app)
# thrift isn't upgraded to use seperate repo and tokens yet,
# so they have to be split here, and then passed to the manager.
tokens = table_name.split('.')
repo = tokens[0]
table = tokens[1]
res = manager.get_schema(repo=repo, table=table)
# apps expect get_schema to return in a backwards way. It's made
# to be backwards here.
thrift_crazy_result = {
'status': True, 'row_count': 0, 'tuples': res,
'fields': [
{'type': 1043, 'name': 'column_name'},
{'type': 1043, 'name': 'data_type'}]}
return construct_result_set(thrift_crazy_result)
except Exception, e:
raise DBException(message=str(e))
class BasicOperations(TestCase):
"""Tests basic operations in manager.py."""
@factory.django.mute_signals(signals.pre_save)
def setUp(self):
self.username = "******"
self.password = "******"
self.email = "*****@*****.**"
self.user = User.objects.create_user(
self.username, self.email, self.password)
self.mock_connection = self.create_patch(
'core.db.manager.DataHubConnection')
self.manager = DataHubManager(user=self.username)
def create_patch(self, name):
# helper method for creating patches
patcher = patch(name)
thing = patcher.start()
self.addCleanup(patcher.stop)
return thing
def test_create_table(self):
con_create_table = self.mock_connection.return_value.create_table
repo = 'repo'
table = 'table'
params = {'id': 'integer', 'words': 'text'}
self.manager.create_table(repo, table, params)
self.assertTrue(con_create_table.called)
def test_list_tables(self):
con_list_tables = self.mock_connection.return_value.list_tables
self.manager.list_tables('repo')
self.assertTrue(con_list_tables.called)
def test_describe_table(self):
con_describe_table = self.mock_connection.return_value.describe_table
self.manager.describe_table(repo='repo', table='table', detail=False)
self.assertTrue(con_describe_table.called)
def test_list_table_permissions(self):
con_tbl_perm = self.mock_connection.return_value.list_table_permissions
self.manager.list_table_permissions(repo='repo', table='table')
self.assertTrue(con_tbl_perm.called)
def test_delete_table(self):
con_delete_table = self.mock_connection.return_value.delete_table
self.manager.delete_table(repo='repo', table='table', force=False)
self.assertTrue(con_delete_table.called)
def test_clone_table(self):
con_clone_table = self.mock_connection.return_value.clone_table
self.manager.clone_table(
repo='repo_name', table='table_name', new_table='new_table_name')
self.assertTrue(con_clone_table.called)
def test_list_views(self):
con_list_views = self.mock_connection.return_value.list_views
self.manager.list_views('repo')
self.assertTrue(con_list_views.called)
def test_create_view(self):
con_create_view = self.mock_connection.return_value.create_view
repo = 'repo_name'
view = 'view_name'
sql = 'SELECT * FROM table_name'
self.manager.create_view(repo, view, sql)
self.assertTrue(con_create_view.called)
def test_describe_view(self):
con_describe_view = self.mock_connection.return_value.describe_view
self.manager.describe_view(
repo='repo_name', view='_name', detail=False)
self.assertTrue(con_describe_view.called)
def test_delete_view(self):
con_delete_view = self.mock_connection.return_value.delete_view
self.manager.delete_view(
repo='repo_name', view='view_name', force=False)
self.assertTrue(con_delete_view.called)
def test_list_repos(self):
con_list_repos = self.mock_connection.return_value.list_repos
self.manager.list_repos()
self.assertTrue(con_list_repos.called)
def test_rename_repo(self):
con_rename_repo = self.mock_connection.return_value.rename_repo
# self.mock_Collaborator = self.create_patch(
# 'inventory.models.Collaborator')
self.manager = DataHubManager(
user=self.username, repo_base=self.username)
self.manager.Collaborator = MagicMock
self.manager.rename_repo('old_name', 'new_name')
self.assertTrue(con_rename_repo.called)
def test_create_repo(self):
con_create_repo = self.mock_connection.return_value.create_repo
self.manager.create_repo('repo')
self.assertTrue(con_create_repo.called)
def test_delete_repo(self):
con_delete_repo = self.mock_connection.return_value.delete_repo
self.manager = DataHubManager(
user=self.username, repo_base=self.username)
self.manager.delete_repo('repo')
self.assertTrue(con_delete_repo.called)
self.assertEqual(con_delete_repo.call_args[1]['force'], False)
def test_add_collaborator(self):
con_add_collab = self.mock_connection.return_value.add_collaborator
self.create_patch('core.db.manager.User')
mock_Collaborator = self.create_patch(
'core.db.manager.Collaborator')
mock_Collaborator.objects.get_or_create.return_value = (
MagicMock(), True)
self.manager.add_collaborator(
repo='reponame',
collaborator='new_collaborator',
db_privileges=['select'], file_privileges=['read', 'write'])
self.assertTrue(con_add_collab.called)
self.assertEqual(con_add_collab.call_args[1]['repo'], 'reponame')
self.assertEqual(
con_add_collab.call_args[1]['collaborator'], 'new_collaborator')
self.assertEqual(
con_add_collab.call_args[1]['db_privileges'], ['SELECT'])
def test_delete_collaborator(self):
self.mock_connection.return_value.list_collaborators.return_value = [
{'username': '******', 'permissions': 'U'},
{'username': self.username, 'permissions': 'UC'}
]
self.mock_User = self.create_patch('core.db.manager.User')
self.mock_Collaborator = self.create_patch(
'core.db.manager.Collaborator')
con_delete_collab = (self.mock_connection
.return_value.delete_collaborator)
self.manager.delete_collaborator('reponame', 'old_collaborator')
self.assertTrue(con_delete_collab.called)
self.assertEqual(con_delete_collab.call_args[1]['repo'], 'reponame')
self.assertEqual(
con_delete_collab.call_args[1]['collaborator'], 'old_collaborator')
def test_get_schema(self):
con_get_schema = self.mock_connection.return_value.get_schema
self.manager.get_schema('reponame', 'tablename')
self.assertTrue(con_get_schema.called)
self.assertEqual(con_get_schema.call_args[1]['repo'], 'reponame')
self.assertEqual(con_get_schema.call_args[1]['table'], 'tablename')
class BasicOperations(TestCase):
"""Tests basic operations in manager.py."""
@factory.django.mute_signals(signals.pre_save)
def setUp(self):
self.username = "******"
self.password = "******"
self.email = "*****@*****.**"
self.user = User.objects.create_user(
self.username, self.email, self.password)
self.mock_connection = self.create_patch(
'core.db.manager.DataHubConnection')
self.manager = DataHubManager(user=self.username)
def create_patch(self, name):
# helper method for creating patches
patcher = patch(name)
thing = patcher.start()
self.addCleanup(patcher.stop)
return thing
def test_list_tables(self):
con_list_tables = self.mock_connection.return_value.list_tables
self.manager.list_tables('repo')
self.assertTrue(con_list_tables.called)
def test_list_views(self):
con_list_views = self.mock_connection.return_value.list_views
self.manager.list_views('repo')
self.assertTrue(con_list_views.called)
def test_list_repos(self):
con_list_repos = self.mock_connection.return_value.list_repos
self.manager.list_repos()
self.assertTrue(con_list_repos.called)
def test_create_repo(self):
con_create_repo = self.mock_connection.return_value.create_repo
self.manager.create_repo('repo')
self.assertTrue(con_create_repo.called)
def test_delete_repo(self):
con_delete_repo = self.mock_connection.return_value.delete_repo
self.manager = DataHubManager(
user=self.username, repo_base=self.username)
self.manager.delete_repo('repo')
self.assertTrue(con_delete_repo.called)
self.assertEqual(con_delete_repo.call_args[1]['force'], False)
def test_add_collaborator(self):
con_add_collab = self.mock_connection.return_value.add_collaborator
mock_User = self.create_patch('core.db.manager.User')
mock_Collaborator = self.create_patch(
'core.db.manager.Collaborator')
mock_Collaborator.objects.get_or_create.return_value = (
MagicMock(), True)
self.manager.add_collaborator(
repo='reponame', collaborator='new_collaborator', privileges='select')
self.assertTrue(con_add_collab.called)
self.assertEqual(con_add_collab.call_args[1]['repo'], 'reponame')
self.assertEqual(
con_add_collab.call_args[1]['collaborator'], 'new_collaborator')
self.assertEqual(con_add_collab.call_args[1]['privileges'], 'select')
def test_delete_collaborator(self):
self.mock_connection.return_value.list_collaborators.return_value = [
'old_collaborator']
self.mock_User = self.create_patch('core.db.manager.User')
self.mock_Collaborator = self.create_patch(
'core.db.manager.Collaborator')
con_delete_collab = (self.mock_connection
.return_value.delete_collaborator)
self.manager.delete_collaborator('reponame', 'old_collaborator')
self.assertTrue(con_delete_collab.called)
self.assertEqual(con_delete_collab.call_args[1]['repo'], 'reponame')
self.assertEqual(
con_delete_collab.call_args[1]['collaborator'], 'old_collaborator')
def test_get_schema(self):
con_get_schema = self.mock_connection.return_value.get_schema
self.manager.get_schema('reponame', 'tablename')
self.assertTrue(con_get_schema.called)
self.assertEqual(con_get_schema.call_args[1]['repo'], 'reponame')
self.assertEqual(con_get_schema.call_args[1]['table'], 'tablename')
class BasicOperations(TestCase):
"""Tests basic operations in manager.py."""
@factory.django.mute_signals(signals.pre_save)
def setUp(self):
self.username = "******"
self.password = "******"
self.email = "*****@*****.**"
self.user = User.objects.create_user(
self.username, self.email, self.password)
self.mock_connection = self.create_patch(
'core.db.manager.DataHubConnection')
self.manager = DataHubManager(user=self.username)
def create_patch(self, name):
# helper method for creating patches
patcher = patch(name)
thing = patcher.start()
self.addCleanup(patcher.stop)
return thing
def test_set_search_paths(self):
con_set_spaths = self.mock_connection.return_value.set_search_paths
search_paths = ['reponame', 'username']
self.manager.set_search_paths(search_paths)
self.assertTrue(con_set_spaths.called)
def test_create_table(self):
con_create_table = self.mock_connection.return_value.create_table
repo = 'repo'
table = 'table'
params = {'id': 'integer', 'words': 'text'}
self.manager.create_table(repo, table, params)
self.assertTrue(con_create_table.called)
def test_list_tables(self):
con_list_tables = self.mock_connection.return_value.list_tables
self.manager.list_tables('repo')
self.assertTrue(con_list_tables.called)
def test_describe_table(self):
con_describe_table = self.mock_connection.return_value.describe_table
self.manager.describe_table(repo='repo', table='table', detail=False)
self.assertTrue(con_describe_table.called)
def test_list_table_permissions(self):
con_tbl_perm = self.mock_connection.return_value.list_table_permissions
self.manager.list_table_permissions(repo='repo', table='table')
self.assertTrue(con_tbl_perm.called)
def test_delete_table(self):
con_delete_table = self.mock_connection.return_value.delete_table
self.manager.delete_table(repo='repo', table='table', force=False)
self.assertTrue(con_delete_table.called)
def test_list_views(self):
con_list_views = self.mock_connection.return_value.list_views
self.manager.list_views('repo')
self.assertTrue(con_list_views.called)
def test_create_view(self):
con_create_view = self.mock_connection.return_value.create_view
repo = 'repo_name'
view = 'view_name'
sql = 'SELECT * FROM table_name'
self.manager.create_view(repo, view, sql)
self.assertTrue(con_create_view.called)
def test_describe_view(self):
con_describe_view = self.mock_connection.return_value.describe_view
self.manager.describe_view(
repo='repo_name', view='_name', detail=False)
self.assertTrue(con_describe_view.called)
def test_delete_view(self):
con_delete_view = self.mock_connection.return_value.delete_view
self.manager.delete_view(
repo='repo_name', view='view_name', force=False)
self.assertTrue(con_delete_view.called)
def test_list_repos(self):
con_list_repos = self.mock_connection.return_value.list_repos
self.manager.list_repos()
self.assertTrue(con_list_repos.called)
def test_rename_repo(self):
con_rename_repo = self.mock_connection.return_value.rename_repo
# self.mock_Collaborator = self.create_patch(
# 'inventory.models.Collaborator')
self.manager = DataHubManager(
user=self.username, repo_base=self.username)
self.manager.Collaborator = MagicMock
self.manager.rename_repo('old_name', 'new_name')
self.assertTrue(con_rename_repo.called)
def test_create_repo(self):
con_create_repo = self.mock_connection.return_value.create_repo
self.manager.create_repo('repo')
self.assertTrue(con_create_repo.called)
def test_delete_repo(self):
con_delete_repo = self.mock_connection.return_value.delete_repo
self.manager = DataHubManager(
user=self.username, repo_base=self.username)
self.manager.delete_repo('repo')
self.assertTrue(con_delete_repo.called)
self.assertEqual(con_delete_repo.call_args[1]['force'], False)
def test_add_collaborator(self):
con_add_collab = self.mock_connection.return_value.add_collaborator
self.create_patch('core.db.manager.User')
mock_Collaborator = self.create_patch(
'core.db.manager.Collaborator')
mock_Collaborator.objects.get_or_create.return_value = (
MagicMock(), True)
self.manager.add_collaborator(
repo='reponame',
collaborator='new_collaborator',
db_privileges=['select'], file_privileges=['read', 'write'])
self.assertTrue(con_add_collab.called)
self.assertEqual(con_add_collab.call_args[1]['repo'], 'reponame')
self.assertEqual(
con_add_collab.call_args[1]['collaborator'], 'new_collaborator')
self.assertEqual(
con_add_collab.call_args[1]['db_privileges'], ['SELECT'])
def test_delete_collaborator(self):
self.mock_connection.return_value.list_collaborators.return_value = [
{'username': '******', 'permissions': 'U'},
{'username': self.username, 'permissions': 'UC'}
]
self.mock_User = self.create_patch('core.db.manager.User')
self.mock_Collaborator = self.create_patch(
'core.db.manager.Collaborator')
con_delete_collab = (self.mock_connection
.return_value.delete_collaborator)
self.manager.delete_collaborator('reponame', 'old_collaborator')
self.assertTrue(con_delete_collab.called)
self.assertEqual(con_delete_collab.call_args[1]['repo'], 'reponame')
self.assertEqual(
con_delete_collab.call_args[1]['collaborator'], 'old_collaborator')
def test_get_schema(self):
con_get_schema = self.mock_connection.return_value.get_schema
self.manager.get_schema('reponame', 'tablename')
self.assertTrue(con_get_schema.called)
self.assertEqual(con_get_schema.call_args[1]['repo'], 'reponame')
self.assertEqual(con_get_schema.call_args[1]['table'], 'tablename')
