def account_register(username, email, password, repo_name, app_id, app_token):
try:
user = User.objects.get(username=username)
raise Exception("Duplicate username (email=%s)" % (user.email))
except User.DoesNotExist:
pass
try:
user = User.objects.get(email=email)
raise Exception("Duplicate email (username=%s)" % (user.username))
except User.DoesNotExist:
pass
hashed_password = hashlib.sha1(password).hexdigest()
user = User(username=username, email=email, password=hashed_password)
user.save()
try:
DataHubManager.create_user(username=username, password=hashed_password)
account_grant_permission(username=username,
repo_name=repo_name,
app_id=app_id,
app_token=app_token)
except Exception, e:
user.delete()
raise e
def get_cache(username):
try:
manager = DataHubManager(user=username)
manager.execute_sql(
"create table _dbwipes_cache(key varchar, val text)")
except:
pass
def repo_tables(request, repo_base, repo):
'''
shows the tables under a repo.
'''
username = request.user.get_username()
# get the base tables and views of the user's repo
manager = DataHubManager(user=username, repo_base=repo_base)
try:
base_tables = manager.list_tables(repo)
views = manager.list_views(repo)
except LookupError:
base_tables = []
views = []
res = {
'login': username,
'repo_base': repo_base,
'repo': repo,
'base_tables': base_tables,
'views': views
}
res.update(csrf(request))
return render_to_response("repo-browse-tables.html", res)
def account_remove(username, app_id, app_token):
if not app_id:
raise Exception("Invalid app_id")
if not app_token:
raise Exception("Invalid app_token")
app = None
try:
app = App.objects.get(app_id=app_id)
except App.DoesNotExist:
raise Exception("Invalid app_id")
if app.app_token != app_token:
raise Exception("Invalid app_token")
app = App.objects.get(app_id=app_id)
if app.app_token != app_token:
raise Exception("Incorrect app token")
DataHubManager.remove_user(username=username)
user = User.objects.get(username=username)
user.delete()
def migrate_tables_and_views(apps, schema_editor):
# This shouldn't run during tests. If it does, list_all_users
# will return actual users, and then crash when it can't find them
# in django's test_db
if 'test' in sys.argv:
return
DataHubManager.execute_sql
all_users = DataHubManager.list_all_users()
# filter out the anonymous user, which doesn't have a db
all_users = [
username for username in all_users
if (username != settings.ANONYMOUS_ROLE)
]
# give users select/update/insert access to their rows in the policy table
for username in all_users:
try:
with DataHubManager(username) as m:
res = m.execute_sql(
"SELECT table_name FROM information_schema.tables "
"WHERE table_schema = 'public'")
tables_and_views = [table[0] for table in res['tuples']]
move_tables_to_new_schema(username, tables_and_views)
except (User.DoesNotExist, OperationalError):
pass
def user(request, repo_base=None):
username = request.user.get_username()
if not repo_base:
repo_base = username
manager = DataHubManager(user=username, repo_base=repo_base)
repos = manager.list_repos()
visible_repos = []
for repo in repos:
collaborators = manager.list_collaborators(repo)
collaborators = filter(lambda x: x != '' and x != repo_base,
collaborators)
visible_repos.append({
'name': repo,
'owner': repo_base,
'public': True if 'PUBLIC' in collaborators else False,
'collaborators': collaborators,
})
collaborator_repos = manager.list_collaborator_repos()
return render_to_response(
"user-browse.html", {
'login': username,
'repo_base': repo_base,
'repos': visible_repos,
'collaborator_repos': collaborator_repos
})
def repo_tables(request, repo_base, repo):
try:
login = get_login(request)
res = DataHubManager.has_repo_privilege(login, repo_base, repo, 'USAGE')
if not (res and res['tuples'][0][0]):
raise Exception('Access denied. Missing required privileges.')
manager = DataHubManager(user=repo_base)
# get base_tables for a given repo
res = manager.list_tables(repo)
base_tables = [t[0] for t in res['tuples']]
# get views for a given repo
res = manager.list_views(repo)
views = [t[0] for t in res['tuples']]
res = {
'login': get_login(request),
'repo_base': repo_base,
'repo': repo,
'base_tables': base_tables,
'views': views}
res.update(csrf(request))
return render_to_response("repo-browse-tables.html", res)
except Exception, e:
return HttpResponse(json.dumps(
{'error': str(e)}),
content_type="application/json")
def api_query(request):
ret = {}
jsonstr = request.GET.get("json", None)
if not jsonstr:
print "query: no json string. giving up"
return ret
args = json.loads(jsonstr)
username = args.get("username")
dbname = args.get("db")
table = args.get("table")
args["table"] = "%s.%s" % (dbname, table)
o, params = create_sql_obj(None, args)
o.limit = 10000
query = str(o)
print query
print params
if not dbname or not table or not query:
print "query: no db/table/query. giving up"
return ret
manager = DataHubManager(user=username)
res = manager.execute_sql(query, params)
rows = res["tuples"]
cols = pick(res["fields"], "name")
data = [dict(zip(cols, vals)) for vals in rows]
ret["data"] = data
ret["schema"] = get_schema(dbname, table, username)
print "%d points returned" % len(ret.get("data", []))
return ret
def create_user_db_and_data_folder_if_needed(sender, instance, **kwargs):
"""
Creates a Postgres role and db and data folder to go with new Django users.
Raises an exception if the role, database, or user data folder exists
before this user.
"""
username = instance.username
hashed_password = instance.password
# The Django user doesn't exist yet, so we can't just try to create a
# DataHubManager connection as the user. We need to act as the db
# superuser and check for any existing db role or database.
db_exists = DataHubManager.database_exists(username)
user_exists = DataHubManager.user_exists(username)
user_data_folder_exists = DataHubManager.user_data_folder_exists(username)
if db_exists and user_exists and user_data_folder_exists:
# Make sure new users don't inherit orphaned roles or databases that
# are missing a matching Django user.
try:
User.objects.get(username=username)
except User.DoesNotExist:
raise IntegrityError("Failed to create user. That name is already"
" in use by an orphaned user.")
elif not db_exists and not user_exists and not user_data_folder_exists:
try:
DataHubManager.create_user(username=username,
password=hashed_password)
except OperationalError:
raise
else:
raise Exception("Failed to create user. That name is already"
" in use by either a role, database, or data folder.")
def dbs(request):
username = request.user.get_username()
manager = DataHubManager(user=username)
dbnames = manager.list_repos()
# q = "SELECT datname FROM pg_database where datistemplate = false;"
# dbnames = [str(row[0]) for row in manager.execute_sql(query=q)['tuples']]
return {"databases": dbnames}
def setUp(self):
self.client = Client(enforce_csrf_checks=False)
# Need to mock out the authentication system here
# So that we aren't actually testing the auth/db systems, too
# ARC
# create the user
self.username = "******"
self.password = "******"
self.hashed_password = hashlib.sha1(self.password).hexdigest()
DataHubManager.create_user(username=self.username,
password=self.hashed_password)
user = User(username=self.username,
email="*****@*****.**",
password=self.hashed_password)
user.save()
# log the user in
login_credentials = {
'login_id': self.username,
'login_password': self.password
}
self.client.post('/account/login', login_credentials)
def delete_repo(self, con, repo_name, force_if_non_empty):
try:
'''
res = DataHubManager.has_base_privilege(
con.user, con.repo_base, 'CREATE')
if not (res and res['tuples'][0][0]):
raise Exception('Access denied. Missing required privileges.')
res = DataHubManager.has_repo_privilege(
con.user, con.repo_base, repo_name, 'CREATE')
if not (res and res['tuples'][0][0]):
raise Exception('Access denied. Missing required privileges.')
'''
manager = DataHubManager(user=con.user,
repo_base=con.repo_base,
is_app=con.is_app)
res = manager.delete_repo(repo=repo_name, force=force_if_non_empty)
thrift_crazy_result = {
'status': res,
'row_count': -1,
'tuples': [],
'fields': []
}
return construct_result_set(thrift_crazy_result)
except Exception as e:
raise DBException(message=str(e))
def card_delete(request, repo_base, repo, card_name):
username = request.user.get_username()
manager = DataHubManager(username, repo_base)
manager.delete_card(repo, card_name)
return HttpResponseRedirect(
reverse('browser-repo_cards', args=(repo_base, repo)))
def list_tables(self, con, repo_name):
try:
'''
res = DataHubManager.has_repo_privilege(
con.user, con.repo_base, repo_name, 'USAGE')
if not (res and res['tuples'][0][0]):
raise Exception('Access denied. Missing required privileges.')
'''
manager = DataHubManager(
user=con.user, repo_base=con.repo_base, is_app=con.is_app)
res = manager.list_tables(repo=repo_name)
# create the crazy thrift tuple, since this is for consumption from
# 3rd party apps, and they haven't upgraded to the new list_tables
# ARC 2015-12-15
tuple_list = zip(res)
length = len(res)
thrift_crazy_result = {
'status': True, 'row_count': length,
'tuples': tuple_list,
'fields': [{'type': 1043, 'name': 'table_name'}]
}
return construct_result_set(thrift_crazy_result)
except Exception, e:
raise DBException(message=str(e))
def create_user_db_and_data_folder_if_needed(sender, instance, **kwargs):
"""
Creates a Postgres role and db and data folder to go with new Django users.
Raises an exception if the role, database, or user data folder exists
before this user.
"""
username = instance.username
hashed_password = instance.password
# The Django user doesn't exist yet, so we can't just try to create a
# DataHubManager connection as the user. We need to act as the db
# superuser and check for any existing db role or database.
db_exists = DataHubManager.database_exists(username)
user_exists = DataHubManager.user_exists(username)
user_data_folder_exists = DataHubManager.user_data_folder_exists(username)
if db_exists and user_exists and user_data_folder_exists:
# Make sure new users don't inherit orphaned roles or databases that
# are missing a matching Django user.
try:
User.objects.get(username=username)
except User.DoesNotExist:
raise IntegrityError("Failed to create user. That name is already"
" in use by an orphaned user.")
elif not db_exists and not user_exists and not user_data_folder_exists:
try:
DataHubManager.create_user(
username=username,
password=hashed_password)
except OperationalError:
raise
else:
raise Exception("Failed to create user. That name is already"
" in use by either a role, database, or data folder.")
def account_remove (username, app_id, app_token):
if not app_id:
raise Exception("Invalid app_id")
if not app_token:
raise Exception("Invalid app_token")
app = None
try:
app = App.objects.get(app_id=app_id)
except App.DoesNotExist:
raise Exception("Invalid app_id")
if app.app_token != app_token:
raise Exception("Invalid app_token")
app = App.objects.get(app_id=app_id)
if app.app_token != app_token:
raise Exception("Incorrect app token")
DataHubManager.remove_user(username=username)
user = User.objects.get(username=username)
user.delete()
def test_has_repo_db_privilege(self):
m_has_db_priv = self.mock_connection.return_value.has_repo_db_privilege
m_has_db_priv.return_value = True
# Not raising a PermissionDenied exception when connection returns True
# means has_repo_db_privilege is behaving correctly.
DataHubManager.has_repo_db_privilege(
self.username, 'repo_base', 'repo', 'privilege')
def validate_unique_username(value):
"""
Validates that a proposed username is not already in use.
Checks User and App models, databases, database roles, and user data
folders.
"""
username = value.lower()
try:
existing_user = User.objects.get(username=username)
except User.DoesNotExist:
existing_user = None
try:
existing_app = App.objects.get(app_id=username)
except App.DoesNotExist:
existing_app = None
db_exists = DataHubManager.database_exists(username)
user_exists = DataHubManager.user_exists(username)
user_data_folder_exists = DataHubManager.user_data_folder_exists(username)
if (existing_user or existing_app or db_exists or user_exists
or user_data_folder_exists):
raise forms.ValidationError(
('The username %(value)s is not available.'),
params={'value': value},
)
return True
def account_register (username, email, password, repo_name, app_id, app_token):
try:
user = User.objects.get(username=username)
raise Exception("Duplicate username (email=%s)" %(user.email))
except User.DoesNotExist:
pass
try:
user = User.objects.get(email=email)
raise Exception("Duplicate email (username=%s)" %(user.username))
except User.DoesNotExist:
pass
hashed_password = hashlib.sha1(password).hexdigest()
user = User(username=username, email=email, password=hashed_password)
user.save()
try:
DataHubManager.create_user(username=username, password=hashed_password)
account_grant_permission(
username=username,
repo_name=repo_name,
app_id=app_id,
app_token=app_token)
except Exception, e:
user.delete()
raise e
def get_cache(username):
try:
manager = DataHubManager(user=username)
manager.execute_sql(
"create table _dbwipes_cache(key varchar, val text)")
except:
pass
def execute_sql(self, con, query, query_params=None):
try:
manager = DataHubManager(user=con.user, repo_base=con.repo_base)
res = manager.execute_sql(query=query, params=query_params)
return construct_result_set(res)
except Exception, e:
raise DBException(message=str(e))
def delete_repo(self, con, repo_name, force_if_non_empty):
try:
manager = DataHubManager(user=con.user, repo_base=con.repo_base)
res = manager.delete_repo(repo=repo_name, force=force_if_non_empty)
return construct_result_set(res)
except Exception, e:
raise DBException(message=str(e))
def list_tables(self, con, repo_name):
try:
manager = DataHubManager(user=con.user, repo_base=con.repo_base)
res = manager.list_tables(repo=repo_name)
return construct_result_set(res)
except Exception, e:
raise DBException(message=str(e))
def repo(request, repo_base, repo):
try:
login = get_login(request)
res = DataHubManager.has_repo_privilege(login, repo_base, repo, 'USAGE')
if not (res and res['tuples'][0][0]):
raise Exception('Access denied. Missing required privileges.')
manager = DataHubManager(user=repo_base)
res = manager.list_tables(repo)
tables = [t[0] for t in res['tuples']]
repo_dir = '/user_data/%s/%s' %(repo_base, repo)
if not os.path.exists(repo_dir):
os.makedirs(repo_dir)
uploaded_files = [f for f in os.listdir(repo_dir)]
res = {
'login': get_login(request),
'repo_base': repo_base,
'repo': repo,
'tables': tables,
'files': uploaded_files}
res.update(csrf(request))
return render_to_response("repo.html", res)
except Exception, e:
return HttpResponse(json.dumps(
{'error': str(e)}),
mimetype="application/json")
def table(request, repo_base, repo, table, page='1'):
try:
login = get_login(request)
dh_table_name = '%s.%s.%s' %(repo_base, repo, table)
res = DataHubManager.has_table_privilege(
login, repo_base, dh_table_name, 'SELECT')
if not (res and res['tuples'][0][0]):
raise Exception('Access denied. Missing required privileges.')
manager = DataHubManager(user=repo_base)
res = manager.execute_sql(
query='SELECT count(*) from %s' %(dh_table_name))
limit = 50
count = res['tuples'][0][0]
total_pages = 1 + (int(count) / limit)
current_page = 1
try:
current_page = int(page)
except:
pass
if current_page < 1:
current_page = 1
start_page = current_page - 5
if start_page < 1:
start_page = 1
end_page = start_page + 10
if end_page > total_pages:
end_page = total_pages
res = manager.execute_sql(
query='SELECT * from %s LIMIT %s OFFSET %s'
%(dh_table_name, limit, (current_page -1) * limit))
column_names = [field['name'] for field in res['fields']]
tuples = res['tuples']
return render_to_response("table.html", {
'login': get_login(request),
'repo_base': repo_base,
'repo': repo,
'table': table,
'column_names': column_names,
'tuples': tuples,
'current_page': current_page,
'next_page': current_page + 1,
'prev_page': current_page - 1,
'total_pages': total_pages,
'pages': range(start_page, end_page + 1)})
except Exception, e:
return HttpResponse(json.dumps(
{'error': str(e)}),
mimetype="application/json")
def card_export(request, repo_base, repo, card_name):
try:
login = get_login(request)
card = Card.objects.get(repo_base=repo_base,
repo_name=repo,
card_name=card_name)
query = card.query
res = DataHubManager.has_repo_privilege(login, repo_base, repo,
'CREATE')
if not (res and res['tuples'][0][0]):
raise Exception('Access denied. Missing required privileges.')
repo_dir = '/user_data/%s/%s' % (repo_base, repo)
if not os.path.exists(repo_dir):
os.makedirs(repo_dir)
file_path = '%s/%s.csv' % (repo_dir, card_name)
DataHubManager.export_query(repo_base=repo_base,
query=query,
file_path=file_path)
return HttpResponseRedirect('/browse/%s/%s/files' % (repo_base, repo))
except Exception, e:
return HttpResponse(json.dumps({'error': str(e)}),
content_type="application/json")
def repo_settings(request, repo_base, repo):
try:
login = get_login(request)
res = DataHubManager.has_repo_privilege(login, repo_base, repo,
'CREATE')
if not (res and res['tuples'][0][0]):
raise Exception('Access denied. Missing required privileges.')
manager = DataHubManager(user=repo_base)
res = manager.list_collaborators(repo_base, repo)
collaborators = [(c[0].split('=')[0]).strip() for c in res['tuples']]
collaborators = filter(lambda x: x != '' and x != repo_base,
collaborators)
res = {
'login': get_login(request),
'repo_base': repo_base,
'repo': repo,
'collaborators': collaborators
}
res.update(csrf(request))
return render_to_response("repo-settings.html", res)
except Exception, e:
return HttpResponse(json.dumps({'error': str(e)}),
content_type="application/json")
def repo_tables(request, repo_base, repo):
try:
login = get_login(request)
res = DataHubManager.has_repo_privilege(login, repo_base, repo,
'USAGE')
if not (res and res['tuples'][0][0]):
raise Exception('Access denied. Missing required privileges.')
manager = DataHubManager(user=repo_base)
# get base_tables for a given repo
res = manager.list_tables(repo)
base_tables = [t[0] for t in res['tuples']]
# get views for a given repo
res = manager.list_views(repo)
views = [t[0] for t in res['tuples']]
res = {
'login': get_login(request),
'repo_base': repo_base,
'repo': repo,
'base_tables': base_tables,
'views': views
}
res.update(csrf(request))
return render_to_response("repo-browse-tables.html", res)
except Exception, e:
return HttpResponse(json.dumps({'error': str(e)}),
content_type="application/json")
def list_tables(self, con, repo_name):
try:
'''
res = DataHubManager.has_repo_privilege(
con.user, con.repo_base, repo_name, 'USAGE')
if not (res and res['tuples'][0][0]):
raise Exception('Access denied. Missing required privileges.')
'''
manager = DataHubManager(user=con.user,
repo_base=con.repo_base,
is_app=con.is_app)
res = manager.list_tables(repo=repo_name)
# create the crazy thrift tuple, since this is for consumption from
# 3rd party apps, and they haven't upgraded to the new list_tables
# ARC 2015-12-15
tuple_list = zip(res)
length = len(res)
thrift_crazy_result = {
'status': True,
'row_count': length,
'tuples': tuple_list,
'fields': [{
'type': 1043,
'name': 'table_name'
}]
}
return construct_result_set(thrift_crazy_result)
except Exception as e:
raise DBException(message=str(e))
def get_schema(self, con, table_name):
try:
manager = DataHubManager(user=con.user, repo_base=con.repo_base)
res = manager.get_schema(table=table_name)
return construct_result_set(res)
except Exception, e:
raise DBException(message=str(e))
def repo_settings(request, repo_base, repo):
try:
login = get_login(request)
res = DataHubManager.has_repo_privilege(login, repo_base, repo, 'CREATE')
if not (res and res['tuples'][0][0]):
raise Exception('Access denied. Missing required privileges.')
manager = DataHubManager(user=repo_base)
res = manager.list_collaborators(repo_base, repo)
collaborators = [(c[0].split('=')[0]).strip() for c in res['tuples']]
collaborators = filter(lambda x: x!='' and x!=repo_base, collaborators)
res = {
'login': get_login(request),
'repo_base': repo_base,
'repo': repo,
'collaborators': collaborators}
res.update(csrf(request))
return render_to_response("repo-settings.html", res)
except Exception, e:
return HttpResponse(json.dumps(
{'error': str(e)}),
content_type="application/json")
def test_has_repo_db_privilege(self):
m_has_db_priv = self.mock_connection.return_value.has_repo_db_privilege
m_has_db_priv.return_value = True
# Not raising a PermissionDenied exception when connection returns True
# means has_repo_db_privilege is behaving correctly.
DataHubManager.has_repo_db_privilege(
self.username, 'repo_base', 'repo', 'privilege')
def test_register_user_manually_sign_in_and_delete(self):
self.browser.get(self.server_url)
self.browser.set_window_size(1024, 768)
# Justin clicks "Sign Up"
self.browser.find_element_by_id('id_sign_up')
# Justin registers a new account
self.sign_up_manually()
# The URL bar now now shows Justin's username
justin_url = self.browser.current_url
self.assertRegexpMatches(justin_url, self.username)
# Justin clicks on the menu item with his name
self.browser.find_element_by_id('id_user_menu').click()
# Justin signs out
self.browser.find_element_by_id('id_sign_out').click()
# The URL bar now shows logout
justin_url = self.browser.current_url
self.assertRegexpMatches(justin_url, 'logout')
# Justin is able to sign back in
self.sign_in_manually()
justin_url = self.browser.current_url
self.assertRegexpMatches(justin_url, self.username)
# DataHub deletes his user and database, somewhat vindictively
DataHubManager.remove_user_and_database(self.username)
def user(request, repo_base=None):
username = request.user.get_username()
if not repo_base:
repo_base = username
manager = DataHubManager(user=username, repo_base=repo_base)
repos = manager.list_repos()
visible_repos = []
for repo in repos:
collaborators = manager.list_collaborators(repo)
collaborators = filter(
lambda x: x != '' and x != repo_base, collaborators)
visible_repos.append({
'name': repo,
'owner': repo_base,
'public': True if 'PUBLIC' in collaborators else False,
'collaborators': collaborators,
})
collaborator_repos = manager.list_collaborator_repos()
return render_to_response("user-browse.html", {
'login': username,
'repo_base': repo_base,
'repos': visible_repos,
'collaborator_repos': collaborator_repos})
def list_repos(self, con):
try:
'''
res = DataHubManager.has_base_privilege(
con.user, con.repo_base, 'CONNECT')
if not (res and res['tuples'][0][0]):
raise Exception('Access denied. Missing required privileges.')
'''
manager = DataHubManager(user=con.user,
repo_base=con.repo_base,
is_app=con.is_app)
res = manager.list_repos()
# prepare dumb thrift stuff
tuple_list = zip(res)
length = len(res)
thrift_crazy_result = {
'status': True,
'row_count': length,
'tuples': tuple_list,
'fields': [{
'type': 1043,
'name': 'repo_name'
}]
}
return construct_result_set(thrift_crazy_result)
except Exception as e:
raise DBException(message=str(e))
def api_query(request):
ret = {}
jsonstr = request.GET.get('json', None)
if not jsonstr:
print "query: no json string. giving up"
return ret
args = json.loads(jsonstr)
username = args.get('username')
dbname = args.get('db')
table = args.get('table')
args['table'] = "%s.%s" % (dbname, table)
o, params = create_sql_obj(None, args)
o.limit = 10000
query = str(o)
print query
print params
if not dbname or not table or not query:
print "query: no db/table/query. giving up"
return ret
manager = DataHubManager(user=username)
res = manager.execute_sql(query, params)
rows = res['tuples']
cols = pick(res['fields'], 'name')
data = [dict(zip(cols, vals)) for vals in rows]
ret['data'] = data
ret['schema'] = get_schema(dbname, table, username)
print "%d points returned" % len(ret.get('data', []))
return ret
def card_delete(request, repo_base, repo, card_name):
username = request.user.get_username()
manager = DataHubManager(username, repo_base)
manager.delete_card(repo, card_name)
return HttpResponseRedirect(
reverse('browser-repo_cards', args=(repo_base, repo)))
def test_register_user_manually_sign_in_and_delete(self):
self.browser.get(self.server_url)
self.browser.set_window_size(1024, 768)
# Justin clicks "Sign Up"
self.browser.find_element_by_id('id_sign_up')
# Justin registers a new account
self.sign_up_manually()
# The URL bar now now shows Justin's username
justin_url = self.browser.current_url
self.assertRegexpMatches(justin_url, self.username)
# Justin clicks on the menu item with his name
self.browser.find_element_by_id('id_user_menu').click()
# Justin signs out
self.browser.find_element_by_id('id_sign_out').click()
# The URL bar now shows logout
justin_url = self.browser.current_url
self.assertRegexpMatches(justin_url, 'logout')
# Justin is able to sign back in
self.sign_in_manually()
justin_url = self.browser.current_url
self.assertRegexpMatches(justin_url, self.username)
# DataHub deletes his user and database, somewhat vindictively
DataHubManager.remove_user_and_database(self.username)
def grant_app_permission(username, repo_name, app_id, app_token):
"""
Grants SELECT, INSERT, UPDATE, and DELETE on given user's repo to app.
Raises exceptions on empty input, if no app matches app_id, if app_token
doesn't match, or if there are any database errors.
"""
if not app_id:
raise Exception("Invalid app_id")
if not app_token:
raise Exception("Invalid app_token")
app = None
try:
app = App.objects.get(app_id=app_id)
except App.DoesNotExist:
raise Exception("Invalid app_id")
if app.app_token != app_token:
raise Exception("Invalid app_token")
try:
manager = DataHubManager(user=username)
manager.create_repo(repo_name)
manager.add_collaborator(
repo_name,
app_id,
privileges=['SELECT', 'INSERT', 'UPDATE', 'DELETE'])
except Exception as e:
raise e
def dbs(request):
username = get_login(request)
manager = DataHubManager(user=username)
dbnames = pick(manager.list_repos()['tuples'], 0)
#q = "SELECT datname FROM pg_database where datistemplate = false;"
#dbnames = [str(row[0]) for row in manager.execute_sql(query=q)['tuples']]
return {'databases': dbnames}
def api_query(request):
ret = { }
jsonstr = request.GET.get('json', None)
if not jsonstr:
print "query: no json string. giving up"
return ret
args = json.loads(jsonstr)
username = args.get('username')
dbname = args.get('db')
table = args.get('table')
args['table'] = "%s.%s" % (dbname, table)
o, params = create_sql_obj(None, args)
o.limit = 10000;
query = str(o)
print query
print params
if not dbname or not table or not query:
print "query: no db/table/query. giving up"
return ret
manager = DataHubManager(user=username)
res = manager.execute_sql(query, params)
rows = res['tuples']
cols = pick(res['fields'], 'name')
data = [dict(zip(cols, vals)) for vals in rows]
ret['data'] = data
ret['schema'] = get_schema(dbname, table, username)
print "%d points returned" % len(ret.get('data', []))
return ret
def app_register(request):
username = request.user.get_username()
if request.method == "POST":
try:
user = User.objects.get(username=username)
app_id = request.POST["app-id"].lower()
app_name = request.POST["app-name"]
app_token = str(uuid.uuid4())
app = App(app_id=app_id,
app_name=app_name,
user=user,
app_token=app_token)
app.save()
try:
hashed_password = hashlib.sha1(app_token).hexdigest()
DataHubManager.create_user(username=app_id,
password=hashed_password,
create_db=False)
except Exception as e:
app.delete()
raise e
return HttpResponseRedirect('/developer/apps')
except Exception as e:
c = {'login': username, 'errors': [str(e)]}
c.update(csrf(request))
return render_to_response('app-create.html', c)
else:
c = {'login': username}
c.update(csrf(request))
return render_to_response('app-create.html', c)
def validate_unique_username(value):
"""
Validates that a proposed username is not already in use.
Checks User and App models, databases, database roles, and user data
folders.
"""
username = value.lower()
try:
existing_user = User.objects.get(username=username)
except User.DoesNotExist:
existing_user = None
try:
existing_app = App.objects.get(app_id=username)
except App.DoesNotExist:
existing_app = None
db_exists = DataHubManager.database_exists(username)
user_exists = DataHubManager.user_exists(username)
user_data_folder_exists = DataHubManager.user_data_folder_exists(username)
if (existing_user or existing_app or
db_exists or user_exists or
user_data_folder_exists):
raise forms.ValidationError(
('The username %(value)s is not available.'),
params={'value': value},
)
return True
def file_import(request, repo_base, repo, file_name):
username = request.user.get_username()
delimiter = str(request.GET['delimiter'])
if delimiter == '':
delimiter = str(request.GET['other_delimiter'])
header = False
if request.GET['has_header'] == 'true':
header = True
quote_character = request.GET['quote_character']
if quote_character == '':
quote_character = request.GET['other_quote_character']
DataHubManager.import_file(
username=username,
repo_base=repo_base,
repo=repo,
table=table,
file_name=file_name,
delimiter=delimiter,
header=header,
quote_character=quote_character)
return HttpResponseRedirect(
reverse('browser-repo', args=(repo_base, repo)))
def file_import(request, repo_base, repo, file_name):
username = request.user.get_username()
delimiter = str(request.GET['delimiter'])
if delimiter == '':
delimiter = str(request.GET['other_delimiter'])
header = False
if request.GET['has_header'] == 'true':
header = True
quote_character = request.GET['quote_character']
if quote_character == '':
quote_character = request.GET['other_quote_character']
DataHubManager.import_file(username=username,
repo_base=repo_base,
repo=repo,
table=table,
file_name=file_name,
delimiter=delimiter,
header=header,
quote_character=quote_character)
return HttpResponseRedirect(reverse('browser-repo',
args=(repo_base, repo)))
def dbs(request):
username = get_login(request)
manager = DataHubManager(user=username)
dbnames = pick(manager.list_repos()['tuples'], 0)
#q = "SELECT datname FROM pg_database where datistemplate = false;"
#dbnames = [str(row[0]) for row in manager.execute_sql(query=q)['tuples']]
return {'databases': dbnames}
def app_register (request):
login = get_login(request)
if request.method == "POST":
try:
user = User.objects.get(username=login)
app_id = request.POST["app-id"].lower()
app_name = request.POST["app-name"]
app_token = str(uuid.uuid4())
app = App(
app_id=app_id, app_name=app_name, user=user, app_token=app_token)
app.save()
try:
hashed_password = hashlib.sha1(app_token).hexdigest()
DataHubManager.create_user(
username=app_id, password=hashed_password, create_db=False)
except Exception, e:
app.delete()
raise e
return HttpResponseRedirect('/developer/apps')
except Exception, e:
c = {
'login': login,
'errors': [str(e)]}
c.update(csrf(request))
return render_to_response('app-create.html', c)
def get_schema(self, con, table_name):
# this should really accept repo and table name as seperate
try:
manager = DataHubManager(user=con.user,
repo_base=con.repo_base,
is_app=con.is_app)
# thrift isn't upgraded to use seperate repo and tokens yet,
# so they have to be split here, and then passed to the manager.
tokens = table_name.split('.')
repo = tokens[0]
table = tokens[1]
res = manager.get_schema(repo=repo, table=table)
# apps expect get_schema to return in a backwards way. It's made
# to be backwards here.
thrift_crazy_result = {
'status':
True,
'row_count':
0,
'tuples':
res,
'fields': [{
'type': 1043,
'name': 'column_name'
}, {
'type': 1043,
'name': 'data_type'
}]
}
return construct_result_set(thrift_crazy_result)
except Exception as e:
raise DBException(message=str(e))
def execute_sql(self, con, query, query_params=None):
try:
manager = DataHubManager(user=con.repo_base,
repo_base=con.repo_base)
res = manager.execute_sql(query=query, params=query_params)
return construct_result_set(res)
except Exception, e:
raise DBException(message=str(e))
def file_upload(request, repo_base, repo):
username = request.user.get_username()
data_file = request.FILES['data_file']
manager = DataHubManager(username, repo_base)
manager.save_file(repo, data_file)
return HttpResponseRedirect(
reverse('browser-repo_files', args=(repo_base, repo)))
def table_export(request, repo_base, repo, table_name):
username = request.user.get_username()
DataHubManager.export_table(
username=username, repo_base=repo_base, repo=repo, table=table_name,
file_format='CSV', delimiter=',', header=True)
return HttpResponseRedirect(
reverse('browser-repo_files', args=(repo_base, repo)))
def test_delete_repo(self):
con_delete_repo = self.mock_connection.return_value.delete_repo
self.manager = DataHubManager(
user=self.username, repo_base=self.username)
self.manager.delete_repo('repo')
self.assertTrue(con_delete_repo.called)
self.assertEqual(con_delete_repo.call_args[1]['force'], False)
def get_schema(self, con, table_name):
try:
manager = DataHubManager(user=con.repo_base,
repo_base=con.repo_base)
res = manager.get_schema(table=table_name)
return construct_result_set(res)
except Exception, e:
raise DBException(message=str(e))
def file_upload(request, repo_base, repo):
username = request.user.get_username()
data_file = request.FILES['data_file']
manager = DataHubManager(username, repo_base)
manager.save_file(repo, data_file)
return HttpResponseRedirect(
reverse('browser-repo_files', args=(repo_base, repo)))
def repo_delete(request, repo_base, repo):
'''
deletes a repo in the current database (repo_base)
'''
username = request.user.get_username()
manager = DataHubManager(user=username, repo_base=repo_base)
manager.delete_repo(repo=repo, force=True)
return HttpResponseRedirect(reverse('browser-user-default'))
def file_download(request, repo_base, repo, file_name):
username = request.user.get_username()
manager = DataHubManager(username, repo_base)
file_to_download = manager.get_file(repo, file_name)
response = HttpResponse(file_to_download,
content_type='application/force-download')
response['Content-Disposition'] = 'attachment; filename="%s"' % (file_name)
return response
def file_download(request, repo_base, repo, file_name):
username = request.user.get_username()
manager = DataHubManager(username, repo_base)
file_to_download = manager.get_file(repo, file_name)
response = HttpResponse(file_to_download,
content_type='application/force-download')
response['Content-Disposition'] = 'attachment; filename="%s"' % (file_name)
return response
def reset(request, encrypted_email):
errors = []
error = False
if request.method == "POST":
try:
user_email = request.POST["user_email"].lower()
password = request.POST["new_password"]
password2 = request.POST["new_password2"]
if password == "":
errors.append("Empty Password.")
error = True
if password2 != password:
errors.append("Password and Confirm Password don't match.")
error = True
if not error:
hashed_password = hashlib.sha1(password).hexdigest()
user = User.objects.get(email=user_email)
try:
DataHubManager.create_user(username=user.username,
password=hashed_password)
except Exception, e:
pass
try:
DataHubManager.change_password(username=user.username,
password=hashed_password)
except Exception, e:
errors.append(str(e))
error = True
if error:
c = {
'user_email': user_email,
'encrypted_email': encrypted_email,
'errors': errors
}
c.update(csrf(request))
return render_to_response('reset.html', c)
else:
hashed_password = hashlib.sha1(password).hexdigest()
user = User.objects.get(email=user_email)
user.password = hashed_password
user.save()
c = {
'msg_title':
'DataHub Reset Password',
'msg_body':
'Your password has been changed successfully.<br /> <br />'
'<a href="/account/login" class="blue bold">Click Here</a>'
' to sign in.'
}
c.update(csrf(request))
return render_to_response('confirmation.html', c)
def repo_delete(request, repo_base, repo):
'''
deletes a repo in the current database (repo_base)
'''
username = request.user.get_username()
manager = DataHubManager(user=username, repo_base=repo_base)
manager.delete_repo(repo=repo, force=True)
return HttpResponseRedirect(reverse('browser-user-default'))
def card_create(request, repo_base, repo):
username = request.user.get_username()
card_name = request.POST['card-name']
query = request.POST['query']
url = reverse('browser-card', args=(repo_base, repo, card_name))
manager = DataHubManager(username, repo_base)
manager.create_card(repo, query, card_name)
return HttpResponseRedirect(url)
