def tables(request, repo_name):
username = request.user.get_username()
manager = DataHubManager(username)
repos = get_repos(manager)
if repos is not None and repo_name in repos:
tables = get_tables(manager, repo_name)
if tables is not None:
return json_response({"tables": tables})
return error_response()
def execute_sql(self, con, query, query_params=None):
try:
manager = DataHubManager(user=con.user,
repo_base=con.repo_base,
is_app=con.is_app)
res = manager.execute_sql(query=query, params=query_params)
return construct_result_set(res)
except Exception, e:
raise DBException(message=str(e))
def file_download(request, repo_base, repo, file_name):
username = request.user.get_username()
manager = DataHubManager(username, repo_base)
file_to_download = manager.get_file(repo, file_name)
response = HttpResponse(file_to_download,
content_type='application/force-download')
response['Content-Disposition'] = 'attachment; filename="%s"' % (file_name)
return response
def card_export(request, repo_base, repo, card_name):
username = request.user.get_username()
file_name = request.GET.get('var_text', card_name)
with DataHubManager(user=username, repo_base=repo_base) as manager:
manager.export_card(repo, file_name, card_name)
return HttpResponseRedirect(
reverse('browser-repo_files', args=(repo_base, repo)))
def repo_delete(request, repo_base, repo):
'''
deletes a repo in the current database (repo_base)
'''
username = request.user.get_username()
manager = DataHubManager(user=username, repo_base=repo_base)
manager.delete_repo(repo=repo, force=True)
return HttpResponseRedirect(reverse('browser-user-default'))
def table_clone(request, repo_base, repo, table):
username = request.user.get_username()
new_table = request.GET.get('var_text', None)
with DataHubManager(user=username, repo_base=repo_base) as manager:
manager.clone_table(repo, table, new_table)
return HttpResponseRedirect(
reverse('browser-repo_tables', args=(repo_base, repo)))
def file_upload(request, repo_base, repo):
username = request.user.get_username()
data_file = request.FILES['data_file']
with DataHubManager(user=username, repo_base=repo_base) as manager:
manager.save_file(repo, data_file)
return HttpResponseRedirect(
reverse('browser-repo_files', args=(repo_base, repo)))
def file_import(request, repo_base, repo, file_name):
try:
login = get_login(request)
res = DataHubManager.has_repo_privilege(login, repo_base, repo,
'CREATE')
if not (res and res['tuples'][0][0]):
raise Exception('Access denied. Missing required privileges.')
delimiter = str(request.GET['delimiter'])
if delimiter == '':
delimiter = str(request.GET['other_delimiter'])
header = True if request.GET['has_header'] == "true" else False
quote_character = request.GET['quote_character']
if quote_character == '':
quote_character = request.GET['other_quote_character']
delimiter = delimiter.decode('string_escape')
repo_dir = '/user_data/%s/%s' % (repo_base, repo)
file_path = '%s/%s' % (repo_dir, file_name)
table_name, _ = os.path.splitext(file_name)
table_name = clean_str(table_name, 'table')
dh_table_name = '%s.%s.%s' % (repo_base, repo, table_name)
f = codecs.open(file_path, 'r', 'ISO-8859-1')
data = csv.reader(f, delimiter=delimiter)
cells = data.next()
columns = [clean_str(str(i), 'col') for i in range(0, len(cells))]
if header:
columns = map(lambda x: clean_str(x, 'col'), cells)
columns = rename_duplicates(columns)
query = 'CREATE TABLE %s (%s text' % (dh_table_name, columns[0])
for i in range(1, len(columns)):
query += ', %s %s' % (columns[i], 'text')
query += ')'
manager = DataHubManager(user=repo_base)
manager.execute_sql(query=query)
manager.import_file(repo_base=repo_base,
table_name=dh_table_name,
file_path=file_path,
delimiter=delimiter,
header=header,
quote_character=quote_character)
return HttpResponseRedirect('/browse/%s/%s' % (repo_base, repo))
except Exception, e:
return HttpResponse(json.dumps({'error': str(e)}),
content_type="application/json")
def tables(request):
""" Returns the tables in a user's repo.
I think this is only used for (disabled) scorpion features.
"""
username = request.user.get_username()
repo_base = request.GET.get('username', None)
manager = DataHubManager(username, repo_base)
repo = request.GET.get('repo')
tables = manager.list_tables(repo)
return {'tables': tables}
def card_create(request, repo_base, repo):
username = request.user.get_username()
card_name = request.POST['card-name']
query = request.POST['query']
url = reverse('browser-card', args=(repo_base, repo, card_name))
with DataHubManager(user=username, repo_base=repo_base) as manager:
manager.create_card(repo, card_name, query)
return HttpResponseRedirect(url)
def move_tables_to_new_schema(username, tables_and_views):
# if they do have tables/views
if len(tables_and_views) > 0:
repo_name = create_and_return_name_for_public_schema(username)
for table in tables_and_views:
with DataHubManager(username) as m:
query = ('Alter table public.%s '
'set schema %s') % (table, repo_name)
m.execute_sql(query)
def test_rename_repo(self):
con_rename_repo = self.mock_connection.return_value.rename_repo
# self.mock_Collaborator = self.create_patch(
# 'inventory.models.Collaborator')
self.manager = DataHubManager(
user=self.username, repo_base=self.username)
self.manager.Collaborator = MagicMock
self.manager.rename_repo('old_name', 'new_name')
self.assertTrue(con_rename_repo.called)
def schema(request, repo, table):
username = request.user.get_username()
manager = DataHubManager(username)
repos = get_repos(manager)
if repos is not None and repo in repos:
tables = get_tables(manager, repo)
if tables is not None and table in tables:
schema = manager.get_schema(repo, table)
if len(schema) > 0:
return json_response({"schema": schema})
return error_response()
def table_export(request, repo_base, repo, table_name):
username = request.user.get_username()
file_name = request.GET.get('var_text', table_name)
with DataHubManager(user=username, repo_base=repo_base) as manager:
manager.export_table(
repo=repo, table=table_name, file_name=file_name,
file_format='CSV', delimiter=',', header=True)
return HttpResponseRedirect(
reverse('browser-repo_files', args=(repo_base, repo)))
def repos(request):
""" return the repos in the user's database.
I think this is only used for (disabled) scorpion features.
"""
username = request.user.get_username()
repo_base = request.GET.get('username', None)
manager = DataHubManager(username, repo_base)
dbnames = manager.list_repos()
# q = "SELECT datname FROM pg_database where datistemplate = false;"
# dbnames = [str(row[0]) for row in manager.execute_sql(query=q)['tuples']]
return {'databases': dbnames}
def setUp(self):
self.username = "******"
self.password = "******"
self.email = "*****@*****.**"
self.user = User.objects.create_user(
self.username, self.email, self.password)
self.mock_connection = self.create_patch(
'core.db.manager.DataHubConnection')
self.manager = DataHubManager(user=self.username)
def schema(request, repo, table):
username = get_login(request)
manager = DataHubManager(username)
repos = get_repos(manager)
if repos is not None and repo in repos:
tables = get_tables(manager, repo)
if tables is not None and table in tables:
schema = manager.get_schema(repo + "." + table)
if schema is not None and 'tuples' in schema:
return json_response({"schema": schema["tuples"]})
return error_response()
def license_view_create(request, repo_base, repo, table, license_id):
'''
creates a new license view for a given table and license_id
'''
username = request.user.get_username()
public_role = settings.PUBLIC_ROLE
with DataHubManager(user=username, repo_base=repo_base) as manager:
collaborators = manager.list_collaborators(repo)
if username != repo_base:
raise PermissionDenied("User does not have access to this repo")
if request.method == 'POST':
# collect parameters that will be used to create the view of the table
removed_columns = request.POST.getlist('removed_columns[]')
view_params = {}
view_params['removed-columns'] = removed_columns
with DataHubManager(user=username, repo_base=repo_base) as manager:
manager.create_license_view(
repo=repo,
table=table,
view_params=view_params,
license_id=license_id)
return HttpResponseRedirect(
reverse('browser-repo_licenses', args=(repo_base, repo)))
elif request.method == 'GET':
collaborators = [c for c in collaborators if c['username']
not in ['', username, settings.PUBLIC_ROLE]]
res = {
'login': username,
'repo_base': repo_base,
'repo': repo,
'collaborators': collaborators,
'public_role': public_role}
res.update(csrf(request))
return render_to_response("license-create.html", res)
def query(request, repo_base, repo):
query = post_or_get(request, key='q', fallback=None)
username = request.user.get_username()
# if the user is just requesting the query page
if not query:
data = {
'login': username,
'repo_base': repo_base,
'repo': repo,
'select_query': False,
'query': None
}
return render_to_response("query.html", data)
# if the user is actually executing a query
current_page = 1
if request.POST.get('page'):
current_page = request.POST.get('page')
url_path = reverse('browser-query', args=(repo_base, repo))
with DataHubManager(user=username, repo_base=repo_base) as manager:
res = manager.paginate_query(query=query,
current_page=current_page,
rows_per_page=50)
# get annotation to the table:
annotation, created = Annotation.objects.get_or_create(url_path=url_path)
annotation_text = annotation.annotation_text
data = {
'login': username,
'repo_base': repo_base,
'repo': repo,
'annotation': annotation_text,
'current_page': current_page,
'next_page': current_page + 1, # the template should relaly do this
'prev_page': current_page - 1, # the template should relaly do this
'url_path': url_path,
'query': query,
'select_query': res['select_query'],
'column_names': res['column_names'],
'tuples': res['rows'],
'total_pages': res['total_pages'],
'pages': range(res['start_page'], res['end_page'] + 1), # template
'num_rows': res['num_rows'],
'time_cost': res['time_cost']
}
data.update(csrf(request))
return render_to_response("query-browse-results.html", data)
def list_repos(self, con):
try:
'''
res = DataHubManager.has_base_privilege(con.user, con.repo_base, 'CONNECT')
if not (res and res['tuples'][0][0]):
raise Exception('Access denied. Missing required privileges.')
'''
manager = DataHubManager(user=con.repo_base,
repo_base=con.repo_base)
res = manager.list_repos()
return construct_result_set(res)
except Exception, e:
raise DBException(message=str(e))
def index(request):
# if the user is authenticated, we pass them the available repos
username = request.user.get_username()
res = {'login': username}
try:
with DataHubManager(username) as m:
repos = m.list_repos()
res['repos'] = repos
except:
pass
res.update(csrf(request))
return render_to_response("refiner.html", res)
def test_create_repo_creates_a_repo(self):
# create the new repo
post_object = {'repo': 'repo_name'}
response = self.client.post('/create/' + self.username + '/repo',
post_object)
# get a list of repos that the user owns
manager = DataHubManager(user=self.username)
res = manager.list_repos()
repos = [t[0] for t in res['tuples']]
# make sure that it's in there
self.assertTrue('repo_name' in repos)
def card_update_public(request, repo_base, repo, card_name):
username = request.user.get_username()
if 'public' in request.POST:
public = request.POST['public'] == 'True'
else:
raise ValueError("Request missing \'public\' parameter.")
with DataHubManager(user=username, repo_base=repo_base) as manager:
manager.update_card(repo, card_name, public=public)
return HttpResponseRedirect(
reverse('browser-card', args=(repo_base, repo, card_name)))
def _f(self, *args, **kwargs):
try:
key = str(
map(str, (f.__name__, self.dbname, self.tablename, self.where,
self.nbuckets, map(str, args))))
manager = DataHubManager(user=self.username)
vals = manager.execute_sql(
'select val from _dbwipes_cache where key = %s',
params=(key, ))['tuples']
if len(vals):
return json.loads(vals[0][0])
except Exception as e:
print e
pdb.set_trace()
res = f(self, *args, **kwargs)
if key:
manager = DataHubManager(user=self.username)
manager.execute_sql('insert into _dbwipes_cache values(%s, %s)',
params=(key,
json.dumps(res, default=json_handler)))
return res
def table_delete(request, repo_base, repo, table_name):
"""
Deletes the given table.
Does not currently allow the user the option to cascade in the case of
dependencies, though the delete_table method does allow cascade (force) to
be passed.
"""
username = request.user.get_username()
manager = DataHubManager(user=username, repo_base=repo_base)
manager.delete_table(repo, table_name)
return HttpResponseRedirect(
reverse('browser-repo_tables', args=(repo_base, repo)))
def handle(self, *args, **options):
# Throw out all of these changes if it fails somehow
with transaction.atomic():
for base in (user.get_username() for user in User.objects.all()):
try:
manager = DataHubManager(user=base, repo_base=base)
except OperationalError as e:
print(e)
continue
repos = manager.list_repos()
for repo in repos:
self._sync_repo_collaborators(manager, base, repo)
print("Added missing {0} users, {1} missing apps.".format(
self.user_count, self.app_count))
def test_initialization(self):
DataHubManager(user=self.username)
# username passed
self.assertEqual(
self.mock_connection.call_args[1]['user'], self.username)
# password passed
self.assertTrue(
self.mock_connection.call_args[1]['password'] is not None)
# repo defaults to username
self.assertEqual(
self.mock_connection.call_args[1]['repo_base'], self.username)
def _create_table_helper(self, username, repo, cases, tables_after):
"""
Tries to create a table in a repo as a user.
Cases should be an array of test case tuples, where each tuple is the
Exception to be raised or None plus the arguments to be passed to
create_table.
"""
with DataHubManager(user=username, repo_base=self.owner_username) as m:
self.assertEquals([], m.list_tables(repo))
for c in cases:
with self._assertRaisesOrNone(c[0], message=c):
m.create_table(*c[1:])
self.assertEquals(tables_after, m.list_tables(repo))
def license_view_delete(request, repo_base, repo, table,
license_view, license_id):
'''
Deletes license view for table and given license_id
'''
username = request.user.get_username()
public_role = settings.PUBLIC_ROLE
with DataHubManager(user=username, repo_base=repo_base) as manager:
collaborators = manager.list_collaborators(repo)
if username != repo_base:
raise PermissionDenied("User does not have access to this repo")
with DataHubManager(user=username, repo_base=repo_base) as manager:
manager.delete_license_view(
repo=repo,
table=table,
license_view=license_view,
license_id=license_id)
return HttpResponseRedirect(reverse('browser-repo_licenses',
args=(repo_base, repo)))
def user(request, repo_base):
try:
login = get_login(request)
res = DataHubManager.has_base_privilege(login, repo_base, 'CONNECT')
if not (res and res['tuples'][0][0]):
raise Exception('Access denied. Missing required privileges.')
manager = DataHubManager(user=repo_base)
res = manager.list_repos()
repos = [t[0] for t in res['tuples']]
visible_repos = []
for repo in repos:
res = manager.list_collaborators(repo_base, repo)
collaborators = [(c[0].split('=')[0]).strip()
for c in res['tuples']]
collaborators = filter(lambda x: x != '' and x != repo_base,
collaborators)
if login not in collaborators and login != repo_base:
continue
visible_repos.append({
'name':
repo,
'owner':
repo_base,
'public':
True if 'PUBLIC' in collaborators else False,
'collaborators':
collaborators,
'collaborators_str':
', '.join(collaborators),
'num_collaborators':
len(collaborators)
})
return render_to_response(
"user-browse.html", {
'login': get_login(request),
'repo_base': repo_base,
'repos': visible_repos
})
except Exception, e:
return HttpResponse(json.dumps({'error': str(e)}),
content_type="application/json")
