def tables(request, repo_name): username = request.user.get_username() manager = DataHubManager(username) repos = get_repos(manager) if repos is not None and repo_name in repos: tables = get_tables(manager, repo_name) if tables is not None: return json_response({"tables": tables}) return error_response()
def execute_sql(self, con, query, query_params=None): try: manager = DataHubManager(user=con.user, repo_base=con.repo_base, is_app=con.is_app) res = manager.execute_sql(query=query, params=query_params) return construct_result_set(res) except Exception, e: raise DBException(message=str(e))
def file_download(request, repo_base, repo, file_name): username = request.user.get_username() manager = DataHubManager(username, repo_base) file_to_download = manager.get_file(repo, file_name) response = HttpResponse(file_to_download, content_type='application/force-download') response['Content-Disposition'] = 'attachment; filename="%s"' % (file_name) return response
def card_export(request, repo_base, repo, card_name): username = request.user.get_username() file_name = request.GET.get('var_text', card_name) with DataHubManager(user=username, repo_base=repo_base) as manager: manager.export_card(repo, file_name, card_name) return HttpResponseRedirect( reverse('browser-repo_files', args=(repo_base, repo)))
def repo_delete(request, repo_base, repo): ''' deletes a repo in the current database (repo_base) ''' username = request.user.get_username() manager = DataHubManager(user=username, repo_base=repo_base) manager.delete_repo(repo=repo, force=True) return HttpResponseRedirect(reverse('browser-user-default'))
def table_clone(request, repo_base, repo, table): username = request.user.get_username() new_table = request.GET.get('var_text', None) with DataHubManager(user=username, repo_base=repo_base) as manager: manager.clone_table(repo, table, new_table) return HttpResponseRedirect( reverse('browser-repo_tables', args=(repo_base, repo)))
def file_upload(request, repo_base, repo): username = request.user.get_username() data_file = request.FILES['data_file'] with DataHubManager(user=username, repo_base=repo_base) as manager: manager.save_file(repo, data_file) return HttpResponseRedirect( reverse('browser-repo_files', args=(repo_base, repo)))
def file_import(request, repo_base, repo, file_name): try: login = get_login(request) res = DataHubManager.has_repo_privilege(login, repo_base, repo, 'CREATE') if not (res and res['tuples'][0][0]): raise Exception('Access denied. Missing required privileges.') delimiter = str(request.GET['delimiter']) if delimiter == '': delimiter = str(request.GET['other_delimiter']) header = True if request.GET['has_header'] == "true" else False quote_character = request.GET['quote_character'] if quote_character == '': quote_character = request.GET['other_quote_character'] delimiter = delimiter.decode('string_escape') repo_dir = '/user_data/%s/%s' % (repo_base, repo) file_path = '%s/%s' % (repo_dir, file_name) table_name, _ = os.path.splitext(file_name) table_name = clean_str(table_name, 'table') dh_table_name = '%s.%s.%s' % (repo_base, repo, table_name) f = codecs.open(file_path, 'r', 'ISO-8859-1') data = csv.reader(f, delimiter=delimiter) cells = data.next() columns = [clean_str(str(i), 'col') for i in range(0, len(cells))] if header: columns = map(lambda x: clean_str(x, 'col'), cells) columns = rename_duplicates(columns) query = 'CREATE TABLE %s (%s text' % (dh_table_name, columns[0]) for i in range(1, len(columns)): query += ', %s %s' % (columns[i], 'text') query += ')' manager = DataHubManager(user=repo_base) manager.execute_sql(query=query) manager.import_file(repo_base=repo_base, table_name=dh_table_name, file_path=file_path, delimiter=delimiter, header=header, quote_character=quote_character) return HttpResponseRedirect('/browse/%s/%s' % (repo_base, repo)) except Exception, e: return HttpResponse(json.dumps({'error': str(e)}), content_type="application/json")
def tables(request): """ Returns the tables in a user's repo. I think this is only used for (disabled) scorpion features. """ username = request.user.get_username() repo_base = request.GET.get('username', None) manager = DataHubManager(username, repo_base) repo = request.GET.get('repo') tables = manager.list_tables(repo) return {'tables': tables}
def card_create(request, repo_base, repo): username = request.user.get_username() card_name = request.POST['card-name'] query = request.POST['query'] url = reverse('browser-card', args=(repo_base, repo, card_name)) with DataHubManager(user=username, repo_base=repo_base) as manager: manager.create_card(repo, card_name, query) return HttpResponseRedirect(url)
def move_tables_to_new_schema(username, tables_and_views): # if they do have tables/views if len(tables_and_views) > 0: repo_name = create_and_return_name_for_public_schema(username) for table in tables_and_views: with DataHubManager(username) as m: query = ('Alter table public.%s ' 'set schema %s') % (table, repo_name) m.execute_sql(query)
def test_rename_repo(self): con_rename_repo = self.mock_connection.return_value.rename_repo # self.mock_Collaborator = self.create_patch( # 'inventory.models.Collaborator') self.manager = DataHubManager( user=self.username, repo_base=self.username) self.manager.Collaborator = MagicMock self.manager.rename_repo('old_name', 'new_name') self.assertTrue(con_rename_repo.called)
def schema(request, repo, table): username = request.user.get_username() manager = DataHubManager(username) repos = get_repos(manager) if repos is not None and repo in repos: tables = get_tables(manager, repo) if tables is not None and table in tables: schema = manager.get_schema(repo, table) if len(schema) > 0: return json_response({"schema": schema}) return error_response()
def table_export(request, repo_base, repo, table_name): username = request.user.get_username() file_name = request.GET.get('var_text', table_name) with DataHubManager(user=username, repo_base=repo_base) as manager: manager.export_table( repo=repo, table=table_name, file_name=file_name, file_format='CSV', delimiter=',', header=True) return HttpResponseRedirect( reverse('browser-repo_files', args=(repo_base, repo)))
def repos(request): """ return the repos in the user's database. I think this is only used for (disabled) scorpion features. """ username = request.user.get_username() repo_base = request.GET.get('username', None) manager = DataHubManager(username, repo_base) dbnames = manager.list_repos() # q = "SELECT datname FROM pg_database where datistemplate = false;" # dbnames = [str(row[0]) for row in manager.execute_sql(query=q)['tuples']] return {'databases': dbnames}
def setUp(self): self.username = "******" self.password = "******" self.email = "*****@*****.**" self.user = User.objects.create_user( self.username, self.email, self.password) self.mock_connection = self.create_patch( 'core.db.manager.DataHubConnection') self.manager = DataHubManager(user=self.username)
def schema(request, repo, table): username = get_login(request) manager = DataHubManager(username) repos = get_repos(manager) if repos is not None and repo in repos: tables = get_tables(manager, repo) if tables is not None and table in tables: schema = manager.get_schema(repo + "." + table) if schema is not None and 'tuples' in schema: return json_response({"schema": schema["tuples"]}) return error_response()
def license_view_create(request, repo_base, repo, table, license_id): ''' creates a new license view for a given table and license_id ''' username = request.user.get_username() public_role = settings.PUBLIC_ROLE with DataHubManager(user=username, repo_base=repo_base) as manager: collaborators = manager.list_collaborators(repo) if username != repo_base: raise PermissionDenied("User does not have access to this repo") if request.method == 'POST': # collect parameters that will be used to create the view of the table removed_columns = request.POST.getlist('removed_columns[]') view_params = {} view_params['removed-columns'] = removed_columns with DataHubManager(user=username, repo_base=repo_base) as manager: manager.create_license_view( repo=repo, table=table, view_params=view_params, license_id=license_id) return HttpResponseRedirect( reverse('browser-repo_licenses', args=(repo_base, repo))) elif request.method == 'GET': collaborators = [c for c in collaborators if c['username'] not in ['', username, settings.PUBLIC_ROLE]] res = { 'login': username, 'repo_base': repo_base, 'repo': repo, 'collaborators': collaborators, 'public_role': public_role} res.update(csrf(request)) return render_to_response("license-create.html", res)
def query(request, repo_base, repo): query = post_or_get(request, key='q', fallback=None) username = request.user.get_username() # if the user is just requesting the query page if not query: data = { 'login': username, 'repo_base': repo_base, 'repo': repo, 'select_query': False, 'query': None } return render_to_response("query.html", data) # if the user is actually executing a query current_page = 1 if request.POST.get('page'): current_page = request.POST.get('page') url_path = reverse('browser-query', args=(repo_base, repo)) with DataHubManager(user=username, repo_base=repo_base) as manager: res = manager.paginate_query(query=query, current_page=current_page, rows_per_page=50) # get annotation to the table: annotation, created = Annotation.objects.get_or_create(url_path=url_path) annotation_text = annotation.annotation_text data = { 'login': username, 'repo_base': repo_base, 'repo': repo, 'annotation': annotation_text, 'current_page': current_page, 'next_page': current_page + 1, # the template should relaly do this 'prev_page': current_page - 1, # the template should relaly do this 'url_path': url_path, 'query': query, 'select_query': res['select_query'], 'column_names': res['column_names'], 'tuples': res['rows'], 'total_pages': res['total_pages'], 'pages': range(res['start_page'], res['end_page'] + 1), # template 'num_rows': res['num_rows'], 'time_cost': res['time_cost'] } data.update(csrf(request)) return render_to_response("query-browse-results.html", data)
def list_repos(self, con): try: ''' res = DataHubManager.has_base_privilege(con.user, con.repo_base, 'CONNECT') if not (res and res['tuples'][0][0]): raise Exception('Access denied. Missing required privileges.') ''' manager = DataHubManager(user=con.repo_base, repo_base=con.repo_base) res = manager.list_repos() return construct_result_set(res) except Exception, e: raise DBException(message=str(e))
def index(request): # if the user is authenticated, we pass them the available repos username = request.user.get_username() res = {'login': username} try: with DataHubManager(username) as m: repos = m.list_repos() res['repos'] = repos except: pass res.update(csrf(request)) return render_to_response("refiner.html", res)
def test_create_repo_creates_a_repo(self): # create the new repo post_object = {'repo': 'repo_name'} response = self.client.post('/create/' + self.username + '/repo', post_object) # get a list of repos that the user owns manager = DataHubManager(user=self.username) res = manager.list_repos() repos = [t[0] for t in res['tuples']] # make sure that it's in there self.assertTrue('repo_name' in repos)
def card_update_public(request, repo_base, repo, card_name): username = request.user.get_username() if 'public' in request.POST: public = request.POST['public'] == 'True' else: raise ValueError("Request missing \'public\' parameter.") with DataHubManager(user=username, repo_base=repo_base) as manager: manager.update_card(repo, card_name, public=public) return HttpResponseRedirect( reverse('browser-card', args=(repo_base, repo, card_name)))
def _f(self, *args, **kwargs): try: key = str( map(str, (f.__name__, self.dbname, self.tablename, self.where, self.nbuckets, map(str, args)))) manager = DataHubManager(user=self.username) vals = manager.execute_sql( 'select val from _dbwipes_cache where key = %s', params=(key, ))['tuples'] if len(vals): return json.loads(vals[0][0]) except Exception as e: print e pdb.set_trace() res = f(self, *args, **kwargs) if key: manager = DataHubManager(user=self.username) manager.execute_sql('insert into _dbwipes_cache values(%s, %s)', params=(key, json.dumps(res, default=json_handler))) return res
def table_delete(request, repo_base, repo, table_name): """ Deletes the given table. Does not currently allow the user the option to cascade in the case of dependencies, though the delete_table method does allow cascade (force) to be passed. """ username = request.user.get_username() manager = DataHubManager(user=username, repo_base=repo_base) manager.delete_table(repo, table_name) return HttpResponseRedirect( reverse('browser-repo_tables', args=(repo_base, repo)))
def handle(self, *args, **options): # Throw out all of these changes if it fails somehow with transaction.atomic(): for base in (user.get_username() for user in User.objects.all()): try: manager = DataHubManager(user=base, repo_base=base) except OperationalError as e: print(e) continue repos = manager.list_repos() for repo in repos: self._sync_repo_collaborators(manager, base, repo) print("Added missing {0} users, {1} missing apps.".format( self.user_count, self.app_count))
def test_initialization(self): DataHubManager(user=self.username) # username passed self.assertEqual( self.mock_connection.call_args[1]['user'], self.username) # password passed self.assertTrue( self.mock_connection.call_args[1]['password'] is not None) # repo defaults to username self.assertEqual( self.mock_connection.call_args[1]['repo_base'], self.username)
def _create_table_helper(self, username, repo, cases, tables_after): """ Tries to create a table in a repo as a user. Cases should be an array of test case tuples, where each tuple is the Exception to be raised or None plus the arguments to be passed to create_table. """ with DataHubManager(user=username, repo_base=self.owner_username) as m: self.assertEquals([], m.list_tables(repo)) for c in cases: with self._assertRaisesOrNone(c[0], message=c): m.create_table(*c[1:]) self.assertEquals(tables_after, m.list_tables(repo))
def license_view_delete(request, repo_base, repo, table, license_view, license_id): ''' Deletes license view for table and given license_id ''' username = request.user.get_username() public_role = settings.PUBLIC_ROLE with DataHubManager(user=username, repo_base=repo_base) as manager: collaborators = manager.list_collaborators(repo) if username != repo_base: raise PermissionDenied("User does not have access to this repo") with DataHubManager(user=username, repo_base=repo_base) as manager: manager.delete_license_view( repo=repo, table=table, license_view=license_view, license_id=license_id) return HttpResponseRedirect(reverse('browser-repo_licenses', args=(repo_base, repo)))
def user(request, repo_base): try: login = get_login(request) res = DataHubManager.has_base_privilege(login, repo_base, 'CONNECT') if not (res and res['tuples'][0][0]): raise Exception('Access denied. Missing required privileges.') manager = DataHubManager(user=repo_base) res = manager.list_repos() repos = [t[0] for t in res['tuples']] visible_repos = [] for repo in repos: res = manager.list_collaborators(repo_base, repo) collaborators = [(c[0].split('=')[0]).strip() for c in res['tuples']] collaborators = filter(lambda x: x != '' and x != repo_base, collaborators) if login not in collaborators and login != repo_base: continue visible_repos.append({ 'name': repo, 'owner': repo_base, 'public': True if 'PUBLIC' in collaborators else False, 'collaborators': collaborators, 'collaborators_str': ', '.join(collaborators), 'num_collaborators': len(collaborators) }) return render_to_response( "user-browse.html", { 'login': get_login(request), 'repo_base': repo_base, 'repos': visible_repos }) except Exception, e: return HttpResponse(json.dumps({'error': str(e)}), content_type="application/json")