class Connection: def __init__(self, user, password): self.backend = PGBackend(user, password) def create_repo(self, repo): return self.backend.create_repo(repo=repo) def list_repos(self): return self.backend.list_repos() def delete_repo(self, repo, force=False): return self.backend.delete_repo(repo=repo, force=force) def list_tables(self, repo): return self.backend.list_tables(repo=repo) def desc_table(self, table): return self.backend.desc_table(table=table) def execute_sql(self, query, params=None): return self.backend.execute_sql(query, params) def close(self): self.backend.close() ''' The following methods run in superuser mode ''' @staticmethod def create_user(username, password): s_backend = PGBackend(user='******', password='******') s_backend.create_user(username, password) s_backend.execute_sql('CREATE DATABASE %s WITH OWNER=%s' % (username, username)) return s_backend.create_user(username, password) @staticmethod def change_password(username, password): s_backend = PGBackend(user='******', password='******') return s_backend.change_password(username, password)
class DataHubConnection: def __init__(self, user, password, repo_base=None): self.backend = PGBackend(user, password, repo_base=repo_base) def reset_connection(self, repo_base): self.backend.reset_connection(repo_base=repo_base) def close_connection(self): self.backend.close_connection() def create_repo(self, repo): return self.backend.create_repo(repo=repo) def list_repos(self): return self.backend.list_repos() def delete_repo(self, repo, force=False): return self.backend.delete_repo(repo=repo, force=force) def add_collaborator(self, repo, username, privileges, auto_in_future=True): return self.backend.add_collaborator(repo=repo, username=username, privileges=privileges, auto_in_future=auto_in_future) def delete_collaborator(self, repo, username): return self.backend.delete_collaborator(repo=repo, username=username) def list_tables(self, repo): return self.backend.list_tables(repo=repo) def get_schema(self, table): return self.backend.get_schema(table=table) def execute_sql(self, query, params=None): return self.backend.execute_sql(query, params) def has_base_privilege(self, login, privilege): return self.backend.has_base_privilege(login=login, privilege=privilege) def has_repo_privilege(self, login, repo, privilege): return self.backend.has_repo_privilege(login=login, repo=repo, privilege=privilege) def has_table_privilege(self, login, table, privilege): return self.backend.has_table_privilege(login=login, table=table, privilege=privilege) def has_column_privilege(self, login, table, column, privilege): return self.backend.has_column_privilege(login=login, table=table, column=column, privilege=privilege) ''' The following methods works only in superuser mode ''' def create_user(self, username, password, create_db): return self.backend.create_user(username, password, create_db) def remove_user(self, username): return self.backend.remove_user(username) def change_password(self, username, password): return self.backend.change_password(username, password) def import_file(self, table_name, file_path, file_format='CSV', delimiter=',', header=True, encoding='ISO-8859-1', quote_character='"'): return self.backend.import_file(table_name=table_name, file_path=file_path, file_format=file_format, delimiter=delimiter, encoding=encoding, quote_character=quote_character) def export_table(self, table_name, file_path, file_format='CSV', delimiter=',', header=True): return self.backend.export_table(table_name=table_name, file_path=file_path, file_format=file_format, delimiter=delimiter) def export_query(self, query, file_path, file_format='CSV', delimiter=',', header=True): return self.backend.export_query(query=query, file_path=file_path, file_format=file_format, delimiter=delimiter) def list_collaborators(self, repo_base, repo): return self.backend.list_collaborators(repo_base=repo_base, repo=repo)
class DataHubConnection: def __init__(self, user, password, repo_base=None): self.backend = PGBackend(user, password, repo_base=repo_base) def change_repo_base(self, repo_base): self.backend.change_repo_base(repo_base=repo_base) def close_connection(self): self.backend.close_connection() def create_repo(self, repo): return self.backend.create_repo(repo=repo) def list_repos(self): return self.backend.list_repos() def rename_repo(self, repo, new_name): return self.backend.rename_repo(repo, new_name) def delete_repo(self, repo, force=False): return self.backend.delete_repo(repo=repo, force=force) def add_collaborator(self, repo, collaborator, db_privileges): return self.backend.add_collaborator(repo=repo, collaborator=collaborator, db_privileges=db_privileges) def delete_collaborator(self, repo, collaborator): return self.backend.delete_collaborator(repo=repo, collaborator=collaborator) def create_table(self, repo, table, params): return self.backend.create_table(repo=repo, table=table, params=params) def list_tables(self, repo): return self.backend.list_tables(repo=repo) def describe_table(self, repo, table, detail=False): return self.backend.describe_table(repo=repo, table=table, detail=detail) def list_table_permissions(self, repo, table): return self.backend.list_table_permissions(repo=repo, table=table) def create_view(self, repo, view, sql): return self.backend.create_view(repo=repo, view=view, sql=sql) def list_views(self, repo): return self.backend.list_views(repo=repo) def describe_view(self, repo, view, detail=False): return self.backend.describe_view(repo=repo, view=view, detail=detail) def delete_view(self, repo, view, force=False): return self.backend.delete_view(repo=repo, view=view, force=force) def delete_table(self, repo, table, force=False): return self.backend.delete_table(repo=repo, table=table, force=force) def get_schema(self, repo, table): return self.backend.get_schema(repo=repo, table=table) def explain_query(self, query): return self.backend.explain_query(query=query) def limit_and_offset_select_query(self, query, limit, offset): return self.backend.limit_and_offset_select_query(query=query, limit=limit, offset=offset) def select_table_query(self, repo_base, repo, table): return self.backend.select_table_query(repo_base=repo_base, repo=repo, table=table) def execute_sql(self, query, params=None): return self.backend.execute_sql(query, params) def has_base_privilege(self, login, privilege): return self.backend.has_base_privilege(login=login, privilege=privilege) def has_repo_db_privilege(self, login, repo, privilege): return self.backend.has_repo_db_privilege(login=login, repo=repo, privilege=privilege) def has_table_privilege(self, login, table, privilege): return self.backend.has_table_privilege(login=login, table=table, privilege=privilege) def has_column_privilege(self, login, table, column, privilege): return self.backend.has_column_privilege(login=login, table=table, column=column, privilege=privilege) ''' The following methods works only in superuser mode ''' def user_exists(self, username): return self.backend.user_exists(username) def database_exists(self, db_name): return self.backend.database_exists(db_name) def create_user(self, username, password, create_db): return self.backend.create_user(username, password, create_db) def remove_user(self, username): return self.backend.remove_user(username) def create_user_database(self, username): return self.backend.create_user_database(username) def drop_owned_by(self, username): return self.backend.drop_owned_by(username) def list_all_users(self): return self.backend.list_all_users() def list_all_databases(self): return self.backend.list_all_databases() def remove_database(self, repo_base, revoke_collaborators=True): return self.backend.remove_database(repo_base, revoke_collaborators) def change_password(self, username, password): return self.backend.change_password(username, password) def import_file(self, table_name, file_path, file_format='CSV', delimiter=',', header=True, encoding='ISO-8859-1', quote_character='"'): return self.backend.import_file(table_name=table_name, file_path=file_path, file_format=file_format, delimiter=delimiter, header=header, encoding=encoding, quote_character=quote_character) def export_table(self, table_name, file_path, file_format='CSV', delimiter=',', header=True): return self.backend.export_table(table_name=table_name, file_path=file_path, file_format=file_format, delimiter=delimiter) def export_view(self, view_name, file_path, file_format='CSV', delimiter=',', header=True): return self.backend.export_view(view_name=view_name, file_path=file_path, file_format=file_format, delimiter=delimiter) def export_query(self, query, file_path, file_format='CSV', delimiter=',', header=True): return self.backend.export_query(query=query, file_path=file_path, file_format=file_format, delimiter=delimiter) def list_collaborators(self, repo): return self.backend.list_collaborators(repo) # Row Level Security Stuff def create_security_policy_schema(self): return self.backend.create_security_policy_schema() def create_security_policy_table(self): return self.backend.create_security_policy_table() def create_security_policy(self, policy, policy_type, grantee, grantor, repo_base, repo, table): return self.backend.create_security_policy(policy=policy, policy_type=policy_type, grantee=grantee, grantor=grantor, repo_base=repo_base, repo=repo, table=table) def find_security_policies(self, repo_base, repo=None, table=None, policy_id=None, policy=None, policy_type=None, grantee=None, grantor=None): return self.backend.find_security_policies(repo_base, repo, table, policy_id, policy, policy_type, grantee, grantor) def find_all_security_policies(self, username): """ Find all security policies that are granted by or to a username """ return self.backend.find_all_security_policies(username) def update_security_policy(self, policy_id, new_policy, new_policy_type, new_grantee): return self.backend.update_security_policy(policy_id, new_policy, new_policy_type, new_grantee) def find_security_policy_by_id(self, policy_id): return self.backend.find_security_policy_by_id(policy_id) def remove_security_policy(self, policy_id): return self.backend.remove_security_policy(policy_id) def can_user_access_rls_table(self, username, permissions=['SELECT', 'UPDATE', 'INSERT']): return self.backend.can_user_access_rls_table(username, permissions)
class DataHubConnection: def __init__(self, user, password, repo_base=None): self.backend = PGBackend(user, password, repo_base=repo_base) def change_repo_base(self, repo_base): self.backend.change_repo_base(repo_base=repo_base) def close_connection(self): self.backend.close_connection() def create_repo(self, repo): return self.backend.create_repo(repo=repo) def list_repos(self): return self.backend.list_repos() def rename_repo(self, repo, new_name): return self.backend.rename_repo(repo, new_name) def delete_repo(self, repo, force=False): return self.backend.delete_repo(repo=repo, force=force) def add_collaborator(self, repo, collaborator, db_privileges): return self.backend.add_collaborator( repo=repo, collaborator=collaborator, db_privileges=db_privileges) def delete_collaborator(self, repo, collaborator): return self.backend.delete_collaborator( repo=repo, collaborator=collaborator) def create_table(self, repo, table, params): return self.backend.create_table(repo=repo, table=table, params=params) def list_tables(self, repo): return self.backend.list_tables(repo=repo) def describe_table(self, repo, table, detail=False): return self.backend.describe_table( repo=repo, table=table, detail=detail) def list_table_permissions(self, repo, table): return self.backend.list_table_permissions( repo=repo, table=table) def create_view(self, repo, view, sql): return self.backend.create_view(repo=repo, view=view, sql=sql) def list_views(self, repo): return self.backend.list_views(repo=repo) def describe_view(self, repo, view, detail=False): return self.backend.describe_view( repo=repo, view=view, detail=detail) def delete_view(self, repo, view, force=False): return self.backend.delete_view(repo=repo, view=view, force=force) def delete_table(self, repo, table, force=False): return self.backend.delete_table(repo=repo, table=table, force=force) def get_schema(self, repo, table): return self.backend.get_schema(repo=repo, table=table) def explain_query(self, query): return self.backend.explain_query(query=query) def limit_and_offset_select_query(self, query, limit, offset): return self.backend.limit_and_offset_select_query( query=query, limit=limit, offset=offset) def select_table_query(self, repo_base, repo, table): return self.backend.select_table_query( repo_base=repo_base, repo=repo, table=table) def execute_sql(self, query, params=None): return self.backend.execute_sql(query, params) def has_base_privilege(self, login, privilege): return self.backend.has_base_privilege( login=login, privilege=privilege) def has_repo_db_privilege(self, login, repo, privilege): return self.backend.has_repo_db_privilege( login=login, repo=repo, privilege=privilege) def has_table_privilege(self, login, table, privilege): return self.backend.has_table_privilege( login=login, table=table, privilege=privilege) def has_column_privilege(self, login, table, column, privilege): return self.backend.has_column_privilege( login=login, table=table, column=column, privilege=privilege) ''' The following methods works only in superuser mode ''' def user_exists(self, username): return self.backend.user_exists(username) def database_exists(self, db_name): return self.backend.database_exists(db_name) def create_user(self, username, password, create_db): return self.backend.create_user(username, password, create_db) def remove_user(self, username): return self.backend.remove_user(username) def create_user_database(self, username): return self.backend.create_user_database(username) def drop_owned_by(self, username): return self.backend.drop_owned_by(username) def list_all_users(self): return self.backend.list_all_users() def list_all_databases(self): return self.backend.list_all_databases() def remove_database(self, repo_base, revoke_collaborators=True): return self.backend.remove_database(repo_base, revoke_collaborators) def change_password(self, username, password): return self.backend.change_password(username, password) def import_file(self, table_name, file_path, file_format='CSV', delimiter=',', header=True, encoding='ISO-8859-1', quote_character='"'): return self.backend.import_file( table_name=table_name, file_path=file_path, file_format=file_format, delimiter=delimiter, header=header, encoding=encoding, quote_character=quote_character) def export_table(self, table_name, file_path, file_format='CSV', delimiter=',', header=True): return self.backend.export_table( table_name=table_name, file_path=file_path, file_format=file_format, delimiter=delimiter) def export_view(self, view_name, file_path, file_format='CSV', delimiter=',', header=True): return self.backend.export_view( view_name=view_name, file_path=file_path, file_format=file_format, delimiter=delimiter) def export_query(self, query, file_path, file_format='CSV', delimiter=',', header=True): return self.backend.export_query( query=query, file_path=file_path, file_format=file_format, delimiter=delimiter) def list_collaborators(self, repo): return self.backend.list_collaborators(repo) # Row Level Security Stuff def create_security_policy_schema(self): return self.backend.create_security_policy_schema() def create_security_policy_table(self): return self.backend.create_security_policy_table() def create_security_policy(self, policy, policy_type, grantee, grantor, repo_base, repo, table): return self.backend.create_security_policy( policy=policy, policy_type=policy_type, grantee=grantee, grantor=grantor, repo_base=repo_base, repo=repo, table=table) def find_security_policies( self, repo_base, repo=None, table=None, policy_id=None, policy=None, policy_type=None, grantee=None, grantor=None): return self.backend.find_security_policies( repo_base, repo, table, policy_id, policy, policy_type, grantee, grantor) def find_all_security_policies(self, username): """ Find all security policies that are granted by or to a username """ return self.backend.find_all_security_policies(username) def update_security_policy(self, policy_id, new_policy, new_policy_type, new_grantee): return self.backend.update_security_policy( policy_id, new_policy, new_policy_type, new_grantee) def find_security_policy_by_id(self, policy_id): return self.backend.find_security_policy_by_id(policy_id) def remove_security_policy(self, policy_id): return self.backend.remove_security_policy(policy_id) def can_user_access_rls_table(self, username, permissions=['SELECT', 'UPDATE', 'INSERT']): return self.backend.can_user_access_rls_table(username, permissions)
class DataHubConnection: def __init__(self, user, password, repo_base=None): self.backend = PGBackend(user, password, repo_base=repo_base) def reset_connection(self, repo_base): self.backend.reset_connection(repo_base=repo_base) def close_connection(self): self.backend.close_connection() def create_repo(self, repo): return self.backend.create_repo(repo=repo) def list_repos(self): return self.backend.list_repos() def delete_repo(self, repo, force=False): return self.backend.delete_repo(repo=repo, force=force) def add_collaborator(self, repo, username, privileges, auto_in_future=True): return self.backend.add_collaborator( repo=repo, username=username, privileges=privileges, auto_in_future=auto_in_future) def delete_collaborator(self, repo, username): return self.backend.delete_collaborator(repo=repo, username=username) def list_tables(self, repo): return self.backend.list_tables(repo=repo) def get_schema(self, table): return self.backend.get_schema(table=table) def execute_sql(self, query, params=None): return self.backend.execute_sql(query, params) def has_base_privilege(self, login, privilege): return self.backend.has_base_privilege( login=login, privilege=privilege) def has_repo_privilege(self, login, repo, privilege): return self.backend.has_repo_privilege( login=login, repo=repo, privilege=privilege) def has_table_privilege(self, login, table, privilege): return self.backend.has_table_privilege( login=login, table=table, privilege=privilege) def has_column_privilege(self, login, table, column, privilege): return self.backend.has_column_privilege( login=login, table=table, column=column, privilege=privilege) ''' The following methods works only in superuser mode ''' def create_user(self, username, password, create_db): return self.backend.create_user(username, password, create_db) def remove_user(self, username): return self.backend.remove_user(username) def change_password(self, username, password): return self.backend.change_password(username, password) def import_file(self, table_name, file_path, file_format='CSV', delimiter=',', header=True, encoding='ISO-8859-1', quote_character='"'): return self.backend.import_file( table_name=table_name, file_path=file_path, file_format=file_format, delimiter=delimiter, encoding=encoding, quote_character=quote_character) def export_table(self, table_name, file_path, file_format='CSV', delimiter=',', header=True): return self.backend.export_table( table_name=table_name, file_path=file_path, file_format=file_format, delimiter=delimiter) def export_query(self, query, file_path, file_format='CSV', delimiter=',', header=True): return self.backend.export_query( query=query, file_path=file_path, file_format=file_format, delimiter=delimiter) def list_collaborators(self, repo_base, repo): return self.backend.list_collaborators(repo_base=repo_base, repo=repo)
class DataHubConnection: def __init__(self, user, password, repo_base=None): self.backend = PGBackend(user, password, repo_base=repo_base) def change_repo_base(self, repo_base): self.backend.change_repo_base(repo_base=repo_base) def close_connection(self): self.backend.close_connection() def create_repo(self, repo): return self.backend.create_repo(repo=repo) def list_repos(self): return self.backend.list_repos() def delete_repo(self, repo, force=False): return self.backend.delete_repo(repo=repo, force=force) def add_collaborator(self, repo, collaborator, privileges): return self.backend.add_collaborator( repo=repo, collaborator=collaborator, privileges=privileges) def delete_collaborator(self, repo, collaborator): return self.backend.delete_collaborator( repo=repo, collaborator=collaborator) def list_tables(self, repo): return self.backend.list_tables(repo=repo) def list_views(self, repo): return self.backend.list_views(repo=repo) def delete_table(self, repo, table, force=False): return self.backend.delete_table(repo=repo, table=table, force=force) def get_schema(self, repo, table): return self.backend.get_schema(repo=repo, table=table) def explain_query(self, query): return self.backend.explain_query(query=query) def limit_and_offset_select_query(self, query, limit, offset): return self.backend.limit_and_offset_select_query( query=query, limit=limit, offset=offset) def select_table_query(self, repo_base, repo, table): return self.backend.select_table_query( repo_base=repo_base, repo=repo, table=table) def execute_sql(self, query, params=None): return self.backend.execute_sql(query, params) def has_base_privilege(self, login, privilege): return self.backend.has_base_privilege( login=login, privilege=privilege) def has_repo_privilege(self, login, repo, privilege): return self.backend.has_repo_privilege( login=login, repo=repo, privilege=privilege) def has_table_privilege(self, login, table, privilege): return self.backend.has_table_privilege( login=login, table=table, privilege=privilege) def has_column_privilege(self, login, table, column, privilege): return self.backend.has_column_privilege( login=login, table=table, column=column, privilege=privilege) ''' The following methods works only in superuser mode ''' def user_exists(self, username): return self.backend.user_exists(username) def database_exists(self, db_name): return self.backend.database_exists(db_name) def create_user(self, username, password, create_db): return self.backend.create_user(username, password, create_db) def remove_user(self, username): return self.backend.remove_user(username) def drop_owned_by(self, username): return self.backend.drop_owned_by(username) def list_all_users(self): return self.backend.list_all_users() def list_all_databases(self): return self.backend.list_all_databases() def remove_database(self, repo_base, revoke_collaborators=True): return self.backend.remove_database(repo_base, revoke_collaborators) def change_password(self, username, password): return self.backend.change_password(username, password) def import_file(self, table_name, file_path, file_format='CSV', delimiter=',', header=True, encoding='ISO-8859-1', quote_character='"'): return self.backend.import_file( table_name=table_name, file_path=file_path, file_format=file_format, delimiter=delimiter, encoding=encoding, quote_character=quote_character) def export_table(self, table_name, file_path, file_format='CSV', delimiter=',', header=True): return self.backend.export_table( table_name=table_name, file_path=file_path, file_format=file_format, delimiter=delimiter) def export_query(self, query, file_path, file_format='CSV', delimiter=',', header=True): return self.backend.export_query( query=query, file_path=file_path, file_format=file_format, delimiter=delimiter) def list_collaborators(self, repo): return self.backend.list_collaborators(repo)
class Connection: GRAPH_PICKLE_BASE = 'version-graph-%s.txt' def __init__(self, user, password): self.backend = PGBackend(user, password) self.current_repo = None self.current_version_file = None self.current_branch = None self.g = None self.table = 'metatest' # Test version table self.create_table = 'CREATE TABLE %s (id int, name varchar(50), state varchar(20), salary int)' self.insert_table = "INSERT INTO %s values (%s, '%s', '%s', %s )" def create_repo(self, repo): return self.backend.create_repo(repo=repo) def list_repos(self): return self.backend.list_repos() def delete_repo(self, repo, force=False): return self.backend.delete_repo(repo=repo, force=force) def reset(self): os.path.dirname(os.path.realpath(__file__)) get_tables_query = "SELECT table_name FROM information_schema.tables WHERE table_schema='public' AND table_type='BASE TABLE' and table_name like '%s%s';" % (self.table,'%') for table in self.backend.execute_sql(get_tables_query)['tuples']: print "dropping table : %s" % table self.backend.execute_sql("drop table %s" % table) base = self.GRAPH_PICKLE_BASE.split('%s')[0] for _file in [f for f in os.listdir('.') if os.path.isfile(f) and base in f]: print "removing file %s" % _file os.remove(_file) for s in self.list_repos()['tuples']: print "dropping : %s" % s[0] self.delete_repo(s[0],True) ############################################################# ### Versioning ############################################################# def check_key_conflict(self, t1, t2): query = "select t1.id from %s t1 inner join %s t2 on t1.id = t2.id" % (t1,t2) print query res = self.backend.execute_sql(query) print res if res['row_count'] > 0: print "Cannot automeger" else: print "No conflicts" def merge(self, branch, merging_branch): if not self.current_repo: raise Exception("No active repo set") try: branch = self.g.vs.find(branch) merging_branch = self.g.vs.find(merging_branch) b_out_tables = self.get_out_links_map(branch, 'table','table_name') m_out_tables = self.get_out_links_map(merging_branch, 'table', 'table_name') if b_out_tables.keys() == m_out_tables.keys(): print "Merging" print "TODO trace to ancestor and find deltas" #TODO for table in b_out_tables.keys(): print "Checking %s" % table self.check_key_conflict(b_out_tables[table]['real_table'], m_out_tables[table]['real_table']) else: raise NotImplementedError("Tables of branches are different.") except ValueError,e: raise ValueError('branch does not exist. %s' % e)