def update_legacy_states(graph_kwargs): """Updates state, file_state on legacy nodes - node.state in {None, 'live'} - node.project_id in {None, <Legacy project_id list>} there is no project_id, or project_id points to a legacy project """ graph = PsqlGraphDriver(**graph_kwargs) with graph.session_scope(): print_cls_query_summary(graph) input_q = Queue() pool = [ Process(target=update_classes, args=(graph_kwargs, input_q)) for _ in range(cpu_count()) ] for cls in CLS_TO_UPDATE: input_q.put(cls) for process in pool: input_q.put(None) # put a no more work signal for each process for process in pool: process.start() for process in pool: process.join()
def test_priv_revoke_read(self): """Test ability to revoke read privs""" self.create_all_tables() try: self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'") g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database) pgadmin.main(pgadmin.get_parser().parse_args([ 'graph-grant', '--read=pytest', ] + self.base_args)) pgadmin.main(pgadmin.get_parser().parse_args([ 'graph-revoke', '--read=pytest', ] + self.base_args)) with self.assertRaises(ProgrammingError): with g.session_scope() as s: g.nodes().count() s.merge(models.Case('1')) finally: self.engine.execute("DROP OWNED BY pytest; DROP USER pytest")
def translate_node_range(_args): args, offset = _args src = PsqlGraphDriver( args.source_host, args.source_user, args.source_password, args.source, **driver_kwargs) dst = PsqlGraphDriver( args.dest_host, args.dest_user, args.dest_password, args.dest, **driver_kwargs) with src.session_scope() as session: with dst.session_scope() as session: for old in src.nodes(OldNode).order_by(OldNode.node_id)\ .offset(offset)\ .limit(BLOCK)\ .yield_per(BLOCK): try: new = PolyNode( node_id=old.node_id, properties=old.properties, system_annotations=old.system_annotations, acl=old.acl, label=old.label, ) new.created = old.created session.merge(new) except Exception as e: logging.error("unable to add node {}, {}".format( old.label, old.node_id)) logging.error(e) try: session.commit() except Exception as e: logging.error(e)
def test_priv_revoke_write(self): """Test ability to revoke read/write privs""" self.create_all_tables() self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'") try: g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database) pgadmin.main(pgadmin.get_parser().parse_args([ 'graph-grant', '--write=pytest', ] + self.base_args)) pgadmin.main(pgadmin.get_parser().parse_args([ 'graph-revoke', '--write=pytest', ] + self.base_args)) with g.session_scope() as s: g.nodes().count() with self.assertRaises(ProgrammingError): with g.session_scope() as s: s.merge(models.Case('1')) finally: self.engine.execute("DROP OWNED BY pytest; DROP USER pytest")
def translate_nodes(args): src = PsqlGraphDriver(args.source_host, args.source_user, args.source_password, args.source, **driver_kwargs) with src.session_scope(): count = src.nodes(OldNode).count() offsets = [i * BLOCK for i in range(count / BLOCK + 1)] pool = Pool(args.nprocs) args = [(args, offset) for offset in offsets] pool.map_async(translate_node_range, args).get(int(1e9))
def translate_nodes(args): src = PsqlGraphDriver( args.source_host, args.source_user, args.source_password, args.source, **driver_kwargs) with src.session_scope(): count = src.nodes(OldNode).count() offsets = [i*BLOCK for i in range(count/BLOCK+1)] pool = Pool(args.nprocs) args = [(args, offset) for offset in offsets] pool.map_async(translate_node_range, args).get(int(1e9))
def test_priv_grant_read(self): """Test ability to grant read but not write privs""" self.create_all_tables() try: self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'") g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database) #: If this failes, this test (not the code) is wrong! with self.assertRaises(ProgrammingError): with g.session_scope(): g.nodes().count() pgadmin.main(pgadmin.get_parser().parse_args([ 'graph-grant', '--read=pytest', ] + self.base_args)) with g.session_scope(): g.nodes().count() with self.assertRaises(ProgrammingError): with g.session_scope() as s: s.merge(models.Case('1')) finally: self.engine.execute("DROP OWNED BY pytest; DROP USER pytest")
def test_custom_application_name(self): cmd = "select application_name from pg_stat_activity;" custom_name = '_CUSTOM_NAME' g_ = PsqlGraphDriver(host, user, password, database, application_name=custom_name) with g_.session_scope() as s: s.merge(Test('a')) app_names = {r[0] for r in g.engine.execute(cmd)} self.assertIn(custom_name, app_names)
def test_priv_grant_read(self): """Test ability to grant read but not write privs""" self.create_all_tables() self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'") try: g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database) #: If this failes, this test (not the code) is wrong! with self.assertRaises(ProgrammingError): with g.session_scope(): g.nodes().count() pgadmin.main(pgadmin.get_parser().parse_args([ 'graph-grant', '--read=pytest', ] + self.base_args)) with g.session_scope(): g.nodes().count() with self.assertRaises(ProgrammingError): with g.session_scope() as s: s.merge(models.Case('1')) finally: self.engine.execute("DROP OWNED BY pytest; DROP USER pytest")
def migrate(host, user, password, database): graph = PsqlGraphDriver(host, user, password, database) with graph.session_scope(): for project, phsids in mapping.PROJECT_TO_PHSID.iteritems(): tokens = project.split('-') program_code = tokens[0] project_code = '-'.join(tokens[1:]) program_phsid = mapping.get_program_level_phsid(project) project_phsid = mapping.get_project_level_phsid(project) program = graph.nodes(Program).props(name=program_code).first() project = graph.nodes(Project).props(code=project_code).first() if program: program.props['dbgap_accession_number'] = program_phsid if project: project.props['dbgap_accession_number'] = project_phsid
def db_init(app): app.logger.info("Initializing PsqlGraph driver") connect_args = {} if app.config.get("PSQLGRAPH") and app.config["PSQLGRAPH"].get("sslmode"): connect_args["sslmode"] = app.config["PSQLGRAPH"]["sslmode"] app.db = PsqlGraphDriver( host=app.config["PSQLGRAPH"]["host"], user=app.config["PSQLGRAPH"]["user"], password=app.config["PSQLGRAPH"]["password"], database=app.config["PSQLGRAPH"]["database"], set_flush_timestamps=True, connect_args=connect_args, isolation_level=app.config["PSQLGRAPH"].get( "isolation_level", "READ_COMMITTED" ), ) if app.config.get("AUTO_MIGRATE_DATABASE"): migrate_database(app) app.oauth_client = oauth2_client.OAuthClient(**app.config["OAUTH2"]) app.logger.info("Initializing index client") app.index_client = IndexClient( app.config["INDEX_CLIENT"]["host"], version=app.config["INDEX_CLIENT"]["version"], auth=app.config["INDEX_CLIENT"]["auth"], )
def db_init(app): app.logger.info('Initializing PsqlGraph driver') app.db = PsqlGraphDriver( host=app.config['PSQLGRAPH']['host'], user=app.config['PSQLGRAPH']['user'], password=app.config['PSQLGRAPH']['password'], database=app.config['PSQLGRAPH']['database'], set_flush_timestamps=True, ) app.userdb = SQLAlchemyDriver(app.config['PSQL_USER_DB_CONNECTION']) flask_scoped_session(app.userdb.Session, app) app.oauth2 = OAuth2Client(**app.config['OAUTH2']) app.logger.info('Initializing Signpost driver') app.signpost = SignpostClient( app.config['SIGNPOST']['host'], version=app.config['SIGNPOST']['version'], auth=app.config['SIGNPOST']['auth']) try: app.logger.info('Initializing Auth driver') app.auth = AuthDriver(app.config["AUTH_ADMIN_CREDS"], app.config["INTERNAL_AUTH"]) except Exception: app.logger.exception("Couldn't initialize auth, continuing anyway")
def create_tables(host, user, password, database): """ create a table """ print('Creating tables in test database') driver = PsqlGraphDriver(host, user, password, database) Base.metadata.create_all(driver.engine)
def pg_driver(request): pg_driver = PsqlGraphDriver(**pg_config()) def closeConnection(): pg_driver.engine.dispose() request.addfinalizer(closeConnection) return pg_driver
def setUpClass(cls): host = 'localhost' user = '******' password = '******' database = 'automated_test' cls.g = PsqlGraphDriver(host, user, password, database) cls._clear_tables()
def translate_node_range(_args): args, offset = _args src = PsqlGraphDriver(args.source_host, args.source_user, args.source_password, args.source, **driver_kwargs) dst = PsqlGraphDriver(args.dest_host, args.dest_user, args.dest_password, args.dest, **driver_kwargs) with src.session_scope() as session: with dst.session_scope() as session: for old in src.nodes(OldNode).order_by(OldNode.node_id)\ .offset(offset)\ .limit(BLOCK)\ .yield_per(BLOCK): try: new = PolyNode( node_id=old.node_id, properties=old.properties, system_annotations=old.system_annotations, acl=old.acl, label=old.label, ) new.created = old.created session.merge(new) except Exception as e: logging.error("unable to add node {}, {}".format( old.label, old.node_id)) logging.error(e) try: session.commit() except Exception as e: logging.error(e)
def test_priv_grant_write(self): """Test ability to grant read/write privs""" self.create_all_tables() self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'") try: g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database) pgadmin.main(pgadmin.get_parser().parse_args([ 'graph-grant', '--write=pytest', ] + self.base_args)) with g.session_scope() as s: g.nodes().count() s.merge(models.Case('1')) finally: self.engine.execute("DROP OWNED BY pytest; DROP USER pytest")
def db_init(app): app.logger.info('Initializing PsqlGraph driver') app.db = PsqlGraphDriver( host=app.config['PSQLGRAPH']['host'], user=app.config['PSQLGRAPH']['user'], password=app.config['PSQLGRAPH']['password'], database=app.config['PSQLGRAPH']['database'], set_flush_timestamps=True, )
def get_driver(host, user, password, database): return PsqlGraphDriver( host=host, user=user, password=password, database=database, set_flush_timestamps=True, connect_args={"application_name": app_name}, )
def db_init(app): app.logger.info("Initializing PsqlGraph driver") app.db = PsqlGraphDriver( host=app.config["PSQLGRAPH"]["host"], user=app.config["PSQLGRAPH"]["user"], password=app.config["PSQLGRAPH"]["password"], database=app.config["PSQLGRAPH"]["database"], set_flush_timestamps=True, )
def update_classes(graph_kwargs, input_q): """Creates a db driver and pulls classes from the queue to update""" graph = PsqlGraphDriver(**graph_kwargs) while True: cls = input_q.get() if cls is None: # none means no more work return update_cls(graph, cls)
def main(): parser = argparse.ArgumentParser() parser.add_argument("-H", "--host", type=str, action="store", required=True, help="psql-server host") parser.add_argument("-U", "--user", type=str, action="store", required=True, help="psql test user") parser.add_argument("-D", "--database", type=str, action="store", required=True, help="psql test database") parser.add_argument("-P", "--password", type=str, action="store", help="psql test password") args = parser.parse_args() prompt = "Password for {}:".format(args.user) password = args.password or getpass.getpass(prompt) g = PsqlGraphDriver(args.host, args.user, password, args.database) with g.session_scope(): projects = g.nodes(md.Project).not_props(state='legacy').all() map(update_project_related_case_cache, projects) print("Done.")
def bg(): """Fixture for database driver""" cfg = { 'host': 'localhost', 'user': '******', 'password': '******', 'database': 'dev_models', 'package_namespace': 'basic', } g = PsqlGraphDriver(**cfg) create_tables(g.engine, namespace="basic") yield g truncate(g.engine, namespace="basic")
def db_init(app): app.logger.info('Initializing PsqlGraph driver') app.db = PsqlGraphDriver( host=app.config['PSQLGRAPH']['host'], user=app.config['PSQLGRAPH']['user'], password=app.config['PSQLGRAPH']['password'], database=app.config['PSQLGRAPH']['database'], set_flush_timestamps=True, ) app.oauth2 = OAuth2Client(**app.config['OAUTH2']) app.logger.info('Initializing Indexd driver') app.signpost = IndexClient(app.config['SIGNPOST']['host'], version=app.config['SIGNPOST']['version'], auth=app.config['SIGNPOST']['auth'])
def db_init(app): app.logger.info('Initializing PsqlGraph driver') app.db = PsqlGraphDriver( host=app.config['PSQLGRAPH']['host'], user=app.config['PSQLGRAPH']['user'], password=app.config['PSQLGRAPH']['password'], database=app.config['PSQLGRAPH']['database'], set_flush_timestamps=True, ) if app.config.get('AUTO_MIGRATE_DATABASE'): migrate_database(app) app.oauth_client = oauth2_client.OAuthClient(**app.config['OAUTH2']) app.logger.info('Initializing Signpost driver') app.signpost = SignpostClient(app.config['SIGNPOST']['host'], version=app.config['SIGNPOST']['version'], auth=app.config['SIGNPOST']['auth'])
def db_init(app): app.logger.info("Initializing PsqlGraph driver") app.db = PsqlGraphDriver( host=app.config["PSQLGRAPH"]["host"], user=app.config["PSQLGRAPH"]["user"], password=app.config["PSQLGRAPH"]["password"], database=app.config["PSQLGRAPH"]["database"], set_flush_timestamps=True, ) app.oauth2 = OAuth2Client(**app.config["OAUTH2"]) app.logger.info("Initializing Indexd driver") app.index_client = IndexClient( app.config["INDEX_CLIENT"]["host"], version=app.config["INDEX_CLIENT"]["version"], auth=app.config["INDEX_CLIENT"]["auth"], )
def translate_edge_range(_args): args, offset = _args src = PsqlGraphDriver(args.source_host, args.source_user, args.source_password, args.source, **driver_kwargs) dst = PsqlGraphDriver(args.dest_host, args.dest_user, args.dest_password, args.dest, **driver_kwargs) print '{}-{}'.format(offset, offset + BLOCK) sys.stdout.flush() with src.session_scope() as session: with dst.session_scope() as session: for old in src.edges(OldEdge)\ .order_by((OldEdge.src_id), (OldEdge.dst_id), (OldEdge.label))\ .options(joinedload(OldEdge.src))\ .options(joinedload(OldEdge.dst))\ .offset(offset)\ .limit(BLOCK).all(): try: Type = dst.get_edge_by_labels(old.src.label, old.label, old.dst.label) print Type.__name__ new = Type( src_id=old.src_id, dst_id=old.dst_id, properties=old.properties, system_annotations=old.system_annotations, label=old.label, ) new.created = old.created session.merge(new) except Exception as e: logging.error("unable to add edge {}, {}".format( old.label, old.src_id, old.dst_id)) logging.error(e) try: session.commit() except Exception as e: logging.error(e)
def db_init(app): app.logger.info("Initializing PsqlGraph driver") app.db = PsqlGraphDriver( host=app.config["PSQLGRAPH"]["host"], user=app.config["PSQLGRAPH"]["user"], password=app.config["PSQLGRAPH"]["password"], database=app.config["PSQLGRAPH"]["database"], set_flush_timestamps=True, ) if app.config.get("AUTO_MIGRATE_DATABASE"): migrate_database(app) app.oauth_client = oauth2_client.OAuthClient(**app.config["OAUTH2"]) app.logger.info("Initializing index client") app.index_client = IndexClient( app.config["INDEX_CLIENT"]["host"], version=app.config["INDEX_CLIENT"]["version"], auth=app.config["INDEX_CLIENT"]["auth"], )
def pg_driver(request, client): pg_driver = PsqlGraphDriver(**pg_config()) def tearDown(): with pg_driver.engine.begin() as conn: for table in models.Node().get_subclass_table_names(): if table != models.Node.__tablename__: conn.execute('delete from {}'.format(table)) for table in models.Edge().get_subclass_table_names(): if table != models.Edge.__tablename__: conn.execute('delete from {}'.format(table)) conn.execute('delete from versioned_nodes') conn.execute('delete from _voided_nodes') conn.execute('delete from _voided_edges') conn.execute('delete from transaction_snapshots') conn.execute('delete from transaction_documents') conn.execute('delete from transaction_logs') tearDown() request.addfinalizer(tearDown) return pg_driver
def pg_driver(request, client, use_ssl, isolation_level): pg_driver = PsqlGraphDriver( **pg_config(use_ssl=use_ssl, isolation_level=isolation_level)) def tearDown(): with pg_driver.engine.begin() as conn: for table in models.Node().get_subclass_table_names(): if table != models.Node.__tablename__: conn.execute("delete from {}".format(table)) # nosec for table in models.Edge().get_subclass_table_names(): if table != models.Edge.__tablename__: conn.execute("delete from {}".format(table)) # nosec conn.execute("delete from versioned_nodes") conn.execute("delete from _voided_nodes") conn.execute("delete from _voided_edges") conn.execute("delete from transaction_snapshots") conn.execute("delete from transaction_documents") conn.execute("delete from transaction_logs") tearDown() request.addfinalizer(tearDown) return pg_driver
def translate_edge_range(_args): args, offset = _args src = PsqlGraphDriver( args.source_host, args.source_user, args.source_password, args.source, **driver_kwargs) dst = PsqlGraphDriver( args.dest_host, args.dest_user, args.dest_password, args.dest, **driver_kwargs) print '{}-{}'.format(offset, offset+BLOCK) sys.stdout.flush() with src.session_scope() as session: with dst.session_scope() as session: for old in src.edges(OldEdge)\ .order_by((OldEdge.src_id), (OldEdge.dst_id), (OldEdge.label))\ .options(joinedload(OldEdge.src))\ .options(joinedload(OldEdge.dst))\ .offset(offset)\ .limit(BLOCK).all(): try: Type = dst.get_edge_by_labels( old.src.label, old.label, old.dst.label) print Type.__name__ new = Type( src_id=old.src_id, dst_id=old.dst_id, properties=old.properties, system_annotations=old.system_annotations, label=old.label, ) new.created = old.created session.merge(new) except Exception as e: logging.error("unable to add edge {}, {}".format( old.label, old.src_id, old.dst_id)) logging.error(e) try: session.commit() except Exception as e: logging.error(e)
def g(db_config, tables_created): """Fixture for database driver""" return PsqlGraphDriver(**db_config)
class TestGDCPostgresAdmin(unittest.TestCase): logger = logging.getLogger('TestGDCPostgresAdmin') logger.setLevel(logging.INFO) host = 'localhost' user = '******' database = 'automated_test' base_args = [ '-H', host, '-U', user, '-D', database, ] g = PsqlGraphDriver(host, user, '', database) root_con_str = "postgres://{user}:{pwd}@{host}/{db}".format(user=user, host=host, pwd='', db=database) engine = pgadmin.create_engine(root_con_str) @classmethod def tearDownClass(cls): """Recreate the database for tests that follow. """ cls.create_all_tables() # Re-grant permissions to test user for scls in Node.__subclasses__() + Edge.__subclasses__(): statment = ("GRANT ALL PRIVILEGES ON TABLE {} TO test".format( scls.__tablename__)) cls.engine.execute('BEGIN; %s; COMMIT;' % statment) @classmethod def drop_all_tables(cls): for scls in Node.__subclasses__(): try: cls.engine.execute("DROP TABLE {} CASCADE".format( scls.__tablename__)) except Exception as e: cls.logger.warning(e) @classmethod def create_all_tables(cls): parser = pgadmin.get_parser() args = parser.parse_args( ['graph-create', '--delay', '1', '--retries', '0'] + cls.base_args) pgadmin.main(args) @classmethod def drop_a_table(cls): cls.engine.execute('DROP TABLE edge_clinicaldescribescase') cls.engine.execute('DROP TABLE node_clinical') def startTestRun(self): self.drop_all_tables() def setUp(self): self.drop_all_tables() def test_args(self): parser = pgadmin.get_parser() parser.parse_args(['graph-create'] + self.base_args) def test_create_single(self): """Test simple table creation""" pgadmin.main(pgadmin.get_parser().parse_args( ['graph-create', '--delay', '1', '--retries', '0'] + self.base_args)) self.engine.execute('SELECT * from node_case') def test_create_double(self): """Test idempotency of table creation""" pgadmin.main(pgadmin.get_parser().parse_args( ['graph-create', '--delay', '1', '--retries', '0'] + self.base_args)) self.engine.execute('SELECT * from node_case') @pytest.mark.skip(reason="Causing race conditions, so skipping for now.") def test_create_fails_blocked_without_force(self): """Test table creation fails when blocked w/o force""" q = Queue() # to communicate with blocking process args = pgadmin.get_parser().parse_args( ['graph-create', '--delay', '1', '--retries', '1'] + self.base_args) pgadmin.main(args) self.drop_a_table() def blocker(): with self.g.session_scope() as s: s.merge(models.Case('1')) q.put(0) # Tell main thread we're ready q.get() # Wait for main thread to tell us to exit p = Process(target=blocker) p.daemon = True p.start() q.get() with self.assertRaises(RuntimeError): pgadmin.main(args) q.put(0) p.terminate() @pytest.mark.skip(reason="This test is causing race conditions.") def test_create_force(self): """Test ability to force table creation""" q = Queue() # to communicate with blocking process args = pgadmin.get_parser().parse_args( ['graph-create', '--delay', '1', '--retries', '1', '--force'] + self.base_args) pgadmin.main(args) self.drop_a_table() def blocker(): with self.g.session_scope() as s: s.merge(models.Case('1')) q.put(0) # Tell main thread we're ready q.get() # This get should block until this prcoess is killed assert False, 'Should not be reachable!' p = Process(target=blocker) p.daemon = True p.start() q.get() try: pgadmin.main(args) except: p.terminate() raise q.put(0) p.terminate() def test_priv_grant_read(self): """Test ability to grant read but not write privs""" self.create_all_tables() try: self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'") g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database) #: If this failes, this test (not the code) is wrong! with self.assertRaises(ProgrammingError): with g.session_scope(): g.nodes().count() pgadmin.main(pgadmin.get_parser().parse_args([ 'graph-grant', '--read=pytest', ] + self.base_args)) with g.session_scope(): g.nodes().count() with self.assertRaises(ProgrammingError): with g.session_scope() as s: s.merge(models.Case('1')) finally: self.engine.execute("DROP OWNED BY pytest; DROP USER pytest") def test_priv_grant_write(self): """Test ability to grant read/write privs""" self.create_all_tables() try: self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'") g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database) pgadmin.main(pgadmin.get_parser().parse_args([ 'graph-grant', '--write=pytest', ] + self.base_args)) with g.session_scope() as s: g.nodes().count() s.merge(models.Case('1')) finally: self.engine.execute("DROP OWNED BY pytest; DROP USER pytest") def test_priv_revoke_read(self): """Test ability to revoke read privs""" self.create_all_tables() try: self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'") g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database) pgadmin.main(pgadmin.get_parser().parse_args([ 'graph-grant', '--read=pytest', ] + self.base_args)) pgadmin.main(pgadmin.get_parser().parse_args([ 'graph-revoke', '--read=pytest', ] + self.base_args)) with self.assertRaises(ProgrammingError): with g.session_scope() as s: g.nodes().count() s.merge(models.Case('1')) finally: self.engine.execute("DROP OWNED BY pytest; DROP USER pytest") def test_priv_revoke_write(self): """Test ability to revoke read/write privs""" self.create_all_tables() try: self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'") g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database) pgadmin.main(pgadmin.get_parser().parse_args([ 'graph-grant', '--write=pytest', ] + self.base_args)) pgadmin.main(pgadmin.get_parser().parse_args([ 'graph-revoke', '--write=pytest', ] + self.base_args)) with g.session_scope() as s: g.nodes().count() with self.assertRaises(ProgrammingError): with g.session_scope() as s: s.merge(models.Case('1')) finally: self.engine.execute("DROP OWNED BY pytest; DROP USER pytest")
class MetadataImporter(object): def __init__(self,metadata={}): self.metadata = metadata self.driver=PsqlGraphDriver(settings.METADATA_DB['HOST'], settings.METADATA_DB['USER'],settings.METADATA_DB['PASSWORD'], settings.METADATA_DB['NAME']) self.signpost = SignpostClient(settings.SIGNPOST_URL,version='v0') def find_props(self,props): return self.driver.nodes().labels('dataset').props(props).count()>0 def validate_metadata(self): for key in required_field: if key not in self.metadata: print "%s not provided" % key return False if 'short_description' not in self.metadata: print 'short description not provided, use description as short description' self.metadata['short_description']= self.metadata['description'] if 'slug' not in self.metadata: self.metadata['slug'] = "-".join(self.metadata['title'].lower().split(" ")) if 'availability_mechanism' not in self.metadata: self.metadata['availability_mechanism']='udr, rsync' if self.find_props({'slug':self.metadata['slug']}): print 'slug "%s" exist before, please change a slug' % self.metadata['slug'] return False if 'url' not in self.metadata or self.metadata['url'].strip()=='': self.metadata['url'] = urlparse.urljoin(ROOT_URL,self.metadata['slug']) return True def search_identifier(self,ark): while self.signpost.search(ark): new_ark = 'ark:/31807/osdc-' + binascii.b2a_hex(os.urandom(8)) print '%s exists, create new ark %s' % (ark,new_ark) ark = new_ark return ark.split(":")[-1] def import_keywords(self): nodes = [] for keyword in self.metadata['keywords'].split(","): keyword=keyword.strip() node = self.driver.nodes().labels('keyword').props({'value':keyword}).first() if not node: doc = self.signpost.create() node = Node(label='keyword',node_id = doc.did,properties = {'value':keyword}) self.driver.node_merge(node=node) print 'create new keyword %s' % keyword nodes.append(node) return nodes def import_metadata(self): with self.driver.session_scope(): if not self.validate_metadata(): return doc = self.signpost.create() doc.urls=[self.metadata['url']] doc.identifiers = { 'ark':self.search_identifier('ark:/31807/osdc-'+doc.did.split('-')[0]) } doc.patch() properties = self.metadata.copy() del properties['url'] del properties['keywords'] node = Node(node_id=doc.did,label='dataset',properties=properties) self.driver.node_merge(node=node) keyword_nodes = self.import_keywords() for keyword in keyword_nodes: self.driver.edge_insert(Edge(node.node_id,keyword.node_id,'member_of')) print 'metadata %s created' % doc.did def delete_metadata(self,did): with self.driver.session_scope(): node = self.driver.nodes().ids(did).first() if node: self.driver.node_delete(node_id=did) doc = self.signpost.get(did) doc.delete()
def __init__(self,metadata={}): self.metadata = metadata self.driver=PsqlGraphDriver(settings.METADATA_DB['HOST'], settings.METADATA_DB['USER'],settings.METADATA_DB['PASSWORD'], settings.METADATA_DB['NAME']) self.signpost = SignpostClient(settings.SIGNPOST_URL,version='v0')
from signpostclient import SignpostClient #This needs work -- a lot of assumptions, lack of error checking and general ugliness osdc_prefix = 'osdc' #does not support time zones -- for now leave it up to the submission scripts to put in UTC. time_format = '%Y-%m-%d %H:%M:%S' valid_keys = [ 'source', 'source_url', 'description', 'short_description', 'keyword', 'size', 'modified', 'license', 'osdc_location', 'osdc_folder', 'osdc_hs_location', 'osdc_hs_folder' ] pg_driver = PsqlGraphDriver(METADATA_DB['HOST'], METADATA_DB['USER'], METADATA_DB['PASSWORD'], METADATA_DB['NAME']) signpost = SignpostClient(SIGNPOST_URL, version='v0') def init_keys(): for key in valid_keys: k = Key(key_name=key, public=True) k.save() def add_dataset(title, prefix): key = str(uuid.uuid4()) slug = slugify(title) d = DataSet(key=key, prefix=prefix, title=title, slug=slug)
class TestGDCPostgresAdmin(unittest.TestCase): logger = logging.getLogger('TestGDCPostgresAdmin') logger.setLevel(logging.INFO) host = 'localhost' user = '******' database = 'automated_test' base_args = [ '-H', host, '-U', user, '-D', database, ] g = PsqlGraphDriver(host, user, '', database) root_con_str = "postgres://{user}:{pwd}@{host}/{db}".format(user=user, host=host, pwd='', db=database) engine = pgadmin.create_engine(root_con_str) @classmethod def tearDownClass(cls): """Recreate the database for tests that follow. """ cls.create_all_tables() # Re-grant permissions to test user for scls in Node.get_subclasses() + Edge.get_subclasses(): statment = ("GRANT ALL PRIVILEGES ON TABLE {} TO test".format( scls.__tablename__)) cls.engine.execute('BEGIN; %s; COMMIT;' % statment) @classmethod def drop_all_tables(cls): for scls in Node.get_subclasses(): try: cls.engine.execute("DROP TABLE {} CASCADE".format( scls.__tablename__)) except Exception as e: cls.logger.warning(e) @classmethod def create_all_tables(cls): parser = pgadmin.get_parser() args = parser.parse_args( ['graph-create', '--delay', '1', '--retries', '0'] + cls.base_args) pgadmin.main(args) @classmethod def drop_a_table(cls): cls.engine.execute('DROP TABLE edge_clinicaldescribescase') cls.engine.execute('DROP TABLE node_clinical') def startTestRun(self): self.drop_all_tables() def setUp(self): self.drop_all_tables() def test_args(self): parser = pgadmin.get_parser() parser.parse_args(['graph-create'] + self.base_args) def test_create_single(self): """Test simple table creation""" pgadmin.main(pgadmin.get_parser().parse_args( ['graph-create', '--delay', '1', '--retries', '0'] + self.base_args)) self.engine.execute('SELECT * from node_case') def test_create_double(self): """Test idempotency of table creation""" pgadmin.main(pgadmin.get_parser().parse_args( ['graph-create', '--delay', '1', '--retries', '0'] + self.base_args)) self.engine.execute('SELECT * from node_case') def test_priv_grant_read(self): """Test ability to grant read but not write privs""" self.create_all_tables() try: self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'") self.engine.execute("GRANT USAGE ON SCHEMA public TO pytest") g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database) #: If this failes, this test (not the code) is wrong! with self.assertRaises(ProgrammingError): with g.session_scope(): g.nodes().count() pgadmin.main(pgadmin.get_parser().parse_args([ 'graph-grant', '--read=pytest', ] + self.base_args)) with g.session_scope(): g.nodes().count() with self.assertRaises(ProgrammingError): with g.session_scope() as s: s.merge(models.Case('1')) finally: self.engine.execute("DROP OWNED BY pytest; DROP USER pytest") def test_priv_grant_write(self): """Test ability to grant read/write privs""" self.create_all_tables() try: self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'") self.engine.execute("GRANT USAGE ON SCHEMA public TO pytest") g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database) pgadmin.main(pgadmin.get_parser().parse_args([ 'graph-grant', '--write=pytest', ] + self.base_args)) with g.session_scope() as s: g.nodes().count() s.merge(models.Case('1')) finally: self.engine.execute("DROP OWNED BY pytest; DROP USER pytest") def test_priv_revoke_read(self): """Test ability to revoke read privs""" self.create_all_tables() try: self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'") self.engine.execute("GRANT USAGE ON SCHEMA public TO pytest") g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database) pgadmin.main(pgadmin.get_parser().parse_args([ 'graph-grant', '--read=pytest', ] + self.base_args)) pgadmin.main(pgadmin.get_parser().parse_args([ 'graph-revoke', '--read=pytest', ] + self.base_args)) with self.assertRaises(ProgrammingError): with g.session_scope() as s: g.nodes().count() s.merge(models.Case('1')) finally: self.engine.execute("DROP OWNED BY pytest; DROP USER pytest") def test_priv_revoke_write(self): """Test ability to revoke read/write privs""" self.create_all_tables() try: self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'") self.engine.execute("GRANT USAGE ON SCHEMA public TO pytest") g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database) pgadmin.main(pgadmin.get_parser().parse_args([ 'graph-grant', '--write=pytest', ] + self.base_args)) pgadmin.main(pgadmin.get_parser().parse_args([ 'graph-revoke', '--write=pytest', ] + self.base_args)) with g.session_scope() as s: g.nodes().count() with self.assertRaises(ProgrammingError): with g.session_scope() as s: s.merge(models.Case('1')) finally: self.engine.execute("DROP OWNED BY pytest; DROP USER pytest")