def update_legacy_states(graph_kwargs):
"""Updates state, file_state on legacy nodes
- node.state in {None, 'live'}
- node.project_id in {None, <Legacy project_id list>}
there is no project_id, or project_id points to a legacy project
"""
graph = PsqlGraphDriver(**graph_kwargs)
with graph.session_scope():
print_cls_query_summary(graph)
input_q = Queue()
pool = [
Process(target=update_classes, args=(graph_kwargs, input_q))
for _ in range(cpu_count())
]
for cls in CLS_TO_UPDATE:
input_q.put(cls)
for process in pool:
input_q.put(None) # put a no more work signal for each process
for process in pool:
process.start()
for process in pool:
process.join()
def test_priv_revoke_read(self):
"""Test ability to revoke read privs"""
self.create_all_tables()
try:
self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'")
g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database)
pgadmin.main(pgadmin.get_parser().parse_args([
'graph-grant',
'--read=pytest',
] + self.base_args))
pgadmin.main(pgadmin.get_parser().parse_args([
'graph-revoke',
'--read=pytest',
] + self.base_args))
with self.assertRaises(ProgrammingError):
with g.session_scope() as s:
g.nodes().count()
s.merge(models.Case('1'))
finally:
self.engine.execute("DROP OWNED BY pytest; DROP USER pytest")
def translate_node_range(_args):
args, offset = _args
src = PsqlGraphDriver(
args.source_host, args.source_user, args.source_password,
args.source, **driver_kwargs)
dst = PsqlGraphDriver(
args.dest_host, args.dest_user, args.dest_password,
args.dest, **driver_kwargs)
with src.session_scope() as session:
with dst.session_scope() as session:
for old in src.nodes(OldNode).order_by(OldNode.node_id)\
.offset(offset)\
.limit(BLOCK)\
.yield_per(BLOCK):
try:
new = PolyNode(
node_id=old.node_id,
properties=old.properties,
system_annotations=old.system_annotations,
acl=old.acl,
label=old.label,
)
new.created = old.created
session.merge(new)
except Exception as e:
logging.error("unable to add node {}, {}".format(
old.label, old.node_id))
logging.error(e)
try:
session.commit()
except Exception as e:
logging.error(e)
def test_priv_revoke_write(self):
"""Test ability to revoke read/write privs"""
self.create_all_tables()
self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'")
try:
g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database)
pgadmin.main(pgadmin.get_parser().parse_args([
'graph-grant', '--write=pytest',
] + self.base_args))
pgadmin.main(pgadmin.get_parser().parse_args([
'graph-revoke', '--write=pytest',
] + self.base_args))
with g.session_scope() as s:
g.nodes().count()
with self.assertRaises(ProgrammingError):
with g.session_scope() as s:
s.merge(models.Case('1'))
finally:
self.engine.execute("DROP OWNED BY pytest; DROP USER pytest")
def translate_nodes(args):
src = PsqlGraphDriver(args.source_host, args.source_user,
args.source_password, args.source, **driver_kwargs)
with src.session_scope():
count = src.nodes(OldNode).count()
offsets = [i * BLOCK for i in range(count / BLOCK + 1)]
pool = Pool(args.nprocs)
args = [(args, offset) for offset in offsets]
pool.map_async(translate_node_range, args).get(int(1e9))
def translate_nodes(args):
src = PsqlGraphDriver(
args.source_host, args.source_user, args.source_password,
args.source, **driver_kwargs)
with src.session_scope():
count = src.nodes(OldNode).count()
offsets = [i*BLOCK for i in range(count/BLOCK+1)]
pool = Pool(args.nprocs)
args = [(args, offset) for offset in offsets]
pool.map_async(translate_node_range, args).get(int(1e9))
def test_priv_grant_read(self):
"""Test ability to grant read but not write privs"""
self.create_all_tables()
try:
self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'")
g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database)
#: If this failes, this test (not the code) is wrong!
with self.assertRaises(ProgrammingError):
with g.session_scope():
g.nodes().count()
pgadmin.main(pgadmin.get_parser().parse_args([
'graph-grant',
'--read=pytest',
] + self.base_args))
with g.session_scope():
g.nodes().count()
with self.assertRaises(ProgrammingError):
with g.session_scope() as s:
s.merge(models.Case('1'))
finally:
self.engine.execute("DROP OWNED BY pytest; DROP USER pytest")
def test_custom_application_name(self):
cmd = "select application_name from pg_stat_activity;"
custom_name = '_CUSTOM_NAME'
g_ = PsqlGraphDriver(host,
user,
password,
database,
application_name=custom_name)
with g_.session_scope() as s:
s.merge(Test('a'))
app_names = {r[0] for r in g.engine.execute(cmd)}
self.assertIn(custom_name, app_names)
def test_priv_grant_read(self):
"""Test ability to grant read but not write privs"""
self.create_all_tables()
self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'")
try:
g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database)
#: If this failes, this test (not the code) is wrong!
with self.assertRaises(ProgrammingError):
with g.session_scope():
g.nodes().count()
pgadmin.main(pgadmin.get_parser().parse_args([
'graph-grant', '--read=pytest',
] + self.base_args))
with g.session_scope():
g.nodes().count()
with self.assertRaises(ProgrammingError):
with g.session_scope() as s:
s.merge(models.Case('1'))
finally:
self.engine.execute("DROP OWNED BY pytest; DROP USER pytest")
def migrate(host, user, password, database):
graph = PsqlGraphDriver(host, user, password, database)
with graph.session_scope():
for project, phsids in mapping.PROJECT_TO_PHSID.iteritems():
tokens = project.split('-')
program_code = tokens[0]
project_code = '-'.join(tokens[1:])
program_phsid = mapping.get_program_level_phsid(project)
project_phsid = mapping.get_project_level_phsid(project)
program = graph.nodes(Program).props(name=program_code).first()
project = graph.nodes(Project).props(code=project_code).first()
if program:
program.props['dbgap_accession_number'] = program_phsid
if project:
project.props['dbgap_accession_number'] = project_phsid
def migrate(host, user, password, database):
graph = PsqlGraphDriver(host, user, password, database)
with graph.session_scope():
for project, phsids in mapping.PROJECT_TO_PHSID.iteritems():
tokens = project.split('-')
program_code = tokens[0]
project_code = '-'.join(tokens[1:])
program_phsid = mapping.get_program_level_phsid(project)
project_phsid = mapping.get_project_level_phsid(project)
program = graph.nodes(Program).props(name=program_code).first()
project = graph.nodes(Project).props(code=project_code).first()
if program:
program.props['dbgap_accession_number'] = program_phsid
if project:
project.props['dbgap_accession_number'] = project_phsid
def db_init(app):
app.logger.info("Initializing PsqlGraph driver")
connect_args = {}
if app.config.get("PSQLGRAPH") and app.config["PSQLGRAPH"].get("sslmode"):
connect_args["sslmode"] = app.config["PSQLGRAPH"]["sslmode"]
app.db = PsqlGraphDriver(
host=app.config["PSQLGRAPH"]["host"],
user=app.config["PSQLGRAPH"]["user"],
password=app.config["PSQLGRAPH"]["password"],
database=app.config["PSQLGRAPH"]["database"],
set_flush_timestamps=True,
connect_args=connect_args,
isolation_level=app.config["PSQLGRAPH"].get(
"isolation_level", "READ_COMMITTED"
),
)
if app.config.get("AUTO_MIGRATE_DATABASE"):
migrate_database(app)
app.oauth_client = oauth2_client.OAuthClient(**app.config["OAUTH2"])
app.logger.info("Initializing index client")
app.index_client = IndexClient(
app.config["INDEX_CLIENT"]["host"],
version=app.config["INDEX_CLIENT"]["version"],
auth=app.config["INDEX_CLIENT"]["auth"],
)
def db_init(app):
app.logger.info('Initializing PsqlGraph driver')
app.db = PsqlGraphDriver(
host=app.config['PSQLGRAPH']['host'],
user=app.config['PSQLGRAPH']['user'],
password=app.config['PSQLGRAPH']['password'],
database=app.config['PSQLGRAPH']['database'],
set_flush_timestamps=True,
)
app.userdb = SQLAlchemyDriver(app.config['PSQL_USER_DB_CONNECTION'])
flask_scoped_session(app.userdb.Session, app)
app.oauth2 = OAuth2Client(**app.config['OAUTH2'])
app.logger.info('Initializing Signpost driver')
app.signpost = SignpostClient(
app.config['SIGNPOST']['host'],
version=app.config['SIGNPOST']['version'],
auth=app.config['SIGNPOST']['auth'])
try:
app.logger.info('Initializing Auth driver')
app.auth = AuthDriver(app.config["AUTH_ADMIN_CREDS"], app.config["INTERNAL_AUTH"])
except Exception:
app.logger.exception("Couldn't initialize auth, continuing anyway")
def create_tables(host, user, password, database):
"""
create a table
"""
print('Creating tables in test database')
driver = PsqlGraphDriver(host, user, password, database)
Base.metadata.create_all(driver.engine)
def pg_driver(request):
pg_driver = PsqlGraphDriver(**pg_config())
def closeConnection():
pg_driver.engine.dispose()
request.addfinalizer(closeConnection)
return pg_driver
def setUpClass(cls):
host = 'localhost'
user = '******'
password = '******'
database = 'automated_test'
cls.g = PsqlGraphDriver(host, user, password, database)
cls._clear_tables()
def translate_node_range(_args):
args, offset = _args
src = PsqlGraphDriver(args.source_host, args.source_user,
args.source_password, args.source, **driver_kwargs)
dst = PsqlGraphDriver(args.dest_host, args.dest_user, args.dest_password,
args.dest, **driver_kwargs)
with src.session_scope() as session:
with dst.session_scope() as session:
for old in src.nodes(OldNode).order_by(OldNode.node_id)\
.offset(offset)\
.limit(BLOCK)\
.yield_per(BLOCK):
try:
new = PolyNode(
node_id=old.node_id,
properties=old.properties,
system_annotations=old.system_annotations,
acl=old.acl,
label=old.label,
)
new.created = old.created
session.merge(new)
except Exception as e:
logging.error("unable to add node {}, {}".format(
old.label, old.node_id))
logging.error(e)
try:
session.commit()
except Exception as e:
logging.error(e)
def test_priv_grant_write(self):
"""Test ability to grant read/write privs"""
self.create_all_tables()
self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'")
try:
g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database)
pgadmin.main(pgadmin.get_parser().parse_args([
'graph-grant', '--write=pytest',
] + self.base_args))
with g.session_scope() as s:
g.nodes().count()
s.merge(models.Case('1'))
finally:
self.engine.execute("DROP OWNED BY pytest; DROP USER pytest")
def db_init(app):
app.logger.info('Initializing PsqlGraph driver')
app.db = PsqlGraphDriver(
host=app.config['PSQLGRAPH']['host'],
user=app.config['PSQLGRAPH']['user'],
password=app.config['PSQLGRAPH']['password'],
database=app.config['PSQLGRAPH']['database'],
set_flush_timestamps=True,
)
def get_driver(host, user, password, database):
return PsqlGraphDriver(
host=host,
user=user,
password=password,
database=database,
set_flush_timestamps=True,
connect_args={"application_name": app_name},
)
def db_init(app):
app.logger.info("Initializing PsqlGraph driver")
app.db = PsqlGraphDriver(
host=app.config["PSQLGRAPH"]["host"],
user=app.config["PSQLGRAPH"]["user"],
password=app.config["PSQLGRAPH"]["password"],
database=app.config["PSQLGRAPH"]["database"],
set_flush_timestamps=True,
)
def update_classes(graph_kwargs, input_q):
"""Creates a db driver and pulls classes from the queue to update"""
graph = PsqlGraphDriver(**graph_kwargs)
while True:
cls = input_q.get()
if cls is None: # none means no more work
return
update_cls(graph, cls)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-H", "--host", type=str, action="store",
required=True, help="psql-server host")
parser.add_argument("-U", "--user", type=str, action="store",
required=True, help="psql test user")
parser.add_argument("-D", "--database", type=str, action="store",
required=True, help="psql test database")
parser.add_argument("-P", "--password", type=str, action="store",
help="psql test password")
args = parser.parse_args()
prompt = "Password for {}:".format(args.user)
password = args.password or getpass.getpass(prompt)
g = PsqlGraphDriver(args.host, args.user, password, args.database)
with g.session_scope():
projects = g.nodes(md.Project).not_props(state='legacy').all()
map(update_project_related_case_cache, projects)
print("Done.")
def bg():
"""Fixture for database driver"""
cfg = {
'host': 'localhost',
'user': '******',
'password': '******',
'database': 'dev_models',
'package_namespace': 'basic',
}
g = PsqlGraphDriver(**cfg)
create_tables(g.engine, namespace="basic")
yield g
truncate(g.engine, namespace="basic")
def db_init(app):
app.logger.info('Initializing PsqlGraph driver')
app.db = PsqlGraphDriver(
host=app.config['PSQLGRAPH']['host'],
user=app.config['PSQLGRAPH']['user'],
password=app.config['PSQLGRAPH']['password'],
database=app.config['PSQLGRAPH']['database'],
set_flush_timestamps=True,
)
app.oauth2 = OAuth2Client(**app.config['OAUTH2'])
app.logger.info('Initializing Indexd driver')
app.signpost = IndexClient(app.config['SIGNPOST']['host'],
version=app.config['SIGNPOST']['version'],
auth=app.config['SIGNPOST']['auth'])
def db_init(app):
app.logger.info('Initializing PsqlGraph driver')
app.db = PsqlGraphDriver(
host=app.config['PSQLGRAPH']['host'],
user=app.config['PSQLGRAPH']['user'],
password=app.config['PSQLGRAPH']['password'],
database=app.config['PSQLGRAPH']['database'],
set_flush_timestamps=True,
)
if app.config.get('AUTO_MIGRATE_DATABASE'):
migrate_database(app)
app.oauth_client = oauth2_client.OAuthClient(**app.config['OAUTH2'])
app.logger.info('Initializing Signpost driver')
app.signpost = SignpostClient(app.config['SIGNPOST']['host'],
version=app.config['SIGNPOST']['version'],
auth=app.config['SIGNPOST']['auth'])
def db_init(app):
app.logger.info("Initializing PsqlGraph driver")
app.db = PsqlGraphDriver(
host=app.config["PSQLGRAPH"]["host"],
user=app.config["PSQLGRAPH"]["user"],
password=app.config["PSQLGRAPH"]["password"],
database=app.config["PSQLGRAPH"]["database"],
set_flush_timestamps=True,
)
app.oauth2 = OAuth2Client(**app.config["OAUTH2"])
app.logger.info("Initializing Indexd driver")
app.index_client = IndexClient(
app.config["INDEX_CLIENT"]["host"],
version=app.config["INDEX_CLIENT"]["version"],
auth=app.config["INDEX_CLIENT"]["auth"],
)
def translate_edge_range(_args):
args, offset = _args
src = PsqlGraphDriver(args.source_host, args.source_user,
args.source_password, args.source, **driver_kwargs)
dst = PsqlGraphDriver(args.dest_host, args.dest_user, args.dest_password,
args.dest, **driver_kwargs)
print '{}-{}'.format(offset, offset + BLOCK)
sys.stdout.flush()
with src.session_scope() as session:
with dst.session_scope() as session:
for old in src.edges(OldEdge)\
.order_by((OldEdge.src_id),
(OldEdge.dst_id),
(OldEdge.label))\
.options(joinedload(OldEdge.src))\
.options(joinedload(OldEdge.dst))\
.offset(offset)\
.limit(BLOCK).all():
try:
Type = dst.get_edge_by_labels(old.src.label, old.label,
old.dst.label)
print Type.__name__
new = Type(
src_id=old.src_id,
dst_id=old.dst_id,
properties=old.properties,
system_annotations=old.system_annotations,
label=old.label,
)
new.created = old.created
session.merge(new)
except Exception as e:
logging.error("unable to add edge {}, {}".format(
old.label, old.src_id, old.dst_id))
logging.error(e)
try:
session.commit()
except Exception as e:
logging.error(e)
def db_init(app):
app.logger.info("Initializing PsqlGraph driver")
app.db = PsqlGraphDriver(
host=app.config["PSQLGRAPH"]["host"],
user=app.config["PSQLGRAPH"]["user"],
password=app.config["PSQLGRAPH"]["password"],
database=app.config["PSQLGRAPH"]["database"],
set_flush_timestamps=True,
)
if app.config.get("AUTO_MIGRATE_DATABASE"):
migrate_database(app)
app.oauth_client = oauth2_client.OAuthClient(**app.config["OAUTH2"])
app.logger.info("Initializing index client")
app.index_client = IndexClient(
app.config["INDEX_CLIENT"]["host"],
version=app.config["INDEX_CLIENT"]["version"],
auth=app.config["INDEX_CLIENT"]["auth"],
)
def pg_driver(request, client):
pg_driver = PsqlGraphDriver(**pg_config())
def tearDown():
with pg_driver.engine.begin() as conn:
for table in models.Node().get_subclass_table_names():
if table != models.Node.__tablename__:
conn.execute('delete from {}'.format(table))
for table in models.Edge().get_subclass_table_names():
if table != models.Edge.__tablename__:
conn.execute('delete from {}'.format(table))
conn.execute('delete from versioned_nodes')
conn.execute('delete from _voided_nodes')
conn.execute('delete from _voided_edges')
conn.execute('delete from transaction_snapshots')
conn.execute('delete from transaction_documents')
conn.execute('delete from transaction_logs')
tearDown()
request.addfinalizer(tearDown)
return pg_driver
def pg_driver(request, client, use_ssl, isolation_level):
pg_driver = PsqlGraphDriver(
**pg_config(use_ssl=use_ssl, isolation_level=isolation_level))
def tearDown():
with pg_driver.engine.begin() as conn:
for table in models.Node().get_subclass_table_names():
if table != models.Node.__tablename__:
conn.execute("delete from {}".format(table)) # nosec
for table in models.Edge().get_subclass_table_names():
if table != models.Edge.__tablename__:
conn.execute("delete from {}".format(table)) # nosec
conn.execute("delete from versioned_nodes")
conn.execute("delete from _voided_nodes")
conn.execute("delete from _voided_edges")
conn.execute("delete from transaction_snapshots")
conn.execute("delete from transaction_documents")
conn.execute("delete from transaction_logs")
tearDown()
request.addfinalizer(tearDown)
return pg_driver
def translate_edge_range(_args):
args, offset = _args
src = PsqlGraphDriver(
args.source_host, args.source_user, args.source_password,
args.source, **driver_kwargs)
dst = PsqlGraphDriver(
args.dest_host, args.dest_user, args.dest_password,
args.dest, **driver_kwargs)
print '{}-{}'.format(offset, offset+BLOCK)
sys.stdout.flush()
with src.session_scope() as session:
with dst.session_scope() as session:
for old in src.edges(OldEdge)\
.order_by((OldEdge.src_id),
(OldEdge.dst_id),
(OldEdge.label))\
.options(joinedload(OldEdge.src))\
.options(joinedload(OldEdge.dst))\
.offset(offset)\
.limit(BLOCK).all():
try:
Type = dst.get_edge_by_labels(
old.src.label, old.label, old.dst.label)
print Type.__name__
new = Type(
src_id=old.src_id,
dst_id=old.dst_id,
properties=old.properties,
system_annotations=old.system_annotations,
label=old.label,
)
new.created = old.created
session.merge(new)
except Exception as e:
logging.error("unable to add edge {}, {}".format(
old.label, old.src_id, old.dst_id))
logging.error(e)
try:
session.commit()
except Exception as e:
logging.error(e)
def g(db_config, tables_created):
"""Fixture for database driver"""
return PsqlGraphDriver(**db_config)
class TestGDCPostgresAdmin(unittest.TestCase):
logger = logging.getLogger('TestGDCPostgresAdmin')
logger.setLevel(logging.INFO)
host = 'localhost'
user = '******'
database = 'automated_test'
base_args = [
'-H',
host,
'-U',
user,
'-D',
database,
]
g = PsqlGraphDriver(host, user, '', database)
root_con_str = "postgres://{user}:{pwd}@{host}/{db}".format(user=user,
host=host,
pwd='',
db=database)
engine = pgadmin.create_engine(root_con_str)
@classmethod
def tearDownClass(cls):
"""Recreate the database for tests that follow.
"""
cls.create_all_tables()
# Re-grant permissions to test user
for scls in Node.__subclasses__() + Edge.__subclasses__():
statment = ("GRANT ALL PRIVILEGES ON TABLE {} TO test".format(
scls.__tablename__))
cls.engine.execute('BEGIN; %s; COMMIT;' % statment)
@classmethod
def drop_all_tables(cls):
for scls in Node.__subclasses__():
try:
cls.engine.execute("DROP TABLE {} CASCADE".format(
scls.__tablename__))
except Exception as e:
cls.logger.warning(e)
@classmethod
def create_all_tables(cls):
parser = pgadmin.get_parser()
args = parser.parse_args(
['graph-create', '--delay', '1', '--retries', '0'] + cls.base_args)
pgadmin.main(args)
@classmethod
def drop_a_table(cls):
cls.engine.execute('DROP TABLE edge_clinicaldescribescase')
cls.engine.execute('DROP TABLE node_clinical')
def startTestRun(self):
self.drop_all_tables()
def setUp(self):
self.drop_all_tables()
def test_args(self):
parser = pgadmin.get_parser()
parser.parse_args(['graph-create'] + self.base_args)
def test_create_single(self):
"""Test simple table creation"""
pgadmin.main(pgadmin.get_parser().parse_args(
['graph-create', '--delay', '1', '--retries', '0'] +
self.base_args))
self.engine.execute('SELECT * from node_case')
def test_create_double(self):
"""Test idempotency of table creation"""
pgadmin.main(pgadmin.get_parser().parse_args(
['graph-create', '--delay', '1', '--retries', '0'] +
self.base_args))
self.engine.execute('SELECT * from node_case')
@pytest.mark.skip(reason="Causing race conditions, so skipping for now.")
def test_create_fails_blocked_without_force(self):
"""Test table creation fails when blocked w/o force"""
q = Queue() # to communicate with blocking process
args = pgadmin.get_parser().parse_args(
['graph-create', '--delay', '1', '--retries', '1'] +
self.base_args)
pgadmin.main(args)
self.drop_a_table()
def blocker():
with self.g.session_scope() as s:
s.merge(models.Case('1'))
q.put(0) # Tell main thread we're ready
q.get() # Wait for main thread to tell us to exit
p = Process(target=blocker)
p.daemon = True
p.start()
q.get()
with self.assertRaises(RuntimeError):
pgadmin.main(args)
q.put(0)
p.terminate()
@pytest.mark.skip(reason="This test is causing race conditions.")
def test_create_force(self):
"""Test ability to force table creation"""
q = Queue() # to communicate with blocking process
args = pgadmin.get_parser().parse_args(
['graph-create', '--delay', '1', '--retries', '1', '--force'] +
self.base_args)
pgadmin.main(args)
self.drop_a_table()
def blocker():
with self.g.session_scope() as s:
s.merge(models.Case('1'))
q.put(0) # Tell main thread we're ready
q.get() # This get should block until this prcoess is killed
assert False, 'Should not be reachable!'
p = Process(target=blocker)
p.daemon = True
p.start()
q.get()
try:
pgadmin.main(args)
except:
p.terminate()
raise
q.put(0)
p.terminate()
def test_priv_grant_read(self):
"""Test ability to grant read but not write privs"""
self.create_all_tables()
try:
self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'")
g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database)
#: If this failes, this test (not the code) is wrong!
with self.assertRaises(ProgrammingError):
with g.session_scope():
g.nodes().count()
pgadmin.main(pgadmin.get_parser().parse_args([
'graph-grant',
'--read=pytest',
] + self.base_args))
with g.session_scope():
g.nodes().count()
with self.assertRaises(ProgrammingError):
with g.session_scope() as s:
s.merge(models.Case('1'))
finally:
self.engine.execute("DROP OWNED BY pytest; DROP USER pytest")
def test_priv_grant_write(self):
"""Test ability to grant read/write privs"""
self.create_all_tables()
try:
self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'")
g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database)
pgadmin.main(pgadmin.get_parser().parse_args([
'graph-grant',
'--write=pytest',
] + self.base_args))
with g.session_scope() as s:
g.nodes().count()
s.merge(models.Case('1'))
finally:
self.engine.execute("DROP OWNED BY pytest; DROP USER pytest")
def test_priv_revoke_read(self):
"""Test ability to revoke read privs"""
self.create_all_tables()
try:
self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'")
g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database)
pgadmin.main(pgadmin.get_parser().parse_args([
'graph-grant',
'--read=pytest',
] + self.base_args))
pgadmin.main(pgadmin.get_parser().parse_args([
'graph-revoke',
'--read=pytest',
] + self.base_args))
with self.assertRaises(ProgrammingError):
with g.session_scope() as s:
g.nodes().count()
s.merge(models.Case('1'))
finally:
self.engine.execute("DROP OWNED BY pytest; DROP USER pytest")
def test_priv_revoke_write(self):
"""Test ability to revoke read/write privs"""
self.create_all_tables()
try:
self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'")
g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database)
pgadmin.main(pgadmin.get_parser().parse_args([
'graph-grant',
'--write=pytest',
] + self.base_args))
pgadmin.main(pgadmin.get_parser().parse_args([
'graph-revoke',
'--write=pytest',
] + self.base_args))
with g.session_scope() as s:
g.nodes().count()
with self.assertRaises(ProgrammingError):
with g.session_scope() as s:
s.merge(models.Case('1'))
finally:
self.engine.execute("DROP OWNED BY pytest; DROP USER pytest")
class MetadataImporter(object):
def __init__(self,metadata={}):
self.metadata = metadata
self.driver=PsqlGraphDriver(settings.METADATA_DB['HOST'],
settings.METADATA_DB['USER'],settings.METADATA_DB['PASSWORD'],
settings.METADATA_DB['NAME'])
self.signpost = SignpostClient(settings.SIGNPOST_URL,version='v0')
def find_props(self,props):
return self.driver.nodes().labels('dataset').props(props).count()>0
def validate_metadata(self):
for key in required_field:
if key not in self.metadata:
print "%s not provided" % key
return False
if 'short_description' not in self.metadata:
print 'short description not provided, use description as short description'
self.metadata['short_description']= self.metadata['description']
if 'slug' not in self.metadata:
self.metadata['slug'] = "-".join(self.metadata['title'].lower().split(" "))
if 'availability_mechanism' not in self.metadata:
self.metadata['availability_mechanism']='udr, rsync'
if self.find_props({'slug':self.metadata['slug']}):
print 'slug "%s" exist before, please change a slug' % self.metadata['slug']
return False
if 'url' not in self.metadata or self.metadata['url'].strip()=='':
self.metadata['url'] = urlparse.urljoin(ROOT_URL,self.metadata['slug'])
return True
def search_identifier(self,ark):
while self.signpost.search(ark):
new_ark = 'ark:/31807/osdc-' + binascii.b2a_hex(os.urandom(8))
print '%s exists, create new ark %s' % (ark,new_ark)
ark = new_ark
return ark.split(":")[-1]
def import_keywords(self):
nodes = []
for keyword in self.metadata['keywords'].split(","):
keyword=keyword.strip()
node = self.driver.nodes().labels('keyword').props({'value':keyword}).first()
if not node:
doc = self.signpost.create()
node = Node(label='keyword',node_id = doc.did,properties = {'value':keyword})
self.driver.node_merge(node=node)
print 'create new keyword %s' % keyword
nodes.append(node)
return nodes
def import_metadata(self):
with self.driver.session_scope():
if not self.validate_metadata():
return
doc = self.signpost.create()
doc.urls=[self.metadata['url']]
doc.identifiers = {
'ark':self.search_identifier('ark:/31807/osdc-'+doc.did.split('-')[0])
}
doc.patch()
properties = self.metadata.copy()
del properties['url']
del properties['keywords']
node = Node(node_id=doc.did,label='dataset',properties=properties)
self.driver.node_merge(node=node)
keyword_nodes = self.import_keywords()
for keyword in keyword_nodes:
self.driver.edge_insert(Edge(node.node_id,keyword.node_id,'member_of'))
print 'metadata %s created' % doc.did
def delete_metadata(self,did):
with self.driver.session_scope():
node = self.driver.nodes().ids(did).first()
if node:
self.driver.node_delete(node_id=did)
doc = self.signpost.get(did)
doc.delete()
def __init__(self,metadata={}):
self.metadata = metadata
self.driver=PsqlGraphDriver(settings.METADATA_DB['HOST'],
settings.METADATA_DB['USER'],settings.METADATA_DB['PASSWORD'],
settings.METADATA_DB['NAME'])
self.signpost = SignpostClient(settings.SIGNPOST_URL,version='v0')
from signpostclient import SignpostClient
#This needs work -- a lot of assumptions, lack of error checking and general ugliness
osdc_prefix = 'osdc'
#does not support time zones -- for now leave it up to the submission scripts to put in UTC.
time_format = '%Y-%m-%d %H:%M:%S'
valid_keys = [
'source', 'source_url', 'description', 'short_description', 'keyword',
'size', 'modified', 'license', 'osdc_location', 'osdc_folder',
'osdc_hs_location', 'osdc_hs_folder'
]
pg_driver = PsqlGraphDriver(METADATA_DB['HOST'], METADATA_DB['USER'],
METADATA_DB['PASSWORD'], METADATA_DB['NAME'])
signpost = SignpostClient(SIGNPOST_URL, version='v0')
def init_keys():
for key in valid_keys:
k = Key(key_name=key, public=True)
k.save()
def add_dataset(title, prefix):
key = str(uuid.uuid4())
slug = slugify(title)
d = DataSet(key=key, prefix=prefix, title=title, slug=slug)
class TestGDCPostgresAdmin(unittest.TestCase):
logger = logging.getLogger('TestGDCPostgresAdmin')
logger.setLevel(logging.INFO)
host = 'localhost'
user = '******'
database = 'automated_test'
base_args = [
'-H',
host,
'-U',
user,
'-D',
database,
]
g = PsqlGraphDriver(host, user, '', database)
root_con_str = "postgres://{user}:{pwd}@{host}/{db}".format(user=user,
host=host,
pwd='',
db=database)
engine = pgadmin.create_engine(root_con_str)
@classmethod
def tearDownClass(cls):
"""Recreate the database for tests that follow.
"""
cls.create_all_tables()
# Re-grant permissions to test user
for scls in Node.get_subclasses() + Edge.get_subclasses():
statment = ("GRANT ALL PRIVILEGES ON TABLE {} TO test".format(
scls.__tablename__))
cls.engine.execute('BEGIN; %s; COMMIT;' % statment)
@classmethod
def drop_all_tables(cls):
for scls in Node.get_subclasses():
try:
cls.engine.execute("DROP TABLE {} CASCADE".format(
scls.__tablename__))
except Exception as e:
cls.logger.warning(e)
@classmethod
def create_all_tables(cls):
parser = pgadmin.get_parser()
args = parser.parse_args(
['graph-create', '--delay', '1', '--retries', '0'] + cls.base_args)
pgadmin.main(args)
@classmethod
def drop_a_table(cls):
cls.engine.execute('DROP TABLE edge_clinicaldescribescase')
cls.engine.execute('DROP TABLE node_clinical')
def startTestRun(self):
self.drop_all_tables()
def setUp(self):
self.drop_all_tables()
def test_args(self):
parser = pgadmin.get_parser()
parser.parse_args(['graph-create'] + self.base_args)
def test_create_single(self):
"""Test simple table creation"""
pgadmin.main(pgadmin.get_parser().parse_args(
['graph-create', '--delay', '1', '--retries', '0'] +
self.base_args))
self.engine.execute('SELECT * from node_case')
def test_create_double(self):
"""Test idempotency of table creation"""
pgadmin.main(pgadmin.get_parser().parse_args(
['graph-create', '--delay', '1', '--retries', '0'] +
self.base_args))
self.engine.execute('SELECT * from node_case')
def test_priv_grant_read(self):
"""Test ability to grant read but not write privs"""
self.create_all_tables()
try:
self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'")
self.engine.execute("GRANT USAGE ON SCHEMA public TO pytest")
g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database)
#: If this failes, this test (not the code) is wrong!
with self.assertRaises(ProgrammingError):
with g.session_scope():
g.nodes().count()
pgadmin.main(pgadmin.get_parser().parse_args([
'graph-grant',
'--read=pytest',
] + self.base_args))
with g.session_scope():
g.nodes().count()
with self.assertRaises(ProgrammingError):
with g.session_scope() as s:
s.merge(models.Case('1'))
finally:
self.engine.execute("DROP OWNED BY pytest; DROP USER pytest")
def test_priv_grant_write(self):
"""Test ability to grant read/write privs"""
self.create_all_tables()
try:
self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'")
self.engine.execute("GRANT USAGE ON SCHEMA public TO pytest")
g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database)
pgadmin.main(pgadmin.get_parser().parse_args([
'graph-grant',
'--write=pytest',
] + self.base_args))
with g.session_scope() as s:
g.nodes().count()
s.merge(models.Case('1'))
finally:
self.engine.execute("DROP OWNED BY pytest; DROP USER pytest")
def test_priv_revoke_read(self):
"""Test ability to revoke read privs"""
self.create_all_tables()
try:
self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'")
self.engine.execute("GRANT USAGE ON SCHEMA public TO pytest")
g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database)
pgadmin.main(pgadmin.get_parser().parse_args([
'graph-grant',
'--read=pytest',
] + self.base_args))
pgadmin.main(pgadmin.get_parser().parse_args([
'graph-revoke',
'--read=pytest',
] + self.base_args))
with self.assertRaises(ProgrammingError):
with g.session_scope() as s:
g.nodes().count()
s.merge(models.Case('1'))
finally:
self.engine.execute("DROP OWNED BY pytest; DROP USER pytest")
def test_priv_revoke_write(self):
"""Test ability to revoke read/write privs"""
self.create_all_tables()
try:
self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'")
self.engine.execute("GRANT USAGE ON SCHEMA public TO pytest")
g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database)
pgadmin.main(pgadmin.get_parser().parse_args([
'graph-grant',
'--write=pytest',
] + self.base_args))
pgadmin.main(pgadmin.get_parser().parse_args([
'graph-revoke',
'--write=pytest',
] + self.base_args))
with g.session_scope() as s:
g.nodes().count()
with self.assertRaises(ProgrammingError):
with g.session_scope() as s:
s.merge(models.Case('1'))
finally:
self.engine.execute("DROP OWNED BY pytest; DROP USER pytest")
