def test_priv_grant_read(self):
"""Test ability to grant read but not write privs"""
self.create_all_tables()
try:
self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'")
g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database)
#: If this failes, this test (not the code) is wrong!
with self.assertRaises(ProgrammingError):
with g.session_scope():
g.nodes().count()
pgadmin.main(pgadmin.get_parser().parse_args([
'graph-grant',
'--read=pytest',
] + self.base_args))
with g.session_scope():
g.nodes().count()
with self.assertRaises(ProgrammingError):
with g.session_scope() as s:
s.merge(models.Case('1'))
finally:
self.engine.execute("DROP OWNED BY pytest; DROP USER pytest")
def test_priv_revoke_write(self):
"""Test ability to revoke read/write privs"""
self.create_all_tables()
try:
self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'")
g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database)
pgadmin.main(pgadmin.get_parser().parse_args([
'graph-grant',
'--write=pytest',
] + self.base_args))
pgadmin.main(pgadmin.get_parser().parse_args([
'graph-revoke',
'--write=pytest',
] + self.base_args))
with g.session_scope() as s:
g.nodes().count()
with self.assertRaises(ProgrammingError):
with g.session_scope() as s:
s.merge(models.Case('1'))
finally:
self.engine.execute("DROP OWNED BY pytest; DROP USER pytest")
def test_priv_grant_read(self):
"""Test ability to grant read but not write privs"""
self.create_all_tables()
self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'")
try:
g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database)
#: If this failes, this test (not the code) is wrong!
with self.assertRaises(ProgrammingError):
with g.session_scope():
g.nodes().count()
pgadmin.main(pgadmin.get_parser().parse_args([
'graph-grant', '--read=pytest',
] + self.base_args))
with g.session_scope():
g.nodes().count()
with self.assertRaises(ProgrammingError):
with g.session_scope() as s:
s.merge(models.Case('1'))
finally:
self.engine.execute("DROP OWNED BY pytest; DROP USER pytest")
def test_priv_revoke_write(self):
"""Test ability to revoke read/write privs"""
self.create_all_tables()
self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'")
try:
g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database)
pgadmin.main(pgadmin.get_parser().parse_args([
'graph-grant', '--write=pytest',
] + self.base_args))
pgadmin.main(pgadmin.get_parser().parse_args([
'graph-revoke', '--write=pytest',
] + self.base_args))
with g.session_scope() as s:
g.nodes().count()
with self.assertRaises(ProgrammingError):
with g.session_scope() as s:
s.merge(models.Case('1'))
finally:
self.engine.execute("DROP OWNED BY pytest; DROP USER pytest")
def translate_node_range(_args):
args, offset = _args
src = PsqlGraphDriver(
args.source_host, args.source_user, args.source_password,
args.source, **driver_kwargs)
dst = PsqlGraphDriver(
args.dest_host, args.dest_user, args.dest_password,
args.dest, **driver_kwargs)
with src.session_scope() as session:
with dst.session_scope() as session:
for old in src.nodes(OldNode).order_by(OldNode.node_id)\
.offset(offset)\
.limit(BLOCK)\
.yield_per(BLOCK):
try:
new = PolyNode(
node_id=old.node_id,
properties=old.properties,
system_annotations=old.system_annotations,
acl=old.acl,
label=old.label,
)
new.created = old.created
session.merge(new)
except Exception as e:
logging.error("unable to add node {}, {}".format(
old.label, old.node_id))
logging.error(e)
try:
session.commit()
except Exception as e:
logging.error(e)
def translate_node_range(_args):
args, offset = _args
src = PsqlGraphDriver(args.source_host, args.source_user,
args.source_password, args.source, **driver_kwargs)
dst = PsqlGraphDriver(args.dest_host, args.dest_user, args.dest_password,
args.dest, **driver_kwargs)
with src.session_scope() as session:
with dst.session_scope() as session:
for old in src.nodes(OldNode).order_by(OldNode.node_id)\
.offset(offset)\
.limit(BLOCK)\
.yield_per(BLOCK):
try:
new = PolyNode(
node_id=old.node_id,
properties=old.properties,
system_annotations=old.system_annotations,
acl=old.acl,
label=old.label,
)
new.created = old.created
session.merge(new)
except Exception as e:
logging.error("unable to add node {}, {}".format(
old.label, old.node_id))
logging.error(e)
try:
session.commit()
except Exception as e:
logging.error(e)
def translate_nodes(args):
src = PsqlGraphDriver(args.source_host, args.source_user,
args.source_password, args.source, **driver_kwargs)
with src.session_scope():
count = src.nodes(OldNode).count()
offsets = [i * BLOCK for i in range(count / BLOCK + 1)]
pool = Pool(args.nprocs)
args = [(args, offset) for offset in offsets]
pool.map_async(translate_node_range, args).get(int(1e9))
def translate_nodes(args):
src = PsqlGraphDriver(
args.source_host, args.source_user, args.source_password,
args.source, **driver_kwargs)
with src.session_scope():
count = src.nodes(OldNode).count()
offsets = [i*BLOCK for i in range(count/BLOCK+1)]
pool = Pool(args.nprocs)
args = [(args, offset) for offset in offsets]
pool.map_async(translate_node_range, args).get(int(1e9))
def test_priv_grant_write(self):
"""Test ability to grant read/write privs"""
self.create_all_tables()
try:
self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'")
self.engine.execute("GRANT USAGE ON SCHEMA public TO pytest")
g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database)
pgadmin.main(pgadmin.get_parser().parse_args([
'graph-grant',
'--write=pytest',
] + self.base_args))
with g.session_scope() as s:
g.nodes().count()
s.merge(models.Case('1'))
finally:
self.engine.execute("DROP OWNED BY pytest; DROP USER pytest")
def migrate(host, user, password, database):
graph = PsqlGraphDriver(host, user, password, database)
with graph.session_scope():
for project, phsids in mapping.PROJECT_TO_PHSID.iteritems():
tokens = project.split('-')
program_code = tokens[0]
project_code = '-'.join(tokens[1:])
program_phsid = mapping.get_program_level_phsid(project)
project_phsid = mapping.get_project_level_phsid(project)
program = graph.nodes(Program).props(name=program_code).first()
project = graph.nodes(Project).props(code=project_code).first()
if program:
program.props['dbgap_accession_number'] = program_phsid
if project:
project.props['dbgap_accession_number'] = project_phsid
def migrate(host, user, password, database):
graph = PsqlGraphDriver(host, user, password, database)
with graph.session_scope():
for project, phsids in mapping.PROJECT_TO_PHSID.iteritems():
tokens = project.split('-')
program_code = tokens[0]
project_code = '-'.join(tokens[1:])
program_phsid = mapping.get_program_level_phsid(project)
project_phsid = mapping.get_project_level_phsid(project)
program = graph.nodes(Program).props(name=program_code).first()
project = graph.nodes(Project).props(code=project_code).first()
if program:
program.props['dbgap_accession_number'] = program_phsid
if project:
project.props['dbgap_accession_number'] = project_phsid
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-H", "--host", type=str, action="store",
required=True, help="psql-server host")
parser.add_argument("-U", "--user", type=str, action="store",
required=True, help="psql test user")
parser.add_argument("-D", "--database", type=str, action="store",
required=True, help="psql test database")
parser.add_argument("-P", "--password", type=str, action="store",
help="psql test password")
args = parser.parse_args()
prompt = "Password for {}:".format(args.user)
password = args.password or getpass.getpass(prompt)
g = PsqlGraphDriver(args.host, args.user, password, args.database)
with g.session_scope():
projects = g.nodes(md.Project).not_props(state='legacy').all()
map(update_project_related_case_cache, projects)
print("Done.")
class MetadataImporter(object):
def __init__(self,metadata={}):
self.metadata = metadata
self.driver=PsqlGraphDriver(settings.METADATA_DB['HOST'],
settings.METADATA_DB['USER'],settings.METADATA_DB['PASSWORD'],
settings.METADATA_DB['NAME'])
self.signpost = SignpostClient(settings.SIGNPOST_URL,version='v0')
def find_props(self,props):
return self.driver.nodes().labels('dataset').props(props).count()>0
def validate_metadata(self):
for key in required_field:
if key not in self.metadata:
print "%s not provided" % key
return False
if 'short_description' not in self.metadata:
print 'short description not provided, use description as short description'
self.metadata['short_description']= self.metadata['description']
if 'slug' not in self.metadata:
self.metadata['slug'] = "-".join(self.metadata['title'].lower().split(" "))
if 'availability_mechanism' not in self.metadata:
self.metadata['availability_mechanism']='udr, rsync'
if self.find_props({'slug':self.metadata['slug']}):
print 'slug "%s" exist before, please change a slug' % self.metadata['slug']
return False
if 'url' not in self.metadata or self.metadata['url'].strip()=='':
self.metadata['url'] = urlparse.urljoin(ROOT_URL,self.metadata['slug'])
return True
def search_identifier(self,ark):
while self.signpost.search(ark):
new_ark = 'ark:/31807/osdc-' + binascii.b2a_hex(os.urandom(8))
print '%s exists, create new ark %s' % (ark,new_ark)
ark = new_ark
return ark.split(":")[-1]
def import_keywords(self):
nodes = []
for keyword in self.metadata['keywords'].split(","):
keyword=keyword.strip()
node = self.driver.nodes().labels('keyword').props({'value':keyword}).first()
if not node:
doc = self.signpost.create()
node = Node(label='keyword',node_id = doc.did,properties = {'value':keyword})
self.driver.node_merge(node=node)
print 'create new keyword %s' % keyword
nodes.append(node)
return nodes
def import_metadata(self):
with self.driver.session_scope():
if not self.validate_metadata():
return
doc = self.signpost.create()
doc.urls=[self.metadata['url']]
doc.identifiers = {
'ark':self.search_identifier('ark:/31807/osdc-'+doc.did.split('-')[0])
}
doc.patch()
properties = self.metadata.copy()
del properties['url']
del properties['keywords']
node = Node(node_id=doc.did,label='dataset',properties=properties)
self.driver.node_merge(node=node)
keyword_nodes = self.import_keywords()
for keyword in keyword_nodes:
self.driver.edge_insert(Edge(node.node_id,keyword.node_id,'member_of'))
print 'metadata %s created' % doc.did
def delete_metadata(self,did):
with self.driver.session_scope():
node = self.driver.nodes().ids(did).first()
if node:
self.driver.node_delete(node_id=did)
doc = self.signpost.get(did)
doc.delete()
