def test_priv_revoke_write(self):
"""Test ability to revoke read/write privs"""
self.create_all_tables()
self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'")
try:
g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database)
pgadmin.main(pgadmin.get_parser().parse_args([
'graph-grant', '--write=pytest',
] + self.base_args))
pgadmin.main(pgadmin.get_parser().parse_args([
'graph-revoke', '--write=pytest',
] + self.base_args))
with g.session_scope() as s:
g.nodes().count()
with self.assertRaises(ProgrammingError):
with g.session_scope() as s:
s.merge(models.Case('1'))
finally:
self.engine.execute("DROP OWNED BY pytest; DROP USER pytest")
def translate_node_range(_args):
args, offset = _args
src = PsqlGraphDriver(
args.source_host, args.source_user, args.source_password,
args.source, **driver_kwargs)
dst = PsqlGraphDriver(
args.dest_host, args.dest_user, args.dest_password,
args.dest, **driver_kwargs)
with src.session_scope() as session:
with dst.session_scope() as session:
for old in src.nodes(OldNode).order_by(OldNode.node_id)\
.offset(offset)\
.limit(BLOCK)\
.yield_per(BLOCK):
try:
new = PolyNode(
node_id=old.node_id,
properties=old.properties,
system_annotations=old.system_annotations,
acl=old.acl,
label=old.label,
)
new.created = old.created
session.merge(new)
except Exception as e:
logging.error("unable to add node {}, {}".format(
old.label, old.node_id))
logging.error(e)
try:
session.commit()
except Exception as e:
logging.error(e)
def test_priv_grant_read(self):
"""Test ability to grant read but not write privs"""
self.create_all_tables()
try:
self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'")
g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database)
#: If this failes, this test (not the code) is wrong!
with self.assertRaises(ProgrammingError):
with g.session_scope():
g.nodes().count()
pgadmin.main(pgadmin.get_parser().parse_args([
'graph-grant',
'--read=pytest',
] + self.base_args))
with g.session_scope():
g.nodes().count()
with self.assertRaises(ProgrammingError):
with g.session_scope() as s:
s.merge(models.Case('1'))
finally:
self.engine.execute("DROP OWNED BY pytest; DROP USER pytest")
def test_priv_grant_read(self):
"""Test ability to grant read but not write privs"""
self.create_all_tables()
self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'")
try:
g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database)
#: If this failes, this test (not the code) is wrong!
with self.assertRaises(ProgrammingError):
with g.session_scope():
g.nodes().count()
pgadmin.main(pgadmin.get_parser().parse_args([
'graph-grant', '--read=pytest',
] + self.base_args))
with g.session_scope():
g.nodes().count()
with self.assertRaises(ProgrammingError):
with g.session_scope() as s:
s.merge(models.Case('1'))
finally:
self.engine.execute("DROP OWNED BY pytest; DROP USER pytest")
def translate_node_range(_args):
args, offset = _args
src = PsqlGraphDriver(args.source_host, args.source_user,
args.source_password, args.source, **driver_kwargs)
dst = PsqlGraphDriver(args.dest_host, args.dest_user, args.dest_password,
args.dest, **driver_kwargs)
with src.session_scope() as session:
with dst.session_scope() as session:
for old in src.nodes(OldNode).order_by(OldNode.node_id)\
.offset(offset)\
.limit(BLOCK)\
.yield_per(BLOCK):
try:
new = PolyNode(
node_id=old.node_id,
properties=old.properties,
system_annotations=old.system_annotations,
acl=old.acl,
label=old.label,
)
new.created = old.created
session.merge(new)
except Exception as e:
logging.error("unable to add node {}, {}".format(
old.label, old.node_id))
logging.error(e)
try:
session.commit()
except Exception as e:
logging.error(e)
def test_priv_revoke_write(self):
"""Test ability to revoke read/write privs"""
self.create_all_tables()
try:
self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'")
g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database)
pgadmin.main(pgadmin.get_parser().parse_args([
'graph-grant',
'--write=pytest',
] + self.base_args))
pgadmin.main(pgadmin.get_parser().parse_args([
'graph-revoke',
'--write=pytest',
] + self.base_args))
with g.session_scope() as s:
g.nodes().count()
with self.assertRaises(ProgrammingError):
with g.session_scope() as s:
s.merge(models.Case('1'))
finally:
self.engine.execute("DROP OWNED BY pytest; DROP USER pytest")
def update_legacy_states(graph_kwargs):
"""Updates state, file_state on legacy nodes
- node.state in {None, 'live'}
- node.project_id in {None, <Legacy project_id list>}
there is no project_id, or project_id points to a legacy project
"""
graph = PsqlGraphDriver(**graph_kwargs)
with graph.session_scope():
print_cls_query_summary(graph)
input_q = Queue()
pool = [
Process(target=update_classes, args=(graph_kwargs, input_q))
for _ in range(cpu_count())
]
for cls in CLS_TO_UPDATE:
input_q.put(cls)
for process in pool:
input_q.put(None) # put a no more work signal for each process
for process in pool:
process.start()
for process in pool:
process.join()
def translate_nodes(args):
src = PsqlGraphDriver(args.source_host, args.source_user,
args.source_password, args.source, **driver_kwargs)
with src.session_scope():
count = src.nodes(OldNode).count()
offsets = [i * BLOCK for i in range(count / BLOCK + 1)]
pool = Pool(args.nprocs)
args = [(args, offset) for offset in offsets]
pool.map_async(translate_node_range, args).get(int(1e9))
def translate_edge_range(_args):
args, offset = _args
src = PsqlGraphDriver(
args.source_host, args.source_user, args.source_password,
args.source, **driver_kwargs)
dst = PsqlGraphDriver(
args.dest_host, args.dest_user, args.dest_password,
args.dest, **driver_kwargs)
print '{}-{}'.format(offset, offset+BLOCK)
sys.stdout.flush()
with src.session_scope() as session:
with dst.session_scope() as session:
for old in src.edges(OldEdge)\
.order_by((OldEdge.src_id),
(OldEdge.dst_id),
(OldEdge.label))\
.options(joinedload(OldEdge.src))\
.options(joinedload(OldEdge.dst))\
.offset(offset)\
.limit(BLOCK).all():
try:
Type = dst.get_edge_by_labels(
old.src.label, old.label, old.dst.label)
print Type.__name__
new = Type(
src_id=old.src_id,
dst_id=old.dst_id,
properties=old.properties,
system_annotations=old.system_annotations,
label=old.label,
)
new.created = old.created
session.merge(new)
except Exception as e:
logging.error("unable to add edge {}, {}".format(
old.label, old.src_id, old.dst_id))
logging.error(e)
try:
session.commit()
except Exception as e:
logging.error(e)
def translate_nodes(args):
src = PsqlGraphDriver(
args.source_host, args.source_user, args.source_password,
args.source, **driver_kwargs)
with src.session_scope():
count = src.nodes(OldNode).count()
offsets = [i*BLOCK for i in range(count/BLOCK+1)]
pool = Pool(args.nprocs)
args = [(args, offset) for offset in offsets]
pool.map_async(translate_node_range, args).get(int(1e9))
def translate_edge_range(_args):
args, offset = _args
src = PsqlGraphDriver(args.source_host, args.source_user,
args.source_password, args.source, **driver_kwargs)
dst = PsqlGraphDriver(args.dest_host, args.dest_user, args.dest_password,
args.dest, **driver_kwargs)
print '{}-{}'.format(offset, offset + BLOCK)
sys.stdout.flush()
with src.session_scope() as session:
with dst.session_scope() as session:
for old in src.edges(OldEdge)\
.order_by((OldEdge.src_id),
(OldEdge.dst_id),
(OldEdge.label))\
.options(joinedload(OldEdge.src))\
.options(joinedload(OldEdge.dst))\
.offset(offset)\
.limit(BLOCK).all():
try:
Type = dst.get_edge_by_labels(old.src.label, old.label,
old.dst.label)
print Type.__name__
new = Type(
src_id=old.src_id,
dst_id=old.dst_id,
properties=old.properties,
system_annotations=old.system_annotations,
label=old.label,
)
new.created = old.created
session.merge(new)
except Exception as e:
logging.error("unable to add edge {}, {}".format(
old.label, old.src_id, old.dst_id))
logging.error(e)
try:
session.commit()
except Exception as e:
logging.error(e)
def test_custom_application_name(self):
cmd = "select application_name from pg_stat_activity;"
custom_name = '_CUSTOM_NAME'
g_ = PsqlGraphDriver(host,
user,
password,
database,
application_name=custom_name)
with g_.session_scope() as s:
s.merge(Test('a'))
app_names = {r[0] for r in g.engine.execute(cmd)}
self.assertIn(custom_name, app_names)
def migrate(host, user, password, database):
graph = PsqlGraphDriver(host, user, password, database)
with graph.session_scope():
for project, phsids in mapping.PROJECT_TO_PHSID.iteritems():
tokens = project.split('-')
program_code = tokens[0]
project_code = '-'.join(tokens[1:])
program_phsid = mapping.get_program_level_phsid(project)
project_phsid = mapping.get_project_level_phsid(project)
program = graph.nodes(Program).props(name=program_code).first()
project = graph.nodes(Project).props(code=project_code).first()
if program:
program.props['dbgap_accession_number'] = program_phsid
if project:
project.props['dbgap_accession_number'] = project_phsid
def migrate(host, user, password, database):
graph = PsqlGraphDriver(host, user, password, database)
with graph.session_scope():
for project, phsids in mapping.PROJECT_TO_PHSID.iteritems():
tokens = project.split('-')
program_code = tokens[0]
project_code = '-'.join(tokens[1:])
program_phsid = mapping.get_program_level_phsid(project)
project_phsid = mapping.get_project_level_phsid(project)
program = graph.nodes(Program).props(name=program_code).first()
project = graph.nodes(Project).props(code=project_code).first()
if program:
program.props['dbgap_accession_number'] = program_phsid
if project:
project.props['dbgap_accession_number'] = project_phsid
def test_priv_grant_write(self):
"""Test ability to grant read/write privs"""
self.create_all_tables()
try:
self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'")
self.engine.execute("GRANT USAGE ON SCHEMA public TO pytest")
g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database)
pgadmin.main(pgadmin.get_parser().parse_args([
'graph-grant',
'--write=pytest',
] + self.base_args))
with g.session_scope() as s:
g.nodes().count()
s.merge(models.Case('1'))
finally:
self.engine.execute("DROP OWNED BY pytest; DROP USER pytest")
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-H", "--host", type=str, action="store",
required=True, help="psql-server host")
parser.add_argument("-U", "--user", type=str, action="store",
required=True, help="psql test user")
parser.add_argument("-D", "--database", type=str, action="store",
required=True, help="psql test database")
parser.add_argument("-P", "--password", type=str, action="store",
help="psql test password")
args = parser.parse_args()
prompt = "Password for {}:".format(args.user)
password = args.password or getpass.getpass(prompt)
g = PsqlGraphDriver(args.host, args.user, password, args.database)
with g.session_scope():
projects = g.nodes(md.Project).not_props(state='legacy').all()
map(update_project_related_case_cache, projects)
print("Done.")
class MetadataImporter(object):
def __init__(self,metadata={}):
self.metadata = metadata
self.driver=PsqlGraphDriver(settings.METADATA_DB['HOST'],
settings.METADATA_DB['USER'],settings.METADATA_DB['PASSWORD'],
settings.METADATA_DB['NAME'])
self.signpost = SignpostClient(settings.SIGNPOST_URL,version='v0')
def find_props(self,props):
return self.driver.nodes().labels('dataset').props(props).count()>0
def validate_metadata(self):
for key in required_field:
if key not in self.metadata:
print "%s not provided" % key
return False
if 'short_description' not in self.metadata:
print 'short description not provided, use description as short description'
self.metadata['short_description']= self.metadata['description']
if 'slug' not in self.metadata:
self.metadata['slug'] = "-".join(self.metadata['title'].lower().split(" "))
if 'availability_mechanism' not in self.metadata:
self.metadata['availability_mechanism']='udr, rsync'
if self.find_props({'slug':self.metadata['slug']}):
print 'slug "%s" exist before, please change a slug' % self.metadata['slug']
return False
if 'url' not in self.metadata or self.metadata['url'].strip()=='':
self.metadata['url'] = urlparse.urljoin(ROOT_URL,self.metadata['slug'])
return True
def search_identifier(self,ark):
while self.signpost.search(ark):
new_ark = 'ark:/31807/osdc-' + binascii.b2a_hex(os.urandom(8))
print '%s exists, create new ark %s' % (ark,new_ark)
ark = new_ark
return ark.split(":")[-1]
def import_keywords(self):
nodes = []
for keyword in self.metadata['keywords'].split(","):
keyword=keyword.strip()
node = self.driver.nodes().labels('keyword').props({'value':keyword}).first()
if not node:
doc = self.signpost.create()
node = Node(label='keyword',node_id = doc.did,properties = {'value':keyword})
self.driver.node_merge(node=node)
print 'create new keyword %s' % keyword
nodes.append(node)
return nodes
def import_metadata(self):
with self.driver.session_scope():
if not self.validate_metadata():
return
doc = self.signpost.create()
doc.urls=[self.metadata['url']]
doc.identifiers = {
'ark':self.search_identifier('ark:/31807/osdc-'+doc.did.split('-')[0])
}
doc.patch()
properties = self.metadata.copy()
del properties['url']
del properties['keywords']
node = Node(node_id=doc.did,label='dataset',properties=properties)
self.driver.node_merge(node=node)
keyword_nodes = self.import_keywords()
for keyword in keyword_nodes:
self.driver.edge_insert(Edge(node.node_id,keyword.node_id,'member_of'))
print 'metadata %s created' % doc.did
def delete_metadata(self,did):
with self.driver.session_scope():
node = self.driver.nodes().ids(did).first()
if node:
self.driver.node_delete(node_id=did)
doc = self.signpost.get(did)
doc.delete()
