def test_priv_grant_read(self): """Test ability to grant read but not write privs""" self.create_all_tables() try: self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'") g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database) #: If this failes, this test (not the code) is wrong! with self.assertRaises(ProgrammingError): with g.session_scope(): g.nodes().count() pgadmin.main(pgadmin.get_parser().parse_args([ 'graph-grant', '--read=pytest', ] + self.base_args)) with g.session_scope(): g.nodes().count() with self.assertRaises(ProgrammingError): with g.session_scope() as s: s.merge(models.Case('1')) finally: self.engine.execute("DROP OWNED BY pytest; DROP USER pytest")
def test_priv_revoke_write(self): """Test ability to revoke read/write privs""" self.create_all_tables() try: self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'") g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database) pgadmin.main(pgadmin.get_parser().parse_args([ 'graph-grant', '--write=pytest', ] + self.base_args)) pgadmin.main(pgadmin.get_parser().parse_args([ 'graph-revoke', '--write=pytest', ] + self.base_args)) with g.session_scope() as s: g.nodes().count() with self.assertRaises(ProgrammingError): with g.session_scope() as s: s.merge(models.Case('1')) finally: self.engine.execute("DROP OWNED BY pytest; DROP USER pytest")
def test_priv_grant_read(self): """Test ability to grant read but not write privs""" self.create_all_tables() self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'") try: g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database) #: If this failes, this test (not the code) is wrong! with self.assertRaises(ProgrammingError): with g.session_scope(): g.nodes().count() pgadmin.main(pgadmin.get_parser().parse_args([ 'graph-grant', '--read=pytest', ] + self.base_args)) with g.session_scope(): g.nodes().count() with self.assertRaises(ProgrammingError): with g.session_scope() as s: s.merge(models.Case('1')) finally: self.engine.execute("DROP OWNED BY pytest; DROP USER pytest")
def test_priv_revoke_write(self): """Test ability to revoke read/write privs""" self.create_all_tables() self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'") try: g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database) pgadmin.main(pgadmin.get_parser().parse_args([ 'graph-grant', '--write=pytest', ] + self.base_args)) pgadmin.main(pgadmin.get_parser().parse_args([ 'graph-revoke', '--write=pytest', ] + self.base_args)) with g.session_scope() as s: g.nodes().count() with self.assertRaises(ProgrammingError): with g.session_scope() as s: s.merge(models.Case('1')) finally: self.engine.execute("DROP OWNED BY pytest; DROP USER pytest")
def translate_node_range(_args): args, offset = _args src = PsqlGraphDriver( args.source_host, args.source_user, args.source_password, args.source, **driver_kwargs) dst = PsqlGraphDriver( args.dest_host, args.dest_user, args.dest_password, args.dest, **driver_kwargs) with src.session_scope() as session: with dst.session_scope() as session: for old in src.nodes(OldNode).order_by(OldNode.node_id)\ .offset(offset)\ .limit(BLOCK)\ .yield_per(BLOCK): try: new = PolyNode( node_id=old.node_id, properties=old.properties, system_annotations=old.system_annotations, acl=old.acl, label=old.label, ) new.created = old.created session.merge(new) except Exception as e: logging.error("unable to add node {}, {}".format( old.label, old.node_id)) logging.error(e) try: session.commit() except Exception as e: logging.error(e)
def translate_node_range(_args): args, offset = _args src = PsqlGraphDriver(args.source_host, args.source_user, args.source_password, args.source, **driver_kwargs) dst = PsqlGraphDriver(args.dest_host, args.dest_user, args.dest_password, args.dest, **driver_kwargs) with src.session_scope() as session: with dst.session_scope() as session: for old in src.nodes(OldNode).order_by(OldNode.node_id)\ .offset(offset)\ .limit(BLOCK)\ .yield_per(BLOCK): try: new = PolyNode( node_id=old.node_id, properties=old.properties, system_annotations=old.system_annotations, acl=old.acl, label=old.label, ) new.created = old.created session.merge(new) except Exception as e: logging.error("unable to add node {}, {}".format( old.label, old.node_id)) logging.error(e) try: session.commit() except Exception as e: logging.error(e)
def translate_nodes(args): src = PsqlGraphDriver(args.source_host, args.source_user, args.source_password, args.source, **driver_kwargs) with src.session_scope(): count = src.nodes(OldNode).count() offsets = [i * BLOCK for i in range(count / BLOCK + 1)] pool = Pool(args.nprocs) args = [(args, offset) for offset in offsets] pool.map_async(translate_node_range, args).get(int(1e9))
def translate_nodes(args): src = PsqlGraphDriver( args.source_host, args.source_user, args.source_password, args.source, **driver_kwargs) with src.session_scope(): count = src.nodes(OldNode).count() offsets = [i*BLOCK for i in range(count/BLOCK+1)] pool = Pool(args.nprocs) args = [(args, offset) for offset in offsets] pool.map_async(translate_node_range, args).get(int(1e9))
def test_priv_grant_write(self): """Test ability to grant read/write privs""" self.create_all_tables() try: self.engine.execute("CREATE USER pytest WITH PASSWORD 'pyt3st'") self.engine.execute("GRANT USAGE ON SCHEMA public TO pytest") g = PsqlGraphDriver(self.host, 'pytest', 'pyt3st', self.database) pgadmin.main(pgadmin.get_parser().parse_args([ 'graph-grant', '--write=pytest', ] + self.base_args)) with g.session_scope() as s: g.nodes().count() s.merge(models.Case('1')) finally: self.engine.execute("DROP OWNED BY pytest; DROP USER pytest")
def migrate(host, user, password, database): graph = PsqlGraphDriver(host, user, password, database) with graph.session_scope(): for project, phsids in mapping.PROJECT_TO_PHSID.iteritems(): tokens = project.split('-') program_code = tokens[0] project_code = '-'.join(tokens[1:]) program_phsid = mapping.get_program_level_phsid(project) project_phsid = mapping.get_project_level_phsid(project) program = graph.nodes(Program).props(name=program_code).first() project = graph.nodes(Project).props(code=project_code).first() if program: program.props['dbgap_accession_number'] = program_phsid if project: project.props['dbgap_accession_number'] = project_phsid
def main(): parser = argparse.ArgumentParser() parser.add_argument("-H", "--host", type=str, action="store", required=True, help="psql-server host") parser.add_argument("-U", "--user", type=str, action="store", required=True, help="psql test user") parser.add_argument("-D", "--database", type=str, action="store", required=True, help="psql test database") parser.add_argument("-P", "--password", type=str, action="store", help="psql test password") args = parser.parse_args() prompt = "Password for {}:".format(args.user) password = args.password or getpass.getpass(prompt) g = PsqlGraphDriver(args.host, args.user, password, args.database) with g.session_scope(): projects = g.nodes(md.Project).not_props(state='legacy').all() map(update_project_related_case_cache, projects) print("Done.")
class MetadataImporter(object): def __init__(self,metadata={}): self.metadata = metadata self.driver=PsqlGraphDriver(settings.METADATA_DB['HOST'], settings.METADATA_DB['USER'],settings.METADATA_DB['PASSWORD'], settings.METADATA_DB['NAME']) self.signpost = SignpostClient(settings.SIGNPOST_URL,version='v0') def find_props(self,props): return self.driver.nodes().labels('dataset').props(props).count()>0 def validate_metadata(self): for key in required_field: if key not in self.metadata: print "%s not provided" % key return False if 'short_description' not in self.metadata: print 'short description not provided, use description as short description' self.metadata['short_description']= self.metadata['description'] if 'slug' not in self.metadata: self.metadata['slug'] = "-".join(self.metadata['title'].lower().split(" ")) if 'availability_mechanism' not in self.metadata: self.metadata['availability_mechanism']='udr, rsync' if self.find_props({'slug':self.metadata['slug']}): print 'slug "%s" exist before, please change a slug' % self.metadata['slug'] return False if 'url' not in self.metadata or self.metadata['url'].strip()=='': self.metadata['url'] = urlparse.urljoin(ROOT_URL,self.metadata['slug']) return True def search_identifier(self,ark): while self.signpost.search(ark): new_ark = 'ark:/31807/osdc-' + binascii.b2a_hex(os.urandom(8)) print '%s exists, create new ark %s' % (ark,new_ark) ark = new_ark return ark.split(":")[-1] def import_keywords(self): nodes = [] for keyword in self.metadata['keywords'].split(","): keyword=keyword.strip() node = self.driver.nodes().labels('keyword').props({'value':keyword}).first() if not node: doc = self.signpost.create() node = Node(label='keyword',node_id = doc.did,properties = {'value':keyword}) self.driver.node_merge(node=node) print 'create new keyword %s' % keyword nodes.append(node) return nodes def import_metadata(self): with self.driver.session_scope(): if not self.validate_metadata(): return doc = self.signpost.create() doc.urls=[self.metadata['url']] doc.identifiers = { 'ark':self.search_identifier('ark:/31807/osdc-'+doc.did.split('-')[0]) } doc.patch() properties = self.metadata.copy() del properties['url'] del properties['keywords'] node = Node(node_id=doc.did,label='dataset',properties=properties) self.driver.node_merge(node=node) keyword_nodes = self.import_keywords() for keyword in keyword_nodes: self.driver.edge_insert(Edge(node.node_id,keyword.node_id,'member_of')) print 'metadata %s created' % doc.did def delete_metadata(self,did): with self.driver.session_scope(): node = self.driver.nodes().ids(did).first() if node: self.driver.node_delete(node_id=did) doc = self.signpost.get(did) doc.delete()