def main(): """Run the servur""" env = os.environ while True: try: neodb.set_connection( f'bolt://{env["DB_USER"]}:{env["DB_PASS"]}@{env["DB_HOST"]}:7687' ) except neo4j.exceptions.ServiceUnavailable: sleep(1) else: break loop = asyncio.get_event_loop() snow = SnowflakeService.service_factory(f'http://{env["SNOW_HOST"]}:8080/') boardds = BoardDeleteService.service_factory() websocket = WebSocketExtension('0.0.0.0', 8000) server = HTTPServer({ 'snowflake': snow, 'board_delete': boardds }, {'ws': websocket}, port=80, security_url=f'http://{env["AUTH_HOST"]}') cogs = [UserCog(), BoardCog(), ChannelCog()] for cog in cogs: server.load_cog(cog) loop.run_until_complete(server()) loop.close()
def main(): parser = ArgumentParser( description=''' Setup indexes and constraints on labels in Neo4j for your neomodel schema. Database credentials can be set by the environment variable NEO4J_BOLT_URL. ''') parser.add_argument( 'apps', metavar='<someapp.models/app.py>', type=str, nargs='+', help='python modules or files to load schema from.') parser.add_argument( '--db', metavar='bolt://*****:*****@localhost:7687', dest='neo4j_bolt_url', type=str, default='', help='address of your neo4j database' ) args = parser.parse_args() bolt_url = args.neo4j_bolt_url if len(bolt_url) == 0: bolt_url = environ.get('NEO4J_BOLT_URL', 'bolt://*****:*****@localhost:7687') for app in args.apps: load_python_module_or_file(app) # Connect after to override any code in the module that may set the connection print('Connecting to {}\n'.format(bolt_url)) db.set_connection(bolt_url) install_all_labels()
def custom_connection(self, **kwargs): if len(kwargs) > 0: variables = kwargs else: variables = self.variables self.uri = "bolt://%s:%s@%s:%s" % \ ( # User:Password variables.get('user', 'neo4j'), variables.get('password'), # Host:Port variables.get('host'), variables.get('port'), ) log.very_verbose("URI IS %s" % re_obscure_pattern(self.uri)) config.DATABASE_URL = self.uri # Ensure all DateTimes are provided with a timezone # before being serialised to UTC epoch config.FORCE_TIMEZONE = True # default False db.url = self.uri db.set_connection(self.uri) client = NeomodelClient(db) return client
def main(verbose): os.environ["PLAYGROUND_VERBOSE"] = str(verbose).upper() neodb.set_connection(os.getenv( 'NEO4J_CS')) #this connection method is a shame and i know it config = configparser.ConfigParser() config.read('./config/config.ini') if os.getenv("PLAYGROUND_VERBOSE") == "TRUE": print( f"Starting Reddit Playground (v{config.get('PLAYGROUND', 'VERSION')})" ) # Initialize the GlobalContext ctx = GlobalContext() sr = ctx.Reddit.subreddit('all') for comment in sr.stream.comments(): ctx.comment_queue.enqueue('reddit_playground.lib.db.insert_comment', comment) ctx.redditor_queue.enqueue('reddit_playground.lib.db.insert_redditor', comment.author) # ctx.submission_queue.enqueue('reddit_playground.lib.db.insert_submission', comment.submission) ctx.subreddit_queue.enqueue( 'reddit_playground.lib.db.insert_subreddit', comment.submission.subreddit) return 0
def connect(self, **kwargs: str) -> "NeoModel": variables = self.variables.copy() variables.update(kwargs) USER = variables.get("user", "neo4j") PWD = variables.get("password") HOST = variables.get("host") PORT = variables.get("port") # Fixed... to be configured? DATABASE = "neo4j" URI = f"bolt://{USER}:{PWD}@{HOST}:{PORT}/{DATABASE}" # https://neomodel.readthedocs.io/en/latest/getting_started.html#connecting # Set config.DATABASE_URL only once if config.DATABASE_URL == UNINITIALIZED_DATABASE_URL: config.DATABASE_URL = URI # Then switch the connection via set_connection db.set_connection(URI) # Ensure all DateTimes are provided with a timezone # before being serialised to UTC epoch config.FORCE_TIMEZONE = True # default False db.url = URI db.set_connection(URI) db.driver.verify_connectivity() StructuredNode.save = catch_db_exceptions(StructuredNode.save) NodeSet.get = catch_db_exceptions(NodeSet.get) self.db = db return self
def add_zVector(cls, uri, data: List[List[float]]): ''' Adds zVector data to the track :param uri: the Spotify URI of the track :param data: the zVector data as a list of lists, not a numpy array :return: the updated node ''' # print(uri) db.set_connection(connection_url()) # query = 'MATCH (n: Track {uri: "%s"}) RETURN n LIMIT 1' % (uri) # results, meta = db.cypher_query(query=query) # track = [cls.inflate(row[0]) for row in results] # # if track: # track = track[0] # print(track.name) # print('Please save', track) # f**k neomodel ffs data = json.dumps({'zVector': data}) query = ''' MATCH (t:Track {uri:'%s'}) WITH t, properties(t) as snapshot SET t.zVector = '%s' RETURN snapshot ''' % (uri, data) # print(query) track = db.cypher_query(query) return track
def pytest_sessionstart(session): """ Provides initial connection to the database and sets up the rest of the test suite :param session: The session object. Please see <https://docs.pytest.org/en/latest/reference.html#_pytest.hookspec.pytest_sessionstart>`_ :type Session object: For more information please see <https://docs.pytest.org/en/latest/reference.html#session>`_ """ warnings.simplefilter('default') config.DATABASE_URL = os.environ.get('NEO4J_BOLT_URL', 'bolt://*****:*****@localhost:7687') config.AUTO_INSTALL_LABELS = True try: # Clear the database if required database_is_populated, _ = db.cypher_query("MATCH (a) return count(a)>0 as database_is_populated") if database_is_populated[0][0] and not session.config.getoption("resetdb"): raise SystemError("Please note: The database seems to be populated.\n\tEither delete all nodes and edges manually, or set the --resetdb parameter when calling pytest\n\n\tpytest --resetdb.") else: clear_neo4j_database(db) except CypherError as ce: # Handle instance without password being changed if 'The credentials you provided were valid, but must be changed before you can use this instance' in str(ce): warnings.warn("New database with no password set, setting password to 'test'") change_neo4j_password(db, 'test') db.set_connection('bolt://*****:*****@localhost:7687') warnings.warn("Please 'export NEO4J_BOLT_URL=bolt://neo4j:test@localhost:7687' for subsequent test runs") else: raise ce
def connect(self, uri: str, login: str, password: str): """ :exception neo4j.exceptions.ServiceUnavailable: Если не подключается """ uri = DataBaseConnection.get_uri(uri, login, password) log.debug(f'Connecting to {uri}') config.DATABASE_URL = uri db.set_connection(uri)
def remove(): bolt_url = environ.get("DB_URL") # Connect after to override any code in the module that may set the connection print('Connecting to {}\n'.format(bolt_url)) db.set_connection(bolt_url) remove_all_labels()
def connect() -> None: """" Connects to the Neo4J instance """ user, pw, host, bolt_port = [N4J_CONF[a] for a in ['user', 'pass', 'host', 'bolt_port']] connection_url = f'bolt://{user}:{pw}@{host}:{bolt_port}' logger.debug(f'Connecting to Neo4J: {connection_url}') db.set_connection(connection_url)
def main(): from model.conv_autoencoder import Autoencoder from neomodel import config, db import pandas as pd import os import numpy as np from ast import literal_eval as eval from tqdm import tqdm from model.graph import connection_url from model.graph.spotify.track import Track from model.graph.spotify.playlist import Playlist config.DATABASE_URL = connection_url() db.set_connection(connection_url()) # Segment analysis def segment_to_vector(segment): return list(map(abs, [segment['duration'], segment['loudness_start'], segment['loudness_max_time'], segment['loudness_max']] \ + segment['pitches'] + segment['timbre'])) stopval = len(Track.nodes) print('Number of tracks:', stopval) print('Number of playlists:', len(Playlist.nodes)) def get_minibatches(stopval, offset=0, interval=500): while offset < stopval: to_analyze: List[Track] = Track.get_songs_not_in_playlists( interval, offset=offset) yield to_analyze offset += len(to_analyze) print(f'{offset}/{stopval}') for i, x in enumerate(get_minibatches(stopval)): tracks = x X = [a.analysis['segments'] for a in x if a.analysis] arrs = [list(map(segment_to_vector, sample)) for sample in X] print(tracks) latest = list( filter(lambda x: 'segment_analysis' in x, os.listdir('models'))) if latest: latest = os.path.join('models', max(latest)) else: latest = None to_pad = 800 arrs = Autoencoder.pad_and_normalize(arrs, to_pad=to_pad) auto = Autoencoder.train('segment_analysis', arrs, weights_path=latest, epochs=10) df = pd.DataFrame({'uri': [a.uri for a in tracks]}) df = Autoencoder.store_zVectors(auto, arrs, df) for i, row in tqdm(df.iterrows()): track = Track.add_zVector(row['uri'], row['zVector'].tolist())
def connect(self, host='localhost', username='******', password='', bolt_port=None, http_port=None, use_bolt=False, timeout=30): """connect - make connection to Neo4j DB :type host: str - hostname or IP of Neo4j database server :type password: str - password for Neo4j database server :type bolt_port: int - port for Neo4j Bolt protocol :type http_port: int - port for Neo4j HTTP protocol :type timeout: int - timeout for waiting for the Neo4j connection""" connected = False if use_bolt: test_port = bolt_port else: test_port = http_port scheme = 'http' if not use_bolt else 'bolt' self.logger.debug("testing if we can connect to {} which should be open for {}".format(test_port, scheme)) while timeout > 0: try: socket.create_connection((host, test_port), 1) except socket.error: timeout -= 1 time.sleep(1) else: connected = True break if not connected: raise socket.timeout('timed out trying to connect to {}'.format( host, test_port)) self.bolt_port = bolt_port self.http_port = http_port self.logger.info( "connecting to http port: {} bolt_port: {} host: {} bolt: {}\n". format(http_port, bolt_port, host, use_bolt)) # wait for server to be up for sure, needed for neo4j in Docker time.sleep(5) if password == '': pass_string = ':' else: pass_string = ':' + password port = bolt_port if use_bolt else http_port self.logger.debug( "connecting graph using {} on port: {} at host: {}".format( scheme, test_port, host)) connect_url = '{scheme}://{user}{pass_string}@{host}:{port}'.format(scheme=scheme, user=username, pass_string=pass_string, host=host, port=port) db.set_connection(connect_url) self.logger.debug("connected") return db
def main(): import pandas as pd import os from ast import literal_eval from scraper.genius import Genius from scraper.spotify import Spotify from model.graph import connection_url from model.graph.billboard.track import Track from neomodel import db, clear_neo4j_database, config url = connection_url() print(url) config.DATABASE_URL = url db.set_connection(url) print('connected') clear_neo4j_database(db) BILLBOARD_DIR = os.path.join('output', 'billboard') weeks = os.listdir(BILLBOARD_DIR) weeks.sort(reverse=True) for week in weeks: df = pd.read_csv(os.path.join(BILLBOARD_DIR, week, 'main.csv')) for i, row in df.iterrows(): billboard_track = Track.inst(**dict(row)) print(billboard_track) # Sort artists by appearance in the title artists = ', '.join( list( map( lambda x: x['artist_name'], sorted(literal_eval(row['credited_artists']), key=lambda x: x['ordinal'])))) search_str = row['title'] + ' ' + artists genius_resp = Genius.search(search_str) print('\n\nSearching for:', search_str) if genius_resp['meta']['status'] != 200: raise ValueError( 'Probably exceeded Genius limits or invalid search') genius_resp = genius_resp['response']['hits'] for hit in genius_resp: hit = hit['result'] song_data = Genius.get_song(hit['id'])['response']['song'] if 'spotify' in [a['provider'] for a in song_data['media']]: print('Spotify exists!') for i, a in enumerate(song_data['media']): print(a) if a['provider'] == 'spotify': print('Spotify Exists -', song_data['full_title']) spotify_data = Spotify.get_track( song_data['media'][i]['native_uri']) print(spotify_data) break quit() print(weeks)
def test_set_connection_works(): assert APerson(name='New guy').save() from socket import gaierror old_url = db.url with raises(AddressError): db.set_connection('bolt://*****:*****@6.6.6.6.6.6.6.6:7687') db.set_connection(old_url) # set connection back assert APerson(name='New guy2').save()
def init_db(neo4j_user_key, neo4j_pass_key, neo4j_host_key, protocol='bolt', port=7687): NEO4J_USER = os.environ[neo4j_user_key] NEO4J_PASS = os.environ[neo4j_pass_key] NEO4J_HOST = os.environ[neo4j_host_key] db.set_connection( f'{protocol}://{NEO4J_USER}:{NEO4J_PASS}@{NEO4J_HOST}:{port}')
def test_set_connection_works(): assert Person(name='New guy').save() from socket import gaierror old_url = db.url with raises(AddressError): db.set_connection('bolt://*****:*****@nowhere:7687') db.set_connection(old_url) # set connection back assert Person(name='New guy2').save()
def test_set_connection_works(): assert APerson(name="New guy").save() from socket import gaierror old_url = db.url with raises(ValueError): db.set_connection("bolt://*****:*****@6.6.6.6.6.6.6.6:7687") db.set_connection(old_url) # set connection back assert APerson(name="New guy2").save()
def install(): bolt_url = environ.get("DB_URL") load_python_module_or_file("main.py") # Connect after to override any code in the module that may set the connection print('Connecting to {}\n'.format(bolt_url)) db.set_connection(bolt_url) install_all_labels()
def get_album(track_id: str): ''' Gets the album given the Spotify ID :param track_id: the Spotify ID of the track :return: the album it's on ''' db.set_connection(connection_url()) results, meta = db.cypher_query(query=''' MATCH(t: Track {spotify_id: "%s"})-[:FROM]->(a: Album) RETURN a ''' % (track_id)) return [Album.inflate(album[0]) for album in results]
def migrate(): matching_dict = {"Act": {'DATE_OF_DOCUMENT': 'date_of_document', 'DATE_OF_EFFECT': 'date_of_effect', 'DATE_OF_VALIDITY': 'date_of_validity', 'DEADLINE': 'deadline', 'CONCERNS': 'concerns', 'CLASSIFICATION': 'classification', 'DIRECTORY': 'directory', 'AUTHOR': 'author', 'FORM': 'form', 'RELEVANT': 'relevant', 'TREATY': 'treaty', 'CITES': 'cites', 'BASED': {'Act': 'based_act', 'Article': 'based_article'}, 'CONSISTS': 'consists', 'REGULATES': 'regulated_entities', 'MENTIONS': 'mentioned_entities'}, "Article": {'MENTIONS': 'mentioned_entities', 'DEFINES': 'defined_entities', 'REGULATES': 'regulated_entities'}} db.set_connection('bolt://*****:*****@localhost:7687') #db.set_connection('bolt://*****:*****@100.25.33.237:38562') results, meta = db.cypher_query("MATCH p=()-->() RETURN p") db.set_connection('bolt://*****:*****@100.26.226.98:34409') for result in results: start_node_label = list(result[0].start_node.labels)[0] start_node_class = getattr(sys.modules[__name__], start_node_label) start_node = start_node_class.get_or_create(result[0].start_node)[0] end_node_label = list(result[0].end_node.labels)[0] end_node_class = getattr(sys.modules[__name__], end_node_label) if end_node_label == "Date": date = datetime.strptime(result[0].end_node._properties['date'], '%Y-%m-%d').date() try: end_node = Date.nodes.get(date=date) except DoesNotExist: end_node = Date(date=date) end_node.save() else: end_node = end_node_class.get_or_create(result[0].end_node)[0] rel_type = result[0].relationships[0].type rel_properties = result[0].relationships[0]._properties rel_attr = matching_dict[start_node_label][rel_type] if not type(rel_attr) is str: rel_attr = rel_attr[end_node_label] try: rel = attrgetter(rel_attr)(start_node).connect(end_node) except: print("failed to get an attr for {0} with attr {1}".format(start_node, rel_attr)) for key, value in rel_properties.items(): rel.__dict__[key] = value rel.save()
def find(cls, uri: str): ''' Finds a track given Spotify URI :param uri: the Spotify URI :return: the track ''' db.set_connection(connection_url()) results, meta = db.cypher_query(query=''' MATCH(t: Track {uri: "%s"}) RETURN t ''' % (uri)) return [cls.inflate(track[0]) for track in results]
def get_playlists(spotify_id) -> List[Playlist]: ''' Gets playlists the given track appears in :param spotify_id: the Spotify ID of the track :return: the list of playlists that the track appears in ''' db.set_connection(connection_url()) results, meta = db.cypher_query( 'MATCH (t:Track {spotify_id: "%s"})-[:`FEATURED IN`]->(p:Playlist) RETURN p' % spotify_id) return [Playlist.inflate(row[0]) for row in results]
def pytest_sessionstart(session): """ Provides initial connection to the database and sets up the rest of the test suite :param session: The session object. Please see <https://docs.pytest.org/en/latest/reference.html#_pytest.hookspec.pytest_sessionstart>`_ :type Session object: For more information please see <https://docs.pytest.org/en/latest/reference.html#session>`_ """ warnings.simplefilter("default") config.DATABASE_URL = os.environ.get( "NEO4J_BOLT_URL", "bolt://*****:*****@localhost:7687") config.AUTO_INSTALL_LABELS = True try: # Clear the database if required database_is_populated, _ = db.cypher_query( "MATCH (a) return count(a)>0 as database_is_populated") if database_is_populated[0][0] and not session.config.getoption( "resetdb"): raise SystemError( "Please note: The database seems to be populated.\n\tEither delete all nodes and edges manually, or set the --resetdb parameter when calling pytest\n\n\tpytest --resetdb." ) else: clear_neo4j_database(db, clear_constraints=True, clear_indexes=True) except (Neo4jError, ClientError) as ce: # Handle instance without password being changed if ("The credentials you provided were valid, but must be changed before you can use this instance" in str(ce)): warnings.warn( "New database with no password set, setting password to 'test'" ) try: change_neo4j_password(db, "test") # Ensures that multiprocessing tests can use the new password config.DATABASE_URL = "bolt://*****:*****@localhost:7687" db.set_connection("bolt://*****:*****@localhost:7687") warnings.warn( "Please 'export NEO4J_BOLT_URL=bolt://neo4j:test@localhost:7687' for subsequent test runs" ) except (Neo4jError, ClientError) as e: if ("The credentials you provided were valid, but must be changed before you can use this instance" in str(e)): warnings.warn( "You appear to be running on version 4.0+ of Neo4j, without having changed the password." "Please manually log in, change your password, then update the config.DATABASE_URL call at line 32 in this file" ) else: raise e else: raise ce
def test_set_connection_works(): assert Person(name='New guy').save() from socket import gaierror old_url = db.url try: db.set_connection('bolt://*****:*****@nowhere:7687') assert False # Shouldnt get here except gaierror: assert True db.set_connection(old_url) # set connection back assert Person(name='New guy2').save()
def paginate(cls, limit, offset=0, query=None): ''' Allows for staging queries so you don't fit the entire query results in one thing :param limit: rows to limit :param offset: rows to offset by :return: the rows as node objects ''' db.set_connection(connection_url()) name = cls.__label__ if query is None: query = f'MATCH (n: {name}) RETURN n' query += f' SKIP {offset} LIMIT {limit}' results, meta = db.cypher_query(query=query) return [cls.inflate(row[0]) for row in results]
def n4j_isready(): while time() - start_time < check_timeout: try: db.set_connection(DATABASE_URL) logger.info("Neo4J is ready!") return True except: logger.info( f"Neo4j isn't ready. Waiting for {check_interval} {interval_unit}..." ) sleep(check_interval) logger.error( f"We could not connect to Neo4j within {check_timeout} seconds.") return False
def get_tracks(cls, spotify_id): s = time() from model.graph.spotify.track import SmallTrack db.set_connection(connection_url()) query = ''' MATCH (p: Playlist {spotify_id: "%s"}) <-[r1: `FEATURED IN`]- (t: Track) RETURN t.uri, t.zVector, t.spotify_id, t.name ''' % spotify_id results, meta = db.cypher_query(query) print('Results fetched', time() - s) keys = ['uri', 'zVector', 'spotify_id', 'name'] kwargs = [{keys[i]: val for i, val in enumerate(result)} for result in results] return [SmallTrack(**result) for result in kwargs]
def get_associated_artists(cls, artist_id): ''' Gets artists associated with the given artist :param artist_id: the Spotify ID of the given artist :return: the list of associated artists ''' db.set_connection(connection_url()) results, meta = db.cypher_query(''' MATCH (t) -[:`FEATURED IN`]-> (p: Playlist) MATCH (a: Artist {spotify_id: "%s"}) <-[r1: BY]- (t: Track) -[r2: BY]-> (similar_artists: Artist) RETURN similar_artists ''' % artist_id) return set([cls.inflate(artist[0]) for artist in results])
def pytest_sessionstart(session): """ Provides initial connection to the database and sets up the rest of the test suite :param session: The session object. Please see <https://docs.pytest.org/en/latest/reference.html#_pytest.hookspec.pytest_sessionstart>`_ :type Session object: For more information please see <https://docs.pytest.org/en/latest/reference.html#session>`_ """ warnings.simplefilter('default') config.DATABASE_URL = os.environ.get('NEO4J_BOLT_URL', 'bolt://*****:*****@localhost:7687') config.AUTO_INSTALL_LABELS = True reset_db = session.config.getoption("resetdb") try: database_is_populated, _ = db.cypher_query( "MATCH (a) return count(a)>0 as database_is_populated") except CypherError as ce: # Handle instance without password being changed if 'The credentials you provided were valid, but must be changed before you can use this instance' in str( ce): warnings.warn( "New database with no password set, setting password to 'test'" ) change_neo4j_password(db, 'test') db.set_connection('bolt://*****:*****@localhost:7687') warnings.warn( "Please 'export NEO4J_BOLT_URL=bolt://neo4j:test@localhost:7687' for subsequent test runs" ) database_is_populated, _ = db.cypher_query( "MATCH (a) return count(a)>0 as database_is_populated") else: raise ce # If database is populated and user has not requested to clear the db, terminate test run with a SystemError if database_is_populated[0][0] and not reset_db: raise SystemError(DATABASE_POPULATED) # Clear the database if requested if reset_db: print("Clearing Neo4j database.") clear_neo4j_database(db)
def get_similar_playlists(track_id: str, playlist_id: str) -> List[Playlist]: ''' Gets similar playlists to the given playlist :param track_id: the Spotify ID of the track currently playing :param playlist_id: the Spotify ID of the playlist currently playing :return: a list of similar playlists ''' query = ''' match (t: Track {spotify_id: "%s"}) -[r1:`FEATURED IN`]-> (p: Playlist {spotify_id: "%s"}) <-[r2:`FEATURED IN`]- (other_tracks: Track) -[r3:`FEATURED IN`]-> (similar_playlists: Playlist) return similar_playlists ''' % (track_id, playlist_id) db.set_connection(connection_url()) results, meta = db.cypher_query(query=query) return [Playlist.inflate(playlist[0]) for playlist in results]
def get_tracks_with_multiple_artists(cls, context, *artist_ids): ''' Gets tracks with multiple artists on it :param artist_ids: the given artist IDs :param context: the Spotify ID of the playlist :return: ''' artists = ','.join([ "(a%s:Artist { uri: '%s' })" % (i, a) for i, a in enumerate(artist_ids) ]) query_constructor = '''MATCH %s, p = allShortestPaths((a0)-[*]-(a1)) WHERE EXISTS((a0) <-[:BY]- (:Track) -[:`FEATURED IN`]-> (:Playlist {spotify_id: "%s"})) RETURN nodes(p) ''' % (artists, context) from model.graph.spotify.track import Track db.set_connection(connection_url()) results, meta = db.cypher_query(query_constructor) return [Track.inflate(result[0][1]) for result in results]
def login(self): db.set_connection(self.url) install_all_labels()
from __future__ import print_function import warnings import os from neomodel import config, db from neo4j.v1.exceptions import CypherError warnings.simplefilter('default') config.DATABASE_URL = os.environ.get('NEO4J_BOLT_URL', 'bolt://*****:*****@localhost:7687') try: db.cypher_query("MATCH (a) DETACH DELETE a") except CypherError as ce: # handle instance without password being changed if 'The credentials you provided were valid, but must be changed before you can use this instance' in str(ce): db.cypher_query("CALL dbms.changePassword('test')") db.set_connection('bolt://*****:*****@localhost:7687') print("New database with no password set, setting password to 'test'") print("Please 'export NEO4J_BOLT_URL=bolt://neo4j:test@localhost:7687' for subsequent test runs") else: raise ce