def db_conn(): from sqlalchemy import create_engine, engine from sqlalchemy.ext.automap import automap_base from sqlalchemy.exc import InterfaceError config = { "drivername": "mysql+mysqlconnector", "username": "******", "password": "******", "host": "mysql" if script_runs_in_container() else "localhost", "port": "3306", "database": "database", } Base = automap_base() try: engine = create_engine(engine.url.URL(**config)) except InterfaceError: print("Something went wrong while connecting to MySQL db") Base.prepare(engine, reflect=True) yield engine print("Disposing engine") engine.dispose()
def __init__(self, filename): echo = False if self.DebugLevel > 2: echo = True self.SessionInstance=None if not filename: return self.FileList=[] self.AttachedDatabases=[] if self.UseAttach: self.Engine=create_engine('sqlite://',echo=False) self.AttachedDatabases.append('Diff') self.Engine.execute("ATTACH DATABASE '%s' AS Diff;" % filename) self.Session=sessionmaker() self.Session.configure(bind=self.Engine) self.SessionInstance = self.Session() metadata=MetaData(self.Engine) type_map={} for file_list in self.SessionInstance.query(FileList).all(): if type_map.has_key(file_list.type): continue type_map[file_list.type]=1 self.FileList.append(file_list) self.AttachedDatabases.append(file_list.type) query="ATTACH DATABASE '%s' AS %s;" % (file_list.filename,file_list.type) self.Engine.execute(query) else: self.Engine = create_engine('sqlite:///' + filename, echo = echo) self.metadata=MetaData(self.Engine) self.Session=sessionmaker() self.Session.configure(bind=self.Engine) self.SessionInstance = self.Session() self.SessionInstancesMap={} for file_list in self.SessionInstance.query(FileList).all(): filename=self.GetFilename(file_list.filename) engine = create_engine('sqlite:///' + filename, echo = echo) metadata=Base.metadata metadata.create_all(engine) self.Session=sessionmaker() self.Session.configure(bind=engine) self.SessionInstancesMap[file_list.type] = self.Session() self.FileList.append(file_list) engine.dispose() if not self.SessionInstancesMap.has_key('Source'): self.SessionInstancesMap['Source']=self.SessionInstance if not self.SessionInstancesMap.has_key('Target'): self.SessionInstancesMap['Target']=self.SessionInstance
def _ping_listener(engine, dbapi_conn, connection_rec, connection_proxy): """Ensures that MySQL and DB2 connections are alive. Borrowed from: http://groups.google.com/group/sqlalchemy/msg/a4ce563d802c929f """ cursor = dbapi_conn.cursor() try: ping_sql = 'select 1' if engine.name == 'ibm_db_sa': # DB2 requires a table expression ping_sql = 'select 1 from (values (1)) AS t1' cursor.execute(ping_sql) except Exception as ex: if engine.dialect.is_disconnect(ex, dbapi_conn, cursor): msg = _('Database server has gone away: %s') % ex LOG.warning(msg) # if the database server has gone away, all connections in the pool # have become invalid and we can safely close all of them here, # rather than waste time on checking of every single connection engine.dispose() # this will be handled by SQLAlchemy and will force it to create # a new connection and retry the original action raise sqla_exc.DisconnectionError(msg) else: raise
def postgres_db(orm_metadata): engine = create_engine( config.get_postgres_uri(), isolation_level="READ_COMMITTED", ) wait_for_engine_to_come_up(engine) orm_metadata.create_all(engine) yield engine engine.dispose()
def sendRating(rating, username): global engine dict = { 'rating': [rating], 'username': [username], 'date': [datetime.datetime.now()] } df = pd.DataFrame.from_dict(dict) df.to_sql('ratings', con=engine, if_exists='append', index=False) engine.dispose() return {"status": "rating sent"}
def destroy_db_engine(db_engine=None): """Destroys the DB SQAlchemy engine. :param db_engine: Engine to destroy :return: Nothing """ global engine if db_engine: db_engine.dispose() else: engine.dispose()
def putDataForUserGermany(): global engine api = configscript.setupTwitterAuth() places = api.geo_search(query="Germany", granularity="country") place_id = places[0].id tweets = tw.Cursor(api.search, q="place:%s" % place_id, tweet_mode='extended', lang='en').items() for tweet in tweets: username = tweet.user.screen_name allTweets = tw.Cursor(api.user_timeline, screen_name=username, tweet_mode="extended", exclude_replies=False, include_rts=False, lang='en').items(150) listAllTweets = list(allTweets) if (len(listAllTweets) < 1): continue tweetsDict = m.getTweetsDict(listAllTweets) score = m.getOverallScore(tweetsDict) if (score == -1): continue tweetsonlyscore = m.tweetsOnlyScore(tweetsDict) scoremax = m.getHappiestTweet(tweetsonlyscore) scoremin = m.getSaddestTweet(tweetsonlyscore) dict = { username: { "score": score, "min": scoremin["score"], "max": scoremax["score"] } } df = pd.DataFrame.from_dict(dict, orient='index') df.index.name = 'username' df.to_sql('germany_users', con=engine, if_exists='append') engine.execute( "DELETE FROM germany_users T1 USING germany_users T2 WHERE T1.ctid < T2.ctid AND T1.username = T2.username;" ) engine.dispose()
def putDataDB(): global engine api = configscript.setupTwitterAuth() places = api.geo_search(query="Denmark", granularity="country") place_id = places[0].id tweets = tw.Cursor(api.search, q="place:%s" % place_id, tweet_mode='extended', lang='en').items() df = pd.DataFrame.from_dict(m.getTweetsDictRaw(tweets), orient='index') df.set_index('id', inplace=True) df.to_sql('tweets', con=engine, if_exists='append') engine.execute( "DELETE FROM tweets T1 USING tweets T2 WHERE T1.ctid < T2.ctid AND T1.id = T2.id AND T1.score = T2.score AND T1.created = T2.created;" ) engine.dispose()
def celebrityScore(username): print(username) global engine api = configscript.setupTwitterAuth() allTweets = tw.Cursor(api.user_timeline, screen_name=username, tweet_mode="extended", exclude_replies=False, include_rts=False, lang='en').items() listAllTweets = list(allTweets) if (len(listAllTweets) == 0): return tweetsDict = m.getTweetsDict(listAllTweets) score = m.getOverallScore(tweetsDict) if (score == -1): return user = api.get_user(username) # Remove _normal from profile image URL profile_image_url = user.profile_image_url_https url = re.sub('_normal', '', profile_image_url) dict = {username: {"score": score, "pic": url}} df = pd.DataFrame.from_dict(dict, orient='index') df.index.name = 'username' df.to_sql('celebrity', con=engine, if_exists='append') engine.execute( "DELETE FROM celebrity T1 USING celebrity T2 WHERE T1.ctid < T2.ctid AND T1.username = T2.username;" ) engine.dispose()
def __init__(self, filename): echo = False if self.DebugLevel > 2: echo = True self.SessionInstance = None if not filename: return self.FileList = [] self.AttachedDatabases = [] if self.UseAttach: self.Engine = create_engine('sqlite://', echo=False) self.AttachedDatabases.append('Diff') self.Engine.execute("ATTACH DATABASE '%s' AS Diff;" % filename) self.Session = sessionmaker() self.Session.configure(bind=self.Engine) self.SessionInstance = self.Session() metadata = MetaData(self.Engine) type_map = {} for file_list in self.SessionInstance.query(FileList).all(): if type_map.has_key(file_list.type): continue type_map[file_list.type] = 1 self.FileList.append(file_list) dgf_filename = self.GetFilename(filename, file_list.filename) try: import sqlite3 conn = sqlite3.connect(dgf_filename) c = conn.cursor() c.execute( "SELECT name FROM sqlite_master WHERE type='table' AND name='OneLocationInfo'" ) data = c.fetchone() if data != None: c.execute( '''ALTER TABLE OneLocationInfo RENAME TO BasicBlock;''' ) conn.close() except: pass self.AttachedDatabases.append(file_list.type) query = "ATTACH DATABASE '%s' AS %s;" % (dgf_filename, file_list.type) self.Engine.execute(query) else: self.Engine = create_engine('sqlite:///' + filename, echo=echo) self.metadata = MetaData(self.Engine) self.Session = sessionmaker() self.Session.configure(bind=self.Engine) self.SessionInstance = self.Session() self.SessionInstancesMap = {} for file_list in self.SessionInstance.query(FileList).all(): filename = self.GetFilename(filename, file_list.filename) engine = create_engine('sqlite:///' + filename, echo=echo) metadata = Base.metadata metadata.create_all(engine) self.Session = sessionmaker() self.Session.configure(bind=engine) self.SessionInstancesMap[file_list.type] = self.Session() self.FileList.append(file_list) engine.dispose() if not self.SessionInstancesMap.has_key('Source'): self.SessionInstancesMap['Source'] = self.SessionInstance if not self.SessionInstancesMap.has_key('Target'): self.SessionInstancesMap['Target'] = self.SessionInstance
def __init__(self, filename): echo = False if self.DebugLevel > 2: echo = True self.SessionInstance=None if not filename: return self.FileList=[] self.AttachedDatabases=[] if self.UseAttach: self.Engine=create_engine('sqlite://',echo=False) self.AttachedDatabases.append('Diff') self.Engine.execute("ATTACH DATABASE '%s' AS Diff;" % filename) self.Session=sessionmaker() self.Session.configure(bind=self.Engine) self.SessionInstance = self.Session() metadata=MetaData(self.Engine) type_map={} for file_list in self.SessionInstance.query(FileList).all(): if type_map.has_key(file_list.type): continue type_map[file_list.type]=1 self.FileList.append(file_list) dgf_filename=self.GetFilename(filename, file_list.filename) try: import sqlite3 conn = sqlite3.connect(dgf_filename) c=conn.cursor() c.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='OneLocationInfo'") data=c.fetchone() if data!=None: c.execute('''ALTER TABLE OneLocationInfo RENAME TO BasicBlock;''') conn.close() except: pass self.AttachedDatabases.append(file_list.type) query="ATTACH DATABASE '%s' AS %s;" % (dgf_filename,file_list.type) self.Engine.execute(query) else: self.Engine = create_engine('sqlite:///' + filename, echo = echo) self.metadata=MetaData(self.Engine) self.Session=sessionmaker() self.Session.configure(bind=self.Engine) self.SessionInstance = self.Session() self.SessionInstancesMap={} for file_list in self.SessionInstance.query(FileList).all(): filename=self.GetFilename(filename, file_list.filename) engine = create_engine('sqlite:///' + filename, echo = echo) metadata=Base.metadata metadata.create_all(engine) self.Session=sessionmaker() self.Session.configure(bind=engine) self.SessionInstancesMap[file_list.type] = self.Session() self.FileList.append(file_list) engine.dispose() if not self.SessionInstancesMap.has_key('Source'): self.SessionInstancesMap['Source']=self.SessionInstance if not self.SessionInstancesMap.has_key('Target'): self.SessionInstancesMap['Target']=self.SessionInstance