def harvest_users_from_tweets(session: sqlalchemy.orm.Session,
FLUSH_LIMIT=10,
startTweet=None):
"""This iterates through tweets from the database and harvests user data from them"""
users = 0
lastTweetId = None
tweetIter = tweets_with_other_data_generator(session)
try:
while True:
tweet = next(tweetIter)
user = update_or_create_user_from_tweet(tweet, session)
users += 1
lastTweetId = tweet.tweetID
if users % FLUSH_LIMIT == 0:
print('flushing at %s users' % users)
session.commit()
except StopIteration:
print("%s users created or updated" % users)
session.commit()
finally:
print("Last processed tweet %s" % lastTweetId)
# session.commit()
session.close()
def transaction_cm(session: sa.orm.Session) -> tp.ContextManager[None]:
"""Provide a transactional scope around a series of operations."""
try:
yield
session.commit()
except Exception:
session.rollback()
raise
finally:
session.close()