class SlackFeedItem(db.Model):
__tablename__ = 'slack_feed_items'
id = db.Column(db.BigInteger, primary_key=True)
slack_feed_id = db.Column(db.Integer,
db.ForeignKey('feed_setting_slack.id'))
timestamp = db.Column(db.DateTime(timezone=True), default=None)
data = db.Column(db.PickleType)
@property
def ts(self):
return self.data['ts']
class TwitterFeedItem(db.Model):
__tablename__ = 'twitter_feed_items'
id = db.Column(db.BigInteger, primary_key=True)
twitter_feed_id = db.Column(db.Integer,
db.ForeignKey('feed_setting_twitter.id'))
tweet_id = db.Column(db.BigInteger)
tweet_retrieved = db.Column(db.DateTime(timezone=True),
default=datetime.now)
tweet_data = db.Column(db.PickleType)
def __init__(self, tweet_id, twitter_feed_id, tweet_data):
self.tweet_id = tweet_id
self.twitter_feed_id = twitter_feed_id
self.tweet_data = tweet_data
@property
def status_url(self):
return ('https://twitter.com/statuses/' + str(self.tweet_id))
class GithubFeedItem(db.Model):
__tablename__ = 'github_feed_items'
id = db.Column(db.BigInteger, primary_key=True)
github_feed_id = db.Column(db.Integer, db.ForeignKey('feed_setting_github.id'))
# todo: need to add an index on this due to the lookups here
sha = db.Column( db.String(40) , default=None, index=True) # git commit id
commit_date = db.Column( db.DateTime(timezone=True), default=None )
date_retrieved = db.Column( db.DateTime(timezone=True), default=datetime.now )
# throwing these all in until decide what is best to use here :/
git_commit_data = db.Column( db.PickleType )
# other potential ideas: github_commit data - includes stuff like author data
def __init__(self, github_feed_id=github_feed_id, git_commit_data=git_commit_data ):
self.github_feed_id = github_feed_id
self.date_retrieved = datetime.now()
self.sha = git_commit_data.sha
self.git_commit_data = git_commit_data.raw_data['commit']
self.commit_date = dateutil.parser.parse( git_commit_data.raw_data['commit']['committer']['date'] ).replace( tzinfo=None )
class TwitterFeedSetting(db.Model):
__tablename__ = 'feed_setting_twitter'
id = db.Column(db.Integer, primary_key=True)
user_id = db.Column(db.Integer, db.ForeignKey('users.uid'))
# handle = db.Column( db.String(64) )
hashtag = db.Column(db.String(128), default=None)
status = db.Column(db.String(64), default=None)
last_updated = db.Column(db.DateTime(timezone=True), default=None)
feed_items = db.relationship('TwitterFeedItem',
backref='feed_setting_twitter',
cascade="all, delete-orphan",
lazy='select')
#todo: move method that starts adding hashtags here
def set_updating(self):
self.status = 'updating'
db.session.commit()
return self
def set_updated(self):
self.status = 'updated'
self.last_updated = datetime.now()
db.session.commit()
return self
def start_populate(self):
# download 200 latest tweets with this hash tag, this can go up to 1500 according to twitter api docs
self.download_tweets(max_tweets=200)
return True
def get_last_tweet_id_downloaded(self):
return db.session.query(db.func.max(
TwitterFeedItem.tweet_id)).filter_by(
twitter_feed_id=self.id).scalar()
def download_tweets(self, since_id=False, max_tweets=100):
query = self.hashtag
self.set_updating()
if since_id:
hashtag_tweets = [
status for status in tweepy.Cursor(
tweepy_API.search, rpp=100, q=query,
since_id=since_id).items(max_tweets)
]
else:
hashtag_tweets = [
status for status in tweepy.Cursor(
tweepy_API.search, rpp=100, q=query).items(max_tweets)
]
for tweet in hashtag_tweets:
feed_item = TwitterFeedItem(tweet.id, self.id, tweet)
db.session.add(feed_item)
self.set_updated()
db.session.commit()
def do_feed_update(self):
last_tweet = self.get_last_tweet_id_downloaded()
if (not (last_tweet)):
self.start_populate()
else:
self.download_tweets(since_id=last_tweet)
return True
@classmethod
def belonging_to_user(cls, user_id):
return cls.query.filter_by(user_id=user_id).all()
@classmethod
def update_items(cls):
items = cls.query.filter_by(status='updated').all()
for item in items:
item.do_feed_update()
db.session.close()
return True
@classmethod
def new_items(cls):
return cls.query.filter_by(status='new').all()
@classmethod
def start_populate_new_items(cls):
new_items = cls.new_items()
for item in new_items:
item.start_populate()
db.session.close()
return True
class GithubFeedSetting(db.Model):
__tablename__ = 'feed_setting_github'
id = db.Column(db.Integer, primary_key=True)
user_id = db.Column(db.Integer, db.ForeignKey('users.uid'))
username = db.Column( db.String(128) )
project = db.Column( db.String(128) , default=None)
branch = db.Column( db.String(128) , default='master')
status = db.Column( db.String(64), default=None)
last_updated = db.Column( db.DateTime(timezone=True), default=None )
feed_items = db.relationship( 'GithubFeedItem' , backref='github_feed_items', cascade="all, delete-orphan", lazy='select')
def set_updating(self):
self.status = 'updating'
db.session.commit()
return self
def set_updated(self):
self.status = 'updated'
self.last_updated = datetime.now()
db.session.commit()
return self
def get_branch(self):
branch = self.branch
if ( not(self.branch) or (self.branch.strip() == '') ):
branch = 'master'
return branch
@property
def latest_feed_item(self):
items = GithubFeedItem.query.order_by(
db.desc( GithubFeedItem.commit_date )
).filter_by(
github_feed_id=self.id
).limit(1).all()
if len(items) == 1:
return items[0]
else:
return False
def download_commits(self, since=False):
self.set_updating()
#todo: if this feed update fails, due to exception like http error, throttle limit, etc, then
# we need revert the status and (probably) log the error
if type(since) is datetime:
commits = self.repo.get_commits(sha=self.branch, since=since )
else:
commits = self.repo.get_commits(sha=self.branch)
for commit in commits:
# when interating over this PaginatedList type of class it results in
# a single request downloading each commit details which may result in too many
# request and end up hitting our 5000 req/s rate limit -
# in that case may wish to use the requests lib to just make a call to api endpoint like
# https://api.github.com/repos/jwenerd/SKTimeline/commits?sha=github_events&per_page=100&since=2016-07-13T19:49:47Z to fetch these details in one go
if not( self.commit_already_stored(commit.sha) ):
feed_item = GithubFeedItem(github_feed_id=self.id, git_commit_data=commit)
db.session.add(feed_item)
self.set_updated()
db.session.commit()
def do_feed_update(self):
since = False
if self.latest_feed_item:
since = self.latest_feed_item.commit_date
self.download_commits(since=since)
def commit_already_stored(self, sha):
count = GithubFeedItem.query.filter_by(github_feed_id=self.id).filter_by(sha=sha).count()
return count > 0
@property
def repo(self):
repo = GithubAPI.get_repo( self.username + '/' + self.project)
return repo
@classmethod
def belonging_to_user(cls, user_id):
return cls.query.filter_by(user_id=user_id).all()
@classmethod
def update_items(cls):
items = cls.query.filter_by(status='updated').all()
for item in items:
item.do_feed_update()
db.session.close()
return True
@classmethod
def new_items(cls):
return cls.query.filter_by(status='new').all()
@classmethod
def start_populate_new_items(cls):
for item in cls.new_items():
item.do_feed_update()
db.session.close()
return True
class SlackFeedSetting(db.Model):
__tablename__ = 'feed_setting_slack'
id = db.Column(db.Integer, primary_key=True)
user_id = db.Column(db.Integer, db.ForeignKey('users.uid'))
status = db.Column(db.String(64), default=None)
last_updated = db.Column(db.DateTime(timezone=True), default=None)
# holds the token info, user id, team id, team name
slack_auth_info = db.Column(db.PickleType)
# slack machine readable id used for api request
channel_id = db.Column(db.String(128), default=None)
# channel info like name/description, members, etc
channel_info = db.Column(db.PickleType)
feed_items = db.relationship('SlackFeedItem',
backref='slack_feed_items',
cascade="all, delete-orphan",
lazy='select')
user_data = db.Column(db.PickleType)
_slack_client = False
def __init__(self):
self.status = 'new'
self.last_updated = datetime.now()
def set_updating(self):
self.status = 'updating'
db.session.commit()
return self
def set_updated(self):
self.status = 'updated'
self.last_updated = datetime.now()
db.session.commit()
return self
def start_populate(self):
# download 200 latest tweets with this hash tag, this can go up to 1500 according to twitter api docs
self.download_history(count=100)
return True
def download_history(self, count=100, oldest=0):
self.set_updating()
response = self.slack_client.api_call('channels.history',
channel=self.channel_id,
count=count,
oldest=oldest)
if response['messages']:
for message in response['messages']:
feed_item = SlackFeedItem()
feed_item.timestamp = datetime.utcfromtimestamp(
float(message['ts']))
feed_item.slack_feed_id = self.id
feed_item.data = message
db.session.add(feed_item)
self.set_updated()
db.session.commit()
return response
@property
def latest_feed_item(self):
items = SlackFeedItem.query.order_by(db.desc(
SlackFeedItem.timestamp)).filter_by(
slack_feed_id=self.id).limit(1).all()
if len(items) == 1:
return items[0]
else:
return False
@classmethod
def new_items(cls):
return cls.query.filter_by(status='new').all()
@classmethod
def start_populate_new_items(cls):
new_items = cls.new_items()
for item in new_items:
item.start_populate()
db.session.close()
return True
def do_update(self):
oldest = 0
if self.latest_feed_item:
oldest = self.latest_feed_item.ts
self.download_history(count=1000, oldest=oldest)
# todo: will need to see how to handle this if there are > 1000 messages to add
return True
@classmethod
def update_items(cls):
items = cls.query.filter_by(status='updated').all()
for item in items:
item.do_update()
db.session.close()
return True
@property
def slack_client(self):
if not self.is_token_info_present:
return False
if not self._slack_client:
self._slack_client = SlackClient(self.token)
return self._slack_client
@property
def team_name(self):
if not self.slack_auth_info:
return False
return self.slack_auth_info['team_name']
@property
def is_channel_info_present(self):
if not self.channel_info:
return False
return True
@property
def channel_name(self):
if not self.is_channel_info_present:
return False
return self.channel_info['name']
@property
def is_token_info_present(self):
if not self.slack_auth_info:
return False
if not self.slack_auth_info['access_token']:
return False
return True
@property
def token(self):
if not self.is_token_info_present:
return False
return self.slack_auth_info['access_token']
@classmethod
# todo: these can be dry by using module I think
def belonging_to_user(cls, user_id):
return cls.query.filter_by(user_id=user_id).all()
def slack_user_info(self, user_id):
if not self.user_data:
self.user_data = {}
if user_id in self.user_data:
# todo: check the _time_retrieved and add and expire length time to see if we should update these records after a certain amount of time
return self.user_data[user_id]
user = self.slack_client.api_call('users.info', user=user_id)
if ('ok' in user) and user['ok']:
user = user['user']
user['_time_retrieved'] = int(
datetime.now().strftime("%s")
) # use this time later for for determining when to update
user_data_cache = self.user_data
user_data_cache[user_id] = user
self.user_data = None
db.session.commit()
self.user_data = user_data_cache
db.session.commit()
return user
return False
def user_name_from_id(self, user_id):
data = self.slack_user_info(user_id)
if not data:
return False
return data['name']
def channel_name_from_id(self, channel_id):
if channel_id == self.channel_id:
return self.channel_name
else:
# for now going to make a request to the API to get the channel name here,
# we should reconsider another approach here later where it can cache the channel info data, so that it does not continually
# repeat the same requests over and over again
channel_info = self.slack_client.api_call('channels.info',
channel=channel_id)
name = ''
if channel_info['ok']:
name = channel_info['channel']['name']
return name