def save_tweet_posted_status(self, _id, status):
# Pass in the ID (tweet_id-account) rather than just the tweet_id.
t = (status, now(), _id, self.env.default_account)
self.c.execute(
'UPDATE tweet set status = ?, posted_at = ? where t_id = ? and account = ?',
t)
self.commit()
def save_retweet(self, tweet_id):
t = (tweet_id, self.env.default_account, now())
try:
self.c.execute(
'insert into retweets (tweet_id, account, retweeted_at) values (?, ?, ?)',
t)
self.commit()
except sqlite3.IntegrityError:
logger.warning(
f'Integrity error. Duplicate in retweets (tweet_id={tweet_id}, '
f'account={self.env.default_account}')
def save_command(command, status_text, db_summary, polling_api):
logger.info(f'{command.id}: {status_text}')
# try:
# polling_api.PostUpdate(status_text,
# in_reply_to_status_id=command.id,
# auto_populate_reply_metadata=True)
# except TwitterError as e:
# logger.error(e.message)
command.status = status_text
command.processed_date = now()
db_summary.save_command(command)
db_summary.commit()
def save_tweet(self, tweet):
self.delete_tweet_if_exists(tweet)
t = (f'{tweet.id}-{tweet.account}', tweet.type, tweet.tweet_id,
tweet.head, tweet.tail, tweet.tweet_screen_name, None,
tweet.status, None, tweet.drafted_at, now(), tweet.image_head,
tweet.date_nkey, tweet.period, tweet.account, None, tweet.trend,
tweet.background_image)
self.c.execute(
'INSERT INTO tweet (t_id, type, tweet_id, head, tail, tweet_screen_name, '
+
'tweet_retweet_count, status, tweet_created_at, drafted_at, submitted_at, '
+
'image_head, date_nkey, period, account, tweeter_type, trend, background_image) '
+ 'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
t)
for item in tweet.items:
self.save_tweet_item(tweet, item)
def main():
# parser = argparse.ArgumentParser(description='Draft stats for the given day and push to cloud for approval.')
# parser.add_argument('date', metavar='yyyy-mm-dd',
# help='the date to process')
#
# args = parser.parse_args()
environment = defaults.get_environment()
db = DB(environment, today())
db_summary = DBSummary(environment)
jdata = Publisher.get_pending(environment)
# c = db_connect(env.summary_database)
trenders_published = list()
trenders_all = list()
already_processed = list()
if 'tweets' in jdata:
for tweet in jdata['tweets']:
tweet_status = db_summary.get_tweet_status(tweet['t_id'])
if tweet_status is None:
db_summary.save_tweet(tweet)
for item in tweet['items']:
db_summary.save_tweet_item(tweet, item)
if tweet['type'] == 'trenders' and item['selected'] == 'Y':
trenders_all.append(item['tweet_text'][1:])
if tweet['status'] == 'pend-post':
trenders_published.append(item['tweet_text'][1:])
elif tweet_status in ['posted', 'rejected']:
tweet['status'] = tweet_status
already_processed.append(tweet)
if len(trenders_published) > 0:
with open(
f'{environment.bot_data_directory}/trenders_all_{yesterday_file()}.txt',
'a') as f:
for sn in trenders_all:
f.write("%s\n" % sn)
with open(
f'{environment.bot_data_directory}/trenders_published_{yesterday_file()}.txt',
'a') as f:
for sn in trenders_published:
f.write("%s\n" % sn)
db_summary.disconnect()
trend_date = now()
# now = now()
# yesterday = (datetime.datetime.now() - timedelta(days=2)).strftime('%Y-%m-%d')
if 'trends' in jdata:
if len(jdata['trends']) > 0:
# c = db_connect()
# t = (yesterday,)
# c.execute('SELECT max(max_id) max_id FROM tag_history where date <= ?', t)
# row = c.fetchone()
# max_id = 0
# if row != None:
# max_id = row[0]
relevant_words = db.get_relevant_words()
generic_words = db.get_generic_words()
trs = list()
for trend in jdata['trends']:
tag = '#' + trend['hashtag'].lower()
tr = {
'hashtag': trend['hashtag'],
'status': 'posted',
'trend_at': trend_date
}
trs.append(tr)
tag_discovery_result = db.get_tag_discovery_result(tag)
status = nvl(tag_discovery_result, 'NONE')
if trend['status'] == 'pend-post' and status in ('NONE',
'AUTO_DEL',
'MAN_DEL'):
logger.info('Adding: ' + tag)
db.save_tag_discovery(tag, 'MAN_ADD')
elif trend['status'] == 'pend-del' and status in ('AUTO_ADD',
'MAN_ADD'):
logger.info('Deleting: ' + tag)
db.save_tag_discovery(tag, 'MAN_DEL')
# Trend relevance
if 'relevance' in trend:
relevance = relevant_words[
trend['hashtag'].lower()] if trend['hashtag'].lower(
) in relevant_words else 'neutral'
if trend['relevance'] != relevance:
new_relevance = None if trend[
'relevance'] == 'neutral' else trend['relevance']
db.set_word_relevance(trend['hashtag'], new_relevance)
# Trend generic
if 'generic' in trend:
generic = generic_words[trend['hashtag'].lower(
)] if trend['hashtag'].lower() in generic_words else ''
if trend['generic'] != generic:
new_relevance = None if trend[
'generic'] == 'neutral' else trend['generic']
db.set_word_generic(trend['hashtag'], new_relevance)
data = {'trends': trs}
Publisher.publish(environment, data, 'trends')
db.commit()
open(f'{environment.temp_file_directory}/compute_daily',
'a').close()
if 'categories' in jdata:
if len(jdata['categories']) > 0:
for cat in jdata['categories']:
db.set_tweeter_category(cat['screen_name'], cat['category'])
logger.info("Category for", cat['screen_name'], "changed to",
cat['category'])
db.commit()
if 'words' in jdata:
if len(jdata['words']) > 0:
for word in jdata['words']:
category = word['category']
if category == '':
category = None
db.set_word_generic(word['word'], category)
logger.info("Generic for", word['word'], "changed to",
category)
db.disconnect()
if len(already_processed) > 0:
data = {'tweets': already_processed}
Publisher.publish(environment, data, 'posted')
def write_tweet(self, count):
period = 'day' # Only worried about daily stats for now
showdate = datetime.datetime.strptime(self.date, '%Y-%m-%d').strftime('%d %B %Y').lstrip('0')
trend = self.trend if self.trend is not None else ''
tweet = DraftStatsTweet(
id='{}-{}'.format(self.date, self.trend if self.type == 'trenders' else self.type),
type=self.type,
head={'mentions': "Top mentions " + showdate + ":\n",
'trends': "Trends " + showdate + ":\n",
'trenders': "Top tweeps for #%s:\n" % trend}[self.type],
tail={'mentions': "\n-\n#PTI #PMLN #PPP",
'trends': "\n-\n#PTI #PMLN #PPP",
'trenders': ""}[self.type],
image_head={'mentions': "Top mentions for the " + period,
'trends': "Top trends for the " + period,
'trenders': "Top tweeps for #%s" % trend}[self.type],
drafted_at=now(),
date_nkey=self.date,
tweet_screen_name=self.env.default_account,
account=self.env.default_account,
period=period,
trend=self.trend,
background_image=None,
status='pend-post',
tweet_id=None
)
i = 0
while i < len(self.items) and i < count:
item = DraftStatsTweetItem(rank=i + 1)
if self.type != 'trends':
screen_name = self.items[i][0][0][0]
# logger.info(f'Screen name: {screen_name}')
score = self.items[i][1]
starcount = int(score / 25) if self.type == 'mentions' else int(score / 100)
# else:
# if screen_name in self.tweeps:
# subitem.update(self.tweeps[screen_name])
(url, name, category) = self.get_tweeter_details(screen_name)
# item['rank'] = i + 1
# item['subitems'] = list()
subitem = DraftStatsTweetSubItem(score=score,
star_count=starcount,
display_image=url,
tweet_text='@' + screen_name,
display_text=name,
category=category)
if self.type == 'trenders' and screen_name in self.tweeps:
tweep = self.tweeps[screen_name]
subitem.tweet_count = tweep.get('tweet_count')
subitem.rt_count = tweep.get('rt_count')
subitem.rt_received_count = tweep.get('rt_received_count')
subitem.botness = tweep.get('botness')
# subitem['a'] = int(starcount / 25)
# subitem['b'] = int((starcount % 25) / 5)
# subitem['c'] = int(starcount % 5)
# subitem['display_image'] = url
# subitem['tweet_text'] = '@' + screen_name
# subitem['display_text'] = name
# subitem['category'] = category
item.subitems.append(subitem)
logger.debug("%3d %5d %s", item.rank, subitem.score, subitem.tweet_text)
else:
# item = dict()
# item['rank'] = i + 1
# item['subitems'] = list()
subrank = 1
for t, s in self.items[i][0]:
subitem = DraftStatsTweetSubItem(score=s,
tweet_text=t,
subrank=subrank,
star_count=int(s / 250))
# subitem['score'] = s
# subitem['tweet_text'] = t
# subitem['subrank'] = subrank
# starcount = int(subitem['score'] / 250)
# subitem['a'] = int(starcount / 25)
# subitem['b'] = int((starcount % 25) / 5)
# subitem['c'] = int(starcount % 5)
word = t.lower()[1:]
english_word = urdu_to_english(word)
if english_word in self.hashtags and word in self.tag_history:
subitem.tweet_count = self.hashtags[english_word]['tweets']
subitem.tweep_count = self.hashtags[english_word]['tweeps']
subrank += 1
item.subitems.append(subitem)
tweet.items.append(item)
i += 1
return tweet