def scan_partial():
kwargs = {}
kwargs['query_filter'] = {'instagrams__null': False}
#kwargs['index'] = Tweet.index_league
#kwargs['league__eq'] = 'nfl'
kwargs['index'] = Tweet.index_site
kwargs['site__eq'] = 'd.com'
return Tweet().query_2(**kwargs)
def scan_all():
kwargs = {
'unknown_instagrams__null': False,
'instagram_name__null': False,
"_ts_ms__gt": str(yesterday),
'conditional_operator': 'AND'
}
kwargs['attributes'] = attribute_list
return Tweet().scan(**kwargs)
def scan_all():
kwargs = {}
if len(sys.argv) > 1:
kwargs['_ts_ms__gt'] = str(gt).split('.')[0]
if len(sys.argv) > 2:
kwargs['_ts_ms__lt'] = str(lt).split('.')[0]
kwargs['instagrams__null'] = False
print 'fix kwargs:', kwargs
return Tweet().scan(**kwargs)
def backfill_twitter(self, client, incoming):
bf = incoming['backfill']
last = incoming['last'] if 'last' in incoming else None
kwargs = self.fill_kwargs(client, bf, last, True)
print 'backfill args:', kwargs
threads.deferToThread(Tweet().query_2,
**kwargs).addCallback(self.filling).addCallback(
self.windout,
client).addErrback(self.errorout, client, bf)
def handle_queue(sq):
print 'handle queue:', ic.page().url().toString()
url = fixed.simpleurl(sq[0])
print 'new url:', url
tweet = Tweet().get_item(_twitter_id=sq[1][Tweet.tweet_user_id],
_tweet_id=sq[1][Tweet.tweet_id])
d = ic.goto_url(url)
d.addCallback(lambda ign: ic.get_image_data(url, tweet))
d.addCallback(ic.post_process, tweet)
d.addBoth(prime_queue)
return d
def last_conversation(l):
try:
tweet = [
t for t in Tweet().query_2(index=Tweet.index_league,
league__eq=l,
limit=1,
reverse=True,
query_filter=last_conversation_filter)
][0]
return tweet._data
except IndexError:
pass
except Exception as e:
print e.__class__.__name__, e
def last_retweet(l):
try:
tweet = [
t for t in Tweet().query_2(index=Tweet.index_league,
league__eq=l,
limit=1,
reverse=True,
query_filter=last_retweet_filter,
conditional_operator='OR')
][0]
return tweet._data
except IndexError:
pass
except Exception as e:
print e.__class__.__name__, e
def check_recover_queue():
rq = RecoverQueue()
print 'check recover queue:', rq
try:
while True:
msg = rq.getMessage()
if msg:
recover = json.loads(msg.get_body())
print 'recover msg:', recover[keys.entity_twitter_id]
for r in Tweet().query_2(
_twitter_id__eq=recover[keys.entity_twitter_id],
limit=1,
reverse=True):
print 'got last tweet:'
do_recover(r)
print 'delete recover msg!'
rq.deleteMessage(msg)
else:
break
except Exception as e:
print 'recover exception:', e
exit()
from amazon.dynamo import Tweet
import time
all_keys = set([])
all_keys_count = 0
for e in Tweet().scan():
for k in e._data.keys():
all_keys.add(k)
if len(all_keys) > all_keys_count:
all_keys_count = len(all_keys)
print ''
print sorted(list(all_keys))
print ''
time.sleep(.25)
def store_tweet(self, tweet):
if not Tweet().put_item(data=tweet):
print 'bad:', tweet
exit(-1)
'source_instagram', 'source_twitter', 'team_href', 'ts_add', 'ts_avatar', 'ts_backup',
'ts_delta', 'ts_following', 'ts_match_twitter', 'ts_retweet',
'ts_tweet', 'roster_url', 'version',
twitter_keys.match_posts,
twitter_keys.match_followers,
instagram_keys.instagram_avi,
ProfileTwitter.following,
instagram_keys.instagram_url,
instagram_keys.instagram_verified
]
update_keys = {'prize_money': 'prizemoney'}
kwargs = {}
for k in remove_keys + update_keys.keys():
kwargs[k + '__null'] = False
kwargs['attributes'] = [Tweet.tweet_id, Tweet.tweet_user_id] + remove_keys + update_keys.keys()
kwargs['conditional_operator'] = 'OR'
for e in Tweet().scan(**kwargs):
for k in remove_keys:
if e[k]:
del e[k]
for k in update_keys.keys():
if e[k]:
e[update_keys[k]] = e[k]
try:
e.partial_save()
pprint.pprint(e._data)
time.sleep(.2)
except:
pass
def run_charts():
print 'run charts'
gsi = Tweet().describe()['Table']['GlobalSecondaryIndexes']
oil = [g for g in gsi
if g['IndexName'] == Tweet.index_league][0]['ProvisionedThroughput']
oil = {
Tweet.index_league: {
'read': oil['ReadCapacityUnits'],
'write': oil['WriteCapacityUnits']
}
}
nil = copy.deepcopy(oil)
nil[Tweet.index_league]['read'] = 200
oil[Tweet.index_league]['read'] = 20
try:
print 'increate throughput:', nil
Tweet().update_global_secondary_index(global_indexes=nil)
except Exception as e:
print 'increase exception:', e
yield task.deferLater(reactor, 15, defer.succeed, True)
for u in User().scan(role__contains='.com', twitter_apps__null=False):
try:
site = u[user_keys.user_role]
chart_tweet = tweets.chart_tweet(u, 'Last 24 Hours')
print 'chart tweet:', chart_tweet
yield task.deferLater(reactor, 1, defer.succeed, True)
print 'site:', site
if not u[keys.entity_description]:
u[keys.entity_description] = 'placeholder'
u.partial_save()
yesterday = int(time.time()) - (24 * 60 * 60)
yesterday = str(yesterday * 1000)
curator = User().get_by_role(site, keys.entity_twitter)
charts = []
categories = list(curator[user_keys.user_site_leagues])
for c in categories:
#tc = SocialBeta().query_count(SocialBeta.index_social_ts_received, league__eq=c, social_ts_received__gt=yesterday, query_filter={'social_act__eq': 'tweet'})
tweet_filter = {
'known_retweet__null': True,
'unknown_retweet__null': True,
'_ts_ms__gt': yesterday
}
tc = Tweet().query_count(index=Tweet.index_league,
league__eq=c,
query_filter=tweet_filter,
scan_index_forward=False)
known_retweet_filter = {
'known_retweet__null': False,
'_ts_ms__gt': yesterday
}
krc = Tweet().query_count(index=Tweet.index_league,
league__eq=c,
query_filter=known_retweet_filter,
scan_index_forward=False)
unknown_retweet_filter = {
'unknown_retweet__null': False,
'_ts_ms__gt': yesterday
}
urc = Tweet().query_count(index=Tweet.index_league,
league__eq=c,
query_filter=unknown_retweet_filter,
scan_index_forward=False)
conversion_filter = {
'known_conversation__null': False,
'_ts_ms__gt': yesterday
}
cc = Tweet().query_count(index=Tweet.index_league,
league__eq=c,
query_filter=conversion_filter,
scan_index_forward=False)
ce = {c: (tc, krc, urc, cc)}
print ce
charts.append(ce)
charts.sort(key=lambda x: x[x.keys()[0]][0])
tweets_list = [t[t.keys()[0]][0] for t in charts]
known_retweets_list = [t[t.keys()[0]][1] for t in charts]
unknown_retweets_list = [t[t.keys()[0]][2] for t in charts]
conversations_list = [t[t.keys()[0]][3] for t in charts]
categories = [t.keys()[0] for t in charts]
height = 200 + (40 * len(categories))
view = QQuickView()
view.setSource(QUrl('qml/render/chart_daily.qml'))
view.setWidth(800)
view.setHeight(height)
view.rootObject().setProperty('categories', categories)
view.rootObject().setProperty('_height', height)
view.rootObject().setProperty(
'title', 'Tweets/Retweets/Conversations - ' + site)
view.rootObject().setProperty('description',
u[keys.entity_description])
view.rootObject().setProperty('conversations', conversations_list)
view.rootObject().setProperty('retweets', known_retweets_list)
view.rootObject().setProperty('unknown_retweets',
unknown_retweets_list)
view.rootObject().setProperty('tweets', tweets_list)
view.show()
yield task.deferLater(reactor, 5, defer.succeed, True)
img = view.grabWindow()
chart_location = '/home/ubuntu/Desktop/' + site + '.png'
res = img.save(chart_location)
print 'save result:', res
yield task.deferLater(reactor, 5, defer.succeed, True)
restful.post_tweet(u, u, chart_tweet, chart_location)
yield task.deferLater(reactor, 15, defer.succeed, True)
os.remove(chart_location)
view.deleteLater()
except Exception as e:
print 'chart exception:', e
try:
print 'decreate throughput:', oil
Tweet().update_global_secondary_index(global_indexes=oil)
except Exception as e:
print 'descrease exception:', e
yield task.deferLater(reactor, 15, defer.succeed, True)
reactor.callLater(0, reactor.stop)