def get_last_league(site, l, retweet, conversation):
lt = last_tweet(l)
try:
print '{:22s} {:15s} {:15s} {:16s} {:20s}'.format(
site, l, fixed.lingo_since_date(int(lt[Tweet.ts_ms]) / 1000),
lt[keys.entity_twitter], lt[Tweet.tweet_id])
if retweet:
lr = last_retweet(l)
if lr:
print '{:23s} {:14s} {:15s} {:16s} {:25s}'.format(
'', 'retweet',
fixed.lingo_since_date(int(lr[Tweet.ts_ms]) / 1000),
lr[keys.entity_twitter], lr[Tweet.tweet_id])
if conversation:
lc = last_conversation(l)
if lc:
print '{:23s} {:14s} {:15s} {:16s} {:25s}'.format(
'', 'conversation',
fixed.lingo_since_date(int(lc[Tweet.ts_ms]) / 1000),
lc[keys.entity_twitter], lc[Tweet.tweet_id])
except:
locked_league = User().get_by_role(l, keys.entity_twitter)
locked = False
if locked_league[user_keys.user_locked]:
locked = locked_league[user_keys.user_locked]
print '{:22s} {:15s} {:20s}'.format(site, l, 'unknown-' + str(locked))
def inflate_service(perform=False):
users = []
for t in [User]:
for u in t().scan():
if u[user_keys.user_twitter_auth] and '.com' not in u[
user_keys.
user_role] and user_keys.user_locked not in u.keys():
ua = auth.user_app(u)
if ua:
users.append(u)
sorted_users = sorted(users, key=lambda item: item[time_keys.ts_inflated])
for i, so in enumerate(sorted_users):
print '{:3s}'.format(str(i + 1)), '{:20s}'.format(
so[user_keys.user_role]), fixed.lingo_since_date(
so[time_keys.ts_inflated]), so.table.table_name
if perform:
inflate_user = sorted_users[0]
inflate_app_user = auth.user_app(inflate_user)
print ''
print inflate_user[user_keys.user_role], inflate_app_user[
user_keys.user_role], inflate_user.table.table_name
restful.Inflate().do_inflate(inflate_user, inflate_app_user)
StalkQueue().createMessage(
{keys.entity_league: inflate_user[user_keys.user_role]})
def stalk_analysis(league_name):
mutuals = []
extras = []
for mf in Entity().query_2(league__eq=league_name):
if 'twitter' in mf and 'ts_followers_' + league_name in mf:
try:
ts_add = None
if mf[time_keys.ts_scout]:
ts_add = mf[time_keys.ts_scout]
slt = ProfileTwitter().profile_last(mf[keys.entity_twitter_id], None, ts_add)
if slt is not None:
print 'append:', mf[keys.entity_twitter], 'since:', fixed.lingo_since_date(ts_add)
mutuals.append( (mf, slt) )
else:
print 'no last stats:', mf[keys.entity_twitter], mf[keys.entity_twitter_id]
except Exception as e:
print 'missing:', e, 'https://twitter.com/' + mf[keys.entity_twitter], fixed.lingo_since(mf, twitter_keys.league_ts_followers(league_name))
else:
extras.append(mf._data)
print 'extras length:', len(extras)
for mutual_seq in mutuals:
try:
mutual = mutual_seq[0]
mutual_slt = mutual_seq[1]
print 'mutual:', mutual[keys.entity_twitter]
tf = set([])
for other_seq in [others for others in mutuals if others[0] != mutual and twitter_keys.league_mutual(league_name) in others[1]]:
if mutual[keys.entity_twitter_id] in other_seq[1][twitter_keys.league_mutual(league_name)]:
tf.add(other_seq[0][keys.entity_twitter_id])
if len(tf) > 0:
print mutual[keys.entity_twitter], 'follows:', len(tf), 'following:', 0 if twitter_keys.league_mutual(league_name) not in mutual_slt else len(mutual_slt[twitter_keys.league_mutual(league_name)])
mutual_slt[twitter_keys.league_follows(league_name)] = tf
mutual_slt.partial_save()
else:
print 'not following anyone:', mutual[keys.entity_twitter], mutual[keys.entity_twitter_id]
except Exception as e:
print 'mutual exception:', e
publish = []
curator = User().get_curator(league_name)
for mutual_seq_2 in mutuals:
try:
p = {}
mutual = mutual_seq_2[0]
p.update(mutual._data)
mutual_slt = mutual_seq_2[1]
p.update(mutual_slt._data)
publish.append(p)
except:
print 'mutual exception:', e
b = s3.bucket_straight(curator[user_keys.user_role])
filename = league_name + '/db/bible.json'
meta = shared.entity_filter(curator._data)
output = shared.dump(publish + extras)
s3.save_s3(b, filename, output , None, content_type='application/json', acl='public-read', meta=meta, encode='gzip')
def bored_loop(seed=None):
print 'bored loop:', seed
for t in scan_all():
#for t in scan_partial():
print 'found:', t[keys.entity_league], t[
keys.entity_twitter], 'length:', len(
t['instagrams']), t[Tweet.tweet_user_id], t[Tweet.tweet_id]
for instagram in t[Tweet.instagrams]:
print 'instagram url:', instagram, 'lingo:', fixed.lingo_since_date(
int(t[Tweet.ts_ms]) / 1000)
yield iw.goto_url(instagram)
yield iw.get_image_data(instagram, t).addErrback(fix_error, t)
yield task.deferLater(reactor, 5, defer.succeed, True)
if t.needs_save():
print 'needs save'
t.partial_save()
def filling(self, query_result):
wo = []
try:
for t in query_result:
ff = {}
ff.update(t._data)
print 'found-1:', ff[
keys.entity_twitter], fixed.lingo_since_date(
int(ff[Tweet.ts_ms]) / 1000)
wo.append(ff)
if len(wo) == 0:
#print 'empty result:', incoming
raise EmptyResults()
print 'wo length:', len(wo)
return wo
except Exception as e:
print 'filling exception:', e
def find_instagrams():
yield cv.goto_url('https://www.instagram.com')
try:
for t in scan_all():
origin_entity = Entity().get_item(league=t[keys.entity_league],
profile=t[keys.entity_profile])
since = fixed.lingo_since_date(int(t[Tweet.ts_ms]) / 1000)
instagram_entity = None
if t[instagram_keys.instagram_name]:
for i_e in Entity().query_2(
index=Entity.index_instagram_league,
instagram__eq=t[instagram_keys.instagram_name],
limit=1):
instagram_entity = i_e
if not instagram_entity and origin_entity[keys.entity_instagram]:
try:
print '{:10s}'.format('unknown'), '{:3d}'.format(
since[1]), '{:5s}'.format(since[0]), '{:50s}'.format(
'https://www.instagram.com/' +
origin_entity[keys.entity_instagram]
), 'of', '{:50s}'.format(
'https://www.instagram.com/' +
t[instagram_keys.instagram_name])
except:
print ''
elif instagram_entity and instagram_entity[
keys.entity_twitter] == t[keys.entity_twitter]:
print 'found self instagram user:'******'{:3d}'.format(
since[1]), '{:5s}'.format(since[0])
reactor.callLater(0, process_match, instagram_entity, t)
elif instagram_entity and instagram_entity[
keys.entity_twitter] != t[
keys.entity_twitter] and origin_entity[
keys.entity_instagram]:
print '{:10s}'.format('known'), '{:3d}'.format(
since[1]), '{:5s}'.format(since[0]), '{:50s}'.format(
'https://www.instagram.com/' +
origin_entity[keys.entity_instagram]
), 'of', '{:50s}'.format(
'https://www.instagram.com/' +
instagram_entity[keys.entity_instagram])
if not t[Tweet.unknown_instagrams] and t[
Tweet.unknown_instagrams] and len(
t[Tweet.unknown_instagrams]) == 1:
t[Tweet.unknown_instagrams] = t[Tweet.unknown_instagrams]
del t[Tweet.unknown_instagrams]
elif t[Tweet.unknown_instagrams] and len(
t[Tweet.unknown_instagrams]) == 1:
del t[Tweet.unknown_instagrams]
del t[instagram_keys.instagram_name]
if t.needs_save():
t.partial_save()
elif not origin_entity[keys.entity_instagram] and t[
keys.entity_twitter] not in skip_find:
yield execute_find(origin_entity, t)
else:
print 'something else!'
except ItemNotFound as e:
pass
except KeyError as e:
print 'key error:', e
pprint.pprint(t._data)
yield defer.Deferred()
except AttributeError as e:
print 'attribute error:', e
pprint.pprint(t._data)
yield defer.Deferred()
except Exception as e:
print 'oh f**k:', e.__class__.__name__
print e
pprint.pprint(t._data)
print ''
pprint.pprint(origin_entity._data)
yield defer.Deferred()
def match_done(self):
print 'match loop complete - updating from:', fixed.lingo_since_date(
self.user[time_keys.ts_match_twitter])
self.user[time_keys.ts_match_twitter] = int(time.time())
self.user.partial_save()
def run_counts():
for site in User().get_sites():
print 'site yield'
yield task.deferLater(reactor, 10, defer.succeed, True)
if True or not site[time_keys.ts_count]:
print 'site counts since:', fixed.lingo_since_date(site[time_keys.ts_count])
_ec = Entity().query_count(index=Entity.index_site_profile, site__eq=site[user_keys.user_role])
_tc = Entity().query_count(index=Entity.index_site_profile, site__eq=site[user_keys.user_role], profile__beginswith='team:')
_lc = Entity().query_count(index=Entity.index_site_profile, site__eq=site[user_keys.user_role], profile__beginswith='league:')
_twc = Entity().query_count(index=Entity.index_site_profile, site__eq=site[user_keys.user_role], query_filter={'twitter__null': False})
_iwc = Entity().query_count(index=Entity.index_site_profile, site__eq=site[user_keys.user_role], query_filter={'instagram__null': False})
_fwc = Entity().query_count(index=Entity.index_site_profile, site__eq=site[user_keys.user_role], query_filter={'facebook__null': False})
site[keys.count('entity')] = _ec
site[keys.count(keys.entity_team)] = _tc
site[keys.count(keys.entity_league)] = _lc
site[keys.count(keys.entity_twitter)] = _twc
site[keys.count(keys.entity_instagram)] = _iwc
site[keys.count(keys.entity_facebook)] = _fwc
site[time_keys.ts_count] = int(time.time())
site.partial_save()
else:
print 'site counts since:', fixed.lingo_since_date(site[time_keys.ts_count])
print_operator_curator(site)
for l in site[user_keys.user_site_leagues]:
print 'league yield'
yield task.deferLater(reactor, 10, defer.succeed, True)
curator = User().get_by_role(l, keys.entity_twitter)
if True or not curator[time_keys.ts_count]:
print 'league counts since:', fixed.lingo_since_date(curator[time_keys.ts_count])
tc = Entity().query_count(league__eq=curator[user_keys.user_role], profile__beginswith='team:')
ec = Entity().query_count(league__eq=curator[user_keys.user_role])
twc = Entity().query_count(league__eq=curator[user_keys.user_role], query_filter={'twitter__null': False})
iwc = Entity().query_count(league__eq=curator[user_keys.user_role], query_filter={'instagram__null': False})
fwc = Entity().query_count(league__eq=curator[user_keys.user_role], query_filter={'facebook__null': False})
curator[keys.count('entity')] = ec
curator[keys.count(keys.entity_team)] = tc
curator[keys.count(keys.entity_twitter)] = twc
curator[keys.count(keys.entity_instagram)] = iwc
curator[keys.count(keys.entity_facebook)] = fwc
curator[time_keys.ts_count] = int(time.time())
curator.partial_save()
#for team in Entity().query_2(league__eq=curator[user_keys.user_role], profile__beginswith='team:'):
# print 'team yield'
# yield task.deferLater(reactor, 10, defer.succeed, True)
# team_url = 'http://service.' + site[user_keys.user_role] + '/site/' + curator[user_keys.user_role] + '/' + team[keys.entity_profile].split(':', 1)[1] + '/players'
# print 'team:', team_url
# r = requests.get(team_url)
# print r.status_code
#teams_url = 'http://service.' + site[user_keys.user_role] + '/site/' + curator[user_keys.user_role] + '/teams'
#print 'teams:', teams_url
#requests.get(teams_url)
else:
print ' league counts since:', fixed.lingo_since_date(curator[time_keys.ts_count])
print_operator_curator(curator)
site_url = 'http://service.' + site[user_keys.user_role] + '/site/curator'
print 'site:', site_url
requests.get(site_url)
operators_url = 'http://service.' + site[user_keys.user_role] + '/site/operators'
print 'operators:', operators_url
requests.get(operators_url)
yield task.deferLater(reactor, 10, defer.succeed, True)
reactor.callLater(0, reactor.stop)
import sys
tw = TwitterView()
role = 'browser'
if len(sys.argv) > 2:
role = sys.argv[2]
try:
tw.role = (role, sys.argv[1])
user_available = False
if len(sys.argv) > 3:
user_available = True
tw.setFixedWidth(1024)
tw.setFixedHeight(768)
tw.show()
tw.twitter_initialize(user_available)
tw.signin()
reactor.run()
except Exception as e:
print 'missing league argument'
for curator in sorted([c for c in User().get_curators()],
key=lambda k: k[time_keys.ts_last_login]):
print 'curator:', curator[
user_keys.user_role], fixed.lingo_since_date(
curator[time_keys.ts_last_login])
users = User().get_leagues(None, keys.entity_twitter)
users.sort(key=lambda k: k[time_keys.ts_last_login])
print 'users:', len(users)
for league in users:
print 'league:', league[
user_keys.user_role], fixed.lingo_since_date(
league[time_keys.ts_last_login])