Example #1
0
def get_last_league(site, l, retweet, conversation):
    lt = last_tweet(l)
    try:
        print '{:22s} {:15s} {:15s} {:16s} {:20s}'.format(
            site, l, fixed.lingo_since_date(int(lt[Tweet.ts_ms]) / 1000),
            lt[keys.entity_twitter], lt[Tweet.tweet_id])
        if retweet:
            lr = last_retweet(l)
            if lr:
                print '{:23s} {:14s} {:15s} {:16s} {:25s}'.format(
                    '', 'retweet',
                    fixed.lingo_since_date(int(lr[Tweet.ts_ms]) / 1000),
                    lr[keys.entity_twitter], lr[Tweet.tweet_id])
        if conversation:
            lc = last_conversation(l)
            if lc:
                print '{:23s} {:14s} {:15s} {:16s} {:25s}'.format(
                    '', 'conversation',
                    fixed.lingo_since_date(int(lc[Tweet.ts_ms]) / 1000),
                    lc[keys.entity_twitter], lc[Tweet.tweet_id])
    except:
        locked_league = User().get_by_role(l, keys.entity_twitter)
        locked = False
        if locked_league[user_keys.user_locked]:
            locked = locked_league[user_keys.user_locked]
        print '{:22s} {:15s} {:20s}'.format(site, l, 'unknown-' + str(locked))
Example #2
0
def inflate_service(perform=False):

    users = []
    for t in [User]:
        for u in t().scan():
            if u[user_keys.user_twitter_auth] and '.com' not in u[
                    user_keys.
                    user_role] and user_keys.user_locked not in u.keys():
                ua = auth.user_app(u)
                if ua:
                    users.append(u)
    sorted_users = sorted(users, key=lambda item: item[time_keys.ts_inflated])
    for i, so in enumerate(sorted_users):
        print '{:3s}'.format(str(i + 1)), '{:20s}'.format(
            so[user_keys.user_role]), fixed.lingo_since_date(
                so[time_keys.ts_inflated]), so.table.table_name

    if perform:
        inflate_user = sorted_users[0]
        inflate_app_user = auth.user_app(inflate_user)

        print ''
        print inflate_user[user_keys.user_role], inflate_app_user[
            user_keys.user_role], inflate_user.table.table_name
        restful.Inflate().do_inflate(inflate_user, inflate_app_user)
        StalkQueue().createMessage(
            {keys.entity_league: inflate_user[user_keys.user_role]})
Example #3
0
def stalk_analysis(league_name):        
    mutuals = []
    extras = []
    for mf in Entity().query_2(league__eq=league_name):
        if 'twitter' in mf and 'ts_followers_' + league_name in mf:
            try:
                ts_add = None
                if mf[time_keys.ts_scout]:
                    ts_add = mf[time_keys.ts_scout]
                slt = ProfileTwitter().profile_last(mf[keys.entity_twitter_id], None, ts_add) 
                if slt is not None:
                    print 'append:', mf[keys.entity_twitter], 'since:', fixed.lingo_since_date(ts_add)                
                    mutuals.append( (mf, slt) )
                else:
                    print 'no last stats:', mf[keys.entity_twitter], mf[keys.entity_twitter_id]
            except Exception as e:
                print 'missing:', e, 'https://twitter.com/' + mf[keys.entity_twitter], fixed.lingo_since(mf, twitter_keys.league_ts_followers(league_name))
        else:
            extras.append(mf._data)
    print 'extras length:', len(extras)
    for mutual_seq in mutuals:
        try:
            mutual = mutual_seq[0]
            mutual_slt = mutual_seq[1]
            print 'mutual:', mutual[keys.entity_twitter]
            tf = set([])            
            for other_seq in [others for others in mutuals if others[0] != mutual and twitter_keys.league_mutual(league_name) in others[1]]:
                if mutual[keys.entity_twitter_id] in other_seq[1][twitter_keys.league_mutual(league_name)]:
                    tf.add(other_seq[0][keys.entity_twitter_id])
            if len(tf) > 0:
                print mutual[keys.entity_twitter], 'follows:', len(tf), 'following:', 0 if twitter_keys.league_mutual(league_name) not in mutual_slt else len(mutual_slt[twitter_keys.league_mutual(league_name)])
                mutual_slt[twitter_keys.league_follows(league_name)] = tf
                mutual_slt.partial_save()                          
            else:
                print 'not following anyone:', mutual[keys.entity_twitter], mutual[keys.entity_twitter_id]
        except Exception as e:
            print 'mutual exception:', e
    publish = []
    curator = User().get_curator(league_name)
    for mutual_seq_2 in mutuals:
        try:
            p = {}
            mutual = mutual_seq_2[0]
            p.update(mutual._data)
            mutual_slt = mutual_seq_2[1]
            p.update(mutual_slt._data)
            publish.append(p)
        except:
            print 'mutual exception:', e
    b = s3.bucket_straight(curator[user_keys.user_role])
    filename = league_name + '/db/bible.json'
    meta = shared.entity_filter(curator._data)
    output = shared.dump(publish + extras)
    s3.save_s3(b, filename, output , None, content_type='application/json', acl='public-read', meta=meta, encode='gzip')    
Example #4
0
def bored_loop(seed=None):
    print 'bored loop:', seed
    for t in scan_all():
        #for t in scan_partial():
        print 'found:', t[keys.entity_league], t[
            keys.entity_twitter], 'length:', len(
                t['instagrams']), t[Tweet.tweet_user_id], t[Tweet.tweet_id]
        for instagram in t[Tweet.instagrams]:
            print 'instagram url:', instagram, 'lingo:', fixed.lingo_since_date(
                int(t[Tweet.ts_ms]) / 1000)
            yield iw.goto_url(instagram)
            yield iw.get_image_data(instagram, t).addErrback(fix_error, t)
            yield task.deferLater(reactor, 5, defer.succeed, True)
            if t.needs_save():
                print 'needs save'
                t.partial_save()
Example #5
0
 def filling(self, query_result):
     wo = []
     try:
         for t in query_result:
             ff = {}
             ff.update(t._data)
             print 'found-1:', ff[
                 keys.entity_twitter], fixed.lingo_since_date(
                     int(ff[Tweet.ts_ms]) / 1000)
             wo.append(ff)
         if len(wo) == 0:
             #print 'empty result:', incoming
             raise EmptyResults()
         print 'wo length:', len(wo)
         return wo
     except Exception as e:
         print 'filling exception:', e
Example #6
0
def find_instagrams():
    yield cv.goto_url('https://www.instagram.com')
    try:
        for t in scan_all():
            origin_entity = Entity().get_item(league=t[keys.entity_league],
                                              profile=t[keys.entity_profile])
            since = fixed.lingo_since_date(int(t[Tweet.ts_ms]) / 1000)
            instagram_entity = None
            if t[instagram_keys.instagram_name]:
                for i_e in Entity().query_2(
                        index=Entity.index_instagram_league,
                        instagram__eq=t[instagram_keys.instagram_name],
                        limit=1):
                    instagram_entity = i_e
            if not instagram_entity and origin_entity[keys.entity_instagram]:
                try:
                    print '{:10s}'.format('unknown'), '{:3d}'.format(
                        since[1]), '{:5s}'.format(since[0]), '{:50s}'.format(
                            'https://www.instagram.com/' +
                            origin_entity[keys.entity_instagram]
                        ), 'of', '{:50s}'.format(
                            'https://www.instagram.com/' +
                            t[instagram_keys.instagram_name])
                except:
                    print ''
            elif instagram_entity and instagram_entity[
                    keys.entity_twitter] == t[keys.entity_twitter]:
                print 'found self instagram user:'******'{:3d}'.format(
                    since[1]), '{:5s}'.format(since[0])
                reactor.callLater(0, process_match, instagram_entity, t)
            elif instagram_entity and instagram_entity[
                    keys.entity_twitter] != t[
                        keys.entity_twitter] and origin_entity[
                            keys.entity_instagram]:
                print '{:10s}'.format('known'), '{:3d}'.format(
                    since[1]), '{:5s}'.format(since[0]), '{:50s}'.format(
                        'https://www.instagram.com/' +
                        origin_entity[keys.entity_instagram]
                    ), 'of', '{:50s}'.format(
                        'https://www.instagram.com/' +
                        instagram_entity[keys.entity_instagram])
                if not t[Tweet.unknown_instagrams] and t[
                        Tweet.unknown_instagrams] and len(
                            t[Tweet.unknown_instagrams]) == 1:
                    t[Tweet.unknown_instagrams] = t[Tweet.unknown_instagrams]
                    del t[Tweet.unknown_instagrams]
                elif t[Tweet.unknown_instagrams] and len(
                        t[Tweet.unknown_instagrams]) == 1:
                    del t[Tweet.unknown_instagrams]
                del t[instagram_keys.instagram_name]
                if t.needs_save():
                    t.partial_save()
            elif not origin_entity[keys.entity_instagram] and t[
                    keys.entity_twitter] not in skip_find:
                yield execute_find(origin_entity, t)
            else:
                print 'something else!'
    except ItemNotFound as e:
        pass
    except KeyError as e:
        print 'key error:', e
        pprint.pprint(t._data)
        yield defer.Deferred()
    except AttributeError as e:
        print 'attribute error:', e
        pprint.pprint(t._data)
        yield defer.Deferred()
    except Exception as e:
        print 'oh f**k:', e.__class__.__name__
        print e
        pprint.pprint(t._data)
        print ''
        pprint.pprint(origin_entity._data)
        yield defer.Deferred()
Example #7
0
 def match_done(self):
     print 'match loop complete - updating from:', fixed.lingo_since_date(
         self.user[time_keys.ts_match_twitter])
     self.user[time_keys.ts_match_twitter] = int(time.time())
     self.user.partial_save()
Example #8
0
def run_counts():
    for site in User().get_sites():
        print 'site yield'
        yield task.deferLater(reactor, 10, defer.succeed, True)    
        if True or not site[time_keys.ts_count]:
            print 'site counts since:', fixed.lingo_since_date(site[time_keys.ts_count])            
            _ec = Entity().query_count(index=Entity.index_site_profile, site__eq=site[user_keys.user_role])
            _tc = Entity().query_count(index=Entity.index_site_profile, site__eq=site[user_keys.user_role], profile__beginswith='team:')
            _lc = Entity().query_count(index=Entity.index_site_profile, site__eq=site[user_keys.user_role], profile__beginswith='league:')
            _twc = Entity().query_count(index=Entity.index_site_profile, site__eq=site[user_keys.user_role], query_filter={'twitter__null': False})
            _iwc = Entity().query_count(index=Entity.index_site_profile, site__eq=site[user_keys.user_role], query_filter={'instagram__null': False})
            _fwc = Entity().query_count(index=Entity.index_site_profile, site__eq=site[user_keys.user_role], query_filter={'facebook__null': False})
            site[keys.count('entity')] = _ec
            site[keys.count(keys.entity_team)] = _tc
            site[keys.count(keys.entity_league)] = _lc
            site[keys.count(keys.entity_twitter)] = _twc
            site[keys.count(keys.entity_instagram)] = _iwc
            site[keys.count(keys.entity_facebook)] = _fwc
            site[time_keys.ts_count] = int(time.time())
            site.partial_save()
        else:
            print 'site counts since:', fixed.lingo_since_date(site[time_keys.ts_count])
        print_operator_curator(site)
        for l in site[user_keys.user_site_leagues]:
            print 'league yield'
            yield task.deferLater(reactor, 10, defer.succeed, True)        
            curator = User().get_by_role(l, keys.entity_twitter)
            if True or not curator[time_keys.ts_count]:
                print 'league counts since:', fixed.lingo_since_date(curator[time_keys.ts_count])                
                tc = Entity().query_count(league__eq=curator[user_keys.user_role], profile__beginswith='team:')
                ec = Entity().query_count(league__eq=curator[user_keys.user_role])
                twc = Entity().query_count(league__eq=curator[user_keys.user_role], query_filter={'twitter__null': False})
                iwc = Entity().query_count(league__eq=curator[user_keys.user_role], query_filter={'instagram__null': False})
                fwc = Entity().query_count(league__eq=curator[user_keys.user_role], query_filter={'facebook__null': False})
                curator[keys.count('entity')] = ec
                curator[keys.count(keys.entity_team)] = tc
                curator[keys.count(keys.entity_twitter)] = twc
                curator[keys.count(keys.entity_instagram)] = iwc
                curator[keys.count(keys.entity_facebook)] = fwc
                curator[time_keys.ts_count] = int(time.time())
                curator.partial_save()
                #for team in Entity().query_2(league__eq=curator[user_keys.user_role], profile__beginswith='team:'):
                #    print 'team yield'
                #    yield task.deferLater(reactor, 10, defer.succeed, True)                    
                #    team_url = 'http://service.' + site[user_keys.user_role] + '/site/' + curator[user_keys.user_role] + '/' + team[keys.entity_profile].split(':', 1)[1] + '/players'
                #    print 'team:', team_url
                #    r = requests.get(team_url)
                #    print r.status_code
                #teams_url = 'http://service.' + site[user_keys.user_role] + '/site/' + curator[user_keys.user_role] + '/teams'
                #print 'teams:', teams_url
                #requests.get(teams_url)                                        
            else:
                print '    league counts since:', fixed.lingo_since_date(curator[time_keys.ts_count])
            print_operator_curator(curator)
        site_url = 'http://service.' + site[user_keys.user_role] + '/site/curator'
        print 'site:', site_url
        requests.get(site_url)
        operators_url = 'http://service.' + site[user_keys.user_role] + '/site/operators'
        print 'operators:', operators_url
        requests.get(operators_url)
    yield task.deferLater(reactor, 10, defer.succeed, True)
    reactor.callLater(0, reactor.stop)
Example #9
0
    import sys
    tw = TwitterView()
    role = 'browser'
    if len(sys.argv) > 2:
        role = sys.argv[2]
    try:
        tw.role = (role, sys.argv[1])
        user_available = False
        if len(sys.argv) > 3:
            user_available = True
        tw.setFixedWidth(1024)
        tw.setFixedHeight(768)
        tw.show()
        tw.twitter_initialize(user_available)
        tw.signin()
        reactor.run()
    except Exception as e:
        print 'missing league argument'
        for curator in sorted([c for c in User().get_curators()],
                              key=lambda k: k[time_keys.ts_last_login]):
            print 'curator:', curator[
                user_keys.user_role], fixed.lingo_since_date(
                    curator[time_keys.ts_last_login])
        users = User().get_leagues(None, keys.entity_twitter)
        users.sort(key=lambda k: k[time_keys.ts_last_login])
        print 'users:', len(users)
        for league in users:
            print 'league:', league[
                user_keys.user_role], fixed.lingo_since_date(
                    league[time_keys.ts_last_login])