def league_lists(league_name, do_add=True): print 'league list:', league_name league_user = User().get_by_role(league_name, keys.entity_twitter) league_user_app = auth.user_app(league_user) oauth = auth.get_oauth( league_user, league_user_app, league_user_app[user_keys.user_twitter_apps].keys()[0]) lists = restful.get_lists(league_user, league_user_app, oauth) print 'lists:', len(lists), [_l['name'] for _l in lists] league_user[user_keys.user_home_lists] = len(lists) league_user.save() for team in Entity().query_2(league__eq=league_name, profile__beginswith='team:', reverse=True): team_name = make_list_name(team[keys.entity_profile].split(':')[1]) if not get_list_id(team_name, lists): print 'missing list:', team_name list_description = u'Tracking {0} {1} from {2}'.format( league_user[keys.entity_name], league_user[keys.entity_lingo]['players'], team_name) restful.create_list(league_user, league_user_app, team_name, list_description, oauth) else: print league_name, 'has:', team_name list_id = get_list_id(team_name, lists) existing_members = restful.list_members(league_user, league_user_app, list_id, oauth) print team_name, 'existing members:', len(existing_members) if do_add: add_members = [] for e in Entity().query_2( index=Entity.index_team_profile, team__eq=team[keys.entity_profile].split(':', 1)[1], query_filter={'twitter__null': False}): check_twitter(e, add_members, existing_members) if len(add_members) == 100: if not do_add_members(league_user, league_user_app, list_id, add_members, oauth): return add_members = [] if len(add_members) > 0: if not do_add_members(league_user, league_user_app, list_id, add_members, oauth): return print 'excess members:', len( existing_members), existing_members remove_members = [] for em in existing_members: remove_members.append(em) if len(remove_members) == 100: do_remove_members(league_user, league_user_app, list_id, remove_members, oauth) remove_members = [] if len(remove_members) > 0: do_remove_members(league_user, league_user_app, list_id, remove_members, oauth)
def curator_lists(curator_name, league_name, do_add=True): print 'curator list:', curator_name, 'league:', league_name curator_user = User().get_by_role(curator_name, keys.entity_twitter) oauth = auth.get_oauth(curator_user, curator_user, curator_user[user_keys.user_twitter_apps].keys()[0]) lists = restful.get_lists(curator_user, curator_user, oauth) curator_user[user_keys.user_home_lists] = len(lists) curator_user.save() u = User().get_by_role(league_name, keys.entity_twitter) list_name = make_list_name(u[keys.entity_name].replace(' ', '-').replace( '/', '_')) if not get_list_id(list_name, lists): print 'missing list:', list_name restful.create_list(curator_user, curator_user, list_name, u[keys.entity_description], oauth) else: print curator_name, league_name, 'has:', list_name list_id = get_list_id(list_name, lists) existing_members = restful.list_members(curator_user, curator_user, list_id, oauth) print 'existing members:', len(existing_members) if do_add: add_members = [] for e in Entity().query_2(league__eq=league_name, query_filter={'twitter__null': False}): check_twitter(e, add_members, existing_members) if len(add_members) == 100: if not do_add_members(curator_user, curator_user, list_id, add_members, oauth): return add_members = [] if len(add_members) > 0: if not do_add_members(curator_user, curator_user, list_id, add_members, oauth): return print 'excess members:', len(existing_members), existing_members remove_members = [] for em in existing_members: remove_members.append(em) if len(remove_members) == 100: do_remove_members(curator_user, curator_user, list_id, remove_members, oauth) remove_members = [] if len(remove_members) > 0: do_remove_members(curator_user, curator_user, list_id, remove_members, oauth)
def get_last_league(site, l, retweet, conversation): lt = last_tweet(l) try: print '{:22s} {:15s} {:15s} {:16s} {:20s}'.format( site, l, fixed.lingo_since_date(int(lt[Tweet.ts_ms]) / 1000), lt[keys.entity_twitter], lt[Tweet.tweet_id]) if retweet: lr = last_retweet(l) if lr: print '{:23s} {:14s} {:15s} {:16s} {:25s}'.format( '', 'retweet', fixed.lingo_since_date(int(lr[Tweet.ts_ms]) / 1000), lr[keys.entity_twitter], lr[Tweet.tweet_id]) if conversation: lc = last_conversation(l) if lc: print '{:23s} {:14s} {:15s} {:16s} {:25s}'.format( '', 'conversation', fixed.lingo_since_date(int(lc[Tweet.ts_ms]) / 1000), lc[keys.entity_twitter], lc[Tweet.tweet_id]) except: locked_league = User().get_by_role(l, keys.entity_twitter) locked = False if locked_league[user_keys.user_locked]: locked = locked_league[user_keys.user_locked] print '{:22s} {:15s} {:20s}'.format(site, l, 'unknown-' + str(locked))
def render_GET(self, request): shared.SharedPath().response_headers(request, 'application/json') kwargs = User().get_role_args(shared.SharedPath().path_site(request), keys.entity_twitter) print 'get curator:', kwargs deferred_user(request, kwargs) return server.NOT_DONE_YET
def get_profile_overrides(league_name): print 'get_profile_overrides:', league_name try: return json.loads(s3.bucket_straight(User().get_curator(league_name)[user_keys.user_role]).lookup('/' + league_name + '/profile_overrides.json').get_contents_as_string()) except Exception as e: print 'profile overrides exception:', e return {}
def publish_loop(): for site in User().get_sites(): b = s3.Bucket(site[user_keys.user_role]) for league_name in [ l for l in site[user_keys.user_site_leagues] if not leagues or l in leagues ]: league = Entity().get_league(league_name) prefix_1 = (league[keys.entity_league] if league[keys.entity_emblem] is None else league[keys.entity_emblem]) + '/logo/' prefix_2 = (league[keys.entity_league] if league[keys.entity_emblem] is None else league[keys.entity_emblem]) + '/logo_standard/' for prefix in [prefix_1, prefix_2]: for logo in os.listdir('/home/ubuntu/' + site[user_keys.user_role] + '/' + prefix): fp = '/home/ubuntu/' + site[ user_keys.user_role] + '/' + prefix + logo objs = list(b.objects.filter(Prefix=prefix + logo)) print b.name, logo if len(objs) > 0 and len(sys.argv) < 2: print 'exists:', logo else: print 'missing or overwrite:', logo data = open(fp, 'rb') b.put_object(ACL='public-read', Key=prefix + logo, Body=data, ContentType='image/svg+xml')
def route_email(self, ee): if ee.broadcast_dict[ 'derived_from'] == '*****@*****.**' and ee.broadcast_dict[ 'subject'] == 'For security purposes, your Twitter account has been locked.': for u in User().query_2( username__eq=ee.broadcast_dict['derived_to']): curator = User().get_curator(u[user_keys.user_role]) instance = u[user_keys.user_role] + '.' + curator[ user_keys.user_role] inst = instances.get_instances() for i in inst: if 'Name' in i.tags: if instance == i.tags['Name']: print 'found instance:', i.id boto.ec2.connect_to_region( 'us-east-1').reboot_instances([i.id])
def step1(request): request_token_url = 'https://api.twitter.com/oauth/request_token' site = shared.SharedPath().path_site(request) curator = User().get_by_role(site, keys.entity_twitter) app_name = curator[user_keys.user_twitter_apps].keys()[0] client_key = curator[user_keys.user_twitter_apps][app_name][user_keys.user_consumer_key] client_secret = curator[user_keys.user_twitter_apps][app_name][user_keys.user_consumer_secret] oauth = OAuth1Session(client_key, client_secret=client_secret) fetch_response = oauth.fetch_request_token(request_token_url) #{ # "oauth_token": "Z6eEdO8MOmk394WozF5oKyuAv855l4Mlqo7hhlSLik", # "oauth_token_secret": "Kd75W4OQfb2oJTV0vzGzeXftVAwgMnEK9MumzYcM" #} resource_owner_key = fetch_response.get('oauth_token') resource_owner_secret = fetch_response.get('oauth_token_secret') print 'oauth_callback_confirmed:', fetch_response.get('oauth_callback_confirmed') #oauth = OAuth1(client_key, client_secret=client_secret) #r = requests.post(url=request_token_url, auth=oauth) #r.content #"oauth_token=Z6eEdO8MOmk394WozF5oKyuAv855l4Mlqo7hhlSLik&oauth_token_secret=Kd75W4OQfb2oJTV0vzGzeXftVAwgMnEK9MumzYcM" #from urlparse import parse_qs #credentials = parse_qs(r.content) #resource_owner_key = credentials.get('oauth_token')[0] #resource_owner_secret = credentials.get('oauth_token_secret')[0] #oauth = OAuth1Session(client_key, client_secret=client_secret) request.write(json.dumps({'oauth_token': resource_owner_key})) #request.redirect("https://api.twitter.com/oauth/authorize?oauth_token=" + resource_owner_key) request.finish()
def identify(self, social_key, entity, request): differences = { social_key + '__add': entity[social_key], 'ts_match_' + social_key: int(time.time()) } print 'valid:', social_key, entity[social_key] if entity['match_' + social_key]: del entity['match_' + social_key] entity.partial_save() EntityHistory().delta(entity, differences) league = Entity().get_item(league=entity[keys.entity_league], profile=Entity().league_profile( entity[keys.entity_league])) curator = User().get_curator(entity[keys.entity_league]) tweet_txt = getattr(tweets, 'id_' + social_key)(entity, curator, league) try: print 'tweet message:', tweet_txt except: pass tweet_message = {} tweet_message.update(entity._data) tweet_message[twitter_keys.message_tweet] = tweet_txt request.write(json.dumps(tweet_message, cls=fixed.SetEncoder)) request.finish() return server.NOT_DONE_YET
def authorize(self): print 'authorize:', self.role[1] univeral = User().get_by_role('me', keys.entity_twitter) try: for univeral_app in univeral[user_keys.user_twitter_apps].keys(): if self.user[user_keys. user_twitter_auth] and univeral_app in self.user[ user_keys.user_twitter_auth]: print univeral_app, 'authorization:', self.user[ user_keys.user_twitter_auth][univeral_app] else: univeral_ans = yield self.create_token( univeral, univeral_app, True) print univeral_app, 'token ans:', univeral_ans except Exception as e: print 'authorize universal exception:', e try: if self.curator[user_keys.user_twitter_apps]: for curator_app in self.curator[ user_keys.user_twitter_apps].keys(): if self.user[ user_keys. user_twitter_auth] and curator_app in self.user[ user_keys.user_twitter_auth]: print curator_app, 'authorization:', self.user[ user_keys.user_twitter_auth][curator_app] else: curator_ans = yield self.create_token( self.curator, curator_app) print curator_app, 'token ans:', curator_ans except Exception as e: print 'authorize curator exception:', e
def league_scouting_report(league_name): from amazon.dynamo import Entity, User s1 = '%s %s' % (Entity().query_count(league__eq=league_name, profile__beginswith='team:'), 'Teams') s2 = '%s %s' % (Entity().query_count(league__eq=league_name, profile__beginswith='http:'), 'Players') s3 = '%s %s' % (Entity().query_count(index=Entity.index_twitter, league__eq=league_name), 'Tweeter(s)') # s4 = time.strftime("%a, %d %H:%M", time.localtime()) announce = u' '.join([User().get_curator(league_name)[user_keys.user_role] + '/#' + league_name, 'curates', league_name, 'with', s1, s2, s3]) return announce
def render_GET(self, request): shared.SharedPath().response_headers(request, 'application/json') if len(request.postpath) > 0: kwargs = User().get_role_args(request.postpath[0], keys.entity_twitter) print 'get operator:', kwargs deferred_user(request, kwargs) else: site = shared.SharedPath().path_site(request) threads.deferToThread( User().get_leagues, site, keys.entity_twitter).addCallback(lambda ans: json.dumps( [shared.treat(shared.entity_filter(u._data)) for u in ans], cls=fixed.SetEncoder)).addCallback( request.write).addCallback( lambda ign: request.finish()) return server.NOT_DONE_YET
def myself(self): print 'myself:', self.role[1] if self.role[1] == 'me': for t in [User]: for user in t().scan(type__eq=keys.entity_twitter): if user[keys.entity_twitter]: try: d = defer.Deferred() self.deferred_cbs.append(d) visit_function = """(function () {{ document.location='{}'; return true }})();""".format( 'https://twitter.com/' + user[keys.entity_twitter]) self.page().runJavaScript(visit_function) yield d yield task.deferLater(reactor, 3, defer.succeed, True) html = yield self.to_html() if self.can_follow( html, user[keys.entity_twitter] ) and not self.am_following( html, user[keys.entity_twitter]): print 'following:', user[keys.entity_twitter] do_follow = twitter_keys.do_follow_fmt % user[ keys.entity_twitter] self.page().runJavaScript( 'document.querySelector(\'' + do_follow + '\').click()') yield task.deferLater(reactor, 30, defer.succeed, True) except Exception as e: print 'myself exception:', e if self.role[1].endswith('.com'): for user in User().get_leagues(self.role[1]): if user[keys.entity_twitter]: try: d = defer.Deferred() self.deferred_cbs.append(d) visit_function = """(function () {{ document.location='{}'; return true }})();""".format( 'https://twitter.com/' + user[keys.entity_twitter]) self.page().runJavaScript(visit_function) yield d yield task.deferLater(reactor, 3, defer.succeed, True) html = yield self.to_html() if self.can_follow( html, user[keys.entity_twitter] ) and not self.am_following(html, user[keys.entity_twitter]): do_follow = twitter_keys.do_follow_fmt % user[ keys.entity_twitter] self.page().runJavaScript( 'document.querySelector(\'' + do_follow + '\').click()') yield task.deferLater(reactor, 60, defer.succeed, True) except Exception as e: print 'myself:', e
def stalk_analysis(league_name): mutuals = [] extras = [] for mf in Entity().query_2(league__eq=league_name): if 'twitter' in mf and 'ts_followers_' + league_name in mf: try: ts_add = None if mf[time_keys.ts_scout]: ts_add = mf[time_keys.ts_scout] slt = ProfileTwitter().profile_last(mf[keys.entity_twitter_id], None, ts_add) if slt is not None: print 'append:', mf[keys.entity_twitter], 'since:', fixed.lingo_since_date(ts_add) mutuals.append( (mf, slt) ) else: print 'no last stats:', mf[keys.entity_twitter], mf[keys.entity_twitter_id] except Exception as e: print 'missing:', e, 'https://twitter.com/' + mf[keys.entity_twitter], fixed.lingo_since(mf, twitter_keys.league_ts_followers(league_name)) else: extras.append(mf._data) print 'extras length:', len(extras) for mutual_seq in mutuals: try: mutual = mutual_seq[0] mutual_slt = mutual_seq[1] print 'mutual:', mutual[keys.entity_twitter] tf = set([]) for other_seq in [others for others in mutuals if others[0] != mutual and twitter_keys.league_mutual(league_name) in others[1]]: if mutual[keys.entity_twitter_id] in other_seq[1][twitter_keys.league_mutual(league_name)]: tf.add(other_seq[0][keys.entity_twitter_id]) if len(tf) > 0: print mutual[keys.entity_twitter], 'follows:', len(tf), 'following:', 0 if twitter_keys.league_mutual(league_name) not in mutual_slt else len(mutual_slt[twitter_keys.league_mutual(league_name)]) mutual_slt[twitter_keys.league_follows(league_name)] = tf mutual_slt.partial_save() else: print 'not following anyone:', mutual[keys.entity_twitter], mutual[keys.entity_twitter_id] except Exception as e: print 'mutual exception:', e publish = [] curator = User().get_curator(league_name) for mutual_seq_2 in mutuals: try: p = {} mutual = mutual_seq_2[0] p.update(mutual._data) mutual_slt = mutual_seq_2[1] p.update(mutual_slt._data) publish.append(p) except: print 'mutual exception:', e b = s3.bucket_straight(curator[user_keys.user_role]) filename = league_name + '/db/bible.json' meta = shared.entity_filter(curator._data) output = shared.dump(publish + extras) s3.save_s3(b, filename, output , None, content_type='application/json', acl='public-read', meta=meta, encode='gzip')
def get_secret_key(self, request): print self.consumer_secret_keys, shared.SharedPath().path_site(request) if shared.SharedPath().path_site(request) in self.consumer_secret_keys: return self.consumer_secret_keys[shared.SharedPath().path_site( request)] else: ln = request.getRequestHostname().split('.')[0] if ln not in self.consumer_secret_keys: self.consumer_secret_keys[ln] = user_secret_key( User().get_by_role(ln, keys.entity_twitter)) return self.consumer_secret_keys[ln]
def update_email(self): if 'mail.ru' not in self.user[user_keys.user_username]: return yield self.goto_url('https://twitter.com/settings/account') try: new_email = ''.join([ c for c in self.user[user_keys.user_username].lower() if not c.isdigit() ]) print 'new_email:', new_email new_email = new_email.split('@')[0] + '@socialcss.com' print 'new_email:', new_email md = {'filter': {'derived_to': new_email}} d = client.mail_listener(mail_domain='mail.scewpt.com', message_filter_dic=md) d.addCallback(client.hearing_back, new_email) d.addErrback(self.error_view) self.page().runJavaScript(js_email_focus) yield task.deferLater(reactor, 1, defer.succeed, True) QTest.keyClicks(qt5.app.opengl, new_email, Qt.NoModifier, 20) yield task.deferLater(reactor, 2, defer.succeed, True) self.page().runJavaScript(js_email) yield task.deferLater(reactor, 2, defer.succeed, True) QTest.keyClicks(qt5.app.opengl, self.user[user_keys.user_password], Qt.NoModifier, 20) yield task.deferLater(reactor, 2, defer.succeed, True) self.page().runJavaScript(js_save) result = yield d print 'result:', result['file_dest'] email_html = etree.HTML( emails.get_html_from_msg(result['file_dest'])) print 'email_html:', email_html for el in email_html.cssselect('a[href]'): if etree.tostring( el, method="text", encoding='UTF-8').lower().strip() == 'confirm now': yield self.goto_url(el.attrib['href']) new_user = {} new_user.update(self.user._data) self.user.delete() new_user[user_keys.user_username] = new_email if self.available: UserAvailable().put_item(data=new_user) else: User().put_item(data=new_user) except Exception as e: print 'update email exception:', e
def add_twitter_ignore(self, ignore_twitter): print 'twitter ignore:', ignore_twitter po = self.get_twitter_ignores() po.append(ignore_twitter) s3.save_s3(s3.bucket_straight(User().get_curator( self.get_league_name())[user_keys.user_role]), '/' + self.get_league_name() + '/twitter_ignores.json', json.dumps(po), None, content_type='application/json', acl='public-read') print 'done save' del self.twitter_ignores
def user_app(user): if user[user_keys.user_twitter_auth]: for k in user[user_keys.user_twitter_auth]: if user_keys.user_read_only not in user[ user_keys.user_twitter_auth][k]: if user[user_keys.user_twitter_apps] and k in user[ user_keys.user_twitter_apps]: return user else: curator = User().get_curator(user[user_keys.user_role]) if curator[user_keys.user_twitter_apps] and k in curator[ user_keys.user_twitter_apps]: return curator
def screenshot(view, site, curator): img = view.grabWindow() il = '/home/ubuntu/' + site + '/curator_background.png' res = img.save(il) print 'save result:', res, il try: bs = s3.bucket_straight(curator[user_keys.user_role]) s3.save_s3(bs, 'curator_background.png', None, il, 'image/png', 'public-read') except Exception as e: print e curator = User().get_by_role(site, keys.entity_twitter) restful.update_background(curator, curator, il)
def identify_instance(): leagues = sorted(Entity().get_leagues(), key=lambda e2: str(e2[keys.entity_league])) groups = [] for league in leagues: if len(groups) < group_max: groups.append(league) if len(groups) == group_max: tn = ', '.join([ l[keys.entity_league] + '.' + User().get_curator( l[keys.entity_league])[user_keys.user_role] for l in groups ]) launch_tag(tn) groups = [] if groups: tn = ', '.join([ l[keys.entity_league] + '.' + User().get_curator(l[keys.entity_league])[user_keys.user_role] for l in groups ]) launch_tag(tn)
def twitter_initialize(self, available=False): self.available = available print 'twitter initialize:', self.role[1], 'available:', self.available try: if available: self.user = UserAvailable().get_by_role( self.role[1], keys.entity_twitter) else: self.user = User().get_by_role(self.role[1], keys.entity_twitter) print 'twitter initialiaze user:'******'twitter initialize user exception:', e try: self.curator = User().get_curator(self.role[1]) print 'twitter initialize curator:', self.curator[ user_keys.user_role] except Exception as e: print 'twitter initialize curator exception:', e self.league_initialize()
def report(): sls = [] for site_leagues_seq in [ (u[user_keys.user_role], u[user_keys.user_site_leagues]) for u in User().scan(site_leagues__null=False, inactive__null=True) ]: sls.append(site_leagues_seq) sls = sorted(sls, key=lambda item: item[0]) for sl_seq in sls: #print site_leagues_seq[0], site_leagues_seq[1] sl = sorted(list(sl_seq[1])) for league in sl: get_last_league(sl_seq[0], league, len(sys.argv) > 1, len(sys.argv) > 2)
def buildandsave(site, exit_on_save=True): agent = Agent(reactor) curator = User().get_by_role(site, keys.entity_twitter) leagues = [] deferreds_league = [] for l in curator[user_keys.user_site_leagues]: league = Entity().get_item(league=l, profile='league:' + l) d = agent.request( "HEAD", str('http://' + league[keys.entity_site] + '/tw/' + league[keys.entity_twitter_id] + '/avatar_large.png')) d.addCallback(add_redirect, league, leagues, 'large') deferreds_league.append(d) yield defer.DeferredList(deferreds_league) print 'leagues length:', len(leagues) players = [] deferreds_small = [] for p in Entity().query_2(index=Entity.index_site_profile, site__eq=curator[user_keys.user_role], query_filter={'twitter__null': False}, limit=200): d = agent.request( "HEAD", str('http://' + p[keys.entity_site] + '/tw/' + p[keys.entity_twitter_id] + '/avatar_small.png')) d.addCallback(add_redirect, p, players, 'small') deferreds_small.append(d) yield defer.DeferredList(deferreds_small) print 'players length:', len(players) view = QQuickView() view.setSource(QUrl('qml/render/curator_twitter_bg.qml')) view.rootObject().setProperty('bgcolor', 'black') view.setWidth(1500) view.setHeight(500) view.show() view.rootObject().setProperty('curator', curator._data) view.rootObject().setProperty('leagues', leagues) view.rootObject().setProperty('players', players) yield task.deferLater(reactor, 30, screenshot, view, site, curator) if exit_on_save: print 'exit on save' reactor.callLater(0, reactor.stop) else: print 'done'
def add_message(self, msg): self.total_msg += 1 if msg['derived_from'].split('@')[1] in save_domain: if msg['derived_from'].split('@')[1] == 'twitter.com': if User().query_count(username__eq=msg['derived_to']) == 0: if UserAvailable().query_count( username__eq=msg['derived_to']) == 0: data = { user_keys.user_username: msg['derived_to'], user_keys.user_type: keys.entity_twitter, user_keys.user_password: '******', } UserAvailable().put_item(data=data) if len(self.all_msg) > MemoryBag.limit: self.all_msg.pop() self.all_msg.insert(0, msg)
def verify_webhooks(): tags = instances.get_instance( requests.get('http://169.254.169.254/latest/meta-data/instance-id'). text).tags['Name'] for tn in tags.split(','): tag = tn.strip() if tag == 'service': print 'verify webhooks:', tag me = User().get_by_role('me', keys.entity_twitter) oauth = auth.get_oauth(me, me, me[user_keys.user_twitter_apps].keys()[0]) #r = requests.post(webhook_url_fmt.format(quote_plus('http://service.socialcss.com/activity')), auth=oauth) r = requests.get(webhook_get_url, auth=oauth) print r.status_code, r.text else: pass
def get_twitter_ignores(self): if hasattr(self, 'twitter_ignores'): return self.twitter_ignores else: try: site_value = User().get_curator( self.get_league_name())[user_keys.user_role] key_value = '/' + self.get_league_name( ) + '/twitter_ignores.json' print 'site:', site_value, 'key value:', key_value ti = s3.bucket_straight(site_value).lookup( key_value).get_contents_as_string() self.twitter_ignores = json.loads(ti) except Exception as e: print e self.twitter_ignores = [] return self.twitter_ignores
def derived_entities(self, players): derived_team_names = [] missing_team = [] found_team = [] lost_team = [] for p in players: if keys.entity_team in p and p[ keys.entity_team] not in derived_team_names: derived_team_names.append(p[keys.entity_team]) for team_name in derived_team_names: try: team_entity = Entity().get_item( league=self.get_league_name(), profile=Entity().team_profile(team_name)) print 'found team:', fixed.team_name(team_entity) found_team.append(team_entity) #players.append(team_entity) except Exception as e: missing_team.append(team_name) print 'derived missing team:', team_name, self.get_league_name( ), e if self.create_missing_teams: Entity().put_item( data={ keys.entity_league: self.get_league_name(), keys.entity_profile: Entity().team_profile(team_name), keys.entity_site: User().get_curator(self.get_league_name())[ user_keys.user_role] }) for e in Entity().query_2(league__eq=self.get_league_name(), profile__beginswith='team:'): if e[keys.entity_profile] not in [ ft[keys.entity_profile] for ft in found_team ]: print 'lost team:', fixed.team_name(e) lost_team.append(e) if self.remove_lost_teams: e.delete() print 'derived_team_names:', len( derived_team_names), 'missing_team:', len( missing_team), 'found team:', len( found_team), 'lost team:', len(lost_team)
def league_compare(self, no_drops=False, dry_run=False): league = Entity().get_league(self.get_league_name()) curator = User().get_curator(league[keys.entity_league]) redirect = s3.get_redirect( s3.bucket_straight(curator[user_keys.user_role]), league[keys.entity_league] + '/db/league.json') key = s3.check_key(s3.bucket_straight(curator[user_keys.user_role]), redirect) print 'redirect:', redirect, 'key:', key, 'meta:', key.metadata if time_keys.ts_compare not in key: entities = [] loaded_entities = json.loads(key.get_contents_as_string()) for le in loaded_entities: if le[keys.entity_profile] not in [e for e in entities]: entities.append(le) else: print 'entity exists:', le[keys.entity_profile] print 'entities:', len(entities) self.entities_league_compare(entities, curator, league, no_drops, dry_run, key)
def trimIt(self, it, tweet, msg): if tweet[twitter_keys.message_tweet] != it: tweet[twitter_keys.message_tweet] = it if keys.entity_site in tweet: try: user = User().get_by_role(tweet[keys.entity_league], keys.entity_twitter) user_app = auth.user_app(user) print 'user:'******'publish api tweet' tweet_delay[tweet[keys.entity_league]] = int( time.time()) local_image_location = None if twitter_keys.message_pic in tweet: local_image_location = '/tmp/' + tweet[ twitter_keys.message_pic].rsplit('/', 1)[1] urllib.urlretrieve(tweet[twitter_keys.message_pic], local_image_location) restful.post_tweet(user, user_app, tweet[twitter_keys.message_tweet], local_image_location) TweetQueue().deleteMessage(msg) else: print 'delay publish api tweet' else: print 'publish queue tweet' TweetQueue(tweet[keys.entity_league]).createMessage(tweet) TweetQueue().deleteMessage(msg) except Exception as e: print 'post tweet exception:', e else: print 'delete tweet' TweetQueue().deleteMessage(msg)
def recover(self, entity): if not entity[keys.entity_twitter_id]: maybe = twitter_keys.validate_twitter(entity, False) if maybe: entity.partial_save() if entity[keys.entity_twitter_id]: print 'twitter id:', entity[keys.entity_twitter_id] u = User().get_by_role('me', keys.entity_twitter) oauth = auth.get_oauth(u, u, 'Curator Login') recover_url = 'https://api.twitter.com/1.1/users/lookup.json?user_id=%s&include_entities=false' % entity[ keys.entity_twitter_id] print 'recover_url:', recover_url r = requests.get(recover_url, auth=oauth) print r.json() try: entity[keys.entity_twitter] = r.json()[0]['screen_name'] entity.partial_save() except KeyError: del entity[keys.entity_twitter] del entity[keys.entity_twitter_id] entity.partial_save() else: print 'could not recover' self.lost(entity)