def process_inline(self): raw_entities = yield self.entities() print 'raw entities:', len(raw_entities) all_entities = self.prune_dups(raw_entities) if len(all_entities) >= self.min_size and len( all_entities) <= self.max_size and self.save_db: print 'accpetable entities', len(all_entities) try: io = json.dumps(all_entities, cls=fixed.SetEncoder) bucket = s3.bucket_straight(User().get_curator( self.get_league_name())[user_keys.user_role]) db_count = '1' try: redirect = s3.get_redirect( bucket, '/' + self.get_league_name() + '/db/league.json') db_count = str( int(redirect[len('/' + self.get_league_name() + '/db/league'):][:-5]) + 1) except Exception as e: print 'db count redirecct exception:', 3 try: index = 0 for key in bucket.list(prefix=self.get_league_name() + '/db/league'): index += 1 db_count = str(index) except: print 'db count prefix exception!' locate = '/' + self.get_league_name( ) + '/db/league' + db_count + '.json' ts_scraped = int(time.time()) bible_tweet = { time_keys.ts_scraped: ts_scraped, keys.entity_count: len(all_entities) } league_for_message = Entity().get_league( self.get_league_name()) bible_tweet.update(league_for_message._data) bible_tweet[twitter_keys.message_tweet] = tweets.bible_tweet( User().get_curator(self.get_league_name()), league_for_message) print locate, bible_tweet s3.save_s3(bucket, locate, io, None, 'application/json', 'public-read', bible_tweet) s3.create_local_redirect( bucket, self.get_league_name() + '/db/league.json', locate) league_user = User().get_by_role(self.get_league_name(), keys.entity_twitter) league_user_app = auth.user_app(league_user) if league_user_app: if not restful.post_tweet( league_user, league_user_app, bible_tweet[twitter_keys.message_tweet]): TweetQueue().createMessage(bible_tweet) else: TweetQueue().createMessage(bible_tweet) try: user_league = User().get_by_role(self.get_league_name(), keys.entity_twitter) user_league[time_keys.ts_bible] = ts_scraped user_league.partial_save() except: print 'missing user:'******'league done:', self.get_league_name( ), 'all entities length:', len(all_entities) defer.returnValue(True) except Exception as e: print 'except:', e
def run_counts(): for site in User().get_sites(): print 'site yield' yield task.deferLater(reactor, 10, defer.succeed, True) if True or not site[time_keys.ts_count]: print 'site counts since:', fixed.lingo_since_date(site[time_keys.ts_count]) _ec = Entity().query_count(index=Entity.index_site_profile, site__eq=site[user_keys.user_role]) _tc = Entity().query_count(index=Entity.index_site_profile, site__eq=site[user_keys.user_role], profile__beginswith='team:') _lc = Entity().query_count(index=Entity.index_site_profile, site__eq=site[user_keys.user_role], profile__beginswith='league:') _twc = Entity().query_count(index=Entity.index_site_profile, site__eq=site[user_keys.user_role], query_filter={'twitter__null': False}) _iwc = Entity().query_count(index=Entity.index_site_profile, site__eq=site[user_keys.user_role], query_filter={'instagram__null': False}) _fwc = Entity().query_count(index=Entity.index_site_profile, site__eq=site[user_keys.user_role], query_filter={'facebook__null': False}) site[keys.count('entity')] = _ec site[keys.count(keys.entity_team)] = _tc site[keys.count(keys.entity_league)] = _lc site[keys.count(keys.entity_twitter)] = _twc site[keys.count(keys.entity_instagram)] = _iwc site[keys.count(keys.entity_facebook)] = _fwc site[time_keys.ts_count] = int(time.time()) site.partial_save() else: print 'site counts since:', fixed.lingo_since_date(site[time_keys.ts_count]) print_operator_curator(site) for l in site[user_keys.user_site_leagues]: print 'league yield' yield task.deferLater(reactor, 10, defer.succeed, True) curator = User().get_by_role(l, keys.entity_twitter) if True or not curator[time_keys.ts_count]: print 'league counts since:', fixed.lingo_since_date(curator[time_keys.ts_count]) tc = Entity().query_count(league__eq=curator[user_keys.user_role], profile__beginswith='team:') ec = Entity().query_count(league__eq=curator[user_keys.user_role]) twc = Entity().query_count(league__eq=curator[user_keys.user_role], query_filter={'twitter__null': False}) iwc = Entity().query_count(league__eq=curator[user_keys.user_role], query_filter={'instagram__null': False}) fwc = Entity().query_count(league__eq=curator[user_keys.user_role], query_filter={'facebook__null': False}) curator[keys.count('entity')] = ec curator[keys.count(keys.entity_team)] = tc curator[keys.count(keys.entity_twitter)] = twc curator[keys.count(keys.entity_instagram)] = iwc curator[keys.count(keys.entity_facebook)] = fwc curator[time_keys.ts_count] = int(time.time()) curator.partial_save() #for team in Entity().query_2(league__eq=curator[user_keys.user_role], profile__beginswith='team:'): # print 'team yield' # yield task.deferLater(reactor, 10, defer.succeed, True) # team_url = 'http://service.' + site[user_keys.user_role] + '/site/' + curator[user_keys.user_role] + '/' + team[keys.entity_profile].split(':', 1)[1] + '/players' # print 'team:', team_url # r = requests.get(team_url) # print r.status_code #teams_url = 'http://service.' + site[user_keys.user_role] + '/site/' + curator[user_keys.user_role] + '/teams' #print 'teams:', teams_url #requests.get(teams_url) else: print ' league counts since:', fixed.lingo_since_date(curator[time_keys.ts_count]) print_operator_curator(curator) site_url = 'http://service.' + site[user_keys.user_role] + '/site/curator' print 'site:', site_url requests.get(site_url) operators_url = 'http://service.' + site[user_keys.user_role] + '/site/operators' print 'operators:', operators_url requests.get(operators_url) yield task.deferLater(reactor, 10, defer.succeed, True) reactor.callLater(0, reactor.stop)
class TwitterView(ChromeView, TwitterSignIn, TwitterAuth, TwitterStatsView): whoami_renew = True def league_initialize(self): try: self.league = Entity().get_item(league=self.role[1], profile='league:' + self.role[1]) print 'league initialize complete:', self.league[ keys.entity_league] except Exception as e: print 'league initialize exception:', e def twitter_initialize(self, available=False): self.available = available print 'twitter initialize:', self.role[1], 'available:', self.available try: if available: self.user = UserAvailable().get_by_role( self.role[1], keys.entity_twitter) else: self.user = User().get_by_role(self.role[1], keys.entity_twitter) print 'twitter initialiaze user:'******'twitter initialize user exception:', e try: self.curator = User().get_curator(self.role[1]) print 'twitter initialize curator:', self.curator[ user_keys.user_role] except Exception as e: print 'twitter initialize curator exception:', e self.league_initialize() def tw_clone(self, window): window.me = self.me window.role = self.role window.user = self.user try: window.curator = self.curator except Exception as e: print 'clone curator exception', e try: window.league = self.league except Exception as e: print 'clone league exception', e def status_twitter(self, check_url): print 'check_url:', check_url check = requests.head(check_url, headers={ 'User-Agent': 'curl/7.35.0', 'Accept': '*/*' }, verify=False) print 'twitter plus response:', check_url, check.status_code return check @defer.inlineCallbacks def notifications(self): yield self.goto_url('https://twitter.com/i/notifications') yield task.deferLater(reactor, 30, defer.succeed, True) reactor.callLater(0, reactor.stop) @defer.inlineCallbacks def developer(self): yield self.goto_url('https://developer.twitter.com') @defer.inlineCallbacks def apps(self): yield self.goto_url('https://apps.twitter.com') try: html = yield self.to_html() #parse.dumpit(html, 'apps.html') for h2 in html.cssselect('div[class="app-details"] h2'): a = h2.cssselect('a')[0] appname = parse.csstext(a) print 'appname:', appname if not self.user[user_keys.user_twitter_apps]: self.user[user_keys.user_twitter_apps] = {appname: {}} self.user[user_keys.user_twitter_auth] = {appname: {}} yield self.goto_url( 'https://apps.twitter.com/' + a.attrib['href'].replace('show', 'keys')) html2 = yield self.to_html() #parse.dumpit(html2, 'apps2.html') for span in html2.cssselect( 'div[class="app-settings"] div[class="row"] span[class="heading"]' ): if parse.csstext(span) == 'Consumer Key (API Key)': self.user[user_keys.user_twitter_apps][appname][ user_keys.user_consumer_key] = parse.csstext( span.getnext()) if parse.csstext( span) == 'Consumer Secret (API Secret)': self.user[user_keys.user_twitter_apps][appname][ user_keys. user_consumer_secret] = parse.csstext( span.getnext()) for span in html2.cssselect( 'div[class="access"] div[class="row"] span[class="heading"]' ): if parse.csstext(span) == 'Access Token': self.user[user_keys.user_twitter_auth][appname][ user_keys.user_auth_token] = parse.csstext( span.getnext()) if parse.csstext(span) == 'Access Token Secret': self.user[user_keys.user_twitter_auth][appname][ user_keys. user_auth_token_secret] = parse.csstext( span.getnext()) self.user.partial_save() except Exception as e: print 'apps exception:', e @defer.inlineCallbacks def update_email(self): if 'mail.ru' not in self.user[user_keys.user_username]: return yield self.goto_url('https://twitter.com/settings/account') try: new_email = ''.join([ c for c in self.user[user_keys.user_username].lower() if not c.isdigit() ]) print 'new_email:', new_email new_email = new_email.split('@')[0] + '@socialcss.com' print 'new_email:', new_email md = {'filter': {'derived_to': new_email}} d = client.mail_listener(mail_domain='mail.scewpt.com', message_filter_dic=md) d.addCallback(client.hearing_back, new_email) d.addErrback(self.error_view) self.page().runJavaScript(js_email_focus) yield task.deferLater(reactor, 1, defer.succeed, True) QTest.keyClicks(qt5.app.opengl, new_email, Qt.NoModifier, 20) yield task.deferLater(reactor, 2, defer.succeed, True) self.page().runJavaScript(js_email) yield task.deferLater(reactor, 2, defer.succeed, True) QTest.keyClicks(qt5.app.opengl, self.user[user_keys.user_password], Qt.NoModifier, 20) yield task.deferLater(reactor, 2, defer.succeed, True) self.page().runJavaScript(js_save) result = yield d print 'result:', result['file_dest'] email_html = etree.HTML( emails.get_html_from_msg(result['file_dest'])) print 'email_html:', email_html for el in email_html.cssselect('a[href]'): if etree.tostring( el, method="text", encoding='UTF-8').lower().strip() == 'confirm now': yield self.goto_url(el.attrib['href']) new_user = {} new_user.update(self.user._data) self.user.delete() new_user[user_keys.user_username] = new_email if self.available: UserAvailable().put_item(data=new_user) else: User().put_item(data=new_user) except Exception as e: print 'update email exception:', e @defer.inlineCallbacks def update_available(self): yield self.authorize() yield self.update_email() @defer.inlineCallbacks def authorize(self): print 'authorize:', self.role[1] univeral = User().get_by_role('me', keys.entity_twitter) try: for univeral_app in univeral[user_keys.user_twitter_apps].keys(): if self.user[user_keys. user_twitter_auth] and univeral_app in self.user[ user_keys.user_twitter_auth]: print univeral_app, 'authorization:', self.user[ user_keys.user_twitter_auth][univeral_app] else: univeral_ans = yield self.create_token( univeral, univeral_app, True) print univeral_app, 'token ans:', univeral_ans except Exception as e: print 'authorize universal exception:', e try: if self.curator[user_keys.user_twitter_apps]: for curator_app in self.curator[ user_keys.user_twitter_apps].keys(): if self.user[ user_keys. user_twitter_auth] and curator_app in self.user[ user_keys.user_twitter_auth]: print curator_app, 'authorization:', self.user[ user_keys.user_twitter_auth][curator_app] else: curator_ans = yield self.create_token( self.curator, curator_app) print curator_app, 'token ans:', curator_ans except Exception as e: print 'authorize curator exception:', e @defer.inlineCallbacks def myself(self): print 'myself:', self.role[1] if self.role[1] == 'me': for t in [User]: for user in t().scan(type__eq=keys.entity_twitter): if user[keys.entity_twitter]: try: d = defer.Deferred() self.deferred_cbs.append(d) visit_function = """(function () {{ document.location='{}'; return true }})();""".format( 'https://twitter.com/' + user[keys.entity_twitter]) self.page().runJavaScript(visit_function) yield d yield task.deferLater(reactor, 3, defer.succeed, True) html = yield self.to_html() if self.can_follow( html, user[keys.entity_twitter] ) and not self.am_following( html, user[keys.entity_twitter]): print 'following:', user[keys.entity_twitter] do_follow = twitter_keys.do_follow_fmt % user[ keys.entity_twitter] self.page().runJavaScript( 'document.querySelector(\'' + do_follow + '\').click()') yield task.deferLater(reactor, 30, defer.succeed, True) except Exception as e: print 'myself exception:', e if self.role[1].endswith('.com'): for user in User().get_leagues(self.role[1]): if user[keys.entity_twitter]: try: d = defer.Deferred() self.deferred_cbs.append(d) visit_function = """(function () {{ document.location='{}'; return true }})();""".format( 'https://twitter.com/' + user[keys.entity_twitter]) self.page().runJavaScript(visit_function) yield d yield task.deferLater(reactor, 3, defer.succeed, True) html = yield self.to_html() if self.can_follow( html, user[keys.entity_twitter] ) and not self.am_following(html, user[keys.entity_twitter]): do_follow = twitter_keys.do_follow_fmt % user[ keys.entity_twitter] self.page().runJavaScript( 'document.querySelector(\'' + do_follow + '\').click()') yield task.deferLater(reactor, 60, defer.succeed, True) except Exception as e: print 'myself:', e def browser(self): print 'browser:', self.role[1] def suspended(self): self.goto_url( "https://support.twitter.com/forms/general?subtopic=suspended") def whoami(self, html): #parse.dumpit(html, 'test.html') try: self.me = html.cssselect( 'a.DashboardProfileCard-screennameLink.u-linkComplex.u-linkClean' )[0].attrib['href'][1:] except IndexError: e = MeException() e.html = html raise e print 'ME:', self.me if self.me != self.user[keys.entity_twitter]: print 'unknown me:', self.me, self.user[keys.entity_twitter] if self.whoami_renew or keys.entity_twitter not in self.user: self.user[keys.entity_twitter] = self.me self.user.partial_save() print 'update me:', self.me, self.user._data else: print 'not renewing:', self.me, self.user[keys.entity_twitter] raise Exception('No Renewing') else: if len( html.cssselect( 'div[class="Banner-textContent"][id="account-suspended"]' )) == 0: print 'known me:', self.me, self.user[keys.entity_twitter] if self.user[user_keys.user_locked]: del self.user[user_keys.user_locked] self.user.partial_save() self.user[time_keys.ts_last_login] = int(time.time()) self.user.partial_save() print self.user[time_keys.ts_last_login] try: self.league[time_keys.ts_last_login] = int(time.time()) self.league.partial_save() except Exception as e: print 'could not save login:', e else: e = MeException() e.html = html raise e return html