def process_inline(self):
raw_entities = yield self.entities()
print 'raw entities:', len(raw_entities)
all_entities = self.prune_dups(raw_entities)
if len(all_entities) >= self.min_size and len(
all_entities) <= self.max_size and self.save_db:
print 'accpetable entities', len(all_entities)
try:
io = json.dumps(all_entities, cls=fixed.SetEncoder)
bucket = s3.bucket_straight(User().get_curator(
self.get_league_name())[user_keys.user_role])
db_count = '1'
try:
redirect = s3.get_redirect(
bucket,
'/' + self.get_league_name() + '/db/league.json')
db_count = str(
int(redirect[len('/' + self.get_league_name() +
'/db/league'):][:-5]) + 1)
except Exception as e:
print 'db count redirecct exception:', 3
try:
index = 0
for key in bucket.list(prefix=self.get_league_name() +
'/db/league'):
index += 1
db_count = str(index)
except:
print 'db count prefix exception!'
locate = '/' + self.get_league_name(
) + '/db/league' + db_count + '.json'
ts_scraped = int(time.time())
bible_tweet = {
time_keys.ts_scraped: ts_scraped,
keys.entity_count: len(all_entities)
}
league_for_message = Entity().get_league(
self.get_league_name())
bible_tweet.update(league_for_message._data)
bible_tweet[twitter_keys.message_tweet] = tweets.bible_tweet(
User().get_curator(self.get_league_name()),
league_for_message)
print locate, bible_tweet
s3.save_s3(bucket, locate, io, None, 'application/json',
'public-read', bible_tweet)
s3.create_local_redirect(
bucket,
self.get_league_name() + '/db/league.json', locate)
league_user = User().get_by_role(self.get_league_name(),
keys.entity_twitter)
league_user_app = auth.user_app(league_user)
if league_user_app:
if not restful.post_tweet(
league_user, league_user_app,
bible_tweet[twitter_keys.message_tweet]):
TweetQueue().createMessage(bible_tweet)
else:
TweetQueue().createMessage(bible_tweet)
try:
user_league = User().get_by_role(self.get_league_name(),
keys.entity_twitter)
user_league[time_keys.ts_bible] = ts_scraped
user_league.partial_save()
except:
print 'missing user:'******'league done:', self.get_league_name(
), 'all entities length:', len(all_entities)
defer.returnValue(True)
except Exception as e:
print 'except:', e
def run_counts():
for site in User().get_sites():
print 'site yield'
yield task.deferLater(reactor, 10, defer.succeed, True)
if True or not site[time_keys.ts_count]:
print 'site counts since:', fixed.lingo_since_date(site[time_keys.ts_count])
_ec = Entity().query_count(index=Entity.index_site_profile, site__eq=site[user_keys.user_role])
_tc = Entity().query_count(index=Entity.index_site_profile, site__eq=site[user_keys.user_role], profile__beginswith='team:')
_lc = Entity().query_count(index=Entity.index_site_profile, site__eq=site[user_keys.user_role], profile__beginswith='league:')
_twc = Entity().query_count(index=Entity.index_site_profile, site__eq=site[user_keys.user_role], query_filter={'twitter__null': False})
_iwc = Entity().query_count(index=Entity.index_site_profile, site__eq=site[user_keys.user_role], query_filter={'instagram__null': False})
_fwc = Entity().query_count(index=Entity.index_site_profile, site__eq=site[user_keys.user_role], query_filter={'facebook__null': False})
site[keys.count('entity')] = _ec
site[keys.count(keys.entity_team)] = _tc
site[keys.count(keys.entity_league)] = _lc
site[keys.count(keys.entity_twitter)] = _twc
site[keys.count(keys.entity_instagram)] = _iwc
site[keys.count(keys.entity_facebook)] = _fwc
site[time_keys.ts_count] = int(time.time())
site.partial_save()
else:
print 'site counts since:', fixed.lingo_since_date(site[time_keys.ts_count])
print_operator_curator(site)
for l in site[user_keys.user_site_leagues]:
print 'league yield'
yield task.deferLater(reactor, 10, defer.succeed, True)
curator = User().get_by_role(l, keys.entity_twitter)
if True or not curator[time_keys.ts_count]:
print 'league counts since:', fixed.lingo_since_date(curator[time_keys.ts_count])
tc = Entity().query_count(league__eq=curator[user_keys.user_role], profile__beginswith='team:')
ec = Entity().query_count(league__eq=curator[user_keys.user_role])
twc = Entity().query_count(league__eq=curator[user_keys.user_role], query_filter={'twitter__null': False})
iwc = Entity().query_count(league__eq=curator[user_keys.user_role], query_filter={'instagram__null': False})
fwc = Entity().query_count(league__eq=curator[user_keys.user_role], query_filter={'facebook__null': False})
curator[keys.count('entity')] = ec
curator[keys.count(keys.entity_team)] = tc
curator[keys.count(keys.entity_twitter)] = twc
curator[keys.count(keys.entity_instagram)] = iwc
curator[keys.count(keys.entity_facebook)] = fwc
curator[time_keys.ts_count] = int(time.time())
curator.partial_save()
#for team in Entity().query_2(league__eq=curator[user_keys.user_role], profile__beginswith='team:'):
# print 'team yield'
# yield task.deferLater(reactor, 10, defer.succeed, True)
# team_url = 'http://service.' + site[user_keys.user_role] + '/site/' + curator[user_keys.user_role] + '/' + team[keys.entity_profile].split(':', 1)[1] + '/players'
# print 'team:', team_url
# r = requests.get(team_url)
# print r.status_code
#teams_url = 'http://service.' + site[user_keys.user_role] + '/site/' + curator[user_keys.user_role] + '/teams'
#print 'teams:', teams_url
#requests.get(teams_url)
else:
print ' league counts since:', fixed.lingo_since_date(curator[time_keys.ts_count])
print_operator_curator(curator)
site_url = 'http://service.' + site[user_keys.user_role] + '/site/curator'
print 'site:', site_url
requests.get(site_url)
operators_url = 'http://service.' + site[user_keys.user_role] + '/site/operators'
print 'operators:', operators_url
requests.get(operators_url)
yield task.deferLater(reactor, 10, defer.succeed, True)
reactor.callLater(0, reactor.stop)
class TwitterView(ChromeView, TwitterSignIn, TwitterAuth, TwitterStatsView):
whoami_renew = True
def league_initialize(self):
try:
self.league = Entity().get_item(league=self.role[1],
profile='league:' + self.role[1])
print 'league initialize complete:', self.league[
keys.entity_league]
except Exception as e:
print 'league initialize exception:', e
def twitter_initialize(self, available=False):
self.available = available
print 'twitter initialize:', self.role[1], 'available:', self.available
try:
if available:
self.user = UserAvailable().get_by_role(
self.role[1], keys.entity_twitter)
else:
self.user = User().get_by_role(self.role[1],
keys.entity_twitter)
print 'twitter initialiaze user:'******'twitter initialize user exception:', e
try:
self.curator = User().get_curator(self.role[1])
print 'twitter initialize curator:', self.curator[
user_keys.user_role]
except Exception as e:
print 'twitter initialize curator exception:', e
self.league_initialize()
def tw_clone(self, window):
window.me = self.me
window.role = self.role
window.user = self.user
try:
window.curator = self.curator
except Exception as e:
print 'clone curator exception', e
try:
window.league = self.league
except Exception as e:
print 'clone league exception', e
def status_twitter(self, check_url):
print 'check_url:', check_url
check = requests.head(check_url,
headers={
'User-Agent': 'curl/7.35.0',
'Accept': '*/*'
},
verify=False)
print 'twitter plus response:', check_url, check.status_code
return check
@defer.inlineCallbacks
def notifications(self):
yield self.goto_url('https://twitter.com/i/notifications')
yield task.deferLater(reactor, 30, defer.succeed, True)
reactor.callLater(0, reactor.stop)
@defer.inlineCallbacks
def developer(self):
yield self.goto_url('https://developer.twitter.com')
@defer.inlineCallbacks
def apps(self):
yield self.goto_url('https://apps.twitter.com')
try:
html = yield self.to_html()
#parse.dumpit(html, 'apps.html')
for h2 in html.cssselect('div[class="app-details"] h2'):
a = h2.cssselect('a')[0]
appname = parse.csstext(a)
print 'appname:', appname
if not self.user[user_keys.user_twitter_apps]:
self.user[user_keys.user_twitter_apps] = {appname: {}}
self.user[user_keys.user_twitter_auth] = {appname: {}}
yield self.goto_url(
'https://apps.twitter.com/' +
a.attrib['href'].replace('show', 'keys'))
html2 = yield self.to_html()
#parse.dumpit(html2, 'apps2.html')
for span in html2.cssselect(
'div[class="app-settings"] div[class="row"] span[class="heading"]'
):
if parse.csstext(span) == 'Consumer Key (API Key)':
self.user[user_keys.user_twitter_apps][appname][
user_keys.user_consumer_key] = parse.csstext(
span.getnext())
if parse.csstext(
span) == 'Consumer Secret (API Secret)':
self.user[user_keys.user_twitter_apps][appname][
user_keys.
user_consumer_secret] = parse.csstext(
span.getnext())
for span in html2.cssselect(
'div[class="access"] div[class="row"] span[class="heading"]'
):
if parse.csstext(span) == 'Access Token':
self.user[user_keys.user_twitter_auth][appname][
user_keys.user_auth_token] = parse.csstext(
span.getnext())
if parse.csstext(span) == 'Access Token Secret':
self.user[user_keys.user_twitter_auth][appname][
user_keys.
user_auth_token_secret] = parse.csstext(
span.getnext())
self.user.partial_save()
except Exception as e:
print 'apps exception:', e
@defer.inlineCallbacks
def update_email(self):
if 'mail.ru' not in self.user[user_keys.user_username]:
return
yield self.goto_url('https://twitter.com/settings/account')
try:
new_email = ''.join([
c for c in self.user[user_keys.user_username].lower()
if not c.isdigit()
])
print 'new_email:', new_email
new_email = new_email.split('@')[0] + '@socialcss.com'
print 'new_email:', new_email
md = {'filter': {'derived_to': new_email}}
d = client.mail_listener(mail_domain='mail.scewpt.com',
message_filter_dic=md)
d.addCallback(client.hearing_back, new_email)
d.addErrback(self.error_view)
self.page().runJavaScript(js_email_focus)
yield task.deferLater(reactor, 1, defer.succeed, True)
QTest.keyClicks(qt5.app.opengl, new_email, Qt.NoModifier, 20)
yield task.deferLater(reactor, 2, defer.succeed, True)
self.page().runJavaScript(js_email)
yield task.deferLater(reactor, 2, defer.succeed, True)
QTest.keyClicks(qt5.app.opengl, self.user[user_keys.user_password],
Qt.NoModifier, 20)
yield task.deferLater(reactor, 2, defer.succeed, True)
self.page().runJavaScript(js_save)
result = yield d
print 'result:', result['file_dest']
email_html = etree.HTML(
emails.get_html_from_msg(result['file_dest']))
print 'email_html:', email_html
for el in email_html.cssselect('a[href]'):
if etree.tostring(
el, method="text",
encoding='UTF-8').lower().strip() == 'confirm now':
yield self.goto_url(el.attrib['href'])
new_user = {}
new_user.update(self.user._data)
self.user.delete()
new_user[user_keys.user_username] = new_email
if self.available:
UserAvailable().put_item(data=new_user)
else:
User().put_item(data=new_user)
except Exception as e:
print 'update email exception:', e
@defer.inlineCallbacks
def update_available(self):
yield self.authorize()
yield self.update_email()
@defer.inlineCallbacks
def authorize(self):
print 'authorize:', self.role[1]
univeral = User().get_by_role('me', keys.entity_twitter)
try:
for univeral_app in univeral[user_keys.user_twitter_apps].keys():
if self.user[user_keys.
user_twitter_auth] and univeral_app in self.user[
user_keys.user_twitter_auth]:
print univeral_app, 'authorization:', self.user[
user_keys.user_twitter_auth][univeral_app]
else:
univeral_ans = yield self.create_token(
univeral, univeral_app, True)
print univeral_app, 'token ans:', univeral_ans
except Exception as e:
print 'authorize universal exception:', e
try:
if self.curator[user_keys.user_twitter_apps]:
for curator_app in self.curator[
user_keys.user_twitter_apps].keys():
if self.user[
user_keys.
user_twitter_auth] and curator_app in self.user[
user_keys.user_twitter_auth]:
print curator_app, 'authorization:', self.user[
user_keys.user_twitter_auth][curator_app]
else:
curator_ans = yield self.create_token(
self.curator, curator_app)
print curator_app, 'token ans:', curator_ans
except Exception as e:
print 'authorize curator exception:', e
@defer.inlineCallbacks
def myself(self):
print 'myself:', self.role[1]
if self.role[1] == 'me':
for t in [User]:
for user in t().scan(type__eq=keys.entity_twitter):
if user[keys.entity_twitter]:
try:
d = defer.Deferred()
self.deferred_cbs.append(d)
visit_function = """(function () {{ document.location='{}'; return true }})();""".format(
'https://twitter.com/' +
user[keys.entity_twitter])
self.page().runJavaScript(visit_function)
yield d
yield task.deferLater(reactor, 3, defer.succeed,
True)
html = yield self.to_html()
if self.can_follow(
html, user[keys.entity_twitter]
) and not self.am_following(
html, user[keys.entity_twitter]):
print 'following:', user[keys.entity_twitter]
do_follow = twitter_keys.do_follow_fmt % user[
keys.entity_twitter]
self.page().runJavaScript(
'document.querySelector(\'' + do_follow +
'\').click()')
yield task.deferLater(reactor, 30,
defer.succeed, True)
except Exception as e:
print 'myself exception:', e
if self.role[1].endswith('.com'):
for user in User().get_leagues(self.role[1]):
if user[keys.entity_twitter]:
try:
d = defer.Deferred()
self.deferred_cbs.append(d)
visit_function = """(function () {{ document.location='{}'; return true }})();""".format(
'https://twitter.com/' + user[keys.entity_twitter])
self.page().runJavaScript(visit_function)
yield d
yield task.deferLater(reactor, 3, defer.succeed, True)
html = yield self.to_html()
if self.can_follow(
html, user[keys.entity_twitter]
) and not self.am_following(html,
user[keys.entity_twitter]):
do_follow = twitter_keys.do_follow_fmt % user[
keys.entity_twitter]
self.page().runJavaScript(
'document.querySelector(\'' + do_follow +
'\').click()')
yield task.deferLater(reactor, 60, defer.succeed,
True)
except Exception as e:
print 'myself:', e
def browser(self):
print 'browser:', self.role[1]
def suspended(self):
self.goto_url(
"https://support.twitter.com/forms/general?subtopic=suspended")
def whoami(self, html):
#parse.dumpit(html, 'test.html')
try:
self.me = html.cssselect(
'a.DashboardProfileCard-screennameLink.u-linkComplex.u-linkClean'
)[0].attrib['href'][1:]
except IndexError:
e = MeException()
e.html = html
raise e
print 'ME:', self.me
if self.me != self.user[keys.entity_twitter]:
print 'unknown me:', self.me, self.user[keys.entity_twitter]
if self.whoami_renew or keys.entity_twitter not in self.user:
self.user[keys.entity_twitter] = self.me
self.user.partial_save()
print 'update me:', self.me, self.user._data
else:
print 'not renewing:', self.me, self.user[keys.entity_twitter]
raise Exception('No Renewing')
else:
if len(
html.cssselect(
'div[class="Banner-textContent"][id="account-suspended"]'
)) == 0:
print 'known me:', self.me, self.user[keys.entity_twitter]
if self.user[user_keys.user_locked]:
del self.user[user_keys.user_locked]
self.user.partial_save()
self.user[time_keys.ts_last_login] = int(time.time())
self.user.partial_save()
print self.user[time_keys.ts_last_login]
try:
self.league[time_keys.ts_last_login] = int(time.time())
self.league.partial_save()
except Exception as e:
print 'could not save login:', e
else:
e = MeException()
e.html = html
raise e
return html
