def inflate_service(perform=False):
users = []
for t in [User]:
for u in t().scan():
if u[user_keys.user_twitter_auth] and '.com' not in u[
user_keys.
user_role] and user_keys.user_locked not in u.keys():
ua = auth.user_app(u)
if ua:
users.append(u)
sorted_users = sorted(users, key=lambda item: item[time_keys.ts_inflated])
for i, so in enumerate(sorted_users):
print '{:3s}'.format(str(i + 1)), '{:20s}'.format(
so[user_keys.user_role]), fixed.lingo_since_date(
so[time_keys.ts_inflated]), so.table.table_name
if perform:
inflate_user = sorted_users[0]
inflate_app_user = auth.user_app(inflate_user)
print ''
print inflate_user[user_keys.user_role], inflate_app_user[
user_keys.user_role], inflate_user.table.table_name
restful.Inflate().do_inflate(inflate_user, inflate_app_user)
StalkQueue().createMessage(
{keys.entity_league: inflate_user[user_keys.user_role]})
def league_lists(league_name, do_add=True):
print 'league list:', league_name
league_user = User().get_by_role(league_name, keys.entity_twitter)
league_user_app = auth.user_app(league_user)
oauth = auth.get_oauth(
league_user, league_user_app,
league_user_app[user_keys.user_twitter_apps].keys()[0])
lists = restful.get_lists(league_user, league_user_app, oauth)
print 'lists:', len(lists), [_l['name'] for _l in lists]
league_user[user_keys.user_home_lists] = len(lists)
league_user.save()
for team in Entity().query_2(league__eq=league_name,
profile__beginswith='team:',
reverse=True):
team_name = make_list_name(team[keys.entity_profile].split(':')[1])
if not get_list_id(team_name, lists):
print 'missing list:', team_name
list_description = u'Tracking {0} {1} from {2}'.format(
league_user[keys.entity_name],
league_user[keys.entity_lingo]['players'], team_name)
restful.create_list(league_user, league_user_app, team_name,
list_description, oauth)
else:
print league_name, 'has:', team_name
list_id = get_list_id(team_name, lists)
existing_members = restful.list_members(league_user,
league_user_app, list_id,
oauth)
print team_name, 'existing members:', len(existing_members)
if do_add:
add_members = []
for e in Entity().query_2(
index=Entity.index_team_profile,
team__eq=team[keys.entity_profile].split(':', 1)[1],
query_filter={'twitter__null': False}):
check_twitter(e, add_members, existing_members)
if len(add_members) == 100:
if not do_add_members(league_user, league_user_app,
list_id, add_members, oauth):
return
add_members = []
if len(add_members) > 0:
if not do_add_members(league_user, league_user_app,
list_id, add_members, oauth):
return
print 'excess members:', len(
existing_members), existing_members
remove_members = []
for em in existing_members:
remove_members.append(em)
if len(remove_members) == 100:
do_remove_members(league_user, league_user_app,
list_id, remove_members, oauth)
remove_members = []
if len(remove_members) > 0:
do_remove_members(league_user, league_user_app, list_id,
remove_members, oauth)
def trimIt(self, it, tweet, msg):
if tweet[twitter_keys.message_tweet] != it:
tweet[twitter_keys.message_tweet] = it
if keys.entity_site in tweet:
try:
user = User().get_by_role(tweet[keys.entity_league],
keys.entity_twitter)
user_app = auth.user_app(user)
print 'user:'******'publish api tweet'
tweet_delay[tweet[keys.entity_league]] = int(
time.time())
local_image_location = None
if twitter_keys.message_pic in tweet:
local_image_location = '/tmp/' + tweet[
twitter_keys.message_pic].rsplit('/', 1)[1]
urllib.urlretrieve(tweet[twitter_keys.message_pic],
local_image_location)
restful.post_tweet(user, user_app,
tweet[twitter_keys.message_tweet],
local_image_location)
TweetQueue().deleteMessage(msg)
else:
print 'delay publish api tweet'
else:
print 'publish queue tweet'
TweetQueue(tweet[keys.entity_league]).createMessage(tweet)
TweetQueue().deleteMessage(msg)
except Exception as e:
print 'post tweet exception:', e
else:
print 'delete tweet'
TweetQueue().deleteMessage(msg)
from app import user_keys
from amazon.dynamo import User
from twitter import auth
for i, l in enumerate(User().get_leagues()):
locked = False
if l[user_keys.user_locked]:
locked = True
app_name = None
ua = auth.user_app(l)
if ua:
app_name = ua[user_keys.user_twitter_apps].keys()[0]
print '{:3s}'.format(str(i + 1)), '{:6s}'.format(
str(locked)), '{:20s}'.format(l[user_keys.user_role]), '{:40s}'.format(
str(app_name))
{keys.entity_league: inflate_user[user_keys.user_role]})
if __name__ == '__main__':
try:
backup = bool(sys.argv[2])
except:
backup = False
#if backup:
# t = UserAvailable
#else:
t = User
if len(sys.argv) > 1:
if sys.argv[1] != 'True':
inflate_user = t().get_by_role(sys.argv[1], keys.entity_twitter)
print 'inflate:', inflate_user[
user_keys.user_role], 'twitter:', inflate_user[
keys.entity_twitter]
inflate_user_app = auth.user_app(inflate_user)
app_name = inflate_user_app[user_keys.user_twitter_apps].keys()[0]
print 'app name:', app_name
if inflate_user[
user_keys.user_twitter_auth] and app_name in inflate_user[
user_keys.user_twitter_auth]:
restful.Inflate().do_inflate(inflate_user, inflate_user_app)
else:
print 'no auth for app_name:', app_name
else:
inflate_service(True)
else:
inflate_service()
def process_inline(self):
raw_entities = yield self.entities()
print 'raw entities:', len(raw_entities)
all_entities = self.prune_dups(raw_entities)
if len(all_entities) >= self.min_size and len(
all_entities) <= self.max_size and self.save_db:
print 'accpetable entities', len(all_entities)
try:
io = json.dumps(all_entities, cls=fixed.SetEncoder)
bucket = s3.bucket_straight(User().get_curator(
self.get_league_name())[user_keys.user_role])
db_count = '1'
try:
redirect = s3.get_redirect(
bucket,
'/' + self.get_league_name() + '/db/league.json')
db_count = str(
int(redirect[len('/' + self.get_league_name() +
'/db/league'):][:-5]) + 1)
except Exception as e:
print 'db count redirecct exception:', 3
try:
index = 0
for key in bucket.list(prefix=self.get_league_name() +
'/db/league'):
index += 1
db_count = str(index)
except:
print 'db count prefix exception!'
locate = '/' + self.get_league_name(
) + '/db/league' + db_count + '.json'
ts_scraped = int(time.time())
bible_tweet = {
time_keys.ts_scraped: ts_scraped,
keys.entity_count: len(all_entities)
}
league_for_message = Entity().get_league(
self.get_league_name())
bible_tweet.update(league_for_message._data)
bible_tweet[twitter_keys.message_tweet] = tweets.bible_tweet(
User().get_curator(self.get_league_name()),
league_for_message)
print locate, bible_tweet
s3.save_s3(bucket, locate, io, None, 'application/json',
'public-read', bible_tweet)
s3.create_local_redirect(
bucket,
self.get_league_name() + '/db/league.json', locate)
league_user = User().get_by_role(self.get_league_name(),
keys.entity_twitter)
league_user_app = auth.user_app(league_user)
if league_user_app:
if not restful.post_tweet(
league_user, league_user_app,
bible_tweet[twitter_keys.message_tweet]):
TweetQueue().createMessage(bible_tweet)
else:
TweetQueue().createMessage(bible_tweet)
try:
user_league = User().get_by_role(self.get_league_name(),
keys.entity_twitter)
user_league[time_keys.ts_bible] = ts_scraped
user_league.partial_save()
except:
print 'missing user:'******'league done:', self.get_league_name(
), 'all entities length:', len(all_entities)
defer.returnValue(True)
except Exception as e:
print 'except:', e