def add_blocked(site_name, social_key, block):
blocks = set(get_blocked(site_name, social_key))
blocks.add(block.lower())
content = json.dumps(blocks,cls=fixed.SetEncoder)
bucket = s3.bucket_straight(site_name)
s3.save_s3(bucket, 'site/' + social_key + '_blocked.json', content, None, 'application/json', 'public-read')
return list(blocks)
def route_email(self, ee):
if save_it(ee.broadcast_dict['derived_from'].split('@')[-1]):
if ee.attachments is not None:
for filename, raw_file, content_type in ee.attachments:
s3.save_s3(s3.bucket_straight('www.scewpt.com'),
ee.broadcast_dict['file_dest'] + "_" + filename,
raw_file, None, content_type, 'public-read')
def html_loaded(result=None):
print 'html_loaded:', result
local_file_name = prefix + '/bg_update_' + background[
keys.entity_twitter] + '.png'
qimg = QImage(cv.width(), cv.height(),
QImage.Format_ARGB32_Premultiplied)
painter = QPainter(qimg)
cv.page().view().render(painter)
qimg.save(local_file_name)
print 'saved:', local_file_name, 'painter:', painter, 'qimg:', qimg
del painter
if background and twitter_keys.message_tweet in background:
upload_png = background[keys.entity_league] + '/tweet/' + ''.join(
random.choice(string.ascii_lowercase)
for _ in range(10)) + '.png'
b = s3.bucket_straight(curator['role'])
s3.save_s3(b, upload_png, None, local_file_name, 'image/png')
bg_message_pic = 'http://' + b.name + '/' + upload_png
print 'bg_message_pic:', bg_message_pic
if background:
background[twitter_keys.message_pic] = bg_message_pic
print 'append to tweet:', background[
twitter_keys.message_tweet], 'url:', background[
twitter_keys.message_pic]
TweetQueue().createMessage(background)
cv.close()
reactor.stop()
def tick():
try:
upload_png = avi[keys.entity_league] + '/tweet/' + str(
int(time.time())) + '.png'
img = view.grabWindow()
print 'img:', img, local_file_name, img.isNull()
render_res = img.save(local_file_name)
print 'render_res:', render_res, 'file name:', upload_png, 'tweet:', avi
if render_res and avi and twitter_keys.message_tweet in avi:
b = s3.bucket_straight(curator['role'])
s3.save_s3(b, upload_png, None, local_file_name, 'image/png',
'public-read', meta)
if avi:
avi[twitter_keys.
message_pic] = 'http://' + b.name + '/' + upload_png
print 'append to tweet:', avi[
twitter_keys.message_tweet], 'url:', avi[
twitter_keys.message_pic]
TweetQueue().createMessage(avi)
else:
print 'skip publish'
except Exception as e:
print 'tick exception:', e
reactor.callLater(0, reactor.stop)
def do_publish(site):
b = s3.bucket_straight(site)
for res in get_publish_list(site):
publish_to = res[len(get_build_dir(site)) + 1:]
key = s3.save_s3(b, publish_to, None, res, None, 'public-read', None,
'gzip')
print publish_to, 'to:', key.name
key = s3.save_s3(b, 'intersection-observer.js', None,
'/home/ubuntu/' + site + '/intersection-observer.js',
None, 'public-read', None, 'gzip')
def stalk_analysis(league_name):
mutuals = []
extras = []
for mf in Entity().query_2(league__eq=league_name):
if 'twitter' in mf and 'ts_followers_' + league_name in mf:
try:
ts_add = None
if mf[time_keys.ts_scout]:
ts_add = mf[time_keys.ts_scout]
slt = ProfileTwitter().profile_last(mf[keys.entity_twitter_id], None, ts_add)
if slt is not None:
print 'append:', mf[keys.entity_twitter], 'since:', fixed.lingo_since_date(ts_add)
mutuals.append( (mf, slt) )
else:
print 'no last stats:', mf[keys.entity_twitter], mf[keys.entity_twitter_id]
except Exception as e:
print 'missing:', e, 'https://twitter.com/' + mf[keys.entity_twitter], fixed.lingo_since(mf, twitter_keys.league_ts_followers(league_name))
else:
extras.append(mf._data)
print 'extras length:', len(extras)
for mutual_seq in mutuals:
try:
mutual = mutual_seq[0]
mutual_slt = mutual_seq[1]
print 'mutual:', mutual[keys.entity_twitter]
tf = set([])
for other_seq in [others for others in mutuals if others[0] != mutual and twitter_keys.league_mutual(league_name) in others[1]]:
if mutual[keys.entity_twitter_id] in other_seq[1][twitter_keys.league_mutual(league_name)]:
tf.add(other_seq[0][keys.entity_twitter_id])
if len(tf) > 0:
print mutual[keys.entity_twitter], 'follows:', len(tf), 'following:', 0 if twitter_keys.league_mutual(league_name) not in mutual_slt else len(mutual_slt[twitter_keys.league_mutual(league_name)])
mutual_slt[twitter_keys.league_follows(league_name)] = tf
mutual_slt.partial_save()
else:
print 'not following anyone:', mutual[keys.entity_twitter], mutual[keys.entity_twitter_id]
except Exception as e:
print 'mutual exception:', e
publish = []
curator = User().get_curator(league_name)
for mutual_seq_2 in mutuals:
try:
p = {}
mutual = mutual_seq_2[0]
p.update(mutual._data)
mutual_slt = mutual_seq_2[1]
p.update(mutual_slt._data)
publish.append(p)
except:
print 'mutual exception:', e
b = s3.bucket_straight(curator[user_keys.user_role])
filename = league_name + '/db/bible.json'
meta = shared.entity_filter(curator._data)
output = shared.dump(publish + extras)
s3.save_s3(b, filename, output , None, content_type='application/json', acl='public-read', meta=meta, encode='gzip')
def screenshot(view, site, curator):
img = view.grabWindow()
il = '/home/ubuntu/' + site + '/curator_background.png'
res = img.save(il)
print 'save result:', res, il
try:
bs = s3.bucket_straight(curator[user_keys.user_role])
s3.save_s3(bs, 'curator_background.png', None, il, 'image/png',
'public-read')
except Exception as e:
print e
curator = User().get_by_role(site, keys.entity_twitter)
restful.update_background(curator, curator, il)
def add_twitter_ignore(self, ignore_twitter):
print 'twitter ignore:', ignore_twitter
po = self.get_twitter_ignores()
po.append(ignore_twitter)
s3.save_s3(s3.bucket_straight(User().get_curator(
self.get_league_name())[user_keys.user_role]),
'/' + self.get_league_name() + '/twitter_ignores.json',
json.dumps(po),
None,
content_type='application/json',
acl='public-read')
print 'done save'
del self.twitter_ignores
def route_email(self, ee):
proxy = Proxy('http://localhost:8001/')
if ee.html:
print 'preview html in browser'
d = proxy.callRemote('preview', ee.html,
fixed.digest(ee.broadcast_dict['file_dest']))
d.addCallback(lambda local_file: s3.save_s3(
s3.bucket_straight('www.scewpt.com'),
str(ee.broadcast_dict['file_dest'] + "_preview.png"), None,
local_file, "image/png", 'public-read'))
d.addErrback(self.browser_error)
return d
else:
print 'not previewed'
return defer.SUCCESS
def route_email(self, ee):
if save_it(ee.broadcast_dict['derived_from'].split('@')[-1]):
s3.save_s3(s3.bucket_straight('www.scewpt.com'),
ee.broadcast_dict['file_dest'] + '.json',
json.dumps(ee.broadcast_dict), None, 'application/json',
'public-read')
def route_email(self, ee):
print 'route email perm'
if save_it(ee.broadcast_dict['derived_from'].split('@')[-1]):
s3.save_s3(s3.bucket_straight('www.scewpt.com'),
ee.broadcast_dict['file_dest'] + '.html', ee.summary,
None, 'text/html', 'public-read')
def route_email(self, ee):
print 'route last/loudest'
s3.save_s3(s3.bucket_straight('www.scewpt.com'),
'inbox/lastloudest.html', ee.summary, None, 'text/html',
'public-read')
import json
from amazon.dynamo import User
from app import user_keys
from amazon import s3
c = boto.ec2.cloudwatch.connect_to_region('us-east-1')
end = datetime.datetime.utcnow()
start = end - datetime.timedelta(days=14)
epoch = datetime.datetime.utcfromtimestamp(0)
average_period = 900
data = c.get_metric_statistics(period=average_period,
start_time=start,
end_time=end,
metric_name='ConsumedWriteCapacityUnits',
namespace='AWS/DynamoDB',
statistics=['Sum'],
dimensions={'TableName': 'tweet'})
print 'data length:', len(data)
social_sum = [{
'timestamp': int((d['Timestamp'] - epoch).total_seconds()),
'sum': int(d['Sum'])
} for d in data]
for site in User().get_sites():
s3.save_s3(s3.bucket_straight(site[user_keys.user_role]),
'chart/social.json', json.dumps(social_sum), None,
'application/json', 'public-read')
def store_data(datapath, data):
s3.save_s3(s3.bucket_straight('ventorta.com'), datapath,
json.dumps(data, indent=4), None, 'application/json')
from amazon.dynamo import User, Entity
from amazon import s3
from app import user_keys, keys
from urllib2 import urlopen
from wand.image import Image
for u in User().get_leagues():
e = Entity().get_league(u[user_keys.user_role])
bg = 'http://' + e[keys.entity_site] + '/tw/' + e[
keys.entity_twitter_id] + '/background.png'
try:
f = urlopen(bg)
except:
f = urlopen('http://s3.amazonaws.com/scewpt.com/1500x500.png')
with Image(file=f) as img:
width = img.width
height = img.height
img.resize(300, 100)
img.save(filename='/tmp/' + u[user_keys.user_role] + '.png')
b = s3.bucket_straight(e[keys.entity_site])
s3.save_s3(b, u[user_keys.user_role] + '/background.png', None,
'/tmp/' + u[user_keys.user_role] + '.png', 'image/png')
f.close()
print u[user_keys.user_role], width, height
def process_inline(self):
raw_entities = yield self.entities()
print 'raw entities:', len(raw_entities)
all_entities = self.prune_dups(raw_entities)
if len(all_entities) >= self.min_size and len(
all_entities) <= self.max_size and self.save_db:
print 'accpetable entities', len(all_entities)
try:
io = json.dumps(all_entities, cls=fixed.SetEncoder)
bucket = s3.bucket_straight(User().get_curator(
self.get_league_name())[user_keys.user_role])
db_count = '1'
try:
redirect = s3.get_redirect(
bucket,
'/' + self.get_league_name() + '/db/league.json')
db_count = str(
int(redirect[len('/' + self.get_league_name() +
'/db/league'):][:-5]) + 1)
except Exception as e:
print 'db count redirecct exception:', 3
try:
index = 0
for key in bucket.list(prefix=self.get_league_name() +
'/db/league'):
index += 1
db_count = str(index)
except:
print 'db count prefix exception!'
locate = '/' + self.get_league_name(
) + '/db/league' + db_count + '.json'
ts_scraped = int(time.time())
bible_tweet = {
time_keys.ts_scraped: ts_scraped,
keys.entity_count: len(all_entities)
}
league_for_message = Entity().get_league(
self.get_league_name())
bible_tweet.update(league_for_message._data)
bible_tweet[twitter_keys.message_tweet] = tweets.bible_tweet(
User().get_curator(self.get_league_name()),
league_for_message)
print locate, bible_tweet
s3.save_s3(bucket, locate, io, None, 'application/json',
'public-read', bible_tweet)
s3.create_local_redirect(
bucket,
self.get_league_name() + '/db/league.json', locate)
league_user = User().get_by_role(self.get_league_name(),
keys.entity_twitter)
league_user_app = auth.user_app(league_user)
if league_user_app:
if not restful.post_tweet(
league_user, league_user_app,
bible_tweet[twitter_keys.message_tweet]):
TweetQueue().createMessage(bible_tweet)
else:
TweetQueue().createMessage(bible_tweet)
try:
user_league = User().get_by_role(self.get_league_name(),
keys.entity_twitter)
user_league[time_keys.ts_bible] = ts_scraped
user_league.partial_save()
except:
print 'missing user:'******'league done:', self.get_league_name(
), 'all entities length:', len(all_entities)
defer.returnValue(True)
except Exception as e:
print 'except:', e