def refresh(self): secrets = app_config.get_secrets() if not self.project.domain: return url = 'http://%s%s' % (self.project.domain, self.project.prefix) response = requests.get('https://free.sharedcount.com/url?apikey=%s&url=%s' % (secrets['SHAREDCOUNT_API_KEY'], url)) if response.status_code != 200: print 'Failed to refresh social data from SharedCount: %i.' % response.status_code return print 'Updating social counts from SharedCount' data = response.json() self.facebook_likes = data['Facebook']['like_count'] or 0 self.facebook_shares = data['Facebook']['share_count'] or 0 self.facebook_comments = data['Facebook']['comment_count'] or 0 self.twitter = data['Twitter'] or 0 self.google = data['GooglePlusOne'] or 0 self.pinterest = data['Pinterest'] or 0 self.linkedin = data['LinkedIn'] or 0 self.stumbleupon = data['StumbleUpon'] or 0 self.last_update = timezone.now() self.save()
def _test_api_init(freq=2627): secrets = app_config.get_secrets() url = secrets.get('VERB8TM_SRT_TEST_API', app_config.PROJECT_SLUG) header = 'Content-Type: application/json' payload = '{"command":"init","millisecondCaptionDelay":"%s"}' % (freq) command = 'curl -H "%s" -X POST -d \'%s\' %s' % (header, payload, url) local(command)
def get_goodreads_id(cls, isbn): """ Use GoodReads search API """ secrets = app_config.get_secrets() goodreads_id = None search_api_tpl = 'https://www.goodreads.com/search/index.xml' params = { 'key': secrets['GOODREADS_API_KEY'], 'q': isbn.encode('utf-8') } query_string = urlencode(params) search_api_url = '%s?%s' % (search_api_tpl, query_string) # Get search api results. r = requests.get(search_api_url, params=params) if r.status_code == 200: tree = ElementTree.fromstring(r.content) best_book = tree.find('.//best_book') if best_book is not None: goodreads_id = best_book.find('id').text else: logger.warning('could not find a matching book for ISBN %s' % isbn) else: logger.warning('did not receive a 200 when using Goodreads search api') return goodreads_id
def refresh(self): secrets = app_config.get_secrets() if not self.project.domain: return url = 'http://%s%s' % (self.project.domain, self.project.prefix) response = requests.get( 'https://free.sharedcount.com/url?apikey=%s&url=%s' % (secrets['SHAREDCOUNT_API_KEY'], url)) if response.status_code != 200: print 'Failed to refresh social data from SharedCount: %i.' % response.status_code return print 'Updating social counts from SharedCount' data = response.json() self.facebook_likes = data['Facebook']['like_count'] or 0 self.facebook_shares = data['Facebook']['share_count'] or 0 self.facebook_comments = data['Facebook']['comment_count'] or 0 self.twitter = data['Twitter'] or 0 self.google = data['GooglePlusOne'] or 0 self.pinterest = data['Pinterest'] or 0 self.linkedin = data['LinkedIn'] or 0 self.stumbleupon = data['StumbleUpon'] or 0 self.last_update = timezone.now() self.save()
def render_confs(): """ Renders server configurations. """ require('settings', provided_by=[production, staging]) with settings(warn_only=True): local('mkdir confs/rendered') context = app_config.get_secrets() context['PROJECT_SLUG'] = app_config.PROJECT_SLUG context['CLOUD_SEARCH_DOMAIN'] = app_config.CLOUD_SEARCH_DOMAIN context['PROJECT_NAME'] = app_config.PROJECT_NAME context['DEPLOYMENT_TARGET'] = env.settings context['CONFIG_NAME'] = env.project_slug.replace('-', '').upper() for service, remote_path, extension in SERVICES: file_path = 'confs/rendered/%s.%s.%s' % (app_config.PROJECT_SLUG, service, extension) with open('confs/%s.%s' % (service, extension), 'r') as read_template: with open(file_path, 'wb') as write_template: payload = Template(read_template.read()) write_template.write(payload.render(**context))
def fetch_posts(): """ Returns a list of all tumblr posts, unsorted. """ print "Starting." # Set constants secrets = app_config.get_secrets() base_url = 'http://api.tumblr.com/v2/blog/%s.tumblr.com/posts/photo' % app_config.TUMBLR_BLOG_ID key_param = '?api_key=%s' % secrets['TUMBLR_APP_KEY'] limit_param = '&limit=20' limit = 20 new_limit = limit post_list = [] print base_url + key_param # Figure out the total number of posts. r = requests.get(base_url + key_param) total_count = int(json.loads(r.content)['response']['total_posts']) print "%s total posts available." % total_count # Do the pagination math. pages_count = (total_count / limit) pages_remainder = (total_count % limit) if pages_remainder > 0: pages_count += 1 pages = range(0, pages_count) print "%s pages required." % len(pages) # Start requesting pages. # Note: Maximum of 20 posts per page. print "Requesting pages." for page in pages: # Update all of the pagination shenanigans. start_number = new_limit - limit end_number = new_limit if end_number > total_count: end_number = total_count new_limit = new_limit + limit page_param = '&offset=%s' % start_number page_url = base_url + key_param + limit_param + page_param # Actually fetch the page URL. r = requests.get(page_url) posts = json.loads(r.content) for post in posts['response']['posts']: try: if 'NSFW' in post['tags']: pass elif 'nsfw' in post['tags']: pass else: post_list.append(post) except KeyError: pass return post_list
def local_reset_db(): secrets = app_config.get_secrets() with settings(warn_only=True): local('dropdb %s' % app_config.PROJECT_SLUG) local('echo "CREATE USER %s WITH PASSWORD \'%s\';" | psql' % (app_config.PROJECT_SLUG, secrets['POSTGRES_PASSWORD'])) local('createdb -O %s %s' % (app_config.PROJECT_SLUG, app_config.PROJECT_SLUG))
def index(): """ Example view demonstrating rendering a simple HTML page. """ context = make_context() context['INSTAGRAM_CLIENT_ID'] = app_config.get_secrets()['INSTAGRAM_CLIENT_ID'] return render_template('index.html', **context)
def index(): """ Example view demonstrating rendering a simple HTML page. """ context = make_context() context['INSTAGRAM_CLIENT_ID'] = app_config.get_secrets( )['INSTAGRAM_CLIENT_ID'] return render_template('index.html', **context)
def authenticate(): secrets = app_config.get_secrets() auth = tweepy.OAuthHandler(secrets['TWITTER_CONSUMER_KEY'], secrets['TWITTER_CONSUMER_SECRET']) auth.set_access_token(secrets['TWITTER_ACCESS_KEY'], secrets['TWITTER_ACCESS_SECRET']) api = tweepy.API(auth) return api
def verb8tm_stop(): """ Stops the generation of content in Verb8tm test API endpoint """ secrets = app_config.get_secrets() url = secrets.get('VERB8TM_SRT_TEST_API', app_config.PROJECT_SLUG) header = 'Content-Type: application/json' payload = '{"command":"stop"}' command = 'curl -H "%s" -X POST -d \'%s\' %s' % (header, payload, url) local(command)
def load_images(): """ Downloads images from Baker and Taylor. Eschews the API for a magic URL pattern, which is faster. """ # Secrets. secrets = app_config.get_secrets() # Open the books JSON. with open("www/static-data/books.json", "rb") as readfile: books = json.loads(readfile.read()) print "Start load_images(): %i books." % len(books) # Loop. for book in books: # Skip books with no title or ISBN. if book["title"] == "": continue if "isbn" not in book or book["isbn"] == "": continue # Construct the URL with secrets and the ISBN. book_url = "http://imagesa.btol.com/ContentCafe/Jacket.aspx?UserID=%s&Password=%s&Return=T&Type=L&Value=%s" % ( secrets["BAKER_TAYLOR_USERID"], secrets["BAKER_TAYLOR_PASSWORD"], book["isbn"], ) # Request the image. r = requests.get(book_url) path = "www/img/cover/%s.jpg" % book["slug"] # Write the image to www using the slug as the filename. with open(path, "wb") as writefile: writefile.write(r.content) file_size = os.path.getsize(path) if file_size < 10000: print "Image not available for ISBN: %s" % book["isbn"] image = Image.open(path) width = 250 height = int((float(width) / image.size[0]) * image.size[1]) image.thumbnail([width, height], Image.ANTIALIAS) image.save(path.replace(".jpg", "-thumb.jpg"), "JPEG") print "End."
def local_reset_db(): """ Reset the database locally. """ secrets = app_config.get_secrets() with settings(warn_only=True): local('dropdb %s' % app_config.PROJECT_SLUG) local('echo "CREATE USER %s WITH PASSWORD \'%s\';" | psql' % (app_config.PROJECT_SLUG, secrets['POSTGRES_PASSWORD'])) local('createdb -O %s %s' % (app_config.PROJECT_SLUG, app_config.PROJECT_SLUG))
def local_reset_db(): secrets = app_config.get_secrets() with settings(warn_only=True): local("dropdb %s" % app_config.PROJECT_SLUG) local( "echo \"CREATE USER %s WITH PASSWORD '%s';\" | psql" % (app_config.PROJECT_SLUG, secrets["POSTGRES_PASSWORD"]) ) local("createdb -O %s %s" % (app_config.PROJECT_SLUG, app_config.PROJECT_SLUG))
def load_images(): """ Downloads images from Baker and Taylor. Eschews the API for a magic URL pattern, which is faster. """ # Secrets. secrets = app_config.get_secrets() # Open the books JSON. with open('www/static-data/books.json', 'rb') as readfile: books = json.loads(readfile.read()) print "Start load_images(): %i books." % len(books) # Loop. for book in books: # Skip books with no title or ISBN. if book['title'] == "": continue if 'isbn' not in book or book['isbn'] == "": continue # Construct the URL with secrets and the ISBN. book_url = "http://imagesa.btol.com/ContentCafe/Jacket.aspx?UserID=%s&Password=%s&Return=T&Type=L&Value=%s" % ( secrets['BAKER_TAYLOR_USERID'], secrets['BAKER_TAYLOR_PASSWORD'], book['isbn']) # Request the image. r = requests.get(book_url) path = 'www/assets/cover/%s.jpg' % book['slug'] # Write the image to www using the slug as the filename. with open(path, 'wb') as writefile: writefile.write(r.content) file_size = os.path.getsize(path) if file_size < 10000: print "Image not available for ISBN: %s" % book['isbn'] image = Image.open(path) width = 250 height = int((float(width) / image.size[0]) * image.size[1]) image.thumbnail([width, height], Image.ANTIALIAS) image.save(path.replace('.jpg', '-thumb.jpg'), 'JPEG') print "End."
def execute_setup(script_name=None, doc_id=None, log_id=None): """ execute script setup: params script_id, document_id, log_id """ require('settings', provided_by=['production', 'staging', 'development']) # Get secrets secrets = app_config.get_secrets() verb8tm_srt_url = secrets.get('VERB8TM_SRT_API', app_config.PROJECT_SLUG) verb8tm_timestamp_url = secrets.get('VERB8TM_TIMESTAMP_API', app_config.PROJECT_SLUG) # Get the script id from the script name and deployment target # prioritize passed in parameters if not script_name: script_name = '%s_%s' % (app_config.DEPLOYMENT_TARGET, app_config.SCRIPT_PROJECT_NAME) script_id = get_gas_project_id(script_name) URL_PREFIX = 'https://script.googleapis.com/v1/scripts/' # url url = '%s%s:run' % (URL_PREFIX, script_id) # Compose payload we pass documentID and logID to setup script properties payload = { 'function': 'setup', 'parameters': [verb8tm_srt_url, verb8tm_timestamp_url, app_config.LIVEBLOG_GDOC_KEY, app_config.GAS_LOG_KEY] } kwargs = { 'credentials': check_credentials(), 'url': url, 'method': 'POST', 'headers': {'Content-Type': 'application/json'}, 'body': json.dumps(payload) } resp = send_api_request(kwargs) if resp.status == 200: return True else: print resp.status exit() return False
def _geckoboard_push(widget_key, data): payload = { 'api_key': app_config.get_secrets()['GECKOBOARD_API_KEY'], 'data': data } response = requests.post( 'https://push.geckoboard.com/v1/send/%s' % widget_key, json.dumps(payload)) if response.status_code != 200: print 'Failed update update Geckoboard widget %s' % widget_key print response.content
def _geckoboard_push(widget_key, data): payload = { 'api_key': app_config.get_secrets()['GECKOBOARD_API_KEY'], 'data': data } response = requests.post( 'https://push.geckoboard.com/v1/send/%s' % widget_key, json.dumps(payload) ) if response.status_code != 200: print 'Failed update update Geckoboard widget %s' % widget_key print response.content
def mount(server=None, path='www', mount_point='mnt'): '''mount especiales server into local disk''' local_path = os.path.join(cwd, '..', mount_point) if os.path.exists(local_path): if os.listdir(local_path): logger.error('Mount point is not empty, choose other mount point') exit(1) else: os.makedirs(local_path) if not server: secrets = app_config.get_secrets() server = secrets.get('DEFAULT_SERVER', app_config.PROJECT_SLUG) command = SMB_MOUNT_TEMPLATE % (server, path, local_path) local(command) return local_path
def fetch_tweets(username, days): """ Get tweets of a specific user """ current_time = datetime.now() secrets = app_config.get_secrets() twitter_api = Twitter( auth=OAuth( secrets['TWITTER_API_OAUTH_TOKEN'], secrets['TWITTER_API_OAUTH_SECRET'], secrets['TWITTER_API_CONSUMER_KEY'], secrets['TWITTER_API_CONSUMER_SECRET'] ) ) out = [] tweets = twitter_api.statuses.user_timeline(screen_name=username, count=TWITTER_BATCH_SIZE) i = 0 while True: if i > (len(tweets)-1): break tweet = tweets[i] created_time = datetime.strptime(tweet['created_at'], '%a %b %d %H:%M:%S +0000 %Y') time_difference = (current_time - created_time).days if time_difference > int(days): break out.extend(_process_tweet(tweet, username)) i += 1 if i > (TWITTER_BATCH_SIZE-1): tweets = twitter_api.statuses.user_timeline(screen_name=username, count=TWITTER_BATCH_SIZE, max_id=tweet['id']) i = 0 out = _dedupe_links(out) return out
def dump_tumblr_json(): secrets = app_config.get_secrets() t = Tumblpy( app_key=secrets['TUMBLR_APP_KEY'], app_secret=secrets['TUMBLR_APP_SECRET'], oauth_token=secrets['TUMBLR_OAUTH_TOKEN'], oauth_token_secret=secrets['TUMBLR_OAUTH_TOKEN_SECRET']) limit = 10 pages = range(0, 20) for page in pages: offset = page * limit posts = t.get('posts', blog_url=app_config.TUMBLR_URL, params={'limit': limit, 'offset': offset}) with open('data/backups/tumblr_prod_%s.json' % page, 'w') as f: f.write(json.dumps(posts))
def deploy_to_tumblr(): now = datetime.now() secrets = app_config.get_secrets() tumblr_api = pytumblr.TumblrRestClient(secrets['TUMBLR_CONSUMER_KEY'], secrets['TUMBLR_CONSUMER_SECRET'], secrets['TUMBLR_TOKEN'], secrets['TUMBLR_TOKEN_SECRET']) body = data.make_tumblr_draft_html() response = tumblr_api.create_text(env.tumblr_blog_name, state='draft', format='html', body=body.encode('utf8')) print "%s: Created tumblr draft (id: %s)" % (now.isoformat(), response['id']) return response
def render_confs(): """ Renders server configurations. """ require('settings', provided_by=['production', 'staging']) # Copy the app_config so that when we load the secrets they don't # get exposed to other management commands context = copy.copy(app_config.__dict__) context.update(app_config.get_secrets()) for service, remote_path, extension in app_config.SERVER_SERVICES: template_path = _get_template_conf_path(service, extension) rendered_path = _get_rendered_conf_path(service, extension) with open(template_path, 'r') as read_template: with open(rendered_path, 'wb') as write_template: payload = Template(read_template.read()) write_template.write(payload.render(**context))
def write_test_posts(): """ Writes test posts to our test blog as defined by app_config.py """ # This is how many posts will be written. TOTAL_NUMBER = 9 secrets = app_config.get_secrets() t = Tumblpy(app_key=secrets['TUMBLR_APP_KEY'], app_secret=secrets['TUMBLR_APP_SECRET'], oauth_token=secrets['TUMBLR_OAUTH_TOKEN'], oauth_token_secret=secrets['TUMBLR_OAUTH_TOKEN_SECRET']) tags = ['featured', ''] images = [ 'http://media.npr.org/assets/img/2013/04/24/habitablezones_custom-fa87578c6e6a97788b92a0ecac956b9098607aa6-s40.jpg', 'http://media.npr.org/assets/img/2013/04/24/ocpack-32260770b4090f86ddeb7502175a631d50c3b4a1-s51.jpg', 'http://media.npr.org/assets/img/2013/04/24/dalrymple-c-karoki-lewis-4c9bd790639c870d51c670cbecbca4b802b82b1a-s51.jpg', 'http://media.npr.org/assets/img/2013/04/24/ap111231019469-46289d097a45801ed2ca3464da14b13be40e5adb-s51.jpg' ] n = 0 while n < TOTAL_NUMBER: image = choice(images) tag = choice(tags) caption = u"<p class='intro'>Introduction,</p><p class='message'>This is a test post.</p><p class='signature-name'>Sincerely,<br/>Test from Test, Test</p>" tumblr_post = t.post('post', blog_url=app_config.TUMBLR_URL, params={ 'type': 'photo', 'caption': caption, 'tags': tag, 'source': image }) print n, tumblr_post['id'] n += 1
def create_tables(): """ Create the databse tables. """ import models secrets = app_config.get_secrets() print 'Creating database tables' models.Race.create_table() models.Candidate.create_table() models.Slide.create_table() models.SlideSequence.create_table() print 'Setting up admin' admin_app.auth.User.create_table() admin_user = admin_app.auth.User(username='******', email='', admin=True, active=True) admin_user.set_password(secrets.get('ADMIN_PASSWORD')) admin_user.save()
def generate_rdio_playlist(): """ Generate a list of Rdio track IDs """ secrets = app_config.get_secrets() state = {} rdio_api = Rdio(secrets['RDIO_CONSUMER_KEY'], secrets['RDIO_CONSUMER_SECRET'], state) songs = [] with open('data/songs.csv') as f: rows = csv.DictReader(f) for row in rows: if row['rdio']: song_object = rdio_api.getObjectFromUrl(url=row['rdio']) if song_object['type'] == 't': songs.append(song_object['key']) print ','.join(songs)
def write_test_posts(): """ Writes test posts to our test blog as defined by app_config.py """ # This is how many posts will be written. TOTAL_NUMBER = 9 secrets = app_config.get_secrets() t = Tumblpy( app_key=secrets['TUMBLR_APP_KEY'], app_secret=secrets['TUMBLR_APP_SECRET'], oauth_token=secrets['TUMBLR_OAUTH_TOKEN'], oauth_token_secret=secrets['TUMBLR_OAUTH_TOKEN_SECRET']) tags = ['featured', ''] images = [ 'http://media.npr.org/assets/img/2013/04/24/habitablezones_custom-fa87578c6e6a97788b92a0ecac956b9098607aa6-s40.jpg', 'http://media.npr.org/assets/img/2013/04/24/ocpack-32260770b4090f86ddeb7502175a631d50c3b4a1-s51.jpg', 'http://media.npr.org/assets/img/2013/04/24/dalrymple-c-karoki-lewis-4c9bd790639c870d51c670cbecbca4b802b82b1a-s51.jpg', 'http://media.npr.org/assets/img/2013/04/24/ap111231019469-46289d097a45801ed2ca3464da14b13be40e5adb-s51.jpg' ] n = 0 while n < TOTAL_NUMBER: image = choice(images) tag = choice(tags) caption = u"<p class='intro'>Introduction,</p><p class='message'>This is a test post.</p><p class='signature-name'>Sincerely,<br/>Test from Test, Test</p>" tumblr_post = t.post('post', blog_url=app_config.TUMBLR_URL, params={ 'type': 'photo', 'caption': caption, 'tags': tag, 'source': image }) print n, tumblr_post['id'] n += 1
def render_confs(): """ Renders server configurations. """ require('settings', provided_by=[production, staging]) with settings(warn_only=True): local('mkdir confs/rendered') context = app_config.get_secrets() context['PROJECT_SLUG'] = app_config.PROJECT_SLUG context['PROJECT_NAME'] = app_config.PROJECT_NAME context['DEPLOYMENT_TARGET'] = env.settings for service, remote_path in SERVICES: file_path = 'confs/rendered/%s.%s.conf' % (app_config.PROJECT_SLUG, service) with open('confs/%s.conf' % service, 'r') as read_template: with open(file_path, 'wb') as write_template: payload = Template(read_template.read()) write_template.write(payload.render(**context))
def generate_spotify_playlist(): """ Generate a list of Spotify track IDs """ sp = spotipy.Spotify() songs = [] secrets = app_config.get_secrets() with open('data/songs.csv') as f: rows = csv.DictReader(f) for row in rows: if row['spotify'] and "track" in row['spotify']: try: song_object = sp.track(track_id=row['spotify']) if song_object['track_number']: songs.append(song_object['uri']) except: print 'invalid ID' print ','.join(songs)
def render_confs(): """ Renders server configurations. """ require("settings", provided_by=[production, staging]) with settings(warn_only=True): local("mkdir confs/rendered") context = app_config.get_secrets() context["PROJECT_SLUG"] = app_config.PROJECT_SLUG context["PROJECT_NAME"] = app_config.PROJECT_NAME context["DEPLOYMENT_TARGET"] = env.settings for service, remote_path in SERVICES: file_path = "confs/rendered/%s.%s.conf" % (app_config.PROJECT_SLUG, service) with open("confs/%s.conf" % service, "r") as read_template: with open(file_path, "wb") as write_template: payload = Template(read_template.read()) write_template.write(payload.render(**context))
def update(): """ Fetch latests posts from Tumblr API. """ secrets = app_config.get_secrets() offset = 0 total_posts = 1 while offset < total_posts: print 'Fetching posts %i-%i' % (offset, offset + LIMIT) response = requests.get('http://api.tumblr.com/v2/blog/%s.tumblr.com/posts' % app_config.TUMBLR_NAME, params={ 'api_key':secrets['TUMBLR_CONSUMER_KEY'], 'limit': LIMIT, 'offset': offset }) data = response.json() total_posts = data['response']['total_posts'] posts = data['response']['posts'] for post in posts: if post['type'] in UNSUPPORTED_TYPES: continue if datetime.fromtimestamp(post['timestamp']) < app_config.TUMBLR_NOT_BEFORE: print 'Skipping old post' continue elif datetime.fromtimestamp(post['timestamp']) > app_config.TUMBLR_NOT_AFTER: print 'Skipping newer post' continue _create_slide(post) offset += LIMIT
def render_confs(): """ Renders server configurations. """ require("settings", provided_by=[production, staging]) with settings(warn_only=True): local("mkdir confs/rendered") # Copy the app_config so that when we load the secrets they don't # get exposed to other management commands context = copy.copy(app_config.__dict__) context.update(app_config.get_secrets()) for service, remote_path, extension in app_config.SERVER_SERVICES: template_path = _get_template_conf_path(service, extension) rendered_path = _get_rendered_conf_path(service, extension) with open(template_path, "r") as read_template: with open(rendered_path, "wb") as write_template: payload = Template(read_template.read()) write_template.write(payload.render(**context))
def update(): """ Fetch latests posts from Tumblr API. """ secrets = app_config.get_secrets() offset = 0 total_posts = 1 while offset < total_posts: print "Fetching posts %i-%i" % (offset, offset + LIMIT) response = requests.get( "http://api.tumblr.com/v2/blog/%s.tumblr.com/posts" % app_config.TUMBLR_NAME, params={"api_key": secrets["TUMBLR_CONSUMER_KEY"], "limit": LIMIT, "offset": offset}, ) data = response.json() total_posts = data["response"]["total_posts"] posts = data["response"]["posts"] for post in posts: if post["type"] in UNSUPPORTED_TYPES: continue if datetime.fromtimestamp(post["timestamp"]) < app_config.TUMBLR_NOT_BEFORE: print "Skipping old post" continue elif datetime.fromtimestamp(post["timestamp"]) > app_config.TUMBLR_NOT_AFTER: print "Skipping newer post" continue _create_slide(post) offset += LIMIT
def render_confs(): """ Renders server configurations. """ require("settings", provided_by=[production, staging]) with settings(warn_only=True): local("mkdir confs/rendered") context = app_config.get_secrets() context["PROJECT_SLUG"] = app_config.PROJECT_SLUG context["CLOUD_SEARCH_DOMAIN"] = app_config.CLOUD_SEARCH_DOMAIN context["PROJECT_NAME"] = app_config.PROJECT_NAME context["DEPLOYMENT_TARGET"] = env.settings context["CONFIG_NAME"] = env.project_slug.replace("-", "").upper() for service, remote_path, extension in SERVICES: file_path = "confs/rendered/%s.%s.%s" % (app_config.PROJECT_SLUG, service, extension) with open("confs/%s.%s" % (service, extension), "r") as read_template: with open(file_path, "wb") as write_template: payload = Template(read_template.read()) write_template.write(payload.render(**context))
def generate_rdio_playlist(): """ Generate a list of Rdio track IDs """ secrets = app_config.get_secrets() state = {} rdio_api = Rdio( secrets['RDIO_CONSUMER_KEY'], secrets['RDIO_CONSUMER_SECRET'], state ) songs = [] with open('data/songs.csv') as f: rows = csv.DictReader(f) for row in rows: if row['rdio']: song_object = rdio_api.getObjectFromUrl(url=row['rdio']) if song_object['type'] == 't': songs.append(song_object['key']) print ','.join(songs)
def update_featured_social(): """ Update featured tweets """ COPY = copytext.Copy(app_config.COPY_PATH) secrets = app_config.get_secrets() # Twitter print 'Fetching tweets...' twitter_api = Twitter(auth=OAuth(secrets['TWITTER_API_OAUTH_TOKEN'], secrets['TWITTER_API_OAUTH_SECRET'], secrets['TWITTER_API_CONSUMER_KEY'], secrets['TWITTER_API_CONSUMER_SECRET'])) tweets = [] for i in range(1, 4): tweet_url = COPY['share']['featured_tweet%i' % i] if isinstance(tweet_url, copytext.Error) or unicode(tweet_url).strip() == '': continue tweet_id = unicode(tweet_url).split('/')[-1] tweet = twitter_api.statuses.show(id=tweet_id) creation_date = datetime.strptime(tweet['created_at'], '%a %b %d %H:%M:%S +0000 %Y') creation_date = '%s %i' % (creation_date.strftime('%b'), creation_date.day) tweet_url = 'http://twitter.com/%s/status/%s' % ( tweet['user']['screen_name'], tweet['id']) photo = None html = tweet['text'] subs = {} for media in tweet['entities'].get('media', []): original = tweet['text'][media['indices'][0]:media['indices'][1]] replacement = '<a href="%s" target="_blank" onclick="_gaq.push([\'_trackEvent\', \'%s\', \'featured-tweet-action\', \'link\', 0, \'%s\']);">%s</a>' % ( media['url'], app_config.PROJECT_SLUG, tweet_url, media['display_url']) subs[original] = replacement if media['type'] == 'photo' and not photo: photo = {'url': media['media_url']} for url in tweet['entities'].get('urls', []): original = tweet['text'][url['indices'][0]:url['indices'][1]] replacement = '<a href="%s" target="_blank" onclick="_gaq.push([\'_trackEvent\', \'%s\', \'featured-tweet-action\', \'link\', 0, \'%s\']);">%s</a>' % ( url['url'], app_config.PROJECT_SLUG, tweet_url, url['display_url']) subs[original] = replacement for hashtag in tweet['entities'].get('hashtags', []): original = tweet['text'][ hashtag['indices'][0]:hashtag['indices'][1]] replacement = '<a href="https://twitter.com/hashtag/%s" target="_blank" onclick="_gaq.push([\'_trackEvent\', \'%s\', \'featured-tweet-action\', \'hashtag\', 0, \'%s\']);">%s</a>' % ( hashtag['text'], app_config.PROJECT_SLUG, tweet_url, '#%s' % hashtag['text']) subs[original] = replacement for original, replacement in subs.items(): html = html.replace(original, replacement) # https://dev.twitter.com/docs/api/1.1/get/statuses/show/%3Aid tweets.append({ 'id': tweet['id'], 'url': tweet_url, 'html': html, 'favorite_count': tweet['favorite_count'], 'retweet_count': tweet['retweet_count'], 'user': { 'id': tweet['user']['id'], 'name': tweet['user']['name'], 'screen_name': tweet['user']['screen_name'], 'profile_image_url': tweet['user']['profile_image_url'], 'url': tweet['user']['url'], }, 'creation_date': creation_date, 'photo': photo }) # Facebook print 'Fetching Facebook posts...' fb_api = GraphAPI(secrets['FACEBOOK_API_APP_TOKEN']) facebook_posts = [] for i in range(1, 4): fb_url = COPY['share']['featured_facebook%i' % i] if isinstance(fb_url, copytext.Error) or unicode(fb_url).strip() == '': continue fb_id = unicode(fb_url).split('/')[-1] post = fb_api.get_object(fb_id) user = fb_api.get_object(post['from']['id']) user_picture = fb_api.get_object('%s/picture' % post['from']['id']) likes = fb_api.get_object('%s/likes' % fb_id, summary='true') comments = fb_api.get_object('%s/comments' % fb_id, summary='true') #shares = fb_api.get_object('%s/sharedposts' % fb_id) creation_date = datetime.strptime(post['created_time'], '%Y-%m-%dT%H:%M:%S+0000') creation_date = '%s %i' % (creation_date.strftime('%b'), creation_date.day) # https://developers.facebook.com/docs/graph-api/reference/v2.0/post facebook_posts.append({ 'id': post['id'], 'message': post['message'], 'link': { 'url': post['link'], 'name': post['name'], 'caption': (post['caption'] if 'caption' in post else None), 'description': post['description'], 'picture': post['picture'] }, 'from': { 'name': user['name'], 'link': user['link'], 'picture': user_picture['url'] }, 'likes': likes['summary']['total_count'], 'comments': comments['summary']['total_count'], #'shares': shares['summary']['total_count'], 'creation_date': creation_date }) # Render to JSON output = {'tweets': tweets, 'facebook_posts': facebook_posts} with open('data/featured.json', 'w') as f: json.dump(output, f)
""" import app_config import dataset import ipdb import logging import requests from datetime import datetime from dateutil import parser from fabric.api import local, task from facebook import GraphAPI from pyquery import PyQuery from scrapers.utils import get_art_root_url, get_seamus_id_from_url from scrapers.seamus.models import Story SECRETS = app_config.get_secrets() FACEBOOK_USER = '******' logging.basicConfig() logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) @task def local_bootstrap(): local('dropdb --if-exists %s' % app_config.PROJECT_SLUG) local('createdb %s' % app_config.PROJECT_SLUG) @task def drop_tables():
""" import app_config import json import logging import requests from datetime import datetime from fabric.api import local, require, task logging.basicConfig() logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) today = datetime.now().strftime('%Y-%m-%d') WEBHOOK = app_config.get_secrets()['WEBHOOK'] URL = 'https://www.federalregister.gov/api/v1/documents.json' COLORS_DICT = { 'Rule': '#5cb85c', 'Proposed Rule': '#5bc0de', 'Presidential Document': '#d9534f' } @task def post_message(): """ Example cron task. Note we use "local" instead of "run" because this will run on the server. """
#!/usr/bin/env python from datetime import datetime import logging import os import requests from pyquery import PyQuery from app_config import get_secrets from models import ApiEntry, Article SECRETS = get_secrets() logging.basicConfig() logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) class HomepageScraper: url = 'http://npr.org' def __init__(self): self.run_time = datetime.utcnow() def scrape_homepage(self, **kwargs): """ Scrape! """ logger.info('Scraping homepage (start time: %s)' % self.run_time) if not kwargs:
import datetime import time from peewee import * from app_config import get_secrets secrets = get_secrets() psql_db = PostgresqlDatabase("breaking", user=secrets["APPS_USER"], password=secrets["APPS_PASS"]) def delete_tables(): try: Event.drop_table() except: pass try: Fact.drop_table() except: pass def create_tables(): Event.create_table() Fact.create_table() class Event(Model): """
def execute_setup(name=None, doc_id=None, log_id=None, cspan=None, cspan_server=None): """ execute script setup params: name - script name document_id - associated google doc log_id - associated google logs spreadsheet cspan - False to use Verb8tm, True to use CSPAN cspan_server - url that will serve the CSPAN captions """ require('settings', provided_by=['production', 'staging', 'development']) # Get secrets secrets = app_config.get_secrets() verb8tm_srt_url = secrets.get('VERB8TM_SRT_API', app_config.PROJECT_SLUG) verb8tm_timestamp_url = secrets.get('VERB8TM_TIMESTAMP_API', app_config.PROJECT_SLUG) if cspan: pass else: cspan = str(app_config.CSPAN) ec2_public_dns_tpl = 'http://ec2-%s.compute-1.amazonaws.com:5000/' if cspan_server: pass else: if len(app_config.SERVERS): ip = app_config.SERVERS[0] ip = ip.replace('.', '-') cspan_server = ec2_public_dns_tpl % (ip) else: logger.error("no servers found, please specify a cspan_server") exit() # Get the script id from the script name and deployment target # prioritize passed in parameters if not name: name = '%s_%s' % (app_config.DEPLOYMENT_TARGET, app_config.SCRIPT_PROJECT_NAME) script_id = get_gas_project_id(name) URL_PREFIX = 'https://script.googleapis.com/v1/scripts/' # url url = '%s%s:run' % (URL_PREFIX, script_id) # Compose payload we pass documentID and logID to setup script properties payload = { 'function': 'setup', 'parameters': [ verb8tm_srt_url, verb8tm_timestamp_url, cspan_server, cspan, app_config.TRANSCRIPT_GDOC_KEY, app_config.GAS_LOG_KEY ] } kwargs = { 'credentials': check_credentials(), 'url': url, 'method': 'POST', 'headers': { 'Content-Type': 'application/json' }, 'body': json.dumps(payload) } resp = send_api_request(kwargs) if resp.status == 200: return True else: print resp.status exit() return False
import argparse import base64 from cStringIO import StringIO import datetime import logging from flask import Flask, abort, redirect, render_template, request, session, url_for from twython import Twython import app_config from render_utils import make_context app = Flask(__name__) app.debug = app_config.DEBUG app.secret_key = app_config.get_secrets()["SESSION_KEY"] file_handler = logging.FileHandler(app_config.APP_LOG_PATH) file_handler.setLevel(logging.INFO) app.logger.addHandler(file_handler) app.logger.setLevel(logging.INFO) secrets = app_config.get_secrets() # Example application views @app.route("/test/", methods=["GET"]) def _test_app(): """ Test route for verifying the application is running. """ app.logger.info("Test URL requested.")
'ERROR', '500', e, app_config.SERVERS[0], svg_path, name)) # These bits build a nicer error page that has the real stack trace on it. context = {} context['title'] = 'CairoSVG is unhappy.' context['message'] = e.output return render_template('500.html', **context) context = { 'name': name, 'location': location, } caption = render_template('caption.html', **context) secrets = app_config.get_secrets() t = Tumblpy( app_key=secrets['TUMBLR_APP_KEY'], app_secret=secrets['TUMBLR_APP_SECRET'], oauth_token=secrets['TUMBLR_OAUTH_TOKEN'], oauth_token_secret=secrets['TUMBLR_OAUTH_TOKEN_SECRET']) params = { "type": "photo", "caption": caption, "tags": app_config.TUMBLR_TAGS, "source": "http://%s%s" % (app_config.SERVERS[0], png_path) } try:
def update_featured_social(): """ Update featured tweets """ COPY = copytext.Copy(app_config.COPY_PATH) secrets = app_config.get_secrets() # Twitter print 'Fetching tweets...' twitter_api = Twitter( auth=OAuth( secrets['TWITTER_API_OAUTH_TOKEN'], secrets['TWITTER_API_OAUTH_SECRET'], secrets['TWITTER_API_CONSUMER_KEY'], secrets['TWITTER_API_CONSUMER_SECRET'] ) ) tweets = [] for i in range(1, 4): tweet_url = COPY['share']['featured_tweet%i' % i] if isinstance(tweet_url, copytext.Error) or unicode(tweet_url).strip() == '': continue tweet_id = unicode(tweet_url).split('/')[-1] tweet = twitter_api.statuses.show(id=tweet_id) creation_date = datetime.strptime(tweet['created_at'],'%a %b %d %H:%M:%S +0000 %Y') creation_date = '%s %i' % (creation_date.strftime('%b'), creation_date.day) tweet_url = 'http://twitter.com/%s/status/%s' % (tweet['user']['screen_name'], tweet['id']) photo = None html = tweet['text'] subs = {} for media in tweet['entities'].get('media', []): original = tweet['text'][media['indices'][0]:media['indices'][1]] replacement = '<a href="%s" target="_blank" onclick="_gaq.push([\'_trackEvent\', \'%s\', \'featured-tweet-action\', \'link\', 0, \'%s\']);">%s</a>' % (media['url'], app_config.PROJECT_SLUG, tweet_url, media['display_url']) subs[original] = replacement if media['type'] == 'photo' and not photo: photo = { 'url': media['media_url'] } for url in tweet['entities'].get('urls', []): original = tweet['text'][url['indices'][0]:url['indices'][1]] replacement = '<a href="%s" target="_blank" onclick="_gaq.push([\'_trackEvent\', \'%s\', \'featured-tweet-action\', \'link\', 0, \'%s\']);">%s</a>' % (url['url'], app_config.PROJECT_SLUG, tweet_url, url['display_url']) subs[original] = replacement for hashtag in tweet['entities'].get('hashtags', []): original = tweet['text'][hashtag['indices'][0]:hashtag['indices'][1]] replacement = '<a href="https://twitter.com/hashtag/%s" target="_blank" onclick="_gaq.push([\'_trackEvent\', \'%s\', \'featured-tweet-action\', \'hashtag\', 0, \'%s\']);">%s</a>' % (hashtag['text'], app_config.PROJECT_SLUG, tweet_url, '#%s' % hashtag['text']) subs[original] = replacement for original, replacement in subs.items(): html = html.replace(original, replacement) # https://dev.twitter.com/docs/api/1.1/get/statuses/show/%3Aid tweets.append({ 'id': tweet['id'], 'url': tweet_url, 'html': html, 'favorite_count': tweet['favorite_count'], 'retweet_count': tweet['retweet_count'], 'user': { 'id': tweet['user']['id'], 'name': tweet['user']['name'], 'screen_name': tweet['user']['screen_name'], 'profile_image_url': tweet['user']['profile_image_url'], 'url': tweet['user']['url'], }, 'creation_date': creation_date, 'photo': photo }) # Facebook print 'Fetching Facebook posts...' fb_api = GraphAPI(secrets['FACEBOOK_API_APP_TOKEN']) facebook_posts = [] for i in range(1, 4): fb_url = COPY['share']['featured_facebook%i' % i] if isinstance(fb_url, copytext.Error) or unicode(fb_url).strip() == '': continue fb_id = unicode(fb_url).split('/')[-1] post = fb_api.get_object(fb_id) user = fb_api.get_object(post['from']['id']) user_picture = fb_api.get_object('%s/picture' % post['from']['id']) likes = fb_api.get_object('%s/likes' % fb_id, summary='true') comments = fb_api.get_object('%s/comments' % fb_id, summary='true') #shares = fb_api.get_object('%s/sharedposts' % fb_id) creation_date = datetime.strptime(post['created_time'],'%Y-%m-%dT%H:%M:%S+0000') creation_date = '%s %i' % (creation_date.strftime('%b'), creation_date.day) # https://developers.facebook.com/docs/graph-api/reference/v2.0/post facebook_posts.append({ 'id': post['id'], 'message': post['message'], 'link': { 'url': post['link'], 'name': post['name'], 'caption': (post['caption'] if 'caption' in post else None), 'description': post['description'], 'picture': post['picture'] }, 'from': { 'name': user['name'], 'link': user['link'], 'picture': user_picture['url'] }, 'likes': likes['summary']['total_count'], 'comments': comments['summary']['total_count'], #'shares': shares['summary']['total_count'], 'creation_date': creation_date }) # Render to JSON output = { 'tweets': tweets, 'facebook_posts': facebook_posts } with open('data/featured.json', 'w') as f: json.dump(output, f)
""" import app_config import json import logging import os import requests from fabric.api import local, require, task from pyquery import PyQuery as pq logging.basicConfig() logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) secrets = app_config.get_secrets() WEBHOOK = secrets['WEBHOOK'] @task def post_message(): message = check_page() if message: r = requests.post(WEBHOOK, data=json.dumps(message)) logger.info(r.text) def check_page(): attachments = [] last_title = get_last_title()
#!/usr/bin/env python from datetime import datetime import logging import os import requests from pyquery import PyQuery from app_config import get_secrets from models import ApiEntry, Article SECRETS = get_secrets() logging.basicConfig() logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) class HomepageScraper: url = 'http://npr.org' def __init__(self): self.run_time = datetime.utcnow() def scrape_homepage(self, **kwargs): """ Scrape! """ logger.info('Scraping homepage (start time: %s)' % self.run_time)
def boundaries(): context = make_context() context['name'] = 'Boundaries' context['id'] = context['name'] context['secrets'] = app_config.get_secrets() return make_response(render_template('boundaries.html', **context))
#!/usr/bin/env python import app_config import json import os import time from datetime import datetime from elections import AP from fabric.api import task from time import sleep SECRETS = app_config.get_secrets() # If main FTP server goes down, uncomment the next line #AP.FTP_HOSTNAME = 'electionsonline2.ap.org' def _generate_candidate_id(candidate, race): """ Makes an unique compound ID """ return '%s-%s' % (candidate.ap_polra_number, race.state_postal) @task def bootstrap(): init() update() @task def init(output_dir='data'): """
def load_images(): """ Downloads images from Baker and Taylor. Eschews the API for a magic URL pattern, which is faster. """ # Secrets. secrets = app_config.get_secrets() # Open the books JSON. with open('www/static-data/books.json', 'rb') as readfile: books = json.loads(readfile.read()) print "Start load_images(): %i books." % len(books) # Loop. for book in books: # Skip books with no title or ISBN. if book['title'] == "": continue if 'isbn' not in book or book['isbn'] == "": continue # Construct the URL with secrets and the ISBN. book_url = "http://imagesa.btol.com/ContentCafe/Jacket.aspx?UserID=%s&Password=%s&Return=T&Type=L&Value=%s" % ( secrets['BAKER_TAYLOR_USERID'], secrets['BAKER_TAYLOR_PASSWORD'], book['isbn']) # Request the image. r = requests.get(book_url) path = 'www/assets/cover/%s.jpg' % book['slug'] # Write the image to www using the slug as the filename. with open(path, 'wb') as writefile: writefile.write(r.content) file_size = os.path.getsize(path) if file_size < 10000: print u'LOG (%s): Image not available from Baker and Taylor, using NPR book page' % book['title'] url = 'http://www.npr.org/%s' % book['book_seamus_id'] npr_r = requests.get(url) soup = BeautifulSoup(npr_r.content) try: if book['title'] == 'The Three-Body Problem': alt_img_url = 'http://media.npr.org/assets/bakertaylor/covers/t/the-three-body-problem/9780765377067_custom-d83e0e334f348e6c52fe5da588ec3448921af64f-s600-c85.jpg' else: alt_img_url = soup.select('.bookedition .image img')[0].attrs.get('src').replace('s99', 's400') print 'LOG (%s): Getting alternate image from %s' % (book['title'], alt_img_url) alt_img_resp = requests.get(alt_img_url) with open(path, 'wb') as writefile: writefile.write(alt_img_resp.content) except IndexError: print u'ERROR (%s): Image not available on NPR book page either (%s)' % (book['title'], url) image = Image.open(path) image.save(path, optimize=True, quality=75) print "End."
def _post_to_tumblr(): """ Handles the POST to Tumblr. """ def strip_html(value): """ Strips HTML from a string. """ return re.compile(r'</?\S([^=]*=(\s*"[^"]*"|\s*\'[^\']*\'|\S*)|[^>])*?>', re.IGNORECASE).sub('', value) def strip_breaks(value): """ Converts newlines, returns and other breaks to <br/>. """ value = re.sub(r'\r\n|\r|\n', '\n', value) return value.replace('\n', do_mark_safe('<br/>')) # Request is a global. Import it down here where we need it. from flask import request # These should match the form fields. message = strip_html(request.form.get('message', None)) message = escape(message) message = strip_breaks(message) name = strip_html(request.form.get('signed_name', None)) email = strip_html(request.form.get('email', None)) context = { 'message': message, 'name': name, 'email': email, 'app_config': app_config } caption = render_template('caption.html', **context) secrets = app_config.get_secrets() t = Tumblpy( app_key=secrets['TUMBLR_APP_KEY'], app_secret=secrets['TUMBLR_APP_SECRET'], oauth_token=secrets['TUMBLR_OAUTH_TOKEN'], oauth_token_secret=secrets['TUMBLR_OAUTH_TOKEN_SECRET']) file_path = '/uploads/%s/%s_%s' % ( app_config.PROJECT_SLUG, str(time.mktime(datetime.datetime.now().timetuple())).replace('.', ''), secure_filename(request.files['image'].filename.replace(' ', '-')) ) with open('/var/www%s' % file_path, 'w') as f: f.write(request.files['image'].read()) params = { "type": "photo", "caption": caption, "tags": app_config.TUMBLR_TAGS, "source": "http://%s%s" % (app_config.SERVERS[0], file_path) } try: tumblr_post = t.post('post', blog_url=app_config.TUMBLR_URL, params=params) tumblr_url = u"http://%s/%s" % (app_config.TUMBLR_URL, tumblr_post['id']) logger.info('200 %s reader(%s %s) (times in EST)' % (tumblr_url, name, email)) return redirect(tumblr_url, code=301) except TumblpyError, e: logger.error('%s %s http://%s%s reader(%s %s) (times in EST)' % ( e.error_code, e.msg, app_config.SERVERS[0], file_path, name, email)) return 'TUMBLR ERROR'
def load_images(): """ Downloads images from Baker and Taylor. Eschews the API for a magic URL pattern, which is faster. """ # Secrets. secrets = app_config.get_secrets() # Open the books JSON. with open('www/static-data/books.json', 'rb') as readfile: books = json.loads(readfile.read()) print "Start load_images(): %i books." % len(books) always_use_npr_cover = set(app_config.ALWAYS_USE_NPR_COVER) # Loop. for book in books: # Skip books with no title or ISBN. if book['title'] == "": logger.warning('found book with no title') continue if 'isbn' not in book or book['isbn'] == "": logger.warning('This book has no isbn: %s' % book['title']) continue # Construct the URL with secrets and the ISBN. book_url = "http://imagesa.btol.com/ContentCafe/Jacket.aspx" params = { 'UserID': secrets['BAKER_TAYLOR_USERID'], 'Password': secrets['BAKER_TAYLOR_PASSWORD'], 'Value': book['isbn'], 'Return': 'T', 'Type': 'L' } # Request the image. r = requests.get(book_url, params=params) path = 'www/assets/cover' if not os.path.exists(path): os.makedirs(path) imagepath = '%s/%s.jpg' % (path, book['slug']) if os.path.exists(imagepath): logger.debug('image already downloaded for: %s' % book['slug']) # Write the image to www using the slug as the filename. with open(imagepath, 'wb') as writefile: writefile.write(r.content) file_size = os.path.getsize(imagepath) use_npr_book_page = ( file_size < 10000 or book['isbn'] in always_use_npr_cover ) if use_npr_book_page: msg = ('(%s): Image not available from Baker and Taylor, ' 'using NPR book page') % book['title'] logger.info(msg) try: alt_img_url = _get_npr_cover_img_url(book) msg = 'LOG (%s): Getting alternate image from %s' % ( book['title'], alt_img_url) logger.info(msg) alt_img_resp = requests.get(alt_img_url) with open(imagepath, 'wb') as writefile: writefile.write(alt_img_resp.content) except ValueError: msg = ( 'ERROR (%s): Image not available on NPR book page either' ) % (book['title']) logger.info(msg) image = Image.open(imagepath) image.save(imagepath, optimize=True, quality=75) logger.info("Load Images End.")