def _templates_detail(slug): """ Renders a parent.html index with child.html embedded as iframe. """ from flask import request template_path = '%s/%s' % (app_config.TEMPLATES_PATH, slug) base_template_path = '%s/%s' % (app_config.TEMPLATES_PATH, '_base') # NOTE: Parent must load pym.js from same source as child to prevent version conflicts! context = make_context(asset_depth=2, root_path=template_path) context['slug'] = slug try: graphic_config = load_graphic_config(template_path, [base_template_path]) context.update(graphic_config.__dict__) if hasattr(graphic_config, 'COPY_GOOGLE_DOC_KEY') and graphic_config.COPY_GOOGLE_DOC_KEY: copy_path = '%s/%s.xlsx' % (template_path, slug) if request.args.get('refresh'): oauth.get_document(graphic_config.COPY_GOOGLE_DOC_KEY, copy_path) context['COPY'] = copytext.Copy(filename=copy_path) except IOError: pass return make_response(render_template('parent.html', **context))
def _graphics_detail(slug): """ Renders a parent.html index with child.html embedded as iframe. """ from flask import request graphic_path = '%s/%s' % (app_config.GRAPHICS_PATH, slug) # NOTE: Parent must load pym.js from same source as child to prevent # version conflicts! context = make_context(asset_depth=2, root_path=graphic_path) context['slug'] = slug context['type'] = 'graphics' # From previous commit context['var_name'] = slug.replace('-', '_') template = 'parent.html' if not os.path.exists('%s/%s/js/lib/pym.js' % (app_config.GRAPHICS_PATH, slug)): template = 'parent_old.html' try: graphic_config = load_graphic_config(graphic_path) context.update(graphic_config.__dict__) if hasattr(graphic_config, 'COPY_GOOGLE_DOC_KEY') and graphic_config.COPY_GOOGLE_DOC_KEY: copy_path = '%s/%s.xlsx' % (graphic_path, slug) if request.args.get('refresh'): oauth.get_document(graphic_config.COPY_GOOGLE_DOC_KEY, copy_path) context['COPY'] = copytext.Copy(filename=copy_path) except IOError: pass return make_response(render_template(template, **context))
def _graphics_detail(slug): """ Renders a parent.html index with child.html embedded as iframe. """ from flask import request graphic_path = '%s/%s' % (app_config.GRAPHICS_PATH, slug) # NOTE: Parent must load pym.js from same source as child to prevent version conflicts! context = make_context(asset_depth=2, root_path=graphic_path) context['slug'] = slug template = 'parent.html' try: graphic_config = load_graphic_config(graphic_path) context.update(graphic_config.__dict__) if hasattr(graphic_config, 'COPY_GOOGLE_DOC_KEY') and graphic_config.COPY_GOOGLE_DOC_KEY: copy_path = '%s/%s.xlsx' % (graphic_path, slug) if request.args.get('refresh'): oauth.get_document(graphic_config.COPY_GOOGLE_DOC_KEY, copy_path) context['COPY'] = copytext.Copy(filename=copy_path) except IOError: pass return make_response(render_template(template, **context))
def _factcheck(): """ Liveblog only contains published posts """ get_document(app_config.COPY_GOOGLE_DOC_KEY, app_config.COPY_PATH) context = get_factcheck_context() return make_response(render_template('factcheck.html', **context))
def scrape_and_load(self, path=app_config.STORIES_PATH): if not POCKY_TRACKER: return get_document(POCKY_TRACKER, path) raw_stories = self.scrape_spreadsheet() stories = self.write(stories=raw_stories, team=self.source['team']) return stories
def update_songs(verify='true'): get_document(app_config.SONGS_GOOGLE_DOC_KEY, app_config.SONGS_DATA_PATH) data = copytext.Copy(app_config.SONGS_DATA_PATH) output = process_songs(data, verify == 'true') with codecs.open('data/songs.json', 'w', 'utf-8') as f: json.dump(output, f)
def scrape_spreadsheet(): """ Scrape 'Did we touch it?' spreadsheet """ db = dataset.connect(app_config.POSTGRES_URL) get_document(app_config.STORIES_GOOGLE_DOC_KEY, app_config.STORIES_PATH) scraper = SpreadsheetScraper() stories = scraper.scrape_spreadsheet(app_config.STORIES_PATH) scraper.write(db, stories)
def __init__(self, *args, **kwargs): try: get_document(POCKY_TRACKER, DATA_PATH) spreadsheet = copytext.Copy(DATA_PATH) self.data = data = spreadsheet['reviews'] except KeyError: pass super(NPRPocky, self).__init__(*args, **kwargs)
def _templates_detail(slug): """ Renders a parent.html index with child.html embedded as iframe. """ from flask import request template_path = '%s/%s' % (app_config.TEMPLATES_PATH, slug) base_template_path = '%s/%s' % (app_config.TEMPLATES_PATH, '_base') # NOTE: Parent must load pym.js from same source as child to prevent version conflicts! context = make_context(asset_depth=2, root_path=template_path) context['slug'] = slug try: graphic_config = load_graphic_config(template_path, [base_template_path]) context.update(graphic_config.__dict__) if hasattr(graphic_config, 'COPY_GOOGLE_DOC_KEY') and graphic_config.COPY_GOOGLE_DOC_KEY: copy_path = '%s/%s.xlsx' % (template_path, slug) if request.args.get('refresh'): oauth.get_document(graphic_config.COPY_GOOGLE_DOC_KEY, copy_path) context['COPY'] = copytext.Copy(filename=copy_path) if hasattr(graphic_config, 'AIRTABLE_ENDPOINTS') and graphic_config.AIRTABLE_ENDPOINTS and request.args.get('refresh'): for airtable_endpoint in graphic_config.AIRTABLE_ENDPOINTS: copy_path = '%s/%s.json' % (template_path, airtable_endpoint['name']) results = [] url = airtable_endpoint['url'] print 'downloading %s' % airtable_endpoint['name'] while True: response = requests.get(url) response.raise_for_status() data = json.loads(response.text) results = results + data['records'] if 'offset' in data: offset = data['offset'] url = '%s&offset=%s' % (airtable_endpoint['url'], offset) else: break print '%s records found' % len(results) with open(copy_path, 'w+') as outfile: json.dump(results, outfile) except IOError: pass return make_response(render_template('parent.html', **context))
def download_google_sheet(slug, graphic_config): """ Downloads a Google Doc as an .xlsx file. """ graphic_path = '%s/%s' % (app_config.GRAPHICS_PATH, slug) if not hasattr(graphic_config, 'COPY_GOOGLE_DOC_KEY') or not graphic_config.COPY_GOOGLE_DOC_KEY: print 'COPY_GOOGLE_DOC_KEY is not defined in %s/graphic_config.py.' % slug return print 'downloading google sheet\n%s' % graphic_config.COPY_GOOGLE_DOC_KEY copy_path = os.path.join(graphic_path, '%s.xlsx' % slug) get_document(graphic_config.COPY_GOOGLE_DOC_KEY, copy_path)
def render_playgrounds(playgrounds=None, compiled_includes=[]): """ Render the playgrounds pages. """ from flask import g, url_for get_document(app_config.COPY_GOOGLE_DOC_KEY, app_config.COPY_PATH) less() jst() if not playgrounds: playgrounds = get_active_playgrounds() slugs = [p.slug for p in playgrounds] app_config_js() updated_paths = [] for slug in slugs: # Silly fix because url_for require a context with app.app.test_request_context(): path = url_for('_playground', playground_slug=slug) with app.app.test_request_context(path=path): print 'Rendering %s' % path g.compile_includes = True g.compiled_includes = compiled_includes view = app.__dict__['_playground'] content = view(slug) compiled_includes = g.compiled_includes path = '.playgrounds_html%s' % path # Ensure path exists head = os.path.split(path)[0] try: os.makedirs(head) except OSError: pass with open(path, 'w') as f: f.write(content.encode('utf-8')) updated_paths.append(path) return updated_paths
def update(): """ Downloads a Google Doc as an Excel file. """ if app_config.COPY_GOOGLE_DOC_KEY == None: print colored('You have set COPY_GOOGLE_DOC_KEY to None. If you want to use a Google Sheet, set COPY_GOOGLE_DOC_KEY to the key of your sheet in app_config.py', 'blue') return credentials = get_credentials() if not credentials: print colored('No Google OAuth credentials file found.', 'yellow') print colored('Run `fab app` and visit `http://localhost:8000` to generate credentials.', 'yellow') return get_document(app_config.COPY_GOOGLE_DOC_KEY, app_config.COPY_PATH)
def update(): """ Downloads a Google Doc as an Excel file. """ if app_config.COPY_GOOGLE_DOC_KEY == None: logger.warn('You have set COPY_GOOGLE_DOC_KEY to None. If you want to use a Google Sheet, set COPY_GOOGLE_DOC_KEY to the key of your sheet in app_config.py') return credentials = get_credentials() if not credentials: print logger.warn('No Google OAuth credentials file found.') print logger.warn('Run `fab app` and visit `http://localhost:8000` to generate credentials.') return get_document(app_config.COPY_GOOGLE_DOC_KEY, app_config.COPY_PATH)
def _graphics_detail(slug): """ Renders a parent.html index with child.html embedded as iframe. """ from flask import request, g alt_path = getattr(g, 'alt_path', None) if alt_path: graphic_path = alt_path else: graphic_path = '%s/%s' % (app_config.GRAPHICS_PATH, slug) # NOTE: Parent must load pym.js from same source as child to prevent version conflicts! context = make_context(asset_depth=2, root_path=graphic_path) context['slug'] = slug context['var_name'] = slug.replace('-', '_') # Use local_pym for legacy graphics local_pym = getattr(g, 'local_pym', None) context['LOCAL_PYM'] = local_pym # warning message custom_location = getattr(g, 'custom_location', None) context['CUSTOM_LOCATION'] = custom_location template = 'parent.html' try: graphic_config = load_graphic_config(graphic_path) context.update(graphic_config.__dict__) if hasattr(graphic_config, 'COPY_GOOGLE_DOC_KEY') and graphic_config.COPY_GOOGLE_DOC_KEY: copy_path = '%s/%s.xlsx' % (graphic_path, slug) if request.args.get('refresh'): oauth.get_document(graphic_config.COPY_GOOGLE_DOC_KEY, copy_path) context['COPY'] = copytext.Copy(filename=copy_path) except IOError: pass try: env = Environment(loader=FileSystemLoader(graphic_path)) template = env.get_template('parent.html') return make_response(template.render(**context)) except TemplateNotFound: return make_response(render_template(template, **context))
def download_copy(slug): """ Downloads a Google Doc as an .xlsx file. """ graphic_path = '%s/%s' % (app_config.GRAPHICS_PATH, slug) try: graphic_config = load_graphic_config(graphic_path) except IOError: print '%s/graphic_config.py does not exist.' % slug return if not hasattr(graphic_config, 'COPY_GOOGLE_DOC_KEY') or not graphic_config.COPY_GOOGLE_DOC_KEY: print 'COPY_GOOGLE_DOC_KEY is not defined in %s/graphic_config.py.' % slug return copy_path = os.path.join(graphic_path, '%s.xlsx' % slug) get_document(graphic_config.COPY_GOOGLE_DOC_KEY, copy_path)
def update(): """ Downloads a Google Doc as an Excel file. """ if app_config.COPY_GOOGLE_DOC_KEY == None: print colored( 'You have set COPY_GOOGLE_DOC_KEY to None. If you want to use a Google Sheet, set COPY_GOOGLE_DOC_KEY to the key of your sheet in app_config.py', 'blue') return cred_file = os.path.expanduser(app_config.GOOGLE_OAUTH_CREDENTIALS_PATH) if not os.path.isfile(cred_file): print colored('No Google OAuth credentials file found.', 'yellow') print colored( 'Run `fab app` and visit `http://localhost:8000` to generate credentials.', 'yellow') return get_document(app_config.COPY_GOOGLE_DOC_KEY, app_config.COPY_PATH)
def update_copy(): """ Downloads a Google Doc as an Excel file. """ print "DOING THINGS" if app_config.COPY_GOOGLE_DOC_KEY == None: print colored( "You have set COPY_GOOGLE_DOC_KEY to None. If you want to use a Google Sheet, set COPY_GOOGLE_DOC_KEY to the key of your sheet in app_config.py", "blue", ) return credentials = get_credentials() if not credentials: print colored("No Google OAuth credentials file found.", "yellow") print colored("Run `fab app` and visit `http://localhost:8000` to generate credentials.", "yellow") return get_document(app_config.COPY_GOOGLE_DOC_KEY, app_config.COPY_PATH)
def get_graphic_template_variables(path, graphic_number): """ Generates the template variables for each graphic """ slug, abspath = utils.parse_path(path) graphic_path = '%s/%s' % (abspath, slug) ## Get Spreadsheet Path try: graphic_config = load_graphic_config(graphic_path) except IOError: print '%s/graphic_config.py does not exist.' % slug return if not hasattr( graphic_config, 'COPY_GOOGLE_DOC_KEY') or not graphic_config.COPY_GOOGLE_DOC_KEY: print 'COPY_GOOGLE_DOC_KEY is not defined in %s/graphic_config.py.' % slug return ## Generate Links From Slug spreadsheet_id = graphic_config.COPY_GOOGLE_DOC_KEY app_id = slug ## Update Spreadsheet copy_path = os.path.join(graphic_path, '%s.xlsx' % slug) get_document(graphic_config.COPY_GOOGLE_DOC_KEY, copy_path) ## Get Sheet Data copy = copytext.Copy(filename=copy_path) sheet = copy['labels'] note = { "spreadsheet_id": spreadsheet_id, "app_id": app_id, "graphic_number": graphic_number + 1, "sheet": sheet, } return note
def update(): """ Downloads a Google Doc as an Excel file. """ if app_config.COPY_GOOGLE_DOC_KEY == None: logger.warn( 'You have set COPY_GOOGLE_DOC_KEY to None. If you want to use a Google Sheet, set COPY_GOOGLE_DOC_KEY to the key of your sheet in app_config.py' ) return credentials = get_credentials() if not credentials: print logger.warn('No Google OAuth credentials file found.') print logger.warn( 'Run `fab app` and visit `http://localhost:8000` to generate credentials.' ) return get_document(app_config.COPY_GOOGLE_DOC_KEY, app_config.COPY_PATH) get_document(app_config.NAVBAR_GOOGLE_DOC_KEY, app_config.NAVBAR_PATH) get_document(app_config.BRIEFING_GOOGLE_DOC_KEY, app_config.BRIEFING_PATH) with open('www/data/extra_data/state-briefings.json', 'w') as f: copy = copytext.Copy(app_config.BRIEFING_PATH) output = copy.json() f.write(output)
def get_graphic_template_variables(path, graphic_number): """ Generates the template variables for each graphic """ slug, abspath = utils.parse_path(path) graphic_path = '%s/%s' % (abspath, slug) ## Get Spreadsheet Path try: graphic_config = load_graphic_config(graphic_path) except IOError: print '%s/graphic_config.py does not exist.' % slug return if not hasattr(graphic_config, 'COPY_GOOGLE_DOC_KEY') or not graphic_config.COPY_GOOGLE_DOC_KEY: print 'COPY_GOOGLE_DOC_KEY is not defined in %s/graphic_config.py.' % slug return ## Generate Links From Slug spreadsheet_id = graphic_config.COPY_GOOGLE_DOC_KEY app_id = slug ## Update Spreadsheet copy_path = os.path.join(graphic_path, '%s.xlsx' % slug) get_document(graphic_config.COPY_GOOGLE_DOC_KEY, copy_path) ## Get Sheet Data copy = copytext.Copy(filename=copy_path) sheet = copy['labels'] note = { "spreadsheet_id": spreadsheet_id, "app_id": app_id, "graphic_number": graphic_number + 1, "sheet": sheet, } return note
def update(): """ Downloads a Google Doc as an Excel file. """ get_document(env.copytext_key, 'data/%s.xlsx' % env.copytext_slug)
def update(): """ Downloads a Google Doc as an Excel file. """ get_document(app_config.COPY_GOOGLE_DOC_KEY, app_config.COPY_PATH) get_document(app_config.DATA_GOOGLE_DOC_KEY, app_config.DATA_PATH)
def update_calendar(): """ Download calendar file. """ get_document(app_config.CALENDAR_GOOGLE_DOC_KEY, app_config.CALENDAR_PATH)
def update_copytext(): """ Downloads a Google Doc as an Excel file. """ get_document(app_config.COPY_GOOGLE_DOC_KEY, app_config.COPY_PATH)
def update_copy(): """ Fetches the latest Google Doc and updates local JSON. """ get_document(app_config.COPY_GOOGLE_DOC_KEY, app_config.COPY_PATH)
def update(): """ Downloads a Google Doc as an Excel file. """ if app_config.COPY_GOOGLE_DOC_KEY == None: logger.warn( 'You have set COPY_GOOGLE_DOC_KEY to None. If you want to use a Google Sheet, set COPY_GOOGLE_DOC_KEY to the key of your sheet in app_config.py' ) return credentials = get_credentials() if not credentials: print logger.warn('No Google OAuth credentials file found.') print logger.warn( 'Run `fab app` and visit `http://localhost:8000` to generate credentials.' ) return get_document(app_config.COPY_GOOGLE_DOC_KEY, app_config.COPY_PATH) with open('www/js/songs.json', 'w') as f: sheet = copytext.Copy(app_config.COPY_PATH) all_lists = list(sheet['best_lists']) + list(sheet['deeper_lists']) all_songs = {} for row in all_lists: if row['slug']: slug = row['slug'].replace('-', '_') songs = sheet[slug] for song in songs: if all_songs.get(song['song_slug']): continue song_obj = {} song_obj['sort'] = song['sort'] song_obj['artist'] = song['artist'] song_obj['title'] = song['title'] if song['song_slug'] == 'big-thief-masterpiece' and song[ 'type'] == 'album': song_obj['song_slug'] = 'big-thief-masterpiece-album' elif song['song_slug'] == 'big-thief-masterpiece' and song[ 'type'] == 'song': song_obj['song_slug'] = 'big-thief-masterpiece-song' else: song_obj['song_slug'] = song['song_slug'] song_obj['description'] = song['description'] song_obj['art'] = song['art'] song_obj['author'] = song['author'] song_obj['author_twitter'] = song['author_twitter'] song_obj['affiliation'] = song['affiliation'] song_obj['explicit'] = song['explicit'] song_obj['filed_under'] = song['filed_under'] song_obj['type'] = song['type'] song_obj['smarturl'] = song['smarturl'] song_obj['embed'] = song['embed'].replace( 'watch?v=', 'embed/') song_obj['embed_type'] = song['embed_type'] song_obj['embed_caption'] = song['embed_caption'] all_songs[song_obj['song_slug']] = song_obj output = json.dumps(all_songs) f.write(output)
def load_spreadsheet(source): get_document(source['doc_key'], app_config.STORIES_PATH) scraper = SpreadsheetScraper() stories = scraper.scrape_spreadsheet(app_config.STORIES_PATH) new_stories = scraper.write(stories, team=source['team']) return new_stories
def scrape_and_load(self, path=app_config.STORIES_PATH): get_document(self.source['doc_key'], path) raw_stories = self.scrape_spreadsheet() stories = self.write(stories=raw_stories, team=self.source['team']) return stories