def scrapevideo_cmd(parser, parsed, args): if not parsed.quiet: parser.print_byline() video_url = parsed.video[0] data = scrapevideo(video_url, parsed.richard, 'object') if parsed.save: cfg = get_project_config() projectpath = cfg.get('project', 'projectpath') jsonpath = os.path.join(projectpath, 'json') if not os.path.exists(jsonpath): os.makedirs(jsonpath) fn = 'json/' + generate_filename(data['title']) + '.json' if os.path.exists(fn): err('It already exists!') return 1 with open(fn, 'w') as fp: fp.write(convert_to_json(data)) print 'Saved as {0}'.format(fn) else: print convert_to_json(data) return 0
def route_home(self, path): config = get_project_config() self.render_response( 200, 'home.html', { 'title': config.get('project', 'category'), 'data_files': load_json_files(config) })
def scrapevideo(ctx, quiet, save, video_url): """Fetches metadata for a video from a site.""" if not quiet: click.echo(VERSION) data = scrape_video(video_url)[0] if save: cfg = get_project_config() jsonpath = cfg.get('project', 'jsonpath') if not os.path.exists(jsonpath): os.makedirs(jsonpath) fn = generate_filename(data['title']) + '.json' full_path = os.path.join(jsonpath, fn) if os.path.exists(fn): raise click.ClickException(u'File "%s" already exists!' % fn) with open(full_path, 'w') as fp: fp.write(convert_to_json(data)) click.echo(u'Saved as {0}'.format(fn)) else: click.echo(convert_to_json(data))
def route_save(self, path): cfg = get_project_config() data_file = get_data(cfg, path[1]) if not data_file: return self.render_error(404) fn, data = data_file form_data = self.parse_form() reqs = get_video_requirements() for req in reqs: key = req['name'] if key in form_data: value = None if req['type'] == 'IntegerField': value = int(form_data[key].value) elif req['type'] == 'BooleanField': value = bool(int(form_data[key].value)) elif req['type'] in ('DateField', 'DateTimeField'): # TODO: Verify the data format value = form_data[key].value elif req['type'] in ('CharField', 'TextField'): value = form_data[key].value.replace('\r\n', '\n') elif req['type'] == 'SlugField': # TODO: Verify the data format. Maybe if there is # no slug field, we create it from the title? value = form_data[key].value elif req['type'] == 'TextArrayField': # Split the field on carriage returns and drop any # empty strings. value = form_data[key].value value = [ mem.strip() for mem in value.split('\n') if mem.strip() ] elif req['type'] in ('URLField'): value = form_data[key].value else: raise NotImplementedError data[key] = value else: if req['type'] in ('CharField', 'TextField'): data[key] = '' elif req['type'] == 'TextArrayField': data[key] = [] # TODO: What to do about other fields? Set to default? # Category is a special case. If it's empty, we should remove # it so verify works well. if 'category' in data and not data['category'].strip(): del data['category'] save_json_file(cfg, fn, data) return self.redirect('/edit/{0}'.format(fn))
def route_save(self, path): cfg = get_project_config() data_file = get_data(cfg, path[1]) if not data_file: return self.render_error(404) fn, data = data_file form_data = self.parse_form() reqs = get_video_requirements() for req in reqs: key = req['name'] if key in form_data: value = None if req['type'] == 'IntegerField': value = int(form_data[key].value) elif req['type'] == 'BooleanField': value = bool(int(form_data[key].value)) elif req['type'] in ('DateField', 'DateTimeField'): # TODO: Verify the data format value = form_data[key].value elif req['type'] in ('CharField', 'TextField'): value = form_data[key].value.replace('\r\n', '\n') elif req['type'] == 'SlugField': # TODO: Verify the data format. Maybe if there is # no slug field, we create it from the title? value = form_data[key].value elif req['type'] == 'TextArrayField': # Split the field on carriage returns and drop any # empty strings. value = form_data[key].value value = [mem.strip() for mem in value.split('\n') if mem.strip()] elif req['type'] in ('URLField'): value = form_data[key].value else: raise NotImplementedError data[key] = value else: if req['type'] in ('CharField', 'TextField'): data[key] = '' elif req['type'] == 'TextArrayField': data[key] = [] # TODO: What to do about other fields? Set to default? # Category is a special case. If it's empty, we should remove # it so verify works well. if 'category' in data and not data['category'].strip(): del data['category'] save_json_file(cfg, fn, data) return self.redirect('/edit/{0}'.format(fn))
def test_ini_config_creation(self, config): runner = CliRunner() with runner.isolated_filesystem(): path = os.getcwd() with pytest.raises(ConfigNotFound): get_project_config() # Create the project directory create_project_config_file(path, config) cfg = get_project_config() assert os.path.exists(config.file_name) tmp_dir = 'tmp' os.mkdir(tmp_dir) os.chdir(tmp_dir) cfg2 = get_project_config() os.chdir(path) assert cfg.get('project', 'jsonpath') == cfg2.get('project', 'jsonpath') with pytest.raises(NoOptionError): cfg.get('project', 'xmlpath')
def route_edit(self, path): cfg = get_project_config() data_file = get_data(cfg, path[1]) if not data_file: return self.render_error(404) fn, data = data_file reqs = get_video_requirements() # TODO: verify the data and add the errors to the fields? all_files = [filename for filename, _ in load_json_files(cfg)] fn_index = all_files.index(fn) prev_fn = all_files[fn_index - 1] if fn_index > 0 else '' next_fn = (all_files[fn_index + 1] if fn_index < len(all_files) - 1 else '') fields = [] category = cfg.get('project', 'category') for req in reqs: key = req['name'] if key == 'category' and category: fields.append( { 'name': req['name'], 'value': category }) else: fields.append( { 'name': req['name'], 'type': req['type'], 'choices': req['choices'], 'md': req['md'], 'value': data.get(key, '') }) self.render_response( 200, 'edit.html', { 'title': u'edit {0}'.format(data['title']), 'fn': fn, 'fields': fields, 'prev_fn': prev_fn, 'next_fn': next_fn })
def route_save(self, path): cfg = get_project_config() data_file = get_data(cfg, path[1]) if not data_file: return self.render_error(404) fn, data = data_file form_data = self.parse_form() reqs = get_video_requirements() for req in reqs: key = req['name'] if key in form_data: value = None if req['type'] == 'IntegerField': value = int(form_data[key].value) elif req['type'] == 'BooleanField': value = bool(int(form_data[key].value)) elif req['type'] in ('DateField', 'DateTimeField'): # TODO: Verify the data format value = form_data[key].value elif req['type'] in ('CharField', 'TextField'): value = form_data[key].value elif req['type'] == 'SlugField': # TODO: Verify the data format. Maybe if there is # no slug field, we create it from the title? value = form_data[key].value elif req['type'] in ('TextArrayField'): # Split the field on carriage returns and drop any # empty strings. value = [ mem.strip() for mem in form_data[key].value.split('\n') if mem.strip() ] data[key] = value else: if req['type'] in ('CharField', 'TextField'): data[key] = '' # TODO: What to do about other fields? Set to default? save_json_file(cfg, fn, data) return self.redirect('/edit/{0}'.format(fn[5:]))
def route_save(self, path): cfg = get_project_config() data_file = get_data(cfg, path[1]) if not data_file: return self.render_error(404) fn, data = data_file form_data = self.parse_form() reqs = get_video_requirements() for req in reqs: key = req['name'] if key in form_data: value = None if req['type'] == 'IntegerField': value = int(form_data[key].value) elif req['type'] == 'BooleanField': value = bool(int(form_data[key].value)) elif req['type'] in ('DateField', 'DateTimeField'): # TODO: Verify the data format value = form_data[key].value elif req['type'] in ('CharField', 'TextField'): value = form_data[key].value elif req['type'] == 'SlugField': # TODO: Verify the data format. Maybe if there is # no slug field, we create it from the title? value = form_data[key].value elif req['type'] in ('TextArrayField'): # Split the field on carriage returns and drop any # empty strings. value = [mem.strip() for mem in form_data[key].value.split('\n') if mem.strip()] data[key] = value else: if req['type'] in ('CharField', 'TextField'): data[key] = '' # TODO: What to do about other fields? Set to default? save_json_file(cfg, fn, data) return self.redirect('/edit/{0}'.format(fn[5:]))
def route_edit(self, path): cfg = get_project_config() data_file = get_data(cfg, path[1]) if not data_file: return self.render_error(404) fn, data = data_file reqs = get_video_requirements() # TODO: verify the data and add the errors to the fields? all_files = [filename for filename, _ in load_json_files(cfg)] fn_index = all_files.index(fn) prev_fn = all_files[fn_index - 1] if fn_index > 0 else '' next_fn = (all_files[fn_index + 1] if fn_index < len(all_files) - 1 else '') fields = [] category = cfg.get('project', 'category') for req in reqs: key = req['name'] if key == 'category' and category: fields.append({'name': req['name'], 'value': category}) else: fields.append({ 'name': req['name'], 'type': req['type'], 'choices': req['choices'], 'md': req['md'], 'value': data.get(key, '') }) self.render_response( 200, 'edit.html', { 'title': u'edit {0}'.format(data['title']), 'fn': fn, 'fields': fields, 'prev_fn': prev_fn, 'next_fn': next_fn })
'slug': '', 'speakers': speakers, 'source_url': 'https://www.youtube.com/watch?v={}'.format(video_id) } return item if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument("--maxresults", help="Max results", default=50) parser.add_argument("-c", "--channel", action='store_true', help="YouTube channel id") parser.add_argument("-p", "--playlist", action='store_true', help="YouTube playlist id") args = parser.parse_args() cfg = get_project_config() scraper = YouTubeScraper(cfg, max_results=args.maxresults) if args.channel: channel_id = get_from_config(cfg, 'channel_id', 'youtube') print("scraping channel {}".format(channel_id)) data = scraper.scrape_channel(channel_id) save_json_files(cfg, data) elif args.playlist: playlist_id = get_from_config(cfg, 'playlist_id', 'youtube') print("scraping playlist {}".format(playlist_id)) data = scraper.scrape_playlist(playlist_id) save_json_files(cfg, data)