def createproject(ctx, quiet, projectname): """Creates a new project.""" if not quiet: click.echo(VERSION) path = os.path.abspath(projectname) if os.path.exists(path): raise click.ClickException( u'{0} exists. Remove it and try again or try again with ' u'a different project name.'.format(path) ) # TODO: this kicks up errors. catch the errors and tell the user # something more useful click.echo(u'Creating directory {0}...'.format(path)) os.makedirs(path) conf_name = get_project_config_file_name() click.echo(u'Creating {0}...'.format(conf_name)) create_project_config_file(path) click.echo(u'{0} created.'.format(path)) click.echo(u'') click.echo(u'Now cd into the directory and edit the {0} file.'.format(conf_name)) click.echo(u'After you do that, you should put your project into version ' u'control. Srsly.')
def fetch(cfg, ctx, quiet, force): """Fetches videos and generates JSON files.""" if not quiet: click.echo(VERSION) jsonpath = cfg.get('project', 'jsonpath') # source_url -> filename source_map = dict( (item['source_url'], fn) for fn, item in load_json_files(cfg) ) if not os.path.exists(jsonpath): os.makedirs(jsonpath) try: url = cfg.get('project', 'url') except NoOptionError: url = '' if not url: raise click.ClickException( u'url not specified in {0} project config file.\n\n' u'Add "url = ..." to [project] section of {0} file.'.format( get_project_config_file_name()) ) click.echo(u'Scraping {0}...'.format(url)) click.echo(u'(This can take a *long* time with no indication of progress.)') videos = scrape_videos(url) click.echo(u'Found {0} videos...'.format(len(videos))) for i, video in enumerate(videos): if video['source_url'] in source_map and not force: click.echo(u'Skipping {0}... already exists.'.format( stringify(video['title']))) continue filename = generate_filename(video['title']) filename = '{index:04d}_{basename}.json'.format( index=i, basename=filename[:40]) click.echo(u'Created {0}... ({1})'.format( stringify(video['title']), filename)) with open(os.path.join(jsonpath, filename), 'w') as fp: fp.write(convert_to_json(video))
def fetch(cfg, ctx, quiet, force): """Fetches videos and generates JSON files.""" if not quiet: click.echo(VERSION) jsonpath = cfg.get('project', 'jsonpath') # source_url -> filename source_map = dict( (item['source_url'], fn) for fn, item in load_json_files(cfg) ) if not os.path.exists(jsonpath): os.makedirs(jsonpath) try: url = cfg.get('project', 'url') except ConfigParser.NoOptionError: url = '' if not url: raise click.ClickException( u'url not specified in {0} project config file.\n\n' u'Add "url = ..." to [project] section of {0} file.'.format( get_project_config_file_name()) ) click.echo(u'Scraping {0}...'.format(url)) click.echo(u'(This can take a *long* time with no indication of progress.)') videos = scrape_videos(url) click.echo(u'Found {0} videos...'.format(len(videos))) for i, video in enumerate(videos): if video['source_url'] in source_map and not force: click.echo(u'Skipping {0}... already exists.'.format( stringify(video['title']))) continue filename = generate_filename(video['title']) filename = '{index:04d}_{basename}.json'.format( index=i, basename=filename[:40]) click.echo(u'Created {0}... ({1})'.format( stringify(video['title']), filename)) with open(os.path.join(jsonpath, filename), 'w') as fp: fp.write(convert_to_json(video))
def pull(cfg, ctx, quiet, apikey): """Pulls data from a richard instance.""" if not quiet: click.echo(VERSION) username = get_from_config(cfg, 'username') api_url = get_from_config(cfg, 'api_url') cat_title = get_from_config(cfg, 'category') # Command line api_key overrides config-set api_key if not apikey: try: apikey = cfg.get('project', 'api_key') except NoOptionError: pass if not apikey: raise click.ClickException( u'Specify an api key either in {0}, on command line, ' u'or in API_KEY file.'.format(get_project_config_file_name()) ) if not username or not api_url or not cat_title or not apikey: raise click.ClickException(u'Missing username, api_url or api_key.') api = steve.restapi.API(api_url) all_categories = steve.restapi.get_content( api.category.get(username=username, api_key=apikey, limit=0)) cat = [cat_item for cat_item in all_categories['objects'] if cat_item['title'] == cat_title] if not cat: raise click.ClickException(u'Category "{0}" does not exist.'.format(cat_title)) # Get the category from the list of 1. cat = cat[0] click.echo('Retrieved category.') data = [] for counter, video_url in enumerate(cat['videos']): # Lame, but good enough for now. video_id = video_url.split('/')[-2] video_data = steve.restapi.get_content( api.video(video_id).get(username=username, api_key=apikey)) click.echo('Working on "{0}"'.format(video_data['slug'])) # Nix some tastypie bits from the data. for bad_key in ('resource_uri',): if bad_key in video_data: del video_data[bad_key] # Add id. video_data['id'] = video_id fn = 'json/{0:4d}_{1}.json'.format(counter, video_data['slug']) data.append((fn, video_data)) click.echo('Saving files....') save_json_files(cfg, data)
def push(cfg, ctx, quiet, apikey, update, overwrite, files): """Pushes metadata to a richard instance.""" if not quiet: click.echo(VERSION) # Get username, api_url and api_key. username = get_from_config(cfg, 'username') api_url = get_from_config(cfg, 'api_url') # Command line api_key overrides config-set api_key if not apikey: try: apikey = cfg.get('project', 'api_key') except NoOptionError: pass if not apikey: raise click.ClickException( u'Specify an api key either in {0}, on command line, ' u'or in API_KEY file.'.format(get_project_config_file_name()) ) if not username or not api_url or not apikey: raise click.ClickException(u'Missing username, api_url or api_key.') data = load_json_files(cfg) if files: data = [(fn, contents) for fn, contents in data if fn in files] # There are two modes: # # 1. User set category in configuration. Then the json files can # either have no category set or they have to have the same # category set. # # 2. User has NOT set category in configuration. Then the json # files must all have the category set. The categories can be # different. # # Go through and make sure there aren't any problems with # categories. all_categories = dict( [(cat['title'], cat) for cat in steve.richardapi.get_all_categories(api_url)]) try: category = cfg.get('project', 'category') category = category.strip() if category not in all_categories: raise click.ClickException( u'Category "{0}" does not exist on server. Build it there ' u'first.'.format(category) ) else: click.echo('Category {0} exists on site.'.format(category)) except NoOptionError: category = None errors = [] for fn, contents in data: if category is None: this_cat = contents.get('category') if not this_cat: errors.append( u'No category set in configuration and {0} has no ' u'category set.'.format(fn) ) elif this_cat != this_cat.strip(): errors.append( u'Category "{0}" has whitespace at beginning or ' u'end.'.format(this_cat) ) elif this_cat not in all_categories: errors.append( u'Category "{0}" does not exist on server. ' u'Build it there first.'.format(this_cat) ) else: this_cat = contents.get('category') if this_cat is not None and str(this_cat).strip() != category: errors.append( u'Category set in configuration ({0}), but {1} has ' u'different category ({2}).'.format(category, fn, this_cat) ) if update: for fn, contents in data: if 'id' not in contents: errors.append( u'id not in contents for "{0}".'.format(fn) ) if errors: raise click.ClickException('\n'.join(errors)) # Everything looks ok. So double-check with the user and push. click.echo('Pushing to: {0}'.format(api_url)) click.echo('Username: {0}'.format(username)) click.echo('api_key: {0}'.format(apikey)) click.echo('update?: {0}'.format(update)) click.echo('# videos: {0}'.format(len(data))) click.echo('Once you push, you can not undo it. Push for realz? Y/N') if not raw_input().strip().lower().startswith('y'): raise click.Abort() for fn, contents in data: contents['category'] = category or contents.get('category') if not update: # Nix any id field since that causes problems. if 'id' in contents: if not overwrite: click.echo(u'Skipping... already exists.') continue del contents['id'] click.echo('Pushing {0}'.format(fn)) try: vid = steve.richardapi.create_video(api_url, apikey, contents) if 'id' in vid: contents['id'] = vid['id'] click.echo(' Now has id {0}'.format(vid['id'])) else: click.echo(' Errors?: {0}'.format(vid), err=True) except steve.restapi.RestAPIException as exc: click.echo(' Error?: {0}'.format(exc), err=True) click.echo(' "{0}"'.format(exc.response.content), err=True) else: click.echo('Updating {0} "{1}" ({2})'.format( contents['id'], contents['title'], fn)) try: vid = steve.richardapi.update_video( api_url, apikey, contents['id'], contents) except steve.restapi.RestAPIException as exc: click.err(' Error?: {0}'.format(exc)) click.err(' "{0}"'.format(exc.response.content)) save_json_file(cfg, fn, contents)
def pull(cfg, ctx, quiet, apikey): """Pulls data from a richard instance.""" if not quiet: click.echo(VERSION) username = get_from_config(cfg, 'username') api_url = get_from_config(cfg, 'api_url') cat_title = get_from_config(cfg, 'category') # Command line api_key overrides config-set api_key if not apikey: try: apikey = cfg.get('project', 'api_key') except ConfigParser.NoOptionError: pass if not apikey: raise click.ClickException( u'Specify an api key either in {0}, on command line, ' u'or in API_KEY file.'.format(get_project_config_file_name()) ) if not username or not api_url or not cat_title or not apikey: raise click.ClickException(u'Missing username, api_url or api_key.') api = steve.restapi.API(api_url) all_categories = steve.restapi.get_content( api.category.get(username=username, api_key=apikey, limit=0)) cat = [cat_item for cat_item in all_categories['objects'] if cat_item['title'] == cat_title] if not cat: raise click.ClickException(u'Category "{0}" does not exist.'.format(cat_title)) # Get the category from the list of 1. cat = cat[0] click.echo('Retrieved category.') data = [] for counter, video_url in enumerate(cat['videos']): video_id = get_video_id(video_url) video_data = steve.restapi.get_content( api.video(video_id).get(username=username, api_key=apikey)) click.echo('Working on "{0}"'.format(video_data['slug'])) # Nix some tastypie bits from the data. for bad_key in ('resource_uri',): if bad_key in video_data: del video_data[bad_key] # Add id. video_data['id'] = video_id fn = 'json/{0:4d}_{1}.json'.format(counter, video_data['slug']) data.append((fn, video_data)) click.echo('Saving files....') save_json_files(cfg, data)
def push(cfg, ctx, quiet, apikey, update, overwrite, files): """Pushes metadata to a richard instance.""" if not quiet: click.echo(VERSION) # Get username, api_url and api_key. username = get_from_config(cfg, 'username') api_url = get_from_config(cfg, 'api_url') # Command line api_key overrides config-set api_key if not apikey: try: apikey = cfg.get('project', 'api_key') except ConfigParser.NoOptionError: pass if not apikey: raise click.ClickException( u'Specify an api key either in {0}, on command line, ' u'or in API_KEY file.'.format(get_project_config_file_name()) ) if not username or not api_url or not apikey: raise click.ClickException(u'Missing username, api_url or api_key.') data = load_json_files(cfg) if files: data = [(fn, contents) for fn, contents in data if fn in files] # There are two modes: # # 1. User set category in configuration. Then the json files can # either have no category set or they have to have the same # category set. # # 2. User has NOT set category in configuration. Then the json # files must all have the category set. The categories can be # different. # # Go through and make sure there aren't any problems with # categories. all_categories = dict( [(cat['title'], cat) for cat in steve.richardapi.get_all_categories(api_url)]) try: category = cfg.get('project', 'category') category = category.strip() if category not in all_categories: raise click.ClickException( u'Category "{0}" does not exist on server. Build it there ' u'first.'.format(category) ) else: click.echo('Category {0} exists on site.'.format(category)) except ConfigParser.NoOptionError: category = None errors = [] for fn, contents in data: if category is None: this_cat = contents.get('category') if not this_cat: errors.append( u'No category set in configuration and {0} has no ' u'category set.'.format(fn) ) elif this_cat != this_cat.strip(): errors.append( u'Category "{0}" has whitespace at beginning or ' u'end.'.format(this_cat) ) elif this_cat not in all_categories: errors.append( u'Category "{0}" does not exist on server. ' u'Build it there first.'.format(this_cat) ) else: this_cat = contents.get('category') if this_cat is not None and str(this_cat).strip() != category: errors.append( u'Category set in configuration ({0}), but {1} has ' u'different category ({2}).'.format(category, fn, this_cat) ) if update: for fn, contents in data: if 'id' not in contents: errors.append( u'id not in contents for "{0}".'.format(fn) ) if errors: raise click.ClickException('\n'.join(errors)) # Everything looks ok. So double-check with the user and push. click.echo('Pushing to: {0}'.format(api_url)) click.echo('Username: {0}'.format(username)) click.echo('api_key: {0}'.format(apikey)) click.echo('update?: {0}'.format(update)) click.echo('# videos: {0}'.format(len(data))) click.echo('Once you push, you can not undo it. Push for realz? Y/N') if not raw_input().strip().lower().startswith('y'): raise click.Abort() for fn, contents in data: contents['category'] = category or contents.get('category') if not update: # Nix any id field since that causes problems. if 'id' in contents: if not overwrite: click.echo(u'Skipping... already exists.') continue del contents['id'] click.echo('Pushing {0}'.format(fn)) try: vid = steve.richardapi.create_video(api_url, apikey, contents) if 'id' in vid: contents['id'] = vid['id'] click.echo(' Now has id {0}'.format(vid['id'])) else: click.echo(' Errors?: {0}'.format(vid), err=True) except steve.restapi.RestAPIException as exc: click.echo(' Error?: {0}'.format(exc), err=True) click.echo(' "{0}"'.format(exc.response.content), err=True) else: click.echo('Updating {0} "{1}" ({2})'.format( contents['id'], contents['title'], fn)) try: vid = steve.richardapi.update_video( api_url, apikey, contents['id'], contents) except steve.restapi.RestAPIException as exc: click.echo(' Error?: {0}'.format(exc), err=True) click.echo(' "{0}"'.format(exc.response.content), err=True) save_json_file(cfg, fn, contents)