Esempio n. 1
0
def createproject(ctx, quiet, projectname):
    """Creates a new project."""
    if not quiet:
        click.echo(VERSION)

    path = os.path.abspath(projectname)
    if os.path.exists(path):
        raise click.ClickException(
            u'{0} exists. Remove it and try again or try again with '
            u'a different project name.'.format(path)
        )

    # TODO: this kicks up errors. catch the errors and tell the user
    # something more useful
    click.echo(u'Creating directory {0}...'.format(path))
    os.makedirs(path)

    conf_name = get_project_config_file_name()
    click.echo(u'Creating {0}...'.format(conf_name))
    create_project_config_file(path)

    click.echo(u'{0} created.'.format(path))
    click.echo(u'')

    click.echo(u'Now cd into the directory and edit the {0} file.'.format(conf_name))
    click.echo(u'After you do that, you should put your project into version '
               u'control. Srsly.')
Esempio n. 2
0
def createproject(ctx, quiet, projectname):
    """Creates a new project."""
    if not quiet:
        click.echo(VERSION)

    path = os.path.abspath(projectname)
    if os.path.exists(path):
        raise click.ClickException(
            u'{0} exists. Remove it and try again or try again with '
            u'a different project name.'.format(path)
        )

    # TODO: this kicks up errors. catch the errors and tell the user
    # something more useful
    click.echo(u'Creating directory {0}...'.format(path))
    os.makedirs(path)

    conf_name = get_project_config_file_name()
    click.echo(u'Creating {0}...'.format(conf_name))
    create_project_config_file(path)

    click.echo(u'{0} created.'.format(path))
    click.echo(u'')

    click.echo(u'Now cd into the directory and edit the {0} file.'.format(conf_name))
    click.echo(u'After you do that, you should put your project into version '
               u'control. Srsly.')
Esempio n. 3
0
def fetch(cfg, ctx, quiet, force):
    """Fetches videos and generates JSON files."""
    if not quiet:
        click.echo(VERSION)

    jsonpath = cfg.get('project', 'jsonpath')

    # source_url -> filename
    source_map = dict(
        (item['source_url'], fn)
        for fn, item in load_json_files(cfg)
    )

    if not os.path.exists(jsonpath):
        os.makedirs(jsonpath)

    try:
        url = cfg.get('project', 'url')
    except NoOptionError:
        url = ''

    if not url:
        raise click.ClickException(
            u'url not specified in {0} project config file.\n\n'
            u'Add "url = ..." to [project] section of {0} file.'.format(
                get_project_config_file_name())
        )

    click.echo(u'Scraping {0}...'.format(url))
    click.echo(u'(This can take a *long* time with no indication of progress.)')
    videos = scrape_videos(url)

    click.echo(u'Found {0} videos...'.format(len(videos)))
    for i, video in enumerate(videos):
        if video['source_url'] in source_map and not force:
            click.echo(u'Skipping {0}... already exists.'.format(
                stringify(video['title'])))
            continue

        filename = generate_filename(video['title'])
        filename = '{index:04d}_{basename}.json'.format(
            index=i, basename=filename[:40])

        click.echo(u'Created {0}... ({1})'.format(
            stringify(video['title']), filename))

        with open(os.path.join(jsonpath, filename), 'w') as fp:
            fp.write(convert_to_json(video))
Esempio n. 4
0
def fetch(cfg, ctx, quiet, force):
    """Fetches videos and generates JSON files."""
    if not quiet:
        click.echo(VERSION)

    jsonpath = cfg.get('project', 'jsonpath')

    # source_url -> filename
    source_map = dict(
        (item['source_url'], fn)
        for fn, item in load_json_files(cfg)
    )

    if not os.path.exists(jsonpath):
        os.makedirs(jsonpath)

    try:
        url = cfg.get('project', 'url')
    except ConfigParser.NoOptionError:
        url = ''

    if not url:
        raise click.ClickException(
            u'url not specified in {0} project config file.\n\n'
            u'Add "url = ..." to [project] section of {0} file.'.format(
                get_project_config_file_name())
        )

    click.echo(u'Scraping {0}...'.format(url))
    click.echo(u'(This can take a *long* time with no indication of progress.)')
    videos = scrape_videos(url)

    click.echo(u'Found {0} videos...'.format(len(videos)))
    for i, video in enumerate(videos):
        if video['source_url'] in source_map and not force:
            click.echo(u'Skipping {0}... already exists.'.format(
                stringify(video['title'])))
            continue

        filename = generate_filename(video['title'])
        filename = '{index:04d}_{basename}.json'.format(
            index=i, basename=filename[:40])

        click.echo(u'Created {0}... ({1})'.format(
            stringify(video['title']), filename))

        with open(os.path.join(jsonpath, filename), 'w') as fp:
            fp.write(convert_to_json(video))
Esempio n. 5
0
def pull(cfg, ctx, quiet, apikey):
    """Pulls data from a richard instance."""
    if not quiet:
        click.echo(VERSION)

    username = get_from_config(cfg, 'username')
    api_url = get_from_config(cfg, 'api_url')
    cat_title = get_from_config(cfg, 'category')

    # Command line api_key overrides config-set api_key
    if not apikey:
        try:
            apikey = cfg.get('project', 'api_key')
        except NoOptionError:
            pass
    if not apikey:
        raise click.ClickException(
            u'Specify an api key either in {0}, on command line, '
            u'or in API_KEY file.'.format(get_project_config_file_name())
        )

    if not username or not api_url or not cat_title or not apikey:
        raise click.ClickException(u'Missing username, api_url or api_key.')

    api = steve.restapi.API(api_url)

    all_categories = steve.restapi.get_content(
        api.category.get(username=username, api_key=apikey,
                         limit=0))
    cat = [cat_item for cat_item in all_categories['objects']
           if cat_item['title'] == cat_title]

    if not cat:
        raise click.ClickException(u'Category "{0}" does not exist.'.format(cat_title))

    # Get the category from the list of 1.
    cat = cat[0]

    click.echo('Retrieved category.')

    data = []

    for counter, video_url in enumerate(cat['videos']):
        # Lame, but good enough for now.
        video_id = video_url.split('/')[-2]

        video_data = steve.restapi.get_content(
            api.video(video_id).get(username=username,
                                    api_key=apikey))

        click.echo('Working on "{0}"'.format(video_data['slug']))

        # Nix some tastypie bits from the data.
        for bad_key in ('resource_uri',):
            if bad_key in video_data:
                del video_data[bad_key]

        # Add id.
        video_data['id'] = video_id

        fn = 'json/{0:4d}_{1}.json'.format(counter, video_data['slug'])
        data.append((fn, video_data))

    click.echo('Saving files....')
    save_json_files(cfg, data)
Esempio n. 6
0
def push(cfg, ctx, quiet, apikey, update, overwrite, files):
    """Pushes metadata to a richard instance."""
    if not quiet:
        click.echo(VERSION)

    # Get username, api_url and api_key.

    username = get_from_config(cfg, 'username')
    api_url = get_from_config(cfg, 'api_url')

    # Command line api_key overrides config-set api_key
    if not apikey:
        try:
            apikey = cfg.get('project', 'api_key')
        except NoOptionError:
            pass
    if not apikey:
        raise click.ClickException(
            u'Specify an api key either in {0}, on command line, '
            u'or in API_KEY file.'.format(get_project_config_file_name())
        )

    if not username or not api_url or not apikey:
        raise click.ClickException(u'Missing username, api_url or api_key.')

    data = load_json_files(cfg)

    if files:
        data = [(fn, contents) for fn, contents in data if fn in files]

    # There are two modes:
    #
    # 1. User set category in configuration. Then the json files can
    #    either have no category set or they have to have the same
    #    category set.
    #
    # 2. User has NOT set category in configuration. Then the json
    #    files must all have the category set. The categories can be
    #    different.
    #
    # Go through and make sure there aren't any problems with
    # categories.

    all_categories = dict(
        [(cat['title'], cat)
         for cat in steve.richardapi.get_all_categories(api_url)])

    try:
        category = cfg.get('project', 'category')
        category = category.strip()
        if category not in all_categories:
            raise click.ClickException(
                u'Category "{0}" does not exist on server. Build it there '
                u'first.'.format(category)
            )
        else:
            click.echo('Category {0} exists on site.'.format(category))
    except NoOptionError:
        category = None

    errors = []
    for fn, contents in data:
        if category is None:
            this_cat = contents.get('category')
            if not this_cat:
                errors.append(
                    u'No category set in configuration and {0} has no '
                    u'category set.'.format(fn)
                )
            elif this_cat != this_cat.strip():
                errors.append(
                    u'Category "{0}" has whitespace at beginning or '
                    u'end.'.format(this_cat)
                )
            elif this_cat not in all_categories:
                errors.append(
                    u'Category "{0}" does not exist on server. '
                    u'Build it there first.'.format(this_cat)
                )

        else:
            this_cat = contents.get('category')
            if this_cat is not None and str(this_cat).strip() != category:
                errors.append(
                    u'Category set in configuration ({0}), but {1} has '
                    u'different category ({2}).'.format(category, fn, this_cat)
                )

    if update:
        for fn, contents in data:
            if 'id' not in contents:
                errors.append(
                    u'id not in contents for "{0}".'.format(fn)
                )

    if errors:
        raise click.ClickException('\n'.join(errors))

    # Everything looks ok. So double-check with the user and push.

    click.echo('Pushing to: {0}'.format(api_url))
    click.echo('Username:   {0}'.format(username))
    click.echo('api_key:    {0}'.format(apikey))
    click.echo('update?:    {0}'.format(update))
    click.echo('# videos:   {0}'.format(len(data)))
    click.echo('Once you push, you can not undo it. Push for realz? Y/N')
    if not raw_input().strip().lower().startswith('y'):
        raise click.Abort()

    for fn, contents in data:
        contents['category'] = category or contents.get('category')

        if not update:
            # Nix any id field since that causes problems.
            if 'id' in contents:
                if not overwrite:
                    click.echo(u'Skipping... already exists.')
                    continue
                del contents['id']

            click.echo('Pushing {0}'.format(fn))
            try:
                vid = steve.richardapi.create_video(api_url, apikey, contents)

                if 'id' in vid:
                    contents['id'] = vid['id']
                    click.echo('   Now has id {0}'.format(vid['id']))
                else:
                    click.echo('   Errors?: {0}'.format(vid), err=True)
            except steve.restapi.RestAPIException as exc:
                click.echo('   Error?: {0}'.format(exc), err=True)
                click.echo('   "{0}"'.format(exc.response.content), err=True)

        else:
            click.echo('Updating {0} "{1}" ({2})'.format(
                contents['id'], contents['title'], fn))
            try:
                vid = steve.richardapi.update_video(
                    api_url, apikey, contents['id'], contents)
            except steve.restapi.RestAPIException as exc:
                click.err('   Error?: {0}'.format(exc))
                click.err('   "{0}"'.format(exc.response.content))

        save_json_file(cfg, fn, contents)
Esempio n. 7
0
def pull(cfg, ctx, quiet, apikey):
    """Pulls data from a richard instance."""
    if not quiet:
        click.echo(VERSION)

    username = get_from_config(cfg, 'username')
    api_url = get_from_config(cfg, 'api_url')
    cat_title = get_from_config(cfg, 'category')

    # Command line api_key overrides config-set api_key
    if not apikey:
        try:
            apikey = cfg.get('project', 'api_key')
        except ConfigParser.NoOptionError:
            pass
    if not apikey:
        raise click.ClickException(
            u'Specify an api key either in {0}, on command line, '
            u'or in API_KEY file.'.format(get_project_config_file_name())
        )

    if not username or not api_url or not cat_title or not apikey:
        raise click.ClickException(u'Missing username, api_url or api_key.')

    api = steve.restapi.API(api_url)

    all_categories = steve.restapi.get_content(
        api.category.get(username=username, api_key=apikey,
                         limit=0))
    cat = [cat_item for cat_item in all_categories['objects']
           if cat_item['title'] == cat_title]

    if not cat:
        raise click.ClickException(u'Category "{0}" does not exist.'.format(cat_title))

    # Get the category from the list of 1.
    cat = cat[0]

    click.echo('Retrieved category.')

    data = []

    for counter, video_url in enumerate(cat['videos']):
        video_id = get_video_id(video_url)

        video_data = steve.restapi.get_content(
            api.video(video_id).get(username=username,
                                    api_key=apikey))

        click.echo('Working on "{0}"'.format(video_data['slug']))

        # Nix some tastypie bits from the data.
        for bad_key in ('resource_uri',):
            if bad_key in video_data:
                del video_data[bad_key]

        # Add id.
        video_data['id'] = video_id

        fn = 'json/{0:4d}_{1}.json'.format(counter, video_data['slug'])
        data.append((fn, video_data))

    click.echo('Saving files....')
    save_json_files(cfg, data)
Esempio n. 8
0
def push(cfg, ctx, quiet, apikey, update, overwrite, files):
    """Pushes metadata to a richard instance."""
    if not quiet:
        click.echo(VERSION)

    # Get username, api_url and api_key.

    username = get_from_config(cfg, 'username')
    api_url = get_from_config(cfg, 'api_url')

    # Command line api_key overrides config-set api_key
    if not apikey:
        try:
            apikey = cfg.get('project', 'api_key')
        except ConfigParser.NoOptionError:
            pass
    if not apikey:
        raise click.ClickException(
            u'Specify an api key either in {0}, on command line, '
            u'or in API_KEY file.'.format(get_project_config_file_name())
        )

    if not username or not api_url or not apikey:
        raise click.ClickException(u'Missing username, api_url or api_key.')

    data = load_json_files(cfg)

    if files:
        data = [(fn, contents) for fn, contents in data if fn in files]

    # There are two modes:
    #
    # 1. User set category in configuration. Then the json files can
    #    either have no category set or they have to have the same
    #    category set.
    #
    # 2. User has NOT set category in configuration. Then the json
    #    files must all have the category set. The categories can be
    #    different.
    #
    # Go through and make sure there aren't any problems with
    # categories.

    all_categories = dict(
        [(cat['title'], cat)
         for cat in steve.richardapi.get_all_categories(api_url)])

    try:
        category = cfg.get('project', 'category')
        category = category.strip()
        if category not in all_categories:
            raise click.ClickException(
                u'Category "{0}" does not exist on server. Build it there '
                u'first.'.format(category)
            )
        else:
            click.echo('Category {0} exists on site.'.format(category))
    except ConfigParser.NoOptionError:
        category = None

    errors = []
    for fn, contents in data:
        if category is None:
            this_cat = contents.get('category')
            if not this_cat:
                errors.append(
                    u'No category set in configuration and {0} has no '
                    u'category set.'.format(fn)
                )
            elif this_cat != this_cat.strip():
                errors.append(
                    u'Category "{0}" has whitespace at beginning or '
                    u'end.'.format(this_cat)
                )
            elif this_cat not in all_categories:
                errors.append(
                    u'Category "{0}" does not exist on server. '
                    u'Build it there first.'.format(this_cat)
                )

        else:
            this_cat = contents.get('category')
            if this_cat is not None and str(this_cat).strip() != category:
                errors.append(
                    u'Category set in configuration ({0}), but {1} has '
                    u'different category ({2}).'.format(category, fn, this_cat)
                )

    if update:
        for fn, contents in data:
            if 'id' not in contents:
                errors.append(
                    u'id not in contents for "{0}".'.format(fn)
                )

    if errors:
        raise click.ClickException('\n'.join(errors))

    # Everything looks ok. So double-check with the user and push.

    click.echo('Pushing to: {0}'.format(api_url))
    click.echo('Username:   {0}'.format(username))
    click.echo('api_key:    {0}'.format(apikey))
    click.echo('update?:    {0}'.format(update))
    click.echo('# videos:   {0}'.format(len(data)))
    click.echo('Once you push, you can not undo it. Push for realz? Y/N')
    if not raw_input().strip().lower().startswith('y'):
        raise click.Abort()

    for fn, contents in data:
        contents['category'] = category or contents.get('category')

        if not update:
            # Nix any id field since that causes problems.
            if 'id' in contents:
                if not overwrite:
                    click.echo(u'Skipping... already exists.')
                    continue
                del contents['id']

            click.echo('Pushing {0}'.format(fn))
            try:
                vid = steve.richardapi.create_video(api_url, apikey, contents)

                if 'id' in vid:
                    contents['id'] = vid['id']
                    click.echo('   Now has id {0}'.format(vid['id']))
                else:
                    click.echo('   Errors?: {0}'.format(vid), err=True)
            except steve.restapi.RestAPIException as exc:
                click.echo('   Error?: {0}'.format(exc), err=True)
                click.echo('   "{0}"'.format(exc.response.content), err=True)

        else:
            click.echo('Updating {0} "{1}" ({2})'.format(
                contents['id'], contents['title'], fn))
            try:
                vid = steve.richardapi.update_video(
                    api_url, apikey, contents['id'], contents)
            except steve.restapi.RestAPIException as exc:
                click.echo('   Error?: {0}'.format(exc), err=True)
                click.echo('   "{0}"'.format(exc.response.content), err=True)

        save_json_file(cfg, fn, contents)