示例#1
0
文件: cmdline.py 项目: B-Rich/steve
def scrapevideo_cmd(parser, parsed, args):
    if not parsed.quiet:
        parser.print_byline()

    video_url = parsed.video[0]
    data = scrapevideo(video_url, parsed.richard, 'object')
    if parsed.save:
        cfg = get_project_config()

        projectpath = cfg.get('project', 'projectpath')
        jsonpath = os.path.join(projectpath, 'json')

        if not os.path.exists(jsonpath):
            os.makedirs(jsonpath)

        fn = 'json/' + generate_filename(data['title']) + '.json'

        if os.path.exists(fn):
            err('It already exists!')
            return 1

        with open(fn, 'w') as fp:
            fp.write(convert_to_json(data))
        print 'Saved as {0}'.format(fn)
    else:
        print convert_to_json(data)
    return 0
示例#2
0
文件: webedit.py 项目: B-Rich/steve
 def route_home(self, path):
     config = get_project_config()
     self.render_response(
         200, 'home.html', {
             'title': config.get('project', 'category'),
             'data_files': load_json_files(config)
         })
示例#3
0
def scrapevideo_cmd(parser, parsed, args):
    if not parsed.quiet:
        parser.print_byline()

    video_url = parsed.video[0]
    data = scrapevideo(video_url, parsed.richard, 'object')
    if parsed.save:
        cfg = get_project_config()

        projectpath = cfg.get('project', 'projectpath')
        jsonpath = os.path.join(projectpath, 'json')

        if not os.path.exists(jsonpath):
            os.makedirs(jsonpath)

        fn = 'json/' + generate_filename(data['title']) + '.json'

        if os.path.exists(fn):
            err('It already exists!')
            return 1

        with open(fn, 'w') as fp:
            fp.write(convert_to_json(data))
        print 'Saved as {0}'.format(fn)
    else:
        print convert_to_json(data)
    return 0
示例#4
0
 def route_home(self, path):
     config = get_project_config()
     self.render_response(
         200, 'home.html', {
             'title': config.get('project', 'category'),
             'data_files': load_json_files(config)
         })
示例#5
0
文件: cmdline.py 项目: AvdN/steve
def scrapevideo(ctx, quiet, save, video_url):
    """Fetches metadata for a video from a site."""
    if not quiet:
        click.echo(VERSION)

    data = scrape_video(video_url)[0]
    if save:
        cfg = get_project_config()

        jsonpath = cfg.get('project', 'jsonpath')

        if not os.path.exists(jsonpath):
            os.makedirs(jsonpath)

        fn = generate_filename(data['title']) + '.json'
        full_path = os.path.join(jsonpath, fn)

        if os.path.exists(fn):
            raise click.ClickException(u'File "%s" already exists!' % fn)

        with open(full_path, 'w') as fp:
            fp.write(convert_to_json(data))
        click.echo(u'Saved as {0}'.format(fn))

    else:
        click.echo(convert_to_json(data))
示例#6
0
文件: cmdline.py 项目: jethar/steve
def scrapevideo(ctx, quiet, save, video_url):
    """Fetches metadata for a video from a site."""
    if not quiet:
        click.echo(VERSION)

    data = scrape_video(video_url)[0]
    if save:
        cfg = get_project_config()

        jsonpath = cfg.get('project', 'jsonpath')

        if not os.path.exists(jsonpath):
            os.makedirs(jsonpath)

        fn = generate_filename(data['title']) + '.json'
        full_path = os.path.join(jsonpath, fn)

        if os.path.exists(fn):
            raise click.ClickException(u'File "%s" already exists!' % fn)

        with open(full_path, 'w') as fp:
            fp.write(convert_to_json(data))
        click.echo(u'Saved as {0}'.format(fn))

    else:
        click.echo(convert_to_json(data))
示例#7
0
文件: webedit.py 项目: jethar/steve
    def route_save(self, path):
        cfg = get_project_config()
        data_file = get_data(cfg, path[1])
        if not data_file:
            return self.render_error(404)

        fn, data = data_file
        form_data = self.parse_form()
        reqs = get_video_requirements()

        for req in reqs:
            key = req['name']
            if key in form_data:
                value = None
                if req['type'] == 'IntegerField':
                    value = int(form_data[key].value)
                elif req['type'] == 'BooleanField':
                    value = bool(int(form_data[key].value))
                elif req['type'] in ('DateField', 'DateTimeField'):
                    # TODO: Verify the data format
                    value = form_data[key].value
                elif req['type'] in ('CharField', 'TextField'):
                    value = form_data[key].value.replace('\r\n', '\n')
                elif req['type'] == 'SlugField':
                    # TODO: Verify the data format. Maybe if there is
                    # no slug field, we create it from the title?
                    value = form_data[key].value
                elif req['type'] == 'TextArrayField':
                    # Split the field on carriage returns and drop any
                    # empty strings.
                    value = form_data[key].value
                    value = [
                        mem.strip() for mem in value.split('\n')
                        if mem.strip()
                    ]
                elif req['type'] in ('URLField'):
                    value = form_data[key].value
                else:
                    raise NotImplementedError

                data[key] = value
            else:
                if req['type'] in ('CharField', 'TextField'):
                    data[key] = ''
                elif req['type'] == 'TextArrayField':
                    data[key] = []

                # TODO: What to do about other fields? Set to default?

        # Category is a special case. If it's empty, we should remove
        # it so verify works well.
        if 'category' in data and not data['category'].strip():
            del data['category']

        save_json_file(cfg, fn, data)

        return self.redirect('/edit/{0}'.format(fn))
示例#8
0
文件: webedit.py 项目: redapple/steve
    def route_save(self, path):
        cfg = get_project_config()
        data_file = get_data(cfg, path[1])
        if not data_file:
            return self.render_error(404)

        fn, data = data_file
        form_data = self.parse_form()
        reqs = get_video_requirements()

        for req in reqs:
            key = req['name']
            if key in form_data:
                value = None
                if req['type'] == 'IntegerField':
                    value = int(form_data[key].value)
                elif req['type'] == 'BooleanField':
                    value = bool(int(form_data[key].value))
                elif req['type'] in ('DateField', 'DateTimeField'):
                    # TODO: Verify the data format
                    value = form_data[key].value
                elif req['type'] in ('CharField', 'TextField'):
                    value = form_data[key].value.replace('\r\n', '\n')
                elif req['type'] == 'SlugField':
                    # TODO: Verify the data format. Maybe if there is
                    # no slug field, we create it from the title?
                    value = form_data[key].value
                elif req['type'] == 'TextArrayField':
                    # Split the field on carriage returns and drop any
                    # empty strings.
                    value = form_data[key].value
                    value = [mem.strip() for mem in value.split('\n')
                             if mem.strip()]
                elif req['type'] in ('URLField'):
                    value = form_data[key].value
                else:
                    raise NotImplementedError

                data[key] = value
            else:
                if req['type'] in ('CharField', 'TextField'):
                    data[key] = ''
                elif req['type'] == 'TextArrayField':
                    data[key] = []

                # TODO: What to do about other fields? Set to default?

        # Category is a special case. If it's empty, we should remove
        # it so verify works well.
        if 'category' in data and not data['category'].strip():
            del data['category']

        save_json_file(cfg, fn, data)

        return self.redirect('/edit/{0}'.format(fn))
示例#9
0
文件: test_utils.py 项目: AvdN/steve
    def test_ini_config_creation(self, config):
        runner = CliRunner()
        with runner.isolated_filesystem():
            path = os.getcwd()

            with pytest.raises(ConfigNotFound):
                get_project_config()
            # Create the project directory
            create_project_config_file(path, config)
            cfg = get_project_config()

            assert os.path.exists(config.file_name)
            tmp_dir = 'tmp'
            os.mkdir(tmp_dir)
            os.chdir(tmp_dir)
            cfg2 = get_project_config()
            os.chdir(path)
            assert cfg.get('project', 'jsonpath') == cfg2.get('project', 'jsonpath')

            with pytest.raises(NoOptionError):
                cfg.get('project', 'xmlpath')
示例#10
0
文件: webedit.py 项目: B-Rich/steve
    def route_edit(self, path):
        cfg = get_project_config()
        data_file = get_data(cfg, path[1])
        if not data_file:
            return self.render_error(404)

        fn, data = data_file
        reqs = get_video_requirements()

        # TODO: verify the data and add the errors to the fields?

        all_files = [filename for filename, _ in load_json_files(cfg)]
        fn_index = all_files.index(fn)
        prev_fn = all_files[fn_index - 1] if fn_index > 0 else ''
        next_fn = (all_files[fn_index + 1] if fn_index < len(all_files) - 1
                   else '')

        fields = []

        category = cfg.get('project', 'category')

        for req in reqs:
            key = req['name']
            if key == 'category' and category:
                fields.append(
                    {
                        'name': req['name'],
                        'value': category
                    })
            else:
                fields.append(
                    {
                        'name': req['name'],
                        'type': req['type'],
                        'choices': req['choices'],
                        'md': req['md'],
                        'value': data.get(key, '')
                    })

        self.render_response(
            200, 'edit.html', {
                'title': u'edit {0}'.format(data['title']),
                'fn': fn,
                'fields': fields,
                'prev_fn': prev_fn,
                'next_fn': next_fn
            })
示例#11
0
    def route_save(self, path):
        cfg = get_project_config()
        data_file = get_data(cfg, path[1])
        if not data_file:
            return self.render_error(404)

        fn, data = data_file
        form_data = self.parse_form()
        reqs = get_video_requirements()

        for req in reqs:
            key = req['name']
            if key in form_data:
                value = None
                if req['type'] == 'IntegerField':
                    value = int(form_data[key].value)
                elif req['type'] == 'BooleanField':
                    value = bool(int(form_data[key].value))
                elif req['type'] in ('DateField', 'DateTimeField'):
                    # TODO: Verify the data format
                    value = form_data[key].value
                elif req['type'] in ('CharField', 'TextField'):
                    value = form_data[key].value
                elif req['type'] == 'SlugField':
                    # TODO: Verify the data format. Maybe if there is
                    # no slug field, we create it from the title?
                    value = form_data[key].value
                elif req['type'] in ('TextArrayField'):
                    # Split the field on carriage returns and drop any
                    # empty strings.
                    value = [
                        mem.strip() for mem in form_data[key].value.split('\n')
                        if mem.strip()
                    ]

                data[key] = value
            else:
                if req['type'] in ('CharField', 'TextField'):
                    data[key] = ''

                # TODO: What to do about other fields? Set to default?

        save_json_file(cfg, fn, data)

        return self.redirect('/edit/{0}'.format(fn[5:]))
示例#12
0
    def route_save(self, path):
        cfg = get_project_config()
        data_file = get_data(cfg, path[1])
        if not data_file:
            return self.render_error(404)

        fn, data = data_file
        form_data = self.parse_form()
        reqs = get_video_requirements()

        for req in reqs:
            key = req['name']
            if key in form_data:
                value = None
                if req['type'] == 'IntegerField':
                    value = int(form_data[key].value)
                elif req['type'] == 'BooleanField':
                    value = bool(int(form_data[key].value))
                elif req['type'] in ('DateField', 'DateTimeField'):
                    # TODO: Verify the data format
                    value = form_data[key].value
                elif req['type'] in ('CharField', 'TextField'):
                    value = form_data[key].value
                elif req['type'] == 'SlugField':
                    # TODO: Verify the data format. Maybe if there is
                    # no slug field, we create it from the title?
                    value = form_data[key].value
                elif req['type'] in ('TextArrayField'):
                    # Split the field on carriage returns and drop any
                    # empty strings.
                    value = [mem.strip()
                             for mem in form_data[key].value.split('\n')
                             if mem.strip()]

                data[key] = value
            else:
                if req['type'] in ('CharField', 'TextField'):
                    data[key] = ''

                # TODO: What to do about other fields? Set to default?

        save_json_file(cfg, fn, data)

        return self.redirect('/edit/{0}'.format(fn[5:]))
示例#13
0
    def route_edit(self, path):
        cfg = get_project_config()
        data_file = get_data(cfg, path[1])
        if not data_file:
            return self.render_error(404)

        fn, data = data_file
        reqs = get_video_requirements()

        # TODO: verify the data and add the errors to the fields?

        all_files = [filename for filename, _ in load_json_files(cfg)]
        fn_index = all_files.index(fn)
        prev_fn = all_files[fn_index - 1] if fn_index > 0 else ''
        next_fn = (all_files[fn_index +
                             1] if fn_index < len(all_files) - 1 else '')

        fields = []

        category = cfg.get('project', 'category')

        for req in reqs:
            key = req['name']
            if key == 'category' and category:
                fields.append({'name': req['name'], 'value': category})
            else:
                fields.append({
                    'name': req['name'],
                    'type': req['type'],
                    'choices': req['choices'],
                    'md': req['md'],
                    'value': data.get(key, '')
                })

        self.render_response(
            200, 'edit.html', {
                'title': u'edit {0}'.format(data['title']),
                'fn': fn,
                'fields': fields,
                'prev_fn': prev_fn,
                'next_fn': next_fn
            })
            'slug': '',
            'speakers': speakers,
            'source_url': 'https://www.youtube.com/watch?v={}'.format(video_id)
        }
        return item



if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("--maxresults", help="Max results", default=50)
    parser.add_argument("-c", "--channel", action='store_true', help="YouTube channel id")
    parser.add_argument("-p", "--playlist", action='store_true',  help="YouTube playlist id")
    args = parser.parse_args()

    cfg = get_project_config()

    scraper = YouTubeScraper(cfg, max_results=args.maxresults)

    if args.channel:
        channel_id = get_from_config(cfg, 'channel_id', 'youtube')
        print("scraping channel {}".format(channel_id))
        data = scraper.scrape_channel(channel_id)
        save_json_files(cfg, data)

    elif args.playlist:
        playlist_id = get_from_config(cfg, 'playlist_id', 'youtube')
        print("scraping playlist {}".format(playlist_id))
        data = scraper.scrape_playlist(playlist_id)
        save_json_files(cfg, data)