Beispiel #1
0
def fetch_objects(options):
    if options.content == "categories":
        if options.provider != None:
            fetch_categories(options.provider)
        else:
            print "Please provider a provider or run: crab.py -a list -c providers"
    elif options.content == "videos":
        categories = get_categories()
        if options.starting != None:
            if options.starting in parse_categories(categories):
                categories = slice_category(options.starting, categories)
            else:
                print "Could not find %s in category list. run crab.py -a list -c categories to see available categories" % options.starting
                exit()
        for c in categories:
            api_urls['rtube']['videos']['params']['category'] = c.title
            # print api_urls['rtube']['videos']['params']['category']
            url = api.format_url(api_urls['rtube']['videos']['url'], api_urls['rtube']['videos']['params'])
            # print url
            # fetch api call
            json = api.fetch(url)
            pages = json['count'] / len(json['videos'])
            print pages
            for i in range(pages):
                api_urls['rtube']['videos']['params']['page'] = i + 1
                p_url = api.format_url(api_urls['rtube']['videos']['url'], api_urls['rtube']['videos']['params'])
                print p_url
                p_json = api.fetch(p_url)
                api.insert_content(p_json, 'videos')
                if i > 5:
                    break
Beispiel #2
0
def main():
    usage = "%prog [-a action] [-c type of content] [-s starting category]"
    version = "%prog 0.0.1a"
    parser = OptionParser(usage=usage, version=version)
    parser.add_option("-a", "--action",
        action="store", type="string", dest="action",
        help="The action you wish to perform: fetch|get|list. [fetch pulls from web, get pulls from database]")
    parser.add_option('-c', "--content",
        action="store", type="string", dest="content", default=None,
        help="Which type of content you want to get: videos|categories")
    parser.add_option('-s', '--start',
        action="store", type="string", dest="starting", default=None,
        help="Starting Category you wish to start at. Try listing to see available categories")
    parser.add_option('-p', '--provider',
        action="store", type="string", dest="provider", default=None,
        help="The content Provider you wish to slurp")
    (options, args) = parser.parse_args()
    categories = []
    if options.action == "fetch":
        fetch_objects(options)
        if options.content == "categories":
            if options.provider == 'rtube':
                categories = fetch_categories()
            else:
                print "Please provider a provider or run: crab.py -a list -c providers"
        if options.content == "videos":
            categories = get_categories()
            if options.starting != None:
                if options.starting in parse_categories(categories):
                    categories = slice_category(options.starting, categories)
                else:
                    print "coud not find %s in category list. try --list" % options.starting
                    exit()
            for c in categories:
                api_urls['rtube']['videos']['params']['category'] = c.title
                # print api_urls['rtube']['videos']['params']['category']
                url = api.format_url(api_urls['rtube']['videos']['url'], api_urls['rtube']['videos']['params'])
                # print url
                # fetch api call
                json = api.fetch(url)
                pages = json['count'] / len(json['videos'])
                print pages
                for i in range(pages):
                    api_urls['rtube']['videos']['params']['page'] = i + 1
                    p_url = api.format_url(api_urls['rtube']['videos']['url'], api_urls['rtube']['videos']['params'])
                    print p_url
                    p_json = api.fetch(p_url)
                    api.insert_content(p_json, 'videos')
                    if i > 5:
                        break
                # break
            # exit()
    elif options.action == "get":
        if options.content == "categories":
            categories = get_categories()
    elif options.action == "list":
        if options.content == "categories":
            categories = get_categories()
            print "Here is a list of the known categories:"
            for c in categories:
                print c.title
        if options.content == "providers":
            print "here are the known providers:"
            for p in PROVIDERS:
                print p
        else:
            print "No content type specified. Try: categories|videos|providers"
    else:
        print "No action speficied. try --help"
Beispiel #3
0
def fetch_categories(provider="rtube"):
    if provider in PROVIDERS:
        url = api.format_url(api_urls[provider]['categories']['url'], api_urls[provider]['categories']['params'])
        json = api.fetch(url)
        api.insert_content(json, 'categories')
        return get_categories()