Python open_url_while_lying_about_agent 예제들

예제 #1

파일: vimeo.py 프로젝트: MechanisM/mirosubs

def bulk_import(parsed_feed):
    match = USERNAME_RE.search(parsed_feed.feed.link)
    username = match.group('name')
    if parsed_feed.feed.link in _cached_video_count:
        count = _cached_video_count[parsed_feed.feed.link]
    else:
        count = video_count(parsed_feed)
    post_url = _post_url(username, match.group('type') or 'videos', 'page=%i')
    parsed_feed = feedparser.FeedParserDict(parsed_feed.copy())
    parsed_feed.entries = []
    for i in range(1, int(math.ceil(count / 20.0)) + 1):
        response = open_url_while_lying_about_agent(post_url % i)
        if response.getcode() != 200:
            break
        data = response.read()
        if not data:
            break
        json_data = simplejson.loads(data)
        for video in json_data:
            parsed_feed.entries.append(feedparser_dict(
                    _json_to_feedparser(video)))

    # clean up cache
    if parsed_feed.feed.link in _cached_video_count:
        del _cached_video_count[parsed_feed.feed.link]

    return parsed_feed

예제 #2

def bulk_import(parsed_feed):
    match = USERNAME_RE.search(parsed_feed.feed.link)
    username = match.group('name')
    if parsed_feed.feed.link in _cached_video_count:
        count = _cached_video_count[parsed_feed.feed.link]
    else:
        count = video_count(parsed_feed)
    post_url = _post_url(username, match.group('type') or 'videos', 'page=%i')
    parsed_feed = feedparser.FeedParserDict(parsed_feed.copy())
    parsed_feed.entries = []
    for i in range(1, int(math.ceil(count / 20.0)) + 1):
        response = open_url_while_lying_about_agent(post_url % i)
        if response.getcode() != 200:
            break
        data = response.read()
        if not data:
            break
        json_data = simplejson.loads(data)
        for video in json_data:
            parsed_feed.entries.append(
                feedparser_dict(_json_to_feedparser(video)))

    # clean up cache
    if parsed_feed.feed.link in _cached_video_count:
        del _cached_video_count[parsed_feed.feed.link]

    return parsed_feed

예제 #3

파일: vimeo.py 프로젝트: MechanisM/mirosubs

def video_count(parsed_feed):
    if not parsed_feed.feed.get('generator', '').endswith('Vimeo'):
        return None
    match = USERNAME_RE.search(parsed_feed.feed.link)
    username = match.group('name')
    url = _post_url(username, 'info')
    json_data = simplejson.load(open_url_while_lying_about_agent(url))
    count = None
    if match.group('type') in ('videos', None):
        if 'total_videos_uploaded' in json_data:
            count = json_data['total_videos_uploaded']
        elif 'total_videos' in json_data:
            count = json_data['total_videos']
    elif match.group('type') == 'likes':
        count = json_data.get('total_videos_liked')
    _cached_video_count[parsed_feed.feed.link] = count
    return count

예제 #4

def video_count(parsed_feed):
    if not parsed_feed.feed.get('generator', '').endswith('Vimeo'):
        return None
    match = USERNAME_RE.search(parsed_feed.feed.link)
    username = match.group('name')
    url = _post_url(username, 'info')
    json_data = simplejson.load(open_url_while_lying_about_agent(url))
    count = None
    if match.group('type') in ('videos', None):
        if 'total_videos_uploaded' in json_data:
            count = json_data['total_videos_uploaded']
        elif 'total_videos' in json_data:
            count = json_data['total_videos']
    elif match.group('type') == 'likes':
        count = json_data.get('total_videos_liked')
    _cached_video_count[parsed_feed.feed.link] = count
    return count

예제 #5

파일: vimeo.py 프로젝트: pombredanne/vidscraper

def video_count(parsed_feed):
    if not parsed_feed.feed.get("generator", "").endswith("Vimeo"):
        return None
    match = USERNAME_RE.search(parsed_feed.feed.link)
    username = match.group("name")
    url = _post_url(username, "info")
    json_data = simplejson.load(open_url_while_lying_about_agent(url))
    count = None
    if match.group("type") in ("videos", None):
        if "total_videos_uploaded" in json_data:
            count = json_data["total_videos_uploaded"]
        elif "total_videos" in json_data:
            count = json_data["total_videos"]
    elif match.group("type") == "likes":
        count = json_data.get("total_videos_liked")
    _cached_video_count[parsed_feed.feed.link] = count
    return count