def new_scraper_func(url, shortmem=None, *args, **kwargs):
if not shortmem.get('api_data'):
video_id = VIMEO_REGEX.match(url).groupdict()['video_id']
url = '%s?%s' % (VIMEO_API_URL,
urllib.urlencode({
'method': 'vimeo.videos.getInfo',
'format': 'json',
'video_id': video_id}))
consumer = oauth2.Consumer(VIMEO_API_KEY, VIMEO_API_SECRET)
client = oauth2.Client(consumer)
backoff = util.random_exponential_backoff(2)
for i in range(5):
try:
api_raw_data = client.request(url)[1]
api_data = simplejson.loads(api_raw_data)
except Exception:
continue
else:
if 'video' in api_data:
shortmem['api_data'] = api_data['video'][0]
break
backoff.next()
if 'api_data' not in shortmem:
return None
return scraper_func(url, shortmem=shortmem, *args,
**kwargs)
def get_shortmem(url):
shortmem = {}
video_id = VIMEO_REGEX.match(url).groupdict()['video_id']
url = '%s?%s' % (VIMEO_API_URL,
urllib.urlencode({
'method': 'vimeo.videos.getInfo',
'format': 'json',
'video_id': video_id
}))
consumer = oauth2.Consumer(settings.VIMEO_API_KEY,
settings.VIMEO_API_SECRET)
client = oauth2.Client(consumer)
backoff = util.random_exponential_backoff(2)
for i in range(5):
try:
api_raw_data = client.request(url)[1]
api_data = simplejson.loads(api_raw_data)
except Exception:
continue
else:
if api_data.get('stat') == u'fail':
error = u'Vimeo API error'
try:
error += ': %s' % api_data['err']['expl']
except KeyError:
pass
raise VimeoError(error)
if 'video' in api_data:
shortmem['api_data'] = api_data['video'][0]
break
backoff.next()
return shortmem
def get_shortmem(url):
shortmem = {}
video_id = VIMEO_REGEX.match(url).groupdict()['video_id']
url = '%s?%s' % (VIMEO_API_URL,
urllib.urlencode({
'method': 'vimeo.videos.getInfo',
'format': 'json',
'video_id': video_id}))
consumer = oauth2.Consumer(settings.VIMEO_API_KEY, settings.VIMEO_API_SECRET)
client = oauth2.Client(consumer)
backoff = util.random_exponential_backoff(2)
for i in range(5):
try:
api_raw_data = client.request(url)[1]
api_data = simplejson.loads(api_raw_data)
except Exception:
continue
else:
if api_data.get('stat') == u'fail':
error = u'Vimeo API error'
try:
error += ': %s' % api_data['err']['expl']
except KeyError:
pass
raise VimeoError(error)
if 'video' in api_data:
shortmem['api_data'] = api_data['video'][0]
break
backoff.next()
return shortmem
def bulk_import(parsed_feed):
match = USERNAME_RE.search(parsed_feed.feed.link)
username = match.group("name")
if parsed_feed.feed.link in _cached_video_count:
count = _cached_video_count[parsed_feed.feed.link]
else:
count = video_count(parsed_feed)
parsed_feed = feedparser.FeedParserDict(parsed_feed.copy())
parsed_feed.entries = []
consumer = oauth2.Consumer(vimeo.VIMEO_API_KEY, vimeo.VIMEO_API_SECRET)
client = oauth2.Client(consumer)
data = {
"format": "json",
"method": "vimeo.videos.getUploaded",
"per_page": 50,
"sort": "newest",
"full_response": "yes",
"user_id": username,
}
if username.startswith("channels"):
del data["user_id"]
data["method"] = "vimeo.channels.getVideos"
data["channel_id"] = username.split("/", 1)[1]
elif username.startswith("groups"):
del data["user_id"]
data["method"] = "vimeo.groups.getVideos"
data["group_id"] = username.split("/", 1)[1]
elif match.group("type") == "likes":
data["method"] = "vimeo.videos.getLikes"
for i in range(1, int(math.ceil(count / 50.0)) + 1):
data["page"] = i
backoff = random_exponential_backoff(2)
api_data = None
for j in range(5):
try:
api_raw_data = client.request("%s?%s" % (vimeo.VIMEO_API_URL, urllib.urlencode(data)))[1]
api_data = simplejson.loads(api_raw_data)
break
except Exception:
continue
else:
if "videos" in api_data:
break
backoff.next()
if api_data is None:
break
for video in api_data["videos"]["video"]:
parsed_feed.entries.append(feedparser_dict(_json_to_feedparser(video)))
# clean up cache
if parsed_feed.feed.link in _cached_video_count:
del _cached_video_count[parsed_feed.feed.link]
return parsed_feed
def get_shortmem(url):
shortmem = {}
video_id = VIMEO_REGEX.match(url).groupdict()['video_id']
url = '%s?%s' % (VIMEO_API_URL,
urllib.urlencode({
'method': 'vimeo.videos.getInfo',
'format': 'json',
'video_id': video_id
}))
consumer = oauth2.Consumer(settings.VIMEO_API_KEY,
settings.VIMEO_API_SECRET)
client = oauth2.Client(consumer)
backoff = util.random_exponential_backoff(2)
for i in range(3):
try:
api_raw_data = client.request(url)[1]
api_data = json.loads(api_raw_data)
except Exception, e:
continue
else:
if api_data.get('stat') == u'fail':
error = u'Vimeo API error'
try:
error += ': %s' % api_data['err']['expl']
except KeyError:
pass
raise VimeoError(error)
if 'video' in api_data:
shortmem['api_data'] = api_data['video'][0]
break
else:
#this is hack to get info from Vimeo API. For some video it return strange error in response
try:
data = api_data['backtrace'][1]['object']["_valStack"][1][
"children"]
if 'description' in data and 'title' in data:
shortmem['api_data'] = data
break
except (KeyError, IndexError):
pass
backoff.next()
def get_shortmem(url):
shortmem = {}
video_id = VIMEO_REGEX.match(url).groupdict()['video_id']
url = '%s?%s' % (VIMEO_API_URL,
urllib.urlencode({
'method': 'vimeo.videos.getInfo',
'format': 'json',
'video_id': video_id}))
consumer = oauth2.Consumer(settings.VIMEO_API_KEY, settings.VIMEO_API_SECRET)
client = oauth2.Client(consumer)
backoff = util.random_exponential_backoff(2)
for i in range(3):
try:
api_raw_data = client.request(url)[1]
api_data = simplejson.loads(api_raw_data)
except Exception, e:
continue
else:
if api_data.get('stat') == u'fail':
error = u'Vimeo API error'
try:
error += ': %s' % api_data['err']['expl']
except KeyError:
pass
raise VimeoError(error)
if 'video' in api_data:
shortmem['api_data'] = api_data['video'][0]
break
else:
#this is hack to get info from Vimeo API. For some video it return strange error in response
try:
data = api_data['backtrace'][1]['object']["_valStack"][1]["children"]
if 'description' in data and 'title' in data:
shortmem['api_data'] = data
break
except (KeyError, IndexError):
pass
backoff.next()
def get_shortmem(url):
shortmem = {}
video_id = VIMEO_REGEX.match(url).groupdict()["video_id"]
url = "%s?%s" % (
VIMEO_API_URL,
urllib.urlencode({"method": "vimeo.videos.getInfo", "format": "json", "video_id": video_id}),
)
consumer = oauth2.Consumer(settings.VIMEO_API_KEY, settings.VIMEO_API_SECRET)
client = oauth2.Client(consumer)
backoff = util.random_exponential_backoff(2)
for i in range(3):
try:
api_raw_data = client.request(url)[1]
api_data = simplejson.loads(api_raw_data)
except Exception, e:
continue
else:
if api_data.get("stat") == u"fail":
error = u"Vimeo API error"
try:
error += ": %s" % api_data["err"]["expl"]
except KeyError:
pass
raise VimeoError(error)
if "video" in api_data:
shortmem["api_data"] = api_data["video"][0]
break
else:
# this is hack to get info from Vimeo API. For some video it return strange error in response
try:
data = api_data["backtrace"][1]["object"]["_valStack"][1]["children"]
if "description" in data and "title" in data:
shortmem["api_data"] = data
break
except (KeyError, IndexError):
pass
backoff.next()
def get_shortmem(url):
shortmem = {}
video_id = WISTIA_REGEX.match(url).groupdict()['video_id']
apiurl = '%s?%s' % (WISTIA_OEMBED_API_URL, urllib.quote(url))
finalexcept = None
backoff = util.random_exponential_backoff(2)
for i in range(3):
try:
reponse = urllib.urlopen(apiurl)
api_raw_data = response.read()
api_data = json.loads(api_raw_data)
except Exception as e:
finalexcept = e
continue
else:
shortmem['oembed'] = api_data
break
backoff.next()
if 'oembed' in shortmem:
return shortmem
errmsg = u'Wistia API error : '
if finalexcept is not None:
"""if isinstance(finalexcept, urllib.HTTPError):
errmsg += finalexcept.code + " - " + HTTPResponseMessages[ finalexcept.code ][0]
elif isinstance(finalexcept, urllib.URLError):
errmsg += "Could not connect - " + finalexcept.reason
else:"""
errmsg += str(finalexcept)
else:
errmsg += u' Unrecognized error. Sorry about that, chief.'
return None
