def populate_database(self, published_before_datetime, page_token, search_params):
if page_token != None: # get next page results
search_params['pageToken'] = page_token
elif published_before_datetime != None: # Continue populating database from where we last left off
search_params['publishedBefore'] = published_before_datetime.strftime('%Y-%m-%dT%H:%M:%SZ')
else: # the database is currently empty. Start a search
# Ensure that we get the updated videos list
search_params['publishedAfter'] = (datetime.now()+timedelta(days=1)).strftime('%Y-%m-%dT%H:%M:%SZ')
response = requests.get(self.request_url, params=search_params)
if str(response.status_code) == '200':
response = response.json() # Convert json response to python dictionary
next_page_token = response['nextPageToken']
for item in response['items']:
video_data = item['snippet']
video = Video() # Video Model instance
video.title = video_data['title']
video.description = video_data['description']
video.publish_datetime = video_data['publishedAt']
video.thumbnail_url = video_data['thumbnails']['default']['url']
video.save()
print(video.publish_datetime)
return next_page_token # needed for next API call
else:
print('failed to fetch data')
print(response.json())
def populate():
client = get_spotify()
errors = []
tracks = Track.objects.filter(populated=False, spotify_id__isnull=False)[:10]
for track in tracks:
try:
data = client._get("audio-features/" + client._get_id("track", track.spotify_id))
except Exception as e:
errors.append(str(e))
continue
track.analysis_url = data["analysis_url"]
track.key = data["key"]
track.time_signature = data["time_signature"]
track.danceability = data["danceability"]
track.energy = data["energy"]
track.loudness = data["loudness"]
track.speechiness = data["speechiness"]
track.acousticness = data["acousticness"]
track.instrumentalness = data["instrumentalness"]
track.liveness = data["liveness"]
track.valence = data["valence"]
track.tempo = data["tempo"]
track.duration_ms = data["duration_ms"]
track.populated = True
track.save()
youtube_videos = YouTubeHelper().search("%s - %s" % (track.artists_to_str, track.title))
for youtube_video in youtube_videos:
try:
# do not load the same video twice
Video.objects.get(video_id=youtube_video["id"]["videoId"])
continue
except Video.DoesNotExist:
pass
except Video.MultipleObjectsReturned:
continue
try:
video = Video()
video.track = track
video.source = "youtube"
video.description = youtube_video["snippet"]["description"]
video.title = youtube_video["snippet"]["title"]
video.channel_id = youtube_video["snippet"]["channelId"]
video.url = "https://www.youtube.com/watch?v=%s" % youtube_video["id"]["videoId"]
video.video_id = youtube_video["id"]["videoId"]
video.save()
except Exception as e:
errors.append(str(e))
continue
return [tracks, errors]
def populate(self):
errors = []
limit = 10
last_id = 0
total = 0
while True:
tracks = Track.objects.filter(
~Exists(Video.objects.filter(track=OuterRef('pk'))),
id__gt=last_id).order_by('id')[:limit]
for track in tracks:
youtube_videos = self.search(
"%s - %s" % (track.artists_to_str, track.title))
if len(youtube_videos) > 0:
for youtube_video in youtube_videos:
try:
# do not load the same video twice
Video.objects.get(
video_id=youtube_video["id"]["videoId"])
continue
except Video.DoesNotExist:
pass
except Video.MultipleObjectsReturned:
continue
try:
video = Video()
video.track = track
video.source = "youtube"
video.description = youtube_video["snippet"][
"description"]
video.title = youtube_video["snippet"]["title"]
video.channel_id = youtube_video["snippet"][
"channelId"]
video.url = "https://www.youtube.com/watch?v=%s" % youtube_video[
"id"]["videoId"]
video.video_id = youtube_video["id"]["videoId"]
video.save()
except Exception as e:
errors.append(str(e))
continue
else:
errors.append("No videos for track %s by %s" %
(track.title, track.artists_to_str))
last_id = track.id
total += 1
if 0 == len(tracks):
break
return [total, errors]
def like_by_url(request):
if not request.user.is_authenticated():
raise Unauthorized()
querydict = request.GET if request.method == 'GET' else request.POST
try:
normalized_url = url_fix(querydict['url'])
except KeyError:
raise BadRequest('Parameter:url missing')
except MalformedURLException:
raise BadRequest('Malformed URL:%s' % url)
try:
video = Video.objects.get(url=normalized_url)
try:
UserVideo.objects.get(user=request.user, video=video, liked_timestamp__isnull=False)
except UserVideo.DoesNotExist:
push_like_to_fb.delay(video, request.user)
user_video = request.user.like_video(video)
task_id = video.task_id
except Video.DoesNotExist:
video = Video(url=normalized_url)
video.save()
user_video = UserVideo(user=request.user,
video=video,
host=request.META.get('HTTP_REFERER'),
liked=True,
liked_timestamp=datetime.utcnow())
user_video.save()
# Fetch video metadata in background
task = fetch.delay(request.user.id,
normalized_url,
user_video.host,
callback=push_like_to_fb.subtask((request.user, )))
task_id = task.task_id
info = as_dict(user_video)
info['task_id'] = task_id
info['first'] = request.user.notifications()['firstlike']
return info
def add(request):
if not request.user.is_authenticated():
raise Unauthorized()
querydict = request.GET if request.method == 'GET' else request.POST
try:
normalized_url = url_fix(querydict['url'])
except KeyError:
raise BadRequest('Parameter:url missing')
except MalformedURLException:
raise BadRequest('Malformed URL:%s' % url)
try:
user_video = UserVideo.objects.get(user=request.user, video__url=normalized_url)
if user_video.saved:
raise BadRequest('Video:%s already saved with id:%s' % (user_video.video.url, user_video.video.id))
except UserVideo.DoesNotExist:
try:
video = Video.objects.get(url=normalized_url)
except Video.DoesNotExist:
video = Video(url=normalized_url)
video.save()
user_video = UserVideo(user=request.user, video=video)
user_video.saved = True
user_video.saved_timestamp = datetime.utcnow()
user_video.host = request.META.get('HTTP_REFERER')
user_video.save()
# Fetch video metadata in background
task = fetch.delay(request.user.id, normalized_url, user_video.host)
info = as_dict(user_video)
info.update({'first': request.user.saved_videos().count() == 1,
'unwatched': request.user.unwatched_videos().count(),
'task_id': task.task_id})
return info
