def populate_database(self, published_before_datetime, page_token, search_params):
if page_token != None: # get next page results
search_params['pageToken'] = page_token
elif published_before_datetime != None: # Continue populating database from where we last left off
search_params['publishedBefore'] = published_before_datetime.strftime('%Y-%m-%dT%H:%M:%SZ')
else: # the database is currently empty. Start a search
# Ensure that we get the updated videos list
search_params['publishedAfter'] = (datetime.now()+timedelta(days=1)).strftime('%Y-%m-%dT%H:%M:%SZ')
response = requests.get(self.request_url, params=search_params)
if str(response.status_code) == '200':
response = response.json() # Convert json response to python dictionary
next_page_token = response['nextPageToken']
for item in response['items']:
video_data = item['snippet']
video = Video() # Video Model instance
video.title = video_data['title']
video.description = video_data['description']
video.publish_datetime = video_data['publishedAt']
video.thumbnail_url = video_data['thumbnails']['default']['url']
video.save()
print(video.publish_datetime)
return next_page_token # needed for next API call
else:
print('failed to fetch data')
print(response.json())
def populate():
client = get_spotify()
errors = []
tracks = Track.objects.filter(populated=False, spotify_id__isnull=False)[:10]
for track in tracks:
try:
data = client._get("audio-features/" + client._get_id("track", track.spotify_id))
except Exception as e:
errors.append(str(e))
continue
track.analysis_url = data["analysis_url"]
track.key = data["key"]
track.time_signature = data["time_signature"]
track.danceability = data["danceability"]
track.energy = data["energy"]
track.loudness = data["loudness"]
track.speechiness = data["speechiness"]
track.acousticness = data["acousticness"]
track.instrumentalness = data["instrumentalness"]
track.liveness = data["liveness"]
track.valence = data["valence"]
track.tempo = data["tempo"]
track.duration_ms = data["duration_ms"]
track.populated = True
track.save()
youtube_videos = YouTubeHelper().search("%s - %s" % (track.artists_to_str, track.title))
for youtube_video in youtube_videos:
try:
# do not load the same video twice
Video.objects.get(video_id=youtube_video["id"]["videoId"])
continue
except Video.DoesNotExist:
pass
except Video.MultipleObjectsReturned:
continue
try:
video = Video()
video.track = track
video.source = "youtube"
video.description = youtube_video["snippet"]["description"]
video.title = youtube_video["snippet"]["title"]
video.channel_id = youtube_video["snippet"]["channelId"]
video.url = "https://www.youtube.com/watch?v=%s" % youtube_video["id"]["videoId"]
video.video_id = youtube_video["id"]["videoId"]
video.save()
except Exception as e:
errors.append(str(e))
continue
return [tracks, errors]
def populate(self):
errors = []
limit = 10
last_id = 0
total = 0
while True:
tracks = Track.objects.filter(
~Exists(Video.objects.filter(track=OuterRef('pk'))),
id__gt=last_id).order_by('id')[:limit]
for track in tracks:
youtube_videos = self.search(
"%s - %s" % (track.artists_to_str, track.title))
if len(youtube_videos) > 0:
for youtube_video in youtube_videos:
try:
# do not load the same video twice
Video.objects.get(
video_id=youtube_video["id"]["videoId"])
continue
except Video.DoesNotExist:
pass
except Video.MultipleObjectsReturned:
continue
try:
video = Video()
video.track = track
video.source = "youtube"
video.description = youtube_video["snippet"][
"description"]
video.title = youtube_video["snippet"]["title"]
video.channel_id = youtube_video["snippet"][
"channelId"]
video.url = "https://www.youtube.com/watch?v=%s" % youtube_video[
"id"]["videoId"]
video.video_id = youtube_video["id"]["videoId"]
video.save()
except Exception as e:
errors.append(str(e))
continue
else:
errors.append("No videos for track %s by %s" %
(track.title, track.artists_to_str))
last_id = track.id
total += 1
if 0 == len(tracks):
break
return [total, errors]
